feat(analytics): adding google analytics for ZFSPV

Whenever a volume is provisioned and de-provisioned we will send a google event with mainly following details :
1.    pvName (will shown as app title in google analytics)
2.    size of the volume
3.    event type : volume-provision, volume-deprovision
4.    storage type zfs-localpv
5.    replicacount as 1
6.    ClientId as default namespace uuid

Apart from this, we send the event once in 24 hr, which will have some info like number of nodes, node type, kubernetes version etc.

This metric is cotrolled by OPENEBS_IO_ENABLE_ANALYTICS env. We can set it to false if we don't want to send the metrics.

Signed-off-by: Pawan <pawan@mayadata.io>

pkg/usage/const.go

@@ -0,0 +1,49 @@
+/*
+Copyright 2020 The OpenEBS Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+package usage
+
+const (
+	// GAclientID is the unique code of OpenEBS project in Google Analytics
+	GAclientID string = "UA-127388617-1"
+
+	// supported events categories
+
+	// Install event is sent on pod starts
+	InstallEvent string = "install"
+	// Ping event is sent periodically
+	Ping string = "zfs-ping"
+	// VolumeProvision event is sent when a volume is created
+	VolumeProvision string = "volume-provision"
+	//VolumeDeprovision event is sent when a volume is deleted
+	VolumeDeprovision string = "volume-deprovision"
+	AppName           string = "OpenEBS"
+
+	// Event labels
+	RunningStatus      string = "running"
+	EventLabelNode     string = "nodes"
+	EventLabelCapacity string = "capacity"
+
+	// Event action
+	Replica             string = "replica:"
+	DefaultReplicaCount string = "replica:1"
+
+	// Event application name constant for volume event
+	DefaultCASType string = "zfs-localpv"
+
+	// LocalPVReplicaCount is the constant used by usage to represent
+	// replication factor in LocalPV
+	LocalPVReplicaCount string = "1"
+)

pkg/usage/googleanalytics.go

@@ -0,0 +1,52 @@
+/*
+Copyright 2020 The OpenEBS Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+package usage
+
+import (
+	analytics "github.com/jpillora/go-ogle-analytics"
+	"k8s.io/klog"
+)
+
+// Send sends a single usage metric to Google Analytics with some
+// compulsory fields defined in Google Analytics API
+// bindings(jpillora/go-ogle-analytics)
+func (u *Usage) Send() {
+	// Instantiate a Gclient with the tracking ID
+	go func() {
+		// Un-wrap the gaClient struct back here
+		gaClient, err := analytics.NewClient(u.Gclient.trackID)
+		if err != nil {
+			return
+		}
+		gaClient.ClientID(u.clientID).
+			CampaignSource(u.campaignSource).
+			CampaignContent(u.clientID).
+			ApplicationID(u.appID).
+			ApplicationVersion(u.appVersion).
+			DataSource(u.dataSource).
+			ApplicationName(u.appName).
+			ApplicationInstallerID(u.appInstallerID).
+			DocumentTitle(u.documentTitle)
+		// Un-wrap the Event struct back here
+		event := analytics.NewEvent(u.category, u.action)
+		event.Label(u.label)
+		event.Value(u.value)
+		if err := gaClient.Send(event); err != nil {
+			klog.Errorf(err.Error())
+			return
+		}
+	}()
+}

pkg/usage/ping.go

@@ -0,0 +1,63 @@
+/*
+Copyright 2020 The OpenEBS Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package usage
+
+import (
+	"time"
+
+	"github.com/openebs/zfs-localpv/pkg/common/env"
+)
+
+var OpenEBSPingPeriod = "OPENEBS_IO_ANALYTICS_PING_INTERVAL"
+
+const (
+	// defaultPingPeriod sets the default ping heartbeat interval
+	defaultPingPeriod time.Duration = 24 * time.Hour
+	// minimumPingPeriod sets the minimum possible configurable
+	// heartbeat period, if a value lower than this will be set, the
+	// defaultPingPeriod will be used
+	minimumPingPeriod time.Duration = 1 * time.Hour
+)
+
+// PingCheck sends ping events to Google Analytics
+func PingCheck() {
+	// Create a new usage field
+	u := New()
+	duration := getPingPeriod()
+	ticker := time.NewTicker(duration)
+	for _ = range ticker.C {
+		u.Build().
+			InstallBuilder(true).
+			SetCategory(Ping).
+			Send()
+	}
+}
+
+// getPingPeriod sets the duration of health events, defaults to 24
+func getPingPeriod() time.Duration {
+	value := env.GetOrDefault(OpenEBSPingPeriod, string(defaultPingPeriod))
+	duration, _ := time.ParseDuration(value)
+	// Sanitychecks for setting time duration of health events
+	// This way, we are checking for negative and zero time duration and we
+	// also have a minimum possible configurable time duration between health events
+	if duration < minimumPingPeriod {
+		// Avoid corner case when the ENV value is undesirable
+		return time.Duration(defaultPingPeriod)
+	} else {
+		return time.Duration(duration)
+	}
+}

pkg/usage/ping_test.go

@@ -0,0 +1,56 @@
+// Copyright 2020 The OpenEBS Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package usage
+
+import (
+	"os"
+	"testing"
+	"time"
+)
+
+func TestGetPingPeriod(t *testing.T) {
+	beforeFunc := func(value string) {
+		if err := os.Setenv(string(OpenEBSPingPeriod), value); err != nil {
+			t.Logf("Unable to set environment variable")
+		}
+	}
+	afterFunc := func() {
+		if err := os.Unsetenv(string(OpenEBSPingPeriod)); err != nil {
+			t.Logf("Unable to unset environment variable")
+		}
+	}
+	testSuite := map[string]struct {
+		OpenEBSPingPeriodValue string
+		ExpectedPeriodValue    time.Duration
+	}{
+		"24 seconds":          {"24s", 86400000000000},
+		"24 minutes":          {"24m", 86400000000000},
+		"24 hours":            {"24h", 86400000000000},
+		"Negative 24 hours":   {"-24h", 86400000000000},
+		"Random string input": {"Apache", 86400000000000},
+		"Two hours":           {"2h", 7200000000000},
+		"Three hundred hours": {"300h", 1080000000000000},
+		"Fifty two seconds":   {"52000000000ns", 86400000000000},
+		"Empty env value":     {"", 86400000000000},
+	}
+	for testKey, testData := range testSuite {
+		beforeFunc(testData.OpenEBSPingPeriodValue)
+		evaluatedValue := getPingPeriod()
+		if evaluatedValue != testData.ExpectedPeriodValue {
+			t.Fatalf("Tests failed for %s, expected=%d, got=%d", testKey, testData.ExpectedPeriodValue, evaluatedValue)
+		}
+		afterFunc()
+	}
+}

pkg/usage/size.go

@@ -0,0 +1,27 @@
+/*
+Copyright 2020 The OpenEBS Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package usage
+
+import units "github.com/docker/go-units"
+
+// toGigaUnits converts a size from xB to bytes where x={k,m,g,t,p...}
+// and return the number of Gigabytes as an integer
+// 1 gigabyte=1000 megabyte
+func toGigaUnits(size string) (int64, error) {
+	sizeInBytes, err := units.FromHumanSize(size)
+	return sizeInBytes / units.GB, err
+}

pkg/usage/size_test.go

@@ -0,0 +1,61 @@
+/*
+Copyright 2020 The OpenEBS Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package usage
+
+import "testing"
+
+func TestToGigaUnits(t *testing.T) {
+	tests := map[string]struct {
+		stringSize    string
+		expectedGsize int64
+		positiveTest  bool
+	}{
+		"One Hundred Twenty Three thousand Four Hundred Fifty Six Teribytes": {
+			"123456 TiB",
+			123456000,
+			true,
+		},
+		"One Gibibyte": {
+			"1 GiB",
+			1,
+			true,
+		},
+		"One Megabyte": {
+			"1 MB",
+			0, // One cannot express <1GB in integer
+			true,
+		},
+		"One Megabyte negative-case": {
+			"1 MB",
+			1,
+			false,
+			// 1 MB isn't 1 GB
+		},
+		"One hundred four point five gigabyte": {
+			"104.5 GB",
+			104,
+			true,
+		},
+	}
+
+	for testKey, testSuite := range tests {
+		gotValue, err := toGigaUnits(testSuite.stringSize)
+		if (gotValue != testSuite.expectedGsize || err != nil) && testSuite.positiveTest {
+			t.Fatalf("Tests failed for %s, expected=%d, got=%d", testKey, testSuite.expectedGsize, gotValue)
+		}
+	}
+}

pkg/usage/usage.go

@@ -0,0 +1,257 @@
+/*
+Copyright 2020 The OpenEBS Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+package usage
+
+import (
+	k8sapi "github.com/openebs/zfs-localpv/pkg/client/k8s/v1alpha1"
+)
+
+// Usage struct represents all information about a usage metric sent to
+// Google Analytics with respect to the application
+type Usage struct {
+	// Embedded Event struct as we are currently only sending hits of type
+	// 'event'
+	Event
+
+	// https://developers.google.com/analytics/devguides/collection/protocol/v1/parameters#an
+	// use-case: cstor or jiva volume, or m-apiserver application
+	// Embedded field for application
+	Application
+
+	// Embedded Gclient struct
+	Gclient
+}
+
+// Event is a represents usage of OpenEBS
+// Event contains all the query param fields when hits is of type='event'
+// Ref: https://developers.google.com/analytics/devguides/collection/protocol/v1/parameters#ec
+type Event struct {
+	// (Required) Event Category, ec
+	category string
+	// (Required) Event Action, ea
+	action string
+	// (Optional) Event Label, el
+	label string
+	// (Optional) Event vallue, ev
+	// Non negative
+	value int64
+}
+
+// NewEvent returns an Event struct with eventCategory, eventAction,
+// eventLabel, eventValue fields
+func (u *Usage) NewEvent(c, a, l string, v int64) *Usage {
+	u.category = c
+	u.action = a
+	u.label = l
+	u.value = v
+	return u
+}
+
+// Application struct holds details about the Application
+type Application struct {
+	// eg. project version
+	appVersion string
+
+	// eg. kubernetes version
+	appInstallerID string
+
+	// Name of the application, usage(OpenEBS/NDM)
+	appID string
+
+	// eg. usage(os-type/architecture) of system or volume's CASType
+	appName string
+}
+
+// Gclient struct represents a Google Analytics hit
+type Gclient struct {
+	// constant tracking-id used to send a hit
+	trackID string
+
+	// anonymous client-id
+	clientID string
+
+	// anonymous campaign source
+	campaignSource string
+
+	// https://developers.google.com/analytics/devguides/collection/protocol/v1/parameters#ds
+	// (usecase) node-detail
+	dataSource string
+
+	// Document-title property in Google Analytics
+	// https://developers.google.com/analytics/devguides/collection/protocol/v1/parameters#dt
+	// use-case: uuid of the volume objects or a uuid to anonymously tell objects apart
+	documentTitle string
+}
+
+// New returns an instance of Usage
+func New() *Usage {
+	return &Usage{}
+}
+
+// SetDataSource : usage(os-type, kernel)
+func (u *Usage) SetDataSource(dataSource string) *Usage {
+	u.dataSource = dataSource
+	return u
+}
+
+// SetTrackingID Sets the GA-code for the project
+func (u *Usage) SetTrackingID(track string) *Usage {
+	u.trackID = track
+	return u
+}
+
+// SetCampaignSource : source of openebs installater like:
+// helm or operator etc. This will have to be configured
+// via ENV variable OPENEBS_IO_INSTALLER_TYPE
+func (u *Usage) SetCampaignSource(campaignSrc string) *Usage {
+	u.campaignSource = campaignSrc
+	return u
+}
+
+// SetDocumentTitle : usecase(anonymous-id)
+func (u *Usage) SetDocumentTitle(documentTitle string) *Usage {
+	u.documentTitle = documentTitle
+	return u
+}
+
+// SetApplicationName : usecase(os-type/arch, volume CASType)
+func (u *Usage) SetApplicationName(appName string) *Usage {
+	u.appName = appName
+	return u
+}
+
+// SetApplicationID : usecase(OpenEBS/NDM)
+func (u *Usage) SetApplicationID(appID string) *Usage {
+	u.appID = appID
+	return u
+}
+
+// SetApplicationVersion : usecase(project-version)
+func (u *Usage) SetApplicationVersion(appVersion string) *Usage {
+	u.appVersion = appVersion
+	return u
+}
+
+// SetApplicationInstallerID : usecase(k8s-version)
+func (u *Usage) SetApplicationInstallerID(appInstallerID string) *Usage {
+	u.appInstallerID = appInstallerID
+	return u
+}
+
+// SetClientID sets the anonymous user id
+func (u *Usage) SetClientID(userID string) *Usage {
+	u.clientID = userID
+	return u
+}
+
+// SetCategory sets the category of an event
+func (u *Usage) SetCategory(c string) *Usage {
+	u.category = c
+	return u
+}
+
+// SetAction sets the action of an event
+func (u *Usage) SetAction(a string) *Usage {
+	u.action = a
+	return u
+}
+
+// SetLabel sets the label for an event
+func (u *Usage) SetLabel(l string) *Usage {
+	u.label = l
+	return u
+}
+
+// SetValue sets the value for an event's label
+func (u *Usage) SetValue(v int64) *Usage {
+	u.value = v
+	return u
+}
+
+// Build is a builder method for Usage struct
+func (u *Usage) Build() *Usage {
+	// Default ApplicationID for openebs project is OpenEBS
+	v := NewVersion()
+	v.getVersion(false)
+	u.SetApplicationID(AppName).
+		SetTrackingID(GAclientID).
+		SetClientID(v.id).
+		SetCampaignSource(v.installerType)
+	// TODO: Add condition for version over-ride
+	// Case: CAS/Jiva version, etc
+	return u
+}
+
+// Application builder is used for adding k8s&openebs environment detail
+// for non install events
+func (u *Usage) ApplicationBuilder() *Usage {
+	v := NewVersion()
+	v.getVersion(false)
+	u.SetApplicationVersion(v.openebsVersion).
+		SetApplicationName(v.k8sArch).
+		SetApplicationInstallerID(v.k8sVersion).
+		SetDataSource(v.nodeType)
+	return u
+}
+
+// SetVolumeCapacity sets the storage capacity of the volume for a volume event
+func (u *Usage) SetVolumeCapacity(volCapG string) *Usage {
+	s, _ := toGigaUnits(volCapG)
+	u.SetValue(s)
+	return u
+}
+
+// Wrapper for setting the default storage-engine for volume-provision event
+func (u *Usage) SetVolumeType(volType, method string) *Usage {
+	if method == VolumeProvision && volType == "" {
+		// Set the default storage engine, if not specified in the request
+		u.SetApplicationName(DefaultCASType)
+	} else {
+		u.SetApplicationName(volType)
+	}
+	return u
+}
+
+// Wrapper for setting replica count for volume events
+// NOTE: This doesn't get the replica count in a volume de-provision event.
+// TODO: Pick the current value of replica-count from the CAS-engine
+func (u *Usage) SetReplicaCount(count, method string) *Usage {
+	if method == VolumeProvision && count == "" {
+		// Case: When volume-provision the replica count isn't specified
+		// it is set to three by default by the m-apiserver
+		u.SetAction(DefaultReplicaCount)
+	} else {
+		// Catch all case for volume-deprovision event and
+		// volume-provision event with an overriden replica-count
+		u.SetAction(Replica + count)
+	}
+	return u
+}
+
+// InstallBuilder is a concrete builder for install events
+func (u *Usage) InstallBuilder(override bool) *Usage {
+	v := NewVersion()
+	clusterSize, _ := k8sapi.NumberOfNodes()
+	v.getVersion(override)
+	u.SetApplicationVersion(v.openebsVersion).
+		SetApplicationName(v.k8sArch).
+		SetApplicationInstallerID(v.k8sVersion).
+		SetDataSource(v.nodeType).
+		SetDocumentTitle(v.id).
+		SetApplicationID(AppName).
+		NewEvent(InstallEvent, RunningStatus, EventLabelNode, int64(clusterSize))
+	return u
+}

pkg/usage/versionset.go

@@ -0,0 +1,111 @@
+/*
+Copyright 2020 The OpenEBS Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+package usage
+
+import (
+	k8sapi "github.com/openebs/zfs-localpv/pkg/client/k8s/v1alpha1"
+	env "github.com/openebs/zfs-localpv/pkg/common/env"
+	openebsversion "github.com/openebs/zfs-localpv/pkg/version"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/klog"
+)
+
+var (
+	clusterUUID    = "OPENEBS_IO_USAGE_UUID"
+	clusterVersion = "OPENEBS_IO_K8S_VERSION"
+	clusterArch    = "OPENEBS_IO_K8S_ARCH"
+	openEBSversion = "OPENEBS_IO_VERSION_TAG"
+	nodeType       = "OPENEBS_IO_NODE_TYPE"
+	installerType  = "OPENEBS_IO_INSTALLER_TYPE"
+)
+
+// versionSet is a struct which stores (sort of) fixed information about a
+// k8s environment
+type versionSet struct {
+	id             string // OPENEBS_IO_USAGE_UUID
+	k8sVersion     string // OPENEBS_IO_K8S_VERSION
+	k8sArch        string // OPENEBS_IO_K8S_ARCH
+	openebsVersion string // OPENEBS_IO_VERSION_TAG
+	nodeType       string // OPENEBS_IO_NODE_TYPE
+	installerType  string // OPENEBS_IO_INSTALLER_TYPE
+}
+
+// NewVersion returns a new versionSet struct
+func NewVersion() *versionSet {
+	return &versionSet{}
+}
+
+// fetchAndSetVersion consumes the Kubernetes API to get environment constants
+// and returns a versionSet struct
+func (v *versionSet) fetchAndSetVersion() error {
+	var err error
+	v.id, err = getUUIDbyNS("default")
+	if err != nil {
+		return err
+	}
+	env.Set(clusterUUID, v.id)
+
+	k8s, err := k8sapi.GetServerVersion()
+	if err != nil {
+		return err
+	}
+	// eg. linux/amd64
+	v.k8sArch = k8s.Platform
+	v.k8sVersion = k8s.GitVersion
+	env.Set(clusterArch, v.k8sArch)
+	env.Set(clusterVersion, v.k8sVersion)
+	v.nodeType, err = k8sapi.GetOSAndKernelVersion()
+	env.Set(nodeType, v.nodeType)
+	if err != nil {
+		return err
+	}
+	v.openebsVersion = openebsversion.GetVersionDetails()
+	env.Set(openEBSversion, v.openebsVersion)
+	return nil
+}
+
+// getVersion is a wrapper over fetchAndSetVersion
+func (v *versionSet) getVersion(override bool) error {
+	// If ENVs aren't set or the override is true, fetch the required
+	// values from the K8s APIserver
+	if _, present := env.Lookup(openEBSversion); !present || override {
+		if err := v.fetchAndSetVersion(); err != nil {
+			klog.Error(err.Error())
+			return err
+		}
+	}
+	// Fetch data from ENV
+	v.id = env.Get(clusterUUID)
+	v.k8sArch = env.Get(clusterArch)
+	v.k8sVersion = env.Get(clusterVersion)
+	v.nodeType = env.Get(nodeType)
+	v.openebsVersion = env.Get(openEBSversion)
+	v.installerType = env.Get(installerType)
+	return nil
+}
+
+// getUUIDbyNS returns the metadata.object.uid of a namespace in Kubernetes
+func getUUIDbyNS(namespace string) (string, error) {
+	ns := k8sapi.Namespace()
+	NSstruct, err := ns.Get(namespace, metav1.GetOptions{})
+	if err != nil {
+		return "", err
+	}
+	if NSstruct != nil {
+		return string(NSstruct.GetObjectMeta().GetUID()), nil
+	}
+	return "", nil
+}