Merge pull request #12071 from grafana/12056_usage_stats

Additional anonymous usage stats

pkg/metrics/metrics.go

@@ -332,6 +332,8 @@ func updateTotalStats() {
 	M_StatTotal_Orgs.Set(float64(statsQuery.Result.Orgs))
 }
 
+var usageStatsURL = "https://stats.grafana.org/grafana-usage-report"
+
 func sendUsageStats() {
 	if !setting.ReportingEnabled {
 		return
@@ -366,6 +368,12 @@ func sendUsageStats() {
 	metrics["stats.active_users.count"] = statsQuery.Result.ActiveUsers
 	metrics["stats.datasources.count"] = statsQuery.Result.Datasources
 	metrics["stats.stars.count"] = statsQuery.Result.Stars
+	metrics["stats.folders.count"] = statsQuery.Result.Folders
+	metrics["stats.dashboard_permissions.count"] = statsQuery.Result.DashboardPermissions
+	metrics["stats.folder_permissions.count"] = statsQuery.Result.FolderPermissions
+	metrics["stats.provisioned_dashboards.count"] = statsQuery.Result.ProvisionedDashboards
+	metrics["stats.snapshots.count"] = statsQuery.Result.Snapshots
+	metrics["stats.teams.count"] = statsQuery.Result.Teams
 
 	dsStats := models.GetDataSourceStatsQuery{}
 	if err := bus.Dispatch(&dsStats); err != nil {
@@ -386,9 +394,38 @@ func sendUsageStats() {
 	}
 	metrics["stats.ds.other.count"] = dsOtherCount
 
+	dsAccessStats := models.GetDataSourceAccessStatsQuery{}
+	if err := bus.Dispatch(&dsAccessStats); err != nil {
+		metricsLogger.Error("Failed to get datasource access stats", "error", err)
+		return
+	}
+
+	// send access counters for each data source
+	// but ignore any custom data sources
+	// as sending that name could be sensitive information
+	dsAccessOtherCount := make(map[string]int64)
+	for _, dsAccessStat := range dsAccessStats.Result {
+		if dsAccessStat.Access == "" {
+			continue
+		}
+
+		access := strings.ToLower(dsAccessStat.Access)
+
+		if models.IsKnownDataSourcePlugin(dsAccessStat.Type) {
+			metrics["stats.ds_access."+dsAccessStat.Type+"."+access+".count"] = dsAccessStat.Count
+		} else {
+			old := dsAccessOtherCount[access]
+			dsAccessOtherCount[access] = old + dsAccessStat.Count
+		}
+	}
+
+	for access, count := range dsAccessOtherCount {
+		metrics["stats.ds_access.other."+access+".count"] = count
+	}
+
 	out, _ := json.MarshalIndent(report, "", " ")
 	data := bytes.NewBuffer(out)
 
 	client := http.Client{Timeout: 5 * time.Second}
-	go client.Post("https://stats.grafana.org/grafana-usage-report", "application/json", data)
+	go client.Post(usageStatsURL, "application/json", data)
 }

pkg/metrics/metrics_test.go

@@ -0,0 +1,222 @@
+package metrics
+
+import (
+	"bytes"
+	"io/ioutil"
+	"runtime"
+	"sync"
+	"testing"
+	"time"
+
+	"net/http"
+	"net/http/httptest"
+
+	"github.com/grafana/grafana/pkg/bus"
+	"github.com/grafana/grafana/pkg/components/simplejson"
+	"github.com/grafana/grafana/pkg/models"
+	"github.com/grafana/grafana/pkg/plugins"
+	"github.com/grafana/grafana/pkg/setting"
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func TestMetrics(t *testing.T) {
+	Convey("Test send usage stats", t, func() {
+		var getSystemStatsQuery *models.GetSystemStatsQuery
+		bus.AddHandler("test", func(query *models.GetSystemStatsQuery) error {
+			query.Result = &models.SystemStats{
+				Dashboards:            1,
+				Datasources:           2,
+				Users:                 3,
+				ActiveUsers:           4,
+				Orgs:                  5,
+				Playlists:             6,
+				Alerts:                7,
+				Stars:                 8,
+				Folders:               9,
+				DashboardPermissions:  10,
+				FolderPermissions:     11,
+				ProvisionedDashboards: 12,
+				Snapshots:             13,
+				Teams:                 14,
+			}
+			getSystemStatsQuery = query
+			return nil
+		})
+
+		var getDataSourceStatsQuery *models.GetDataSourceStatsQuery
+		bus.AddHandler("test", func(query *models.GetDataSourceStatsQuery) error {
+			query.Result = []*models.DataSourceStats{
+				{
+					Type:  models.DS_ES,
+					Count: 9,
+				},
+				{
+					Type:  models.DS_PROMETHEUS,
+					Count: 10,
+				},
+				{
+					Type:  "unknown_ds",
+					Count: 11,
+				},
+				{
+					Type:  "unknown_ds2",
+					Count: 12,
+				},
+			}
+			getDataSourceStatsQuery = query
+			return nil
+		})
+
+		var getDataSourceAccessStatsQuery *models.GetDataSourceAccessStatsQuery
+		bus.AddHandler("test", func(query *models.GetDataSourceAccessStatsQuery) error {
+			query.Result = []*models.DataSourceAccessStats{
+				{
+					Type:   models.DS_ES,
+					Access: "direct",
+					Count:  1,
+				},
+				{
+					Type:   models.DS_ES,
+					Access: "proxy",
+					Count:  2,
+				},
+				{
+					Type:   models.DS_PROMETHEUS,
+					Access: "proxy",
+					Count:  3,
+				},
+				{
+					Type:   "unknown_ds",
+					Access: "proxy",
+					Count:  4,
+				},
+				{
+					Type:   "unknown_ds2",
+					Access: "",
+					Count:  5,
+				},
+				{
+					Type:   "unknown_ds3",
+					Access: "direct",
+					Count:  6,
+				},
+				{
+					Type:   "unknown_ds4",
+					Access: "direct",
+					Count:  7,
+				},
+				{
+					Type:   "unknown_ds5",
+					Access: "proxy",
+					Count:  8,
+				},
+			}
+			getDataSourceAccessStatsQuery = query
+			return nil
+		})
+
+		var wg sync.WaitGroup
+		var responseBuffer *bytes.Buffer
+		var req *http.Request
+		ts := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
+			req = r
+			buf, err := ioutil.ReadAll(r.Body)
+			if err != nil {
+				t.Fatalf("Failed to read response body, err=%v", err)
+			}
+			responseBuffer = bytes.NewBuffer(buf)
+			wg.Done()
+		}))
+		usageStatsURL = ts.URL
+
+		sendUsageStats()
+
+		Convey("Given reporting not enabled and sending usage stats", func() {
+			setting.ReportingEnabled = false
+			sendUsageStats()
+
+			Convey("Should not gather stats or call http endpoint", func() {
+				So(getSystemStatsQuery, ShouldBeNil)
+				So(getDataSourceStatsQuery, ShouldBeNil)
+				So(getDataSourceAccessStatsQuery, ShouldBeNil)
+				So(req, ShouldBeNil)
+			})
+		})
+
+		Convey("Given reporting enabled and sending usage stats", func() {
+			setting.ReportingEnabled = true
+			setting.BuildVersion = "5.0.0"
+			wg.Add(1)
+			sendUsageStats()
+
+			Convey("Should gather stats and call http endpoint", func() {
+				if waitTimeout(&wg, 2*time.Second) {
+					t.Fatalf("Timed out waiting for http request")
+				}
+
+				So(getSystemStatsQuery, ShouldNotBeNil)
+				So(getDataSourceStatsQuery, ShouldNotBeNil)
+				So(getDataSourceAccessStatsQuery, ShouldNotBeNil)
+				So(req, ShouldNotBeNil)
+				So(req.Method, ShouldEqual, http.MethodPost)
+				So(req.Header.Get("Content-Type"), ShouldEqual, "application/json")
+
+				So(responseBuffer, ShouldNotBeNil)
+
+				j, err := simplejson.NewFromReader(responseBuffer)
+				So(err, ShouldBeNil)
+
+				So(j.Get("version").MustString(), ShouldEqual, "5_0_0")
+				So(j.Get("os").MustString(), ShouldEqual, runtime.GOOS)
+				So(j.Get("arch").MustString(), ShouldEqual, runtime.GOARCH)
+
+				metrics := j.Get("metrics")
+				So(metrics.Get("stats.dashboards.count").MustInt(), ShouldEqual, getSystemStatsQuery.Result.Dashboards)
+				So(metrics.Get("stats.users.count").MustInt(), ShouldEqual, getSystemStatsQuery.Result.Users)
+				So(metrics.Get("stats.orgs.count").MustInt(), ShouldEqual, getSystemStatsQuery.Result.Orgs)
+				So(metrics.Get("stats.playlist.count").MustInt(), ShouldEqual, getSystemStatsQuery.Result.Playlists)
+				So(metrics.Get("stats.plugins.apps.count").MustInt(), ShouldEqual, len(plugins.Apps))
+				So(metrics.Get("stats.plugins.panels.count").MustInt(), ShouldEqual, len(plugins.Panels))
+				So(metrics.Get("stats.plugins.datasources.count").MustInt(), ShouldEqual, len(plugins.DataSources))
+				So(metrics.Get("stats.alerts.count").MustInt(), ShouldEqual, getSystemStatsQuery.Result.Alerts)
+				So(metrics.Get("stats.active_users.count").MustInt(), ShouldEqual, getSystemStatsQuery.Result.ActiveUsers)
+				So(metrics.Get("stats.datasources.count").MustInt(), ShouldEqual, getSystemStatsQuery.Result.Datasources)
+				So(metrics.Get("stats.stars.count").MustInt(), ShouldEqual, getSystemStatsQuery.Result.Stars)
+				So(metrics.Get("stats.folders.count").MustInt(), ShouldEqual, getSystemStatsQuery.Result.Folders)
+				So(metrics.Get("stats.dashboard_permissions.count").MustInt(), ShouldEqual, getSystemStatsQuery.Result.DashboardPermissions)
+				So(metrics.Get("stats.folder_permissions.count").MustInt(), ShouldEqual, getSystemStatsQuery.Result.FolderPermissions)
+				So(metrics.Get("stats.provisioned_dashboards.count").MustInt(), ShouldEqual, getSystemStatsQuery.Result.ProvisionedDashboards)
+				So(metrics.Get("stats.snapshots.count").MustInt(), ShouldEqual, getSystemStatsQuery.Result.Snapshots)
+				So(metrics.Get("stats.teams.count").MustInt(), ShouldEqual, getSystemStatsQuery.Result.Teams)
+
+				So(metrics.Get("stats.ds."+models.DS_ES+".count").MustInt(), ShouldEqual, 9)
+				So(metrics.Get("stats.ds."+models.DS_PROMETHEUS+".count").MustInt(), ShouldEqual, 10)
+				So(metrics.Get("stats.ds.other.count").MustInt(), ShouldEqual, 11+12)
+
+				So(metrics.Get("stats.ds_access."+models.DS_ES+".direct.count").MustInt(), ShouldEqual, 1)
+				So(metrics.Get("stats.ds_access."+models.DS_ES+".proxy.count").MustInt(), ShouldEqual, 2)
+				So(metrics.Get("stats.ds_access."+models.DS_PROMETHEUS+".proxy.count").MustInt(), ShouldEqual, 3)
+				So(metrics.Get("stats.ds_access.other.direct.count").MustInt(), ShouldEqual, 6+7)
+				So(metrics.Get("stats.ds_access.other.proxy.count").MustInt(), ShouldEqual, 4+8)
+			})
+		})
+
+		Reset(func() {
+			ts.Close()
+		})
+	})
+}
+
+func waitTimeout(wg *sync.WaitGroup, timeout time.Duration) bool {
+	c := make(chan struct{})
+	go func() {
+		defer close(c)
+		wg.Wait()
+	}()
+	select {
+	case <-c:
+		return false // completed normally
+	case <-time.After(timeout):
+		return true // timed out
+	}
+}

pkg/models/stats.go

@@ -1,14 +1,20 @@
 package models
 
 type SystemStats struct {
-	Dashboards  int64
-	Datasources int64
-	Users       int64
-	ActiveUsers int64
-	Orgs        int64
-	Playlists   int64
-	Alerts      int64
-	Stars       int64
+	Dashboards            int64
+	Datasources           int64
+	Users                 int64
+	ActiveUsers           int64
+	Orgs                  int64
+	Playlists             int64
+	Alerts                int64
+	Stars                 int64
+	Snapshots             int64
+	Teams                 int64
+	DashboardPermissions  int64
+	FolderPermissions     int64
+	Folders               int64
+	ProvisionedDashboards int64
 }
 
 type DataSourceStats struct {
@@ -24,6 +30,16 @@ type GetDataSourceStatsQuery struct {
 	Result []*DataSourceStats
 }
 
+type DataSourceAccessStats struct {
+	Type   string
+	Access string
+	Count  int64
+}
+
+type GetDataSourceAccessStatsQuery struct {
+	Result []*DataSourceAccessStats
+}
+
 type AdminStats struct {
 	Users       int `json:"users"`
 	Orgs        int `json:"orgs"`
@@ -40,3 +56,11 @@ type AdminStats struct {
 type GetAdminStatsQuery struct {
 	Result *AdminStats
 }
+
+type SystemUserCountStats struct {
+	Count int64
+}
+
+type GetSystemUserCountStatsQuery struct {
+	Result *SystemUserCountStats
+}

pkg/services/sqlstore/sqlstore.go

@@ -86,13 +86,13 @@ func (ss *SqlStore) Init() error {
 }
 
 func (ss *SqlStore) ensureAdminUser() error {
-	statsQuery := m.GetSystemStatsQuery{}
+	systemUserCountQuery := m.GetSystemUserCountStatsQuery{}
 
-	if err := bus.Dispatch(&statsQuery); err != nil {
+	if err := bus.Dispatch(&systemUserCountQuery); err != nil {
 		fmt.Errorf("Could not determine if admin user exists: %v", err)
 	}
 
-	if statsQuery.Result.Users > 0 {
+	if systemUserCountQuery.Result.Count > 0 {
 		return nil
 	}
 

pkg/services/sqlstore/stats.go

@@ -10,7 +10,9 @@ import (
 func init() {
 	bus.AddHandler("sql", GetSystemStats)
 	bus.AddHandler("sql", GetDataSourceStats)
+	bus.AddHandler("sql", GetDataSourceAccessStats)
 	bus.AddHandler("sql", GetAdminStats)
+	bus.AddHandler("sql", GetSystemUserCountStats)
 }
 
 var activeUserTimeLimit = time.Hour * 24 * 30
@@ -22,43 +24,51 @@ func GetDataSourceStats(query *m.GetDataSourceStatsQuery) error {
 	return err
 }
 
+func GetDataSourceAccessStats(query *m.GetDataSourceAccessStatsQuery) error {
+	var rawSql = `SELECT COUNT(*) as count, type, access FROM data_source GROUP BY type, access`
+	query.Result = make([]*m.DataSourceAccessStats, 0)
+	err := x.SQL(rawSql).Find(&query.Result)
+	return err
+}
+
 func GetSystemStats(query *m.GetSystemStatsQuery) error {
-	var rawSql = `SELECT
-			(
-				SELECT COUNT(*)
-		FROM ` + dialect.Quote("user") + `
-	  ) AS users,
-			(
-				SELECT COUNT(*)
-		FROM ` + dialect.Quote("org") + `
-	  ) AS orgs,
-	  (
-		SELECT COUNT(*)
-		FROM ` + dialect.Quote("dashboard") + `
-	  ) AS dashboards,
-		(
-		SELECT COUNT(*)
-		FROM ` + dialect.Quote("data_source") + `
-	  ) AS datasources,
-	  (
-		SELECT COUNT(*) FROM ` + dialect.Quote("star") + `
-	  ) AS stars,
-	  (
-		SELECT COUNT(*)
-		FROM ` + dialect.Quote("playlist") + `
-	  ) AS playlists,
-	  (
-		SELECT COUNT(*)
-		FROM ` + dialect.Quote("alert") + `
-	  ) AS alerts,
-			(
-				SELECT COUNT(*) FROM ` + dialect.Quote("user") + ` where last_seen_at > ?
-	  ) as active_users
-			`
+	sb := &SqlBuilder{}
+	sb.Write("SELECT ")
+	sb.Write(`(SELECT COUNT(*) FROM ` + dialect.Quote("user") + `) AS users,`)
+	sb.Write(`(SELECT COUNT(*) FROM ` + dialect.Quote("org") + `) AS orgs,`)
+	sb.Write(`(SELECT COUNT(*) FROM ` + dialect.Quote("dashboard") + `) AS dashboards,`)
+	sb.Write(`(SELECT COUNT(*) FROM ` + dialect.Quote("data_source") + `) AS datasources,`)
+	sb.Write(`(SELECT COUNT(*) FROM ` + dialect.Quote("star") + `) AS stars,`)
+	sb.Write(`(SELECT COUNT(*) FROM ` + dialect.Quote("playlist") + `) AS playlists,`)
+	sb.Write(`(SELECT COUNT(*) FROM ` + dialect.Quote("alert") + `) AS alerts,`)
 
 	activeUserDeadlineDate := time.Now().Add(-activeUserTimeLimit)
+	sb.Write(`(SELECT COUNT(*) FROM `+dialect.Quote("user")+` where last_seen_at > ?) AS active_users,`, activeUserDeadlineDate)
+
+	sb.Write(`(SELECT COUNT(id) FROM `+dialect.Quote("dashboard")+` where is_folder = ?) AS folders,`, dialect.BooleanStr(true))
+
+	sb.Write(`(
+		SELECT COUNT(acl.id)
+		FROM `+dialect.Quote("dashboard_acl")+` as acl
+			inner join `+dialect.Quote("dashboard")+` as d
+			on d.id = acl.dashboard_id
+		WHERE d.is_folder = ?
+	) AS dashboard_permissions,`, dialect.BooleanStr(false))
+
+	sb.Write(`(
+		SELECT COUNT(acl.id)
+		FROM `+dialect.Quote("dashboard_acl")+` as acl
+			inner join `+dialect.Quote("dashboard")+` as d
+			on d.id = acl.dashboard_id
+		WHERE d.is_folder = ?
+	) AS folder_permissions,`, dialect.BooleanStr(true))
+
+	sb.Write(`(SELECT COUNT(id) FROM ` + dialect.Quote("dashboard_provisioning") + `) AS provisioned_dashboards,`)
+	sb.Write(`(SELECT COUNT(id) FROM ` + dialect.Quote("dashboard_snapshot") + `) AS snapshots,`)
+	sb.Write(`(SELECT COUNT(id) FROM ` + dialect.Quote("team") + `) AS teams`)
+
 	var stats m.SystemStats
-	_, err := x.SQL(rawSql, activeUserDeadlineDate).Get(&stats)
+	_, err := x.SQL(sb.GetSqlString(), sb.params...).Get(&stats)
 	if err != nil {
 		return err
 	}
@@ -122,3 +132,16 @@ func GetAdminStats(query *m.GetAdminStatsQuery) error {
 	query.Result = &stats
 	return err
 }
+
+func GetSystemUserCountStats(query *m.GetSystemUserCountStatsQuery) error {
+	var rawSql = `SELECT COUNT(id) AS Count FROM ` + dialect.Quote("user")
+	var stats m.SystemUserCountStats
+	_, err := x.SQL(rawSql).Get(&stats)
+	if err != nil {
+		return err
+	}
+
+	query.Result = &stats
+
+	return err
+}

pkg/services/sqlstore/stats_test.go

@@ -0,0 +1,39 @@
+package sqlstore
+
+import (
+	"testing"
+
+	m "github.com/grafana/grafana/pkg/models"
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func TestStatsDataAccess(t *testing.T) {
+
+	Convey("Testing Stats Data Access", t, func() {
+		InitTestDB(t)
+
+		Convey("Get system stats should not results in error", func() {
+			query := m.GetSystemStatsQuery{}
+			err := GetSystemStats(&query)
+			So(err, ShouldBeNil)
+		})
+
+		Convey("Get system user count stats should not results in error", func() {
+			query := m.GetSystemUserCountStatsQuery{}
+			err := GetSystemUserCountStats(&query)
+			So(err, ShouldBeNil)
+		})
+
+		Convey("Get datasource stats should not results in error", func() {
+			query := m.GetDataSourceStatsQuery{}
+			err := GetDataSourceStats(&query)
+			So(err, ShouldBeNil)
+		})
+
+		Convey("Get datasource access stats should not results in error", func() {
+			query := m.GetDataSourceAccessStatsQuery{}
+			err := GetDataSourceAccessStats(&query)
+			So(err, ShouldBeNil)
+		})
+	})
+}