dashboards as cfg: create dashboard folders if missing

closes #10259

pkg/models/dashboards.go

@@ -139,8 +139,12 @@ func (dash *Dashboard) GetString(prop string, defaultValue string) string {
 
 // UpdateSlug updates the slug
 func (dash *Dashboard) UpdateSlug() {
-	title := strings.ToLower(dash.Data.Get("title").MustString())
-	dash.Slug = slug.Make(title)
+	title := dash.Data.Get("title").MustString()
+	dash.Slug = SlugifyTitle(title)
+}
+
+func SlugifyTitle(title string) string {
+	return slug.Make(strings.ToLower(title))
 }
 
 //

pkg/models/dashboards_test.go

@@ -16,6 +16,12 @@ func TestDashboardModel(t *testing.T) {
 		So(dashboard.Slug, ShouldEqual, "grafana-play-home")
 	})
 
+	Convey("Can slugify title", t, func() {
+		slug := SlugifyTitle("Grafana Play Home")
+
+		So(slug, ShouldEqual, "grafana-play-home")
+	})
+
 	Convey("Given a dashboard json", t, func() {
 		json := simplejson.New()
 		json.Set("title", "test dash")

pkg/services/provisioning/dashboards/file_reader.go

@@ -2,6 +2,7 @@ package dashboards
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"os"
 	"path/filepath"
@@ -15,21 +16,21 @@ import (
 	"github.com/grafana/grafana/pkg/components/simplejson"
 	"github.com/grafana/grafana/pkg/log"
 	"github.com/grafana/grafana/pkg/models"
-	gocache "github.com/patrickmn/go-cache"
 )
 
 var (
 	checkDiskForChangesInterval time.Duration = time.Second * 3
+
+	ErrFolderNameMissing error = errors.New("Folder name missing")
 )
 
 type fileReader struct {
-	Cfg            *DashboardsAsConfig
-	Path           string
-	FolderId       int64
-	log            log.Logger
-	dashboardRepo  dashboards.Repository
-	cache          *gocache.Cache
-	createWalkFunc func(fr *fileReader) filepath.WalkFunc
+	Cfg           *DashboardsAsConfig
+	Path          string
+	log           log.Logger
+	dashboardRepo dashboards.Repository
+	cache         *DashboardCache
+	createWalk    func(fr *fileReader, folderId int64) filepath.WalkFunc
 }
 
 func NewDashboardFileReader(cfg *DashboardsAsConfig, log log.Logger) (*fileReader, error) {
@@ -43,19 +44,19 @@ func NewDashboardFileReader(cfg *DashboardsAsConfig, log log.Logger) (*fileReade
 	}
 
 	return &fileReader{
-		Cfg:            cfg,
-		Path:           path,
-		log:            log,
-		dashboardRepo:  dashboards.GetRepository(),
-		cache:          gocache.New(5*time.Minute, 30*time.Minute),
-		createWalkFunc: createWalkFn,
+		Cfg:           cfg,
+		Path:          path,
+		log:           log,
+		dashboardRepo: dashboards.GetRepository(),
+		cache:         NewDashboardCache(),
+		createWalk:    createWalkFn,
 	}, nil
 }
 
 func (fr *fileReader) ReadAndListen(ctx context.Context) error {
 	ticker := time.NewTicker(checkDiskForChangesInterval)
 
-	if err := fr.walkFolder(); err != nil {
+	if err := fr.startWalkingDisk(); err != nil {
 		fr.log.Error("failed to search for dashboards", "error", err)
 	}
 
@@ -67,7 +68,9 @@ func (fr *fileReader) ReadAndListen(ctx context.Context) error {
 			if !running { // avoid walking the filesystem in parallel. incase fs is very slow.
 				running = true
 				go func() {
-					fr.walkFolder()
+					if err := fr.startWalkingDisk(); err != nil {
+						fr.log.Error("failed to search for dashboards", "error", err)
+					}
 					running = false
 				}()
 			}
@@ -77,17 +80,56 @@ func (fr *fileReader) ReadAndListen(ctx context.Context) error {
 	}
 }
 
-func (fr *fileReader) walkFolder() error {
+func (fr *fileReader) startWalkingDisk() error {
 	if _, err := os.Stat(fr.Path); err != nil {
 		if os.IsNotExist(err) {
 			return err
 		}
 	}
 
-	return filepath.Walk(fr.Path, fr.createWalkFunc(fr)) //omg this is so ugly :(
+	folderId, err := getOrCreateFolder(fr.Cfg, fr.dashboardRepo)
+	if err != nil && err != ErrFolderNameMissing {
+		return err
+	}
+
+	return filepath.Walk(fr.Path, fr.createWalk(fr, folderId))
 }
 
-func createWalkFn(fr *fileReader) filepath.WalkFunc {
+func getOrCreateFolder(cfg *DashboardsAsConfig, repo dashboards.Repository) (int64, error) {
+	if cfg.Folder == "" {
+		return 0, ErrFolderNameMissing
+	}
+
+	cmd := &models.GetDashboardQuery{Slug: models.SlugifyTitle(cfg.Folder), OrgId: cfg.OrgId}
+	err := bus.Dispatch(cmd)
+
+	if err != nil && err != models.ErrDashboardNotFound {
+		return 0, err
+	}
+
+	// dashboard folder not found. create one.
+	if err == models.ErrDashboardNotFound {
+		dash := &dashboards.SaveDashboardItem{}
+		dash.Dashboard = models.NewDashboard(cfg.Folder)
+		dash.Dashboard.IsFolder = true
+		dash.Overwrite = true
+		dash.OrgId = cfg.OrgId
+		dbDash, err := repo.SaveDashboard(dash)
+		if err != nil {
+			return 0, err
+		}
+
+		return dbDash.Id, nil
+	}
+
+	if !cmd.Result.IsFolder {
+		return 0, fmt.Errorf("Got invalid response. Expected folder, found dashboard")
+	}
+
+	return cmd.Result.Id, nil
+}
+
+func createWalkFn(fr *fileReader, folderId int64) filepath.WalkFunc {
 	return func(path string, fileInfo os.FileInfo, err error) error {
 		if err != nil {
 			return err
@@ -103,12 +145,12 @@ func createWalkFn(fr *fileReader) filepath.WalkFunc {
 			return nil
 		}
 
-		cachedDashboard, exist := fr.getCache(path)
+		cachedDashboard, exist := fr.cache.getCache(path)
 		if exist && cachedDashboard.UpdatedAt == fileInfo.ModTime() {
 			return nil
 		}
 
-		dash, err := fr.readDashboardFromFile(path)
+		dash, err := fr.readDashboardFromFile(path, folderId)
 		if err != nil {
 			fr.log.Error("failed to load dashboard from ", "file", path, "error", err)
 			return nil
@@ -143,7 +185,7 @@ func createWalkFn(fr *fileReader) filepath.WalkFunc {
 	}
 }
 
-func (fr *fileReader) readDashboardFromFile(path string) (*dashboards.SaveDashboardItem, error) {
+func (fr *fileReader) readDashboardFromFile(path string, folderId int64) (*dashboards.SaveDashboardItem, error) {
 	reader, err := os.Open(path)
 	if err != nil {
 		return nil, err
@@ -160,30 +202,12 @@ func (fr *fileReader) readDashboardFromFile(path string) (*dashboards.SaveDashbo
 		return nil, err
 	}
 
-	dash, err := createDashboardJson(data, stat.ModTime(), fr.Cfg)
+	dash, err := createDashboardJson(data, stat.ModTime(), fr.Cfg, folderId)
 	if err != nil {
 		return nil, err
 	}
 
-	fr.addDashboardCache(path, dash)
+	fr.cache.addDashboardCache(path, dash)
 
 	return dash, nil
 }
-
-func (fr *fileReader) addDashboardCache(key string, json *dashboards.SaveDashboardItem) {
-	fr.cache.Add(key, json, time.Minute*10)
-}
-
-func (fr *fileReader) getCache(key string) (*dashboards.SaveDashboardItem, bool) {
-	obj, exist := fr.cache.Get(key)
-	if !exist {
-		return nil, exist
-	}
-
-	dash, ok := obj.(*dashboards.SaveDashboardItem)
-	if !ok {
-		return nil, ok
-	}
-
-	return dash, ok
-}

pkg/services/provisioning/dashboards/file_reader_test.go

@@ -24,14 +24,14 @@ var (
 
 func TestDashboardFileReader(t *testing.T) {
 	Convey("Dashboard file reader", t, func() {
-		Convey("Reading dashboards from disk", func() {
+		bus.ClearBusHandlers()
+		fakeRepo = &fakeDashboardRepo{}
 
-			bus.ClearBusHandlers()
-			fakeRepo = &fakeDashboardRepo{}
+		bus.AddHandler("test", mockGetDashboardQuery)
+		dashboards.SetRepository(fakeRepo)
+		logger := log.New("test.logger")
 
-			bus.AddHandler("test", mockGetDashboardQuery)
-			dashboards.SetRepository(fakeRepo)
-			logger := log.New("test.logger")
+		Convey("Reading dashboards from disk", func() {
 
 			cfg := &DashboardsAsConfig{
 				Name:    "Default",
@@ -43,14 +43,27 @@ func TestDashboardFileReader(t *testing.T) {
 
 			Convey("Can read default dashboard", func() {
 				cfg.Options["folder"] = defaultDashboards
+				cfg.Folder = "Team A"
 
 				reader, err := NewDashboardFileReader(cfg, logger)
 				So(err, ShouldBeNil)
 
-				err = reader.walkFolder()
+				err = reader.startWalkingDisk()
 				So(err, ShouldBeNil)
 
-				So(len(fakeRepo.inserted), ShouldEqual, 2)
+				folders := 0
+				dashboards := 0
+
+				for _, i := range fakeRepo.inserted {
+					if i.Dashboard.IsFolder {
+						folders++
+					} else {
+						dashboards++
+					}
+				}
+
+				So(dashboards, ShouldEqual, 2)
+				So(folders, ShouldEqual, 1)
 			})
 
 			Convey("Should not update dashboards when db is newer", func() {
@@ -64,7 +77,7 @@ func TestDashboardFileReader(t *testing.T) {
 				reader, err := NewDashboardFileReader(cfg, logger)
 				So(err, ShouldBeNil)
 
-				err = reader.walkFolder()
+				err = reader.startWalkingDisk()
 				So(err, ShouldBeNil)
 
 				So(len(fakeRepo.inserted), ShouldEqual, 0)
@@ -83,7 +96,7 @@ func TestDashboardFileReader(t *testing.T) {
 				reader, err := NewDashboardFileReader(cfg, logger)
 				So(err, ShouldBeNil)
 
-				err = reader.walkFolder()
+				err = reader.startWalkingDisk()
 				So(err, ShouldBeNil)
 
 				So(len(fakeRepo.inserted), ShouldEqual, 1)
@@ -102,22 +115,52 @@ func TestDashboardFileReader(t *testing.T) {
 			})
 
 			Convey("Broken dashboards should not cause error", func() {
-				cfg := &DashboardsAsConfig{
-					Name:   "Default",
-					Type:   "file",
-					OrgId:  1,
-					Folder: "",
-					Options: map[string]interface{}{
-						"folder": brokenDashboards,
-					},
-				}
+				cfg.Options["folder"] = brokenDashboards
 
 				_, err := NewDashboardFileReader(cfg, logger)
 				So(err, ShouldBeNil)
 			})
 		})
 
-		Convey("Walking", func() {
+		Convey("Should not create new folder if folder name is missing", func() {
+			cfg := &DashboardsAsConfig{
+				Name:   "Default",
+				Type:   "file",
+				OrgId:  1,
+				Folder: "",
+				Options: map[string]interface{}{
+					"folder": defaultDashboards,
+				},
+			}
+
+			_, err := getOrCreateFolder(cfg, fakeRepo)
+			So(err, ShouldEqual, ErrFolderNameMissing)
+		})
+
+		Convey("can get or Create dashboard folder", func() {
+			cfg := &DashboardsAsConfig{
+				Name:   "Default",
+				Type:   "file",
+				OrgId:  1,
+				Folder: "TEAM A",
+				Options: map[string]interface{}{
+					"folder": defaultDashboards,
+				},
+			}
+
+			folderId, err := getOrCreateFolder(cfg, fakeRepo)
+			So(err, ShouldBeNil)
+			inserted := false
+			for _, d := range fakeRepo.inserted {
+				if d.Dashboard.IsFolder && d.Dashboard.Id == folderId {
+					inserted = true
+				}
+			}
+			So(len(fakeRepo.inserted), ShouldEqual, 1)
+			So(inserted, ShouldBeTrue)
+		})
+
+		Convey("Walking the folder with dashboards", func() {
 			cfg := &DashboardsAsConfig{
 				Name:   "Default",
 				Type:   "file",
@@ -132,12 +175,12 @@ func TestDashboardFileReader(t *testing.T) {
 			So(err, ShouldBeNil)
 
 			Convey("should skip dirs that starts with .", func() {
-				shouldSkip := reader.createWalkFunc(reader)("path", &FakeFileInfo{isDirectory: true, name: ".folder"}, nil)
+				shouldSkip := reader.createWalk(reader, 0)("path", &FakeFileInfo{isDirectory: true, name: ".folder"}, nil)
 				So(shouldSkip, ShouldEqual, filepath.SkipDir)
 			})
 
 			Convey("should keep walking if file is not .json", func() {
-				shouldSkip := reader.createWalkFunc(reader)("path", &FakeFileInfo{isDirectory: true, name: "folder"}, nil)
+				shouldSkip := reader.createWalk(reader, 0)("path", &FakeFileInfo{isDirectory: true, name: "folder"}, nil)
 				So(shouldSkip, ShouldBeNil)
 			})
 		})

pkg/services/provisioning/dashboards/types.go

@@ -7,6 +7,7 @@ import (
 	"github.com/grafana/grafana/pkg/services/dashboards"
 
 	"github.com/grafana/grafana/pkg/models"
+	gocache "github.com/patrickmn/go-cache"
 )
 
 type DashboardsAsConfig struct {
@@ -18,14 +19,42 @@ type DashboardsAsConfig struct {
 	Options  map[string]interface{} `json:"options" yaml:"options"`
 }
 
-func createDashboardJson(data *simplejson.Json, lastModified time.Time, cfg *DashboardsAsConfig) (*dashboards.SaveDashboardItem, error) {
+type DashboardCache struct {
+	internalCache *gocache.Cache
+}
+
+func NewDashboardCache() *DashboardCache {
+	return &DashboardCache{internalCache: gocache.New(5*time.Minute, 30*time.Minute)}
+}
+
+func (fr *DashboardCache) addDashboardCache(key string, json *dashboards.SaveDashboardItem) {
+	fr.internalCache.Add(key, json, time.Minute*10)
+}
+
+func (fr *DashboardCache) getCache(key string) (*dashboards.SaveDashboardItem, bool) {
+	obj, exist := fr.internalCache.Get(key)
+	if !exist {
+		return nil, exist
+	}
+
+	dash, ok := obj.(*dashboards.SaveDashboardItem)
+	if !ok {
+		return nil, ok
+	}
 
+	return dash, ok
+}
+
+func createDashboardJson(data *simplejson.Json, lastModified time.Time, cfg *DashboardsAsConfig, folderId int64) (*dashboards.SaveDashboardItem, error) {
 	dash := &dashboards.SaveDashboardItem{}
 	dash.Dashboard = models.NewDashboardFromJson(data)
 	dash.UpdatedAt = lastModified
 	dash.Overwrite = true
 	dash.OrgId = cfg.OrgId
-	dash.Dashboard.Data.Set("editable", cfg.Editable)
+	dash.Dashboard.FolderId = folderId
+	if !cfg.Editable {
+		dash.Dashboard.Data.Set("editable", cfg.Editable)
+	}
 
 	if dash.Dashboard.Title == "" {
 		return nil, models.ErrDashboardTitleEmpty