Progress on json file index and search #960

pkg/api/search.go

@@ -35,6 +35,7 @@ func Search(c *middleware.Context) {
 		result.TagsOnly = true
 
 	} else {
+
 		query := search.Query{
 			Title:     query,
 			Tag:       tag,

pkg/models/dashboards.go

@@ -54,12 +54,10 @@ func (dash *Dashboard) GetTags() []string {
 	return b
 }
 
-// GetDashboardModel turns the command into the savable model
-func (cmd *SaveDashboardCommand) GetDashboardModel() *Dashboard {
+func NewDashboardFromJson(data map[string]interface{}) *Dashboard {
 	dash := &Dashboard{}
-	dash.Data = cmd.Dashboard
+	dash.Data = data
 	dash.Title = dash.Data["title"].(string)
-	dash.OrgId = cmd.OrgId
 	dash.UpdateSlug()
 
 	if dash.Data["id"] != nil {
@@ -75,6 +73,14 @@ func (cmd *SaveDashboardCommand) GetDashboardModel() *Dashboard {
 	return dash
 }
 
+// GetDashboardModel turns the command into the savable model
+func (cmd *SaveDashboardCommand) GetDashboardModel() *Dashboard {
+	dash := NewDashboardFromJson(cmd.Dashboard)
+	dash.OrgId = cmd.OrgId
+	dash.UpdateSlug()
+	return dash
+}
+
 // GetString a
 func (dash *Dashboard) GetString(prop string) string {
 	return dash.Data[prop].(string)

pkg/services/search/json_index.go

@@ -0,0 +1,114 @@
+package search
+
+import (
+	"encoding/json"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/grafana/grafana/pkg/log"
+	m "github.com/grafana/grafana/pkg/models"
+)
+
+type JsonDashIndex struct {
+	path    string
+	orgsIds []int64
+	items   []*JsonDashIndexItem
+}
+
+type JsonDashIndexItem struct {
+	Title      string
+	TitleLower string
+	Tags       []string
+	TagsCsv    string
+	FilePath   string
+}
+
+func NewJsonDashIndex(path string, orgIds string) *JsonDashIndex {
+	index := JsonDashIndex{}
+	index.path = path
+	// if orgIds != ""  || orgIds != "*" {
+	// }
+
+	index.updateIndex()
+	return &index
+}
+
+func (index *JsonDashIndex) Search(query *Query) ([]*m.DashboardSearchHit, error) {
+	results := make([]*m.DashboardSearchHit, 0)
+
+	for _, item := range index.items {
+		if strings.Contains(item.TitleLower, query.Title) {
+			results = append(results, &m.DashboardSearchHit{
+				Title: item.Title,
+				Tags:  item.Tags,
+				Slug:  item.FilePath,
+			})
+		}
+	}
+
+	return results, nil
+}
+
+func (index *JsonDashIndex) updateIndex() error {
+	log.Info("Updating JSON dashboard index, path: %v", index.path)
+
+	index.items = make([]*JsonDashIndexItem, 0)
+
+	visitor := func(path string, f os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if f.IsDir() {
+			return nil
+		}
+		if strings.HasSuffix(f.Name(), ".json") {
+			err = index.loadDashboardIntoCache(path)
+			if err != nil {
+				return err
+			}
+		}
+		return nil
+	}
+
+	if err := filepath.Walk(index.path, visitor); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (index *JsonDashIndex) loadDashboardIntoCache(filename string) error {
+	dash, err := loadDashboardFromFile(filename)
+	if err != nil {
+		return err
+	}
+
+	index.items = append(index.items, dash)
+	return nil
+}
+
+func loadDashboardFromFile(filename string) (*JsonDashIndexItem, error) {
+	reader, err := os.Open(filename)
+	if err != nil {
+		return nil, err
+	}
+	defer reader.Close()
+
+	jsonParser := json.NewDecoder(reader)
+	var data map[string]interface{}
+
+	if err := jsonParser.Decode(&data); err != nil {
+		return nil, err
+	}
+
+	dash := m.NewDashboardFromJson(data)
+
+	item := &JsonDashIndexItem{}
+	item.Title = dash.Title
+	item.TitleLower = strings.ToLower(item.Title)
+	item.Tags = dash.GetTags()
+	item.TagsCsv = strings.Join(item.Tags, ",")
+
+	return item, nil
+}

pkg/services/search/json_index_test.go

@@ -0,0 +1,35 @@
+package search
+
+import (
+	"testing"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func TestJsonDashIndex(t *testing.T) {
+
+	Convey("Given the json dash index", t, func() {
+		index := NewJsonDashIndex("../../../public/dashboards/", "*")
+
+		Convey("Should be able to update index", func() {
+			err := index.updateIndex()
+			So(err, ShouldBeNil)
+		})
+
+		Convey("Should be able to search index", func() {
+			res, err := index.Search(&Query{Title: "", Tag: ""})
+			So(err, ShouldBeNil)
+
+			So(len(res), ShouldEqual, 4)
+		})
+
+		Convey("Should be able to search index by title", func() {
+			res, err := index.Search(&Query{Title: "home", Tag: ""})
+			So(err, ShouldBeNil)
+
+			So(len(res), ShouldEqual, 1)
+			So(res[0].Title, ShouldEqual, "Home")
+		})
+
+	})
+}

pkg/services/search/jsonfiles.go

@@ -1,12 +0,0 @@
-package search
-
-var (
-	// settings
-	DashboardsJsonEnabled bool
-	DashboardsJsonPath    string
-	DashboardJsonOrgs     string
-)
-
-func initJsonFileIndex() {
-
-}

pkg/services/search/search.go

@@ -1,8 +1,11 @@
 package search
 
 import (
+	"path/filepath"
+
 	"github.com/grafana/grafana/pkg/bus"
 	m "github.com/grafana/grafana/pkg/models"
+	"github.com/grafana/grafana/pkg/setting"
 )
 
 type Query struct {
@@ -16,12 +19,28 @@ type Query struct {
 	Result []*m.DashboardSearchHit
 }
 
+var jsonDashIndex *JsonDashIndex
+
 func Init() {
 	bus.AddHandler("search", searchHandler)
-	initJsonFileIndex()
+
+	jsonIndexCfg, _ := setting.Cfg.GetSection("dashboards.json")
+	jsonIndexEnabled := jsonIndexCfg.Key("enabled").MustBool(false)
+
+	if jsonIndexEnabled {
+		jsonFilesPath := jsonIndexCfg.Key("path").String()
+		if !filepath.IsAbs(jsonFilesPath) {
+			jsonFilesPath = filepath.Join(setting.HomePath, jsonFilesPath)
+		}
+
+		orgIds := jsonIndexCfg.Key("org_ids").String()
+		jsonDashIndex = NewJsonDashIndex(jsonFilesPath, orgIds)
+	}
 }
 
 func searchHandler(query *Query) error {
+	hits := make([]*m.DashboardSearchHit, 0)
+
 	dashQuery := m.SearchDashboardsQuery{
 		Title:     query.Title,
 		Tag:       query.Tag,
@@ -35,11 +54,22 @@ func searchHandler(query *Query) error {
 		return err
 	}
 
-	if err := setIsStarredFlagOnSearchResults(query.UserId, query.Result); err != nil {
+	hits = append(hits, dashQuery.Result...)
+
+	if jsonDashIndex != nil {
+		jsonHits, err := jsonDashIndex.Search(query)
+		if err != nil {
+			return err
+		}
+
+		hits = append(hits, jsonHits...)
+	}
+
+	if err := setIsStarredFlagOnSearchResults(query.UserId, hits); err != nil {
 		return err
 	}
 
-	query.Result = dashQuery.Result
+	query.Result = hits
 	return nil
 }
 

pkg/setting/setting.go

@@ -258,11 +258,13 @@ func loadSpecifedConfigFile(configFile string) {
 
 			defaultSec, err := Cfg.GetSection(section.Name())
 			if err != nil {
-				log.Fatal(3, "Unknown config section %s defined in %s", section.Name(), configFile)
+				log.Error(3, "Unknown config section %s defined in %s", section.Name(), configFile)
+				continue
 			}
 			defaultKey, err := defaultSec.GetKey(key.Name())
 			if err != nil {
-				log.Fatal(3, "Unknown config key %s defined in section %s, in file", key.Name(), section.Name(), configFile)
+				log.Error(3, "Unknown config key %s defined in section %s, in file", key.Name(), section.Name(), configFile)
+				continue
 			}
 			defaultKey.SetValue(key.Value())
 		}