Merge branch 'master' of github.com:grafana/grafana

CHANGELOG.md

@@ -14,6 +14,8 @@
 * **OAuth**: Add support for generic oauth, closes [#4718](https://github.com/grafana/grafana/pull/4718)
 * **Cloudwatch**: Add support to expand multi select template variable, closes [#5003](https://github.com/grafana/grafana/pull/5003)
 * **Graph Panel**: Now supports flexible lower/upper bounds on Y-Max and Y-Min, PR [#5720](https://github.com/grafana/grafana/pull/5720)
+* **Background Tasks**: Now support automatic purging of old snapshots, closes [#4087](https://github.com/grafana/grafana/issues/4087)
+* **Background Tasks**: Now support automatic purging of old rendered images, closes [#2172](https://github.com/grafana/grafana/issues/2172)
 
 ### Breaking changes
 * **SystemD**: Change systemd description, closes [#5971](https://github.com/grafana/grafana/pull/5971)

conf/defaults.ini

@@ -161,6 +161,12 @@ external_enabled = true
 external_snapshot_url = https://snapshots-origin.raintank.io
 external_snapshot_name = Publish to snapshot.raintank.io
 
+# remove expired snapshot
+snapshot_remove_expired = true
+
+# remove snapshots after 90 days
+snapshot_TTL_days = 90
+
 #################################### Users ####################################
 [users]
 # disable user signup / registration
@@ -267,6 +273,9 @@ from_address = admin@grafana.localhost
 welcome_email_on_sign_up = false
 templates_pattern = emails/*.html
 
+[tmp.files]
+rendered_image_ttl_days = 14
+
 #################################### Logging ##########################
 [log]
 # Either "console", "file", "syslog". Default is console and  file

conf/sample.ini

@@ -149,6 +149,12 @@ check_for_updates = true
 ;external_snapshot_url = https://snapshots-origin.raintank.io
 ;external_snapshot_name = Publish to snapshot.raintank.io
 
+# remove expired snapshot
+;snapshot_remove_expired = true
+
+# remove snapshots after 90 days
+;snapshot_TTL_days = 90
+
 #################################### Users ####################################
 [users]
 # disable user signup / registration

docs/sources/installation/configuration.md

@@ -525,3 +525,9 @@ Set root url to a Grafana instance where you want to publish external snapshots
 
 ### external_snapshot_name
 Set name for external snapshot button. Defaults to `Publish to snapshot.raintank.io`
+
+### remove expired snapshot
+Enabled to automatically remove expired snapshots
+
+### remove snapshots after 90 days
+Time to live for snapshots.

pkg/api/render.go

@@ -14,7 +14,7 @@ func RenderToPng(c *middleware.Context) {
 	queryParams := fmt.Sprintf("?%s", c.Req.URL.RawQuery)
 
 	renderOpts := &renderer.RenderOpts{
-		Url:     c.Params("*") + queryParams,
+		Path:    c.Params("*") + queryParams,
 		Width:   queryReader.Get("width", "800"),
 		Height:  queryReader.Get("height", "400"),
 		OrgId:   c.OrgId,

pkg/cmd/grafana-server/main.go

@@ -17,6 +17,7 @@ import (
 	"github.com/grafana/grafana/pkg/metrics"
 	"github.com/grafana/grafana/pkg/plugins"
 	alertingInit "github.com/grafana/grafana/pkg/services/alerting/init"
+	"github.com/grafana/grafana/pkg/services/backgroundtasks"
 	"github.com/grafana/grafana/pkg/services/eventpublisher"
 	"github.com/grafana/grafana/pkg/services/notifications"
 	"github.com/grafana/grafana/pkg/services/search"
@@ -62,13 +63,13 @@ func main() {
 	writePIDFile()
 	initRuntime()
 	metrics.Init()
-
 	search.Init()
 	login.Init()
 	social.NewOAuthService()
 	eventpublisher.Init()
 	plugins.Init()
 	alertingInit.Init()
+	backgroundtasks.Init()
 
 	if err := notifications.Init(); err != nil {
 		log.Fatal(3, "Notification service failed to initialize", err)

pkg/components/renderer/renderer.go

@@ -18,7 +18,7 @@ import (
 )
 
 type RenderOpts struct {
-	Url     string
+	Path    string
 	Width   string
 	Height  string
 	Timeout string
@@ -28,14 +28,14 @@ type RenderOpts struct {
 var rendererLog log.Logger = log.New("png-renderer")
 
 func RenderToPng(params *RenderOpts) (string, error) {
-	rendererLog.Info("Rendering", "url", params.Url)
+	rendererLog.Info("Rendering", "path", params.Path)
 
 	var executable = "phantomjs"
 	if runtime.GOOS == "windows" {
 		executable = executable + ".exe"
 	}
 
-	params.Url = fmt.Sprintf("%s://localhost:%s/%s", setting.Protocol, setting.HttpPort, params.Url)
+	url := fmt.Sprintf("%s://localhost:%s/%s", setting.Protocol, setting.HttpPort, params.Path)
 
 	binPath, _ := filepath.Abs(filepath.Join(setting.PhantomDir, executable))
 	scriptPath, _ := filepath.Abs(filepath.Join(setting.PhantomDir, "render.js"))
@@ -48,7 +48,7 @@ func RenderToPng(params *RenderOpts) (string, error) {
 	cmdArgs := []string{
 		"--ignore-ssl-errors=true",
 		scriptPath,
-		"url=" + params.Url,
+		"url=" + url,
 		"width=" + params.Width,
 		"height=" + params.Height,
 		"png=" + pngPath,

pkg/models/timer.go

@@ -0,0 +1,7 @@
+package models
+
+import "time"
+
+type HourCommand struct {
+	Time time.Time
+}

pkg/plugins/update_checker.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/grafana/grafana/pkg/log"
 	"github.com/grafana/grafana/pkg/setting"
+	"github.com/hashicorp/go-version"
 )
 
 var (
@@ -85,7 +86,15 @@ func checkForUpdates() {
 		for _, gplug := range gNetPlugins {
 			if gplug.Slug == plug.Id {
 				plug.GrafanaNetVersion = gplug.Version
-				plug.GrafanaNetHasUpdate = plug.Info.Version != plug.GrafanaNetVersion
+
+				plugVersion, err1 := version.NewVersion(plug.Info.Version)
+				gplugVersion, err2 := version.NewVersion(gplug.Version)
+
+				if err1 != nil || err2 != nil {
+					plug.GrafanaNetHasUpdate = plug.Info.Version != plug.GrafanaNetVersion
+				} else {
+					plug.GrafanaNetHasUpdate = plugVersion.LessThan(gplugVersion)
+				}
 			}
 		}
 	}
@@ -117,4 +126,11 @@ func checkForUpdates() {
 		GrafanaLatestVersion = githubLatest.Stable
 		GrafanaHasUpdate = githubLatest.Stable != setting.BuildVersion
 	}
+
+	currVersion, err1 := version.NewVersion(setting.BuildVersion)
+	latestVersion, err2 := version.NewVersion(GrafanaLatestVersion)
+
+	if err1 == nil && err2 == nil {
+		GrafanaHasUpdate = currVersion.LessThan(latestVersion)
+	}
 }

pkg/services/alerting/conditions/query.go

@@ -184,5 +184,6 @@ func validateToValue(to string) error {
 		}
 	}
 
-	return fmt.Errorf("cannot parse to value %s", to)
+	_, err := time.ParseDuration(to)
+	return err
 }

pkg/services/alerting/engine.go

@@ -93,14 +93,18 @@ func (e *Engine) executeJob(job *Job) {
 }
 
 func (e *Engine) resultDispatcher() {
+	for result := range e.resultQueue {
+		go e.handleResponse(result)
+	}
+}
+
+func (e *Engine) handleResponse(result *EvalContext) {
 	defer func() {
 		if err := recover(); err != nil {
 			e.log.Error("Panic in resultDispatcher", "error", err, "stack", log.Stack(1))
 		}
 	}()
 
-	for result := range e.resultQueue {
-		e.log.Debug("Alert Rule Result", "ruleId", result.Rule.Id, "firing", result.Firing)
-		e.resultHandler.Handle(result)
-	}
+	e.log.Debug("Alert Rule Result", "ruleId", result.Rule.Id, "firing", result.Firing)
+	e.resultHandler.Handle(result)
 }

pkg/services/alerting/eval_context.go

@@ -71,7 +71,7 @@ func (c *EvalContext) GetNotificationTitle() string {
 	return "[" + c.GetStateModel().Text + "] " + c.Rule.Name
 }
 
-func (c *EvalContext) getDashboardSlug() (string, error) {
+func (c *EvalContext) GetDashboardSlug() (string, error) {
 	if c.dashboardSlug != "" {
 		return c.dashboardSlug, nil
 	}
@@ -86,7 +86,7 @@ func (c *EvalContext) getDashboardSlug() (string, error) {
 }
 
 func (c *EvalContext) GetRuleUrl() (string, error) {
-	if slug, err := c.getDashboardSlug(); err != nil {
+	if slug, err := c.GetDashboardSlug(); err != nil {
 		return "", err
 	} else {
 		ruleUrl := fmt.Sprintf("%sdashboard/db/%s?fullscreen&edit&tab=alert&panelId=%d", setting.AppUrl, slug, c.Rule.PanelId)
@@ -94,15 +94,6 @@ func (c *EvalContext) GetRuleUrl() (string, error) {
 	}
 }
 
-func (c *EvalContext) GetImageUrl() (string, error) {
-	if slug, err := c.getDashboardSlug(); err != nil {
-		return "", err
-	} else {
-		ruleUrl := fmt.Sprintf("%sdashboard-solo/db/%s?&panelId=%d", setting.AppUrl, slug, c.Rule.PanelId)
-		return ruleUrl, nil
-	}
-}
-
 func NewEvalContext(rule *Rule) *EvalContext {
 	return &EvalContext{
 		StartTime:   time.Now(),

pkg/services/alerting/eval_handler.go

@@ -20,7 +20,7 @@ type DefaultEvalHandler struct {
 func NewEvalHandler() *DefaultEvalHandler {
 	return &DefaultEvalHandler{
 		log:             log.New("alerting.evalHandler"),
-		alertJobTimeout: time.Second * 10,
+		alertJobTimeout: time.Second * 15,
 	}
 }
 

pkg/services/alerting/notifier.go

@@ -2,6 +2,7 @@ package alerting
 
 import (
 	"errors"
+	"fmt"
 
 	"github.com/grafana/grafana/pkg/bus"
 	"github.com/grafana/grafana/pkg/components/imguploader"
@@ -60,22 +61,22 @@ func (n *RootNotifier) sendNotifications(notifiers []Notifier, context *EvalCont
 	}
 }
 
-func (n *RootNotifier) uploadImage(context *EvalContext) error {
+func (n *RootNotifier) uploadImage(context *EvalContext) (err error) {
 	uploader, _ := imguploader.NewImageUploader()
 
-	imageUrl, err := context.GetImageUrl()
-	if err != nil {
-		return err
-	}
-
 	renderOpts := &renderer.RenderOpts{
-		Url:     imageUrl,
 		Width:   "800",
 		Height:  "400",
 		Timeout: "30",
 		OrgId:   context.Rule.OrgId,
 	}
 
+	if slug, err := context.GetDashboardSlug(); err != nil {
+		return err
+	} else {
+		renderOpts.Path = fmt.Sprintf("dashboard-solo/db/%s?&panelId=%d", slug, context.Rule.PanelId)
+	}
+
 	if imagePath, err := renderer.RenderToPng(renderOpts); err != nil {
 		return err
 	} else {

pkg/services/alerting/notifiers/webhook.go

@@ -52,9 +52,8 @@ func (this *WebhookNotifier) Notify(context *alerting.EvalContext) {
 		bodyJSON.Set("rule_url", ruleUrl)
 	}
 
-	imageUrl, err := context.GetImageUrl()
-	if err == nil {
-		bodyJSON.Set("image_url", imageUrl)
+	if context.ImagePublicUrl != "" {
+		bodyJSON.Set("image_url", context.ImagePublicUrl)
 	}
 
 	body, _ := bodyJSON.MarshalJSON()

pkg/services/backgroundtasks/background_tasks.go

@@ -0,0 +1,39 @@
+//"I want to be a cleaner, just like you," said Mathilda
+//"Okay," replied Leon
+
+package backgroundtasks
+
+import (
+	"time"
+
+	"github.com/grafana/grafana/pkg/bus"
+	"github.com/grafana/grafana/pkg/log"
+	"github.com/grafana/grafana/pkg/models"
+)
+
+var (
+	tlog log.Logger = log.New("ticker")
+)
+
+func Init() {
+	go start()
+}
+
+func start() {
+	go cleanup(time.Now())
+
+	ticker := time.NewTicker(time.Hour * 1)
+	for {
+		select {
+		case tick := <-ticker.C:
+			go cleanup(tick)
+		}
+	}
+}
+
+func cleanup(now time.Time) {
+	err := bus.Publish(&models.HourCommand{Time: now})
+	if err != nil {
+		tlog.Error("Cleanup job failed", "error", err)
+	}
+}

pkg/services/backgroundtasks/remove_tmp_images.go

@@ -0,0 +1,38 @@
+package backgroundtasks
+
+import (
+	"io/ioutil"
+	"os"
+	"path"
+
+	"github.com/grafana/grafana/pkg/bus"
+	"github.com/grafana/grafana/pkg/models"
+	"github.com/grafana/grafana/pkg/setting"
+)
+
+func init() {
+	bus.AddEventListener(CleanTmpFiles)
+}
+
+func CleanTmpFiles(cmd *models.HourCommand) error {
+	files, err := ioutil.ReadDir(setting.ImagesDir)
+
+	var toDelete []os.FileInfo
+	for _, file := range files {
+		if file.ModTime().AddDate(0, 0, setting.RenderedImageTTLDays).Before(cmd.Time) {
+			toDelete = append(toDelete, file)
+		}
+	}
+
+	for _, file := range toDelete {
+		fullPath := path.Join(setting.ImagesDir, file.Name())
+		err := os.Remove(fullPath)
+		if err != nil {
+			return err
+		}
+	}
+
+	tlog.Debug("Found old rendered image to delete", "deleted", len(toDelete), "keept", len(files))
+
+	return err
+}

pkg/services/sqlstore/dashboard_snapshot.go

@@ -5,7 +5,9 @@ import (
 
 	"github.com/go-xorm/xorm"
 	"github.com/grafana/grafana/pkg/bus"
+	"github.com/grafana/grafana/pkg/log"
 	m "github.com/grafana/grafana/pkg/models"
+	"github.com/grafana/grafana/pkg/setting"
 )
 
 func init() {
@@ -13,6 +15,31 @@ func init() {
 	bus.AddHandler("sql", GetDashboardSnapshot)
 	bus.AddHandler("sql", DeleteDashboardSnapshot)
 	bus.AddHandler("sql", SearchDashboardSnapshots)
+	bus.AddEventListener(DeleteExpiredSnapshots)
+}
+
+func DeleteExpiredSnapshots(cmd *m.HourCommand) error {
+	return inTransaction(func(sess *xorm.Session) error {
+		var expiredCount int64 = 0
+		var oldCount int64 = 0
+
+		if setting.SnapShotRemoveExpired {
+			deleteExpiredSql := "DELETE FROM dashboard_snapshot WHERE expires < ?"
+			expiredResponse, err := x.Exec(deleteExpiredSql, cmd.Time)
+			if err != nil {
+				return err
+			}
+			expiredCount, _ = expiredResponse.RowsAffected()
+		}
+
+		oldSnapshotsSql := "DELETE FROM dashboard_snapshot WHERE created < ?"
+		oldResponse, err := x.Exec(oldSnapshotsSql, cmd.Time.AddDate(0, 0, setting.SnapShotTTLDays*-1))
+		oldCount, _ = oldResponse.RowsAffected()
+
+		log.Debug2("Deleted old/expired snaphots", "to old", oldCount, "expired", expiredCount)
+
+		return err
+	})
 }
 
 func CreateDashboardSnapshot(cmd *m.CreateDashboardSnapshotCommand) error {

pkg/setting/setting.go

@@ -78,9 +78,11 @@ var (
 	DataProxyWhiteList    map[string]bool
 
 	// Snapshots
-	ExternalSnapshotUrl  string
-	ExternalSnapshotName string
-	ExternalEnabled      bool
+	ExternalSnapshotUrl   string
+	ExternalSnapshotName  string
+	ExternalEnabled       bool
+	SnapShotTTLDays       int
+	SnapShotRemoveExpired bool
 
 	// User settings
 	AllowUserSignUp    bool
@@ -118,8 +120,9 @@ var (
 	IsWindows    bool
 
 	// PhantomJs Rendering
-	ImagesDir  string
-	PhantomDir string
+	ImagesDir            string
+	PhantomDir           string
+	RenderedImageTTLDays int
 
 	// for logging purposes
 	configFiles                  []string
@@ -495,6 +498,8 @@ func NewConfigContext(args *CommandLineArgs) error {
 	ExternalSnapshotUrl = snapshots.Key("external_snapshot_url").String()
 	ExternalSnapshotName = snapshots.Key("external_snapshot_name").String()
 	ExternalEnabled = snapshots.Key("external_enabled").MustBool(true)
+	SnapShotRemoveExpired = snapshots.Key("snapshot_remove_expired").MustBool(true)
+	SnapShotTTLDays = snapshots.Key("snapshot_TTL_days").MustInt(90)
 
 	//  read data source proxy white list
 	DataProxyWhiteList = make(map[string]bool)
@@ -535,6 +540,9 @@ func NewConfigContext(args *CommandLineArgs) error {
 	ImagesDir = filepath.Join(DataPath, "png")
 	PhantomDir = filepath.Join(HomePath, "vendor/phantomjs")
 
+	tmpFilesSection := Cfg.Section("tmp.files")
+	RenderedImageTTLDays = tmpFilesSection.Key("rendered_image_ttl_days").MustInt(14)
+
 	analytics := Cfg.Section("analytics")
 	ReportingEnabled = analytics.Key("reporting_enabled").MustBool(true)
 	CheckForUpdates = analytics.Key("check_for_updates").MustBool(true)

pkg/tsdb/graphite/graphite.go

@@ -38,7 +38,7 @@ func init() {
 	}
 
 	HttpClient = http.Client{
-		Timeout:   time.Duration(10 * time.Second),
+		Timeout:   time.Duration(15 * time.Second),
 		Transport: tr,
 	}
 }
@@ -102,9 +102,9 @@ func (e *GraphiteExecutor) parseResponse(res *http.Response) ([]TargetResponseDT
 		return nil, err
 	}
 
-	if res.StatusCode == http.StatusUnauthorized {
-		glog.Info("Request is Unauthorized", "status", res.Status, "body", string(body))
-		return nil, fmt.Errorf("Request is Unauthorized status: %v body: %s", res.Status, string(body))
+	if res.StatusCode/100 != 2 {
+		glog.Info("Request failed", "status", res.Status, "body", string(body))
+		return nil, fmt.Errorf("Request failed status: %v", res.Status)
 	}
 
 	var data []TargetResponseDTO

public/views/index.html

@@ -64,13 +64,13 @@
 							<a href="http://grafana.org" target="_blank">Grafana</a>
 							<span>v[[.BuildVersion]] (commit: [[.BuildCommit]])</span>
 						</li>
-						<li>
-							[[if .NewGrafanaVersionExists]]
+						[[if .NewGrafanaVersionExists]]
+							<li>
 								<a href="http://grafana.org/download" target="_blank" bs-tooltip="'[[.NewGrafanaVersion]]'">
 									New version available!
 								</a>
-							[[end]]
-						</li>
+							</li>
+						[[end]]
 					</ul>
 				</div>
 			</footer>