CLI: Add command to migrate all datasources to use encrypted password fields (#17118)

closes: #17107

pkg/cmd/grafana-cli/commands/commands.go

@@ -7,14 +7,16 @@ import (
 	"github.com/codegangsta/cli"
 	"github.com/fatih/color"
 	"github.com/grafana/grafana/pkg/bus"
+	"github.com/grafana/grafana/pkg/cmd/grafana-cli/commands/datamigrations"
 	"github.com/grafana/grafana/pkg/cmd/grafana-cli/logger"
+	"github.com/grafana/grafana/pkg/cmd/grafana-cli/utils"
 	"github.com/grafana/grafana/pkg/services/sqlstore"
 	"github.com/grafana/grafana/pkg/setting"
 )
 
-func runDbCommand(command func(commandLine CommandLine) error) func(context *cli.Context) {
+func runDbCommand(command func(commandLine utils.CommandLine, sqlStore *sqlstore.SqlStore) error) func(context *cli.Context) {
 	return func(context *cli.Context) {
-		cmd := &contextCommandLine{context}
+		cmd := &utils.ContextCommandLine{Context: context}
 
 		cfg := setting.NewCfg()
 		cfg.Load(&setting.CommandLineArgs{
@@ -28,7 +30,7 @@ func runDbCommand(command func(commandLine CommandLine) error) func(context *cli
 		engine.Bus = bus.GetBus()
 		engine.Init()
 
-		if err := command(cmd); err != nil {
+		if err := command(cmd, engine); err != nil {
 			logger.Errorf("\n%s: ", color.RedString("Error"))
 			logger.Errorf("%s\n\n", err)
 
@@ -40,10 +42,10 @@ func runDbCommand(command func(commandLine CommandLine) error) func(context *cli
 	}
 }
 
-func runPluginCommand(command func(commandLine CommandLine) error) func(context *cli.Context) {
+func runPluginCommand(command func(commandLine utils.CommandLine) error) func(context *cli.Context) {
 	return func(context *cli.Context) {
 
-		cmd := &contextCommandLine{context}
+		cmd := &utils.ContextCommandLine{Context: context}
 		if err := command(cmd); err != nil {
 			logger.Errorf("\n%s: ", color.RedString("Error"))
 			logger.Errorf("%s %s\n\n", color.RedString("✗"), err)
@@ -107,6 +109,17 @@ var adminCommands = []cli.Command{
 			},
 		},
 	},
+	{
+		Name:  "data-migration",
+		Usage: "Runs a script that migrates or cleanups data in your db",
+		Subcommands: []cli.Command{
+			{
+				Name:   "encrypt-datasource-passwords",
+				Usage:  "Migrates passwords from unsecured fields to secure_json_data field. Return ok unless there is an error. Safe to execute multiple times.",
+				Action: runDbCommand(datamigrations.EncryptDatasourcePaswords),
+			},
+		},
+	},
 }
 
 var Commands = []cli.Command{

pkg/cmd/grafana-cli/commands/datamigrations/encrypt_datasource_passwords.go

@@ -0,0 +1,126 @@
+package datamigrations
+
+import (
+	"context"
+	"encoding/json"
+
+	"github.com/fatih/color"
+	"github.com/grafana/grafana/pkg/cmd/grafana-cli/logger"
+
+	"github.com/grafana/grafana/pkg/cmd/grafana-cli/utils"
+	"github.com/grafana/grafana/pkg/services/sqlstore"
+	"github.com/grafana/grafana/pkg/setting"
+	"github.com/grafana/grafana/pkg/util"
+	"github.com/grafana/grafana/pkg/util/errutil"
+)
+
+var (
+	datasourceTypes = []string{
+		"mysql",
+		"influxdb",
+		"elasticsearch",
+		"graphite",
+		"prometheus",
+		"opentsdb",
+	}
+)
+
+// EncryptDatasourcePaswords migrates un-encrypted secrets on datasources
+// to the secureJson Column.
+func EncryptDatasourcePaswords(c utils.CommandLine, sqlStore *sqlstore.SqlStore) error {
+	return sqlStore.WithDbSession(context.Background(), func(session *sqlstore.DBSession) error {
+		passwordsUpdated, err := migrateColumn(session, "password")
+		if err != nil {
+			return err
+		}
+
+		basicAuthUpdated, err := migrateColumn(session, "basic_auth_password")
+		if err != nil {
+			return err
+		}
+
+		logger.Info("\n")
+		if passwordsUpdated > 0 {
+			logger.Infof("%s Encrypted password field for %d datasources \n", color.GreenString("✔"), passwordsUpdated)
+		}
+
+		if basicAuthUpdated > 0 {
+			logger.Infof("%s Encrypted basic_auth_password field for %d datasources \n", color.GreenString("✔"), basicAuthUpdated)
+		}
+
+		if passwordsUpdated == 0 && basicAuthUpdated == 0 {
+			logger.Infof("%s All datasources secrets are allready encrypted\n", color.GreenString("✔"))
+		}
+
+		logger.Info("\n")
+
+		logger.Warn("Warning: Datasource provisioning files need to be manually changed to prevent overwriting of " +
+			"the data during provisioning. See https://grafana.com/docs/installation/upgrading/#upgrading-to-v6-2 for " +
+			"details")
+		return nil
+	})
+}
+
+func migrateColumn(session *sqlstore.DBSession, column string) (int, error) {
+	var rows []map[string]string
+
+	session.Cols("id", column, "secure_json_data")
+	session.Table("data_source")
+	session.In("type", datasourceTypes)
+	session.Where(column + " IS NOT NULL AND " + column + " != ''")
+	err := session.Find(&rows)
+
+	if err != nil {
+		return 0, errutil.Wrapf(err, "failed to select column: %s", column)
+	}
+
+	rowsUpdated, err := updateRows(session, rows, column)
+	return rowsUpdated, errutil.Wrapf(err, "failed to update column: %s", column)
+}
+
+func updateRows(session *sqlstore.DBSession, rows []map[string]string, passwordFieldName string) (int, error) {
+	var rowsUpdated int
+
+	for _, row := range rows {
+		newSecureJSONData, err := getUpdatedSecureJSONData(row, passwordFieldName)
+		if err != nil {
+			return 0, err
+		}
+
+		data, err := json.Marshal(newSecureJSONData)
+		if err != nil {
+			return 0, errutil.Wrap("marshaling newSecureJsonData failed", err)
+		}
+
+		newRow := map[string]interface{}{"secure_json_data": data, passwordFieldName: ""}
+		session.Table("data_source")
+		session.Where("id = ?", row["id"])
+		// Setting both columns while having value only for secure_json_data should clear the [passwordFieldName] column
+		session.Cols("secure_json_data", passwordFieldName)
+
+		_, err = session.Update(newRow)
+		if err != nil {
+			return 0, err
+		}
+
+		rowsUpdated++
+	}
+	return rowsUpdated, nil
+}
+
+func getUpdatedSecureJSONData(row map[string]string, passwordFieldName string) (map[string]interface{}, error) {
+	encryptedPassword, err := util.Encrypt([]byte(row[passwordFieldName]), setting.SecretKey)
+	if err != nil {
+		return nil, err
+	}
+
+	var secureJSONData map[string]interface{}
+
+	if err := json.Unmarshal([]byte(row["secure_json_data"]), &secureJSONData); err != nil {
+		return nil, err
+	}
+
+	jsonFieldName := util.ToCamelCase(passwordFieldName)
+	secureJSONData[jsonFieldName] = encryptedPassword
+	return secureJSONData, nil
+}

pkg/cmd/grafana-cli/commands/datamigrations/encrypt_datasource_passwords_test.go

@@ -0,0 +1,67 @@
+package datamigrations
+
+import (
+	"testing"
+	"time"
+
+	"github.com/grafana/grafana/pkg/cmd/grafana-cli/commands/commandstest"
+	"github.com/grafana/grafana/pkg/components/securejsondata"
+	"github.com/grafana/grafana/pkg/models"
+	"github.com/grafana/grafana/pkg/services/sqlstore"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestPasswordMigrationCommand(t *testing.T) {
+	//setup datasources with password, basic_auth and none
+	sqlstore := sqlstore.InitTestDB(t)
+	session := sqlstore.NewSession()
+	defer session.Close()
+
+	datasources := []*models.DataSource{
+		{Type: "influxdb", Name: "influxdb", Password: "foobar"},
+		{Type: "graphite", Name: "graphite", BasicAuthPassword: "foobar"},
+		{Type: "prometheus", Name: "prometheus", SecureJsonData: securejsondata.GetEncryptedJsonData(map[string]string{})},
+	}
+
+	// set required default values
+	for _, ds := range datasources {
+		ds.Created = time.Now()
+		ds.Updated = time.Now()
+		ds.SecureJsonData = securejsondata.GetEncryptedJsonData(map[string]string{})
+	}
+
+	_, err := session.Insert(&datasources)
+	assert.Nil(t, err)
+
+	//run migration
+	err = EncryptDatasourcePaswords(&commandstest.FakeCommandLine{}, sqlstore)
+	assert.Nil(t, err)
+
+	//verify that no datasources still have password or basic_auth
+	var dss []*models.DataSource
+	err = session.SQL("select * from data_source").Find(&dss)
+	assert.Nil(t, err)
+	assert.Equal(t, len(dss), 3)
+
+	for _, ds := range dss {
+		sj := ds.SecureJsonData.Decrypt()
+
+		if ds.Name == "influxdb" {
+			assert.Equal(t, ds.Password, "")
+			v, exist := sj["password"]
+			assert.True(t, exist)
+			assert.Equal(t, v, "foobar", "expected password to be moved to securejson")
+		}
+
+		if ds.Name == "graphite" {
+			assert.Equal(t, ds.BasicAuthPassword, "")
+			v, exist := sj["basicAuthPassword"]
+			assert.True(t, exist)
+			assert.Equal(t, v, "foobar", "expected basic_auth_password to be moved to securejson")
+		}
+
+		if ds.Name == "prometheus" {
+			assert.Equal(t, len(sj), 0)
+		}
+	}
+}

pkg/cmd/grafana-cli/commands/install_command.go

@@ -14,13 +14,14 @@ import (
 	"strings"
 
 	"github.com/fatih/color"
+	"github.com/grafana/grafana/pkg/cmd/grafana-cli/utils"
 
 	"github.com/grafana/grafana/pkg/cmd/grafana-cli/logger"
 	m "github.com/grafana/grafana/pkg/cmd/grafana-cli/models"
 	s "github.com/grafana/grafana/pkg/cmd/grafana-cli/services"
 )
 
-func validateInput(c CommandLine, pluginFolder string) error {
+func validateInput(c utils.CommandLine, pluginFolder string) error {
 	arg := c.Args().First()
 	if arg == "" {
 		return errors.New("please specify plugin to install")
@@ -46,7 +47,7 @@ func validateInput(c CommandLine, pluginFolder string) error {
 	return nil
 }
 
-func installCommand(c CommandLine) error {
+func installCommand(c utils.CommandLine) error {
 	pluginFolder := c.PluginDirectory()
 	if err := validateInput(c, pluginFolder); err != nil {
 		return err
@@ -60,7 +61,7 @@ func installCommand(c CommandLine) error {
 
 // InstallPlugin downloads the plugin code as a zip file from the Grafana.com API
 // and then extracts the zip into the plugins directory.
-func InstallPlugin(pluginName, version string, c CommandLine) error {
+func InstallPlugin(pluginName, version string, c utils.CommandLine) error {
 	pluginFolder := c.PluginDirectory()
 	downloadURL := c.PluginURL()
 	if downloadURL == "" {

pkg/cmd/grafana-cli/commands/listremote_command.go

@@ -3,9 +3,10 @@ package commands
 import (
 	"github.com/grafana/grafana/pkg/cmd/grafana-cli/logger"
 	s "github.com/grafana/grafana/pkg/cmd/grafana-cli/services"
+	"github.com/grafana/grafana/pkg/cmd/grafana-cli/utils"
 )
 
-func listremoteCommand(c CommandLine) error {
+func listremoteCommand(c utils.CommandLine) error {
 	plugin, err := s.ListAllPlugins(c.RepoDirectory())
 
 	if err != nil {

pkg/cmd/grafana-cli/commands/listversions_command.go

@@ -5,9 +5,10 @@ import (
 
 	"github.com/grafana/grafana/pkg/cmd/grafana-cli/logger"
 	s "github.com/grafana/grafana/pkg/cmd/grafana-cli/services"
+	"github.com/grafana/grafana/pkg/cmd/grafana-cli/utils"
 )
 
-func validateVersionInput(c CommandLine) error {
+func validateVersionInput(c utils.CommandLine) error {
 	arg := c.Args().First()
 	if arg == "" {
 		return errors.New("please specify plugin to list versions for")
@@ -16,7 +17,7 @@ func validateVersionInput(c CommandLine) error {
 	return nil
 }
 
-func listversionsCommand(c CommandLine) error {
+func listversionsCommand(c utils.CommandLine) error {
 	if err := validateVersionInput(c); err != nil {
 		return err
 	}

pkg/cmd/grafana-cli/commands/ls_command.go

@@ -8,6 +8,7 @@ import (
 	"github.com/grafana/grafana/pkg/cmd/grafana-cli/logger"
 	m "github.com/grafana/grafana/pkg/cmd/grafana-cli/models"
 	s "github.com/grafana/grafana/pkg/cmd/grafana-cli/services"
+	"github.com/grafana/grafana/pkg/cmd/grafana-cli/utils"
 )
 
 var ls_getPlugins func(path string) []m.InstalledPlugin = s.GetLocalPlugins
@@ -31,7 +32,7 @@ var validateLsCommand = func(pluginDir string) error {
 	return nil
 }
 
-func lsCommand(c CommandLine) error {
+func lsCommand(c utils.CommandLine) error {
 	pluginDir := c.PluginDirectory()
 	if err := validateLsCommand(pluginDir); err != nil {
 		return err

pkg/cmd/grafana-cli/commands/remove_command.go

@@ -5,12 +5,13 @@ import (
 	"fmt"
 	"strings"
 
-	services "github.com/grafana/grafana/pkg/cmd/grafana-cli/services"
+	"github.com/grafana/grafana/pkg/cmd/grafana-cli/services"
+	"github.com/grafana/grafana/pkg/cmd/grafana-cli/utils"
 )
 
 var removePlugin func(pluginPath, id string) error = services.RemoveInstalledPlugin
 
-func removeCommand(c CommandLine) error {
+func removeCommand(c utils.CommandLine) error {
 	pluginPath := c.PluginDirectory()
 
 	plugin := c.Args().First()

pkg/cmd/grafana-cli/commands/reset_password_command.go

@@ -6,13 +6,15 @@ import (
 	"github.com/fatih/color"
 	"github.com/grafana/grafana/pkg/bus"
 	"github.com/grafana/grafana/pkg/cmd/grafana-cli/logger"
+	"github.com/grafana/grafana/pkg/cmd/grafana-cli/utils"
 	"github.com/grafana/grafana/pkg/models"
+	"github.com/grafana/grafana/pkg/services/sqlstore"
 	"github.com/grafana/grafana/pkg/util"
 )
 
 const AdminUserId = 1
 
-func resetPasswordCommand(c CommandLine) error {
+func resetPasswordCommand(c utils.CommandLine, sqlStore *sqlstore.SqlStore) error {
 	newPassword := c.Args().First()
 
 	password := models.Password(newPassword)

pkg/cmd/grafana-cli/commands/upgrade_all_command.go

@@ -4,6 +4,7 @@ import (
 	"github.com/grafana/grafana/pkg/cmd/grafana-cli/logger"
 	m "github.com/grafana/grafana/pkg/cmd/grafana-cli/models"
 	s "github.com/grafana/grafana/pkg/cmd/grafana-cli/services"
+	"github.com/grafana/grafana/pkg/cmd/grafana-cli/utils"
 	"github.com/hashicorp/go-version"
 )
 
@@ -27,7 +28,7 @@ func ShouldUpgrade(installed string, remote m.Plugin) bool {
 	return false
 }
 
-func upgradeAllCommand(c CommandLine) error {
+func upgradeAllCommand(c utils.CommandLine) error {
 	pluginsDir := c.PluginDirectory()
 
 	localPlugins := s.GetLocalPlugins(pluginsDir)

pkg/cmd/grafana-cli/commands/upgrade_command.go

@@ -4,9 +4,10 @@ import (
 	"github.com/fatih/color"
 	"github.com/grafana/grafana/pkg/cmd/grafana-cli/logger"
 	s "github.com/grafana/grafana/pkg/cmd/grafana-cli/services"
+	"github.com/grafana/grafana/pkg/cmd/grafana-cli/utils"
 )
 
-func upgradeCommand(c CommandLine) error {
+func upgradeCommand(c utils.CommandLine) error {
 	pluginsDir := c.PluginDirectory()
 	pluginName := c.Args().First()
 

pkg/cmd/grafana-cli/commands/command_line.go → pkg/cmd/grafana-cli/utils/command_line.go

@@ -1,4 +1,4 @@
-package commands
+package utils
 
 import (
 	"github.com/codegangsta/cli"
@@ -22,30 +22,30 @@ type CommandLine interface {
 	PluginURL() string
 }
 
-type contextCommandLine struct {
+type ContextCommandLine struct {
 	*cli.Context
 }
 
-func (c *contextCommandLine) ShowHelp() {
+func (c *ContextCommandLine) ShowHelp() {
 	cli.ShowCommandHelp(c.Context, c.Command.Name)
 }
 
-func (c *contextCommandLine) ShowVersion() {
+func (c *ContextCommandLine) ShowVersion() {
 	cli.ShowVersion(c.Context)
 }
 
-func (c *contextCommandLine) Application() *cli.App {
+func (c *ContextCommandLine) Application() *cli.App {
 	return c.App
 }
 
-func (c *contextCommandLine) PluginDirectory() string {
+func (c *ContextCommandLine) PluginDirectory() string {
 	return c.GlobalString("pluginsDir")
 }
 
-func (c *contextCommandLine) RepoDirectory() string {
+func (c *ContextCommandLine) RepoDirectory() string {
 	return c.GlobalString("repo")
 }
 
-func (c *contextCommandLine) PluginURL() string {
+func (c *ContextCommandLine) PluginURL() string {
 	return c.GlobalString("pluginUrl")
 }

pkg/util/strings.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"math"
 	"regexp"
+	"strings"
 	"time"
 )
 
@@ -66,3 +67,19 @@ func GetAgeString(t time.Time) string {
 
 	return "< 1m"
 }
+
+// ToCamelCase changes kebab case, snake case or mixed strings to camel case. See unit test for examples.
+func ToCamelCase(str string) string {
+	var finalParts []string
+	parts := strings.Split(str, "_")
+
+	for _, part := range parts {
+		finalParts = append(finalParts, strings.Split(part, "-")...)
+	}
+
+	for index, part := range finalParts[1:] {
+		finalParts[index+1] = strings.Title(part)
+	}
+
+	return strings.Join(finalParts, "")
+}

pkg/util/strings_test.go

@@ -37,3 +37,12 @@ func TestDateAge(t *testing.T) {
 		So(GetAgeString(time.Now().Add(-time.Hour*24*409)), ShouldEqual, "1y")
 	})
 }
+
+func TestToCamelCase(t *testing.T) {
+	Convey("ToCamelCase", t, func() {
+		So(ToCamelCase("kebab-case-string"), ShouldEqual, "kebabCaseString")
+		So(ToCamelCase("snake_case_string"), ShouldEqual, "snakeCaseString")
+		So(ToCamelCase("mixed-case_string"), ShouldEqual, "mixedCaseString")
+		So(ToCamelCase("alreadyCamelCase"), ShouldEqual, "alreadyCamelCase")
+	})
+}