Added migration log and migration id, do not execute already executed migrations

pkg/services/sqlstore/migrations/builder.go

@@ -13,6 +13,8 @@ const (
 
 type Migration interface {
 	Sql(dialect Dialect) string
+	Id() string
+	SetId(string)
 }
 
 type ColumnType string
@@ -22,7 +24,15 @@ const (
 )
 
 type MigrationBase struct {
-	desc string
+	id string
+}
+
+func (m *MigrationBase) Id() string {
+	return m.id
+}
+
+func (m *MigrationBase) SetId(id string) {
+	m.id = id
 }
 
 type RawSqlMigration struct {
@@ -53,11 +63,6 @@ func (m *RawSqlMigration) Mysql(sql string) *RawSqlMigration {
 	return m
 }
 
-func (m *RawSqlMigration) Desc(desc string) *RawSqlMigration {
-	m.desc = desc
-	return m
-}
-
 type AddColumnMigration struct {
 	MigrationBase
 	tableName  string
@@ -90,16 +95,12 @@ func (m *AddColumnMigration) Sql(dialect Dialect) string {
 	return fmt.Sprintf("ALTER TABLE %s ADD COLUMN %s %s", m.tableName, m.columnName, dialect.ToDBTypeSql(m.columnType, m.length))
 }
 
-func (m *AddColumnMigration) Desc(desc string) *AddColumnMigration {
-	m.desc = desc
-	return m
-}
-
 type AddIndexMigration struct {
 	MigrationBase
 	tableName string
 	columns   string
 	indexName string
+	unique    string
 }
 
 func (m *AddIndexMigration) Name(name string) *AddIndexMigration {
@@ -112,11 +113,16 @@ func (m *AddIndexMigration) Table(tableName string) *AddIndexMigration {
 	return m
 }
 
+func (m *AddIndexMigration) Unique() *AddIndexMigration {
+	m.unique = "UNIQUE"
+	return m
+}
+
 func (m *AddIndexMigration) Columns(columns ...string) *AddIndexMigration {
 	m.columns = strings.Join(columns, ",")
 	return m
 }
 
 func (m *AddIndexMigration) Sql(dialect Dialect) string {
-	return fmt.Sprintf("CREATE UNIQUE INDEX %s ON %s(%s)", m.indexName, m.tableName, m.columns)
+	return fmt.Sprintf("CREATE %s INDEX %s ON %s(%s)", m.unique, m.indexName, m.tableName, m.columns)
 }

pkg/services/sqlstore/migrations/engine.go

@@ -1,99 +0,0 @@
-package migrations
-
-import (
-	_ "github.com/go-sql-driver/mysql"
-	"github.com/go-xorm/xorm"
-	_ "github.com/lib/pq"
-	_ "github.com/mattn/go-sqlite3"
-	"github.com/torkelo/grafana-pro/pkg/log"
-)
-
-var x *xorm.Engine
-var dialect Dialect
-
-func getSchemaVersion() (int, error) {
-	exists, err := x.IsTableExist(new(SchemaVersion))
-	if err != nil {
-		return 0, err
-	}
-
-	if !exists {
-		if err := x.CreateTables(new(SchemaVersion)); err != nil {
-			return 0, err
-		}
-		return 0, nil
-	}
-
-	v := SchemaVersion{}
-	_, err = x.Table("schema_version").Limit(1, 0).Desc("version").Get(&v)
-	return v.Version, err
-}
-
-func setEngineAndDialect(engine *xorm.Engine) {
-	x = engine
-	switch x.DriverName() {
-	case MYSQL:
-		dialect = new(Mysql)
-	case SQLITE:
-		dialect = new(Sqlite3)
-	}
-}
-
-func StartMigration(engine *xorm.Engine) error {
-	log.Info("Starting database schema migration: DB: %v", engine.DriverName())
-
-	setEngineAndDialect(engine)
-
-	_, err := getSchemaVersion()
-	if err != nil {
-		return err
-	}
-
-	for _, m := range migrationList {
-		if err := execMigration(m); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func execMigration(m Migration) error {
-	err := inTransaction(func(sess *xorm.Session) error {
-		_, err := sess.Exec(m.Sql(dialect))
-		if err != nil {
-			return err
-		}
-		return nil
-	})
-
-	if err != nil {
-		return err
-	}
-
-	return nil
-}
-
-type dbTransactionFunc func(sess *xorm.Session) error
-
-func inTransaction(callback dbTransactionFunc) error {
-	var err error
-
-	sess := x.NewSession()
-	defer sess.Close()
-
-	if err = sess.Begin(); err != nil {
-		return err
-	}
-
-	err = callback(sess)
-
-	if err != nil {
-		sess.Rollback()
-		return err
-	} else if err = sess.Commit(); err != nil {
-		return err
-	}
-
-	return nil
-}

pkg/services/sqlstore/migrations/migrations.go

@@ -1,6 +1,6 @@
 package migrations
 
-var migrationList []Migration
+import "time"
 
 // Id              int64
 // Login           string `xorm:"UNIQUE NOT NULL"`
@@ -16,9 +16,11 @@ var migrationList []Migration
 // Created         time.Time
 // Updated         time.Time
 
-func init() {
-	// ------------------------------
-	addMigration(new(RawSqlMigration).Desc("Create account table").
+func AddMigrations(mg *Migrator) {
+
+	// TABLE Account
+	// -------------------------------
+	mg.AddMigration("create account table", new(RawSqlMigration).
 		Sqlite(`
 		  CREATE TABLE account (
 		  	id INTEGER PRIMARY KEY AUTOINCREMENT,
@@ -34,25 +36,18 @@ func init() {
 		  )
 		`))
 	// ------------------------------
-	addMigration(new(AddIndexMigration).
+	mg.AddMigration("add index UIX_account.login", new(AddIndexMigration).
 		Name("UIX_account_login").Table("account").Columns("login"))
 	// ------------------------------
-	addMigration(new(AddColumnMigration).Desc("Add name column").
+	mg.AddMigration("add column", new(AddColumnMigration).
 		Table("account").Column("name").Type(DB_TYPE_STRING).Length(255))
 }
 
-func addMigration(m Migration) {
-	migrationList = append(migrationList, m)
-}
-
-type SchemaVersion struct {
-	Version int
-}
-
-type SchemaLog struct {
-	Id      int64
-	Version int64
-	Desc    string
-	Info    string
-	Error   bool
+type MigrationLog struct {
+	Id          int64
+	MigrationId string
+	Sql         string
+	Success     bool
+	Error       string
+	Timestamp   time.Time
 }

pkg/services/sqlstore/migrations/migrations_test.go

@@ -6,6 +6,7 @@ import (
 	"testing"
 
 	"github.com/go-xorm/xorm"
+	"github.com/torkelo/grafana-pro/pkg/log"
 
 	. "github.com/smartystreets/goconvey/convey"
 )
@@ -31,8 +32,10 @@ func cleanDB(x *xorm.Engine) {
 var indexTypes = []string{"Unknown", "", "UNIQUE"}
 
 func TestMigrations(t *testing.T) {
+	log.NewLogger(0, "console", `{"level": 0}`)
+
 	testDBs := [][]string{
-		//[]string{"mysql", "grafana:password@tcp(localhost:3306)/grafana_tests?charset=utf8"},
+		[]string{"mysql", "grafana:password@tcp(localhost:3306)/grafana_tests?charset=utf8"},
 		[]string{"sqlite3", ":memory:"},
 	}
 
@@ -46,7 +49,10 @@ func TestMigrations(t *testing.T) {
 				cleanDB(x)
 			}
 
-			err = StartMigration(x)
+			mg := NewMigrator(x)
+			AddMigrations(mg)
+
+			err = mg.Start()
 			So(err, ShouldBeNil)
 
 			tables, err := x.DBMetas()

pkg/services/sqlstore/migrations/migrator.go

@@ -0,0 +1,143 @@
+package migrations
+
+import (
+	"time"
+
+	_ "github.com/go-sql-driver/mysql"
+	"github.com/go-xorm/xorm"
+	_ "github.com/lib/pq"
+	_ "github.com/mattn/go-sqlite3"
+	"github.com/torkelo/grafana-pro/pkg/log"
+)
+
+type Migrator struct {
+	x          *xorm.Engine
+	dialect    Dialect
+	migrations []Migration
+}
+
+func NewMigrator(engine *xorm.Engine) *Migrator {
+	mg := &Migrator{}
+	mg.x = engine
+	mg.migrations = make([]Migration, 0)
+
+	switch mg.x.DriverName() {
+	case MYSQL:
+		mg.dialect = new(Mysql)
+	case SQLITE:
+		mg.dialect = new(Sqlite3)
+	}
+
+	return mg
+}
+
+func (mg *Migrator) AddMigration(id string, m Migration) {
+	m.SetId(id)
+	mg.migrations = append(mg.migrations, m)
+}
+
+func (mg *Migrator) GetMigrationLog() (map[string]MigrationLog, error) {
+	exists, err := mg.x.IsTableExist(new(MigrationLog))
+	if err != nil {
+		return nil, err
+	}
+
+	if !exists {
+		if err := mg.x.CreateTables(new(MigrationLog)); err != nil {
+			return nil, err
+		}
+		return nil, nil
+	}
+
+	logMap := make(map[string]MigrationLog)
+	logItems := make([]MigrationLog, 0)
+	if err = mg.x.Find(&logItems); err != nil {
+		return nil, err
+	}
+
+	for _, logItem := range logItems {
+		if !logItem.Success {
+			continue
+		}
+		logMap[logItem.MigrationId] = logItem
+	}
+
+	return logMap, nil
+}
+
+func (mg *Migrator) Start() error {
+	log.Info("Migrator::Start DB migration")
+
+	logMap, err := mg.GetMigrationLog()
+	if err != nil {
+		return err
+	}
+
+	for _, m := range mg.migrations {
+		_, exists := logMap[m.Id()]
+		if exists {
+			log.Info("Migrator:: Skipping migration: %v, Already executed", m.Id())
+			continue
+		}
+
+		record := MigrationLog{
+			MigrationId: m.Id(),
+			Sql:         m.Sql(mg.dialect),
+			Timestamp:   time.Now(),
+		}
+
+		if err := mg.exec(m); err != nil {
+			record.Error = err.Error()
+			mg.x.Insert(&record)
+			return err
+		} else {
+			record.Success = true
+			mg.x.Insert(&record)
+		}
+	}
+
+	return nil
+}
+
+func (mg *Migrator) exec(m Migration) error {
+	log.Info("Migrator::exec migration id: %v", m.Id())
+
+	err := mg.inTransaction(func(sess *xorm.Session) error {
+		_, err := sess.Exec(m.Sql(mg.dialect))
+		if err != nil {
+			log.Error(3, "Migrator::exec FAILED migration id: %v, err: %v", m.Id(), err)
+			return err
+		}
+		return nil
+	})
+
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+type dbTransactionFunc func(sess *xorm.Session) error
+
+func (mg *Migrator) inTransaction(callback dbTransactionFunc) error {
+	var err error
+
+	sess := mg.x.NewSession()
+	defer sess.Close()
+
+	if err = sess.Begin(); err != nil {
+		return err
+	}
+
+	err = callback(sess)
+
+	if err != nil {
+		sess.Rollback()
+		return err
+	} else if err = sess.Commit(); err != nil {
+		return err
+	}
+
+	return nil
+}

pkg/services/sqlstore/migrations/migrator_test.go

@@ -0,0 +1,78 @@
+package migrations
+
+import (
+	"testing"
+
+	"github.com/go-xorm/xorm"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+// func cleanDB(x *xorm.Engine) {
+// 	tables, _ := x.DBMetas()
+// 	sess := x.NewSession()
+// 	defer sess.Close()
+//
+// 	for _, table := range tables {
+// 		if _, err := sess.Exec("SET FOREIGN_KEY_CHECKS = 0"); err != nil {
+// 			panic("Failed to disable foreign key checks")
+// 		}
+// 		if _, err := sess.Exec("DROP TABLE " + table.Name); err != nil {
+// 			panic(fmt.Sprintf("Failed to delete table: %v, err: %v", table.Name, err))
+// 		}
+// 		if _, err := sess.Exec("SET FOREIGN_KEY_CHECKS = 1"); err != nil {
+// 			panic("Failed to disable foreign key checks")
+// 		}
+// 	}
+// }
+//
+// var indexTypes = []string{"Unknown", "", "UNIQUE"}
+//
+
+func TestMigrator(t *testing.T) {
+
+	Convey("Migrator", t, func() {
+		x, err := xorm.NewEngine(SQLITE, ":memory:")
+		So(err, ShouldBeNil)
+
+		mg := NewMigrator(x)
+
+		Convey("Given one migration", func() {
+			mg.AddMigration("test migration", new(RawSqlMigration).
+				Sqlite(`
+			    CREATE TABLE account (
+		       	id INTEGER PRIMARY KEY AUTOINCREMENT
+				  )`).
+				Mysql(`
+			   	CREATE TABLE account (
+						id BIGINT NOT NULL AUTO_INCREMENT, PRIMARY KEY (id)
+					)`))
+
+			err := mg.Start()
+			So(err, ShouldBeNil)
+
+			log, err := mg.GetMigrationLog()
+			So(err, ShouldBeNil)
+			So(len(log), ShouldEqual, 1)
+		})
+
+		// So(err, ShouldBeNil)
+		//
+		// So(len(tables), ShouldEqual, 2)
+		// fmt.Printf("\nDB Schema after migration: table count: %v\n", len(tables))
+		//
+		// for _, table := range tables {
+		// 	fmt.Printf("\nTable: %v \n", table.Name)
+		// 	for _, column := range table.Columns() {
+		// 		fmt.Printf("\t %v \n", column.String(x.Dialect()))
+		// 	}
+		//
+		// 	if len(table.Indexes) > 0 {
+		// 		fmt.Printf("\n\tIndexes:\n")
+		// 		for _, index := range table.Indexes {
+		// 			fmt.Printf("\t %v (%v) %v \n", index.Name, strings.Join(index.Cols, ","), indexTypes[index.Type])
+		// 		}
+		// 	}
+		// }
+	})
+}