Provisioning: Interpolate env vars in provisioning files (#16499)

* Add value types with custom unmarshalling logic

* Add env support for notifications config

* Use env vars in json data tests for values

* Add some more complexities to value tests

* Update comment with example usage

* Set env directly in the tests, removing patching

* Update documentation

* Add env var to the file reader tests

* Add raw value

* Post merge fixes

* Add comment

docs/sources/administration/provisioning.md

@@ -30,33 +30,19 @@ Checkout the [configuration](/installation/configuration) page for more informat
 
 ### Using Environment Variables
 
-All options in the configuration file (listed below) can be overridden
-using environment variables using the syntax:
+It is possible to use environment variable interpolation in all 3 provisioning config types. Allowed syntax
+is either `$ENV_VAR_NAME` or `${ENV_VAR_NAME}` and can be used only for values not for keys or bigger parts
+of the configs. It is not available in the dashboards definition files just the dashboard provisioning
+configuration.
+Example:
 
-```bash
-GF_<SectionName>_<KeyName>
-```
-
-Where the section name is the text within the brackets. Everything
-should be upper case and `.` should be replaced by `_`. For example, given these configuration settings:
-
-```bash
-# default section
-instance_name = ${HOSTNAME}
-
-[security]
-admin_user = admin
-
-[auth.google]
-client_secret = 0ldS3cretKey
-```
-
-Overriding will be done like so:
-
-```bash
-export GF_DEFAULT_INSTANCE_NAME=my-instance
-export GF_SECURITY_ADMIN_USER=true
-export GF_AUTH_GOOGLE_CLIENT_SECRET=newS3cretKey
+```yaml
+datasources:
+- name: Graphite
+  url: http://localhost:$PORT
+  user: $USER
+  secureJsonData:
+    password: $PASSWORD
 ```
 
 <hr />

pkg/services/provisioning/dashboards/config_reader_test.go

@@ -1,6 +1,7 @@
 package dashboards
 
 import (
+	"os"
 	"testing"
 
 	"github.com/grafana/grafana/pkg/log"
@@ -18,8 +19,10 @@ func TestDashboardsAsConfig(t *testing.T) {
 		logger := log.New("test-logger")
 
 		Convey("Can read config file version 1 format", func() {
+			_ = os.Setenv("TEST_VAR", "general")
 			cfgProvider := configReader{path: simpleDashboardConfig, log: logger}
 			cfg, err := cfgProvider.readConfig()
+			_ = os.Unsetenv("TEST_VAR")
 			So(err, ShouldBeNil)
 
 			validateDashboardAsConfig(t, cfg)

pkg/services/provisioning/dashboards/testdata/test-configs/dashboards-from-disk/dev-dashboards.yaml

@@ -1,7 +1,7 @@
 apiVersion: 1
 
 providers:
-- name: 'general dashboards'
+- name: '$TEST_VAR dashboards'
   orgId: 2
   folder: 'developers'
   folderUid: 'xyz'

pkg/services/provisioning/dashboards/types.go

@@ -1,6 +1,7 @@
 package dashboards
 
 import (
+	"github.com/grafana/grafana/pkg/services/provisioning/values"
 	"time"
 
 	"github.com/grafana/grafana/pkg/components/simplejson"
@@ -42,15 +43,15 @@ type DashboardAsConfigV1 struct {
 }
 
 type DashboardProviderConfigs struct {
-	Name                  string                 `json:"name" yaml:"name"`
-	Type                  string                 `json:"type" yaml:"type"`
-	OrgId                 int64                  `json:"orgId" yaml:"orgId"`
-	Folder                string                 `json:"folder" yaml:"folder"`
-	FolderUid             string                 `json:"folderUid" yaml:"folderUid"`
-	Editable              bool                   `json:"editable" yaml:"editable"`
-	Options               map[string]interface{} `json:"options" yaml:"options"`
-	DisableDeletion       bool                   `json:"disableDeletion" yaml:"disableDeletion"`
-	UpdateIntervalSeconds int64                  `json:"updateIntervalSeconds" yaml:"updateIntervalSeconds"`
+	Name                  values.StringValue `json:"name" yaml:"name"`
+	Type                  values.StringValue `json:"type" yaml:"type"`
+	OrgId                 values.Int64Value  `json:"orgId" yaml:"orgId"`
+	Folder                values.StringValue `json:"folder" yaml:"folder"`
+	FolderUid             values.StringValue `json:"folderUid" yaml:"folderUid"`
+	Editable              values.BoolValue   `json:"editable" yaml:"editable"`
+	Options               values.JSONValue   `json:"options" yaml:"options"`
+	DisableDeletion       values.BoolValue   `json:"disableDeletion" yaml:"disableDeletion"`
+	UpdateIntervalSeconds values.Int64Value  `json:"updateIntervalSeconds" yaml:"updateIntervalSeconds"`
 }
 
 func createDashboardJson(data *simplejson.Json, lastModified time.Time, cfg *DashboardsAsConfig, folderId int64) (*dashboards.SaveDashboardDTO, error) {
@@ -94,15 +95,15 @@ func (dc *DashboardAsConfigV1) mapToDashboardAsConfig() []*DashboardsAsConfig {
 
 	for _, v := range dc.Providers {
 		r = append(r, &DashboardsAsConfig{
-			Name:                  v.Name,
-			Type:                  v.Type,
-			OrgId:                 v.OrgId,
-			Folder:                v.Folder,
-			FolderUid:             v.FolderUid,
-			Editable:              v.Editable,
-			Options:               v.Options,
-			DisableDeletion:       v.DisableDeletion,
-			UpdateIntervalSeconds: v.UpdateIntervalSeconds,
+			Name:                  v.Name.Value(),
+			Type:                  v.Type.Value(),
+			OrgId:                 v.OrgId.Value(),
+			Folder:                v.Folder.Value(),
+			FolderUid:             v.FolderUid.Value(),
+			Editable:              v.Editable.Value(),
+			Options:               v.Options.Value(),
+			DisableDeletion:       v.DisableDeletion.Value(),
+			UpdateIntervalSeconds: v.UpdateIntervalSeconds.Value(),
 		})
 	}
 

pkg/services/provisioning/datasources/config_reader_test.go

@@ -1,6 +1,7 @@
 package datasources
 
 import (
+	"os"
 	"testing"
 
 	"github.com/grafana/grafana/pkg/bus"
@@ -146,8 +147,10 @@ func TestDatasourceAsConfig(t *testing.T) {
 		})
 
 		Convey("can read all properties from version 1", func() {
+			_ = os.Setenv("TEST_VAR", "name")
 			cfgProvifer := &configReader{log: log.New("test logger")}
 			cfg, err := cfgProvifer.readConfig(allProperties)
+			_ = os.Unsetenv("TEST_VAR")
 			if err != nil {
 				t.Fatalf("readConfig return an error %v", err)
 			}

pkg/services/provisioning/datasources/testdata/all-properties/all-properties.yaml

@@ -1,7 +1,7 @@
 apiVersion: 1
 
 datasources:
-  - name: name
+  - name: $TEST_VAR
     type: type
     access: proxy
     orgId: 2

pkg/services/provisioning/datasources/types.go

@@ -4,6 +4,7 @@ import (
 	"github.com/grafana/grafana/pkg/components/simplejson"
 	"github.com/grafana/grafana/pkg/log"
 	"github.com/grafana/grafana/pkg/models"
+	"github.com/grafana/grafana/pkg/services/provisioning/values"
 )
 
 type ConfigVersion struct {
@@ -64,8 +65,8 @@ type DeleteDatasourceConfigV0 struct {
 }
 
 type DeleteDatasourceConfigV1 struct {
-	OrgId int64  `json:"orgId" yaml:"orgId"`
-	Name  string `json:"name" yaml:"name"`
+	OrgId values.Int64Value  `json:"orgId" yaml:"orgId"`
+	Name  values.StringValue `json:"name" yaml:"name"`
 }
 
 type DataSourceFromConfigV0 struct {
@@ -89,23 +90,23 @@ type DataSourceFromConfigV0 struct {
 }
 
 type DataSourceFromConfigV1 struct {
-	OrgId             int64                  `json:"orgId" yaml:"orgId"`
-	Version           int                    `json:"version" yaml:"version"`
-	Name              string                 `json:"name" yaml:"name"`
-	Type              string                 `json:"type" yaml:"type"`
-	Access            string                 `json:"access" yaml:"access"`
-	Url               string                 `json:"url" yaml:"url"`
-	Password          string                 `json:"password" yaml:"password"`
-	User              string                 `json:"user" yaml:"user"`
-	Database          string                 `json:"database" yaml:"database"`
-	BasicAuth         bool                   `json:"basicAuth" yaml:"basicAuth"`
-	BasicAuthUser     string                 `json:"basicAuthUser" yaml:"basicAuthUser"`
-	BasicAuthPassword string                 `json:"basicAuthPassword" yaml:"basicAuthPassword"`
-	WithCredentials   bool                   `json:"withCredentials" yaml:"withCredentials"`
-	IsDefault         bool                   `json:"isDefault" yaml:"isDefault"`
-	JsonData          map[string]interface{} `json:"jsonData" yaml:"jsonData"`
-	SecureJsonData    map[string]string      `json:"secureJsonData" yaml:"secureJsonData"`
-	Editable          bool                   `json:"editable" yaml:"editable"`
+	OrgId             values.Int64Value     `json:"orgId" yaml:"orgId"`
+	Version           values.IntValue       `json:"version" yaml:"version"`
+	Name              values.StringValue    `json:"name" yaml:"name"`
+	Type              values.StringValue    `json:"type" yaml:"type"`
+	Access            values.StringValue    `json:"access" yaml:"access"`
+	Url               values.StringValue    `json:"url" yaml:"url"`
+	Password          values.StringValue    `json:"password" yaml:"password"`
+	User              values.StringValue    `json:"user" yaml:"user"`
+	Database          values.StringValue    `json:"database" yaml:"database"`
+	BasicAuth         values.BoolValue      `json:"basicAuth" yaml:"basicAuth"`
+	BasicAuthUser     values.StringValue    `json:"basicAuthUser" yaml:"basicAuthUser"`
+	BasicAuthPassword values.StringValue    `json:"basicAuthPassword" yaml:"basicAuthPassword"`
+	WithCredentials   values.BoolValue      `json:"withCredentials" yaml:"withCredentials"`
+	IsDefault         values.BoolValue      `json:"isDefault" yaml:"isDefault"`
+	JsonData          values.JSONValue      `json:"jsonData" yaml:"jsonData"`
+	SecureJsonData    values.StringMapValue `json:"secureJsonData" yaml:"secureJsonData"`
+	Editable          values.BoolValue      `json:"editable" yaml:"editable"`
 }
 
 func (cfg *DatasourcesAsConfigV1) mapToDatasourceFromConfig(apiVersion int64) *DatasourcesAsConfig {
@@ -119,36 +120,47 @@ func (cfg *DatasourcesAsConfigV1) mapToDatasourceFromConfig(apiVersion int64) *D
 
 	for _, ds := range cfg.Datasources {
 		r.Datasources = append(r.Datasources, &DataSourceFromConfig{
-			OrgId:             ds.OrgId,
-			Name:              ds.Name,
-			Type:              ds.Type,
-			Access:            ds.Access,
-			Url:               ds.Url,
-			Password:          ds.Password,
-			User:              ds.User,
-			Database:          ds.Database,
-			BasicAuth:         ds.BasicAuth,
-			BasicAuthUser:     ds.BasicAuthUser,
-			BasicAuthPassword: ds.BasicAuthPassword,
-			WithCredentials:   ds.WithCredentials,
-			IsDefault:         ds.IsDefault,
-			JsonData:          ds.JsonData,
-			SecureJsonData:    ds.SecureJsonData,
-			Editable:          ds.Editable,
-			Version:           ds.Version,
+			OrgId:             ds.OrgId.Value(),
+			Name:              ds.Name.Value(),
+			Type:              ds.Type.Value(),
+			Access:            ds.Access.Value(),
+			Url:               ds.Url.Value(),
+			Password:          ds.Password.Value(),
+			User:              ds.User.Value(),
+			Database:          ds.Database.Value(),
+			BasicAuth:         ds.BasicAuth.Value(),
+			BasicAuthUser:     ds.BasicAuthUser.Value(),
+			BasicAuthPassword: ds.BasicAuthPassword.Value(),
+			WithCredentials:   ds.WithCredentials.Value(),
+			IsDefault:         ds.IsDefault.Value(),
+			JsonData:          ds.JsonData.Value(),
+			SecureJsonData:    ds.SecureJsonData.Value(),
+			Editable:          ds.Editable.Value(),
+			Version:           ds.Version.Value(),
 		})
-		if ds.Password != "" {
-			cfg.log.Warn("[Deprecated] the use of password field is deprecated. Please use secureJsonData.password", "datasource name", ds.Name)
+
+		// Using Raw value for the warnings here so that even if it uses env interpolation and the env var is empty
+		// it will still warn
+		if len(ds.Password.Raw) > 0 {
+			cfg.log.Warn(
+				"[Deprecated] the use of password field is deprecated. Please use secureJsonData.password",
+				"datasource name",
+				ds.Name.Value(),
+			)
 		}
-		if ds.BasicAuthPassword != "" {
-			cfg.log.Warn("[Deprecated] the use of basicAuthPassword field is deprecated. Please use secureJsonData.basicAuthPassword", "datasource name", ds.Name)
+		if len(ds.BasicAuthPassword.Raw) > 0 {
+			cfg.log.Warn(
+				"[Deprecated] the use of basicAuthPassword field is deprecated. Please use secureJsonData.basicAuthPassword",
+				"datasource name",
+				ds.Name.Value(),
+			)
 		}
 	}
 
 	for _, ds := range cfg.DeleteDatasources {
 		r.DeleteDatasources = append(r.DeleteDatasources, &DeleteDatasourceConfig{
-			OrgId: ds.OrgId,
-			Name:  ds.Name,
+			OrgId: ds.OrgId.Value(),
+			Name:  ds.Name.Value(),
 		})
 	}
 

pkg/services/provisioning/notifiers/alert_notifications.go

@@ -131,39 +131,6 @@ func (dc *NotificationProvisioner) mergeNotifications(notificationToMerge []*not
 	return nil
 }
 
-func (cfg *notificationsAsConfig) mapToNotificationFromConfig() *notificationsAsConfig {
-	r := &notificationsAsConfig{}
-	if cfg == nil {
-		return r
-	}
-
-	for _, notification := range cfg.Notifications {
-		r.Notifications = append(r.Notifications, &notificationFromConfig{
-			Uid:                   notification.Uid,
-			OrgId:                 notification.OrgId,
-			OrgName:               notification.OrgName,
-			Name:                  notification.Name,
-			Type:                  notification.Type,
-			IsDefault:             notification.IsDefault,
-			Settings:              notification.Settings,
-			DisableResolveMessage: notification.DisableResolveMessage,
-			Frequency:             notification.Frequency,
-			SendReminder:          notification.SendReminder,
-		})
-	}
-
-	for _, notification := range cfg.DeleteNotifications {
-		r.DeleteNotifications = append(r.DeleteNotifications, &deleteNotificationConfig{
-			Uid:     notification.Uid,
-			OrgId:   notification.OrgId,
-			OrgName: notification.OrgName,
-			Name:    notification.Name,
-		})
-	}
-
-	return r
-}
-
 func (dc *NotificationProvisioner) applyChanges(configPath string) error {
 	configs, err := dc.cfgProvider.readConfig(configPath)
 	if err != nil {

pkg/services/provisioning/notifiers/config_reader.go

@@ -63,7 +63,7 @@ func (cr *configReader) parseNotificationConfig(path string, file os.FileInfo) (
 		return nil, err
 	}
 
-	var cfg *notificationsAsConfig
+	var cfg *notificationsAsConfigV0
 	err = yaml.Unmarshal(yamlFile, &cfg)
 	if err != nil {
 		return nil, err

pkg/services/provisioning/notifiers/config_reader_test.go

@@ -1,6 +1,7 @@
 package notifiers
 
 import (
+	"os"
 	"testing"
 
 	"github.com/grafana/grafana/pkg/log"
@@ -43,8 +44,10 @@ func TestNotificationAsConfig(t *testing.T) {
 		})
 
 		Convey("Can read correct properties", func() {
+			_ = os.Setenv("TEST_VAR", "default")
 			cfgProvifer := &configReader{log: log.New("test logger")}
 			cfg, err := cfgProvifer.readConfig(correct_properties)
+			_ = os.Unsetenv("TEST_VAR")
 			if err != nil {
 				t.Fatalf("readConfig return an error %v", err)
 			}

pkg/services/provisioning/notifiers/testdata/test-configs/correct-properties/correct-properties.yaml

@@ -1,5 +1,5 @@
 notifiers:
-  - name: default-slack-notification
+  - name: $TEST_VAR-slack-notification
     type: slack
     uid: notifier1
     org_id: 2
@@ -39,4 +39,4 @@ delete_notifiers:
     org_id: 0
     uid: "notifier3"
   - name: Deleted notification with whitespaces in name
-    uid: "notifier4"
+    uid: "notifier4"

pkg/services/provisioning/notifiers/types.go

@@ -1,30 +1,61 @@
 package notifiers
 
-import "github.com/grafana/grafana/pkg/components/simplejson"
+import (
+	"github.com/grafana/grafana/pkg/components/simplejson"
+	"github.com/grafana/grafana/pkg/services/provisioning/values"
+)
 
+// notificationsAsConfig is normalized data object for notifications config data. Any config version should be mappable
+// to this type.
 type notificationsAsConfig struct {
-	Notifications       []*notificationFromConfig   `json:"notifiers" yaml:"notifiers"`
-	DeleteNotifications []*deleteNotificationConfig `json:"delete_notifiers" yaml:"delete_notifiers"`
+	Notifications       []*notificationFromConfig
+	DeleteNotifications []*deleteNotificationConfig
 }
 
 type deleteNotificationConfig struct {
-	Uid     string `json:"uid" yaml:"uid"`
-	Name    string `json:"name" yaml:"name"`
-	OrgId   int64  `json:"org_id" yaml:"org_id"`
-	OrgName string `json:"org_name" yaml:"org_name"`
+	Uid     string
+	Name    string
+	OrgId   int64
+	OrgName string
 }
 
 type notificationFromConfig struct {
-	Uid                   string                 `json:"uid" yaml:"uid"`
-	OrgId                 int64                  `json:"org_id" yaml:"org_id"`
-	OrgName               string                 `json:"org_name" yaml:"org_name"`
-	Name                  string                 `json:"name" yaml:"name"`
-	Type                  string                 `json:"type" yaml:"type"`
-	SendReminder          bool                   `json:"send_reminder" yaml:"send_reminder"`
-	DisableResolveMessage bool                   `json:"disable_resolve_message" yaml:"disable_resolve_message"`
-	Frequency             string                 `json:"frequency" yaml:"frequency"`
-	IsDefault             bool                   `json:"is_default" yaml:"is_default"`
-	Settings              map[string]interface{} `json:"settings" yaml:"settings"`
+	Uid                   string
+	OrgId                 int64
+	OrgName               string
+	Name                  string
+	Type                  string
+	SendReminder          bool
+	DisableResolveMessage bool
+	Frequency             string
+	IsDefault             bool
+	Settings              map[string]interface{}
+}
+
+// notificationsAsConfigV0 is mapping for zero version configs. This is mapped to its normalised version.
+type notificationsAsConfigV0 struct {
+	Notifications       []*notificationFromConfigV0   `json:"notifiers" yaml:"notifiers"`
+	DeleteNotifications []*deleteNotificationConfigV0 `json:"delete_notifiers" yaml:"delete_notifiers"`
+}
+
+type deleteNotificationConfigV0 struct {
+	Uid     values.StringValue `json:"uid" yaml:"uid"`
+	Name    values.StringValue `json:"name" yaml:"name"`
+	OrgId   values.Int64Value  `json:"org_id" yaml:"org_id"`
+	OrgName values.StringValue `json:"org_name" yaml:"org_name"`
+}
+
+type notificationFromConfigV0 struct {
+	Uid                   values.StringValue `json:"uid" yaml:"uid"`
+	OrgId                 values.Int64Value  `json:"org_id" yaml:"org_id"`
+	OrgName               values.StringValue `json:"org_name" yaml:"org_name"`
+	Name                  values.StringValue `json:"name" yaml:"name"`
+	Type                  values.StringValue `json:"type" yaml:"type"`
+	SendReminder          values.BoolValue   `json:"send_reminder" yaml:"send_reminder"`
+	DisableResolveMessage values.BoolValue   `json:"disable_resolve_message" yaml:"disable_resolve_message"`
+	Frequency             values.StringValue `json:"frequency" yaml:"frequency"`
+	IsDefault             values.BoolValue   `json:"is_default" yaml:"is_default"`
+	Settings              values.JSONValue   `json:"settings" yaml:"settings"`
 }
 
 func (notification notificationFromConfig) SettingsToJson() *simplejson.Json {
@@ -36,3 +67,38 @@ func (notification notificationFromConfig) SettingsToJson() *simplejson.Json {
 	}
 	return settings
 }
+
+// mapToNotificationFromConfig maps config syntax to normalized notificationsAsConfig object. Every version
+// of the config syntax should have this function.
+func (cfg *notificationsAsConfigV0) mapToNotificationFromConfig() *notificationsAsConfig {
+	r := &notificationsAsConfig{}
+	if cfg == nil {
+		return r
+	}
+
+	for _, notification := range cfg.Notifications {
+		r.Notifications = append(r.Notifications, &notificationFromConfig{
+			Uid:                   notification.Uid.Value(),
+			OrgId:                 notification.OrgId.Value(),
+			OrgName:               notification.OrgName.Value(),
+			Name:                  notification.Name.Value(),
+			Type:                  notification.Type.Value(),
+			IsDefault:             notification.IsDefault.Value(),
+			Settings:              notification.Settings.Value(),
+			DisableResolveMessage: notification.DisableResolveMessage.Value(),
+			Frequency:             notification.Frequency.Value(),
+			SendReminder:          notification.SendReminder.Value(),
+		})
+	}
+
+	for _, notification := range cfg.DeleteNotifications {
+		r.DeleteNotifications = append(r.DeleteNotifications, &deleteNotificationConfig{
+			Uid:     notification.Uid.Value(),
+			OrgId:   notification.OrgId.Value(),
+			OrgName: notification.OrgName.Value(),
+			Name:    notification.Name.Value(),
+		})
+	}
+
+	return r
+}

pkg/services/provisioning/values/values.go

@@ -0,0 +1,206 @@
+// A set of value types to use in provisioning. They add custom unmarshaling logic that puts the string values
+// through os.ExpandEnv.
+// Usage:
+// type Data struct {
+//   Field StringValue `yaml:"field"` // Instead of string
+// }
+// d := &Data{}
+// // unmarshal into d
+// d.Field.Value() // returns the final interpolated value from the yaml file
+//
+package values
+
+import (
+	"github.com/pkg/errors"
+	"os"
+	"reflect"
+	"strconv"
+)
+
+type IntValue struct {
+	value int
+	Raw   string
+}
+
+func (val *IntValue) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	interpolated, err := getInterpolated(unmarshal)
+	if err != nil {
+		return err
+	}
+	if len(interpolated.value) == 0 {
+		// To keep the same behaviour as the yaml lib which just does not set the value if it is empty.
+		return nil
+	}
+	val.Raw = interpolated.raw
+	val.value, err = strconv.Atoi(interpolated.value)
+	return errors.Wrap(err, "cannot convert value int")
+}
+
+func (val *IntValue) Value() int {
+	return val.value
+}
+
+type Int64Value struct {
+	value int64
+	Raw   string
+}
+
+func (val *Int64Value) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	interpolated, err := getInterpolated(unmarshal)
+	if err != nil {
+		return err
+	}
+	if len(interpolated.value) == 0 {
+		// To keep the same behaviour as the yaml lib which just does not set the value if it is empty.
+		return nil
+	}
+	val.Raw = interpolated.raw
+	val.value, err = strconv.ParseInt(interpolated.value, 10, 64)
+	return err
+}
+
+func (val *Int64Value) Value() int64 {
+	return val.value
+}
+
+type StringValue struct {
+	value string
+	Raw   string
+}
+
+func (val *StringValue) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	interpolated, err := getInterpolated(unmarshal)
+	if err != nil {
+		return err
+	}
+	val.Raw = interpolated.raw
+	val.value = interpolated.value
+	return err
+}
+
+func (val *StringValue) Value() string {
+	return val.value
+}
+
+type BoolValue struct {
+	value bool
+	Raw   string
+}
+
+func (val *BoolValue) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	interpolated, err := getInterpolated(unmarshal)
+	if err != nil {
+		return err
+	}
+	val.Raw = interpolated.raw
+	val.value, err = strconv.ParseBool(interpolated.value)
+	return err
+}
+
+func (val *BoolValue) Value() bool {
+	return val.value
+}
+
+type JSONValue struct {
+	value map[string]interface{}
+	Raw   map[string]interface{}
+}
+
+func (val *JSONValue) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	unmarshaled := make(map[string]interface{})
+	err := unmarshal(unmarshaled)
+	if err != nil {
+		return err
+	}
+	val.Raw = unmarshaled
+	interpolated := make(map[string]interface{})
+	for key, val := range unmarshaled {
+		interpolated[key] = tranformInterface(val)
+	}
+	val.value = interpolated
+	return err
+}
+
+func (val *JSONValue) Value() map[string]interface{} {
+	return val.value
+}
+
+type StringMapValue struct {
+	value map[string]string
+	Raw   map[string]string
+}
+
+func (val *StringMapValue) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	unmarshaled := make(map[string]string)
+	err := unmarshal(unmarshaled)
+	if err != nil {
+		return err
+	}
+	val.Raw = unmarshaled
+	interpolated := make(map[string]string)
+	for key, val := range unmarshaled {
+		interpolated[key] = interpolateValue(val)
+	}
+	val.value = interpolated
+	return err
+}
+
+func (val *StringMapValue) Value() map[string]string {
+	return val.value
+}
+
+// tranformInterface tries to transform any interface type into proper value with env expansion. It travers maps and
+// slices and the actual interpolation is done on all simple string values in the structure. It returns a copy of any
+// map or slice value instead of modifying them in place.
+func tranformInterface(i interface{}) interface{} {
+	switch reflect.TypeOf(i).Kind() {
+	case reflect.Slice:
+		return transformSlice(i.([]interface{}))
+	case reflect.Map:
+		return transformMap(i.(map[interface{}]interface{}))
+	case reflect.String:
+		return interpolateValue(i.(string))
+	default:
+		// Was int, float or some other value that we do not need to do any transform on.
+		return i
+	}
+}
+
+func transformSlice(i []interface{}) interface{} {
+	var transformed []interface{}
+	for _, val := range i {
+		transformed = append(transformed, tranformInterface(val))
+	}
+	return transformed
+}
+
+func transformMap(i map[interface{}]interface{}) interface{} {
+	transformed := make(map[interface{}]interface{})
+	for key, val := range i {
+		transformed[key] = tranformInterface(val)
+	}
+	return transformed
+}
+
+// interpolateValue returns final value after interpolation. At the moment only env var interpolation is done
+// here but in the future something like interpolation from file could be also done here.
+func interpolateValue(val string) string {
+	return os.ExpandEnv(val)
+}
+
+type interpolated struct {
+	value string
+	raw   string
+}
+
+// getInterpolated unmarshals the value as string and runs interpolation on it. It is the responsibility of each
+// value type to convert this string value to appropriate type.
+func getInterpolated(unmarshal func(interface{}) error) (*interpolated, error) {
+	var raw string
+	err := unmarshal(&raw)
+	if err != nil {
+		return &interpolated{}, err
+	}
+	value := interpolateValue(raw)
+	return &interpolated{raw: raw, value: value}, nil
+}

pkg/services/provisioning/values/values_test.go

@@ -0,0 +1,210 @@
+package values
+
+import (
+	. "github.com/smartystreets/goconvey/convey"
+	"gopkg.in/yaml.v2"
+	"os"
+	"testing"
+)
+
+func TestValues(t *testing.T) {
+	Convey("Values", t, func() {
+		os.Setenv("INT", "1")
+		os.Setenv("STRING", "test")
+		os.Setenv("BOOL", "true")
+
+		Convey("IntValue", func() {
+			type Data struct {
+				Val IntValue `yaml:"val"`
+			}
+			d := &Data{}
+
+			Convey("Should unmarshal simple number", func() {
+				unmarshalingTest(`val: 1`, d)
+				So(d.Val.Value(), ShouldEqual, 1)
+				So(d.Val.Raw, ShouldEqual, "1")
+			})
+
+			Convey("Should unmarshal env var", func() {
+				unmarshalingTest(`val: $INT`, d)
+				So(d.Val.Value(), ShouldEqual, 1)
+				So(d.Val.Raw, ShouldEqual, "$INT")
+			})
+
+			Convey("Should ignore empty value", func() {
+				unmarshalingTest(`val: `, d)
+				So(d.Val.Value(), ShouldEqual, 0)
+				So(d.Val.Raw, ShouldEqual, "")
+			})
+		})
+
+		Convey("StringValue", func() {
+			type Data struct {
+				Val StringValue `yaml:"val"`
+			}
+			d := &Data{}
+
+			Convey("Should unmarshal simple string", func() {
+				unmarshalingTest(`val: test`, d)
+				So(d.Val.Value(), ShouldEqual, "test")
+				So(d.Val.Raw, ShouldEqual, "test")
+			})
+
+			Convey("Should unmarshal env var", func() {
+				unmarshalingTest(`val: $STRING`, d)
+				So(d.Val.Value(), ShouldEqual, "test")
+				So(d.Val.Raw, ShouldEqual, "$STRING")
+			})
+
+			Convey("Should ignore empty value", func() {
+				unmarshalingTest(`val: `, d)
+				So(d.Val.Value(), ShouldEqual, "")
+				So(d.Val.Raw, ShouldEqual, "")
+			})
+		})
+
+		Convey("BoolValue", func() {
+			type Data struct {
+				Val BoolValue `yaml:"val"`
+			}
+			d := &Data{}
+
+			Convey("Should unmarshal bool value", func() {
+				unmarshalingTest(`val: true`, d)
+				So(d.Val.Value(), ShouldBeTrue)
+				So(d.Val.Raw, ShouldEqual, "true")
+			})
+
+			Convey("Should unmarshal explicit string", func() {
+				unmarshalingTest(`val: "true"`, d)
+				So(d.Val.Value(), ShouldBeTrue)
+				So(d.Val.Raw, ShouldEqual, "true")
+			})
+
+			Convey("Should unmarshal env var", func() {
+				unmarshalingTest(`val: $BOOL`, d)
+				So(d.Val.Value(), ShouldBeTrue)
+				So(d.Val.Raw, ShouldEqual, "$BOOL")
+			})
+
+			Convey("Should ignore empty value", func() {
+				unmarshalingTest(`val: `, d)
+				So(d.Val.Value(), ShouldBeFalse)
+				So(d.Val.Raw, ShouldEqual, "")
+			})
+		})
+
+		Convey("JSONValue", func() {
+
+			type Data struct {
+				Val JSONValue `yaml:"val"`
+			}
+			d := &Data{}
+
+			Convey("Should unmarshal variable nesting", func() {
+				doc := `
+                 val:
+                   one: 1
+                   two: $STRING
+                   three:
+                     - 1
+                     - two
+                     - three:
+                         inside: $STRING
+                   four:
+                     nested:
+                       onemore: $INT
+                   multiline: >
+                     Some text with $STRING
+                   anchor: &label $INT
+                   anchored: *label
+               `
+				unmarshalingTest(doc, d)
+
+				type anyMap = map[interface{}]interface{}
+				So(d.Val.Value(), ShouldResemble, map[string]interface{}{
+					"one": 1,
+					"two": "test",
+					"three": []interface{}{
+						1, "two", anyMap{
+							"three": anyMap{
+								"inside": "test",
+							},
+						},
+					},
+					"four": anyMap{
+						"nested": anyMap{
+							"onemore": "1",
+						},
+					},
+					"multiline": "Some text with test\n",
+					"anchor":    "1",
+					"anchored":  "1",
+				})
+
+				So(d.Val.Raw, ShouldResemble, map[string]interface{}{
+					"one": 1,
+					"two": "$STRING",
+					"three": []interface{}{
+						1, "two", anyMap{
+							"three": anyMap{
+								"inside": "$STRING",
+							},
+						},
+					},
+					"four": anyMap{
+						"nested": anyMap{
+							"onemore": "$INT",
+						},
+					},
+					"multiline": "Some text with $STRING\n",
+					"anchor":    "$INT",
+					"anchored":  "$INT",
+				})
+			})
+		})
+
+		Convey("StringMapValue", func() {
+			type Data struct {
+				Val StringMapValue `yaml:"val"`
+			}
+			d := &Data{}
+
+			Convey("Should unmarshal mapping", func() {
+				doc := `
+                 val:
+                   one: 1
+                   two: "test string"
+                   three: $STRING
+                   four: true
+               `
+				unmarshalingTest(doc, d)
+				So(d.Val.Value(), ShouldResemble, map[string]string{
+					"one":   "1",
+					"two":   "test string",
+					"three": "test",
+					"four":  "true",
+				})
+
+				So(d.Val.Raw, ShouldResemble, map[string]string{
+					"one":   "1",
+					"two":   "test string",
+					"three": "$STRING",
+					"four":  "true",
+				})
+
+			})
+		})
+
+		Reset(func() {
+			os.Unsetenv("INT")
+			os.Unsetenv("STRING")
+			os.Unsetenv("BOOL")
+		})
+	})
+}
+
+func unmarshalingTest(document string, out interface{}) {
+	err := yaml.Unmarshal([]byte(document), out)
+	So(err, ShouldBeNil)
+}