Cloudwatch: Correctly obtain IAM roles within ECS container tasks. (#7892)

We now obtain credentials based on the container task's role
rather than just relying on the credentials of the enclosing
container instance.

Fixes #6700.

pkg/api/cloudwatch/cloudwatch.go

@@ -3,7 +3,9 @@ package cloudwatch
 import (
 	"encoding/json"
 	"errors"
+	"fmt"
 	"io/ioutil"
+	"os"
 	"strings"
 	"sync"
 	"time"
@@ -12,6 +14,7 @@ import (
 	"github.com/aws/aws-sdk-go/aws/awsutil"
 	"github.com/aws/aws-sdk-go/aws/credentials"
 	"github.com/aws/aws-sdk-go/aws/credentials/ec2rolecreds"
+	"github.com/aws/aws-sdk-go/aws/credentials/endpointcreds"
 	"github.com/aws/aws-sdk-go/aws/ec2metadata"
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/cloudwatch"
@@ -122,7 +125,7 @@ func getCredentials(dsInfo *datasourceInfo) (*credentials.Credentials, error) {
 			[]credentials.Provider{
 				&credentials.EnvProvider{},
 				&credentials.SharedCredentialsProvider{Filename: "", Profile: dsInfo.Profile},
-				&ec2rolecreds.EC2RoleProvider{Client: ec2metadata.New(stsSess), ExpiryWindow: 5 * time.Minute},
+				remoteCredProvider(stsSess),
 			})
 		stsConfig := &aws.Config{
 			Region:      aws.String(dsInfo.Region),
@@ -176,6 +179,30 @@ func getCredentials(dsInfo *datasourceInfo) (*credentials.Credentials, error) {
 	return creds, nil
 }
 
+func remoteCredProvider(sess *session.Session) credentials.Provider {
+	ecsCredURI := os.Getenv("AWS_CONTAINER_CREDENTIALS_RELATIVE_URI")
+
+	if len(ecsCredURI) > 0 {
+		return ecsCredProvider(sess, ecsCredURI)
+	}
+	return ec2RoleProvider(sess)
+}
+
+func ecsCredProvider(sess *session.Session, uri string) credentials.Provider {
+	const host = `169.254.170.2`
+
+	c := ec2metadata.New(sess)
+	return endpointcreds.NewProviderClient(
+		c.Client.Config,
+		c.Client.Handlers,
+		fmt.Sprintf("http://%s%s", host, uri),
+		func(p *endpointcreds.Provider) { p.ExpiryWindow = 5 * time.Minute })
+}
+
+func ec2RoleProvider(sess *session.Session) credentials.Provider {
+	return &ec2rolecreds.EC2RoleProvider{Client: ec2metadata.New(sess), ExpiryWindow: 5 * time.Minute}
+}
+
 func getAwsConfig(req *cwRequest) (*aws.Config, error) {
 	creds, err := getCredentials(req.GetDatasourceInfo())
 	if err != nil {

pkg/api/cloudwatch/cloudwatch_test.go

@@ -0,0 +1,41 @@
+package cloudwatch
+
+import (
+	"fmt"
+	"os"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws/credentials/ec2rolecreds"
+	"github.com/aws/aws-sdk-go/aws/credentials/endpointcreds"
+	"github.com/aws/aws-sdk-go/aws/session"
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func TestECSCredProvider(t *testing.T) {
+	Convey("Running in an ECS container task", t, func() {
+		defer os.Clearenv()
+		os.Setenv("AWS_CONTAINER_CREDENTIALS_RELATIVE_URI", "/abc/123")
+
+		provider := remoteCredProvider(&session.Session{})
+
+		So(provider, ShouldNotBeNil)
+
+		ecsProvider, ok := provider.(*endpointcreds.Provider)
+		So(ecsProvider, ShouldNotBeNil)
+		So(ok, ShouldBeTrue)
+
+		So(ecsProvider.Client.Endpoint, ShouldEqual, fmt.Sprintf("http://169.254.170.2/abc/123"))
+	})
+}
+
+func TestDefaultEC2RoleProvider(t *testing.T) {
+	Convey("Running outside an ECS container task", t, func() {
+		provider := remoteCredProvider(&session.Session{})
+
+		So(provider, ShouldNotBeNil)
+
+		ec2Provider, ok := provider.(*ec2rolecreds.EC2RoleProvider)
+		So(ec2Provider, ShouldNotBeNil)
+		So(ok, ShouldBeTrue)
+	})
+}