add timeout test for alert handling.

pkg/services/alerting/engine_test.go

@@ -4,7 +4,11 @@ import (
 	"context"
 	"errors"
 	"math"
+	"net"
+	"net/http"
+	"net/http/httptest"
 	"testing"
+	"time"
 
 	. "github.com/smartystreets/goconvey/convey"
 )
@@ -34,6 +38,106 @@ func (handler *FakeResultHandler) Handle(evalContext *EvalContext) error {
 	return nil
 }
 
+type FakeCommonTimeoutHandler struct {
+	TransportTimeoutDuration time.Duration
+	ServerBusySleepDuration  time.Duration
+	EvalSucceed              bool
+	ResultHandleSucceed      bool
+}
+
+func NewFakeCommonTimeoutHandler(transportTimeoutDuration time.Duration, serverBusySleepDuration time.Duration) *FakeCommonTimeoutHandler {
+	return &FakeCommonTimeoutHandler{
+		TransportTimeoutDuration: transportTimeoutDuration,
+		ServerBusySleepDuration:  serverBusySleepDuration,
+		EvalSucceed:              false,
+		ResultHandleSucceed:      false,
+	}
+}
+
+func (handler *FakeCommonTimeoutHandler) Eval(evalContext *EvalContext) {
+	// 1. prepare mock server
+	path := "/evaltimeout"
+	srv := runBusyServer(path, handler.ServerBusySleepDuration)
+	defer srv.Close()
+
+	// 2. send requests
+	url := srv.URL + path
+	res, err := sendRequest(evalContext.Ctx, url, handler.TransportTimeoutDuration)
+	if res != nil {
+		defer res.Body.Close()
+	}
+
+	if err != nil {
+		evalContext.Error = errors.New("Fake evaluation timeout test failure")
+		return
+	}
+
+	if res.StatusCode == 200 {
+		handler.EvalSucceed = true
+	}
+
+	evalContext.Error = errors.New("Fake evaluation timeout test failure; wrong response")
+}
+
+func (handler *FakeCommonTimeoutHandler) Handle(evalContext *EvalContext) error {
+	// 1. prepare mock server
+	path := "/resulthandle"
+	srv := runBusyServer(path, handler.ServerBusySleepDuration)
+	defer srv.Close()
+
+	// 2. send requests
+	url := srv.URL + path
+	res, err := sendRequest(evalContext.Ctx, url, handler.TransportTimeoutDuration)
+	if res != nil {
+		defer res.Body.Close()
+	}
+
+	if err != nil {
+		evalContext.Error = errors.New("Fake result handle timeout test failure")
+		return evalContext.Error
+	}
+
+	if res.StatusCode == 200 {
+		handler.ResultHandleSucceed = true
+		return nil
+	}
+
+	evalContext.Error = errors.New("Fake result handle timeout test failure; wrong response")
+
+	return evalContext.Error
+}
+
+func runBusyServer(path string, serverBusySleepDuration time.Duration) *httptest.Server {
+	mux := http.NewServeMux()
+	server := httptest.NewServer(mux)
+
+	mux.HandleFunc(path, func(w http.ResponseWriter, r *http.Request) {
+		time.Sleep(serverBusySleepDuration)
+	})
+
+	return server
+}
+
+func sendRequest(context context.Context, url string, transportTimeoutInterval time.Duration) (resp *http.Response, err error) {
+	req, err := http.NewRequest("GET", url, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	req = req.WithContext(context)
+
+	transport := http.Transport{
+		Dial: (&net.Dialer{
+			Timeout:   transportTimeoutInterval,
+			KeepAlive: transportTimeoutInterval,
+		}).Dial,
+	}
+	client := http.Client{
+		Transport: &transport,
+	}
+
+	return client.Do(req)
+}
 func TestEngineProcessJob(t *testing.T) {
 	Convey("Alerting engine job processing", t, func() {
 		engine := NewEngine()
@@ -113,6 +217,29 @@ func TestEngineProcessJob(t *testing.T) {
 				engine.processJobWithRetry(context.TODO(), job)
 				So(evalHandler.CallNb, ShouldEqual, expectedAttempts)
 			})
+
+			Convey("pended alert for datasource -> result handler should be worked", func() {
+				// reduce alert timeout to test quickly
+				originAlertTimeout := alertTimeout
+				alertTimeout = 5 * time.Second
+				transportTimeoutInterval := 5 * time.Second
+				serverBusySleepDuration := 4 * time.Second
+
+				evalHandler := NewFakeCommonTimeoutHandler(transportTimeoutInterval, serverBusySleepDuration)
+				resultHandler := NewFakeCommonTimeoutHandler(transportTimeoutInterval, serverBusySleepDuration)
+				engine.evalHandler = evalHandler
+				engine.resultHandler = resultHandler
+
+				engine.processJobWithRetry(context.TODO(), job)
+
+				So(evalHandler.EvalSucceed, ShouldEqual, true)
+				So(resultHandler.ResultHandleSucceed, ShouldEqual, true)
+
+				// initialize for other tests.
+				alertTimeout = originAlertTimeout
+				engine.resultHandler = &FakeResultHandler{}
+			})
+
 		})
 	})
 }