|
@@ -1,10 +1,17 @@
|
|
|
package es
|
|
package es
|
|
|
|
|
|
|
|
import (
|
|
import (
|
|
|
|
|
+ "bytes"
|
|
|
|
|
+ "context"
|
|
|
|
|
+ "fmt"
|
|
|
|
|
+ "io/ioutil"
|
|
|
"net/http"
|
|
"net/http"
|
|
|
|
|
+ "net/http/httptest"
|
|
|
"testing"
|
|
"testing"
|
|
|
|
|
+ "time"
|
|
|
|
|
|
|
|
"github.com/grafana/grafana/pkg/components/simplejson"
|
|
"github.com/grafana/grafana/pkg/components/simplejson"
|
|
|
|
|
+ "github.com/grafana/grafana/pkg/tsdb"
|
|
|
|
|
|
|
|
"github.com/grafana/grafana/pkg/models"
|
|
"github.com/grafana/grafana/pkg/models"
|
|
|
. "github.com/smartystreets/goconvey/convey"
|
|
. "github.com/smartystreets/goconvey/convey"
|
|
@@ -85,131 +92,213 @@ func TestClient(t *testing.T) {
|
|
|
})
|
|
})
|
|
|
})
|
|
})
|
|
|
|
|
|
|
|
- Convey("v2", func() {
|
|
|
|
|
- ds := &models.DataSource{
|
|
|
|
|
- JsonData: simplejson.NewFromAny(map[string]interface{}{
|
|
|
|
|
- "esVersion": 2,
|
|
|
|
|
- }),
|
|
|
|
|
|
|
+ Convey("Given a fake http client", func() {
|
|
|
|
|
+ var responseBuffer *bytes.Buffer
|
|
|
|
|
+ var req *http.Request
|
|
|
|
|
+ ts := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
|
|
|
|
|
+ req = r
|
|
|
|
|
+ buf, err := ioutil.ReadAll(r.Body)
|
|
|
|
|
+ if err != nil {
|
|
|
|
|
+ t.Fatalf("Failed to read response body, err=%v", err)
|
|
|
|
|
+ }
|
|
|
|
|
+ responseBuffer = bytes.NewBuffer(buf)
|
|
|
|
|
+ }))
|
|
|
|
|
+
|
|
|
|
|
+ currentNewDatasourceHttpClient := newDatasourceHttpClient
|
|
|
|
|
+
|
|
|
|
|
+ newDatasourceHttpClient = func(ds *models.DataSource) (*http.Client, error) {
|
|
|
|
|
+ return ts.Client(), nil
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
- c, err := newV2Client(newFakeBaseClient(ds, []string{"test-*"}))
|
|
|
|
|
- So(err, ShouldBeNil)
|
|
|
|
|
- So(c, ShouldNotBeNil)
|
|
|
|
|
|
|
+ from := time.Date(2018, 5, 15, 17, 50, 0, 0, time.UTC)
|
|
|
|
|
+ to := time.Date(2018, 5, 15, 17, 55, 0, 0, time.UTC)
|
|
|
|
|
+ fromStr := fmt.Sprintf("%d", from.UnixNano()/int64(time.Millisecond))
|
|
|
|
|
+ toStr := fmt.Sprintf("%d", to.UnixNano()/int64(time.Millisecond))
|
|
|
|
|
+ timeRange := tsdb.NewTimeRange(fromStr, toStr)
|
|
|
|
|
|
|
|
- Convey("When creating multisearch requests should have correct headers", func() {
|
|
|
|
|
- multiRequests := c.createMultiSearchRequests([]*SearchRequest{
|
|
|
|
|
- {Index: "test-*"},
|
|
|
|
|
- })
|
|
|
|
|
- So(multiRequests, ShouldHaveLength, 1)
|
|
|
|
|
- header := multiRequests[0].header
|
|
|
|
|
- So(header, ShouldHaveLength, 3)
|
|
|
|
|
- So(header["index"], ShouldEqual, "test-*")
|
|
|
|
|
- So(header["ignore_unavailable"], ShouldEqual, true)
|
|
|
|
|
- So(header["search_type"], ShouldEqual, "count")
|
|
|
|
|
- })
|
|
|
|
|
- })
|
|
|
|
|
|
|
+ Convey("and a v2.x client", func() {
|
|
|
|
|
+ ds := models.DataSource{
|
|
|
|
|
+ Database: "[metrics-]YYYY.MM.DD",
|
|
|
|
|
+ Url: ts.URL,
|
|
|
|
|
+ JsonData: simplejson.NewFromAny(map[string]interface{}{
|
|
|
|
|
+ "esVersion": 2,
|
|
|
|
|
+ "timeField": "@timestamp",
|
|
|
|
|
+ "interval": "Daily",
|
|
|
|
|
+ }),
|
|
|
|
|
+ }
|
|
|
|
|
|
|
|
- Convey("v5", func() {
|
|
|
|
|
- ds := &models.DataSource{
|
|
|
|
|
- JsonData: simplejson.NewFromAny(map[string]interface{}{
|
|
|
|
|
- "esVersion": 5,
|
|
|
|
|
- }),
|
|
|
|
|
- }
|
|
|
|
|
|
|
+ c, err := NewClient(context.Background(), &ds, timeRange)
|
|
|
|
|
+ So(err, ShouldBeNil)
|
|
|
|
|
+ So(c, ShouldNotBeNil)
|
|
|
|
|
|
|
|
- c, err := newV5Client(newFakeBaseClient(ds, []string{"test-*"}))
|
|
|
|
|
- So(err, ShouldBeNil)
|
|
|
|
|
- So(c, ShouldNotBeNil)
|
|
|
|
|
|
|
+ Convey("When executing multi search", func() {
|
|
|
|
|
+ ms, err := createMultisearchForTest(c)
|
|
|
|
|
+ So(err, ShouldBeNil)
|
|
|
|
|
+ c.ExecuteMultisearch(ms)
|
|
|
|
|
|
|
|
- Convey("When creating multisearch requests should have correct headers", func() {
|
|
|
|
|
- multiRequests := c.createMultiSearchRequests([]*SearchRequest{
|
|
|
|
|
- {Index: "test-*"},
|
|
|
|
|
|
|
+ Convey("Should send correct request and payload", func() {
|
|
|
|
|
+ So(req, ShouldNotBeNil)
|
|
|
|
|
+ So(req.Method, ShouldEqual, http.MethodPost)
|
|
|
|
|
+ So(req.URL.Path, ShouldEqual, "/_msearch")
|
|
|
|
|
+
|
|
|
|
|
+ So(responseBuffer, ShouldNotBeNil)
|
|
|
|
|
+
|
|
|
|
|
+ headerBytes, err := responseBuffer.ReadBytes('\n')
|
|
|
|
|
+ So(err, ShouldBeNil)
|
|
|
|
|
+ bodyBytes := responseBuffer.Bytes()
|
|
|
|
|
+
|
|
|
|
|
+ jHeader, err := simplejson.NewJson(headerBytes)
|
|
|
|
|
+ So(err, ShouldBeNil)
|
|
|
|
|
+
|
|
|
|
|
+ jBody, err := simplejson.NewJson(bodyBytes)
|
|
|
|
|
+ So(err, ShouldBeNil)
|
|
|
|
|
+
|
|
|
|
|
+ fmt.Println("body", string(headerBytes))
|
|
|
|
|
+
|
|
|
|
|
+ So(jHeader.Get("index").MustString(), ShouldEqual, "metrics-2018.05.15")
|
|
|
|
|
+ So(jHeader.Get("ignore_unavailable").MustBool(false), ShouldEqual, true)
|
|
|
|
|
+ So(jHeader.Get("search_type").MustString(), ShouldEqual, "count")
|
|
|
|
|
+ So(jHeader.Get("max_concurrent_shard_requests").MustInt(10), ShouldEqual, 10)
|
|
|
|
|
+
|
|
|
|
|
+ Convey("and replace $__interval variable", func() {
|
|
|
|
|
+ So(jBody.GetPath("aggs", "2", "aggs", "1", "avg", "script").MustString(), ShouldEqual, "15000*@hostname")
|
|
|
|
|
+ })
|
|
|
|
|
+
|
|
|
|
|
+ Convey("and replace $__interval_ms variable", func() {
|
|
|
|
|
+ So(jBody.GetPath("aggs", "2", "date_histogram", "interval").MustString(), ShouldEqual, "15s")
|
|
|
|
|
+ })
|
|
|
|
|
+ })
|
|
|
})
|
|
})
|
|
|
- So(multiRequests, ShouldHaveLength, 1)
|
|
|
|
|
- header := multiRequests[0].header
|
|
|
|
|
- So(header, ShouldHaveLength, 3)
|
|
|
|
|
- So(header["index"], ShouldEqual, "test-*")
|
|
|
|
|
- So(header["ignore_unavailable"], ShouldEqual, true)
|
|
|
|
|
- So(header["search_type"], ShouldEqual, "query_then_fetch")
|
|
|
|
|
})
|
|
})
|
|
|
- })
|
|
|
|
|
|
|
|
|
|
- Convey("v5.6", func() {
|
|
|
|
|
- Convey("With default settings", func() {
|
|
|
|
|
|
|
+ Convey("and a v5.x client", func() {
|
|
|
ds := models.DataSource{
|
|
ds := models.DataSource{
|
|
|
|
|
+ Database: "[metrics-]YYYY.MM.DD",
|
|
|
|
|
+ Url: ts.URL,
|
|
|
JsonData: simplejson.NewFromAny(map[string]interface{}{
|
|
JsonData: simplejson.NewFromAny(map[string]interface{}{
|
|
|
- "esVersion": 56,
|
|
|
|
|
|
|
+ "esVersion": 5,
|
|
|
|
|
+ "maxConcurrentShardRequests": 100,
|
|
|
|
|
+ "timeField": "@timestamp",
|
|
|
|
|
+ "interval": "Daily",
|
|
|
}),
|
|
}),
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
- c, err := newV56Client(newFakeBaseClient(&ds, []string{"test-*"}))
|
|
|
|
|
|
|
+ c, err := NewClient(context.Background(), &ds, timeRange)
|
|
|
So(err, ShouldBeNil)
|
|
So(err, ShouldBeNil)
|
|
|
So(c, ShouldNotBeNil)
|
|
So(c, ShouldNotBeNil)
|
|
|
|
|
|
|
|
- Convey("When creating multisearch requests should have correct headers", func() {
|
|
|
|
|
- multiRequests := c.createMultiSearchRequests([]*SearchRequest{
|
|
|
|
|
- {Index: "test-*"},
|
|
|
|
|
|
|
+ Convey("When executing multi search", func() {
|
|
|
|
|
+ ms, err := createMultisearchForTest(c)
|
|
|
|
|
+ So(err, ShouldBeNil)
|
|
|
|
|
+ c.ExecuteMultisearch(ms)
|
|
|
|
|
+
|
|
|
|
|
+ Convey("Should send correct request and payload", func() {
|
|
|
|
|
+ So(req, ShouldNotBeNil)
|
|
|
|
|
+ So(req.Method, ShouldEqual, http.MethodPost)
|
|
|
|
|
+ So(req.URL.Path, ShouldEqual, "/_msearch")
|
|
|
|
|
+
|
|
|
|
|
+ So(responseBuffer, ShouldNotBeNil)
|
|
|
|
|
+
|
|
|
|
|
+ headerBytes, err := responseBuffer.ReadBytes('\n')
|
|
|
|
|
+ So(err, ShouldBeNil)
|
|
|
|
|
+ bodyBytes := responseBuffer.Bytes()
|
|
|
|
|
+
|
|
|
|
|
+ jHeader, err := simplejson.NewJson(headerBytes)
|
|
|
|
|
+ So(err, ShouldBeNil)
|
|
|
|
|
+
|
|
|
|
|
+ jBody, err := simplejson.NewJson(bodyBytes)
|
|
|
|
|
+ So(err, ShouldBeNil)
|
|
|
|
|
+
|
|
|
|
|
+ fmt.Println("body", string(headerBytes))
|
|
|
|
|
+
|
|
|
|
|
+ So(jHeader.Get("index").MustString(), ShouldEqual, "metrics-2018.05.15")
|
|
|
|
|
+ So(jHeader.Get("ignore_unavailable").MustBool(false), ShouldEqual, true)
|
|
|
|
|
+ So(jHeader.Get("search_type").MustString(), ShouldEqual, "query_then_fetch")
|
|
|
|
|
+ So(jHeader.Get("max_concurrent_shard_requests").MustInt(10), ShouldEqual, 10)
|
|
|
|
|
+
|
|
|
|
|
+ Convey("and replace $__interval variable", func() {
|
|
|
|
|
+ So(jBody.GetPath("aggs", "2", "aggs", "1", "avg", "script").MustString(), ShouldEqual, "15000*@hostname")
|
|
|
|
|
+ })
|
|
|
|
|
+
|
|
|
|
|
+ Convey("and replace $__interval_ms variable", func() {
|
|
|
|
|
+ So(jBody.GetPath("aggs", "2", "date_histogram", "interval").MustString(), ShouldEqual, "15s")
|
|
|
|
|
+ })
|
|
|
})
|
|
})
|
|
|
- So(multiRequests, ShouldHaveLength, 1)
|
|
|
|
|
- header := multiRequests[0].header
|
|
|
|
|
- So(header, ShouldHaveLength, 4)
|
|
|
|
|
- So(header["index"], ShouldEqual, "test-*")
|
|
|
|
|
- So(header["ignore_unavailable"], ShouldEqual, true)
|
|
|
|
|
- So(header["search_type"], ShouldEqual, "query_then_fetch")
|
|
|
|
|
- So(header["max_concurrent_shard_requests"], ShouldEqual, 256)
|
|
|
|
|
})
|
|
})
|
|
|
})
|
|
})
|
|
|
|
|
|
|
|
- Convey("With custom settings", func() {
|
|
|
|
|
|
|
+ Convey("and a v5.6 client", func() {
|
|
|
ds := models.DataSource{
|
|
ds := models.DataSource{
|
|
|
|
|
+ Database: "[metrics-]YYYY.MM.DD",
|
|
|
|
|
+ Url: ts.URL,
|
|
|
JsonData: simplejson.NewFromAny(map[string]interface{}{
|
|
JsonData: simplejson.NewFromAny(map[string]interface{}{
|
|
|
"esVersion": 56,
|
|
"esVersion": 56,
|
|
|
"maxConcurrentShardRequests": 100,
|
|
"maxConcurrentShardRequests": 100,
|
|
|
|
|
+ "timeField": "@timestamp",
|
|
|
|
|
+ "interval": "Daily",
|
|
|
}),
|
|
}),
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
- c, err := newV56Client(newFakeBaseClient(&ds, []string{"test-*"}))
|
|
|
|
|
|
|
+ c, err := NewClient(context.Background(), &ds, timeRange)
|
|
|
So(err, ShouldBeNil)
|
|
So(err, ShouldBeNil)
|
|
|
So(c, ShouldNotBeNil)
|
|
So(c, ShouldNotBeNil)
|
|
|
- Convey("When creating multisearch requests should have correct headers", func() {
|
|
|
|
|
- multiRequests := c.createMultiSearchRequests([]*SearchRequest{
|
|
|
|
|
- {Index: "test-*"},
|
|
|
|
|
|
|
+
|
|
|
|
|
+ Convey("When executing multi search", func() {
|
|
|
|
|
+ ms, err := createMultisearchForTest(c)
|
|
|
|
|
+ So(err, ShouldBeNil)
|
|
|
|
|
+ c.ExecuteMultisearch(ms)
|
|
|
|
|
+
|
|
|
|
|
+ Convey("Should send correct request and payload", func() {
|
|
|
|
|
+ So(req, ShouldNotBeNil)
|
|
|
|
|
+ So(req.Method, ShouldEqual, http.MethodPost)
|
|
|
|
|
+ So(req.URL.Path, ShouldEqual, "/_msearch")
|
|
|
|
|
+
|
|
|
|
|
+ So(responseBuffer, ShouldNotBeNil)
|
|
|
|
|
+
|
|
|
|
|
+ headerBytes, err := responseBuffer.ReadBytes('\n')
|
|
|
|
|
+ So(err, ShouldBeNil)
|
|
|
|
|
+ bodyBytes := responseBuffer.Bytes()
|
|
|
|
|
+
|
|
|
|
|
+ jHeader, err := simplejson.NewJson(headerBytes)
|
|
|
|
|
+ So(err, ShouldBeNil)
|
|
|
|
|
+
|
|
|
|
|
+ jBody, err := simplejson.NewJson(bodyBytes)
|
|
|
|
|
+ So(err, ShouldBeNil)
|
|
|
|
|
+
|
|
|
|
|
+ fmt.Println("body", string(headerBytes))
|
|
|
|
|
+
|
|
|
|
|
+ So(jHeader.Get("index").MustString(), ShouldEqual, "metrics-2018.05.15")
|
|
|
|
|
+ So(jHeader.Get("ignore_unavailable").MustBool(false), ShouldEqual, true)
|
|
|
|
|
+ So(jHeader.Get("search_type").MustString(), ShouldEqual, "query_then_fetch")
|
|
|
|
|
+ So(jHeader.Get("max_concurrent_shard_requests").MustInt(), ShouldEqual, 100)
|
|
|
|
|
+
|
|
|
|
|
+ Convey("and replace $__interval variable", func() {
|
|
|
|
|
+ So(jBody.GetPath("aggs", "2", "aggs", "1", "avg", "script").MustString(), ShouldEqual, "15000*@hostname")
|
|
|
|
|
+ })
|
|
|
|
|
+
|
|
|
|
|
+ Convey("and replace $__interval_ms variable", func() {
|
|
|
|
|
+ So(jBody.GetPath("aggs", "2", "date_histogram", "interval").MustString(), ShouldEqual, "15s")
|
|
|
|
|
+ })
|
|
|
})
|
|
})
|
|
|
- So(multiRequests, ShouldHaveLength, 1)
|
|
|
|
|
- header := multiRequests[0].header
|
|
|
|
|
- So(header, ShouldHaveLength, 4)
|
|
|
|
|
- So(header["index"], ShouldEqual, "test-*")
|
|
|
|
|
- So(header["ignore_unavailable"], ShouldEqual, true)
|
|
|
|
|
- So(header["search_type"], ShouldEqual, "query_then_fetch")
|
|
|
|
|
- So(header["max_concurrent_shard_requests"], ShouldEqual, 100)
|
|
|
|
|
})
|
|
})
|
|
|
})
|
|
})
|
|
|
|
|
+
|
|
|
|
|
+ Reset(func() {
|
|
|
|
|
+ newDatasourceHttpClient = currentNewDatasourceHttpClient
|
|
|
|
|
+ })
|
|
|
})
|
|
})
|
|
|
})
|
|
})
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
-type fakeBaseClient struct {
|
|
|
|
|
- *baseClientImpl
|
|
|
|
|
- ds *models.DataSource
|
|
|
|
|
-}
|
|
|
|
|
-
|
|
|
|
|
-func newFakeBaseClient(ds *models.DataSource, indices []string) baseClient {
|
|
|
|
|
- return &fakeBaseClient{
|
|
|
|
|
- baseClientImpl: &baseClientImpl{
|
|
|
|
|
- ds: ds,
|
|
|
|
|
- indices: indices,
|
|
|
|
|
- },
|
|
|
|
|
- ds: ds,
|
|
|
|
|
- }
|
|
|
|
|
-}
|
|
|
|
|
-
|
|
|
|
|
-func (c *fakeBaseClient) executeBatchRequest(uriPath string, requests []*multiRequest) (*http.Response, error) {
|
|
|
|
|
- return nil, nil
|
|
|
|
|
-}
|
|
|
|
|
|
|
+func createMultisearchForTest(c Client) (*MultiSearchRequest, error) {
|
|
|
|
|
+ msb := c.MultiSearch()
|
|
|
|
|
+ s := msb.Search(tsdb.Interval{Value: 15 * time.Second, Text: "15s"})
|
|
|
|
|
+ s.Agg().DateHistogram("2", "@timestamp", func(a *DateHistogramAgg, ab AggBuilder) {
|
|
|
|
|
+ a.Interval = "$__interval"
|
|
|
|
|
|
|
|
-func (c *fakeBaseClient) executeRequest(method, uriPath string, body []byte) (*http.Response, error) {
|
|
|
|
|
- return nil, nil
|
|
|
|
|
-}
|
|
|
|
|
-
|
|
|
|
|
-func (c *fakeBaseClient) executeMultisearch(searchRequests []*SearchRequest) ([]*SearchResponse, error) {
|
|
|
|
|
- return nil, nil
|
|
|
|
|
|
|
+ ab.Metric("1", "avg", "@hostname", func(a *MetricAggregation) {
|
|
|
|
|
+ a.Settings["script"] = "$__interval_ms*@hostname"
|
|
|
|
|
+ })
|
|
|
|
|
+ })
|
|
|
|
|
+ return msb.Build()
|
|
|
}
|
|
}
|