|
|
@@ -1,491 +0,0 @@
|
|
|
-package goreq
|
|
|
-
|
|
|
-import (
|
|
|
- "bufio"
|
|
|
- "bytes"
|
|
|
- "compress/gzip"
|
|
|
- "compress/zlib"
|
|
|
- "crypto/tls"
|
|
|
- "encoding/json"
|
|
|
- "errors"
|
|
|
- "fmt"
|
|
|
- "io"
|
|
|
- "io/ioutil"
|
|
|
- "log"
|
|
|
- "net"
|
|
|
- "net/http"
|
|
|
- "net/http/httputil"
|
|
|
- "net/url"
|
|
|
- "reflect"
|
|
|
- "strings"
|
|
|
- "time"
|
|
|
-)
|
|
|
-
|
|
|
-type itimeout interface {
|
|
|
- Timeout() bool
|
|
|
-}
|
|
|
-type Request struct {
|
|
|
- headers []headerTuple
|
|
|
- cookies []*http.Cookie
|
|
|
- Method string
|
|
|
- Uri string
|
|
|
- Body interface{}
|
|
|
- QueryString interface{}
|
|
|
- Timeout time.Duration
|
|
|
- ContentType string
|
|
|
- Accept string
|
|
|
- Host string
|
|
|
- UserAgent string
|
|
|
- Insecure bool
|
|
|
- MaxRedirects int
|
|
|
- RedirectHeaders bool
|
|
|
- Proxy string
|
|
|
- Compression *compression
|
|
|
- BasicAuthUsername string
|
|
|
- BasicAuthPassword string
|
|
|
- CookieJar http.CookieJar
|
|
|
- ShowDebug bool
|
|
|
- OnBeforeRequest func(goreq *Request, httpreq *http.Request)
|
|
|
-}
|
|
|
-
|
|
|
-type compression struct {
|
|
|
- writer func(buffer io.Writer) (io.WriteCloser, error)
|
|
|
- reader func(buffer io.Reader) (io.ReadCloser, error)
|
|
|
- ContentEncoding string
|
|
|
-}
|
|
|
-
|
|
|
-type Response struct {
|
|
|
- *http.Response
|
|
|
- Uri string
|
|
|
- Body *Body
|
|
|
- req *http.Request
|
|
|
-}
|
|
|
-
|
|
|
-func (r Response) CancelRequest() {
|
|
|
- cancelRequest(DefaultTransport, r.req)
|
|
|
-
|
|
|
-}
|
|
|
-
|
|
|
-func cancelRequest(transport interface{}, r *http.Request) {
|
|
|
- if tp, ok := transport.(transportRequestCanceler); ok {
|
|
|
- tp.CancelRequest(r)
|
|
|
- }
|
|
|
-}
|
|
|
-
|
|
|
-type headerTuple struct {
|
|
|
- name string
|
|
|
- value string
|
|
|
-}
|
|
|
-
|
|
|
-type Body struct {
|
|
|
- reader io.ReadCloser
|
|
|
- compressedReader io.ReadCloser
|
|
|
-}
|
|
|
-
|
|
|
-type Error struct {
|
|
|
- timeout bool
|
|
|
- Err error
|
|
|
-}
|
|
|
-
|
|
|
-type transportRequestCanceler interface {
|
|
|
- CancelRequest(*http.Request)
|
|
|
-}
|
|
|
-
|
|
|
-func (e *Error) Timeout() bool {
|
|
|
- return e.timeout
|
|
|
-}
|
|
|
-
|
|
|
-func (e *Error) Error() string {
|
|
|
- return e.Err.Error()
|
|
|
-}
|
|
|
-
|
|
|
-func (b *Body) Read(p []byte) (int, error) {
|
|
|
- if b.compressedReader != nil {
|
|
|
- return b.compressedReader.Read(p)
|
|
|
- }
|
|
|
- return b.reader.Read(p)
|
|
|
-}
|
|
|
-
|
|
|
-func (b *Body) Close() error {
|
|
|
- err := b.reader.Close()
|
|
|
- if b.compressedReader != nil {
|
|
|
- return b.compressedReader.Close()
|
|
|
- }
|
|
|
- return err
|
|
|
-}
|
|
|
-
|
|
|
-func (b *Body) FromJsonTo(o interface{}) error {
|
|
|
- return json.NewDecoder(b).Decode(o)
|
|
|
-}
|
|
|
-
|
|
|
-func (b *Body) ToString() (string, error) {
|
|
|
- body, err := ioutil.ReadAll(b)
|
|
|
- if err != nil {
|
|
|
- return "", err
|
|
|
- }
|
|
|
- return string(body), nil
|
|
|
-}
|
|
|
-
|
|
|
-func Gzip() *compression {
|
|
|
- reader := func(buffer io.Reader) (io.ReadCloser, error) {
|
|
|
- return gzip.NewReader(buffer)
|
|
|
- }
|
|
|
- writer := func(buffer io.Writer) (io.WriteCloser, error) {
|
|
|
- return gzip.NewWriter(buffer), nil
|
|
|
- }
|
|
|
- return &compression{writer: writer, reader: reader, ContentEncoding: "gzip"}
|
|
|
-}
|
|
|
-
|
|
|
-func Deflate() *compression {
|
|
|
- reader := func(buffer io.Reader) (io.ReadCloser, error) {
|
|
|
- return zlib.NewReader(buffer)
|
|
|
- }
|
|
|
- writer := func(buffer io.Writer) (io.WriteCloser, error) {
|
|
|
- return zlib.NewWriter(buffer), nil
|
|
|
- }
|
|
|
- return &compression{writer: writer, reader: reader, ContentEncoding: "deflate"}
|
|
|
-}
|
|
|
-
|
|
|
-func Zlib() *compression {
|
|
|
- return Deflate()
|
|
|
-}
|
|
|
-
|
|
|
-func paramParse(query interface{}) (string, error) {
|
|
|
- switch query.(type) {
|
|
|
- case url.Values:
|
|
|
- return query.(url.Values).Encode(), nil
|
|
|
- case *url.Values:
|
|
|
- return query.(*url.Values).Encode(), nil
|
|
|
- default:
|
|
|
- var v = &url.Values{}
|
|
|
- err := paramParseStruct(v, query)
|
|
|
- return v.Encode(), err
|
|
|
- }
|
|
|
-}
|
|
|
-
|
|
|
-func paramParseStruct(v *url.Values, query interface{}) error {
|
|
|
- var (
|
|
|
- s = reflect.ValueOf(query)
|
|
|
- t = reflect.TypeOf(query)
|
|
|
- )
|
|
|
- for t.Kind() == reflect.Ptr || t.Kind() == reflect.Interface {
|
|
|
- s = s.Elem()
|
|
|
- t = s.Type()
|
|
|
- }
|
|
|
-
|
|
|
- if t.Kind() != reflect.Struct {
|
|
|
- return errors.New("Can not parse QueryString.")
|
|
|
- }
|
|
|
-
|
|
|
- for i := 0; i < t.NumField(); i++ {
|
|
|
- var name string
|
|
|
-
|
|
|
- field := s.Field(i)
|
|
|
- typeField := t.Field(i)
|
|
|
-
|
|
|
- if !field.CanInterface() {
|
|
|
- continue
|
|
|
- }
|
|
|
-
|
|
|
- urlTag := typeField.Tag.Get("url")
|
|
|
- if urlTag == "-" {
|
|
|
- continue
|
|
|
- }
|
|
|
-
|
|
|
- name, opts := parseTag(urlTag)
|
|
|
-
|
|
|
- var omitEmpty, squash bool
|
|
|
- omitEmpty = opts.Contains("omitempty")
|
|
|
- squash = opts.Contains("squash")
|
|
|
-
|
|
|
- if squash {
|
|
|
- err := paramParseStruct(v, field.Interface())
|
|
|
- if err != nil {
|
|
|
- return err
|
|
|
- }
|
|
|
- continue
|
|
|
- }
|
|
|
-
|
|
|
- if urlTag == "" {
|
|
|
- name = strings.ToLower(typeField.Name)
|
|
|
- }
|
|
|
-
|
|
|
- if val := fmt.Sprintf("%v", field.Interface()); !(omitEmpty && len(val) == 0) {
|
|
|
- v.Add(name, val)
|
|
|
- }
|
|
|
- }
|
|
|
- return nil
|
|
|
-}
|
|
|
-
|
|
|
-func prepareRequestBody(b interface{}) (io.Reader, error) {
|
|
|
- switch b.(type) {
|
|
|
- case string:
|
|
|
- // treat is as text
|
|
|
- return strings.NewReader(b.(string)), nil
|
|
|
- case io.Reader:
|
|
|
- // treat is as text
|
|
|
- return b.(io.Reader), nil
|
|
|
- case []byte:
|
|
|
- //treat as byte array
|
|
|
- return bytes.NewReader(b.([]byte)), nil
|
|
|
- case nil:
|
|
|
- return nil, nil
|
|
|
- default:
|
|
|
- // try to jsonify it
|
|
|
- j, err := json.Marshal(b)
|
|
|
- if err == nil {
|
|
|
- return bytes.NewReader(j), nil
|
|
|
- }
|
|
|
- return nil, err
|
|
|
- }
|
|
|
-}
|
|
|
-
|
|
|
-var DefaultDialer = &net.Dialer{Timeout: 1000 * time.Millisecond}
|
|
|
-var DefaultTransport http.RoundTripper = &http.Transport{Dial: DefaultDialer.Dial, Proxy: http.ProxyFromEnvironment}
|
|
|
-var DefaultClient = &http.Client{Transport: DefaultTransport}
|
|
|
-
|
|
|
-var proxyTransport http.RoundTripper
|
|
|
-var proxyClient *http.Client
|
|
|
-
|
|
|
-func SetConnectTimeout(duration time.Duration) {
|
|
|
- DefaultDialer.Timeout = duration
|
|
|
-}
|
|
|
-
|
|
|
-func (r *Request) AddHeader(name string, value string) {
|
|
|
- if r.headers == nil {
|
|
|
- r.headers = []headerTuple{}
|
|
|
- }
|
|
|
- r.headers = append(r.headers, headerTuple{name: name, value: value})
|
|
|
-}
|
|
|
-
|
|
|
-func (r Request) WithHeader(name string, value string) Request {
|
|
|
- r.AddHeader(name, value)
|
|
|
- return r
|
|
|
-}
|
|
|
-
|
|
|
-func (r *Request) AddCookie(c *http.Cookie) {
|
|
|
- r.cookies = append(r.cookies, c)
|
|
|
-}
|
|
|
-
|
|
|
-func (r Request) WithCookie(c *http.Cookie) Request {
|
|
|
- r.AddCookie(c)
|
|
|
- return r
|
|
|
-
|
|
|
-}
|
|
|
-
|
|
|
-func (r Request) Do() (*Response, error) {
|
|
|
- var client = DefaultClient
|
|
|
- var transport = DefaultTransport
|
|
|
- var resUri string
|
|
|
- var redirectFailed bool
|
|
|
-
|
|
|
- r.Method = valueOrDefault(r.Method, "GET")
|
|
|
-
|
|
|
- // use a client with a cookie jar if necessary. We create a new client not
|
|
|
- // to modify the default one.
|
|
|
- if r.CookieJar != nil {
|
|
|
- client = &http.Client{
|
|
|
- Transport: transport,
|
|
|
- Jar: r.CookieJar,
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- if r.Proxy != "" {
|
|
|
- proxyUrl, err := url.Parse(r.Proxy)
|
|
|
- if err != nil {
|
|
|
- // proxy address is in a wrong format
|
|
|
- return nil, &Error{Err: err}
|
|
|
- }
|
|
|
-
|
|
|
- //If jar is specified new client needs to be built
|
|
|
- if proxyTransport == nil || client.Jar != nil {
|
|
|
- proxyTransport = &http.Transport{Dial: DefaultDialer.Dial, Proxy: http.ProxyURL(proxyUrl)}
|
|
|
- proxyClient = &http.Client{Transport: proxyTransport, Jar: client.Jar}
|
|
|
- } else if proxyTransport, ok := proxyTransport.(*http.Transport); ok {
|
|
|
- proxyTransport.Proxy = http.ProxyURL(proxyUrl)
|
|
|
- }
|
|
|
- transport = proxyTransport
|
|
|
- client = proxyClient
|
|
|
- }
|
|
|
-
|
|
|
- client.CheckRedirect = func(req *http.Request, via []*http.Request) error {
|
|
|
-
|
|
|
- if len(via) > r.MaxRedirects {
|
|
|
- redirectFailed = true
|
|
|
- return errors.New("Error redirecting. MaxRedirects reached")
|
|
|
- }
|
|
|
-
|
|
|
- resUri = req.URL.String()
|
|
|
-
|
|
|
- //By default Golang will not redirect request headers
|
|
|
- // https://code.google.com/p/go/issues/detail?id=4800&q=request%20header
|
|
|
- if r.RedirectHeaders {
|
|
|
- for key, val := range via[0].Header {
|
|
|
- req.Header[key] = val
|
|
|
- }
|
|
|
- }
|
|
|
- return nil
|
|
|
- }
|
|
|
-
|
|
|
- if transport, ok := transport.(*http.Transport); ok {
|
|
|
- if r.Insecure {
|
|
|
- if transport.TLSClientConfig != nil {
|
|
|
- transport.TLSClientConfig.InsecureSkipVerify = true
|
|
|
- } else {
|
|
|
- transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
|
|
|
- }
|
|
|
- } else if transport.TLSClientConfig != nil {
|
|
|
- // the default TLS client (when transport.TLSClientConfig==nil) is
|
|
|
- // already set to verify, so do nothing in that case
|
|
|
- transport.TLSClientConfig.InsecureSkipVerify = false
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- req, err := r.NewRequest()
|
|
|
-
|
|
|
- if err != nil {
|
|
|
- // we couldn't parse the URL.
|
|
|
- return nil, &Error{Err: err}
|
|
|
- }
|
|
|
-
|
|
|
- timeout := false
|
|
|
- if r.Timeout > 0 {
|
|
|
- client.Timeout = r.Timeout
|
|
|
- }
|
|
|
-
|
|
|
- if r.ShowDebug {
|
|
|
- dump, err := httputil.DumpRequest(req, true)
|
|
|
- if err != nil {
|
|
|
- log.Println(err)
|
|
|
- }
|
|
|
- log.Println(string(dump))
|
|
|
- }
|
|
|
-
|
|
|
- if r.OnBeforeRequest != nil {
|
|
|
- r.OnBeforeRequest(&r, req)
|
|
|
- }
|
|
|
- res, err := client.Do(req)
|
|
|
-
|
|
|
- if err != nil {
|
|
|
- if !timeout {
|
|
|
- if t, ok := err.(itimeout); ok {
|
|
|
- timeout = t.Timeout()
|
|
|
- }
|
|
|
- if ue, ok := err.(*url.Error); ok {
|
|
|
- if t, ok := ue.Err.(itimeout); ok {
|
|
|
- timeout = t.Timeout()
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- var response *Response
|
|
|
- //If redirect fails we still want to return response data
|
|
|
- if redirectFailed {
|
|
|
- if res != nil {
|
|
|
- response = &Response{res, resUri, &Body{reader: res.Body}, req}
|
|
|
- } else {
|
|
|
- response = &Response{res, resUri, nil, req}
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- //If redirect fails and we haven't set a redirect count we shouldn't return an error
|
|
|
- if redirectFailed && r.MaxRedirects == 0 {
|
|
|
- return response, nil
|
|
|
- }
|
|
|
-
|
|
|
- return response, &Error{timeout: timeout, Err: err}
|
|
|
- }
|
|
|
-
|
|
|
- if r.Compression != nil && strings.Contains(res.Header.Get("Content-Encoding"), r.Compression.ContentEncoding) {
|
|
|
- compressedReader, err := r.Compression.reader(res.Body)
|
|
|
- if err != nil {
|
|
|
- return nil, &Error{Err: err}
|
|
|
- }
|
|
|
- return &Response{res, resUri, &Body{reader: res.Body, compressedReader: compressedReader}, req}, nil
|
|
|
- }
|
|
|
-
|
|
|
- return &Response{res, resUri, &Body{reader: res.Body}, req}, nil
|
|
|
-}
|
|
|
-
|
|
|
-func (r Request) addHeaders(headersMap http.Header) {
|
|
|
- if len(r.UserAgent) > 0 {
|
|
|
- headersMap.Add("User-Agent", r.UserAgent)
|
|
|
- }
|
|
|
- if r.Accept != "" {
|
|
|
- headersMap.Add("Accept", r.Accept)
|
|
|
- }
|
|
|
- if r.ContentType != "" {
|
|
|
- headersMap.Add("Content-Type", r.ContentType)
|
|
|
- }
|
|
|
-}
|
|
|
-
|
|
|
-func (r Request) NewRequest() (*http.Request, error) {
|
|
|
-
|
|
|
- b, e := prepareRequestBody(r.Body)
|
|
|
- if e != nil {
|
|
|
- // there was a problem marshaling the body
|
|
|
- return nil, &Error{Err: e}
|
|
|
- }
|
|
|
-
|
|
|
- if r.QueryString != nil {
|
|
|
- param, e := paramParse(r.QueryString)
|
|
|
- if e != nil {
|
|
|
- return nil, &Error{Err: e}
|
|
|
- }
|
|
|
- r.Uri = r.Uri + "?" + param
|
|
|
- }
|
|
|
-
|
|
|
- var bodyReader io.Reader
|
|
|
- if b != nil && r.Compression != nil {
|
|
|
- buffer := bytes.NewBuffer([]byte{})
|
|
|
- readBuffer := bufio.NewReader(b)
|
|
|
- writer, err := r.Compression.writer(buffer)
|
|
|
- if err != nil {
|
|
|
- return nil, &Error{Err: err}
|
|
|
- }
|
|
|
- _, e = readBuffer.WriteTo(writer)
|
|
|
- writer.Close()
|
|
|
- if e != nil {
|
|
|
- return nil, &Error{Err: e}
|
|
|
- }
|
|
|
- bodyReader = buffer
|
|
|
- } else {
|
|
|
- bodyReader = b
|
|
|
- }
|
|
|
-
|
|
|
- req, err := http.NewRequest(r.Method, r.Uri, bodyReader)
|
|
|
- if err != nil {
|
|
|
- return nil, err
|
|
|
- }
|
|
|
- // add headers to the request
|
|
|
- req.Host = r.Host
|
|
|
-
|
|
|
- r.addHeaders(req.Header)
|
|
|
- if r.Compression != nil {
|
|
|
- req.Header.Add("Content-Encoding", r.Compression.ContentEncoding)
|
|
|
- req.Header.Add("Accept-Encoding", r.Compression.ContentEncoding)
|
|
|
- }
|
|
|
- if r.headers != nil {
|
|
|
- for _, header := range r.headers {
|
|
|
- req.Header.Add(header.name, header.value)
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- //use basic auth if required
|
|
|
- if r.BasicAuthUsername != "" {
|
|
|
- req.SetBasicAuth(r.BasicAuthUsername, r.BasicAuthPassword)
|
|
|
- }
|
|
|
-
|
|
|
- for _, c := range r.cookies {
|
|
|
- req.AddCookie(c)
|
|
|
- }
|
|
|
- return req, nil
|
|
|
-}
|
|
|
-
|
|
|
-// Return value if nonempty, def otherwise.
|
|
|
-func valueOrDefault(value, def string) string {
|
|
|
- if value != "" {
|
|
|
- return value
|
|
|
- }
|
|
|
- return def
|
|
|
-}
|
bergquist