From 1eba1730d1af50ed545f4fde78b22d6fb62ca11e Mon Sep 17 00:00:00 2001 From: Frédéric Guillot Date: Sat, 28 Apr 2018 10:51:07 -0700 Subject: Move HTTP client to its own package --- http/client.go | 218 ------------------------------------------ http/client/client.go | 223 +++++++++++++++++++++++++++++++++++++++++++ http/client/response.go | 68 +++++++++++++ http/client/response_test.go | 56 +++++++++++ http/doc.go | 10 -- http/response.go | 68 ------------- http/response_test.go | 56 ----------- 7 files changed, 347 insertions(+), 352 deletions(-) delete mode 100644 http/client.go create mode 100644 http/client/client.go create mode 100644 http/client/response.go create mode 100644 http/client/response_test.go delete mode 100644 http/doc.go delete mode 100644 http/response.go delete mode 100644 http/response_test.go (limited to 'http') diff --git a/http/client.go b/http/client.go deleted file mode 100644 index 1884224..0000000 --- a/http/client.go +++ /dev/null @@ -1,218 +0,0 @@ -// Copyright 2017 Frédéric Guillot. All rights reserved. -// Use of this source code is governed by the Apache 2.0 -// license that can be found in the LICENSE file. - -package http - -import ( - "bytes" - "crypto/tls" - "crypto/x509" - "encoding/json" - "fmt" - "io" - "net" - "net/http" - "net/url" - "strings" - "time" - - "github.com/miniflux/miniflux/errors" - "github.com/miniflux/miniflux/logger" - "github.com/miniflux/miniflux/timer" - "github.com/miniflux/miniflux/version" -) - -const ( - // 20 seconds max. - requestTimeout = 20 - - // 15MB max. - maxBodySize = 1024 * 1024 * 15 -) - -var ( - errInvalidCertificate = "Invalid SSL certificate (original error: %q)" - errTemporaryNetworkOperation = "This website is temporarily unreachable (original error: %q)" - errPermanentNetworkOperation = "This website is permanently unreachable (original error: %q)" - errRequestTimeout = "Website unreachable, the request timed out after %d seconds" -) - -// Client is a HTTP Client :) -type Client struct { - url string - etagHeader string - lastModifiedHeader string - authorizationHeader string - username string - password string - Insecure bool -} - -// Get execute a GET HTTP request. -func (c *Client) Get() (*Response, error) { - request, err := c.buildRequest(http.MethodGet, nil) - if err != nil { - return nil, err - } - - return c.executeRequest(request) -} - -// PostForm execute a POST HTTP request with form values. -func (c *Client) PostForm(values url.Values) (*Response, error) { - request, err := c.buildRequest(http.MethodPost, strings.NewReader(values.Encode())) - if err != nil { - return nil, err - } - - request.Header.Add("Content-Type", "application/x-www-form-urlencoded") - return c.executeRequest(request) -} - -// PostJSON execute a POST HTTP request with JSON payload. -func (c *Client) PostJSON(data interface{}) (*Response, error) { - b, err := json.Marshal(data) - if err != nil { - return nil, err - } - - request, err := c.buildRequest(http.MethodPost, bytes.NewReader(b)) - if err != nil { - return nil, err - } - - request.Header.Add("Content-Type", "application/json") - return c.executeRequest(request) -} - -func (c *Client) executeRequest(request *http.Request) (*Response, error) { - defer timer.ExecutionTime(time.Now(), fmt.Sprintf("[HttpClient] url=%s", c.url)) - - client := c.buildClient() - resp, err := client.Do(request) - if err != nil { - if uerr, ok := err.(*url.Error); ok { - switch uerr.Err.(type) { - case x509.CertificateInvalidError, x509.HostnameError: - err = errors.NewLocalizedError(errInvalidCertificate, uerr.Err) - case *net.OpError: - if uerr.Err.(*net.OpError).Temporary() { - err = errors.NewLocalizedError(errTemporaryNetworkOperation, uerr.Err) - } else { - err = errors.NewLocalizedError(errPermanentNetworkOperation, uerr.Err) - } - case net.Error: - nerr := uerr.Err.(net.Error) - if nerr.Timeout() { - err = errors.NewLocalizedError(errRequestTimeout, requestTimeout) - } else if nerr.Temporary() { - err = errors.NewLocalizedError(errTemporaryNetworkOperation, nerr) - } - } - } - - return nil, err - } - - if resp.ContentLength > maxBodySize { - return nil, fmt.Errorf("client: response too large (%d bytes)", resp.ContentLength) - } - - response := &Response{ - Body: resp.Body, - StatusCode: resp.StatusCode, - EffectiveURL: resp.Request.URL.String(), - LastModified: resp.Header.Get("Last-Modified"), - ETag: resp.Header.Get("ETag"), - ContentType: resp.Header.Get("Content-Type"), - ContentLength: resp.ContentLength, - } - - logger.Debug("[HttpClient:%s] URL=%s, EffectiveURL=%s, Code=%d, Length=%d, Type=%s, ETag=%s, LastMod=%s, Expires=%s", - request.Method, - c.url, - response.EffectiveURL, - response.StatusCode, - resp.ContentLength, - response.ContentType, - response.ETag, - response.LastModified, - resp.Header.Get("Expires"), - ) - - // Ignore caching headers for feeds that do not want any cache. - if resp.Header.Get("Expires") == "0" { - logger.Debug("[HttpClient] Ignore caching headers for %q", response.EffectiveURL) - response.ETag = "" - response.LastModified = "" - } - - return response, err -} - -func (c *Client) buildRequest(method string, body io.Reader) (*http.Request, error) { - request, err := http.NewRequest(method, c.url, body) - if err != nil { - return nil, err - } - - request.Header = c.buildHeaders() - - if c.username != "" && c.password != "" { - request.SetBasicAuth(c.username, c.password) - } - - return request, nil -} - -func (c *Client) buildClient() http.Client { - client := http.Client{Timeout: time.Duration(requestTimeout * time.Second)} - if c.Insecure { - client.Transport = &http.Transport{ - TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, - } - } - - return client -} - -func (c *Client) buildHeaders() http.Header { - headers := make(http.Header) - headers.Add("User-Agent", "Mozilla/5.0 (compatible; Miniflux/"+version.Version+"; +https://miniflux.net)") - headers.Add("Accept", "*/*") - - if c.etagHeader != "" { - headers.Add("If-None-Match", c.etagHeader) - } - - if c.lastModifiedHeader != "" { - headers.Add("If-Modified-Since", c.lastModifiedHeader) - } - - if c.authorizationHeader != "" { - headers.Add("Authorization", c.authorizationHeader) - } - - return headers -} - -// NewClient returns a new HTTP client. -func NewClient(url string) *Client { - return &Client{url: url, Insecure: false} -} - -// NewClientWithCredentials returns a new HTTP client that requires authentication. -func NewClientWithCredentials(url, username, password string) *Client { - return &Client{url: url, Insecure: false, username: username, password: password} -} - -// NewClientWithAuthorization returns a new client with a custom authorization header. -func NewClientWithAuthorization(url, authorization string) *Client { - return &Client{url: url, Insecure: false, authorizationHeader: authorization} -} - -// NewClientWithCacheHeaders returns a new HTTP client that send cache headers. -func NewClientWithCacheHeaders(url, etagHeader, lastModifiedHeader string) *Client { - return &Client{url: url, etagHeader: etagHeader, lastModifiedHeader: lastModifiedHeader, Insecure: false} -} diff --git a/http/client/client.go b/http/client/client.go new file mode 100644 index 0000000..7663064 --- /dev/null +++ b/http/client/client.go @@ -0,0 +1,223 @@ +// Copyright 2018 Frédéric Guillot. All rights reserved. +// Use of this source code is governed by the Apache 2.0 +// license that can be found in the LICENSE file. + +package client + +import ( + "bytes" + "crypto/tls" + "crypto/x509" + "encoding/json" + "fmt" + "io" + "net" + "net/http" + "net/url" + "strings" + "time" + + "github.com/miniflux/miniflux/errors" + "github.com/miniflux/miniflux/logger" + "github.com/miniflux/miniflux/timer" + "github.com/miniflux/miniflux/version" +) + +const ( + // 20 seconds max. + requestTimeout = 20 + + // 15MB max. + maxBodySize = 1024 * 1024 * 15 +) + +var ( + errInvalidCertificate = "Invalid SSL certificate (original error: %q)" + errTemporaryNetworkOperation = "This website is temporarily unreachable (original error: %q)" + errPermanentNetworkOperation = "This website is permanently unreachable (original error: %q)" + errRequestTimeout = "Website unreachable, the request timed out after %d seconds" +) + +// Client is a HTTP Client :) +type Client struct { + url string + etagHeader string + lastModifiedHeader string + authorizationHeader string + username string + password string + Insecure bool +} + +// WithCredentials defines the username/password for HTTP Basic authentication. +func (c *Client) WithCredentials(username, password string) *Client { + c.username = username + c.password = password + return c +} + +// WithAuthorization defines authorization header value. +func (c *Client) WithAuthorization(authorization string) *Client { + c.authorizationHeader = authorization + return c +} + +// WithCacheHeaders defines caching headers. +func (c *Client) WithCacheHeaders(etagHeader, lastModifiedHeader string) *Client { + c.etagHeader = etagHeader + c.lastModifiedHeader = lastModifiedHeader + return c +} + +// Get execute a GET HTTP request. +func (c *Client) Get() (*Response, error) { + request, err := c.buildRequest(http.MethodGet, nil) + if err != nil { + return nil, err + } + + return c.executeRequest(request) +} + +// PostForm execute a POST HTTP request with form values. +func (c *Client) PostForm(values url.Values) (*Response, error) { + request, err := c.buildRequest(http.MethodPost, strings.NewReader(values.Encode())) + if err != nil { + return nil, err + } + + request.Header.Add("Content-Type", "application/x-www-form-urlencoded") + return c.executeRequest(request) +} + +// PostJSON execute a POST HTTP request with JSON payload. +func (c *Client) PostJSON(data interface{}) (*Response, error) { + b, err := json.Marshal(data) + if err != nil { + return nil, err + } + + request, err := c.buildRequest(http.MethodPost, bytes.NewReader(b)) + if err != nil { + return nil, err + } + + request.Header.Add("Content-Type", "application/json") + return c.executeRequest(request) +} + +func (c *Client) executeRequest(request *http.Request) (*Response, error) { + defer timer.ExecutionTime(time.Now(), fmt.Sprintf("[HttpClient] url=%s", c.url)) + + client := c.buildClient() + resp, err := client.Do(request) + if err != nil { + if uerr, ok := err.(*url.Error); ok { + switch uerr.Err.(type) { + case x509.CertificateInvalidError, x509.HostnameError: + err = errors.NewLocalizedError(errInvalidCertificate, uerr.Err) + case *net.OpError: + if uerr.Err.(*net.OpError).Temporary() { + err = errors.NewLocalizedError(errTemporaryNetworkOperation, uerr.Err) + } else { + err = errors.NewLocalizedError(errPermanentNetworkOperation, uerr.Err) + } + case net.Error: + nerr := uerr.Err.(net.Error) + if nerr.Timeout() { + err = errors.NewLocalizedError(errRequestTimeout, requestTimeout) + } else if nerr.Temporary() { + err = errors.NewLocalizedError(errTemporaryNetworkOperation, nerr) + } + } + } + + return nil, err + } + + if resp.ContentLength > maxBodySize { + return nil, fmt.Errorf("client: response too large (%d bytes)", resp.ContentLength) + } + + response := &Response{ + Body: resp.Body, + StatusCode: resp.StatusCode, + EffectiveURL: resp.Request.URL.String(), + LastModified: resp.Header.Get("Last-Modified"), + ETag: resp.Header.Get("ETag"), + ContentType: resp.Header.Get("Content-Type"), + ContentLength: resp.ContentLength, + } + + logger.Debug("[HttpClient:%s] URL=%s, EffectiveURL=%s, Code=%d, Length=%d, Type=%s, ETag=%s, LastMod=%s, Expires=%s", + request.Method, + c.url, + response.EffectiveURL, + response.StatusCode, + resp.ContentLength, + response.ContentType, + response.ETag, + response.LastModified, + resp.Header.Get("Expires"), + ) + + // Ignore caching headers for feeds that do not want any cache. + if resp.Header.Get("Expires") == "0" { + logger.Debug("[HttpClient] Ignore caching headers for %q", response.EffectiveURL) + response.ETag = "" + response.LastModified = "" + } + + return response, err +} + +func (c *Client) buildRequest(method string, body io.Reader) (*http.Request, error) { + request, err := http.NewRequest(method, c.url, body) + if err != nil { + return nil, err + } + + request.Header = c.buildHeaders() + + if c.username != "" && c.password != "" { + request.SetBasicAuth(c.username, c.password) + } + + return request, nil +} + +func (c *Client) buildClient() http.Client { + client := http.Client{Timeout: time.Duration(requestTimeout * time.Second)} + if c.Insecure { + client.Transport = &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + } + } + + return client +} + +func (c *Client) buildHeaders() http.Header { + headers := make(http.Header) + headers.Add("User-Agent", "Mozilla/5.0 (compatible; Miniflux/"+version.Version+"; +https://miniflux.net)") + headers.Add("Accept", "*/*") + + if c.etagHeader != "" { + headers.Add("If-None-Match", c.etagHeader) + } + + if c.lastModifiedHeader != "" { + headers.Add("If-Modified-Since", c.lastModifiedHeader) + } + + if c.authorizationHeader != "" { + headers.Add("Authorization", c.authorizationHeader) + } + + return headers +} + +// New returns a new HTTP client. +func New(url string) *Client { + return &Client{url: url, Insecure: false} +} diff --git a/http/client/response.go b/http/client/response.go new file mode 100644 index 0000000..f033d61 --- /dev/null +++ b/http/client/response.go @@ -0,0 +1,68 @@ +// Copyright 2017 Frédéric Guillot. All rights reserved. +// Use of this source code is governed by the Apache 2.0 +// license that can be found in the LICENSE file. + +package client + +import ( + "io" + "mime" + "strings" + + "github.com/miniflux/miniflux/logger" + "golang.org/x/net/html/charset" +) + +// Response wraps a server response. +type Response struct { + Body io.Reader + StatusCode int + EffectiveURL string + LastModified string + ETag string + ContentType string + ContentLength int64 +} + +// HasServerFailure returns true if the status code represents a failure. +func (r *Response) HasServerFailure() bool { + return r.StatusCode >= 400 +} + +// IsModified returns true if the resource has been modified. +func (r *Response) IsModified(etag, lastModified string) bool { + if r.StatusCode == 304 { + return false + } + + if r.ETag != "" && r.ETag == etag { + return false + } + + if r.LastModified != "" && r.LastModified == lastModified { + return false + } + + return true +} + +// NormalizeBodyEncoding make sure the body is encoded in UTF-8. +// +// If a charset other than UTF-8 is detected, we convert the document to UTF-8. +// This is used by the scraper and feed readers. +// +// Do not forget edge cases: +// - Some non-utf8 feeds specify encoding only in Content-Type, not in XML document. +func (r *Response) NormalizeBodyEncoding() (io.Reader, error) { + _, params, err := mime.ParseMediaType(r.ContentType) + if err == nil { + if enc, found := params["charset"]; found { + enc = strings.ToLower(enc) + if enc != "utf-8" && enc != "utf8" && enc != "" { + logger.Debug("[NormalizeBodyEncoding] Convert body to UTF-8 from %s", enc) + return charset.NewReader(r.Body, r.ContentType) + } + } + } + return r.Body, nil +} diff --git a/http/client/response_test.go b/http/client/response_test.go new file mode 100644 index 0000000..f3402a8 --- /dev/null +++ b/http/client/response_test.go @@ -0,0 +1,56 @@ +// Copyright 2017 Frédéric Guillot. All rights reserved. +// Use of this source code is governed by the Apache 2.0 +// license that can be found in the LICENSE file. + +package client + +import "testing" + +func TestHasServerFailureWith200Status(t *testing.T) { + r := &Response{StatusCode: 200} + if r.HasServerFailure() { + t.Error("200 is not a failure") + } +} + +func TestHasServerFailureWith404Status(t *testing.T) { + r := &Response{StatusCode: 404} + if !r.HasServerFailure() { + t.Error("404 is a failure") + } +} + +func TestHasServerFailureWith500Status(t *testing.T) { + r := &Response{StatusCode: 500} + if !r.HasServerFailure() { + t.Error("500 is a failure") + } +} + +func TestIsModifiedWith304Status(t *testing.T) { + r := &Response{StatusCode: 304} + if r.IsModified("etag", "lastModified") { + t.Error("The resource should not be considered modified") + } +} + +func TestIsModifiedWithIdenticalEtag(t *testing.T) { + r := &Response{StatusCode: 200, ETag: "etag"} + if r.IsModified("etag", "lastModified") { + t.Error("The resource should not be considered modified") + } +} + +func TestIsModifiedWithIdenticalLastModified(t *testing.T) { + r := &Response{StatusCode: 200, LastModified: "lastModified"} + if r.IsModified("etag", "lastModified") { + t.Error("The resource should not be considered modified") + } +} + +func TestIsModifiedWithDifferentHeaders(t *testing.T) { + r := &Response{StatusCode: 200, ETag: "some etag", LastModified: "some date"} + if !r.IsModified("etag", "lastModified") { + t.Error("The resource should be considered modified") + } +} diff --git a/http/doc.go b/http/doc.go deleted file mode 100644 index f0eb1fa..0000000 --- a/http/doc.go +++ /dev/null @@ -1,10 +0,0 @@ -// Copyright 2018 Frédéric Guillot. All rights reserved. -// Use of this source code is governed by the MIT license -// that can be found in the LICENSE file. - -/* - -Package http implements a set of utilities related to the HTTP protocol. - -*/ -package http diff --git a/http/response.go b/http/response.go deleted file mode 100644 index a0cfc3f..0000000 --- a/http/response.go +++ /dev/null @@ -1,68 +0,0 @@ -// Copyright 2017 Frédéric Guillot. All rights reserved. -// Use of this source code is governed by the Apache 2.0 -// license that can be found in the LICENSE file. - -package http - -import ( - "io" - "mime" - "strings" - - "github.com/miniflux/miniflux/logger" - "golang.org/x/net/html/charset" -) - -// Response wraps a server response. -type Response struct { - Body io.Reader - StatusCode int - EffectiveURL string - LastModified string - ETag string - ContentType string - ContentLength int64 -} - -// HasServerFailure returns true if the status code represents a failure. -func (r *Response) HasServerFailure() bool { - return r.StatusCode >= 400 -} - -// IsModified returns true if the resource has been modified. -func (r *Response) IsModified(etag, lastModified string) bool { - if r.StatusCode == 304 { - return false - } - - if r.ETag != "" && r.ETag == etag { - return false - } - - if r.LastModified != "" && r.LastModified == lastModified { - return false - } - - return true -} - -// NormalizeBodyEncoding make sure the body is encoded in UTF-8. -// -// If a charset other than UTF-8 is detected, we convert the document to UTF-8. -// This is used by the scraper and feed readers. -// -// Do not forget edge cases: -// - Some non-utf8 feeds specify encoding only in Content-Type, not in XML document. -func (r *Response) NormalizeBodyEncoding() (io.Reader, error) { - _, params, err := mime.ParseMediaType(r.ContentType) - if err == nil { - if enc, found := params["charset"]; found { - enc = strings.ToLower(enc) - if enc != "utf-8" && enc != "utf8" && enc != "" { - logger.Debug("[NormalizeBodyEncoding] Convert body to UTF-8 from %s", enc) - return charset.NewReader(r.Body, r.ContentType) - } - } - } - return r.Body, nil -} diff --git a/http/response_test.go b/http/response_test.go deleted file mode 100644 index c5f6a1c..0000000 --- a/http/response_test.go +++ /dev/null @@ -1,56 +0,0 @@ -// Copyright 2017 Frédéric Guillot. All rights reserved. -// Use of this source code is governed by the Apache 2.0 -// license that can be found in the LICENSE file. - -package http - -import "testing" - -func TestHasServerFailureWith200Status(t *testing.T) { - r := &Response{StatusCode: 200} - if r.HasServerFailure() { - t.Error("200 is not a failure") - } -} - -func TestHasServerFailureWith404Status(t *testing.T) { - r := &Response{StatusCode: 404} - if !r.HasServerFailure() { - t.Error("404 is a failure") - } -} - -func TestHasServerFailureWith500Status(t *testing.T) { - r := &Response{StatusCode: 500} - if !r.HasServerFailure() { - t.Error("500 is a failure") - } -} - -func TestIsModifiedWith304Status(t *testing.T) { - r := &Response{StatusCode: 304} - if r.IsModified("etag", "lastModified") { - t.Error("The resource should not be considered modified") - } -} - -func TestIsModifiedWithIdenticalEtag(t *testing.T) { - r := &Response{StatusCode: 200, ETag: "etag"} - if r.IsModified("etag", "lastModified") { - t.Error("The resource should not be considered modified") - } -} - -func TestIsModifiedWithIdenticalLastModified(t *testing.T) { - r := &Response{StatusCode: 200, LastModified: "lastModified"} - if r.IsModified("etag", "lastModified") { - t.Error("The resource should not be considered modified") - } -} - -func TestIsModifiedWithDifferentHeaders(t *testing.T) { - r := &Response{StatusCode: 200, ETag: "some etag", LastModified: "some date"} - if !r.IsModified("etag", "lastModified") { - t.Error("The resource should be considered modified") - } -} -- cgit v1.2.3