Make sure XML feeds are always encoded in UTF-8

5 files changed, 60 insertions, 33 deletions

diff --git a/reader/feed/handler.go b/reader/feed/handler.go
index 27ff126..c046ad9 100644
--- a/reader/feed/handler.go
+++ b/reader/feed/handler.go
@@ -6,14 +6,15 @@ package feed
 
 import (
 	"fmt"
+	"log"
+	"time"
+
 	"github.com/miniflux/miniflux2/errors"
 	"github.com/miniflux/miniflux2/helper"
 	"github.com/miniflux/miniflux2/model"
 	"github.com/miniflux/miniflux2/reader/http"
 	"github.com/miniflux/miniflux2/reader/icon"
 	"github.com/miniflux/miniflux2/storage"
-	"log"
-	"time"
 )
 
 var (
@@ -21,6 +22,7 @@ var (
 	errServerFailure = "Unable to fetch feed (statusCode=%d)."
 	errDuplicate     = "This feed already exists (%s)."
 	errNotFound      = "Feed %d not found"
+	errEncoding      = "Unable to normalize encoding: %v."
 )
 
 // Handler contains all the logic to create and refresh feeds.
@@ -32,7 +34,7 @@ type Handler struct {
 func (h *Handler) CreateFeed(userID, categoryID int64, url string) (*model.Feed, error) {
 	defer helper.ExecutionTime(time.Now(), fmt.Sprintf("[Handler:CreateFeed] feedUrl=%s", url))
 
-	client := http.NewHttpClient(url)
+	client := http.NewClient(url)
 	response, err := client.Get()
 	if err != nil {
 		return nil, errors.NewLocalizedError(errRequestFailed, err)
@@ -46,7 +48,12 @@ func (h *Handler) CreateFeed(userID, categoryID int64, url string) (*model.Feed,
 		return nil, errors.NewLocalizedError(errDuplicate, response.EffectiveURL)
 	}
 
-	subscription, err := parseFeed(response.Body)
+	body, err := response.NormalizeBodyEncoding()
+	if err != nil {
+		return nil, errors.NewLocalizedError(errEncoding, err)
+	}
+
+	subscription, err := parseFeed(body)
 	if err != nil {
 		return nil, err
 	}
@@ -89,7 +96,7 @@ func (h *Handler) RefreshFeed(userID, feedID int64) error {
 		return errors.NewLocalizedError(errNotFound, feedID)
 	}
 
-	client := http.NewHttpClientWithCacheHeaders(originalFeed.FeedURL, originalFeed.EtagHeader, originalFeed.LastModifiedHeader)
+	client := http.NewClientWithCacheHeaders(originalFeed.FeedURL, originalFeed.EtagHeader, originalFeed.LastModifiedHeader)
 	response, err := client.Get()
 	if err != nil {
 		customErr := errors.NewLocalizedError(errRequestFailed, err)
@@ -111,8 +118,12 @@ func (h *Handler) RefreshFeed(userID, feedID int64) error {
 
 	if response.IsModified(originalFeed.EtagHeader, originalFeed.LastModifiedHeader) {
 		log.Printf("[Handler:RefreshFeed] Feed #%d has been modified\n", feedID)
+		body, err := response.NormalizeBodyEncoding()
+		if err != nil {
+			return errors.NewLocalizedError(errEncoding, err)
+		}
 
-		subscription, err := parseFeed(response.Body)
+		subscription, err := parseFeed(body)
 		if err != nil {
 			originalFeed.ParsingErrorCount++
 			originalFeed.ParsingErrorMsg = err.Error()
diff --git a/reader/http/client.go b/reader/http/client.go
index 745ff0d..edb3c86 100644
--- a/reader/http/client.go
+++ b/reader/http/client.go
@@ -7,23 +7,26 @@ package http
 import (
 	"crypto/tls"
 	"fmt"
-	"github.com/miniflux/miniflux2/helper"
 	"log"
 	"net/http"
 	"net/url"
 	"time"
+
+	"github.com/miniflux/miniflux2/helper"
 )
 
-const HTTP_USER_AGENT = "Miniflux <https://miniflux.net/>"
+const userAgent = "Miniflux <https://miniflux.net/>"
 
-type HttpClient struct {
+// Client is a HTTP Client :)
+type Client struct {
 	url                string
 	etagHeader         string
 	lastModifiedHeader string
 	Insecure           bool
 }
 
-func (h *HttpClient) Get() (*ServerResponse, error) {
+// Get execute a GET HTTP request.
+func (h *Client) Get() (*Response, error) {
 	defer helper.ExecutionTime(time.Now(), fmt.Sprintf("[HttpClient:Get] url=%s", h.url))
 	u, _ := url.Parse(h.url)
 
@@ -39,7 +42,7 @@ func (h *HttpClient) Get() (*ServerResponse, error) {
 		return nil, err
 	}
 
-	response := &ServerResponse{
+	response := &Response{
 		Body:         resp.Body,
 		StatusCode:   resp.StatusCode,
 		EffectiveURL: resp.Request.URL.String(),
@@ -59,7 +62,7 @@ func (h *HttpClient) Get() (*ServerResponse, error) {
 	return response, err
 }
 
-func (h *HttpClient) buildClient() http.Client {
+func (h *Client) buildClient() http.Client {
 	if h.Insecure {
 		transport := &http.Transport{
 			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
@@ -71,9 +74,9 @@ func (h *HttpClient) buildClient() http.Client {
 	return http.Client{}
 }
 
-func (h *HttpClient) buildHeaders() http.Header {
+func (h *Client) buildHeaders() http.Header {
 	headers := make(http.Header)
-	headers.Add("User-Agent", HTTP_USER_AGENT)
+	headers.Add("User-Agent", userAgent)
 
 	if h.etagHeader != "" {
 		headers.Add("If-None-Match", h.etagHeader)
@@ -86,10 +89,12 @@ func (h *HttpClient) buildHeaders() http.Header {
 	return headers
 }
 
-func NewHttpClient(url string) *HttpClient {
-	return &HttpClient{url: url, Insecure: false}
+// NewClient returns a new HTTP client.
+func NewClient(url string) *Client {
+	return &Client{url: url, Insecure: false}
 }
 
-func NewHttpClientWithCacheHeaders(url, etagHeader, lastModifiedHeader string) *HttpClient {
-	return &HttpClient{url: url, etagHeader: etagHeader, lastModifiedHeader: lastModifiedHeader, Insecure: false}
+// NewClientWithCacheHeaders returns a new HTTP client that send cache headers.
+func NewClientWithCacheHeaders(url, etagHeader, lastModifiedHeader string) *Client {
+	return &Client{url: url, etagHeader: etagHeader, lastModifiedHeader: lastModifiedHeader, Insecure: false}
 }
diff --git a/reader/http/response.go b/reader/http/response.go
index 49e9f19..b2dfb54 100644
--- a/reader/http/response.go
+++ b/reader/http/response.go
@@ -5,8 +5,10 @@
 package http
 
 import "io"
+import "golang.org/x/net/html/charset"
 
-type ServerResponse struct {
+// Response wraps a server response.
+type Response struct {
 	Body         io.Reader
 	StatusCode   int
 	EffectiveURL string
@@ -15,18 +17,25 @@ type ServerResponse struct {
 	ContentType  string
 }
 
-func (s *ServerResponse) HasServerFailure() bool {
-	return s.StatusCode >= 400
+// HasServerFailure returns true if the status code represents a failure.
+func (r *Response) HasServerFailure() bool {
+	return r.StatusCode >= 400
 }
 
-func (s *ServerResponse) IsModified(etag, lastModified string) bool {
-	if s.StatusCode == 304 {
+// IsModified returns true if the resource has been modified.
+func (r *Response) IsModified(etag, lastModified string) bool {
+	if r.StatusCode == 304 {
 		return false
 	}
 
-	if s.ETag != "" && s.LastModified != "" && (s.ETag == etag || s.LastModified == lastModified) {
+	if r.ETag != "" && r.LastModified != "" && (r.ETag == etag || r.LastModified == lastModified) {
 		return false
 	}
 
 	return true
 }
+
+// NormalizeBodyEncoding make sure the body is encoded in UTF-8.
+func (r *Response) NormalizeBodyEncoding() (io.Reader, error) {
+	return charset.NewReader(r.Body, r.ContentType)
+}
diff --git a/reader/icon/finder.go b/reader/icon/finder.go
index 54d509f..fe6e86d 100644
--- a/reader/icon/finder.go
+++ b/reader/icon/finder.go
@@ -6,13 +6,14 @@ package icon
 
 import (
 	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+
 	"github.com/miniflux/miniflux2/helper"
 	"github.com/miniflux/miniflux2/model"
 	"github.com/miniflux/miniflux2/reader/http"
 	"github.com/miniflux/miniflux2/reader/url"
-	"io"
-	"io/ioutil"
-	"log"
 
 	"github.com/PuerkitoBio/goquery"
 )
@@ -20,7 +21,7 @@ import (
 // FindIcon try to find the website's icon.
 func FindIcon(websiteURL string) (*model.Icon, error) {
 	rootURL := url.GetRootURL(websiteURL)
-	client := http.NewHttpClient(rootURL)
+	client := http.NewClient(rootURL)
 	response, err := client.Get()
 	if err != nil {
 		return nil, fmt.Errorf("unable to download website index page: %v", err)
@@ -80,7 +81,7 @@ func parseDocument(websiteURL string, data io.Reader) (string, error) {
 }
 
 func downloadIcon(iconURL string) (*model.Icon, error) {
-	client := http.NewHttpClient(iconURL)
+	client := http.NewClient(iconURL)
 	response, err := client.Get()
 	if err != nil {
 		return nil, fmt.Errorf("unable to download iconURL: %v", err)
diff --git a/reader/subscription/finder.go b/reader/subscription/finder.go
index 7314644..cb6fbf2 100644
--- a/reader/subscription/finder.go
+++ b/reader/subscription/finder.go
@@ -7,14 +7,15 @@ package subscription
 import (
 	"bytes"
 	"fmt"
+	"io"
+	"log"
+	"time"
+
 	"github.com/miniflux/miniflux2/errors"
 	"github.com/miniflux/miniflux2/helper"
 	"github.com/miniflux/miniflux2/reader/feed"
 	"github.com/miniflux/miniflux2/reader/http"
 	"github.com/miniflux/miniflux2/reader/url"
-	"io"
-	"log"
-	"time"
 
 	"github.com/PuerkitoBio/goquery"
 )
@@ -28,7 +29,7 @@ var (
 func FindSubscriptions(websiteURL string) (Subscriptions, error) {
 	defer helper.ExecutionTime(time.Now(), fmt.Sprintf("[FindSubscriptions] url=%s", websiteURL))
 
-	client := http.NewHttpClient(websiteURL)
+	client := http.NewClient(websiteURL)
 	response, err := client.Get()
 	if err != nil {
 		return nil, errors.NewLocalizedError(errConnectionFailure, err)