aboutsummaryrefslogtreecommitdiffhomepage
path: root/reader
diff options
context:
space:
mode:
Diffstat (limited to 'reader')
-rw-r--r--reader/feed/handler.go2
-rw-r--r--reader/http/client.go115
-rw-r--r--reader/http/response.go52
-rw-r--r--reader/http/response_test.go56
-rw-r--r--reader/icon/finder.go4
-rw-r--r--reader/sanitizer/sanitizer.go2
-rw-r--r--reader/subscription/finder.go4
-rw-r--r--reader/url/url.go71
-rw-r--r--reader/url/url_test.go117
9 files changed, 6 insertions, 417 deletions
diff --git a/reader/feed/handler.go b/reader/feed/handler.go
index f406c13..e978deb 100644
--- a/reader/feed/handler.go
+++ b/reader/feed/handler.go
@@ -11,8 +11,8 @@ import (
"github.com/miniflux/miniflux2/errors"
"github.com/miniflux/miniflux2/helper"
+ "github.com/miniflux/miniflux2/http"
"github.com/miniflux/miniflux2/model"
- "github.com/miniflux/miniflux2/reader/http"
"github.com/miniflux/miniflux2/reader/icon"
"github.com/miniflux/miniflux2/storage"
)
diff --git a/reader/http/client.go b/reader/http/client.go
deleted file mode 100644
index 3b4488f..0000000
--- a/reader/http/client.go
+++ /dev/null
@@ -1,115 +0,0 @@
-// Copyright 2017 Frédéric Guillot. All rights reserved.
-// Use of this source code is governed by the Apache 2.0
-// license that can be found in the LICENSE file.
-
-package http
-
-import (
- "crypto/tls"
- "fmt"
- "log"
- "net/http"
- "net/url"
- "time"
-
- "github.com/miniflux/miniflux2/helper"
-)
-
-const userAgent = "Miniflux <https://miniflux.net/>"
-const requestTimeout = 300
-
-// Client is a HTTP Client :)
-type Client struct {
- url string
- etagHeader string
- lastModifiedHeader string
- username string
- password string
- Insecure bool
-}
-
-// Get execute a GET HTTP request.
-func (c *Client) Get() (*Response, error) {
- defer helper.ExecutionTime(time.Now(), fmt.Sprintf("[HttpClient:Get] url=%s", c.url))
-
- client := c.buildClient()
- resp, err := client.Do(c.buildRequest())
- if err != nil {
- return nil, err
- }
-
- response := &Response{
- Body: resp.Body,
- StatusCode: resp.StatusCode,
- EffectiveURL: resp.Request.URL.String(),
- LastModified: resp.Header.Get("Last-Modified"),
- ETag: resp.Header.Get("ETag"),
- ContentType: resp.Header.Get("Content-Type"),
- }
-
- log.Println("[HttpClient:Get]",
- "OriginalURL:", c.url,
- "StatusCode:", response.StatusCode,
- "ETag:", response.ETag,
- "LastModified:", response.LastModified,
- "EffectiveURL:", response.EffectiveURL,
- )
-
- return response, err
-}
-
-func (c *Client) buildRequest() *http.Request {
- link, _ := url.Parse(c.url)
- request := &http.Request{
- URL: link,
- Method: http.MethodGet,
- Header: c.buildHeaders(),
- }
-
- if c.username != "" && c.password != "" {
- request.SetBasicAuth(c.username, c.password)
- }
-
- return request
-}
-
-func (c *Client) buildClient() http.Client {
- client := http.Client{Timeout: time.Duration(requestTimeout * time.Second)}
- if c.Insecure {
- client.Transport = &http.Transport{
- TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
- }
- }
-
- return client
-}
-
-func (c *Client) buildHeaders() http.Header {
- headers := make(http.Header)
- headers.Add("User-Agent", userAgent)
-
- if c.etagHeader != "" {
- headers.Add("If-None-Match", c.etagHeader)
- }
-
- if c.lastModifiedHeader != "" {
- headers.Add("If-Modified-Since", c.lastModifiedHeader)
- }
-
- return headers
-}
-
-// NewClient returns a new HTTP client.
-func NewClient(url string) *Client {
- return &Client{url: url, Insecure: false}
-}
-
-// NewClientWithCredentials returns a new HTTP client that require authentication.
-func NewClientWithCredentials(url, username, password string) *Client {
- return &Client{url: url, Insecure: false, username: username, password: password}
-}
-
-// NewClientWithCacheHeaders returns a new HTTP client that send cache headers.
-func NewClientWithCacheHeaders(url, etagHeader, lastModifiedHeader string) *Client {
- return &Client{url: url, etagHeader: etagHeader, lastModifiedHeader: lastModifiedHeader, Insecure: false}
-}
diff --git a/reader/http/response.go b/reader/http/response.go
deleted file mode 100644
index 7213be7..0000000
--- a/reader/http/response.go
+++ /dev/null
@@ -1,52 +0,0 @@
-// Copyright 2017 Frédéric Guillot. All rights reserved.
-// Use of this source code is governed by the Apache 2.0
-// license that can be found in the LICENSE file.
-
-package http
-
-import (
- "io"
- "strings"
-
- "golang.org/x/net/html/charset"
-)
-
-// Response wraps a server response.
-type Response struct {
- Body io.Reader
- StatusCode int
- EffectiveURL string
- LastModified string
- ETag string
- ContentType string
-}
-
-// HasServerFailure returns true if the status code represents a failure.
-func (r *Response) HasServerFailure() bool {
- return r.StatusCode >= 400
-}
-
-// IsModified returns true if the resource has been modified.
-func (r *Response) IsModified(etag, lastModified string) bool {
- if r.StatusCode == 304 {
- return false
- }
-
- if r.ETag != "" && r.ETag == etag {
- return false
- }
-
- if r.LastModified != "" && r.LastModified == lastModified {
- return false
- }
-
- return true
-}
-
-// NormalizeBodyEncoding make sure the body is encoded in UTF-8.
-func (r *Response) NormalizeBodyEncoding() (io.Reader, error) {
- if strings.Contains(r.ContentType, "charset=") {
- return charset.NewReader(r.Body, r.ContentType)
- }
- return r.Body, nil
-}
diff --git a/reader/http/response_test.go b/reader/http/response_test.go
deleted file mode 100644
index c5f6a1c..0000000
--- a/reader/http/response_test.go
+++ /dev/null
@@ -1,56 +0,0 @@
-// Copyright 2017 Frédéric Guillot. All rights reserved.
-// Use of this source code is governed by the Apache 2.0
-// license that can be found in the LICENSE file.
-
-package http
-
-import "testing"
-
-func TestHasServerFailureWith200Status(t *testing.T) {
- r := &Response{StatusCode: 200}
- if r.HasServerFailure() {
- t.Error("200 is not a failure")
- }
-}
-
-func TestHasServerFailureWith404Status(t *testing.T) {
- r := &Response{StatusCode: 404}
- if !r.HasServerFailure() {
- t.Error("404 is a failure")
- }
-}
-
-func TestHasServerFailureWith500Status(t *testing.T) {
- r := &Response{StatusCode: 500}
- if !r.HasServerFailure() {
- t.Error("500 is a failure")
- }
-}
-
-func TestIsModifiedWith304Status(t *testing.T) {
- r := &Response{StatusCode: 304}
- if r.IsModified("etag", "lastModified") {
- t.Error("The resource should not be considered modified")
- }
-}
-
-func TestIsModifiedWithIdenticalEtag(t *testing.T) {
- r := &Response{StatusCode: 200, ETag: "etag"}
- if r.IsModified("etag", "lastModified") {
- t.Error("The resource should not be considered modified")
- }
-}
-
-func TestIsModifiedWithIdenticalLastModified(t *testing.T) {
- r := &Response{StatusCode: 200, LastModified: "lastModified"}
- if r.IsModified("etag", "lastModified") {
- t.Error("The resource should not be considered modified")
- }
-}
-
-func TestIsModifiedWithDifferentHeaders(t *testing.T) {
- r := &Response{StatusCode: 200, ETag: "some etag", LastModified: "some date"}
- if !r.IsModified("etag", "lastModified") {
- t.Error("The resource should be considered modified")
- }
-}
diff --git a/reader/icon/finder.go b/reader/icon/finder.go
index c55e4d1..c5f9a07 100644
--- a/reader/icon/finder.go
+++ b/reader/icon/finder.go
@@ -11,9 +11,9 @@ import (
"log"
"github.com/miniflux/miniflux2/helper"
+ "github.com/miniflux/miniflux2/http"
"github.com/miniflux/miniflux2/model"
- "github.com/miniflux/miniflux2/reader/http"
- "github.com/miniflux/miniflux2/reader/url"
+ "github.com/miniflux/miniflux2/url"
"github.com/PuerkitoBio/goquery"
)
diff --git a/reader/sanitizer/sanitizer.go b/reader/sanitizer/sanitizer.go
index 1c1c7c1..49fb1cb 100644
--- a/reader/sanitizer/sanitizer.go
+++ b/reader/sanitizer/sanitizer.go
@@ -10,7 +10,7 @@ import (
"io"
"strings"
- "github.com/miniflux/miniflux2/reader/url"
+ "github.com/miniflux/miniflux2/url"
"golang.org/x/net/html"
)
diff --git a/reader/subscription/finder.go b/reader/subscription/finder.go
index 506fe1a..239baf7 100644
--- a/reader/subscription/finder.go
+++ b/reader/subscription/finder.go
@@ -13,9 +13,9 @@ import (
"github.com/miniflux/miniflux2/errors"
"github.com/miniflux/miniflux2/helper"
+ "github.com/miniflux/miniflux2/http"
"github.com/miniflux/miniflux2/reader/feed"
- "github.com/miniflux/miniflux2/reader/http"
- "github.com/miniflux/miniflux2/reader/url"
+ "github.com/miniflux/miniflux2/url"
"github.com/PuerkitoBio/goquery"
)
diff --git a/reader/url/url.go b/reader/url/url.go
deleted file mode 100644
index e5a7c62..0000000
--- a/reader/url/url.go
+++ /dev/null
@@ -1,71 +0,0 @@
-// Copyright 2017 Frédéric Guillot. All rights reserved.
-// Use of this source code is governed by the Apache 2.0
-// license that can be found in the LICENSE file.
-
-package url
-
-import "net/url"
-import "fmt"
-import "strings"
-
-// AbsoluteURL converts the input URL as absolute URL if necessary.
-func AbsoluteURL(baseURL, input string) (string, error) {
- if strings.HasPrefix(input, "//") {
- input = "https://" + input[2:]
- }
-
- u, err := url.Parse(input)
- if err != nil {
- return "", fmt.Errorf("unable to parse input URL: %v", err)
- }
-
- if u.IsAbs() {
- return u.String(), nil
- }
-
- base, err := url.Parse(baseURL)
- if err != nil {
- return "", fmt.Errorf("unable to parse base URL: %v", err)
- }
-
- return base.ResolveReference(u).String(), nil
-}
-
-// RootURL returns absolute URL without the path.
-func RootURL(websiteURL string) string {
- if strings.HasPrefix(websiteURL, "//") {
- websiteURL = "https://" + websiteURL[2:]
- }
-
- absoluteURL, err := AbsoluteURL(websiteURL, "")
- if err != nil {
- return websiteURL
- }
-
- u, err := url.Parse(absoluteURL)
- if err != nil {
- return absoluteURL
- }
-
- return u.Scheme + "://" + u.Host + "/"
-}
-
-// IsHTTPS returns true if the URL is using HTTPS.
-func IsHTTPS(websiteURL string) bool {
- parsedURL, err := url.Parse(websiteURL)
- if err != nil {
- return false
- }
-
- return strings.ToLower(parsedURL.Scheme) == "https"
-}
-
-// Domain returns only the domain part of the given URL.
-func Domain(websiteURL string) string {
- parsedURL, err := url.Parse(websiteURL)
- if err != nil {
- return websiteURL
- }
-
- return parsedURL.Host
-}
diff --git a/reader/url/url_test.go b/reader/url/url_test.go
deleted file mode 100644
index 13c723a..0000000
--- a/reader/url/url_test.go
+++ /dev/null
@@ -1,117 +0,0 @@
-package url
-
-import "testing"
-
-func TestGetAbsoluteURLWithAbsolutePath(t *testing.T) {
- expected := `https://example.org/path/file.ext`
- input := `/path/file.ext`
- output, err := AbsoluteURL("https://example.org/folder/", input)
-
- if err != nil {
- t.Error(err)
- }
-
- if expected != output {
- t.Errorf(`Unexpected output, got "%s" instead of "%s"`, output, expected)
- }
-}
-
-func TestGetAbsoluteURLWithRelativePath(t *testing.T) {
- expected := `https://example.org/folder/path/file.ext`
- input := `path/file.ext`
- output, err := AbsoluteURL("https://example.org/folder/", input)
-
- if err != nil {
- t.Error(err)
- }
-
- if expected != output {
- t.Errorf(`Unexpected output, got "%s" instead of "%s"`, output, expected)
- }
-}
-
-func TestGetAbsoluteURLWithRelativePaths(t *testing.T) {
- expected := `https://example.org/path/file.ext`
- input := `path/file.ext`
- output, err := AbsoluteURL("https://example.org/folder", input)
-
- if err != nil {
- t.Error(err)
- }
-
- if expected != output {
- t.Errorf(`Unexpected output, got "%s" instead of "%s"`, output, expected)
- }
-}
-
-func TestWhenInputIsAlreadyAbsolute(t *testing.T) {
- expected := `https://example.org/path/file.ext`
- input := `https://example.org/path/file.ext`
- output, err := AbsoluteURL("https://example.org/folder/", input)
-
- if err != nil {
- t.Error(err)
- }
-
- if expected != output {
- t.Errorf(`Unexpected output, got "%s" instead of "%s"`, output, expected)
- }
-}
-
-func TestGetAbsoluteURLWithProtocolRelative(t *testing.T) {
- expected := `https://static.example.org/path/file.ext`
- input := `//static.example.org/path/file.ext`
- output, err := AbsoluteURL("https://www.example.org/", input)
-
- if err != nil {
- t.Error(err)
- }
-
- if expected != output {
- t.Errorf(`Unexpected output, got "%s" instead of "%s"`, output, expected)
- }
-}
-
-func TestGetRootURL(t *testing.T) {
- expected := `https://example.org/`
- input := `https://example.org/path/file.ext`
- output := RootURL(input)
-
- if expected != output {
- t.Errorf(`Unexpected output, got "%s" instead of "%s"`, output, expected)
- }
-}
-
-func TestGetRootURLWithProtocolRelativePath(t *testing.T) {
- expected := `https://static.example.org/`
- input := `//static.example.org/path/file.ext`
- output := RootURL(input)
-
- if expected != output {
- t.Errorf(`Unexpected output, got "%s" instead of "%s"`, output, expected)
- }
-}
-
-func TestIsHTTPS(t *testing.T) {
- if !IsHTTPS("https://example.org/") {
- t.Error("Unable to recognize HTTPS URL")
- }
-
- if IsHTTPS("http://example.org/") {
- t.Error("Unable to recognize HTTP URL")
- }
-
- if IsHTTPS("") {
- t.Error("Unable to recognize malformed URL")
- }
-}
-
-func TestGetDomain(t *testing.T) {
- expected := `static.example.org`
- input := `http://static.example.org/`
- output := Domain(input)
-
- if expected != output {
- t.Errorf(`Unexpected output, got "%s" instead of "%s"`, output, expected)
- }
-}
generated by cgit v1.2.3 (git 2.39.1) at 2024-06-26 06:25:21 +0000