1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
|
// Copyright 2017 Frédéric Guillot. All rights reserved.
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.
package http
import (
"bytes"
"crypto/tls"
"encoding/json"
"fmt"
"io"
"net/http"
"net/url"
"strings"
"time"
"github.com/miniflux/miniflux/logger"
"github.com/miniflux/miniflux/timer"
)
// Note: Some websites have a user agent filter.
const userAgent = "Mozilla/5.0 (like Gecko, like Safari, like Chrome) - Miniflux <https://miniflux.net/>"
const requestTimeout = 300
const maxBodySize = 1024 * 1024 * 15
// Client is a HTTP Client :)
type Client struct {
url string
etagHeader string
lastModifiedHeader string
authorizationHeader string
username string
password string
Insecure bool
}
// Get execute a GET HTTP request.
func (c *Client) Get() (*Response, error) {
request, err := c.buildRequest(http.MethodGet, nil)
if err != nil {
return nil, err
}
return c.executeRequest(request)
}
// PostForm execute a POST HTTP request with form values.
func (c *Client) PostForm(values url.Values) (*Response, error) {
request, err := c.buildRequest(http.MethodPost, strings.NewReader(values.Encode()))
if err != nil {
return nil, err
}
request.Header.Add("Content-Type", "application/x-www-form-urlencoded")
return c.executeRequest(request)
}
// PostJSON execute a POST HTTP request with JSON payload.
func (c *Client) PostJSON(data interface{}) (*Response, error) {
b, err := json.Marshal(data)
if err != nil {
return nil, err
}
request, err := c.buildRequest(http.MethodPost, bytes.NewReader(b))
if err != nil {
return nil, err
}
request.Header.Add("Content-Type", "application/json")
return c.executeRequest(request)
}
func (c *Client) executeRequest(request *http.Request) (*Response, error) {
defer timer.ExecutionTime(time.Now(), fmt.Sprintf("[HttpClient] url=%s", c.url))
client := c.buildClient()
resp, err := client.Do(request)
if err != nil {
return nil, err
}
if resp.ContentLength > maxBodySize {
return nil, fmt.Errorf("client: response too large (%d bytes)", resp.ContentLength)
}
response := &Response{
Body: resp.Body,
StatusCode: resp.StatusCode,
EffectiveURL: resp.Request.URL.String(),
LastModified: resp.Header.Get("Last-Modified"),
ETag: resp.Header.Get("ETag"),
ContentType: resp.Header.Get("Content-Type"),
ContentLength: resp.ContentLength,
}
logger.Debug("[HttpClient:%s] OriginalURL=%s, StatusCode=%d, ContentLength=%d, ETag=%s, LastModified=%s, EffectiveURL=%s",
request.Method,
c.url,
response.StatusCode,
resp.ContentLength,
response.ETag,
response.LastModified,
response.EffectiveURL,
)
return response, err
}
func (c *Client) buildRequest(method string, body io.Reader) (*http.Request, error) {
request, err := http.NewRequest(method, c.url, body)
if err != nil {
return nil, err
}
if c.username != "" && c.password != "" {
request.SetBasicAuth(c.username, c.password)
}
request.Header = c.buildHeaders()
return request, nil
}
func (c *Client) buildClient() http.Client {
client := http.Client{Timeout: time.Duration(requestTimeout * time.Second)}
if c.Insecure {
client.Transport = &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
}
}
return client
}
func (c *Client) buildHeaders() http.Header {
headers := make(http.Header)
headers.Add("User-Agent", userAgent)
headers.Add("Accept", "*/*")
if c.etagHeader != "" {
headers.Add("If-None-Match", c.etagHeader)
}
if c.lastModifiedHeader != "" {
headers.Add("If-Modified-Since", c.lastModifiedHeader)
}
if c.authorizationHeader != "" {
headers.Add("Authorization", c.authorizationHeader)
}
return headers
}
// NewClient returns a new HTTP client.
func NewClient(url string) *Client {
return &Client{url: url, Insecure: false}
}
// NewClientWithCredentials returns a new HTTP client that requires authentication.
func NewClientWithCredentials(url, username, password string) *Client {
return &Client{url: url, Insecure: false, username: username, password: password}
}
// NewClientWithAuthorization returns a new client with a custom authorization header.
func NewClientWithAuthorization(url, authorization string) *Client {
return &Client{url: url, Insecure: false, authorizationHeader: authorization}
}
// NewClientWithCacheHeaders returns a new HTTP client that send cache headers.
func NewClientWithCacheHeaders(url, etagHeader, lastModifiedHeader string) *Client {
return &Client{url: url, etagHeader: etagHeader, lastModifiedHeader: lastModifiedHeader, Insecure: false}
}
|
