diff options
author | Frédéric Guillot <fred@miniflux.net> | 2019-12-26 15:26:23 -0800 |
---|---|---|
committer | Frédéric Guillot <fred@miniflux.net> | 2019-12-26 15:56:59 -0800 |
commit | 3debf75eb9229144a05701e03ba59408a75dd815 (patch) | |
tree | 9e9eb6569db3234b514f798d4278b20793b79833 /url | |
parent | 200b1c304b999191a29f36d4122e7aa05481125c (diff) |
Normalize URL query string before executing HTTP requests
- Make sure query strings parameters are encoded
- As opposed to the standard library, do not append equal sign
for query parameters with empty value
- Strip URL fragments like Web browsers
Diffstat (limited to 'url')
-rw-r--r-- | url/url.go | 57 | ||||
-rw-r--r-- | url/url_test.go | 22 |
2 files changed, 76 insertions, 3 deletions
@@ -4,9 +4,12 @@ package url // import "miniflux.app/url" -import "net/url" -import "fmt" -import "strings" +import ( + "fmt" + "net/url" + "sort" + "strings" +) // AbsoluteURL converts the input URL as absolute URL if necessary. func AbsoluteURL(baseURL, input string) (string, error) { @@ -69,3 +72,51 @@ func Domain(websiteURL string) string { return parsedURL.Host } + +// RequestURI returns the encoded URI to be used in HTTP requests. +func RequestURI(websiteURL string) string { + u, err := url.Parse(websiteURL) + if err != nil { + return websiteURL + } + + queryValues := u.Query() + u.RawQuery = "" // Clear RawQuery to make sure it's encoded properly. + u.Fragment = "" // Clear fragment because Web browsers strip #fragment before sending the URL to a web server. + + var buf strings.Builder + buf.WriteString(u.String()) + + if len(queryValues) > 0 { + buf.WriteByte('?') + + // Sort keys. + keys := make([]string, 0, len(queryValues)) + for k := range queryValues { + keys = append(keys, k) + } + sort.Strings(keys) + + i := 0 + for _, key := range keys { + keyEscaped := url.QueryEscape(key) + values := queryValues[key] + for _, value := range values { + if i > 0 { + buf.WriteByte('&') + } + buf.WriteString(keyEscaped) + + // As opposed to the standard library, we append the = only if the value is not empty. + if value != "" { + buf.WriteByte('=') + buf.WriteString(url.QueryEscape(value)) + } + + i++ + } + } + } + + return buf.String() +} diff --git a/url/url_test.go b/url/url_test.go index 9a14b20..54868a9 100644 --- a/url/url_test.go +++ b/url/url_test.go @@ -71,3 +71,25 @@ func TestDomain(t *testing.T) { } } } + +func TestRequestURI(t *testing.T) { + scenarios := map[string]string{ + "https://www.example.org": "https://www.example.org", + "https://user:password@www.example.org": "https://user:password@www.example.org", + "https://www.example.org/path with spaces": "https://www.example.org/path%20with%20spaces", + "https://www.example.org/path#test": "https://www.example.org/path", + "https://www.example.org/path?abc#test": "https://www.example.org/path?abc", + "https://www.example.org/path?a=b&a=c": "https://www.example.org/path?a=b&a=c", + "https://www.example.org/path?a=b&a=c&d": "https://www.example.org/path?a=b&a=c&d", + "https://www.example.org/path?atom": "https://www.example.org/path?atom", + "https://www.example.org/path?测试=测试": "https://www.example.org/path?%E6%B5%8B%E8%AF%95=%E6%B5%8B%E8%AF%95", + "https://www.example.org/url=http%3A%2F%2Fwww.example.com%2Ffeed%2F&max=20": "https://www.example.org/url=http%3A%2F%2Fwww.example.com%2Ffeed%2F&max=20", + } + + for input, expected := range scenarios { + actual := RequestURI(input) + if actual != expected { + t.Errorf(`Unexpected result, got %q instead of %q`, actual, expected) + } + } +} |