From 2538eea1776e1d03d33465ad2001512caca93937 Mon Sep 17 00:00:00 2001 From: Patrick Date: Thu, 20 Sep 2018 03:19:24 +0200 Subject: Add the possibility to override default user agent for each feed --- api/feed.go | 1 + api/payload.go | 13 ++++++++++--- api/subscription.go | 1 + client/core.go | 2 ++ database/migration.go | 2 +- database/sql.go | 2 ++ database/sql/schema_version_21.sql | 1 + http/client/client.go | 16 ++++++++++++++-- model/feed.go | 1 + reader/feed/handler.go | 6 +++++- reader/processor/processor.go | 8 +++++++- reader/scraper/scraper.go | 6 +++++- reader/subscription/finder.go | 3 ++- storage/entry_query_builder.go | 3 ++- storage/feed.go | 18 +++++++++++------- template/html/add_subscription.html | 3 +++ template/html/choose_subscription.html | 1 + template/html/edit_feed.html | 3 +++ template/views.go | 13 ++++++++++--- tests/feed_test.go | 25 +++++++++++++++++++++++++ ui/entry_scraper.go | 2 +- ui/feed_edit.go | 3 +++ ui/feed_update.go | 2 ++ ui/form/feed.go | 3 +++ ui/form/subscription.go | 2 ++ ui/subscription_add.go | 2 ++ ui/subscription_bookmarklet.go | 2 ++ ui/subscription_choose.go | 3 +++ ui/subscription_submit.go | 4 ++++ 29 files changed, 129 insertions(+), 22 deletions(-) create mode 100644 database/sql/schema_version_21.sql diff --git a/api/feed.go b/api/feed.go index e303f58..17bb73d 100644 --- a/api/feed.go +++ b/api/feed.go @@ -47,6 +47,7 @@ func (c *Controller) CreateFeed(w http.ResponseWriter, r *http.Request) { feedInfo.CategoryID, feedInfo.FeedURL, feedInfo.Crawler, + feedInfo.UserAgent, feedInfo.Username, feedInfo.Password, ) diff --git a/api/payload.go b/api/payload.go index ee1e41b..5acf0bb 100644 --- a/api/payload.go +++ b/api/payload.go @@ -26,15 +26,17 @@ type entriesResponse struct { type feedCreation struct { FeedURL string `json:"feed_url"` CategoryID int64 `json:"category_id"` + UserAgent string `json:"user_agent"` Username string `json:"username"` Password string `json:"password"` Crawler bool `json:"crawler"` } type subscriptionDiscovery struct { - URL string `json:"url"` - Username string `json:"username"` - Password string `json:"password"` + URL string `json:"url"` + UserAgent string `json:"user_agent"` + Username string `json:"username"` + Password string `json:"password"` } type feedModification struct { @@ -44,6 +46,7 @@ type feedModification struct { ScraperRules *string `json:"scraper_rules"` RewriteRules *string `json:"rewrite_rules"` Crawler *bool `json:"crawler"` + UserAgent *string `json:"user_agent"` Username *string `json:"username"` Password *string `json:"password"` CategoryID *int64 `json:"category_id"` @@ -74,6 +77,10 @@ func (f *feedModification) Update(feed *model.Feed) { feed.Crawler = *f.Crawler } + if f.UserAgent != nil { + feed.UserAgent = *f.UserAgent + } + if f.Username != nil { feed.Username = *f.Username } diff --git a/api/subscription.go b/api/subscription.go index 603b932..ff4c7cf 100644 --- a/api/subscription.go +++ b/api/subscription.go @@ -22,6 +22,7 @@ func (c *Controller) GetSubscriptions(w http.ResponseWriter, r *http.Request) { subscriptions, err := subscription.FindSubscriptions( subscriptionInfo.URL, + subscriptionInfo.UserAgent, subscriptionInfo.Username, subscriptionInfo.Password, ) diff --git a/client/core.go b/client/core.go index 437af51..53d1e13 100644 --- a/client/core.go +++ b/client/core.go @@ -91,6 +91,7 @@ type Feed struct { ScraperRules string `json:"scraper_rules"` RewriteRules string `json:"rewrite_rules"` Crawler bool `json:"crawler"` + UserAgent string `json:"user_agent"` Username string `json:"username"` Password string `json:"password"` Category *Category `json:"category,omitempty"` @@ -105,6 +106,7 @@ type FeedModification struct { ScraperRules *string `json:"scraper_rules"` RewriteRules *string `json:"rewrite_rules"` Crawler *bool `json:"crawler"` + UserAgent *string `json:"user_agent"` Username *string `json:"username"` Password *string `json:"password"` CategoryID *int64 `json:"category_id"` diff --git a/database/migration.go b/database/migration.go index 3179705..39d15b7 100644 --- a/database/migration.go +++ b/database/migration.go @@ -12,7 +12,7 @@ import ( "miniflux.app/logger" ) -const schemaVersion = 20 +const schemaVersion = 21 // Migrate executes database migrations. func Migrate(db *sql.DB) { diff --git a/database/sql.go b/database/sql.go index 34fecd8..be033fb 100644 --- a/database/sql.go +++ b/database/sql.go @@ -144,6 +144,7 @@ create index users_extra_idx on users using gin(extra); "schema_version_20": `alter table entries add column document_vectors tsvector; update entries set document_vectors = to_tsvector(title || ' ' || coalesce(content, '')); create index document_vectors_idx on entries using gin(document_vectors);`, + "schema_version_21": `alter table feeds add column user_agent text default '';`, "schema_version_3": `create table tokens ( id text not null, value text not null, @@ -192,6 +193,7 @@ var SqlMapChecksums = map[string]string{ "schema_version_19": "a83f77b41cc213d282805a5b518f15abbf96331599119f0ef4aca4be037add7b", "schema_version_2": "e8e9ff32478df04fcddad10a34cba2e8bb1e67e7977b5bd6cdc4c31ec94282b4", "schema_version_20": "6c4e9b2c5bccdc3243c239c390fb1caa5e15624e669b2c07e14c126f6d2e2cd6", + "schema_version_21": "77da01ee38918ff4fe33985fbb20ed3276a717a7584c2ca9ebcf4d4ab6cb6910", "schema_version_3": "a54745dbc1c51c000f74d4e5068f1e2f43e83309f023415b1749a47d5c1e0f12", "schema_version_4": "216ea3a7d3e1704e40c797b5dc47456517c27dbb6ca98bf88812f4f63d74b5d9", "schema_version_5": "46397e2f5f2c82116786127e9f6a403e975b14d2ca7b652a48cd1ba843e6a27c", diff --git a/database/sql/schema_version_21.sql b/database/sql/schema_version_21.sql new file mode 100644 index 0000000..2c96d76 --- /dev/null +++ b/database/sql/schema_version_21.sql @@ -0,0 +1 @@ +alter table feeds add column user_agent text default ''; \ No newline at end of file diff --git a/http/client/client.go b/http/client/client.go index 201b136..2dce15c 100644 --- a/http/client/client.go +++ b/http/client/client.go @@ -33,6 +33,9 @@ const ( ) var ( + // DefaultUserAgent sets the User-Agent header used for any requests by miniflux. + DefaultUserAgent = "Mozilla/5.0 (compatible; Miniflux/" + version.Version + "; +https://miniflux.app)" + errInvalidCertificate = "Invalid SSL certificate (original error: %q)" errTemporaryNetworkOperation = "This website is temporarily unreachable (original error: %q)" errPermanentNetworkOperation = "This website is permanently unreachable (original error: %q)" @@ -47,6 +50,7 @@ type Client struct { authorizationHeader string username string password string + userAgent string Insecure bool } @@ -72,6 +76,14 @@ func (c *Client) WithCacheHeaders(etagHeader, lastModifiedHeader string) *Client return c } +// WithUserAgent defines the User-Agent header to use for outgoing requests. +func (c *Client) WithUserAgent(userAgent string) *Client { + if userAgent != "" { + c.userAgent = userAgent + } + return c +} + // Get execute a GET HTTP request. func (c *Client) Get() (*Response, error) { request, err := c.buildRequest(http.MethodGet, nil) @@ -212,7 +224,7 @@ func (c *Client) buildClient() http.Client { func (c *Client) buildHeaders() http.Header { headers := make(http.Header) - headers.Add("User-Agent", "Mozilla/5.0 (compatible; Miniflux/"+version.Version+"; +https://miniflux.app)") + headers.Add("User-Agent", c.userAgent) headers.Add("Accept", "*/*") if c.etagHeader != "" { @@ -233,5 +245,5 @@ func (c *Client) buildHeaders() http.Header { // New returns a new HTTP client. func New(url string) *Client { - return &Client{url: url, Insecure: false} + return &Client{url: url, userAgent: DefaultUserAgent, Insecure: false} } diff --git a/model/feed.go b/model/feed.go index d6ce0d1..0ace098 100644 --- a/model/feed.go +++ b/model/feed.go @@ -24,6 +24,7 @@ type Feed struct { ScraperRules string `json:"scraper_rules"` RewriteRules string `json:"rewrite_rules"` Crawler bool `json:"crawler"` + UserAgent string `json:"user_agent"` Username string `json:"username"` Password string `json:"password"` Category *Category `json:"category,omitempty"` diff --git a/reader/feed/handler.go b/reader/feed/handler.go index fa09cb6..252d178 100644 --- a/reader/feed/handler.go +++ b/reader/feed/handler.go @@ -37,7 +37,7 @@ type Handler struct { } // CreateFeed fetch, parse and store a new feed. -func (h *Handler) CreateFeed(userID, categoryID int64, url string, crawler bool, username, password string) (*model.Feed, error) { +func (h *Handler) CreateFeed(userID, categoryID int64, url string, crawler bool, userAgent, username, password string) (*model.Feed, error) { defer timer.ExecutionTime(time.Now(), fmt.Sprintf("[Handler:CreateFeed] feedUrl=%s", url)) if !h.store.CategoryExists(userID, categoryID) { @@ -46,6 +46,7 @@ func (h *Handler) CreateFeed(userID, categoryID int64, url string, crawler bool, clt := client.New(url) clt.WithCredentials(username, password) + clt.WithUserAgent(userAgent) response, err := clt.Get() if err != nil { if _, ok := err.(*errors.LocalizedError); ok { @@ -87,6 +88,7 @@ func (h *Handler) CreateFeed(userID, categoryID int64, url string, crawler bool, subscription.FeedURL = response.EffectiveURL subscription.UserID = userID subscription.Crawler = crawler + subscription.UserAgent = userAgent subscription.Username = username subscription.Password = password @@ -136,6 +138,7 @@ func (h *Handler) RefreshFeed(userID, feedID int64) error { clt := client.New(originalFeed.FeedURL) clt.WithCredentials(originalFeed.Username, originalFeed.Password) clt.WithCacheHeaders(originalFeed.EtagHeader, originalFeed.LastModifiedHeader) + clt.WithUserAgent(originalFeed.UserAgent) response, err := clt.Get() if err != nil { var customErr errors.LocalizedError @@ -196,6 +199,7 @@ func (h *Handler) RefreshFeed(userID, feedID int64) error { feedProcessor := processor.NewFeedProcessor(userID, h.store, subscription) feedProcessor.WithScraperRules(originalFeed.ScraperRules) + feedProcessor.WithUserAgent(originalFeed.UserAgent) feedProcessor.WithRewriteRules(originalFeed.RewriteRules) feedProcessor.WithCrawler(originalFeed.Crawler) feedProcessor.Process() diff --git a/reader/processor/processor.go b/reader/processor/processor.go index 002f7e8..f57e6cd 100644 --- a/reader/processor/processor.go +++ b/reader/processor/processor.go @@ -21,6 +21,7 @@ type FeedProcessor struct { scraperRules string rewriteRules string crawler bool + userAgent string } // WithCrawler enables the crawler. @@ -33,6 +34,11 @@ func (f *FeedProcessor) WithScraperRules(rules string) { f.scraperRules = rules } +// WithUserAgent sets the User-Agent header for fetching article content. +func (f *FeedProcessor) WithUserAgent(userAgent string) { + f.userAgent = userAgent +} + // WithRewriteRules adds rewrite rules to the processing. func (f *FeedProcessor) WithRewriteRules(rules string) { f.rewriteRules = rules @@ -45,7 +51,7 @@ func (f *FeedProcessor) Process() { if f.store.EntryURLExists(f.userID, entry.URL) { logger.Debug(`[FeedProcessor] Do not crawl existing entry URL: "%s"`, entry.URL) } else { - content, err := scraper.Fetch(entry.URL, f.scraperRules) + content, err := scraper.Fetch(entry.URL, f.scraperRules, f.userAgent) if err != nil { logger.Error("[FeedProcessor] %v", err) } else { diff --git a/reader/scraper/scraper.go b/reader/scraper/scraper.go index d2cccdb..7aa7084 100644 --- a/reader/scraper/scraper.go +++ b/reader/scraper/scraper.go @@ -19,8 +19,12 @@ import ( ) // Fetch downloads a web page a returns relevant contents. -func Fetch(websiteURL, rules string) (string, error) { +func Fetch(websiteURL, rules, userAgent string) (string, error) { clt := client.New(websiteURL) + if userAgent != "" { + clt.WithUserAgent(userAgent) + } + response, err := clt.Get() if err != nil { return "", err diff --git a/reader/subscription/finder.go b/reader/subscription/finder.go index 8be6f73..027e810 100644 --- a/reader/subscription/finder.go +++ b/reader/subscription/finder.go @@ -29,11 +29,12 @@ var ( ) // FindSubscriptions downloads and try to find one or more subscriptions from an URL. -func FindSubscriptions(websiteURL, username, password string) (Subscriptions, error) { +func FindSubscriptions(websiteURL, userAgent, username, password string) (Subscriptions, error) { defer timer.ExecutionTime(time.Now(), fmt.Sprintf("[FindSubscriptions] url=%s", websiteURL)) clt := client.New(websiteURL) clt.WithCredentials(username, password) + clt.WithUserAgent(userAgent) response, err := clt.Get() if err != nil { if _, ok := err.(errors.LocalizedError); ok { diff --git a/storage/entry_query_builder.go b/storage/entry_query_builder.go index 18cf735..192f515 100644 --- a/storage/entry_query_builder.go +++ b/storage/entry_query_builder.go @@ -192,7 +192,7 @@ func (e *EntryQueryBuilder) GetEntries() (model.Entries, error) { e.id, e.user_id, e.feed_id, e.hash, e.published_at at time zone u.timezone, e.title, e.url, e.comments_url, e.author, e.content, e.status, e.starred, f.title as feed_title, f.feed_url, f.site_url, f.checked_at, - f.category_id, c.title as category_title, f.scraper_rules, f.rewrite_rules, f.crawler, + f.category_id, c.title as category_title, f.scraper_rules, f.rewrite_rules, f.crawler, f.user_agent, fi.icon_id, u.timezone FROM entries e @@ -247,6 +247,7 @@ func (e *EntryQueryBuilder) GetEntries() (model.Entries, error) { &entry.Feed.ScraperRules, &entry.Feed.RewriteRules, &entry.Feed.Crawler, + &entry.Feed.UserAgent, &iconID, &tz, ) diff --git a/storage/feed.go b/storage/feed.go index be312ac..c60c11c 100644 --- a/storage/feed.go +++ b/storage/feed.go @@ -66,7 +66,7 @@ func (s *Storage) Feeds(userID int64) (model.Feeds, error) { f.id, f.feed_url, f.site_url, f.title, f.etag_header, f.last_modified_header, f.user_id, f.checked_at at time zone u.timezone, f.parsing_error_count, f.parsing_error_msg, - f.scraper_rules, f.rewrite_rules, f.crawler, + f.scraper_rules, f.rewrite_rules, f.crawler, f.user_agent, f.username, f.password, f.category_id, c.title as category_title, fi.icon_id, @@ -104,6 +104,7 @@ func (s *Storage) Feeds(userID int64) (model.Feeds, error) { &feed.ScraperRules, &feed.RewriteRules, &feed.Crawler, + &feed.UserAgent, &feed.Username, &feed.Password, &feed.Category.ID, @@ -141,7 +142,7 @@ func (s *Storage) FeedByID(userID, feedID int64) (*model.Feed, error) { f.id, f.feed_url, f.site_url, f.title, f.etag_header, f.last_modified_header, f.user_id, f.checked_at at time zone u.timezone, f.parsing_error_count, f.parsing_error_msg, - f.scraper_rules, f.rewrite_rules, f.crawler, + f.scraper_rules, f.rewrite_rules, f.crawler, f.user_agent, f.username, f.password, f.category_id, c.title as category_title, fi.icon_id, @@ -166,6 +167,7 @@ func (s *Storage) FeedByID(userID, feedID int64) (*model.Feed, error) { &feed.ScraperRules, &feed.RewriteRules, &feed.Crawler, + &feed.UserAgent, &feed.Username, &feed.Password, &feed.Category.ID, @@ -194,8 +196,8 @@ func (s *Storage) CreateFeed(feed *model.Feed) error { defer timer.ExecutionTime(time.Now(), fmt.Sprintf("[Storage:CreateFeed] feedURL=%s", feed.FeedURL)) sql := ` INSERT INTO feeds - (feed_url, site_url, title, category_id, user_id, etag_header, last_modified_header, crawler, username, password) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) + (feed_url, site_url, title, category_id, user_id, etag_header, last_modified_header, crawler, user_agent, username, password) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11) RETURNING id ` @@ -209,6 +211,7 @@ func (s *Storage) CreateFeed(feed *model.Feed) error { feed.EtagHeader, feed.LastModifiedHeader, feed.Crawler, + feed.UserAgent, feed.Username, feed.Password, ).Scan(&feed.ID) @@ -234,9 +237,9 @@ func (s *Storage) UpdateFeed(feed *model.Feed) (err error) { query := `UPDATE feeds SET feed_url=$1, site_url=$2, title=$3, category_id=$4, etag_header=$5, last_modified_header=$6, checked_at=$7, - parsing_error_msg=$8, parsing_error_count=$9, scraper_rules=$10, rewrite_rules=$11, crawler=$12, - username=$13, password=$14 - WHERE id=$15 AND user_id=$16` + parsing_error_msg=$8, parsing_error_count=$9, scraper_rules=$10, rewrite_rules=$11, crawler=$12, user_agent=$13, + username=$14, password=$15 + WHERE id=$16 AND user_id=$17` _, err = s.db.Exec(query, feed.FeedURL, @@ -251,6 +254,7 @@ func (s *Storage) UpdateFeed(feed *model.Feed) (err error) { feed.ScraperRules, feed.RewriteRules, feed.Crawler, + feed.UserAgent, feed.Username, feed.Password, feed.ID, diff --git a/template/html/add_subscription.html b/template/html/add_subscription.html index 5b46549..7ccbc20 100644 --- a/template/html/add_subscription.html +++ b/template/html/add_subscription.html @@ -41,6 +41,9 @@ + + + diff --git a/template/html/choose_subscription.html b/template/html/choose_subscription.html index ad1c38a..2b053a1 100644 --- a/template/html/choose_subscription.html +++ b/template/html/choose_subscription.html @@ -19,6 +19,7 @@
+ {{ if .form.Crawler }} diff --git a/template/html/edit_feed.html b/template/html/edit_feed.html index 9c57590..0e0017c 100644 --- a/template/html/edit_feed.html +++ b/template/html/edit_feed.html @@ -58,6 +58,9 @@ --> + + + diff --git a/template/views.go b/template/views.go index 4c5c9ba..ef50ef3 100644 --- a/template/views.go +++ b/template/views.go @@ -87,6 +87,9 @@ var templateViewsMap = map[string]string{ + + + @@ -265,6 +268,7 @@ var templateViewsMap = map[string]string{ + {{ if .form.Crawler }} @@ -453,6 +457,9 @@ var templateViewsMap = map[string]string{ --> + + + @@ -1359,15 +1366,15 @@ var templateViewsMap = map[string]string{ var templateViewsMapChecksums = map[string]string{ "about": "ad2fb778fc73c39b733b3f81b13e5c7d689b041fadd24ee2d4577f545aa788ad", - "add_subscription": "3fbcffefc94fb0fccfcf870d602f5ba78ce3ab7ebaeacd04198a6e529143cb29", + "add_subscription": "dc6593913e6ff14a6f957349c2ce43f0e3e095e1985f9d739929a88f5f6550bd", "bookmark_entries": "49423f84c05d77368e20c8e14c53ad237308cdaf4143413487d1b0e11c18d148", "categories": "ca1280cd157bb527d4fc907da67b05a8347378f6dce965b9389d4bcdf3600a11", "category_entries": "d219d4bd5376c526c00a3da49b511fb73e812be5d1e12acadeceee8dfa4bbfe2", - "choose_subscription": "7266b269ddbe145e757a24a57f3fbc7611e34a20383fbd887988204cebce2681", + "choose_subscription": "57280e5b6ba3118c14ef1238e8003f5e92401a671dd6e6e198c864ebbd0b8030", "create_category": "2b82af5d2dcd67898dc5daa57a6461e6ff8121a6089b2a2a1be909f35e4a2275", "create_user": "1ef0a1f9bf119d44929c81f13073a257d69650cf5064960cf06a63fe51923e86", "edit_category": "cee720faadcec58289b707ad30af623d2ee66c1ce23a732965463250d7ff41c5", - "edit_feed": "1a8e342e4fac80e8b9c73537c7fe8aaf7f9e3e7af22f411927010897dd37e9c3", + "edit_feed": "6ff765208828e7533f865009de26edd87d8109b547bdff08132d862aea0e223b", "edit_user": "7373e09f805e6c017167001519b9feb04226be6c81c2875cbacd5ce94f2c24bf", "entry": "82a0a4e715da94b12370b380072f1175c9f0e07b37e7f54a9adca4ed1fe015c0", "feed_entries": "bebc42317ca9e908fcdb98cc1c4a2dc3f4bb7ef6d4c288d3d3fba8f8339403b6", diff --git a/tests/feed_test.go b/tests/feed_test.go index c930826..eba68ba 100644 --- a/tests/feed_test.go +++ b/tests/feed_test.go @@ -195,6 +195,31 @@ func TestUpdateFeedRewriteRules(t *testing.T) { } } +func TestUpdateFeedUserAgent(t *testing.T) { + client := createClient(t) + feed, _ := createFeed(t, client) + + userAgent := "test" + updatedFeed, err := client.UpdateFeed(feed.ID, &miniflux.FeedModification{UserAgent: &userAgent}) + if err != nil { + t.Fatal(err) + } + + if updatedFeed.UserAgent != userAgent { + t.Fatalf(`Wrong UserAgent value, got "%v" instead of "%v"`, updatedFeed.UserAgent, userAgent) + } + + userAgent = "" + updatedFeed, err = client.UpdateFeed(feed.ID, &miniflux.FeedModification{UserAgent: &userAgent}) + if err != nil { + t.Fatal(err) + } + + if updatedFeed.UserAgent != userAgent { + t.Fatalf(`Wrong UserAgent value, got "%v" instead of "%v"`, updatedFeed.UserAgent, userAgent) + } +} + func TestUpdateFeedUsername(t *testing.T) { client := createClient(t) feed, _ := createFeed(t, client) diff --git a/ui/entry_scraper.go b/ui/entry_scraper.go index 0082307..b4a290f 100644 --- a/ui/entry_scraper.go +++ b/ui/entry_scraper.go @@ -38,7 +38,7 @@ func (c *Controller) FetchContent(w http.ResponseWriter, r *http.Request) { return } - content, err := scraper.Fetch(entry.URL, entry.Feed.ScraperRules) + content, err := scraper.Fetch(entry.URL, entry.Feed.ScraperRules, entry.Feed.UserAgent) if err != nil { json.ServerError(w, err) return diff --git a/ui/feed_edit.go b/ui/feed_edit.go index c78ca5e..9063d51 100644 --- a/ui/feed_edit.go +++ b/ui/feed_edit.go @@ -7,6 +7,7 @@ package ui // import "miniflux.app/ui" import ( "net/http" + "miniflux.app/http/client" "miniflux.app/http/request" "miniflux.app/http/response/html" "miniflux.app/ui/form" @@ -52,6 +53,7 @@ func (c *Controller) EditFeed(w http.ResponseWriter, r *http.Request) { ScraperRules: feed.ScraperRules, RewriteRules: feed.RewriteRules, Crawler: feed.Crawler, + UserAgent: feed.UserAgent, CategoryID: feed.Category.ID, Username: feed.Username, Password: feed.Password, @@ -66,6 +68,7 @@ func (c *Controller) EditFeed(w http.ResponseWriter, r *http.Request) { view.Set("user", user) view.Set("countUnread", c.store.CountUnreadEntries(user.ID)) view.Set("countErrorFeeds", c.store.CountErrorFeeds(user.ID)) + view.Set("defaultUserAgent", client.DefaultUserAgent) html.OK(w, r, view.Render("edit_feed")) } diff --git a/ui/feed_update.go b/ui/feed_update.go index fc3a9a2..402daa2 100644 --- a/ui/feed_update.go +++ b/ui/feed_update.go @@ -7,6 +7,7 @@ package ui // import "miniflux.app/ui" import ( "net/http" + "miniflux.app/http/client" "miniflux.app/http/request" "miniflux.app/http/response" "miniflux.app/http/response/html" @@ -59,6 +60,7 @@ func (c *Controller) UpdateFeed(w http.ResponseWriter, r *http.Request) { view.Set("user", user) view.Set("countUnread", c.store.CountUnreadEntries(user.ID)) view.Set("countErrorFeeds", c.store.CountErrorFeeds(user.ID)) + view.Set("defaultUserAgent", client.DefaultUserAgent) if err := feedForm.ValidateModification(); err != nil { view.Set("errorMessage", err.Error()) diff --git a/ui/form/feed.go b/ui/form/feed.go index a8a4343..3645664 100644 --- a/ui/form/feed.go +++ b/ui/form/feed.go @@ -20,6 +20,7 @@ type FeedForm struct { ScraperRules string RewriteRules string Crawler bool + UserAgent string CategoryID int64 Username string Password string @@ -42,6 +43,7 @@ func (f FeedForm) Merge(feed *model.Feed) *model.Feed { feed.ScraperRules = f.ScraperRules feed.RewriteRules = f.RewriteRules feed.Crawler = f.Crawler + feed.UserAgent = f.UserAgent feed.ParsingErrorCount = 0 feed.ParsingErrorMsg = "" feed.Username = f.Username @@ -61,6 +63,7 @@ func NewFeedForm(r *http.Request) *FeedForm { SiteURL: r.FormValue("site_url"), Title: r.FormValue("title"), ScraperRules: r.FormValue("scraper_rules"), + UserAgent: r.FormValue("user_agent"), RewriteRules: r.FormValue("rewrite_rules"), Crawler: r.FormValue("crawler") == "1", CategoryID: int64(categoryID), diff --git a/ui/form/subscription.go b/ui/form/subscription.go index a1a73b9..9bb14fd 100644 --- a/ui/form/subscription.go +++ b/ui/form/subscription.go @@ -16,6 +16,7 @@ type SubscriptionForm struct { URL string CategoryID int64 Crawler bool + UserAgent string Username string Password string } @@ -40,6 +41,7 @@ func NewSubscriptionForm(r *http.Request) *SubscriptionForm { URL: r.FormValue("url"), Crawler: r.FormValue("crawler") == "1", CategoryID: int64(categoryID), + UserAgent: r.FormValue("user_agent"), Username: r.FormValue("feed_username"), Password: r.FormValue("feed_password"), } diff --git a/ui/subscription_add.go b/ui/subscription_add.go index 1639d0d..387f340 100644 --- a/ui/subscription_add.go +++ b/ui/subscription_add.go @@ -7,6 +7,7 @@ package ui // import "miniflux.app/ui" import ( "net/http" + "miniflux.app/http/client" "miniflux.app/http/response/html" "miniflux.app/http/request" "miniflux.app/ui/session" @@ -35,6 +36,7 @@ func (c *Controller) AddSubscription(w http.ResponseWriter, r *http.Request) { view.Set("user", user) view.Set("countUnread", c.store.CountUnreadEntries(user.ID)) view.Set("countErrorFeeds", c.store.CountErrorFeeds(user.ID)) + view.Set("defaultUserAgent", client.DefaultUserAgent) html.OK(w, r, view.Render("add_subscription")) } diff --git a/ui/subscription_bookmarklet.go b/ui/subscription_bookmarklet.go index dd0d11f..3aa1392 100644 --- a/ui/subscription_bookmarklet.go +++ b/ui/subscription_bookmarklet.go @@ -7,6 +7,7 @@ package ui // import "miniflux.app/ui" import ( "net/http" + "miniflux.app/http/client" "miniflux.app/http/request" "miniflux.app/http/response/html" "miniflux.app/ui/form" @@ -39,6 +40,7 @@ func (c *Controller) Bookmarklet(w http.ResponseWriter, r *http.Request) { view.Set("user", user) view.Set("countUnread", c.store.CountUnreadEntries(user.ID)) view.Set("countErrorFeeds", c.store.CountErrorFeeds(user.ID)) + view.Set("defaultUserAgent", client.DefaultUserAgent) html.OK(w, r, view.Render("add_subscription")) } diff --git a/ui/subscription_choose.go b/ui/subscription_choose.go index f1cdfcb..10e46ad 100644 --- a/ui/subscription_choose.go +++ b/ui/subscription_choose.go @@ -7,6 +7,7 @@ package ui // import "miniflux.app/ui" import ( "net/http" + "miniflux.app/http/client" "miniflux.app/http/response" "miniflux.app/http/response/html" "miniflux.app/http/request" @@ -38,6 +39,7 @@ func (c *Controller) ChooseSubscription(w http.ResponseWriter, r *http.Request) view.Set("user", user) view.Set("countUnread", c.store.CountUnreadEntries(user.ID)) view.Set("countErrorFeeds", c.store.CountErrorFeeds(user.ID)) + view.Set("defaultUserAgent", client.DefaultUserAgent) subscriptionForm := form.NewSubscriptionForm(r) if err := subscriptionForm.Validate(); err != nil { @@ -52,6 +54,7 @@ func (c *Controller) ChooseSubscription(w http.ResponseWriter, r *http.Request) subscriptionForm.CategoryID, subscriptionForm.URL, subscriptionForm.Crawler, + subscriptionForm.UserAgent, subscriptionForm.Username, subscriptionForm.Password, ) diff --git a/ui/subscription_submit.go b/ui/subscription_submit.go index c6e8a95..c2c1298 100644 --- a/ui/subscription_submit.go +++ b/ui/subscription_submit.go @@ -7,6 +7,7 @@ package ui // import "miniflux.app/ui" import ( "net/http" + "miniflux.app/http/client" "miniflux.app/http/response" "miniflux.app/http/response/html" "miniflux.app/http/request" @@ -40,6 +41,7 @@ func (c *Controller) SubmitSubscription(w http.ResponseWriter, r *http.Request) v.Set("user", user) v.Set("countUnread", c.store.CountUnreadEntries(user.ID)) v.Set("countErrorFeeds", c.store.CountErrorFeeds(user.ID)) + v.Set("defaultUserAgent", client.DefaultUserAgent) subscriptionForm := form.NewSubscriptionForm(r) if err := subscriptionForm.Validate(); err != nil { @@ -51,6 +53,7 @@ func (c *Controller) SubmitSubscription(w http.ResponseWriter, r *http.Request) subscriptions, err := subscription.FindSubscriptions( subscriptionForm.URL, + subscriptionForm.UserAgent, subscriptionForm.Username, subscriptionForm.Password, ) @@ -76,6 +79,7 @@ func (c *Controller) SubmitSubscription(w http.ResponseWriter, r *http.Request) subscriptionForm.CategoryID, subscriptions[0].URL, subscriptionForm.Crawler, + subscriptionForm.UserAgent, subscriptionForm.Username, subscriptionForm.Password, ) -- cgit v1.2.3