From 69aa650203087ad6365fcd6769a49bdf327e9efb Mon Sep 17 00:00:00 2001 From: Frédéric Guillot Date: Fri, 29 Nov 2019 11:17:14 -0800 Subject: Add the possibility to add rules during feed creation --- api/feed.go | 2 ++ api/payload.go | 14 ++++++++------ model/feed.go | 4 +++- model/feed_test.go | 10 +++++++++- reader/feed/handler.go | 8 ++++---- reader/rewrite/rewriter.go | 2 +- storage/feed.go | 8 ++++++-- template/html/add_subscription.html | 6 ++++++ template/html/choose_subscription.html | 2 ++ template/views.go | 12 ++++++++++-- ui/form/subscription.go | 28 ++++++++++++++++------------ ui/subscription_choose.go | 4 +++- ui/subscription_submit.go | 6 ++++-- 13 files changed, 74 insertions(+), 32 deletions(-) diff --git a/api/feed.go b/api/feed.go index fdc1560..bcf5884 100644 --- a/api/feed.go +++ b/api/feed.go @@ -49,6 +49,8 @@ func (h *handler) createFeed(w http.ResponseWriter, r *http.Request) { feedInfo.UserAgent, feedInfo.Username, feedInfo.Password, + feedInfo.ScraperRules, + feedInfo.RewriteRules, ) if err != nil { json.ServerError(w, r, err) diff --git a/api/payload.go b/api/payload.go index 011a516..becbb47 100644 --- a/api/payload.go +++ b/api/payload.go @@ -24,12 +24,14 @@ type entriesResponse struct { } type feedCreation struct { - FeedURL string `json:"feed_url"` - CategoryID int64 `json:"category_id"` - UserAgent string `json:"user_agent"` - Username string `json:"username"` - Password string `json:"password"` - Crawler bool `json:"crawler"` + FeedURL string `json:"feed_url"` + CategoryID int64 `json:"category_id"` + UserAgent string `json:"user_agent"` + Username string `json:"username"` + Password string `json:"password"` + Crawler bool `json:"crawler"` + ScraperRules string `json:"scraper_rules"` + RewriteRules string `json:"rewrite_rules"` } type subscriptionDiscovery struct { diff --git a/model/feed.go b/model/feed.go index 9d0ab27..5782c4f 100644 --- a/model/feed.go +++ b/model/feed.go @@ -61,11 +61,13 @@ func (f *Feed) WithCategoryID(categoryID int64) { } // WithBrowsingParameters defines browsing parameters. -func (f *Feed) WithBrowsingParameters(crawler bool, userAgent, username, password string) { +func (f *Feed) WithBrowsingParameters(crawler bool, userAgent, username, password, scraperRules, rewriteRules string) { f.Crawler = crawler f.UserAgent = userAgent f.Username = username f.Password = password + f.ScraperRules = scraperRules + f.RewriteRules = rewriteRules } // WithError adds a new error message and increment the error counter. diff --git a/model/feed_test.go b/model/feed_test.go index 8cbb00b..27ef897 100644 --- a/model/feed_test.go +++ b/model/feed_test.go @@ -44,7 +44,7 @@ func TestFeedCategorySetter(t *testing.T) { func TestFeedBrowsingParams(t *testing.T) { feed := &Feed{} - feed.WithBrowsingParameters(true, "Custom User Agent", "Username", "Secret") + feed.WithBrowsingParameters(true, "Custom User Agent", "Username", "Secret", "Some Rule", "Another Rule") if !feed.Crawler { t.Error(`The crawler must be activated`) @@ -61,6 +61,14 @@ func TestFeedBrowsingParams(t *testing.T) { if feed.Password != "Secret" { t.Error(`The password must be set`) } + + if feed.ScraperRules != "Some Rule" { + t.Errorf(`The scraper rules must be set`) + } + + if feed.RewriteRules != "Another Rule" { + t.Errorf(`The rewrite rules must be set`) + } } func TestFeedErrorCounter(t *testing.T) { diff --git a/reader/feed/handler.go b/reader/feed/handler.go index 3da0d46..b272ea1 100644 --- a/reader/feed/handler.go +++ b/reader/feed/handler.go @@ -29,11 +29,11 @@ var ( // Handler contains all the logic to create and refresh feeds. type Handler struct { - store *storage.Storage + store *storage.Storage } // CreateFeed fetch, parse and store a new feed. -func (h *Handler) CreateFeed(userID, categoryID int64, url string, crawler bool, userAgent, username, password string) (*model.Feed, error) { +func (h *Handler) CreateFeed(userID, categoryID int64, url string, crawler bool, userAgent, username, password, scraperRules, rewriteRules string) (*model.Feed, error) { defer timer.ExecutionTime(time.Now(), fmt.Sprintf("[Handler:CreateFeed] feedUrl=%s", url)) if !h.store.CategoryExists(userID, categoryID) { @@ -59,7 +59,7 @@ func (h *Handler) CreateFeed(userID, categoryID int64, url string, crawler bool, subscription.UserID = userID subscription.WithCategoryID(categoryID) - subscription.WithBrowsingParameters(crawler, userAgent, username, password) + subscription.WithBrowsingParameters(crawler, userAgent, username, password, scraperRules, rewriteRules) subscription.WithClientResponse(response) subscription.CheckedNow() @@ -160,4 +160,4 @@ func checkFeedIcon(store *storage.Storage, feedID int64, websiteURL string) { } } } -} \ No newline at end of file +} diff --git a/reader/rewrite/rewriter.go b/reader/rewrite/rewriter.go index c6b796a..f894ce8 100644 --- a/reader/rewrite/rewriter.go +++ b/reader/rewrite/rewriter.go @@ -37,7 +37,7 @@ func Rewriter(entryURL, entryContent, customRewriteRules string) string { entryContent = addPDFLink(entryURL, entryContent) case "nl2br": entryContent = replaceLineFeeds(entryContent) - case "convert_text_link": + case "convert_text_link", "convert_text_links": entryContent = replaceTextLinks(entryContent) } } diff --git a/storage/feed.go b/storage/feed.go index 6db3488..c5ba464 100644 --- a/storage/feed.go +++ b/storage/feed.go @@ -346,10 +346,12 @@ func (s *Storage) CreateFeed(feed *model.Feed) error { user_agent, username, password, - disabled + disabled, + scraper_rules, + rewrite_rules ) VALUES - ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) + ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14) RETURNING id ` @@ -367,6 +369,8 @@ func (s *Storage) CreateFeed(feed *model.Feed) error { feed.Username, feed.Password, feed.Disabled, + feed.ScraperRules, + feed.RewriteRules, ).Scan(&feed.ID) if err != nil { return fmt.Errorf(`store: unable to create feed %q: %v`, feed.FeedURL, err) diff --git a/template/html/add_subscription.html b/template/html/add_subscription.html index 4a6fe7d..c3e8076 100644 --- a/template/html/add_subscription.html +++ b/template/html/add_subscription.html @@ -56,6 +56,12 @@ - Using a different input name doesn't change anything --> + + + + + + diff --git a/template/html/choose_subscription.html b/template/html/choose_subscription.html index bac7083..ee4a1dc 100644 --- a/template/html/choose_subscription.html +++ b/template/html/choose_subscription.html @@ -22,6 +22,8 @@ + + {{ if .form.Crawler }} {{ end }} diff --git a/template/views.go b/template/views.go index 193f485..477b7e5 100644 --- a/template/views.go +++ b/template/views.go @@ -87,6 +87,12 @@ var templateViewsMap = map[string]string{ - Using a different input name doesn't change anything --> + + + + + + @@ -311,6 +317,8 @@ var templateViewsMap = map[string]string{ + + {{ if .form.Crawler }} {{ end }} @@ -1343,12 +1351,12 @@ var templateViewsMap = map[string]string{ var templateViewsMapChecksums = map[string]string{ "about": "4035658497363d7af7f79be83190404eb21ec633fe8ec636bdfc219d9fc78cfc", - "add_subscription": "a0f1d2bc02b6adc83dbeae593f74d9b936102cd6dd73302cdbec2137cafdcdd9", + "add_subscription": "5bae8c60989593257f515f9c73b35c854a94bee4b2fb08ad38aa93b17be9c1b5", "bookmark_entries": "65588da78665699dd3f287f68325e9777d511f1a57fee4131a5bb6d00bb68df8", "categories": "2c5dd0ed6355bd5acc393bbf6117d20458b5581aab82036008324f6bbbe2af75", "category_entries": "dee7b9cd60c6c46f01dd4289940679df31c1fce28ce4aa7249fa459023e1eeb4", "category_feeds": "527c2ffbc4fcec775071424ba1022ae003525dba53a28cc41f48fb7b30aa984b", - "choose_subscription": "33c04843d7c1b608d034e605e52681822fc6d79bc6b900c04915dd9ebae584e2", + "choose_subscription": "5f21556e6cecfd64b1cff30e22ef313d2f09698cebb03f711cbac8ec7f2c1d04", "create_category": "6b22b5ce51abf4e225e23a79f81be09a7fb90acb265e93a8faf9446dff74018d", "create_user": "9b73a55233615e461d1f07d99ad1d4d3b54532588ab960097ba3e090c85aaf3a", "edit_category": "b1c0b38f1b714c5d884edcd61e5b5295a5f1c8b71c469b35391e4dcc97cc6d36", diff --git a/ui/form/subscription.go b/ui/form/subscription.go index 16a2bf9..f6348e2 100644 --- a/ui/form/subscription.go +++ b/ui/form/subscription.go @@ -13,12 +13,14 @@ import ( // SubscriptionForm represents the subscription form. type SubscriptionForm struct { - URL string - CategoryID int64 - Crawler bool - UserAgent string - Username string - Password string + URL string + CategoryID int64 + Crawler bool + UserAgent string + Username string + Password string + ScraperRules string + RewriteRules string } // Validate makes sure the form values are valid. @@ -38,11 +40,13 @@ func NewSubscriptionForm(r *http.Request) *SubscriptionForm { } return &SubscriptionForm{ - URL: r.FormValue("url"), - Crawler: r.FormValue("crawler") == "1", - CategoryID: int64(categoryID), - UserAgent: r.FormValue("user_agent"), - Username: r.FormValue("feed_username"), - Password: r.FormValue("feed_password"), + URL: r.FormValue("url"), + Crawler: r.FormValue("crawler") == "1", + CategoryID: int64(categoryID), + UserAgent: r.FormValue("user_agent"), + Username: r.FormValue("feed_username"), + Password: r.FormValue("feed_password"), + ScraperRules: r.FormValue("scraper_rules"), + RewriteRules: r.FormValue("rewrite_rules"), } } diff --git a/ui/subscription_choose.go b/ui/subscription_choose.go index b554b03..37c7461 100644 --- a/ui/subscription_choose.go +++ b/ui/subscription_choose.go @@ -8,8 +8,8 @@ import ( "net/http" "miniflux.app/http/client" - "miniflux.app/http/response/html" "miniflux.app/http/request" + "miniflux.app/http/response/html" "miniflux.app/http/route" "miniflux.app/ui/form" "miniflux.app/ui/session" @@ -55,6 +55,8 @@ func (h *handler) showChooseSubscriptionPage(w http.ResponseWriter, r *http.Requ subscriptionForm.UserAgent, subscriptionForm.Username, subscriptionForm.Password, + subscriptionForm.ScraperRules, + subscriptionForm.RewriteRules, ) if err != nil { view.Set("form", subscriptionForm) diff --git a/ui/subscription_submit.go b/ui/subscription_submit.go index 8d3c7ed..0a3c1af 100644 --- a/ui/subscription_submit.go +++ b/ui/subscription_submit.go @@ -2,14 +2,14 @@ // Use of this source code is governed by the Apache 2.0 // license that can be found in the LICENSE file. -package ui // import "miniflux.app/ui" +package ui // import "miniflux.app/ui" import ( "net/http" "miniflux.app/http/client" - "miniflux.app/http/response/html" "miniflux.app/http/request" + "miniflux.app/http/response/html" "miniflux.app/http/route" "miniflux.app/logger" "miniflux.app/reader/subscription" @@ -80,6 +80,8 @@ func (h *handler) submitSubscription(w http.ResponseWriter, r *http.Request) { subscriptionForm.UserAgent, subscriptionForm.Username, subscriptionForm.Password, + subscriptionForm.ScraperRules, + subscriptionForm.RewriteRules, ) if err != nil { v.Set("form", subscriptionForm) -- cgit v1.2.3