Add scraper rules

5 files changed, 27 insertions, 21 deletions

diff --git a/server/template/html/edit_feed.html b/server/template/html/edit_feed.html
index fac2a9b..0495092 100644
--- a/server/template/html/edit_feed.html
+++ b/server/template/html/edit_feed.html
@@ -45,6 +45,9 @@
         <label for="form-feed-url">{{ t "Feed URL" }}</label>
         <input type="url" name="feed_url" id="form-feed-url" placeholder="https://domain.tld/" value="{{ .form.FeedURL }}" required>
 
+        <label for="form-scraper-rules">{{ t "Scraper Rules" }}</label>
+        <input type="text" name="scraper_rules" id="form-scraper-rules" value="{{ .form.ScraperRules }}">
+
         <label for="form-category">{{ t "Category" }}</label>
         <select id="form-category" name="category_id">
         {{ range .categories }}
diff --git a/server/template/views.go b/server/template/views.go
index 80d956d..420a344 100644
--- a/server/template/views.go
+++ b/server/template/views.go
@@ -1,5 +1,5 @@
 // Code generated by go generate; DO NOT EDIT.
-// 2017-12-10 18:56:24.375327888 -0800 PST m=+0.017306975
+// 2017-12-10 20:08:14.428877093 -0800 PST m=+0.021859548
 
 package template
 
@@ -395,6 +395,9 @@ var templateViewsMap = map[string]string{
         <label for="form-feed-url">{{ t "Feed URL" }}</label>
         <input type="url" name="feed_url" id="form-feed-url" placeholder="https://domain.tld/" value="{{ .form.FeedURL }}" required>
 
+        <label for="form-scraper-rules">{{ t "Scraper Rules" }}</label>
+        <input type="text" name="scraper_rules" id="form-scraper-rules" value="{{ .form.ScraperRules }}">
+
         <label for="form-category">{{ t "Category" }}</label>
         <select id="form-category" name="category_id">
         {{ range .categories }}
@@ -1181,7 +1184,7 @@ var templateViewsMapChecksums = map[string]string{
 	"create_category":     "2b82af5d2dcd67898dc5daa57a6461e6ff8121a6089b2a2a1be909f35e4a2275",
 	"create_user":         "45e226df757126d5fe7c464e295e9a34f07952cfdb71e31e49839850d35af139",
 	"edit_category":       "cee720faadcec58289b707ad30af623d2ee66c1ce23a732965463250d7ff41c5",
-	"edit_feed":           "c5bc4c22bf7e8348d880395250545595d21fb8c8e723fc5d7cca68e25d250884",
+	"edit_feed":           "b3c7dd5e93d58e051abcd59da31217d8e9b50587014b895d1b7c9172247b35f8",
 	"edit_user":           "82d9749d76ddbd2352816d813c4b1f6d92f2222de678b4afe5821090246735c7",
 	"entry":               "ebcf9bb35812dd02759718f7f7411267e6a6c8efd59a9aa0a0e735bcb88efeff",
 	"feed_entries":        "547c19eb36b20e350ce70ed045173b064cdcd6b114afb241c9f2dda9d88fcc27",
diff --git a/server/ui/controller/entry.go b/server/ui/controller/entry.go
index eb47201..5c1d3f5 100644
--- a/server/ui/controller/entry.go
+++ b/server/ui/controller/entry.go
@@ -40,18 +40,14 @@ func (c *Controller) FetchContent(ctx *core.Context, request *core.Request, resp
 		return
 	}
 
-	content, err := scraper.Fetch(entry.URL)
+	content, err := scraper.Fetch(entry.URL, entry.Feed.ScraperRules)
 	if err != nil {
 		response.JSON().ServerError(err)
 		return
 	}
 
-	if len(content) > len(entry.Content) {
-		entry.Content = content
-		c.store.UpdateEntryContent(entry)
-	} else {
-		content = entry.Content
-	}
+	entry.Content = content
+	c.store.UpdateEntryContent(entry)
 
 	response.JSON().Created(map[string]string{"content": content})
 }
diff --git a/server/ui/controller/feed.go b/server/ui/controller/feed.go
index eeb66c4..d5bc858 100644
--- a/server/ui/controller/feed.go
+++ b/server/ui/controller/feed.go
@@ -217,10 +217,11 @@ func (c *Controller) getFeedFormTemplateArgs(ctx *core.Context, user *model.User
 
 	if feedForm == nil {
 		args["form"] = form.FeedForm{
-			SiteURL:    feed.SiteURL,
-			FeedURL:    feed.FeedURL,
-			Title:      feed.Title,
-			CategoryID: feed.Category.ID,
+			SiteURL:      feed.SiteURL,
+			FeedURL:      feed.FeedURL,
+			Title:        feed.Title,
+			ScraperRules: feed.ScraperRules,
+			CategoryID:   feed.Category.ID,
 		}
 	} else {
 		args["form"] = feedForm
diff --git a/server/ui/form/feed.go b/server/ui/form/feed.go
index 8a8cf20..7d07375 100644
--- a/server/ui/form/feed.go
+++ b/server/ui/form/feed.go
@@ -14,10 +14,11 @@ import (
 
 // FeedForm represents a feed form in the UI
 type FeedForm struct {
-	FeedURL    string
-	SiteURL    string
-	Title      string
-	CategoryID int64
+	FeedURL      string
+	SiteURL      string
+	Title        string
+	ScraperRules string
+	CategoryID   int64
 }
 
 // ValidateModification validates FeedForm fields
@@ -34,6 +35,7 @@ func (f FeedForm) Merge(feed *model.Feed) *model.Feed {
 	feed.Title = f.Title
 	feed.SiteURL = f.SiteURL
 	feed.FeedURL = f.FeedURL
+	feed.ScraperRules = f.ScraperRules
 	feed.ParsingErrorCount = 0
 	feed.ParsingErrorMsg = ""
 	return feed
@@ -47,9 +49,10 @@ func NewFeedForm(r *http.Request) *FeedForm {
 	}
 
 	return &FeedForm{
-		FeedURL:    r.FormValue("feed_url"),
-		SiteURL:    r.FormValue("site_url"),
-		Title:      r.FormValue("title"),
-		CategoryID: int64(categoryID),
+		FeedURL:      r.FormValue("feed_url"),
+		SiteURL:      r.FormValue("site_url"),
+		Title:        r.FormValue("title"),
+		ScraperRules: r.FormValue("scraper_rules"),
+		CategoryID:   int64(categoryID),
 	}
 }