Add the possibility to override default user agent for each feed

1 files changed, 7 insertions, 1 deletions

diff --git a/reader/processor/processor.go b/reader/processor/processor.go
index 002f7e8..f57e6cd 100644
--- a/reader/processor/processor.go
+++ b/reader/processor/processor.go
@@ -21,6 +21,7 @@ type FeedProcessor struct {
 	scraperRules string
 	rewriteRules string
 	crawler      bool
+	userAgent    string
 }
 
 // WithCrawler enables the crawler.
@@ -33,6 +34,11 @@ func (f *FeedProcessor) WithScraperRules(rules string) {
 	f.scraperRules = rules
 }
 
+// WithUserAgent sets the User-Agent header for fetching article content.
+func (f *FeedProcessor) WithUserAgent(userAgent string) {
+	f.userAgent = userAgent
+}
+
 // WithRewriteRules adds rewrite rules to the processing.
 func (f *FeedProcessor) WithRewriteRules(rules string) {
 	f.rewriteRules = rules
@@ -45,7 +51,7 @@ func (f *FeedProcessor) Process() {
 			if f.store.EntryURLExists(f.userID, entry.URL) {
 				logger.Debug(`[FeedProcessor] Do not crawl existing entry URL: "%s"`, entry.URL)
 			} else {
-				content, err := scraper.Fetch(entry.URL, f.scraperRules)
+				content, err := scraper.Fetch(entry.URL, f.scraperRules, f.userAgent)
 				if err != nil {
 					logger.Error("[FeedProcessor] %v", err)
 				} else {