diff options
Diffstat (limited to 'reader/processor/processor.go')
-rw-r--r-- | reader/processor/processor.go | 8 |
1 files changed, 7 insertions, 1 deletions
diff --git a/reader/processor/processor.go b/reader/processor/processor.go index 002f7e8..f57e6cd 100644 --- a/reader/processor/processor.go +++ b/reader/processor/processor.go @@ -21,6 +21,7 @@ type FeedProcessor struct { scraperRules string rewriteRules string crawler bool + userAgent string } // WithCrawler enables the crawler. @@ -33,6 +34,11 @@ func (f *FeedProcessor) WithScraperRules(rules string) { f.scraperRules = rules } +// WithUserAgent sets the User-Agent header for fetching article content. +func (f *FeedProcessor) WithUserAgent(userAgent string) { + f.userAgent = userAgent +} + // WithRewriteRules adds rewrite rules to the processing. func (f *FeedProcessor) WithRewriteRules(rules string) { f.rewriteRules = rules @@ -45,7 +51,7 @@ func (f *FeedProcessor) Process() { if f.store.EntryURLExists(f.userID, entry.URL) { logger.Debug(`[FeedProcessor] Do not crawl existing entry URL: "%s"`, entry.URL) } else { - content, err := scraper.Fetch(entry.URL, f.scraperRules) + content, err := scraper.Fetch(entry.URL, f.scraperRules, f.userAgent) if err != nil { logger.Error("[FeedProcessor] %v", err) } else { |