// Copyright 2017 Frédéric Guillot. All rights reserved. // Use of this source code is governed by the Apache 2.0 // license that can be found in the LICENSE file. package feed // import "miniflux.app/reader/feed" import ( "fmt" "time" "miniflux.app/errors" "miniflux.app/http/client" "miniflux.app/locale" "miniflux.app/logger" "miniflux.app/model" "miniflux.app/reader/icon" "miniflux.app/reader/parser" "miniflux.app/reader/processor" "miniflux.app/storage" "miniflux.app/timer" ) var ( errRequestFailed = "Unable to execute request: %v" errServerFailure = "Unable to fetch feed (Status Code = %d)" errDuplicate = "This feed already exists (%s)" errNotFound = "Feed %d not found" errEncoding = "Unable to normalize encoding: %q" errCategoryNotFound = "Category not found for this user" errEmptyFeed = "This feed is empty" errResourceNotFound = "Resource not found (404), this feed doesn't exists anymore, check the feed URL" ) // Handler contains all the logic to create and refresh feeds. type Handler struct { store *storage.Storage } // CreateFeed fetch, parse and store a new feed. func (h *Handler) CreateFeed(userID, categoryID int64, url string, crawler bool, userAgent, username, password string) (*model.Feed, error) { defer timer.ExecutionTime(time.Now(), fmt.Sprintf("[Handler:CreateFeed] feedUrl=%s", url)) if !h.store.CategoryExists(userID, categoryID) { return nil, errors.NewLocalizedError(errCategoryNotFound) } clt := client.New(url) clt.WithCredentials(username, password) clt.WithUserAgent(userAgent) response, err := clt.Get() if err != nil { if _, ok := err.(*errors.LocalizedError); ok { return nil, err } return nil, errors.NewLocalizedError(errRequestFailed, err) } if response.HasServerFailure() { return nil, errors.NewLocalizedError(errServerFailure, response.StatusCode) } // Content-Length = -1 when no Content-Length header is sent if response.ContentLength == 0 { return nil, errors.NewLocalizedError(errEmptyFeed) } if h.store.FeedURLExists(userID, response.EffectiveURL) { return nil, errors.NewLocalizedError(errDuplicate, response.EffectiveURL) } if err := response.EnsureUnicodeBody(); err != nil { return nil, errors.NewLocalizedError(errEncoding, err) } subscription, feedErr := parser.ParseFeed(response.String()) if feedErr != nil { return nil, feedErr } feedProcessor := processor.NewFeedProcessor(userID, h.store, subscription) feedProcessor.WithCrawler(crawler) feedProcessor.Process() subscription.Category = &model.Category{ID: categoryID} subscription.EtagHeader = response.ETag subscription.LastModifiedHeader = response.LastModified subscription.FeedURL = response.EffectiveURL subscription.UserID = userID subscription.Crawler = crawler subscription.UserAgent = userAgent subscription.Username = username subscription.Password = password if subscription.SiteURL == "" { subscription.SiteURL = subscription.FeedURL } err = h.store.CreateFeed(subscription) if err != nil { return nil, err } logger.Debug("[Handler:CreateFeed] Feed saved with ID: %d", subscription.ID) icon, err := icon.FindIcon(subscription.SiteURL) if err != nil { logger.Error("[Handler:CreateFeed] %v", err) } else if icon == nil { logger.Info("No icon found for feedID=%d", subscription.ID) } else { h.store.CreateFeedIcon(subscription, icon) } return subscription, nil } // RefreshFeed fetch and update a feed if necessary. func (h *Handler) RefreshFeed(userID, feedID int64) error { defer timer.ExecutionTime(time.Now(), fmt.Sprintf("[Handler:RefreshFeed] feedID=%d", feedID)) userLanguage, err := h.store.UserLanguage(userID) if err != nil { logger.Error("[Handler:RefreshFeed] %v", err) userLanguage = "en_US" } printer := locale.NewPrinter(userLanguage) originalFeed, err := h.store.FeedByID(userID, feedID) if err != nil { return err } if originalFeed == nil { return errors.NewLocalizedError(errNotFound, feedID) } clt := client.New(originalFeed.FeedURL) clt.WithCredentials(originalFeed.Username, originalFeed.Password) clt.WithCacheHeaders(originalFeed.EtagHeader, originalFeed.LastModifiedHeader) clt.WithUserAgent(originalFeed.UserAgent) response, err := clt.Get() if err != nil { var customErr errors.LocalizedError if lerr, ok := err.(*errors.LocalizedError); ok { customErr = *lerr } else { customErr = *errors.NewLocalizedError(errRequestFailed, err) } originalFeed.ParsingErrorCount++ originalFeed.ParsingErrorMsg = customErr.Localize(printer) h.store.UpdateFeed(originalFeed) return customErr } originalFeed.CheckedAt = time.Now() if response.IsNotFound() { err := errors.NewLocalizedError(errResourceNotFound) originalFeed.ParsingErrorCount++ originalFeed.ParsingErrorMsg = err.Localize(printer) h.store.UpdateFeed(originalFeed) return err } if response.HasServerFailure() { err := errors.NewLocalizedError(errServerFailure, response.StatusCode) originalFeed.ParsingErrorCount++ originalFeed.ParsingErrorMsg = err.Localize(printer) h.store.UpdateFeed(originalFeed) return err } if response.IsModified(originalFeed.EtagHeader, originalFeed.LastModifiedHeader) { logger.Debug("[Handler:RefreshFeed] Feed #%d has been modified", feedID) // Content-Length = -1 when no Content-Length header is sent if response.ContentLength == 0 { err := errors.NewLocalizedError(errEmptyFeed) originalFeed.ParsingErrorCount++ originalFeed.ParsingErrorMsg = err.Localize(printer) h.store.UpdateFeed(originalFeed) return err } if err := response.EnsureUnicodeBody(); err != nil { return errors.NewLocalizedError(errEncoding, err) } subscription, parseErr := parser.ParseFeed(response.String()) if parseErr != nil { originalFeed.ParsingErrorCount++ originalFeed.ParsingErrorMsg = parseErr.Localize(printer) h.store.UpdateFeed(originalFeed) return err } feedProcessor := processor.NewFeedProcessor(userID, h.store, subscription) feedProcessor.WithScraperRules(originalFeed.ScraperRules) feedProcessor.WithUserAgent(originalFeed.UserAgent) feedProcessor.WithRewriteRules(originalFeed.RewriteRules) feedProcessor.WithCrawler(originalFeed.Crawler) feedProcessor.Process() originalFeed.EtagHeader = response.ETag originalFeed.LastModifiedHeader = response.LastModified // Note: We don't update existing entries when the crawler is enabled (we crawl only inexisting entries). if err := h.store.UpdateEntries(originalFeed.UserID, originalFeed.ID, subscription.Entries, !originalFeed.Crawler); err != nil { return err } if !h.store.HasIcon(originalFeed.ID) { logger.Debug("[Handler:RefreshFeed] Looking for feed icon") icon, err := icon.FindIcon(originalFeed.SiteURL) if err != nil { logger.Debug("[Handler:RefreshFeed] %v", err) } else { h.store.CreateFeedIcon(originalFeed, icon) } } } else { logger.Debug("[Handler:RefreshFeed] Feed #%d not modified", feedID) } originalFeed.ParsingErrorCount = 0 originalFeed.ParsingErrorMsg = "" if originalFeed.SiteURL == "" { originalFeed.SiteURL = originalFeed.FeedURL } return h.store.UpdateFeed(originalFeed) } // NewFeedHandler returns a feed handler. func NewFeedHandler(store *storage.Storage) *Handler { return &Handler{store} }