Add readability package to fetch original content

2 files changed, 79 insertions, 40 deletions

diff --git a/server/ui/controller/entry.go b/server/ui/controller/entry.go
index 309fa09..eb47201 100644
--- a/server/ui/controller/entry.go
+++ b/server/ui/controller/entry.go
@@ -8,12 +8,91 @@ import (
 	"errors"
 	"log"
 
+	"github.com/miniflux/miniflux2/integration"
 	"github.com/miniflux/miniflux2/model"
+	"github.com/miniflux/miniflux2/reader/scraper"
 	"github.com/miniflux/miniflux2/server/core"
 	"github.com/miniflux/miniflux2/server/ui/payload"
 	"github.com/miniflux/miniflux2/storage"
 )
 
+// FetchContent downloads the original HTML page and returns relevant contents.
+func (c *Controller) FetchContent(ctx *core.Context, request *core.Request, response *core.Response) {
+	entryID, err := request.IntegerParam("entryID")
+	if err != nil {
+		response.HTML().BadRequest(err)
+		return
+	}
+
+	user := ctx.LoggedUser()
+	builder := c.store.GetEntryQueryBuilder(user.ID, user.Timezone)
+	builder.WithEntryID(entryID)
+	builder.WithoutStatus(model.EntryStatusRemoved)
+
+	entry, err := builder.GetEntry()
+	if err != nil {
+		response.JSON().ServerError(err)
+		return
+	}
+
+	if entry == nil {
+		response.JSON().NotFound(errors.New("Entry not found"))
+		return
+	}
+
+	content, err := scraper.Fetch(entry.URL)
+	if err != nil {
+		response.JSON().ServerError(err)
+		return
+	}
+
+	if len(content) > len(entry.Content) {
+		entry.Content = content
+		c.store.UpdateEntryContent(entry)
+	} else {
+		content = entry.Content
+	}
+
+	response.JSON().Created(map[string]string{"content": content})
+}
+
+// SaveEntry send the link to external services.
+func (c *Controller) SaveEntry(ctx *core.Context, request *core.Request, response *core.Response) {
+	entryID, err := request.IntegerParam("entryID")
+	if err != nil {
+		response.HTML().BadRequest(err)
+		return
+	}
+
+	user := ctx.LoggedUser()
+	builder := c.store.GetEntryQueryBuilder(user.ID, user.Timezone)
+	builder.WithEntryID(entryID)
+	builder.WithoutStatus(model.EntryStatusRemoved)
+
+	entry, err := builder.GetEntry()
+	if err != nil {
+		response.JSON().ServerError(err)
+		return
+	}
+
+	if entry == nil {
+		response.JSON().NotFound(errors.New("Entry not found"))
+		return
+	}
+
+	settings, err := c.store.Integration(user.ID)
+	if err != nil {
+		response.JSON().ServerError(err)
+		return
+	}
+
+	go func() {
+		integration.SendEntry(entry, settings)
+	}()
+
+	response.JSON().Created(map[string]string{"message": "saved"})
+}
+
 // ShowFeedEntry shows a single feed entry in "feed" mode.
 func (c *Controller) ShowFeedEntry(ctx *core.Context, request *core.Request, response *core.Response) {
 	user := ctx.LoggedUser()
diff --git a/server/ui/controller/integrations.go b/server/ui/controller/integrations.go
index ceac09a..2035817 100644
--- a/server/ui/controller/integrations.go
+++ b/server/ui/controller/integrations.go
@@ -6,11 +6,8 @@ package controller
 
 import (
 	"crypto/md5"
-	"errors"
 	"fmt"
 
-	"github.com/miniflux/miniflux2/integration"
-	"github.com/miniflux/miniflux2/model"
 	"github.com/miniflux/miniflux2/server/core"
 	"github.com/miniflux/miniflux2/server/ui/form"
 )
@@ -73,40 +70,3 @@ func (c *Controller) UpdateIntegration(ctx *core.Context, request *core.Request,
 
 	response.Redirect(ctx.Route("integrations"))
 }
-
-// SaveEntry send the link to external services.
-func (c *Controller) SaveEntry(ctx *core.Context, request *core.Request, response *core.Response) {
-	entryID, err := request.IntegerParam("entryID")
-	if err != nil {
-		response.HTML().BadRequest(err)
-		return
-	}
-
-	user := ctx.LoggedUser()
-	builder := c.store.GetEntryQueryBuilder(user.ID, user.Timezone)
-	builder.WithEntryID(entryID)
-	builder.WithoutStatus(model.EntryStatusRemoved)
-
-	entry, err := builder.GetEntry()
-	if err != nil {
-		response.JSON().ServerError(err)
-		return
-	}
-
-	if entry == nil {
-		response.JSON().NotFound(errors.New("Entry not found"))
-		return
-	}
-
-	settings, err := c.store.Integration(user.ID)
-	if err != nil {
-		response.JSON().ServerError(err)
-		return
-	}
-
-	go func() {
-		integration.SendEntry(entry, settings)
-	}()
-
-	response.JSON().Created(map[string]string{"message": "saved"})
-}