summaryrefslogtreecommitdiff
path: root/Command/ImportFeed.hs
diff options
context:
space:
mode:
authorGravatar Joey Hess <joey@kitenet.net>2014-07-03 13:46:09 -0400
committerGravatar Joey Hess <joey@kitenet.net>2014-07-03 14:15:00 -0400
commit38e2750137330da8fcc7067a2fa2e5aa74be5125 (patch)
tree22b7d238cbe66ab5ed3444b91f448b7de5097dc5 /Command/ImportFeed.hs
parentff0cf010c169e7d4868994a7eca428496f9e54e6 (diff)
import metadata from feeds
When annex.genmetadata is set, metadata from the feed is added to files that are imported from it. Reused the same feedtitle and itemtitle, feedauthor, itemauthor, etc names that are used in --template. Also added title and author, which are the item title/author if available, falling back to the feed title/author. These are more likely to be common metadata fields. (There is a small bit of dupication here, but once git gets around to packing the object, it will compress it away.) The itempubdate field is not included in the metadata as a string; instead it is used to generate year and month fields, same as is done when adding files with annex.genmetadata set. This commit was sponsored by Amitai Schlair, who cooincidentially is responsible for ikiwiki generating nice feed metadata!
Diffstat (limited to 'Command/ImportFeed.hs')
-rw-r--r--Command/ImportFeed.hs73
1 files changed, 50 insertions, 23 deletions
diff --git a/Command/ImportFeed.hs b/Command/ImportFeed.hs
index 29f2fb148..71cd0dc82 100644
--- a/Command/ImportFeed.hs
+++ b/Command/ImportFeed.hs
@@ -33,6 +33,9 @@ import Annex.Quvi
import qualified Utility.Quvi as Quvi
import Command.AddUrl (addUrlFileQuvi)
#endif
+import Types.MetaData
+import Logs.MetaData
+import Annex.MetaData
def :: [Command]
def = [notBareRepo $ withOptions [templateOption, relaxedOption] $
@@ -165,12 +168,14 @@ performDownload relaxed cache todownload = case location todownload of
Nothing -> return True
Just f -> do
showStart "addurl" f
- ok <- getter f
- if ok
- then do
+ mk <- getter f
+ case mk of
+ Just key -> do
+ whenM (annexGenMetaData <$> Annex.getGitConfig) $
+ addMetaData key $ extractMetaData todownload
showEndOk
return True
- else do
+ Nothing -> do
showEndFail
checkFeedBroken (feedurl todownload)
@@ -198,32 +203,19 @@ performDownload relaxed cache todownload = case location todownload of
( return Nothing
, tryanother
)
-
+
defaultTemplate :: String
defaultTemplate = "${feedtitle}/${itemtitle}${extension}"
{- Generates a filename to use for a feed item by filling out the template.
- The filename may not be unique. -}
feedFile :: Utility.Format.Format -> ToDownload -> String -> FilePath
-feedFile tmpl i extension = Utility.Format.format tmpl $ M.fromList
- [ field "feedtitle" $ getFeedTitle $ feed i
- , fieldMaybe "itemtitle" $ getItemTitle $ item i
- , fieldMaybe "feedauthor" $ getFeedAuthor $ feed i
- , fieldMaybe "itemauthor" $ getItemAuthor $ item i
- , fieldMaybe "itemsummary" $ getItemSummary $ item i
- , fieldMaybe "itemdescription" $ getItemDescription $ item i
- , fieldMaybe "itemrights" $ getItemRights $ item i
- , fieldMaybe "itemid" $ snd <$> getItemId (item i)
- , fieldMaybe "itempubdate" $ pubdate $ item i
- , ("extension", sanitizeFilePath extension)
- ]
+feedFile tmpl i extension = Utility.Format.format tmpl $
+ M.map sanitizeFilePath $ M.fromList $ extractFields i ++
+ [ ("extension", extension)
+ , extractField "itempubdate" [pubdate $ item i]
+ ]
where
- field k v =
- let s = sanitizeFilePath v in
- if null s then (k, "none") else (k, s)
- fieldMaybe k Nothing = (k, "none")
- fieldMaybe k (Just v) = field k v
-
#if MIN_VERSION_feed(0,3,9)
pubdate itm = case getItemPublishDate itm :: Maybe (Maybe UTCTime) of
Just (Just d) -> Just $
@@ -234,6 +226,41 @@ feedFile tmpl i extension = Utility.Format.format tmpl $ M.fromList
pubdate _ = Nothing
#endif
+extractMetaData :: ToDownload -> MetaData
+extractMetaData i = case getItemPublishDate (item i) :: Maybe (Maybe UTCTime) of
+ Just (Just d) -> addDateMetaData d meta
+ _ -> meta
+ where
+ tometa (k, v) = (mkMetaFieldUnchecked k, S.singleton (toMetaValue v))
+ meta = MetaData $ M.fromList $ map tometa $ extractFields i
+
+{- Extract fields from the feed and item, that are both used as metadata,
+ - and to generate the filename. -}
+extractFields :: ToDownload -> [(String, String)]
+extractFields i = map (uncurry extractField)
+ [ ("feedtitle", [feedtitle])
+ , ("itemtitle", [itemtitle])
+ , ("feedauthor", [feedauthor])
+ , ("itemauthor", [itemauthor])
+ , ("itemsummary", [getItemSummary $ item i])
+ , ("itemdescription", [getItemDescription $ item i])
+ , ("itemrights", [getItemRights $ item i])
+ , ("itemid", [snd <$> getItemId (item i)])
+ , ("title", [itemtitle, feedtitle])
+ , ("author", [itemauthor, feedauthor])
+ ]
+ where
+ feedtitle = Just $ getFeedTitle $ feed i
+ itemtitle = getItemTitle $ item i
+ feedauthor = getFeedAuthor $ feed i
+ itemauthor = getItemAuthor $ item i
+
+extractField :: String -> [Maybe String] -> (String, String)
+extractField k [] = (k, "none")
+extractField k (Just v:_)
+ | not (null v) = (k, v)
+extractField k (_:rest) = extractField k rest
+
{- Called when there is a problem with a feed.
- Throws an error if the feed is broken, otherwise shows a warning. -}
feedProblem :: URLString -> String -> Annex ()