git annex get from exports

Straightforward enough, except for the needed belt-and-suspenders sanity
checks to avoid foot shooting due to exports not being key/value stores.

* Even when annex.verify=false, always verify from exports.
* Only get files from exports that use a backend that supports
  checksum verification.
* Never trust exports, even if the user says to, because then
  `git annex drop` would drop content if the export seemed to contain
  a copy.

This commit was supported by the NSF-funded DataLad project.

2 files changed, 59 insertions, 9 deletions

diff --git a/Remote/Directory.hs b/Remote/Directory.hs
index 6adf6477a..7769eddd2 100644
--- a/Remote/Directory.hs
+++ b/Remote/Directory.hs
@@ -44,7 +44,7 @@ gen :: Git.Repo -> UUID -> RemoteConfig -> RemoteGitConfig -> Annex (Maybe Remot
 gen r u c gc = do
 	cst <- remoteCost gc cheapRemoteCost
 	let chunkconfig = getChunkConfig c
-	return $ Just $ exportableRemote $ specialRemote c
+	exportableRemote $ specialRemote c
 		(prepareStore dir chunkconfig)
 		(retrieve dir chunkconfig)
 		(simplyPrepare $ remove dir)
diff --git a/Remote/Helper/Export.hs b/Remote/Helper/Export.hs
index 9bbbb1f59..73ebb9141 100644
--- a/Remote/Helper/Export.hs
+++ b/Remote/Helper/Export.hs
@@ -8,9 +8,15 @@
 module Remote.Helper.Export where
 
 import Annex.Common
+import qualified Annex
 import Types.Remote
 import Types.Creds
+import Types.Backend
+import Types.Key
+import Types.TrustLevel
+import Backend
 import Remote.Helper.Encryptable (isEncrypted)
+import Database.Export
 
 import qualified Data.Map as M
 
@@ -27,15 +33,59 @@ exportUnsupported = ExportActions
 
 -- | A remote that supports exports when configured with exporttree=yes,
 -- and otherwise does not.
-exportableRemote :: Remote -> Remote
+exportableRemote :: Remote -> Annex (Maybe Remote)
 exportableRemote r = case M.lookup "exporttree" (config r) of
-	Just "yes" -> r
-		{ storeKey = \_ _ _ -> do
-			warning "remote is configured with exporttree=yes; use `git-annex export` to store content on it"
-			return False
-		}
-	_ -> r
-		{ exportActions = exportUnsupported }
+	Just "yes" -> do
+		db <- openDb (uuid r)
+
+		return $ Just $ r
+			-- Storing a key on an export would need a way to
+			-- look up the file(s) that the currently exported
+			-- tree uses for a key; there's not currently an
+			-- inexpensive way to do that (getExportLocation
+			-- only finds files that have been stored on the
+			-- export already).
+			{ storeKey = \_ _ _ -> do
+				warning "remote is configured with exporttree=yes; use `git-annex export` to store content on it"
+				return False
+			-- Keys can be retrieved, but since an export
+			-- is not a true key/value store, the content of
+			-- the key has to be able to be strongly verified.
+			, retrieveKeyFile = \k _af dest p ->
+				if maybe False (isJust . verifyKeyContent) (maybeLookupBackendVariety (keyVariety k))
+					then do
+						locs <- liftIO $ getExportLocation db k
+						case locs of
+							[] -> do
+								warning "unknown export location"
+								return (False, UnVerified)
+							(l:_) -> retrieveExport (exportActions r) k l dest p
+					else do
+						warning $ "exported content cannot be verified due to using the " ++ formatKeyVariety (keyVariety k) ++ " backend"
+						return (False, UnVerified)
+			, retrieveKeyFileCheap = \_ _ _ -> return False
+			-- Remove all files a key was exported to.
+			, removeKey = \k -> do
+				locs <- liftIO $ getExportLocation db k
+				oks <- forM locs $ \loc -> do
+					ok <- removeExport (exportActions r) k loc
+					when ok $
+						liftIO $ removeExportLocation db k loc
+					return ok
+				liftIO $ flushDbQueue db
+				return (and oks)
+			-- Can't lock content on exports, since they're
+			-- not key/value stores, and someone else could
+			-- change what's exported to a file at any time.
+			, lockContent = Nothing
+			-- Check if any of the files a key was exported
+			-- to are present. This doesn't guarantee the
+			-- export contains the right content.
+			, checkPresent = \k ->
+				anyM (checkPresentExport (exportActions r) k)
+					=<< liftIO (getExportLocation db k)
+			}
+	_ -> return $ Just $ r { exportActions = exportUnsupported }
 
 exportableRemoteSetup :: (SetupStage -> Maybe UUID -> Maybe CredPair -> RemoteConfig -> RemoteGitConfig -> Annex (RemoteConfig, UUID)) -> SetupStage -> Maybe UUID -> Maybe CredPair -> RemoteConfig -> RemoteGitConfig -> Annex (RemoteConfig, UUID)
 exportableRemoteSetup setupaction st mu cp c gc = case st of