better streaming while encrypting/decrypting

Both the directory and webdav special remotes used to have to buffer
the whole file contents before it could be decrypted, as they read
from chunks. Now the chunks are streamed through gpg with no buffering.

7 files changed, 38 insertions, 29 deletions

diff --git a/Remote/Bup.hs b/Remote/Bup.hs
index f5bcc4f45..62db01a7b 100644
--- a/Remote/Bup.hs
+++ b/Remote/Bup.hs
@@ -125,7 +125,7 @@ storeEncrypted r buprepo (cipher, enck) k _p = do
 	src <- inRepo $ gitAnnexLocation k
 	params <- bupSplitParams r buprepo enck []
 	liftIO $ catchBoolIO $
-		withEncryptedHandle cipher (L.readFile src) $ \h ->
+		encrypt cipher (feedFile src) $ \h ->
 			pipeBup params (Just h) Nothing
 
 retrieve :: BupRepo -> Key -> AssociatedFile -> FilePath -> Annex Bool
@@ -141,7 +141,8 @@ retrieveCheap _ _ _ = return False
 retrieveEncrypted :: BupRepo -> (Cipher, Key) -> Key -> FilePath -> Annex Bool
 retrieveEncrypted buprepo (cipher, enck) _ f = liftIO $ catchBoolIO $
 	withHandle StdoutHandle createProcessSuccess p $ \h -> do
-		withDecryptedContent cipher (L.hGetContents h) $ L.writeFile f
+		decrypt cipher (\toh -> L.hPut toh =<< L.hGetContents h) $
+			readBytes $ L.writeFile f
 		return True
   where
 	params = bupParams "join" buprepo [Param $ bupRef enck]
diff --git a/Remote/Directory.hs b/Remote/Directory.hs
index 794a8c468..0527270a1 100644
--- a/Remote/Directory.hs
+++ b/Remote/Directory.hs
@@ -118,13 +118,13 @@ storeEncrypted d chunksize (cipher, enck) k p = do
 	src <- inRepo $ gitAnnexLocation k
 	metered (Just p) k $ \meterupdate ->
 		storeHelper d chunksize enck $ \dests ->
-			withEncryptedContent cipher (L.readFile src) $ \s ->
+			encrypt cipher (feedFile src) $ readBytes $ \b ->
 				case chunksize of
 					Nothing -> do
 						let dest = Prelude.head dests
-						meteredWriteFile meterupdate dest s
+						meteredWriteFile meterupdate dest b
 						return [dest]
-					Just _ -> storeSplit meterupdate chunksize dests s
+					Just _ -> storeSplit meterupdate chunksize dests b
 
 {- Splits a ByteString into chunks and writes to dests, obeying configured
  - chunk size (not to be confused with the L.ByteString chunk size).
@@ -192,9 +192,11 @@ retrieveEncrypted :: FilePath -> ChunkSize -> (Cipher, Key) -> Key -> FilePath -
 retrieveEncrypted d chunksize (cipher, enck) k f = metered Nothing k $ \meterupdate ->
 	liftIO $ withStoredFiles chunksize d enck $ \files ->
 		catchBoolIO $ do
-			withDecryptedContent cipher (L.concat <$> mapM L.readFile files) $
-				meteredWriteFile meterupdate f
+			decrypt cipher (feeder files) $
+				readBytes $ meteredWriteFile meterupdate f
 			return True
+  where
+	feeder files h = forM_ files $ \file -> L.hPut h =<< L.readFile file
 
 retrieveCheap :: FilePath -> ChunkSize -> Key -> FilePath -> Annex Bool
 retrieveCheap _ (Just _) _ _ = return False -- no cheap retrieval for chunks
diff --git a/Remote/Helper/Encryptable.hs b/Remote/Helper/Encryptable.hs
index 12c7d37e9..a48ec813a 100644
--- a/Remote/Helper/Encryptable.hs
+++ b/Remote/Helper/Encryptable.hs
@@ -81,12 +81,11 @@ remoteCipher c = go $ extractCipher c
 		cache <- Annex.getState Annex.ciphers
 		case M.lookup encipher cache of
 			Just cipher -> return $ Just cipher
-			Nothing -> decrypt encipher cache
-	decrypt encipher cache = do
-		showNote "gpg"
-		cipher <- liftIO $ decryptCipher encipher
-		Annex.changeState (\s -> s { Annex.ciphers = M.insert encipher cipher cache })
-		return $ Just cipher
+			Nothing -> do
+				showNote "gpg"
+				cipher <- liftIO $ decryptCipher encipher
+				Annex.changeState (\s -> s { Annex.ciphers = M.insert encipher cipher cache })
+				return $ Just cipher
 
 {- Checks if there is a trusted (non-shared) cipher. -}
 isTrustedCipher :: RemoteConfig -> Bool
@@ -96,9 +95,9 @@ isTrustedCipher c =
 {- Gets encryption Cipher, and encrypted version of Key. -}
 cipherKey :: Maybe RemoteConfig -> Key -> Annex (Maybe (Cipher, Key))
 cipherKey Nothing _ = return Nothing
-cipherKey (Just c) k = maybe Nothing encrypt <$> remoteCipher c
+cipherKey (Just c) k = maybe Nothing make <$> remoteCipher c
   where
-	encrypt ciphertext = Just (ciphertext, encryptKey ciphertext k)
+	make ciphertext = Just (ciphertext, encryptKey ciphertext k)
 
 {- Stores an StorableCipher in a remote's configuration. -}
 storeCipher :: RemoteConfig -> StorableCipher -> RemoteConfig
diff --git a/Remote/Hook.hs b/Remote/Hook.hs
index f9a143ccd..7173a5b80 100644
--- a/Remote/Hook.hs
+++ b/Remote/Hook.hs
@@ -108,7 +108,8 @@ store h k _f _p = do
 storeEncrypted :: String -> (Cipher, Key) -> Key -> MeterUpdate -> Annex Bool
 storeEncrypted h (cipher, enck) k _p = withTmp enck $ \tmp -> do
 	src <- inRepo $ gitAnnexLocation k
-	liftIO $ withEncryptedContent cipher (L.readFile src) $ L.writeFile tmp
+	liftIO $ encrypt cipher (feedFile src) $
+		readBytes $ L.writeFile tmp
 	runHook h "store" enck (Just tmp) $ return True
 
 retrieve :: String -> Key -> AssociatedFile -> FilePath -> Annex Bool
@@ -120,7 +121,8 @@ retrieveCheap _ _ _ = return False
 retrieveEncrypted :: String -> (Cipher, Key) -> Key -> FilePath -> Annex Bool
 retrieveEncrypted h (cipher, enck) _ f = withTmp enck $ \tmp ->
 	runHook h "retrieve" enck (Just tmp) $ liftIO $ catchBoolIO $ do
-		withDecryptedContent cipher (L.readFile tmp) $ L.writeFile f
+		decrypt cipher (feedFile tmp) $
+			readBytes $ L.writeFile f
 		return True
 
 remove :: String -> Key -> Annex Bool
diff --git a/Remote/Rsync.hs b/Remote/Rsync.hs
index 1d5f2d28c..c48a9c14a 100644
--- a/Remote/Rsync.hs
+++ b/Remote/Rsync.hs
@@ -110,7 +110,8 @@ store o k _f p = rsyncSend o p k <=< inRepo $ gitAnnexLocation k
 storeEncrypted :: RsyncOpts -> (Cipher, Key) -> Key -> MeterUpdate -> Annex Bool
 storeEncrypted o (cipher, enck) k p = withTmp enck $ \tmp -> do
 	src <- inRepo $ gitAnnexLocation k
-	liftIO $ withEncryptedContent cipher (L.readFile src) $ L.writeFile tmp
+	liftIO $ decrypt cipher (feedFile src) $
+		readBytes $ L.writeFile tmp
 	rsyncSend o p enck tmp
 
 retrieve :: RsyncOpts -> Key -> AssociatedFile -> FilePath -> Annex Bool
@@ -128,7 +129,8 @@ retrieveEncrypted :: RsyncOpts -> (Cipher, Key) -> Key -> FilePath -> Annex Bool
 retrieveEncrypted o (cipher, enck) _ f = withTmp enck $ \tmp -> do
 	ifM (retrieve o enck undefined tmp)
 		( liftIO $ catchBoolIO $ do
-			withDecryptedContent cipher (L.readFile tmp) $ L.writeFile f
+			decrypt cipher (feedFile tmp) $
+				readBytes $ L.writeFile f
 			return True
 		, return False
 		)
diff --git a/Remote/S3.hs b/Remote/S3.hs
index 93620dfd0..54136f8a0 100644
--- a/Remote/S3.hs
+++ b/Remote/S3.hs
@@ -122,7 +122,8 @@ storeEncrypted r (cipher, enck) k _p = s3Action r False $ \(conn, bucket) ->
 	-- (An alternative would be chunking to to a constant size.)
 	withTmp enck $ \tmp -> do
 		f <- inRepo $ gitAnnexLocation k
-		liftIO $ withEncryptedContent cipher (L.readFile f) $ L.writeFile tmp
+		liftIO $ encrypt cipher (feedFile f) $
+			readBytes $ L.writeFile tmp
 		res <- liftIO $ storeHelper (conn, bucket) r enck tmp
 		s3Bool res
 
@@ -162,8 +163,8 @@ retrieveEncrypted :: Remote -> (Cipher, Key) -> Key -> FilePath -> Annex Bool
 retrieveEncrypted r (cipher, enck) _ f = s3Action r False $ \(conn, bucket) -> do
 	res <- liftIO $ getObject conn $ bucketKey r bucket enck
 	case res of
-		Right o -> liftIO $ 
-			withDecryptedContent cipher (return $ obj_data o) $ \content -> do
+		Right o -> liftIO $ decrypt cipher (feedBytes $ obj_data o) $ 
+			readBytes $ \content -> do
 				L.writeFile f content
 				return True
 		Left e -> s3Warning e
diff --git a/Remote/WebDAV.hs b/Remote/WebDAV.hs
index aab5e7efb..ea4800c9d 100644
--- a/Remote/WebDAV.hs
+++ b/Remote/WebDAV.hs
@@ -93,8 +93,8 @@ storeEncrypted :: Remote -> (Cipher, Key) -> Key -> MeterUpdate -> Annex Bool
 storeEncrypted r (cipher, enck) k _p = davAction r False $ \(baseurl, user, pass) -> do
 	let url = davLocation baseurl enck
 	f <- inRepo $ gitAnnexLocation k
-	liftIO $ withEncryptedContent cipher (L.readFile f) $
-		storeHelper r url user pass
+	liftIO $ encrypt cipher (feedFile f) $
+		readBytes $ storeHelper r url user pass
 
 storeHelper :: Remote -> DavUrl -> DavUser -> DavPass -> L.ByteString -> IO Bool
 storeHelper r urlbase user pass b = catchBoolIO $ do
@@ -133,18 +133,20 @@ retrieveEncrypted :: Remote -> (Cipher, Key) -> Key -> FilePath -> Annex Bool
 retrieveEncrypted r (cipher, enck) k d = metered Nothing k $ \meterupdate ->
 	davAction r False $ \(baseurl, user, pass) -> liftIO $ catchBoolIO $
 		withStoredFiles r enck baseurl user pass onerr $ \urls -> do
-			withDecryptedContent cipher (L.concat <$> feeder user pass urls []) $
-				meteredWriteFile meterupdate d
+			decrypt cipher (feeder user pass urls) $
+				readBytes $ meteredWriteFile meterupdate d
 			return True
   where
 	onerr _ = return False
 
-	feeder _ _ [] c = return $ reverse c
-	feeder user pass (url:urls) c = do
+	feeder _ _ [] _ = noop
+	feeder user pass (url:urls) h = do
 		mb <- davGetUrlContent url user pass
 		case mb of
 			Nothing -> throwIO "download failed"
-			Just b -> feeder user pass urls (b:c)
+			Just b -> do
+				L.hPut h b
+				feeder user pass urls h
 
 remove :: Remote -> Key -> Annex Bool
 remove r k = davAction r False $ \(baseurl, user, pass) -> liftIO $ do