summaryrefslogtreecommitdiff
path: root/Remote
diff options
context:
space:
mode:
authorGravatar Joey Hess <joey@kitenet.net>2012-11-18 15:27:44 -0400
committerGravatar Joey Hess <joey@kitenet.net>2012-11-18 15:27:44 -0400
commit677aab525a7023642f4b2e9d96db3c3481e8f0b1 (patch)
treee6584a8f1663f364001ad452fe8d09b83fda11a4 /Remote
parentcb2ec900ae8aa60b4ccf35adeb287823d976be07 (diff)
better streaming while encrypting/decrypting
Both the directory and webdav special remotes used to have to buffer the whole file contents before it could be decrypted, as they read from chunks. Now the chunks are streamed through gpg with no buffering.
Diffstat (limited to 'Remote')
-rw-r--r--Remote/Bup.hs5
-rw-r--r--Remote/Directory.hs12
-rw-r--r--Remote/Helper/Encryptable.hs15
-rw-r--r--Remote/Hook.hs6
-rw-r--r--Remote/Rsync.hs6
-rw-r--r--Remote/S3.hs7
-rw-r--r--Remote/WebDAV.hs16
7 files changed, 38 insertions, 29 deletions
diff --git a/Remote/Bup.hs b/Remote/Bup.hs
index f5bcc4f45..62db01a7b 100644
--- a/Remote/Bup.hs
+++ b/Remote/Bup.hs
@@ -125,7 +125,7 @@ storeEncrypted r buprepo (cipher, enck) k _p = do
src <- inRepo $ gitAnnexLocation k
params <- bupSplitParams r buprepo enck []
liftIO $ catchBoolIO $
- withEncryptedHandle cipher (L.readFile src) $ \h ->
+ encrypt cipher (feedFile src) $ \h ->
pipeBup params (Just h) Nothing
retrieve :: BupRepo -> Key -> AssociatedFile -> FilePath -> Annex Bool
@@ -141,7 +141,8 @@ retrieveCheap _ _ _ = return False
retrieveEncrypted :: BupRepo -> (Cipher, Key) -> Key -> FilePath -> Annex Bool
retrieveEncrypted buprepo (cipher, enck) _ f = liftIO $ catchBoolIO $
withHandle StdoutHandle createProcessSuccess p $ \h -> do
- withDecryptedContent cipher (L.hGetContents h) $ L.writeFile f
+ decrypt cipher (\toh -> L.hPut toh =<< L.hGetContents h) $
+ readBytes $ L.writeFile f
return True
where
params = bupParams "join" buprepo [Param $ bupRef enck]
diff --git a/Remote/Directory.hs b/Remote/Directory.hs
index 794a8c468..0527270a1 100644
--- a/Remote/Directory.hs
+++ b/Remote/Directory.hs
@@ -118,13 +118,13 @@ storeEncrypted d chunksize (cipher, enck) k p = do
src <- inRepo $ gitAnnexLocation k
metered (Just p) k $ \meterupdate ->
storeHelper d chunksize enck $ \dests ->
- withEncryptedContent cipher (L.readFile src) $ \s ->
+ encrypt cipher (feedFile src) $ readBytes $ \b ->
case chunksize of
Nothing -> do
let dest = Prelude.head dests
- meteredWriteFile meterupdate dest s
+ meteredWriteFile meterupdate dest b
return [dest]
- Just _ -> storeSplit meterupdate chunksize dests s
+ Just _ -> storeSplit meterupdate chunksize dests b
{- Splits a ByteString into chunks and writes to dests, obeying configured
- chunk size (not to be confused with the L.ByteString chunk size).
@@ -192,9 +192,11 @@ retrieveEncrypted :: FilePath -> ChunkSize -> (Cipher, Key) -> Key -> FilePath -
retrieveEncrypted d chunksize (cipher, enck) k f = metered Nothing k $ \meterupdate ->
liftIO $ withStoredFiles chunksize d enck $ \files ->
catchBoolIO $ do
- withDecryptedContent cipher (L.concat <$> mapM L.readFile files) $
- meteredWriteFile meterupdate f
+ decrypt cipher (feeder files) $
+ readBytes $ meteredWriteFile meterupdate f
return True
+ where
+ feeder files h = forM_ files $ \file -> L.hPut h =<< L.readFile file
retrieveCheap :: FilePath -> ChunkSize -> Key -> FilePath -> Annex Bool
retrieveCheap _ (Just _) _ _ = return False -- no cheap retrieval for chunks
diff --git a/Remote/Helper/Encryptable.hs b/Remote/Helper/Encryptable.hs
index 12c7d37e9..a48ec813a 100644
--- a/Remote/Helper/Encryptable.hs
+++ b/Remote/Helper/Encryptable.hs
@@ -81,12 +81,11 @@ remoteCipher c = go $ extractCipher c
cache <- Annex.getState Annex.ciphers
case M.lookup encipher cache of
Just cipher -> return $ Just cipher
- Nothing -> decrypt encipher cache
- decrypt encipher cache = do
- showNote "gpg"
- cipher <- liftIO $ decryptCipher encipher
- Annex.changeState (\s -> s { Annex.ciphers = M.insert encipher cipher cache })
- return $ Just cipher
+ Nothing -> do
+ showNote "gpg"
+ cipher <- liftIO $ decryptCipher encipher
+ Annex.changeState (\s -> s { Annex.ciphers = M.insert encipher cipher cache })
+ return $ Just cipher
{- Checks if there is a trusted (non-shared) cipher. -}
isTrustedCipher :: RemoteConfig -> Bool
@@ -96,9 +95,9 @@ isTrustedCipher c =
{- Gets encryption Cipher, and encrypted version of Key. -}
cipherKey :: Maybe RemoteConfig -> Key -> Annex (Maybe (Cipher, Key))
cipherKey Nothing _ = return Nothing
-cipherKey (Just c) k = maybe Nothing encrypt <$> remoteCipher c
+cipherKey (Just c) k = maybe Nothing make <$> remoteCipher c
where
- encrypt ciphertext = Just (ciphertext, encryptKey ciphertext k)
+ make ciphertext = Just (ciphertext, encryptKey ciphertext k)
{- Stores an StorableCipher in a remote's configuration. -}
storeCipher :: RemoteConfig -> StorableCipher -> RemoteConfig
diff --git a/Remote/Hook.hs b/Remote/Hook.hs
index f9a143ccd..7173a5b80 100644
--- a/Remote/Hook.hs
+++ b/Remote/Hook.hs
@@ -108,7 +108,8 @@ store h k _f _p = do
storeEncrypted :: String -> (Cipher, Key) -> Key -> MeterUpdate -> Annex Bool
storeEncrypted h (cipher, enck) k _p = withTmp enck $ \tmp -> do
src <- inRepo $ gitAnnexLocation k
- liftIO $ withEncryptedContent cipher (L.readFile src) $ L.writeFile tmp
+ liftIO $ encrypt cipher (feedFile src) $
+ readBytes $ L.writeFile tmp
runHook h "store" enck (Just tmp) $ return True
retrieve :: String -> Key -> AssociatedFile -> FilePath -> Annex Bool
@@ -120,7 +121,8 @@ retrieveCheap _ _ _ = return False
retrieveEncrypted :: String -> (Cipher, Key) -> Key -> FilePath -> Annex Bool
retrieveEncrypted h (cipher, enck) _ f = withTmp enck $ \tmp ->
runHook h "retrieve" enck (Just tmp) $ liftIO $ catchBoolIO $ do
- withDecryptedContent cipher (L.readFile tmp) $ L.writeFile f
+ decrypt cipher (feedFile tmp) $
+ readBytes $ L.writeFile f
return True
remove :: String -> Key -> Annex Bool
diff --git a/Remote/Rsync.hs b/Remote/Rsync.hs
index 1d5f2d28c..c48a9c14a 100644
--- a/Remote/Rsync.hs
+++ b/Remote/Rsync.hs
@@ -110,7 +110,8 @@ store o k _f p = rsyncSend o p k <=< inRepo $ gitAnnexLocation k
storeEncrypted :: RsyncOpts -> (Cipher, Key) -> Key -> MeterUpdate -> Annex Bool
storeEncrypted o (cipher, enck) k p = withTmp enck $ \tmp -> do
src <- inRepo $ gitAnnexLocation k
- liftIO $ withEncryptedContent cipher (L.readFile src) $ L.writeFile tmp
+ liftIO $ decrypt cipher (feedFile src) $
+ readBytes $ L.writeFile tmp
rsyncSend o p enck tmp
retrieve :: RsyncOpts -> Key -> AssociatedFile -> FilePath -> Annex Bool
@@ -128,7 +129,8 @@ retrieveEncrypted :: RsyncOpts -> (Cipher, Key) -> Key -> FilePath -> Annex Bool
retrieveEncrypted o (cipher, enck) _ f = withTmp enck $ \tmp -> do
ifM (retrieve o enck undefined tmp)
( liftIO $ catchBoolIO $ do
- withDecryptedContent cipher (L.readFile tmp) $ L.writeFile f
+ decrypt cipher (feedFile tmp) $
+ readBytes $ L.writeFile f
return True
, return False
)
diff --git a/Remote/S3.hs b/Remote/S3.hs
index 93620dfd0..54136f8a0 100644
--- a/Remote/S3.hs
+++ b/Remote/S3.hs
@@ -122,7 +122,8 @@ storeEncrypted r (cipher, enck) k _p = s3Action r False $ \(conn, bucket) ->
-- (An alternative would be chunking to to a constant size.)
withTmp enck $ \tmp -> do
f <- inRepo $ gitAnnexLocation k
- liftIO $ withEncryptedContent cipher (L.readFile f) $ L.writeFile tmp
+ liftIO $ encrypt cipher (feedFile f) $
+ readBytes $ L.writeFile tmp
res <- liftIO $ storeHelper (conn, bucket) r enck tmp
s3Bool res
@@ -162,8 +163,8 @@ retrieveEncrypted :: Remote -> (Cipher, Key) -> Key -> FilePath -> Annex Bool
retrieveEncrypted r (cipher, enck) _ f = s3Action r False $ \(conn, bucket) -> do
res <- liftIO $ getObject conn $ bucketKey r bucket enck
case res of
- Right o -> liftIO $
- withDecryptedContent cipher (return $ obj_data o) $ \content -> do
+ Right o -> liftIO $ decrypt cipher (feedBytes $ obj_data o) $
+ readBytes $ \content -> do
L.writeFile f content
return True
Left e -> s3Warning e
diff --git a/Remote/WebDAV.hs b/Remote/WebDAV.hs
index aab5e7efb..ea4800c9d 100644
--- a/Remote/WebDAV.hs
+++ b/Remote/WebDAV.hs
@@ -93,8 +93,8 @@ storeEncrypted :: Remote -> (Cipher, Key) -> Key -> MeterUpdate -> Annex Bool
storeEncrypted r (cipher, enck) k _p = davAction r False $ \(baseurl, user, pass) -> do
let url = davLocation baseurl enck
f <- inRepo $ gitAnnexLocation k
- liftIO $ withEncryptedContent cipher (L.readFile f) $
- storeHelper r url user pass
+ liftIO $ encrypt cipher (feedFile f) $
+ readBytes $ storeHelper r url user pass
storeHelper :: Remote -> DavUrl -> DavUser -> DavPass -> L.ByteString -> IO Bool
storeHelper r urlbase user pass b = catchBoolIO $ do
@@ -133,18 +133,20 @@ retrieveEncrypted :: Remote -> (Cipher, Key) -> Key -> FilePath -> Annex Bool
retrieveEncrypted r (cipher, enck) k d = metered Nothing k $ \meterupdate ->
davAction r False $ \(baseurl, user, pass) -> liftIO $ catchBoolIO $
withStoredFiles r enck baseurl user pass onerr $ \urls -> do
- withDecryptedContent cipher (L.concat <$> feeder user pass urls []) $
- meteredWriteFile meterupdate d
+ decrypt cipher (feeder user pass urls) $
+ readBytes $ meteredWriteFile meterupdate d
return True
where
onerr _ = return False
- feeder _ _ [] c = return $ reverse c
- feeder user pass (url:urls) c = do
+ feeder _ _ [] _ = noop
+ feeder user pass (url:urls) h = do
mb <- davGetUrlContent url user pass
case mb of
Nothing -> throwIO "download failed"
- Just b -> feeder user pass urls (b:c)
+ Just b -> do
+ L.hPut h b
+ feeder user pass urls h
remove :: Remote -> Key -> Annex Bool
remove r k = davAction r False $ \(baseurl, user, pass) -> liftIO $ do