diff options
| -rw-r--r-- | Backend/SHA.hs | 18 | ||||
| -rw-r--r-- | Git/CatFile.hs | 4 | ||||
| -rw-r--r-- | Remote/Bup.hs | 6 | ||||
| -rw-r--r-- | Utility/ExternalSHA.hs | 3 | ||||
| -rw-r--r-- | Utility/Hash.hs | 29 | ||||
| -rw-r--r-- | Utility/WebApp.hs | 4 | ||||
| -rw-r--r-- | debian/changelog | 8 | ||||
| -rw-r--r-- | git-annex.cabal | 2 |
8 files changed, 57 insertions, 17 deletions
diff --git a/Backend/SHA.hs b/Backend/SHA.hs index a735ce1e5..7150a4202 100644 --- a/Backend/SHA.hs +++ b/Backend/SHA.hs @@ -1,6 +1,6 @@ {- git-annex SHA backends - - - Copyright 2011,2012 Joey Hess <joey@kitenet.net> + - Copyright 2011-2013 Joey Hess <joey@kitenet.net> - - Licensed under the GNU GPL version 3 or higher. -} @@ -12,10 +12,10 @@ import qualified Annex import Types.Backend import Types.Key import Types.KeySource +import Utility.Hash import Utility.ExternalSHA import qualified Build.SysConfig as SysConfig -import Data.Digest.Pure.SHA import qualified Data.ByteString.Lazy as L import Data.Char @@ -70,12 +70,14 @@ shaCommand shasize filesize | shasize == 512 = use SysConfig.sha512 sha512 | otherwise = error $ "bad sha size " ++ show shasize where - use Nothing sha = Left $ showDigest . sha - use (Just c) sha - {- use builtin, but slower sha for small files - - benchmarking indicates it's faster up to - - and slightly beyond 50 kb files -} - | filesize < 51200 = use Nothing sha + use Nothing hasher = Left $ show . hasher + use (Just c) hasher + {- Use builtin, but slightly slower hashing for + - smallish files. Cryptohash benchmarks 90 to 101% + - faster than external hashers, depending on the hash + - and system. So there is no point forking an external + - process unless the file is large. -} + | filesize < 1048576 = use Nothing hasher | otherwise = Right c {- A key is a checksum of its contents. -} diff --git a/Git/CatFile.hs b/Git/CatFile.hs index bd86ff326..f7438b410 100644 --- a/Git/CatFile.hs +++ b/Git/CatFile.hs @@ -18,7 +18,6 @@ module Git.CatFile ( import System.IO import qualified Data.ByteString as S import qualified Data.ByteString.Lazy as L -import Data.Digest.Pure.SHA import Data.Char import System.Process (std_out, std_err) import Numeric @@ -31,6 +30,7 @@ import Git.Command import Git.Types import Git.FilePath import qualified Utility.CoProcess as CoProcess +import Utility.Hash data CatFileHandle = CatFileHandle CoProcess.CoProcessHandle Repo @@ -103,7 +103,7 @@ catObjectDetails (CatFileHandle hdl repo) object = CoProcess.query hdl send rece } fileEncoding h content <- L.hGetContents h - let sha = (\s -> length s `seq` s) (showDigest $ sha1 content) + let sha = (\s -> length s `seq` s) (show $ sha1 content) ok <- checkSuccessProcess pid return $ if ok then Just (content, Ref sha) diff --git a/Remote/Bup.hs b/Remote/Bup.hs index 960ed4ada..16fe8c8c5 100644 --- a/Remote/Bup.hs +++ b/Remote/Bup.hs @@ -10,6 +10,7 @@ module Remote.Bup (remote) where import qualified Data.ByteString.Lazy as L import qualified Data.Map as M import System.Process +import Data.ByteString.Lazy.UTF8 (fromString) import Common.Annex import Types.Remote @@ -25,8 +26,7 @@ import Remote.Helper.Ssh import Remote.Helper.Special import Remote.Helper.Encryptable import Crypto -import Data.ByteString.Lazy.UTF8 (fromString) -import Data.Digest.Pure.SHA +import Utility.Hash import Utility.UserInfo import Annex.Content import Annex.UUID @@ -277,7 +277,7 @@ bup2GitRemote r bupRef :: Key -> String bupRef k | Git.Ref.legal True shown = shown - | otherwise = "git-annex-" ++ showDigest (sha256 (fromString shown)) + | otherwise = "git-annex-" ++ show (sha256 (fromString shown)) where shown = key2file k diff --git a/Utility/ExternalSHA.hs b/Utility/ExternalSHA.hs index 21241d302..adbde795a 100644 --- a/Utility/ExternalSHA.hs +++ b/Utility/ExternalSHA.hs @@ -1,6 +1,7 @@ {- Calculating a SHA checksum with an external command. - - - This is often faster than using Haskell libraries. + - This is typically a bit faster than using Haskell libraries, + - by around 1% to 10%. Worth it for really big files. - - Copyright 2011-2013 Joey Hess <joey@kitenet.net> - diff --git a/Utility/Hash.hs b/Utility/Hash.hs new file mode 100644 index 000000000..31a36462c --- /dev/null +++ b/Utility/Hash.hs @@ -0,0 +1,29 @@ +{- Convenience wrapper around cryptohash. + - + - The resulting Digests can be shown to get a canonical hash encoding. -} + +module Utility.Hash where + +import Crypto.Hash +import qualified Data.ByteString.Lazy as L + +sha1 :: L.ByteString -> Digest SHA1 +sha1 = hashlazy + +sha224 :: L.ByteString -> Digest SHA224 +sha224 = hashlazy + +sha256 :: L.ByteString -> Digest SHA256 +sha256 = hashlazy + +sha384 :: L.ByteString -> Digest SHA384 +sha384 = hashlazy + +sha512 :: L.ByteString -> Digest SHA512 +sha512 = hashlazy + +-- sha3 is not yet fully standardized +--sha3 :: L.ByteString -> Digest SHA3 +--sha3 = hashlazy + + diff --git a/Utility/WebApp.hs b/Utility/WebApp.hs index f3c0d3a6b..c078a5643 100644 --- a/Utility/WebApp.hs +++ b/Utility/WebApp.hs @@ -12,6 +12,7 @@ module Utility.WebApp where import Common import Utility.Tmp import Utility.FileMode +import Utility.Hash import qualified Yesod import qualified Network.Wai as Wai @@ -24,7 +25,6 @@ import qualified Data.CaseInsensitive as CI import Network.Socket import Control.Exception import Crypto.Random -import Data.Digest.Pure.SHA import qualified Web.ClientSession as CS import qualified Data.ByteString.Lazy as L import qualified Data.ByteString.Lazy.UTF8 as L8 @@ -214,7 +214,7 @@ genRandomToken = do return $ case genBytes 512 g of Left e -> error $ "failed to generate secret token: " ++ show e - Right (s, _) -> showDigest $ sha512 $ L.fromChunks [s] + Right (s, _) -> show $ sha512 $ L.fromChunks [s] {- A Yesod isAuthorized method, which checks the auth cgi parameter - against a token extracted from the Yesod application. diff --git a/debian/changelog b/debian/changelog index 91d23876e..8c28323f8 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,11 @@ +git-annex (4.20130921) UNRELEASED; urgency=low + + * Use cryptohash rather than SHA for hashing when no external hash program + is available. This is a significant speedup for SHA256 on OSX, for + example. + + -- Joey Hess <joeyh@debian.org> Sun, 22 Sep 2013 19:42:29 -0400 + git-annex (4.20130920) unstable; urgency=low * webapp: Initial support for setting up encrypted removable drives. diff --git a/git-annex.cabal b/git-annex.cabal index 0846cca39..df4097ac2 100644 --- a/git-annex.cabal +++ b/git-annex.cabal @@ -76,7 +76,7 @@ Executable git-annex Build-Depends: MissingH, hslogger, directory, filepath, containers, utf8-string, network (>= 2.0), mtl (>= 2), bytestring, old-locale, time, HTTP, - extensible-exceptions, dataenc, SHA, process, json, + extensible-exceptions, dataenc, SHA, cryptohash, process, json, base (>= 4.5 && < 4.8), monad-control, MonadCatchIO-transformers, IfElse, text, QuickCheck >= 2.1, bloomfilter, edit-distance, process, SafeSemaphore, uuid, random, dlist, unix-compat |