aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorGravatar Joey Hess <joey@kitenet.net>2013-09-22 19:45:08 -0400
committerGravatar Joey Hess <joey@kitenet.net>2013-09-22 20:06:02 -0400
commitca7695f9e746ef0d9cc70ffed6c2590dba29db79 (patch)
tree2291c29ab8d6126fea397fdd3d1e502cff064b1c
parent978d0362b0469236c848882df9b3a7d412fc96f9 (diff)
Use cryptohash rather than SHA for hashing.
This is a massive win on OSX, which doesn't have a sha256sum normally. Only use external hash commands when the file is > 1 mb, since cryptohash is quite close to them in speed. SHA is still used to calculate HMACs. I don't quite understand cryptohash's API for those. Used the following benchmark to arrive at the 1 mb number. 1 mb file: benchmarking sha256/internal mean: 13.86696 ms, lb 13.83010 ms, ub 13.93453 ms, ci 0.950 std dev: 249.3235 us, lb 162.0448 us, ub 458.1744 us, ci 0.950 found 5 outliers among 100 samples (5.0%) 4 (4.0%) high mild 1 (1.0%) high severe variance introduced by outliers: 10.415% variance is moderately inflated by outliers benchmarking sha256/external mean: 14.20670 ms, lb 14.17237 ms, ub 14.27004 ms, ci 0.950 std dev: 230.5448 us, lb 150.7310 us, ub 427.6068 us, ci 0.950 found 3 outliers among 100 samples (3.0%) 2 (2.0%) high mild 1 (1.0%) high severe 2 mb file: benchmarking sha256/internal mean: 26.44270 ms, lb 26.23701 ms, ub 26.63414 ms, ci 0.950 std dev: 1.012303 ms, lb 925.8921 us, ub 1.122267 ms, ci 0.950 variance introduced by outliers: 35.540% variance is moderately inflated by outliers benchmarking sha256/external mean: 26.84521 ms, lb 26.77644 ms, ub 26.91433 ms, ci 0.950 std dev: 347.7867 us, lb 210.6283 us, ub 571.3351 us, ci 0.950 found 6 outliers among 100 samples (6.0%) import Crypto.Hash import Data.ByteString.Lazy as L import Criterion.Main import Common testfile :: FilePath testfile = "/run/shm/data" -- on ram disk main = defaultMain [ bgroup "sha256" [ bench "internal" $ whnfIO internal , bench "external" $ whnfIO external ] ] sha256 :: L.ByteString -> Digest SHA256 sha256 = hashlazy internal :: IO String internal = show . sha256 <$> L.readFile testfile external :: IO String external = do s <- readProcess "sha256sum" [testfile] return $ fst $ separate (== ' ') s
-rw-r--r--Backend/SHA.hs18
-rw-r--r--Git/CatFile.hs4
-rw-r--r--Remote/Bup.hs6
-rw-r--r--Utility/ExternalSHA.hs3
-rw-r--r--Utility/Hash.hs29
-rw-r--r--Utility/WebApp.hs4
-rw-r--r--debian/changelog8
-rw-r--r--git-annex.cabal2
8 files changed, 57 insertions, 17 deletions
diff --git a/Backend/SHA.hs b/Backend/SHA.hs
index a735ce1e5..7150a4202 100644
--- a/Backend/SHA.hs
+++ b/Backend/SHA.hs
@@ -1,6 +1,6 @@
{- git-annex SHA backends
-
- - Copyright 2011,2012 Joey Hess <joey@kitenet.net>
+ - Copyright 2011-2013 Joey Hess <joey@kitenet.net>
-
- Licensed under the GNU GPL version 3 or higher.
-}
@@ -12,10 +12,10 @@ import qualified Annex
import Types.Backend
import Types.Key
import Types.KeySource
+import Utility.Hash
import Utility.ExternalSHA
import qualified Build.SysConfig as SysConfig
-import Data.Digest.Pure.SHA
import qualified Data.ByteString.Lazy as L
import Data.Char
@@ -70,12 +70,14 @@ shaCommand shasize filesize
| shasize == 512 = use SysConfig.sha512 sha512
| otherwise = error $ "bad sha size " ++ show shasize
where
- use Nothing sha = Left $ showDigest . sha
- use (Just c) sha
- {- use builtin, but slower sha for small files
- - benchmarking indicates it's faster up to
- - and slightly beyond 50 kb files -}
- | filesize < 51200 = use Nothing sha
+ use Nothing hasher = Left $ show . hasher
+ use (Just c) hasher
+ {- Use builtin, but slightly slower hashing for
+ - smallish files. Cryptohash benchmarks 90 to 101%
+ - faster than external hashers, depending on the hash
+ - and system. So there is no point forking an external
+ - process unless the file is large. -}
+ | filesize < 1048576 = use Nothing hasher
| otherwise = Right c
{- A key is a checksum of its contents. -}
diff --git a/Git/CatFile.hs b/Git/CatFile.hs
index bd86ff326..f7438b410 100644
--- a/Git/CatFile.hs
+++ b/Git/CatFile.hs
@@ -18,7 +18,6 @@ module Git.CatFile (
import System.IO
import qualified Data.ByteString as S
import qualified Data.ByteString.Lazy as L
-import Data.Digest.Pure.SHA
import Data.Char
import System.Process (std_out, std_err)
import Numeric
@@ -31,6 +30,7 @@ import Git.Command
import Git.Types
import Git.FilePath
import qualified Utility.CoProcess as CoProcess
+import Utility.Hash
data CatFileHandle = CatFileHandle CoProcess.CoProcessHandle Repo
@@ -103,7 +103,7 @@ catObjectDetails (CatFileHandle hdl repo) object = CoProcess.query hdl send rece
}
fileEncoding h
content <- L.hGetContents h
- let sha = (\s -> length s `seq` s) (showDigest $ sha1 content)
+ let sha = (\s -> length s `seq` s) (show $ sha1 content)
ok <- checkSuccessProcess pid
return $ if ok
then Just (content, Ref sha)
diff --git a/Remote/Bup.hs b/Remote/Bup.hs
index 960ed4ada..16fe8c8c5 100644
--- a/Remote/Bup.hs
+++ b/Remote/Bup.hs
@@ -10,6 +10,7 @@ module Remote.Bup (remote) where
import qualified Data.ByteString.Lazy as L
import qualified Data.Map as M
import System.Process
+import Data.ByteString.Lazy.UTF8 (fromString)
import Common.Annex
import Types.Remote
@@ -25,8 +26,7 @@ import Remote.Helper.Ssh
import Remote.Helper.Special
import Remote.Helper.Encryptable
import Crypto
-import Data.ByteString.Lazy.UTF8 (fromString)
-import Data.Digest.Pure.SHA
+import Utility.Hash
import Utility.UserInfo
import Annex.Content
import Annex.UUID
@@ -277,7 +277,7 @@ bup2GitRemote r
bupRef :: Key -> String
bupRef k
| Git.Ref.legal True shown = shown
- | otherwise = "git-annex-" ++ showDigest (sha256 (fromString shown))
+ | otherwise = "git-annex-" ++ show (sha256 (fromString shown))
where
shown = key2file k
diff --git a/Utility/ExternalSHA.hs b/Utility/ExternalSHA.hs
index 21241d302..adbde795a 100644
--- a/Utility/ExternalSHA.hs
+++ b/Utility/ExternalSHA.hs
@@ -1,6 +1,7 @@
{- Calculating a SHA checksum with an external command.
-
- - This is often faster than using Haskell libraries.
+ - This is typically a bit faster than using Haskell libraries,
+ - by around 1% to 10%. Worth it for really big files.
-
- Copyright 2011-2013 Joey Hess <joey@kitenet.net>
-
diff --git a/Utility/Hash.hs b/Utility/Hash.hs
new file mode 100644
index 000000000..31a36462c
--- /dev/null
+++ b/Utility/Hash.hs
@@ -0,0 +1,29 @@
+{- Convenience wrapper around cryptohash.
+ -
+ - The resulting Digests can be shown to get a canonical hash encoding. -}
+
+module Utility.Hash where
+
+import Crypto.Hash
+import qualified Data.ByteString.Lazy as L
+
+sha1 :: L.ByteString -> Digest SHA1
+sha1 = hashlazy
+
+sha224 :: L.ByteString -> Digest SHA224
+sha224 = hashlazy
+
+sha256 :: L.ByteString -> Digest SHA256
+sha256 = hashlazy
+
+sha384 :: L.ByteString -> Digest SHA384
+sha384 = hashlazy
+
+sha512 :: L.ByteString -> Digest SHA512
+sha512 = hashlazy
+
+-- sha3 is not yet fully standardized
+--sha3 :: L.ByteString -> Digest SHA3
+--sha3 = hashlazy
+
+
diff --git a/Utility/WebApp.hs b/Utility/WebApp.hs
index f3c0d3a6b..c078a5643 100644
--- a/Utility/WebApp.hs
+++ b/Utility/WebApp.hs
@@ -12,6 +12,7 @@ module Utility.WebApp where
import Common
import Utility.Tmp
import Utility.FileMode
+import Utility.Hash
import qualified Yesod
import qualified Network.Wai as Wai
@@ -24,7 +25,6 @@ import qualified Data.CaseInsensitive as CI
import Network.Socket
import Control.Exception
import Crypto.Random
-import Data.Digest.Pure.SHA
import qualified Web.ClientSession as CS
import qualified Data.ByteString.Lazy as L
import qualified Data.ByteString.Lazy.UTF8 as L8
@@ -214,7 +214,7 @@ genRandomToken = do
return $
case genBytes 512 g of
Left e -> error $ "failed to generate secret token: " ++ show e
- Right (s, _) -> showDigest $ sha512 $ L.fromChunks [s]
+ Right (s, _) -> show $ sha512 $ L.fromChunks [s]
{- A Yesod isAuthorized method, which checks the auth cgi parameter
- against a token extracted from the Yesod application.
diff --git a/debian/changelog b/debian/changelog
index 91d23876e..8c28323f8 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,11 @@
+git-annex (4.20130921) UNRELEASED; urgency=low
+
+ * Use cryptohash rather than SHA for hashing when no external hash program
+ is available. This is a significant speedup for SHA256 on OSX, for
+ example.
+
+ -- Joey Hess <joeyh@debian.org> Sun, 22 Sep 2013 19:42:29 -0400
+
git-annex (4.20130920) unstable; urgency=low
* webapp: Initial support for setting up encrypted removable drives.
diff --git a/git-annex.cabal b/git-annex.cabal
index 0846cca39..df4097ac2 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -76,7 +76,7 @@ Executable git-annex
Build-Depends: MissingH, hslogger, directory, filepath,
containers, utf8-string, network (>= 2.0), mtl (>= 2),
bytestring, old-locale, time, HTTP,
- extensible-exceptions, dataenc, SHA, process, json,
+ extensible-exceptions, dataenc, SHA, cryptohash, process, json,
base (>= 4.5 && < 4.8), monad-control, MonadCatchIO-transformers,
IfElse, text, QuickCheck >= 2.1, bloomfilter, edit-distance, process,
SafeSemaphore, uuid, random, dlist, unix-compat