From bca3fd65b9cd3c2224144349505ddda20361b59e Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 9 Mar 2012 19:26:02 -0400 Subject: fix key directory hash calculation code Fix Key directory hash calculation code to behave as it did before version 3.20120227 when a key contains non-ascii. The hash directories for a given Key are based on its md5sum. Prior to ghc 7.4, Keys contained raw, undecoded bytes, so the md5sum was taken of each byte in turn. With the ghc 7.4 filename encoding change, keys contains decoded unicode characters (possibly with surrigates for undecodable bytes). This changes the result of the md5sum, since the md5sum used is pure haskell and supports unicode. And that won't do, as git-annex will start looking in a different hash directory for the content of a key. The surrigates are particularly bad, since that's essentially a ghc implementation detail, so could change again at any time. Also, changing the locale changes how the bytes are decoded, which can also change the md5sum. Symptoms would include things like: * git annex fsck would complain that no copies existed of a file, despite its symlink pointing to the content that was locally present * git annex fix would change the symlink to use the wrong hash directory. Only WORM backend is likely to have been affected, since only it tends to include much filename data (SHA1E could in theory also be affected). I have not tried to support the hash directories used by git-annex versions 3.20120227 to 3.20120308, so things added with those versions with WORM will require manual fixups. Sorry for the inconvenience! --- Locations.hs | 4 ++-- Utility/FileSystemEncoding.hs | 28 +++++++++++++++++----------- Utility/StatFS.hsc | 2 -- debian/changelog | 8 ++++++++ 4 files changed, 27 insertions(+), 15 deletions(-) diff --git a/Locations.hs b/Locations.hs index 18df06004..2e3372551 100644 --- a/Locations.hs +++ b/Locations.hs @@ -218,12 +218,12 @@ hashDirMixed :: Hasher hashDirMixed k = addTrailingPathSeparator $ take 2 dir drop 2 dir where dir = take 4 $ display_32bits_as_dir =<< [a,b,c,d] - ABCD (a,b,c,d) = md5 $ Str $ show k + ABCD (a,b,c,d) = md5 $ Str $ encodeFilePath $ show k hashDirLower :: Hasher hashDirLower k = addTrailingPathSeparator $ take 3 dir drop 3 dir where - dir = take 6 $ md5s $ Str $ show k + dir = take 6 $ md5s $ Str $ encodeFilePath $ show k {- modified version of display_32bits_as_hex from Data.Hash.MD5 - Copyright (C) 2001 Ian Lynagh diff --git a/Utility/FileSystemEncoding.hs b/Utility/FileSystemEncoding.hs index 048323ee3..fe4202a75 100644 --- a/Utility/FileSystemEncoding.hs +++ b/Utility/FileSystemEncoding.hs @@ -7,31 +7,37 @@ module Utility.FileSystemEncoding where -import System.IO +import qualified GHC.Foreign as GHC +import qualified GHC.IO.Encoding as Encoding import Foreign.C -import GHC.Foreign as GHC -import GHC.IO.Encoding +import System.IO +import System.IO.Unsafe {- Sets a Handle to use the filesystem encoding. This causes data - written or read from it to be encoded/decoded the same - as ghc 7.4 does to filenames etc. This special encoding - allows "arbitrary undecodable bytes to be round-tripped through it". -} fileEncoding :: Handle -> IO () -fileEncoding h = hSetEncoding h =<< getFileSystemEncoding +fileEncoding h = hSetEncoding h =<< Encoding.getFileSystemEncoding {- Marshal a Haskell FilePath into a NUL terminated C string using temporary - storage. The FilePath is encoded using the filesystem encoding, - reversing the decoding that should have been done when the FilePath - was obtained. -} withFilePath :: FilePath -> (CString -> IO a) -> IO a -withFilePath fp f = getFileSystemEncoding >>= \enc -> GHC.withCString enc fp f +withFilePath fp f = Encoding.getFileSystemEncoding + >>= \enc -> GHC.withCString enc fp f {- Encodes a FilePath into a String of encoded bytes, applying the - filesystem encoding. - - - This does not do any IO, beyond allocating a C buffer. GHC does not - - seem to provide a pure way to do this conversion. -} -encodeFilePath :: FilePath -> IO String -encodeFilePath fp = do - enc <- getFileSystemEncoding - GHC.withCString enc fp $ GHC.peekCString enc + - This use of unsafePerformIO is belived to be safe; GHC's interface + - only allows doing this conversion with CStrings, and the CString buffer + - is allocated, used, and deallocated within the call, with no side + - effects. + -} +{-# NOINLINE encodeFilePath #-} +encodeFilePath :: FilePath -> String +encodeFilePath fp = unsafePerformIO $ do + enc <- Encoding.getFileSystemEncoding + GHC.withCString enc fp $ GHC.peekCString Encoding.char8 diff --git a/Utility/StatFS.hsc b/Utility/StatFS.hsc index 58d0b3e02..ed4c9f1cb 100644 --- a/Utility/StatFS.hsc +++ b/Utility/StatFS.hsc @@ -52,8 +52,6 @@ import Utility.FileSystemEncoding import Foreign import Foreign.C.Types import Foreign.C.String -import GHC.IO.Encoding (getFileSystemEncoding) -import GHC.Foreign as GHC #if defined (__FreeBSD__) || defined (__FreeBSD_kernel__) || defined (__APPLE__) # include diff --git a/debian/changelog b/debian/changelog index 97cf38348..a7eb55272 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,11 @@ +git-annex (3.20120231) UNRELEASED; urgency=low + + * Fix key directory hash calculation code to behave as it did before + version 3.20120227 when a key contains non-ascii characters (only + WORM backend is likely to have been affected). + + -- Joey Hess Fri, 09 Mar 2012 19:19:44 -0400 + git-annex (3.20120230) unstable; urgency=low * "here" can be used to refer to the current repository, -- cgit v1.2.3