add ChunkMethod type and make Logs.Chunk use it, rather than assuming fixed size chunks (so eg, rolling hash chunks can be supported later)

If a newer git-annex starts logging something else in the chunk log, it
won't be used by this version, but it will be preserved when updating the
log.

3 files changed, 42 insertions, 21 deletions

diff --git a/Logs/Chunk.hs b/Logs/Chunk.hs
index 76da50947..a3e18efc1 100644
--- a/Logs/Chunk.hs
+++ b/Logs/Chunk.hs
@@ -15,7 +15,14 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module Logs.Chunk where
+module Logs.Chunk (
+	ChunkMethod(..),
+	ChunkSize,
+	ChunkCount,
+	chunksStored,
+	chunksRemoved,
+	getCurrentChunks,
+) where
 
 import Common.Annex
 import Logs
@@ -26,19 +33,19 @@ import Logs.Chunk.Pure
 import qualified Data.Map as M
 import Data.Time.Clock.POSIX
 
-chunksStored :: UUID -> Key -> ChunkSize -> ChunkCount -> Annex ()
-chunksStored u k chunksize chunkcount = do
+chunksStored :: UUID -> Key -> ChunkMethod -> ChunkCount -> Annex ()
+chunksStored u k chunkmethod chunkcount = do
 	ts <- liftIO getPOSIXTime
 	Annex.Branch.change (chunkLogFile k) $
-		showLog . changeMapLog ts (u, chunksize) chunkcount . parseLog
+		showLog . changeMapLog ts (u, chunkmethod) chunkcount . parseLog
 
-chunksRemoved :: UUID -> Key -> ChunkSize -> Annex ()
-chunksRemoved u k chunksize = chunksStored u k chunksize 0
+chunksRemoved :: UUID -> Key -> ChunkMethod -> Annex ()
+chunksRemoved u k chunkmethod = chunksStored u k chunkmethod 0
 
-getCurrentChunks :: UUID -> Key -> Annex [(ChunkSize, ChunkCount)]
+getCurrentChunks :: UUID -> Key -> Annex [(ChunkMethod, ChunkCount)]
 getCurrentChunks u k = select . parseLog <$> Annex.Branch.get (chunkLogFile k)
   where
-	select = filter (\(_sz, ct) -> ct > 0)
-		. map (\((_ku, sz), l) -> (sz, value l))
+	select = filter (\(_m, ct) -> ct > 0)
+		. map (\((_ku, m), l) -> (m, value l))
 		. M.toList
-		. M.filterWithKey (\(ku, _sz) _ -> ku == u)
+		. M.filterWithKey (\(ku, _m) _ -> ku == u)
diff --git a/Logs/Chunk/Pure.hs b/Logs/Chunk/Pure.hs
index 9bbfb868c..080a5a08b 100644
--- a/Logs/Chunk/Pure.hs
+++ b/Logs/Chunk/Pure.hs
@@ -6,7 +6,8 @@
  -}
 
 module Logs.Chunk.Pure 
-	( ChunkSize
+	( ChunkMethod(..)
+	, ChunkSize
 	, ChunkCount
 	, ChunkLog
 	, parseLog
@@ -17,24 +18,37 @@ import Common.Annex
 import Logs.MapLog
 import Data.Int
 
+-- Currently chunks are all fixed size, but other chunking methods
+-- may be added.
+data ChunkMethod = FixedSizeChunks ChunkSize | UnknownChunks String
+	deriving (Ord, Eq)
+
 type ChunkSize = Int64
 
+-- 0 when chunks are no longer present
 type ChunkCount = Integer
 
-type ChunkLog = MapLog (UUID, ChunkSize) ChunkCount
+type ChunkLog = MapLog (UUID, ChunkMethod) ChunkCount
+
+parseChunkMethod :: String -> ChunkMethod
+parseChunkMethod s = maybe (UnknownChunks s) FixedSizeChunks (readish s)
+
+showChunkMethod :: ChunkMethod -> String
+showChunkMethod (FixedSizeChunks sz) = show sz
+showChunkMethod (UnknownChunks s) = s
 
 parseLog :: String -> ChunkLog
 parseLog = parseMapLog fieldparser valueparser
   where
 	fieldparser s =
-		let (u,sz) = separate (== sep) s
-		in (,) <$> pure (toUUID u) <*> readish sz
+		let (u,m) = separate (== sep) s
+		in Just (toUUID u, parseChunkMethod m)
 	valueparser = readish
 
 showLog :: ChunkLog -> String
 showLog = showMapLog fieldshower valueshower
   where
-	fieldshower (u, sz) = fromUUID u ++ sep : show sz
+	fieldshower (u, m) = fromUUID u ++ sep : showChunkMethod m
 	valueshower = show
 
 sep :: Char
diff --git a/Remote/Helper/Chunked.hs b/Remote/Helper/Chunked.hs
index 9ba6d9cbd..11cd42c90 100644
--- a/Remote/Helper/Chunked.hs
+++ b/Remote/Helper/Chunked.hs
@@ -19,7 +19,6 @@ import Common.Annex
 import Utility.DataUnits
 import Types.Remote
 import Types.Key
-import Logs.Chunk.Pure (ChunkSize, ChunkCount)
 import Logs.Chunk
 import Utility.Metered
 import Crypto (EncKey)
@@ -108,7 +107,7 @@ storeChunks u chunkconfig k f p storer = metered (Just p) k $ \meterupdate ->
 			| L.null chunk && numchunks > 0 = do
  				-- Once all chunks are successfully
 				-- stored, update the chunk log.
-				chunksStored u k chunksize numchunks
+				chunksStored u k (FixedSizeChunks chunksize) numchunks
 				return True
 			| otherwise = do
 				let (chunkkey, chunkkeys') = nextChunkKeyStream chunkkeys
@@ -140,7 +139,7 @@ removeChunks remover u chunkconfig encryptor k = do
 	ok <- allM (remover . encryptor) (concat ls)
 	when ok $ do
 		let chunksizes = catMaybes $ map (keyChunkSize <=< headMaybe) ls
-		forM_ chunksizes $ chunksRemoved u k . fromIntegral
+		forM_ chunksizes $ chunksRemoved u k . FixedSizeChunks . fromIntegral
 	return ok
 
 {- Retrieves a key from a remote, using a retriever action that
@@ -313,6 +312,7 @@ chunkKeys u chunkconfig k = do
 chunkKeysOnly :: UUID -> Key -> Annex [[Key]]
 chunkKeysOnly u k = map (toChunkList k) <$> getCurrentChunks u k
 
-toChunkList :: Key -> (ChunkSize, ChunkCount) -> [Key]
-toChunkList k (chunksize, chunkcount) = takeChunkKeyStream chunkcount $
-	chunkKeyStream k chunksize
+toChunkList :: Key -> (ChunkMethod, ChunkCount) -> [Key]
+toChunkList k (FixedSizeChunks chunksize, chunkcount) =
+	takeChunkKeyStream chunkcount $ chunkKeyStream k chunksize
+toChunkList _ (UnknownChunks _, _) = []