upgrade: Handle upgrade to v6 when the repository already contains v6 unlocked files whose content is already present.

Closes https://github.com/datalad/datalad/issues/1020

The use of runWriter in scanUnlockedFiles broke due to this change;
it failed with blocked indefinitely in mvar, because the database write
handle was taken while linkFromAnnex needed to also write to it (to update
the inode cache). So, switched to using a separate runWriter for each call
to addAssociatedFileFast. A little less efficient, but not greatly; the
writes should all still be cached.

4 files changed, 43 insertions, 16 deletions

diff --git a/Annex/WorkTree.hs b/Annex/WorkTree.hs
index fe42ab726..d62e15aee 100644
--- a/Annex/WorkTree.hs
+++ b/Annex/WorkTree.hs
@@ -11,7 +11,10 @@ import Annex.Common
 import Annex.Link
 import Annex.CatFile
 import Annex.Version
+import Annex.Content
+import Annex.ReplaceFile
 import Config
+import Git.FilePath
 import qualified Git.Ref
 import qualified Git.Branch
 import qualified Git.LsTree
@@ -54,24 +57,37 @@ ifAnnexed file yes no = maybe no yes =<< lookupFile file
  - This is expensive, and so normally the associated files are updated
  - incrementally when changes are noticed. So, this only needs to be done
  - when initializing/upgrading a v6 mode repository.
+ -
+ - Also, the content for the unlocked file may already be present as
+ - an annex object. If so, make the unlocked file use that content.
  -}
 scanUnlockedFiles :: Annex ()
-scanUnlockedFiles = whenM (isJust <$> inRepo Git.Branch.current) $ 
-	Database.Keys.runWriter $ \h -> do
-		showSideAction "scanning for unlocked files"
-		liftIO $ Database.Keys.SQL.dropAllAssociatedFiles h
-		(l, cleanup) <- inRepo $ Git.LsTree.lsTree Git.Ref.headRef
-		forM_ l $ \i -> 
-			when (isregfile i) $
-				maybe noop (add h i)
-					=<< catKey (Git.LsTree.sha i)
-		liftIO $ void cleanup
+scanUnlockedFiles = whenM (isJust <$> inRepo Git.Branch.current) $ do
+	showSideAction "scanning for unlocked files"
+	Database.Keys.runWriter $
+		liftIO . Database.Keys.SQL.dropAllAssociatedFiles
+	(l, cleanup) <- inRepo $ Git.LsTree.lsTree Git.Ref.headRef
+	forM_ l $ \i -> 
+		when (isregfile i) $
+			maybe noop (add i)
+				=<< catKey (Git.LsTree.sha i)
+	liftIO $ void cleanup
   where
 	isregfile i = case Git.Types.toBlobType (Git.LsTree.mode i) of
 		Just Git.Types.FileBlob -> True
 		Just Git.Types.ExecutableBlob -> True
 		_ -> False
-	add h i k = liftIO $ Database.Keys.SQL.addAssociatedFileFast 
-		(toIKey k)
-		(Git.LsTree.file i)
-		h
+	add i k = do
+		let tf = Git.LsTree.file i
+		Database.Keys.runWriter $
+			liftIO . Database.Keys.SQL.addAssociatedFileFast (toIKey k) tf
+		whenM (inAnnex k) $ do
+			f <- fromRepo $ fromTopFilePath tf
+			destmode <- liftIO $ catchMaybeIO $ fileMode <$> getFileStatus f
+			replaceFile f $ \tmp -> do
+				r <- linkFromAnnex k tmp destmode
+				case r of
+					LinkAnnexOk -> return ()
+					LinkAnnexNoop -> return ()
+					LinkAnnexFailed -> liftIO $
+						writePointerFile tmp k destmode
diff --git a/CHANGELOG b/CHANGELOG
index 7dd7b8d2b..db291e15a 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -2,6 +2,8 @@ git-annex (6.20161013) UNRELEASED; urgency=medium
 
   * lock, smudge: Fix edge cases where data loss could occur in v6 mode
     when the keys database was not populated.
+  * upgrade: Handle upgrade to v6 when the repository already contains
+    v6 unlocked files whose content is already present.
 
  -- Joey Hess <id@joeyh.name>  Mon, 17 Oct 2016 12:46:54 -0400
 
diff --git a/Command/Unlock.hs b/Command/Unlock.hs
index dfd8350f1..0ac221c1f 100644
--- a/Command/Unlock.hs
+++ b/Command/Unlock.hs
@@ -56,7 +56,7 @@ performNew dest key = do
 				case r of
 					LinkAnnexOk -> return ()
 					LinkAnnexNoop -> return ()
-					_ -> error "unlock failed"
+					LinkAnnexFailed -> error "unlock failed"
 			, liftIO $ writePointerFile tmp key destmode
 			)
 	next $ cleanupNew dest key destmode
diff --git a/doc/bugs/remote_repository_must_be_version_6_as_well_to_have_copied_files_appear_correctly.mdwn b/doc/bugs/remote_repository_must_be_version_6_as_well_to_have_copied_files_appear_correctly.mdwn
index 08df3b13c..57a174455 100644
--- a/doc/bugs/remote_repository_must_be_version_6_as_well_to_have_copied_files_appear_correctly.mdwn
+++ b/doc/bugs/remote_repository_must_be_version_6_as_well_to_have_copied_files_appear_correctly.mdwn
@@ -35,4 +35,13 @@ $> ./v6-push.sh 6 2>&1 | grep '>>>'
 > 
 > I am not sure yet why the keys database lacked an entry for the file;
 > perhaps something to do with it being a v6 unlocked file in a v5
-> repository. --[[Joey]]
+> repository.
+> 
+> Ok.. Seems that cloning to a v5 repository, and then copying/getting
+> objects into it, and then upgrading to v6 reproduces the problem with the
+> keys database. The inode cache does not get populated for unlocked files
+> on upgrade. Also, unlocked files stay as pointer files even when their
+> content is present in annex/objects. Fixed the upgrade process to handle
+> this case.
+> 
+> [[fixed|done]]] --[[Joey]]