diff options
author | Joey Hess <joeyh@joeyh.name> | 2015-12-21 18:41:15 -0400 |
---|---|---|
committer | Joey Hess <joeyh@joeyh.name> | 2015-12-21 18:41:15 -0400 |
commit | 0ac35d9b0ea997f9d6051d86889f4bb4df914498 (patch) | |
tree | 2a840c98bcf644f97c4092cb350a3171cd0365d4 | |
parent | 5c0d394c39f0cc08abad5de12d797a18661b6c9d (diff) |
wip v6 support for assistant
Files are not yet added to v6 repos in unlocked mode.
-rw-r--r-- | Annex/Content/Direct.hs | 4 | ||||
-rw-r--r-- | Annex/InodeSentinal.hs | 3 | ||||
-rw-r--r-- | Assistant/Threads/Committer.hs | 83 | ||||
-rw-r--r-- | Assistant/Threads/Watcher.hs | 62 | ||||
-rw-r--r-- | debian/changelog | 8 | ||||
-rw-r--r-- | doc/todo/smudge.mdwn | 2 |
6 files changed, 103 insertions, 59 deletions
diff --git a/Annex/Content/Direct.hs b/Annex/Content/Direct.hs index 59bea8f99..3b9d1aea2 100644 --- a/Annex/Content/Direct.hs +++ b/Annex/Content/Direct.hs @@ -21,7 +21,6 @@ module Annex.Content.Direct ( addInodeCache, writeInodeCache, compareInodeCaches, - compareInodeCachesWith, sameInodeCache, elemInodeCaches, sameFileStatus, @@ -172,9 +171,6 @@ sameFileStatus key f status = do ([], Nothing) -> return True _ -> return False -compareInodeCachesWith :: Annex InodeComparisonType -compareInodeCachesWith = ifM inodesChanged ( return Weakly, return Strongly ) - {- Copies the contentfile to the associated file, if the associated - file has no content. If the associated file does have content, - even if the content differs, it's left unchanged. -} diff --git a/Annex/InodeSentinal.hs b/Annex/InodeSentinal.hs index 8b48094df..412a7accc 100644 --- a/Annex/InodeSentinal.hs +++ b/Annex/InodeSentinal.hs @@ -24,6 +24,9 @@ compareInodeCaches x y , return False ) +compareInodeCachesWith :: Annex InodeComparisonType +compareInodeCachesWith = ifM inodesChanged ( return Weakly, return Strongly ) + {- Checks if one of the provided old InodeCache matches the current - version of a file. -} sameInodeCache :: FilePath -> [InodeCache] -> Annex Bool diff --git a/Assistant/Threads/Committer.hs b/Assistant/Threads/Committer.hs index 745047d9d..891df8419 100644 --- a/Assistant/Threads/Committer.hs +++ b/Assistant/Threads/Committer.hs @@ -31,9 +31,11 @@ import Annex.Content import Annex.Link import Annex.CatFile import Annex.InodeSentinal +import Annex.Version import qualified Annex import Utility.InodeCache import Annex.Content.Direct +import qualified Database.Keys import qualified Command.Sync import qualified Git.Branch @@ -228,12 +230,11 @@ commitStaged msg = do return ok {- OSX needs a short delay after a file is added before locking it down, - - when using a non-direct mode repository, as pasting a file seems to - - try to set file permissions or otherwise access the file after closing - - it. -} + - as pasting a file seems to try to set file permissions or otherwise + - access the file after closing it. -} delayaddDefault :: Annex (Maybe Seconds) #ifdef darwin_HOST_OS -delayaddDefault = ifM isDirect +delayaddDefault = ifM (isDirect || versionSupportsUnlockedPointers) ( return Nothing , return $ Just $ Seconds 1 ) @@ -250,12 +251,11 @@ delayaddDefault = return Nothing - for write by some other process, and faster checking with git-ls-files - that the files are not already checked into git. - - - When a file is added, Inotify will notice the new symlink. So this waits - - for additional Changes to arrive, so that the symlink has hopefully been - - staged before returning, and will be committed immediately. - - - - OTOH, for kqueue, eventsCoalesce, so instead the symlink is directly - - created and staged. + - When a file is added in locked mode, Inotify will notice the new symlink. + - So this waits for additional Changes to arrive, so that the symlink has + - hopefully been staged before returning, and will be committed immediately. + - (OTOH, for kqueue, eventsCoalesce, so instead the symlink is directly + - created and staged.) - - Returns a list of all changes that are ready to be committed. - Any pending adds that are not ready yet are put back into the ChangeChan, @@ -265,7 +265,8 @@ handleAdds :: Bool -> Maybe Seconds -> [Change] -> Assistant [Change] handleAdds havelsof delayadd cs = returnWhen (null incomplete) $ do let (pending, inprocess) = partition isPendingAddChange incomplete direct <- liftAnnex isDirect - (pending', cleanup) <- if direct + unlocked <- liftAnnex versionSupportsUnlockedPointers + (pending', cleanup) <- if unlocked || direct then return (pending, noop) else findnew pending (postponed, toadd) <- partitionEithers <$> safeToAdd havelsof delayadd pending' inprocess @@ -276,10 +277,11 @@ handleAdds havelsof delayadd cs = returnWhen (null incomplete) $ do returnWhen (null toadd) $ do added <- addaction toadd $ - catMaybes <$> if direct - then adddirect toadd - else forM toadd add - if DirWatcher.eventsCoalesce || null added || direct + catMaybes <$> + if unlocked || direct + then addunlocked direct toadd + else forM toadd add + if DirWatcher.eventsCoalesce || null added || unlocked || direct then return $ added ++ otherchanges else do r <- handleAdds havelsof delayadd =<< getChanges @@ -316,15 +318,15 @@ handleAdds havelsof delayadd cs = returnWhen (null incomplete) $ do maybe (failedingest change) (done change mcache $ keyFilename ks) mkey add _ = return Nothing - {- In direct mode, avoid overhead of re-injesting a renamed - - file, by examining the other Changes to see if a removed - - file has the same InodeCache as the new file. If so, - - we can just update bookkeeping, and stage the file in git. + {- Avoid overhead of re-injesting a renamed unlocked file, by + - examining the other Changes to see if a removed file has the + - same InodeCache as the new file. If so, we can just update + - bookkeeping, and stage the file in git. -} - adddirect :: [Change] -> Assistant [Maybe Change] - adddirect toadd = do + addunlocked :: Bool -> [Change] -> Assistant [Maybe Change] + addunlocked isdirect toadd = do ct <- liftAnnex compareInodeCachesWith - m <- liftAnnex $ removedKeysMap ct cs + m <- liftAnnex $ removedKeysMap isdirect ct cs delta <- liftAnnex getTSDelta if M.null m then forM toadd add @@ -335,22 +337,33 @@ handleAdds havelsof delayadd cs = returnWhen (null incomplete) $ do Just cache -> case M.lookup (inodeCacheToKey ct cache) m of Nothing -> add c - Just k -> fastadd c k + Just k -> if isdirect + then fastadddirect c k + else fastaddunlocked c k - fastadd :: Change -> Key -> Assistant (Maybe Change) - fastadd change key = do + fastadddirect :: Change -> Key -> Assistant (Maybe Change) + fastadddirect change key = do let source = keySource change liftAnnex $ Command.Add.finishIngestDirect key source done change Nothing (keyFilename source) key + + fastaddunlocked :: Change -> Key -> Assistant (Maybe Change) + fastaddunlocked change key = do + let source = keySource change + liftAnnex $ do + Database.Keys.addAssociatedFile key (keyFilename source) + done change Nothing (keyFilename source) key - removedKeysMap :: InodeComparisonType -> [Change] -> Annex (M.Map InodeCacheKey Key) - removedKeysMap ct l = do + removedKeysMap :: Bool -> InodeComparisonType -> [Change] -> Annex (M.Map InodeCacheKey Key) + removedKeysMap isdirect ct l = do mks <- forM (filter isRmChange l) $ \c -> catKeyFile $ changeFile c M.fromList . concat <$> mapM mkpairs (catMaybes mks) where mkpairs k = map (\c -> (inodeCacheToKey ct c, k)) <$> - recordedInodeCache k + if isdirect + then recordedInodeCache k + else Database.Keys.getInodeCaches k failedingest change = do refill [retryChange change] @@ -359,12 +372,16 @@ handleAdds havelsof delayadd cs = returnWhen (null incomplete) $ do done change mcache file key = liftAnnex $ do logStatus key InfoPresent - link <- ifM isDirect - ( calcRepo $ gitAnnexLink file key - , Command.Add.link file key mcache + ifM versionSupportsUnlockedPointers + ( stagePointerFile file =<< hashPointerFile key + , do + link <- ifM isDirect + ( calcRepo $ gitAnnexLink file key + , Command.Add.link file key mcache + ) + whenM (pure DirWatcher.eventsCoalesce <||> isDirect) $ + stageSymlink file =<< hashSymlink link ) - whenM (pure DirWatcher.eventsCoalesce <||> isDirect) $ - stageSymlink file =<< hashSymlink link showEndOk return $ Just $ finishedChange change key diff --git a/Assistant/Threads/Watcher.hs b/Assistant/Threads/Watcher.hs index 37e0154b4..bb9659b7c 100644 --- a/Assistant/Threads/Watcher.hs +++ b/Assistant/Threads/Watcher.hs @@ -1,6 +1,6 @@ {- git-annex assistant tree watcher - - - Copyright 2012-2013 Joey Hess <id@joeyh.name> + - Copyright 2012-2015 Joey Hess <id@joeyh.name> - - Licensed under the GNU GPL version 3 or higher. -} @@ -36,10 +36,15 @@ import Annex.CheckIgnore import Annex.Link import Annex.FileMatcher import Types.FileMatcher +import Annex.Content import Annex.ReplaceFile +import Annex.Version +import Annex.InodeSentinal import Git.Types import Config import Utility.ThreadScheduler +import Logs.Location +import qualified Database.Keys #ifndef mingw32_HOST_OS import qualified Utility.Lsof as Lsof #endif @@ -88,10 +93,13 @@ runWatcher = do startup <- asIO1 startupScan matcher <- liftAnnex largeFilesMatcher direct <- liftAnnex isDirect + unlocked <- liftAnnex versionSupportsUnlockedPointers symlinkssupported <- liftAnnex $ coreSymlinks <$> Annex.getGitConfig - addhook <- hook $ if direct - then onAddDirect symlinkssupported matcher - else onAdd matcher + addhook <- hook $ if unlocked + then onAddUnlocked symlinkssupported matcher + else if direct + then onAddDirect symlinkssupported matcher + else onAdd matcher delhook <- hook onDel addsymlinkhook <- hook $ onAddSymlink direct deldirhook <- hook onDelDir @@ -216,15 +224,33 @@ onAdd matcher file filestatus shouldRestage :: DaemonStatus -> Bool shouldRestage ds = scanComplete ds || forceRestage ds +onAddUnlocked :: Bool -> FileMatcher Annex -> Handler +onAddUnlocked = onAddUnlocked' False contentchanged Database.Keys.addAssociatedFile samefilestatus + where + samefilestatus key file status = do + cache <- Database.Keys.getInodeCaches key + curr <- withTSDelta $ \delta -> liftIO $ toInodeCache delta file status + case (cache, curr) of + (_, Just c) -> elemInodeCaches c cache + ([], Nothing) -> return True + _ -> return False + contentchanged oldkey file = do + Database.Keys.removeAssociatedFile oldkey file + unlessM (inAnnex oldkey) $ + logStatus oldkey InfoMissing + {- In direct mode, add events are received for both new files, and - modified existing files. -} onAddDirect :: Bool -> FileMatcher Annex -> Handler -onAddDirect symlinkssupported matcher file fs = do +onAddDirect = onAddUnlocked' True changedDirect (\k f -> void $ addAssociatedFile k f) sameFileStatus + +onAddUnlocked' :: Bool -> (Key -> FilePath -> Annex ()) -> (Key -> FilePath -> Annex ()) -> (Key -> FilePath -> FileStatus -> Annex Bool) -> Bool -> FileMatcher Annex -> Handler +onAddUnlocked' isdirect contentchanged addassociatedfile samefilestatus symlinkssupported matcher file fs = do v <- liftAnnex $ catKeyFile file case (v, fs) of (Just key, Just filestatus) -> - ifM (liftAnnex $ sameFileStatus key file filestatus) + ifM (liftAnnex $ samefilestatus key file filestatus) {- It's possible to get an add event for - an existing file that is not - really modified, but it might have @@ -237,13 +263,13 @@ onAddDirect symlinkssupported matcher file fs = do , noChange ) , guardSymlinkStandin (Just key) $ do - debug ["changed direct", file] - liftAnnex $ changedDirect key file + debug ["changed", file] + liftAnnex $ contentchanged key file add matcher file ) _ -> unlessIgnored file $ guardSymlinkStandin Nothing $ do - debug ["add direct", file] + debug ["add", file] add matcher file where {- On a filesystem without symlinks, we'll get changes for regular @@ -259,9 +285,9 @@ onAddDirect symlinkssupported matcher file fs = do Just lt -> do case fileKey $ takeFileName lt of Nothing -> noop - Just key -> void $ liftAnnex $ - addAssociatedFile key file - onAddSymlink' linktarget mk True file fs + Just key -> liftAnnex $ + addassociatedfile key file + onAddSymlink' linktarget mk isdirect file fs {- A symlink might be an arbitrary symlink, which is just added. - Or, if it is a git-annex symlink, ensure it points to the content @@ -330,13 +356,15 @@ onDel file _ = do onDel' :: FilePath -> Annex () onDel' file = do - whenM isDirect $ do - mkey <- catKeyFile file - case mkey of - Nothing -> noop - Just key -> void $ removeAssociatedFile key file + ifM versionSupportsUnlockedPointers + ( withkey $ flip Database.Keys.removeAssociatedFile file + , whenM isDirect $ + withkey $ \key -> void $ removeAssociatedFile key file + ) Annex.Queue.addUpdateIndex =<< inRepo (Git.UpdateIndex.unstageFile file) + where + withkey a = maybe noop a =<< catKeyFile file {- A directory has been deleted, or moved, so tell git to remove anything - that was inside it from its cache. Since it could reappear at any time, diff --git a/debian/changelog b/debian/changelog index 0488c2eb1..fb12926b6 100644 --- a/debian/changelog +++ b/debian/changelog @@ -4,16 +4,18 @@ git-annex (6.20151225) unstable; urgency=medium * The upgrade to version 6 is not done fully automatically, because upgrading a direct mode repository to version 6 will prevent old versions of git-annex from working in other clones of that repository. + * init: --version parameter added to control which supported repository + version to use. * smudge: New command, used for git smudge filter. This will replace direct mode. - * init: Configure .git/info/attributes to use git-annex as a smudge + * init, upgrade: Configure .git/info/attributes to use git-annex as a smudge filter. Note that this changes the default behavior of git add in a newly initialized repository; it will add files to the annex. * unlock, lock: In v6 mode, unlocking a file changes it from a symlink to a pointer file, and this change can be committed to the git repository. * add: In v6 mode, adds modified files to the annex. - * init: --version parameter added to control which supported repository - version to use. + * assistant: In v6 mode, adds files in unlocked mode, so they can + continue to be modified. TODO -- Joey Hess <id@joeyh.name> Tue, 08 Dec 2015 11:14:03 -0400 diff --git a/doc/todo/smudge.mdwn b/doc/todo/smudge.mdwn index 63f05c42b..fe938bfbe 100644 --- a/doc/todo/smudge.mdwn +++ b/doc/todo/smudge.mdwn @@ -323,8 +323,6 @@ files to be unlocked, while the indirect upgrades don't touch the files. * Still a few test suite failues for v6 with locked files. * Test suite should make pass for v6 with unlocked files. -* assistant: In v6 mode, adds files in unlocked mode, so they can - continue to be modified. TODO * When the webapp creates a repo, it forces it into direct mode. But that will fail when annex.version=6. Long-term, the assistant should make v6 repos, but short-term, the assistant should make v5 repos in direct mode. |