diff options
22 files changed, 180 insertions, 44 deletions
diff --git a/Annex/Branch.hs b/Annex/Branch.hs index 5436132d8..ad96a2073 100644 --- a/Annex/Branch.hs +++ b/Annex/Branch.hs @@ -18,6 +18,7 @@ module Annex.Branch ( get, getHistorical, change, + maybeChange, commit, forceCommit, files, @@ -224,7 +225,15 @@ getRef ref file = withIndex $ decodeBS <$> catFile ref file - modifes the current content of the file on the branch. -} change :: FilePath -> (String -> String) -> Annex () -change file a = lockJournal $ \jl -> a <$> getLocal file >>= set jl file +change file f = lockJournal $ \jl -> f <$> getLocal file >>= set jl file + +{- Applies a function which can modify the content of a file, or not. -} +maybeChange :: FilePath -> (String -> Maybe String) -> Annex () +maybeChange file f = lockJournal $ \jl -> do + v <- getLocal file + case f v of + Just v' | v' /= v -> set jl file v' + _ -> noop {- Records new content of a file into the journal -} set :: JournalLocked -> FilePath -> String -> Annex () diff --git a/Annex/Init.hs b/Annex/Init.hs index 26502511e..03915904e 100644 --- a/Annex/Init.hs +++ b/Annex/Init.hs @@ -209,7 +209,7 @@ checkSharedClone = inRepo Git.Objects.isSharedClone initSharedClone :: Bool -> Annex () initSharedClone False = return () initSharedClone True = do - showSideAction "Repository was cloned with --shared; setting annex.hardlink=true and making repository untrusted." + showLongNote "Repository was cloned with --shared; setting annex.hardlink=true and making repository untrusted." u <- getUUID trustSet u UnTrusted setConfig (annexConfig "hardlink") (Git.Config.boolConfig True) diff --git a/Backend/Hash.hs b/Backend/Hash.hs index e6e5210a1..7f61c4f3e 100644 --- a/Backend/Hash.hs +++ b/Backend/Hash.hs @@ -116,7 +116,7 @@ checkKeyChecksum hash key file = go `catchHardwareFault` hwfault case (mstat, fast) of (Just stat, False) -> do filesize <- liftIO $ getFileSize' file stat - showSideAction "checksum" + showAction "checksum" check <$> hashFile hash file filesize _ -> return True expected = keyHash key diff --git a/Command/ImportFeed.hs b/Command/ImportFeed.hs index 647cd269e..2db6e608a 100644 --- a/Command/ImportFeed.hs +++ b/Command/ImportFeed.hs @@ -115,7 +115,7 @@ getCache :: Maybe String -> Annex Cache getCache opttemplate = ifM (Annex.getState Annex.force) ( ret S.empty S.empty , do - showSideAction "checking known urls" + showAction "checking known urls" (is, us) <- unzip <$> (mapM knownItems =<< knownUrls) ret (S.fromList us) (S.fromList (concat is)) ) diff --git a/Command/TestRemote.hs b/Command/TestRemote.hs index be1b9a324..f66a77ba7 100644 --- a/Command/TestRemote.hs +++ b/Command/TestRemote.hs @@ -63,7 +63,7 @@ start :: Int -> RemoteName -> CommandStart start basesz name = do showStart "testremote" name r <- either error id <$> Remote.byName' name - showSideAction "generating test keys" + showAction "generating test keys" fast <- Annex.getState Annex.fast ks <- mapM randKey (keySizes basesz fast) rs <- catMaybes <$> mapM (adjustChunkSize r) (chunkSizes basesz fast) diff --git a/Command/View.hs b/Command/View.hs index b39aef7d9..3fdbbb9c5 100644 --- a/Command/View.hs +++ b/Command/View.hs @@ -39,7 +39,7 @@ start ps = do perform :: View -> CommandPerform perform view = do - showSideAction "searching" + showAction "searching" next $ checkoutViewBranch view applyView paramView :: String diff --git a/Database/Handle.hs b/Database/Handle.hs index 1fd9f7834..439e7c18b 100644 --- a/Database/Handle.hs +++ b/Database/Handle.hs @@ -137,7 +137,8 @@ queryDb (DbHandle _ jobs _) a = do res <- newEmptyMVar putMVar jobs $ QueryJob $ liftIO . putMVar res =<< tryNonAsync a - either throwIO return =<< takeMVar res + (either throwIO return =<< takeMVar res) + `catchNonAsync` (const $ error "sqlite query crashed") closeDb :: DbHandle -> IO () closeDb h@(DbHandle worker jobs _) = do diff --git a/Logs/Location.hs b/Logs/Location.hs index ba9c31abf..89100805b 100644 --- a/Logs/Location.hs +++ b/Logs/Location.hs @@ -48,7 +48,7 @@ logChange = logChange' logNow logChange' :: (LogStatus -> String -> Annex LogLine) -> Key -> UUID -> LogStatus -> Annex () logChange' mklog key (UUID u) s = do config <- Annex.getGitConfig - addLog (locationLogFile config key) =<< mklog s u + maybeAddLog (locationLogFile config key) =<< mklog s u logChange' _ _ NoUUID _ = noop {- Returns a list of repository UUIDs that, according to the log, have diff --git a/Logs/Presence.hs b/Logs/Presence.hs index 60e0c542a..f90253421 100644 --- a/Logs/Presence.hs +++ b/Logs/Presence.hs @@ -4,7 +4,7 @@ - a way that can be union merged. - - A line of the log will look like: "date N INFO" - - Where N=1 when the INFO is present, and 0 otherwise. + - Where N=1 when the INFO is present, 0 otherwise. - - Copyright 2010-2014 Joey Hess <id@joeyh.name> - @@ -14,6 +14,7 @@ module Logs.Presence ( module X, addLog, + maybeAddLog, readLog, logNow, currentLog, @@ -28,10 +29,21 @@ import Common.Annex import qualified Annex.Branch import Git.Types (RefDate) +{- Adds a LogLine to the log, removing any LogLines that are obsoleted by + - adding it. -} addLog :: FilePath -> LogLine -> Annex () -addLog file line = Annex.Branch.change file $ \s -> +addLog file line = Annex.Branch.change file $ \s -> showLog $ compactLog (line : parseLog s) +{- When a LogLine already exists with the same status and info, but an + - older timestamp, that LogLine is preserved, rather than updating the log + - with a newer timestamp. + -} +maybeAddLog :: FilePath -> LogLine -> Annex () +maybeAddLog file line = Annex.Branch.maybeChange file $ \s -> do + m <- insertNewStatus line $ logMap $ parseLog s + return $ showLog $ mapLog m + {- Reads a log file. - Note that the LogLines returned may be in any order. -} readLog :: FilePath -> Annex [LogLine] diff --git a/Logs/Presence/Pure.hs b/Logs/Presence/Pure.hs index b1fc212fd..4e5ff68c0 100644 --- a/Logs/Presence/Pure.hs +++ b/Logs/Presence/Pure.hs @@ -61,21 +61,39 @@ filterPresent = filter (\l -> InfoPresent == status l) . compactLog {- Compacts a set of logs, returning a subset that contains the current - status. -} compactLog :: [LogLine] -> [LogLine] -compactLog = M.elems . foldr mapLog M.empty +compactLog = mapLog . logMap type LogMap = M.Map String LogLine -{- Inserts a log into a map of logs, if the log has better (ie, newer) - - information than the other logs in the map -} -mapLog :: LogLine -> LogMap -> LogMap -mapLog l m - | better = M.insert i l m - | otherwise = m +mapLog :: LogMap -> [LogLine] +mapLog = M.elems + +logMap :: [LogLine] -> LogMap +logMap = foldr insertNewerLogLine M.empty + +insertBetter :: (LogLine -> Bool) -> LogLine -> LogMap -> Maybe LogMap +insertBetter betterthan l m + | better = Just (M.insert i l m) + | otherwise = Nothing where - better = maybe True newer $ M.lookup i m - newer l' = date l' <= date l + better = maybe True betterthan (M.lookup i m) i = info l +{- Inserts a log into a map of logs, if the log has newer + - information than the other logs in the map for the same info. -} +insertNewerLogLine :: LogLine -> LogMap -> LogMap +insertNewerLogLine l m = fromMaybe m $ insertBetter newer l m + where + newer l' = date l' <= date l + +{- Inserts the log unless there's already one in the map with + - the same status for its info, in which case there's no need to + - change anything, to avoid log churn. -} +insertNewStatus :: LogLine -> LogMap -> Maybe LogMap +insertNewStatus l m = insertBetter diffstatus l m + where + diffstatus l' = status l' /= status l + instance Arbitrary LogLine where arbitrary = LogLine <$> arbitrary diff --git a/Remote/WebDAV.hs b/Remote/WebDAV.hs index 7f4173d03..4c5edef72 100644 --- a/Remote/WebDAV.hs +++ b/Remote/WebDAV.hs @@ -185,7 +185,7 @@ toDavPass = B8.fromString -} testDav :: URLString -> Maybe CredPair -> Annex () testDav url (Just (u, p)) = do - showSideAction "testing WebDAV server" + showAction "testing WebDAV server" test $ liftIO $ evalDAVT url $ do prepDAV user pass makeParentDirs diff --git a/Utility/FileMode.hs b/Utility/FileMode.hs index 83d53388e..b4a6b536d 100644 --- a/Utility/FileMode.hs +++ b/Utility/FileMode.hs @@ -8,30 +8,8 @@ {-# LANGUAGE CPP #-} module Utility.FileMode ( + module Utility.FileMode, FileMode, - modifyFileMode, - addModes, - removeModes, - writeModes, - readModes, - executeModes, - otherGroupModes, - preventWrite, - allowWrite, - allowRead, - groupSharedModes, - groupWriteRead, - checkMode, - isSymLink, - isExecutable, - noUmask, - withUmask, - combineModes, - isSticky, - stickyMode, - setSticky, - writeFileProtected, - writeFileProtected' ) where import System.IO diff --git a/debian/changelog b/debian/changelog index 250e183a6..1f336b9ec 100644 --- a/debian/changelog +++ b/debian/changelog @@ -30,11 +30,13 @@ git-annex (5.20150931) UNRELEASED; urgency=medium * copy --auto was checking the wrong repo's preferred content. (--from was checking what --to should, and vice-versa.) Fixed this bug, which was introduced in version 5.20150727. + * Avoid unncessary write to the location log when a file is unlocked + and then added back with unchanged content. * Debian: Add torrent library to build-depends as it's packaged now, and stop recommending bittornado | bittorrent. * Debian: Remove dependency on transformers library, as it is now included in ghc. - + -- Joey Hess <id@joeyh.name> Thu, 01 Oct 2015 12:42:56 -0400 git-annex (5.20150930) unstable; urgency=medium diff --git a/doc/bugs/Android:_Several_error_messages___47___warnings/comment_4_b38b675a2862d527fad28b7477f7ddde._comment b/doc/bugs/Android:_Several_error_messages___47___warnings/comment_4_b38b675a2862d527fad28b7477f7ddde._comment new file mode 100644 index 000000000..81e0c0fa0 --- /dev/null +++ b/doc/bugs/Android:_Several_error_messages___47___warnings/comment_4_b38b675a2862d527fad28b7477f7ddde._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 4""" + date="2015-10-12T17:15:30Z" + content=""" +The assistant will use whatever git remotes and git-annex configuration is +present in the repository when it's started up. +"""]] diff --git a/doc/bugs/SSL_repos_does_not_show_up_in_Assistant/comment_1_5a8e599d6e22e7c1960bd1715666657b._comment b/doc/bugs/SSL_repos_does_not_show_up_in_Assistant/comment_1_5a8e599d6e22e7c1960bd1715666657b._comment new file mode 100644 index 000000000..d394ca039 --- /dev/null +++ b/doc/bugs/SSL_repos_does_not_show_up_in_Assistant/comment_1_5a8e599d6e22e7c1960bd1715666657b._comment @@ -0,0 +1,17 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2015-10-12T18:52:42Z" + content=""" +You seem to be talking about using https to access a git repository. + +git-annex is not generally able to use https as a transport -- ie, it +can maybe download files from a git repository over https, if +the repository is set up just right. But it can't upload file to that +repository, nor can it promptly notice when changes are synced to that +repository from elsewhere. + +So, there's not much that the assistant can do with such a repository. + +You'll be better off using ssh for your git-annex repositories. +"""]] diff --git a/doc/bugs/STANDARD__95__IA_for_S3_remote_not_working/comment_2_1c153ecedc231f5cd4d6b2f081721494._comment b/doc/bugs/STANDARD__95__IA_for_S3_remote_not_working/comment_2_1c153ecedc231f5cd4d6b2f081721494._comment new file mode 100644 index 000000000..ca1d9a98f --- /dev/null +++ b/doc/bugs/STANDARD__95__IA_for_S3_remote_not_working/comment_2_1c153ecedc231f5cd4d6b2f081721494._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 2""" + date="2015-10-12T17:08:06Z" + content=""" +The feature is supported in git-annex, but it needs a version of the aws +library that has not seen a stable release yet. +"""]] diff --git a/doc/bugs/thread_blocked_indefinitely_in_an_MVar_operation_durin_fsck.mdwn b/doc/bugs/thread_blocked_indefinitely_in_an_MVar_operation_durin_fsck.mdwn index f30bc1d52..49156e366 100644 --- a/doc/bugs/thread_blocked_indefinitely_in_an_MVar_operation_durin_fsck.mdwn +++ b/doc/bugs/thread_blocked_indefinitely_in_an_MVar_operation_durin_fsck.mdwn @@ -33,3 +33,4 @@ It runs on Raspbian Testing ### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) Of course, The movie archive seems to be filling up nicely I just have some problems automating the checks. +> [[done]] diff --git a/doc/bugs/thread_blocked_indefinitely_in_an_MVar_operation_durin_fsck/comment_1_34df6b7f9b96351130e60a5952816cc7._comment b/doc/bugs/thread_blocked_indefinitely_in_an_MVar_operation_durin_fsck/comment_1_34df6b7f9b96351130e60a5952816cc7._comment new file mode 100644 index 000000000..7c9c7640a --- /dev/null +++ b/doc/bugs/thread_blocked_indefinitely_in_an_MVar_operation_durin_fsck/comment_1_34df6b7f9b96351130e60a5952816cc7._comment @@ -0,0 +1,20 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2015-10-12T16:49:11Z" + content=""" +Seems your guess is right: + + joey@darkstar:~/tmp/b4/b>sudo chown root.root -R .git/annex/fsck/4625a6de-d8f5-4036-83a5-6e202ad346da/ + joey@darkstar:~/tmp/b4/b>git annex fsck --incremental-schedule 182d + + sqlite worker thread crashed: SQLite3 returned ErrorCan'tOpen while attempting to perform prepare "SELECT null from fscked limit 1": unable to open database file + git-annex: thread blocked indefinitely in an MVar operation + +So, fix the owner/permissions (or delete .git/annex/fsck) and you should be good to go. + +Cleaned up the error message about the MVar, which is something of a red herring. + +BTW, I double-checked, and core.sharedRepository is honored to set the mode +of the fsck database file, so can be used to share a repo amoung multiple users. +"""]] diff --git a/doc/special_remotes/S3/comment_19_9d0d622b6202698f09eb06c097579fbb._comment b/doc/special_remotes/S3/comment_19_9d0d622b6202698f09eb06c097579fbb._comment new file mode 100644 index 000000000..e38def8ba --- /dev/null +++ b/doc/special_remotes/S3/comment_19_9d0d622b6202698f09eb06c097579fbb._comment @@ -0,0 +1,19 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 19""" + date="2015-10-12T17:10:21Z" + content=""" +@darkfeline I suppose you're talking about two completely disjoint git +repositories, and not two clones of the same parent repo. + +If you don't use fileprefix, and have the same file in two disjoint +repositories, git-annex will pick the same key for it in both cases, and +so you'll get deduplication, but only if you don't use different +fileprefixes. + +And this will mostly work pretty well. The danger is, if you drop the file +from the S3 repo, because say, it's not used anymore in one repository, +then you're also removing it from the S3 repo as used by the other +repository. If that was the last copy of the file, that may not be what you +want. +"""]] diff --git a/doc/todo/wishlist:_allow_re-adding_without_generating_log_entry.mdwn b/doc/todo/wishlist:_allow_re-adding_without_generating_log_entry.mdwn index d9ea181a8..258b82f6a 100644 --- a/doc/todo/wishlist:_allow_re-adding_without_generating_log_entry.mdwn +++ b/doc/todo/wishlist:_allow_re-adding_without_generating_log_entry.mdwn @@ -7,3 +7,5 @@ I keep a database file in git-annex so it stays synced across several machines. git annex sync --content This works fine except it creates an entry in the log for the database file even if that stays unchanged. I would like an option that just returns to the state before `git annex unlock` if the file has not been changed. Maybe `git annex add --restore-unchanged`? + +> [[done]] --[[Joey]] diff --git a/doc/todo/wishlist:_allow_re-adding_without_generating_log_entry/comment_1_ef1dbde0fbf20b7fd91503d9df50dcab._comment b/doc/todo/wishlist:_allow_re-adding_without_generating_log_entry/comment_1_ef1dbde0fbf20b7fd91503d9df50dcab._comment new file mode 100644 index 000000000..3c6df0477 --- /dev/null +++ b/doc/todo/wishlist:_allow_re-adding_without_generating_log_entry/comment_1_ef1dbde0fbf20b7fd91503d9df50dcab._comment @@ -0,0 +1,33 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2015-10-12T17:17:12Z" + content=""" +I want to think a little bit about why the location log is updated in this +case before thinking about adding an option. It might make sense to just +not update the location log when it already says the file is present and +only the timestamp is being changed. + +I can think of 2 reasons not to do it: + +1. If it has to query the current state of the log, that might slow down + `git annex add` in the common case, just for this less common case. + + But, updating the log necessarily involves reading it in and outputting + an updated one, so that could probably be finessed. + + (Or, git annex add makes a separate pass to add unlocked files anyway, + so it could only do the query in that case.) + +2. There might be good reasons to want to update the timestamp in the log, + since it's just verified that the content is still present. Maybe. + + But then, fsck doesn't update the timestamps when it does the same kind + of verification. And, the only thing that updates a given repo's entry + in the log is that repo, or another repo that is sending or dropping + content from that repo. + + There don't seem to be any reasons of + distributed consistency to need to worry about updating the timestamp + just to reflect current facts. +"""]] diff --git a/doc/todo/wishlist:_allow_re-adding_without_generating_log_entry/comment_2_3a3d793b32b8440a8213b38bc83ea94a._comment b/doc/todo/wishlist:_allow_re-adding_without_generating_log_entry/comment_2_3a3d793b32b8440a8213b38bc83ea94a._comment new file mode 100644 index 000000000..aeb98f06e --- /dev/null +++ b/doc/todo/wishlist:_allow_re-adding_without_generating_log_entry/comment_2_3a3d793b32b8440a8213b38bc83ea94a._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 2""" + date="2015-10-12T17:37:59Z" + content=""" +Ok, I found a way to implement it with no added overhead in the common +case! +"""]] |