diff options
169 files changed, 2746 insertions, 465 deletions
diff --git a/Annex/CatFile.hs b/Annex/CatFile.hs index 8de0e536c..254dafddb 100644 --- a/Annex/CatFile.hs +++ b/Annex/CatFile.hs @@ -10,6 +10,7 @@ module Annex.CatFile ( catFileDetails, catObject, catTree, + catCommit, catObjectDetails, catFileHandle, catFileStop, @@ -52,6 +53,11 @@ catTree ref = do h <- catFileHandle liftIO $ Git.CatFile.catTree h ref +catCommit :: Git.Ref -> Annex (Maybe Commit) +catCommit ref = do + h <- catFileHandle + liftIO $ Git.CatFile.catCommit h ref + catObjectDetails :: Git.Ref -> Annex (Maybe (L.ByteString, Sha, ObjectType)) catObjectDetails ref = do h <- catFileHandle diff --git a/Annex/FileMatcher.hs b/Annex/FileMatcher.hs index e97b80c82..fa46e64b1 100644 --- a/Annex/FileMatcher.hs +++ b/Annex/FileMatcher.hs @@ -33,6 +33,7 @@ import Git.CheckAttr (unspecifiedAttr) #ifdef WITH_MAGICMIME import Magic +import Utility.Env #endif import Data.Either @@ -129,8 +130,15 @@ preferredContentParser matchstandard matchgroupwanted getgroupmap configmap mu e mkLargeFilesParser :: Annex (String -> [ParseResult]) mkLargeFilesParser = do #ifdef WITH_MAGICMIME - magicmime <- liftIO $ magicOpen [MagicMimeType] - liftIO $ magicLoadDefault magicmime + magicmime <- liftIO $ catchMaybeIO $ do + m <- magicOpen [MagicMimeType] + liftIO $ do + md <- getEnv "GIT_ANNEX_DIR" + case md of + Nothing -> magicLoadDefault m + Just d -> magicLoad m + (d </> "magic" </> "magic.mgc") + return m #endif let parse = parseToken $ commonTokens #ifdef WITH_MAGICMIME diff --git a/Annex/Index.hs b/Annex/Index.hs index ee27ce5f6..b3323ff3a 100644 --- a/Annex/Index.hs +++ b/Annex/Index.hs @@ -5,19 +5,14 @@ - Licensed under the GNU GPL version 3 or higher. -} -{-# LANGUAGE CPP #-} - -module Annex.Index ( - withIndexFile, - addGitEnv, -) where +module Annex.Index (withIndexFile) where import qualified Control.Exception as E import Annex.Common import Git.Types +import Git.Env import qualified Annex -import Utility.Env {- Runs an action using a different git index file. -} withIndexFile :: FilePath -> Annex a -> Annex a @@ -30,23 +25,3 @@ withIndexFile f a = do a Annex.changeState $ \s -> s { Annex.repo = (Annex.repo s) { gitEnv = gitEnv g} } either E.throw return r - -addGitEnv :: Repo -> String -> String -> IO Repo -addGitEnv g var val = do - e <- maybe copyenv return (gitEnv g) - let e' = addEntry var val e - return $ g { gitEnv = Just e' } - where - copyenv = do -#ifdef __ANDROID__ - {- This should not be necessary on Android, but there is some - - weird getEnvironment breakage. See - - https://github.com/neurocyte/ghc-android/issues/7 - - Use getEnv to get some key environment variables that - - git expects to have. -} - let keyenv = words "USER PATH GIT_EXEC_PATH HOSTNAME HOME" - let getEnvPair k = maybe Nothing (\v -> Just (k, v)) <$> getEnv k - liftIO $ catMaybes <$> forM keyenv getEnvPair -#else - liftIO getEnvironment -#endif diff --git a/Annex/Ingest.hs b/Annex/Ingest.hs index 68db3eef0..b80f0e1e0 100644 --- a/Annex/Ingest.hs +++ b/Annex/Ingest.hs @@ -1,6 +1,6 @@ {- git-annex content ingestion - - - Copyright 2010-2015 Joey Hess <id@joeyh.name> + - Copyright 2010-2016 Joey Hess <id@joeyh.name> - - Licensed under the GNU GPL version 3 or higher. -} @@ -9,14 +9,17 @@ module Annex.Ingest ( LockedDown(..), LockDownConfig(..), lockDown, + ingestAdd, ingest, finishIngestDirect, finishIngestUnlocked, cleanOldKeys, addLink, makeLink, + addUnlocked, restoreFile, forceParams, + addAnnexedFile, ) where import Annex.Common @@ -27,6 +30,7 @@ import Annex.Content.Direct import Annex.Perms import Annex.Link import Annex.MetaData +import Annex.Version import Logs.Location import qualified Annex import qualified Annex.Queue @@ -105,11 +109,30 @@ lockDown' cfg file = ifM (pure (not (hardlinkFileTmp cfg)) <||> crippledFileSyst , inodeCache = cache } -{- Ingests a locked down file into the annex. - - - - The file may be added to the git repository as a locked or an unlocked - - file. When unlocked, the work tree file is left alone. When locked, - - the work tree file is deleted, in preparation for adding the symlink. +{- Ingests a locked down file into the annex. Updates the work tree and + - index. -} +ingestAdd :: Maybe LockedDown -> Annex (Maybe Key) +ingestAdd Nothing = return Nothing +ingestAdd ld@(Just (LockedDown cfg source)) = do + (mk, mic) <- ingest ld + case mk of + Nothing -> return Nothing + Just k -> do + let f = keyFilename source + if lockingFile cfg + then do + liftIO $ nukeFile f + addLink f k mic + else ifM isDirect + ( do + l <- calcRepo $ gitAnnexLink f k + stageSymlink f =<< hashSymlink l + , stagePointerFile f =<< hashPointerFile k + ) + return (Just k) + +{- Ingests a locked down file into the annex. Does not update the working + - tree or the index. -} ingest :: Maybe LockedDown -> Annex (Maybe Key, Maybe InodeCache) ingest Nothing = return (Nothing, Nothing) @@ -135,7 +158,6 @@ ingest (Just (LockedDown cfg source)) = withTSDelta $ \delta -> do golocked key mcache s = do catchNonAsync (moveAnnex key $ contentLocation source) (restoreFile (keyFilename source) key) - liftIO $ nukeFile $ keyFilename source populateAssociatedFiles key source success key mcache s @@ -285,3 +307,50 @@ forceParams = ifM (Annex.getState Annex.force) ( return [Param "-f"] , return [] ) + +{- Whether a file should be added unlocked or not. Default is to not, + - unless symlinks are not supported. annex.addunlocked can override that. -} +addUnlocked :: Annex Bool +addUnlocked = isDirect <||> + (versionSupportsUnlockedPointers <&&> + ((not . coreSymlinks <$> Annex.getGitConfig) <||> + (annexAddUnlocked <$> Annex.getGitConfig) + ) + ) + +{- Adds a file to the work tree for the key, and stages it in the index. + - The content of the key may be provided in a temp file, which will be + - moved into place. -} +addAnnexedFile :: FilePath -> Key -> Maybe FilePath -> Annex () +addAnnexedFile file key mtmp = ifM (addUnlocked <&&> not <$> isDirect) + ( do + stagePointerFile file =<< hashPointerFile key + Database.Keys.addAssociatedFile key =<< inRepo (toTopFilePath file) + case mtmp of + Just tmp -> do + moveAnnex key tmp + linkunlocked + Nothing -> ifM (inAnnex key) + ( linkunlocked + , writepointer + ) + , do + addLink file key Nothing + whenM isDirect $ do + void $ addAssociatedFile key file + case mtmp of + Just tmp -> do + {- For moveAnnex to work in direct mode, the + - symlink must already exist, so flush the queue. -} + whenM isDirect $ + Annex.Queue.flush + moveAnnex key tmp + Nothing -> return () + ) + where + writepointer = liftIO $ writeFile file (formatPointer key) + linkunlocked = do + r <- linkFromAnnex key file + case r of + LinkAnnexFailed -> writepointer + _ -> return () diff --git a/Annex/Init.hs b/Annex/Init.hs index ece6e3a31..7501d9b8f 100644 --- a/Annex/Init.hs +++ b/Annex/Init.hs @@ -14,6 +14,7 @@ module Annex.Init ( initialize', uninitialize, probeCrippledFileSystem, + probeCrippledFileSystem', ) where import Annex.Common @@ -36,11 +37,11 @@ import Annex.Environment import Annex.Hook import Annex.InodeSentinal import Upgrade +import Annex.Perms import qualified Database.Keys #ifndef mingw32_HOST_OS import Utility.UserInfo import Utility.FileMode -import Annex.Perms import System.Posix.User import qualified Utility.LockFile.Posix as Posix #endif @@ -91,7 +92,7 @@ initialize' mversion = do whenM versionSupportsUnlockedPointers $ do configureSmudgeFilter Database.Keys.scanAssociatedFiles - ifM (crippledFileSystem <&&> not <$> isBare) + ifM (crippledFileSystem <&&> (not <$> isBare) <&&> (not <$> versionSupportsUnlockedPointers)) ( do enableDirectMode setDirect True @@ -134,16 +135,20 @@ isBare = fromRepo Git.repoIsLocalBare - or removing write access from files. -} probeCrippledFileSystem :: Annex Bool probeCrippledFileSystem = do + tmp <- fromRepo gitAnnexTmpMiscDir + createAnnexDirectory tmp + liftIO $ probeCrippledFileSystem' tmp + +probeCrippledFileSystem' :: FilePath -> IO Bool +probeCrippledFileSystem' tmp = do #ifdef mingw32_HOST_OS return True #else - tmp <- fromRepo gitAnnexTmpMiscDir let f = tmp </> "gaprobe" - createAnnexDirectory tmp - liftIO $ writeFile f "" - uncrippled <- liftIO $ probe f - void $ liftIO $ tryIO $ allowWrite f - liftIO $ removeFile f + writeFile f "" + uncrippled <- probe f + void $ tryIO $ allowWrite f + removeFile f return $ not uncrippled where probe f = catchBoolIO $ do diff --git a/Annex/Link.hs b/Annex/Link.hs index 7fd0098ef..40e56f23e 100644 --- a/Annex/Link.hs +++ b/Annex/Link.hs @@ -12,6 +12,8 @@ - Licensed under the GNU GPL version 3 or higher. -} +{-# LANGUAGE CPP, BangPatterns #-} + module Annex.Link where import Annex.Common @@ -23,6 +25,7 @@ import Git.Types import Git.FilePath import qualified Data.ByteString.Lazy as L +import Data.Int type LinkTarget = String @@ -130,28 +133,50 @@ stagePointerFile file sha = - Only looks at the first line, as pointer files can have subsequent - lines. -} parseLinkOrPointer :: L.ByteString -> Maybe Key -parseLinkOrPointer = parseLinkOrPointer' . decodeBS . L.take maxsz +parseLinkOrPointer = parseLinkOrPointer' . decodeBS . L.take maxPointerSz where - {- Want to avoid buffering really big files in git into - - memory when reading files that may be pointers. - - - - 8192 bytes is plenty for a pointer to a key. - - Pad some more to allow for any pointer files that might have - - lines after the key explaining what the file is used for. -} - maxsz = 81920 + +{- Want to avoid buffering really big files in git into + - memory when reading files that may be pointers. + - + - 8192 bytes is plenty for a pointer to a key. + - Pad some more to allow for any pointer files that might have + - lines after the key explaining what the file is used for. -} +maxPointerSz :: Int64 +maxPointerSz = 81920 parseLinkOrPointer' :: String -> Maybe Key -parseLinkOrPointer' s = headMaybe (lines (fromInternalGitPath s)) >>= go +parseLinkOrPointer' = go . fromInternalGitPath . takeWhile (not . lineend) where go l - | isLinkToAnnex l = file2key $ takeFileName l + | isLinkToAnnex l = fileKey $ takeFileName l | otherwise = Nothing + lineend '\n' = True + lineend '\r' = True + lineend _ = False formatPointer :: Key -> String formatPointer k = - toInternalGitPath (pathSeparator:objectDir </> key2file k) ++ "\n" + toInternalGitPath (pathSeparator:objectDir </> keyFile k) ++ "\n" {- Checks if a file is a pointer to a key. -} isPointerFile :: FilePath -> IO (Maybe Key) -isPointerFile f = catchDefaultIO Nothing $ - parseLinkOrPointer <$> L.readFile f +isPointerFile f = catchDefaultIO Nothing $ do + b <- L.take maxPointerSz <$> L.readFile f + let !mk = parseLinkOrPointer' (decodeBS b) + return mk + +{- Checks a symlink target or pointer file first line to see if it + - appears to point to annexed content. + - + - We only look for paths inside the .git directory, and not at the .git + - directory itself, because GIT_DIR may cause a directory name other + - than .git to be used. + -} +isLinkToAnnex :: FilePath -> Bool +isLinkToAnnex s = (pathSeparator:objectDir) `isInfixOf` s +#ifdef mingw32_HOST_OS + -- '/' is still used inside pointer files on Windows, not the native + -- '\' + || ('/':objectDir) `isInfixOf` s +#endif diff --git a/Annex/Locations.hs b/Annex/Locations.hs index 322165aee..4c2816fa9 100644 --- a/Annex/Locations.hs +++ b/Annex/Locations.hs @@ -63,7 +63,6 @@ module Annex.Locations ( gitAnnexSshDir, gitAnnexRemotesDir, gitAnnexAssistantDefaultDir, - isLinkToAnnex, HashLevels(..), hashDirMixed, hashDirLower, @@ -386,15 +385,6 @@ gitAnnexRemotesDir r = addTrailingPathSeparator $ gitAnnexDir r </> "remotes" gitAnnexAssistantDefaultDir :: FilePath gitAnnexAssistantDefaultDir = "annex" -{- Checks a symlink target to see if it appears to point to annexed content. - - - - We only look at paths inside the .git directory, and not at the .git - - directory itself, because GIT_DIR may cause a directory name other - - than .git to be used. - -} -isLinkToAnnex :: FilePath -> Bool -isLinkToAnnex s = (pathSeparator:objectDir) `isInfixOf` s - {- Sanitizes a String that will be used as part of a Key's keyName, - dealing with characters that cause problems on substandard filesystems. - diff --git a/Annex/LockPool/PosixOrPid.hs b/Annex/LockPool/PosixOrPid.hs index ecf96d51f..c788f6fa0 100644 --- a/Annex/LockPool/PosixOrPid.hs +++ b/Annex/LockPool/PosixOrPid.hs @@ -47,8 +47,14 @@ tryLockExclusive :: Maybe FileMode -> LockFile -> Annex (Maybe LockHandle) tryLockExclusive m f = tryPidLock m f $ Posix.tryLockExclusive m f checkLocked :: LockFile -> Annex (Maybe Bool) -checkLocked f = Posix.checkLocked f - `pidLockCheck` Pid.checkLocked +checkLocked f = Posix.checkLocked f `pidLockCheck` checkpid + where + checkpid pidlock = do + v <- Pid.checkLocked pidlock + case v of + -- Only return true when the posix lock file exists. + Just _ -> Posix.checkLocked f + Nothing -> return Nothing getLockStatus :: LockFile -> Annex LockStatus getLockStatus f = Posix.getLockStatus f @@ -88,6 +94,6 @@ tryPidLock m f posixlock = liftIO . go =<< pidLockFile -- The posix lock file is created even when using pid locks, in order to -- avoid complicating any code that might expect to be able to see that --- lock file. +-- lock file. But, it's not locked. dummyPosixLock :: Maybe FileMode -> LockFile -> IO () dummyPosixLock m f = closeFd =<< openLockFile ReadLock m f diff --git a/Annex/MetaData.hs b/Annex/MetaData.hs index 1f6a484ff..d8d8177c4 100644 --- a/Annex/MetaData.hs +++ b/Annex/MetaData.hs @@ -1,6 +1,6 @@ {- git-annex metadata - - - Copyright 2014 Joey Hess <id@joeyh.name> + - Copyright 2014-2016 Joey Hess <id@joeyh.name> - - Licensed under the GNU GPL version 3 or higher. -} @@ -8,6 +8,8 @@ module Annex.MetaData ( genMetaData, dateMetaData, + parseModMeta, + parseMetaDataMatcher, module X ) where @@ -17,6 +19,7 @@ import Types.MetaData as X import Annex.MetaData.StandardFields as X import Logs.MetaData import Annex.CatFile +import Utility.Glob import qualified Data.Set as S import qualified Data.Map as M @@ -53,3 +56,37 @@ dateMetaData mtime old = MetaData $ M.fromList $ filter isnew where isnew (f, _) = S.null (currentMetaDataValues f old) (y, m, _d) = toGregorian $ utctDay mtime + +{- Parses field=value, field+=value, field-=value, field?=value -} +parseModMeta :: String -> Either String ModMeta +parseModMeta p = case lastMaybe f of + Just '+' -> AddMeta <$> mkMetaField f' <*> v + Just '-' -> DelMeta <$> mkMetaField f' <*> (Just <$> v) + Just '?' -> MaybeSetMeta <$> mkMetaField f' <*> v + _ -> SetMeta <$> mkMetaField f <*> v + where + (f, sv) = separate (== '=') p + f' = beginning f + v = pure (toMetaValue sv) + +{- Parses field=value, field<value, field<=value, field>value, field>=value -} +parseMetaDataMatcher :: String -> Either String (MetaField, MetaValue -> Bool) +parseMetaDataMatcher p = (,) + <$> mkMetaField f + <*> pure matcher + where + (f, op_v) = break (`elem` "=<>") p + matcher = case op_v of + ('=':v) -> checkglob v + ('<':'=':v) -> checkcmp (<=) v + ('<':v) -> checkcmp (<) v + ('>':'=':v) -> checkcmp (>=) v + ('>':v) -> checkcmp (>) v + _ -> checkglob "" + checkglob v = + let cglob = compileGlob v CaseInsensative + in matchGlob cglob . fromMetaValue + checkcmp cmp v v' = case (doubleval v, doubleval (fromMetaValue v')) of + (Just d, Just d') -> d' `cmp` d + _ -> False + doubleval v = readish v :: Maybe Double diff --git a/Annex/Ssh.hs b/Annex/Ssh.hs index 5c6a9dcea..a97134c79 100644 --- a/Annex/Ssh.hs +++ b/Annex/Ssh.hs @@ -34,7 +34,7 @@ import Config import Annex.Path import Utility.Env import Types.CleanupActions -import Annex.Index (addGitEnv) +import Git.Env #ifndef mingw32_HOST_OS import Annex.Perms import Annex.LockPool diff --git a/Annex/Transfer.hs b/Annex/Transfer.hs index af9d89f4d..213de0648 100644 --- a/Annex/Transfer.hs +++ b/Annex/Transfer.hs @@ -105,10 +105,10 @@ runTransfer' ignorelock t file shouldretry transferobserver transferaction = do , return (Nothing, True) ) #else - prep tfile _mode info = catchPermissionDenied (const prepfailed) liftIO $ do + prep tfile _mode info = catchPermissionDenied (const prepfailed) $ do let lck = transferLockFile tfile createAnnexDirectory $ takeDirectory lck - v <- catchMaybeIO $ lockExclusive lck + v <- catchMaybeIO $ liftIO $ lockExclusive lck case v of Nothing -> return (Nothing, False) Just Nothing -> return (Nothing, True) diff --git a/Backend/Hash.hs b/Backend/Hash.hs index 363467a2a..6f42cac88 100644 --- a/Backend/Hash.hs +++ b/Backend/Hash.hs @@ -5,7 +5,7 @@ - Licensed under the GNU GPL version 3 or higher. -} -{-# LANGUAGE BangPatterns, CPP #-} +{-# LANGUAGE CPP #-} module Backend.Hash ( backends, @@ -171,8 +171,9 @@ hashFile hash file filesize = go hash go (SkeinHash hashsize) = use (skeinHasher hashsize) use hasher = liftIO $ do - !h <- hasher <$> L.readFile file - return h + h <- hasher <$> L.readFile file + -- Force full evaluation so file is read and closed. + return (length h `seq` h) usehasher hashsize = case shaHasher hashsize filesize of Left sha -> use sha diff --git a/Build/BundledPrograms.hs b/Build/BundledPrograms.hs index bd1c25359..4e6d2081e 100644 --- a/Build/BundledPrograms.hs +++ b/Build/BundledPrograms.hs @@ -17,7 +17,31 @@ import Build.SysConfig as SysConfig - - These may be just the command name, or the full path to it. -} bundledPrograms :: [FilePath] -bundledPrograms = catMaybes +bundledPrograms = preferredBundledPrograms ++ extraBundledPrograms + +{- Programs that are only included in the bundle in case the system + - doesn't have them. These come after the system PATH. + -} +extraBundledPrograms :: [FilePath] +extraBundledPrograms = catMaybes + -- The system gpg is probably better, because it may better + -- integrate with the system gpg-agent, etc. + -- On Windows, gpg is bundled with git for windows. +#ifndef mingw32_HOST_OS + [ SysConfig.gpg +#else + [ +#endif + ] + +{- Programs that should be preferred for use from the bundle, over + - any that might be installed on the system otherwise. These come before + - the system PATH. + - + - For example, git-annex is built for a specific version of git. + -} +preferredBundledPrograms :: [FilePath] +preferredBundledPrograms = catMaybes [ Nothing #ifndef mingw32_HOST_OS -- git is not included in the windows bundle; git for windows is used @@ -56,7 +80,6 @@ bundledPrograms = catMaybes #ifndef mingw32_HOST_OS -- All these utilities are included in git for Windows , ifset SysConfig.curl "curl" - , SysConfig.gpg , SysConfig.sha1 , SysConfig.sha256 , SysConfig.sha512 diff --git a/Build/Standalone.hs b/Build/Standalone.hs index 2a6c04e6d..a3a4bac48 100644 --- a/Build/Standalone.hs +++ b/Build/Standalone.hs @@ -26,6 +26,9 @@ progDir topdir = topdir progDir topdir = topdir </> "bin" #endif +extraProgDir :: FilePath -> FilePath +extraProgDir topdir = topdir </> "extra" + installProg :: FilePath -> FilePath -> IO (FilePath, FilePath) installProg dir prog = searchPath prog >>= go where @@ -41,7 +44,9 @@ main = getArgs >>= go where go [] = error "specify topdir" go (topdir:_) = do - let dir = progDir topdir - createDirectoryIfMissing True dir - installed <- forM bundledPrograms $ installProg dir - writeFile "tmp/standalone-installed" (show installed) + installed <- forM + [ (progDir topdir, preferredBundledPrograms) + , (extraProgDir topdir, extraBundledPrograms) ] $ \(dir, progs) -> do + createDirectoryIfMissing True dir + forM progs $ installProg dir + writeFile "tmp/standalone-installed" (show (concat installed)) diff --git a/Command/Add.hs b/Command/Add.hs index 7a1150d10..194e34de0 100644 --- a/Command/Add.hs +++ b/Command/Add.hs @@ -12,12 +12,10 @@ import Annex.Ingest import Logs.Location import Annex.Content import Annex.Content.Direct -import Annex.Link import qualified Annex import qualified Annex.Queue import qualified Database.Keys import Config -import Utility.InodeCache import Annex.FileMatcher import Annex.Version @@ -99,48 +97,42 @@ start file = ifAnnexed file addpresent add ( do ms <- liftIO $ catchMaybeIO $ getSymbolicLinkStatus file case ms of - Just s | isSymbolicLink s -> fixup key + Just s | isSymbolicLink s -> fixuplink key _ -> ifM (sameInodeCache file =<< Database.Keys.getInodeCaches key) ( stop, add ) , ifM isDirect ( do ms <- liftIO $ catchMaybeIO $ getSymbolicLinkStatus file case ms of - Just s | isSymbolicLink s -> fixup key + Just s | isSymbolicLink s -> fixuplink key _ -> ifM (goodContent key file) ( stop , add ) - , fixup key + , fixuplink key ) ) - fixup key = do + fixuplink key = do -- the annexed symlink is present but not yet added to git showStart "add" file liftIO $ removeFile file - whenM isDirect $ - void $ addAssociatedFile key file - next $ next $ cleanup file key Nothing =<< inAnnex key + next $ next $ do + addLink file key Nothing + cleanup key =<< inAnnex key perform :: FilePath -> CommandPerform perform file = do - lockingfile <- not <$> isDirect + lockingfile <- not <$> addUnlocked let cfg = LockDownConfig { lockingFile = lockingfile , hardlinkFileTmp = True } - lockDown cfg file >>= ingest >>= go + lockDown cfg file >>= ingestAdd >>= finish where - go (Just key, cache) = next $ cleanup file key cache True - go (Nothing, _) = stop + finish (Just key) = next $ cleanup key True + finish Nothing = stop -cleanup :: FilePath -> Key -> Maybe InodeCache -> Bool -> CommandCleanup -cleanup file key mcache hascontent = do +cleanup :: Key -> Bool -> CommandCleanup +cleanup key hascontent = do maybeShowJSON [("key", key2file key)] - ifM (isDirect <&&> pure hascontent) - ( do - l <- calcRepo $ gitAnnexLink file key - stageSymlink file =<< hashSymlink l - , addLink file key mcache - ) when hascontent $ logStatus key InfoPresent return True diff --git a/Command/AddUrl.hs b/Command/AddUrl.hs index 333ca494d..0370a2fc9 100644 --- a/Command/AddUrl.hs +++ b/Command/AddUrl.hs @@ -12,7 +12,6 @@ import Network.URI import Command import Backend import qualified Annex -import qualified Annex.Queue import qualified Annex.Url as Url import qualified Backend.URL import qualified Remote @@ -24,8 +23,6 @@ import Annex.UUID import Logs.Web import Types.KeySource import Types.UrlContents -import Config -import Annex.Content.Direct import Annex.FileMatcher import Logs.Location import Utility.Metered @@ -363,13 +360,7 @@ cleanup u url file key mtmp = case mtmp of when (isJust mtmp) $ logStatus key InfoPresent setUrlPresent u key url - addLink file key Nothing - whenM isDirect $ do - void $ addAssociatedFile key file - {- For moveAnnex to work in direct mode, the symlink - - must already exist, so flush the queue. -} - Annex.Queue.flush - maybe noop (moveAnnex key) mtmp + addAnnexedFile file key mtmp nodownload :: URLString -> Url.UrlInfo -> FilePath -> Annex (Maybe Key) nodownload url urlinfo file diff --git a/Command/Fsck.hs b/Command/Fsck.hs index 3b31b7211..2e7579b5b 100644 --- a/Command/Fsck.hs +++ b/Command/Fsck.hs @@ -409,7 +409,9 @@ checkBackendOr' bad backend key file postcheck = checkKeyNumCopies :: Key -> AssociatedFile -> NumCopies -> Annex Bool checkKeyNumCopies key afile numcopies = do let file = fromMaybe (key2file key) afile - (untrustedlocations, safelocations) <- trustPartition UnTrusted =<< Remote.keyLocations key + locs <- loggedLocations key + (untrustedlocations, otherlocations) <- trustPartition UnTrusted locs + (deadlocations, safelocations) <- trustPartition DeadTrusted otherlocations let present = NumCopies (length safelocations) if present < numcopies then ifM (pure (isNothing afile) <&&> checkDead key) @@ -417,29 +419,35 @@ checkKeyNumCopies key afile numcopies = do showLongNote $ "This key is dead, skipping." return True , do - ppuuids <- Remote.prettyPrintUUIDs "untrusted" untrustedlocations - warning $ missingNote file present numcopies ppuuids + untrusted <- Remote.prettyPrintUUIDs "untrusted" untrustedlocations + dead <- Remote.prettyPrintUUIDs "dead" deadlocations + warning $ missingNote file present numcopies untrusted dead when (fromNumCopies present == 0 && isNothing afile) $ showLongNote "(Avoid this check by running: git annex dead --key )" return False ) else return True -missingNote :: String -> NumCopies -> NumCopies -> String -> String -missingNote file (NumCopies 0) _ [] = - "** No known copies exist of " ++ file -missingNote file (NumCopies 0) _ untrusted = +missingNote :: String -> NumCopies -> NumCopies -> String -> String -> String +missingNote file (NumCopies 0) _ [] dead = + "** No known copies exist of " ++ file ++ honorDead dead +missingNote file (NumCopies 0) _ untrusted dead = "Only these untrusted locations may have copies of " ++ file ++ "\n" ++ untrusted ++ - "Back it up to trusted locations with git-annex copy." -missingNote file present needed [] = + "Back it up to trusted locations with git-annex copy." ++ honorDead dead +missingNote file present needed [] _ = "Only " ++ show (fromNumCopies present) ++ " of " ++ show (fromNumCopies needed) ++ " trustworthy copies exist of " ++ file ++ "\nBack it up with git-annex copy." -missingNote file present needed untrusted = - missingNote file present needed [] ++ +missingNote file present needed untrusted dead = + missingNote file present needed [] dead ++ "\nThe following untrusted locations may also have copies: " ++ "\n" ++ untrusted + +honorDead :: String -> String +honorDead dead + | null dead = "" + | otherwise = "\nThese dead repositories used to have copies\n" ++ dead {- Bad content is moved aside. -} badContent :: Key -> Annex String diff --git a/Command/Info.hs b/Command/Info.hs index ede61a369..05d6a01d3 100644 --- a/Command/Info.hs +++ b/Command/Info.hs @@ -22,6 +22,7 @@ import qualified Types.Remote as Remote import Utility.DataUnits import Utility.DiskFree import Annex.Content +import Annex.UUID import Logs.UUID import Logs.Trust import Logs.Location @@ -111,6 +112,9 @@ start o ps = do globalInfo :: InfoOptions -> Annex () globalInfo o = do + u <- getUUID + whenM ((==) DeadTrusted <$> lookupTrust u) $ + earlyWarning "Warning: This repository is currently marked as dead." stats <- selStats global_fast_stats global_slow_stats showCustom "info" $ do evalStateT (mapM_ showStat stats) (emptyStatInfo o) diff --git a/Command/MetaData.hs b/Command/MetaData.hs index db1179701..e2afccb9b 100644 --- a/Command/MetaData.hs +++ b/Command/MetaData.hs @@ -46,10 +46,14 @@ optParser desc = MetaDataOptions ( long "tag" <> short 't' <> metavar "TAG" <> help "set a tag" )) - <|> (AddMeta tagMetaField . mkMetaValue (CurrentlySet False) <$> strOption + <|> (DelMeta tagMetaField . Just . toMetaValue <$> strOption ( long "untag" <> short 'u' <> metavar "TAG" <> help "remove a tag" )) + <|> option (eitherReader (\f -> DelMeta <$> mkMetaField f <*> pure Nothing)) + ( long "remove" <> short 'r' <> metavar "FIELD" + <> help "remove all values of a field" + ) seek :: MetaDataOptions -> CommandSeek seek o = do diff --git a/Git/Branch.hs b/Git/Branch.hs index a2225dc73..ff209d44d 100644 --- a/Git/Branch.hs +++ b/Git/Branch.hs @@ -23,7 +23,7 @@ import qualified Git.BuildVersion - branch is not created yet. So, this also looks at show-ref HEAD - to double-check. -} -current :: Repo -> IO (Maybe Git.Ref) +current :: Repo -> IO (Maybe Branch) current r = do v <- currentUnsafe r case v of @@ -35,7 +35,7 @@ current r = do ) {- The current branch, which may not really exist yet. -} -currentUnsafe :: Repo -> IO (Maybe Git.Ref) +currentUnsafe :: Repo -> IO (Maybe Branch) currentUnsafe r = parse . firstLine <$> pipeReadStrict [Param "symbolic-ref", Param "-q", Param $ fromRef Git.Ref.headRef] r where @@ -144,26 +144,32 @@ commit commitmode allowempty message branch parentrefs repo = do pipeReadStrict [Param "write-tree"] repo ifM (cancommit tree) ( do - sha <- getSha "commit-tree" $ - pipeWriteRead ([Param "commit-tree", Param (fromRef tree)] ++ ps) sendmsg repo + sha <- commitTree commitmode message parentrefs tree repo update branch sha repo return $ Just sha , return Nothing ) where - ps = applyCommitMode commitmode $ - map Param $ concatMap (\r -> ["-p", fromRef r]) parentrefs cancommit tree | allowempty = return True | otherwise = case parentrefs of [p] -> maybe False (tree /=) <$> Git.Ref.tree p repo _ -> return True - sendmsg = Just $ flip hPutStr message commitAlways :: CommitMode -> String -> Branch -> [Ref] -> Repo -> IO Sha commitAlways commitmode message branch parentrefs repo = fromJust <$> commit commitmode True message branch parentrefs repo +commitTree :: CommitMode -> String -> [Ref] -> Ref -> Repo -> IO Sha +commitTree commitmode message parentrefs tree repo = + getSha "commit-tree" $ + pipeWriteRead ([Param "commit-tree", Param (fromRef tree)] ++ ps) + sendmsg repo + where + ps = applyCommitMode commitmode $ + map Param $ concatMap (\r -> ["-p", fromRef r]) parentrefs + sendmsg = Just $ flip hPutStr message + {- A leading + makes git-push force pushing a branch. -} forcePush :: String -> String forcePush b = "+" ++ b diff --git a/Git/CatFile.hs b/Git/CatFile.hs index d213bef06..455f192a0 100644 --- a/Git/CatFile.hs +++ b/Git/CatFile.hs @@ -1,6 +1,6 @@ {- git cat-file interface - - - Copyright 2011, 2013 Joey Hess <id@joeyh.name> + - Copyright 2011-2016 Joey Hess <id@joeyh.name> - - Licensed under the GNU GPL version 3 or higher. -} @@ -13,6 +13,7 @@ module Git.CatFile ( catFile, catFileDetails, catTree, + catCommit, catObject, catObjectDetails, ) where @@ -20,6 +21,10 @@ module Git.CatFile ( import System.IO import qualified Data.ByteString as S import qualified Data.ByteString.Lazy as L +import qualified Data.ByteString.Lazy.Char8 as L8 +import qualified Data.Map as M +import Data.String +import Data.Char import Data.Tuple.Utils import Numeric import System.Posix.Types @@ -110,3 +115,43 @@ catTree h treeref = go <$> catObjectDetails h treeref let (modestr, file) = separate (== ' ') (decodeBS b) in (file, readmode modestr) readmode = fromMaybe 0 . fmap fst . headMaybe . readOct + +catCommit :: CatFileHandle -> Ref -> IO (Maybe Commit) +catCommit h commitref = go <$> catObjectDetails h commitref + where + go (Just (b, _, CommitObject)) = parseCommit b + go _ = Nothing + +parseCommit :: L.ByteString -> Maybe Commit +parseCommit b = Commit + <$> (extractSha . L8.unpack =<< field "tree") + <*> (parsemetadata <$> field "author") + <*> (parsemetadata <$> field "committer") + <*> Just (L8.unpack $ L.intercalate (L.singleton nl) message) + where + field n = M.lookup (fromString n) fields + fields = M.fromList ((map breakfield) header) + breakfield l = + let (k, sp_v) = L.break (== sp) l + in (k, L.drop 1 sp_v) + (header, message) = separate L.null ls + ls = L.split nl b + + -- author and committer lines have the form: "name <email> date" + -- The email is always present, even if empty "<>" + parsemetadata l = CommitMetaData + { commitName = whenset $ L.init name_sp + , commitEmail = whenset email + , commitDate = whenset $ L.drop 2 gt_sp_date + } + where + (name_sp, rest) = L.break (== lt) l + (email, gt_sp_date) = L.break (== gt) (L.drop 1 rest) + whenset v + | L.null v = Nothing + | otherwise = Just (L8.unpack v) + + nl = fromIntegral (ord '\n') + sp = fromIntegral (ord ' ') + lt = fromIntegral (ord '<') + gt = fromIntegral (ord '>') diff --git a/Git/Env.hs b/Git/Env.hs new file mode 100644 index 000000000..b0070e9a1 --- /dev/null +++ b/Git/Env.hs @@ -0,0 +1,57 @@ +{- Adjusting the environment while running git commands. + - + - Copyright 2014-2016 Joey Hess <id@joeyh.name> + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP #-} + +module Git.Env where + +import Git +import Git.Types +import Utility.Env +#ifdef __ANDROID__ +import Common +#endif + +{- Adjusts the gitEnv of a Repo. Copies the system environment if the repo + - does not have any gitEnv yet. -} +adjustGitEnv :: Repo -> ([(String, String)] -> [(String, String)]) -> IO Repo +adjustGitEnv g adj = do + e <- maybe copyenv return (gitEnv g) + let e' = adj e + return $ g { gitEnv = Just e' } + where + copyenv = do +#ifdef __ANDROID__ + {- This should not be necessary on Android, but there is some + - weird getEnvironment breakage. See + - https://github.com/neurocyte/ghc-android/issues/7 + - Use getEnv to get some key environment variables that + - git expects to have. -} + let keyenv = words "USER PATH GIT_EXEC_PATH HOSTNAME HOME" + let getEnvPair k = maybe Nothing (\v -> Just (k, v)) <$> getEnv k + catMaybes <$> forM keyenv getEnvPair +#else + getEnvironment +#endif + +addGitEnv :: Repo -> String -> String -> IO Repo +addGitEnv g var val = adjustGitEnv g (addEntry var val) + +{- Use with any action that makes a commit to set metadata. -} +commitWithMetaData :: CommitMetaData -> CommitMetaData -> (Repo -> IO a) -> Repo -> IO a +commitWithMetaData authormetadata committermetadata a g = + a =<< adjustGitEnv g adj + where + adj = mkadj "AUTHOR" authormetadata + . mkadj "COMMITTER" committermetadata + mkadj p md = go "NAME" commitName + . go "EMAIL" commitEmail + . go "DATE" commitDate + where + go s getv = case getv md of + Nothing -> id + Just v -> addEntry ("GIT_" ++ p ++ "_" ++ s) v diff --git a/Git/FilePath.hs b/Git/FilePath.hs index 2085f287b..5af74c067 100644 --- a/Git/FilePath.hs +++ b/Git/FilePath.hs @@ -31,7 +31,7 @@ import qualified System.FilePath.Posix {- A FilePath, relative to the top of the git repository. -} newtype TopFilePath = TopFilePath { getTopFilePath :: FilePath } - deriving (Show) + deriving (Show, Eq) {- Path to a TopFilePath, within the provided git repo. -} fromTopFilePath :: TopFilePath -> Git.Repo -> FilePath diff --git a/Git/LsTree.hs b/Git/LsTree.hs index b1d9190d0..2060fa793 100644 --- a/Git/LsTree.hs +++ b/Git/LsTree.hs @@ -5,12 +5,15 @@ - Licensed under the GNU GPL version 3 or higher. -} +{-# LANGUAGE BangPatterns #-} + module Git.LsTree ( TreeItem(..), lsTree, + lsTree', lsTreeParams, lsTreeFiles, - parseLsTree + parseLsTree, ) where import Common @@ -33,8 +36,11 @@ data TreeItem = TreeItem {- Lists the complete contents of a tree, recursing into sub-trees, - with lazy output. -} lsTree :: Ref -> Repo -> IO ([TreeItem], IO Bool) -lsTree t repo = do - (l, cleanup) <- pipeNullSplit (lsTreeParams t []) repo +lsTree = lsTree' [] + +lsTree' :: [CommandParam] -> Ref -> Repo -> IO ([TreeItem], IO Bool) +lsTree' ps t repo = do + (l, cleanup) <- pipeNullSplit (lsTreeParams t ps) repo return (map parseLsTree l, cleanup) lsTreeParams :: Ref -> [CommandParam] -> [CommandParam] @@ -64,16 +70,18 @@ lsTreeFiles t fs repo = map parseLsTree <$> pipeNullSplitStrict ps repo - (The --long format is not currently supported.) -} parseLsTree :: String -> TreeItem parseLsTree l = TreeItem - { mode = fst $ Prelude.head $ readOct m + { mode = smode , typeobj = t , sha = Ref s - , file = asTopFilePath $ Git.Filename.decode f + , file = sfile } where -- l = <mode> SP <type> SP <sha> TAB <file> -- All fields are fixed, so we can pull them out of -- specific positions in the line. (m, past_m) = splitAt 7 l - (t, past_t) = splitAt 4 past_m - (s, past_s) = splitAt shaSize $ Prelude.tail past_t - f = Prelude.tail past_s + (!t, past_t) = splitAt 4 past_m + (!s, past_s) = splitAt shaSize $ Prelude.tail past_t + !f = Prelude.tail past_s + !smode = fst $ Prelude.head $ readOct m + !sfile = asTopFilePath $ Git.Filename.decode f diff --git a/Git/Tree.hs b/Git/Tree.hs new file mode 100644 index 000000000..5cc72ec8a --- /dev/null +++ b/Git/Tree.hs @@ -0,0 +1,180 @@ +{- git trees + - + - Copyright 2016 Joey Hess <id@joeyh.name> + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE BangPatterns #-} + +module Git.Tree ( + Tree(..), + TreeContent(..), + getTree, + recordTree, + TreeItem(..), + adjustTree, +) where + +import Common +import Git +import Git.FilePath +import Git.Types +import Git.Command +import Git.Sha +import qualified Git.LsTree as LsTree +import qualified Utility.CoProcess as CoProcess + +import Numeric +import System.Posix.Types +import Control.Monad.IO.Class + +newtype Tree = Tree [TreeContent] + deriving (Show) + +data TreeContent + = TreeBlob TopFilePath FileMode Sha + -- A subtree that is already recorded in git, with a known sha. + | RecordedSubTree TopFilePath Sha [TreeContent] + -- A subtree that has not yet been recorded in git. + | NewSubTree TopFilePath [TreeContent] + deriving (Show) + +{- Gets the Tree for a Ref. -} +getTree :: Ref -> Repo -> IO Tree +getTree r repo = do + (l, cleanup) <- lsTreeWithObjects r repo + let !t = either (\e -> error ("ls-tree parse error:" ++ e)) id + (extractTree l) + void cleanup + return t + +lsTreeWithObjects :: Ref -> Repo -> IO ([LsTree.TreeItem], IO Bool) +lsTreeWithObjects = LsTree.lsTree' [Param "-t"] + +newtype MkTreeHandle = MkTreeHandle CoProcess.CoProcessHandle + +withMkTreeHandle :: (MonadIO m, MonadMask m) => Repo -> (MkTreeHandle -> m a) -> m a +withMkTreeHandle repo a = bracketIO setup cleanup (a . MkTreeHandle) + where + setup = CoProcess.rawMode =<< gitCoProcessStart False ps repo + ps = [Param "mktree", Param "--batch", Param "-z"] + cleanup = CoProcess.stop + +{- Records a Tree in the Repo, returning its Sha. + - + - Efficiently handles subtrees, by only recording ones that have not + - already been recorded before. And even when many subtrees need to be + - recorded, it's done with a single call to git mktree, using its batch + - interface. + -} +recordTree :: Tree -> Repo -> IO Sha +recordTree t repo = withMkTreeHandle repo $ \h -> recordTree' h t + +recordTree' :: MkTreeHandle -> Tree -> IO Sha +recordTree' h (Tree l) = mkTree h =<< mapM (recordSubTree h) l + +{- Note that the returned RecordedSubTree does not have its [TreeContent] + - list populated. This is a memory optimisation, since the list is not + - used. -} +recordSubTree :: MkTreeHandle -> TreeContent -> IO TreeContent +recordSubTree h (NewSubTree d l) = do + sha <- mkTree h =<< mapM (recordSubTree h) l + return (RecordedSubTree d sha []) +recordSubTree _ alreadyrecorded = return alreadyrecorded + +mkTree :: MkTreeHandle -> [TreeContent] -> IO Sha +mkTree (MkTreeHandle cp) l = CoProcess.query cp send receive + where + send h = do + forM_ l $ \i -> hPutStr h $ case i of + TreeBlob f fm s -> mkTreeOutput fm BlobObject s f + RecordedSubTree f s _ -> mkTreeOutput 0o040000 TreeObject s f + NewSubTree _ _ -> error "recordSubTree internal error; unexpected NewSubTree" + hPutStr h "\NUL" -- signal end of tree to --batch + receive h = getSha "mktree" (hGetLine h) + +mkTreeOutput :: FileMode -> ObjectType -> Sha -> TopFilePath -> String +mkTreeOutput fm ot s f = concat + [ showOct fm "" + , " " + , show ot + , " " + , fromRef s + , "\t" + , takeFileName (getTopFilePath f) + , "\NUL" + ] + +data TreeItem = TreeItem TopFilePath FileMode Sha + deriving (Eq) + +{- Applies an adjustment to items in a tree. + - + - While less flexible than using getTree and recordTree, this avoids + - buffering the whole tree in memory. + -} +adjustTree :: (MonadIO m, MonadMask m) => (TreeItem -> m (Maybe TreeItem)) -> Ref -> Repo -> m Sha +adjustTree adjust r repo = withMkTreeHandle repo $ \h -> do + (l, cleanup) <- liftIO $ lsTreeWithObjects r repo + (l', _, _) <- go h False [] topTree l + sha <- liftIO $ mkTree h l' + void $ liftIO cleanup + return sha + where + go _ wasmodified c _ [] = return (c, wasmodified, []) + go h wasmodified c intree (i:is) + | intree i = + case readObjectType (LsTree.typeobj i) of + Just BlobObject -> do + let ti = TreeItem (LsTree.file i) (LsTree.mode i) (LsTree.sha i) + v <- adjust ti + case v of + Nothing -> go h True c intree is + Just ti'@(TreeItem f m s) -> + let !modified = wasmodified || ti' /= ti + blob = TreeBlob f m s + in go h modified (blob:c) intree is + Just TreeObject -> do + (sl, modified, is') <- go h False [] (subTree i) is + subtree <- if modified + then liftIO $ recordSubTree h $ NewSubTree (LsTree.file i) sl + else return $ RecordedSubTree (LsTree.file i) (LsTree.sha i) [] + let !modified' = modified || wasmodified + go h modified' (subtree : c) intree is' + _ -> error ("unexpected object type \"" ++ LsTree.typeobj i ++ "\"") + | otherwise = return (c, wasmodified, i:is) + +{- Assumes the list is ordered, with tree objects coming right before their + - contents. -} +extractTree :: [LsTree.TreeItem] -> Either String Tree +extractTree l = case go [] topTree l of + Right (t, []) -> Right (Tree t) + Right _ -> parseerr "unexpected tree form" + Left e -> parseerr e + where + go t _ [] = Right (t, []) + go t intree (i:is) + | intree i = + case readObjectType (LsTree.typeobj i) of + Just BlobObject -> + let b = TreeBlob (LsTree.file i) (LsTree.mode i) (LsTree.sha i) + in go (b:t) intree is + Just TreeObject -> case go [] (subTree i) is of + Right (subtree, is') -> + let st = RecordedSubTree (LsTree.file i) (LsTree.sha i) subtree + in go (st:t) intree is' + Left e -> Left e + _ -> parseerr ("unexpected object type \"" ++ LsTree.typeobj i ++ "\"") + | otherwise = Right (t, i:is) + parseerr = Left + +type InTree = LsTree.TreeItem -> Bool + +topTree :: InTree +topTree = notElem '/' . getTopFilePath . LsTree.file + +subTree :: LsTree.TreeItem -> InTree +subTree t = + let prefix = getTopFilePath (LsTree.file t) ++ "/" + in (\i -> prefix `isPrefixOf` getTopFilePath (LsTree.file i)) diff --git a/Git/Types.hs b/Git/Types.hs index bb91a1703..4fa49be5c 100644 --- a/Git/Types.hs +++ b/Git/Types.hs @@ -11,7 +11,6 @@ import Network.URI import qualified Data.Map as M import System.Posix.Types import Utility.SafeCommand -import Utility.URI () {- Support repositories on local disk, and repositories accessed via an URL. - @@ -98,3 +97,23 @@ toBlobType 0o100644 = Just FileBlob toBlobType 0o100755 = Just ExecutableBlob toBlobType 0o120000 = Just SymlinkBlob toBlobType _ = Nothing + +fromBlobType :: BlobType -> FileMode +fromBlobType FileBlob = 0o100644 +fromBlobType ExecutableBlob = 0o100755 +fromBlobType SymlinkBlob = 0o120000 + +data Commit = Commit + { commitTree :: Sha + , commitAuthorMetaData :: CommitMetaData + , commitCommitterMetaData :: CommitMetaData + , commitMessage :: String + } + deriving (Show) + +data CommitMetaData = CommitMetaData + { commitName :: Maybe String + , commitEmail :: Maybe String + , commitDate :: Maybe String -- In raw git form, "epoch -tzoffset" + } + deriving (Show) @@ -23,6 +23,7 @@ import Types.TrustLevel import Types.Group import Types.FileMatcher import Types.MetaData +import Annex.MetaData import Logs.MetaData import Logs.Group import Logs.Unused @@ -97,14 +98,15 @@ matchGlobFile glob = go go (MatchingInfo af _ _ _) = matchGlob cglob <$> getInfo af #ifdef WITH_MAGICMIME -matchMagic :: Magic -> MkLimit Annex -matchMagic magic glob = Right $ const go +matchMagic :: Maybe Magic -> MkLimit Annex +matchMagic (Just magic) glob = Right $ const go where cglob = compileGlob glob CaseSensative -- memoized go (MatchingKey _) = pure False go (MatchingFile fi) = liftIO $ catchBoolIO $ matchGlob cglob <$> magicFile magic (matchFile fi) go (MatchingInfo _ _ _ mimeval) = matchGlob cglob <$> getInfo mimeval +matchMagic Nothing _ = Left "unable to load magic database; \"mimetype\" cannot be used" #endif {- Adds a limit to skip files not believed to be present @@ -277,14 +279,12 @@ addMetaData :: String -> Annex () addMetaData = addLimit . limitMetaData limitMetaData :: MkLimit Annex -limitMetaData s = case parseMetaData s of +limitMetaData s = case parseMetaDataMatcher s of Left e -> Left e - Right (f, v) -> - let cglob = compileGlob (fromMetaValue v) CaseInsensative - in Right $ const $ checkKey (check f cglob) + Right (f, matching) -> Right $ const $ checkKey (check f matching) where - check f cglob k = not . S.null - . S.filter (matchGlob cglob . fromMetaValue) + check f matching k = not . S.null + . S.filter matching . metaDataValues f <$> getCurrentMetaData k addTimeLimit :: String -> Annex () @@ -147,6 +147,8 @@ linuxstandalone-nobuild: Build/Standalone Build/LinuxMkLibs install -d "$(LINUXSTANDALONE_DEST)/git-core" (cd "$(shell git --exec-path)" && tar c .) | (cd "$(LINUXSTANDALONE_DEST)"/git-core && tar x) install -d "$(LINUXSTANDALONE_DEST)/templates" + install -d "$(LINUXSTANDALONE_DEST)/magic" + cp /usr/share/file/magic.mgc "$(LINUXSTANDALONE_DEST)/magic" ./Build/LinuxMkLibs "$(LINUXSTANDALONE_DEST)" @@ -199,6 +201,12 @@ osxapp: Build/Standalone Build/OSXMkLibs (cd "$(shell git --exec-path)" && tar c .) | (cd "$(OSXAPP_BASE)" && tar x) install -d "$(OSXAPP_BASE)/templates" + install -d "$(OSXAPP_BASE)/magic" + if [ -e "$(OSX_MAGIC_FILE)" ]; then \ + cp "$(OSX_MAGIC_FILE)" "$(OSXAPP_BASE)/magic/magic.mgc"; \ + else \ + echo "** OSX_MAGIC_FILE not set; not including it" >&2; \ + fi # OSX looks in man dir nearby the bin $(MAKE) install-mans DESTDIR="$(OSXAPP_BASE)/.." SHAREDIR="" PREFIX="" @@ -258,10 +266,12 @@ fast: dist/cabalbuild @ln -sf dist/build/git-annex/git-annex git-annex @$(MAKE) tags >/dev/null 2>&1 & -dist/cabalbuild: git-annex.cabal +dist/cabalbuild: dist/caballog + grep 'ghc --make' dist/caballog | tail -n 1 > dist/cabalbuild + +dist/caballog: git-annex.cabal $(BUILDER) configure -f"-Production" -O0 --enable-executable-dynamic $(BUILDER) build -v2 --ghc-options="-O0 -j" | tee dist/caballog - grep 'ghc --make' dist/caballog | tail -n 1 > dist/cabalbuild # Hardcoded command line to make hdevtools start up and work. # You will need some memory. It's worth it. diff --git a/Messages/Concurrent.hs b/Messages/Concurrent.hs index ee81109f1..0e39114dd 100644 --- a/Messages/Concurrent.hs +++ b/Messages/Concurrent.hs @@ -11,16 +11,18 @@ module Messages.Concurrent where import Annex +import Types.Messages #ifdef WITH_CONCURRENTOUTPUT import Common -import Types.Messages import qualified System.Console.Concurrent as Console import qualified System.Console.Regions as Regions import Control.Concurrent.STM import qualified Data.Text as T +#ifndef mingw32_HOST_OS import GHC.IO.Encoding #endif +#endif {- Outputs a message in a concurrency safe way. - @@ -30,8 +32,8 @@ import GHC.IO.Encoding - instead. -} concurrentMessage :: OutputType -> Bool -> String -> Annex () -> Annex () -#ifdef WITH_CONCURRENTOUTPUT concurrentMessage o iserror msg fallback +#ifdef WITH_CONCURRENTOUTPUT | concurrentOutputEnabled o = go =<< consoleRegion <$> Annex.getState Annex.output #endif diff --git a/Messages/Progress.hs b/Messages/Progress.hs index 6bbf43f4c..1583d922b 100644 --- a/Messages/Progress.hs +++ b/Messages/Progress.hs @@ -45,8 +45,8 @@ metered combinemeterupdate key a = case keySize key of maybe noop (\m -> m n) combinemeterupdate liftIO $ clearMeter stdout meter return r -#if WITH_CONCURRENTOUTPUT go size o@(ConcurrentOutput {}) +#if WITH_CONCURRENTOUTPUT | concurrentOutputEnabled o = withProgressRegion $ \r -> do (progress, meter) <- mkmeter size a $ \n -> liftIO $ do @@ -71,6 +71,7 @@ import qualified Annex.WorkTree import qualified Annex.Link import qualified Annex.Init import qualified Annex.CatFile +import qualified Annex.Path import qualified Annex.View import qualified Annex.View.ViewedFile import qualified Logs.View @@ -102,22 +103,25 @@ import qualified Utility.Gpg optParser :: Parser TestOptions optParser = TestOptions - <$> suiteOptionParser ingredients (tests mempty) + <$> suiteOptionParser ingredients (tests False mempty) <*> switch ( long "keep-failures" <> help "preserve repositories on test failure" - ) + ) runner :: Maybe (TestOptions -> IO ()) -runner = Just $ \opts -> case tryIngredients ingredients (tastyOptionSet opts) (tests opts) of - Nothing -> error "No tests found!?" - Just act -> ifM act - ( exitSuccess - , do - putStrLn " (This could be due to a bug in git-annex, or an incompatability" - putStrLn " with utilities, such as git, installed on this system.)" - exitFailure - ) +runner = Just $ \opts -> do + ensuretmpdir + crippledfilesystem <- Annex.Init.probeCrippledFileSystem' tmpdir + case tryIngredients ingredients (tastyOptionSet opts) (tests crippledfilesystem opts) of + Nothing -> error "No tests found!?" + Just act -> ifM act + ( exitSuccess + , do + putStrLn " (This could be due to a bug in git-annex, or an incompatability" + putStrLn " with utilities, such as git, installed on this system.)" + exitFailure + ) ingredients :: [Ingredient] ingredients = @@ -125,19 +129,19 @@ ingredients = , rerunningTests [consoleTestReporter] ] -tests :: TestOptions -> TestTree -tests opts = testGroup "Tests" $ properties : +tests :: Bool -> TestOptions -> TestTree +tests crippledfilesystem opts = testGroup "Tests" $ properties : map (\(d, te) -> withTestMode te (unitTests d)) testmodes where - testmodes = - [ ("v6 unlocked", (testMode opts "6") { unlockedFiles = True }) - , ("v6 locked", testMode opts "6") - , ("v5", testMode opts "5") -#ifndef mingw32_HOST_OS - -- Windows will only use direct mode, so don't test twice. - , ("v5 direct", (testMode opts "5") { forceDirect = True }) -#endif + testmodes = catMaybes + [ Just ("v6 unlocked", (testMode opts "6") { unlockedFiles = True }) + , unlesscrippled ("v5", testMode opts "5") + , unlesscrippled ("v6 locked", testMode opts "6") + , Just ("v5 direct", (testMode opts "5") { forceDirect = True }) ] + unlesscrippled v + | crippledfilesystem = Nothing + | otherwise = Just v properties :: TestTree properties = localOption (QuickCheckTests 1000) $ testGroup "QuickCheck" @@ -323,12 +327,12 @@ test_import :: Assertion test_import = intmpclonerepo $ Utility.Tmp.withTmpDir "importtest" $ \importdir -> do (toimport1, importf1, imported1) <- mktoimport importdir "import1" git_annex "import" [toimport1] @? "import failed" - annexed_present_locked imported1 + annexed_present_imported imported1 checkdoesnotexist importf1 (toimport2, importf2, imported2) <- mktoimport importdir "import2" git_annex "import" [toimport2] @? "import of duplicate failed" - annexed_present_locked imported2 + annexed_present_imported imported2 checkdoesnotexist importf2 (toimport3, importf3, imported3) <- mktoimport importdir "import3" @@ -348,11 +352,11 @@ test_import = intmpclonerepo $ Utility.Tmp.withTmpDir "importtest" $ \importdir (toimport5, importf5, imported5) <- mktoimport importdir "import5" git_annex "import" ["--duplicate", toimport5] @? "import --duplicate failed" - annexed_present_locked imported5 + annexed_present_imported imported5 checkexists importf5 git_annex "drop" ["--force", imported1, imported2, imported5] @? "drop failed" - annexed_notpresent_locked imported2 + annexed_notpresent_imported imported2 (toimportdup, importfdup, importeddup) <- mktoimport importdir "importdup" git_annex "import" ["--clean-duplicates", toimportdup] @? "import of missing duplicate with --clean-duplicates failed" @@ -364,6 +368,14 @@ test_import = intmpclonerepo $ Utility.Tmp.withTmpDir "importtest" $ \importdir let importf = subdir </> "f" writeFile (importdir </> importf) (content importf) return (importdir </> subdir, importdir </> importf, importf) + annexed_present_imported f = ifM (annexeval Config.crippledFileSystem) + ( annexed_present_unlocked f + , annexed_present_locked f + ) + annexed_notpresent_imported f = ifM (annexeval Config.crippledFileSystem) + ( annexed_notpresent_unlocked f + , annexed_notpresent_locked f + ) test_reinject :: Assertion test_reinject = intmpclonerepoInDirect $ do @@ -373,8 +385,11 @@ test_reinject = intmpclonerepoInDirect $ do key <- Types.Key.key2file <$> getKey backendSHA1 tmp git_annex "reinject" [tmp, sha1annexedfile] @? "reinject failed" annexed_present sha1annexedfile - git_annex "fromkey" [key, sha1annexedfiledup] @? "fromkey failed for dup" - annexed_present_locked sha1annexedfiledup + -- fromkey can't be used on a crippled filesystem, since it makes a + -- symlink + unlessM (annexeval Config.crippledFileSystem) $ do + git_annex "fromkey" [key, sha1annexedfiledup] @? "fromkey failed for dup" + annexed_present_locked sha1annexedfiledup where tmp = "tmpfile" @@ -814,16 +829,20 @@ test_unused = intmpclonerepoInDirect $ do checkunused [] "after dropunused" not <$> git_annex "dropunused" ["--force", "10", "501"] @? "dropunused failed to fail on bogus numbers" - -- unused used to miss renamed symlinks that were not staged - -- and pointed at annexed content, and think that content was unused - writeFile "unusedfile" "unusedcontent" - git_annex "add" ["unusedfile"] @? "add of unusedfile failed" - unusedfilekey <- getKey backendSHA256E "unusedfile" - renameFile "unusedfile" "unusedunstagedfile" - boolSystem "git" [Param "rm", Param "-qf", File "unusedfile"] @? "git rm failed" - checkunused [] "with unstaged link" - removeFile "unusedunstagedfile" - checkunused [unusedfilekey] "with renamed link deleted" + -- Unused used to miss renamed symlinks that were not staged + -- and pointed at annexed content, and think that content was unused. + -- This is only relevant when using locked files; if the file is + -- unlocked, the work tree file has the content, and there's no way + -- to associate it with the key. + unlessM (unlockedFiles <$> getTestMode) $ do + writeFile "unusedfile" "unusedcontent" + git_annex "add" ["unusedfile"] @? "add of unusedfile failed" + unusedfilekey <- getKey backendSHA256E "unusedfile" + renameFile "unusedfile" "unusedunstagedfile" + boolSystem "git" [Param "rm", Param "-qf", File "unusedfile"] @? "git rm failed" + checkunused [] "with unstaged link" + removeFile "unusedunstagedfile" + checkunused [unusedfilekey] "with renamed link deleted" -- unused used to miss symlinks that were deleted or modified -- manually @@ -1269,7 +1288,7 @@ test_uncommitted_conflict_resolution = do - lost track of whether a file was a symlink. -} test_conflict_resolution_symlink_bit :: Assertion -test_conflict_resolution_symlink_bit = +test_conflict_resolution_symlink_bit = unlessM (unlockedFiles <$> getTestMode) $ withtmpclonerepo $ \r1 -> withtmpclonerepo $ \r2 -> withtmpclonerepo $ \r3 -> do @@ -1542,9 +1561,10 @@ test_add_subdirs = intmpclonerepo $ do {- Regression test for Windows bug where symlinks were not - calculated correctly for files in subdirs. -} - git_annex "sync" [] @? "sync failed" - l <- annexeval $ decodeBS <$> Annex.CatFile.catObject (Git.Types.Ref "HEAD:dir/foo") - "../.git/annex/" `isPrefixOf` l @? ("symlink from subdir to .git/annex is wrong: " ++ l) + unlessM (unlockedFiles <$> getTestMode) $ do + git_annex "sync" [] @? "sync failed" + l <- annexeval $ decodeBS <$> Annex.CatFile.catObject (Git.Types.Ref "HEAD:dir/foo") + "../.git/annex/" `isPrefixOf` l @? ("symlink from subdir to .git/annex is wrong: " ++ l) createDirectory "dir2" writeFile ("dir2" </> "foo") $ content annexedfile @@ -1577,7 +1597,8 @@ git_annex command params = do {- Runs git-annex and returns its output. -} git_annex_output :: String -> [String] -> IO String git_annex_output command params = do - got <- Utility.Process.readProcess "git-annex" (command:params) + pp <- Annex.Path.programPath + got <- Utility.Process.readProcess pp (command:params) -- Since the above is a separate process, code coverage stats are -- not gathered for things run in it. -- Run same command again, to get code coverage. @@ -1748,12 +1769,17 @@ cleanup' final dir = whenM (doesDirectoryExist dir) $ do removeDirectoryRecursive dir checklink :: FilePath -> Assertion -checklink f = do - s <- getSymbolicLinkStatus f +checklink f = -- in direct mode, it may be a symlink, or not, depending -- on whether the content is present. unlessM (annexeval Config.isDirect) $ - isSymbolicLink s @? f ++ " is not a symlink" + ifM (annexeval Config.crippledFileSystem) + ( (isJust <$> annexeval (Annex.Link.getAnnexLinkTarget f)) + @? f ++ " is not a (crippled) symlink" + , do + s <- getSymbolicLinkStatus f + isSymbolicLink s @? f ++ " is not a symlink" + ) checkregularfile :: FilePath -> Assertion checkregularfile f = do @@ -1855,8 +1881,10 @@ annexed_present f = ifM (unlockedFiles <$> getTestMode) ) annexed_present_locked :: FilePath -> Assertion -annexed_present_locked = runchecks - [checklink, checkcontent, checkunwritable, inlocationlog] +annexed_present_locked f = ifM (annexeval Config.crippledFileSystem) + ( runchecks [checklink, inlocationlog] f + , runchecks [checklink, checkcontent, checkunwritable, inlocationlog] f + ) annexed_present_unlocked :: FilePath -> Assertion annexed_present_unlocked = runchecks diff --git a/Types/GitConfig.hs b/Types/GitConfig.hs index fbc3484d0..b5e3d795e 100644 --- a/Types/GitConfig.hs +++ b/Types/GitConfig.hs @@ -69,6 +69,7 @@ data GitConfig = GitConfig , annexVerify :: Bool , annexPidLock :: Bool , annexPidLockTimeout :: Seconds + , annexAddUnlocked :: Bool , coreSymlinks :: Bool , coreSharedRepository :: SharedRepository , gcryptId :: Maybe String @@ -118,6 +119,7 @@ extractGitConfig r = GitConfig , annexPidLock = getbool (annex "pidlock") False , annexPidLockTimeout = Seconds $ fromMaybe 300 $ getmayberead (annex "pidlocktimeout") + , annexAddUnlocked = getbool (annex "addunlocked") False , coreSymlinks = getbool "core.symlinks" True , coreSharedRepository = getSharedRepository r , gcryptId = getmaybe "core.gcrypt-id" diff --git a/Types/MetaData.hs b/Types/MetaData.hs index cf2811b22..198fe5d7d 100644 --- a/Types/MetaData.hs +++ b/Types/MetaData.hs @@ -36,8 +36,6 @@ module Types.MetaData ( metaDataValues, ModMeta(..), modMeta, - parseModMeta, - parseMetaData, prop_metadata_sane, prop_metadata_serialize ) where @@ -221,9 +219,13 @@ metaDataValues f (MetaData m) = fromMaybe S.empty (M.lookup f m) {- Ways that existing metadata can be modified -} data ModMeta = AddMeta MetaField MetaValue - | DelMeta MetaField MetaValue - | SetMeta MetaField MetaValue -- removes any existing values - | MaybeSetMeta MetaField MetaValue -- when field has no existing value + | DelMeta MetaField (Maybe MetaValue) + -- ^ delete value of a field. With Just, only that specific value + -- is deleted; with Nothing, all current values are deleted. + | SetMeta MetaField MetaValue + -- ^ removes any existing values + | MaybeSetMeta MetaField MetaValue + -- ^ set when field has no existing value deriving (Show) {- Applies a ModMeta, generating the new MetaData. @@ -231,7 +233,10 @@ data ModMeta - values set in the input metadata. It only contains changed values. -} modMeta :: MetaData -> ModMeta -> MetaData modMeta _ (AddMeta f v) = updateMetaData f v emptyMetaData -modMeta _ (DelMeta f oldv) = updateMetaData f (unsetMetaValue oldv) emptyMetaData +modMeta _ (DelMeta f (Just oldv)) = + updateMetaData f (unsetMetaValue oldv) emptyMetaData +modMeta m (DelMeta f Nothing) = MetaData $ M.singleton f $ + S.fromList $ map unsetMetaValue $ S.toList $ currentMetaDataValues f m modMeta m (SetMeta f v) = updateMetaData f v $ foldr (updateMetaData f) emptyMetaData $ map unsetMetaValue $ S.toList $ currentMetaDataValues f m @@ -239,26 +244,6 @@ modMeta m (MaybeSetMeta f v) | S.null (currentMetaDataValues f m) = updateMetaData f v emptyMetaData | otherwise = emptyMetaData -{- Parses field=value, field+=value, field-=value, field?=value -} -parseModMeta :: String -> Either String ModMeta -parseModMeta p = case lastMaybe f of - Just '+' -> AddMeta <$> mkMetaField f' <*> v - Just '-' -> DelMeta <$> mkMetaField f' <*> v - Just '?' -> MaybeSetMeta <$> mkMetaField f' <*> v - _ -> SetMeta <$> mkMetaField f <*> v - where - (f, sv) = separate (== '=') p - f' = beginning f - v = pure (toMetaValue sv) - -{- Parses field=value -} -parseMetaData :: String -> Either String (MetaField, MetaValue) -parseMetaData p = (,) - <$> mkMetaField f - <*> pure (toMetaValue v) - where - (f, v) = separate (== '=') p - {- Avoid putting too many fields in the map; extremely large maps make - the seriaization test slow due to the sheer amount of data. - It's unlikely that more than 100 fields of metadata will be used. -} diff --git a/Upgrade/V1.hs b/Upgrade/V1.hs index 165d44912..90e1a62a0 100644 --- a/Upgrade/V1.hs +++ b/Upgrade/V1.hs @@ -13,6 +13,7 @@ import Data.Default import Annex.Common import Annex.Content +import Annex.Link import Logs.Presence import qualified Annex.Queue import qualified Git diff --git a/Utility/LockFile/PidLock.hs b/Utility/LockFile/PidLock.hs index 53eb5a54f..a21014c0e 100644 --- a/Utility/LockFile/PidLock.hs +++ b/Utility/LockFile/PidLock.hs @@ -201,7 +201,7 @@ checkInsaneLustre dest = do -- -- Uses a 1 second wait-loop. -- --- May wait untie timeout if the lock file is stale and is on a network file +-- May wait until timeout if the lock file is stale and is on a network file -- system, or on a system where the side lock cannot be taken. waitLock :: Seconds -> LockFile -> IO LockHandle waitLock (Seconds timeout) lockfile = go timeout diff --git a/Utility/LockPool.hs b/Utility/LockPool.hs index 2a4ac5101..7dbabb91a 100644 --- a/Utility/LockPool.hs +++ b/Utility/LockPool.hs @@ -7,15 +7,13 @@ - the lock will be released, despite the first thread still having the - lockfile open. - - - Or, if a process is already holding an exclusive lock on a file, an + - Or, if a process is already holding an exclusive lock on a file, and - re-opens it and tries to take another exclusive lock, it won't block - on the first lock. - - To avoid these problems, this implements a lock pool. This keeps track - - of which lock files are being used by the process, and avoids - - re-opening them. Instead, if a lockfile is in use by the current - - process, STM is used to handle further concurrent uses of that lock - - file. + - of which lock files are being used by the process, using STM to handle + - inter-process locking. - - Note that, like Utility.LockFile, this does *not* attempt to be a - portability shim; the native locking of the OS is used. diff --git a/Utility/LockPool/LockHandle.hs b/Utility/LockPool/LockHandle.hs index 68c979b5d..34446ff52 100644 --- a/Utility/LockPool/LockHandle.hs +++ b/Utility/LockPool/LockHandle.hs @@ -7,20 +7,24 @@ {-# LANGUAGE CPP #-} -module Utility.LockPool.LockHandle where +module Utility.LockPool.LockHandle ( + LockHandle, + FileLockOps(..), + dropLock, +#ifndef mingw32_HOST_OS + checkSaneLock, +#endif + makeLockHandle, + tryMakeLockHandle, +) where import qualified Utility.LockPool.STM as P -#ifndef mingw32_HOST_OS import Utility.LockPool.STM (LockFile) -#endif import Control.Concurrent.STM import Control.Exception -data LockHandle = LockHandle - { poolHandle :: P.LockHandle - , fileLockOps :: FileLockOps - } +data LockHandle = LockHandle P.LockHandle FileLockOps data FileLockOps = FileLockOps { fDropLock :: IO () @@ -30,7 +34,7 @@ data FileLockOps = FileLockOps } dropLock :: LockHandle -> IO () -dropLock h = P.releaseLock (poolHandle h) (fDropLock (fileLockOps h)) +dropLock (LockHandle ph _) = P.releaseLock ph #ifndef mingw32_HOST_OS checkSaneLock :: LockFile -> LockHandle -> IO Bool @@ -40,26 +44,30 @@ checkSaneLock lockfile (LockHandle _ flo) = fCheckSaneLock flo lockfile -- Take a lock, by first updating the lock pool, and then taking the file -- lock. If taking the file lock fails for any reason, take care to -- release the lock in the lock pool. -makeLockHandle :: STM P.LockHandle -> IO FileLockOps -> IO LockHandle -makeLockHandle pa fa = bracketOnError setup cleanup go +makeLockHandle :: P.LockPool -> LockFile -> (P.LockPool -> LockFile -> STM P.LockHandle) -> (LockFile -> IO FileLockOps) -> IO LockHandle +makeLockHandle pool file pa fa = bracketOnError setup cleanup go where - setup = atomically pa - cleanup ph = P.releaseLock ph (return ()) - go ph = do - fo <- fa - return $ LockHandle ph fo + setup = atomically (pa pool file) + cleanup ph = P.releaseLock ph + go ph = mkLockHandle pool file ph =<< fa file -tryMakeLockHandle :: STM (Maybe P.LockHandle) -> IO (Maybe FileLockOps) -> IO (Maybe LockHandle) -tryMakeLockHandle pa fa = bracketOnError setup cleanup go +tryMakeLockHandle :: P.LockPool -> LockFile -> (P.LockPool -> LockFile -> STM (Maybe P.LockHandle)) -> (LockFile -> IO (Maybe FileLockOps)) -> IO (Maybe LockHandle) +tryMakeLockHandle pool file pa fa = bracketOnError setup cleanup go where - setup = atomically pa + setup = atomically (pa pool file) cleanup Nothing = return () - cleanup (Just ph) = P.releaseLock ph (return ()) + cleanup (Just ph) = P.releaseLock ph go Nothing = return Nothing go (Just ph) = do - mfo <- fa + mfo <- fa file case mfo of Nothing -> do cleanup (Just ph) return Nothing - Just fo -> return $ Just $ LockHandle ph fo + Just fo -> Just <$> mkLockHandle pool file ph fo + +mkLockHandle :: P.LockPool -> LockFile -> P.LockHandle -> FileLockOps -> IO LockHandle +mkLockHandle pool file ph fo = do + atomically $ P.registerCloseLockFile pool file (fDropLock fo) + return $ LockHandle ph fo + diff --git a/Utility/LockPool/PidLock.hs b/Utility/LockPool/PidLock.hs index dca353fdf..26ed96f3c 100644 --- a/Utility/LockPool/PidLock.hs +++ b/Utility/LockPool/PidLock.hs @@ -32,15 +32,17 @@ import Prelude -- Takes a pid lock, blocking until the lock is available or the timeout. waitLock :: Seconds -> LockFile -> IO LockHandle -waitLock timeout file = makeLockHandle - (P.waitTakeLock P.lockPool file LockExclusive) - (mk <$> F.waitLock timeout file) +waitLock timeout file = makeLockHandle P.lockPool file + -- LockShared for STM lock, because a pid lock can be the top-level + -- lock with various other STM level locks gated behind it. + (\p f -> P.waitTakeLock p f LockShared) + (\f -> mk <$> F.waitLock timeout f) -- Tries to take a pid lock, but does not block. tryLock :: LockFile -> IO (Maybe LockHandle) -tryLock file = tryMakeLockHandle - (P.tryTakeLock P.lockPool file LockShared) - (fmap mk <$> F.tryLock file) +tryLock file = tryMakeLockHandle P.lockPool file + (\p f -> P.tryTakeLock p f LockShared) + (\f -> fmap mk <$> F.tryLock f) checkLocked :: LockFile -> IO (Maybe Bool) checkLocked file = P.getLockStatus P.lockPool file diff --git a/Utility/LockPool/Posix.hs b/Utility/LockPool/Posix.hs index a77ed8f01..2c0b7c78e 100644 --- a/Utility/LockPool/Posix.hs +++ b/Utility/LockPool/Posix.hs @@ -33,27 +33,27 @@ import Prelude -- Takes a shared lock, blocking until the lock is available. lockShared :: Maybe FileMode -> LockFile -> IO LockHandle -lockShared mode file = makeLockHandle - (P.waitTakeLock P.lockPool file LockShared) - (mk <$> F.lockShared mode file) +lockShared mode file = makeLockHandle P.lockPool file + (\p f -> P.waitTakeLock p f LockShared) + (\f -> mk <$> F.lockShared mode f) -- Takes an exclusive lock, blocking until the lock is available. lockExclusive :: Maybe FileMode -> LockFile -> IO LockHandle -lockExclusive mode file = makeLockHandle - (P.waitTakeLock P.lockPool file LockExclusive) - (mk <$> F.lockExclusive mode file) +lockExclusive mode file = makeLockHandle P.lockPool file + (\p f -> P.waitTakeLock p f LockExclusive) + (\f -> mk <$> F.lockExclusive mode f) -- Tries to take a shared lock, but does not block. tryLockShared :: Maybe FileMode -> LockFile -> IO (Maybe LockHandle) -tryLockShared mode file = tryMakeLockHandle - (P.tryTakeLock P.lockPool file LockShared) - (fmap mk <$> F.tryLockShared mode file) +tryLockShared mode file = tryMakeLockHandle P.lockPool file + (\p f -> P.tryTakeLock p f LockShared) + (\f -> fmap mk <$> F.tryLockShared mode f) -- Tries to take an exclusive lock, but does not block. tryLockExclusive :: Maybe FileMode -> LockFile -> IO (Maybe LockHandle) -tryLockExclusive mode file = tryMakeLockHandle - (P.tryTakeLock P.lockPool file LockExclusive) - (fmap mk <$> F.tryLockExclusive mode file) +tryLockExclusive mode file = tryMakeLockHandle P.lockPool file + (\p f -> P.tryTakeLock p f LockExclusive) + (\f -> fmap mk <$> F.tryLockExclusive mode f) -- Returns Nothing when the file doesn't exist, for cases where -- that is different from it not being locked. diff --git a/Utility/LockPool/STM.hs b/Utility/LockPool/STM.hs index 1dc30b09b..d1ee0dbaf 100644 --- a/Utility/LockPool/STM.hs +++ b/Utility/LockPool/STM.hs @@ -15,8 +15,12 @@ module Utility.LockPool.STM ( tryTakeLock, getLockStatus, releaseLock, + CloseLockFile, + registerCloseLockFile, ) where +import Utility.Monad + import System.IO.Unsafe (unsafePerformIO) import qualified Data.Map.Strict as M import Control.Concurrent.STM @@ -36,7 +40,9 @@ type LockHandle = TMVar (LockPool, LockFile) type LockCount = Integer -data LockStatus = LockStatus LockMode LockCount +data LockStatus = LockStatus LockMode LockCount CloseLockFile + +type CloseLockFile = IO () -- This TMVar is normally kept full. type LockPool = TMVar (M.Map LockFile LockStatus) @@ -59,11 +65,11 @@ waitTakeLock :: LockPool -> LockFile -> LockMode -> STM LockHandle waitTakeLock pool file mode = do m <- takeTMVar pool v <- case M.lookup file m of - Just (LockStatus mode' n) + Just (LockStatus mode' n closelockfile) | mode == LockShared && mode' == LockShared -> - return $ LockStatus mode (succ n) + return $ LockStatus mode (succ n) closelockfile | n > 0 -> retry -- wait for lock - _ -> return $ LockStatus mode 1 + _ -> return $ LockStatus mode 1 noop putTMVar pool (M.insert file v m) newTMVar (pool, file) @@ -74,6 +80,16 @@ tryTakeLock pool file mode = `orElse` return Nothing +-- Call after waitTakeLock or tryTakeLock, to register a CloseLockFile +-- action to run when releasing the lock. +registerCloseLockFile :: LockPool -> LockFile -> CloseLockFile -> STM () +registerCloseLockFile pool file closelockfile = do + m <- takeTMVar pool + putTMVar pool (M.update go file m) + where + go (LockStatus mode n closelockfile') = Just $ + LockStatus mode n (closelockfile' >> closelockfile) + -- Checks if a lock is being held. If it's held by the current process, -- runs the getdefault action; otherwise runs the checker action. -- @@ -87,7 +103,7 @@ getLockStatus pool file getdefault checker = do v <- atomically $ do m <- takeTMVar pool let threadlocked = case M.lookup file m of - Just (LockStatus _ n) | n > 0 -> True + Just (LockStatus _ n _) | n > 0 -> True _ -> False if threadlocked then do @@ -99,25 +115,24 @@ getLockStatus pool file getdefault checker = do Just restore -> bracket_ (return ()) restore checker -- Only runs action to close underlying lock file when this is the last --- user of the lock, and when the handle has not already been closed. +-- user of the lock, and when the lock has not already been closed. -- --- Note that the lock pool is left empty while the closelockfile action +-- Note that the lock pool is left empty while the CloseLockFile action -- is run, to avoid race with another thread trying to open the same lock -- file. -releaseLock :: LockHandle -> IO () -> IO () -releaseLock h closelockfile = go =<< atomically (tryTakeTMVar h) +releaseLock :: LockHandle -> IO () +releaseLock h = go =<< atomically (tryTakeTMVar h) where go (Just (pool, file)) = do - (m, unused) <- atomically $ do + (m, closelockfile) <- atomically $ do m <- takeTMVar pool return $ case M.lookup file m of - Just (LockStatus mode n) - | n == 1 -> (M.delete file m, True) + Just (LockStatus mode n closelockfile) + | n == 1 -> (M.delete file m, closelockfile) | otherwise -> - (M.insert file (LockStatus mode (pred n)) m, False) - Nothing -> (m, True) - when unused - closelockfile + (M.insert file (LockStatus mode (pred n) closelockfile) m, noop) + Nothing -> (m, noop) + closelockfile atomically $ putTMVar pool m -- The LockHandle was already closed. go Nothing = return () diff --git a/Utility/LockPool/Windows.hs b/Utility/LockPool/Windows.hs index 2641ac37d..0ca3c8116 100644 --- a/Utility/LockPool/Windows.hs +++ b/Utility/LockPool/Windows.hs @@ -22,9 +22,9 @@ import Utility.LockPool.STM (LockFile, LockMode(..)) {- Tries to lock a file with a shared lock, which allows other processes to - also lock it shared. Fails if the file is exclusively locked. -} lockShared :: LockFile -> IO (Maybe LockHandle) -lockShared file = tryMakeLockHandle - (P.tryTakeLock P.lockPool file LockShared) - (fmap mk <$> F.lockShared file) +lockShared file = tryMakeLockHandle P.lockPool file + (\p f -> P.tryTakeLock p f LockShared) + (\f -> fmap mk <$> F.lockShared f) {- Tries to take an exclusive lock on a file. Fails if another process has - a shared or exclusive lock. @@ -33,9 +33,9 @@ lockShared file = tryMakeLockHandle - read or write by any other process. So for advisory locking of a file's - content, a separate LockFile should be used. -} lockExclusive :: LockFile -> IO (Maybe LockHandle) -lockExclusive file = tryMakeLockHandle - (P.tryTakeLock P.lockPool file LockExclusive) - (fmap mk <$> F.lockExclusive file) +lockExclusive file = tryMakeLockHandle P.lockPool file + (\p f -> P.tryTakeLock f LockExclusive) + (\f -> fmap mk <$> F.lockExclusive f) {- If the initial lock fails, this is a BUSY wait, and does not - guarentee FIFO order of waiters. In other news, Windows is a POS. -} diff --git a/Utility/URI.hs b/Utility/URI.hs deleted file mode 100644 index e68fda58d..000000000 --- a/Utility/URI.hs +++ /dev/null @@ -1,18 +0,0 @@ -{- Network.URI - - - - Copyright 2014 Joey Hess <id@joeyh.name> - - - - License: BSD-2-clause - -} - -{-# LANGUAGE CPP #-} - -module Utility.URI where - --- Old versions of network lacked an Ord for URI -#if ! MIN_VERSION_network(2,4,0) -import Network.URI - -instance Ord URI where - a `compare` b = show a `compare` show b -#endif diff --git a/Utility/UserInfo.hs b/Utility/UserInfo.hs index 7e94cafa4..6f0c59d04 100644 --- a/Utility/UserInfo.hs +++ b/Utility/UserInfo.hs @@ -58,6 +58,6 @@ myVal envvars extract = go envvars #ifndef mingw32_HOST_OS go [] = extract <$> (getUserEntryForID =<< getEffectiveUserID) #else - go [] = error $ "environment not set: " ++ show envvars + go [] = extract <$> error ("environment not set: " ++ show envvars) #endif go (v:vs) = maybe (go vs) return =<< getEnv v diff --git a/debian/changelog b/debian/changelog index b93d7ba91..3ea3cc1af 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,4 +1,42 @@ -git-annex (6.20160212) UNRELEASED; urgency=medium +git-annex (6.20160230) UNRELEASED; urgency=medium + + * metadata: Added -r to remove all current values of a field. + * Fix data loss that can occur when annex.pidlock is set in a repository. + * Fix bug preventing moving files to/from a repository with annex.pidlock set. + * Fix shared lock file FD leak. + * Fix metadata hook behavior when multiple files are added at once. + Thanks, Klaus Ethgen. + + -- Joey Hess <id@joeyh.name> Mon, 29 Feb 2016 13:00:30 -0400 + +git-annex (6.20160229) unstable; urgency=medium + + * Update perlmagick build dependency. Closes: #789225 + * Fix memory leak in last release, which affected commands like + git-annex status when a large non-annexed file is present in the work + tree. + * fsck: When the only copy of a file is in a dead repository, mention + the repository. + * info: Mention when run in a dead repository. + * Linux and OSX standalone builds put the bundled gpg last in PATH, + so any system gpg will be preferred over it. + * Avoid crashing when built with MagicMime support, but when the magic + database cannot be loaded. + * Include magic database in the linux and OSX standalone builds. + * Fix memory leak when hashing files, which triggered during fsck + when an external hash program was not used. + (This leak was introduced in version 6.20160114.) + * Support --metadata field<number, --metadata field>number etc + to match ranges of numeric values. + * Similarly, support preferred content expressions like + metadata=field<number and metadata=field>number + * The pre-commit-annex hook script that automatically extracts + metadata has been updated to also use exiftool. + Thanks, Klaus Ethgen. + + -- Joey Hess <id@joeyh.name> Mon, 29 Feb 2016 12:41:49 -0400 + +git-annex (6.20160217) unstable; urgency=medium * Support getting files from read-only repositories. * checkpresentkey: Allow to be run without an explicit remote. @@ -11,8 +49,17 @@ git-annex (6.20160212) UNRELEASED; urgency=medium * fsck: Detect and fix missing associated file mappings in v6 repositories. * fsck: Populate unlocked files in v6 repositories whose content is present in annex/objects but didn't reach the work tree. - - -- Joey Hess <id@joeyh.name> Fri, 12 Feb 2016 14:03:46 -0400 + * When initializing a v6 repo on a crippled filesystem, don't force it + into direct mode. + * Windows: Fix v6 unlocked files to actually work. + * add, addurl, import, importfeed: When in a v6 repository on a crippled + filesystem, add files unlocked. + * annex.addunlocked: New configuration setting, makes files always be + added unlocked. (v6 only) + * Improve format of v6 unlocked pointer files to support keys containing + slashes. + + -- Joey Hess <id@joeyh.name> Wed, 17 Feb 2016 14:48:51 -0400 git-annex (6.20160211) unstable; urgency=medium diff --git a/debian/control b/debian/control index 3ebfbc2e4..25d6d9c92 100644 --- a/debian/control +++ b/debian/control @@ -75,7 +75,7 @@ Build-Depends: libghc-magic-dev, lsof [linux-any], ikiwiki, - perlmagick, + libimage-magick-perl, git (>= 1:1.8.1), rsync, wget, diff --git a/debian/copyright b/debian/copyright index 4f89d811b..6620962b6 100644 --- a/debian/copyright +++ b/debian/copyright @@ -40,6 +40,11 @@ Copyright: 2001 Ian Lynagh 2010-2015 Joey Hess <id@joeyh.name> License: GPL-3+ +Files: doc/tips/automatically_adding_metadata/pre-commit-annex +Copyright: 2014 Joey Hess <id@joeyh.name> + 2016 Klaus Ethgen <Klaus@Ethgen.ch> +License: GPL-3+ + Files: static/jquery* Copyright: © 2005-2011 by John Resig, Branden Aaron & Jörn Zaefferer © 2011 The Dojo Foundation diff --git a/doc/bugs/Hard_links_not_synced_in_direct_mode.mdwn b/doc/bugs/Hard_links_not_synced_in_direct_mode.mdwn index 2c8bd3588..6ede61860 100644 --- a/doc/bugs/Hard_links_not_synced_in_direct_mode.mdwn +++ b/doc/bugs/Hard_links_not_synced_in_direct_mode.mdwn @@ -1,3 +1,5 @@ +[[!meta title="hard links not synced"]] + ### Please describe the problem. Direct mode repositories seem to initially ignore hard linked files and then when changes are done to them sync them as separate files. However, changes to one file are only propagated to that file and not to any of the others that are hardlinked to it. diff --git a/doc/bugs/OSX:_addurl_--batch_--json_spits_out_shortened_output_string__dies_off_with_4.mdwn b/doc/bugs/OSX:_addurl_--batch_--json_spits_out_shortened_output_string__dies_off_with_4.mdwn new file mode 100644 index 000000000..5c72b0095 --- /dev/null +++ b/doc/bugs/OSX:_addurl_--batch_--json_spits_out_shortened_output_string__dies_off_with_4.mdwn @@ -0,0 +1,41 @@ +### Please describe the problem. + +See transcript. Didn't bisect fully but 6.20160128-g599a3ba seems to work ok, 6.20160211-g479a094 already dies off + +[[!format sh """ + +datalads-imac:~ yoh$ git annex version +git-annex version: 6.20160221-g3d1895e +... +datalads-imac:~ yoh$ mkdir -p /tmp/123; cd /tmp/123; git init; git annex init; +Initialized empty Git repository in /private/tmp/123/.git/ +init ok +(recording state in git...) + +datalads-imac:123 yoh$ rm -f /tmp/pipe; mkfifo /tmp/pipe; cat /tmp/pipe | git -c receive.autogc=false annex addurl -c annex.largefiles=exclude=*.txt --with-files --json --batch +{"command":"addurl","file":"1.png","note":"downloading http://www.onerussian.com/tmp/banner.png ..."error: git-annex died of signal 4 + +# now with --debug +datalads-imac:123 yoh$ rm -f /tmp/pipe; mkfifo /tmp/pipe; cat /tmp/pipe | git -c receive.autogc=false annex addurl -c annex.largefiles=exclude=*.txt --with-files --json --batch --debug +[2016-02-23 08:32:49.160686] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","-c","annex.largefiles=exclude=*.txt","show-ref","git-annex"] +[2016-02-23 08:32:49.167372] process done ExitSuccess +[2016-02-23 08:32:49.167479] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","-c","annex.largefiles=exclude=*.txt","show-ref","--hash","refs/heads/git-annex"] +[2016-02-23 08:32:49.172795] process done ExitSuccess +[2016-02-23 08:32:49.173039] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","-c","annex.largefiles=exclude=*.txt","log","refs/heads/git-annex..b2892e67682c5240dadb5d439da810edbdb882df","-n1","--pretty=%H"] +[2016-02-23 08:32:49.178586] process done ExitSuccess +[2016-02-23 08:32:49.179085] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","-c","annex.largefiles=exclude=*.txt","cat-file","--batch"] +[2016-02-23 08:32:49.183985] read: quvi ["--version"] +{"command":"addurl","file":"1.png","note":"downloading http://www.onerussian.com/tmp/banner.png ..."[2016-02-23 08:32:49.255949] call: curl ["-s","-f","-L","-C","-","-#","-o","/private/tmp/123/.git/annex/tmp/URL-s25319--http&c%%www.onerussian.com%tmp%banner.png","http://www.onerussian.com/tmp/banner.png","--user-agent","git-annex/6.20160221-g3d1895e"] +[2016-02-23 08:32:49.302577] process done ExitSuccess +[2016-02-23 08:32:49.303332] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","-c","annex.largefiles=exclude=*.txt","check-attr","-z","--stdin","annex.backend","annex.numcopies","annex.largefiles","--"] +[2016-02-23 08:32:49.305653] read: git ["--version"] +[2016-02-23 08:32:49.310314] process done ExitSuccess +error: git-annex died of signal 4 + +"""]] + +[[!meta author=yoh]] + +> [[done]]; this was indeed due to libmagic, and I've fixed it using brew +> --build-bottle to make a portable one. Then had to disable the build +> flag. --[[Joey]] diff --git a/doc/bugs/OSX:_addurl_--batch_--json_spits_out_shortened_output_string__dies_off_with_4/comment_1_36e75e72a1257db3cf8a48a025d42122._comment b/doc/bugs/OSX:_addurl_--batch_--json_spits_out_shortened_output_string__dies_off_with_4/comment_1_36e75e72a1257db3cf8a48a025d42122._comment new file mode 100644 index 000000000..774381d00 --- /dev/null +++ b/doc/bugs/OSX:_addurl_--batch_--json_spits_out_shortened_output_string__dies_off_with_4/comment_1_36e75e72a1257db3cf8a48a025d42122._comment @@ -0,0 +1,20 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2016-02-23T15:03:19Z" + content=""" +OSX signal 4 is SIGILL; git-annex test fails similarly, but so far I've only +had the problem be reported on this same datalads mac; I don't know how +widespread the problem is. + +Occurs to me it could well be libmagic that's dying. That would be +consistent with the point it fails, and IIRC with where git-annex test was +failing. Any use of annex.largefiles will cause the magic library to be +initialized. + +I've seen libraries taken from homebrew (as libmagic is) +be aggressively optimised for the host CPU and sigill on others. + +I've re-installed that using `brew install --build-bottle` now. +Hopefully that will fix the problem. +"""]] diff --git a/doc/bugs/Three_tests_fail_when_annex.backends_is_defined.mdwn b/doc/bugs/Three_tests_fail_when_annex.backends_is_defined.mdwn new file mode 100644 index 000000000..34096dbc9 --- /dev/null +++ b/doc/bugs/Three_tests_fail_when_annex.backends_is_defined.mdwn @@ -0,0 +1,107 @@ +### Please describe the problem. + +I noticed three tests failed when running "git annex test", and it seems +it's because I have `annex.backends` set to `SHA256` in `~/.gitconfig`. +When I disable that option, all tests succeed. + +### What steps will reproduce the problem? + +Add + + [annex] + backends = SHA256 + +to `~/.gitconfig`. + +### What version of git-annex are you using? On what operating system? + +Newest git-annex (6.20160211 amd64) from downloads.kitenet.net. + +- Debian GNU/Linux 8.3 (64 bit). Installed yesterday, so it's pretty pristine. +- git version 2.7.1.287.g4943984 (Newest version from 'master' in + git.git) + +The first version with this problem is 6.20160114. 5.20151218 works. + +### Please provide any additional information below. + +[[!format text """ +# If you can, paste a complete transcript of the problem occurring here. +# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log + +$ git annex test +Tests + +[Removed 188 lines] + + migrate (via gitattributes): /etc/magic, 4: Warning: using regular magic file `/usr/share/misc/magic' +OK (3.11s) + unused: /etc/magic, 4: Warning: using regular magic file `/usr/share/misc/magic' +FAIL (1.84s) + unused keys differ after origin branches are gone + expected: [Key {keyName = "e394a389d787383843decc5d3d99b6d184ffa5fddeec23b911f9ee7fc8b9ea77", keyBackendName = "SHA256E", keySize = Just 20, keyMtime = Nothing, keyChunkSize = Nothing, keyChunkNum = Nothing}] + but got: [Key {keyName = "e394a389d787383843decc5d3d99b6d184ffa5fddeec23b911f9ee7fc8b9ea77", keyBackendName = "SHA256", keySize = Just 20, keyMtime = Nothing, keyChunkSize = Nothing, keyChunkNum = Nothing}] + describe: /etc/magic, 4: Warning: using regular magic file `/usr/share/misc/magic' +OK (0.84s) + +[Removed 1108 lines] + + migrate (via gitattributes): /etc/magic, 4: Warning: using regular magic file `/usr/share/misc/magic' +OK (3.34s) + unused: /etc/magic, 4: Warning: using regular magic file `/usr/share/misc/magic' +/etc/magic, 4: Warning: using regular magic file `/usr/share/misc/magic' +/etc/magic, 4: Warning: using regular magic file `/usr/share/misc/magic' +FAIL (1.82s) + unused keys differ after origin branches are gone + expected: [Key {keyName = "e394a389d787383843decc5d3d99b6d184ffa5fddeec23b911f9ee7fc8b9ea77", keyBackendName = "SHA256E", keySize = Just 20, keyMtime = Nothing, keyChunkSize = Nothing, keyChunkNum = Nothing}] + but got: [Key {keyName = "e394a389d787383843decc5d3d99b6d184ffa5fddeec23b911f9ee7fc8b9ea77", keyBackendName = "SHA256", keySize = Just 20, keyMtime = Nothing, keyChunkSize = Nothing, keyChunkNum = Nothing}] + describe: /etc/magic, 4: Warning: using regular magic file `/usr/share/misc/magic' +OK (0.78s) + +[Removed 1062 lines] + + migrate (via gitattributes): OK (2.64s) + unused: FAIL (1.53s) + unused keys differ after origin branches are gone + expected: [Key {keyName = "e394a389d787383843decc5d3d99b6d184ffa5fddeec23b911f9ee7fc8b9ea77", keyBackendName = "SHA256E", keySize = Just 20, keyMtime = Nothing, keyChunkSize = Nothing, keyChunkNum = Nothing}] + but got: [Key {keyName = "e394a389d787383843decc5d3d99b6d184ffa5fddeec23b911f9ee7fc8b9ea77", keyBackendName = "SHA256", keySize = Just 20, keyMtime = Nothing, keyChunkSize = Nothing, keyChunkNum = Nothing}] + describe: OK (0.62s) + +[Removed 1667 lines] + +3 out of 269 tests failed (640.58s) + (This could be due to a bug in git-annex, or an incompatability + with utilities, such as git, installed on this system.) +$ + +# End of transcript or log. +"""]] + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) + +Roses are red<br/> +Violets are blue<br/> +git-annex is awesome<br/> +and so are you + +`;-)` + +But bloody hell, it's hard to get this thing to build. As you've +mentioned earlier, building on Debian 7 isn't supported anymore, so I +installed Debian 8.3 yesterday for the sole purpose of compiling git-annex, +but still no luck: + + [32 of 32] Compiling Main ( dist/setup/setup.hs, dist/setup/Main.o ) + Linking ./dist/setup/setup ... + unrecognized option `--extra-prog-path=/home/sunny/.cabal/bin' + Makefile:19: recipe for target 'Build/SysConfig.hs' failed + make: *** [Build/SysConfig.hs] Error 1 + +I was hoping to get it to build so I could find the exact commit that +introduced this, and also provide some patches with some functionality +I've been thinking of. Is building on Debian 8.3 (jessie) supported? I +found a bug report from you on +<https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=778987> that seems to +be related, but I don't know if that's the case. The newest version I'm +able to build is 5.20150219, somewhere after that various things start +to fail. diff --git a/doc/bugs/Three_tests_fail_when_annex.backends_is_defined/comment_1_153ba2b98b65fcc6156b1f2f967839b2._comment b/doc/bugs/Three_tests_fail_when_annex.backends_is_defined/comment_1_153ba2b98b65fcc6156b1f2f967839b2._comment new file mode 100644 index 000000000..194439668 --- /dev/null +++ b/doc/bugs/Three_tests_fail_when_annex.backends_is_defined/comment_1_153ba2b98b65fcc6156b1f2f967839b2._comment @@ -0,0 +1,23 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2016-02-17T16:16:53Z" + content=""" +The solution to your build failure is to edit ~/.cabal/config and remove +the extra-prog-path line. (It's unfortunate that Debian shipped 8.3 with +that bug. I told them about it before the release, but they didn't get it +fixed in time.) + +There are probably several other config settings that also break +the test suite if set in a global gitconfig. For example the +annex.addunlocked setting I added yesterday will. + +I think the best approach is to isolate the the test suite from global git +config settings. Making the test suite set `GIT_CONFIG` to the config of +each test repo it runs git-annex in seems like it would +work; this prevents git from reading the user and global configs. + +The test suite could set all config values to defaults +in the test git repos. But this would need some restructuring of the code to +be able to not just query, but also set those values. +"""]] diff --git a/doc/bugs/Three_tests_fail_when_annex.backends_is_defined/comment_2_20bd7de1902f7e3e4afdb8a8ff0ca1b7._comment b/doc/bugs/Three_tests_fail_when_annex.backends_is_defined/comment_2_20bd7de1902f7e3e4afdb8a8ff0ca1b7._comment new file mode 100644 index 000000000..917a88c1f --- /dev/null +++ b/doc/bugs/Three_tests_fail_when_annex.backends_is_defined/comment_2_20bd7de1902f7e3e4afdb8a8ff0ca1b7._comment @@ -0,0 +1,9 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 2""" + date="2016-02-17T19:19:18Z" + content=""" +Setting `GIT_CONFIG` won't work well, because sometimes git-annex in one +repo needs to look at the config of another repo, which would be prevented +by that setting. +"""]] diff --git a/doc/bugs/__34__Adding_4923_files__34___is_really_slow.mdwn b/doc/bugs/__34__Adding_4923_files__34___is_really_slow.mdwn index 5495662b9..6a0b5ced9 100644 --- a/doc/bugs/__34__Adding_4923_files__34___is_really_slow.mdwn +++ b/doc/bugs/__34__Adding_4923_files__34___is_really_slow.mdwn @@ -100,3 +100,4 @@ add music/Pop/Various/Like, Omigod! The 80s Pop Culture Box (totally)/._4-08 Tal [[!meta title="direct mode mappings scale badly with thousands of identical files"]] [[!tag confirmed]] +[[!meta tag=deprecateddirectmode]] diff --git a/doc/bugs/__39__add__39___results_in_max_cpu__44___long_run_and_huge_repo.mdwn b/doc/bugs/__39__add__39___results_in_max_cpu__44___long_run_and_huge_repo.mdwn new file mode 100644 index 000000000..cb3c7ef81 --- /dev/null +++ b/doc/bugs/__39__add__39___results_in_max_cpu__44___long_run_and_huge_repo.mdwn @@ -0,0 +1,40 @@ +### Please describe the problem. +massive repo, max cpu using + + git annex add . + +had to interrupt the job as it was processing 1 small file per 5 seconds after about 3h run. + +I am running it on the root of a large (currently 1TB) exFAT-based drive used for archiving + +The repo grew to 28G. + +Is this a regular issue with exFAT? I've done quite a bit of searching. I'll do more. + +### What steps will reproduce the problem? +- install on El Capitan (latest) via homebrew +- create 1TB exFAT file store +- follow walk through to setup annex locally and on external +- add + +### What version of git-annex are you using? On what operating system? +git-annex version: 6.20160126 +build flags: Assistant Webapp Pairing Testsuite S3(multipartupload)(storageclasses) WebDAV FsEvents XMPP ConcurrentOutput TorrentParser Feeds Quvi +key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 SHA1E SHA1 MD5E MD5 WORM URL +remote types: git gcrypt S3 bup directory rsync web bittorrent webdav tahoe glacier ddar hook external + +El Capitan 10.11.3 + + +### Please provide any additional information below. + +[[!format sh """ +# If you can, paste a complete transcript of the problem occurring here. +# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log + + +# End of transcript or log. +"""]] + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) +I'd love to say I have. You'll hear my shout of joy when I do. diff --git a/doc/bugs/__39__git_annex_get__39___fails_for_unlocked_files_with_special_characters___40__e.g._umlauts__41___when_using_precompiled_version_6.20160126-g2336107_/comment_3_205784e6385bc8cdd21af4773c57a6f3._comment b/doc/bugs/__39__git_annex_get__39___fails_for_unlocked_files_with_special_characters___40__e.g._umlauts__41___when_using_precompiled_version_6.20160126-g2336107_/comment_3_205784e6385bc8cdd21af4773c57a6f3._comment new file mode 100644 index 000000000..c78cce50e --- /dev/null +++ b/doc/bugs/__39__git_annex_get__39___fails_for_unlocked_files_with_special_characters___40__e.g._umlauts__41___when_using_precompiled_version_6.20160126-g2336107_/comment_3_205784e6385bc8cdd21af4773c57a6f3._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="christoph.schober@b8a66d38d9dfeeb6d7cd83888079d5870014ad7a" + nickname="christoph.schober" + subject="comment 3" + date="2016-02-18T21:47:39Z" + content=""" +Perfect, thank you! +"""]] diff --git a/doc/bugs/after_git_annex_add_and_commit__44___git_annex_fsck_fails:_no_known_copies.mdwn b/doc/bugs/after_git_annex_add_and_commit__44___git_annex_fsck_fails:_no_known_copies.mdwn index 9cc0c031c..0d7804a43 100644 --- a/doc/bugs/after_git_annex_add_and_commit__44___git_annex_fsck_fails:_no_known_copies.mdwn +++ b/doc/bugs/after_git_annex_add_and_commit__44___git_annex_fsck_fails:_no_known_copies.mdwn @@ -9,3 +9,8 @@ Unfortunately, I am not able to reproduce the problem on a toy-example repositor I use git v1.9.1 on Ubuntu 14.04 LTS, and git-annex version: 5.20150406-gb2814bc OK, in all honesty, I did a 'git annex sync' between the 'add' and the 'commit' above. But I synced with a clone of the repository that I keep on an external drive, which is less updated than my laptop. It is less updated because I always add content on my laptop and then sync/get from the external disk. So the sync did no harm, apparently. + +> Since this seems to be only a problem with messaging about accidentially +> marked dead repositories, I've made fsck mention when a file is only +> located in a dead repo, and I've made info tell when it's run in a +> supposedly-dead repo. [[done]] --[[Joey]] diff --git a/doc/bugs/after_git_annex_add_and_commit__44___git_annex_fsck_fails:_no_known_copies/comment_3_063e27ec1f2dd23fbf914a08213316df._comment b/doc/bugs/after_git_annex_add_and_commit__44___git_annex_fsck_fails:_no_known_copies/comment_3_063e27ec1f2dd23fbf914a08213316df._comment new file mode 100644 index 000000000..6146fad5c --- /dev/null +++ b/doc/bugs/after_git_annex_add_and_commit__44___git_annex_fsck_fails:_no_known_copies/comment_3_063e27ec1f2dd23fbf914a08213316df._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="emanuele.olivetti@47d88ed185b03191e25329caa6fabc2efb3118b2" + nickname="emanuele.olivetti" + subject="comment 3" + date="2016-02-15T09:47:24Z" + content=""" +Yes, the content is locally present. In the sense that each symlink in the current directory points to an actual file in .git/annex/objects/ . That is why, in my opinion, fsck incorrectly thinks the content is not present, despite it being present. + +Initially I thought that a possible explanation was that 'git commit' went wrong, so that I would have to do it again. I tried to git commit again, but git says there is nothing to commit. + +My repository is in indirect mode. All files in the repository, including the troubling ones are in .git/annex/objects. +"""]] diff --git a/doc/bugs/after_git_annex_add_and_commit__44___git_annex_fsck_fails:_no_known_copies/comment_4_8688477bb694dbc9e6c7768f5f375f3f._comment b/doc/bugs/after_git_annex_add_and_commit__44___git_annex_fsck_fails:_no_known_copies/comment_4_8688477bb694dbc9e6c7768f5f375f3f._comment new file mode 100644 index 000000000..1fa409f9c --- /dev/null +++ b/doc/bugs/after_git_annex_add_and_commit__44___git_annex_fsck_fails:_no_known_copies/comment_4_8688477bb694dbc9e6c7768f5f375f3f._comment @@ -0,0 +1,28 @@ +[[!comment format=mdwn + username="emanuele.olivetti@47d88ed185b03191e25329caa6fabc2efb3118b2" + nickname="emanuele.olivetti" + subject="repository marked as dead (!)" + date="2016-02-18T14:33:56Z" + content=""" +Update: I've just tried to add a 'test' directory to the repository with only a 'foo' file inside, then git commit. Surprisingly, if I do 'git annex fsck .' of it, the repository is marked as **dead** and no known copies of foo are available: + + (master)> mkdir test ; cd test + test (master) > cat > foo + bar + test (master)> git annex add . + add foo ok + (recording state in git...) + test (master)> git commit -m \"added fake content to test git annex repo\" + [master b9f0a8f] added fake content to test git annex repo + 1 file changed, 1 insertion(+) + create mode 120000 events/2015/test/foo + test (master)> git annex fsck . + Warning: Fscking a repository that is currently marked as dead. + fsck foo (checksum...) + ** No known copies exist of foo + failed + (recording state in git...) + git-annex: fsck: 1 failed + +At this point, I suspect that the repository on my laptop has some serious problem. Shall I move this discussion to the forum? It may be a git-annex bug but it can just be that the repository is damaged for other reasons. Moreover, what is the best course of action, at this point? +"""]] diff --git a/doc/bugs/after_git_annex_add_and_commit__44___git_annex_fsck_fails:_no_known_copies/comment_5_a256583bd9b3815a23cd1ca40d6c19ca._comment b/doc/bugs/after_git_annex_add_and_commit__44___git_annex_fsck_fails:_no_known_copies/comment_5_a256583bd9b3815a23cd1ca40d6c19ca._comment new file mode 100644 index 000000000..785d6c7b7 --- /dev/null +++ b/doc/bugs/after_git_annex_add_and_commit__44___git_annex_fsck_fails:_no_known_copies/comment_5_a256583bd9b3815a23cd1ca40d6c19ca._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="emanuele.olivetti@47d88ed185b03191e25329caa6fabc2efb3118b2" + nickname="emanuele.olivetti" + subject="'git-annex semitrust' seems to solve the issue" + date="2016-02-18T21:51:00Z" + content=""" +Given that the repository is marked as dead, I tried to bring it back with 'git-annex semitrust laptop' - 'laptop' being the name of the git annex repository on my laptop. Apparently, this action solved the problem. Fsck now works as expected, and I could sync and get the new files on the clone on the external hard disk as usual. + +Now, if this the correct solution to my problem, I am wondering how it is possible that the repository was marked as dead. I carefully checked the history of previous command, but no sign of 'git annex dead' o similar. Are there scenarios where a repository is automatically marked as dead? +"""]] diff --git a/doc/bugs/after_git_annex_add_and_commit__44___git_annex_fsck_fails:_no_known_copies/comment_6_0999f9ec9c3d6f51889141344d4cfcb6._comment b/doc/bugs/after_git_annex_add_and_commit__44___git_annex_fsck_fails:_no_known_copies/comment_6_0999f9ec9c3d6f51889141344d4cfcb6._comment new file mode 100644 index 000000000..a2e678c1b --- /dev/null +++ b/doc/bugs/after_git_annex_add_and_commit__44___git_annex_fsck_fails:_no_known_copies/comment_6_0999f9ec9c3d6f51889141344d4cfcb6._comment @@ -0,0 +1,16 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 6""" + date="2016-02-19T18:52:25Z" + content=""" +Ok, the repository being dead certianly explains why fsck found a problem. + +It may be that fsck should have a better message in this case. Like +"only dead repository had copies of the file". + +Repositories are only ever marked dead on user request. There are a few +ways besides `git annex dead` that you could have done it. Perhaps trying +to delete the repo in the webapp, or changing it in `git annex vicfg`. +You can check out the git-annex branch and examine the history of the +trust.log file to see when it was changed to dead. +"""]] diff --git a/doc/bugs/cannot_remove___96__.t__96___directory.mdwn b/doc/bugs/cannot_remove___96__.t__96___directory.mdwn index 3c2b1aad7..cd7c15daf 100644 --- a/doc/bugs/cannot_remove___96__.t__96___directory.mdwn +++ b/doc/bugs/cannot_remove___96__.t__96___directory.mdwn @@ -32,3 +32,6 @@ git HEAD (self-built). On Linux. On /tmp the self-test works. Maybe it is related to NFS? +> Turns out that git-annex was keeping files in .t open in some +> circumstances after deleting them. I have fixed some of this. Possibly +> not all. --[[Joey]] diff --git a/doc/bugs/checksum_loads_whole_file_into_memory.mdwn b/doc/bugs/checksum_loads_whole_file_into_memory.mdwn new file mode 100644 index 000000000..ece328105 --- /dev/null +++ b/doc/bugs/checksum_loads_whole_file_into_memory.mdwn @@ -0,0 +1,24 @@ +Using eg, fsck with the MD5 backend loads whole files into memory. + +May only happen for very large files (40 gb) or in other specific +circumstances, including ghc version used for buildd etc. + +Observed with 6.20160217-g95bbdb8, linux standalone amd64. + +Not observed with 5.20151218-g5008846. + +Commit 7482853ddddc21f2696dcfbc82d737f03032134a may be relevant, +but I don't understand how yet. A small test program using the same +code doesn't exhibit the problem, even when built in the identical build +environment as the 6.20160217-g95bbdb8 that has the problem. + +> Update: Reverted 7482853ddddc21f2696dcfbc82d737f03032134a and indeed the +> problem got fixed. But, reverting that commit breaks the test suite on +> windows and has a FD leak, so is not desirable. This needs more +> investigation. --[[Joey]] + +>> I see it now, the checksum is a String and it was only forced to WHNF, +>> so the hashing didn't fully complete and the file got buffered. +>> Probably only occurred when fscking, and not when adding a file, +>> due to differing use patterns of the checksum. +>> [[fixed|done]] --[[Joey]] diff --git a/doc/bugs/direct_command_leaves_repository_inconsistent_if_interrupted.mdwn b/doc/bugs/direct_command_leaves_repository_inconsistent_if_interrupted.mdwn index c19db9727..8a8881173 100644 --- a/doc/bugs/direct_command_leaves_repository_inconsistent_if_interrupted.mdwn +++ b/doc/bugs/direct_command_leaves_repository_inconsistent_if_interrupted.mdwn @@ -43,3 +43,5 @@ Similar issues and discussions: * [[forum/git-status_typechange_in_direct_mode/]] [[!meta title="git annex lock --force deletes only copy of content after interrupted switch to direct mode"] + +[[!meta tag=deprecateddirectmode]] diff --git a/doc/bugs/direct_mode_fails__44___left_in_an_inconsistent_state.mdwn b/doc/bugs/direct_mode_fails__44___left_in_an_inconsistent_state.mdwn index 664511b6c..42746b966 100644 --- a/doc/bugs/direct_mode_fails__44___left_in_an_inconsistent_state.mdwn +++ b/doc/bugs/direct_mode_fails__44___left_in_an_inconsistent_state.mdwn @@ -58,3 +58,4 @@ git-annex: direct: 1 failed """]] [[!tag moreinfo]] +[[!meta tag=deprecateddirectmode]] diff --git a/doc/bugs/direct_mode_merge_interrupt.mdwn b/doc/bugs/direct_mode_merge_interrupt.mdwn index 7cbb44208..5c26e1773 100644 --- a/doc/bugs/direct_mode_merge_interrupt.mdwn +++ b/doc/bugs/direct_mode_merge_interrupt.mdwn @@ -52,3 +52,5 @@ merge. (I assume this is how `git merge` normally works.) --[[Joey]] > then run mergeDirectCleanup to recover, before any commits can be made > from the inconsistent state. This approach seems to get complicated > quickly.. --[[Joey]] + +[[!meta tag=deprecateddirectmode]] diff --git a/doc/bugs/direct_mode_should_refuse_to_merge_with_illegal_filenames.mdwn b/doc/bugs/direct_mode_should_refuse_to_merge_with_illegal_filenames.mdwn index 102404be4..f90710277 100644 --- a/doc/bugs/direct_mode_should_refuse_to_merge_with_illegal_filenames.mdwn +++ b/doc/bugs/direct_mode_should_refuse_to_merge_with_illegal_filenames.mdwn @@ -35,4 +35,4 @@ So, how to detect when `git merge` has skipped creating illegal filenames? Alternatively, git-annex could learn/probe the full set of characters not allowed in filenames, and examine merges before performing them, and refuse to do anything if the merge added an illegal filename.a [[!tag confirmed]] - +[[!meta tag=deprecateddirectmode]] diff --git a/doc/bugs/duplicate_progress_reports_in_parallel___39__get__39__.mdwn b/doc/bugs/duplicate_progress_reports_in_parallel___39__get__39__.mdwn new file mode 100644 index 000000000..398eb04ca --- /dev/null +++ b/doc/bugs/duplicate_progress_reports_in_parallel___39__get__39__.mdwn @@ -0,0 +1,22 @@ +[[!format sh """ +$> git annex version +git-annex version: 6.20160213+gitg9597a21-1~ndall+1 +... +$> git annex get -J 5 . +get docs/freesurfer.groupanalysis.ppt (from origin...) (checksum...) ok +get docs/freesurfer.future_directions.2007.ppt (from origin...) (checksum...) ok +get docs/freesurfer.groupanalysis.short.ppt (from origin...) (checksum...) ok +get distribution/trctrain/trctraindata.tar.gz (from origin...) +47% 80.2MB/s 3s +47% 80.2MB/s 3s +get docs/freesurfer.inferring_architectonics.ppt (from origin...) +33% 8.1MB/s 4s +33% 8.1MB/s 4s +get docs/freesurfer.intro.2011.ppt (from origin...) +64% 7.3MB/s 1s +64% 7.3MB/s 1s +get docs/freesurfer.intro.mmclass.ppt (from origin...) +# End of transcript or log. +"""]] + +[[!meta author=yoh]] diff --git a/doc/bugs/fatal:_Cannot_handle_files_this_big.mdwn b/doc/bugs/fatal:_Cannot_handle_files_this_big.mdwn index f4e8b7f91..7272bfc29 100644 --- a/doc/bugs/fatal:_Cannot_handle_files_this_big.mdwn +++ b/doc/bugs/fatal:_Cannot_handle_files_this_big.mdwn @@ -94,5 +94,3 @@ ok """]] - -> provisionally [[done]]. --[[Joey]] diff --git a/doc/bugs/fatal:_Cannot_handle_files_this_big/comment_2_546782c644230741470f9a9de23bd019._comment b/doc/bugs/fatal:_Cannot_handle_files_this_big/comment_2_546782c644230741470f9a9de23bd019._comment new file mode 100644 index 000000000..0ea9dc4d1 --- /dev/null +++ b/doc/bugs/fatal:_Cannot_handle_files_this_big/comment_2_546782c644230741470f9a9de23bd019._comment @@ -0,0 +1,24 @@ +[[!comment format=mdwn + username="bvaa" + subject="similar problem" + date="2016-03-01T08:12:27Z" + content=""" +I have a similar problem on Windows 7 64bit trying to add files that are around 5GB in size. I tried repository version 5 and 6 with same results. + +``` +$ git annex add bigfile +add bigfile ok +(recording state in git...) + +$ git annex status +fatal: Cannot handle files this big +``` +git-annex version: 6.20160229-g37a89cc +build flags: Assistant Webapp Pairing Testsuite S3(multipartupload) WebDAV ConcurrentOutput TorrentParser Feeds Quvi +key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 SHA1E SHA1 MD5E MD5 WORM URL +remote types: git gcrypt S3 bup directory rsync web bittorrent webdav tahoe glacier ddar hook external +local repository version: 5 +supported repository versions: 5 6 +upgrade supported from repository versions: 2 3 4 5 + +"""]] diff --git a/doc/bugs/fatal:_Cannot_handle_files_this_big/comment_3_151e7cf96c7d168e1397d111aa47f279._comment b/doc/bugs/fatal:_Cannot_handle_files_this_big/comment_3_151e7cf96c7d168e1397d111aa47f279._comment new file mode 100644 index 000000000..e6ad551e4 --- /dev/null +++ b/doc/bugs/fatal:_Cannot_handle_files_this_big/comment_3_151e7cf96c7d168e1397d111aa47f279._comment @@ -0,0 +1,20 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 3""" + date="2016-03-01T14:41:45Z" + content=""" +git (not git-annex) will throw this error if a file size is greater than +`size_t`. + +This bug report seemed to originally concern git add being run on such a +file, but I can't see how git-annex would do that, it doesn't add large +files to git. + +I think that in the case of git-annex status, when it runs git status, that +looks at work tree files, and so falls over if they're large, even if +what's checked into git is a nice small git-annex symlink. This would also +probably affect other places where git looks at worktree files, perhaps git +diff (in v6 repo mode). + +Reopening bug report. +"""]] diff --git a/doc/bugs/git-annex:_content_is_locked__while_trying_to_move_under_NFS_and_pidlock.mdwn b/doc/bugs/git-annex:_content_is_locked__while_trying_to_move_under_NFS_and_pidlock.mdwn new file mode 100644 index 000000000..d3d75e8c8 --- /dev/null +++ b/doc/bugs/git-annex:_content_is_locked__while_trying_to_move_under_NFS_and_pidlock.mdwn @@ -0,0 +1,31 @@ +### Please describe the problem. + +Ideally annex should detect all "paranormal" cases such as running on NFS mounted partition, but according to [https://git-annex.branchable.com/bugs/huge_multiple_copies_of___39__.nfs__42____39___and___39__.panfs__42____39___being_created/](https://git-annex.branchable.com/bugs/huge_multiple_copies_of___39__.nfs__42____39___and___39__.panfs__42____39___being_created/). Happily ignorant we were running annex (5.20151116-g76139a9) on NFS mounted partition until we filled up 2TB of allocated to us space with .nfs* files. Well -- apparently according to above we should have tried pidlock... trying now but doesn't work :-/ + +[[!format sh """ +*$> git clone smaug:/tmp/123 123-clone && cd 123-clone && git config annex.pidlock true && echo 124 > 124.dat && git annex add 124.dat && git commit -m 'added 124' && git annex move --to=origin 124.dat +Initialized empty Git repository in /home/yhalchen/123-clone/.git/ +remote: Counting objects: 22, done. +remote: Compressing objects: 100% (16/16), done. +remote: Total 22 (delta 3), reused 0 (delta 0) +Receiving objects: 100% (22/22), done. +Resolving deltas: 100% (3/3), done. +total 1 +1 123.dat@ 1 README.txt +(merging origin/git-annex into git-annex...) +(recording state in git...) +add 124.dat ok +(recording state in git...) +[master 0f1092a] added 124 + 1 files changed, 1 insertions(+), 0 deletions(-) + create mode 120000 124.dat +move 124.dat (checking origin...) git-annex: content is locked + +$> echo $? +1 + +"""]] + +BTW running move in our old now somewhat screwed up annex, results in a differently expressed error: [http://www.onerussian.com/tmp/2016-02-29.png](http://www.onerussian.com/tmp/2016-02-29.png) + +[[!meta author=yoh]] diff --git a/doc/bugs/git-annex:_content_is_locked__while_trying_to_move_under_NFS_and_pidlock/comment_10_d44de6a250694b25ce9c3169d62db8d1._comment b/doc/bugs/git-annex:_content_is_locked__while_trying_to_move_under_NFS_and_pidlock/comment_10_d44de6a250694b25ce9c3169d62db8d1._comment new file mode 100644 index 000000000..f2ba4335c --- /dev/null +++ b/doc/bugs/git-annex:_content_is_locked__while_trying_to_move_under_NFS_and_pidlock/comment_10_d44de6a250694b25ce9c3169d62db8d1._comment @@ -0,0 +1,20 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 10""" + date="2016-03-01T20:52:38Z" + content=""" + 2456732 openat(AT_FDCWD, ".git/annex/ssh/", O_RDONLY|O_NONBLOCK|O_DIRECTORY|O_CLOEXEC) = -1 ENOENT (No such file or directory) + 2456732 mkdir(".git/annex/ssh", 0777) = 0 + 2456732 open(".git/annex/ssh/smaug.lock", O_RDONLY|O_CREAT, 0666) = 11 + 2456732 fcntl(11, F_GETFD) = 0 + 2456732 fcntl(11, F_SETFD, FD_CLOEXEC) = 0 + 2456732 close(11) = 0 + +Backs up what I thought git-annex should be doing; it's not fcntl locking that file. + +Ah, I'll bet it's not git-annex at all this time. +It runs ssh with -S .git/annex/ssh/smaug, and ssh probably +does its own locking around setting up that control socket. + +If so, disabling annex.sshcaching will avoid the problem. +"""]] diff --git a/doc/bugs/git-annex:_content_is_locked__while_trying_to_move_under_NFS_and_pidlock/comment_11_56ae0f15bbdea2331df3b261b74d0b0b._comment b/doc/bugs/git-annex:_content_is_locked__while_trying_to_move_under_NFS_and_pidlock/comment_11_56ae0f15bbdea2331df3b261b74d0b0b._comment new file mode 100644 index 000000000..2ee2e6aa8 --- /dev/null +++ b/doc/bugs/git-annex:_content_is_locked__while_trying_to_move_under_NFS_and_pidlock/comment_11_56ae0f15bbdea2331df3b261b74d0b0b._comment @@ -0,0 +1,7 @@ +[[!comment format=mdwn + username="https://me.yahoo.com/a/EbvxpTI_xP9Aod7Mg4cwGhgjrCrdM5s-#7c0f4" + subject="comment 11" + date="2016-03-01T22:40:44Z" + content=""" +would then may be annex not to use sshcaching if operating under pidlock, unless some nfs specific flag is used to tease it apart +"""]] diff --git a/doc/bugs/git-annex:_content_is_locked__while_trying_to_move_under_NFS_and_pidlock/comment_1_a98a54c04fa4e81f35fe958e746d61cb._comment b/doc/bugs/git-annex:_content_is_locked__while_trying_to_move_under_NFS_and_pidlock/comment_1_a98a54c04fa4e81f35fe958e746d61cb._comment new file mode 100644 index 000000000..21304e2e8 --- /dev/null +++ b/doc/bugs/git-annex:_content_is_locked__while_trying_to_move_under_NFS_and_pidlock/comment_1_a98a54c04fa4e81f35fe958e746d61cb._comment @@ -0,0 +1,7 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2016-03-01T14:36:25Z" + content=""" +FYI, I think you could remove the .nfs files to free up space. +"""]] diff --git a/doc/bugs/git-annex:_content_is_locked__while_trying_to_move_under_NFS_and_pidlock/comment_2_18169e7bbd2caba5ee4bb0286961ac95._comment b/doc/bugs/git-annex:_content_is_locked__while_trying_to_move_under_NFS_and_pidlock/comment_2_18169e7bbd2caba5ee4bb0286961ac95._comment new file mode 100644 index 000000000..42b3265bd --- /dev/null +++ b/doc/bugs/git-annex:_content_is_locked__while_trying_to_move_under_NFS_and_pidlock/comment_2_18169e7bbd2caba5ee4bb0286961ac95._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 2""" + date="2016-03-01T15:35:48Z" + content=""" +Oddly, I cannot reproduce this, although I can reproduce the behavior in +<http://git-annex.branchable.com/bugs/thread_blocked_indefinitely_in_an_STM_transaction__while_moving_within__a_local_clone/>a + +(smaug:/tmp/123 has permissions that do not let me access it.) +"""]] diff --git a/doc/bugs/git-annex:_content_is_locked__while_trying_to_move_under_NFS_and_pidlock/comment_3_e3b623ff6714a9fe5fa0d332c72fe32f._comment b/doc/bugs/git-annex:_content_is_locked__while_trying_to_move_under_NFS_and_pidlock/comment_3_e3b623ff6714a9fe5fa0d332c72fe32f._comment new file mode 100644 index 000000000..51f71c7ac --- /dev/null +++ b/doc/bugs/git-annex:_content_is_locked__while_trying_to_move_under_NFS_and_pidlock/comment_3_e3b623ff6714a9fe5fa0d332c72fe32f._comment @@ -0,0 +1,9 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 3""" + date="2016-03-01T16:52:16Z" + content=""" +I've fixed the STM transaction bug. Need either more info to reproduce this +bug, or you could test and see if it still occurs when git-annex is +upgraded to ad888a6b760e8f9d31f8d99c51912bcdaa7fb0c1 +"""]] diff --git a/doc/bugs/git-annex:_content_is_locked__while_trying_to_move_under_NFS_and_pidlock/comment_4_58eebd8cfd664b32ef6fd0ddc34c5e86._comment b/doc/bugs/git-annex:_content_is_locked__while_trying_to_move_under_NFS_and_pidlock/comment_4_58eebd8cfd664b32ef6fd0ddc34c5e86._comment new file mode 100644 index 000000000..7a562f7ef --- /dev/null +++ b/doc/bugs/git-annex:_content_is_locked__while_trying_to_move_under_NFS_and_pidlock/comment_4_58eebd8cfd664b32ef6fd0ddc34c5e86._comment @@ -0,0 +1,9 @@ +[[!comment format=mdwn + username="https://me.yahoo.com/a/EbvxpTI_xP9Aod7Mg4cwGhgjrCrdM5s-#7c0f4" + subject="more info" + date="2016-03-01T17:31:29Z" + content=""" +If we could remove those .nfs* files, it would indeed be not that bad but we can't + +smaug:/tmp/123 -- sorry about permissions but it is a regular annex nothing special, so the bug should show itself with other repos as well I think. I gave you access to it now and also there is /tmp/123.tar.gz archive of it just in case. +"""]] diff --git a/doc/bugs/git-annex:_content_is_locked__while_trying_to_move_under_NFS_and_pidlock/comment_5_e5e24428ac02b78d38cd4f197ae3807b._comment b/doc/bugs/git-annex:_content_is_locked__while_trying_to_move_under_NFS_and_pidlock/comment_5_e5e24428ac02b78d38cd4f197ae3807b._comment new file mode 100644 index 000000000..75a45bdad --- /dev/null +++ b/doc/bugs/git-annex:_content_is_locked__while_trying_to_move_under_NFS_and_pidlock/comment_5_e5e24428ac02b78d38cd4f197ae3807b._comment @@ -0,0 +1,31 @@ +[[!comment format=mdwn + username="https://me.yahoo.com/a/EbvxpTI_xP9Aod7Mg4cwGhgjrCrdM5s-#7c0f4" + subject="recent snapshot seems has fixed it" + date="2016-03-01T18:52:27Z" + content=""" +[[!format sh \"\"\" +$> git clone smaug:/tmp/123 123-clone && cd 123-clone && git config annex.pidlock true && echo 124 > 124.dat && git annex add 124.dat && git commit -m 'added 124' && git annex move --to=origin 124.dat +Cloning into '123-clone'... +remote: Counting objects: 22, done. +remote: Compressing objects: 100% (16/16), done. +remote: Total 22 (delta 3), reused 0 (delta 0) +Receiving objects: 100% (22/22), done. +Resolving deltas: 100% (3/3), done. +Checking connectivity... done. +total 1 +1 123.dat@ 1 README.txt +(merging origin/git-annex into git-annex...) +(recording state in git...) +add 124.dat ok +(recording state in git...) +[master 6eca577] added 124 + 1 file changed, 1 insertion(+) + create mode 120000 124.dat +move 124.dat (checking origin...) ok +(recording state in git...) + +$> git annex version +git-annex version: 6.20160301+gitg647fffd-1~ndall+1 + +\"\"\"]] +"""]] diff --git a/doc/bugs/git-annex:_content_is_locked__while_trying_to_move_under_NFS_and_pidlock/comment_6_01dc7a1ff67783ce672d72cefe7b4bb5._comment b/doc/bugs/git-annex:_content_is_locked__while_trying_to_move_under_NFS_and_pidlock/comment_6_01dc7a1ff67783ce672d72cefe7b4bb5._comment new file mode 100644 index 000000000..20eca2f62 --- /dev/null +++ b/doc/bugs/git-annex:_content_is_locked__while_trying_to_move_under_NFS_and_pidlock/comment_6_01dc7a1ff67783ce672d72cefe7b4bb5._comment @@ -0,0 +1,7 @@ +[[!comment format=mdwn + username="https://me.yahoo.com/a/EbvxpTI_xP9Aod7Mg4cwGhgjrCrdM5s-#7c0f4" + subject="comment 6" + date="2016-03-01T18:54:15Z" + content=""" +but then I found ./.git/annex/ssh/.nfs000000000000f41600003608.lock left behind (removable, luckily to me ;) ) +"""]] diff --git a/doc/bugs/git-annex:_content_is_locked__while_trying_to_move_under_NFS_and_pidlock/comment_7_458518805b8d6613930b38b9ccc3c1bc._comment b/doc/bugs/git-annex:_content_is_locked__while_trying_to_move_under_NFS_and_pidlock/comment_7_458518805b8d6613930b38b9ccc3c1bc._comment new file mode 100644 index 000000000..756b64250 --- /dev/null +++ b/doc/bugs/git-annex:_content_is_locked__while_trying_to_move_under_NFS_and_pidlock/comment_7_458518805b8d6613930b38b9ccc3c1bc._comment @@ -0,0 +1,7 @@ +[[!comment format=mdwn + username="https://me.yahoo.com/a/EbvxpTI_xP9Aod7Mg4cwGhgjrCrdM5s-#7c0f4" + subject="comment 7" + date="2016-03-01T18:58:20Z" + content=""" +and those are breeding with next subsequent --move +"""]] diff --git a/doc/bugs/git-annex:_content_is_locked__while_trying_to_move_under_NFS_and_pidlock/comment_8_853bc273b19bd6d84ca8f5da6c3dfb56._comment b/doc/bugs/git-annex:_content_is_locked__while_trying_to_move_under_NFS_and_pidlock/comment_8_853bc273b19bd6d84ca8f5da6c3dfb56._comment new file mode 100644 index 000000000..d24d78d51 --- /dev/null +++ b/doc/bugs/git-annex:_content_is_locked__while_trying_to_move_under_NFS_and_pidlock/comment_8_853bc273b19bd6d84ca8f5da6c3dfb56._comment @@ -0,0 +1,20 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 8""" + date="2016-03-01T20:17:35Z" + content=""" +That ssh lock file is created by this code: + + -- The posix lock file is created even when using pid locks, in order to + -- avoid complicating any code that might expect to be able to see that + -- lock file. But, it's not locked. + dummyPosixLock :: Maybe FileMode -> LockFile -> IO () + dummyPosixLock m f = closeFd =<< openLockFile ReadLock m f + +But, that does not ever actually take a lock on the file, so +NFS should not make its .nfs thing in this case. Unless NFS does it when a +FD is simply opened with close-on-exec set. + +Can you get a strace of the creation of files under .git/annex/ssh/ +that result in these .nfs things? +"""]] diff --git a/doc/bugs/git-annex:_content_is_locked__while_trying_to_move_under_NFS_and_pidlock/comment_9_86656a409ab25c7fa24de8ac3e68b254._comment b/doc/bugs/git-annex:_content_is_locked__while_trying_to_move_under_NFS_and_pidlock/comment_9_86656a409ab25c7fa24de8ac3e68b254._comment new file mode 100644 index 000000000..b862174b1 --- /dev/null +++ b/doc/bugs/git-annex:_content_is_locked__while_trying_to_move_under_NFS_and_pidlock/comment_9_86656a409ab25c7fa24de8ac3e68b254._comment @@ -0,0 +1,7 @@ +[[!comment format=mdwn + username="https://me.yahoo.com/a/EbvxpTI_xP9Aod7Mg4cwGhgjrCrdM5s-#7c0f4" + subject="comment 9" + date="2016-03-01T20:43:17Z" + content=""" +ok -- see on smaug /mnt/nfs/scrap/datalad/test_nfs/123-clone-move.strace . Now you can experiment there as well -- the entire /mnt/btrfs/scrap is mounted also via nfs (under /mnt/nfs/scrap) +"""]] diff --git a/doc/bugs/git-annex_confuses_Git_with_nested_submodules.mdwn b/doc/bugs/git-annex_confuses_Git_with_nested_submodules.mdwn new file mode 100644 index 000000000..83d74f79f --- /dev/null +++ b/doc/bugs/git-annex_confuses_Git_with_nested_submodules.mdwn @@ -0,0 +1,37 @@ +### Please describe the problem. +The way git-annex deals with submodules (replacing the .git file in the submodule, with a link to the corresponding gitdir of the submodule) seems to confuse Git when creating another submodule in an annex-init'ed submodule. + +### What steps will reproduce the problem? + % mkdir some ; cd some; git init + Initialized empty Git repository in /tmp/some/.git/ + % git submodule add /src/somegitrepo sub_lvl1 + Cloning into 'sub_lvl1'... + done. + % cd sub_lvl1 + % git annex init + init (merging origin/git-annex into git-annex...) + (recording state in git...) + ok + (recording state in git...) + % git submodule add /src/somegitrepo sub_lvl2 + Cloning into 'sub_lvl2'... + done. + fatal: Could not chdir to '../../../sub_lvl2': No such file or directory + Unable to checkout submodule 'sub_lvl2' + +### What version of git-annex are you using? On what operating system? + % apt-cache policy git-annex-standalone + git-annex-standalone: + Installed: 6.20160213+gitg9597a21-1~ndall+1 + +Debian stretch, git-annex from NeuroDebian. + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) + +Yes, lots! Using it for some of its original use cases for more than five years now -- I was actually surprised to learn, just now, that the oldest commit in my music repository is exactly 5 years and 6 days old. Thanks for longevity and reliability! + +More recently I aim exploring the use of git annex for managing datasets and their dependencies, i.e. going from raw to some processed state over multiple levels, where each level is a useful starting point for some analysis, and each previous level is a dependency (input) to the next. With just one level above "raw" this has massively improved collaboration workflows in student/teacher settings for me. Deeper nesting levels would allow for even more interesting applications, but see above ;-) I think Git seems needlessly confused, but I don't fully grasp what is happening yet. I'd appreciate any insight you may have. Although it is Git that shows the undesired behavior, it seems it is git-annex that ultimately confused it. Hence I came here first. + +BTW: What a nice idea to ask for something like this in a bug report. + + diff --git a/doc/bugs/git-annex_confuses_Git_with_nested_submodules/comment_1_fb01d4b5af500affc08a5c3b3b1849dd._comment b/doc/bugs/git-annex_confuses_Git_with_nested_submodules/comment_1_fb01d4b5af500affc08a5c3b3b1849dd._comment new file mode 100644 index 000000000..41068b5a9 --- /dev/null +++ b/doc/bugs/git-annex_confuses_Git_with_nested_submodules/comment_1_fb01d4b5af500affc08a5c3b3b1849dd._comment @@ -0,0 +1,16 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2016-03-01T20:25:13Z" + content=""" +Reproduced this. + +This really does feel like a git bug. git is supposed to treat "gitlink" +files and .git symlinks the same. While modern versions of git set up +gitlink files for submodules, older versions of git used .git symlinks, and +git should still support that. + +Looks like the problem can be worked around, by setting +`GIT_DIR`. In your example, `GIT_DIR=../.git/modules/sub_lvl1/ git +submodule add /src/somegitrepo sub_lvl2` +"""]] diff --git a/doc/bugs/git-annex_confuses_Git_with_nested_submodules/comment_2_094baf6c3738691879fd907dd1729c56._comment b/doc/bugs/git-annex_confuses_Git_with_nested_submodules/comment_2_094baf6c3738691879fd907dd1729c56._comment new file mode 100644 index 000000000..50d2f709a --- /dev/null +++ b/doc/bugs/git-annex_confuses_Git_with_nested_submodules/comment_2_094baf6c3738691879fd907dd1729c56._comment @@ -0,0 +1,17 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 2""" + date="2016-03-01T20:36:43Z" + content=""" +Here's a more minimal test case, not involving git-annex at all: + + git init gitdir + mkdir worktree + cd worktree + ln -s ../gitdir/.git .git + git submodule add /any/git/repo sub + + fatal: Could not chdir to '../../../sub': No such file or directory + +I have forwarded that test case to the git ML. +"""]] diff --git a/doc/bugs/git-annex_confuses_Git_with_nested_submodules/comment_3_e1bc8eb7f6ce0d6f2d2f2b0ea6f20862._comment b/doc/bugs/git-annex_confuses_Git_with_nested_submodules/comment_3_e1bc8eb7f6ce0d6f2d2f2b0ea6f20862._comment new file mode 100644 index 000000000..49e883c7c --- /dev/null +++ b/doc/bugs/git-annex_confuses_Git_with_nested_submodules/comment_3_e1bc8eb7f6ce0d6f2d2f2b0ea6f20862._comment @@ -0,0 +1,29 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 3""" + date="2016-03-02T16:48:24Z" + content=""" +[git bug report](http://news.gmane.org/find-root.php?message_id=20160301204218.GA4083%40kitenet.net) + +So far, the git devs admit this is a problem, but don't seem too keen on fixing +it, even though it breaks backwards compatability with repositories git +submodule add created (circa 2012). + +It might be that git-annex init could work around git's bugginess by, +instead of making submodule/.git a symlink to ../.git/modules/dir, making +submodule/.git be the git directory, and converting ../.git/modules/dir +to a symlink. In very limited testing, that setup seems to work. + +I don't know if all the submodule stuff would work, perhaps it would break moving +submodules etc. And, since git likes to chdir around (not the best idea), +if it expected to be able to chdir from .git/modules to dir and chdir .. to +get back, changing that to a symlink would defeat it. + +BTW, I found another way, unrelated to git-annex or symlinks at all, +that git submodule add's broken path handling makes it fall over with +nested submodules. +<http://news.gmane.org/find-root.php?message_id=20160302165240.GA17654%40kitenet.net>. + +(It's almost like myrepos was a better idea than this submodule stuff, or +something...) +""]] diff --git a/doc/bugs/git-annex_confuses_Git_with_nested_submodules/comment_4_4bcd571dcd6c1e709e83e519135519b3._comment b/doc/bugs/git-annex_confuses_Git_with_nested_submodules/comment_4_4bcd571dcd6c1e709e83e519135519b3._comment new file mode 100644 index 000000000..d663f3370 --- /dev/null +++ b/doc/bugs/git-annex_confuses_Git_with_nested_submodules/comment_4_4bcd571dcd6c1e709e83e519135519b3._comment @@ -0,0 +1,9 @@ +[[!comment format=mdwn + username="mih" + subject="Thanks" + date="2016-03-02T19:30:49Z" + content=""" +Thanks for investigating this further. + +One aspect that may make switching the location of the .git directory into the worktree of the submodule less desirable is this: With the actual .git in ../.git/modules/... one can easily rm -rf the submodule, deinit it, and re-init/update from the (still present) ../.git/modules/... at a later point in time. Especially, when a submodule is a more complicated beast (e.g. with multiple configured remotes) the required steps to regenerate the same setup get more complex. +"""]] diff --git a/doc/bugs/git_annex_sync_in_direct_mode_does_not_honor_skip-worktree.mdwn b/doc/bugs/git_annex_sync_in_direct_mode_does_not_honor_skip-worktree.mdwn index fbe46ab2d..7ff82b69d 100644 --- a/doc/bugs/git_annex_sync_in_direct_mode_does_not_honor_skip-worktree.mdwn +++ b/doc/bugs/git_annex_sync_in_direct_mode_does_not_honor_skip-worktree.mdwn @@ -35,3 +35,4 @@ I did a little digging in the code, and it looks like the source of this is the I wonder if this would have side effects, or if there are other places in the code where skip-worktree files would need to be handled, though. I'm particularly motivated to solve this, so let me know if it doesn't look like it would get looked at right away, and I'll have an excuse to get a Haskell dev environment setup again and shake the rust off. [[!tag confirmed]] +[[!meta tag=deprecateddirectmode]] diff --git a/doc/bugs/put_gpg_last_in_OSX_dmg_PATH.mdwn b/doc/bugs/put_gpg_last_in_OSX_dmg_PATH.mdwn index 2f30352f9..dfaec26f1 100644 --- a/doc/bugs/put_gpg_last_in_OSX_dmg_PATH.mdwn +++ b/doc/bugs/put_gpg_last_in_OSX_dmg_PATH.mdwn @@ -6,3 +6,5 @@ A solution might be to move the gpg binary to a different directory and put it at the end of PATH, not the front. So system one is used if available. This should also be considered for the linux standalone builds. + +> [[done]] for both OSX and linux. --[[Joey]] diff --git a/doc/bugs/shouldn__39__t_keep_permissions_of_the_ssh_remote__63__.mdwn b/doc/bugs/shouldn__39__t_keep_permissions_of_the_ssh_remote__63__.mdwn new file mode 100644 index 000000000..fc9c3965e --- /dev/null +++ b/doc/bugs/shouldn__39__t_keep_permissions_of_the_ssh_remote__63__.mdwn @@ -0,0 +1,9 @@ +### Please describe the problem. + +Initially generated an annex while having a restrictive umask 077. Then cloned that repository to another host for public consumption so directory had proper/good permissions set allowing group to access. And everything is accessible but not the load which I 'annex get'ed. Key directories were readable but not the content. I guess there is somewhere 'preserve permissions' setting for rsync/scp which imho shouldn't be there and content should inherit local/environment settings + +### What version of git-annex are you using? On what operating system? + +6.20160208+gitg1ac9034-1~ndall+1 + +[[!meta author=yoh]] diff --git a/doc/bugs/symlinks_to_absent_files_remain_upon_switching_to_direct_mode2.mdwn b/doc/bugs/symlinks_to_absent_files_remain_upon_switching_to_direct_mode2.mdwn index d040e9a4f..a33550bb3 100644 --- a/doc/bugs/symlinks_to_absent_files_remain_upon_switching_to_direct_mode2.mdwn +++ b/doc/bugs/symlinks_to_absent_files_remain_upon_switching_to_direct_mode2.mdwn @@ -88,3 +88,7 @@ lrwxrwxrwx 1 yoh yoh 186 Jul 28 16:11 test-annex.dat -> .git/annex/objects/zk/71 # End of transcript or log. """]] + +> Closing as I think we're in agreement this is the right behavior. +> But, see <http://git-annex.branchable.com/todo/hide_missing_files/>. +> [[done]] --[[Joey]] diff --git a/doc/bugs/thread_blocked_indefinitely_in_an_STM_transaction__while_moving_within__a_local_clone.mdwn b/doc/bugs/thread_blocked_indefinitely_in_an_STM_transaction__while_moving_within__a_local_clone.mdwn new file mode 100644 index 000000000..eaf79a862 --- /dev/null +++ b/doc/bugs/thread_blocked_indefinitely_in_an_STM_transaction__while_moving_within__a_local_clone.mdwn @@ -0,0 +1,40 @@ +relates to having pidlock true + +[[!format sh """ +$> mkdir 123; cd 123; git init; git annex init; git config annex.pidlock true && echo "123" > 123.dat; git annex add 123.dat; git commit -m 'added'; +W: git-annex repositories not (yet) supported in the prompt +Initialized empty Git repository in /tmp/123/.git/ +init ok +(recording state in git...) +add 123.dat ok +(recording state in git...) +[master (root-commit) 9449f1b] added + 1 file changed, 1 insertion(+) + create mode 120000 123.dat + +$> git clone . ../123-clone && git remote add clone ../123-clone && git fetch clone && cd ../123-clone && git config annex.pidlock true && cd - && git annex move --to=clone . +Cloning into '../123-clone'... +done. +From ../123-clone + * [new branch] master -> clone/master +move 123.dat git-annex: thread blocked indefinitely in an STM transaction + +$> echo $? +1 + +$> git annex version +git-annex version: 6.20160226+gitg01f1de0-1~ndall+1 +build flags: Assistant Webapp Pairing Testsuite S3(multipartupload) WebDAV Inotify DBus DesktopNotify XMPP ConcurrentOutput TorrentParser MagicMime Feeds Quvi +key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 SHA1E SHA1 MD5E MD5 WORM URL +remote types: git gcrypt S3 bup directory rsync web bittorrent webdav tahoe glacier ddar hook external +local repository version: 5 +supported repository versions: 5 6 +upgrade supported from repository versions: 0 1 2 4 5 + +"""]] + +and it works ok without pidlock enabled + +[[!meta author=yoh]] + +> [[fixed|done]] --[[Joey]] diff --git a/doc/bugs/thread_blocked_indefinitely_in_an_STM_transaction__while_moving_within__a_local_clone/comment_1_0cb6b5d69cc47bfbab8fb5e87e6e2bad._comment b/doc/bugs/thread_blocked_indefinitely_in_an_STM_transaction__while_moving_within__a_local_clone/comment_1_0cb6b5d69cc47bfbab8fb5e87e6e2bad._comment new file mode 100644 index 000000000..7afc0eb30 --- /dev/null +++ b/doc/bugs/thread_blocked_indefinitely_in_an_STM_transaction__while_moving_within__a_local_clone/comment_1_0cb6b5d69cc47bfbab8fb5e87e6e2bad._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2016-03-01T15:40:12Z" + content=""" +I can reproduce this. But, when I change the origin remote to use ssh, it +works around the problem. +"""]] diff --git a/doc/bugs/thread_blocked_indefinitely_in_an_STM_transaction__while_moving_within__a_local_clone/comment_2_aa8c82f27965df44e69fd06b34be0ece._comment b/doc/bugs/thread_blocked_indefinitely_in_an_STM_transaction__while_moving_within__a_local_clone/comment_2_aa8c82f27965df44e69fd06b34be0ece._comment new file mode 100644 index 000000000..2c06811d8 --- /dev/null +++ b/doc/bugs/thread_blocked_indefinitely_in_an_STM_transaction__while_moving_within__a_local_clone/comment_2_aa8c82f27965df44e69fd06b34be0ece._comment @@ -0,0 +1,11 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 2""" + date="2016-03-01T16:11:37Z" + content=""" +A worse problem with annex.pidlock is that it completly broke checking +whether a key is present in the repository. That could lead to data loss +when eg, moving --to a repo with annex.pidlock set. + +I've fixed that related bug. +"""]] diff --git a/doc/bugs/thread_blocked_indefinitely_in_an_STM_transaction__while_moving_within__a_local_clone/comment_3_7e6b3ab0beaca49d7d68c9e610c1d147._comment b/doc/bugs/thread_blocked_indefinitely_in_an_STM_transaction__while_moving_within__a_local_clone/comment_3_7e6b3ab0beaca49d7d68c9e610c1d147._comment new file mode 100644 index 000000000..becf5a1b3 --- /dev/null +++ b/doc/bugs/thread_blocked_indefinitely_in_an_STM_transaction__while_moving_within__a_local_clone/comment_3_7e6b3ab0beaca49d7d68c9e610c1d147._comment @@ -0,0 +1,17 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 3""" + date="2016-03-01T16:21:31Z" + content=""" +Analysis: What's crashing is Utility.LockPool.PidLock.waitLock after a call +to Utility.LockPool.PidLock.tryLock. The former takes an exclusive STM lock +of the pid lock file; the latter takes a shared STM lock. + +Since the pid lock stands in for multiple more fine-grained locks, waitLock +will be called while a lock from tryLock (or a previous waitLock perhaps) +is still open. + +The fix seems as simple as making waitLock take a shared STM lock of the +pid lock file, leaving the exclusive lock for the later, more fine-grained +STM lock checking that's done after taking the pid lock. +"""]] diff --git a/doc/bugs/using_regular_magic_file__warning_pollutes_stderr.mdwn b/doc/bugs/using_regular_magic_file__warning_pollutes_stderr.mdwn new file mode 100644 index 000000000..c94a56247 --- /dev/null +++ b/doc/bugs/using_regular_magic_file__warning_pollutes_stderr.mdwn @@ -0,0 +1,25 @@ +### Please describe the problem. + +Although probably not an annex issue but thought to let you know since you might see +a quick resolution on your end + +Here is the log from datalad: + +[[!format sh """ +2016-02-25 22:53:32,886 [DEBUG] Running: ['git', '-c', 'receive.autogc=false', '-c', 'annex.alwayscommit=false', 'annex', 'add', '--debug', '--json', '-c', 'annex.largefiles=exclude=CHANGES* and exclude=README* and exclude=*.[mc] and exclude=dataset*.json and (exclude=*.txt or include=*/*.txt) and (exclude=*.json or include=*/*.json) and (exclude=*.tsv or include=*/*.tsv)', 'README.txt'] (cmd.py:351) +2016-02-25 22:53:32,979 [ERROR] stderr| /etc/magic, 4: Warning: using regular magic file `/usr/share/misc/magic' (cmd.py:351) + + +or outside (file is already under annex) + +$> git annex --debug add -c 'annex.largefiles=exclude=CHANGES* and exclude=README*' README.txt +/etc/magic, 4: Warning: using regular magic file `/usr/share/misc/magic' + +"""]] + +annex is up to date: 6.20160225+gitg229db26-1~ndall+1 + +edit1: that is happening on jessie with file 1:5.22+15-2+deb8u1 if that is relevant +[[!meta author=yoh]] + +> [[fixed|done]] --[[Joey]] diff --git a/doc/bugs/using_regular_magic_file__warning_pollutes_stderr/comment_1_407787292dd6e6d1aff7193634f8a7bf._comment b/doc/bugs/using_regular_magic_file__warning_pollutes_stderr/comment_1_407787292dd6e6d1aff7193634f8a7bf._comment new file mode 100644 index 000000000..a0f7e03c8 --- /dev/null +++ b/doc/bugs/using_regular_magic_file__warning_pollutes_stderr/comment_1_407787292dd6e6d1aff7193634f8a7bf._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2016-02-26T15:52:43Z" + content=""" +Reproduced in a clean chroot using the linux standalone tarball. + +I think it does this when it can't find any magic file. + +Fixing by including the magic database in the bundle. This will also let me +re-enable magicmime on OSX. +"""]] diff --git a/doc/design/adjusted_branches.mdwn b/doc/design/adjusted_branches.mdwn index 6cb282f1b..f79a7f1ab 100644 --- a/doc/design/adjusted_branches.mdwn +++ b/doc/design/adjusted_branches.mdwn @@ -20,30 +20,158 @@ play out.] [[!toc]] -## merge +## filtering + + master adjusted/master + A + |--------------->A' + | | + +When generating commit A', reuse the date of A and use a standard author, +committer, and message. This means that two users with the adjusted branch +checked out and using the same filters will get identical shas for A', and +so can collaborate on them. + +## commit + +When committing changes, a commit is made as usual to the adjusted branch. +So, the user can `git commit` as usual. This does not touch the +original branch yet. + +Then we need to get from that commit to one with the filters reversed, +which should be the same as if the adjusted branch had not been used. +This commit gets added onto the original branch. + +So, the branches would look like this: -When merging changes from a remote, apply the filter to the head of the -remote branch, resulting in a commit with its changes. Merge in that -commit. Note that it's possible to control the metadata of the commit such -that 2 users who have the same adjusted branch checked out, both generate -the same commit sha. + master adjusted/master + A + |--------------->A' + | | + | C (new commit) + B < - - - - - - - + | + |--------------->B' + | | + +Note particularly that B does not have A' or C in its history; +the adjusted branch is not evident from outside. + +Also note that B gets filtered and the adjusted branch is rebased on top of +it, so C does not remain in the adjusted branch history either. This will +make other checkouts that are in the same adjusted branch end up with the +same B' commit when they pull B. + +It might be useful to have a post-commit hook that generates B and B' +and updates the branches. And/or `git-annex sync` could do it. + +There may be multiple commits made to the adjusted branch before any get +applied back to the original branch. This is handled by reverse filtering +one at a time and rebasing the others on top. + + master adjusted/master + A + |--------------->A' + | | + | C1 + | | + | C2 + + + master adjusted/master + A + |--------------->A' + | | + | C1 + B1< - - - - - - - + | + |--------------->B1' + | | + | C2' + B2< - - - - - - - + | + |--------------->B2' -This would be done by `git annex merge` and `git annex sync`. + +[WORKTREE: A pre-commit hook would be needed to update the staged changes, +reversing the filter before the commit is made. All the other complications +above are avoided.] + +## merge + +This would be done by `git annex merge` and `git annex sync`, with the goal +of merging origin/master into master, and updating adjusted/master. + +Note that the adjusted files db needs to be updated to reflect the changes +that are merged in, for object add/remove to work as described below. + +When merging, there should never be any commits present on the +adjusted/master branch that have not yet been filtered over to the master +branch. If there are any such commits, just filter them into master before +beginning the merge. There may be staged changes, or changes in the work tree. + +First filter the new commit: + + origin/master adjusted/master + A + |--------------->A' + | | + | | + B + | + |---------->B' + +Then, merge that into adjusted/master: + + origin/master adjusted/master + A + |--------------->A' + | | + | | + B | + | | + |----------->B'->B'' + +That merge will take care of updating the work tree. + +(What if there is a merge conflict between A' and B'? Normally such a merge +conflict should only affect the work tree/index, so can be resolved without +making a commit, but B'' may end up being made to resolve a merge +conflict.) + +Once the merge is done, we have a commit B'' on adjusted/master. To finish, +adjust that commit so it does not have adjusted/master as its parent. + + origin/master adjusted/master + A + |--------------->A' + | | + | | + B + | + |--------------->B'' + | | + +Finally, update master to point to B''. + +Notice how similar this is to the commit graph. So, "fast-forward" +merging the same B commit from origin/master will lead to an identical +sha for B' as the original committer got. Since the adjusted/master branch is not present on the remote, if the user does a `git pull`, it won't merge in changes from origin/master. Which is good because the filter needs to be applied first. -[WORKTREE: `git pull` would update the work tree, and may lead to conflicts -between the adjusted work tree and pulled changes. A post-merge hook would -be needed to re-adjust the work tree, and there would be a window where eg, -not present files would appear in the work tree.] - However, if the user does `git merge origin/master`, they'll get into a state where the filter has not been applied. The post-merge hook could be used to clean up after that. Or, let the user foot-shoot this way; they can always reset back once they notice the mistake. +[WORKTREE: `git pull` would update the work tree, and may lead to conflicts +between the adjusted work tree and pulled changes. A post-merge hook would +be needed to re-adjust the work tree, and there would be a window where eg, +not present files would appear in the work tree.] + ## annex object add/remove When objects are added/removed from the annex, the associated file has to @@ -57,59 +185,35 @@ These changes would need to be committed to the adjusted branch, otherwise [WORKTREE: Simply adjust the work tree (and index) per the filter.] -## commit - -When committing changes, a commit is made as usual to the adjusted branch. -So, the user can `git commit` (or `git annex sync`). This does not touch -the original branch yet. - -Then we need to get from that commit to one with the filters reversed, -which should be the same as if the adjusted branch had not been used. -This commit gets added onto the original branch. - -So, the branches would look like this: - - master adjusted/master - A ---filter----> A - | | - | A' - | | - | B' - B <--rev filter- | - | B - | ---filter----> | - | B'' - -Note particularly that B does not have A' in its history; the adjusted -branch is not evident from outside. So, we need a way to detect commits -like A'. - -Also note that B gets merged back to the adjusted branch, re-applying the -filter. This will make other checkouts that are in the same adjusted branch -end up with the same B'' commit when they pull B. - -It might be useful to have a post-commit hook that generates the -reverse-filtered commit and updates the original branch. And/or -`git-annex sync` could do it. - -[WORKTREE: A pre-commit hook would be needed to update the staged changes, -reversing the filter before the commit is made. All the other complications -above are avoided.] - ## reverse filtering Reversing filter #1 would mean only converting pointer files to symlinks when the file was originally a symlink. This is problimatic when a file is renamed. Would it be ok, if foo is renamed to bar and bar is committed, for it to be committed as an unlocked file, even if foo was -originally locked? +originally locked? Probably. Reversing filter #2 would mean not deleting removed files whose content was not present. When the commit includes deletion of files that were removed due to their content not being present, those deletions are not propigated. When the user deletes an unlocked file, the content is still present in annex, so reversing the filter should propigate the file -deletion. +deletion. + +What if an object was sent to the annex (or removed from the annex) +after the commit and before the reverse filtering? This would cause the +reverse filter to draw the wrong conclusion. Maybe look at a list of what +objects were not present when applying the filter, and use that to decide +which to not delete when reversing it? + +So, a reverse filter may need some state that was collected when running +the filter forwards, in order to decide what to do. + +Alternatively, instead of reverse filtering the whole adjusted tree, +look at just the new commit that's being propigated back from the +adjusted to master branch. Get the diff from it to the previous +commit; the changes that were made. Then de-adjust those changes, +and apply the changes to the master branch. ## push @@ -178,6 +282,8 @@ adjusting filter, albeit an extreme one. This might improve view branches. For example, it's not currently possible to update a view branch with changes fetched from a remote, and this could get us there. +This would need the reverse filter to be able to change metadata. + [WORKTREE: Wouldn't be able to integrate, unless view branches are changed into adjusted view worktrees.] @@ -220,5 +326,3 @@ like this, at its most simple: -- Generate a version of the commit made on the filter branch -- with the filtering of modified files reversed. unfilteredCommit :: Filter -> Git.Commit -> Git.Commit - - isFilteredCommit :: Git.Commit -> Bool diff --git a/doc/design/adjusted_branches/comment_1_5bdbf54ae75f77c69c8f557f4dbac13d._comment b/doc/design/adjusted_branches/comment_1_5bdbf54ae75f77c69c8f557f4dbac13d._comment new file mode 100644 index 000000000..36f2db306 --- /dev/null +++ b/doc/design/adjusted_branches/comment_1_5bdbf54ae75f77c69c8f557f4dbac13d._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="https://me.yahoo.com/a/EbvxpTI_xP9Aod7Mg4cwGhgjrCrdM5s-#7c0f4" + subject="would it be possible to get a branch only with annexed content?" + date="2016-02-25T03:04:26Z" + content=""" +I am yet to grasp all the glorious plans here, but wondered to ask: it seems like it then should be possible to establish a branch only with annexed archive? may be matching some preferred content expression? +Usecase: a relatively large git/annex repository mixing in code, large data files, and some pre-built binaries. If I could seamlessly and reproducibly (with the progress of their master) tease those apart (for separate Debian packaging ;)), that would be really handy + +[[!meta author=yoh]] +"""]] diff --git a/doc/design/assistant/polls/prioritizing_special_remotes.mdwn b/doc/design/assistant/polls/prioritizing_special_remotes.mdwn index 36aa2b83f..43fd2bed3 100644 --- a/doc/design/assistant/polls/prioritizing_special_remotes.mdwn +++ b/doc/design/assistant/polls/prioritizing_special_remotes.mdwn @@ -6,7 +6,7 @@ locally paired systems, and remote servers with rsync. Help me prioritize my work: What special remote would you most like to use with the git-annex assistant? -[[!poll open=yes 18 "Amazon S3 (done)" 13 "Amazon Glacier (done)" 10 "Box.com (done)" 75 "My phone (or MP3 player)" 27 "Tahoe-LAFS" 16 "OpenStack SWIFT" 37 "Google Drive"]] +[[!poll open=yes 18 "Amazon S3 (done)" 13 "Amazon Glacier (done)" 10 "Box.com (done)" 76 "My phone (or MP3 player)" 27 "Tahoe-LAFS" 16 "OpenStack SWIFT" 37 "Google Drive"]] This poll is ordered with the options I consider easiest to build listed first. Mostly because git-annex already supports them and they diff --git a/doc/devblog/day_360__annex.largefiles_mimetype/comment_2_78a22b96a82f76ab7217d37918a19024._comment b/doc/devblog/day_360__annex.largefiles_mimetype/comment_2_78a22b96a82f76ab7217d37918a19024._comment new file mode 100644 index 000000000..3e6f070a5 --- /dev/null +++ b/doc/devblog/day_360__annex.largefiles_mimetype/comment_2_78a22b96a82f76ab7217d37918a19024._comment @@ -0,0 +1,9 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 2""" + date="2016-02-19T15:53:46Z" + content=""" +Don't know, unless your system somehow has the magic C library installed +but no magic database file installed. You should file a bug report with +details about your system and how you installed the magic C library. +"""]] diff --git a/doc/devblog/day_360__annex.largefiles_mimetype/comment_3_3145e5843e20a037351e5a14fa3d6e9a._comment b/doc/devblog/day_360__annex.largefiles_mimetype/comment_3_3145e5843e20a037351e5a14fa3d6e9a._comment new file mode 100644 index 000000000..659f02f94 --- /dev/null +++ b/doc/devblog/day_360__annex.largefiles_mimetype/comment_3_3145e5843e20a037351e5a14fa3d6e9a._comment @@ -0,0 +1,17 @@ +[[!comment format=mdwn + username="davho" + subject="Why would it fail?" + date="2016-02-19T15:49:00Z" + content=""" +Yeah! I've waited for this option for a half a year :-) Thumbs up for finally including it! + +Doesn't work out of the box, though; I do have the MagicMime build flag but it fails to add files now! + +I get this error message for every file I try to add: + + /etc/magic, 0: Warning: using regular magic file `/usr/share/misc/magic' + + git-annex: user error (magicLoadDefault: could not find any valid magic files!) + failed + +"""]] diff --git a/doc/devblog/day_363__snow_day.mdwn b/doc/devblog/day_363__snow_day.mdwn new file mode 100644 index 000000000..4744816da --- /dev/null +++ b/doc/devblog/day_363__snow_day.mdwn @@ -0,0 +1,11 @@ +Made a `no-cbits` branch that removes several things that use C code and +the FFI. I moved one of them out to a new haskell library, +<http://hackage.haskell.org/package/mountpoints>. Others were replaced with +other existing libraries. This will simplify git-annex's build process, and +more library use is good. Planning to merge this branch in a week or two. + +v6 unlocked files don't work on Windows. I had assumed that since the build +was succeeding, the test suite was passing there. But, it turns out the +test suite was failing and somehow not failing the build. Have now fixed +several problems with v6 on Windows. Still a couple test suite problems to +address. diff --git a/doc/devblog/day_364__more_v6_unlocked.mdwn b/doc/devblog/day_364__more_v6_unlocked.mdwn new file mode 100644 index 000000000..38b053136 --- /dev/null +++ b/doc/devblog/day_364__more_v6_unlocked.mdwn @@ -0,0 +1,8 @@ +In a v6 repository on a filesystem not supporting symlinks, +it makes sense for commands like `git annex add` and `git annex import` +to add the files unlocked, since locked files are not usable there. +After implementing that, I also added an `annex.addunlocked` config setting, +so that the same behavior can be configured in other repositories. + +Rest of the day was spent fixing up the test suite's v6 repository tests +to work on FAT and Windows. diff --git a/doc/devblog/day_365__some_kind_of_milestone.mdwn b/doc/devblog/day_365__some_kind_of_milestone.mdwn new file mode 100644 index 000000000..fdbd4f610 --- /dev/null +++ b/doc/devblog/day_365__some_kind_of_milestone.mdwn @@ -0,0 +1,6 @@ +Should mention that there was a release two days ago. The main reason for +the timing of that release is because the Linux wstandalone builds include +glibc, which recently had a nasty security hole and had to be updated. + +Today, fixed a memory leak, and worked on getting caught up with backlog, +which now stands at 112 messages. diff --git a/doc/devblog/day_366__starting_adjusted_branches.mdwn b/doc/devblog/day_366__starting_adjusted_branches.mdwn new file mode 100644 index 000000000..3f50d76bf --- /dev/null +++ b/doc/devblog/day_366__starting_adjusted_branches.mdwn @@ -0,0 +1,33 @@ +Getting started on [[design/adjusted_branches]], taking a top-down and +bottom-up approach. Yesterday I worked on improving the design. Today, +built a `git mktree` interface that supports recursive tree generation and +filtering, which is the low-level core of what's needed to implement the +adjusted branches. + +To test that, wrote a fun program that generates a git tree with all +the filenames reversed. + +[[!format haskell """ +import Git.Tree +import Git.CurrentRepo +import Git.FilePath +import Git.Types +import System.FilePath + +main = do + r <- Git.CurrentRepo.get + (Tree t, cleanup) <- getTree (Ref "HEAD") r + print =<< recordTree r (Tree (map reverseTree t)) + cleanup + +reverseTree :: TreeContent -> TreeContent +reverseTree (TreeBlob f m s) = TreeBlob (reverseFile f) m s +reverseTree (RecordedSubTree f s l) = NewSubTree (reverseFile f) (map reverseTree l) + +reverseFile :: TopFilePath -> TopFilePath +reverseFile = asTopFilePath . joinPath . map reverse . splitPath . getTopFilePath +"""]] + +Also, fixed problems with the Android, Windows, and OSX builds today. +Made a point release of the OSX dmg, because the last several releases +of it will SIGILL on some hardware. diff --git a/doc/devblog/day_367__adjusted_branches_proof_of_concept.mdwn b/doc/devblog/day_367__adjusted_branches_proof_of_concept.mdwn new file mode 100644 index 000000000..80576005a --- /dev/null +++ b/doc/devblog/day_367__adjusted_branches_proof_of_concept.mdwn @@ -0,0 +1,18 @@ +Now I have a proof of concept [[design/adjusted_branches]] +implementation, that creates a branch where all locked files +are adjusted to be unlocked. It works! + +Building the adjusted branch is pretty fast; around 2 thousand files +per second. And, I have a trick in my back pocket that could double that +speed. It's important this be quite fast, because it'll be done often. + +Checking out the adjusted branch can be bit slow though, since git runs +`git annex smudge` once per unlocked file. So that might need to be +optimised somehow. On the other hand, this should be done only rarely. + +I like that it generates reproducible git commits so the same adjustments +of the same branch will always have the same sha, no matter when and where +it's done. Implementing that involved parsing git commit objects. + +Next step will be merging pulled changes into the adjusted branch, while +maintaining the desired adjustments. diff --git a/doc/devblog/day_368__leap.mdwn b/doc/devblog/day_368__leap.mdwn new file mode 100644 index 000000000..69766b5e3 --- /dev/null +++ b/doc/devblog/day_368__leap.mdwn @@ -0,0 +1,9 @@ +Pushed out a release today, could not resist the leap day in the version +number, and also there were enough bug fixes accumulated to make it worth +doing. + +I now have `git-annex sync` working inside adjusted branches, so pulls +get adjusted appropriately before being merged into the adjusted branch. +Seems to mostly work well, I did just find one bug in it though. Only +propigating adjusted commits remains to be done to finish my adjusted +branches prototype. diff --git a/doc/devblog/day_369-370__paddling_furiously.mdwn b/doc/devblog/day_369-370__paddling_furiously.mdwn new file mode 100644 index 000000000..532107900 --- /dev/null +++ b/doc/devblog/day_369-370__paddling_furiously.mdwn @@ -0,0 +1,72 @@ +Tuesday was spent dealing with lock files. Turned out there were some bugs +in the `annex.pidlock` configuration that prevented it from working, and +could even lead to data loss. + +And then more lock files today, since I needed to lock git's index file the +same way git does. This involved finding out how to emulate `O_EXCL` under +Windows. Urgh. + +Finally got back to working on [[design/adjusted_branches]] today. And, I've just +gotten syncing of commits from adjusted branches back to the orginal branch +working! Time for short demo of what I've been building for the past couple +weeks: + + joey@darkstar:~/tmp/demo>ls -l + total 4 + lrwxrwxrwx 1 joey joey 190 Mar 3 17:09 bigfile -> .git/annex/objects/zx/X8/SHA256E-s1048576--44ee9fdd91d4bc567355f8b2becd5fe137b9e3aafdfe804341ce2bcc73b8013f/SHA256E-s1048576--44ee9fdd91d4bc567355f8b2becd5fe137b9e3aafdfe804341ce2bcc73b8013f + joey@darkstar:~/tmp/demo>git annex adjust + Switched to branch 'adjusted/master(unlocked)' + ok + joey@darkstar:~/tmp/demo#master(unlocked)>ls -l + total 4 + -rw-r--r-- 1 joey joey 1048576 Mar 3 17:09 bigfile + +Entering the adjusted branch unlocked all the files. + + joey@darkstar:~/tmp/demo#master(unlocked)>git mv bigfile newname + joey@darkstar:~/tmp/demo#master(unlocked)>git commit -m rename + [adjusted/master(unlocked) 29e1bc8] rename + 1 file changed, 0 insertions(+), 0 deletions(-) + rename bigfile => newname (100%) + joey@darkstar:~/tmp/demo#master(unlocked)>git log --pretty=oneline + 29e1bc835080298bbeeaa4a9faf42858c050cad5 rename + a195537dc5beeee73fc026246bd102bae9770389 git-annex adjusted branch + 5dc1d94d40af4bf4a88b52805e2a3ae855122958 add + joey@darkstar:~/tmp/demo#master(unlocked)>git log --pretty=oneline master + 5dc1d94d40af4bf4a88b52805e2a3ae855122958 add + +The commit was made on top of the commit that generated the adjusted branch. +It's not yet reached the master branch. + + joey@darkstar:~/tmp/demo#master(unlocked)>git annex sync + commit ok + joey@darkstar:~/tmp/demo#master(unlocked)>git log --pretty=oneline + b60c5d6dfe55107431b80382596f14f4dcd259c9 git-annex adjusted branch + 9c36848f078a2bb7a304010e962a2b7318c0877c rename + 5dc1d94d40af4bf4a88b52805e2a3ae855122958 add + joey@darkstar:~/tmp/demo#master(unlocked)>git log --pretty=oneline master + 9c36848f078a2bb7a304010e962a2b7318c0877c rename + 5dc1d94d40af4bf4a88b52805e2a3ae855122958 add + +Now the commit has reached master. Notice how the history of the adjusted +branch was rebased on top of the updated master branch as well. + + joey@darkstar:~/tmp/demo#master(unlocked)>ls -l + total 1024 + -rw-r--r-- 1 joey joey 1048576 Mar 3 17:09 newname + joey@darkstar:~/tmp/demo#master(unlocked)>git checkout master + Switched to branch 'master' + joey@darkstar:~/tmp/demo>ls -l + total 4 + lrwxrwxrwx 1 joey joey 190 Mar 3 17:12 newname -> .git/annex/objects/zx/X8/SHA256E-s1048576--44ee9fdd91d4bc567355f8b2becd5fe137b9e3aafdfe804341ce2bcc73b8013f/SHA256E-s1048576--44ee9fdd91d4bc567355f8b2becd5fe137b9e3aafdfe804341ce2bcc73b8013f + +Just as we'd want, the file is locked in master, and unlocked in +the adjusted branch. + +(Not shown: git annex sync will also merge in and adjust changes from remotes.) + +So, that all looks great! But, it's cheating a bit, because it locks +all files when updating the master branch. I need to make it remember, +somehow, when files were originally unlocked, and keep them unlocked. Also +want to implement other adjustments, like hiding files whose content is not +present. diff --git a/doc/forum/Help_with_assistant_and_Adobe_Lightroom.mdwn b/doc/forum/Help_with_assistant_and_Adobe_Lightroom.mdwn new file mode 100644 index 000000000..d3cf4e7a1 --- /dev/null +++ b/doc/forum/Help_with_assistant_and_Adobe_Lightroom.mdwn @@ -0,0 +1,3 @@ +I am trying to use the git-annex assistant to automatically sync photos from my Lightroom catalog on Mac OS X to a remote archive server. I hit a snag due to a race condition combined with some peculiar behavior of Adobe Lightroom: it dereferences symlinks and uses the name of the target when importing (see https://forums.adobe.com/thread/568863?tstart=0) I observe the following problematic workflow: 1. Files are imported using the original filename. 2. The assistant eventually moves the content to .git/annex/objects and creates symlinks to replace the originals. 3. The photos no longer appear in the Library, however Lightroom will not allow me to re-import as it thinks the files have already been imported and are duplicates. + +I know a few other folks are using git-annex and Lightroom together, but I cannot tell if they are using the assistant or manually managing the annex. This implementation is for my wife who is not technical enough to use git on the command line so that's out of the question. It seems like direct mode is my best option for now, however, I'd like to avoid it since it's being deprecated. Is there some other option that I'm missing? Is anyone else out there successfully using the assistant and Lightroom together? I'd love some advice. diff --git a/doc/forum/How_to_improve_performance_with_v6_repos__63__.mdwn b/doc/forum/How_to_improve_performance_with_v6_repos__63__.mdwn new file mode 100644 index 000000000..6a4f5e8cf --- /dev/null +++ b/doc/forum/How_to_improve_performance_with_v6_repos__63__.mdwn @@ -0,0 +1,3 @@ +I've been using git-annex to manage a repo of ~4000 files (an MP3 collection). I used to use direct mode, but switched to v6 and am using unlocked files instead. However, even small commits have been taking 10-20 minutes. Looking at `ps` output shows `git-annex smudge` processes running against all of the files in the repo, one by one. + +Is there any way I can speed things up? diff --git a/doc/forum/How_to_improve_performance_with_v6_repos__63__/comment_1_56e0fead15a88ea12262e6091972c6d6._comment b/doc/forum/How_to_improve_performance_with_v6_repos__63__/comment_1_56e0fead15a88ea12262e6091972c6d6._comment new file mode 100644 index 000000000..c5f1866b1 --- /dev/null +++ b/doc/forum/How_to_improve_performance_with_v6_repos__63__/comment_1_56e0fead15a88ea12262e6091972c6d6._comment @@ -0,0 +1,7 @@ +[[!comment format=mdwn + username="page" + subject="Some additional notes" + date="2016-02-28T11:44:39Z" + content=""" +Just to add some info here. I just started using git-annex and v6 sounded like the way to go. I tried to add my photo collection (around 50k files) and all operations take a huge amount of time. I did 'git add photos/' (to add them in unlocked mode), which took hours. Then I run 'git annex sync', which took >10h again to do the commit. After that i run 'git annex sync' from another remote, and it's syncing items at ~2/s (smudge processes as the op mentions, not blocked either by cpu nor io). I'm not sure if I'm doing something wrong, but this is completely unusable for me. I'm all ears if I can try something, otherwise I'm gonna give v5 a try, I guess. +"""]] diff --git a/doc/forum/How_to_shrink_transfer_repo__63__.mdwn b/doc/forum/How_to_shrink_transfer_repo__63__.mdwn new file mode 100644 index 000000000..8e368087f --- /dev/null +++ b/doc/forum/How_to_shrink_transfer_repo__63__.mdwn @@ -0,0 +1,27 @@ +Hello, + +I have two repositories (Asaru and Horus) that are both ```group=client``` and ```wanted=standard```. The other one, Astarte is ```group=transfer``` and ```wanted=standard```. Pretty standard I think. + +``` +repository mode: direct +trusted repositories: 0 +semitrusted repositories: 5 + 00000000-0000-0000-0000-000000000001 -- web + 00000000-0000-0000-0000-000000000002 -- bittorrent + 58001764-966d-4076-ae99-4ef6de25df39 -- Asaru [here] + 8165bdf1-907e-4bbe-9c35-22fbf6f8cb00 -- Astarte [astarte] + cca0c3c8-593a-4395-936c-1093f0f762e8 -- Horus +untrusted repositories: 0 +``` + +I always sync on the two client repos like that ```git annex add . && git annex sync --content```. The transfer repo is growing larger and larger. ```git annex dropunused N``` says, that it ```could only verify the existence of 0 out of 1 necessary copies```. + +What is the best way to clean up the transfer repo? + +1. Make the two client repos trusted? The three repos have been created manually, not through the assistant. Is that what the assistant does, too? +2. Try to get the two client repos into touch with each other and try to use ```dropunsed --from=astarte```? + +What is the recommended way for that? + +Thanks, +Florian diff --git a/doc/forum/How_to_shrink_transfer_repo__63__/comment_1_6b110311f4c147dc315c4f610cf56afa._comment b/doc/forum/How_to_shrink_transfer_repo__63__/comment_1_6b110311f4c147dc315c4f610cf56afa._comment new file mode 100644 index 000000000..5c1c71475 --- /dev/null +++ b/doc/forum/How_to_shrink_transfer_repo__63__/comment_1_6b110311f4c147dc315c4f610cf56afa._comment @@ -0,0 +1,19 @@ +[[!comment format=mdwn + username="Horus" + subject="comment 1" + date="2016-03-03T18:17:35Z" + content=""" +Ok, I try to do 2) but it still fails: + +``` +dropunused 265 (from astarte...) (unsafe) + Could only verify the existence of 0 out of 1 necessary copies + + Rather than dropping this file, try using: git annex move + + (Use --force to override this check, or adjust numcopies.) +failed +``` + +all repositories are reachable directly. +"""]] diff --git a/doc/forum/Is_there_any_delta_handling_for_file_modifications__63__/comment_2_e46cb391ce377c3a8eb40155a5d0f001._comment b/doc/forum/Is_there_any_delta_handling_for_file_modifications__63__/comment_2_e46cb391ce377c3a8eb40155a5d0f001._comment new file mode 100644 index 000000000..2883d29e3 --- /dev/null +++ b/doc/forum/Is_there_any_delta_handling_for_file_modifications__63__/comment_2_e46cb391ce377c3a8eb40155a5d0f001._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="git-annex@c063eec406e857662ac6bbb814defa6aebde6417" + nickname="git-annex" + subject="any plans to add this?" + date="2016-02-21T12:12:43Z" + content=""" +Presumably this is a desirable feature for faster syncing of modifications to large files, so are there any plans to add it? +"""]] diff --git a/doc/forum/Multiple_remotes_with_the_same_path.mdwn b/doc/forum/Multiple_remotes_with_the_same_path.mdwn new file mode 100644 index 000000000..55d9ee1df --- /dev/null +++ b/doc/forum/Multiple_remotes_with_the_same_path.mdwn @@ -0,0 +1,67 @@ +This is a followup to <https://git-annex.branchable.com/forum/basic_usage_questions/> – I'm actually testing the "two repos at same URL" situation now, with git-annex 6.20160211 and 6.20160221. + +So I have a git-annex repo on two hosts at `rain:~/Attic/Software`, `frost:~/Attic/Software`, and a third clone at `/mnt/portable_HD/Attic/Software`. This means that the repo on the portable HD has two remotes with identical paths, but corresponding to different repositories: + + rain /home/grawity/Attic/Software + frost /home/grawity/Attic/Software + +[I was told earlier](/forum/basic_usage_questions) that this configuration would work fine and git-annex would not get confused. However, that doesn't seem to be the case (unless I misunderstood what it considers to be the "right thing"?) I see that whenever I run a command like `git annex info` on the portable\_HD repo, it overrides `remote.{rain,frost}.annex-uuid` with whatever UUID it sees right now – resulting in `git annex info` output such as: + + ┌ frost /run/media/grawity/vol4_grimoire/Attic/Software master + ┘ git config -l | grep annex-uuid + remote.origin.annex-uuid=0ebc2083-f95e-4637-bd3e-09db8471daf3 + remote.rain.annex-uuid=3e342a37-6c35-40e5-99a1-1f140e6c363d <-- + remote.frost.annex-uuid=524b8690-9b3e-48d4-b1a8-c0edb35c1ccf <-- + remote.fs1.annex-uuid=ed83c81c-1a95-4acb-b76c-b6724fe88873 + + ┌ frost /run/media/grawity/vol4_grimoire/Attic/Software master + ┘ git annex info --fast --verbose --debug + repository mode: indirect + trusted repositories: [2016-02-29 08:09:29.284136] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","git-annex"] + [2016-02-29 08:09:29.286872] process done ExitSuccess + [2016-02-29 08:09:29.286954] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","--hash","refs/heads/git-annex"] + [2016-02-29 08:09:29.289167] process done ExitSuccess + [2016-02-29 08:09:29.289316] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","log","refs/heads/git-annex..fc873fc472a8c5d0db3c91a3868d9adc072f7076","-n1","--pretty=%H"] + [2016-02-29 08:09:29.291836] process done ExitSuccess + [2016-02-29 08:09:29.291941] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","log","refs/heads/git-annex..23c27db2f2a02515ba39e7d0bb8653fa786b6aea","-n1","--pretty=%H"] + [2016-02-29 08:09:29.294762] process done ExitSuccess + [2016-02-29 08:09:29.294869] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","log","refs/heads/git-annex..690a90da7fc28be67d5053c0a6c3050cca614eaa","-n1","--pretty=%H"] + [2016-02-29 08:09:29.301018] process done ExitSuccess + [2016-02-29 08:09:29.30112] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","log","refs/heads/git-annex..94375fb892d1c9f5b70bdd5917c85c5c18a13168","-n1","--pretty=%H"] + [2016-02-29 08:09:29.3031] process done ExitSuccess + [2016-02-29 08:09:29.303182] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","log","refs/heads/git-annex..4a7f17e644e86776b510606beb6f3cb3819d8256","-n1","--pretty=%H"] + [2016-02-29 08:09:29.304916] process done ExitSuccess + [2016-02-29 08:09:29.305455] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","cat-file","--batch"] + [2016-02-29 08:09:29.308946] read: git ["config","--null","--list"] + [2016-02-29 08:09:29.312347] process done ExitSuccess + [2016-02-29 08:09:29.312939] read: git ["config","--null","--list"] + [2016-02-29 08:09:29.3179] process done ExitSuccess + [2016-02-29 08:09:29.318494] call: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","config","remote.rain.annex-uuid","524b8690-9b3e-48d4-b1a8-c0edb35c1ccf"] + [2016-02-29 08:09:29.320648] process done ExitSuccess + [2016-02-29 08:09:29.320735] read: git ["config","--null","--list"] + [2016-02-29 08:09:29.327697] process done ExitSuccess + 0 + semitrusted repositories: 9 + 00000000-0000-0000-0000-000000000001 -- web + 00000000-0000-0000-0000-000000000002 -- bittorrent + 0ebc2083-f95e-4637-bd3e-09db8471daf3 -- origin + 3e14eb23-1b16-4b52-8798-56efb550ab00 -- [vol4_grimoire]:/Attic/Software [here] + 3e342a37-6c35-40e5-99a1-1f140e6c363d -- grawity@rain:~/Attic/Software <-- + 524b8690-9b3e-48d4-b1a8-c0edb35c1ccf -- grawity@frost:~/Downloads/Software [rain] <-- + b9c0c485-07e5-4166-b5ac-ba971faab98a -- [vol3_tombstone]:/Attic/Software + c3c6dd39-ebc7-475a-9992-f99ca01a7f3a -- grawity@wolke:~/Attic/Software + ed83c81c-1a95-4acb-b76c-b6724fe88873 -- fs1:/Attic/Software [fs1] + untrusted repositories: 0 + transfers in progress: none + available local disk space: 920.96 gigabytes (+1 megabyte reserved) + + ┌ frost /run/media/grawity/vol4_grimoire/Attic/Software master + ┘ git config -l | grep annex-uuid + remote.origin.annex-uuid=0ebc2083-f95e-4637-bd3e-09db8471daf3 + remote.rain.annex-uuid=524b8690-9b3e-48d4-b1a8-c0edb35c1ccf <-- + remote.frost.annex-uuid=524b8690-9b3e-48d4-b1a8-c0edb35c1ccf <-- + remote.fs1.annex-uuid=ed83c81c-1a95-4acb-b76c-b6724fe88873 + +Notice how the `[rain]` remote tag now shows up next to the wrong remote. Among other things, **this means I cannot use `git annex find --in frost --not --in rain`** and similar commands (unless I specify all the UUIDs by hand). And even though I trust git-annex to not lose any data, this behavior is a bit confusing. + +Is it possible that git-annex could detect such situations and avoid updating an UUID if it's _already attached_ to another remote? diff --git a/doc/forum/Multiple_remotes_with_the_same_path/comment_1_78f2d8e3906c11ba5753478d08973bd2._comment b/doc/forum/Multiple_remotes_with_the_same_path/comment_1_78f2d8e3906c11ba5753478d08973bd2._comment new file mode 100644 index 000000000..5f9f7426a --- /dev/null +++ b/doc/forum/Multiple_remotes_with_the_same_path/comment_1_78f2d8e3906c11ba5753478d08973bd2._comment @@ -0,0 +1,18 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2016-02-29T15:47:39Z" + content=""" +git-annex is not getting confused; it's looking at the current content of +the remote and using that uuid. Failing to do so would constitute +confusion. + +If you want the remote to only work when the drive is plugged into a single +host, you need to make the remote path only reach a repository on that +host. One way would be a symlink, eg `/home/grawity/Attic/rain-Software -> +Software` on rain. + +Or, just use the repository description, eg +`--in grawity@rain:~/Attic/Software`. Note that the description can be +changed to something easier to type. +"""]] diff --git a/doc/forum/Multiple_remotes_with_the_same_path/comment_2_8cd3edf2e71e904f1b651abdfd7a4499._comment b/doc/forum/Multiple_remotes_with_the_same_path/comment_2_8cd3edf2e71e904f1b651abdfd7a4499._comment new file mode 100644 index 000000000..f18404a6d --- /dev/null +++ b/doc/forum/Multiple_remotes_with_the_same_path/comment_2_8cd3edf2e71e904f1b651abdfd7a4499._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="grawity@2ea26be48562f66fcb9b66307da72b1e2e37453f" + nickname="grawity" + subject="comment 2" + date="2016-02-29T17:25:17Z" + content=""" +Hmm, I still think that avoiding duplicating uuids would be smarter behavior, but the host symlinks will do just fine. Thanks for the suggestion. +"""]] diff --git a/doc/forum/Undo_git_merge_git-annex.mdwn b/doc/forum/Undo_git_merge_git-annex.mdwn new file mode 100644 index 000000000..bc299174c --- /dev/null +++ b/doc/forum/Undo_git_merge_git-annex.mdwn @@ -0,0 +1,3 @@ +After accidentally typing git merge git-annex, I am now wondering how to clean up the resulting chaos... + +Any tips? diff --git a/doc/forum/Undo_git_merge_git-annex/comment_1_0b15f7a8c3bca87dcbf748a229e13a4b._comment b/doc/forum/Undo_git_merge_git-annex/comment_1_0b15f7a8c3bca87dcbf748a229e13a4b._comment new file mode 100644 index 000000000..bbc6117f7 --- /dev/null +++ b/doc/forum/Undo_git_merge_git-annex/comment_1_0b15f7a8c3bca87dcbf748a229e13a4b._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="frederik@ffbea6a549cb3f460d110386c0f634c1ddc6a68a" + nickname="frederik" + subject="comment 1" + date="2016-03-04T14:06:17Z" + content=""" +git revert -m 1 [sha_of_merge] seems to have done the trick +"""]] diff --git a/doc/git-annex-add.mdwn b/doc/git-annex-add.mdwn index 5f1ab33b0..b65ed5132 100644 --- a/doc/git-annex-add.mdwn +++ b/doc/git-annex-add.mdwn @@ -21,7 +21,8 @@ to the git repository, instead of to the annex. Large files are added to the annex in locked form, which prevents further modification of their content unless unlocked by [[git-annex-unlock]](1). -(This is not the case however when a repository is in direct mode.) +(This is not the case however when a repository is in a filesystem not +supporting symlinks, or is in direct mode.) To add a file to the annex in unlocked form, `git add` can be used instead (that only works when the repository has annex.version 6 or higher). diff --git a/doc/git-annex-matching-options.mdwn b/doc/git-annex-matching-options.mdwn index 34cc05c6a..d94c0faf1 100644 --- a/doc/git-annex-matching-options.mdwn +++ b/doc/git-annex-matching-options.mdwn @@ -115,6 +115,15 @@ file contents are present at either of two repositories. matches the glob. The values of metadata fields are matched case insensitively. +* `--metadata field<number` / `--metadata field>number` +* `--metadata field<=number` / `--metadata field>=number` + + Matches only files that have a metadata field attached with a value that + is a number and is less than or greater than the specified number. + + (Note that you will need to quote the second parameter to avoid + the shell doing redirection.) + * `--want-get` Matches files that the preferred content settings for the repository diff --git a/doc/git-annex-metadata.mdwn b/doc/git-annex-metadata.mdwn index 39bb7f2fd..448784b50 100644 --- a/doc/git-annex-metadata.mdwn +++ b/doc/git-annex-metadata.mdwn @@ -19,34 +19,39 @@ When run without any -s or -t parameters, displays the current metadata. # OPTIONS -* `-g field` +* `-g field` / `--get field` Get the value(s) of a single field. The values will be output one per line, with no other output, so this is suitable for use in a script. -* `-s field=value` +* `-s field=value` / `--set field=value` Set a field's value, removing any old values. -* `-s field+=value` +* `-s field+=value` / `--set field+=value` Add an additional value, preserving any old values. -* `-s field-=value` +* `-s field?=value` / `--set field?=value` - Remove a value. + Set a value, but only if the field does not already have a value set. -* `-s field?=value` +* `-s field-=value` / `--set field-=value` - Set a value, but only if the field does not already have a value set. - -* `-t tag` + Remove a value from a field, leaving any other values that the field has + set. + +* `-r field` / `--remove field` + + Remove all current values of the field. + +* `-t tag` / `--tag tag` Set a tag. Note that a tag is just a value of the "tag" field. -* `-u tag` +* `-u tag` / `--unset tag` Unset a tag. diff --git a/doc/git-annex-preferred-content.mdwn b/doc/git-annex-preferred-content.mdwn index bea82654f..5e2a7a9c5 100644 --- a/doc/git-annex-preferred-content.mdwn +++ b/doc/git-annex-preferred-content.mdwn @@ -119,6 +119,15 @@ elsewhere to allow removing it). To match author metadata, use `metadata=author=*Smith` +* `metadata=field<number` / `metadata=field>number` +* `metadata=field<=number` / `metadata=field>=number` + + Matches only files that have a metadata field attached with a value that + is a number and is less than or greater than the specified number. + + To match PDFs with between 100 and 200 pages (assuming something has set + that metadata), use `metadata=pagecount>=100 and metadata=pagecount<=200` + * `present` Makes content be wanted if it's present, but not otherwise. diff --git a/doc/git-annex-view.mdwn b/doc/git-annex-view.mdwn index 56c7e9b85..d8d0ce5b2 100644 --- a/doc/git-annex-view.mdwn +++ b/doc/git-annex-view.mdwn @@ -27,9 +27,9 @@ There are fields corresponding to the path to the file. So a file These location fields can be used the same as other metadata to construct the view. -For example, `/=podcasts` will only include files from the podcasts -directory in the view, while `podcasts/=*` will preserve the -subdirectories of the podcasts directory in the view. +For example, `/=foo` will only include files from the foo +directory in the view, while `foo/=*` will preserve the +subdirectories of the foo directory in the view. # SEE ALSO diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index cb27ec11a..6830f741f 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -823,6 +823,15 @@ Here are all the supported configuration settings. should be checked into git by `git annex add`. Defaults to true; set to false to instead make small files be skipped. +* `annex.addunlocked` + + Set to true to make commands like `git-annex add` that add files to the + repository add them in unlocked form. The default is to add files in + locked form. This only has effect in version 6 repositories. + + When a repository has core.symlinks set to false, it implicitly + sets annex.addunlocked to true. + * `annex.numcopies` This is a deprecated setting. You should instead use the diff --git a/doc/install/Homebrew-cask.mdwn b/doc/install/OSX/Homebrew-cask.mdwn index a7517f7ec..a7517f7ec 100644 --- a/doc/install/Homebrew-cask.mdwn +++ b/doc/install/OSX/Homebrew-cask.mdwn diff --git a/doc/news/version_5.20151218.mdwn b/doc/news/version_5.20151218.mdwn deleted file mode 100644 index a1543742b..000000000 --- a/doc/news/version_5.20151218.mdwn +++ /dev/null @@ -1,10 +0,0 @@ -git-annex 5.20151218 released with [[!toggle text="these changes"]] -[[!toggleable text=""" - * Add S3 features to git-annex version output. - * webdav: When testing the WebDAV server, send a file with content. - The empty file it was sending tickled bugs in some php WebDAV server. - * fsck: Failed to honor annex.diskreserve when checking a remote. - * Debian: Build depend on concurrent-output. - * Fix insecure temporary permissions when git-annex repair is used in - in a corrupted git repository. - * Fix potential denial of service attack when creating temp dirs."""]]
\ No newline at end of file diff --git a/doc/news/version_6.20160217.mdwn b/doc/news/version_6.20160217.mdwn new file mode 100644 index 000000000..598f11f3b --- /dev/null +++ b/doc/news/version_6.20160217.mdwn @@ -0,0 +1,22 @@ +git-annex 6.20160217 released with [[!toggle text="these changes"]] +[[!toggleable text=""" + * Support getting files from read-only repositories. + * checkpresentkey: Allow to be run without an explicit remote. + * checkpresentkey: Added --batch. + * Work around problem with concurrent-output when in a non-unicode locale + by avoiding use of it in such a locale. Instead -J will behave as if + it was built without concurrent-output support in this situation. + * Fix storing of filenames of v6 unlocked files when the filename is not + representable in the current locale. + * fsck: Detect and fix missing associated file mappings in v6 repositories. + * fsck: Populate unlocked files in v6 repositories whose content is + present in annex/objects but didn't reach the work tree. + * When initializing a v6 repo on a crippled filesystem, don't force it + into direct mode. + * Windows: Fix v6 unlocked files to actually work. + * add, addurl, import, importfeed: When in a v6 repository on a crippled + filesystem, add files unlocked. + * annex.addunlocked: New configuration setting, makes files always be + added unlocked. (v6 only) + * Improve format of v6 unlocked pointer files to support keys containing + slashes."""]]
\ No newline at end of file diff --git a/doc/news/version_6.20160229.mdwn b/doc/news/version_6.20160229.mdwn new file mode 100644 index 000000000..a6708464f --- /dev/null +++ b/doc/news/version_6.20160229.mdwn @@ -0,0 +1,24 @@ +git-annex 6.20160229 released with [[!toggle text="these changes"]] +[[!toggleable text=""" + * Update perlmagick build dependency. Closes: #[789225](http://bugs.debian.org/789225) + * Fix memory leak in last release, which affected commands like + git-annex status when a large non-annexed file is present in the work + tree. + * fsck: When the only copy of a file is in a dead repository, mention + the repository. + * info: Mention when run in a dead repository. + * Linux and OSX standalone builds put the bundled gpg last in PATH, + so any system gpg will be preferred over it. + * Avoid crashing when built with MagicMime support, but when the magic + database cannot be loaded. + * Include magic database in the linux and OSX standalone builds. + * Fix memory leak when hashing files, which triggered during fsck + when an external hash program was not used. + (This leak was introduced in version 6.20160114.) + * Support --metadata field<number, --metadata field>number etc + to match ranges of numeric values. + * Similarly, support preferred content expressions like + metadata=field<number and metadata=field>number + * The pre-commit-annex hook script that automatically extracts + metadata has been updated to also use exiftool. + Thanks, Klaus Ethgen."""]]
\ No newline at end of file diff --git a/doc/preferred_content.mdwn b/doc/preferred_content.mdwn index f8c6ce6cd..d7b9870e5 100644 --- a/doc/preferred_content.mdwn +++ b/doc/preferred_content.mdwn @@ -67,3 +67,4 @@ they were added in. * "metadata=" 5.20140221 * "lackingcopies=", "approxlackingcopies=", "unused=" 5.20140127 * "inpreferreddir=" 4.20130501 +* "metadata=field<number" etc 6.20160227 diff --git a/doc/tips/automatically_adding_metadata.mdwn b/doc/tips/automatically_adding_metadata.mdwn index e6d02defa..b3118a75e 100644 --- a/doc/tips/automatically_adding_metadata.mdwn +++ b/doc/tips/automatically_adding_metadata.mdwn @@ -2,23 +2,48 @@ git-annex's [[metadata]] works best when files have a lot of useful metadata attached to them. To make git-annex automatically set the year and month when adding files, -run `git config annex.genmetadata true`. +run: `git config annex.genmetadata true` + +## git commit hook A git commit hook can be set up to extract lots of metadata from files -like photos, mp3s, etc. +like photos, mp3s, etc. Whenever annexed files are committed, their +metadata will be extracted and stored. + +Download [[pre-commit-annex]] and install it in your git-annex repository +as `.git/hooks/pre-commit-annex` +Remember to make the script executable! `chmod +x .git/hooks/pre-commit-annex` + +### using extract -1. Install the `extract` utility, from <http://www.gnu.org/software/libextractor/> - `apt-get install extract` -2. Download [[pre-commit-annex]] and install it in your git-annex repository - as `.git/hooks/pre-commit-annex`. - Remember to make the script executable! -3. Run: `git config metadata.extract "artist album title camera_make video_dimensions"` +The git commit hook can use extract to get metadata. -Now any fields you list in metadata.extract to will be extracted and -stored when files are committed. +Install it from <http://www.gnu.org/software/libextractor/> +`apt-get install extract` + +Configure which metadata fields to ask extract for: `git config metadata.extract "artist album title camera_make video_dimensions"` To get a list of all possible fields, run: `extract -L | sed 's/ /_/g'` +### using exiftool + +The git commit hook can also use exiftool to get metadata. + +Install it from <http://owl.phy.queensu.ca/~phil/exiftool/> +`apt-get install libimage-exiftool-perl` + +Configure which metadata fields to ask exiftool for: `git config metadata.exiftool "Model ImageSize FocusRange GPSAltitude GPSCoordinates"` + +To get a list of all possible fields, run: `exiftool -list` + +### using both extract and exiftool + +If you want some metadata that extract knows about, and other metadata +that exiftool knows about, just install them both, and set both +`metadata.extract` and `metadata.exiftool`. + +### overwriting existing metadata + By default, if a git-annex already has a metadata field for a file, its value will not be overwritten with metadata taken from files. To allow overwriting, run: `git config metadata.overwrite true` diff --git a/doc/tips/automatically_adding_metadata/pre-commit-annex b/doc/tips/automatically_adding_metadata/pre-commit-annex index 873284d2c..2e07e3bf4 100755 --- a/doc/tips/automatically_adding_metadata/pre-commit-annex +++ b/doc/tips/automatically_adding_metadata/pre-commit-annex @@ -1,72 +1,118 @@ #!/bin/sh # +# Copyright (C) 2014 Joey Hess <id@joeyh.name> +# Copyright (C) 2016 Klaus Ethgen <Klaus@Ethgen.ch> +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +# # This script can be used to add git-annex metadata to files when they're # committed. It is typically installed as .git/hooks/pre-commit-annex # # You can also run this script by hand, passing it the names of files # already checked into git-annex, and it will extract/refresh the git-annex # metadata from the files. -# -# Copyright 2014 Joey Hess <id@joeyh.name> -# License: GPL-3+ -extract="$(git config metadata.extract || true)" -want="$(perl -e 'print (join("|", map {s/_/ /g; "^$_ - "} (split " ", shift())))' "$extract")" +tool="$(git config metadata.tool || :)" +if [ -z "$tool" ]; then + tool=extract +fi +case "$tool" in + exiftool) + tool_exec="exiftool -unknown -zip -veryShort -ignoreMinorErrors -use MWG -dateFormat '%Y-%m-%dT%H:%M:%S'" + ;; + *) + tool_exec="$tool" + ;; +esac -if [ -z "$want" ]; then +extract_fields="$(git config metadata.extract || :)" +if [ -n "$extract_fields" ]; then + tools=extract + extract_want="^($(echo "$extract_fields" | sed -e 's/ /|/g' -e 's/_/ /g'))" +fi +exiftool_fields="$(git config metadata.exiftool || :)" +if [ -n "$exiftool_fields" ]; then + tools="exiftool $tools" + exiftool_want="^($(echo "$exiftool_fields" | sed -e 's/ /|/g' -e 's/_/ /g'))" +fi +if [ -z "$tools" ]; then exit 0 fi -case "$(git config --bool metadata.overwrite || true)" in +case "$(git config --bool metadata.overwrite || :)" in true) - overwrite=1 + equal="=" ;; *) - overwrite="" + equal="?=" ;; esac -addmeta () { - file="$1" - field="$2" - value="$3" - afield="$(echo "$field" | tr ' ' _)" - if [ "$overwrite" ]; then - p="$afield=$value" - - else - p="$afield?=$value" - fi - git -c annex.alwayscommit=false annex metadata "$file" -s "$p" --quiet -} - if git rev-parse --verify HEAD >/dev/null 2>&1; then - against=HEAD + against="HEAD" else # Initial commit: diff against an empty tree object - against=4b825dc642cb6eb9a060e54bf8d69288fbee4904 + against="4b825dc642cb6eb9a060e54bf8d69288fbee4904" fi -IFS=" -" +addmeta() { + file="$1" + field="$2" + value="$3" + afield="$(echo "$field" | tr ' ' '_')" + git -c annex.alwayscommit=false annex metadata \ + --set "$afield$equal$value" --quiet -- "$file" +} -process () { +process() { if [ -e "$f" ]; then echo "adding metadata for $f" - for l in $(extract "$f" | egrep "$want"); do - field="${l%% - *}" - value="${l#* - }" - addmeta "$f" "$field" "$value" + for tool in $tools; do + case "$tool" in + exiftool) + tool_exec="exiftool -unknown -zip -veryShort -ignoreMinorErrors -use MWG -dateFormat '%Y-%m-%dT%H:%M:%S'" + ;; + *) + tool_exec="$tool" + ;; + esac + LC_ALL=C $tool_exec "./$f" | eval egrep --text -i \""\$${tool}_want"\" | while read line; do + case "$tool" in + extract) + field="${line%% - *}" + value="${line#* - }" + ;; + exiftool) + field="${line%%: *}" + value="${line#*: }" + ;; + esac + + if [ -n "$value" ]; then + addmeta "$f" "$field" "$value" + fi + done done fi } if [ -n "$*" ]; then - for f in $@; do + for f in "$@"; do process "$f" done else - for f in $(git diff-index --name-only --cached $against); do + git diff-index --name-only --cached $against | while read f; do process "$f" done fi diff --git a/doc/tips/centralized_git_repository_tutorial/on_your_own_server/comment_1_ae5af47faf95f008f1b07dbed5181286._comment b/doc/tips/centralized_git_repository_tutorial/on_your_own_server/comment_1_ae5af47faf95f008f1b07dbed5181286._comment new file mode 100644 index 000000000..253410566 --- /dev/null +++ b/doc/tips/centralized_git_repository_tutorial/on_your_own_server/comment_1_ae5af47faf95f008f1b07dbed5181286._comment @@ -0,0 +1,28 @@ +[[!comment format=mdwn + username="tom_clune" + subject="git annex using the "wrong" ssh socket" + date="2016-02-24T19:23:13Z" + content=""" +To avoid frequent typing of pin + RSA passcode + password, we typically establish an ssh control master just once. This works fine with regular git commands, but the git-annex command apparently try to create a different socket. Even that would be ok, except that apparently it is a new socket each time we enter a command. + +With sufficient \"-vvvv\" we see things like: + + ... + debug1: Executing proxy command: exec ssh -l fred proxy.xxx.yyy direct host + ... + +(Note I have eliminated references to the actual machine names and userid's.) + +If the command had instead been: + + exec ssh -l fred proxy.xxx.yyy direct /home/fred/.ssh/master_host:22 + +everything would have worked fine. In fact, we are now using: + + git config remote.origin.annex-ssh-options '-S /home/fred/.ssh/master_host:22' + +and this eliminates the issue. But it would be nice if git annex could somehow automatically use +the pre-existing connection. Is there a better way to achieve this? + + +"""]] diff --git a/doc/tips/largefiles/comment_1_b84e5cb8316c30fc7b355234384a5899._comment b/doc/tips/largefiles/comment_1_b84e5cb8316c30fc7b355234384a5899._comment new file mode 100644 index 000000000..79374cd14 --- /dev/null +++ b/doc/tips/largefiles/comment_1_b84e5cb8316c30fc7b355234384a5899._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="acbg@8d58f794ce92f6be28b5dbd402059bc30262eb92" + nickname="acbg" + subject="Since which version does this apply?" + date="2016-03-03T16:16:28Z" + content=""" +I use version 5.20140412, and I've tried the annex.largefiles in .gitattributes, but doesn't work. Every file is added to .git/annex/objects, including the ones that are excluded in .gitattributes. + +I've just started using git-annex so maybe I'm doing something wrong... +"""]] diff --git a/doc/tips/unlocked_files.mdwn b/doc/tips/unlocked_files.mdwn index 31f9e9756..fd103940e 100644 --- a/doc/tips/unlocked_files.mdwn +++ b/doc/tips/unlocked_files.mdwn @@ -88,6 +88,13 @@ git-annex converts between the content of the big file and a pointer file, which is what gets committed to git. All the regular git-annex commands (get, drop, etc) can be used on unlocked files too. +[[!template id=note text=""" +By default, git-annex commands will add files in locked mode, +unless used on a filesystem that does not support symlinks, when unlocked +mode is used. To make them always use unlocked mode, run: +`git config annex.addunlocked true` +"""]] + A v6 repository can contain both locked and unlocked files. You can switch a file back and forth using the `git annex lock` and `git annex unlock` commands. This changes what's stored in git between a git-annex symlink diff --git a/doc/todo/Add_confirmation_dialog_to_the_restart_option.mdwn b/doc/todo/Add_confirmation_dialog_to_the_restart_option.mdwn new file mode 100644 index 000000000..1ff909b8c --- /dev/null +++ b/doc/todo/Add_confirmation_dialog_to_the_restart_option.mdwn @@ -0,0 +1,4 @@ +I have four git-annex repositories and I often use the option "Switch repository" in the webapp, but this option is just above the "Restart daemon" option and I have clicked it quite a few time unintentionally. +Could it be possible to add a confirmation dialog just like in the "Shutdown daemon" option? + +> spacer added; [[done]] --[[Joey]] diff --git a/doc/todo/Add_confirmation_dialog_to_the_restart_option/comment_1_00da59dc2d9b86a51eb462c481ab665d._comment b/doc/todo/Add_confirmation_dialog_to_the_restart_option/comment_1_00da59dc2d9b86a51eb462c481ab665d._comment new file mode 100644 index 000000000..ca60ec5fd --- /dev/null +++ b/doc/todo/Add_confirmation_dialog_to_the_restart_option/comment_1_00da59dc2d9b86a51eb462c481ab665d._comment @@ -0,0 +1,13 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2016-02-15T16:54:54Z" + content=""" +Restarting the daemon should just take a second and the web browser should +redirect to the new instance. I guess it does move away from the current +page to the dashboard, and in a way that the back button can't undo, but I +see no other harm in accidentially restarting the daemon. + +So, I think I'll just add a spacer to separate the repository part of that +menu from the daemon part. +"""]] diff --git a/doc/todo/add_magicmime_support_to_OSX_dmg.mdwn b/doc/todo/add_magicmime_support_to_OSX_dmg.mdwn new file mode 100644 index 000000000..9e4e8f387 --- /dev/null +++ b/doc/todo/add_magicmime_support_to_OSX_dmg.mdwn @@ -0,0 +1,3 @@ +The OSX .dmg is built without the MagicMime build flag. Turning it on will +take some work to ship a copy of the magic database inside the dmg. +--[[Joey]] diff --git a/doc/todo/cloning_direct_mode_repo_over_http.mdwn b/doc/todo/cloning_direct_mode_repo_over_http.mdwn index 947bf1e24..8f471f6e4 100644 --- a/doc/todo/cloning_direct_mode_repo_over_http.mdwn +++ b/doc/todo/cloning_direct_mode_repo_over_http.mdwn @@ -33,3 +33,4 @@ mappings would lead to faster code. So, this feature seems best blocked on the direct mode database either way! --[[Joey]] +[[!meta tag=deprecateddirectmode]] diff --git a/doc/todo/cloning_direct_mode_repo_over_http/comment_1_36701696af41f8b71bb2fd4829f05b7c._comment b/doc/todo/cloning_direct_mode_repo_over_http/comment_1_36701696af41f8b71bb2fd4829f05b7c._comment new file mode 100644 index 000000000..a1abe3f2b --- /dev/null +++ b/doc/todo/cloning_direct_mode_repo_over_http/comment_1_36701696af41f8b71bb2fd4829f05b7c._comment @@ -0,0 +1,13 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2016-02-19T19:55:20Z" + content=""" +v6 unlocked files don't have this problem; a repo containing them can be +accessed over http. + +So, I think that's a better way to go than complicating the http support +with direct mode stuff. Especially since direct mode is going away. + +Won't close this yet, but will when I get around to removing direct mode.. +"""]] diff --git a/doc/todo/git_annex_push.mdwn b/doc/todo/git_annex_push.mdwn index 079c83e46..4abdf253c 100644 --- a/doc/todo/git_annex_push.mdwn +++ b/doc/todo/git_annex_push.mdwn @@ -1 +1,6 @@ A common use case for me is to use annex as really just an addition to git to store additional content. What I am ending up with is two stage procedure to submit my changes to our shared repository: git push REMOTE; git annex copy --to=REMOTE . IMHO it would only be logical if there was "git annex push REMOTE [GITPUSHOPTIONS]" which would be merely an alias for "git push REMOTE [GITPUSHOPTIONS]; git annex copy --to=REMOTE" but which will make use of annex for such common usecase simple(r) + +> After this was opened, some options were added, so use: +> `git-annex sync --no-pull --content` +> +> [[done]] --[[Joey]] diff --git a/doc/todo/import_--reinject.mdwn b/doc/todo/import_--reinject.mdwn new file mode 100644 index 000000000..b34a2b1f9 --- /dev/null +++ b/doc/todo/import_--reinject.mdwn @@ -0,0 +1,5 @@ +There's `git annex reinject <src> <dst>` for re-adding one file's contents to the local annex. But what if I have a whole bunch of files, and want git-annex itself to decide whether & where it needs to reinject them? (And if the file doesn't need to be reinjected, it would remain in its original place.) + +None of the `git annex import` modes work properly in this case. By default, importing adds another, unnecessary copy of the imported file (which I have to `rm` after importing). The `--clean-duplicates` mode seems close, but it insists on verifying the content in other repositories rather than just reinjecting it locally. (Let's assume that the main reason I'm trying to reinject is that I cannot access other repos.) + +So I'm hoping for something like `git annex import --reinject <src>...`. Or are there other existing ways to achieve the same? I couldn't find any. diff --git a/doc/todo/import_--reinject/comment_1_659f0d18de4f5ee94adaf85ca106e196._comment b/doc/todo/import_--reinject/comment_1_659f0d18de4f5ee94adaf85ca106e196._comment new file mode 100644 index 000000000..2aeb49cfa --- /dev/null +++ b/doc/todo/import_--reinject/comment_1_659f0d18de4f5ee94adaf85ca106e196._comment @@ -0,0 +1,9 @@ +[[!comment format=mdwn + username="CandyAngel" + subject="comment 1" + date="2016-02-23T12:24:48Z" + content=""" +git-annex verifies the content in other repositories when you use --clean-duplicates because if it did not, it could delete the only copy of a file you had, because it was deleting files it knew about, but didn't have. + +As for what you are attempting, [maybe something like this](https://git-annex.branchable.com/tips/recover_data_from_lost+found/)? +"""]] diff --git a/doc/todo/import_--reinject/comment_2_04074324f866420ebf0d39ddfae85ff7._comment b/doc/todo/import_--reinject/comment_2_04074324f866420ebf0d39ddfae85ff7._comment new file mode 100644 index 000000000..eed8434b5 --- /dev/null +++ b/doc/todo/import_--reinject/comment_2_04074324f866420ebf0d39ddfae85ff7._comment @@ -0,0 +1,22 @@ +[[!comment format=mdwn + username="grawity@2ea26be48562f66fcb9b66307da72b1e2e37453f" + nickname="grawity" + subject="comment 2" + date="2016-03-01T07:10:55Z" + content=""" +Thanks, but you missed my point entirely... I wasn't asking for a mode that would delete data without checking. I was asking for the complete opposite – a mode that would _inject an extra copy_ of the data without checking. + +Yeah, I guess I could `annex add` the files, then un-annex them, and _then_ `annex import --clean-duplicates`, but that's a somewhat long-winded approach, needing twice the space and twice the time. + +(...speaking of losing data, it seems that `git annex reinject` is perfectly happy to delete files if I accidentally give it the wrong target. I.e. after failing content verification, it still throws away the source.) + +--- + +It doesn't have to be part of git-annex; I could _script_ this feature myself, though there aren't nearly enough plumbing commands either. (For example, a command to hash a file and give its key (like `git hash-object`), or a command to find all paths for a key.) + +Having an equivalent of `git hash-object -w` (inject an arbitrary object) would make it even easier, but I couldn't find anything like that either. + +--- + +Anyway, let's cancel this todo, I'll find other ways. +"""]] diff --git a/doc/todo/make_annex_info_more_efficient.mdwn b/doc/todo/make_annex_info_more_efficient.mdwn new file mode 100644 index 000000000..49a711965 --- /dev/null +++ b/doc/todo/make_annex_info_more_efficient.mdwn @@ -0,0 +1,3 @@ +ATM it takes about a minute for 'git annex info' on a sizeable but not huge repository with only ~450 files under annex but a few thousand of files (~7000) in the tree. I am not quite sure why it takes that long since it seems to care only about annexed files. Also it might be of benefit to parallelize some traversal operations to take advantage of multiple cpu/cores + +[[!meta author=yoh]] diff --git a/doc/todo/make_annex_info_more_efficient/comment_1_afd5c806f7285442401b027f82a8c629._comment b/doc/todo/make_annex_info_more_efficient/comment_1_afd5c806f7285442401b027f82a8c629._comment new file mode 100644 index 000000000..77090b680 --- /dev/null +++ b/doc/todo/make_annex_info_more_efficient/comment_1_afd5c806f7285442401b027f82a8c629._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://me.yahoo.com/a/EbvxpTI_xP9Aod7Mg4cwGhgjrCrdM5s-#7c0f4" + subject="comment 1" + date="2016-02-15T15:28:40Z" + content=""" +actually it does indeed relate with all the files, since \"annexed files in working tree\"... if interested -- try on this repo https://surfer.nmr.mgh.harvard.edu/pub/dist/freesurfer/repo/freesurfer.git . On a clean new clone it took only 30 seconds ;) + +"""]] diff --git a/doc/todo/make_annex_info_more_efficient/comment_2_0770ef5c4c261949a565723073480dcb._comment b/doc/todo/make_annex_info_more_efficient/comment_2_0770ef5c4c261949a565723073480dcb._comment new file mode 100644 index 000000000..16f0bea93 --- /dev/null +++ b/doc/todo/make_annex_info_more_efficient/comment_2_0770ef5c4c261949a565723073480dcb._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 2""" + date="2016-02-15T16:51:20Z" + content=""" +There's --fast if you don't need the expensive to obtain data. + +AFAIK, the working tree traversal that makes up most of the overhead of +info is mostly disk-bound. +"""]] diff --git a/doc/todo/make_annex_info_more_efficient/comment_3_c022caab61061b1f77b78485089a9052._comment b/doc/todo/make_annex_info_more_efficient/comment_3_c022caab61061b1f77b78485089a9052._comment new file mode 100644 index 000000000..6bea4c6a6 --- /dev/null +++ b/doc/todo/make_annex_info_more_efficient/comment_3_c022caab61061b1f77b78485089a9052._comment @@ -0,0 +1,18 @@ +[[!comment format=mdwn + username="https://me.yahoo.com/a/EbvxpTI_xP9Aod7Mg4cwGhgjrCrdM5s-#7c0f4" + subject="comment 3" + date="2016-02-15T17:02:59Z" + content=""" +yeap, saw fast but it is kinda of limited use. doesn't even go through git-annex branch to report amount of storage occupied. I didn't check but \"working tree\" is probably a current checked out branch, and that is why requires traversal and thus slow. What is often also desired to know what is the total number/size of annexed files I have locally in any given annex. So something along of 'du -scm .git/annex/objects' (but via info in git-annex branch instead). May be that could be included in the --fast portion? + +as for disk-bound: in my case /tmp is in ram, there is no disk activities, git-annex is 100% (1 core) busy, FWIW: + +[[!format sh \"\"\" + Command being timed: \"git annex info\" + User time (seconds): 27.69 + System time (seconds): 0.68 + Percent of CPU this job got: 99% + Elapsed (wall clock) time (h:mm:ss or m:ss): 0:28.42 +\"\"\"]] + +"""]] diff --git a/doc/todo/make_annex_info_more_efficient/comment_4_923dd7c22920b389488ca2625225164c._comment b/doc/todo/make_annex_info_more_efficient/comment_4_923dd7c22920b389488ca2625225164c._comment new file mode 100644 index 000000000..54ecf5aa8 --- /dev/null +++ b/doc/todo/make_annex_info_more_efficient/comment_4_923dd7c22920b389488ca2625225164c._comment @@ -0,0 +1,7 @@ +[[!comment format=mdwn + username="https://me.yahoo.com/a/EbvxpTI_xP9Aod7Mg4cwGhgjrCrdM5s-#7c0f4" + subject="comment 4" + date="2016-02-15T17:05:08Z" + content=""" +FWIW the acctual git ls-tree command annex invokes takes 0.01 sec ;) +"""]] diff --git a/doc/todo/make_annex_info_more_efficient/comment_5_c6c8850aefe3ab81f1f113daa734695b._comment b/doc/todo/make_annex_info_more_efficient/comment_5_c6c8850aefe3ab81f1f113daa734695b._comment new file mode 100644 index 000000000..3e1cacc80 --- /dev/null +++ b/doc/todo/make_annex_info_more_efficient/comment_5_c6c8850aefe3ab81f1f113daa734695b._comment @@ -0,0 +1,17 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 5""" + date="2016-02-19T15:58:17Z" + content=""" +When run on a local repository, git-annex info does not look at the +git-annex branch. That would be slower than traversing the directories. +(Asking for info about a remote does look at the git-annex branch.) + +git ls-tree does not have to look at files on disk, so is not comprable. +The work `git annex info` does is roughly the same as a du of +.git/annex/objects and a readlink of each symlink in the work tree. + +If you just want the .git/annex/objects size, perhaps it would make sense +to have a way to get only one stat from git-annex status. Or perhaps +it would be as good to `du .git/annex/objects`? +"""]] diff --git a/doc/todo/make_annex_info_more_efficient/comment_6_804dbb72757b09e3abad3e249f704da0._comment b/doc/todo/make_annex_info_more_efficient/comment_6_804dbb72757b09e3abad3e249f704da0._comment new file mode 100644 index 000000000..f4bedf504 --- /dev/null +++ b/doc/todo/make_annex_info_more_efficient/comment_6_804dbb72757b09e3abad3e249f704da0._comment @@ -0,0 +1,11 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 6""" + date="2016-02-19T18:49:38Z" + content=""" +Also, around the time you filed this, there was a change which turned out +to cause all files in the work tree to be read in full, and buffered in +memory for some time. This may have to do with some of the slowdown you +saw, especially since you have a lot of non-annexed files in the tree. +I fixed that problem in commit b0081598c7c580d6760374c42765e94e4750e793. +"""]] diff --git a/doc/todo/metadata_--batch/comment_1_5cfe3b1a2847cfddd46ab6be249a74e4._comment b/doc/todo/metadata_--batch/comment_1_5cfe3b1a2847cfddd46ab6be249a74e4._comment new file mode 100644 index 000000000..54cbe6fa6 --- /dev/null +++ b/doc/todo/metadata_--batch/comment_1_5cfe3b1a2847cfddd46ab6be249a74e4._comment @@ -0,0 +1,16 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2016-02-19T19:18:41Z" + content=""" +The metadata command is pretty complex. A batch interface would need to +cover both setting and getting. + +There are four different ways to change a value, so it would probably +make sense for the input to be of the form "file field=value" +or "file field+=value" or "file field-=value" or "file field?=value" + +A field can have multiple values, including "". So, a batch interface +to querying the value would need to handle that in its output. Perhaps +json output? +"""]] diff --git a/doc/todo/metadata_--batch/comment_2_6230fe7d6a7b6156b74b7f4e3641c200._comment b/doc/todo/metadata_--batch/comment_2_6230fe7d6a7b6156b74b7f4e3641c200._comment new file mode 100644 index 000000000..5b39175a9 --- /dev/null +++ b/doc/todo/metadata_--batch/comment_2_6230fe7d6a7b6156b74b7f4e3641c200._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://me.yahoo.com/a/EbvxpTI_xP9Aod7Mg4cwGhgjrCrdM5s-#7c0f4" + subject="comment 2" + date="2016-02-19T20:06:19Z" + content=""" +yes -- json output would probably be the best +I know that other --batch functions do not take json input, but I wonder if it might make sense here as well? smth like {'file': 'filename', 'add-field': {'field1': 'value1'}, 'remove-field': ..., 'add-tag': ['tag1', 'tag2'], 'remove-tag': ['... and so on. +"""]] diff --git a/doc/todo/metadata_--batch/comment_3_23c88eab56628fd2f75aceb21f59992e._comment b/doc/todo/metadata_--batch/comment_3_23c88eab56628fd2f75aceb21f59992e._comment new file mode 100644 index 000000000..38dcd7d8c --- /dev/null +++ b/doc/todo/metadata_--batch/comment_3_23c88eab56628fd2f75aceb21f59992e._comment @@ -0,0 +1,14 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 3""" + date="2016-02-19T20:47:28Z" + content=""" +See +<http://git-annex.branchable.com/todo/ability_to_set_metadata_from_json/> + +If it does support json input, the simplest interface would be to use the +same json objects for both output and input. + +Both aeson and json are currently dependencies, and either could be used to +parse json input. +"""]] diff --git a/doc/todo/standalone__47__android__47__buildchroot-inchroot_should_install_libmagic-dev.mdwn b/doc/todo/standalone__47__android__47__buildchroot-inchroot_should_install_libmagic-dev.mdwn new file mode 100644 index 000000000..0b0faea90 --- /dev/null +++ b/doc/todo/standalone__47__android__47__buildchroot-inchroot_should_install_libmagic-dev.mdwn @@ -0,0 +1,3 @@ +I'm setting up an Android build environment, and the standalone/android/install-haskell-packages script fails while installing the Magic package. The standalone/android/buildchroot-inchroot package should run `apt-get -y install libmagic-dev` to install this dependency. + +> [[done]] --[[Joey]] diff --git a/doc/todo/standalone__47__android__47__buildchroot-inchroot_should_install_libmagic-dev/comment_1_daef0416d365b60d36e211adcdb42ffb._comment b/doc/todo/standalone__47__android__47__buildchroot-inchroot_should_install_libmagic-dev/comment_1_daef0416d365b60d36e211adcdb42ffb._comment new file mode 100644 index 000000000..9993b6332 --- /dev/null +++ b/doc/todo/standalone__47__android__47__buildchroot-inchroot_should_install_libmagic-dev/comment_1_daef0416d365b60d36e211adcdb42ffb._comment @@ -0,0 +1,19 @@ +[[!comment format=mdwn + username="divergentdave@5c17d06f6d67c6f157b76a4cc95ca764b7d2f899" + nickname="divergentdave" + subject="comment 1" + date="2016-02-21T05:15:57Z" + content=""" +I got further along in the build process, but cabal couldn't install the \"arm-linux-androideabi-4.8\" versions of `magic` and `terminal-size`. The magic library couldn't be installed because there wasn't an appropriate C library available. The terminal-size library failed with the following compilation error. + +``` +Posix.hsc: In function '_hsc2hs_test11': +Posix.hsc:32:39: error: invalid application of 'sizeof' to incomplete type 'struct winsize' +In file included from /home/builder/.ghc/android-14/arm-linux-androideabi-4.8/sysroot/usr/include/sys/types.h:33:0, + from /home/builder/.ghc/android-14/arm-linux-androideabi-4.8/sysroot/usr/include/unistd.h:33, + from Posix.hsc:21: +Posix.hsc: In function '_hsc2hs_test15': +Posix.hsc:35:48: error: invalid use of undefined type 'struct winsize' +... +``` +"""]] diff --git a/doc/todo/standalone__47__android__47__buildchroot-inchroot_should_install_libmagic-dev/comment_2_b56653f1c57acf70eb2ed3abb4546844._comment b/doc/todo/standalone__47__android__47__buildchroot-inchroot_should_install_libmagic-dev/comment_2_b56653f1c57acf70eb2ed3abb4546844._comment new file mode 100644 index 000000000..4c626bba9 --- /dev/null +++ b/doc/todo/standalone__47__android__47__buildchroot-inchroot_should_install_libmagic-dev/comment_2_b56653f1c57acf70eb2ed3abb4546844._comment @@ -0,0 +1,18 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 3""" + date="2016-02-23T14:55:36Z" + content=""" +My android builds include neither magic nor concurrent-output (which is +what's using terminal-size). I don't feel it's worth trying to get +these features working on androd. + +So, I've adjusted install-haskell-packages to disable the flags. + +If you are interested in working on the android port, I'd be very excited +for any help. As you can see, there is much room for improvement in the +build system for it. (It would be nice to rework it to use stack, rather +than the current approach, and using stack would automatically disable +these flags that are not portable.) And of course, there are many other +parts of the android port that need improvement. +"""]] diff --git a/doc/todo/standalone__47__android__47__buildchroot-inchroot_should_install_libmagic-dev/comment_3_501a89c819d0e1531a8bc6548c837a77._comment b/doc/todo/standalone__47__android__47__buildchroot-inchroot_should_install_libmagic-dev/comment_3_501a89c819d0e1531a8bc6548c837a77._comment new file mode 100644 index 000000000..1ffb94cf0 --- /dev/null +++ b/doc/todo/standalone__47__android__47__buildchroot-inchroot_should_install_libmagic-dev/comment_3_501a89c819d0e1531a8bc6548c837a77._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="divergentdave@5c17d06f6d67c6f157b76a4cc95ca764b7d2f899" + nickname="divergentdave" + subject="comment 3" + date="2016-02-26T03:56:41Z" + content=""" +It looks like the flag in that change was incorrect. I had to change it to `--flags=\"-magicmime -concurrentoutput\"`. (no hyphen between current and output) After that, I was able to reinstall the libraries. + +I'd be happy to lend a hand, expect more patches! +"""]] diff --git a/doc/todo/unwanted_repository_version_upgrades.mdwn b/doc/todo/unwanted_repository_version_upgrades.mdwn index 189550803..88468ec6f 100644 --- a/doc/todo/unwanted_repository_version_upgrades.mdwn +++ b/doc/todo/unwanted_repository_version_upgrades.mdwn @@ -23,3 +23,11 @@ upgrade supported from repository versions: 0 1 2 """]] I'm guessing that one of the machines with access to this repository was running a newer version of git-annex, and that the repository was upgraded in the course of some action. + +> I took this onboard with v6; upgrade to it is currently optional and will +> only become the default once enough years have passed that it's +> reasonable to assume most people have git-annex 6.x installed. +> +> I think at this point it's reasonable to assume most people have +> git-annex 5.x installed, so there's no point in trying to change to v3 to +> v5 forced upgrade at this point. So, [[done]] --[[Joey]] diff --git a/doc/todo/wishlist:_pack_metadata_in_direct_mode.mdwn b/doc/todo/wishlist:_pack_metadata_in_direct_mode.mdwn index ddf15a144..1d52ff512 100644 --- a/doc/todo/wishlist:_pack_metadata_in_direct_mode.mdwn +++ b/doc/todo/wishlist:_pack_metadata_in_direct_mode.mdwn @@ -1,3 +1,5 @@ The metadata storage for direct mode (V3) is this. In directory .git/annex/objects, there is one .map for all annexed file, and one .cache for all files in the working tree. Both are small files, containing only 1 line or a few lines. I have a repo with lots of photos, and this created lots of small files. I believe this will cause many performance issues. It would be great if these files are packed, maybe also in the git pack files format. + +[[!meta tag=deprecateddirectmode]] diff --git a/doc/walkthrough/unused_data/comment_4_2110ed6316b6a0df4ef3e1c8bc97ab99._comment b/doc/walkthrough/unused_data/comment_4_2110ed6316b6a0df4ef3e1c8bc97ab99._comment new file mode 100644 index 000000000..8c6f04e27 --- /dev/null +++ b/doc/walkthrough/unused_data/comment_4_2110ed6316b6a0df4ef3e1c8bc97ab99._comment @@ -0,0 +1,11 @@ +[[!comment format=mdwn + username="tom_clune" + subject="dropping files after changing branches/tags" + date="2016-02-24T22:19:00Z" + content=""" +I have a use case in which a colleague wishes to have a working copy of my data repository to use with the current version of my model. When a new version of the model is available they would likewise update their git-annex clone of my data. The colleague wants to drop any files that have been made obsolete by this change, but I do not see an efficient way to make this determination. They could of course drop everything and then do git annex get . but that could be very expensive if only a small subset of the files have actually changed. + +I'm probably just missing something basic, as this seems to be a reasonably frequent use case. + + +"""]] @@ -1,4 +1,4 @@ #!/bin/sh # ghci using objects built by cabal make dist/caballog -$(grep 'ghc --make' dist/caballog | head -n 1 | perl -pe 's/--make/--interactive/; s/.\/[^\.\s]+.hs//; s/-package-id [^\s]+//g; s/-hide-all-packages//; s/-threaded//; s/-O//') $@ +$(grep 'ghc --make' dist/caballog | head -n 1 | perl -pe 's/--make/--interactive/; s/.\/[^\.\s]+.hs//; s/-package-id [^\s]+//g; s/-hide-all-packages//; s/-threaded//') $@ diff --git a/git-annex.cabal b/git-annex.cabal index 4a1e28f17..dcb038d93 100644 --- a/git-annex.cabal +++ b/git-annex.cabal @@ -1,5 +1,5 @@ Name: git-annex -Version: 6.20160211 +Version: 6.20160229 Cabal-Version: >= 1.8 License: GPL-3 Maintainer: Joey Hess <id@joeyh.name> @@ -131,7 +131,7 @@ Executable git-annex if flag(network-uri) Build-Depends: network-uri (>= 2.6), network (>= 2.6) else - Build-Depends: network (< 2.6), network (>= 2.0) + Build-Depends: network (< 2.6), network (>= 2.4) if flag(Cryptonite) Build-Depends: cryptonite @@ -228,8 +228,9 @@ Executable git-annex CPP-Options: -DWITH_TORRENTPARSER if flag(MagicMime) - Build-Depends: magic - CPP-Options: -DWITH_MAGICMIME + if (! os(windows)) + Build-Depends: magic + CPP-Options: -DWITH_MAGICMIME if flag(ConcurrentOutput) Build-Depends: concurrent-output (>= 1.6) diff --git a/standalone/android/install-haskell-packages b/standalone/android/install-haskell-packages index f338cc6c7..ea5f160d2 100755 --- a/standalone/android/install-haskell-packages +++ b/standalone/android/install-haskell-packages @@ -65,7 +65,7 @@ patched () { installgitannexdeps () { pushd ../.. ln -sf standalone/android/cabal.config - cabal install --only-dependencies "$@" # --force-reinstalls --reinstall + cabal install --only-dependencies --flags="-magicmime -concurrent-output" "$@" # --force-reinstalls --reinstall rm -f cabal.config popd } diff --git a/standalone/licences.gz b/standalone/licences.gz Binary files differindex e85b038fb..b126f88c0 100644 --- a/standalone/licences.gz +++ b/standalone/licences.gz diff --git a/standalone/linux/skel/runshell b/standalone/linux/skel/runshell index 8c1db4247..fff0f7082 100755 --- a/standalone/linux/skel/runshell +++ b/standalone/linux/skel/runshell @@ -81,10 +81,10 @@ if [ -z "$GIT_ANNEX_PACKAGE_INSTALL" ]; then fi # Put our binaries first, to avoid issues with out of date or incompatable -# system binaries. +# system binaries. Extra binaries come after system path. ORIG_PATH="$PATH" export ORIG_PATH -PATH="$base/bin:$PATH" +PATH="$base/bin:$PATH:$base/extra" export PATH # These env vars are used by the shim wrapper around each binary. diff --git a/standalone/osx/git-annex.app/Contents/MacOS/runshell b/standalone/osx/git-annex.app/Contents/MacOS/runshell index b48017f43..17c95698c 100755 --- a/standalone/osx/git-annex.app/Contents/MacOS/runshell +++ b/standalone/osx/git-annex.app/Contents/MacOS/runshell @@ -61,10 +61,10 @@ if [ ! -e "$HOME/.ssh/git-annex-wrapper" ]; then fi # Put our binaries first, to avoid issues with out of date or incompatable -# system binaries. +# system binaries. Extra binaries come after system path. ORIG_PATH="$PATH" export ORIG_PATH -PATH="$bundle:$PATH" +PATH="$bundle:$PATH:$bundle/extra" export PATH ORIG_GIT_EXEC_PATH="$GIT_EXEC_PATH" @@ -77,7 +77,11 @@ export ORIG_GIT_TEMPLATE_DIR GIT_TEMPLATE_DIR="$bundle/templates" export GIT_TEMPLATE_DIR -# Indicate which variables were exported above. +GIT_ANNEX_DIR="$bundle" +export GIT_ANNEX_DIR + +# Indicate which variables were exported above and should be cleaned +# when running non-bundled programs. GIT_ANNEX_STANDLONE_ENV="PATH GIT_EXEC_PATH GIT_TEMPLATE_DIR" export GIT_ANNEX_STANDLONE_ENV diff --git a/standalone/windows/build.sh b/standalone/windows/build.sh index 7f577c90e..f43f041d2 100755 --- a/standalone/windows/build.sh +++ b/standalone/windows/build.sh @@ -101,10 +101,5 @@ PATH="$(pwd)/dist/build/git-annex/:$PATH" export PATH mkdir -p c:/WINDOWS/Temp/git-annex-test/ cd c:/WINDOWS/Temp/git-annex-test/ -if withcyg git-annex.exe test; then - rm -rf .t -else - rm -rf .t - echo "Test suite failure; failing build!" - false -fi +rm -rf .t +withcyg git-annex.exe test |