diff options
257 files changed, 4269 insertions, 1092 deletions
@@ -44,6 +44,7 @@ import Git.CatFile import Git.CheckAttr import Git.CheckIgnore import Git.SharedRepository +import qualified Git.Hook import qualified Git.Queue import Types.Key import Types.Backend @@ -62,6 +63,7 @@ import Types.MetaData import qualified Utility.Matcher import qualified Data.Map as M import qualified Data.Set as S +import Utility.Quvi (QuviVersion) {- git-annex's monad is a ReaderT around an AnnexState stored in a MVar. - This allows modifying the state in an exception-safe fashion. @@ -86,6 +88,7 @@ data AnnexState = AnnexState , gitconfig :: GitConfig , backends :: [BackendA Annex] , remotes :: [Types.Remote.RemoteA Annex] + , remoteannexstate :: M.Map UUID AnnexState , output :: MessageState , force :: Bool , fast :: Bool @@ -116,6 +119,8 @@ data AnnexState = AnnexState , useragent :: Maybe String , errcounter :: Integer , unusedkeys :: Maybe (S.Set Key) + , quviversion :: Maybe QuviVersion + , existinghooks :: M.Map Git.Hook.Hook Bool } newState :: GitConfig -> Git.Repo -> AnnexState @@ -124,6 +129,7 @@ newState c r = AnnexState , gitconfig = c , backends = [] , remotes = [] + , remoteannexstate = M.empty , output = defaultMessageState , force = False , fast = False @@ -154,6 +160,8 @@ newState c r = AnnexState , useragent = Nothing , errcounter = 0 , unusedkeys = Nothing + , quviversion = Nothing + , existinghooks = M.empty } {- Makes an Annex state object for the specified git repo. diff --git a/Annex/AutoMerge.hs b/Annex/AutoMerge.hs new file mode 100644 index 000000000..2ed26b78f --- /dev/null +++ b/Annex/AutoMerge.hs @@ -0,0 +1,179 @@ +{- git-annex automatic merge conflict resolution + - + - Copyright 2012-2014 Joey Hess <joey@kitenet.net> + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Annex.AutoMerge (autoMergeFrom) where + +import Common.Annex +import qualified Annex.Queue +import Annex.Direct +import Annex.CatFile +import Annex.Link +import qualified Git.Command +import qualified Git.LsFiles as LsFiles +import qualified Git.UpdateIndex as UpdateIndex +import qualified Git.Merge +import qualified Git.Ref +import qualified Git.Sha +import qualified Git +import Git.Types (BlobType(..)) +import Config +import Annex.ReplaceFile +import Git.FileMode +import Annex.VariantFile + +import qualified Data.Set as S + +{- Merges from a branch into the current branch + - (which may not exist yet), + - with automatic merge conflict resolution. -} +autoMergeFrom :: Git.Ref -> (Maybe Git.Ref) -> Annex Bool +autoMergeFrom branch currbranch = do + showOutput + case currbranch of + Nothing -> go Nothing + Just b -> go =<< inRepo (Git.Ref.sha b) + where + go old = ifM isDirect + ( do + d <- fromRepo gitAnnexMergeDir + r <- inRepo (mergeDirect d branch) + <||> resolveMerge old branch + mergeDirectCleanup d (fromMaybe Git.Sha.emptyTree old) Git.Ref.headRef + return r + , inRepo (Git.Merge.mergeNonInteractive branch) + <||> resolveMerge old branch + ) + +{- Resolves a conflicted merge. It's important that any conflicts be + - resolved in a way that itself avoids later merge conflicts, since + - multiple repositories may be doing this concurrently. + - + - Only merge conflicts where at least one side is an annexed file + - is resolved. + - + - This uses the Keys pointed to by the files to construct new + - filenames. So when both sides modified annexed file foo, + - it will be deleted, and replaced with files foo.variant-A and + - foo.variant-B. + - + - On the other hand, when one side deleted foo, and the other modified it, + - it will be deleted, and the modified version stored as file + - foo.variant-A (or B). + - + - It's also possible that one side has foo as an annexed file, and + - the other as a directory or non-annexed file. The annexed file + - is renamed to resolve the merge, and the other object is preserved as-is. + - + - In indirect mode, the merge is resolved in the work tree and files + - staged, to clean up from a conflicted merge that was run in the work + - tree. In direct mode, the work tree is not touched here; files are + - staged to the index, and written to the gitAnnexMergeDir, and later + - mergeDirectCleanup handles updating the work tree. + -} +resolveMerge :: Maybe Git.Ref -> Git.Ref -> Annex Bool +resolveMerge us them = do + top <- fromRepo Git.repoPath + (fs, cleanup) <- inRepo (LsFiles.unmerged [top]) + mergedfs <- catMaybes <$> mapM (resolveMerge' us them) fs + let merged = not (null mergedfs) + void $ liftIO cleanup + + unlessM isDirect $ do + (deleted, cleanup2) <- inRepo (LsFiles.deleted [top]) + unless (null deleted) $ + Annex.Queue.addCommand "rm" [Params "--quiet -f --"] deleted + void $ liftIO cleanup2 + + when merged $ do + unlessM isDirect $ + cleanConflictCruft mergedfs top + Annex.Queue.flush + whenM isDirect $ + void preCommitDirect + void $ inRepo $ Git.Command.runBool + [ Param "commit" + , Param "--no-verify" + , Param "-m" + , Param "git-annex automatic merge conflict fix" + ] + showLongNote "Merge conflict was automatically resolved; you may want to examine the result." + return merged + +resolveMerge' :: Maybe Git.Ref -> Git.Ref -> LsFiles.Unmerged -> Annex (Maybe FilePath) +resolveMerge' Nothing _ _ = return Nothing +resolveMerge' (Just us) them u = do + kus <- getkey LsFiles.valUs LsFiles.valUs + kthem <- getkey LsFiles.valThem LsFiles.valThem + case (kus, kthem) of + -- Both sides of conflict are annexed files + (Just keyUs, Just keyThem) + | keyUs /= keyThem -> resolveby $ do + makelink keyUs + makelink keyThem + | otherwise -> resolveby $ + makelink keyUs + -- Our side is annexed file, other side is not. + (Just keyUs, Nothing) -> resolveby $ do + graftin them file + makelink keyUs + -- Our side is not annexed file, other side is. + (Nothing, Just keyThem) -> resolveby $ do + graftin us file + makelink keyThem + -- Neither side is annexed file; cannot resolve. + (Nothing, Nothing) -> return Nothing + where + file = LsFiles.unmergedFile u + + getkey select select' + | select (LsFiles.unmergedBlobType u) == Just SymlinkBlob = + case select' (LsFiles.unmergedSha u) of + Nothing -> return Nothing + Just sha -> catKey sha symLinkMode + | otherwise = return Nothing + + makelink key = do + let dest = variantFile file key + l <- inRepo $ gitAnnexLink dest key + ifM isDirect + ( do + d <- fromRepo gitAnnexMergeDir + replaceFile (d </> dest) $ makeAnnexLink l + , replaceFile dest $ makeAnnexLink l + ) + stageSymlink dest =<< hashSymlink l + + {- stage a graft of a directory or file from a branch -} + graftin b item = Annex.Queue.addUpdateIndex + =<< fromRepo (UpdateIndex.lsSubTree b item) + + resolveby a = do + {- Remove conflicted file from index so merge can be resolved. -} + Annex.Queue.addCommand "rm" [Params "--quiet -f --cached --"] [file] + void a + return (Just file) + +{- git-merge moves conflicting files away to files + - named something like f~HEAD or f~branch, but the + - exact name chosen can vary. Once the conflict is resolved, + - this cruft can be deleted. To avoid deleting legitimate + - files that look like this, only delete files that are + - A) not staged in git and B) look like git-annex symlinks. + -} +cleanConflictCruft :: [FilePath] -> FilePath -> Annex () +cleanConflictCruft resolvedfs top = do + (fs, cleanup) <- inRepo $ LsFiles.notInRepo False [top] + mapM_ clean fs + void $ liftIO cleanup + where + clean f + | matchesresolved f = whenM (isJust <$> isAnnexLink f) $ + liftIO $ nukeFile f + | otherwise = noop + s = S.fromList resolvedfs + matchesresolved f = S.member (base f) s + base f = reverse $ drop 1 $ dropWhile (/= '~') $ reverse f diff --git a/Annex/CatFile.hs b/Annex/CatFile.hs index 54a4d1099..fc722c8e7 100644 --- a/Annex/CatFile.hs +++ b/Annex/CatFile.hs @@ -7,6 +7,7 @@ module Annex.CatFile ( catFile, + catFileDetails, catObject, catTree, catObjectDetails, @@ -34,6 +35,11 @@ catFile branch file = do h <- catFileHandle liftIO $ Git.CatFile.catFile h branch file +catFileDetails :: Git.Branch -> FilePath -> Annex (Maybe (L.ByteString, Sha, ObjectType)) +catFileDetails branch file = do + h <- catFileHandle + liftIO $ Git.CatFile.catFileDetails h branch file + catObject :: Git.Ref -> Annex L.ByteString catObject ref = do h <- catFileHandle @@ -87,8 +93,7 @@ catKey' modeguaranteed ref mode | modeguaranteed = catObject ref | otherwise = L.take 8192 <$> catObject ref -{- Looks up the file mode corresponding to the Ref using the running - - cat-file. +{- Looks up the key corresponding to the Ref using the running cat-file. - - Currently this always has to look in HEAD, because cat-file --batch - does not offer a way to specify that we want to look up a tree object diff --git a/Annex/Content.hs b/Annex/Content.hs index bffef19f4..740ed8bbc 100644 --- a/Annex/Content.hs +++ b/Annex/Content.hs @@ -221,7 +221,7 @@ getViaTmpChecked check key action = -} prepGetViaTmpChecked :: Key -> Annex Bool -> Annex Bool prepGetViaTmpChecked key getkey = do - tmp <- fromRepo $ gitAnnexTmpLocation key + tmp <- fromRepo $ gitAnnexTmpObjectLocation key e <- liftIO $ doesFileExist tmp alreadythere <- if e @@ -250,7 +250,7 @@ finishGetViaTmp check key action = do prepTmp :: Key -> Annex FilePath prepTmp key = do - tmp <- fromRepo $ gitAnnexTmpLocation key + tmp <- fromRepo $ gitAnnexTmpObjectLocation key createAnnexDirectory (parentDir tmp) return tmp @@ -514,10 +514,8 @@ saveState nocommit = doSideAction $ do downloadUrl :: [Url.URLString] -> FilePath -> Annex Bool downloadUrl urls file = go =<< annexWebDownloadCommand <$> Annex.getGitConfig where - go Nothing = do - opts <- map Param . annexWebOptions <$> Annex.getGitConfig - headers <- getHttpHeaders - anyM (\u -> Url.withUserAgent $ Url.download u headers opts file) urls + go Nothing = Url.withUrlOptions $ \uo -> + anyM (\u -> Url.download u file uo) urls go (Just basecmd) = liftIO $ anyM (downloadcmd basecmd) urls downloadcmd basecmd url = boolSystem "sh" [Param "-c", Param $ gencmd url basecmd] diff --git a/Annex/Direct.hs b/Annex/Direct.hs index 4a23fcc6c..2f583fd94 100644 --- a/Annex/Direct.hs +++ b/Annex/Direct.hs @@ -33,6 +33,7 @@ import Utility.CopyFile import Annex.Perms import Annex.ReplaceFile import Annex.Exception +import Annex.VariantFile {- Uses git ls-files to find files that need to be committed, and stages - them into the index. Returns True if some changes were staged. -} @@ -142,9 +143,6 @@ addDirect file cache = do {- In direct mode, git merge would usually refuse to do anything, since it - sees present direct mode files as type changed files. To avoid this, - merge is run with the work tree set to a temp directory. - - - - This should only be used once any changes to the real working tree have - - already been committed, because it overwrites files in the working tree. -} mergeDirect :: FilePath -> Git.Ref -> Git.Repo -> IO Bool mergeDirect d branch g = do @@ -172,39 +170,88 @@ mergeDirectCleanup d oldsha newsha = do makeabs <- flip fromTopFilePath <$> gitRepo let fsitems = zip (map (makeabs . DiffTree.file) items) items forM_ fsitems $ - go DiffTree.srcsha DiffTree.srcmode moveout moveout_raw + go makeabs DiffTree.srcsha DiffTree.srcmode moveout moveout_raw forM_ fsitems $ - go DiffTree.dstsha DiffTree.dstmode movein movein_raw + go makeabs DiffTree.dstsha DiffTree.dstmode movein movein_raw void $ liftIO cleanup liftIO $ removeDirectoryRecursive d where - go getsha getmode a araw (f, item) + go makeabs getsha getmode a araw (f, item) | getsha item == nullSha = noop | otherwise = void $ - tryAnnex . maybe (araw f item) (\k -> void $ a k f) + tryAnnex . maybe (araw item makeabs f) (\k -> void $ a item makeabs k f) =<< catKey (getsha item) (getmode item) - moveout = removeDirect + moveout _ _ = removeDirect {- Files deleted by the merge are removed from the work tree. - Empty work tree directories are removed, per git behavior. -} - moveout_raw f _item = liftIO $ do + moveout_raw _ _ f = liftIO $ do nukeFile f void $ tryIO $ removeDirectory $ parentDir f {- If the file is already present, with the right content for the - - key, it's left alone. Otherwise, create the symlink and then - - if possible, replace it with the content. -} - movein k f = unlessM (goodContent k f) $ do + - key, it's left alone. + - + - If the file is already present, and does not exist in the + - oldsha branch, preserve this local file. + - + - Otherwise, create the symlink and then if possible, replace it + - with the content. -} + movein item makeabs k f = unlessM (goodContent k f) $ do + preserveUnannexed item makeabs f oldsha l <- inRepo $ gitAnnexLink f k replaceFile f $ makeAnnexLink l toDirect k f {- Any new, modified, or renamed files were written to the temp - directory by the merge, and are moved to the real work tree. -} - movein_raw f item = liftIO $ do - createDirectoryIfMissing True $ parentDir f - void $ tryIO $ rename (d </> getTopFilePath (DiffTree.file item)) f + movein_raw item makeabs f = do + preserveUnannexed item makeabs f oldsha + liftIO $ do + createDirectoryIfMissing True $ parentDir f + void $ tryIO $ rename (d </> getTopFilePath (DiffTree.file item)) f + +{- If the file that's being moved in is already present in the work + - tree, but did not exist in the oldsha branch, preserve this + - local, unannexed file (or directory), as "variant-local". + - + - It's also possible that the file that's being moved in + - is in a directory that collides with an exsting, non-annexed + - file (not a directory), which should be preserved. + -} +preserveUnannexed :: DiffTree.DiffTreeItem -> (TopFilePath -> FilePath) -> FilePath -> Ref -> Annex () +preserveUnannexed item makeabs absf oldsha = do + whenM (liftIO (collidingitem absf) <&&> unannexed absf) $ + liftIO $ findnewname absf 0 + checkdirs (DiffTree.file item) + where + checkdirs from = do + let p = parentDir (getTopFilePath from) + let d = asTopFilePath p + unless (null p) $ do + let absd = makeabs d + whenM (liftIO (colliding_nondir absd) <&&> unannexed absd) $ + liftIO $ findnewname absd 0 + checkdirs d + + collidingitem f = isJust + <$> catchMaybeIO (getSymbolicLinkStatus f) + colliding_nondir f = maybe False (not . isDirectory) + <$> catchMaybeIO (getSymbolicLinkStatus f) + + unannexed f = (isNothing <$> isAnnexLink f) + <&&> (isNothing <$> catFileDetails oldsha f) + + findnewname :: FilePath -> Int -> IO () + findnewname f n = do + let localf = mkVariant f + ("local" ++ if n > 0 then show n else "") + ifM (collidingitem localf) + ( findnewname f (n+1) + , rename f localf + `catchIO` const (findnewname f (n+1)) + ) {- If possible, converts a symlink in the working tree into a direct - mode file. If the content is not available, leaves the symlink diff --git a/Annex/Hook.hs b/Annex/Hook.hs index 7301a0958..4848e2d61 100644 --- a/Annex/Hook.hs +++ b/Annex/Hook.hs @@ -1,9 +1,10 @@ {- git-annex git hooks - - - Note that it's important that the scripts not change, otherwise - - removing old hooks using an old version of the script would fail. + - Note that it's important that the scripts installed by git-annex + - not change, otherwise removing old hooks using an old version of + - the script would fail. - - - Copyright 2013 Joey Hess <joey@kitenet.net> + - Copyright 2013-2014 Joey Hess <joey@kitenet.net> - - Licensed under the GNU GPL version 3 or higher. -} @@ -12,12 +13,19 @@ module Annex.Hook where import Common.Annex import qualified Git.Hook as Git -import Utility.Shell import Config +import qualified Annex +import Utility.Shell +import Utility.FileMode + +import qualified Data.Map as M preCommitHook :: Git.Hook preCommitHook = Git.Hook "pre-commit" (mkHookScript "git annex pre-commit .") +preCommitAnnexHook :: Git.Hook +preCommitAnnexHook = Git.Hook "pre-commit-annex" "" + mkHookScript :: String -> String mkHookScript s = unlines [ shebang_local @@ -40,3 +48,24 @@ hookWarning :: Git.Hook -> String -> Annex () hookWarning h msg = do r <- gitRepo warning $ Git.hookName h ++ " hook (" ++ Git.hookFile h r ++ ") " ++ msg + +{- Runs a hook. To avoid checking if the hook exists every time, + - the existing hooks are cached. -} +runAnnexHook :: Git.Hook -> Annex () +runAnnexHook hook = do + cmd <- fromRepo $ Git.hookFile hook + m <- Annex.getState Annex.existinghooks + case M.lookup hook m of + Just True -> run cmd + Just False -> noop + Nothing -> do + exists <- hookexists cmd + Annex.changeState $ \s -> s + { Annex.existinghooks = M.insert hook exists m } + when exists $ + run cmd + where + hookexists f = liftIO $ catchBoolIO $ + isExecutable . fileMode <$> getFileStatus f + run cmd = unlessM (liftIO $ boolSystem cmd []) $ + warning $ cmd ++ " failed" diff --git a/Annex/Init.hs b/Annex/Init.hs index 43f24031c..57379535d 100644 --- a/Annex/Init.hs +++ b/Annex/Init.hs @@ -120,7 +120,7 @@ probeCrippledFileSystem = do #ifdef mingw32_HOST_OS return True #else - tmp <- fromRepo gitAnnexTmpDir + tmp <- fromRepo gitAnnexTmpMiscDir let f = tmp </> "gaprobe" createAnnexDirectory tmp liftIO $ writeFile f "" @@ -157,7 +157,7 @@ probeFifoSupport = do #ifdef mingw32_HOST_OS return False #else - tmp <- fromRepo gitAnnexTmpDir + tmp <- fromRepo gitAnnexTmpMiscDir let f = tmp </> "gaprobe" createAnnexDirectory tmp liftIO $ do diff --git a/Annex/Journal.hs b/Annex/Journal.hs index 3f31cb941..395e81d29 100644 --- a/Annex/Journal.hs +++ b/Annex/Journal.hs @@ -35,11 +35,11 @@ import Utility.WinLock -} setJournalFile :: JournalLocked -> FilePath -> String -> Annex () setJournalFile _jl file content = do + tmp <- fromRepo gitAnnexTmpMiscDir createAnnexDirectory =<< fromRepo gitAnnexJournalDir - createAnnexDirectory =<< fromRepo gitAnnexTmpDir + createAnnexDirectory tmp -- journal file is written atomically jfile <- fromRepo $ journalFile file - tmp <- fromRepo gitAnnexTmpDir let tmpfile = tmp </> takeFileName jfile liftIO $ do writeBinaryFile tmpfile content diff --git a/Annex/MetaData.hs b/Annex/MetaData.hs new file mode 100644 index 000000000..68aef33f1 --- /dev/null +++ b/Annex/MetaData.hs @@ -0,0 +1,61 @@ +{- git-annex metadata + - + - Copyright 2014 Joey Hess <joey@kitenet.net> + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Annex.MetaData where + +import Common.Annex +import qualified Annex +import Types.MetaData +import Logs.MetaData +import Annex.CatFile + +import qualified Data.Set as S +import qualified Data.Map as M +import Data.Time.Calendar +import Data.Time.Clock +import Data.Time.Clock.POSIX + +tagMetaField :: MetaField +tagMetaField = mkMetaFieldUnchecked "tag" + +yearMetaField :: MetaField +yearMetaField = mkMetaFieldUnchecked "year" + +monthMetaField :: MetaField +monthMetaField = mkMetaFieldUnchecked "month" + +{- Adds metadata for a file that has just been ingested into the + - annex, but has not yet been committed to git. + - + - When the file has been modified, the metadata is copied over + - from the old key to the new key. Note that it looks at the old key as + - committed to HEAD -- the new key may or may not have already been staged + - in th annex. + - + - Also, can generate new metadata, if configured to do so. + -} +genMetaData :: Key -> FilePath -> FileStatus -> Annex () +genMetaData key file status = do + maybe noop (flip copyMetaData key) =<< catKeyFileHEAD file + whenM (annexGenMetaData <$> Annex.getGitConfig) $ do + metadata <- getCurrentMetaData key + let metadata' = genMetaData' status metadata + unless (metadata' == emptyMetaData) $ + addMetaData key metadata' + +{- Generates metadata from the FileStatus. + - Does not overwrite any existing metadata values. -} +genMetaData' :: FileStatus -> MetaData -> MetaData +genMetaData' status old = MetaData $ M.fromList $ filter isnew + [ (yearMetaField, S.singleton $ toMetaValue $ show y) + , (monthMetaField, S.singleton $ toMetaValue $ show m) + ] + where + isnew (f, _) = S.null (currentMetaDataValues f old) + (y, m, _d) = toGregorian $ utctDay $ + posixSecondsToUTCTime $ realToFrac $ + modificationTime status diff --git a/Annex/Quvi.hs b/Annex/Quvi.hs index b0725bae7..1a2edf6b8 100644 --- a/Annex/Quvi.hs +++ b/Annex/Quvi.hs @@ -14,7 +14,20 @@ import qualified Annex import Utility.Quvi import Utility.Url -withQuviOptions :: forall a. Query a -> [CommandParam] -> URLString -> Annex a +withQuviOptions :: forall a. Query a -> [QuviParam] -> URLString -> Annex a withQuviOptions a ps url = do + v <- quviVersion opts <- map Param . annexQuviOptions <$> Annex.getGitConfig - liftIO $ a (ps++opts) url + liftIO $ a v (map (\mkp -> mkp v) ps++opts) url + +quviSupported :: URLString -> Annex Bool +quviSupported u = liftIO . flip supported u =<< quviVersion + +quviVersion :: Annex QuviVersion +quviVersion = go =<< Annex.getState Annex.quviversion + where + go (Just v) = return v + go Nothing = do + v <- liftIO probeVersion + Annex.changeState $ \s -> s { Annex.quviversion = Just v } + return v diff --git a/Annex/ReplaceFile.hs b/Annex/ReplaceFile.hs index dd93b471c..8b15f5ce3 100644 --- a/Annex/ReplaceFile.hs +++ b/Annex/ReplaceFile.hs @@ -24,7 +24,7 @@ import Annex.Exception -} replaceFile :: FilePath -> (FilePath -> Annex ()) -> Annex () replaceFile file a = do - tmpdir <- fromRepo gitAnnexTmpDir + tmpdir <- fromRepo gitAnnexTmpMiscDir void $ createAnnexDirectory tmpdir bracketIO (setup tmpdir) nukeFile $ \tmpfile -> do a tmpfile @@ -36,4 +36,4 @@ replaceFile file a = do return tmpfile fallback tmpfile _ = do createDirectoryIfMissing True $ parentDir file - rename tmpfile file + moveFile tmpfile file diff --git a/Annex/Url.hs b/Annex/Url.hs index 0401ffe07..397a7910b 100644 --- a/Annex/Url.hs +++ b/Annex/Url.hs @@ -1,13 +1,15 @@ -{- Url downloading, with git-annex user agent. +{- Url downloading, with git-annex user agent and configured http + - headers and wget/curl options. - - - Copyright 2013 Joey Hess <joey@kitenet.net> + - Copyright 2013-2014 Joey Hess <joey@kitenet.net> - - Licensed under the GNU GPL version 3 or higher. -} module Annex.Url ( module U, - withUserAgent, + withUrlOptions, + getUrlOptions, getUserAgent, ) where @@ -23,5 +25,18 @@ getUserAgent :: Annex (Maybe U.UserAgent) getUserAgent = Annex.getState $ Just . fromMaybe defaultUserAgent . Annex.useragent -withUserAgent :: (Maybe U.UserAgent -> IO a) -> Annex a -withUserAgent a = liftIO . a =<< getUserAgent +getUrlOptions :: Annex U.UrlOptions +getUrlOptions = U.UrlOptions + <$> getUserAgent + <*> headers + <*> options + where + headers = do + v <- annexHttpHeadersCommand <$> Annex.getGitConfig + case v of + Just cmd -> lines <$> liftIO (readProcess "sh" ["-c", cmd]) + Nothing -> annexHttpHeaders <$> Annex.getGitConfig + options = map Param . annexWebOptions <$> Annex.getGitConfig + +withUrlOptions :: (U.UrlOptions -> IO a) -> Annex a +withUrlOptions a = liftIO . a =<< getUrlOptions diff --git a/Annex/VariantFile.hs b/Annex/VariantFile.hs new file mode 100644 index 000000000..7c849c59f --- /dev/null +++ b/Annex/VariantFile.hs @@ -0,0 +1,45 @@ +{- git-annex .variant files for automatic merge conflict resolution + - + - Copyright 2014 Joey Hess <joey@kitenet.net> + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Annex.VariantFile where + +import Common.Annex +import Types.Key + +import Data.Hash.MD5 + +variantMarker :: String +variantMarker = ".variant-" + +mkVariant :: FilePath -> String -> FilePath +mkVariant file variant = takeDirectory file + </> dropExtension (takeFileName file) + ++ variantMarker ++ variant + ++ takeExtension file + +{- The filename to use when resolving a conflicted merge of a file, + - that points to a key. + - + - Something derived from the key needs to be included in the filename, + - but rather than exposing the whole key to the user, a very weak hash + - is used. There is a very real, although still unlikely, chance of + - conflicts using this hash. + - + - In the event that there is a conflict with the filename generated + - for some other key, that conflict will itself be handled by the + - conflicted merge resolution code. That case is detected, and the full + - key is used in the filename. + -} +variantFile :: FilePath -> Key -> FilePath +variantFile file key + | doubleconflict = mkVariant file (key2file key) + | otherwise = mkVariant file (shortHash $ key2file key) + where + doubleconflict = variantMarker `isInfixOf` file + +shortHash :: String -> String +shortHash = take 4 . md5s . md5FilePath diff --git a/Annex/View.hs b/Annex/View.hs index 78b4da589..7c187befd 100644 --- a/Annex/View.hs +++ b/Annex/View.hs @@ -5,13 +5,13 @@ - Licensed under the GNU GPL version 3 or higher. -} -{-# LANGUAGE CPP #-} - module Annex.View where import Common.Annex +import Annex.View.ViewedFile import Types.View import Types.MetaData +import Annex.MetaData import qualified Git import qualified Git.DiffTree as DiffTree import qualified Git.Branch @@ -28,22 +28,16 @@ import Annex.Link import Annex.CatFile import Logs.MetaData import Logs.View +import Utility.Glob import Utility.FileMode import Types.Command import Config import CmdLine.Action import qualified Data.Set as S -import System.Path.WildMatch +import qualified Data.Map as M import "mtl" Control.Monad.Writer -#ifdef WITH_TDFA -import Text.Regex.TDFA -import Text.Regex.TDFA.String -#else -import Text.Regex -#endif - {- Each visible ViewFilter in a view results in another level of - subdirectory nesting. When a file matches multiple ways, it will appear - in multiple subdirectories. This means there is a bit of an exponential @@ -58,52 +52,85 @@ viewTooLarge view = visibleViewSize view > 5 visibleViewSize :: View -> Int visibleViewSize = length . filter viewVisible . viewComponents +{- Parses field=value, field!=value, tag, and !tag + - + - Note that the field may not be a legal metadata field name, + - but it's let through anyway. + - This is useful when matching on directory names with spaces, + - which are not legal MetaFields. + -} +parseViewParam :: String -> (MetaField, ViewFilter) +parseViewParam s = case separate (== '=') s of + ('!':tag, []) | not (null tag) -> + ( tagMetaField + , mkExcludeValues tag + ) + (tag, []) -> + ( tagMetaField + , mkFilterValues tag + ) + (field, wanted) + | end field == "!" -> + ( mkMetaFieldUnchecked (beginning field) + , mkExcludeValues wanted + ) + | otherwise -> + ( mkMetaFieldUnchecked field + , mkFilterValues wanted + ) + where + mkFilterValues v + | any (`elem` v) "*?" = FilterGlob v + | otherwise = FilterValues $ S.singleton $ toMetaValue v + mkExcludeValues = ExcludeValues . S.singleton . toMetaValue + data ViewChange = Unchanged | Narrowing | Widening deriving (Ord, Eq, Show) {- Updates a view, adding new fields to filter on (Narrowing), - or allowing new values in an existing field (Widening). -} -refineView :: View -> [(MetaField, String)] -> (View, ViewChange) -refineView = go Unchanged +refineView :: View -> [(MetaField, ViewFilter)] -> (View, ViewChange) +refineView origview = checksize . calc Unchanged origview where - go c v [] = (v, c) - go c v ((f, s):rest) = - let (v', c') = refineView' v f s - in go (max c c') v' rest + calc c v [] = (v, c) + calc c v ((f, vf):rest) = + let (v', c') = refine v f vf + in calc (max c c') v' rest + + refine view field vf + | field `elem` map viewField (viewComponents view) = + let (components', viewchanges) = runWriter $ + mapM (\c -> updateViewComponent c field vf) (viewComponents view) + viewchange = if field `elem` map viewField (viewComponents origview) + then maximum viewchanges + else Narrowing + in (view { viewComponents = components' }, viewchange) + | otherwise = + let component = mkViewComponent field vf + view' = view { viewComponents = component : viewComponents view } + in (view', Narrowing) + + checksize r@(v, _) + | viewTooLarge v = error $ "View is too large (" ++ show (visibleViewSize v) ++ " levels of subdirectories)" + | otherwise = r + +updateViewComponent :: ViewComponent -> MetaField -> ViewFilter -> Writer [ViewChange] ViewComponent +updateViewComponent c field vf + | viewField c == field = do + let (newvf, viewchange) = combineViewFilter (viewFilter c) vf + tell [viewchange] + return $ mkViewComponent field newvf + | otherwise = return c {- Adds an additional filter to a view. This can only result in narrowing - the view. Multivalued filters are added in non-visible form. -} -filterView :: View -> [(MetaField, String)] -> View +filterView :: View -> [(MetaField, ViewFilter)] -> View filterView v vs = v { viewComponents = viewComponents f' ++ viewComponents v} where f = fst $ refineView (v {viewComponents = []}) vs f' = f { viewComponents = map toinvisible (viewComponents f) } toinvisible c = c { viewVisible = False } -refineView' :: View -> MetaField -> String -> (View, ViewChange) -refineView' view field wanted - | field `elem` (map viewField components) = - let (components', viewchanges) = runWriter $ mapM updatefield components - in (view { viewComponents = components' }, maximum viewchanges) - | otherwise = - let component = ViewComponent field viewfilter (multiValue viewfilter) - view' = view { viewComponents = component : components } - in if viewTooLarge view' - then error $ "View is too large (" ++ show (visibleViewSize view') ++ " levels of subdirectories)" - else (view', Narrowing) - where - components = viewComponents view - viewfilter - | any (`elem` wanted) "*?" = FilterGlob wanted - | otherwise = FilterValues $ S.singleton $ toMetaValue wanted - updatefield :: ViewComponent -> Writer [ViewChange] ViewComponent - updatefield v - | viewField v == field = do - let (newvf, viewchange) = combineViewFilter (viewFilter v) viewfilter - tell [viewchange] - return $ v { viewFilter = newvf } - | otherwise = return v - {- Combine old and new ViewFilters, yielding a result that matches - either old+new, or only new. - @@ -124,44 +151,24 @@ combineViewFilter old@(FilterValues olds) (FilterValues news) | otherwise = (combined, Widening) where combined = FilterValues (S.union olds news) +combineViewFilter old@(ExcludeValues olds) (ExcludeValues news) + | combined == old = (combined, Unchanged) + | otherwise = (combined, Narrowing) + where + combined = ExcludeValues (S.union olds news) combineViewFilter (FilterValues _) newglob@(FilterGlob _) = (newglob, Widening) combineViewFilter (FilterGlob oldglob) new@(FilterValues s) - | all (matchGlob (compileGlob oldglob) . fromMetaValue) (S.toList s) = (new, Narrowing) + | all (matchGlob (compileGlob oldglob CaseInsensative) . fromMetaValue) (S.toList s) = (new, Narrowing) | otherwise = (new, Widening) combineViewFilter (FilterGlob old) newglob@(FilterGlob new) | old == new = (newglob, Unchanged) - | matchGlob (compileGlob old) new = (newglob, Narrowing) + | matchGlob (compileGlob old CaseInsensative) new = (newglob, Narrowing) | otherwise = (newglob, Widening) - -{- Converts a filepath used in a reference branch to the - - filename that will be used in the view. - - - - No two filepaths from the same branch should yeild the same result, - - so all directory structure needs to be included in the output file - - in some way. However, the branch's directory structure is not relevant - - in the view. - - - - So, from dir/subdir/file.foo, generate file_{dir;subdir}.foo - - - - (To avoid collisions with a filename that already contains {foo}, - - that is doubled to {{foo}}.) - -} -fileViewFromReference :: MkFileView -fileViewFromReference f = concat - [ double base - , if null dirs then "" else "_{" ++ double (intercalate ";" dirs) ++ "}" - , double $ concat extensions - ] - where - (path, basefile) = splitFileName f - dirs = filter (/= ".") $ map dropTrailingPathSeparator (splitPath path) - (base, extensions) = splitShortExtensions basefile - - double = replace "{" "{{" . replace "}" "}}" - -fileViewReuse :: MkFileView -fileViewReuse = takeFileName +combineViewFilter (FilterGlob _) new@(ExcludeValues _) = (new, Narrowing) +combineViewFilter (ExcludeValues _) new@(FilterGlob _) = (new, Widening) +combineViewFilter (FilterValues _) new@(ExcludeValues _) = (new, Narrowing) +combineViewFilter (ExcludeValues _) new@(FilterValues _) = (new, Widening) {- Generates views for a file from a branch, based on its metadata - and the filename used in the branch. @@ -176,10 +183,10 @@ fileViewReuse = takeFileName - evaluate this function with the view parameter and reuse - the result. The globs in the view will then be compiled and memoized. -} -fileViews :: View -> MkFileView -> FilePath -> MetaData -> [FileView] -fileViews view = +viewedFiles :: View -> MkViewedFile -> FilePath -> MetaData -> [ViewedFile] +viewedFiles view = let matchers = map viewComponentMatcher (viewComponents view) - in \mkfileview file metadata -> + in \mkviewedfile file metadata -> let matches = map (\m -> m metadata) matchers in if any isNothing matches then [] @@ -187,8 +194,8 @@ fileViews view = let paths = pathProduct $ map (map toViewPath) (visible matches) in if null paths - then [mkfileview file] - else map (</> mkfileview file) paths + then [mkviewedfile file] + else map (</> mkviewedfile file) paths where visible = map (fromJust . snd) . filter (viewVisible . fst) . @@ -198,38 +205,23 @@ fileViews view = - returns the value, or values that match. Self-memoizing on ViewComponent. -} viewComponentMatcher :: ViewComponent -> (MetaData -> Maybe [MetaValue]) viewComponentMatcher viewcomponent = \metadata -> - let s = matcher (currentMetaDataValues metafield metadata) - in if S.null s then Nothing else Just (S.toList s) + matcher (currentMetaDataValues metafield metadata) where metafield = viewField viewcomponent matcher = case viewFilter viewcomponent of - FilterValues s -> \values -> S.intersection s values + FilterValues s -> \values -> setmatches $ + S.intersection s values FilterGlob glob -> - let regex = compileGlob glob - in \values -> - S.filter (matchGlob regex . fromMetaValue) values - -compileGlob :: String -> Regex -compileGlob glob = -#ifdef WITH_TDFA - case compile (defaultCompOpt {caseSensitive = False}) defaultExecOpt regex of - Right r -> r - Left _ -> error $ "failed to compile regex: " ++ regex -#else - mkRegexWithOpts regex False True -#endif - where - regex = '^':wildToRegex glob - -matchGlob :: Regex -> String -> Bool -matchGlob regex val = -#ifdef WITH_TDFA - case execute regex val of - Right (Just _) -> True - _ -> False -#else - isJust $ matchRegex regex val -#endif + let cglob = compileGlob glob CaseInsensative + in \values -> setmatches $ + S.filter (matchGlob cglob . fromMetaValue) values + ExcludeValues excludes -> \values -> + if S.null (S.intersection values excludes) + then Just [] + else Nothing + setmatches s + | S.null s = Nothing + | otherwise = Just (S.toList s) toViewPath :: MetaValue -> FilePath toViewPath = concatMap escapeslash . fromMetaValue @@ -268,23 +260,28 @@ pathProduct (l:ls) = foldl combinel l ls where combinel xs ys = [combine x y | x <- xs, y <- ys] -{- Extracts the metadata from a fileview, based on the view that was used - - to construct it. -} -fromView :: View -> FileView -> MetaData -fromView view f = foldr (uncurry updateMetaData) newMetaData (zip fields values) +{- Extracts the metadata from a ViewedFile, based on the view that was used + - to construct it. + - + - Derived metadata is excluded. + -} +fromView :: View -> ViewedFile -> MetaData +fromView view f = MetaData $ + M.fromList (zip fields values) `M.difference` derived where visible = filter viewVisible (viewComponents view) fields = map viewField visible - paths = splitDirectories $ dropFileName f - values = map fromViewPath paths + paths = splitDirectories (dropFileName f) + values = map (S.singleton . fromViewPath) paths + MetaData derived = getViewedFileMetaData f {- Constructing a view that will match arbitrary metadata, and applying - - it to a file yields a set of FileViews which all contain the same + - it to a file yields a set of ViewedFile which all contain the same - MetaFields that were present in the input metadata - (excluding fields that are not visible). -} prop_view_roundtrips :: FilePath -> MetaData -> Bool -> Bool prop_view_roundtrips f metadata visible = null f || viewTooLarge view || - all hasfields (fileViews view fileViewFromReference f metadata) + all hasfields (viewedFiles view viewedFileFromReference f metadata) where view = View (Git.Ref "master") $ map (\(mf, mv) -> ViewComponent mf (FilterValues $ S.filter (not . null . fromMetaValue) mv) visible) @@ -292,11 +289,32 @@ prop_view_roundtrips f metadata visible = null f || viewTooLarge view || visiblefields = sort (map viewField $ filter viewVisible (viewComponents view)) hasfields fv = sort (map fst (fromMetaData (fromView view fv))) == visiblefields +{- A directory foo/bar/baz/ is turned into metadata fields + - /=foo, foo/=bar, foo/bar/=baz. + - + - Note that this may generate MetaFields that legalField rejects. + - This is necessary to have a 1:1 mapping between directory names and + - fields. So this MetaData cannot safely be serialized. -} +getDirMetaData :: FilePath -> MetaData +getDirMetaData d = MetaData $ M.fromList $ zip fields values + where + dirs = splitDirectories d + fields = map (mkMetaFieldUnchecked . addTrailingPathSeparator . joinPath) + (inits dirs) + values = map (S.singleton . toMetaValue . fromMaybe "" . headMaybe) + (tails dirs) + +getWorkTreeMetaData :: FilePath -> MetaData +getWorkTreeMetaData = getDirMetaData . dropFileName + +getViewedFileMetaData :: FilePath -> MetaData +getViewedFileMetaData = getDirMetaData . dirFromViewedFile . takeFileName + {- Applies a view to the currently checked out branch, generating a new - branch for the view. -} applyView :: View -> Annex Git.Branch -applyView view = applyView' fileViewFromReference view +applyView view = applyView' viewedFileFromReference getWorkTreeMetaData view {- Generates a new branch for a View, which must be a more narrow - version of the View originally used to generate the currently @@ -304,18 +322,18 @@ applyView view = applyView' fileViewFromReference view - in view, not any others. -} narrowView :: View -> Annex Git.Branch -narrowView = applyView' fileViewReuse +narrowView = applyView' viewedFileReuse getViewedFileMetaData {- Go through each file in the currently checked out branch. - If the file is not annexed, skip it, unless it's a dotfile in the top. - - Look up the metadata of annexed files, and generate any FileViews, + - Look up the metadata of annexed files, and generate any ViewedFiles, - and stage them. - - Currently only works in indirect mode. Must be run from top of - repository. -} -applyView' :: MkFileView -> View -> Annex Git.Branch -applyView' mkfileview view = do +applyView' :: MkViewedFile -> (FilePath -> MetaData) -> View -> Annex Git.Branch +applyView' mkviewedfile getfilemetadata view = do top <- fromRepo Git.repoPath (l, clean) <- inRepo $ Git.LsFiles.inRepo [top] liftIO . nukeFile =<< fromRepo gitAnnexViewIndex @@ -329,10 +347,11 @@ applyView' mkfileview view = do void $ stopUpdateIndex uh void clean where - genfileviews = fileViews view mkfileview -- enables memoization + genviewedfiles = viewedFiles view mkviewedfile -- enables memoization go uh hasher f (Just (k, _)) = do metadata <- getCurrentMetaData k - forM_ (genfileviews f metadata) $ \fv -> do + let metadata' = getfilemetadata f `unionMetaData` metadata + forM_ (genviewedfiles f metadata') $ \fv -> do stagesymlink uh hasher fv =<< inRepo (gitAnnexLink fv k) go uh hasher f Nothing | "." `isPrefixOf` f = do @@ -381,7 +400,7 @@ updateView view ref oldref = genViewBranch view $ do - Note that removes must be handled before adds. This is so - that moving a file from x/foo/ to x/bar/ adds back the metadata for x. -} -withViewChanges :: (FileView -> Key -> CommandStart) -> (FileView -> Key -> CommandStart) -> Annex () +withViewChanges :: (ViewedFile -> Key -> CommandStart) -> (ViewedFile -> Key -> CommandStart) -> Annex () withViewChanges addmeta removemeta = do makeabs <- flip fromTopFilePath <$> gitRepo (diffs, cleanup) <- inRepo $ DiffTree.diffIndex Git.Ref.headRef diff --git a/Annex/View/ViewedFile.hs b/Annex/View/ViewedFile.hs new file mode 100644 index 000000000..25ac16a34 --- /dev/null +++ b/Annex/View/ViewedFile.hs @@ -0,0 +1,75 @@ +{- filenames (not paths) used in views + - + - Copyright 2014 Joey Hess <joey@kitenet.net> + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Annex.View.ViewedFile ( + ViewedFile, + MkViewedFile, + viewedFileFromReference, + viewedFileReuse, + dirFromViewedFile, + prop_viewedFile_roundtrips, +) where + +import Common.Annex + +type FileName = String +type ViewedFile = FileName + +type MkViewedFile = FilePath -> ViewedFile + +{- Converts a filepath used in a reference branch to the + - filename that will be used in the view. + - + - No two filepaths from the same branch should yeild the same result, + - so all directory structure needs to be included in the output filename + - in some way. + - + - So, from dir/subdir/file.foo, generate file_%dir%subdir%.foo + -} +viewedFileFromReference :: MkViewedFile +viewedFileFromReference f = concat + [ escape base + , if null dirs then "" else "_%" ++ intercalate "%" (map escape dirs) ++ "%" + , escape $ concat extensions + ] + where + (path, basefile) = splitFileName f + dirs = filter (/= ".") $ map dropTrailingPathSeparator (splitPath path) + (base, extensions) = splitShortExtensions basefile + + {- To avoid collisions with filenames or directories that contain + - '%', and to allow the original directories to be extracted + - from the ViewedFile, '%' is escaped to '\%' (and '\' to '\\'). + -} + escape :: String -> String + escape = replace "%" "\\%" . replace "\\" "\\\\" + +{- For use when operating already within a view, so whatever filepath + - is present in the work tree is already a ViewedFile. -} +viewedFileReuse :: MkViewedFile +viewedFileReuse = takeFileName + +{- Extracts from a ViewedFile the directory where the file is located on + - in the reference branch. -} +dirFromViewedFile :: ViewedFile -> FilePath +dirFromViewedFile = joinPath . drop 1 . sep [] "" + where + sep l _ [] = reverse l + sep l curr (c:cs) + | c == '%' = sep (reverse curr:l) "" cs + | c == '\\' = case cs of + (c':cs') -> sep l (c':curr) cs' + [] -> sep l curr cs + | otherwise = sep l (c:curr) cs + +prop_viewedFile_roundtrips :: FilePath -> Bool +prop_viewedFile_roundtrips f + -- Relative filenames wanted, not directories. + | any (isPathSeparator) (end f ++ beginning f) = True + | otherwise = dir == dirFromViewedFile (viewedFileFromReference f) + where + dir = joinPath $ beginning $ splitDirectories f diff --git a/Assistant.hs b/Assistant.hs index c66a1b73b..3c79c476c 100644 --- a/Assistant.hs +++ b/Assistant.hs @@ -49,11 +49,13 @@ import Assistant.Threads.XMPPPusher import Assistant.Types.UrlRenderer #endif import qualified Utility.Daemon -import Utility.LogFile import Utility.ThreadScheduler import Utility.HumanTime -import Annex.Perms import qualified Build.SysConfig as SysConfig +#ifndef mingw32_HOST_OS +import Utility.LogFile +import Annex.Perms +#endif import System.Log.Logger import Network.Socket (HostName) @@ -70,8 +72,8 @@ startDaemon :: Bool -> Bool -> Maybe Duration -> Maybe String -> Maybe HostName startDaemon assistant foreground startdelay cannotrun listenhost startbrowser = do Annex.changeState $ \s -> s { Annex.daemon = True } pidfile <- fromRepo gitAnnexPidFile - logfile <- fromRepo gitAnnexLogFile #ifndef mingw32_HOST_OS + logfile <- fromRepo gitAnnexLogFile createAnnexDirectory (parentDir logfile) logfd <- liftIO $ openLog logfile if foreground @@ -93,11 +95,12 @@ startDaemon assistant foreground startdelay cannotrun listenhost startbrowser = start (Utility.Daemon.daemonize logfd (Just pidfile) False) Nothing #else -- Windows is always foreground, and has no log file. - liftIO $ Utility.Daemon.lockPidFile pidfile - start id $ do - case startbrowser of - Nothing -> Nothing - Just a -> Just $ a Nothing Nothing + when (foreground || not foreground) $ do + liftIO $ Utility.Daemon.lockPidFile pidfile + start id $ do + case startbrowser of + Nothing -> Nothing + Just a -> Just $ a Nothing Nothing #endif where desc @@ -121,7 +124,7 @@ startDaemon assistant foreground startdelay cannotrun listenhost startbrowser = notice ["starting", desc, "version", SysConfig.packageversion] urlrenderer <- liftIO newUrlRenderer #ifdef WITH_WEBAPP - let webappthread = [ assist $ webAppThread d urlrenderer False cannotrun listenhost Nothing webappwaiter ] + let webappthread = [ assist $ webAppThread d urlrenderer False cannotrun Nothing listenhost webappwaiter ] #else let webappthread = [] #endif diff --git a/Assistant/Ssh.hs b/Assistant/Ssh.hs index 82da9e33a..acb2fc11c 100644 --- a/Assistant/Ssh.hs +++ b/Assistant/Ssh.hs @@ -143,6 +143,8 @@ addAuthorizedKeys :: Bool -> FilePath -> SshPubKey -> IO Bool addAuthorizedKeys gitannexshellonly dir pubkey = boolSystem "sh" [ Param "-c" , Param $ addAuthorizedKeysCommand gitannexshellonly dir pubkey ] +{- Should only be used within the same process that added the line; + - the layout of the line is not kepy stable across versions. -} removeAuthorizedKeys :: Bool -> FilePath -> SshPubKey -> IO () removeAuthorizedKeys gitannexshellonly dir pubkey = do let keyline = authorizedKeysLine gitannexshellonly dir pubkey @@ -195,7 +197,7 @@ authorizedKeysLine gitannexshellonly dir pubkey - long perl script. -} | otherwise = pubkey where - limitcommand = "command=\"GIT_ANNEX_SHELL_DIRECTORY="++shellEscape dir++" ~/.ssh/git-annex-shell\",no-agent-forwarding,no-port-forwarding,no-X11-forwarding " + limitcommand = "command=\"GIT_ANNEX_SHELL_DIRECTORY="++shellEscape dir++" ~/.ssh/git-annex-shell\",no-agent-forwarding,no-port-forwarding,no-X11-forwarding,no-pty " {- Generates a ssh key pair. -} genSshKeyPair :: IO SshKeyPair diff --git a/Assistant/Threads/Committer.hs b/Assistant/Threads/Committer.hs index e8d17b13f..cb98b017f 100644 --- a/Assistant/Threads/Committer.hs +++ b/Assistant/Threads/Committer.hs @@ -448,7 +448,7 @@ safeToAdd havelsof delayadd pending inprocess = do let segments = segmentXargs $ map keyFilename keysources concat <$> forM segments (\fs -> Lsof.query $ "--" : fs) , do - tmpdir <- fromRepo gitAnnexTmpDir + tmpdir <- fromRepo gitAnnexTmpMiscDir liftIO $ Lsof.queryDir tmpdir ) diff --git a/Assistant/Threads/Merger.hs b/Assistant/Threads/Merger.hs index 8c406990a..03bcf0aad 100644 --- a/Assistant/Threads/Merger.hs +++ b/Assistant/Threads/Merger.hs @@ -17,7 +17,7 @@ import Utility.DirWatcher.Types import qualified Annex.Branch import qualified Git import qualified Git.Branch -import qualified Command.Sync +import Annex.AutoMerge import Annex.TaggedPush import Remote (remoteFromUUID) @@ -39,7 +39,7 @@ mergeThread = namedThread "Merger" $ do , modifyHook = changehook , errHook = errhook } - void $ liftIO $ watchDir dir (const False) hooks id + void $ liftIO $ watchDir dir (const False) True hooks id debug ["watching", dir] type Handler = FilePath -> Assistant () @@ -83,7 +83,7 @@ onChange file [ "merging", Git.fromRef changedbranch , "into", Git.fromRef current ] - void $ liftAnnex $ Command.Sync.mergeFrom changedbranch + void $ liftAnnex $ autoMergeFrom changedbranch (Just current) mergecurrent _ = noop handleDesynced = case fromTaggedBranch changedbranch of diff --git a/Assistant/Threads/NetWatcher.hs b/Assistant/Threads/NetWatcher.hs index a7124fa01..0b009647c 100644 --- a/Assistant/Threads/NetWatcher.hs +++ b/Assistant/Threads/NetWatcher.hs @@ -24,8 +24,10 @@ import DBus import Data.Word (Word32) import Assistant.NetMessager #else +#ifdef linux_HOST_OS #warning Building without dbus support; will poll for network connection changes #endif +#endif netWatcherThread :: NamedThread #if WITH_DBUS diff --git a/Assistant/Threads/SanityChecker.hs b/Assistant/Threads/SanityChecker.hs index b94d010e3..d7a71d477 100644 --- a/Assistant/Threads/SanityChecker.hs +++ b/Assistant/Threads/SanityChecker.hs @@ -27,7 +27,6 @@ import qualified Git.Command import qualified Git.Config import Utility.ThreadScheduler import qualified Assistant.Threads.Watcher as Watcher -import Utility.LogFile import Utility.Batch import Utility.NotificationBroadcaster import Config @@ -39,10 +38,14 @@ import Assistant.Unused import Logs.Unused import Logs.Transfer import Config.Files +import Utility.DiskFree import qualified Annex #ifdef WITH_WEBAPP import Assistant.WebApp.Types #endif +#ifndef mingw32_HOST_OS +import Utility.LogFile +#endif import Data.Time.Clock.POSIX import qualified Data.Text as T @@ -201,23 +204,33 @@ hourlyCheck = do #endif #ifndef mingw32_HOST_OS -{- Rotate logs until log file size is < 1 mb. -} +{- Rotate logs once when total log file size is > 2 mb. + - + - If total log size is larger than the amount of free disk space, + - continue rotating logs until size is < 2 mb, even if this + - results in immediately losing the just logged data. + -} checkLogSize :: Int -> Assistant () checkLogSize n = do f <- liftAnnex $ fromRepo gitAnnexLogFile logs <- liftIO $ listLogs f totalsize <- liftIO $ sum <$> mapM filesize logs - when (totalsize > oneMegabyte) $ do + when (totalsize > 2 * oneMegabyte) $ do notice ["Rotated logs due to size:", show totalsize] liftIO $ openLog f >>= redirLog - when (n < maxLogs + 1) $ - checkLogSize $ n + 1 + when (n < maxLogs + 1) $ do + df <- liftIO $ getDiskFree $ takeDirectory f + case df of + Just free + | free < fromIntegral totalsize -> + checkLogSize (n + 1) + _ -> noop where filesize f = fromIntegral . fileSize <$> liftIO (getFileStatus f) -#endif -oneMegabyte :: Int -oneMegabyte = 1000000 + oneMegabyte :: Int + oneMegabyte = 1000000 +#endif oneHour :: Int oneHour = 60 * 60 diff --git a/Assistant/Threads/TransferPoller.hs b/Assistant/Threads/TransferPoller.hs index 68075cac8..71bfe3676 100644 --- a/Assistant/Threads/TransferPoller.hs +++ b/Assistant/Threads/TransferPoller.hs @@ -35,7 +35,7 @@ transferPollerThread = namedThread "TransferPoller" $ do {- Downloads are polled by checking the size of the - temp file being used for the transfer. -} | transferDirection t == Download = do - let f = gitAnnexTmpLocation (transferKey t) g + let f = gitAnnexTmpObjectLocation (transferKey t) g sz <- liftIO $ catchMaybeIO $ fromIntegral . fileSize <$> getFileStatus f newsize t info sz diff --git a/Assistant/Threads/TransferWatcher.hs b/Assistant/Threads/TransferWatcher.hs index cd7282865..6e8791732 100644 --- a/Assistant/Threads/TransferWatcher.hs +++ b/Assistant/Threads/TransferWatcher.hs @@ -35,7 +35,7 @@ transferWatcherThread = namedThread "TransferWatcher" $ do , modifyHook = modifyhook , errHook = errhook } - void $ liftIO $ watchDir dir (const False) hooks id + void $ liftIO $ watchDir dir (const False) True hooks id debug ["watching for transfers"] type Handler = FilePath -> Assistant () diff --git a/Assistant/Threads/UpgradeWatcher.hs b/Assistant/Threads/UpgradeWatcher.hs index 80f2040a0..ffad09d3d 100644 --- a/Assistant/Threads/UpgradeWatcher.hs +++ b/Assistant/Threads/UpgradeWatcher.hs @@ -50,8 +50,9 @@ upgradeWatcherThread urlrenderer = namedThread "UpgradeWatcher" $ do let dir = parentDir flagfile let depth = length (splitPath dir) + 1 let nosubdirs f = length (splitPath f) == depth - void $ liftIO $ watchDir dir nosubdirs hooks (startup mvar) + void $ liftIO $ watchDir dir nosubdirs False hooks (startup mvar) -- Ignore bogus events generated during the startup scan. + -- We ask the watcher to not generate them, but just to be safe.. startup mvar scanner = do r <- scanner void $ swapMVar mvar Started diff --git a/Assistant/Threads/Upgrader.hs b/Assistant/Threads/Upgrader.hs index f0c47e844..60aeec70b 100644 --- a/Assistant/Threads/Upgrader.hs +++ b/Assistant/Threads/Upgrader.hs @@ -89,10 +89,10 @@ canUpgrade urgency urlrenderer d = ifM autoUpgradeEnabled getDistributionInfo :: Assistant (Maybe GitAnnexDistribution) getDistributionInfo = do - ua <- liftAnnex Url.getUserAgent + uo <- liftAnnex Url.getUrlOptions liftIO $ withTmpFile "git-annex.tmp" $ \tmpfile h -> do hClose h - ifM (Url.downloadQuiet distributionInfoUrl [] [] tmpfile ua) + ifM (Url.downloadQuiet distributionInfoUrl tmpfile uo) ( readish <$> readFileStrict tmpfile , return Nothing ) diff --git a/Assistant/Threads/Watcher.hs b/Assistant/Threads/Watcher.hs index 50a0efdd5..8a8e8faf0 100644 --- a/Assistant/Threads/Watcher.hs +++ b/Assistant/Threads/Watcher.hs @@ -23,7 +23,6 @@ import Assistant.Types.Changes import Assistant.Alert import Utility.DirWatcher import Utility.DirWatcher.Types -import qualified Utility.Lsof as Lsof import qualified Annex import qualified Annex.Queue import qualified Git @@ -40,6 +39,9 @@ import Annex.ReplaceFile import Git.Types import Config import Utility.ThreadScheduler +#ifndef mingw32_HOST_OS +import qualified Utility.Lsof as Lsof +#endif import Data.Bits.Utils import Data.Typeable @@ -100,7 +102,8 @@ runWatcher = do , delDirHook = deldirhook , errHook = errhook } - handle <- liftIO $ watchDir "." ignored hooks startup + scanevents <- liftAnnex $ annexStartupScan <$> Annex.getGitConfig + handle <- liftIO $ watchDir "." ignored scanevents hooks startup debug [ "watching", "."] {- Let the DirWatcher thread run until signalled to pause it, diff --git a/Assistant/Threads/WebApp.hs b/Assistant/Threads/WebApp.hs index d2c2afd47..7b24ccea1 100644 --- a/Assistant/Threads/WebApp.hs +++ b/Assistant/Threads/WebApp.hs @@ -1,6 +1,6 @@ {- git-annex assistant webapp thread - - - Copyright 2012 Joey Hess <joey@kitenet.net> + - Copyright 2012-2014 Joey Hess <joey@kitenet.net> - - Licensed under the GNU GPL version 3 or higher. -} @@ -41,10 +41,12 @@ import Utility.WebApp import Utility.Tmp import Utility.FileMode import Git +import qualified Annex import Yesod import Network.Socket (SockAddr, HostName) import Data.Text (pack, unpack) +import qualified Network.Wai.Handler.WarpTLS as TLS mkYesodDispatch "WebApp" $(parseRoutesFile "Assistant/WebApp/routes") @@ -55,13 +57,17 @@ webAppThread -> UrlRenderer -> Bool -> Maybe String - -> Maybe HostName -> Maybe (IO Url) + -> Maybe HostName -> Maybe (Url -> FilePath -> IO ()) -> NamedThread -webAppThread assistantdata urlrenderer noannex cannotrun listenhost postfirstrun onstartup = thread $ liftIO $ do +webAppThread assistantdata urlrenderer noannex cannotrun postfirstrun listenhost onstartup = thread $ liftIO $ do + listenhost' <- if isJust listenhost + then pure listenhost + else getAnnex $ annexListen <$> Annex.getGitConfig + tlssettings <- getAnnex getTlsSettings #ifdef __ANDROID__ - when (isJust listenhost) $ + when (isJust listenhost') $ -- See Utility.WebApp error "Sorry, --listen is not currently supported on Android" #endif @@ -73,22 +79,21 @@ webAppThread assistantdata urlrenderer noannex cannotrun listenhost postfirstrun <*> pure postfirstrun <*> pure cannotrun <*> pure noannex - <*> pure listenhost + <*> pure listenhost' setUrlRenderer urlrenderer $ yesodRender webapp (pack "") app <- toWaiAppPlain webapp app' <- ifM debugEnabled ( return $ httpDebugLogger app , return app ) - runWebApp listenhost app' $ \addr -> if noannex + runWebApp tlssettings listenhost' app' $ \addr -> if noannex then withTmpFile "webapp.html" $ \tmpfile h -> do hClose h - go addr webapp tmpfile Nothing + go tlssettings addr webapp tmpfile Nothing else do - let st = threadState assistantdata - htmlshim <- runThreadState st $ fromRepo gitAnnexHtmlShim - urlfile <- runThreadState st $ fromRepo gitAnnexUrlFile - go addr webapp htmlshim (Just urlfile) + htmlshim <- getAnnex' $ fromRepo gitAnnexHtmlShim + urlfile <- getAnnex' $ fromRepo gitAnnexUrlFile + go tlssettings addr webapp htmlshim (Just urlfile) where -- The webapp thread does not wait for the startupSanityCheckThread -- to finish, so that the user interface remains responsive while @@ -98,14 +103,31 @@ webAppThread assistantdata urlrenderer noannex cannotrun listenhost postfirstrun | noannex = return Nothing | otherwise = Just <$> (relHome =<< absPath - =<< runThreadState (threadState assistantdata) (fromRepo repoPath)) - go addr webapp htmlshim urlfile = do - let url = myUrl webapp addr + =<< getAnnex' (fromRepo repoPath)) + go tlssettings addr webapp htmlshim urlfile = do + let url = myUrl tlssettings webapp addr maybe noop (`writeFileProtected` url) urlfile writeHtmlShim "Starting webapp..." url htmlshim maybe noop (\a -> a url htmlshim) onstartup -myUrl :: WebApp -> SockAddr -> Url -myUrl webapp addr = unpack $ yesodRender webapp urlbase DashboardR [] + getAnnex a + | noannex = pure Nothing + | otherwise = getAnnex' a + getAnnex' = runThreadState (threadState assistantdata) + +myUrl :: Maybe TLS.TLSSettings -> WebApp -> SockAddr -> Url +myUrl tlssettings webapp addr = unpack $ yesodRender webapp urlbase DashboardR [] where - urlbase = pack $ "http://" ++ show addr + urlbase = pack $ proto ++ "://" ++ show addr + proto + | isJust tlssettings = "https" + | otherwise = "http" + +getTlsSettings :: Annex (Maybe TLS.TLSSettings) +getTlsSettings = do + cert <- fromRepo gitAnnexWebCertificate + privkey <- fromRepo gitAnnexWebPrivKey + ifM (liftIO $ allM doesFileExist [cert, privkey]) + ( return $ Just $ TLS.tlsSettings cert privkey + , return Nothing + ) diff --git a/Assistant/XMPP/Git.hs b/Assistant/XMPP/Git.hs index 22c3a125e..ab34dce1e 100644 --- a/Assistant/XMPP/Git.hs +++ b/Assistant/XMPP/Git.hs @@ -187,7 +187,7 @@ xmppPush cid gitpush = do v <- liftIO $ getEnv "GIT_ANNEX_TMP_DIR" case v of Nothing -> do - tmp <- liftAnnex $ fromRepo gitAnnexTmpDir + tmp <- liftAnnex $ fromRepo gitAnnexTmpMiscDir return $ tmp </> "xmppgit" Just d -> return $ d </> "xmppgit" diff --git a/Build/Configure.hs b/Build/Configure.hs index 487ed9b10..593e3ada7 100644 --- a/Build/Configure.hs +++ b/Build/Configure.hs @@ -35,8 +35,6 @@ tests = , TestCase "curl" $ testCmd "curl" "curl --version >/dev/null" , TestCase "wget" $ testCmd "wget" "wget --version >/dev/null" , TestCase "bup" $ testCmd "bup" "bup --version >/dev/null" - , TestCase "quvi" $ testCmd "quvi" "quvi --version >/dev/null" - , TestCase "newquvi" $ testCmd "newquvi" "quvi info >/dev/null" , TestCase "nice" $ testCmd "nice" "nice true >/dev/null" , TestCase "ionice" $ testCmd "ionice" "ionice -c3 true >/dev/null" , TestCase "nocache" $ testCmd "nocache" "nocache true >/dev/null" diff --git a/Build/DistributionUpdate.hs b/Build/DistributionUpdate.hs index 2c4d82495..a681ec2ed 100644 --- a/Build/DistributionUpdate.hs +++ b/Build/DistributionUpdate.hs @@ -21,8 +21,14 @@ main = do makeinfos :: Annex () makeinfos = do - basedir <- liftIO getRepoDir version <- liftIO getChangelogVersion + void $ inRepo $ runBool + [ Param "commit" + , Param "-a" + , Param "-m" + , Param $ "publishing git-annex " ++ version + ] + basedir <- liftIO getRepoDir now <- liftIO getCurrentTime liftIO $ putStrLn $ "building info files for version " ++ version ++ " in " ++ basedir fs <- liftIO $ dirContentsRecursiveSkipping (const False) True (basedir </> "git-annex") @@ -44,7 +50,7 @@ makeinfos = do void $ inRepo $ runBool [ Param "commit" , Param "-m" - , Param $ "publishing git-annex " ++ version + , Param $ "updated info files for git-annex " ++ version ] void $ inRepo $ runBool [ Param "annex" diff --git a/Command/Add.hs b/Command/Add.hs index d1dcb6025..3361a430a 100644 --- a/Command/Add.hs +++ b/Command/Add.hs @@ -19,6 +19,7 @@ import Annex.Content import Annex.Content.Direct import Annex.Perms import Annex.Link +import Annex.MetaData import qualified Annex import qualified Annex.Queue #ifdef WITH_CLIBS @@ -95,7 +96,7 @@ lockDown :: FilePath -> Annex (Maybe KeySource) lockDown file = ifM crippledFileSystem ( liftIO $ catchMaybeIO nohardlink , do - tmp <- fromRepo gitAnnexTmpDir + tmp <- fromRepo gitAnnexTmpMiscDir createAnnexDirectory tmp eitherToMaybe <$> tryAnnexIO (go tmp) ) @@ -145,26 +146,32 @@ ingest Nothing = return (Nothing, Nothing) ingest (Just source) = do backend <- chooseBackend $ keyFilename source k <- genKey source backend - cache <- liftIO $ genInodeCache $ contentLocation source - case (cache, inodeCache source) of - (_, Nothing) -> go k cache - (Just newc, Just c) | compareStrong c newc -> go k cache + ms <- liftIO $ catchMaybeIO $ getFileStatus $ contentLocation source + let mcache = toInodeCache =<< ms + case (mcache, inodeCache source) of + (_, Nothing) -> go k mcache ms + (Just newc, Just c) | compareStrong c newc -> go k mcache ms _ -> failure "changed while it was being added" where - go k cache = ifM isDirect ( godirect k cache , goindirect k cache ) + go k mcache ms = ifM isDirect + ( godirect k mcache ms + , goindirect k mcache ms + ) - goindirect (Just (key, _)) mcache = do + goindirect (Just (key, _)) mcache ms = do catchAnnex (moveAnnex key $ contentLocation source) (undo (keyFilename source) key) + maybe noop (genMetaData key (keyFilename source)) ms liftIO $ nukeFile $ keyFilename source return $ (Just key, mcache) - goindirect Nothing _ = failure "failed to generate a key" + goindirect _ _ _ = failure "failed to generate a key" - godirect (Just (key, _)) (Just cache) = do + godirect (Just (key, _)) (Just cache) ms = do addInodeCache key cache + maybe noop (genMetaData key (keyFilename source)) ms finishIngestDirect key source return $ (Just key, Just cache) - godirect _ _ = failure "failed to generate a key" + godirect _ _ _ = failure "failed to generate a key" failure msg = do warning $ keyFilename source ++ " " ++ msg @@ -211,15 +218,15 @@ link file key mcache = flip catchAnnex (undo file key) $ do l <- inRepo $ gitAnnexLink file key replaceFile file $ makeAnnexLink l -#ifdef WITH_CLIBS -#ifndef __ANDROID__ -- touch symlink to have same time as the original file, -- as provided in the InodeCache case mcache of +#if defined(WITH_CLIBS) && ! defined(__ANDROID__) Just c -> liftIO $ touch file (TimeSpec $ inodeCacheToMtime c) False - Nothing -> noop -#endif +#else + Just _ -> noop #endif + Nothing -> noop return l diff --git a/Command/AddUrl.hs b/Command/AddUrl.hs index 82b04f07b..a0978a88d 100644 --- a/Command/AddUrl.hs +++ b/Command/AddUrl.hs @@ -64,7 +64,7 @@ start relaxed optfile pathdepth s = go $ fromMaybe bad $ parseURI s QuviDownloader -> usequvi DefaultDownloader -> #ifdef WITH_QUVI - ifM (liftIO $ Quvi.supported s') + ifM (quviSupported s') ( usequvi , regulardownload url ) @@ -114,7 +114,7 @@ addUrlFileQuvi relaxed quviurl videourl file = do - it later. -} sizedkey <- addSizeUrlKey videourl key prepGetViaTmpChecked sizedkey $ do - tmp <- fromRepo $ gitAnnexTmpLocation key + tmp <- fromRepo $ gitAnnexTmpObjectLocation key showOutput ok <- Transfer.download webUUID key (Just file) Transfer.forwardRetry $ const $ do liftIO $ createDirectoryIfMissing True (parentDir tmp) @@ -134,8 +134,7 @@ perform relaxed url file = ifAnnexed file addurl geturl setUrlPresent key url next $ return True | otherwise = do - headers <- getHttpHeaders - (exists, samesize) <- Url.withUserAgent $ Url.check url headers $ keySize key + (exists, samesize) <- Url.withUrlOptions $ Url.check url (keySize key) if exists && samesize then do setUrlPresent key url @@ -163,7 +162,7 @@ download url file = do - downloads, as the dummy key for a given url is stable. -} dummykey <- addSizeUrlKey url =<< Backend.URL.fromUrl url Nothing prepGetViaTmpChecked dummykey $ do - tmp <- fromRepo $ gitAnnexTmpLocation dummykey + tmp <- fromRepo $ gitAnnexTmpObjectLocation dummykey showOutput ifM (runtransfer dummykey tmp) ( do @@ -192,8 +191,7 @@ download url file = do -} addSizeUrlKey :: URLString -> Key -> Annex Key addSizeUrlKey url key = do - headers <- getHttpHeaders - size <- snd <$> Url.withUserAgent (Url.exists url headers) + size <- snd <$> Url.withUrlOptions (Url.exists url) return $ key { keySize = size } cleanup :: URLString -> FilePath -> Key -> Maybe FilePath -> Annex Bool @@ -212,10 +210,9 @@ cleanup url file key mtmp = do nodownload :: Bool -> URLString -> FilePath -> Annex Bool nodownload relaxed url file = do - headers <- getHttpHeaders (exists, size) <- if relaxed then pure (True, Nothing) - else Url.withUserAgent $ Url.exists url headers + else Url.withUrlOptions (Url.exists url) if exists then do key <- Backend.URL.fromUrl url size diff --git a/Command/Assistant.hs b/Command/Assistant.hs index 7a05cdfba..496df1dd2 100644 --- a/Command/Assistant.hs +++ b/Command/Assistant.hs @@ -19,7 +19,7 @@ import System.Environment def :: [Command] def = [noRepo checkAutoStart $ dontCheck repoExists $ withOptions options $ - command "assistant" paramNothing seek SectionCommon + notBareRepo $ command "assistant" paramNothing seek SectionCommon "automatically handle changes"] options :: [Option] diff --git a/Command/DropUnused.hs b/Command/DropUnused.hs index a3409ab1b..345d03032 100644 --- a/Command/DropUnused.hs +++ b/Command/DropUnused.hs @@ -27,7 +27,7 @@ seek ps = do withUnusedMaps (start numcopies) ps start :: NumCopies -> UnusedMaps -> Int -> CommandStart -start numcopies = startUnused "dropunused" (perform numcopies) (performOther gitAnnexBadLocation) (performOther gitAnnexTmpLocation) +start numcopies = startUnused "dropunused" (perform numcopies) (performOther gitAnnexBadLocation) (performOther gitAnnexTmpObjectLocation) perform :: NumCopies -> Key -> CommandPerform perform numcopies key = maybe droplocal dropremote =<< Remote.byNameWithUUID =<< from diff --git a/Command/Fsck.hs b/Command/Fsck.hs index 6cf444967..059f3e91e 100644 --- a/Command/Fsck.hs +++ b/Command/Fsck.hs @@ -147,7 +147,7 @@ performRemote key file backend numcopies remote = ] withtmp a = do pid <- liftIO getPID - t <- fromRepo gitAnnexTmpDir + t <- fromRepo gitAnnexTmpObjectDir createAnnexDirectory t let tmp = t </> "fsck" ++ show pid ++ "." ++ keyFile key let cleanup = liftIO $ catchIO (removeFile tmp) (const noop) @@ -466,7 +466,8 @@ getFsckTime key = do - To guard against time stamp damange (for example, if an annex directory - is copied without -a), the fsckstate file contains a time that should - be identical to its modification time. - - (This is not possible to do on Windows.) + - (This is not possible to do on Windows, and so the timestamp in + - the file will only be equal or greater than the modification time.) -} recordStartTime :: Annex () recordStartTime = do @@ -477,10 +478,10 @@ recordStartTime = do withFile f WriteMode $ \h -> do #ifndef mingw32_HOST_OS t <- modificationTime <$> getFileStatus f - hPutStr h $ showTime $ realToFrac t #else - noop + t <- getPOSIXTime #endif + hPutStr h $ showTime $ realToFrac t where showTime :: POSIXTime -> String showTime = show @@ -494,15 +495,18 @@ getStartTime = do f <- fromRepo gitAnnexFsckState liftIO $ catchDefaultIO Nothing $ do timestamp <- modificationTime <$> getFileStatus f -#ifndef mingw32_HOST_OS - t <- readishTime <$> readFile f - return $ if Just (realToFrac timestamp) == t + let fromstatus = Just (realToFrac timestamp) + fromfile <- readishTime <$> readFile f + return $ if matchingtimestamp fromfile fromstatus then Just timestamp else Nothing -#else - return $ Just timestamp -#endif where readishTime :: String -> Maybe POSIXTime readishTime s = utcTimeToPOSIXSeconds <$> parseTime defaultTimeLocale "%s%Qs" s + matchingtimestamp fromfile fromstatus = +#ifndef mingw32_HOST_OS + fromfile == fromstatus +#else + fromfile >= fromstatus +#endif diff --git a/Command/ImportFeed.hs b/Command/ImportFeed.hs index dfa89b344..50f4278b6 100644 --- a/Command/ImportFeed.hs +++ b/Command/ImportFeed.hs @@ -108,7 +108,7 @@ findDownloads u = go =<< downloadFeed u Nothing -> mkquvi f i #ifdef WITH_QUVI mkquvi f i = case getItemLink i of - Just link -> ifM (liftIO $ Quvi.supported link) + Just link -> ifM (quviSupported link) ( return $ Just $ ToDownload f u i $ QuviLink link , return Nothing ) @@ -121,10 +121,10 @@ findDownloads u = go =<< downloadFeed u downloadFeed :: URLString -> Annex (Maybe Feed) downloadFeed url = do showOutput - ua <- Url.getUserAgent + uo <- Url.getUrlOptions liftIO $ withTmpFile "feed" $ \f h -> do fileEncoding h - ifM (Url.download url [] [] f ua) + ifM (Url.download url f uo) ( parseFeedString <$> hGetContentsStrict h , return Nothing ) diff --git a/Command/Info.hs b/Command/Info.hs index 98cc49efa..f27fdfb1d 100644 --- a/Command/Info.hs +++ b/Command/Info.hs @@ -201,7 +201,7 @@ known_annex_size = stat "size of annexed files in working tree" $ json id $ showSizeKeys <$> cachedReferencedData tmp_size :: Stat -tmp_size = staleSize "temporary directory size" gitAnnexTmpDir +tmp_size = staleSize "temporary object directory size" gitAnnexTmpObjectDir bad_data_size :: Stat bad_data_size = staleSize "bad keys size" gitAnnexBadDir diff --git a/Command/MetaData.hs b/Command/MetaData.hs index 6112dd095..55d67c6b7 100644 --- a/Command/MetaData.hs +++ b/Command/MetaData.hs @@ -10,6 +10,7 @@ module Command.MetaData where import Common.Annex import qualified Annex import Command +import Annex.MetaData import Logs.MetaData import Types.MetaData @@ -17,7 +18,7 @@ import qualified Data.Set as S import Data.Time.Clock.POSIX def :: [Command] -def = [withOptions [setOption, tagOption, untagOption] $ +def = [withOptions [setOption, tagOption, untagOption, jsonOption] $ command "metadata" paramPaths seek SectionMetaData "sets metadata of a file"] @@ -55,14 +56,16 @@ perform :: POSIXTime -> [ModMeta] -> Key -> CommandPerform perform _ [] k = next $ cleanup k perform now ms k = do oldm <- getCurrentMetaData k - let m = foldl' unionMetaData newMetaData $ map (modMeta oldm) ms + let m = foldl' unionMetaData emptyMetaData $ map (modMeta oldm) ms addMetaData' k m now next $ cleanup k cleanup :: Key -> CommandCleanup cleanup k = do - m <- getCurrentMetaData k - showLongNote $ unlines $ concatMap showmeta $ fromMetaData $ currentMetaData m + l <- map unwrapmeta . fromMetaData <$> getCurrentMetaData k + maybeShowJSON l + showLongNote $ unlines $ concatMap showmeta l return True where - showmeta (f, vs) = map (\v -> fromMetaField f ++ "=" ++ fromMetaValue v) $ S.toList vs + unwrapmeta (f, v) = (fromMetaField f, map fromMetaValue (S.toList v)) + showmeta (f, vs) = map ((f ++ "=") ++) vs diff --git a/Command/PreCommit.hs b/Command/PreCommit.hs index 4b90b5c2e..fa34ad245 100644 --- a/Command/PreCommit.hs +++ b/Command/PreCommit.hs @@ -13,7 +13,9 @@ import Config import qualified Command.Add import qualified Command.Fix import Annex.Direct +import Annex.Hook import Annex.View +import Annex.View.ViewedFile import Logs.View import Logs.MetaData import Types.View @@ -27,13 +29,16 @@ def = [command "pre-commit" paramPaths seek SectionPlumbing seek :: CommandSeek seek ps = ifM isDirect - -- update direct mode mappings for committed files - ( withWords startDirect ps + ( do + -- update direct mode mappings for committed files + withWords startDirect ps + runAnnexHook preCommitAnnexHook , do -- fix symlinks to files being committed withFilesToBeCommitted (whenAnnexed Command.Fix.start) ps -- inject unlocked files into the annex withFilesUnlockedToBeCommitted startIndirect ps + runAnnexHook preCommitAnnexHook -- committing changes to a view updates metadata mv <- currentView case mv of @@ -42,6 +47,7 @@ seek ps = ifM isDirect (addViewMetaData v) (removeViewMetaData v) ) + startIndirect :: FilePath -> CommandStart startIndirect f = next $ do @@ -52,12 +58,12 @@ startIndirect f = next $ do startDirect :: [String] -> CommandStart startDirect _ = next $ next $ preCommitDirect -addViewMetaData :: View -> FileView -> Key -> CommandStart +addViewMetaData :: View -> ViewedFile -> Key -> CommandStart addViewMetaData v f k = do showStart "metadata" f next $ next $ changeMetaData k $ fromView v f -removeViewMetaData :: View -> FileView -> Key -> CommandStart +removeViewMetaData :: View -> ViewedFile -> Key -> CommandStart removeViewMetaData v f k = do showStart "metadata" f next $ next $ changeMetaData k $ unsetMetaData $ fromView v f diff --git a/Command/Sync.hs b/Command/Sync.hs index f041b5d23..07006ef28 100644 --- a/Command/Sync.hs +++ b/Command/Sync.hs @@ -12,25 +12,18 @@ import Common.Annex import Command import qualified Annex import qualified Annex.Branch -import qualified Annex.Queue import qualified Remote import qualified Types.Remote as Remote import Annex.Direct -import Annex.CatFile -import Annex.Link +import Annex.Hook import qualified Git.Command import qualified Git.LsFiles as LsFiles -import qualified Git.Merge import qualified Git.Branch import qualified Git.Ref import qualified Git -import Git.Types (BlobType(..)) import qualified Types.Remote import qualified Remote.Git -import Types.Key import Config -import Annex.ReplaceFile -import Git.FileMode import Annex.Wanted import Annex.Content import Command.Get (getKeyFile') @@ -38,9 +31,8 @@ import qualified Command.Move import Logs.Location import Annex.Drop import Annex.UUID +import Annex.AutoMerge -import qualified Data.Set as S -import Data.Hash.MD5 import Control.Concurrent.MVar def :: [Command] @@ -156,6 +148,7 @@ commitStaged commitmessage = go =<< inRepo Git.Branch.currentUnsafe where go Nothing = return False go (Just branch) = do + runAnnexHook preCommitAnnexHook parent <- inRepo $ Git.Ref.sha branch void $ inRepo $ Git.Branch.commit False commitmessage branch (maybeToList parent) @@ -176,7 +169,7 @@ mergeLocal (Just branch) = go =<< needmerge go False = stop go True = do showStart "merge" $ Git.Ref.describe syncbranch - next $ next $ mergeFrom syncbranch + next $ next $ autoMergeFrom syncbranch (Just branch) pushLocal :: Maybe Git.Ref -> CommandStart pushLocal Nothing = stop @@ -220,10 +213,11 @@ mergeRemote :: Remote -> Maybe Git.Ref -> CommandCleanup mergeRemote remote b = case b of Nothing -> do branch <- inRepo Git.Branch.currentUnsafe - and <$> mapM merge (branchlist branch) - Just _ -> and <$> (mapM merge =<< tomerge (branchlist b)) + and <$> mapM (merge Nothing) (branchlist branch) + Just thisbranch -> + and <$> (mapM (merge (Just thisbranch)) =<< tomerge (branchlist b)) where - merge = mergeFrom . remoteBranch remote + merge thisbranch = flip autoMergeFrom thisbranch . remoteBranch remote tomerge = filterM (changed remote) branchlist Nothing = [] branchlist (Just branch) = [branch, syncBranch branch] @@ -304,206 +298,6 @@ mergeAnnex = do void Annex.Branch.forceUpdate stop -{- Merges from a branch into the current branch. -} -mergeFrom :: Git.Ref -> Annex Bool -mergeFrom branch = do - showOutput - ifM isDirect - ( maybe go godirect =<< inRepo Git.Branch.current - , go - ) - where - go = runmerge $ inRepo $ Git.Merge.mergeNonInteractive branch - godirect currbranch = do - old <- inRepo $ Git.Ref.sha currbranch - d <- fromRepo gitAnnexMergeDir - r <- runmerge $ inRepo $ mergeDirect d branch - new <- inRepo $ Git.Ref.sha currbranch - case (old, new) of - (Just oldsha, Just newsha) -> - mergeDirectCleanup d oldsha newsha - _ -> noop - return r - runmerge a = ifM a - ( return True - , resolveMerge - ) - -{- Resolves a conflicted merge. It's important that any conflicts be - - resolved in a way that itself avoids later merge conflicts, since - - multiple repositories may be doing this concurrently. - - - - Only annexed files are resolved; other files are left for the user to - - handle. - - - - This uses the Keys pointed to by the files to construct new - - filenames. So when both sides modified file foo, - - it will be deleted, and replaced with files foo.variant-A and - - foo.variant-B. - - - - On the other hand, when one side deleted foo, and the other modified it, - - it will be deleted, and the modified version stored as file - - foo.variant-A (or B). - - - - It's also possible that one side has foo as an annexed file, and - - the other as a directory or non-annexed file. The annexed file - - is renamed to resolve the merge, and the other object is preserved as-is. - -} -resolveMerge :: Annex Bool -resolveMerge = do - top <- fromRepo Git.repoPath - (fs, cleanup) <- inRepo (LsFiles.unmerged [top]) - mergedfs <- catMaybes <$> mapM resolveMerge' fs - let merged = not (null mergedfs) - void $ liftIO cleanup - - (deleted, cleanup2) <- inRepo (LsFiles.deleted [top]) - unless (null deleted) $ - Annex.Queue.addCommand "rm" [Params "--quiet -f --"] deleted - void $ liftIO cleanup2 - - when merged $ do - unlessM isDirect $ - cleanConflictCruft mergedfs top - Annex.Queue.flush - void $ inRepo $ Git.Command.runBool - [ Param "commit" - , Param "-m" - , Param "git-annex automatic merge conflict fix" - ] - showLongNote "Merge conflict was automatically resolved; you may want to examine the result." - return merged - -resolveMerge' :: LsFiles.Unmerged -> Annex (Maybe FilePath) -resolveMerge' u - | issymlink LsFiles.valUs && issymlink LsFiles.valThem = do - kus <- getKey LsFiles.valUs - kthem <- getKey LsFiles.valThem - case (kus, kthem) of - -- Both sides of conflict are annexed files - (Just keyUs, Just keyThem) -> do - removeoldfile keyUs - if keyUs == keyThem - then makelink keyUs - else do - makelink keyUs - makelink keyThem - return $ Just file - -- Our side is annexed, other side is not. - (Just keyUs, Nothing) -> do - ifM isDirect - ( do - removeoldfile keyUs - makelink keyUs - movefromdirectmerge file - , do - unstageoldfile - makelink keyUs - ) - return $ Just file - -- Our side is not annexed, other side is. - (Nothing, Just keyThem) -> do - makelink keyThem - unstageoldfile - return $ Just file - -- Neither side is annexed; cannot resolve. - (Nothing, Nothing) -> return Nothing - | otherwise = return Nothing - where - file = LsFiles.unmergedFile u - issymlink select = select (LsFiles.unmergedBlobType u) `elem` [Just SymlinkBlob, Nothing] - makelink key = do - let dest = mergeFile file key - l <- inRepo $ gitAnnexLink dest key - replaceFile dest $ makeAnnexLink l - stageSymlink dest =<< hashSymlink l - whenM isDirect $ - toDirect key dest - removeoldfile keyUs = do - ifM isDirect - ( removeDirect keyUs file - , liftIO $ nukeFile file - ) - Annex.Queue.addCommand "rm" [Params "--quiet -f --"] [file] - unstageoldfile = Annex.Queue.addCommand "rm" [Params "--quiet -f --cached --"] [file] - getKey select = case select (LsFiles.unmergedSha u) of - Nothing -> return Nothing - Just sha -> catKey sha symLinkMode - - {- Move something out of the direct mode merge directory and into - - the git work tree. - - - - On a filesystem not supporting symlinks, this is complicated - - because a directory may contain annex links, but just - - moving them into the work tree will not let git know they are - - symlinks. - - - - Also, if the content of the file is available, make it available - - in direct mode. - -} - movefromdirectmerge item = do - d <- fromRepo gitAnnexMergeDir - liftIO $ rename (d </> item) item - mapM_ setuplink =<< liftIO (dirContentsRecursive item) - setuplink f = do - v <- getAnnexLinkTarget f - case v of - Nothing -> noop - Just target -> do - unlessM (coreSymlinks <$> Annex.getGitConfig) $ - addAnnexLink target f - maybe noop (`toDirect` f) - (fileKey (takeFileName target)) - -{- git-merge moves conflicting files away to files - - named something like f~HEAD or f~branch, but the - - exact name chosen can vary. Once the conflict is resolved, - - this cruft can be deleted. To avoid deleting legitimate - - files that look like this, only delete files that are - - A) not staged in git and B) look like git-annex symlinks. - -} -cleanConflictCruft :: [FilePath] -> FilePath -> Annex () -cleanConflictCruft resolvedfs top = do - (fs, cleanup) <- inRepo $ LsFiles.notInRepo False [top] - mapM_ clean fs - void $ liftIO cleanup - where - clean f - | matchesresolved f = whenM (isJust <$> isAnnexLink f) $ - liftIO $ nukeFile f - | otherwise = noop - s = S.fromList resolvedfs - matchesresolved f = S.member (base f) s - base f = reverse $ drop 1 $ dropWhile (/= '~') $ reverse f - -{- The filename to use when resolving a conflicted merge of a file, - - that points to a key. - - - - Something derived from the key needs to be included in the filename, - - but rather than exposing the whole key to the user, a very weak hash - - is used. There is a very real, although still unlikely, chance of - - conflicts using this hash. - - - - In the event that there is a conflict with the filename generated - - for some other key, that conflict will itself be handled by the - - conflicted merge resolution code. That case is detected, and the full - - key is used in the filename. - -} -mergeFile :: FilePath -> Key -> FilePath -mergeFile file key - | doubleconflict = go $ key2file key - | otherwise = go $ shortHash $ key2file key - where - varmarker = ".variant-" - doubleconflict = varmarker `isInfixOf` file - go v = takeDirectory file - </> dropExtension (takeFileName file) - ++ varmarker ++ v - ++ takeExtension file - -shortHash :: String -> String -shortHash = take 4 . md5s . md5FilePath - changed :: Remote -> Git.Ref -> Annex Bool changed remote b = do let r = remoteBranch remote b diff --git a/Command/Unlock.hs b/Command/Unlock.hs index 9f2c257fb..4cfe39307 100644 --- a/Command/Unlock.hs +++ b/Command/Unlock.hs @@ -36,7 +36,7 @@ perform dest key = do unlessM (checkDiskSpace Nothing key 0) $ error "cannot unlock" src <- calcRepo $ gitAnnexLocation key - tmpdest <- fromRepo $ gitAnnexTmpLocation key + tmpdest <- fromRepo $ gitAnnexTmpObjectLocation key liftIO $ createDirectoryIfMissing True (parentDir tmpdest) showAction "copying" ifM (liftIO $ copyFileExternal src tmpdest) diff --git a/Command/Unused.hs b/Command/Unused.hs index d48956920..6b4475568 100644 --- a/Command/Unused.hs +++ b/Command/Unused.hs @@ -63,7 +63,7 @@ checkUnused :: CommandPerform checkUnused = chain 0 [ check "" unusedMsg $ findunused =<< Annex.getState Annex.fast , check "bad" staleBadMsg $ staleKeysPrune gitAnnexBadDir False - , check "tmp" staleTmpMsg $ staleKeysPrune gitAnnexTmpDir True + , check "tmp" staleTmpMsg $ staleKeysPrune gitAnnexTmpObjectDir True ] where findunused True = do diff --git a/Command/VAdd.hs b/Command/VAdd.hs index 3dc1fd4cf..e3726a051 100644 --- a/Command/VAdd.hs +++ b/Command/VAdd.hs @@ -10,11 +10,11 @@ module Command.VAdd where import Common.Annex import Command import Annex.View -import Command.View (paramView, parseViewParam, checkoutViewBranch) +import Command.View (checkoutViewBranch) def :: [Command] -def = [notBareRepo $ notDirect $ - command "vadd" paramView seek SectionMetaData "add subdirs to current view"] +def = [notBareRepo $ notDirect $ command "vadd" (paramRepeating "FIELD=GLOB") + seek SectionMetaData "add subdirs to current view"] seek :: CommandSeek seek = withWords start diff --git a/Command/VCycle.hs b/Command/VCycle.hs index b41e099a4..f7da47fa2 100644 --- a/Command/VCycle.hs +++ b/Command/VCycle.hs @@ -36,6 +36,6 @@ start = go =<< currentView else next $ next $ checkoutViewBranch v' narrowView vcycle rest (c:cs) - | multiValue (viewFilter c) = rest ++ cs ++ [c] + | viewVisible c = rest ++ cs ++ [c] | otherwise = vcycle (c:rest) cs vcycle rest c = rest ++ c diff --git a/Command/VFilter.hs b/Command/VFilter.hs index c16b28956..bd17aca45 100644 --- a/Command/VFilter.hs +++ b/Command/VFilter.hs @@ -10,7 +10,7 @@ module Command.VFilter where import Common.Annex import Command import Annex.View -import Command.View (paramView, parseViewParam, checkoutViewBranch) +import Command.View (paramView, checkoutViewBranch) def :: [Command] def = [notBareRepo $ notDirect $ diff --git a/Command/View.hs b/Command/View.hs index 17e136f7b..93b045c39 100644 --- a/Command/View.hs +++ b/Command/View.hs @@ -13,7 +13,6 @@ import qualified Git import qualified Git.Command import qualified Git.Ref import qualified Git.Branch -import Types.MetaData import Types.View import Annex.View import Logs.View @@ -35,7 +34,7 @@ start params = do go view Nothing = next $ perform view go view (Just v) | v == view = stop - | otherwise = error "Already in a view. Use 'git annex vadd' to further refine this view." + | otherwise = error "Already in a view. Use the vfilter and vadd commands to further refine this view." perform :: View -> CommandPerform perform view = do @@ -43,21 +42,14 @@ perform view = do next $ checkoutViewBranch view applyView paramView :: String -paramView = paramPair (paramRepeating "FIELD=VALUE") (paramRepeating "TAG") - -parseViewParam :: String -> (MetaField, String) -parseViewParam s = case separate (== '=') s of - (tag, []) -> (tagMetaField, tag) - (field, wanted) -> either error (\f -> (f, wanted)) (mkMetaField field) +paramView = paramPair (paramRepeating "TAG") (paramRepeating "FIELD=VALUE") mkView :: [String] -> Annex View -mkView params = do - v <- View <$> viewbranch <*> pure [] - return $ fst $ refineView v $ - map parseViewParam $ reverse params +mkView params = go =<< inRepo Git.Branch.current where - viewbranch = fromMaybe (error "not on any branch!") - <$> inRepo Git.Branch.current + go Nothing = error "not on any branch!" + go (Just b) = return $ fst $ refineView (View b []) $ + map parseViewParam $ reverse params checkoutViewBranch :: View -> (View -> Annex Git.Branch) -> CommandCleanup checkoutViewBranch view mkbranch = do diff --git a/Command/WebApp.hs b/Command/WebApp.hs index d5f43432c..91c9afcd0 100644 --- a/Command/WebApp.hs +++ b/Command/WebApp.hs @@ -68,18 +68,24 @@ start' allowauto listenhost = do cannotrun <- needsUpgrade . fromMaybe (error "no version") =<< getVersion browser <- fromRepo webBrowser f <- liftIO . absPath =<< fromRepo gitAnnexHtmlShim + listenhost' <- if isJust listenhost + then pure listenhost + else annexListen <$> Annex.getGitConfig ifM (checkpid <&&> checkshim f) ( if isJust listenhost then error "The assistant is already running, so --listen cannot be used." else do url <- liftIO . readFile =<< fromRepo gitAnnexUrlFile - liftIO $ openBrowser browser f url Nothing Nothing - , startDaemon True True Nothing cannotrun listenhost $ Just $ - \origout origerr url htmlshim -> - if isJust listenhost - then maybe noop (`hPutStrLn` url) origout - else openBrowser browser htmlshim url origout origerr + liftIO $ if isJust listenhost' + then putStrLn url + else liftIO $ openBrowser browser f url Nothing Nothing + , do + startDaemon True True Nothing cannotrun listenhost' $ Just $ + \origout origerr url htmlshim -> + if isJust listenhost' + then maybe noop (`hPutStrLn` url) origout + else openBrowser browser htmlshim url origout origerr ) auto | allowauto = liftIO $ startNoRepo [] @@ -107,8 +113,11 @@ startNoRepo _ = do (d:_) -> do setCurrentDirectory d state <- Annex.new =<< Git.CurrentRepo.get - void $ Annex.eval state $ callCommandAction $ - start' False listenhost + void $ Annex.eval state $ do + whenM (fromRepo Git.repoIsLocalBare) $ + error $ d ++ " is a bare git repository, cannot run the webapp in it" + callCommandAction $ + start' False listenhost {- Run the webapp without a repository, which prompts the user, makes one, - changes to it, starts the regular assistant, and redirects the @@ -139,8 +148,9 @@ firstRun listenhost = do let callback a = Just $ a v runAssistant d $ do startNamedThread urlrenderer $ - webAppThread d urlrenderer True Nothing listenhost + webAppThread d urlrenderer True Nothing (callback signaler) + listenhost (callback mainthread) waitNamedThreads where @@ -153,7 +163,8 @@ firstRun listenhost = do hFlush stdout go | otherwise = do - browser <- maybe Nothing webBrowser <$> Git.Config.global + browser <- maybe Nothing webBrowser + <$> catchDefaultIO Nothing Git.Config.global openBrowser browser htmlshim url Nothing Nothing go where @@ -79,11 +79,3 @@ setCrippledFileSystem :: Bool -> Annex () setCrippledFileSystem b = do setConfig (annexConfig "crippledfilesystem") (Git.Config.boolConfig b) Annex.changeGitConfig $ \c -> c { annexCrippledFileSystem = b } - -{- Gets the http headers to use. -} -getHttpHeaders :: Annex [String] -getHttpHeaders = do - v <- annexHttpHeadersCommand <$> Annex.getGitConfig - case v of - Just cmd -> lines <$> liftIO (readProcess "sh" ["-c", cmd]) - Nothing -> annexHttpHeaders <$> Annex.getGitConfig diff --git a/Git/CatFile.hs b/Git/CatFile.hs index c8cb76d59..c7c51b894 100644 --- a/Git/CatFile.hs +++ b/Git/CatFile.hs @@ -11,6 +11,7 @@ module Git.CatFile ( catFileStart', catFileStop, catFile, + catFileDetails, catTree, catObject, catObjectDetails, @@ -52,6 +53,10 @@ catFile :: CatFileHandle -> Branch -> FilePath -> IO L.ByteString catFile h branch file = catObject h $ Ref $ fromRef branch ++ ":" ++ toInternalGitPath file +catFileDetails :: CatFileHandle -> Branch -> FilePath -> IO (Maybe (L.ByteString, Sha, ObjectType)) +catFileDetails h branch file = catObjectDetails h $ Ref $ + fromRef branch ++ ":" ++ toInternalGitPath file + {- Uses a running git cat-file read the content of an object. - Objects that do not exist will have "" returned. -} catObject :: CatFileHandle -> Ref -> IO L.ByteString diff --git a/Git/Command.hs b/Git/Command.hs index 0fa3d1b3b..a0c7c4b2a 100644 --- a/Git/Command.hs +++ b/Git/Command.hs @@ -15,9 +15,6 @@ import Common import Git import Git.Types import qualified Utility.CoProcess as CoProcess -#ifdef mingw32_HOST_OS -import Git.FilePath -#endif import Utility.Batch {- Constructs a git command line operating on the specified repo. -} diff --git a/Git/Hook.hs b/Git/Hook.hs index d56a4a565..6245a292d 100644 --- a/Git/Hook.hs +++ b/Git/Hook.hs @@ -15,6 +15,10 @@ data Hook = Hook { hookName :: FilePath , hookScript :: String } + deriving (Ord) + +instance Eq Hook where + a == b = hookName a == hookName b hookFile :: Hook -> Repo -> FilePath hookFile h r = localGitDir r </> "hooks" </> hookName h diff --git a/Git/Queue.hs b/Git/Queue.hs index 9bb7f77d1..5f7b142c0 100644 --- a/Git/Queue.hs +++ b/Git/Queue.hs @@ -17,16 +17,17 @@ module Git.Queue ( flush, ) where -import qualified Data.Map as M -import System.IO -import System.Process - import Utility.SafeCommand import Common import Git import Git.Command import qualified Git.UpdateIndex +import qualified Data.Map as M +#ifndef mingw32_HOST_OS +import System.Process +#endif + {- Queable actions that can be performed in a git repository. -} data Action @@ -147,8 +148,9 @@ runAction :: Repo -> Action -> IO () runAction repo (UpdateIndexAction streamers) = -- list is stored in reverse order Git.UpdateIndex.streamUpdateIndex repo $ reverse streamers -runAction repo action@(CommandAction {}) = +runAction repo action@(CommandAction {}) = do #ifndef mingw32_HOST_OS + let p = (proc "xargs" $ "-0":"git":toCommand gitparams) { env = gitEnv repo } withHandle StdinHandle createProcessSuccess p $ \h -> do fileEncoding h hPutStr h $ intercalate "\0" $ toCommand $ getFiles action @@ -162,6 +164,5 @@ runAction repo action@(CommandAction {}) = void $ boolSystem "git" (gitparams ++ [f]) #endif where - p = (proc "xargs" $ "-0":"git":toCommand gitparams) { env = gitEnv repo } gitparams = gitCommandLine (Param (getSubcommand action):getParams action) repo diff --git a/Git/Repair.hs b/Git/Repair.hs index 96da5ffe7..cdd70329d 100644 --- a/Git/Repair.hs +++ b/Git/Repair.hs @@ -75,24 +75,35 @@ removeLoose r s = do return True else return False +{- Explodes all pack files, and deletes them. + - + - First moves all pack files to a temp dir, before unpacking them each in + - turn. + - + - This is because unpack-objects will not unpack a pack file if it's in the + - git repo. + - + - Also, this prevents unpack-objects from possibly looking at corrupt + - pack files to see if they contain an object, while unpacking a + - non-corrupt pack file. + -} explodePacks :: Repo -> IO Bool -explodePacks r = do - packs <- listPackFiles r - if null packs - then return False - else do - putStrLn "Unpacking all pack files." - mapM_ go packs - return True +explodePacks r = go =<< listPackFiles r where - go packfile = withTmpFileIn (localGitDir r) "pack" $ \tmp _ -> do - moveFile packfile tmp - nukeFile $ packIdxFile packfile - allowRead tmp - -- May fail, if pack file is corrupt. - void $ tryIO $ - pipeWrite [Param "unpack-objects", Param "-r"] r $ \h -> + go [] = return False + go packs = withTmpDir "packs" $ \tmpdir -> do + putStrLn "Unpacking all pack files." + forM_ packs $ \packfile -> do + moveFile packfile (tmpdir </> takeFileName packfile) + nukeFile $ packIdxFile packfile + forM_ packs $ \packfile -> do + let tmp = tmpdir </> takeFileName packfile + allowRead tmp + -- May fail, if pack file is corrupt. + void $ tryIO $ + pipeWrite [Param "unpack-objects", Param "-r"] r $ \h -> L.hPut h =<< L.readFile tmp + return True {- Try to retrieve a set of missing objects, from the remotes of a - repository. Returns any that could not be retreived. diff --git a/Git/UpdateIndex.hs b/Git/UpdateIndex.hs index 6d1ff2548..4ecd77363 100644 --- a/Git/UpdateIndex.hs +++ b/Git/UpdateIndex.hs @@ -15,6 +15,7 @@ module Git.UpdateIndex ( startUpdateIndex, stopUpdateIndex, lsTree, + lsSubTree, updateIndexLine, stageFile, unstageFile, @@ -74,6 +75,13 @@ lsTree (Ref x) repo streamer = do void $ cleanup where params = map Param ["ls-tree", "-z", "-r", "--full-tree", x] +lsSubTree :: Ref -> FilePath -> Repo -> Streamer +lsSubTree (Ref x) p repo streamer = do + (s, cleanup) <- pipeNullSplit params repo + mapM_ streamer s + void $ cleanup + where + params = map Param ["ls-tree", "-z", "-r", "--full-tree", x, p] {- Generates a line suitable to be fed into update-index, to add - a given file with a given sha. -} @@ -5,8 +5,6 @@ - Licensed under the GNU GPL version 3 or higher. -} -{-# LANGUAGE CPP #-} - module Limit where import Common.Annex @@ -29,18 +27,13 @@ import Logs.Group import Logs.Unused import Logs.Location import Git.Types (RefDate(..)) +import Utility.Glob import Utility.HumanTime import Utility.DataUnits import Data.Time.Clock.POSIX import qualified Data.Set as S import qualified Data.Map as M -import System.Path.WildMatch - -#ifdef WITH_TDFA -import Text.Regex.TDFA -import Text.Regex.TDFA.String -#endif {- Checks if there are user-specified limits. -} limited :: Annex Bool @@ -82,33 +75,21 @@ addInclude :: String -> Annex () addInclude = addLimit . limitInclude limitInclude :: MkLimit -limitInclude glob = Right $ const $ return . matchglob glob +limitInclude glob = Right $ const $ return . matchGlobFile glob {- Add a limit to skip files that match the glob. -} addExclude :: String -> Annex () addExclude = addLimit . limitExclude limitExclude :: MkLimit -limitExclude glob = Right $ const $ return . not . matchglob glob - -{- Could just use wildCheckCase, but this way the regex is only compiled - - once. Also, we use regex-TDFA when available, because it's less buggy - - in its support of non-unicode characters. -} -matchglob :: String -> MatchInfo -> Bool -matchglob glob (MatchingFile fi) = -#ifdef WITH_TDFA - case cregex of - Right r -> case execute r (matchFile fi) of - Right (Just _) -> True - _ -> False - Left _ -> error $ "failed to compile regex: " ++ regex - where - cregex = compile defaultCompOpt defaultExecOpt regex - regex = '^':wildToRegex glob -#else - wildCheckCase glob (matchFile fi) -#endif -matchglob _ (MatchingKey _) = False +limitExclude glob = Right $ const $ return . not . matchGlobFile glob + +matchGlobFile :: String -> (MatchInfo -> Bool) +matchGlobFile glob = go + where + cglob = compileGlob glob CaseSensative -- memoized + go (MatchingKey _) = False + go (MatchingFile fi) = matchGlob cglob (matchFile fi) {- Adds a limit to skip files not believed to be present - in a specfied repository. Optionally on a prior date. -} @@ -270,9 +251,13 @@ addMetaData = addLimit . limitMetaData limitMetaData :: MkLimit limitMetaData s = case parseMetaData s of Left e -> Left e - Right (f, v) -> Right $ const $ checkKey (check f v) + Right (f, v) -> + let cglob = compileGlob (fromMetaValue v) CaseInsensative + in Right $ const $ checkKey (check f cglob) where - check f v k = S.member v . metaDataValues f <$> getCurrentMetaData k + check f cglob k = not . S.null + . S.filter (matchGlob cglob . fromMetaValue) + . metaDataValues f <$> getCurrentMetaData k addTimeLimit :: String -> Annex () addTimeLimit s = do diff --git a/Locations.hs b/Locations.hs index 8189b8a07..74cace156 100644 --- a/Locations.hs +++ b/Locations.hs @@ -23,8 +23,9 @@ module Locations ( annexLocation, gitAnnexDir, gitAnnexObjectDir, - gitAnnexTmpDir, - gitAnnexTmpLocation, + gitAnnexTmpMiscDir, + gitAnnexTmpObjectDir, + gitAnnexTmpObjectLocation, gitAnnexBadDir, gitAnnexBadLocation, gitAnnexUnusedLog, @@ -33,6 +34,8 @@ module Locations ( gitAnnexScheduleState, gitAnnexTransferDir, gitAnnexCredsDir, + gitAnnexWebCertificate, + gitAnnexWebPrivKey, gitAnnexFeedStateDir, gitAnnexFeedState, gitAnnexMergeDir, @@ -180,13 +183,17 @@ gitAnnexDir r = addTrailingPathSeparator $ Git.localGitDir r </> annexDir gitAnnexObjectDir :: Git.Repo -> FilePath gitAnnexObjectDir r = addTrailingPathSeparator $ Git.localGitDir r </> objectDir -{- .git/annex/tmp/ is used for temp files -} -gitAnnexTmpDir :: Git.Repo -> FilePath -gitAnnexTmpDir r = addTrailingPathSeparator $ gitAnnexDir r </> "tmp" +{- .git/annex/misctmp/ is used for random temp files -} +gitAnnexTmpMiscDir :: Git.Repo -> FilePath +gitAnnexTmpMiscDir r = addTrailingPathSeparator $ gitAnnexDir r </> "misctmp" + +{- .git/annex/tmp/ is used for temp files for key's contents -} +gitAnnexTmpObjectDir :: Git.Repo -> FilePath +gitAnnexTmpObjectDir r = addTrailingPathSeparator $ gitAnnexDir r </> "tmp" {- The temp file to use for a given key's content. -} -gitAnnexTmpLocation :: Key -> Git.Repo -> FilePath -gitAnnexTmpLocation key r = gitAnnexTmpDir r </> keyFile key +gitAnnexTmpObjectLocation :: Key -> Git.Repo -> FilePath +gitAnnexTmpObjectLocation key r = gitAnnexTmpObjectDir r </> keyFile key {- .git/annex/bad/ is used for bad files found during fsck -} gitAnnexBadDir :: Git.Repo -> FilePath @@ -218,6 +225,13 @@ gitAnnexScheduleState r = gitAnnexDir r </> "schedulestate" gitAnnexCredsDir :: Git.Repo -> FilePath gitAnnexCredsDir r = addTrailingPathSeparator $ gitAnnexDir r </> "creds" +{- .git/annex/certificate.pem and .git/annex/key.pem are used by the webapp + - when HTTPS is enabled -} +gitAnnexWebCertificate :: Git.Repo -> FilePath +gitAnnexWebCertificate r = gitAnnexDir r </> "certificate.pem" +gitAnnexWebPrivKey :: Git.Repo -> FilePath +gitAnnexWebPrivKey r = gitAnnexDir r </> "privkey.pem" + {- .git/annex/feeds/ is used to record per-key (url) state by importfeeds -} gitAnnexFeedStateDir :: Git.Repo -> FilePath gitAnnexFeedStateDir r = addTrailingPathSeparator $ gitAnnexDir r </> "feedstate" diff --git a/Logs/MetaData.hs b/Logs/MetaData.hs index 807b50afa..6702c3733 100644 --- a/Logs/MetaData.hs +++ b/Logs/MetaData.hs @@ -24,13 +24,14 @@ -} {-# OPTIONS_GHC -fno-warn-orphans #-} +{-# LANGUAGE CPP #-} module Logs.MetaData ( getCurrentMetaData, - getMetaData, addMetaData, addMetaData', currentMetaData, + copyMetaData, ) where import Common.Annex @@ -54,7 +55,7 @@ getMetaData = readLog . metaDataLogFile getCurrentMetaData :: Key -> Annex MetaData getCurrentMetaData = currentMetaData . collect <$$> getMetaData where - collect = foldl' unionMetaData newMetaData . map value . S.toAscList + collect = foldl' unionMetaData emptyMetaData . map value . S.toAscList {- Adds in some metadata, which can override existing values, or unset - them, but otherwise leaves any existing metadata as-is. -} @@ -120,13 +121,34 @@ simplifyLog s = case sl of else s _ -> s where +#if MIN_VERSION_containers(0,5,0) sl = S.toDescList s +#else + sl = reverse (S.toAscList s) +#endif go c _ [] = c go c newer (l:ls) - | unique == newMetaData = go c newer ls + | unique == emptyMetaData = go c newer ls | otherwise = go (l { value = unique } : c) (unionMetaData unique newer) ls where older = value l unique = older `differenceMetaData` newer + +{- Copies the metadata from the old key to the new key. + - + - The exact content of the metadata file is copied, so that the timestamps + - remain the same, and because this is more space-efficient in the git + - repository. + - + - Any metadata already attached to the new key is not preserved. + -} +copyMetaData :: Key -> Key -> Annex () +copyMetaData oldkey newkey + | oldkey == newkey = noop + | otherwise = do + l <- getMetaData oldkey + unless (S.null l) $ + Annex.Branch.change (metaDataLogFile newkey) $ + const $ showLog l diff --git a/Logs/Transfer.hs b/Logs/Transfer.hs index ebbb153ac..742bdc7b9 100644 --- a/Logs/Transfer.hs +++ b/Logs/Transfer.hs @@ -186,7 +186,7 @@ runTransfer t file shouldretry a = do | transferDirection t == Upload = liftIO $ readMVar metervar | otherwise = do - f <- fromRepo $ gitAnnexTmpLocation (transferKey t) + f <- fromRepo $ gitAnnexTmpObjectLocation (transferKey t) liftIO $ catchDefaultIO 0 $ fromIntegral . fileSize <$> getFileStatus f diff --git a/Logs/View.hs b/Logs/View.hs index 63590d5e9..79c2556b3 100644 --- a/Logs/View.hs +++ b/Logs/View.hs @@ -75,12 +75,14 @@ branchView view | otherwise = "(" ++ branchcomp' c ++ ")" branchcomp' (ViewComponent metafield viewfilter _) =concat [ forcelegal (fromMetaField metafield) - , "=" , branchvals viewfilter ] - branchvals (FilterValues set) = intercalate "," $ - map (forcelegal . fromMetaValue) $ S.toList set - branchvals (FilterGlob glob) = forcelegal glob + branchvals (FilterValues set) = '=' : branchset set + branchvals (FilterGlob glob) = '=' : forcelegal glob + branchvals (ExcludeValues set) = "!=" ++ branchset set + branchset = intercalate "," + . map (forcelegal . fromMetaValue) + . S.toList forcelegal s | Git.Ref.legal True s = s | otherwise = map (\c -> if isAlphaNum c then c else '_') s @@ -253,7 +253,7 @@ hdevtools: distributionupdate: git pull cabal configure - ghc --make Build/DistributionUpdate + ghc --make Build/DistributionUpdate -XPackageImports ./Build/DistributionUpdate .PHONY: git-annex git-union-merge git-recover-repository tags build-stamp diff --git a/Remote/Git.hs b/Remote/Git.hs index d714cfec5..14157f498 100644 --- a/Remote/Git.hs +++ b/Remote/Git.hs @@ -144,7 +144,7 @@ repoAvail r else return True | Git.repoIsUrl r = return True | Git.repoIsLocalUnknown r = return False - | otherwise = liftIO $ catchBoolIO $ onLocal r $ return True + | otherwise = liftIO $ isJust <$> catchMaybeIO (Git.Config.read r) {- Tries to read the config for a specified remote, updates state, and - returns the updated repo. -} @@ -158,14 +158,15 @@ tryGitConfigRead r | haveconfig r' -> return r' | otherwise -> configlist_failed Left _ -> configlist_failed - | Git.repoIsHttp r = do - headers <- getHttpHeaders - store $ geturlconfig headers + | Git.repoIsHttp r = store geturlconfig | Git.GCrypt.isEncrypted r = handlegcrypt =<< getConfigMaybe (remoteConfig r "uuid") | Git.repoIsUrl r = return r - | otherwise = store $ safely $ onLocal r $ do - ensureInitialized - Annex.getState Annex.repo + | otherwise = store $ safely $ do + s <- Annex.new r + Annex.eval s $ do + Annex.BranchState.disableUpdate + ensureInitialized + Annex.getState Annex.repo where haveconfig = not . M.null . Git.config @@ -185,11 +186,11 @@ tryGitConfigRead r return $ Right r' Left l -> return $ Left l - geturlconfig headers = do - ua <- Url.getUserAgent + geturlconfig = do + uo <- Url.getUrlOptions v <- liftIO $ withTmpFile "git-annex.tmp" $ \tmpfile h -> do hClose h - ifM (Url.downloadQuiet (Git.repoLocation r ++ "/config") headers [] tmpfile ua) + ifM (Url.downloadQuiet (Git.repoLocation r ++ "/config") tmpfile uo) ( pipedconfig "git" [Param "config", Param "--null", Param "--list", Param "--file", File tmpfile] , return $ Left undefined ) @@ -255,22 +256,22 @@ tryGitConfigRead r -} inAnnex :: Remote -> Key -> Annex (Either String Bool) inAnnex rmt key - | Git.repoIsHttp r = checkhttp =<< getHttpHeaders + | Git.repoIsHttp r = checkhttp | Git.repoIsUrl r = checkremote | otherwise = checklocal where r = repo rmt - checkhttp headers = do + checkhttp = do showChecking r - ifM (anyM (\u -> Url.withUserAgent $ Url.checkBoth u headers (keySize key)) (keyUrls rmt key)) + ifM (Url.withUrlOptions $ \uo -> anyM (\u -> Url.checkBoth u (keySize key) uo) (keyUrls rmt key)) ( return $ Right True , return $ Left "not found" ) checkremote = Ssh.inAnnex r key checklocal = guardUsable r (cantCheck r) $ dispatch <$> check where - check = liftIO $ catchMsgIO $ onLocal r $ - Annex.Content.inAnnexSafe key + check = either (Left . show) Right + <$> tryAnnex (onLocal rmt $ Annex.Content.inAnnexSafe key) dispatch (Left e) = Left e dispatch (Right (Just b)) = Right b dispatch (Right Nothing) = cantCheck r @@ -287,13 +288,13 @@ keyUrls r key = map tourl locs' #ifndef mingw32_HOST_OS locs' = locs #else - locs' = map (replace "\\" "/") (annexLocations key) + locs' = map (replace "\\" "/") locs #endif dropKey :: Remote -> Key -> Annex Bool dropKey r key | not $ Git.repoIsUrl (repo r) = - guardUsable (repo r) False $ commitOnCleanup r $ liftIO $ onLocal (repo r) $ do + guardUsable (repo r) False $ commitOnCleanup r $ onLocal r $ do ensureInitialized whenM (Annex.Content.inAnnex key) $ do Annex.Content.lockContent key $ @@ -313,7 +314,7 @@ copyFromRemote' r key file dest let params = Ssh.rsyncParams r Download u <- getUUID -- run copy from perspective of remote - liftIO $ onLocal (repo r) $ do + onLocal r $ do ensureInitialized v <- Annex.Content.prepSendAnnex key case v of @@ -412,7 +413,7 @@ copyToRemote r key file p let params = Ssh.rsyncParams r Upload u <- getUUID -- run copy from perspective of remote - liftIO $ onLocal (repo r) $ ifM (Annex.Content.inAnnex key) + onLocal r $ ifM (Annex.Content.inAnnex key) ( return True , do ensureInitialized @@ -441,19 +442,40 @@ fsckOnRemote r params {- The passed repair action is run in the Annex monad of the remote. -} repairRemote :: Git.Repo -> Annex Bool -> Annex (IO Bool) -repairRemote r a = return $ Remote.Git.onLocal r a - -{- Runs an action on a local repository inexpensively, by making an annex - - monad using that repository. -} -onLocal :: Git.Repo -> Annex a -> IO a -onLocal r a = do +repairRemote r a = return $ do s <- Annex.new r Annex.eval s $ do - -- No need to update the branch; its data is not used - -- for anything onLocal is used to do. Annex.BranchState.disableUpdate + ensureInitialized a +{- Runs an action from the perspective of a local remote. + - + - The AnnexState is cached for speed and to avoid resource leaks. + - + - The repository's git-annex branch is not updated, as an optimisation. + - No caller of onLocal can query data from the branch and be ensured + - it gets a current value. Caller of onLocal can make changes to + - the branch, however. + -} +onLocal :: Remote -> Annex a -> Annex a +onLocal r a = do + m <- Annex.getState Annex.remoteannexstate + case M.lookup (uuid r) m of + Nothing -> do + st <- liftIO $ Annex.new (repo r) + go st $ do + Annex.BranchState.disableUpdate + a + Just st -> go st a + where + cache st = Annex.changeState $ \s -> s + { Annex.remoteannexstate = M.insert (uuid r) st (Annex.remoteannexstate s) } + go st a' = do + (ret, st') <- liftIO $ Annex.run st a' + cache st' + return ret + {- Copys a file with rsync unless both locations are on the same - filesystem. Then cp could be faster. -} rsyncOrCopyFile :: [CommandParam] -> FilePath -> FilePath -> MeterUpdate -> Annex Bool @@ -490,7 +512,7 @@ commitOnCleanup r a = go `after` a where go = Annex.addCleanup (Git.repoLocation $ repo r) cleanup cleanup - | not $ Git.repoIsUrl (repo r) = liftIO $ onLocal (repo r) $ + | not $ Git.repoIsUrl (repo r) = onLocal r $ doQuietSideAction $ Annex.Branch.commit "update" | otherwise = void $ do diff --git a/Remote/Glacier.hs b/Remote/Glacier.hs index 84557851b..fe6f53a77 100644 --- a/Remote/Glacier.hs +++ b/Remote/Glacier.hs @@ -73,16 +73,16 @@ gen r u c gc = new <$> remoteCost gc veryExpensiveRemoteCost glacierSetup :: Maybe UUID -> Maybe CredPair -> RemoteConfig -> Annex (RemoteConfig, UUID) glacierSetup mu mcreds c = do u <- maybe (liftIO genUUID) return mu - glacierSetup' (isJust mu) u mcreds c -glacierSetup' :: Bool -> UUID -> Maybe CredPair -> RemoteConfig -> Annex (RemoteConfig, UUID) -glacierSetup' enabling u mcreds c = do + c' <- setRemoteCredPair c (AWS.creds u) mcreds + glacierSetup' (isJust mu) u c' +glacierSetup' :: Bool -> UUID -> RemoteConfig -> Annex (RemoteConfig, UUID) +glacierSetup' enabling u c = do c' <- encryptionSetup c let fullconfig = c' `M.union` defaults unless enabling $ genVault fullconfig u gitConfigSpecialRemote u fullconfig "glacier" "true" - c'' <- setRemoteCredPair fullconfig (AWS.creds u) mcreds - return (c'', u) + return (c', u) where remotename = fromJust (M.lookup "name" c) defvault = remotename ++ "-" ++ fromUUID u @@ -196,7 +196,7 @@ checkPresent r k = do else return $ Right False Left err -> return $ Left err - params = + params = glacierParams (config r) [ Param "archive" , Param "checkpresent" , Param $ getVault $ config r diff --git a/Remote/Rsync.hs b/Remote/Rsync.hs index 570725bcf..8f00a767e 100644 --- a/Remote/Rsync.hs +++ b/Remote/Rsync.hs @@ -112,26 +112,26 @@ genRsyncOpts c gc transport url = RsyncOpts | otherwise = True rsyncTransport :: RemoteGitConfig -> RsyncUrl -> Annex ([CommandParam], RsyncUrl) -rsyncTransport gc rawurl - | rsyncUrlIsShell rawurl = - (\rsh -> return (rsyncShell rsh, resturl)) =<< +rsyncTransport gc url + | rsyncUrlIsShell url = + (\rsh -> return (rsyncShell rsh, url)) =<< case fromNull ["ssh"] (remoteAnnexRsyncTransport gc) of "ssh":sshopts -> do let (port, sshopts') = sshReadPort sshopts - host = takeWhile (/=':') resturl + userhost = takeWhile (/=':') url -- Connection caching (Param "ssh":) <$> sshCachingOptions - (host, port) + (userhost, port) (map Param $ loginopt ++ sshopts') "rsh":rshopts -> return $ map Param $ "rsh" : loginopt ++ rshopts rsh -> error $ "Unknown Rsync transport: " ++ unwords rsh - | otherwise = return ([], rawurl) + | otherwise = return ([], url) where - (login,resturl) = case separate (=='@') rawurl of - (h, "") -> (Nothing, h) - (l, h) -> (Just l, h) + login = case separate (=='@') url of + (_h, "") -> Nothing + (l, _) -> Just l loginopt = maybe [] (\l -> ["-l",l]) login fromNull as xs = if null xs then as else xs @@ -247,7 +247,7 @@ sendParams = ifM crippledFileSystem withRsyncScratchDir :: (FilePath -> Annex a) -> Annex a withRsyncScratchDir a = do p <- liftIO getPID - t <- fromRepo gitAnnexTmpDir + t <- fromRepo gitAnnexTmpObjectDir createAnnexDirectory t let tmp = t </> "rsynctmp" </> show p nuke tmp diff --git a/Remote/S3.hs b/Remote/S3.hs index b217892e7..c1a99abcd 100644 --- a/Remote/S3.hs +++ b/Remote/S3.hs @@ -76,9 +76,10 @@ gen r u c gc = new <$> remoteCost gc expensiveRemoteCost s3Setup :: Maybe UUID -> Maybe CredPair -> RemoteConfig -> Annex (RemoteConfig, UUID) s3Setup mu mcreds c = do u <- maybe (liftIO genUUID) return mu - s3Setup' u mcreds c -s3Setup' :: UUID -> Maybe CredPair -> RemoteConfig -> Annex (RemoteConfig, UUID) -s3Setup' u mcreds c = if isIA c then archiveorg else defaulthost + c' <- setRemoteCredPair c (AWS.creds u) mcreds + s3Setup' u c' +s3Setup' :: UUID -> RemoteConfig -> Annex (RemoteConfig, UUID) +s3Setup' u c = if isIA c then archiveorg else defaulthost where remotename = fromJust (M.lookup "name" c) defbucket = remotename ++ "-" ++ fromUUID u @@ -92,8 +93,7 @@ s3Setup' u mcreds c = if isIA c then archiveorg else defaulthost use fullconfig = do gitConfigSpecialRemote u fullconfig "s3" "true" - c' <- setRemoteCredPair fullconfig (AWS.creds u) mcreds - return (c', u) + return (fullconfig, u) defaulthost = do c' <- encryptionSetup c diff --git a/Remote/Web.hs b/Remote/Web.hs index 2863d9d5e..ddd1fc1cc 100644 --- a/Remote/Web.hs +++ b/Remote/Web.hs @@ -14,7 +14,6 @@ import Types.Remote import qualified Git import qualified Git.Construct import Annex.Content -import Config import Config.Cost import Logs.Web import Types.Key @@ -117,9 +116,8 @@ checkKey' key us = firsthit us (Right False) $ \u -> do return $ Left "quvi support needed for this url" #endif DefaultDownloader -> do - headers <- getHttpHeaders - Url.withUserAgent $ catchMsgIO . - Url.checkBoth u' headers (keySize key) + Url.withUrlOptions $ catchMsgIO . + Url.checkBoth u' (keySize key) where firsthit [] miss _ = return miss firsthit (u:rest) _ a = do diff --git a/Remote/WebDAV.hs b/Remote/WebDAV.hs index 6ce83470b..91b83053c 100644 --- a/Remote/WebDAV.hs +++ b/Remote/WebDAV.hs @@ -14,10 +14,15 @@ import qualified Data.Map as M import qualified Data.ByteString.UTF8 as B8 import qualified Data.ByteString.Lazy.UTF8 as L8 import qualified Data.ByteString.Lazy as L -import Network.URI (normalizePathSegments) import qualified Control.Exception as E +import qualified Control.Exception.Lifted as EL +#if MIN_VERSION_DAV(0,6,0) +import Network.HTTP.Client (HttpException(..)) +#else import Network.HTTP.Conduit (HttpException(..)) +#endif import Network.HTTP.Types +import System.Log.Logger (debugM) import System.IO.Error import Common.Annex @@ -33,8 +38,8 @@ import Creds import Utility.Metered import Annex.Content import Annex.UUID +import Remote.WebDAV.DavUrl -type DavUrl = String type DavUser = B8.ByteString type DavPass = B8.ByteString @@ -82,10 +87,10 @@ webdavSetup mu mcreds c = do let url = fromMaybe (error "Specify url=") $ M.lookup "url" c c' <- encryptionSetup c - creds <- getCreds c' u + creds <- maybe (getCreds c' u) (return . Just) mcreds testDav url creds gitConfigSpecialRemote u c' "webdav" "true" - c'' <- setRemoteCredPair c' (davCreds u) mcreds + c'' <- setRemoteCredPair c' (davCreds u) creds return (c'', u) store :: Remote -> Key -> AssociatedFile -> MeterUpdate -> Annex Bool @@ -105,7 +110,7 @@ storeEncrypted r (cipher, enck) k p = metered (Just p) k $ \meterupdate -> storeHelper :: Remote -> Key -> DavUrl -> DavUser -> DavPass -> L.ByteString -> IO Bool storeHelper r k baseurl user pass b = catchBoolIO $ do - davMkdir tmpurl user pass + mkdirRecursiveDAV tmpurl user pass storeChunks k tmpurl keyurl chunksize storer recorder finalizer where tmpurl = tmpLocation baseurl k @@ -114,11 +119,10 @@ storeHelper r k baseurl user pass b = catchBoolIO $ do storer urls = storeChunked chunksize urls storehttp b recorder url s = storehttp url (L8.fromString s) finalizer srcurl desturl = do - void $ catchMaybeHttp (deleteContent desturl user pass) - davMkdir (urlParent desturl) user pass - moveContent srcurl (B8.fromString desturl) user pass - storehttp url v = putContent url user pass - (contentType, v) + void $ tryNonAsync (deleteDAV desturl user pass) + mkdirRecursiveDAV (urlParent desturl) user pass + moveDAV srcurl desturl user pass + storehttp url = putDAV url user pass retrieveCheap :: Remote -> Key -> FilePath -> Annex Bool retrieveCheap _ _ _ = return False @@ -128,7 +132,7 @@ retrieve r k _f d p = metered (Just p) k $ \meterupdate -> davAction r False $ \(baseurl, user, pass) -> liftIO $ catchBoolIO $ withStoredFiles r k baseurl user pass onerr $ \urls -> do meteredWriteFileChunks meterupdate d urls $ \url -> do - mb <- davGetUrlContent url user pass + mb <- getDAV url user pass case mb of Nothing -> throwIO "download failed" Just b -> return b @@ -148,7 +152,7 @@ retrieveEncrypted r (cipher, enck) k d p = metered (Just p) k $ \meterupdate -> feeder _ _ [] _ = noop feeder user pass (url:urls) h = do - mb <- davGetUrlContent url user pass + mb <- getDAV url user pass case mb of Nothing -> throwIO "download failed" Just b -> do @@ -160,7 +164,7 @@ remove r k = davAction r False $ \(baseurl, user, pass) -> liftIO $ do -- Delete the key's whole directory, including any chunked -- files, etc, in a single action. let url = davLocation baseurl k - isJust <$> catchMaybeHttp (deleteContent url user pass) + isJust . eitherToMaybe <$> tryNonAsync (deleteDAV url user pass) checkPresent :: Remote -> Key -> Annex (Either String Bool) checkPresent r k = davAction r noconn go @@ -173,7 +177,7 @@ checkPresent r k = davAction r noconn go where check [] = return $ Right True check (url:urls) = do - v <- davUrlExists url user pass + v <- existsDAV url user pass if v == Right True then check urls else return v @@ -182,7 +186,7 @@ checkPresent r k = davAction r noconn go - or if there's a problem accessing it, - or perhaps this was an intermittent error. -} onerr url = do - v <- davUrlExists url user pass + v <- existsDAV url user pass return $ if v == Right True then Left $ "failed to read " ++ url else v @@ -199,11 +203,11 @@ withStoredFiles withStoredFiles r k baseurl user pass onerr a | isJust $ chunkSize $ config r = do let chunkcount = keyurl ++ chunkCount - v <- davGetUrlContent chunkcount user pass + v <- getDAV chunkcount user pass case v of Just s -> a $ listChunks keyurl $ L8.toString s Nothing -> do - chunks <- probeChunks keyurl $ \u -> (== Right True) <$> davUrlExists u user pass + chunks <- probeChunks keyurl $ \u -> (== Right True) <$> existsDAV u user pass if null chunks then onerr chunkcount else a chunks @@ -231,46 +235,12 @@ toDavUser = B8.fromString toDavPass :: String -> DavPass toDavPass = B8.fromString -{- The directory where files(s) for a key are stored. -} -davLocation :: DavUrl -> Key -> DavUrl -davLocation baseurl k = addTrailingPathSeparator $ - davUrl baseurl $ hashDirLower k </> keyFile k - -{- Where we store temporary data for a key as it's being uploaded. -} -tmpLocation :: DavUrl -> Key -> DavUrl -tmpLocation baseurl k = addTrailingPathSeparator $ - davUrl baseurl $ "tmp" </> keyFile k - -davUrl :: DavUrl -> FilePath -> DavUrl -davUrl baseurl file = baseurl </> file - -davUrlExists :: DavUrl -> DavUser -> DavPass -> IO (Either String Bool) -davUrlExists url user pass = decode <$> catchHttp get - where - decode (Right _) = Right True -#if ! MIN_VERSION_http_conduit(1,9,0) - decode (Left (Left (StatusCodeException status _))) -#else - decode (Left (Left (StatusCodeException status _ _))) -#endif - | statusCode status == statusCode notFound404 = Right False - decode (Left e) = Left $ showEitherException e -#if ! MIN_VERSION_DAV(0,4,0) - get = getProps url user pass -#else - get = getProps url user pass Nothing -#endif - -davGetUrlContent :: DavUrl -> DavUser -> DavPass -> IO (Maybe L.ByteString) -davGetUrlContent url user pass = fmap (snd . snd) <$> - catchMaybeHttp (getPropsAndContent url user pass) - {- Creates a directory in WebDAV, if not already present; also creating - any missing parent directories. -} -davMkdir :: DavUrl -> DavUser -> DavPass -> IO () -davMkdir url user pass = go url +mkdirRecursiveDAV :: DavUrl -> DavUser -> DavPass -> IO () +mkdirRecursiveDAV url user pass = go url where - make u = makeCollection u user pass + make u = mkdirDAV u user pass go u = do r <- E.try (make u) :: IO (Either E.SomeException Bool) @@ -287,64 +257,25 @@ davMkdir url user pass = go url - to use this directory will fail. -} Left _ -> return () -{- Catches HTTP and IO exceptions. -} -catchMaybeHttp :: IO a -> IO (Maybe a) -catchMaybeHttp a = (Just <$> a) `E.catches` - [ E.Handler $ \(_e :: HttpException) -> return Nothing - , E.Handler $ \(_e :: E.IOException) -> return Nothing - ] - -{- Catches HTTP and IO exceptions -} -catchHttp :: IO a -> IO (Either EitherException a) -catchHttp a = (Right <$> a) `E.catches` - [ E.Handler $ \(e :: HttpException) -> return $ Left $ Left e - , E.Handler $ \(e :: E.IOException) -> return $ Left $ Right e - ] - -type EitherException = Either HttpException E.IOException - -showEitherException :: EitherException -> String -#if ! MIN_VERSION_http_conduit(1,9,0) -showEitherException (Left (StatusCodeException status _)) = -#else -showEitherException (Left (StatusCodeException status _ _)) = -#endif - show $ statusMessage status -showEitherException (Left httpexception) = show httpexception -showEitherException (Right ioexception) = show ioexception - -throwIO :: String -> IO a -throwIO msg = ioError $ mkIOError userErrorType msg Nothing Nothing - -urlParent :: DavUrl -> DavUrl -urlParent url = dropTrailingPathSeparator $ - normalizePathSegments (dropTrailingPathSeparator url ++ "/..") - where - {- Test if a WebDAV store is usable, by writing to a test file, and then - deleting the file. Exits with an IO error if not. -} testDav :: String -> Maybe CredPair -> Annex () testDav baseurl (Just (u, p)) = do showSideAction "testing WebDAV server" - test "make directory" $ davMkdir baseurl user pass - test "write file" $ putContent testurl user pass - (contentType, L.empty) - test "delete file" $ deleteContent testurl user pass + test "make directory" $ mkdirRecursiveDAV baseurl user pass + test "write file" $ putDAV testurl user pass L.empty + test "delete file" $ deleteDAV testurl user pass where test desc a = liftIO $ - either (\e -> throwIO $ "WebDAV failed to " ++ desc ++ ": " ++ showEitherException e) + either (\e -> throwIO $ "WebDAV failed to " ++ desc ++ ": " ++ show e) (const noop) - =<< catchHttp a + =<< tryNonAsync a user = toDavUser u pass = toDavPass p testurl = davUrl baseurl "git-annex-test" testDav _ Nothing = error "Need to configure webdav username and password." -{- Content-Type to use for files uploaded to WebDAV. -} -contentType :: Maybe B8.ByteString -contentType = Just $ B8.fromString "application/octet-stream" - getCreds :: RemoteConfig -> UUID -> Annex (Maybe CredPair) getCreds c u = getRemoteCredPairFor "webdav" c (davCreds u) @@ -354,3 +285,115 @@ davCreds u = CredPairStorage , credPairEnvironment = ("WEBDAV_USERNAME", "WEBDAV_PASSWORD") , credPairRemoteKey = Just "davcreds" } + +{- Content-Type to use for files uploaded to WebDAV. -} +contentType :: Maybe B8.ByteString +contentType = Just $ B8.fromString "application/octet-stream" + +throwIO :: String -> IO a +throwIO msg = ioError $ mkIOError userErrorType msg Nothing Nothing + +debugDAV :: DavUrl -> String -> IO () +debugDAV msg url = debugM "DAV" $ msg ++ " " ++ url + +{--------------------------------------------------------------------- + - Low-level DAV operations, using the new DAV monad when available. + ---------------------------------------------------------------------} + +putDAV :: DavUrl -> DavUser -> DavPass -> L.ByteString -> IO () +putDAV url user pass b = do + debugDAV "PUT" url +#if MIN_VERSION_DAV(0,6,0) + goDAV url user pass $ putContentM (contentType, b) +#else + putContent url user pass (contentType, b) +#endif + +getDAV :: DavUrl -> DavUser -> DavPass -> IO (Maybe L.ByteString) +getDAV url user pass = do + debugDAV "GET" url + eitherToMaybe <$> tryNonAsync go + where +#if MIN_VERSION_DAV(0,6,0) + go = goDAV url user pass $ snd <$> getContentM +#else + go = snd . snd <$> getPropsAndContent url user pass +#endif + +deleteDAV :: DavUrl -> DavUser -> DavPass -> IO () +deleteDAV url user pass = do + debugDAV "DELETE" url +#if MIN_VERSION_DAV(0,6,0) + goDAV url user pass delContentM +#else + deleteContent url user pass +#endif + +moveDAV :: DavUrl -> DavUrl -> DavUser -> DavPass -> IO () +moveDAV url newurl user pass = do + debugDAV ("MOVE to " ++ newurl ++ " from ") url +#if MIN_VERSION_DAV(0,6,0) + goDAV url user pass $ moveContentM newurl' +#else + moveContent url newurl' user pass +#endif + where + newurl' = B8.fromString newurl + +mkdirDAV :: DavUrl -> DavUser -> DavPass -> IO Bool +mkdirDAV url user pass = do + debugDAV "MKDIR" url +#if MIN_VERSION_DAV(0,6,0) + goDAV url user pass mkCol +#else + makeCollection url user pass +#endif + +existsDAV :: DavUrl -> DavUser -> DavPass -> IO (Either String Bool) +existsDAV url user pass = do + debugDAV "EXISTS" url + either (Left . show) id <$> tryNonAsync check + where + ispresent = return . Right +#if MIN_VERSION_DAV(0,6,0) + check = goDAV url user pass $ do + setDepth Nothing + EL.catchJust + (matchStatusCodeException notFound404) + (getPropsM >> ispresent True) + (const $ ispresent False) +#else + check = E.catchJust + (matchStatusCodeException notFound404) +#if ! MIN_VERSION_DAV(0,4,0) + (getProps url user pass >> ispresent True) +#else + (getProps url user pass Nothing >> ispresent True) +#endif + (const $ ispresent False) +#endif + +matchStatusCodeException :: Status -> HttpException -> Maybe () +#if MIN_VERSION_DAV(0,6,0) +matchStatusCodeException want (StatusCodeException s _ _) +#else +matchStatusCodeException want (StatusCodeException s _) +#endif + | s == want = Just () + | otherwise = Nothing +matchStatusCodeException _ _ = Nothing + +#if MIN_VERSION_DAV(0,6,0) +goDAV :: DavUrl -> DavUser -> DavPass -> DAVT IO a -> IO a +goDAV url user pass a = choke $ evalDAVT url $ do + setResponseTimeout Nothing -- disable default (5 second!) timeout + setCreds user pass + a + where + choke :: IO (Either String a) -> IO a + choke f = do + x <- f + case x of + Left e -> error e + Right r -> return r +#endif diff --git a/Remote/WebDAV/DavUrl.hs b/Remote/WebDAV/DavUrl.hs new file mode 100644 index 000000000..4862c4f37 --- /dev/null +++ b/Remote/WebDAV/DavUrl.hs @@ -0,0 +1,44 @@ +{- WebDAV urls. + - + - Copyright 2014 Joey Hess <joey@kitenet.net> + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP #-} + +module Remote.WebDAV.DavUrl where + +import Types +import Locations + +import Network.URI (normalizePathSegments) +import System.FilePath.Posix +#ifdef mingw32_HOST_OS +import Data.String.Utils +#endif + +type DavUrl = String + +{- The directory where files(s) for a key are stored. -} +davLocation :: DavUrl -> Key -> DavUrl +davLocation baseurl k = addTrailingPathSeparator $ + davUrl baseurl $ hashdir </> keyFile k + where +#ifndef mingw32_HOST_OS + hashdir = hashDirLower k +#else + hashdir = replace "\\" "/" (hashDirLower k) +#endif + +{- Where we store temporary data for a key as it's being uploaded. -} +tmpLocation :: DavUrl -> Key -> DavUrl +tmpLocation baseurl k = addTrailingPathSeparator $ + davUrl baseurl $ "tmp" </> keyFile k + +davUrl :: DavUrl -> FilePath -> DavUrl +davUrl baseurl file = baseurl </> file + +urlParent :: DavUrl -> DavUrl +urlParent url = dropTrailingPathSeparator $ + normalizePathSegments (dropTrailingPathSeparator url ++ "/..") @@ -32,7 +32,11 @@ import qualified Annex.UUID import qualified Backend import qualified Git.CurrentRepo import qualified Git.Filename +import qualified Git.Construct import qualified Git.Types +import qualified Git.Ref +import qualified Git.LsTree +import qualified Git.FilePath import qualified Locations import qualified Types.KeySource import qualified Types.Backend @@ -55,6 +59,7 @@ import qualified Crypto import qualified Annex.Init import qualified Annex.CatFile import qualified Annex.View +import qualified Annex.View.ViewedFile import qualified Logs.View import qualified Utility.Path import qualified Utility.FileMode @@ -151,6 +156,7 @@ properties = localOption (QuickCheckTests 1000) $ testGroup "QuickCheck" , testProperty "prop_metadata_serialize" Types.MetaData.prop_metadata_serialize , testProperty "prop_branchView_legal" Logs.View.prop_branchView_legal , testProperty "prop_view_roundtrips" Annex.View.prop_view_roundtrips + , testProperty "prop_viewedFile_rountrips" Annex.View.ViewedFile.prop_viewedFile_roundtrips ] {- These tests set up the test environment, but also test some basic parts @@ -195,9 +201,13 @@ unitTests note getenv = testGroup ("Unit Tests " ++ note) , check "version" test_version , check "sync" test_sync , check "union merge regression" test_union_merge_regression - , check "conflict resolution" test_conflict_resolution_movein_bug - , check "conflict_resolution (mixed directory and file)" test_mixed_conflict_resolution - , check "conflict_resolution (mixed directory and file) 2" test_mixed_conflict_resolution2 + , check "conflict resolution" test_conflict_resolution + , check "conflict resolution movein regression" test_conflict_resolution_movein_regression + , check "conflict resolution (mixed directory and file)" test_mixed_conflict_resolution + , check "conflict resolution symlinks" test_conflict_resolution_symlinks + , check "conflict resolution (uncommitted local file)" test_uncommitted_conflict_resolution + , check "conflict resolution (removed file)" test_remove_conflict_resolution + , check "conflict resolution (nonannexed)" test_nonannexed_conflict_resolution , check "map" test_map , check "uninit" test_uninit , check "uninit (in git-annex branch)" test_uninit_inbranch @@ -772,8 +782,8 @@ test_union_merge_regression env = {- Regression test for the automatic conflict resolution bug fixed - in f4ba19f2b8a76a1676da7bb5850baa40d9c388e2. -} -test_conflict_resolution_movein_bug :: TestEnv -> Assertion -test_conflict_resolution_movein_bug env = withtmpclonerepo env False $ \r1 -> +test_conflict_resolution_movein_regression :: TestEnv -> Assertion +test_conflict_resolution_movein_regression env = withtmpclonerepo env False $ \r1 -> withtmpclonerepo env False $ \r2 -> do let rname r = if r == r1 then "r1" else "r2" forM_ [r1, r2] $ \r -> indir env r $ do @@ -804,14 +814,48 @@ test_conflict_resolution_movein_bug env = withtmpclonerepo env False $ \r1 -> forM_ [r1, r2] $ \r -> indir env r $ do git_annex env "get" [] @? "unable to get all files after merge conflict resolution in " ++ rname r +{- Simple case of conflict resolution; 2 different versions of annexed + - file. -} +test_conflict_resolution :: TestEnv -> Assertion +test_conflict_resolution env = + withtmpclonerepo env False $ \r1 -> + withtmpclonerepo env False $ \r2 -> do + indir env r1 $ do + disconnectOrigin + writeFile conflictor "conflictor1" + git_annex env "add" [conflictor] @? "add conflicter failed" + git_annex env "sync" [] @? "sync failed in r1" + indir env r2 $ do + disconnectOrigin + writeFile conflictor "conflictor2" + git_annex env "add" [conflictor] @? "add conflicter failed" + git_annex env "sync" [] @? "sync failed in r2" + pair env r1 r2 + forM_ [r1,r2,r1] $ \r -> indir env r $ + git_annex env "sync" [] @? "sync failed" + checkmerge "r1" r1 + checkmerge "r2" r2 + where + conflictor = "conflictor" + variantprefix = conflictor ++ ".variant" + checkmerge what d = do + l <- getDirectoryContents d + let v = filter (variantprefix `isPrefixOf`) l + length v == 2 + @? (what ++ " not exactly 2 variant files in: " ++ show l) + indir env d $ do + git_annex env "get" v @? "get failed" + git_annex_expectoutput env "find" v v + + {- Check merge conflict resolution when one side is an annexed - file, and the other is a directory. -} test_mixed_conflict_resolution :: TestEnv -> Assertion test_mixed_conflict_resolution env = do - check_mixed_conflict True - check_mixed_conflict False + check True + check False where - check_mixed_conflict inr1 = withtmpclonerepo env False $ \r1 -> + check inr1 = withtmpclonerepo env False $ \r1 -> withtmpclonerepo env False $ \r2 -> do indir env r1 $ do disconnectOrigin @@ -821,7 +865,7 @@ test_mixed_conflict_resolution env = do indir env r2 $ do disconnectOrigin createDirectory conflictor - writeFile (conflictor </> "subfile") "subfile" + writeFile subfile "subfile" git_annex env "add" [conflictor] @? "add conflicter failed" git_annex env "sync" [] @? "sync failed in r2" pair env r1 r2 @@ -829,39 +873,192 @@ test_mixed_conflict_resolution env = do forM_ l $ \r -> indir env r $ git_annex env "sync" [] @? "sync failed in mixed conflict" checkmerge "r1" r1 - checkmerge "r1" r2 + checkmerge "r2" r2 conflictor = "conflictor" + subfile = conflictor </> "subfile" variantprefix = conflictor ++ ".variant" checkmerge what d = do doesDirectoryExist (d </> conflictor) @? (d ++ " conflictor directory missing") l <- getDirectoryContents d - any (variantprefix `isPrefixOf`) l - @? (what ++ " conflictor file missing in: " ++ show l ) - -{- - - During conflict resolution, one of the annexed files in git is - - accidentially converted from a symlink to a regular file. - - This only happens on crippled filesystems. - - - - This test case happens to detect the problem when it tries the next - - pass of conflict resolution, since it's unable to resolve a conflict - - between an annexed and non-annexed file. - -} -test_mixed_conflict_resolution2 :: TestEnv -> Assertion -test_mixed_conflict_resolution2 env = go >> go + let v = filter (variantprefix `isPrefixOf`) l + not (null v) + @? (what ++ " conflictor variant file missing in: " ++ show l ) + length v == 1 + @? (what ++ " too many variant files in: " ++ show v) + indir env d $ do + git_annex env "get" (conflictor:v) @? ("get failed in " ++ what) + git_annex_expectoutput env "find" [conflictor] [Git.FilePath.toInternalGitPath subfile] + git_annex_expectoutput env "find" v v + +{- Check merge conflict resolution when both repos start with an annexed + - file; one modifies it, and the other deletes it. -} +test_remove_conflict_resolution :: TestEnv -> Assertion +test_remove_conflict_resolution env = do + check True + check False where - go = withtmpclonerepo env False $ \r1 -> + check inr1 = withtmpclonerepo env False $ \r1 -> withtmpclonerepo env False $ \r2 -> do indir env r1 $ do + disconnectOrigin writeFile conflictor "conflictor" git_annex env "add" [conflictor] @? "add conflicter failed" git_annex env "sync" [] @? "sync failed in r1" + indir env r2 $ + disconnectOrigin + pair env r1 r2 indir env r2 $ do - createDirectory conflictor - writeFile (conflictor </> "subfile") "subfile" - git_annex env "add" [conflictor] @? "add conflicter failed" git_annex env "sync" [] @? "sync failed in r2" + git_annex env "get" [conflictor] + @? "get conflictor failed" + unlessM (annexeval Config.isDirect) $ do + git_annex env "unlock" [conflictor] + @? "unlock conflictor failed" + writeFile conflictor "newconflictor" + indir env r1 $ + nukeFile conflictor + let l = if inr1 then [r1, r2, r1] else [r2, r1, r2] + forM_ l $ \r -> indir env r $ + git_annex env "sync" [] @? "sync failed" + checkmerge "r1" r1 + checkmerge "r2" r2 + conflictor = "conflictor" + variantprefix = conflictor ++ ".variant" + checkmerge what d = do + l <- getDirectoryContents d + let v = filter (variantprefix `isPrefixOf`) l + not (null v) + @? (what ++ " conflictor variant file missing in: " ++ show l ) + length v == 1 + @? (what ++ " too many variant files in: " ++ show v) + +{- Check merge confalict resolution when a file is annexed in one repo, + - and checked directly into git in the other repo. + - + - This test requires indirect mode to set it up, but tests both direct and + - indirect mode. + -} +test_nonannexed_conflict_resolution :: TestEnv -> Assertion +test_nonannexed_conflict_resolution env = do + check True False + check False False + check True True + check False True + where + check inr1 switchdirect = withtmpclonerepo env False $ \r1 -> + withtmpclonerepo env False $ \r2 -> do + whenM (isInDirect r1 <&&> isInDirect r2) $ do + indir env r1 $ do + disconnectOrigin + writeFile conflictor "conflictor" + git_annex env "add" [conflictor] @? "add conflicter failed" + git_annex env "sync" [] @? "sync failed in r1" + indir env r2 $ do + disconnectOrigin + writeFile conflictor nonannexed_content + boolSystem "git" [Params "add", File conflictor] @? "git add conflictor failed" + git_annex env "sync" [] @? "sync failed in r2" + pair env r1 r2 + let l = if inr1 then [r1, r2] else [r2, r1] + forM_ l $ \r -> indir env r $ do + when switchdirect $ + git_annex env "direct" [] @? "failed switching to direct mode" + git_annex env "sync" [] @? "sync failed" + checkmerge ("r1" ++ show switchdirect) r1 + checkmerge ("r2" ++ show switchdirect) r2 + conflictor = "conflictor" + nonannexed_content = "nonannexed" + variantprefix = conflictor ++ ".variant" + checkmerge what d = do + l <- getDirectoryContents d + let v = filter (variantprefix `isPrefixOf`) l + not (null v) + @? (what ++ " conflictor variant file missing in: " ++ show l ) + length v == 1 + @? (what ++ " too many variant files in: " ++ show v) + conflictor `elem` l @? (what ++ " conflictor file missing in: " ++ show l) + s <- catchMaybeIO (readFile (d </> conflictor)) + s == Just nonannexed_content + @? (what ++ " wrong content for nonannexed file: " ++ show s) + +{- Check merge conflict resolution when there is a local file, + - that is not staged or committed, that conflicts with what's being added + - from the remmote. + - + - Case 1: Remote adds file named conflictor; local has a file named + - conflictor. + - + - Case 2: Remote adds conflictor/file; local has a file named conflictor. + -} +test_uncommitted_conflict_resolution :: TestEnv -> Assertion +test_uncommitted_conflict_resolution env = do + check conflictor + check (conflictor </> "file") + where + check remoteconflictor = withtmpclonerepo env False $ \r1 -> + withtmpclonerepo env False $ \r2 -> do + indir env r1 $ do + disconnectOrigin + createDirectoryIfMissing True (parentDir remoteconflictor) + writeFile remoteconflictor annexedcontent + git_annex env "add" [conflictor] @? "add remoteconflicter failed" + git_annex env "sync" [] @? "sync failed in r1" + indir env r2 $ do + disconnectOrigin + writeFile conflictor localcontent + pair env r1 r2 + indir env r2 $ ifM (annexeval Config.isDirect) + ( do + git_annex env "sync" [] @? "sync failed" + let local = conflictor ++ localprefix + doesFileExist local @? (local ++ " missing after merge") + s <- readFile local + s == localcontent @? (local ++ " has wrong content: " ++ s) + git_annex env "get" [conflictor] @? "get failed" + doesFileExist remoteconflictor @? (remoteconflictor ++ " missing after merge") + s' <- readFile remoteconflictor + s' == annexedcontent @? (remoteconflictor ++ " has wrong content: " ++ s) + -- this case is intentionally not handled + -- in indirect mode, since the user + -- can recover on their own easily + , not <$> git_annex env "sync" [] @? "sync failed to fail" + ) conflictor = "conflictor" + localprefix = ".variant-local" + localcontent = "local" + annexedcontent = "annexed" + +{- On Windows/FAT, repeated conflict resolution sometimes + - lost track of whether a file was a symlink. + -} +test_conflict_resolution_symlinks :: TestEnv -> Assertion +test_conflict_resolution_symlinks env = do + withtmpclonerepo env False $ \r1 -> + withtmpclonerepo env False $ \r2 -> do + withtmpclonerepo env False $ \r3 -> do + indir env r1 $ do + writeFile conflictor "conflictor" + git_annex env "add" [conflictor] @? "add conflicter failed" + git_annex env "sync" [] @? "sync failed in r1" + check_is_link conflictor "r1" + indir env r2 $ do + createDirectory conflictor + writeFile (conflictor </> "subfile") "subfile" + git_annex env "add" [conflictor] @? "add conflicter failed" + git_annex env "sync" [] @? "sync failed in r2" + check_is_link (conflictor </> "subfile") "r2" + indir env r3 $ do + writeFile conflictor "conflictor" + git_annex env "add" [conflictor] @? "add conflicter failed" + git_annex env "sync" [] @? "sync failed in r1" + check_is_link (conflictor </> "subfile") "r3" + where + conflictor = "conflictor" + check_is_link f what = do + git_annex_expectoutput env "find" ["--include=*", f] [Git.FilePath.toInternalGitPath f] + l <- annexeval $ Annex.inRepo $ Git.LsTree.lsTreeFiles Git.Ref.headRef [f] + all (\i -> Git.Types.toBlobType (Git.LsTree.mode i) == Just Git.Types.SymlinkBlob) l + @? (what ++ " " ++ f ++ " lost symlink bit after merge: " ++ show l) {- Set up repos as remotes of each other. -} pair :: TestEnv -> FilePath -> FilePath -> Assertion @@ -1149,6 +1346,11 @@ intmpclonerepoInDirect env a = intmpclonerepo env $ Annex.Init.initialize Nothing Config.isDirect +isInDirect :: FilePath -> IO Bool +isInDirect d = do + s <- Annex.new =<< Git.Construct.fromPath d + not <$> Annex.eval s Config.isDirect + intmpbareclonerepo :: TestEnv -> Assertion -> Assertion intmpbareclonerepo env a = withtmpclonerepo env True $ \r -> indir env r a @@ -1342,7 +1544,8 @@ prepareTestEnv forcedirect = do cwd <- getCurrentDirectory p <- Utility.Env.getEnvDefault "PATH" "" - let env = + env <- Utility.Env.getEnvironment + let newenv = -- Ensure that the just-built git annex is used. [ ("PATH", cwd ++ [searchPathSeparator] ++ p) , ("TOPDIR", cwd) @@ -1358,7 +1561,7 @@ prepareTestEnv forcedirect = do , ("FORCEDIRECT", if forcedirect then "1" else "") ] - return $ M.fromList env + return $ M.fromList newenv `M.union` M.fromList env changeToTmpDir :: TestEnv -> FilePath -> IO () changeToTmpDir env t = do diff --git a/Types/GitConfig.hs b/Types/GitConfig.hs index ab3dbd7b9..b49f3d762 100644 --- a/Types/GitConfig.hs +++ b/Types/GitConfig.hs @@ -49,6 +49,9 @@ data GitConfig = GitConfig , annexAutoUpgrade :: AutoUpgrade , annexExpireUnused :: Maybe (Maybe Duration) , annexSecureEraseCommand :: Maybe String + , annexGenMetaData :: Bool + , annexListen :: Maybe String + , annexStartupScan :: Bool , coreSymlinks :: Bool , gcryptId :: Maybe String } @@ -81,6 +84,9 @@ extractGitConfig r = GitConfig , annexExpireUnused = maybe Nothing Just . parseDuration <$> getmaybe (annex "expireunused") , annexSecureEraseCommand = getmaybe (annex "secure-erase-command") + , annexGenMetaData = getbool (annex "genmetadata") False + , annexListen = getmaybe (annex "listen") + , annexStartupScan = getbool (annex "startupscan") True , coreSymlinks = getbool "core.symlinks" True , gcryptId = getmaybe "core.gcrypt-id" } diff --git a/Types/Key.hs b/Types/Key.hs index 598d5ed20..26af6220f 100644 --- a/Types/Key.hs +++ b/Types/Key.hs @@ -78,8 +78,12 @@ file2key s findfields _ v = v addbackend k v = Just k { keyBackendName = v } - addfield 's' k v = Just k { keySize = readish v } - addfield 'm' k v = Just k { keyMtime = readish v } + addfield 's' k v = do + sz <- readish v + return $ k { keySize = Just sz } + addfield 'm' k v = do + mtime <- readish v + return $ k { keyMtime = Just mtime } addfield _ _ _ = Nothing instance Arbitrary Key where @@ -93,4 +97,12 @@ prop_idempotent_key_encode :: Key -> Bool prop_idempotent_key_encode k = Just k == (file2key . key2file) k prop_idempotent_key_decode :: FilePath -> Bool -prop_idempotent_key_decode f = maybe True (\k -> key2file k == f) (file2key f) +prop_idempotent_key_decode f + | normalfieldorder = maybe True (\k -> key2file k == f) (file2key f) + | otherwise = True + where + -- file2key will accept the fields in any order, so don't + -- try the test unless the fields are in the normal order + normalfieldorder = fields `isPrefixOf` "sm" + fields = map (f !!) $ filter (< length f) $ map succ $ + elemIndices fieldSep f diff --git a/Types/MetaData.hs b/Types/MetaData.hs index 617c122a6..c37b31c51 100644 --- a/Types/MetaData.hs +++ b/Types/MetaData.hs @@ -17,7 +17,7 @@ module Types.MetaData ( MetaSerializable, toMetaField, mkMetaField, - tagMetaField, + mkMetaFieldUnchecked, fromMetaField, toMetaValue, mkMetaValue, @@ -25,7 +25,7 @@ module Types.MetaData ( unsetMetaData, fromMetaValue, fromMetaData, - newMetaData, + emptyMetaData, updateMetaData, unionMetaData, differenceMetaData, @@ -48,6 +48,7 @@ import Utility.QuickCheck import qualified Data.Set as S import qualified Data.Map as M import Data.Char +import qualified Data.CaseInsensitive as CI newtype MetaData = MetaData (M.Map MetaField (S.Set MetaValue)) deriving (Show, Eq, Ord) @@ -57,7 +58,8 @@ newtype MetaData = MetaData (M.Map MetaField (S.Set MetaValue)) newtype CurrentlySet = CurrentlySet Bool deriving (Read, Show, Eq, Ord, Arbitrary) -newtype MetaField = MetaField String +{- Fields are case insensitive. -} +newtype MetaField = MetaField (CI.CI String) deriving (Read, Show, Eq, Ord) data MetaValue = MetaValue CurrentlySet String @@ -81,7 +83,7 @@ instance MetaSerializable MetaData where serialize (MetaData m) = unwords $ concatMap go $ M.toList m where go (f, vs) = serialize f : map serialize (S.toList vs) - deserialize = Just . getfield newMetaData . words + deserialize = Just . getfield emptyMetaData . words where getfield m [] = m getfield m (w:ws) = maybe m (getvalues m ws) (deserialize w) @@ -91,8 +93,8 @@ instance MetaSerializable MetaData where Nothing -> getfield m l instance MetaSerializable MetaField where - serialize (MetaField f) = f - deserialize = Just . MetaField + serialize (MetaField f) = CI.original f + deserialize = Just . mkMetaFieldUnchecked {- Base64 problimatic values. -} instance MetaSerializable MetaValue where @@ -116,19 +118,39 @@ instance MetaSerializable CurrentlySet where deserialize "-" = Just (CurrentlySet False) deserialize _ = Nothing -{- Fields cannot be empty, contain whitespace, or start with "+-" as - - that would break the serialization. -} +mkMetaField :: String -> Either String MetaField +mkMetaField f = maybe (Left $ badField f) Right (toMetaField f) + +badField :: String -> String +badField f = "Illegal metadata field name, \"" ++ f ++ "\"" + +{- Does not check that the field name is valid. Use with caution. -} +mkMetaFieldUnchecked :: String -> MetaField +mkMetaFieldUnchecked = MetaField . CI.mk + toMetaField :: String -> Maybe MetaField toMetaField f - | legalField f = Just $ MetaField f + | legalField f = Just $ MetaField $ CI.mk f | otherwise = Nothing +{- Fields cannot be empty, contain whitespace, or start with "+-" as + - that would break the serialization. + - + - Additionally, fields should not contain any form of path separator, as + - that would break views. + - + - So, require they have an alphanumeric first letter, with the remainder + - being either alphanumeric or a small set of shitelisted common punctuation. + -} legalField :: String -> Bool -legalField f - | null f = False - | any isSpace f = False - | any (`isPrefixOf` f) ["+", "-"] = False - | otherwise = True +legalField [] = False +legalField (c1:cs) + | not (isAlphaNum c1) = False + | otherwise = all legalchars cs + where + legalchars c + | isAlphaNum c = True + | otherwise = c `elem` "_-." toMetaValue :: String -> MetaValue toMetaValue = MetaValue (CurrentlySet True) @@ -144,7 +166,7 @@ unsetMetaData :: MetaData -> MetaData unsetMetaData (MetaData m) = MetaData $ M.map (S.map unsetMetaValue) m fromMetaField :: MetaField -> String -fromMetaField (MetaField f) = f +fromMetaField (MetaField f) = CI.original f fromMetaValue :: MetaValue -> String fromMetaValue (MetaValue _ f) = f @@ -152,8 +174,8 @@ fromMetaValue (MetaValue _ f) = f fromMetaData :: MetaData -> [(MetaField, S.Set MetaValue)] fromMetaData (MetaData m) = M.toList m -newMetaData :: MetaData -newMetaData = MetaData M.empty +emptyMetaData :: MetaData +emptyMetaData = MetaData M.empty {- Can be used to set a value, or to unset it, depending on whether - the MetaValue has CurrentlySet or not. -} @@ -197,22 +219,27 @@ data ModMeta = AddMeta MetaField MetaValue | DelMeta MetaField MetaValue | SetMeta MetaField MetaValue -- removes any existing values + | MaybeSetMeta MetaField MetaValue -- when field has no existing value {- Applies a ModMeta, generating the new MetaData. - Note that the new MetaData does not include all the - values set in the input metadata. It only contains changed values. -} modMeta :: MetaData -> ModMeta -> MetaData -modMeta _ (AddMeta f v) = updateMetaData f v newMetaData -modMeta _ (DelMeta f oldv) = updateMetaData f (unsetMetaValue oldv) newMetaData +modMeta _ (AddMeta f v) = updateMetaData f v emptyMetaData +modMeta _ (DelMeta f oldv) = updateMetaData f (unsetMetaValue oldv) emptyMetaData modMeta m (SetMeta f v) = updateMetaData f v $ - foldr (updateMetaData f) newMetaData $ + foldr (updateMetaData f) emptyMetaData $ map unsetMetaValue $ S.toList $ currentMetaDataValues f m +modMeta m (MaybeSetMeta f v) + | S.null (currentMetaDataValues f m) = updateMetaData f v emptyMetaData + | otherwise = emptyMetaData -{- Parses field=value, field+=value, field-=value -} +{- Parses field=value, field+=value, field-=value, field?=value -} parseModMeta :: String -> Either String ModMeta parseModMeta p = case lastMaybe f of Just '+' -> AddMeta <$> mkMetaField f' <*> v Just '-' -> DelMeta <$> mkMetaField f' <*> v + Just '?' -> MaybeSetMeta <$> mkMetaField f' <*> v _ -> SetMeta <$> mkMetaField f <*> v where (f, sv) = separate (== '=') p @@ -227,15 +254,6 @@ parseMetaData p = (,) where (f, v) = separate (== '=') p -mkMetaField :: String -> Either String MetaField -mkMetaField f = maybe (Left $ badField f) Right (toMetaField f) - -badField :: String -> String -badField f = "Illegal metadata field name, \"" ++ f ++ "\"" - -tagMetaField :: MetaField -tagMetaField = MetaField "tag" - {- Avoid putting too many fields in the map; extremely large maps make - the seriaization test slow due to the sheer amount of data. - It's unlikely that more than 100 fields of metadata will be used. -} @@ -248,13 +266,13 @@ instance Arbitrary MetaValue where arbitrary = MetaValue <$> arbitrary <*> arbitrary instance Arbitrary MetaField where - arbitrary = MetaField <$> arbitrary `suchThat` legalField + arbitrary = MetaField . CI.mk <$> arbitrary `suchThat` legalField prop_metadata_sane :: MetaData -> MetaField -> MetaValue -> Bool prop_metadata_sane m f v = and [ S.member v $ metaDataValues f m' , not (isSet v) || S.member v (currentMetaDataValues f m') - , differenceMetaData m' newMetaData == m' + , differenceMetaData m' emptyMetaData == m' ] where m' = updateMetaData f v m diff --git a/Types/StandardGroups.hs b/Types/StandardGroups.hs index d95f28ee1..2f5cd4b30 100644 --- a/Types/StandardGroups.hs +++ b/Types/StandardGroups.hs @@ -72,6 +72,10 @@ associatedDirectory (Just c) PublicGroup = Just $ associatedDirectory Nothing PublicGroup = Just "public" associatedDirectory _ _ = Nothing +specialRemoteOnly :: StandardGroup -> Bool +specialRemoteOnly PublicGroup = True +specialRemoteOnly _ = False + {- See doc/preferred_content.mdwn for explanations of these expressions. -} preferredContent :: StandardGroup -> PreferredContentExpression preferredContent ClientGroup = lastResort $ diff --git a/Types/View.hs b/Types/View.hs index 04b002879..43afdb8c8 100644 --- a/Types/View.hs +++ b/Types/View.hs @@ -35,21 +35,26 @@ data ViewComponent = ViewComponent instance Arbitrary ViewComponent where arbitrary = ViewComponent <$> arbitrary <*> arbitrary <*> arbitrary -{- Only files with metadata matching the view are displayed. -} -type FileView = FilePath -type MkFileView = FilePath -> FileView - data ViewFilter = FilterValues (S.Set MetaValue) | FilterGlob String + | ExcludeValues (S.Set MetaValue) deriving (Eq, Read, Show) instance Arbitrary ViewFilter where arbitrary = do size <- arbitrarySizedBoundedIntegral `suchThat` (< 100) - FilterValues . S.fromList <$> vector size + s <- S.fromList <$> vector size + ifM arbitrary + ( return (FilterValues s) + , return (ExcludeValues s) + ) + +mkViewComponent :: MetaField -> ViewFilter -> ViewComponent +mkViewComponent f vf = ViewComponent f vf (multiValue vf) {- Can a ViewFilter match multiple different MetaValues? -} multiValue :: ViewFilter -> Bool multiValue (FilterValues s) = S.size s > 1 multiValue (FilterGlob _) = True +multiValue (ExcludeValues _) = False diff --git a/Utility/DirWatcher.hs b/Utility/DirWatcher.hs index 9eeddce3d..077410575 100644 --- a/Utility/DirWatcher.hs +++ b/Utility/DirWatcher.hs @@ -104,33 +104,33 @@ modifyTracked = undefined - to shutdown later. -} #if WITH_INOTIFY type DirWatcherHandle = INotify.INotify -watchDir :: FilePath -> Pruner -> WatchHooks -> (IO () -> IO ()) -> IO DirWatcherHandle -watchDir dir prune hooks runstartup = do +watchDir :: FilePath -> Pruner -> Bool -> WatchHooks -> (IO () -> IO ()) -> IO DirWatcherHandle +watchDir dir prune scanevents hooks runstartup = do i <- INotify.initINotify - runstartup $ INotify.watchDir i dir prune hooks + runstartup $ INotify.watchDir i dir prune scanevents hooks return i #else #if WITH_KQUEUE type DirWatcherHandle = ThreadId -watchDir :: FilePath -> Pruner -> WatchHooks -> (IO Kqueue.Kqueue -> IO Kqueue.Kqueue) -> IO DirWatcherHandle -watchDir dir prune hooks runstartup = do +watchDir :: FilePath -> Pruner -> Bool -> WatchHooks -> (IO Kqueue.Kqueue -> IO Kqueue.Kqueue) -> IO DirWatcherHandle +watchDir dir prune _scanevents hooks runstartup = do kq <- runstartup $ Kqueue.initKqueue dir prune forkIO $ Kqueue.runHooks kq hooks #else #if WITH_FSEVENTS type DirWatcherHandle = FSEvents.EventStream -watchDir :: FilePath -> Pruner -> WatchHooks -> (IO FSEvents.EventStream -> IO FSEvents.EventStream) -> IO DirWatcherHandle -watchDir dir prune hooks runstartup = - runstartup $ FSEvents.watchDir dir prune hooks +watchDir :: FilePath -> Pruner -> Bool -> WatchHooks -> (IO FSEvents.EventStream -> IO FSEvents.EventStream) -> IO DirWatcherHandle +watchDir dir prune scanevents hooks runstartup = + runstartup $ FSEvents.watchDir dir prune scanevents hooks #else #if WITH_WIN32NOTIFY type DirWatcherHandle = Win32Notify.WatchManager -watchDir :: FilePath -> Pruner -> WatchHooks -> (IO Win32Notify.WatchManager -> IO Win32Notify.WatchManager) -> IO DirWatcherHandle -watchDir dir prune hooks runstartup = - runstartup $ Win32Notify.watchDir dir prune hooks +watchDir :: FilePath -> Pruner -> Bool -> WatchHooks -> (IO Win32Notify.WatchManager -> IO Win32Notify.WatchManager) -> IO DirWatcherHandle +watchDir dir prune scanevents hooks runstartup = + runstartup $ Win32Notify.watchDir dir prune scanevents hooks #else type DirWatcherHandle = () -watchDir :: FilePath -> Pruner -> WatchHooks -> (IO () -> IO ()) -> IO DirWatcherHandle +watchDir :: FilePath -> Pruner -> Bool -> WatchHooks -> (IO () -> IO ()) -> IO DirWatcherHandle watchDir = undefined #endif #endif diff --git a/Utility/DirWatcher/FSEvents.hs b/Utility/DirWatcher/FSEvents.hs index db6ac0434..26e1f7671 100644 --- a/Utility/DirWatcher/FSEvents.hs +++ b/Utility/DirWatcher/FSEvents.hs @@ -14,8 +14,8 @@ import System.OSX.FSEvents import qualified System.Posix.Files as Files import Data.Bits ((.&.)) -watchDir :: FilePath -> (FilePath -> Bool) -> WatchHooks -> IO EventStream -watchDir dir ignored hooks = do +watchDir :: FilePath -> (FilePath -> Bool) -> Bool -> WatchHooks -> IO EventStream +watchDir dir ignored scanevents hooks = do unlessM fileLevelEventsSupported $ error "Need at least OSX 10.7.0 for file-level FSEvents" scan dir @@ -79,9 +79,11 @@ watchDir dir ignored hooks = do Nothing -> noop Just s | Files.isSymbolicLink s -> - runhook addSymlinkHook ms + when scanevents $ + runhook addSymlinkHook ms | Files.isRegularFile s -> - runhook addHook ms + when scanevents $ + runhook addHook ms | otherwise -> noop where diff --git a/Utility/DirWatcher/INotify.hs b/Utility/DirWatcher/INotify.hs index 922f202a4..016858b1b 100644 --- a/Utility/DirWatcher/INotify.hs +++ b/Utility/DirWatcher/INotify.hs @@ -46,8 +46,8 @@ import Control.Exception (throw) - So this will fail if there are too many subdirectories. The - errHook is called when this happens. -} -watchDir :: INotify -> FilePath -> (FilePath -> Bool) -> WatchHooks -> IO () -watchDir i dir ignored hooks +watchDir :: INotify -> FilePath -> (FilePath -> Bool) -> Bool -> WatchHooks -> IO () +watchDir i dir ignored scanevents hooks | ignored dir = noop | otherwise = do -- Use a lock to make sure events generated during initial @@ -61,7 +61,7 @@ watchDir i dir ignored hooks mapM_ scan =<< filter (not . dirCruft) <$> getDirectoryContents dir where - recurse d = watchDir i d ignored hooks + recurse d = watchDir i d ignored scanevents hooks -- Select only inotify events required by the enabled -- hooks, but always include Create so new directories can @@ -85,9 +85,11 @@ watchDir i dir ignored hooks | Files.isDirectory s -> recurse $ indir f | Files.isSymbolicLink s -> - runhook addSymlinkHook f ms + when scanevents $ + runhook addSymlinkHook f ms | Files.isRegularFile s -> - runhook addHook f ms + when scanevents $ + runhook addHook f ms | otherwise -> noop diff --git a/Utility/DirWatcher/Win32Notify.hs b/Utility/DirWatcher/Win32Notify.hs index ba786839c..f095e5d0e 100644 --- a/Utility/DirWatcher/Win32Notify.hs +++ b/Utility/DirWatcher/Win32Notify.hs @@ -13,8 +13,8 @@ import Utility.DirWatcher.Types import System.Win32.Notify import qualified Utility.PosixFiles as Files -watchDir :: FilePath -> (FilePath -> Bool) -> WatchHooks -> IO WatchManager -watchDir dir ignored hooks = do +watchDir :: FilePath -> (FilePath -> Bool) -> Bool -> WatchHooks -> IO WatchManager +watchDir dir ignored scanevents hooks = do scan dir wm <- initWatchManager void $ watchDirectory wm dir True [Create, Delete, Modify, Move] handle @@ -52,7 +52,8 @@ watchDir dir ignored hooks = do Nothing -> noop Just s | Files.isRegularFile s -> - runhook addHook ms + when scanevents $ + runhook addHook ms | otherwise -> noop where diff --git a/Utility/Glob.hs b/Utility/Glob.hs new file mode 100644 index 000000000..1a77da7d3 --- /dev/null +++ b/Utility/Glob.hs @@ -0,0 +1,58 @@ +{- file globbing + - + - This uses TDFA when available, with a fallback to regex-compat. + - TDFA is less buggy in its support for non-unicode characters. + - + - Copyright 2014 Joey Hess <joey@kitenet.net> + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP #-} + +module Utility.Glob ( + Glob, + GlobCase(..), + compileGlob, + matchGlob +) where + +import System.Path.WildMatch + +#ifdef WITH_TDFA +import Text.Regex.TDFA +import Text.Regex.TDFA.String +#else +import Text.Regex +import Data.Maybe +#endif + +newtype Glob = Glob Regex + +data GlobCase = CaseSensative | CaseInsensative + +{- Compiles a glob to a regex, that can be repeatedly used. -} +compileGlob :: String -> GlobCase -> Glob +compileGlob glob globcase = Glob $ +#ifdef WITH_TDFA + case compile (defaultCompOpt {caseSensitive = casesentitive}) defaultExecOpt regex of + Right r -> r + Left _ -> error $ "failed to compile regex: " ++ regex +#else + mkRegexWithOpts regex casesentitive True +#endif + where + regex = '^':wildToRegex glob + casesentitive = case globcase of + CaseSensative -> True + CaseInsensative -> False + +matchGlob :: Glob -> String -> Bool +matchGlob (Glob regex) val = +#ifdef WITH_TDFA + case execute regex val of + Right (Just _) -> True + _ -> False +#else + isJust $ matchRegex regex val +#endif diff --git a/Utility/LogFile.hs b/Utility/LogFile.hs index 4e76116ba..73fbf820e 100644 --- a/Utility/LogFile.hs +++ b/Utility/LogFile.hs @@ -11,7 +11,9 @@ module Utility.LogFile where import Common +#ifndef mingw32_HOST_OS import System.Posix.Types +#endif #ifndef mingw32_HOST_OS openLog :: FilePath -> IO Fd diff --git a/Utility/Path.hs b/Utility/Path.hs index e22d0c3f7..570350d61 100644 --- a/Utility/Path.hs +++ b/Utility/Path.hs @@ -18,7 +18,6 @@ import Data.Char import Control.Applicative #ifdef mingw32_HOST_OS -import Data.Char import qualified System.FilePath.Posix as Posix #else import System.Posix.Files diff --git a/Utility/Quvi.hs b/Utility/Quvi.hs index 4039167ac..bb4975cbe 100644 --- a/Utility/Quvi.hs +++ b/Utility/Quvi.hs @@ -11,7 +11,6 @@ module Utility.Quvi where import Common import Utility.Url -import Build.SysConfig (newquvi) import Data.Aeson import Data.ByteString.Lazy.UTF8 (fromString) @@ -19,6 +18,11 @@ import qualified Data.Map as M import Network.URI (uriAuthority, uriRegName) import Data.Char +data QuviVersion + = Quvi04 + | Quvi09 + | NoQuvi + data Page = Page { pageTitle :: String , pageLinks :: [Link] @@ -56,11 +60,19 @@ parseEnum s = Page get = flip M.lookup m m = M.fromList $ map (separate (== '=')) $ lines s -type Query a = [CommandParam] -> URLString -> IO a +probeVersion :: IO QuviVersion +probeVersion = examine <$> processTranscript "quvi" ["--version"] Nothing + where + examine (s, True) + | "quvi v0.4" `isInfixOf` s = Quvi04 + | otherwise = Quvi09 + examine _ = NoQuvi + +type Query a = QuviVersion -> [CommandParam] -> URLString -> IO a {- Throws an error when quvi is not installed. -} forceQuery :: Query (Maybe Page) -forceQuery ps url = query' ps url `catchNonAsync` onerr +forceQuery v ps url = query' v ps url `catchNonAsync` onerr where onerr _ = ifM (inPath "quvi") ( error "quvi failed" @@ -70,33 +82,36 @@ forceQuery ps url = query' ps url `catchNonAsync` onerr {- Returns Nothing if the page is not a video page, or quvi is not - installed. -} query :: Query (Maybe Page) -query ps url = flip catchNonAsync (const $ return Nothing) (query' ps url) +query v ps url = flip catchNonAsync (const $ return Nothing) (query' v ps url) query' :: Query (Maybe Page) -query' ps url - | newquvi = parseEnum - <$> readProcess "quvi" (toCommand $ [Param "dump", Param "-p", Param "enum"] ++ ps ++ [Param url]) - | otherwise = decode . fromString - <$> readProcess "quvi" (toCommand $ ps ++ [Param url]) +query' Quvi09 ps url = parseEnum + <$> readProcess "quvi" (toCommand $ [Param "dump", Param "-p", Param "enum"] ++ ps ++ [Param url]) +query' Quvi04 ps url = decode . fromString + <$> readProcess "quvi" (toCommand $ ps ++ [Param url]) +query' NoQuvi _ _ = return Nothing queryLinks :: Query [URLString] -queryLinks ps url = maybe [] (map linkUrl . pageLinks) <$> query ps url +queryLinks v ps url = maybe [] (map linkUrl . pageLinks) <$> query v ps url {- Checks if quvi can still find a download link for an url. - If quvi is not installed, returns False. -} check :: Query Bool -check ps url = maybe False (not . null . pageLinks) <$> query ps url +check v ps url = maybe False (not . null . pageLinks) <$> query v ps url {- Checks if an url is supported by quvi, as quickly as possible - (without hitting it if possible), and without outputting - anything. Also returns False if quvi is not installed. -} -supported :: URLString -> IO Bool -supported url - {- Use quvi-info to see if the url's domain is supported. - - If so, have to do a online verification of the url. -} - | newquvi = (firstlevel <&&> secondlevel) +supported :: QuviVersion -> URLString -> IO Bool +supported NoQuvi _ = return False +supported Quvi04 url = boolSystem "quvi" + [ Params "--verbosity mute --support" + , Param url + ] +{- Use quvi-info to see if the url's domain is supported. + - If so, have to do a online verification of the url. -} +supported Quvi09 url = (firstlevel <&&> secondlevel) `catchNonAsync` (\_ -> return False) - | otherwise = boolSystem "quvi" [Params "--verbosity mute --support", Param url] where firstlevel = case uriAuthority =<< parseURIRelaxed url of Nothing -> return False @@ -104,30 +119,30 @@ supported url let domain = map toLower $ uriRegName auth let basedomain = intercalate "." $ reverse $ take 2 $ reverse $ split "." domain any (\h -> domain `isSuffixOf` h || basedomain `isSuffixOf` h) - . map (map toLower) <$> listdomains + . map (map toLower) <$> listdomains Quvi09 secondlevel = snd <$> processTranscript "quvi" (toCommand [Param "dump", Param "-o", Param url]) Nothing -listdomains :: IO [String] -listdomains - | newquvi = concatMap (split ",") - . concatMap (drop 1 . words) - . filter ("domains: " `isPrefixOf`) . lines - <$> readProcess "quvi" - (toCommand [Param "info", Param "-p", Param "domains"]) - | otherwise = return [] +listdomains :: QuviVersion -> IO [String] +listdomains Quvi09 = concatMap (split ",") + . concatMap (drop 1 . words) + . filter ("domains: " `isPrefixOf`) . lines + <$> readProcess "quvi" + (toCommand [Param "info", Param "-p", Param "domains"]) +listdomains _ = return [] + +type QuviParam = QuviVersion -> CommandParam {- Disables progress, but not information output. -} -quiet :: CommandParam -quiet - -- Cannot use quiet as it now disables informational output. - -- No way to disable progress. - | newquvi = Params "--verbosity verbose" - | otherwise = Params "--verbosity quiet" +quiet :: QuviParam +-- Cannot use quiet as it now disables informational output. +-- No way to disable progress. +quiet Quvi09 = Params "--verbosity verbose" +quiet Quvi04 = Params "--verbosity quiet" +quiet NoQuvi = Params "" {- Only return http results, not streaming protocols. -} -httponly :: CommandParam -httponly - -- No way to do it with 0.9? - | newquvi = Params "" - | otherwise = Params "-c http" +httponly :: QuviParam +-- No way to do it with 0.9? +httponly Quvi04 = Params "-c http" +httponly _ = Params "" -- No way to do it with 0.9? diff --git a/Utility/Url.hs b/Utility/Url.hs index 2cbab77c8..3ab14ebe4 100644 --- a/Utility/Url.hs +++ b/Utility/Url.hs @@ -10,6 +10,7 @@ module Utility.Url ( URLString, UserAgent, + UrlOptions(..), check, checkBoth, exists, @@ -23,6 +24,7 @@ import Network.URI import qualified Network.Browser as Browser import Network.HTTP import Data.Either +import Data.Default import qualified Build.SysConfig @@ -32,14 +34,24 @@ type Headers = [String] type UserAgent = String +data UrlOptions = UrlOptions + { userAgent :: Maybe UserAgent + , reqHeaders :: Headers + , reqParams :: [CommandParam] + } + +instance Default UrlOptions + where + def = UrlOptions Nothing [] [] + {- Checks that an url exists and could be successfully downloaded, - also checking that its size, if available, matches a specified size. -} -checkBoth :: URLString -> Headers -> Maybe Integer -> Maybe UserAgent -> IO Bool -checkBoth url headers expected_size ua = do - v <- check url headers expected_size ua +checkBoth :: URLString -> Maybe Integer -> UrlOptions -> IO Bool +checkBoth url expected_size uo = do + v <- check url expected_size uo return (fst v && snd v) -check :: URLString -> Headers -> Maybe Integer -> Maybe UserAgent -> IO (Bool, Bool) -check url headers expected_size = handle <$$> exists url headers +check :: URLString -> Maybe Integer -> UrlOptions -> IO (Bool, Bool) +check url expected_size = handle <$$> exists url where handle (False, _) = (False, False) handle (True, Nothing) = (True, True) @@ -55,8 +67,8 @@ check url headers expected_size = handle <$$> exists url headers - Uses curl otherwise, when available, since curl handles https better - than does Haskell's Network.Browser. -} -exists :: URLString -> Headers -> Maybe UserAgent -> IO (Bool, Maybe Integer) -exists url headers ua = case parseURIRelaxed url of +exists :: URLString -> UrlOptions -> IO (Bool, Maybe Integer) +exists url uo = case parseURIRelaxed url of Just u | uriScheme u == "file:" -> do s <- catchMaybeIO $ getFileStatus (unEscapeString $ uriPath u) @@ -70,7 +82,7 @@ exists url headers ua = case parseURIRelaxed url of Just ('2':_:_) -> return (True, extractsize output) _ -> dne else do - r <- request u headers HEAD ua + r <- request u HEAD uo case rspCode r of (2,_,_) -> return (True, size r) _ -> return (False, Nothing) @@ -78,12 +90,12 @@ exists url headers ua = case parseURIRelaxed url of where dne = return (False, Nothing) - curlparams = addUserAgent ua $ + curlparams = addUserAgent uo $ [ Param "-s" , Param "--head" , Param "-L", Param url , Param "-w", Param "%{http_code}" - ] ++ concatMap (\h -> [Param "-H", Param h]) headers + ] ++ concatMap (\h -> [Param "-H", Param h]) (reqHeaders uo) ++ (reqParams uo) extractsize s = case lastMaybe $ filter ("Content-Length:" `isPrefixOf`) (lines s) of Just l -> case lastMaybe $ words l of @@ -94,9 +106,10 @@ exists url headers ua = case parseURIRelaxed url of size = liftM Prelude.read . lookupHeader HdrContentLength . rspHeaders -- works for both wget and curl commands -addUserAgent :: Maybe UserAgent -> [CommandParam] -> [CommandParam] -addUserAgent Nothing ps = ps -addUserAgent (Just ua) ps = ps ++ [Param "--user-agent", Param ua] +addUserAgent :: UrlOptions -> [CommandParam] -> [CommandParam] +addUserAgent uo ps = case userAgent uo of + Nothing -> ps + Just ua -> ps ++ [Param "--user-agent", Param ua] {- Used to download large files, such as the contents of keys. - @@ -105,15 +118,15 @@ addUserAgent (Just ua) ps = ps ++ [Param "--user-agent", Param ua] - would not be appropriate to test at configure time and build support - for only one in. -} -download :: URLString -> Headers -> [CommandParam] -> FilePath -> Maybe UserAgent -> IO Bool +download :: URLString -> FilePath -> UrlOptions -> IO Bool download = download' False {- No output, even on error. -} -downloadQuiet :: URLString -> Headers -> [CommandParam] -> FilePath -> Maybe UserAgent -> IO Bool +downloadQuiet :: URLString -> FilePath -> UrlOptions -> IO Bool downloadQuiet = download' True -download' :: Bool -> URLString -> Headers -> [CommandParam] -> FilePath -> Maybe UserAgent -> IO Bool -download' quiet url headers options file ua = +download' :: Bool -> URLString -> FilePath -> UrlOptions -> IO Bool +download' quiet url file uo = case parseURIRelaxed url of Just u | uriScheme u == "file:" -> do @@ -124,7 +137,7 @@ download' quiet url headers options file ua = | otherwise -> ifM (inPath "wget") (wget , curl) _ -> return False where - headerparams = map (\h -> Param $ "--header=" ++ h) headers + headerparams = map (\h -> Param $ "--header=" ++ h) (reqHeaders uo) wget = go "wget" $ headerparams ++ quietopt "-q" ++ wgetparams {- Regular wget needs --clobber to continue downloading an existing - file. On Android, busybox wget is used, which does not @@ -142,7 +155,7 @@ download' quiet url headers options file ua = curl = go "curl" $ headerparams ++ quietopt "-s" ++ [Params "-f -L -C - -# -o"] go cmd opts = boolSystem cmd $ - addUserAgent ua $ options++opts++[File file, File url] + addUserAgent uo $ reqParams uo++opts++[File file, File url] quietopt s | quiet = [Param s] | otherwise = [] @@ -157,14 +170,14 @@ download' quiet url headers options file ua = - Unfortunately, does not handle https, so should only be used - when curl is not available. -} -request :: URI -> Headers -> RequestMethod -> Maybe UserAgent -> IO (Response String) -request url headers requesttype ua = go 5 url +request :: URI -> RequestMethod -> UrlOptions -> IO (Response String) +request url requesttype uo = go 5 url where go :: Int -> URI -> IO (Response String) go 0 _ = error "Too many redirects " go n u = do rsp <- Browser.browse $ do - maybe noop Browser.setUserAgent ua + maybe noop Browser.setUserAgent (userAgent uo) Browser.setErrHandler ignore Browser.setOutHandler ignore Browser.setAllowRedirects False @@ -174,7 +187,7 @@ request url headers requesttype ua = go 5 url (3,0,x) | x /= 5 -> redir (n - 1) u rsp _ -> return rsp addheaders req = setHeaders req (rqHeaders req ++ userheaders) - userheaders = rights $ map parseHeader headers + userheaders = rights $ map parseHeader (reqHeaders uo) ignore = const noop redir n u rsp = case retrieveHeaders HdrLocation rsp of [] -> return rsp diff --git a/Utility/WebApp.hs b/Utility/WebApp.hs index f92c21e4e..c9cb32106 100644 --- a/Utility/WebApp.hs +++ b/Utility/WebApp.hs @@ -17,13 +17,13 @@ import Utility.Hash import qualified Yesod import qualified Network.Wai as Wai import Network.Wai.Handler.Warp +import Network.Wai.Handler.WarpTLS import Network.Wai.Logger import Control.Monad.IO.Class import Network.HTTP.Types import System.Log.Logger import qualified Data.CaseInsensitive as CI import Network.Socket -import Control.Exception import "crypto-api" Crypto.Random import qualified Web.ClientSession as CS import qualified Data.ByteString.Lazy as L @@ -39,6 +39,10 @@ import Control.Concurrent #ifdef __ANDROID__ import Data.Endian #endif +#if defined(__ANDROID__) || defined (mingw32_HOST_OS) +#else +import Control.Exception (bracketOnError) +#endif localhost :: HostName localhost = "localhost" @@ -67,10 +71,12 @@ browserProc url = proc "xdg-open" [url] - An IO action can also be run, to do something with the address, - such as start a web browser to view the webapp. -} -runWebApp :: Maybe HostName -> Wai.Application -> (SockAddr -> IO ()) -> IO () -runWebApp h app observer = withSocketsDo $ do +runWebApp :: Maybe TLSSettings -> Maybe HostName -> Wai.Application -> (SockAddr -> IO ()) -> IO () +runWebApp tlssettings h app observer = withSocketsDo $ do sock <- getSocket h - void $ forkIO $ runSettingsSocket webAppSettings sock app + void $ forkIO $ + (maybe runSettingsSocket (\ts -> runTLSSocket ts) tlssettings) + webAppSettings sock app sockaddr <- fixSockAddr <$> getSocketName sock observer sockaddr @@ -109,13 +115,13 @@ getSocket h = do use sock where #else - addrs <- getAddrInfo (Just hints) (Just hostname) port + addrs <- getAddrInfo (Just hints) (Just hostname) Nothing case (partition (\a -> addrFamily a == AF_INET) addrs) of (v4addr:_, _) -> go v4addr (_, v6addr:_) -> go v6addr _ -> error "unable to bind to a local socket" where - (hostname, port) = maybe (localhost, Nothing) splitHostPort h + hostname = fromMaybe localhost h hints = defaultHints { addrSocketType = Stream } {- Repeated attempts because bind sometimes fails for an - unknown reason on OSX. -} @@ -136,18 +142,6 @@ getSocket h = do listen sock maxListenQueue return sock -{- Splits address:port. For IPv6, use [address]:port. The port is optional. -} -splitHostPort :: String -> (HostName, Maybe ServiceName) -splitHostPort s - | "[" `isPrefixOf` s = let (h, p) = break (== ']') (drop 1 s) - in if "]:" `isPrefixOf` p - then (h, Just $ drop 2 p) - else (h, Nothing) - | otherwise = let (h, p) = separate (== ':') s - in if null p - then (h, Nothing) - else (h, Just p) - {- Checks if debugging is actually enabled. -} debugEnabled :: IO Bool debugEnabled = do diff --git a/Utility/WinProcess.hs b/Utility/WinProcess.hs index 5c6d4cfce..7a566dcba 100644 --- a/Utility/WinProcess.hs +++ b/Utility/WinProcess.hs @@ -11,9 +11,5 @@ module Utility.WinProcess where import Utility.PID -import System.Win32.Process -import Foreign.C -import Control.Exception - foreign import ccall unsafe "terminatepid" terminatePID :: PID -> IO () diff --git a/debian/changelog b/debian/changelog index 872786d83..b33f4c9cd 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,4 +1,84 @@ -git-annex (5.20140211) UNRELEASED; urgency=medium +git-annex (5.20140307) UNRELEASED; urgency=medium + + * Fix zombie leak and general inneficiency when copying files to a + local git repo. + * webapp: Added a "Sync now" item to each repository's menu. + + -- Joey Hess <joeyh@debian.org> Thu, 06 Mar 2014 16:17:01 -0400 + +git-annex (5.20140306) unstable; urgency=high + + * sync: Fix bug in direct mode that caused a file that was not + checked into git to be deleted when there was a conflicting + merge with a remote. + * webapp: Now supports HTTPS. + * webapp: No longer supports a port specified after --listen, since + it was buggy, and that use case is better supported by setting up HTTPS. + * annex.listen can be configured, instead of using --listen + * annex.startupscan can be set to false to disable the assistant's startup + scan. + * Probe for quvi version at run time. + * webapp: Filter out from Switch Repository list any + repositories listed in autostart file that don't have a + git directory anymore. (Or are bare) + * webapp: Refuse to start in a bare git repository. + * assistant --autostart: Refuse to start in a bare git repository. + * webapp: Don't list the public repository group when editing a + git repository; it only makes sense for special remotes. + * view, vfilter: Add support for filtering tags and values out of a view, + using !tag and field!=value. + * vadd: Allow listing multiple desired values for a field. + * view: Refuse to enter a view when no branch is currently checked out. + * metadata: To only set a field when it's not already got a value, use + -s field?=value + * Run .git/hooks/pre-commit-annex whenever a commit is made. + * sync: Automatically resolve merge conflict between and annexed file + and a regular git file. + * glacier: Pass --region to glacier checkpresent. + * webdav: When built with a new enough haskell DAV (0.6), disable + the http response timeout, which was only 5 seconds. + * webapp: Include no-pty in ssh authorized_keys lines. + * assistant: Smarter log file rotation, which takes free disk space + into account. + + -- Joey Hess <joeyh@debian.org> Thu, 06 Mar 2014 12:28:04 -0400 + +git-annex (5.20140227) unstable; urgency=medium + + * metadata: Field names limited to alphanumerics and a few whitelisted + punctuation characters to avoid issues with views, etc. + * metadata: Field names are now case insensative. + * When constructing views, metadata is available about the location of the + file in the view's reference branch. Allows incorporating parts of the + directory hierarchy in a view. + For example `git annex view tag=* podcasts/=*` makes a view in the form + tag/showname. + * --metadata field=value can now use globs to match, and matches + case insensatively, the same as git annex view field=value does. + * annex.genmetadata can be set to make git-annex automatically set + metadata (year and month) when adding files. + * Make annex.web-options be used in several places that call curl. + * Fix handling of rsync remote urls containing a username, + including rsync.net. + * Preserve metadata when staging a new version of an annexed file. + * metadata: Support --json + * webapp: Fix creation of box.com and Amazon S3 and Glacier + repositories, broken in 5.20140221. + * webdav: When built with DAV 0.6.0, use the new DAV monad to avoid + locking files, which is not needed by git-annex's use of webdav, and + does not work on Box.com. + * webdav: Fix path separator bug when used on Windows. + * repair: Optimise unpacking of pack files, and avoid repeated error + messages about corrupt pack files. + * Add build dep on regex-compat to fix build on mipsel, which lacks + regex-tdfa. + * Disable test suite on sparc, which is missing optparse-applicative. + * Put non-object tmp files in .git/annex/misctmp, leaving .git/annex/tmp + for only partially transferred objects. + + -- Joey Hess <joeyh@debian.org> Thu, 27 Feb 2014 11:34:19 -0400 + +git-annex (5.20140221) unstable; urgency=medium * metadata: New command that can attach metadata to files. * --metadata can be used to limit commands to acting on files @@ -27,7 +107,7 @@ git-annex (5.20140211) UNRELEASED; urgency=medium * Windows webapp: Can create repos on removable drives. * Windows: Ensure HOME is set, as needed by bundled cygwin utilities. - -- Joey Hess <joeyh@debian.org> Mon, 10 Feb 2014 21:33:03 -0400 + -- Joey Hess <joeyh@debian.org> Fri, 21 Feb 2014 11:23:59 -0400 git-annex (5.20140210) unstable; urgency=medium diff --git a/debian/control b/debian/control index 110d160cb..adcbb8255 100644 --- a/debian/control +++ b/debian/control @@ -6,6 +6,7 @@ Build-Depends: ghc (>= 7.4), libghc-mtl-dev (>= 2.1.1), libghc-missingh-dev, + libghc-data-default-dev, libghc-hslogger-dev, libghc-pcre-light-dev, libghc-sha-dev, @@ -35,6 +36,7 @@ Build-Depends: libghc-hamlet-dev [i386 amd64 kfreebsd-i386 kfreebsd-amd64 powerpc sparc], libghc-clientsession-dev [i386 amd64 kfreebsd-i386 kfreebsd-amd64 powerpc sparc], libghc-warp-dev [i386 amd64 kfreebsd-i386 kfreebsd-amd64 powerpc sparc], + libghc-warp-tls-dev [i386 amd64 kfreebsd-i386 kfreebsd-amd64 powerpc sparc], libghc-wai-dev [i386 amd64 kfreebsd-i386 kfreebsd-amd64 powerpc sparc], libghc-wai-logger-dev [i386 amd64 kfreebsd-i386 kfreebsd-amd64 powerpc sparc], libghc-case-insensitive-dev, @@ -51,11 +53,12 @@ Build-Depends: libghc-http-dev, libghc-feed-dev, libghc-regex-tdfa-dev [!mipsel !s390], + libghc-regex-compat-dev [mipsel s390], libghc-tasty-dev (>= 0.7) [!mipsel !sparc], libghc-tasty-hunit-dev [!mipsel !sparc], libghc-tasty-quickcheck-dev [!mipsel !sparc], libghc-tasty-rerun-dev [!mipsel !sparc], - libghc-optparse-applicative-dev, + libghc-optparse-applicative-dev [!sparc], lsof [!kfreebsd-i386 !kfreebsd-amd64], ikiwiki, perlmagick, diff --git a/doc/Android/comment_6_97704e0d89bb87155e019e09e54fc9bf._comment b/doc/Android/comment_6_97704e0d89bb87155e019e09e54fc9bf._comment new file mode 100644 index 000000000..39c44a8fb --- /dev/null +++ b/doc/Android/comment_6_97704e0d89bb87155e019e09e54fc9bf._comment @@ -0,0 +1,13 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawk2BFMjJW081uX4aJdhStmSFPBUGzL92ZU" + nickname="Frédéric" + subject="where do I put the SSH keys on Android" + date="2014-02-26T20:26:38Z" + content=""" +@mebus : +You can put your SSH keys here : + +/sdcard/git-annex.home/.ssh/id_rsa + +/sdcard/git-annex.home/.ssh/id_rsa.pub +"""]] diff --git a/doc/assistant/release_notes.mdwn b/doc/assistant/release_notes.mdwn index 5c91907b1..13b7c62ab 100644 --- a/doc/assistant/release_notes.mdwn +++ b/doc/assistant/release_notes.mdwn @@ -1,3 +1,10 @@ +## version 5.20140221 + +The Windows port of the assistant and webapp is now considered to be beta +quality. There are important missing features (notably Jabber), documented +on [[todo/windows_support]], but the webapp is broadly usable on Windows +now. + ## version 5.20131221 There is now a arm [[install/linux_standalone]] build of git-annex, diff --git a/doc/automatic_conflict_resolution/comment_1_307898855f91a2a189d4fa5eae62cce1._comment b/doc/automatic_conflict_resolution/comment_1_307898855f91a2a189d4fa5eae62cce1._comment new file mode 100644 index 000000000..69e136b44 --- /dev/null +++ b/doc/automatic_conflict_resolution/comment_1_307898855f91a2a189d4fa5eae62cce1._comment @@ -0,0 +1,10 @@ +[[!comment format=txt + username="https://www.google.com/accounts/o8/id?id=AItOawnJTqmRu1YCKS2Hsm4vtOflLhP4fU-k98w" + nickname="Ahmed" + subject="Customise conflict resolution behaviour" + date="2014-02-25T11:42:08Z" + content=""" +How to customise git-annex conflict resolution behaviour, such that for example: change naming convention of conflicted files with suffix or prefix, move conflicted files to another directory structure, overwrite conflicted files from preferred content ... + + +"""]] diff --git a/doc/bugs/Assistant_lost_dbus_connection_spamming_log/comment_4_f4e0fa25b7f466228622a6da02b157e7._comment b/doc/bugs/Assistant_lost_dbus_connection_spamming_log/comment_4_f4e0fa25b7f466228622a6da02b157e7._comment new file mode 100644 index 000000000..65e2f43f8 --- /dev/null +++ b/doc/bugs/Assistant_lost_dbus_connection_spamming_log/comment_4_f4e0fa25b7f466228622a6da02b157e7._comment @@ -0,0 +1,24 @@ +[[!comment format=mdwn + username="http://schnouki.net/" + nickname="Schnouki" + subject="comment 4" + date="2014-03-06T00:10:09Z" + content=""" +Same issue here: Arch chroot on a Synology DS413j NAS. Kernel 2.6.32.12 (by Synology), glibc 2.18-12.1, git-annex 5.20140227 (standalone armel). DBus is running as root, but not for the git-annex user. + +Ran it through strace, here is a hopefully relevant trace: + + socket(PF_INET, SOCK_STREAM, IPPROTO_TCP) = 11 + fcntl64(11, F_GETFL) = 0x2 (flags O_RDWR) + fcntl64(11, F_SETFL, O_RDWR|O_NONBLOCK) = 0 + setsockopt(11, SOL_SOCKET, SO_REUSEADDR, [1], 4) = 0 + bind(11, {sa_family=AF_INET, sin_port=htons(0), sin_addr=inet_addr(\"127.0.0.1\")}, 16) = 0 + listen(11, 128) = 0 + getsockname(11, {sa_family=AF_INET, sin_port=htons(50664), sin_addr=inet_addr(\"127.0.0.1\")}, [16]) = 0 + ... + accept4(11, 0x40b7d3e0, [16], SOCK_NONBLOCK) = -1 ENOSYS (Function not implemented) + +According to `man accept`, `accept4()` is available since Linux 2.6.28 and glibc 2.10. + +And I can't connect to the webapp (neither to localhost using a SSH tunnel, nor to the LAN IP when using --listen=...). +"""]] diff --git a/doc/bugs/Assistant_lost_dbus_connection_spamming_log/comment_5_6b9b87bfb8b94171b3dba51919fd1ceb._comment b/doc/bugs/Assistant_lost_dbus_connection_spamming_log/comment_5_6b9b87bfb8b94171b3dba51919fd1ceb._comment new file mode 100644 index 000000000..d612cf35f --- /dev/null +++ b/doc/bugs/Assistant_lost_dbus_connection_spamming_log/comment_5_6b9b87bfb8b94171b3dba51919fd1ceb._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.146" + subject="comment 5" + date="2014-03-06T18:12:57Z" + content=""" +Again the accept message does not seem to be related to dbus. A dbus client has no reason to do that; a web server does. The use of `O_NONBLOCK` with accept4 seems likely to be the culprit to me. + +How frequently is dbus mentioned in the log? +"""]] diff --git a/doc/bugs/Auto-repair_greatly_slows_down_the_machine.mdwn b/doc/bugs/Auto-repair_greatly_slows_down_the_machine.mdwn new file mode 100644 index 000000000..58d436898 --- /dev/null +++ b/doc/bugs/Auto-repair_greatly_slows_down_the_machine.mdwn @@ -0,0 +1,19 @@ +### Please describe the problem. + +The assistant regulary ends up trying to perform repair (I don't know why, it happens fairly often, once a week or so). When it does so, it ends up creating a huge (2.4G) .git/objects directory, and a git prune-packed process uses so much I/O the machine really slows down. + +### What steps will reproduce the problem? + +I don't have any reliable way to reproduce it. The repository ends up being attempted to be repaired around once a week. This week the repair (and the slowdown) also happened on a second computer. + +### What version of git-annex are you using? On what operating system? + +git-annex version: 5.20140221-gbdfc8e1 (using the standalone 64bit builds) + +This is on an up-to-date Arch Linux. It also happened on Fedora 20. + +### Please provide any additional information below. + +The daemon.log is fairly long, but not particulary interesting: [[https://ssl.zerodogg.org/~zerodogg/private/tmp/daemon.log-2014-02-25.1]] + +The «resource vanished (Broken pipe)» at the end is the result of me killing the prune-packed in order to be able to use the machine again. diff --git a/doc/bugs/Auto-repair_greatly_slows_down_the_machine/comment_1_a52e4ef04209d0a2449165e2b4cb9ccc._comment b/doc/bugs/Auto-repair_greatly_slows_down_the_machine/comment_1_a52e4ef04209d0a2449165e2b4cb9ccc._comment new file mode 100644 index 000000000..b50a0cfb8 --- /dev/null +++ b/doc/bugs/Auto-repair_greatly_slows_down_the_machine/comment_1_a52e4ef04209d0a2449165e2b4cb9ccc._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.146" + subject="comment 1" + date="2014-02-26T17:38:05Z" + content=""" +Auto repair is not intended to be a common occurrance. It means something went badly, horribly wrong on your machine, and it lost data that git wrote to disk. It's more important in such a scenraio to get back to a working system eventually than to do something fast or inexpensively. + +If you're seeing the need for auto repair on a weekly basis, your computer is failing in a horrible way horribly frequently, and the thing to do is to find out why, and fix that. Perhaps you need to start cleanly shutting down the system. Perhaps something is causing your computer to crash, and you need to fix that. +"""]] diff --git a/doc/bugs/Auto-repair_greatly_slows_down_the_machine/comment_2_9f5340ab1012f335af0c246b82c1a777._comment b/doc/bugs/Auto-repair_greatly_slows_down_the_machine/comment_2_9f5340ab1012f335af0c246b82c1a777._comment new file mode 100644 index 000000000..6cbd58f2f --- /dev/null +++ b/doc/bugs/Auto-repair_greatly_slows_down_the_machine/comment_2_9f5340ab1012f335af0c246b82c1a777._comment @@ -0,0 +1,16 @@ +[[!comment format=mdwn + username="EskildHustvedt" + ip="80.202.213.223" + subject="comment 2" + date="2014-02-27T14:16:11Z" + content=""" +That's the thing. The drive is fine, I've fscked it, and the machine is always shut down cleanly (and it is very stable, can't remember the last time it crashed). So there's no reason why this should be happening, and since git-annex doesn't say anything about why it started the auto-repair, I'm unable to track it down further. + +It's also always this repository. I have several assistant managed repos on the same machine, and this is the only one that git-annex regulary starts repairing (and the only one that it has auto-repaired on another box as well). No files in the repository itself has ever been noticed as corrupt, nor has there been any indications anywhere else about problems, hardware or filesystem (nothing wrong in the 40 or so git repos on the machine, no problems with software installed on the drive, never seen anything in dmesg). + +It's happening often enough for me to have to consider dropping use of the assistant. It could still be a problem with the machine (though it seems unlikely, given that it's consistently the same repo, and that it has done it with the same repo on other machines), but git-annex doesn't provide any information about it. + +Could you add some more information about what it thinks is wrong to the logging? + +Funnily enough, git-annex started repairing it again as I was writing this. +"""]] diff --git a/doc/bugs/Auto-repair_greatly_slows_down_the_machine/comment_3_67bfccf0934075559d439b1deafc001e._comment b/doc/bugs/Auto-repair_greatly_slows_down_the_machine/comment_3_67bfccf0934075559d439b1deafc001e._comment new file mode 100644 index 000000000..edf7c781b --- /dev/null +++ b/doc/bugs/Auto-repair_greatly_slows_down_the_machine/comment_3_67bfccf0934075559d439b1deafc001e._comment @@ -0,0 +1,9 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawnX1msQxnLoSeu7q-i-c9BWghonsN7Qmns" + nickname="Jan Ulrich" + subject="comment 3" + date="2014-03-05T11:47:06Z" + content=""" +I observe something similar. Today git-annex started to repair a repository. CPU is at 100%, I can still work because I have a multikernel system, so I'll wait and see if it comes to an end. + +"""]] diff --git a/doc/bugs/Auto-repair_greatly_slows_down_the_machine/comment_4_5fa785aa759d1a1917f2a292324fe5ec._comment b/doc/bugs/Auto-repair_greatly_slows_down_the_machine/comment_4_5fa785aa759d1a1917f2a292324fe5ec._comment new file mode 100644 index 000000000..3a43c9260 --- /dev/null +++ b/doc/bugs/Auto-repair_greatly_slows_down_the_machine/comment_4_5fa785aa759d1a1917f2a292324fe5ec._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawnX1msQxnLoSeu7q-i-c9BWghonsN7Qmns" + nickname="Jan Ulrich" + subject="comment 4" + date="2014-03-06T06:21:59Z" + content=""" +Just for the records. I just shutdown the daemon because it ran all night long with 100%. +"""]] diff --git a/doc/bugs/Auto-repair_greatly_slows_down_the_machine/comment_5_9fe529034ad0115792b58d7da99c167e._comment b/doc/bugs/Auto-repair_greatly_slows_down_the_machine/comment_5_9fe529034ad0115792b58d7da99c167e._comment new file mode 100644 index 000000000..4683165c3 --- /dev/null +++ b/doc/bugs/Auto-repair_greatly_slows_down_the_machine/comment_5_9fe529034ad0115792b58d7da99c167e._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.146" + subject="comment 5" + date="2014-03-06T18:14:37Z" + content=""" +auto-repair is only done if git fsck detects a problem. You can run git fsck yourself to see. +"""]] diff --git a/doc/bugs/Box.com_ReposnseTimeout.mdwn b/doc/bugs/Box.com_ReposnseTimeout.mdwn new file mode 100644 index 000000000..4beb6999c --- /dev/null +++ b/doc/bugs/Box.com_ReposnseTimeout.mdwn @@ -0,0 +1,12 @@ +Box.com is still not working properly in version 5.20140227 (I'm using it in Debian testing and sid). + +I created a new clean repository and configured Box.com (everything from the webapp). At first it seamed to work, files where being uploaded and the logs where fine. Then I created another clean repository in another computer and started syncing. Downloading files worked properly, but when trying to upload a file from the second computer I got this: + + copy my_file (gpg) (checking box.com...) (to box.com...) + 100% 0.0 B/s 0sResponseTimeout + failed + git-annex: copy: 1 failed + +When I got back to the first computer I saw the same behavior, uploading files wasn't working any more. + +> [[duplicate|done]] --[[Joey]] diff --git a/doc/bugs/Box.com_ReposnseTimeout/comment_1_4ac0bf61fb4b2ac335a8a1f29e9d882d._comment b/doc/bugs/Box.com_ReposnseTimeout/comment_1_4ac0bf61fb4b2ac335a8a1f29e9d882d._comment new file mode 100644 index 000000000..062fbfac6 --- /dev/null +++ b/doc/bugs/Box.com_ReposnseTimeout/comment_1_4ac0bf61fb4b2ac335a8a1f29e9d882d._comment @@ -0,0 +1,20 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawk9nck8WX8-ADF3Fdh5vFo4Qrw1I_bJcR8" + nickname="Jon Ander" + subject="comment 1" + date="2014-03-01T11:23:05Z" + content=""" +I have another error log: + + 29% 653.2KB/s 1m49sgpg: [stdout]: write error: Broken pipe + gpg: DBG: deflate: iobuf_write failed + gpg: build_packet failed: file write error + gpg: [stdout]: write error: Broken pipe + gpg: iobuf_flush failed on close: file write error + gpg: symmetric encryption of `[stdin]' failed: file write error + ResponseTimeout + git-annex: fd:96: hPutBuf: resource vanished (Broken pipe) + ResponseTimeout + +Some files (very few) are being uploaded properly, so the error is not always reproducible. +"""]] diff --git a/doc/bugs/Box.com_ReposnseTimeout/comment_2_29d8a9fa8d385a08fa70337baaba462c._comment b/doc/bugs/Box.com_ReposnseTimeout/comment_2_29d8a9fa8d385a08fa70337baaba462c._comment new file mode 100644 index 000000000..864731a47 --- /dev/null +++ b/doc/bugs/Box.com_ReposnseTimeout/comment_2_29d8a9fa8d385a08fa70337baaba462c._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.146" + subject="comment 2" + date="2014-03-05T03:15:43Z" + content=""" +I suspect this is just box.com's webdav support being flakey. + +It's possible there's a timeout that's too aggressive in the http library. I have had some trouble with that when trying to use DAV over a slow connection. +"""]] diff --git a/doc/bugs/Box.com_ReposnseTimeout/comment_3_b73450b3a9728ac6f34f0e63255f6fa9._comment b/doc/bugs/Box.com_ReposnseTimeout/comment_3_b73450b3a9728ac6f34f0e63255f6fa9._comment new file mode 100644 index 000000000..e6585b6a2 --- /dev/null +++ b/doc/bugs/Box.com_ReposnseTimeout/comment_3_b73450b3a9728ac6f34f0e63255f6fa9._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawk9nck8WX8-ADF3Fdh5vFo4Qrw1I_bJcR8" + nickname="Jon Ander" + subject="comment 3" + date="2014-03-05T08:10:25Z" + content=""" +My connection is pretty fast, I've just copied a 77MB folder through Box's webdav with my file manager in ~90sec and I have never seen a timeout when using my file manager, all uploads finish properly. +"""]] diff --git a/doc/bugs/Box.com_ReposnseTimeout/comment_4_0bd9eb5947a21d0657e79cf276923bb5._comment b/doc/bugs/Box.com_ReposnseTimeout/comment_4_0bd9eb5947a21d0657e79cf276923bb5._comment new file mode 100644 index 000000000..c95d3ceb4 --- /dev/null +++ b/doc/bugs/Box.com_ReposnseTimeout/comment_4_0bd9eb5947a21d0657e79cf276923bb5._comment @@ -0,0 +1,9 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.146" + subject="comment 4" + date="2014-03-05T17:11:40Z" + content=""" +Previous bug report about this with much more information, please do any followup there: +[[bugs/box.com_never_stops_syncing.]] +"""]] diff --git a/doc/forum/Can_not_Drop_Unused_Files_With_Spaces.mdwn b/doc/bugs/Can_not_Drop_Unused_Files_With_Spaces.mdwn index 70a573ff4..c40a90feb 100644 --- a/doc/forum/Can_not_Drop_Unused_Files_With_Spaces.mdwn +++ b/doc/bugs/Can_not_Drop_Unused_Files_With_Spaces.mdwn @@ -18,3 +18,5 @@ show these then running, git annex dropunused 1-3 --force reports ok for each drop operation but rerunning git annex unused --from cloud still shows these three files as unused. I am using git-annex on mac os x (current dmg) on a direct repo. I have similar problems dropping files on the current repo even though I drop unused they still show up as unused. + +> [[fixed|done]] --[[Joey]] diff --git a/doc/forum/Can_not_Drop_Unused_Files_With_Spaces/comment_1_b909ed9f474601587b2adad7ad4f674d._comment b/doc/bugs/Can_not_Drop_Unused_Files_With_Spaces/comment_1_b909ed9f474601587b2adad7ad4f674d._comment index fa41b59a7..fa41b59a7 100644 --- a/doc/forum/Can_not_Drop_Unused_Files_With_Spaces/comment_1_b909ed9f474601587b2adad7ad4f674d._comment +++ b/doc/bugs/Can_not_Drop_Unused_Files_With_Spaces/comment_1_b909ed9f474601587b2adad7ad4f674d._comment diff --git a/doc/forum/Can_not_Drop_Unused_Files_With_Spaces/comment_2_b2735a6e03db3f77a87a0f7d87347685._comment b/doc/bugs/Can_not_Drop_Unused_Files_With_Spaces/comment_2_b2735a6e03db3f77a87a0f7d87347685._comment index 5f5694c00..5f5694c00 100644 --- a/doc/forum/Can_not_Drop_Unused_Files_With_Spaces/comment_2_b2735a6e03db3f77a87a0f7d87347685._comment +++ b/doc/bugs/Can_not_Drop_Unused_Files_With_Spaces/comment_2_b2735a6e03db3f77a87a0f7d87347685._comment diff --git a/doc/forum/Can_not_Drop_Unused_Files_With_Spaces/comment_3_dd82a0cd698b0688ff08f0462af0275f._comment b/doc/bugs/Can_not_Drop_Unused_Files_With_Spaces/comment_3_dd82a0cd698b0688ff08f0462af0275f._comment index 86e3bd2c1..86e3bd2c1 100644 --- a/doc/forum/Can_not_Drop_Unused_Files_With_Spaces/comment_3_dd82a0cd698b0688ff08f0462af0275f._comment +++ b/doc/bugs/Can_not_Drop_Unused_Files_With_Spaces/comment_3_dd82a0cd698b0688ff08f0462af0275f._comment diff --git a/doc/forum/Can_not_Drop_Unused_Files_With_Spaces/comment_4_bbebb1d0dc5fbc1f6a0bb75b47bd4986._comment b/doc/bugs/Can_not_Drop_Unused_Files_With_Spaces/comment_4_bbebb1d0dc5fbc1f6a0bb75b47bd4986._comment index 6459ee8d7..6459ee8d7 100644 --- a/doc/forum/Can_not_Drop_Unused_Files_With_Spaces/comment_4_bbebb1d0dc5fbc1f6a0bb75b47bd4986._comment +++ b/doc/bugs/Can_not_Drop_Unused_Files_With_Spaces/comment_4_bbebb1d0dc5fbc1f6a0bb75b47bd4986._comment diff --git a/doc/forum/Can_not_Drop_Unused_Files_With_Spaces/comment_5_106c271d5174342055910bf57c0a34c5._comment b/doc/bugs/Can_not_Drop_Unused_Files_With_Spaces/comment_5_106c271d5174342055910bf57c0a34c5._comment index 4ad4d6f8b..4ad4d6f8b 100644 --- a/doc/forum/Can_not_Drop_Unused_Files_With_Spaces/comment_5_106c271d5174342055910bf57c0a34c5._comment +++ b/doc/bugs/Can_not_Drop_Unused_Files_With_Spaces/comment_5_106c271d5174342055910bf57c0a34c5._comment diff --git a/doc/forum/Can_not_Drop_Unused_Files_With_Spaces/comment_6_3a2d3cc3e018beaf2eb44b86ce7e1a7f._comment b/doc/bugs/Can_not_Drop_Unused_Files_With_Spaces/comment_6_3a2d3cc3e018beaf2eb44b86ce7e1a7f._comment index fbd9ed55c..fbd9ed55c 100644 --- a/doc/forum/Can_not_Drop_Unused_Files_With_Spaces/comment_6_3a2d3cc3e018beaf2eb44b86ce7e1a7f._comment +++ b/doc/bugs/Can_not_Drop_Unused_Files_With_Spaces/comment_6_3a2d3cc3e018beaf2eb44b86ce7e1a7f._comment diff --git a/doc/bugs/Crash_when_adding_jabber_account_/comment_5_9bfd8df548d5866599dfc69fb3aaf94a._comment b/doc/bugs/Crash_when_adding_jabber_account_/comment_5_9bfd8df548d5866599dfc69fb3aaf94a._comment new file mode 100644 index 000000000..40bc68dcb --- /dev/null +++ b/doc/bugs/Crash_when_adding_jabber_account_/comment_5_9bfd8df548d5866599dfc69fb3aaf94a._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawlm_3m5gLhML9bHbZ8FpJ-HBZhWaRfFeO8" + nickname="Corey" + subject="I got this error too." + date="2014-02-25T17:23:33Z" + content=""" +With the armel tarbell downloaded 2014-02-21 on an Ubuntu (precise) chroot running on a Chromebook. +"""]] diff --git a/doc/bugs/Creating_a_box.com_repository_fails.mdwn b/doc/bugs/Creating_a_box.com_repository_fails.mdwn index 75d59c9bc..ecebd7a00 100644 --- a/doc/bugs/Creating_a_box.com_repository_fails.mdwn +++ b/doc/bugs/Creating_a_box.com_repository_fails.mdwn @@ -35,3 +35,7 @@ ubuntu 13.10 (saucy), i686 > Seems that [DAV-0.6 is badly broken](http://bugs.debian.org/737902). > I have adjusted the cabal file to refuse to build with that broken > version. +> +>> Update: Had to work around additional breakage in DAV-0.6. It's +>> fully tested and working now, although not yet uploaded to Debian +>> unstable. [[done]] --[[Joey]] diff --git a/doc/bugs/Creating_a_box.com_repository_fails/comment_7_73f71386f8eafbb65f4cc9769021710f._comment b/doc/bugs/Creating_a_box.com_repository_fails/comment_7_73f71386f8eafbb65f4cc9769021710f._comment new file mode 100644 index 000000000..016371346 --- /dev/null +++ b/doc/bugs/Creating_a_box.com_repository_fails/comment_7_73f71386f8eafbb65f4cc9769021710f._comment @@ -0,0 +1,13 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawk9nck8WX8-ADF3Fdh5vFo4Qrw1I_bJcR8" + nickname="Jon Ander" + subject="comment 7" + date="2014-02-24T13:20:27Z" + content=""" +This is what I get in the log in version 5.20140221 in Debian Sid: + + 100% 46.5KB/s 0sInternalIOException <socket: 28>: hPutBuf: illegal operation (handle is closed) + InternalIOException <socket: 25>: hPutBuf: illegal operation (handle is closed) + +It seams that the file is being uploaded (folders are being created in box.com) but it crashes when reaching 100% +"""]] diff --git a/doc/bugs/Creating_a_box.com_repository_fails/comment_8_109e37051beb729834e05997c023b849._comment b/doc/bugs/Creating_a_box.com_repository_fails/comment_8_109e37051beb729834e05997c023b849._comment new file mode 100644 index 000000000..24c9153a8 --- /dev/null +++ b/doc/bugs/Creating_a_box.com_repository_fails/comment_8_109e37051beb729834e05997c023b849._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="fmarier" + ip="192.81.133.89" + subject="Re: No working ubuntu package" + date="2014-02-26T02:06:24Z" + content=""" +I have just uploaded version 5.20140221.1 of the git-annex package in my PPA. It should re-enable DAV support. +"""]] diff --git a/doc/bugs/Glacier_remote_doesn__39__t_pass_the_--region_parameter_to_glacier-cli_on_hasKey.mdwn b/doc/bugs/Glacier_remote_doesn__39__t_pass_the_--region_parameter_to_glacier-cli_on_hasKey.mdwn new file mode 100644 index 000000000..7756db866 --- /dev/null +++ b/doc/bugs/Glacier_remote_doesn__39__t_pass_the_--region_parameter_to_glacier-cli_on_hasKey.mdwn @@ -0,0 +1,38 @@ +### Please describe the problem. +hasKey check fails when using a Glacier special remote with a non-default region setting. + +### What steps will reproduce the problem? +1. Configure a Glacier special remote with a region other than us-east-1 +2. Move some files to the remote +3. Somehow let galcier-cli cache expire. In my case this was waiting for more than 60 hours. +4. Try to copy the file to the glacier remote again. + +### What version of git-annex are you using? On what operating system? +5.20140227 on Ubuntu 12.04 + +### Please provide any additional information below. + +[[!format sh """ +user@server:$ git annex copy my_file.txt --to glacier +copy my_file.txt (checking glacier...) Traceback (most recent call last): + File "/bin/glacier", line 730, in <module> + App().main() + File "/bin/glacier", line 716, in main + self.args.func() + File "/bin/glacier", line 620, in archive_checkpresent + wait=self.args.wait) + File "/bin/glacier", line 442, in _vault_sync + vault = self.connection.get_vault(vault_name) + File "/usr/local/lib/python2.7/dist-packages/boto/glacier/layer2.py", line 82, in get_vault + response_data = self.layer1.describe_vault(name) + File "/usr/local/lib/python2.7/dist-packages/boto/glacier/layer1.py", line 195, in describe_vault + return self.make_request('GET', uri) + File "/usr/local/lib/python2.7/dist-packages/boto/glacier/layer1.py", line 118, in make_request + raise UnexpectedHTTPResponseError(ok_responses, response) +boto.glacier.exceptions.UnexpectedHTTPResponseError: Expected 200, got (404, code=ResourceNotFoundException, message=Vault not found for ARN: arn:aws:glacier:us-east-1:111111111111:vaults/myvault) +(user error (glacier ["archive","checkpresent","myvault","--quiet","SHA256E-s1111111--aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.JPG"] exited 1)) failed +git-annex: copy: 1 failed +"""]] + +> [[fixed|done]]; made it use the same glacierParams as everything else. +> --[[Joey]] diff --git a/doc/bugs/Hangs_on_creating_repository_when_using_--listen.mdwn b/doc/bugs/Hangs_on_creating_repository_when_using_--listen.mdwn index 913e02cd5..fe0fe80f3 100644 --- a/doc/bugs/Hangs_on_creating_repository_when_using_--listen.mdwn +++ b/doc/bugs/Hangs_on_creating_repository_when_using_--listen.mdwn @@ -44,3 +44,6 @@ WebApp crashed: unable to bind to local socket > to use when something else is already listening there. --[[Joey]] [[!tag /design/assistant]] + +>> --listen no longer accepts a port. Use the new HTTPS support instead. +>> [[done]] --[[Joey]] diff --git a/doc/bugs/Log_rotation_loses_large_logs.mdwn b/doc/bugs/Log_rotation_loses_large_logs.mdwn new file mode 100644 index 000000000..c38c198b6 --- /dev/null +++ b/doc/bugs/Log_rotation_loses_large_logs.mdwn @@ -0,0 +1,69 @@ +### Please describe the problem. + +I have a large git-annex repository created using the assistant. It has thousands of files in it and is about 50GB in size. +Yesterday I added a number of new files, and I also created a new "removable drive" repository for it to sync to. + +During these operations I could see a large amount of data being added to the git-annex log files. +I left my computer on overnight to finish the sync. + +Today I went to check the log files but there was no useful information in them. + +Looking at the source I suspect I had a single log file > 1 Megabyte in size, which caused the rotation to occur repeatedly until it rolled off the end. +However, I would have preferred if this large logfile had been kept for more than a couple of hours. + +See file contents below + +### What steps will reproduce the problem? + +See above + +### What version of git-annex are you using? On what operating system? + +Debian Wheezy, git-annex version: 5.20140210~bpo70+2 from wheezy-backports + +### Please provide any additional information below. + +[[!format sh """ +# If you can, paste a complete transcript of the problem occurring here. +# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log + +When I look at the log files now, I see the following: +-rw-r--r-- 1 pgl users 443 Mar 5 21:48 daemon.log +-rw-r--r-- 1 pgl users 443 Mar 5 21:47 daemon.log.1 +-rw-r--r-- 1 pgl users 81 Mar 5 01:16 daemon.log.10 +-rw-r--r-- 1 pgl users 69 Mar 5 07:29 daemon.log.2 +-rw-r--r-- 1 pgl users 81 Mar 5 01:16 daemon.log.3 +-rw-r--r-- 1 pgl users 81 Mar 5 01:16 daemon.log.4 +-rw-r--r-- 1 pgl users 81 Mar 5 01:16 daemon.log.5 +-rw-r--r-- 1 pgl users 81 Mar 5 01:16 daemon.log.6 +-rw-r--r-- 1 pgl users 81 Mar 5 01:16 daemon.log.7 +-rw-r--r-- 1 pgl users 81 Mar 5 01:16 daemon.log.8 +-rw-r--r-- 1 pgl users 81 Mar 5 01:16 daemon.log.9 + +Looking at some of these log files. + +pgl@....:/....../.git/annex$ cat daemon.log.3 +[2014-03-05 01:16:15 GMT] SanityCheckerHourly: Rotated logs due to size: 1026416 +pgl@....:/....../.git/annex$ cat daemon.log.4 +[2014-03-05 01:16:15 GMT] SanityCheckerHourly: Rotated logs due to size: 1026416 +pgl@....:/....../.git/annex$ cat daemon.log.5 +[2014-03-05 01:16:15 GMT] SanityCheckerHourly: Rotated logs due to size: 1026416 +pgl@....:/....../.git/annex$ cat daemon.log.6 +[2014-03-05 01:16:15 GMT] SanityCheckerHourly: Rotated logs due to size: 1026416 +pgl@....:/....../.git/annex$ cat daemon.log.7 +[2014-03-05 01:16:15 GMT] SanityCheckerHourly: Rotated logs due to size: 1026416 +pgl@....:/....../.git/annex$ cat daemon.log.8 +[2014-03-05 01:16:15 GMT] SanityCheckerHourly: Rotated logs due to size: 1026416 +pgl@....:/....../.git/annex$ cat daemon.log.9 +[2014-03-05 01:16:15 GMT] SanityCheckerHourly: Rotated logs due to size: 1026416 + +# End of transcript or log. +"""]] + +> Changed log rotation to only rotate 1 log per hour max, +> unless the total size of the log files is larger than the +> free disk space on the filesystem containing them. +> +> This way, runaway log growth will still be contained, +> but logs will generally rotate slowly enough to give plenty of time +> to see what's in them. [[done]] --[[Joey]] diff --git a/doc/bugs/Mac_OS_git_version_still_too_old_for_.gitignore__63__/comment_2_888fb193072cf05a34943db072eb7a3b._comment b/doc/bugs/Mac_OS_git_version_still_too_old_for_.gitignore__63__/comment_2_888fb193072cf05a34943db072eb7a3b._comment new file mode 100644 index 000000000..1bd1e037e --- /dev/null +++ b/doc/bugs/Mac_OS_git_version_still_too_old_for_.gitignore__63__/comment_2_888fb193072cf05a34943db072eb7a3b._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawmZgZuUhZlHpd_AbbcixY0QQiutb2I7GWY" + nickname="Jimmy" + subject="comment 2" + date="2014-02-21T07:03:20Z" + content=""" +Thanks Joey and Kevin. Glad to have the bug really fixed and glad to know that I wasn't missing something obvious! +"""]] diff --git a/doc/bugs/Mac_OS_git_version_still_too_old_for_.gitignore__63__/comment_3_f199ac6ae2448949ef0779177cf0ef58._comment b/doc/bugs/Mac_OS_git_version_still_too_old_for_.gitignore__63__/comment_3_f199ac6ae2448949ef0779177cf0ef58._comment new file mode 100644 index 000000000..591d4e80f --- /dev/null +++ b/doc/bugs/Mac_OS_git_version_still_too_old_for_.gitignore__63__/comment_3_f199ac6ae2448949ef0779177cf0ef58._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawmZgZuUhZlHpd_AbbcixY0QQiutb2I7GWY" + nickname="Jimmy" + subject="comment 3" + date="2014-02-21T22:05:06Z" + content=""" +And yep, it's fixed in 5.20140221-g1a47f5f. Thanks guys! +"""]] diff --git a/doc/bugs/Switching_repositories_in_webapp_on_a_remote_server_is_not_honoring_--listen_parameter.mdwn b/doc/bugs/Switching_repositories_in_webapp_on_a_remote_server_is_not_honoring_--listen_parameter.mdwn index 4829cde08..faa7a36de 100644 --- a/doc/bugs/Switching_repositories_in_webapp_on_a_remote_server_is_not_honoring_--listen_parameter.mdwn +++ b/doc/bugs/Switching_repositories_in_webapp_on_a_remote_server_is_not_honoring_--listen_parameter.mdwn @@ -19,3 +19,6 @@ I am running my "origin" repositories on a headless server. Managing these with # End of transcript or log. """]] + +>> --listen no longer accepts a port. Use the new HTTPS support instead. +>> [[done]] --[[Joey]] diff --git a/doc/bugs/amd64_i386_standalone:_no_SKEIN.mdwn b/doc/bugs/amd64_i386_standalone:_no_SKEIN.mdwn index 5dd230118..628cb13bd 100644 --- a/doc/bugs/amd64_i386_standalone:_no_SKEIN.mdwn +++ b/doc/bugs/amd64_i386_standalone:_no_SKEIN.mdwn @@ -35,4 +35,7 @@ key/value backends: SHA256E SHA1E SHA512E SHA224E SHA384E SHA256 SHA1 SHA512 SHA remote types: git gcrypt S3 bup directory rsync web webdav glacier hook external """]] --- [[clacke]]
\ No newline at end of file +-- [[clacke]] + +> [[done]] The autobuilds are now running debian unstable, and SKEIN is included +> now. --[[Joey]] diff --git a/doc/bugs/assistant_eats_all_CPU.mdwn b/doc/bugs/assistant_eats_all_CPU.mdwn index 4cd19782c..4939bf456 100644 --- a/doc/bugs/assistant_eats_all_CPU.mdwn +++ b/doc/bugs/assistant_eats_all_CPU.mdwn @@ -1,3 +1,5 @@ +[[!meta title="debian system runs assistant in tight loop, rather than using select"]] + ### Please describe the problem. After running for a while, the assistant maxes my CPU. I have evidence diff --git a/doc/bugs/assistant_eats_all_CPU/comment_19_2b8241800ae265260506ac9c73cca209._comment b/doc/bugs/assistant_eats_all_CPU/comment_19_2b8241800ae265260506ac9c73cca209._comment new file mode 100644 index 000000000..9c39baad2 --- /dev/null +++ b/doc/bugs/assistant_eats_all_CPU/comment_19_2b8241800ae265260506ac9c73cca209._comment @@ -0,0 +1,56 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawntVnR-Z5ghYInvsElbDeADPSuCsF18iTY" + nickname="Thomas" + subject="XMPP something to do with it?" + date="2014-03-02T13:48:22Z" + content=""" +High (between 30 and 100%) processor load in my case seems to be related to lots of these logfile entries: + + [2014-03-02 14:36:54 CET] XMPPClient: sending: Pushing \"s34\" (SendPackOutput 75 \"<elided>\") + [2014-03-02 14:36:54 CET] XMPPClient: to client: s6/a303a6cfb26b8e08 + [2014-03-02 14:36:54 CET] XMPPClient: sending: Pushing \"s34\" (SendPackOutput 76 \"<elided>\") + [2014-03-02 14:36:54 CET] XMPPClient: received: [\"Ignorable Presence from s6/941d024438f7cd11 Just (Element {elementName = Name {nameLocalName = \\"git-annex\\", nameNamespace = Just \\"git-annex\\", namePrefix = Nothing}, elementAttributes = [], elementNodes = []})\"] + [2014-03-02 14:36:54 CET] XMPPClient: to client: s6/a303a6cfb26b8e08 + [2014-03-02 14:36:54 CET] XMPPClient: received: [\"Pushing \\"s34\\" (CanPush (UUID \\"005cc6c3-a4de-44d5-847e-58b5bc4e4c26\\") [Ref \\"d39375b2da76a05c13a8fdc2197a58af0a9812fc\\",Ref \\"5c0108dcad557ab48bfe335e533f13f29777ca7b\\"])\"] + [2014-03-02 14:36:54 CET] XMPPClient: sending: Pushing \"s34\" (SendPackOutput 77 \"<elided>\") + [2014-03-02 14:36:54 CET] XMPPClient: to client: s6/a303a6cfb26b8e08 + [2014-03-02 14:36:54 CET] XMPPClient: sending: Pushing \"s34\" (SendPackOutput 78 \"<elided>\") + [2014-03-02 14:36:54 CET] XMPPClient: to client: s6/a303a6cfb26b8e08 + [2014-03-02 14:36:55 CET] XMPPClient: sending: Pushing \"s34\" (SendPackOutput 79 \"<elided>\") + [2014-03-02 14:36:55 CET] XMPPClient: to client: s6/a303a6cfb26b8e08 + [2014-03-02 14:36:55 CET] XMPPClient: sending: Pushing \"s34\" (SendPackOutput 80 \"<elided>\") + [2014-03-02 14:36:55 CET] XMPPClient: to client: s6/a303a6cfb26b8e08 + [2014-03-02 14:36:55 CET] XMPPClient: sending: Pushing \"s34\" (SendPackOutput 81 \"<elided>\") + [2014-03-02 14:36:55 CET] XMPPClient: to client: s6/a303a6cfb26b8e08 + [2014-03-02 14:36:55 CET] XMPPClient: sending: Pushing \"s34\" (SendPackOutput 82 \"<elided>\") + [2014-03-02 14:36:55 CET] XMPPClient: to client: s6/a303a6cfb26b8e08 + [2014-03-02 14:36:56 CET] XMPPClient: sending: Pushing \"s34\" (SendPackOutput 83 \"<elided>\") + [2014-03-02 14:36:56 CET] XMPPClient: to client: s6/a303a6cfb26b8e08 + [2014-03-02 14:36:56 CET] XMPPClient: sending: Pushing \"s34\" (SendPackOutput 84 \"<elided>\") + [2014-03-02 14:36:56 CET] XMPPClient: to client: s6/a303a6cfb26b8e08 + [2014-03-02 14:36:56 CET] XMPPClient: received: [\"Presence from s6/a303a6cfb26b8e08 Just (Element {elementName = Name {nameLocalName = \\"git-annex\\", nameNamespace = Just \\"git-annex\\", namePrefix = Nothing}, elementAttributes = [(Name {nameLocalName = \\"query\\", nameNamespace = Nothing, namePrefix = Nothing},[ContentText \\"\\"])], elementNodes = []})\",\"QueryPresence\"] + [2014-03-02 14:36:56 CET] XMPPClient: sending: Pushing \"s34\" (SendPackOutput 85 \"<elided>\") + [2014-03-02 14:36:56 CET] XMPPClient: received: [\"Pushing \\"s34\\" (CanPush (UUID \\"005cc6c3-a4de-44d5-847e-58b5bc4e4c26\\") [Ref \\"d39375b2da76a05c13a8fdc2197a58af0a9812fc\\",Ref \\"f43c777aed404e1e98dcbf9b6499f4d056b6662a\\"])\"] + [2014-03-02 14:36:56 CET] XMPPClient: to client: s6/a303a6cfb26b8e08 + [2014-03-02 14:36:56 CET] XMPPClient: sending: Pushing \"s34\" (SendPackOutput 86 \"<elided>\") + [2014-03-02 14:36:56 CET] XMPPClient: to client: s6/a303a6cfb26b8e08 + [2014-03-02 14:36:57 CET] XMPPClient: sending: Pushing \"s34\" (SendPackOutput 87 \"<elided>\") + [2014-03-02 14:36:57 CET] XMPPClient: received: [\"Ignorable Presence from s6/941d024438f7cd11 Just (Element {elementName = Name {nameLocalName = \\"git-annex\\", nameNamespace = Just \\"git-annex\\", namePrefix = Nothing}, elementAttributes = [], elementNodes = []})\"] + [2014-03-02 14:36:57 CET] XMPPClient: to client: s6/a303a6cfb26b8e08 + [2014-03-02 14:36:57 CET] XMPPClient: received: [\"Pushing \\"s34\\" (CanPush (UUID \\"005cc6c3-a4de-44d5-847e-58b5bc4e4c26\\") [Ref \\"d39375b2da76a05c13a8fdc2197a58af0a9812fc\\",Ref \\"f43c777aed404e1e98dcbf9b6499f4d056b6662a\\"])\"] + [2014-03-02 14:36:57 CET] XMPPClient: sending: Pushing \"s34\" (SendPackOutput 88 \"<elided>\") + [2014-03-02 14:36:57 CET] XMPPClient: to client: s6/a303a6cfb26b8e08 + [2014-03-02 14:36:57 CET] XMPPClient: sending: Pushing \"s34\" (SendPackOutput 89 \"<elided>\") + [2014-03-02 14:36:57 CET] XMPPClient: to client: s6/a303a6cfb26b8e08 + [2014-03-02 14:36:57 CET] XMPPClient: sending: Pushing \"s34\" (SendPackOutput 90 \"<elided>\") + [2014-03-02 14:36:57 CET] XMPPClient: to client: s6/a303a6cfb26b8e08 + +This is after everything is uploaded to a ssh server (in the cloud, used for transfer) and while another computer is downloading from there (?). + +git-annex version: 5.20140227 +build flags: Assistant Webapp Pairing Testsuite S3 WebDAV Inotify DBus XMPP DNS Feeds Quvi TDFA CryptoHash +key/value backends: SHA256E SHA1E SHA512E SHA224E SHA384E SKEIN256E SKEIN512E SHA256 SHA1 SHA512 SHA224 SHA384 SKEIN256 SKEIN512 WORM URL +remote types: git gcrypt S3 bup directory rsync web webdav tahoe glacier hook external + +Debian testing using jabber.org for xmpp +"""]] diff --git a/doc/bugs/assistant_eats_all_CPU/comment_20_1d9020679d66e6b4742df067cb9da4f1._comment b/doc/bugs/assistant_eats_all_CPU/comment_20_1d9020679d66e6b4742df067cb9da4f1._comment new file mode 100644 index 000000000..c228560f2 --- /dev/null +++ b/doc/bugs/assistant_eats_all_CPU/comment_20_1d9020679d66e6b4742df067cb9da4f1._comment @@ -0,0 +1,9 @@ +[[!comment format=mdwn + username="https://id.koumbit.net/anarcat" + ip="2001:1928:1:9::1" + subject="comment 20" + date="2014-03-02T16:33:57Z" + content=""" +This is still a problem in Debian jessie and version `5.20140210` from the debian packages. + +"""]] diff --git a/doc/bugs/assistant_eats_all_CPU/comment_21_8ce65a701604b9d13941844c62f62f23._comment b/doc/bugs/assistant_eats_all_CPU/comment_21_8ce65a701604b9d13941844c62f62f23._comment new file mode 100644 index 000000000..2e74010df --- /dev/null +++ b/doc/bugs/assistant_eats_all_CPU/comment_21_8ce65a701604b9d13941844c62f62f23._comment @@ -0,0 +1,14 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.146" + subject="strace or it didn't happen" + date="2014-03-05T21:02:17Z" + content=""" +This bug is about a particular sequence of system calls. It is not about every possible way that git-annex can use CPU! If you think you have reproduced this bug, you need to include a strace demonstrating the problem, or you're just adding noise. + +@Thomas your log shows git-annex doing a git pull over XMPP. This will take CPU, as will any git pull. It is not related to this bug. + +@myownlittl the assistant has to scan your repository when it's started to find any files you have modified since the last time it was run. That is not related to this bug. However, I will probably be adding a config knob to disable the startup scan. + +@Oliver, you said you had seen this same bug on Ubuntu, but did not show a strace, so I don't know if you are really experiencing the same bug. +"""]] diff --git a/doc/bugs/assistant_eats_all_CPU/comment_22_0d8de9a8e4b8e2ef3b9c7d839fbcad0c._comment b/doc/bugs/assistant_eats_all_CPU/comment_22_0d8de9a8e4b8e2ef3b9c7d839fbcad0c._comment new file mode 100644 index 000000000..ccaf726b7 --- /dev/null +++ b/doc/bugs/assistant_eats_all_CPU/comment_22_0d8de9a8e4b8e2ef3b9c7d839fbcad0c._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.146" + subject="comment 22" + date="2014-03-05T21:08:34Z" + content=""" +@anarcat, it seems to me that you need to find a minumal way to reproduce this. + +What architecture, what specfic model of CPU? + +Can you reproduce it by running the assistant in a new, empty git annex repository? If not, find the difference between that repository and the repository it happens in. If so, it would probably make sense to build git-annex from source with most features disabled to see if one of them causes it, and if not, try to get it down to a minimal test case suitable to being submitted as a GHC bug. +"""]] diff --git a/doc/bugs/box.com_never_stops_syncing..mdwn b/doc/bugs/box.com_never_stops_syncing..mdwn index 42b2eaf1a..d8e5391b5 100644 --- a/doc/bugs/box.com_never_stops_syncing..mdwn +++ b/doc/bugs/box.com_never_stops_syncing..mdwn @@ -61,3 +61,14 @@ failed """]] More to come(full log) + +> This is [[fixed|done]] in git; when built with a new enough +> version of the haskell DAV library, git-annex disables the default 5 +> second timeout. +> +> It'll still be present in the Debian stable backports, which are +> built with an old version of DAV. Not much I can do about that; +> backporting DAV would be difficult. +> +> The daily builds are updated to use the new version. +> --[[Joey]] diff --git a/doc/bugs/can__39__t_get.mdwn b/doc/bugs/can__39__t_get.mdwn index 42a15e8b6..d5a29a2ae 100644 --- a/doc/bugs/can__39__t_get.mdwn +++ b/doc/bugs/can__39__t_get.mdwn @@ -73,3 +73,9 @@ was git annex unused somewhere before) # End of transcript or log. """]] + +[[!tag moreinfo]] + +> Tagged moreinfo since I have a workable theory about how this happened, +> which would make it user configuration error and not a bug, but +> that has not been confirmed. --[[Joey]] diff --git a/doc/bugs/copy_fails_for_some_fails_without_explanation.mdwn b/doc/bugs/copy_fails_for_some_fails_without_explanation.mdwn new file mode 100644 index 000000000..f4489de27 --- /dev/null +++ b/doc/bugs/copy_fails_for_some_fails_without_explanation.mdwn @@ -0,0 +1,7 @@ +I have a large direct-mode repository whose files I'm trying to copy to a non-direct-mode repository. Both repositories live on an HDD attached to an rpi. + +When I do $ git annex copy --to pi dirs/to/copy, the copy starts out OK, but eventually many files fail to copy. The only diagnostic I get is "failed". Judging from the backscroll, I don't see a strong pattern to the files which fail to copy; they're kind of interspersed amongst files which were successfully copied. If I try to copy one of these failed files explicitly (git annex copy --to pi file/which/failed), this succeeds. I have plenty of free space on the disk. + +Is there a way to get more diagnostics out of git annex so I can see why these files are failing to copy? + +> [[fixed|done]] --[[Joey]] diff --git a/doc/bugs/copy_fails_for_some_fails_without_explanation/comment_10_0c1a5837305b721fc4a529cae3f4c3fb._comment b/doc/bugs/copy_fails_for_some_fails_without_explanation/comment_10_0c1a5837305b721fc4a529cae3f4c3fb._comment new file mode 100644 index 000000000..b0127ed82 --- /dev/null +++ b/doc/bugs/copy_fails_for_some_fails_without_explanation/comment_10_0c1a5837305b721fc4a529cae3f4c3fb._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawmUJBh1lYmvfCCiGr3yrdx-QhuLCSRnU5c" + nickname="Justin" + subject="comment 10" + date="2014-03-06T21:38:53Z" + content=""" +Thanks a lot, Joey. + +Compiling all of the dependencies for git-annex was taking forever on my pi, so I'll probably wait until the next release to test this out. But I'll report back here if I have any problems. +"""]] diff --git a/doc/bugs/copy_fails_for_some_fails_without_explanation/comment_1_e456604b26ed9c72b0a88cfb57f1a475._comment b/doc/bugs/copy_fails_for_some_fails_without_explanation/comment_1_e456604b26ed9c72b0a88cfb57f1a475._comment new file mode 100644 index 000000000..dba07bfd5 --- /dev/null +++ b/doc/bugs/copy_fails_for_some_fails_without_explanation/comment_1_e456604b26ed9c72b0a88cfb57f1a475._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawmUJBh1lYmvfCCiGr3yrdx-QhuLCSRnU5c" + nickname="Justin" + subject="comment 1" + date="2014-03-05T16:11:27Z" + content=""" +I tried git annex sync --content, and it failed to copy some files with + + git: createProcess: resource exhausted (Resource temporarily unavailable) + +So this sounds like fork is failing; I'm probably exhausting my poor pi's RAM. Maybe the same thing is happening for git annex copy. I'll run strace to see. +"""]] diff --git a/doc/bugs/copy_fails_for_some_fails_without_explanation/comment_2_4823d66bfb569605868af5cefe0d94dc._comment b/doc/bugs/copy_fails_for_some_fails_without_explanation/comment_2_4823d66bfb569605868af5cefe0d94dc._comment new file mode 100644 index 000000000..5bcd0e92e --- /dev/null +++ b/doc/bugs/copy_fails_for_some_fails_without_explanation/comment_2_4823d66bfb569605868af5cefe0d94dc._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.146" + subject="comment 2" + date="2014-03-05T20:31:57Z" + content=""" +How many files copied are we talking about before it begins to fail? + +You can try passing --debug, which will make git-annex show every external command it runs, which includes `cp` for a copy to another repo on the same machine. + +Might also check memory usage in top during the run. +"""]] diff --git a/doc/bugs/copy_fails_for_some_fails_without_explanation/comment_3_46305aa2d43da000c1a7cb003c822572._comment b/doc/bugs/copy_fails_for_some_fails_without_explanation/comment_3_46305aa2d43da000c1a7cb003c822572._comment new file mode 100644 index 000000000..297c28a40 --- /dev/null +++ b/doc/bugs/copy_fails_for_some_fails_without_explanation/comment_3_46305aa2d43da000c1a7cb003c822572._comment @@ -0,0 +1,18 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawmUJBh1lYmvfCCiGr3yrdx-QhuLCSRnU5c" + nickname="Justin" + subject="comment 3" + date="2014-03-06T18:21:53Z" + content=""" +> How many files copied are we talking about before it begins to fail? + +Tens of thousands of files processed, but many of them were already on the other remote so didn't invoke cp (or anything else). ~3300 invocations of cp. + +I saved a log of ps aux, and, while the memory used by git annex remains relatively constant, I do observe /tons/ of zombie processes. 3300, actually. + +I didn't check all of them, but all of the zombie pids I checked appear to have corresponded to this command: + + /home/pi/git-annex.linux/shimmed/git/git --git-dir=/home/pi/hdd/annex/.git --work-tree=/home/pi/hdd/annex cat-file --batch + +Perhaps git annex is forgetting to reap this processes? +"""]] diff --git a/doc/bugs/copy_fails_for_some_fails_without_explanation/comment_4_1dbdeded7f587e8fc2d1ac5170ecb928._comment b/doc/bugs/copy_fails_for_some_fails_without_explanation/comment_4_1dbdeded7f587e8fc2d1ac5170ecb928._comment new file mode 100644 index 000000000..537736a72 --- /dev/null +++ b/doc/bugs/copy_fails_for_some_fails_without_explanation/comment_4_1dbdeded7f587e8fc2d1ac5170ecb928._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.146" + subject="comment 4" + date="2014-03-06T18:32:33Z" + content=""" +Old versions of git-annex have known bugs involving zombies. What version? +"""]] diff --git a/doc/bugs/copy_fails_for_some_fails_without_explanation/comment_5_1e0c06a07345d85b3712339e6f0d9a9f._comment b/doc/bugs/copy_fails_for_some_fails_without_explanation/comment_5_1e0c06a07345d85b3712339e6f0d9a9f._comment new file mode 100644 index 000000000..ad2b80d66 --- /dev/null +++ b/doc/bugs/copy_fails_for_some_fails_without_explanation/comment_5_1e0c06a07345d85b3712339e6f0d9a9f._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawmUJBh1lYmvfCCiGr3yrdx-QhuLCSRnU5c" + nickname="Justin" + subject="comment 5" + date="2014-03-06T18:35:00Z" + content=""" +5.20140221-g1a47f5f -- I just downloaded it a week or two ago. +"""]] diff --git a/doc/bugs/copy_fails_for_some_fails_without_explanation/comment_6_41798e92068eb227c5e75cae2edef68a._comment b/doc/bugs/copy_fails_for_some_fails_without_explanation/comment_6_41798e92068eb227c5e75cae2edef68a._comment new file mode 100644 index 000000000..a0554cd8a --- /dev/null +++ b/doc/bugs/copy_fails_for_some_fails_without_explanation/comment_6_41798e92068eb227c5e75cae2edef68a._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.146" + subject="comment 6" + date="2014-03-06T18:38:43Z" + content=""" +Hmm, that version should only start git cat-file --batch a maximum of 10 times (if it is crashing for some reason), and appears to wait on the process if it does crash. And if not, should only start one. + +I think you need to post some git-annex --debug output , to show when it's running this command. +"""]] diff --git a/doc/bugs/copy_fails_for_some_fails_without_explanation/comment_7_1f33d694a08d8dcbf04595e3442b8cd5._comment b/doc/bugs/copy_fails_for_some_fails_without_explanation/comment_7_1f33d694a08d8dcbf04595e3442b8cd5._comment new file mode 100644 index 000000000..f846e290d --- /dev/null +++ b/doc/bugs/copy_fails_for_some_fails_without_explanation/comment_7_1f33d694a08d8dcbf04595e3442b8cd5._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.146" + subject="comment 7" + date="2014-03-06T18:40:26Z" + content=""" +Actually, NM, I have reproduced the bug. +"""]] diff --git a/doc/bugs/copy_fails_for_some_fails_without_explanation/comment_8_884f31ce917c8e5ce9a32a55da9b42d6._comment b/doc/bugs/copy_fails_for_some_fails_without_explanation/comment_8_884f31ce917c8e5ce9a32a55da9b42d6._comment new file mode 100644 index 000000000..5ba7442dc --- /dev/null +++ b/doc/bugs/copy_fails_for_some_fails_without_explanation/comment_8_884f31ce917c8e5ce9a32a55da9b42d6._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.146" + subject="comment 8" + date="2014-03-06T20:21:16Z" + content=""" +Analysis: Remote.Git's onLocal calls Annex.new to make a new AnnexState for the local remote. This state is not cached, and is regenerated for each file. Since it runs a Annex.Branch check of the location log on the remote, it needs to start catFile, and since the state is not reused, a new CatFileHandle is allocated each time. I'm not sure, but there may have been a recent-ish change that caused the location log to get checked and so catfile to be run; the general inneficiency of making a new AnnexState each time is not new. + +Fixing this by caching the AnnexState will not only fix the resource leak, but should speed up local to local copies significantly! +"""]] diff --git a/doc/bugs/copy_fails_for_some_fails_without_explanation/comment_9_ab770dafee3bd9212f553db222adbfe6._comment b/doc/bugs/copy_fails_for_some_fails_without_explanation/comment_9_ab770dafee3bd9212f553db222adbfe6._comment new file mode 100644 index 000000000..e6a23eed5 --- /dev/null +++ b/doc/bugs/copy_fails_for_some_fails_without_explanation/comment_9_ab770dafee3bd9212f553db222adbfe6._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.146" + subject="comment 9" + date="2014-03-06T21:17:14Z" + content=""" +Fixed in git. Also reduced the non-data-transfer work done by `git-annex copy` by around 8%. + +I'm going to move this thread to [[bugs]] so I can close it. ;) +"""]] diff --git a/doc/bugs/copy_unused_and_unused_not_agreeing/comment_5_792ab128a91c66e4ddeaa69d09430a78._comment b/doc/bugs/copy_unused_and_unused_not_agreeing/comment_5_792ab128a91c66e4ddeaa69d09430a78._comment new file mode 100644 index 000000000..d5e56151d --- /dev/null +++ b/doc/bugs/copy_unused_and_unused_not_agreeing/comment_5_792ab128a91c66e4ddeaa69d09430a78._comment @@ -0,0 +1,20 @@ +[[!comment format=mdwn + username="http://grossmeier.net/" + nickname="greg" + subject="comment 5" + date="2014-03-05T04:34:49Z" + content=""" +Still seeing it with version 5.20140227 + +git-annex unused shows 110 keys. + +git-annex copy --unused --to $anyremote just returns very quickly without doing anything. + +git-annex copy --all --to $anyremote does go through every key and checks. + +git-annex copy --key=$some_key_listed_in_unused --to $anyremote also just returns quickly without checking. + +I even untrusted the repository via vicfg and passing the --untrust=remote argument. + +Again, a direct mode repo that is also assistant'd. +"""]] diff --git a/doc/bugs/copy_unused_and_unused_not_agreeing/comment_6_e44a16ef3358a6fbcc6ed6b3a31f3273._comment b/doc/bugs/copy_unused_and_unused_not_agreeing/comment_6_e44a16ef3358a6fbcc6ed6b3a31f3273._comment new file mode 100644 index 000000000..eae21f79d --- /dev/null +++ b/doc/bugs/copy_unused_and_unused_not_agreeing/comment_6_e44a16ef3358a6fbcc6ed6b3a31f3273._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://grossmeier.net/" + nickname="greg" + subject="comment 6" + date="2014-03-05T04:38:26Z" + content=""" +Also, in another non-direct/non-assistant'd repo, things work as expected. +"""]] diff --git a/doc/bugs/copy_unused_and_unused_not_agreeing/comment_7_635acd64b524c682c58f26ae96ae0d7d._comment b/doc/bugs/copy_unused_and_unused_not_agreeing/comment_7_635acd64b524c682c58f26ae96ae0d7d._comment new file mode 100644 index 000000000..52cfc575c --- /dev/null +++ b/doc/bugs/copy_unused_and_unused_not_agreeing/comment_7_635acd64b524c682c58f26ae96ae0d7d._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.146" + subject="comment 7" + date="2014-03-05T17:08:19Z" + content=""" +The only reason copy --key would do nothing is if the key is not locally present in your repository. + +Please check if the keys listed by `git annex unused` are present in `.git/annex/objects/` +"""]] diff --git a/doc/bugs/copy_unused_and_unused_not_agreeing/comment_8_1aaeb808e20c67f89eaac5e45d9309f0._comment b/doc/bugs/copy_unused_and_unused_not_agreeing/comment_8_1aaeb808e20c67f89eaac5e45d9309f0._comment new file mode 100644 index 000000000..3480ffa7e --- /dev/null +++ b/doc/bugs/copy_unused_and_unused_not_agreeing/comment_8_1aaeb808e20c67f89eaac5e45d9309f0._comment @@ -0,0 +1,15 @@ +[[!comment format=mdwn + username="http://grossmeier.net/" + nickname="greg" + subject="comment 8" + date="2014-03-06T18:54:08Z" + content=""" +The last key listed by unused (111 of 111): + +[[!format sh \"\"\" +greg@x200s:~/Documents/.git/annex/objects/q4/22/SHA256E-s12289--68a93144e03274664d50754882bdaf196134e06ec2b912157bdccae436d577d6.ods$ ls +SHA256E-s12289--68a93144e03274664d50754882bdaf196134e06ec2b912157bdccae436d577d6.ods.cache +SHA256E-s12289--68a93144e03274664d50754882bdaf196134e06ec2b912157bdccae436d577d6.ods.map +greg@x200s:~/Documents/.git/annex/objects/q4/22/SHA256E-s12289--68a93144e03274664d50754882bdaf196134e06ec2b912157bdccae436d577d6.ods$ +\"\"\"]] +"""]] diff --git a/doc/bugs/copy_unused_and_unused_not_agreeing/comment_9_6abca5f4927e09089cdc5f0bd27b798f._comment b/doc/bugs/copy_unused_and_unused_not_agreeing/comment_9_6abca5f4927e09089cdc5f0bd27b798f._comment new file mode 100644 index 000000000..25edfd5ed --- /dev/null +++ b/doc/bugs/copy_unused_and_unused_not_agreeing/comment_9_6abca5f4927e09089cdc5f0bd27b798f._comment @@ -0,0 +1,26 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.146" + subject="analysis" + date="2014-03-06T21:37:15Z" + content=""" +[[!format haskell \"\"\" + {- In indirect mode, look for the key. In direct mode, + - the inode cache file is only present when a key's content + - is present, so can be used as a surrogate if the content + - is not located in the annex directory. -} +\"\"\"]] + +Seems that is wrong. + +I think that comment was based on removeAnnex calling removeInodeCache, which it does do. +And that is, afaik, the only point in git-annex where content files are removed from the annex. + +However, in direct mode, removeAnnex is not the only way to delete a key's content -- the user can simply delete the file! +(Or a disk error could put it in lost+found, or whatever.) + +That leaves the inode cache file present. unused will then behave as you describe. Also, uninit throws an ugly warning message +due to getKeysPresent giving it bad data. The only other caller of getKeysPresent is info, which will also operate on bad data and so generate slightly wrong stats. + +I'm leaning toward making getKeysPresent do a full check of the cache and map, checking that the work tree still contains a key's content. This will make it somewhat slower (by 2 file reads and a stat() per key). So it would make sense to make a variant that only lists keys with content present in .git/annex/objects/. That could be used by `unused`, since by definition unused keys cannot have their content located in the work tree, so must have it in the object directory. uninit could also use it, since it's only interested in cleaning out .git/annex/objects. Only `info` will be slowed down. +"""]] diff --git a/doc/bugs/direct_mode_merge_can_overwrite_local__44___non-annexed_files.mdwn b/doc/bugs/direct_mode_merge_can_overwrite_local__44___non-annexed_files.mdwn new file mode 100644 index 000000000..081dd414b --- /dev/null +++ b/doc/bugs/direct_mode_merge_can_overwrite_local__44___non-annexed_files.mdwn @@ -0,0 +1,14 @@ +Direct mode merge handles the case where there's a conflict between local and remote files, that are checked into git. + +However, the the local file (or directory, or symlink, whatever) +is not checked into git, the merge will overwrite it with the remote file from git. + +> That's fixed; now this is detected and the local variant +> is renamed with ".variant-local", and possibly a number to make it +> unique. + +New problem: If the merge pulls in a directory, and a file exists with +the name of the directory, locally, not annexed, the file is left alone, +but the directory is thus not checked out, and will be deleted on commit. + +> [[fixed|done]]; regression test in place. --[[Joey]] diff --git a/doc/bugs/enableremote_broken_with_direct_mode__63__.mdwn b/doc/bugs/enableremote_broken_with_direct_mode__63__.mdwn new file mode 100644 index 000000000..a44cef431 --- /dev/null +++ b/doc/bugs/enableremote_broken_with_direct_mode__63__.mdwn @@ -0,0 +1,41 @@ +### Please describe the problem. +I have 2 regular annex locations plus one on glacier. I recently cloned one location, converted it to direct mode, then tried to enableremote glacier. + +When I ran the webapp everything seemed to be ok, it showed file transfers to glacier which appeared to be completing very quickly. When I looked in the log file however, I saw the files weren't actually transferring and I was getting lots of errors: + + Set both AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY to use glacier + +In order to get glacier working I had to manually copy the creds file to the new repo. The transfer errors should also made obvious in the webapp. + +### What steps will reproduce the problem? +[[!format sh """ +git clone server:/location +git annex init "laptop" +git remote add "server" server:/location +git annex direct + +Then I ran: +git annex enableremote glacier +Which prints: +(merging origin/git-annex origin/synced/git-annex into git-annex...) +(Recording state in git...) +enableremote glacier (gpg) ok +(Recording state in git...) +"""]] +But in fact it hasn't actually copied the creds to use it. + + +### What version of git-annex are you using? On what operating system? +5.20140221, debian. + +### Please provide any additional information below. + +[[!format sh """ +# If you can, paste a complete transcript of the problem occurring here. +# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log + + +# End of transcript or log. +"""]] + +> Should be fixed in 5.20140227. [[done]] --[[Joey]] diff --git a/doc/bugs/enableremote_broken_with_direct_mode__63__/comment_1_a2e61f5de7a28498de0c2d5e3d51eab4._comment b/doc/bugs/enableremote_broken_with_direct_mode__63__/comment_1_a2e61f5de7a28498de0c2d5e3d51eab4._comment new file mode 100644 index 000000000..84cf2d4bf --- /dev/null +++ b/doc/bugs/enableremote_broken_with_direct_mode__63__/comment_1_a2e61f5de7a28498de0c2d5e3d51eab4._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.146" + subject="comment 1" + date="2014-02-27T15:35:28Z" + content=""" +5.20140221 had a bug in passing the creds on when setting up remotes in the webapp. This is fixed in 5.20140227. +"""]] diff --git a/doc/bugs/git-annex_sucking_up_all_available_RAM_after_startup/comment_5_1a07f15eb0353768c1e67a1e47e2e494._comment b/doc/bugs/git-annex_sucking_up_all_available_RAM_after_startup/comment_5_1a07f15eb0353768c1e67a1e47e2e494._comment new file mode 100644 index 000000000..d94eda816 --- /dev/null +++ b/doc/bugs/git-annex_sucking_up_all_available_RAM_after_startup/comment_5_1a07f15eb0353768c1e67a1e47e2e494._comment @@ -0,0 +1,9 @@ +[[!comment format=mdwn + username="Hanno" + ip="85.183.3.94" + subject="comment 5" + date="2014-02-27T16:10:54Z" + content=""" +Is there anything I can do to help making progress on this issue? +Right now I'm afraid to reinstall git-annex on this machine. +"""]] diff --git a/doc/bugs/git_annex_sync_--content_not_syncing_all_objects/comment_3_d7349af488008e3ca6557e0c1fbfc5b6._comment b/doc/bugs/git_annex_sync_--content_not_syncing_all_objects/comment_3_d7349af488008e3ca6557e0c1fbfc5b6._comment new file mode 100644 index 000000000..34c2c4c16 --- /dev/null +++ b/doc/bugs/git_annex_sync_--content_not_syncing_all_objects/comment_3_d7349af488008e3ca6557e0c1fbfc5b6._comment @@ -0,0 +1,9 @@ +[[!comment format=mdwn + username="stp" + ip="84.56.21.11" + subject="Ídea" + date="2014-02-23T14:25:22Z" + content=""" +I thought about the implementation need for git annex sync --content --all. If preferred content expressions would work it would be needed. Everything else. could be done via a split usage. +Run \"git annex sync --content\" to satisfy the preferred content expressions on the working tree and the numcopies on the working tree and then loop through all backup/archive repositories with \"git annex get --auto\" this should at least prevent archives from getting objects numcopies is already satisfying and sync the objects not yet satisfied right? +"""]] diff --git a/doc/bugs/pages_of_packfile_errors.mdwn b/doc/bugs/pages_of_packfile_errors.mdwn new file mode 100644 index 000000000..9d60dd2aa --- /dev/null +++ b/doc/bugs/pages_of_packfile_errors.mdwn @@ -0,0 +1,30 @@ +### Please describe the problem. + +A repair that runs for ages. In the log file, pages and pages and pages of: + +error: packfile /Volumes/BandZbackup2/annex/.git/objects/pack/pack-f0ae2f5cc83f11eab406518b9f06a344acf9c93c.pack does not match index +warning: packfile /Volumes/BandZbackup2/annex/.git/objects/pack/pack-f0ae2f5cc83f11eab406518b9f06a344acf9c93c.pack cannot be accessed +error: packfile /Volumes/BandZbackup2/annex/.git/objects/pack/pack-f0ae2f5cc83f11eab406518b9f06a344acf9c93c.pack does not match index +warning: packfile /Volumes/BandZbackup2/annex/.git/objects/pack/pack-f0ae2f5cc83f11eab406518b9f06a344acf9c93c.pack cannot be accessed +error: packfile /Volumes/BandZbackup2/annex/.git/objects/pack/pack-f0ae2f5cc83f11eab406518b9f06a344acf9c93c.pack does not match index +warning: packfile /Volumes/BandZbackup2/annex/.git/objects/pack/pack-f0ae2f5cc83f11eab406518b9f06a344acf9c93c.pack cannot be accessed +error: packfile /Volumes/BandZbackup2/annex/.git/objects/pack/pack-f0ae2f5cc83f11eab406518b9f06a344acf9c93c.pack does not match index +warning: packfile /Volumes/BandZbackup2/annex/.git/objects/pack/pack-f0ae2f5cc83f11eab406518b9f06a344acf9c93c.pack cannot be accessed + +### What steps will reproduce the problem? + +Running git-annex, plugging in my external drive + +### What version of git-annex are you using? On what operating system? + +Auto-updated latest, I thought, but the about page says: Version: 5.20131230-g9a495e6 + +### Please provide any additional information below. + +[[!format sh """ +# If you can, paste a complete transcript of the problem occurring here. +# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log + + +# End of transcript or log. +"""]] diff --git a/doc/bugs/pages_of_packfile_errors/comment_1_eb2989112b38bb27ce8f691dd5d318e5._comment b/doc/bugs/pages_of_packfile_errors/comment_1_eb2989112b38bb27ce8f691dd5d318e5._comment new file mode 100644 index 000000000..d74470ffd --- /dev/null +++ b/doc/bugs/pages_of_packfile_errors/comment_1_eb2989112b38bb27ce8f691dd5d318e5._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.172" + subject="comment 1" + date="2014-02-24T18:32:31Z" + content=""" +Well, you seem to have a corrupt git repository on your removable drive. git-annex seems to be in the process of repairing it, which can take some time. + +I don't see a bug here, from what you've described so far.. +"""]] diff --git a/doc/bugs/pages_of_packfile_errors/comment_2_69fba53035ebea213ae1c11be5326690._comment b/doc/bugs/pages_of_packfile_errors/comment_2_69fba53035ebea213ae1c11be5326690._comment new file mode 100644 index 000000000..facae6496 --- /dev/null +++ b/doc/bugs/pages_of_packfile_errors/comment_2_69fba53035ebea213ae1c11be5326690._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkQafKy7hNSEolLs6TvbgUnkklTctUY9LI" + nickname="Zellyn" + subject="sounds good" + date="2014-02-24T19:39:12Z" + content=""" +Is it normal for the same error to repeat thousands of times like that in the log? +"""]] diff --git a/doc/bugs/pages_of_packfile_errors/comment_3_73b9f574e8ce36d5e0d0f6c6a89006b7._comment b/doc/bugs/pages_of_packfile_errors/comment_3_73b9f574e8ce36d5e0d0f6c6a89006b7._comment new file mode 100644 index 000000000..f0e6bce0b --- /dev/null +++ b/doc/bugs/pages_of_packfile_errors/comment_3_73b9f574e8ce36d5e0d0f6c6a89006b7._comment @@ -0,0 +1,39 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.172" + subject="comment 3" + date="2014-02-24T23:39:46Z" + content=""" +Well, if there's a bug here, it might be that this particular problem has caused the repair process to loop repeatedly trying to unpack a pack file. +I don't see how that could happen, looking at the code it will try to unpack each pack file only once. + +If you run `git annex repair --debug`, you can see the git commands it runs, and so see if it's somehow looping. When I do this with some corrupt pack files (actually, I swapped one pack file for another one), I see, for example: + +<pre> +[2014-02-24 19:11:42 JEST] feed: git [\"--git-dir=/home/joey/tmp/git/.git\",\"--work-tree=/home/joey/tmp/git\",\"unpack-objects\",\"-r\"] +error: packfile /home/joey/tmp/git/.git/objects/pack/pack-857c07e35d98e8f063fdae6846d1f6f7453e1312.pack claims to have 862 objects while index indicates 1431 objects +warning: packfile /home/joey/tmp/git/.git/objects/pack/pack-857c07e35d98e8f063fdae6846d1f6f7453e1312.pack cannot be accessed +error: packfile /home/joey/tmp/git/.git/objects/pack/pack-857c07e35d98e8f063fdae6846d1f6f7453e1312.pack claims to have 862 objects while index indicates 1431 objects +warning: packfile /home/joey/tmp/git/.git/objects/pack/pack-857c07e35d98e8f063fdae6846d1f6f7453e1312.pack cannot be accessed +error: packfile /home/joey/tmp/git/.git/objects/pack/pack-857c07e35d98e8f063fdae6846d1f6f7453e1312.pack claims to have 862 objects while index indicates 1431 objects +... +</pre> + +Which shows that git-annex only ran `git unpack-objects -r` once, and yet it printed out the same error repeatedly. + +One possibility is a problem using `-r`, which makes it keep going on errors. Which seemed like a good idea at the time to unpack as much as possible from a damaged file. It might be that `git unpack-objects` is itself getting stuck in some kind of loop with the -r. + +In my case, it did not get stuck; it eventually quit and it moved on to the next pack file, after 900-some repitions of the error message: + +<pre> +[2014-02-24 19:16:47 JEST] feed: git [\"--git-dir=/home/joey/tmp/git/.git\",\"--work-tree=/home/joey/tmp/git\",\"unpack-objects\",\"-r\"] +error: packfile /home/joey/tmp/git/.git/objects/pack/pack-857c07e35d98e8f063fdae6846d1f6f7453e1312.pack claims to have 862 objects while index indicates 1431 objects +warning: packfile /home/joey/tmp/git/.git/objects/pack/pack-857c07e35d98e8f063fdae6846d1f6f7453e1312.pack cannot be accessed +</pre> + +Intesting that it's again complaining about the same pack file, despite having moved from one pack file on to the next one. I think what's going on here is while unpacking pack files A..Y (which may all be fine), it's checking pack file Z, which is corrupt, to see if the objects exist in it, and complaining each time. + +So, I can improve this a lot by moving *all* the pack files out of the way before trying to unpack any of them. In my test case, that completely eliminated the errors, and probably also sped it up a bit. + +If I were you, I'd either try stopping your running git-annex and run `git annex repair --debug` and analize the log like I did above, or get the next daily build which has that change, and see if it helps in your case. +"""]] diff --git a/doc/bugs/problems_with_android_and_gpg/comment_2_2e1ae66bac4f55b74074b09e22ab270d._comment b/doc/bugs/problems_with_android_and_gpg/comment_2_2e1ae66bac4f55b74074b09e22ab270d._comment new file mode 100644 index 000000000..c7709f0fc --- /dev/null +++ b/doc/bugs/problems_with_android_and_gpg/comment_2_2e1ae66bac4f55b74074b09e22ab270d._comment @@ -0,0 +1,16 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawm78jq1Uo-ZbyOPG3diJUWVvEiM0kyAcvk" + nickname="Dorian" + subject="any ideas or questions?" + date="2014-02-25T13:41:16Z" + content=""" +Hey Joey, + +I was wondering if you had any idea how we could fix this problem or if you need further information on this. +Any response would be appreciated. + +Thanks for your great work on git-annex! + +Cheers, +Dorian +"""]] diff --git a/doc/bugs/problems_with_android_and_gpg/comment_3_47510400e8e45a71a1581aed99a157a1._comment b/doc/bugs/problems_with_android_and_gpg/comment_3_47510400e8e45a71a1581aed99a157a1._comment new file mode 100644 index 000000000..2f663d943 --- /dev/null +++ b/doc/bugs/problems_with_android_and_gpg/comment_3_47510400e8e45a71a1581aed99a157a1._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.146" + subject="comment 3" + date="2014-02-26T18:34:18Z" + content=""" +AFAICS, the messages about locking are a red herring, since shared encryption is being used, the public and secret key rings are not used. + +This problem seems to be explained here: <http://crypto.stackexchange.com/questions/1468/how-does-gpg-verify-succesful-decryption> + +It seems there must be a problem with the symmetric key used for shared encryption. Either the repository somehow has the wrong key in it, or the key is not extracted from the repository correctly somehow, or it's not fed into gpg correctly somehow. +"""]] diff --git a/doc/bugs/problems_with_android_and_gpg/comment_4_d28d773450d09e03160548d99f12256d._comment b/doc/bugs/problems_with_android_and_gpg/comment_4_d28d773450d09e03160548d99f12256d._comment new file mode 100644 index 000000000..8815f03c9 --- /dev/null +++ b/doc/bugs/problems_with_android_and_gpg/comment_4_d28d773450d09e03160548d99f12256d._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawm78jq1Uo-ZbyOPG3diJUWVvEiM0kyAcvk" + nickname="Dorian" + subject="comment 4" + date="2014-03-03T14:05:35Z" + content=""" +Is it correct, that the shared key is transfered through the git repo, meaning in this case XMPP? + +Then I guess this might simply be the problem of unreliable XMPP. +Same problem as here: <http://git-annex.branchable.com/bugs/problems_with_android_and_xmpp/> + +"""]] diff --git a/doc/bugs/problems_with_android_and_gpg/comment_5_74f1177d6dd42bab5ddfc040cbfb035e._comment b/doc/bugs/problems_with_android_and_gpg/comment_5_74f1177d6dd42bab5ddfc040cbfb035e._comment new file mode 100644 index 000000000..a9d24b97d --- /dev/null +++ b/doc/bugs/problems_with_android_and_gpg/comment_5_74f1177d6dd42bab5ddfc040cbfb035e._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.146" + subject="comment 5" + date="2014-03-05T20:49:20Z" + content=""" +Yes, the key is in the git repository. + +While XMPP is an unreliable transport, it either manages to git pull over it, or it fails, git does not allow partial or corrupt pulls. +"""]] diff --git a/doc/bugs/problems_with_android_and_xmpp/comment_2_ae4554fadfc3ea1913a36aa535815cfb._comment b/doc/bugs/problems_with_android_and_xmpp/comment_2_ae4554fadfc3ea1913a36aa535815cfb._comment new file mode 100644 index 000000000..48f945d85 --- /dev/null +++ b/doc/bugs/problems_with_android_and_xmpp/comment_2_ae4554fadfc3ea1913a36aa535815cfb._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.146" + subject="comment 2" + date="2014-02-26T18:52:17Z" + content=""" +I don't see any indications that this problem is specific to Android. + +Relying on git push over XMPP is known to be somewhat fragile, and some XMPP servers are known to do things with the XMPP protcol that make it impossible to reliably do git push over it. [[design/assistant/xmpp]] discusses these problems. Since this seems a basically intractable problem to solve, git-annex will be moving away from XMPP as soon as something better is available. (eg [[design/assistant/telehash]]) + +The best way to use XMPP currently is as a simple signaling mechanism to tell when a push has been made to a git repository. Your ssh server seems to have an encrypted rsync repository on it, so no git repository. If you can put up a git repository someplace both the android and linux machine can access, I think that you'll find it greatly improves the reliability of the syncing. +"""]] diff --git a/doc/bugs/problems_with_android_and_xmpp/comment_3_128702a7974bd00337c3304e49a74f0b._comment b/doc/bugs/problems_with_android_and_xmpp/comment_3_128702a7974bd00337c3304e49a74f0b._comment new file mode 100644 index 000000000..29839cd50 --- /dev/null +++ b/doc/bugs/problems_with_android_and_xmpp/comment_3_128702a7974bd00337c3304e49a74f0b._comment @@ -0,0 +1,23 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawm78jq1Uo-ZbyOPG3diJUWVvEiM0kyAcvk" + nickname="Dorian" + subject="comment 3" + date="2014-03-03T14:01:10Z" + content=""" +You're right, with an unencrypted git repo in between it works. +I was hoping to be able to avoid that, though. + +My very first try was actually to switch XMPP servers, because I thought they might be the problem. But after trying both servers you mentioned in the assistant and on the instruction pages (Google and jabber.me) I figured it should at least work with one of them... + +So if XMPP can actually only be used as a simple signaling mechanism, this should be mentioned explicitly in the \"remote sharing\" and \"share with a friend\" walkthroughs. +After reading/watching them I was under the impression, that after setting up XMPP, you can use any cloud storage to exchange files. +But if I now understand correctly this cloud storage needs to have a full git repo, not only the encrypted files. +And for Android you cannot use a gcrypt remote, so only unencrypted git repos are possible, making any public cloud service unfeasible. + +Is my conclusion correct, that this leaves only one option when sharing with Android? +1) unencrypted git repo on trusted/private special remote + +Or am I missing something? + +Thanks again for this! +"""]] diff --git a/doc/bugs/quvi_0.9.5_does_not_work_with_git-annex.mdwn b/doc/bugs/quvi_0.9.5_does_not_work_with_git-annex.mdwn new file mode 100644 index 000000000..7e760bb70 --- /dev/null +++ b/doc/bugs/quvi_0.9.5_does_not_work_with_git-annex.mdwn @@ -0,0 +1,87 @@ +### Please describe the problem. +The syntax of quvi has changed somewhat, breaking use in git-annex. + +quvi now requires one of four «commands» to be supplied: dump, get, info, scan + +### What steps will reproduce the problem? +Install quvi 0.9.5, attempt to download a video. + +### What version of git-annex are you using? On what operating system? +git-annex version: 5.20140227-gd872677 + +On ArchLinux up-to-date as of 28th of February 2014 + +### Please provide any additional information below. + +[[!format text """ +[0 zerodogg@browncoats Dokumentar]$ git annex addurl 'quvi:https://www.youtube.com/watch?v=de20gulo78g' --debug +[2014-02-28 09:33:22 CET] read: quvi ["--verbosity","quiet","-c","http","https://www.youtube.com/watch?v=de20gulo78g"] +error: `--verbosity' is not a quvi command. See 'quvi help'. +git-annex: quvi failed +[1 zerodogg@browncoats Dokumentar]$ quvi --version +quvi v0.9.5 + built on 2013-11-12 17:02:06 +0000 for x86_64-unknown-linux-gnu + with gcc, -march=x86-64 -mtune=generic -O2 -pipe -fstack-protector --param=ssp-buffer-size=4 + configuration: --prefix=/usr +libquvi v0.9.4 + built on 2013-12-17 11:27:41 +0000 for x86_64-unknown-linux-gnu + with gcc, -march=x86-64 -mtune=generic -O2 -pipe -fstack-protector --param=ssp-buffer-size=4 + configuration: --prefix=/usr +libquvi-scripts v0.9.20131130 + configuration: --prefix=/usr --with-nsfw --with-geoblocked + +Copyright (C) 2012,2013 Toni Gundogdu <legatvs@gmail.com> +quvi comes with ABSOLUTELY NO WARRANTY. You may redistribute copies of +quvi under the terms of the GNU Affero General Public License version 3 +or later. For more information, see <http://www.gnu.org/licenses/agpl.html>. + +To contact the developers, please mail to <quvi-devel@lists.sourceforge.net> +[0 zerodogg@browncoats Dokumentar]$ quvi +Usage: quvi [--version] [--help] COMMAND [ARGS] + +quvi commands are: + dump Query and print the property values + get Save media stream to a file + info Inspect the configuration and the script properties + scan Scan and print the found embedded media URLs + +See 'quvi help COMMAND' for more information on a specific command. +[0 zerodogg@browncoats Dokumentar]$ git annex version +git-annex version: 5.20140227-gd872677 +build flags: Assistant Webapp Pairing S3 Inotify DBus XMPP Feeds Quvi TDFA +key/value backends: SHA256E SHA1E SHA512E SHA224E SHA384E SHA256 SHA1 SHA512 SHA224 SHA384 WORM URL +remote types: git gcrypt S3 bup directory rsync web tahoe glacier hook external +local repository version: 5 +supported repository version: 5 +upgrade supported from repository versions: 0 1 2 4 +"""]] + +quvi dump is probably something you could use + +[[!format text """ +[0 zerodogg@browncoats Dokumentar]$ quvi dump 'https://www.youtube.com/watch?v=de20gulo78g' +QUVI_MEDIA_PROPERTY_THUMBNAIL_URL=https://i1.ytimg.com/vi/de20gulo78g/default.jpg +QUVI_MEDIA_PROPERTY_TITLE=[Linux.conf.au 2013] - Git-annex +QUVI_MEDIA_PROPERTY_ID=de20gulo78g +QUVI_MEDIA_PROPERTY_START_TIME_MS=0 +QUVI_MEDIA_PROPERTY_DURATION_MS=2328000 +QUVI_MEDIA_STREAM_PROPERTY_VIDEO_ENCODING=vp8.0 +QUVI_MEDIA_STREAM_PROPERTY_AUDIO_ENCODING=vorbis +QUVI_MEDIA_STREAM_PROPERTY_CONTAINER=webm +QUVI_MEDIA_STREAM_PROPERTY_URL=[Long googlevideo.com URL] +QUVI_MEDIA_STREAM_PROPERTY_ID=medium_webm_i43_360p +QUVI_MEDIA_STREAM_PROPERTY_VIDEO_BITRATE_KBIT_S=0 +QUVI_MEDIA_STREAM_PROPERTY_AUDIO_BITRATE_KBIT_S=0 +QUVI_MEDIA_STREAM_PROPERTY_VIDEO_HEIGHT=360 +QUVI_MEDIA_STREAM_PROPERTY_VIDEO_WIDTH=640 +"""]] + +It does however output some status messages to STDERR (which it removes later) that doesn't look to be possible to suppress. + +[[!format text """ +[0 zerodogg@browncoats Dokumentar]$ quvi dump 'https://www.youtube.com/watch?v=de20gulo78g' >/dev/null 2>stderr +[0 zerodogg@browncoats Dokumentar]$ cat -v stderr +status: o--- resolve <url> ... ^M ^Mstatus: -o-- fetch <url> ... ^M ^M% [0 zerodogg@browncoats Dokumentar]$ +""" ]] + +> quvi version now probed at runtime. [[done]] --[[Joey]] diff --git a/doc/bugs/quvi_0.9.5_does_not_work_with_git-annex/comment_1_d0d2bcd97ef5c9bce8a57c4184a176e0._comment b/doc/bugs/quvi_0.9.5_does_not_work_with_git-annex/comment_1_d0d2bcd97ef5c9bce8a57c4184a176e0._comment new file mode 100644 index 000000000..9f8650e95 --- /dev/null +++ b/doc/bugs/quvi_0.9.5_does_not_work_with_git-annex/comment_1_d0d2bcd97ef5c9bce8a57c4184a176e0._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.146" + subject="comment 1" + date="2014-02-28T17:52:07Z" + content=""" +git-annex supports quvi 0.9.4 since 5.20131127. Unless the syntax has changed in some breaking way between 0.9.4 and 0.9.5, it seems most likely to me that your system is using a git-annex compiled with support for the old quvi, and it just needs to be recompiled to use the new one. + +I could do run-time quvi version detection, but there's an obvious performance penalty. +"""]] diff --git a/doc/bugs/quvi_0.9.5_does_not_work_with_git-annex/comment_2_ff9661198257b8c5e2e8ca3d85a7471c._comment b/doc/bugs/quvi_0.9.5_does_not_work_with_git-annex/comment_2_ff9661198257b8c5e2e8ca3d85a7471c._comment new file mode 100644 index 000000000..f05b9b069 --- /dev/null +++ b/doc/bugs/quvi_0.9.5_does_not_work_with_git-annex/comment_2_ff9661198257b8c5e2e8ca3d85a7471c._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="EskildHustvedt" + ip="80.202.213.223" + subject="comment 2" + date="2014-02-28T17:56:42Z" + content=""" +That explains it. I'm using the standalone build, which I suppose then is built with the old quvi. I guess I could write a shell script wrapper that converts the old syntax to the new. +"""]] diff --git a/doc/bugs/rsync_transport:_username_not_respected.mdwn b/doc/bugs/rsync_transport:_username_not_respected.mdwn index f4db3b70c..467e67643 100644 --- a/doc/bugs/rsync_transport:_username_not_respected.mdwn +++ b/doc/bugs/rsync_transport:_username_not_respected.mdwn @@ -36,3 +36,8 @@ Type: prebuilt # End of transcript or log. """]] + +> Argh! How did that break? I know it used to work. +> I have fixed it, unfortunately the fix was too late for today's release, +> but it will be available in autobuilds shortly. +> [[fixed|done]] --[[Joey]] diff --git a/doc/bugs/variant-_files_are_created_even_though_the_content_has_the_same_hash.mdwn b/doc/bugs/variant-_files_are_created_even_though_the_content_has_the_same_hash.mdwn new file mode 100644 index 000000000..8bf3c3d20 --- /dev/null +++ b/doc/bugs/variant-_files_are_created_even_though_the_content_has_the_same_hash.mdwn @@ -0,0 +1,11 @@ +After (probably, doesn't hepl anymore) adding the same file into multiple clones and syncing them back, I get a couple of these `.variant-` files. The target of these symlinks is different, but the actual file contents of the real data is the same. + +[[!format txt """ +jkt@svist /mnt/storage5/fotky[master] $ ls -al 2011-02-05\ Svatba\ Maruška\ Pavel/jkt/cam1/IMG_8962*variant* +lrwxrwxrwx 1 jkt jkt 335 Mar 3 02:08 2011-02-05 Svatba Maruška Pavel/jkt/cam1/IMG_8962.JPG.variant-847f.cam -> +../../../.git/annex/objects/P4/F7/SHA512E-s5534099--bf29fad838adb755b98351477e9a57c456b66ca393652f3b5df2093fc5448b3eeb0235420a715bfb84fb0cccfbbcea4514d4558467b64a39dc7daaf43c045672.cam/SHA512E-s5534099--bf29fad838adb755b98351477e9a57c456b66ca393652f3b5df2093fc5448b3eeb0235420a715bfb84fb0cccfbbcea4514d4558467b64a39dc7daaf43c045672.cam +lrwxrwxrwx 1 jkt jkt 343 Mar 3 02:08 2011-02-05 Svatba Maruška Pavel/jkt/cam1/IMG_8962.JPG.variant-c8d0.cam -> +../../../.git/annex/objects/38/pG/SHA512E-s5534099--bf29fad838adb755b98351477e9a57c456b66ca393652f3b5df2093fc5448b3eeb0235420a715bfb84fb0cccfbbcea4514d4558467b64a39dc7daaf43c045672.JPG.cam/SHA512E-s5534099--bf29fad838adb755b98351477e9a57c456b66ca393652f3b5df2093fc5448b3eeb0235420a715bfb84fb0cccfbbcea4514d4558467b64a39dc7daaf43c045672.JPG.cam +"""]] + +I don't know what determines the `P4/F7/` vs `38/pG/` directory names, but I would prefer if these duplicates were not created. diff --git a/doc/bugs/variant-_files_are_created_even_though_the_content_has_the_same_hash/comment_1_ffc5f79368b8927817e0e35a7a8f057b._comment b/doc/bugs/variant-_files_are_created_even_though_the_content_has_the_same_hash/comment_1_ffc5f79368b8927817e0e35a7a8f057b._comment new file mode 100644 index 000000000..3d3531fce --- /dev/null +++ b/doc/bugs/variant-_files_are_created_even_though_the_content_has_the_same_hash/comment_1_ffc5f79368b8927817e0e35a7a8f057b._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.146" + subject="comment 1" + date="2014-03-03T21:58:20Z" + content=""" +You have 2 different keys, so get variant files. This is expected behavior. + +You're right that the 2 files have the same content. But two different keys were generated for them when they were added in their respective repositories. This can happen when you use the SHA512E (or SHA256E) backend, because it uses the extension as part of the key. One of the files you added had an extension of \".cam\", and the other \".JPG.cam\" + +If there is any bug here, it's perhaps that the \"E\" backends are smart about multi-component file extensions, so will treat eg .tar.gz as an extension, rather than just .gz. If this was not done, both files would have had a .cam extension, and the conflict would not have occurred. But while being less smart would have avoided it in this case, it won't in other cases where files can have different extensions but the same content (for example, \".jpeg\" vs \".jpg\"). +"""]] diff --git a/doc/bugs/variant-_files_are_created_even_though_the_content_has_the_same_hash/comment_2_1382b486d198d707db760ae119f33ad1._comment b/doc/bugs/variant-_files_are_created_even_though_the_content_has_the_same_hash/comment_2_1382b486d198d707db760ae119f33ad1._comment new file mode 100644 index 000000000..1ee533347 --- /dev/null +++ b/doc/bugs/variant-_files_are_created_even_though_the_content_has_the_same_hash/comment_2_1382b486d198d707db760ae119f33ad1._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawlmOvQXLmGkQAUOIkPBQtsd9d45NqcX80M" + nickname="Jan" + subject="comment 2" + date="2014-03-04T11:01:12Z" + content=""" +I indeed use the SHA512E backend, but the files got the same name -- `IMG_8962.JPG.cam`. +"""]] diff --git a/doc/bugs/variant-_files_are_created_even_though_the_content_has_the_same_hash/comment_3_3c4074a1d4d7f63f6c07a05ca9717ce8._comment b/doc/bugs/variant-_files_are_created_even_though_the_content_has_the_same_hash/comment_3_3c4074a1d4d7f63f6c07a05ca9717ce8._comment new file mode 100644 index 000000000..0f20d9164 --- /dev/null +++ b/doc/bugs/variant-_files_are_created_even_though_the_content_has_the_same_hash/comment_3_3c4074a1d4d7f63f6c07a05ca9717ce8._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.146" + subject="comment 3" + date="2014-03-05T16:56:14Z" + content=""" +Older versions of git-annex did not preserve more than the very end of the extension when using SHA*E backends, so I guess you added the respective files using two different versions. +"""]] diff --git a/doc/bugs/variant-_files_are_created_even_though_the_content_has_the_same_hash/comment_4_deb848e50f6767d8a5f4348137744ec2._comment b/doc/bugs/variant-_files_are_created_even_though_the_content_has_the_same_hash/comment_4_deb848e50f6767d8a5f4348137744ec2._comment new file mode 100644 index 000000000..e53e5c2dd --- /dev/null +++ b/doc/bugs/variant-_files_are_created_even_though_the_content_has_the_same_hash/comment_4_deb848e50f6767d8a5f4348137744ec2._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawlmOvQXLmGkQAUOIkPBQtsd9d45NqcX80M" + nickname="Jan" + subject="comment 4" + date="2014-03-05T17:07:05Z" + content=""" +Ah, so it's a matter of `.JPG.cam` vs `.cam` as an extension -- that makes sense, I do use different versions on these two remotes. Is there some easy way of getting them back to a single underlying content apart from doing that by hand? +"""]] diff --git a/doc/bugs/variant-_files_are_created_even_though_the_content_has_the_same_hash/comment_5_b426c4569b7a788fbf963f787590e051._comment b/doc/bugs/variant-_files_are_created_even_though_the_content_has_the_same_hash/comment_5_b426c4569b7a788fbf963f787590e051._comment new file mode 100644 index 000000000..c50d75476 --- /dev/null +++ b/doc/bugs/variant-_files_are_created_even_though_the_content_has_the_same_hash/comment_5_b426c4569b7a788fbf963f787590e051._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.146" + subject="comment 5" + date="2014-03-05T18:02:08Z" + content=""" +Well, the easy way, assuming that you don't need the extension in the key at all, is to run `git annex migrate --backend=SHA512 *.cam` in both repositories. + +You can also play tricks with `git annex rekey`. I don't know if I'd recommend doing so. +"""]] diff --git a/doc/bugs/whereis_claims_file_is_not_here__44___but_it_is_available_both_here_and_in_another_remote.mdwn b/doc/bugs/whereis_claims_file_is_not_here__44___but_it_is_available_both_here_and_in_another_remote.mdwn new file mode 100644 index 000000000..ee69014d2 --- /dev/null +++ b/doc/bugs/whereis_claims_file_is_not_here__44___but_it_is_available_both_here_and_in_another_remote.mdwn @@ -0,0 +1,28 @@ +`git annex whereis` claims that I have zero copies of two particular files. However, it is wrong; both files are present: + +[[!format txt """ +[jkt@vorvan fotky]$ git annex whereis '2011-08-13 Svatba Anička Fellnerová a výlet s Julií/IMG_4301'* +(merging synced/git-annex into git-annex...) +whereis 2011-08-13 Svatba Anička Fellnerová a výlet s Julií/IMG_4301.CR2 (0 copies) failed +whereis 2011-08-13 Svatba Anička Fellnerová a výlet s Julií/IMG_4301.JPG (0 copies) failed +git-annex: whereis: 2 failed +[jkt@vorvan fotky]$ ll 2011-08-13\ Svatba\ Anička\ Fellnerová\ a\ výlet\ s\ Julií/IMG_4301* +lrwxrwxrwx. 1 jkt jkt 331 Aug 13 2011 2011-08-13 Svatba Anička Fellnerová a výlet s Julií/IMG_4301.CR2 -> ../.git/annex/objects/mg/qq/SHA512E-s19424303--9bdd6917fe85a450b50233a444c1814d407d41b550b66661272728aaf139576cc7c09de158f47a33ff263d7d25253cec193d0e44ed5fb4a1581c28a9a05e75e1.CR2/SHA512E-s19424303--9bdd6917fe85a450b50233a444c1814d407d41b550b66661272728aaf139576cc7c09de158f47a33ff263d7d25253cec193d0e44ed5fb4a1581c28a9a05e75e1.CR2 +lrwxrwxrwx. 1 jkt jkt 329 Aug 13 2011 2011-08-13 Svatba Anička Fellnerová a výlet s Julií/IMG_4301.JPG -> ../.git/annex/objects/zG/xq/SHA512E-s4744850--da9b1e19f0ecdf08d2f5564232a9c7d4b5ec9fbc9c67033121e5c90468e09e5573ddac42469cfd51fb0d8328dff925830e2d753022365df6d92b115b77831228.JPG/SHA512E-s4744850--da9b1e19f0ecdf08d2f5564232a9c7d4b5ec9fbc9c67033121e5c90468e09e5573ddac42469cfd51fb0d8328dff925830e2d753022365df6d92b115b77831228.JPG +"""]] + +Looking at another repo, the situation is the same: + +[[!format txt """ +jkt@svist /mnt/storage5/fotky[master] $ git annex whereis '2011-08-13 Svatba Anička Fellnerová a výlet s Julií/IMG_4301'* +whereis 2011-08-13 Svatba Anička Fellnerová a výlet s Julií/IMG_4301.CR2 (0 copies) failed +whereis 2011-08-13 Svatba Anička Fellnerová a výlet s Julií/IMG_4301.JPG (0 copies) failed +git-annex: whereis: 2 failed +jkt@svist /mnt/storage5/fotky[master] $ ll '2011-08-13 Svatba Anička Fellnerová a výlet s Julií/IMG_4301'* +lrwxrwxrwx 1 jkt jkt 331 Mar 3 02:08 2011-08-13 Svatba Anička Fellnerová a výlet s Julií/IMG_4301.CR2 -> ../.git/annex/objects/mg/qq/SHA512E-s19424303--9bdd6917fe85a450b50233a444c1814d407d41b550b66661272728aaf139576cc7c09de158f47a33ff263d7d25253cec193d0e44ed5fb4a1581c28a9a05e75e1.CR2/SHA512E-s19424303--9bdd6917fe85a450b50233a444c1814d407d41b550b66661272728aaf139576cc7c09de158f47a33ff263d7d25253cec193d0e44ed5fb4a1581c28a9a05e75e1.CR2 +lrwxrwxrwx 1 jkt jkt 329 Mar 3 02:08 2011-08-13 Svatba Anička Fellnerová a výlet s Julií/IMG_4301.JPG -> ../.git/annex/objects/zG/xq/SHA512E-s4744850--da9b1e19f0ecdf08d2f5564232a9c7d4b5ec9fbc9c67033121e5c90468e09e5573ddac42469cfd51fb0d8328dff925830e2d753022365df6d92b115b77831228.JPG/SHA512E-s4744850--da9b1e19f0ecdf08d2f5564232a9c7d4b5ec9fbc9c67033121e5c90468e09e5573ddac42469cfd51fb0d8328dff925830e2d753022365df6d92b115b77831228.JPG +"""]] + +The directory names are valid UTF-8. These are very common on my machine and there is a ton of directories with these funny names here -- all working without any real trouble. + +As far as I know, the file which the links point to is absolutely correct and not corrupted. Looking at the files in the directory chronologically, it also appears that the symlinks point to a correct file. diff --git a/doc/bugs/whereis_claims_file_is_not_here__44___but_it_is_available_both_here_and_in_another_remote/comment_1_d823b7ee32183fbadd4a49f65e1a3a8b._comment b/doc/bugs/whereis_claims_file_is_not_here__44___but_it_is_available_both_here_and_in_another_remote/comment_1_d823b7ee32183fbadd4a49f65e1a3a8b._comment new file mode 100644 index 000000000..11c55bb5b --- /dev/null +++ b/doc/bugs/whereis_claims_file_is_not_here__44___but_it_is_available_both_here_and_in_another_remote/comment_1_d823b7ee32183fbadd4a49f65e1a3a8b._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.146" + subject="comment 1" + date="2014-03-05T03:14:09Z" + content=""" +Have you tried running `git annex fsck`? +"""]] diff --git a/doc/design/metadata.mdwn b/doc/design/metadata.mdwn index db0d51c5c..10e79b9f8 100644 --- a/doc/design/metadata.mdwn +++ b/doc/design/metadata.mdwn @@ -18,59 +18,49 @@ See [[tips/metadata_driven_views]] The reason to use specially named filtered branches is because it makes self-documenting how the repository is currently filtered. -TODO: Files not matching the view should be able to be included. -For example, it could make a "unsorted" directory containing files -without a tag when viewing by tag. If also viewing by author, the unsorted -directories nest. +## unmatched files in filtered branches + +TODO Files not matching the view should be able to be included in +the filtered branch, in a special location, an "other" directory. + +For example, it could make a "other" directory containing files +without a tag when viewing by tag. + +It might be nice, if in a two level view, for the other directories +to nest. For example, `other/2014/file`. However, that leads to a +performance problem: When adding a level to a view, it has to look at each +file in the "other" directory and generate a view for it too. With a lot +of files, that'd be slow. + +Instead, why not replicate the parent branch's directory structure inside +the "other" directory? Then the directory tree only has to be constructed +once, and can be left alone when refining a view. ## operations while on filtered branch * If files are removed and git commit called, git-annex should remove the - relevant metadata from the files. - TODO: It's not clear that - removing a file should nuke all the metadata used to filter it into the - branch (especially if it's derived metadata like the year). - Currently, only metadata used for visible subdirs is added and removed - this way. - Also, this is not usable in direct mode because deleting the - file.. actually deletes it. + relevant metadata from the files. **done** + (Currently, only metadata used for visible subdirs is added and removed + this way.) + (Also, this is not usable in direct mode because deleting the + file.. actually deletes it...) * If a file is moved into a new subdirectory while in a view branch, a tag is added with the subdir name. This allows on the fly tagging. **done** * `git annex sync` should avoid pushing out the view branch, but it should check if there are changes to the metadata pulled in, and update the branch to reflect them. -* TODO: If `git annex add` adds a file, it gets all the metadata of the filter - branch it's added to. If it's in a relevent directory (like fosdem-2014), - it gets that metadata automatically recorded as well. ## automatically added metadata -TODO git annex add should automatically attach the current mtime of a file -when adding it. - -Could also automatically attach permissions. - -TODO A git hook could be run by git annex add to gather more metadata. -For example, by examining MP3 metadata. - -Also auto add metadata when adding files to view branches. See below. +When annex.genmetadata is set, git annex add automatically attaches +some metadata to a file. Currently year and month fields, from its mtime. -## derived metadata +There's also a post-commit-annex hook script. -This is probably not stored anywhere. It's computed on demand by a pure -function from the other metadata. -(Should be a general mechanism for this. (It probably generalizes to -sql queries if we want to go that far.)) +## directory hierarchy metadata -### data metadata - -TODO From the ctime, some additional -metadata is derived, at least year=yyyy and probably also month, etc. - -### directory hierarchy metadata - -TODO From the original filename used in the master branch, when +From the original filename used in the master branch, when constructing a view, generate fields. For example foo/bar/baz.mp3 would get /=foo, foo/=bar, foo/bar/=baz, and .=mp3. @@ -82,11 +72,10 @@ This allows using whatever directory hierarchy exists to inform the view, without locking the view into using it. Complication: When refining a view, it only looks at the filenames in -the view, so it would need to map from +the view, so it has to map from those filenames to derive the same metadata, unless there is persistent storage. Luckily, the filenames used in the views currently include the -subdirs (although not quite in a parseable format, would need some small -changes). +subdirs. # other uses for metadata @@ -185,21 +174,15 @@ So, possible approaches: * Git has a complex set of rules for what is legal in a ref name. View branch names will need to filter out any illegal stuff. **done** +* Metadata should be copied to the new key when adding a modified version + of a file. **done** + * Filesystems that are not case sensative (including case preserving OSX) will cause problems if view branches try to use different cases for - 2 directories representing the value of some metadata. But, users - probably want at least case-preserving metadata values. + 2 directories representing a metadata field. - Solution might be to compare metadata case-insensitively, and - pick one representation consistently, so if, for example an author - field uses mixed case, it will be used in the view branch. - - Alternatively, it could escape `A` to `_A` when such a filesystem - is detected and avoid collisions that way (double `_` to escape it). - This latter option is ugly, but so are non-posix filesystems.. and it - also solves any similar issues with case-colliding filenames. - - TODO: Check current state of this. + Solution might be to compare fields names case-insensitively, and + pick one representation consistently. **done** * Assistant needs to know about views, so it can update metadata when files are moved around inside them. TODO @@ -207,3 +190,7 @@ So, possible approaches: * What happens if git annex add or the assistant add a new file while on a view? If the file is not also added to the master branch, it will be lost when exiting the view. TODO + +* The filename mangling can result in a filename in a view + that is too long for its containing filesystem. Should detect and do + something reasonable to avoid. TODO diff --git a/doc/design/metadata/comment_1_22ed80bd8eabaa836e9dfc2432531f04._comment b/doc/design/metadata/comment_1_22ed80bd8eabaa836e9dfc2432531f04._comment new file mode 100644 index 000000000..493db4339 --- /dev/null +++ b/doc/design/metadata/comment_1_22ed80bd8eabaa836e9dfc2432531f04._comment @@ -0,0 +1,22 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawm3vKzS4eOWYpKMoYXqMIjNsIg_nYF-loU" + nickname="Konubinix" + subject="Already existing metadata implementation " + date="2014-02-22T21:45:25Z" + content=""" +Hi, + +I love the idea behing storing metadata. + +I suggest to exchange ideas (and maybe code) with projects already implementing metadata systems. + +I have tried several implementations and particularly noticed tmsu (http://tmsu.org/). This tool stores tags into a sqlite database and uses also a SHA-256 fingerprint of the file to be aware of file moves. It provides a fuse view of the tags with the ability to change tags by moving files (like in the git annex metadata view). + +Paul Ruane is particularly responsive on the mailing list and he already supports git annexed files (with SHAE-256 fingerprint) (see the end of the thread https://groups.google.com/forum/#!topic/tmsu/A5EGpnCcJ2w). + +Even if you cannot reuse the project, they are interresting ideas that might be worth looking at like the implications of tags: a file tagged \"film\" being automatically tagged \"video\". + +Tagsistant (http://www.tagsistant.net/) may also be a good source of inspirations. I just don't like the fact that it uses a backstore of tagged files. + +Thanks for reading. +"""]] diff --git a/doc/design/metadata/comment_2_03ae28acedbe1fa45c366b30b58fcf48._comment b/doc/design/metadata/comment_2_03ae28acedbe1fa45c366b30b58fcf48._comment new file mode 100644 index 000000000..c222f75d3 --- /dev/null +++ b/doc/design/metadata/comment_2_03ae28acedbe1fa45c366b30b58fcf48._comment @@ -0,0 +1,14 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" + nickname="Jimmy" + subject="comment 2" + date="2014-02-25T09:51:17Z" + content=""" +Some additional ideas for metadata... + +Instead of having a simplistic scheme like 'field=value' it might be advantageous to consider a scheme like 'attribute=XXX, value=YYY, unit=ZZZ' that way you could do intesting things with the metadata like adding counters to things, and allow for doing interesting queries like give me all 'things' tagged with a unit of \"audio_file\", this assumes one had trawled through an entire annex and then tagged all files based on type with the unix file tool or something like that. + +The above idea is already in use in irods and its a really nice and powerful way to let users add meta-data and to build up more interesting use cases and tools. + +btw, I plan on taking a look at seeing if I can map some of the meta that we have in work into this new git-annex feature to see how well/bad it works. Either way this feature looks cool! +1!!! +"""]] diff --git a/doc/design/metadata/comment_3_ee850df7d3fa4c56194f13a6e3890a30._comment b/doc/design/metadata/comment_3_ee850df7d3fa4c56194f13a6e3890a30._comment new file mode 100644 index 000000000..f77cd8611 --- /dev/null +++ b/doc/design/metadata/comment_3_ee850df7d3fa4c56194f13a6e3890a30._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" + nickname="Jimmy" + subject="comment 3" + date="2014-02-25T09:57:09Z" + content=""" +actually in your mp3 example you could have .... + +ATTRIBUTE=sample_rate, VALUE=22100, UNIT=Hertz + +another example use case is to always be consistent with the AVU order then you could stick in ntriples from RDF to do other cool things by looking up various linked data sources -- see http://www.w3.org/2001/sw/RDFCore/ntriples/ and http://www.freebase.com/, actually this would be quite cool if git-annex examined the mp3's id3 tag, the created an ntriple styled entry can be automatically parsed with the web-based annex gui and automatically pull in additional meta-data from the likes of freebase. I guess the list of ideas can just only get bigger with this potential metadata capability. +"""]] diff --git a/doc/design/roadmap.mdwn b/doc/design/roadmap.mdwn index e6ad21fee..7458468f0 100644 --- a/doc/design/roadmap.mdwn +++ b/doc/design/roadmap.mdwn @@ -6,11 +6,11 @@ Now in the * Month 1 [[!traillink assistant/encrypted_git_remotes]] * Month 2 [[!traillink assistant/disaster_recovery]] -* Month 3 user-driven features and polishing [[todo/direct_mode_guard]] [[assistant/upgrading]] -* Month 4 [[Windows_webapp|assistant/Windows]], Linux arm, [[todo/support_for_writing_external_special_remotes]] +* Month 3 user-driven features and polishing [[!traillink todo/direct_mode_guard]] [[!traillink assistant/upgrading]] +* Month 4 [[!traillink assistant/windows text="Windows webapp"]], Linux arm, [[!traillink todo/support_for_writing_external_special_remotes]] * Month 5 user-driven features and polishing -* **Month 6 get Windows out of beta, [[metadata and views|design/metadata]]** -* Month 7 user-driven features and polishing +* Month 6 get Windows out of beta, [[!traillink design/metadata text="metadata and views"]] +* **Month 7 user-driven features and polishing** * Month 8 [[!traillink assistant/telehash]] * Month 9 [[!traillink assistant/gpgkeys]] [[!traillink assistant/sshpassword]] * Month 10 get [[assistant/Android]] out of beta diff --git a/doc/devblog/day_-4__forgetting/comment_7_a865216046aa91a47d0d2b2f0668ea89._comment b/doc/devblog/day_-4__forgetting/comment_7_a865216046aa91a47d0d2b2f0668ea89._comment new file mode 100644 index 000000000..dc142edc0 --- /dev/null +++ b/doc/devblog/day_-4__forgetting/comment_7_a865216046aa91a47d0d2b2f0668ea89._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="stp" + ip="84.56.21.11" + subject="New findings" + date="2014-02-24T12:28:03Z" + content=""" +Another thing I found, which was annoying is that I have objects in my annex not tracked anywhere it seems. +\"git annex fsck --all\" complains about not having access to the object. \"git log --stat -S '$key'\" doesn't have any record. \"git annex fsck\" has no issues and \"git annex unused\" comes up empty too. +I'm not sure where these objects still reside or why how to remove this annoying failure. + +So not only should \"git annex forget $key\" remove references from within all branches, but should also clean up the aforementioned loose objects, which are neither unused, nor available, nor referenced. +"""]] diff --git a/doc/devblog/day_108__new_use_for_location_tracking.mdwn b/doc/devblog/day_108__new_use_for_location_tracking.mdwn index cf0d3e096..62e40672f 100644 --- a/doc/devblog/day_108__new_use_for_location_tracking.mdwn +++ b/doc/devblog/day_108__new_use_for_location_tracking.mdwn @@ -4,7 +4,7 @@ option can now be told to match files that were in a repository at some point in the past. For example, `git annex get --in=here@{yesterday}` will get any files that have been dropped over the past day. -While git-annex's location tracking info is stored in git and so versioned, +While git-annex's location tracking info is stored in git and thus versioned, very little of it makes use of past versions of the location tracking info (only `git annex log`). I'm happy to have finally found a use for it! diff --git a/doc/devblog/day_119__catching_up/comment_2_db31d08690730836ce6277e797fcae1d._comment b/doc/devblog/day_119__catching_up/comment_2_db31d08690730836ce6277e797fcae1d._comment new file mode 100644 index 000000000..532b7a4d5 --- /dev/null +++ b/doc/devblog/day_119__catching_up/comment_2_db31d08690730836ce6277e797fcae1d._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawneiQ3iR9VXOPEP34u7m_L3Qr28H1nEfE0" + nickname="Ethan" + subject="LFS" + date="2014-02-21T00:03:59Z" + content=""" +You might be interested in the Logic File System at http://www.padator.org/wiki/wiki-LFS/doku.php or http://www.padator.org/papers/ which has a similar idea with views and metadata. +"""]] diff --git a/doc/devblog/day_119__catching_up/comment_3_d44da76b18f53a5f51b46e3e15090a48._comment b/doc/devblog/day_119__catching_up/comment_3_d44da76b18f53a5f51b46e3e15090a48._comment new file mode 100644 index 000000000..9d97aff2d --- /dev/null +++ b/doc/devblog/day_119__catching_up/comment_3_d44da76b18f53a5f51b46e3e15090a48._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawmZgZuUhZlHpd_AbbcixY0QQiutb2I7GWY" + nickname="Jimmy" + subject="comment 3" + date="2014-02-21T07:05:34Z" + content=""" +I agree with Kevin as to the potential usefulness for photos. Particularly if there's some way of automatically extracting and using tags or other EXIF metadata. +"""]] diff --git a/doc/devblog/day_120__more_metadata.mdwn b/doc/devblog/day_120__more_metadata.mdwn new file mode 100644 index 000000000..daff68e37 --- /dev/null +++ b/doc/devblog/day_120__more_metadata.mdwn @@ -0,0 +1,17 @@ +When generating a view, there's now a way to reuse part of the directory +hierarchy of the parent branch. For example, `git annex view tag=* podcasts/=*` +makes a view where the first level is the tags, and the second level is +whatever `podcasts/*` directories the files were in. + +Also, year and month metadata can be automatically recorded when +adding files to the annex. I made this only be done when annex.genmetadata +is turned on, to avoid polluting repositories that don't want to use metadata. + +It would be nice if there was a way to add a hook script that's run +when files are added, to collect their metadata. I am not sure yet if +I am going to add that to git-annex though. It's already possible to do via +the regular git `post-commit` hook. Just make it look at the commit to see +what files were added, and then run `git annex metadata` to set their +metadata appropriately. It would be good to at least have an example of +such a script to eg, extract EXIF or ID3 metadata. Perhaps someone can +contribute one? diff --git a/doc/devblog/day_121__special_remote_maintenance.mdwn b/doc/devblog/day_121__special_remote_maintenance.mdwn new file mode 100644 index 000000000..551704885 --- /dev/null +++ b/doc/devblog/day_121__special_remote_maintenance.mdwn @@ -0,0 +1,23 @@ +Turns out that in the last release I broke making box.com, Amazon S3 and +Glacier remotes from the webapp. Fixed that. + +Also, dealt with changes in the haskell DAV library that broke support for +box.com, and worked around an exception handling bug in the library. + +I think I should try to enhance the test suite so it can run live tests +on special remotes, which would at least have caught the some of these +recent problems... + +---- + +Since metadata is tied to a particular key, editing an annexed file, +which causes the key to change, made the metadata seem to get lost. + +I've now fixed this; it copies the metadata from the old version to the new +one. (Taking care to copy the log file identically, so git can reuse its +blob.) + +That meant that `git annex add` has to check every file it adds to see if +there's an old version. Happily, that check is fairly fast; I benchmarked my +laptop running 2500 such checks a second. So it's not going to slow things +down appreciably. diff --git a/doc/devblog/day_122_more_windows_porting.mdwn b/doc/devblog/day_122_more_windows_porting.mdwn new file mode 100644 index 000000000..7b496f670 --- /dev/null +++ b/doc/devblog/day_122_more_windows_porting.mdwn @@ -0,0 +1,4 @@ +More Windows porting. Made the build completely `-Wall` safe on Windows. +Fixed some DOS path separator bugs that were preventing WebDav from +working. Have now tested both box.com and Amazon S3 to be completely +working in the webapp on Windows. diff --git a/doc/devblog/day_123__stuck.mdwn b/doc/devblog/day_123__stuck.mdwn new file mode 100644 index 000000000..c48507cf0 --- /dev/null +++ b/doc/devblog/day_123__stuck.mdwn @@ -0,0 +1,13 @@ +Not a lot accomplished today. Some release prep, followed up to a few bug +reports. + +Split git-annex's .git/annex/tmp into two directories. .git/annex/tmp will +now be used only for partially transferred objects, while +.git/annex/misctmp will be used for everything else. In particular this +allows symlinking .git/annex/tmp to a ram disk, if you want to do that. +(It's not possible for .git/annex/misctmp to be on a different filesystem +from the rest of the repository for various reasons.) + +Beat on Windows XMPP for several more painful hours. Got all the haskell +bindings installed, except for gnuidn. And patched network-client-xmpp to +build without gnuidn. Have not managed to get it to link. diff --git a/doc/devblog/day_124__day_off.mdwn b/doc/devblog/day_124__day_off.mdwn new file mode 100644 index 000000000..c8c8f0109 --- /dev/null +++ b/doc/devblog/day_124__day_off.mdwn @@ -0,0 +1,13 @@ +Did not plan to work on git-annex today.. + +Unexpectedly ended up making the webapp support HTTPS. Not by default, +but if a key and certificate are provided, it'll use them. Great for +using the webapp remotely! See the new tip: [[tips/remote_webapp_setup]]. + +Also removed support for --listen with a port, which was buggy and not +necessary with HTTPS. + +Also fixed several webapp/assistant bugs, including one that let it be run in +a bare git repository. + +And, made the quvi version be probed at runtime, rather than compile time. diff --git a/doc/devblog/day_125__metadata_and_views.mdwn b/doc/devblog/day_125__metadata_and_views.mdwn new file mode 100644 index 000000000..471457355 --- /dev/null +++ b/doc/devblog/day_125__metadata_and_views.mdwn @@ -0,0 +1,11 @@ +Worked on metadata and views. Besides bugfixes, two features of note: + +Made git-annex run a hook script, pre-commit-annex. And I wrote a +sample script that extracts metadata from lots of kinds of files, +including photos and sound files, using extract(1) to do the heavy lifting. +See [[tips/automatically_adding_metadata]]. + +Views can be filtered to not include a tag or a field. +For example, `git annex view tag=* !old year!=2013` + +Today's work was sponsored by Stephan Schulz diff --git a/doc/devblog/day_128__release_prep.mdwn b/doc/devblog/day_128__release_prep.mdwn new file mode 100644 index 000000000..d43d2f877 --- /dev/null +++ b/doc/devblog/day_128__release_prep.mdwn @@ -0,0 +1,27 @@ +Preparing for a release (probably tomorrow or Friday). + +Part of that was updating the autobuilders. Had to deal with the gnutls +security hole fix, and upgrading that on the OSX autobuilder turned out to +be quite complicated due to library version skew. Also, I switched the +linux autobuilders over to building from Debian unstable, rather than +stable. That should be ok to do now that the standalone build bundles all +the libraries it needs... And the arm build has always used unstable, and +has been reported working on a lot of systems. So I think this will be +safe, but have backed up the old autobuilder chroots just in case. + +Also been catching up on bug reports and traffic and +and dealt with quite a lot of things today. Smarter log file +rotation for the assistant, better webapp behavior when git is not +installed, and a fix for the webdav 5 second timeout problem. + +Perhaps the most interesting change is a new `annex.startupscan` setting, +which can be disabled to prevent the assistant from doing the expensive +startup scan. This means it misses noticing any files that changed since it +last run, but this should be useful for those really big repositories. + +(Last night, did more work on the test suite, including even more checking +of merge conflict resolution.) + +---- + +Today's work was sponsored by Michael Alan Dorman. diff --git a/doc/devblog/day__126-127__merge_fixes.mdwn b/doc/devblog/day__126-127__merge_fixes.mdwn new file mode 100644 index 000000000..f4a57a09d --- /dev/null +++ b/doc/devblog/day__126-127__merge_fixes.mdwn @@ -0,0 +1,61 @@ +Yesterday I learned of a nasty bug in handling of merges in direct mode. It +turns out that if the remote repository has added a file, and there is a +conflicting file in the local work tree, which has not been added to git, the +local file was overwritten when git-annex did a merge. That's really bad, I'm +very unhappy this bug lurked undetected for so long. + +Understanding the bug was easy. Fixing it turned out to be hard, because +the automatic merge conflict resolution code was quite a mess. In +particular, it wrote files to the work tree, which made it difficult for a +later stage to detect and handle the abovementioned case. Also, the +automatic merge resolution code had weird asymmetric structure that I never +fully understood, and generally needed to be stared at for an hour to begin +to understand it. + +In the process of cleaning that up, I wrote several more tests, +to ensure that every case was handled correctly. Coverage was about 50% +of the cases, and should now be 100%. + +To add to the fun, a while ago I had dealt with a bug on FAT/Windows where +it sometimes lost the symlink bit during automatic merge resolution. Except +it turned out my test case for it had a heisenbug, and I had not actually +fixed it (I think). In any case, my old fix for it was a large part +of the ugliness I was cleaning up, and had to be rewritten. +Fully tracking down and dealing with that took a large part of today. + +Finally this evening, I added support for automatically handling merge +conflicts where one side is an annexed file, and the other side has the +same filename committed to git in the normal way. This is not an important +case, but it's worth it for completeness. There was an unexpected benefit +to doing it; it turned out that the weird asymmetric part of the code went +away. + +The final core of the automatic merge conflict resolver has morphed from +a mess I'd not want to paste here to a quite consise and easy to follow +bit of code. + +[[!format haskell """ + case (kus, kthem) of + -- Both sides of conflict are annexed files + (Just keyUs, Just keyThem) -> resolveby $ + if keyUs == keyThem + then makelink keyUs + else do + makelink keyUs + makelink keyThem + -- Our side is annexed file, other side is not. + (Just keyUs, Nothing) -> resolveby $ do + graftin them file + makelink keyUs + -- Our side is not annexed file, other side is. + (Nothing, Just keyThem) -> resolveby $ do + graftin us file + makelink keyThem + -- Neither side is annexed file; cannot resolve. + (Nothing, Nothing) -> return Nothing +"""]] + +Since the bug that started all this is so bad, I want to make a release +pretty soon.. But I will probably let it soak and whale on the test suite +a bit more first. (This bug is also probably worth backporting to old +versions of git-annex in eg Debian stable.) diff --git a/doc/devblog/whither_XMPP.mdwn b/doc/devblog/whither_XMPP.mdwn new file mode 100644 index 000000000..9e99ba1ac --- /dev/null +++ b/doc/devblog/whither_XMPP.mdwn @@ -0,0 +1,30 @@ +Pushed a release today. Rest of day spent beating head against Windows XMPP +brick wall. + +Actually made a lot of progress -- Finally found the right approach, and +got a clean build of the XMPP haskell libraries. But.. ghc fails to +load the libraries when running Template Haskell. +"Misaligned section: 18206e5b". +Filed [a bug report](https://ghc.haskell.org/trac/ghc/ticket/8830), +and I'm sure this alignment problem can be fixed, +but I'm not hopeful about fixing it myself. + +One workaround would be to use the EvilSplicer, building once without the +XMPP library linked in, to get the TH splices expanded, and then a second +time with the XMPP library and no TH. Made a `winsplicehack` branch with +tons of ifdefs that allows doing this. However, several dozen haskell libraries +would need to be patched to get it to work. I have the patches from +Android, but would rather avoid doing all that again on Windows. + +Another workaround would be to move XMPP into a separate process from the +webapp. This is not very appealing either, the IPC between them would be +fairly complicated since the webapp does stuff like show lists of XMPP +buddies, etc. But, one thing this idea has to recommend it is I am already +considering using a separate helper daemon like this for +[[design/assistant/Telehash]]. + +So there could be synergies between XMPP and Telehash support, possibly +leading to some kind of plugin interface in git-annex for this sort of +thing. But then, once Telehash or something like it is available and +working well, I plan to deprecate XMPP entirely. It's been a flakey pain +from the start, so that can't come too soon. diff --git a/doc/forum/Assistant_Droping_Files.mdwn b/doc/forum/Assistant_Droping_Files.mdwn new file mode 100644 index 000000000..e26edc92d --- /dev/null +++ b/doc/forum/Assistant_Droping_Files.mdwn @@ -0,0 +1,8 @@ +I have a setup as follows, + +3 drives on a remote ssh server regular git annex repos. +One untrusted clone of this repository on my laptop. + +On my laptop in webapp I have set num of copies to 2 (each file does have two copies on the server already). all repos plus laptop is set to manual mode. What happens is whenever I get a file to the laptop it would drop it from the disk on the server. Even though laptop is marked as untrusted this does not happen when using annex at the command line when I do a git annex get . --auto on the server it does not take the files on the untrusted repos into account which is what I want so I can drop files without moving them back to the server. + +Is this the intended behavior or am I doing something wrong? diff --git a/doc/forum/Auto_sync_with_music_player.mdwn b/doc/forum/Auto_sync_with_music_player.mdwn new file mode 100644 index 000000000..e4a5543b4 --- /dev/null +++ b/doc/forum/Auto_sync_with_music_player.mdwn @@ -0,0 +1 @@ +I have a music directory under Nexus 5 which is a git-annex repository. If I sync the same with remote repository, newly added songs or modified songs(modifed id3 tags) will not sync with music player. But if I use Android File Transfer to transfer songs, it will sync music player. The songs which are transferred using git-annex will reflect in music player only after restart. Do we have to execute any command which will sync music player. diff --git a/doc/forum/Auto_sync_with_music_player/comment_1_81ad1c15cfd753531c01dae8945d1caf._comment b/doc/forum/Auto_sync_with_music_player/comment_1_81ad1c15cfd753531c01dae8945d1caf._comment new file mode 100644 index 000000000..3ed75f9e9 --- /dev/null +++ b/doc/forum/Auto_sync_with_music_player/comment_1_81ad1c15cfd753531c01dae8945d1caf._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.146" + subject="comment 1" + date="2014-03-05T20:40:50Z" + content=""" +This is probably done via intents or a similar interface. git-annex is not yet able to use such android api's, but if there is a command line way to send the message that a file has changed, which there probably is, I could look into making git-annex use it. +"""]] diff --git a/doc/forum/Can_not_delete_Repository.mdwn b/doc/forum/Can_not_delete_Repository.mdwn new file mode 100644 index 000000000..549b47c3d --- /dev/null +++ b/doc/forum/Can_not_delete_Repository.mdwn @@ -0,0 +1,3 @@ +I have one repository that I deleted a while back. When I mark it as dead in command line it disappears from git annex info however when I run webapp it pops back webapp shows it as syncing disabled. When I try to delete it from the webapp it does not delete. I tried shutting down the daemon mark it as dead again then run git annex forget --drop-dead --force but running it makes the repo active again instead of deleting it. + +Repo in question was a S3 repo. I tried deleting it using both its name and uuid. diff --git a/doc/forum/Can_not_delete_Repository/comment_1_b1a9420974e2e50c9c86a379ad62502c._comment b/doc/forum/Can_not_delete_Repository/comment_1_b1a9420974e2e50c9c86a379ad62502c._comment new file mode 100644 index 000000000..050127c24 --- /dev/null +++ b/doc/forum/Can_not_delete_Repository/comment_1_b1a9420974e2e50c9c86a379ad62502c._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.172" + subject="comment 1" + date="2014-02-21T17:36:03Z" + content=""" +Marking a repository as dead does not mean it is deleted. The way this is supposed to work, using the webapp is you tell it to start deleting the repository, which causes it to mark it as dead, and also, crucially, puts it into the \"unwanted\" group. This does not remove it from the list yet, but it causes all files located in the repository to be moved off it (assuming the repository can still be accessed). + +Then once the webapp detects that the repository is empty it will prompt you to continue the deletion process and truely remove it. It can take quite a while for the webapp to get around to this last step, since it's done during the expensive transfer scan, which it tries to avoid running unnecessarily. + +In any case, you can always edit .git/config and delete the remote, which should make the webapp no longer show it, as long as it's marked as dead. +"""]] diff --git a/doc/forum/Controlling_content_on_mobile_device/comment_2_dba1a1b0917332a1dee387b1183bd2cb._comment b/doc/forum/Controlling_content_on_mobile_device/comment_2_dba1a1b0917332a1dee387b1183bd2cb._comment new file mode 100644 index 000000000..60aea0bdd --- /dev/null +++ b/doc/forum/Controlling_content_on_mobile_device/comment_2_dba1a1b0917332a1dee387b1183bd2cb._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawnBJ6Dv1glxzzi4qIzGFNa6F-mfHIvv9Ck" + nickname="Jim" + subject="Metadata?" + date="2014-02-21T18:44:10Z" + content=""" +I suspect the new [metadata](http://git-annex.branchable.com/design/metadata/) support could be used to help implement this. +"""]] diff --git a/doc/forum/Convert_regular_git-annex_repo_to_a_rsync_repo.mdwn b/doc/forum/Convert_regular_git-annex_repo_to_a_rsync_repo.mdwn new file mode 100644 index 000000000..c94172535 --- /dev/null +++ b/doc/forum/Convert_regular_git-annex_repo_to_a_rsync_repo.mdwn @@ -0,0 +1 @@ +Is it possible to convert a regular git annex repo (git clone then git annex init in the folder), to an rsync remote. I have an annex with alot of remotes which makes the sync operation take a really long time. I would like to convert some of those remotes to rsync. This particular repo has a TB of data so I would like to avoid dropping content from the remote than re download everything. diff --git a/doc/forum/Convert_regular_git-annex_repo_to_a_rsync_repo/comment_1_e6065f9c44c85030c7628e2cfa0fd0fa._comment b/doc/forum/Convert_regular_git-annex_repo_to_a_rsync_repo/comment_1_e6065f9c44c85030c7628e2cfa0fd0fa._comment new file mode 100644 index 000000000..46ba67c30 --- /dev/null +++ b/doc/forum/Convert_regular_git-annex_repo_to_a_rsync_repo/comment_1_e6065f9c44c85030c7628e2cfa0fd0fa._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.172" + subject="comment 1" + date="2014-02-23T18:57:42Z" + content=""" +This is doable. It works best if the remote repo is a bare git repository, because then the filenames line up 100% with the filenames used in a rsync special remote. If the git repo is not bare, the rsync special remote will first try the paths it expects, and only then fall back to the right paths, so a little extra work done. (If this became a big problem, it would not be infesable to move the files around with a script.) + +Anyway, if it's a bare repo, then repo.git/annex/objects is where you want to point the rsync special remote at. With a non-bare repo, repo/.git/annex/objects/ is the location. I'd recommend moving the objects directory out to a new location, and pointing the rsyncurl at that. This way, there's no possibility of git-annex thinking one files accessed 2 ways is 2 copies. + +Of course, you can't use encryption for the rsync special remote. +"""]] diff --git a/doc/forum/Convert_regular_git-annex_repo_to_a_rsync_repo/comment_2_76bfb11396dc20a5105376b22e7e773b._comment b/doc/forum/Convert_regular_git-annex_repo_to_a_rsync_repo/comment_2_76bfb11396dc20a5105376b22e7e773b._comment new file mode 100644 index 000000000..8ed4f6508 --- /dev/null +++ b/doc/forum/Convert_regular_git-annex_repo_to_a_rsync_repo/comment_2_76bfb11396dc20a5105376b22e7e773b._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.172" + subject="comment 2" + date="2014-02-23T19:07:59Z" + content=""" +However, if the only problem is that pushing and pulling with a git repository makes `git annex sync` take too long, another option is setting `git config remote.$foo.annex-sync false`. You can still then use git-annex commands to get and push data to the remote, and can even `git annex sync $foo` from time to time, but it won't slow down the normal `git annex sync`. + +However, this also prevents the assistant from uploading new files to the remote automatically. +"""]] diff --git a/doc/forum/Convert_regular_git-annex_repo_to_a_rsync_repo/comment_3_b34d6ae0718ab0ff6bc1d7b8f2470b9b._comment b/doc/forum/Convert_regular_git-annex_repo_to_a_rsync_repo/comment_3_b34d6ae0718ab0ff6bc1d7b8f2470b9b._comment new file mode 100644 index 000000000..a8040cd3f --- /dev/null +++ b/doc/forum/Convert_regular_git-annex_repo_to_a_rsync_repo/comment_3_b34d6ae0718ab0ff6bc1d7b8f2470b9b._comment @@ -0,0 +1,16 @@ +[[!comment format=mdwn + username="https://me.yahoo.com/a/FHnTlSBo1eCGJRwueeKeB6.RCaPbGMPr5jxx8A--#ce0d8" + nickname="Hamza" + subject="comment 3" + date="2014-02-23T19:39:28Z" + content=""" +Thanks for the reply, just to make sure I got you right, + +It is indeed a non bare git repo. So I will move the folder repo/.git/annex/objects/ to repo/ + +then run, + +git annex initremote myrsync type=rsync rsyncurl=ssh.example.com:~/repo + +and enable the remote on other annexes (disks are connected to an ssh server there is no encryption setup right now so I do not mind not having it.). And everything should be setup correctly. +"""]] diff --git a/doc/forum/Convert_regular_git-annex_repo_to_a_rsync_repo/comment_4_8f5e323b29745591f9f2f0f867353f69._comment b/doc/forum/Convert_regular_git-annex_repo_to_a_rsync_repo/comment_4_8f5e323b29745591f9f2f0f867353f69._comment new file mode 100644 index 000000000..c8305aaa9 --- /dev/null +++ b/doc/forum/Convert_regular_git-annex_repo_to_a_rsync_repo/comment_4_8f5e323b29745591f9f2f0f867353f69._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://me.yahoo.com/a/FHnTlSBo1eCGJRwueeKeB6.RCaPbGMPr5jxx8A--#ce0d8" + nickname="Hamza" + subject="comment 4" + date="2014-02-23T19:45:09Z" + content=""" +and as a follow up do I have rename the repos or can I reuse the same names for the repos? +"""]] diff --git a/doc/forum/Convert_regular_git-annex_repo_to_a_rsync_repo/comment_5_9824c953694770afa0611ff7276737bf._comment b/doc/forum/Convert_regular_git-annex_repo_to_a_rsync_repo/comment_5_9824c953694770afa0611ff7276737bf._comment new file mode 100644 index 000000000..c6052232b --- /dev/null +++ b/doc/forum/Convert_regular_git-annex_repo_to_a_rsync_repo/comment_5_9824c953694770afa0611ff7276737bf._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.172" + subject="comment 5" + date="2014-02-24T19:07:33Z" + content=""" +That looks all-right, although initremote will ask you to tell it what encryption to use, and you'll need to specify `encryption=none` + +One thing I forgot to mention is that the UUID of the new rsync repository won't be the same, so git-annex won't know about the files in there. This can be fixed by `git annex fsck --fast --from myrsync`. Which doesn't re-download all the files, but you still may want to run it on a repository close to or on the server for speed. + +You can re-use the name you're currently using for the git remote for the new rsync special remote if you like. +"""]] diff --git a/doc/forum/Convert_regular_git-annex_repo_to_a_rsync_repo/comment_6_5899741cb7f83e1b22c5ee3509c5ff21._comment b/doc/forum/Convert_regular_git-annex_repo_to_a_rsync_repo/comment_6_5899741cb7f83e1b22c5ee3509c5ff21._comment new file mode 100644 index 000000000..8b7c3b52e --- /dev/null +++ b/doc/forum/Convert_regular_git-annex_repo_to_a_rsync_repo/comment_6_5899741cb7f83e1b22c5ee3509c5ff21._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://me.yahoo.com/a/FHnTlSBo1eCGJRwueeKeB6.RCaPbGMPr5jxx8A--#ce0d8" + nickname="Hamza" + subject="comment 6" + date="2014-02-25T09:34:16Z" + content=""" +assuming the remote I am converting is called some-repo should mark it as dead before converting and reinitting as rsync some-repo again? +"""]] diff --git a/doc/forum/Feature_Request:_Sync_Now_Button_in_Webapp.mdwn b/doc/forum/Feature_Request:_Sync_Now_Button_in_Webapp.mdwn new file mode 100644 index 000000000..01bec0e21 --- /dev/null +++ b/doc/forum/Feature_Request:_Sync_Now_Button_in_Webapp.mdwn @@ -0,0 +1 @@ +One Problem I am having is that I could never get the xmpp pairing to work so whenever I switch machines I have to manually run sync once on the command line to get the changes. Is it possible to have a sync now button of some sort that will trigger a sync on the repos? diff --git a/doc/forum/Feature_Request:_Sync_Now_Button_in_Webapp/comment_1_0d5c90eb0e8fe61b82a19c5fea343613._comment b/doc/forum/Feature_Request:_Sync_Now_Button_in_Webapp/comment_1_0d5c90eb0e8fe61b82a19c5fea343613._comment new file mode 100644 index 000000000..a5f631d50 --- /dev/null +++ b/doc/forum/Feature_Request:_Sync_Now_Button_in_Webapp/comment_1_0d5c90eb0e8fe61b82a19c5fea343613._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawnR6E5iUghMWdUGlbA9CCs8DKaoigMjJXw" + nickname="Efraim" + subject="comment 1" + date="2014-03-06T20:37:36Z" + content=""" +not quite a sync button, but when I want to force sync now I turn off and turn on sync for one of the repos from the webapp and then it syncs. +"""]] diff --git a/doc/forum/Find_files_that_lack_a_certain_field_in_metadata.mdwn b/doc/forum/Find_files_that_lack_a_certain_field_in_metadata.mdwn new file mode 100644 index 000000000..b0fe9ddaa --- /dev/null +++ b/doc/forum/Find_files_that_lack_a_certain_field_in_metadata.mdwn @@ -0,0 +1,5 @@ +Is there any way to find all files that do not have a certain field assigned in metadata. E.g. I want to find all files that do not have an author field set and + + git-annex find --not --metadata "author=*" + +doesn't give any results. diff --git a/doc/forum/Find_files_that_lack_a_certain_field_in_metadata/comment_1_476e52563ccd3ad1b43e3a2da4dfaa82._comment b/doc/forum/Find_files_that_lack_a_certain_field_in_metadata/comment_1_476e52563ccd3ad1b43e3a2da4dfaa82._comment new file mode 100644 index 000000000..fbfb6ebb6 --- /dev/null +++ b/doc/forum/Find_files_that_lack_a_certain_field_in_metadata/comment_1_476e52563ccd3ad1b43e3a2da4dfaa82._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.172" + subject="comment 1" + date="2014-02-21T22:36:51Z" + content=""" +--metadata does not support globs, so your example is asking for all files that don't have an author field with a literal \"*\" value. When I try that command, it lists all files ... as expected. + +It seems that adding glob support to it would get to the result you want, and makes sense to parallel git annex view. Change made in git! +"""]] diff --git a/doc/forum/How_does_one_change_git-annex_assistant__39__s_web_browser__63__/comment_7_71ff45948487e9ac8de809a5ccc3d874._comment b/doc/forum/How_does_one_change_git-annex_assistant__39__s_web_browser__63__/comment_7_71ff45948487e9ac8de809a5ccc3d874._comment new file mode 100644 index 000000000..115d402ad --- /dev/null +++ b/doc/forum/How_does_one_change_git-annex_assistant__39__s_web_browser__63__/comment_7_71ff45948487e9ac8de809a5ccc3d874._comment @@ -0,0 +1,29 @@ +[[!comment format=mdwn + username="rupi" + ip="80.110.152.13" + subject="xdg-open" + date="2014-02-28T17:48:23Z" + content=""" +Since I discovered the same behavior mentioned above (git annex webapp +tries to use 'winebrowser'). There is an easy way to work with the difference +between http +url and html files: enter preferences for both: + +In ~/.local/share/applications/mimeapps.list: + + [Added Associations] + .. + x-scheme-handler/http=iceweasel.desktop; + x-scheme-handler/https=iceweasel.desktop; + text/html=icewesel.desktop; + + + [Default Applications] + x-scheme-handler/http=iceweasel.desktop + x-scheme-handler/https=iceweasel.desktop + text/html=iceweasel.desktop + + +Your preferred browser might be a different one off course. And I agree that this has to be the most logical place to store that information ever.. + +"""]] diff --git a/doc/forum/Too_big_to_fsck.mdwn b/doc/forum/Too_big_to_fsck.mdwn new file mode 100644 index 000000000..975674b5c --- /dev/null +++ b/doc/forum/Too_big_to_fsck.mdwn @@ -0,0 +1,20 @@ +Hi, + +My Webapp isn't working: + + $ git-annex webapp error: refs/gcrypt/gitception+ does not point to a valid object! + error: refs/remotes/Beta/git-annex does not point to a valid object! + error: refs/remotes/Beta/master does not point to a valid object! + fatal: unable to read tree 656e7db5be172f01c0b6994d01f1a08d1273af12 + +So I tried to repair it: + + $ git-annex repair Running git fsck ... + Stack space overflow: current size 8388608 bytes. Use `+RTS -Ksize -RTS' to increase it. + +So I tried to follow your advice here and increase the stack: + + $ git-annex +RTS -K35000000 -RTS fsck + git-annex: Most RTS options are disabled. Link with -rtsopts to enable them. + +I wasn't sure what to do next, so any help would be appreciated. diff --git a/doc/forum/Too_big_to_fsck/comment_1_490b8bfe95b01a23408ecb5d63dcd40b._comment b/doc/forum/Too_big_to_fsck/comment_1_490b8bfe95b01a23408ecb5d63dcd40b._comment new file mode 100644 index 000000000..73d218bac --- /dev/null +++ b/doc/forum/Too_big_to_fsck/comment_1_490b8bfe95b01a23408ecb5d63dcd40b._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.172" + subject="comment 1" + date="2014-02-23T18:51:45Z" + content=""" +I suspect that git fsck is outputting so many lines about problems that it's taking more memory than it's limited to using to hold them all. + +Can you paste the output of: git fsck --no-dangling --no-reflogs +"""]] diff --git a/doc/forum/Too_big_to_fsck/comment_2_2666c135dd3378cf6301aa4957049fbd._comment b/doc/forum/Too_big_to_fsck/comment_2_2666c135dd3378cf6301aa4957049fbd._comment new file mode 100644 index 000000000..f2028c91f --- /dev/null +++ b/doc/forum/Too_big_to_fsck/comment_2_2666c135dd3378cf6301aa4957049fbd._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.172" + subject="comment 2" + date="2014-02-23T19:09:30Z" + content=""" +Erm, that output is liable to be big, I only care how many lines and characters of output there are! + + git fsck --no-dangling --no-reflogs |wc +"""]] diff --git a/doc/forum/Too_big_to_fsck/comment_3_dfb169c441215b671f8c971184de3e16._comment b/doc/forum/Too_big_to_fsck/comment_3_dfb169c441215b671f8c971184de3e16._comment new file mode 100644 index 000000000..96b1ac9cd --- /dev/null +++ b/doc/forum/Too_big_to_fsck/comment_3_dfb169c441215b671f8c971184de3e16._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.172" + subject="comment 3" + date="2014-02-23T19:12:10Z" + content=""" +Also, you can build git-annex from source with the RTS options enabled by running `cabal install git-annex --ghc-options=-rtsopts` + +(or just build git-repair which has the repository repair parts of git-annex) +"""]] diff --git a/doc/forum/Too_big_to_fsck/comment_4_19ef90803aa7ce158bce02378e18ea0f._comment b/doc/forum/Too_big_to_fsck/comment_4_19ef90803aa7ce158bce02378e18ea0f._comment new file mode 100644 index 000000000..69774073d --- /dev/null +++ b/doc/forum/Too_big_to_fsck/comment_4_19ef90803aa7ce158bce02378e18ea0f._comment @@ -0,0 +1,30 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawnFjuvfPpi1kf6l54bxfFUm0Aw_Gf_IO0o" + nickname="Aaron" + subject="Didn't work" + date="2014-03-01T22:40:36Z" + content=""" +Many thanks for your input, Joey. + +That didn't seem to work. + +$ git fsck --no-dangling --no-reflogs |wc +error: unknown option `no-dangling' +usage: git fsck [options] [<object>...] + + -v, --verbose be verbose + --unreachable show unreachable objects + --tags report tags + --root report root nodes + --cache make index objects head nodes + --reflogs make reflogs head nodes (default) + --full also consider packs and alternate objects + --strict enable more strict checking + --lost-found write dangling objects in .git/lost-found + --progress show progress + + 0 0 0 + +$ git --version +git version 1.7.9.5 +"""]] diff --git a/doc/forum/Too_big_to_fsck/comment_5_2b5406768fff2834f7aefa76ef949de2._comment b/doc/forum/Too_big_to_fsck/comment_5_2b5406768fff2834f7aefa76ef949de2._comment new file mode 100644 index 000000000..923e1ccba --- /dev/null +++ b/doc/forum/Too_big_to_fsck/comment_5_2b5406768fff2834f7aefa76ef949de2._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.146" + subject="comment 5" + date="2014-03-05T16:53:33Z" + content=""" +You have an older version of git, which does not support --no-dangling. + +git-annex will detect that, and omit the option. This does make it more likely that git fsck is outputing a ton of information, so I continue to think that's the most likely cause of the memory use. + +Please verify with: git fsck --no-reflogs |wc +"""]] diff --git a/doc/forum/WARNING:_linker:git-annex_has_text_relocations....mdwn b/doc/forum/WARNING:_linker:git-annex_has_text_relocations....mdwn new file mode 100644 index 000000000..2d9ee1324 --- /dev/null +++ b/doc/forum/WARNING:_linker:git-annex_has_text_relocations....mdwn @@ -0,0 +1,7 @@ +I have configured git-annex on my Nexus 5. Even though it works fantastic, it shows some warning messages. + +1: 'git-annex sync' : WARNING: linker: git-annex has text relocations. This is wasting memory and is a security risk. Please fix. + +2: 'git log': error: cannot run less. No such file or directory. + +Both the commands will work has expected with these warnings. What could be the issue here. diff --git a/doc/forum/WARNING:_linker:git-annex_has_text_relocations.../comment_1_fee360353f0b46aab6ee7a902c0837bb._comment b/doc/forum/WARNING:_linker:git-annex_has_text_relocations.../comment_1_fee360353f0b46aab6ee7a902c0837bb._comment new file mode 100644 index 000000000..8d9af9624 --- /dev/null +++ b/doc/forum/WARNING:_linker:git-annex_has_text_relocations.../comment_1_fee360353f0b46aab6ee7a902c0837bb._comment @@ -0,0 +1,11 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.146" + subject="comment 1" + date="2014-03-05T20:38:03Z" + content=""" +<http://stackoverflow.com/questions/20141538/mylib-so-has-text-relocations-this-is-wasting-memory-and-is-a-security-risk-pl> +Says that should have been fixed in ndk-r8c's gcc, but git-annex is already built with ndk-r9b. It might be though, that the llvm compiler still has the problem. In any case, I doubt there is a real security issue here, it's probably just a binary exploit hardening thing. + +As for the git log, git users the less pager by default, but of course it's not necessary and so not included on Android. You can `git config core.pager cat` to get around this. +"""]] diff --git a/doc/forum/git-annex_on_osx_10.9.1_just_crashes__47__closes__47__doesn__39__t_run_on_launch.mdwn b/doc/forum/git-annex_on_osx_10.9.1_just_crashes__47__closes__47__doesn__39__t_run_on_launch.mdwn new file mode 100644 index 000000000..c5d4c40bc --- /dev/null +++ b/doc/forum/git-annex_on_osx_10.9.1_just_crashes__47__closes__47__doesn__39__t_run_on_launch.mdwn @@ -0,0 +1,7 @@ +on a Mac OSX 10.9.1 client I would like to try out git-annex. + +I download the latest Mavericks build, extract from the .dmg and it just does not run. I open the program, one "bounce" on the dock and it is gone, nothing happens. + +I sadly do not have another OSX client to try it on. But nothing shows in the Console.app or anything. + +Am I missing something? diff --git a/doc/forum/git-annex_on_osx_10.9.1_just_crashes__47__closes__47__doesn__39__t_run_on_launch/comment_1_a47174f8438bfaa42fb8067bca77bf4c._comment b/doc/forum/git-annex_on_osx_10.9.1_just_crashes__47__closes__47__doesn__39__t_run_on_launch/comment_1_a47174f8438bfaa42fb8067bca77bf4c._comment new file mode 100644 index 000000000..3c9361cbd --- /dev/null +++ b/doc/forum/git-annex_on_osx_10.9.1_just_crashes__47__closes__47__doesn__39__t_run_on_launch/comment_1_a47174f8438bfaa42fb8067bca77bf4c._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.146" + subject="comment 1" + date="2014-03-05T22:30:00Z" + content=""" +The thing to do is to try to start git-annex from the command line in a terminal. This should get an error message that will help find out what's going wrong. +"""]] diff --git a/doc/forum/git_annex_with_local_apache_webdav_server.mdwn b/doc/forum/git_annex_with_local_apache_webdav_server.mdwn new file mode 100644 index 000000000..db489fdfa --- /dev/null +++ b/doc/forum/git_annex_with_local_apache_webdav_server.mdwn @@ -0,0 +1,23 @@ +Hi, + +trying to make git annex work locally with an apache webdav server. + +I have the webdav server working without issue on computers. When we try to init the repository there we get the following error: + +WEBDAV_USERNAME=user WEBDAV_PASSWORD=xxxxxx git annex initremote webdavtest type=webdav url=http://webdavserver/webdavsgare/annextest4 encryption=none +initremote webdavtest (testing WebDAV server...) + +git-annex: WebDAV failed to delete file: "Locked": user error +failed +git-annex: initremote: 1 failed + + +Does anyone have any thoughts? I can post config of webdav if it helps, though the error I receive in the error_log of apache is as follows: + +This resource is locked and an "If:" header was not supplied to allow access to the resource. [423, #0] + +but I can manage the files through command line, web interface and mounted drive with no issue. + +thank you in advance. + +Damien diff --git a/doc/forum/git_annex_with_local_apache_webdav_server/comment_1_a3b89f90f9ac70e0a9b0711ede1cb810._comment b/doc/forum/git_annex_with_local_apache_webdav_server/comment_1_a3b89f90f9ac70e0a9b0711ede1cb810._comment new file mode 100644 index 000000000..7963f64f0 --- /dev/null +++ b/doc/forum/git_annex_with_local_apache_webdav_server/comment_1_a3b89f90f9ac70e0a9b0711ede1cb810._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.146" + subject="comment 1" + date="2014-02-26T19:40:18Z" + content=""" +The webdav library that git-annex uses does some webdav file locking. When deleting a file, it first locks it. It sounds like your webdav server does not like to delete a file that has been locked. Whether this is in spec I don't know. + +I do know that got rid of all the webdav locking stuff in a revamp of git-annex's webdav support 2 days ago! git-annex does not need locking the way it uses webdav. + +However, the daily builds are still built with an old version of the DAV library, which still does the locking ... and will probably remain like that for some time. To get the new version without locking, you need to build git-annex from its git master on a system with http://hackage.haskell.org/package/DAV version 0.6.1. Building from source with cabal is one way to do it. I'll be releasing a build for Debian unstable in the next day or two. +"""]] diff --git a/doc/forum/new_linux_arm_tarball_build/comment_9_2802b24ccb24f1615c9d61904f916d05._comment b/doc/forum/new_linux_arm_tarball_build/comment_9_2802b24ccb24f1615c9d61904f916d05._comment new file mode 100644 index 000000000..fbb0e2f21 --- /dev/null +++ b/doc/forum/new_linux_arm_tarball_build/comment_9_2802b24ccb24f1615c9d61904f916d05._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawmUJBh1lYmvfCCiGr3yrdx-QhuLCSRnU5c" + nickname="Justin" + subject="comment 9" + date="2014-03-01T19:13:16Z" + content=""" +This seems to work great for me on Raspbian, except I get many errors of the form + + ERROR: ld.so: object '/usr/lib/arm-linux-gnueabihf/libcofi_rpi.so' from /etc/ld.so.preload cannot be preloaded: ignored. + +I've tried, but I haven't been able to figure out why ld doesn't like this particular lib. I can keep ignoring the error, but help would certainly be appreciated! +"""]] diff --git a/doc/forum/performance_and_multiple_replication_problems/comment_3_ad7cb4c510e2ab26959ea7cb40a43fef._comment b/doc/forum/performance_and_multiple_replication_problems/comment_3_ad7cb4c510e2ab26959ea7cb40a43fef._comment new file mode 100644 index 000000000..6e4e1b1c6 --- /dev/null +++ b/doc/forum/performance_and_multiple_replication_problems/comment_3_ad7cb4c510e2ab26959ea7cb40a43fef._comment @@ -0,0 +1,14 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawnNqLKszWk9EoD4CDCqNXJRIklKFBCN1Ao" + nickname="maurizio" + subject="the startup check is not a small issue" + date="2014-02-25T11:37:15Z" + content=""" +I would like to add that this startup check has probably been a blocker for my use case for a long long time. I tried to use git-annex to synchronize a huge number of files, most of them never changing. My plan was to have a few tens of GB of data which more or less never change in an archive directory and then add from time to time new data (by batches of a few hundreds of files, each of them not necessarily very large) to the annex. Once this new data has been processed or otherwise become less immediately useful, it would be shifted to the archive. It would have been very useful to have such a setup, because the amount of data is too large to be replicated everywhere, especially on a laptop. After finding this post I finally understand that the seemingly never ending \"performing startup scan\" that I observed are probably not due to the assistant somehow hanging, contrary to what I thought. It seems it is just normal operation. The problem is that this normal operation makes it unusable for the use case I was considering, since it does not make much sense to have git-annex scanning about 10^6 files or links on every boot of a laptop. On my workstation this \"startup scan\" has now been running for close to one hour now and is not finished yet, this is not thinkable on laptop boot. + +Maybe an analysis of how well git-annex operation scales with number of files should be part of the documentation, since \"large files\" is not the only issue when trying to sync different computers. One finds references to \"very large number of files\" about annex.queuesize, but \"very large\" has no clear meaning. One also finds a reference to \"1 million files\" being a bit of a git limitation on comments of a bug report <https://git-annex.branchable.com/bugs/Stress_test/>. + +Orders of magnitude of the number of files that git-annex is supposed to be able to handle would be very useful. + + +"""]] diff --git a/doc/forum/performance_and_multiple_replication_problems/comment_4_23a6dc7ea569944ca55bd21851dd770d._comment b/doc/forum/performance_and_multiple_replication_problems/comment_4_23a6dc7ea569944ca55bd21851dd770d._comment new file mode 100644 index 000000000..77fe564fc --- /dev/null +++ b/doc/forum/performance_and_multiple_replication_problems/comment_4_23a6dc7ea569944ca55bd21851dd770d._comment @@ -0,0 +1,14 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.146" + subject="comment 4" + date="2014-03-05T22:06:00Z" + content=""" +@maurizio, that's a good motivating example. + +So, I have made `git config annex.startupscan false` disable the startup scan (except for a minor tree walk that needs to be done to get inotify working). + +Of course, if you set that, the assistant won't notice any changes that are made when it's not running. It would work well to set it if the assistant is started at boot and left running until shutdown. + +My goal for git-annex is for it to scale to how ever many files git scales to. So I try to always make git be the limiting factor. I feel that git scales fairly well into the 100k file range. +"""]] diff --git a/doc/forum/workspace.xml_file_disappeared__44___broken_symlink_showed_up/comment_5_50526283b35997cece2f087507cdd4ee._comment b/doc/forum/workspace.xml_file_disappeared__44___broken_symlink_showed_up/comment_5_50526283b35997cece2f087507cdd4ee._comment new file mode 100644 index 000000000..ebf175cf2 --- /dev/null +++ b/doc/forum/workspace.xml_file_disappeared__44___broken_symlink_showed_up/comment_5_50526283b35997cece2f087507cdd4ee._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.146" + subject="comment 5" + date="2014-03-06T18:22:31Z" + content=""" +Do you edit this file in multiple places? This could be an occurance of this bug: [[bugs/direct_mode_merge_can_overwrite_local,_non-annexed_files]] which is fixed in the latest release. + +If the program that writes the workspace.xml file did so by first deleting it, and then writing the new version, this could result in the assistant committing the deletion, which makes the new version a local, non-annexed file, and then if a pull is received that modified the file, I think the above bug could happen. + +You could tell if this was the case by looking at the git log of the directory containing the file, and see if it has been repeatedly deleted and added back to the repository. The git log snippet you pasted unfortunately does not let me tell this information. +"""]] diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index 0912e0b2a..30494e95c 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -307,11 +307,19 @@ subdirectories). By default, the webapp can only be accessed from localhost, and running it opens a browser window. - With the `--listen=address[:port]` option, the webapp can be made to listen - for connections on the specified address. This disables running a - local web browser, and outputs the url you can use to open the webapp - from a remote computer. - Note that this does not yet use HTTPS for security, so use with caution! + To use the webapp on a remote computer, use the `--listen=address` + option to specify the address the web server should listen on + (or set annex.listen). + This disables running a local web browser, and outputs the url you + can use to open the webapp. + + When using the webapp on a remote computer, you'll almost certianly + want to enable HTTPS. The webapp will use HTTPS if it finds + a .git/annex/privkey.pem and .git/annex/certificate.pem. Here's + one way to generate those files, using a self-signed certificate: + + openssl genrsa -out .git/annex/privkey.pem 4096 + openssl req -new -x509 -key .git/annex/privkey.pem > .git/annex/certificate.pem # REPOSITORY SETUP COMMANDS @@ -709,13 +717,16 @@ subdirectories). To remove a value, use -s field-=value. + To set a value, only if the field does not already have a value, + use -s field?=value + To set a tag, use -t tag, and use -u tag to remove a tag. For example, to set some tags on a file and also its author: git annex metadata annexscreencast.ogv -t video -t screencast -s author+=Alice -* `view [field=value ...] [tag ...]` +* `view [tag ...] [field=value ...] [field=glob ...] [!tag ...] [field!=value ...]` Uses metadata to build a view branch of the files in the current branch, and checks out the view branch. Only files in the current branch whose @@ -726,10 +737,19 @@ subdirectories). a glob (`field="*"`) or by listing each wanted value. The resulting view will put files in subdirectories according to the value of their fields. - Once within a view, you can make additional directories, and + Once within such a view, you can make additional directories, and copy or move files into them. When you commit, the metadata will be updated to correspond to your changes. + There are fields corresponding to the path to the file. So a file + "foo/bar/baz/file" has fields "/=foo", "foo/=bar", and "foo/bar/=baz". + These location fields can be used the same as other metadata to construct + the view. + + For example, `/=podcasts` will only include files from the podcasts + directory in the view, while `podcasts/=*` will preserve the + subdirectories of the podcasts directory in the view. + * `vpop [N]` Switches from the currently active view back to the previous view. @@ -737,12 +757,12 @@ subdirectories). The optional number tells how many views to pop. -* `vfilter [field=value ...] [tag ...]` +* `vfilter [tag ...] [field=value ...] [!tag ...] [field!=value ...]` Filters the current view to only the files that have the - specified values and tags. + specified field values and tags. -* `vadd [field=glob ...]` +* `vadd [field=glob ...] [field=value ...] [tag ...]` Changes the current view, adding an additional level of directories to categorize the files. @@ -942,7 +962,7 @@ subdirectories). Rather than the normal output, generate JSON. This is intended to be parsed by programs that use git-annex. Each line of output is a JSON object. Note that JSON output is only usable with some git-annex commands, - like info, find, and whereis. + like info, find, whereis, and metadata. * `--debug` @@ -1133,10 +1153,11 @@ file contents are present at either of two repositories. The size can be specified with any commonly used units, for example, "0.5 gb" or "100 KiloBytes" -* `--metadata field=value` +* `--metadata field=glob` - Matches only files that have a metadata field attached with the specified - value. + Matches only files that have a metadata field attached with a value that + matches the glob. The values of metadata fields are matched case + insensitively. * `--want-get` @@ -1269,6 +1290,12 @@ Here are all the supported configuration settings. Note that setting numcopies to 0 is very unsafe. +* `annex.genmetadata` + + Set this to `true` to make git-annex automatically generate some metadata + when adding files to the repository. In particular, it stores + year and month metadata, from the file's modification date. + * `annex.queuesize` git-annex builds a queue of git commands, in order to combine similar @@ -1353,6 +1380,19 @@ Here are all the supported configuration settings. Set to false to prevent the git-annex assistant from automatically committing changes to files in the repository. +* `annex.startupscan` + + Set to false to prevent the git-annex assistant from scanning the + repository for new and changed files on startup. This will prevent it + from noticing changes that were made while it was not running, but can be + a useful performance tweak for a large repository. + +* `annex.listen` + + Configures which address the webapp listens on. The default is localhost. + Can be either an IP address, or a hostname that resolves to the desired + address. + * `annex.debug` Set to true to enable debug logging by default. @@ -1509,8 +1549,7 @@ Here are all the supported configuration settings. * `annex.web-options` - Options to use when using wget or curl to download a file from the web. - (wget is always used in preference to curl if available.) + Options to pass when running wget or curl. For example, to force ipv4 only, set it to "-4" * `annex.quvi-options` @@ -1649,6 +1688,10 @@ used by git-annex. `~/.config/git-annex/autostart` is a list of git repositories to start the git-annex assistant in. +`.git/hooks/pre-commit-annex` in your git repsitory will be run whenever +a commit is made, either by git commit, git-annex sync, or the git-annex +assistant. + # SEE ALSO Most of git-annex's documentation is available on its web site, diff --git a/doc/install/Fedora.mdwn b/doc/install/Fedora.mdwn index 9a6006e97..c1769a10d 100644 --- a/doc/install/Fedora.mdwn +++ b/doc/install/Fedora.mdwn @@ -1,5 +1,4 @@ -git-annex is available in recent versions of Fedora. Although it is -not currently a very recent version, it should work ok. +git-annex is available in recent versions of Fedora. [status](http://koji.fedoraproject.org/koji/packageinfo?packageID=14145) Should be as simple as: `yum install git-annex` diff --git a/doc/install/Linux_standalone/comment_1_1adc00aecc51f1e74701bd67cd74155d._comment b/doc/install/Linux_standalone/comment_1_1adc00aecc51f1e74701bd67cd74155d._comment new file mode 100644 index 000000000..8218aa40d --- /dev/null +++ b/doc/install/Linux_standalone/comment_1_1adc00aecc51f1e74701bd67cd74155d._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawmUJBh1lYmvfCCiGr3yrdx-QhuLCSRnU5c" + nickname="Justin" + subject="Are the ARM binaries ARMv6 or ARMv7?" + date="2014-02-26T21:04:44Z" + content=""" +I'm trying to run git-annex on my Raspberry PI, which is an ARMv6 device. But I can't tell if the prebuilt binaries are ARMv6 or ARMv7. The binaries seem to work, but it's not clear to me how early I should expect things to fail if they're compiled for ARMv7. +"""]] diff --git a/doc/install/Linux_standalone/comment_2_7983285b56fd10359a7cc3615fd1e2fe._comment b/doc/install/Linux_standalone/comment_2_7983285b56fd10359a7cc3615fd1e2fe._comment new file mode 100644 index 000000000..996263b18 --- /dev/null +++ b/doc/install/Linux_standalone/comment_2_7983285b56fd10359a7cc3615fd1e2fe._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.146" + subject="comment 2" + date="2014-02-26T21:07:20Z" + content=""" +The arm binaries are ARMv6; they're built on a Debian armel system. + +Obviously there is a lot of variation in ARM instructions sets. Like Debian, we're shooting for lowest common denominator here. + +(Note: I'm going to be moving your comment and this one to the place you should have posted it...) +"""]] diff --git a/doc/install/fromscratch.mdwn b/doc/install/fromscratch.mdwn index 2c8bf4b71..3843c31fa 100644 --- a/doc/install/fromscratch.mdwn +++ b/doc/install/fromscratch.mdwn @@ -5,6 +5,7 @@ quite a lot. * [The Haskell Platform](http://haskell.org/platform/) (GHC 7.4 or newer) * [mtl](http://hackage.haskell.org.package/mtl) (2.1.1 or newer) * [MissingH](http://github.com/jgoerzen/missingh/wiki) + * [data-default](http://hackage.haskell.org/package/data-default) * [utf8-string](http://hackage.haskell.org/package/utf8-string) * [SHA](http://hackage.haskell.org/package/SHA) * [cryptohash](http://hackage.haskell.org/package/cryptohash) (optional but recommended) @@ -25,6 +26,7 @@ quite a lot. * [extensible-exceptions](http://hackage.haskell.org/package/extensible-exceptions) * [feed](http://hackage.haskell.org/package/feed) * [async](http://hackage.haskell.org/package/async) + * [case-insensitive](http://hackage.haskell.org/package/case-insensitive) * [stm](http://hackage.haskell.org/package/stm) (version 2.3 or newer) * Optional haskell stuff, used by the [[assistant]] and its webapp @@ -35,11 +37,11 @@ quite a lot. * [yesod-static](http://hackage.haskell.org/package/yesod-static) * [yesod-default](http://hackage.haskell.org/package/yesod-default) * [data-default](http://hackage.haskell.org/package/data-default) - * [case-insensitive](http://hackage.haskell.org/package/case-insensitive) * [http-types](http://hackage.haskell.org/package/http-types) * [wai](http://hackage.haskell.org/package/wai) * [wai-logger](http://hackage.haskell.org/package/wai-logger) * [warp](http://hackage.haskell.org/package/warp) + * [warp-tls](http://hackage.haskell.org/package/warp-tls) * [blaze-builder](http://hackage.haskell.org/package/blaze-builder) * [crypto-api](http://hackage.haskell.org/package/crypto-api) * [hamlet](http://hackage.haskell.org/package/hamlet) diff --git a/doc/internals.mdwn b/doc/internals.mdwn index 970e88ba0..4e003d9bc 100644 --- a/doc/internals.mdwn +++ b/doc/internals.mdwn @@ -27,6 +27,42 @@ Also in [[direct_mode]], some additional data is stored in these directories. changed, and `.map` files contain a list of file(s) in the work directory that contain the key. +# `.git/annex/tmp/` + +This directory contains partially transferred objects. + +# `.git/annex/misctmp/` + +This is a temp directory for miscellaneous other temp files. + +While .git/annex/objects and .git/annex/tmp can be put on different +filesystems if desired, .git/annex/misctmp +has to be on the same filesystem as the work tree and git repository. + +# `.git/annex/bad/` + +git-annex fsck puts any bad objects it finds in here. + +# `.git/annex/transfers/` + +Contains information files for uploads and downloads that are in progress, +as well as any that have failed. Used especially by the assistant. +It is safe to delete these files. + +# `.git/annex/ssh/` + +ssh connection caching files are written in here. + +# `.git/annex/index` + +This is a git index file which git-annex uses for commits to the git-annex +branch. + +# `.git/annex/journal/` + +git-annex uses this to journal changes to the git-annex branch, +before committing a set of changes. + ## The git-annex branch This branch is managed by git-annex, with the contents listed below. diff --git a/doc/metadata.mdwn b/doc/metadata.mdwn new file mode 100644 index 000000000..df873c4c1 --- /dev/null +++ b/doc/metadata.mdwn @@ -0,0 +1,42 @@ +git-annex allows you to store arbitrary metadata about files stored in the +git-annex repository. The metadata is stored in the `git-annex` branch, and +so is automatically kept in sync with the rest of git-annex's state, such +as [[location_tracking]] information. + +Some of the things you can do with metadata include: + +* Using `git annex metadata file` to show all + the metadata associated with a file. +* [[tips/metadata_driven_views]] +* Limiting the files git-annex commands act on to those with + or without particular metadata. + For example `git annex find --metadata tag=foo --or --metadata tag=bar` +* Using it in [[preferred_content]] expressions. + For example "tag=important or not author=me" + +Each file (actually the underlying key) can have any number of metadata +fields, which each can have any number of values. For example, to tag +files, the `tag` field is typically used, with values set to each tag that +applies to the file. + +The field names are limited to alphanumerics (and `[_-.]`), and are case +insensitive. The metadata values can contain absolutely anything you +like -- but you're recommended to keep it simple and reasonably short. + +Here are some recommended metadata fields to use: + +* `tag` - With each tag being a different value. +* `year`, `month` - When this particular version of the file came into + being. + +To make git-annex automatically set the year and month when adding files, +run `git config annex.genmetadata true`. Also, see +[[tips/automatically_adding_metadata]]. + +git-annex's metadata can be updated in a distributed fashion. For example, +two users, each with their own clone of a repository, can set and unset +metadata at the same time, even for the same field of the same file. +When they push their changes, `git annex merge` will combine their +metadata changes in a consistent and (probably) intuitive way. + +See [[the metadata design page|design/metadata]] for more details. diff --git a/doc/news/version_5.20140116.mdwn b/doc/news/version_5.20140116.mdwn deleted file mode 100644 index f107f2672..000000000 --- a/doc/news/version_5.20140116.mdwn +++ /dev/null @@ -1,21 +0,0 @@ -git-annex 5.20140116 released with [[!toggle text="these changes"]] -[[!toggleable text=""" - * Added tahoe special remote. - * external special remote protocol: Added GETGITDIR, and GETAVAILABILITY. - * Refuse to build with git older than 1.7.1.1, which is needed for - git checkout -B - * map: Fix display of v5 direct mode repos. - * repair: Support old git versions from before git fsck --no-dangling was - implemented. - * Fix a long-standing bug that could cause the wrong index file to be used - when committing to the git-annex branch, if GIT\_INDEX\_FILE is set in the - environment. This typically resulted in git-annex branch log files being - committed to the master branch and later showing up in the work tree. - (These log files can be safely removed.) - * assistant: Detect if .git/annex/index is corrupt at startup, and - recover. - * repair: Fix bug in packed refs file exploding code that caused a .gitrefs - directory to be created instead of .git/refs - * Fix FTBFS on mipsel and sparc due to test suite not being available - on those architectures. - * Android: Avoid passing --clobber to busybox wget."""]]
\ No newline at end of file diff --git a/doc/news/version_5.20140127.mdwn b/doc/news/version_5.20140127.mdwn deleted file mode 100644 index 7c5748f35..000000000 --- a/doc/news/version_5.20140127.mdwn +++ /dev/null @@ -1,41 +0,0 @@ -git-annex 5.20140127 released with [[!toggle text="these changes"]] -[[!toggleable text=""" - * sync --content: New option that makes the content of annexed files be - transferred. Similar to the assistant, this honors any configured - preferred content expressions. - * Remove --json option from commands not supporting it. - * status: Support --json. - * list: Fix specifying of files to list. - * Allow --all to be mixed with matching options like --copies and --in - (but not --include and --exclude). - * numcopies: New command, sets global numcopies value that is seen by all - clones of a repository. - * The annex.numcopies git config setting is deprecated. Once the numcopies - command is used to set the global number of copies, any annex.numcopies - git configs will be ignored. - * assistant: Make the prefs page set the global numcopies. - * Add lackingcopies, approxlackingcopies, and unused to - preferred content expressions. - * Client, transfer, incremental backup, and archive repositories - now want to get content that does not yet have enough copies. - * Client, transfer, and source repositories now do not want to retain - unused file contents. - * assistant: Checks daily for unused file contents, and when possible - moves them to a repository (such as a backup repository) that - wants to retain them. - * assistant: annex.expireunused can be configured to cause unused - file contents to be deleted after some period of time. - * webapp: Nudge user to see if they want to expire old unused file - contents when a lot of them seem to be piling up in the repository. - * repair: Check git version at run time. - * assistant: Run the periodic git gc in batch mode. - * added annex.secure-erase-command config option. - * Optimise non-bare http remotes; no longer does a 404 to the wrong - url every time before trying the right url. Needs annex-bare to be - set to false, which is done when initially probing the uuid of a - http remote. - * webapp: After upgrading a git repository to git-annex, fix - bug that made it temporarily not be synced with. - * whereis: Support --all. - * All commands that support --all also support a --key option, - which limits them to acting on a single key."""]]
\ No newline at end of file diff --git a/doc/news/version_5.20140221.mdwn b/doc/news/version_5.20140221.mdwn new file mode 100644 index 000000000..50f85496e --- /dev/null +++ b/doc/news/version_5.20140221.mdwn @@ -0,0 +1,28 @@ +git-annex 5.20140221 released with [[!toggle text="these changes"]] +[[!toggleable text=""" + * metadata: New command that can attach metadata to files. + * --metadata can be used to limit commands to acting on files + that have particular metadata. + * Preferred content expressions can use metadata=field=value + to limit them to acting on files that have particular metadata. + * view: New command that creates and checks out a branch that provides + a structured view of selected metadata. + * vfilter, vadd, vpop, vcycle: New commands for operating within views. + * pre-commit: Update metadata when committing changes to locations + of annexed files within a view. + * Add progress display for transfers to/from external special remotes. + * unused: Fix to actually detect unused keys when in direct mode. + * fsck: When run with --all or --unused, while .gitattributes + annex.numcopies cannot be honored since it's operating on keys + instead of files, make it honor the global numcopies setting, + and the annex.numcopies git config setting. + * trust, untrust, semitrust, dead: Warn when the trust level is + overridden in .git/config. + * glacier: Do not try to run glacier value create when an existing glacier + remote is enabled. + * fsck: Refuse to do anything if more than one of --incremental, --more, + and --incremental-schedule are given, since it's not clear which option + should win. + * Windows webapp: Can set up box.com, Amazon S3, and rsync.net remotes + * Windows webapp: Can create repos on removable drives. + * Windows: Ensure HOME is set, as needed by bundled cygwin utilities."""]]
\ No newline at end of file diff --git a/doc/news/version_5.20140221/comment_1_d50bff4ee026db3397333e8ded7c5940._comment b/doc/news/version_5.20140221/comment_1_d50bff4ee026db3397333e8ded7c5940._comment new file mode 100644 index 000000000..7b28c905f --- /dev/null +++ b/doc/news/version_5.20140221/comment_1_d50bff4ee026db3397333e8ded7c5940._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawnWvnTWY6LrcPB4BzYEBn5mRTpNhg5EtEg" + nickname="Bence" + subject="Metadata for the win! " + date="2014-02-26T07:03:32Z" + content=""" +Awesome. Finally one can store the public url of a file which is in a special remote. Think about Google Drive: a file uploaded there can be shared from the command line if it's public url is known (and stored in metadata when uploaded). The flickr special remote could also make benefit of this: a file doesn't have to be searched to locate in flickr if the ids are stored in metadata. +"""]] diff --git a/doc/news/version_5.20140227.mdwn b/doc/news/version_5.20140227.mdwn new file mode 100644 index 000000000..57f0c9a53 --- /dev/null +++ b/doc/news/version_5.20140227.mdwn @@ -0,0 +1,32 @@ +git-annex 5.20140227 released with [[!toggle text="these changes"]] +[[!toggleable text=""" + * metadata: Field names limited to alphanumerics and a few whitelisted + punctuation characters to avoid issues with views, etc. + * metadata: Field names are now case insensative. + * When constructing views, metadata is available about the location of the + file in the view's reference branch. Allows incorporating parts of the + directory hierarchy in a view. + For example `git annex view tag=* podcasts/=*` makes a view in the form + tag/showname. + * --metadata field=value can now use globs to match, and matches + case insensatively, the same as git annex view field=value does. + * annex.genmetadata can be set to make git-annex automatically set + metadata (year and month) when adding files. + * Make annex.web-options be used in several places that call curl. + * Fix handling of rsync remote urls containing a username, + including rsync.net. + * Preserve metadata when staging a new version of an annexed file. + * metadata: Support --json + * webapp: Fix creation of box.com and Amazon S3 and Glacier + repositories, broken in 5.20140221. + * webdav: When built with DAV 0.6.0, use the new DAV monad to avoid + locking files, which is not needed by git-annex's use of webdav, and + does not work on Box.com. + * webdav: Fix path separator bug when used on Windows. + * repair: Optimise unpacking of pack files, and avoid repeated error + messages about corrupt pack files. + * Add build dep on regex-compat to fix build on mipsel, which lacks + regex-tdfa. + * Disable test suite on sparc, which is missing optparse-applicative. + * Put non-object tmp files in .git/annex/misctmp, leaving .git/annex/tmp + for only partially transferred objects."""]]
\ No newline at end of file diff --git a/doc/news/version_5.20140306.mdwn b/doc/news/version_5.20140306.mdwn new file mode 100644 index 000000000..ef302495b --- /dev/null +++ b/doc/news/version_5.20140306.mdwn @@ -0,0 +1,34 @@ +git-annex 5.20140306 released with [[!toggle text="these changes"]] +[[!toggleable text=""" + * sync: Fix bug in direct mode that caused a file that was not + checked into git to be deleted when there was a conflicting + merge with a remote. + * webapp: Now supports HTTPS. + * webapp: No longer supports a port specified after --listen, since + it was buggy, and that use case is better supported by setting up HTTPS. + * annex.listen can be configured, instead of using --listen + * annex.startupscan can be set to false to disable the assistant's startup + scan. + * Probe for quvi version at run time. + * webapp: Filter out from Switch Repository list any + repositories listed in autostart file that don't have a + git directory anymore. (Or are bare) + * webapp: Refuse to start in a bare git repository. + * assistant --autostart: Refuse to start in a bare git repository. + * webapp: Don't list the public repository group when editing a + git repository; it only makes sense for special remotes. + * view, vfilter: Add support for filtering tags and values out of a view, + using !tag and field!=value. + * vadd: Allow listing multiple desired values for a field. + * view: Refuse to enter a view when no branch is currently checked out. + * metadata: To only set a field when it's not already got a value, use + -s field?=value + * Run .git/hooks/pre-commit-annex whenever a commit is made. + * sync: Automatically resolve merge conflict between and annexed file + and a regular git file. + * glacier: Pass --region to glacier checkpresent. + * webdav: When built with a new enough haskell DAV (0.6), disable + the http response timeout, which was only 5 seconds. + * webapp: Include no-pty in ssh authorized\_keys lines. + * assistant: Smarter log file rotation, which takes free disk space + into account."""]]
\ No newline at end of file diff --git a/doc/related_software.mdwn b/doc/related_software.mdwn index 024a155e3..66abad8df 100644 --- a/doc/related_software.mdwn +++ b/doc/related_software.mdwn @@ -10,3 +10,4 @@ designed to interoperate with it. * [sizes](http://hackage.haskell.org/package/sizes) is another du-like utility, with a `-A` switch that enables git-annex support. * Emacs Org mode can auto-commit attached files to git-annex. +* [git annex darktable integration](https://github.com/xxv/darktable-git-annex) diff --git a/doc/tips/Internet_Archive_via_S3.mdwn b/doc/tips/Internet_Archive_via_S3.mdwn index eba28961d..15f241c9f 100644 --- a/doc/tips/Internet_Archive_via_S3.mdwn +++ b/doc/tips/Internet_Archive_via_S3.mdwn @@ -15,7 +15,12 @@ from archive.org, without needing any login or password info. This makes the Internet Archive a nice way to publish the large files associated with a public git repository. ----- +## webapp setup + +Just go to "Add Another Repository", pick "Internet Archive", +and you're on your way. + +## basic setup Sign up for an account, and get your access keys here: <http://www.archive.org/account/s3.php> @@ -40,7 +45,7 @@ specify `x-archive-meta*` headers to add metadata as explained in their Then you can annex files and copy them to the remote as usual: - # git annex add photo1.jpeg --backend=SHA1E + # git annex add photo1.jpeg --backend=SHA256E add photo1.jpeg (checksum...) ok # git annex copy photo1.jpeg --fast --to archive-panama copy (to archive-panama...) ok @@ -50,9 +55,31 @@ from it. Also, git-annex whereis will tell you a public url for the file on archive.org. (It may take a while for archive.org to make the file publically visibile.) -Note the use of the SHA1E [[backend|backends]] when adding files. That is +Note the use of the SHA256E [[backend|backends]] when adding files. That is the default backend used by git-annex, but even if you don't normally use -it, it makes most sense to use the WORM or SHA1E backend for files that +it, it makes most sense to use the WORM or SHA256E backend for files that will be stored in the Internet Archive, since the key name will be exposed as the filename there, and since the Archive does special processing of files based on their extension. + +## publishing only one subdirectory + +Perhaps you have a repository with lots of files in it, and only want +to publish some of them to a particular Internet Archive item. Of course +you can specify which files to send manually, but it's useful to +configure [[preferred_content]] settings so git-annex knows what content +you want to store in the Internet Archive. + +One way to do this is using the "public" repository type. + + git annex enableremote archive-panama preferreddir=panama + git annex wanted archive-panama standard + git annex group archive-panama public + +Now anything in a "panama" directory will be sent to that remote, +and anything else won't. You can use `git annex copy --auto` or the +assistant and it'll do the right thing. + +When setting up an Internet Archive item using the webapp, this +configuration is automatically done, using an item name that the user +enters as the name of the subdirectory. diff --git a/doc/tips/automatically_adding_metadata.mdwn b/doc/tips/automatically_adding_metadata.mdwn new file mode 100644 index 000000000..c3f50bb39 --- /dev/null +++ b/doc/tips/automatically_adding_metadata.mdwn @@ -0,0 +1,24 @@ +git-annex's [[metadata]] works best when files have a lot of useful +metadata attached to them. + +To make git-annex automatically set the year and month when adding files, +run `git config annex.genmetadata true`. + +A git commit hook can be set up to extract lots of metadata from files +like photos, mp3s, etc. + +1. Install the `extract` utility, from <http://www.gnu.org/software/libextractor/> + `apt-get install extract` +2. Download [[pre-commit-annex]] and install it in your git-annex repository + as `.git/hooks/pre-commit-annex`. + Remember to make the script executable! +3. Run: `git config metadata.extract "artist album title camera_make video_dimensions"` + +Now any fields you list in metadata.extract to will be extracted and +stored when files are committed. + +To get a list of all possible fields, run: `extract -L | sed ' ' _` + +By default, if a git-annex already has a metadata field for a file, +its value will not be overwritten with metadata taken from files. +To allow overwriting, run: `git config metadata.overwrite true` diff --git a/doc/tips/automatically_adding_metadata/comment_1_ffc308cc6aedabbc55820db4f401e0fb._comment b/doc/tips/automatically_adding_metadata/comment_1_ffc308cc6aedabbc55820db4f401e0fb._comment new file mode 100644 index 000000000..59f0e958a --- /dev/null +++ b/doc/tips/automatically_adding_metadata/comment_1_ffc308cc6aedabbc55820db4f401e0fb._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" + nickname="Jimmy" + subject="comment 1" + date="2014-03-03T07:44:29Z" + content=""" +http://projects.iq.harvard.edu/fits might be an even better choice than libextractor. We use it in work and its not too bad, but it can be slow to startup due to the JVM. +"""]] diff --git a/doc/tips/automatically_adding_metadata/pre-commit-annex b/doc/tips/automatically_adding_metadata/pre-commit-annex new file mode 100755 index 000000000..f300bd731 --- /dev/null +++ b/doc/tips/automatically_adding_metadata/pre-commit-annex @@ -0,0 +1,57 @@ +#!/bin/sh +# This script can be used to add git-annex metadata to files when they're +# committed. +# +# Copyright 2014 Joey Hess <id@joeyh.name> +# License: GPL-3+ + +extract="$(git config metadata.extract || true)" +want="$(perl -e 'print (join("|", map {s/_/ /g; "^$_ - "} (split " ", shift())))' "$extract")" + +if [ -z "$want" ]; then + exit 0 +fi + +echo "$want" + +case "$(git config --bool metadata.overwrite || true)" in + true) + overwrite=1 + ;; + *) + overwrite="" + ;; +esac + +addmeta () { + file="$1" + field="$2" + value="$3" + afield="$(echo "$field" | tr ' ' _)" + if [ "$overwrite" ]; then + p="$afield=$value" + + else + p="$afield?=$value" + fi + git -c annex.alwayscommit=false annex metadata "$file" -s "$p" --quiet +} + +if git rev-parse --verify HEAD >/dev/null 2>&1; then + against=HEAD +else + # Initial commit: diff against an empty tree object + against=4b825dc642cb6eb9a060e54bf8d69288fbee4904 +fi + +IFS=" +" +for f in $(git diff-index --name-only --cached $against); do + if [ -e "$f" ]; then + for l in $(extract "$f" | egrep "$want"); do + field="${l%% - *}" + value="${l#* - }" + addmeta "$f" "$field" "$value" + done + fi +done diff --git a/doc/tips/metadata_driven_views.mdwn b/doc/tips/metadata_driven_views.mdwn index 7b46ca974..17ebc6869 100644 --- a/doc/tips/metadata_driven_views.mdwn +++ b/doc/tips/metadata_driven_views.mdwn @@ -1,5 +1,5 @@ git-annex now has support for storing -[[arbitrary metadata|design/metadata]] about annexed files. For example, this can be +[[arbitrary metadata|metadata]] about annexed files. For example, this can be used to tag files, to record the author of a file, etc. The metadata is synced around between repositories with the other information git-annex keeps track of. @@ -14,6 +14,12 @@ refine or reorder a view. Let's get started by setting some tags on files. No views yet, just some metadata: +[[!template id=note text=""" +To avoid needing to manually tag files with the year (and month), +run `annex.genmetadata true`, and git-annex will do it for you +when adding files. +"""]] + # git annex metadata --tag todo work/2014/* # git annex metadata --untag todo work/2014/done/* # git annex metadata --tag urgent work/2014/presentation_for_tomorrow.odt @@ -24,8 +30,8 @@ metadata: # git annex metadata --tag done videos/old # git annex metadata --tag new videos/lotsofcats.ogv # git annex metadata --tag sound podcasts - # git annex metadata --tag done podcasts/old - # git annex metadata --tag new podcasts/recent + # git annex metadata --tag done podcasts/*/old + # git annex metadata --tag new podcasts/*/recent So, you had a bunch of different kinds of files sorted into a directory structure. But that didn't really reflect how you approach the files. @@ -39,6 +45,12 @@ Ok, metadata is in place, but how to use it? Time to change views! Switched to branch 'views/_' ok +[[!template id=note text=""" +Notice that a single file may appear in multiple directories +depending on its tags. For example, `lotsofcats.ogv` is in +both `new/` and `video/`. +"""]] + This searched for all files with any tag, and created a new git branch that sorts the files according to their tags. @@ -51,10 +63,6 @@ that sorts the files according to their tags. video sound -Notice that a single file may appear in multiple directories -depending on its tags. For example, `lotsofcats.ogv` is in -both `new/` and `video/`. - Ah, but you're at work now, and don't want to be distracted by cat videos. Time to filter the view: @@ -81,9 +89,11 @@ all the way out of all views, you'll be back on the regular git branch you originally started from. You can also use `git checkout` to switch between views and other branches. -Beyond simple tags, you can add whatever kinds of metadata you like, and -use that metadata in more elaborate views. For example, let's add a year -field. +## fields + +Beyond simple tags and directories, you can add whatever kinds of metadata +you like, and use that metadata in more elaborate views. For example, let's +add a year field. # git checkout master # git annex metadata --set year=2014 work/2014 @@ -118,4 +128,25 @@ Oh, did you want it the other way around? Easy! |-- 2014 `-- 2013 +## location fields + +Let's switch to a view containing only new podcasts. And since the +podcasts are organized into one subdirectory per show, let's +include those subdirectories in the view. + + # git checkout master + # git annex view tag=new podcasts/=* + # tree -d + This_Developers_Life + Escape_Pod + GitMinutes + The_Haskell_Cast + StarShipSofa + +That's an example of using part of the directory layout of the original +branch to inform the view. Every file gets fields automatically set up +corresponding to the directory it's in. So a file"foo/bar/baz/file" has +fields "/=foo", "foo/=bar", and "foo/bar/=baz". These location fields +can be used the same as other metadata to construct the view. + This has probably only scratched the surface of what you can do with views. diff --git a/doc/tips/remote_webapp_setup.mdwn b/doc/tips/remote_webapp_setup.mdwn new file mode 100644 index 000000000..57993df76 --- /dev/null +++ b/doc/tips/remote_webapp_setup.mdwn @@ -0,0 +1,49 @@ +Here's the scenario: You have a remote server you can ssh into, +and you want to use the git-annex webapp there, displaying back on your local +web browser. + +Sure, no problem! It can even be done securely! + +Let's start by making the git-annex repository on the remote server. + + git init annex + cd annex + git annex init + +Now, you need to generate a private key and a certificate for HTTPS. +These files are stored in `.git/annex/privkey.pem` and +`.git/annex/certificate.pem` inside the git repository. Here's +one way to generate those files, using a self-signed certificate: + + (umask 077 ; openssl genrsa -out .git/annex/privkey.pem 4096) + openssl req -new -x509 -key .git/annex/privkey.pem > .git/annex/certificate.pem + +With those files in place, git-annex will automatically only accept HTTPS +connections. That's good, since HTTP connections are not secure over the +big bad internet. + +All that remains is to make the webapp listen on the external interface +of the server. Normally, for security, git-annex only listens on localhost. +Tell it what hostname to listen on: + + git config annex.listen host.example.com + +(If your hostname doesn't work, its IP address certianly will..) + +When you run the webapp configured like that, it'll print out the +URL to use to open it. You can paste that into your web browser. + + git annex webapp + http://host.example.com:42232/?auth=ea7857ad... + +Notice that the URL has a big jumble of letters at the end -- this is a +secret token that the webapp uses to verify you're you. So random attackers +can't find your webapp and do bad things with it. + +If you like, you can make the server run `git annex assistant --autostart` +on boot. + +To automate opening the remote server's webapp in your local browser, +just run this: + + firefox "$(ssh host.example.com git annex webapp)" diff --git a/doc/tips/using_Amazon_Glacier/comment_5_7788890f58f714b0cdf1462c718ea536._comment b/doc/tips/using_Amazon_Glacier/comment_5_7788890f58f714b0cdf1462c718ea536._comment new file mode 100644 index 000000000..5e0e6c716 --- /dev/null +++ b/doc/tips/using_Amazon_Glacier/comment_5_7788890f58f714b0cdf1462c718ea536._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.172" + subject="good" + date="2014-02-21T15:20:39Z" + content=""" +If you mean the creds are not remembered, that's controlled by the embedcreds= option to initremote, and it only defaults to embedding them for gacier when using strong encryption (not encryption=shared). +"""]] diff --git a/doc/tips/using_Amazon_Glacier/comment_6_0fbe528a57552622e8128196ad80c863._comment b/doc/tips/using_Amazon_Glacier/comment_6_0fbe528a57552622e8128196ad80c863._comment new file mode 100644 index 000000000..65128b72c --- /dev/null +++ b/doc/tips/using_Amazon_Glacier/comment_6_0fbe528a57552622e8128196ad80c863._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://grossmeier.net/" + nickname="greg" + subject="confirmed" + date="2014-02-21T17:53:02Z" + content=""" +Yeah, I choose no encryption for this one for worst case scenario reasons (I still want photos of my kid even if I loss my gpg key and my house burns down). Now about setting up http://git.kitenet.net/?p=gpg.git;a=blob;f=README.sss;hb=HEAD ...... +"""]] diff --git a/doc/todo/Views_Demo.mdwn b/doc/todo/Views_Demo.mdwn new file mode 100644 index 000000000..2587642e3 --- /dev/null +++ b/doc/todo/Views_Demo.mdwn @@ -0,0 +1,13 @@ +Joey, + +I've been thinking about leveraging git-annex for a workgroup document repository and I have just watched your views demo. The timing of the demo is great because I need to deploy a document repository with per-document metadata and your views concept seems like a great mechanism for associating metadata to documents and for displaying that metadata. + +While I don't expect to use your views concept for my workgroup repostory, a later iteration might do. + +The metadata in my use case begins with all the weird metadata seen on a book's copyright page. In addition, per-document provenance, like how one found the document and (if we're lucky) a URL where the latest version of the document may be found. Metadata values may be simple strings or may be markdown text. + +So, are you considering a metadata syntax that can support complex metadata? One example is multiple authors. Another issue is complex metadata values, like key=abstract and value="markdown text...". + +FWIW, + +Bob diff --git a/doc/todo/Views_Demo/comment_1_d7c83a0e9a83e4a05aa74a34a7e1cf19._comment b/doc/todo/Views_Demo/comment_1_d7c83a0e9a83e4a05aa74a34a7e1cf19._comment new file mode 100644 index 000000000..4c9b05635 --- /dev/null +++ b/doc/todo/Views_Demo/comment_1_d7c83a0e9a83e4a05aa74a34a7e1cf19._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.172" + subject="comment 1" + date="2014-02-24T18:17:04Z" + content=""" +All that should work fine. All metadata fields are multivalued, and the value can be any arbitrary data. +"""]] diff --git a/doc/todo/ctrl_c_handling.mdwn b/doc/todo/ctrl_c_handling.mdwn new file mode 100644 index 000000000..7101d578f --- /dev/null +++ b/doc/todo/ctrl_c_handling.mdwn @@ -0,0 +1,5 @@ +Sometimes I start off a large file transfer to a new remote (a la "git-annex copy . --to glacier"). + +I believe all of the special remotes transfer the files one at a time, which is good, and provides a sensible place to interrupt a copy/move operation. + +Wish: When I press ctrl+c in the terminal, git-annex will catch that and finish it's current transfer and then exit cleanly (ie: no odd backtraces in the special remote code). For the case where the file currently being transfered also needs to be killed (ie: it's a big .iso) then subsequent ctrl+c's can do that. diff --git a/doc/todo/ctrl_c_handling/comment_1_3addbe33817db5de836c014287b14c07._comment b/doc/todo/ctrl_c_handling/comment_1_3addbe33817db5de836c014287b14c07._comment new file mode 100644 index 000000000..16139c78d --- /dev/null +++ b/doc/todo/ctrl_c_handling/comment_1_3addbe33817db5de836c014287b14c07._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.172" + subject="comment 1" + date="2014-02-21T21:36:14Z" + content=""" +This really depends on the remote, some can resume where they were interrupted, such as rsync, and some cannot, such as glacier (and, er, encrypted rsync). +"""]] diff --git a/doc/todo/ctrl_c_handling/comment_2_cc2776dc4805421180edcdf96a89fcaa._comment b/doc/todo/ctrl_c_handling/comment_2_cc2776dc4805421180edcdf96a89fcaa._comment new file mode 100644 index 000000000..827b99afa --- /dev/null +++ b/doc/todo/ctrl_c_handling/comment_2_cc2776dc4805421180edcdf96a89fcaa._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://grossmeier.net/" + nickname="greg" + subject="very remote specific" + date="2014-02-21T22:11:16Z" + content=""" +Yeah, this is very remote specific and probably means adding the functionality there as well (eg: in the glacier.py code, not only in git-annex haskell). Maybe I should file bugs there accordingly :) +"""]] diff --git a/doc/todo/ctrl_c_handling/comment_3_8d7d357368987f5d5d59b4d8d99a0e06._comment b/doc/todo/ctrl_c_handling/comment_3_8d7d357368987f5d5d59b4d8d99a0e06._comment new file mode 100644 index 000000000..ed7e4d3b6 --- /dev/null +++ b/doc/todo/ctrl_c_handling/comment_3_8d7d357368987f5d5d59b4d8d99a0e06._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.172" + subject="comment 3" + date="2014-02-21T22:34:14Z" + content=""" +Hmm, I forget if it's possible for git-annex to mask SIGINT when it runs glacier or rsync, so that the child process does not receive it, but the parent git-annex does. +"""]] diff --git a/doc/todo/custom_f-droid_repo.mdwn b/doc/todo/custom_f-droid_repo.mdwn new file mode 100644 index 000000000..d4ec3e1f3 --- /dev/null +++ b/doc/todo/custom_f-droid_repo.mdwn @@ -0,0 +1,3 @@ +It would be great to have a custom f-droid repo "alla guardianproject.info" (before getting git-annex into the main f-droid repo). + +See https://github.com/guardianproject/fdroid-repo (https://guardianproject.info/repo/). diff --git a/doc/todo/custom_f-droid_repo/comment_1_d2bdc001584d4b5f925390910ec1ef73._comment b/doc/todo/custom_f-droid_repo/comment_1_d2bdc001584d4b5f925390910ec1ef73._comment new file mode 100644 index 000000000..f1c682fbc --- /dev/null +++ b/doc/todo/custom_f-droid_repo/comment_1_d2bdc001584d4b5f925390910ec1ef73._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.146" + subject="comment 1" + date="2014-03-05T20:45:32Z" + content=""" +The f-droid developers are willing to add git-annex in principle, their build environment just needs to have its build dependencies added to it. I've completely automated (except for occasional breakage due to changes on hackage) to setup of the necessary cross build environment. I have not had time to look into how to integrate that into the f-droid build system, and it would be great if someone with interest could do so. + +I suspect that setting up a custom f-droid repo would be just as much work, plus more work on an ongoing basis. +"""]] diff --git a/doc/todo/custom_f-droid_repo/comment_2_20eebe13b76d5279a3d09b346b65ff6e._comment b/doc/todo/custom_f-droid_repo/comment_2_20eebe13b76d5279a3d09b346b65ff6e._comment new file mode 100644 index 000000000..34aa063f4 --- /dev/null +++ b/doc/todo/custom_f-droid_repo/comment_2_20eebe13b76d5279a3d09b346b65ff6e._comment @@ -0,0 +1,9 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawnR6E5iUghMWdUGlbA9CCs8DKaoigMjJXw" + nickname="Efraim" + subject="comment 2" + date="2014-03-06T10:06:02Z" + content=""" +I've noticed that fdroidserver is available in the main debian repos, and not having really looked at it too much yet, if it shares similar code to wannabuild then it should be possible to continue everything as you're already doing it, just in addition rsync or git push the code base to a local fdroidserver instance and also copy the binary over once it's set up. +(if I understood wannabuild correctly, if you upload new source and a prebuilt binary it will mark that architecture as built.) +"""]] diff --git a/doc/todo/required_content.mdwn b/doc/todo/required_content.mdwn new file mode 100644 index 000000000..851e652ae --- /dev/null +++ b/doc/todo/required_content.mdwn @@ -0,0 +1,7 @@ +We have preferred content, which is advisory, and numcopies, which is +enforced (except by `git annex move`). What is missing is an expression +like preferred content, which is enforced. So, required content. + +For example, I might want a repository that is required to contain +`*.jpeg`. This would make get --auto get it (it's implicitly part of the +preferred content), and would make drop refuse to drop it. diff --git a/doc/todo/windows_support.mdwn b/doc/todo/windows_support.mdwn index ea532dfc1..17accd62e 100644 --- a/doc/todo/windows_support.mdwn +++ b/doc/todo/windows_support.mdwn @@ -8,21 +8,7 @@ now! --[[Joey]] or perhaps easier, <http://hackage.haskell.org/package/Win32-services-wrapper> -* XMPP library not yet built. - - This should work to install the deps, using libs from cygwin - - cabal install libxml-sax --extra-lib-dirs=C:\\cygwin\\lib --extra-include-dirs=C:\\cygwin\\usr\\include\\libxml2 - cabal install gnuidn --extra-lib-dirs=C:\\cygwin\\lib --extra-include-dirs=C:\\cygwin\\usr\\include\\ - cabal install gnutls --extra-lib-dirs=C:\\cygwin\\lib --extra-include-dirs=C:\\cygwin\\usr\\include\\ - - While the 1st line works, the rest fail oddly. Looks like lack of - quoting when cabal runs c2hs and gcc, as "Haskell Platform" is - taken as 2 filenames. Needs investigation why this happens here - and not other times.. - - Also needs gsasl, which is not in cygwin. - See <http://josefsson.org/gsasl4win/README.html> +* XMPP library not yet built. (See below.) * View debug log is empty in windows -- all logs go to console. This messes up a few parts of UI that direct user to the debug log. @@ -30,6 +16,8 @@ now! --[[Joey]] (and possibly gpg) are not prompted there anymore. * Local pairing seems to fail, after acking on Linux box, it stalls. + (Also, of course, the Windows box is unlikely to have a ssh server, + so only pairing with a !Windows box will work.) * gcrypt is not ported to windows (and as a shell script, may need to be rewritten) @@ -43,9 +31,6 @@ now! --[[Joey]] ## minor problems -* Does not work with Cygwin's build of git (that git does not consistently - support use of DOS style paths, which git-annex uses on Windows). - Must use Msysgit. * rsync special remotes with a rsyncurl of a local directory are known buggy. (git-annex tells rsync C:foo and it thinks it means a remote host named C...) @@ -55,10 +40,54 @@ now! --[[Joey]] has to connect twice to the remote system over ssh per file, which is much slower than on systems supporting connection caching. * glacier-cli is not easily available (probably) +* user feedback: "Git on windows installer provides openssh 4.6. git-annex installer + provides openssh 6.2 . This seems to create problems regarding how + `known_hosts` file path is setup. Setting `GIT_SSH=` to the git-annex + openssh version fixes the problem." + However, I don't know how to determine what that location is after + it's been installed. Maybe look for ssh.exe in same directory as + git-annex.exe? --[[Joey]] ## stuff needing testing -* test S3 and box.com setup in webapp now that they should work.. * test that adding a repo on a removable drive works; that git is synced to it and files can be transferred to it and back * Does stopping in progress transfers work in the webapp? + +## trying to build XMPP + +Lots of library deps: + +1. gsasl-$LATEST.zip from <http://josefsson.org/gnutls4win/> (includes + gnuidn and gnutls) +2. pkg-config from + <http://sourceforge.net/projects/pkgconfiglite/files/latest/download?source=files> +3. libxml2 from mingw: + <http://sourceforge.net/projects/mingw/files/MSYS/Extension/libxml2/libxml2-2.7.6-1/> + both the -dll and the -dev +3. Extract all the above into the Haskell platform's mingw directory. Note + that pkg-config needs to be moved out of a named subdirectory. +4. Run in DOS prompt (not cygwin!): cabal install network-protocol-xmpp + +Current FAIL: + +<pre> +Loading package gnutls-0.1.5 ... ghc.exe: internal error: Misaligned section: 18206e5b + (GHC version 7.6.3 for i386_unknown_mingw32) + Please report this as a GHC bug: + http://www.haskell.org/ghc/reportabug +</pre> + +<https://ghc.haskell.org/trac/ghc/ticket/8830> + +Note: This only happens in the TH link stage. So building w/o the webapp +works with XMPP. + +Options: + +1. Use EvilSplicer, building first without XMPP library, but with its UI, + and a second time without TH, but with the XMPP library. Partially done + on the `winsplicehack` branch, but requires building patched versions + of lots of yesod dependency chain to export modules referenced by TH + splices, like had to be done on Android. Horrible pain. Ugly as hell. +2. Make a helper program with the XMPP support in it, that does not use TH. diff --git a/git-annex.cabal b/git-annex.cabal index d982b6d06..2f3b7a2eb 100644 --- a/git-annex.cabal +++ b/git-annex.cabal @@ -1,5 +1,5 @@ Name: git-annex -Version: 5.20140210 +Version: 5.20140306 Cabal-Version: >= 1.8 License: GPL-3 Maintainer: Joey Hess <joey@kitenet.net> @@ -93,7 +93,8 @@ Executable git-annex extensible-exceptions, dataenc, SHA, process, json, base (>= 4.5 && < 4.9), monad-control, MonadCatchIO-transformers, IfElse, text, QuickCheck >= 2.1, bloomfilter, edit-distance, process, - SafeSemaphore, uuid, random, dlist, unix-compat, async, stm (>= 2.3) + SafeSemaphore, uuid, random, dlist, unix-compat, async, stm (>= 2.3), + data-default, case-insensitive CC-Options: -Wall GHC-Options: -Wall Extensions: PackageImports @@ -122,6 +123,8 @@ Executable git-annex if flag(TDFA) Build-Depends: regex-tdfa CPP-Options: -DWITH_TDFA + else + Build-Depends: regex-compat if flag(CryptoHash) Build-Depends: cryptohash (>= 0.10.0) @@ -133,7 +136,7 @@ Executable git-annex if flag(WebDAV) Build-Depends: DAV ((>= 0.3 && < 0.6) || > 0.6), - http-conduit, xml-conduit, http-types + http-client, http-conduit, http-types, lifted-base CPP-Options: -DWITH_WEBDAV if flag(Assistant) && ! os(solaris) @@ -173,7 +176,7 @@ Executable git-annex if flag(Webapp) Build-Depends: yesod, yesod-default, yesod-static, yesod-form, yesod-core, - case-insensitive, http-types, transformers, wai, wai-logger, warp, + http-types, transformers, wai, wai-logger, warp, warp-tls, blaze-builder, crypto-api, hamlet, clientsession, template-haskell, data-default, aeson, network-conduit CPP-Options: -DWITH_WEBAPP diff --git a/standalone/windows/build.sh b/standalone/windows/build.sh index 9473b9257..8751e4573 100755 --- a/standalone/windows/build.sh +++ b/standalone/windows/build.sh @@ -28,7 +28,7 @@ UPGRADE_LOCATION=http://downloads.kitenet.net/git-annex/windows/current/git-anne # Don't allow build artifact from a past successful build to be extracted # if we fail. -#rm -f git-annex-installer.exe +rm -f git-annex-installer.exe # Install haskell dependencies. # cabal install is not run in cygwin, because we don't want configure scripts |