diff options
| -rw-r--r-- | Annex/CheckAttr.hs | 1 | ||||
| -rw-r--r-- | Annex/FileMatcher.hs | 24 | ||||
| -rw-r--r-- | Assistant/Threads/Watcher.hs | 13 | ||||
| -rw-r--r-- | Command/Import.hs | 3 | ||||
| -rw-r--r-- | Git/CheckAttr.hs | 3 | ||||
| -rw-r--r-- | Limit.hs | 6 | ||||
| -rw-r--r-- | debian/changelog | 4 | ||||
| -rw-r--r-- | doc/git-annex.mdwn | 91 |
8 files changed, 107 insertions, 38 deletions
diff --git a/Annex/CheckAttr.hs b/Annex/CheckAttr.hs index 28c5ffedd..2be95483e 100644 --- a/Annex/CheckAttr.hs +++ b/Annex/CheckAttr.hs @@ -20,6 +20,7 @@ annexAttrs :: [Git.Attr] annexAttrs = [ "annex.backend" , "annex.numcopies" + , "annex.largefiles" ] checkAttr :: Git.Attr -> FilePath -> Annex String diff --git a/Annex/FileMatcher.hs b/Annex/FileMatcher.hs index a8e431acf..4c7541b46 100644 --- a/Annex/FileMatcher.hs +++ b/Annex/FileMatcher.hs @@ -1,6 +1,6 @@ {- git-annex file matching - - - Copyright 2012-2014 Joey Hess <id@joeyh.name> + - Copyright 2012-2016 Joey Hess <id@joeyh.name> - - Licensed under the GNU GPL version 3 or higher. -} @@ -19,12 +19,18 @@ import qualified Annex import Types.FileMatcher import Git.FilePath import Types.Remote (RemoteConfig) +import Annex.CheckAttr +import Git.CheckAttr (unspecifiedAttr) import Data.Either import qualified Data.Set as S -checkFileMatcher :: FileMatcher Annex -> FilePath -> Annex Bool -checkFileMatcher matcher file = checkMatcher matcher Nothing (Just file) S.empty True +type GetFileMatcher = FilePath -> Annex (FileMatcher Annex) + +checkFileMatcher :: GetFileMatcher -> FilePath -> Annex Bool +checkFileMatcher getmatcher file = do + matcher <- getmatcher file + checkMatcher matcher Nothing (Just file) S.empty True checkMatcher :: FileMatcher Annex -> Maybe Key -> AssociatedFile -> AssumeNotPresent -> Bool -> Annex Bool checkMatcher matcher mkey afile notpresent d @@ -104,11 +110,19 @@ tokenizeMatcher = filter (not . null ) . concatMap splitparens . words {- Generates a matcher for files large enough (or meeting other criteria) - to be added to the annex, rather than directly to git. -} -largeFilesMatcher :: Annex (FileMatcher Annex) +largeFilesMatcher :: Annex GetFileMatcher largeFilesMatcher = go =<< annexLargeFiles <$> Annex.getGitConfig where - go Nothing = return matchAll go (Just expr) = do + matcher <- mkmatcher expr + return $ const $ return matcher + go Nothing = return $ \file -> do + expr <- checkAttr "annex.largefiles" file + if null expr || expr == unspecifiedAttr + then return matchAll + else mkmatcher expr + + mkmatcher expr = do u <- getUUID -- No need to read remote configs, that's only needed for -- inpreferreddir, which is used in preferred content diff --git a/Assistant/Threads/Watcher.hs b/Assistant/Threads/Watcher.hs index 1fc20a906..f0a639984 100644 --- a/Assistant/Threads/Watcher.hs +++ b/Assistant/Threads/Watcher.hs @@ -35,7 +35,6 @@ import Annex.CatFile import Annex.CheckIgnore import Annex.Link import Annex.FileMatcher -import Types.FileMatcher import Annex.Content import Annex.ReplaceFile import Annex.Version @@ -202,8 +201,8 @@ runHandler handler file filestatus = void $ do | otherwise = f {- Small files are added to git as-is, while large ones go into the annex. -} -add :: FileMatcher Annex -> FilePath -> Assistant (Maybe Change) -add bigfilematcher file = ifM (liftAnnex $ checkFileMatcher bigfilematcher file) +add :: GetFileMatcher -> FilePath -> Assistant (Maybe Change) +add largefilematcher file = ifM (liftAnnex $ checkFileMatcher largefilematcher file) ( pendingAddChange file , do liftAnnex $ Annex.Queue.addCommand "add" @@ -211,7 +210,7 @@ add bigfilematcher file = ifM (liftAnnex $ checkFileMatcher bigfilematcher file) madeChange file AddFileChange ) -onAdd :: FileMatcher Annex -> Handler +onAdd :: GetFileMatcher -> Handler onAdd matcher file filestatus | maybe False isRegularFile filestatus = unlessIgnored file $ @@ -221,7 +220,7 @@ onAdd matcher file filestatus shouldRestage :: DaemonStatus -> Bool shouldRestage ds = scanComplete ds || forceRestage ds -onAddUnlocked :: Bool -> FileMatcher Annex -> Handler +onAddUnlocked :: Bool -> GetFileMatcher -> Handler onAddUnlocked = onAddUnlocked' False contentchanged addassociatedfile samefilestatus where addassociatedfile key file = @@ -243,10 +242,10 @@ onAddUnlocked = onAddUnlocked' False contentchanged addassociatedfile samefilest {- In direct mode, add events are received for both new files, and - modified existing files. -} -onAddDirect :: Bool -> FileMatcher Annex -> Handler +onAddDirect :: Bool -> GetFileMatcher -> Handler onAddDirect = onAddUnlocked' True changedDirect (\k f -> void $ addAssociatedFile k f) sameFileStatus -onAddUnlocked' :: Bool -> (Key -> FilePath -> Annex ()) -> (Key -> FilePath -> Annex ()) -> (Key -> FilePath -> FileStatus -> Annex Bool) -> Bool -> FileMatcher Annex -> Handler +onAddUnlocked' :: Bool -> (Key -> FilePath -> Annex ()) -> (Key -> FilePath -> Annex ()) -> (Key -> FilePath -> FileStatus -> Annex Bool) -> Bool -> GetFileMatcher -> Handler onAddUnlocked' isdirect contentchanged addassociatedfile samefilestatus symlinkssupported matcher file fs = do v <- liftAnnex $ catKeyFile file case (v, fs) of diff --git a/Command/Import.hs b/Command/Import.hs index 847841930..d5a2feed5 100644 --- a/Command/Import.hs +++ b/Command/Import.hs @@ -17,7 +17,6 @@ import Remote import Types.KeySource import Annex.CheckIgnore import Annex.NumCopies -import Types.FileMatcher import Annex.FileMatcher cmd :: Command @@ -67,7 +66,7 @@ seek o = allowConcurrentOutput $ do largematcher <- largeFilesMatcher withPathContents (start largematcher (duplicateMode o)) (importFiles o) -start :: FileMatcher Annex -> DuplicateMode -> (FilePath, FilePath) -> CommandStart +start :: GetFileMatcher -> DuplicateMode -> (FilePath, FilePath) -> CommandStart start largematcher mode (srcfile, destfile) = ifM (liftIO $ isRegularFile <$> getSymbolicLinkStatus srcfile) ( do diff --git a/Git/CheckAttr.hs b/Git/CheckAttr.hs index 23ed22621..465bbbad3 100644 --- a/Git/CheckAttr.hs +++ b/Git/CheckAttr.hs @@ -92,3 +92,6 @@ checkAttr (h, attrs, oldgit, currdir) want file = do sep = ": " ++ attr ++ ": " getattrvalues (_filename:attr:val:rest) c = getattrvalues rest ((attr,val):c) getattrvalues _ c = c + +unspecifiedAttr :: String +unspecifiedAttr = "!" @@ -200,10 +200,14 @@ limitUnused _ (MatchingInfo _ ak _) = do k <- getInfo ak S.member k <$> unusedKeys -{- Limit that matches any version of any file. -} +{- Limit that matches any version of any file or key. -} limitAnything :: MatchFiles Annex limitAnything _ _ = return True +{- Limit that never matches. -} +limitNothing :: MatchFiles Annex +limitNothing _ _ = return False + {- Adds a limit to skip files not believed to be present in all - repositories in the specified group. -} addInAllGroup :: String -> Annex () diff --git a/debian/changelog b/debian/changelog index b3491f63c..a834e3de1 100644 --- a/debian/changelog +++ b/debian/changelog @@ -6,6 +6,10 @@ git-annex (6.20160127) UNRELEASED; urgency=medium caused them to sometimes output side messages. * webapp: Fix deletion of current repository directory. * Added "nothing" to preferred content expression syntax. + * annex.largefiles can be configured in .gitattributes too; + this is particulary useful for v6 repositories, since the + .gitattributes configuration will apply in all clones of the + repository. -- Joey Hess <id@joeyh.name> Thu, 28 Jan 2016 13:53:09 -0400 diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index 471f21957..77a2b38e9 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -795,6 +795,9 @@ Here are all the supported configuration settings. Space-separated list of names of the key-value backends to use. The first listed is used to store new files by default. + This is overridden by annex annex.backend configuration in the + .gitattributes files. + * `annex.diskreserve` Amount of disk space to reserve. Disk space is checked when transferring @@ -807,26 +810,7 @@ Here are all the supported configuration settings. * `annex.largefiles` - Allows configuring which files are considered to be large enough to - need to be added to the annex. By default, all specified files are added - to the annex, but configuring this can cause the small files to be - checked into git, without using the annex. - - The value is a preferred content expression. See [[git-annex-preferred-content]](1) - for details. - - Example: - - annex.largefiles = largerthan=100kb and not (include=*.c or include=*.h) - - This setting is checked by `git annex add`, `git annex import` and the assistant. - It's also used by `git annex addurl` and `git annex importfeed` when - downloading files. - - It can be useful to temporarily override it via -c at the command line. - For example: - - git annex add -c annex.largefiles='include=*' 99kbfile + See "LARGE FILES CONFIGURATION" below. * `annex.addsmallfiles` @@ -838,10 +822,12 @@ Here are all the supported configuration settings. This is a deprecated setting. You should instead use the `git annex numcopies` command to configure how many copies of files - are kept across all repositories. + are kept across all repositories, or the annex.numcopies .gitattributes + setting. This config setting is only looked at when `git annex numcopies` has - never been configured. + never been configured, and when there's no annex.numcopies setting in the + .gitattributes file. Note that setting numcopies to 0 is very unsafe. @@ -1304,6 +1290,9 @@ but the SHA256E backend for ogg files: * annex.backend=WORM *.ogg annex.backend=SHA256E +There is a annex.largefiles attribute; see "LARGE FILES CONFIGURATION" +below. + The numcopies setting can also be configured on a per-file-type basis via the `annex.numcopies` attribute in `.gitattributes` files. This overrides other numcopies settings. @@ -1318,12 +1307,68 @@ Note that setting numcopies to 0 is very unsafe. These settings are honored by git-annex whenever it's operating on a matching file. However, when using --all, --unused, or --key to specify keys to operate on, git-annex is operating on keys and not files, so will -not honor the settings from .gitattributes. +not honor the settings from .gitattributes. For this reason, the `git annex +numcopies` command is useful to configure a global default for numcopies. Also note that when using views, only the toplevel .gitattributes file is preserved in the view, so other settings in other files won't have any effect. +# LARGE FILES CONFIGURATION + +Normally commands like `git annex add` always add files to the annex. +And when using the v6 repository mode, even `git add` and `git commit -a` +will add files to the annex. + +However, sometimes it's useful to keep the content of some smaller files in +git, any only annex the larger files. For example, a game's code should be +committed to git while its artwork is stored in the annex. + +The annex.largefiles configuration meets this need. It's checked by +`git annex add`, by `git add` and `git commit -a` (in v6 repositories), +by `git annex import` and the assistant. It's also used by +`git annex addurl` and `git annex importfeed` when downloading files. +When a file does not match annex.largefiles, it will be added to git instead +of to the annex. + +There are two ways to configure annex.largefiles. Setting it in the +`.gitattributes` file is recommended to consistently use the same +configuration across different checkouts of the repository. Setting the +annex.largefiles git configuration lets different checkouts behave +differently. The git configuration overrides the `.gitattributes` +configuration. + +For example, in the game scenario, here's how to make only files of a +certian size be annexed, and never source code files: + + git config annex.largefiles 'largerthan=100kb and (not include=*.c)' + +The value is a preferred content expression. +See [[git-annex-preferred-content]](1) for details. + +To configure the same thing in the `.gitattributes` file, looks a little +bit different: + + * annex.largefiles=(largerthan=100kb) + *.c annex.largefiles=nothing + +That has the same effect as the git configuration, because the attribute for +*.c overrides the previous attribute. + +Note that, since git attribute values cannot contain whitespace, +it's useful to instead parenthesize the terms of the preferred content +expression. This trick allows setting the annex.largefiles attribute to more +complicated expressions. For example, this is the same as the git config +shown earlier: + + * annex.largefiles=(largerthan=100kb)and(not(include=*.c)) + +By the way, if you've set up an annex.largefiles configuration but want to +force a file to be stored in the annex, you can temporarily override the +configuration like this: + + git annex add -c annex.largefiles=anything smallfile + # EXIT STATUS git-annex, when called as a git subcommand, may return exit codes 0 or 1 |