diff options
| -rw-r--r-- | Remote.hs | 2 | ||||
| -rw-r--r-- | Remote/Bup.hs | 133 | ||||
| -rw-r--r-- | TestConfig.hs | 30 | ||||
| -rw-r--r-- | configure.hs | 9 | ||||
| -rw-r--r-- | debian/changelog | 1 | ||||
| -rw-r--r-- | debian/control | 2 | ||||
| -rw-r--r-- | doc/git-annex.mdwn | 10 | ||||
| -rw-r--r-- | doc/not.mdwn | 6 | ||||
| -rw-r--r-- | doc/special_remotes.mdwn | 1 | ||||
| -rw-r--r-- | doc/special_remotes/bup.mdwn | 34 | ||||
| -rw-r--r-- | doc/walkthrough.mdwn | 1 | ||||
| -rw-r--r-- | doc/walkthrough/using_bup.mdwn | 22 |
12 files changed, 226 insertions, 25 deletions
@@ -46,12 +46,14 @@ import Config import qualified Remote.Git import qualified Remote.S3 +import qualified Remote.Bup import qualified Remote.Directory remoteTypes :: [RemoteType Annex] remoteTypes = [ Remote.Git.remote , Remote.S3.remote + , Remote.Bup.remote , Remote.Directory.remote ] diff --git a/Remote/Bup.hs b/Remote/Bup.hs new file mode 100644 index 000000000..ef34e2c63 --- /dev/null +++ b/Remote/Bup.hs @@ -0,0 +1,133 @@ +{- Using bup as a remote. + - + - Copyright 2011 Joey Hess <joey@kitenet.net> + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Remote.Bup (remote) where + +import IO +import Control.Exception.Extensible (IOException) +import qualified Data.Map as M +import Control.Monad (unless) +import Control.Monad.State (liftIO) +import System.Process +import System.Exit + +import RemoteClass +import Types +import qualified GitRepo as Git +import qualified Annex +import UUID +import Locations +import LocationLog +import Config +import Utility +import Messages +import Remote.Special + +remote :: RemoteType Annex +remote = RemoteType { + typename = "bup", + enumerate = findSpecialRemotes "bupremote", + generate = gen, + setup = bupSetup +} + +gen :: Git.Repo -> UUID -> Maybe (M.Map String String) -> Annex (Remote Annex) +gen r u c = do + cst <- remoteCost r expensiveRemoteCost + bupremote <- getConfig r "bupremote" (error "missing bupremote") + return $ this cst bupremote + where + this cst bupremote = Remote { + uuid = u, + cost = cst, + name = Git.repoDescribe r, + storeKey = store r bupremote, + retrieveKeyFile = retrieve bupremote, + removeKey = remove, + hasKey = checkPresent u, + hasKeyCheap = True, + config = c + } + +bupSetup :: UUID -> M.Map String String -> Annex (M.Map String String) +bupSetup u c = do + -- verify configuration is sane + let bupremote = case M.lookup "remote" c of + Nothing -> error "Specify remote=" + Just r -> r + case M.lookup "encryption" c of + Nothing -> error "Specify encryption=key or encryption=none" + Just "none" -> return () + Just _ -> error "encryption keys not yet supported" + + -- bup init will create the repository. + -- (If the repository already exists, bup init again appears safe.) + showNote "bup init" + ok <- bup "init" bupremote [] + unless ok $ error "bup init failed" + + -- The bup remote is stored in git config, as well as this remote's + -- persistant state, so it can vary between hosts. + gitConfigSpecialRemote u c "bupremote" bupremote + + return $ M.delete "directory" c + +bupParams :: String -> String -> [CommandParam] -> [CommandParam] +bupParams command bupremote params = + (Param command) : [Param "-r", Param bupremote] ++ params + +bup :: String -> String -> [CommandParam] -> Annex Bool +bup command bupremote params = do + showProgress -- make way for bup output + liftIO $ boolSystem "bup" $ bupParams command bupremote params + +store :: Git.Repo -> String -> Key -> Annex Bool +store r bupremote k = do + g <- Annex.gitRepo + let src = gitAnnexLocation g k + o <- getConfig r "bup-split-options" "" + let os = map Param $ words o + bup "split" bupremote $ os ++ [Param "-n", Param (show k), File src] + +retrieve :: String -> Key -> FilePath -> Annex Bool +retrieve bupremote k f = do + let params = bupParams "join" bupremote [Param $ show k] + ret <- liftIO $ try $ do + -- pipe bup's stdout directly to file + tofile <- openFile f WriteMode + p <- runProcess "bup" (toCommand params) + Nothing Nothing Nothing (Just tofile) Nothing + r <- waitForProcess p + case r of + ExitSuccess -> return True + _ -> return False + case ret of + Right r -> return r + Left e -> return False + +remove :: Key -> Annex Bool +remove _ = do + warning "content cannot be removed from bup remote" + return False + +{- Bup does not provide a way to tell if a given dataset is present + - in a bup repository. One way it to check if the git repository has + - a branch matching the name (as created by bup split -n). + - + - However, git-annex's ususal reasons for checking if a remote really + - has a key also don't really apply in the case of bup, since, short + - of deleting bup's git repository, data cannot be removed from it. + - + - So, trust git-annex's location log; if it says a bup repository has + - content, assume it's right. + -} +checkPresent :: UUID -> Key -> Annex (Either IOException Bool) +checkPresent u k = do + g <- Annex.gitRepo + liftIO $ try $ do + uuids <- keyLocations g k + return $ u `elem` uuids diff --git a/TestConfig.hs b/TestConfig.hs index 9b2759e19..bab297003 100644 --- a/TestConfig.hs +++ b/TestConfig.hs @@ -72,26 +72,28 @@ testCmd k cmdline = do {- Ensures that one of a set of commands is available by running each in - turn. The Config is set to the first one found. -} -selectCmd :: Bool -> ConfigKey -> [String] -> String -> Test -selectCmd required k cmds param = search cmds +selectCmd :: ConfigKey -> [String] -> String -> Test +selectCmd k = searchCmd + (\match -> return $ Config k $ StringConfig match) + (\cmds -> do + testEnd $ Config k $ BoolConfig False + error $ "* need one of these commands, but none are available: " ++ show cmds + ) + +maybeSelectCmd :: ConfigKey -> [String] -> String -> Test +maybeSelectCmd k = searchCmd + (\match -> return $ Config k $ MaybeStringConfig $ Just match) + (\_ -> return $ Config k $ MaybeStringConfig Nothing) + +searchCmd :: (String -> Test) -> ([String] -> Test) -> [String] -> String -> Test +searchCmd success failure cmds param = search cmds where - search [] = failure + search [] = failure cmds search (c:cs) = do ret <- system $ quiet c ++ " " ++ param if (ret == ExitSuccess) then success c else search cs - success c - | required == True = return $ Config k (StringConfig c) - | otherwise = return $ Config k (MaybeStringConfig $ Just c) - failure - | required == True = do - testEnd $ Config k (BoolConfig False) - error $ "* need one of these commands, but none are available: " ++ show cmds - | otherwise = do - let r = Config k (MaybeStringConfig Nothing) - testEnd r - return r quiet :: String -> String quiet s = s ++ " >/dev/null 2>&1" diff --git a/configure.hs b/configure.hs index 0661813ae..4ab305239 100644 --- a/configure.hs +++ b/configure.hs @@ -6,15 +6,16 @@ import Data.List import TestConfig tests :: [TestCase] -tests = [ - TestCase "version" $ getVersion +tests = + [ TestCase "version" $ getVersion , testCp "cp_a" "-a" , testCp "cp_p" "-p" , testCp "cp_reflink_auto" "--reflink=auto" - , TestCase "uuid generator" $ selectCmd True "uuid" ["uuid", "uuidgen"] "" + , TestCase "uuid generator" $ selectCmd "uuid" ["uuid", "uuidgen"] "" , TestCase "xargs -0" $ requireCmd "xargs_0" "xargs -0 </dev/null" , TestCase "rsync" $ requireCmd "rsync" "rsync --version >/dev/null" , TestCase "curl" $ testCmd "curl" "curl --version >/dev/null" + , TestCase "bup" $ testCmd "bup" "bup --version >/dev/null" , TestCase "unicode FilePath support" $ unicodeFilePath ] ++ shaTestCases [1, 256, 512, 224, 384] @@ -24,7 +25,7 @@ shaTestCases l = map make l let cmds = map (\x -> "sha" ++ show n ++ x) ["", "sum"] key = "sha" ++ show n - in TestCase key $ selectCmd False key cmds "</dev/null" + in TestCase key $ maybeSelectCmd key cmds "</dev/null" tmpDir :: String tmpDir = "tmp" diff --git a/debian/changelog b/debian/changelog index 7f104be10..91c0c8f4b 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,5 +1,6 @@ git-annex (0.20110402) UNRELEASED; urgency=low + * bup is now supported as a special type of remote. * Use lowercase hash directories for locationlog files, to avoid some issues with git on OSX with the mixed-case directories. No migration is needed; the old mixed case hash directories are still diff --git a/debian/control b/debian/control index 37e622043..15155b9b4 100644 --- a/debian/control +++ b/debian/control @@ -11,7 +11,7 @@ Package: git-annex Architecture: any Section: utils Depends: ${misc:Depends}, ${shlibs:Depends}, git | git-core, uuid, openssh-client, rsync -Suggests: graphviz +Suggests: graphviz, bup Description: manage files with git, without checking their contents into git git-annex allows managing files with git, without checking the file contents into git. While that may seem paradoxical, it is useful when diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index 7d0fb3e79..3514002a4 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -402,9 +402,15 @@ Here are all the supported configuration settings. to or from this remote. For example, to force ipv6, and limit the bandwidth to 100Kbyte/s, set it to "-6 --bwlimit 100" -* `annex.ssh-options`, `annex.rsync-options` +* `remote.<name>.annex-bup-split-options` - Default ssh and rsync options to use if a remote does not have + Options to pass to bup split when storing content in this remote. + For example, to limit the bandwidth to 100Kbye/s, set it to "--bwlimit 100k" + (There is no corresponding option for bup join.) + +* `annex.ssh-options`, `annex.rsync-options`, `annex.bup-split-options` + + Default ssh, rsync, and bup options to use if a remote does not have specific options. * `annex.diskreserve` diff --git a/doc/not.mdwn b/doc/not.mdwn index fea8b3e96..009e1a79d 100644 --- a/doc/not.mdwn +++ b/doc/not.mdwn @@ -2,10 +2,8 @@ * git-annex is not a backup system. It may be a useful component of an [[archival|use_case/bob]] system, or a way to deliver files to a backup - system. - - For a backup system that uses git, take a look at - [bup](http://github.com/apenwarr/bup). + system. For a backup system that uses git and that git-annex supports + storing data in, see [[special_remotes/bup]]. * git-annex is not unison, but if you're finding unison's checksumming too slow, or its strict mirroring of everything to both places too diff --git a/doc/special_remotes.mdwn b/doc/special_remotes.mdwn index f4d479aa9..a33d3f612 100644 --- a/doc/special_remotes.mdwn +++ b/doc/special_remotes.mdwn @@ -7,6 +7,7 @@ types of remotes. These can be used just like any normal remote by git-annex. They cannot be used by other git commands though. * [[Amazon_S3]] +* [[bup]] * [[directory]] ## Unused content on special remotes diff --git a/doc/special_remotes/bup.mdwn b/doc/special_remotes/bup.mdwn new file mode 100644 index 000000000..09eadd008 --- /dev/null +++ b/doc/special_remotes/bup.mdwn @@ -0,0 +1,34 @@ +This special remote type stores file contents in a +[bup](http://github.com/apenwarr/bup) repository. By using git-annex +in the front-end, and bup as a remote, you get an easy git-style +interface to large files, and easy backups of the file contents using git. + +See [[walkthrough/using_bup]] for usage examples. + +## configuration + +These parameters can be passed to `git annex initremote` to configure bup: + +* `encryption` - Required. Either "none" to disable encryption, + or a value that can be looked up (using gpg -k) to find a gpg encryption + key that will be given access to the remote. Note that additional gpg + keys can be given access to a remote by rerunning initremote with + the new key id. + +* `remote` - Required. This is passed to `bup` as the `--remote` + to use to store data. `bup init` will be run to create the + repository. Example: "remote=example.com:/big/mybup" + +Options to pass to `bup split` when sending content to bup can also +be specified, by using `git config annex.bup-split-options`. This +can be used to, for example, limit its bandwidth. + +## data security + +When encryption=none, there is **no** protection against your data being read +by anyone who can access the bup remote. However, bup does transfer data +using ssh, and if you trust the security of the remote, that's fine. + +** Encryption is not yet supported. ** + +See [[design/encryption]]. diff --git a/doc/walkthrough.mdwn b/doc/walkthrough.mdwn index 53f0be6bb..c64880749 100644 --- a/doc/walkthrough.mdwn +++ b/doc/walkthrough.mdwn @@ -15,6 +15,7 @@ A walkthrough of the basic features of git-annex. using_ssh_remotes moving_file_content_between_repositories using_Amazon_S3 + using_bup using_the_URL_backend using_the_SHA1_backend migrating_data_to_a_new_backend diff --git a/doc/walkthrough/using_bup.mdwn b/doc/walkthrough/using_bup.mdwn new file mode 100644 index 000000000..7e1562d12 --- /dev/null +++ b/doc/walkthrough/using_bup.mdwn @@ -0,0 +1,22 @@ +Another [[special_remote|special_remotes]] that git-annex can use is +a [[special_remotes/bup]] repository. Bup stores large file contents +in a git repository of its own, with deduplication. Combined with +git-annex, you can have git on both the frontend and the backend. + +Here's how to create a bup remote, and describe it. + + # git annex initremote mybup type=bup encryption=none remote=example.com/big/mybup + initremote bup (bup init) + Initialized empty Git repository in /big/mybup/ + ok + # git annex describe mybup "my bup repository at example.com" + describe mybup ok + +Now the remote can be used like any other remote. + + # git annex move my_cool_big_file --to mybup + move my_cool_big_file (to mybup...) + Receiving index from server: 1100/1100, done. + ok + +See [[special_remotes/bup]] for details. |