diff options
| -rw-r--r-- | Remote/Git.hs | 4 | ||||
| -rw-r--r-- | Remote/Helper/Ssh.hs | 14 | ||||
| -rw-r--r-- | Remote/Rsync.hs | 31 | ||||
| -rw-r--r-- | Types/GitConfig.hs | 4 | ||||
| -rw-r--r-- | debian/changelog | 2 | ||||
| -rw-r--r-- | doc/git-annex.mdwn | 24 | ||||
| -rw-r--r-- | doc/todo/separate_rsync_bwlimit_options_for_upload_and_download.mdwn | 2 |
7 files changed, 60 insertions, 21 deletions
diff --git a/Remote/Git.hs b/Remote/Git.hs index 93110745c..d714cfec5 100644 --- a/Remote/Git.hs +++ b/Remote/Git.hs @@ -310,7 +310,7 @@ copyFromRemote r key file dest _p = copyFromRemote' r key file dest copyFromRemote' :: Remote -> Key -> AssociatedFile -> FilePath -> Annex Bool copyFromRemote' r key file dest | not $ Git.repoIsUrl (repo r) = guardUsable (repo r) False $ do - let params = Ssh.rsyncParams r + let params = Ssh.rsyncParams r Download u <- getUUID -- run copy from perspective of remote liftIO $ onLocal (repo r) $ do @@ -409,7 +409,7 @@ copyToRemote r key file p -- the remote's Annex, but it needs access to the current -- Annex monad's state. checksuccessio <- Annex.withCurrentState checksuccess - let params = Ssh.rsyncParams r + let params = Ssh.rsyncParams r Upload u <- getUUID -- run copy from perspective of remote liftIO $ onLocal (repo r) $ ifM (Annex.Content.inAnnex key) diff --git a/Remote/Helper/Ssh.hs b/Remote/Helper/Ssh.hs index 964c8355a..8de88953f 100644 --- a/Remote/Helper/Ssh.hs +++ b/Remote/Helper/Ssh.hs @@ -122,7 +122,7 @@ rsyncParamsRemote direct r direction key file afile = do fields -- Convert the ssh command into rsync command line. let eparam = rsyncShell (Param shellcmd:shellparams) - let o = rsyncParams r + let o = rsyncParams r direction return $ if direction == Download then o ++ rsyncopts eparam dummy (File file) else o ++ rsyncopts eparam (File file) dummy @@ -140,7 +140,11 @@ rsyncParamsRemote direct r direction key file afile = do dummy = Param "dummy:" -- --inplace to resume partial files -rsyncParams :: Remote -> [CommandParam] -rsyncParams r = Params "--progress --inplace" : - map Param (remoteAnnexRsyncOptions $ gitconfig r) - +rsyncParams :: Remote -> Direction -> [CommandParam] +rsyncParams r direction = Params "--progress --inplace" : + map Param (remoteAnnexRsyncOptions gc ++ dps) + where + dps + | direction == Download = remoteAnnexRsyncDownloadOptions gc + | otherwise = remoteAnnexRsyncUploadOptions gc + gc = gitconfig r diff --git a/Remote/Rsync.hs b/Remote/Rsync.hs index 430554ab8..e27286d5a 100644 --- a/Remote/Rsync.hs +++ b/Remote/Rsync.hs @@ -41,12 +41,15 @@ import Utility.Rsync import Utility.CopyFile import Utility.Metered import Annex.Perms +import Logs.Transfer type RsyncUrl = String data RsyncOpts = RsyncOpts { rsyncUrl :: RsyncUrl , rsyncOptions :: [CommandParam] + , rsyncUploadOptions :: [CommandParam] + , rsyncDownloadOptions :: [CommandParam] , rsyncShellEscape :: Bool } @@ -93,10 +96,16 @@ gen r u c gc = do } genRsyncOpts :: RemoteConfig -> RemoteGitConfig -> [CommandParam] -> RsyncUrl -> RsyncOpts -genRsyncOpts c gc transport url = RsyncOpts url (transport ++ opts) escape +genRsyncOpts c gc transport url = RsyncOpts + { rsyncUrl = url + , rsyncOptions = opts [] + , rsyncUploadOptions = transport ++ opts (remoteAnnexRsyncUploadOptions gc) + , rsyncDownloadOptions = transport ++ opts (remoteAnnexRsyncDownloadOptions gc) + , rsyncShellEscape = M.lookup "shellescape" c /= Just "no" + } where - opts = map Param $ filter safe $ remoteAnnexRsyncOptions gc - escape = M.lookup "shellescape" c /= Just "no" + opts specificopts = map Param $ filter safe $ + remoteAnnexRsyncOptions gc ++ specificopts safe opt -- Don't allow user to pass --delete to rsync; -- that could cause it to delete other keys @@ -257,7 +266,7 @@ withRsyncScratchDir a = do rsyncRetrieve :: RsyncOpts -> Key -> FilePath -> Maybe MeterUpdate -> Annex Bool rsyncRetrieve o k dest callback = - showResumable $ untilTrue (rsyncUrls o k) $ \u -> rsyncRemote o callback + showResumable $ untilTrue (rsyncUrls o k) $ \u -> rsyncRemote Download o callback -- use inplace when retrieving to support resuming [ Param "--inplace" , Param u @@ -272,13 +281,15 @@ showResumable a = ifM a return False ) -rsyncRemote :: RsyncOpts -> Maybe MeterUpdate -> [CommandParam] -> Annex Bool -rsyncRemote o callback params = do +rsyncRemote :: Direction -> RsyncOpts -> Maybe MeterUpdate -> [CommandParam] -> Annex Bool +rsyncRemote direction o callback params = do showOutput -- make way for progress bar - liftIO $ (maybe rsync rsyncProgress callback) ps + liftIO $ (maybe rsync rsyncProgress callback) $ + opts ++ [Params "--progress"] ++ params where - defaultParams = [Params "--progress"] - ps = rsyncOptions o ++ defaultParams ++ params + opts + | direction == Download = rsyncDownloadOptions o + | otherwise = rsyncUploadOptions o {- To send a single key is slightly tricky; need to build up a temporary - directory structure to pass to rsync so it can create the hash @@ -301,7 +312,7 @@ rsyncSend o callback k canrename src = withRsyncScratchDir $ \tmp -> do else createLinkOrCopy src dest ps <- sendParams if ok - then showResumable $ rsyncRemote o (Just callback) $ ps ++ + then showResumable $ rsyncRemote Upload o (Just callback) $ ps ++ [ Param "--recursive" , partialParams -- tmp/ to send contents of tmp dir diff --git a/Types/GitConfig.hs b/Types/GitConfig.hs index e54915a68..ab3dbd7b9 100644 --- a/Types/GitConfig.hs +++ b/Types/GitConfig.hs @@ -115,6 +115,8 @@ data RemoteGitConfig = RemoteGitConfig - including special remotes. -} , remoteAnnexSshOptions :: [String] , remoteAnnexRsyncOptions :: [String] + , remoteAnnexRsyncUploadOptions :: [String] + , remoteAnnexRsyncDownloadOptions :: [String] , remoteAnnexRsyncTransport :: [String] , remoteAnnexGnupgOptions :: [String] , remoteAnnexRsyncUrl :: Maybe String @@ -144,6 +146,8 @@ extractRemoteGitConfig r remotename = RemoteGitConfig , remoteAnnexSshOptions = getoptions "ssh-options" , remoteAnnexRsyncOptions = getoptions "rsync-options" + , remoteAnnexRsyncDownloadOptions = getoptions "rsync-download-options" + , remoteAnnexRsyncUploadOptions = getoptions "rsync-upload-options" , remoteAnnexRsyncTransport = getoptions "rsync-transport" , remoteAnnexGnupgOptions = getoptions "gnupg-options" , remoteAnnexRsyncUrl = notempty $ getmaybe "rsyncurl" diff --git a/debian/changelog b/debian/changelog index 2f979aa69..fdd909c53 100644 --- a/debian/changelog +++ b/debian/changelog @@ -12,6 +12,8 @@ git-annex (5.20140128) UNRELEASED; urgency=medium * sync --content: Re-pull from remotes after downloading content, since that can take a while and other changes may be pushed in the meantime. + * Added ways to configure rsync options to be used only when uploading + or downloading from a remote. Useful to eg limit upload bandwidth. -- Joey Hess <joeyh@debian.org> Tue, 28 Jan 2014 13:57:19 -0400 diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index fa1175fb6..ef935921b 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -1379,6 +1379,21 @@ Here are all the supported configuration settings. to or from this remote. For example, to force ipv6, and limit the bandwidth to 100Kbyte/s, set it to `-6 --bwlimit 100` +* `remote.<name>.annex-rsync-upload-options` + + Options to use when using rsync to upload a file to a remote. + + These options are passed after other applicable rsync options, + so can be used to override them. For example, to limit upload bandwidth + to 10Kbye/s, set `--bwlimit 10`. + +* `remote.<name>.annex-rsync-download-options` + + Options to use when using rsync to download a file from a remote. + + These options are passed after other applicable rsync options, + so can be used to override them. + * `remote.<name>.annex-rsync-transport` The remote shell to use to connect to the rsync remote. Possible @@ -1402,11 +1417,12 @@ Here are all the supported configuration settings. precedence over the default GnuPG configuration, which is otherwise used.) -* `annex.ssh-options`, `annex.rsync-options`, `annex.bup-split-options`, - `annex.gnupg-options` +* `annex.ssh-options`, `annex.rsync-options`, + `annex.rsync-upload-options`, `annex.rsync-download-options`, + `annex.bup-split-options`, `annex.gnupg-options` - Default ssh, rsync, wget/curl, bup, and GnuPG options to use if a - remote does not have specific options. + Default options to use if a remote does not have more specific options + as described above. * `annex.web-options` diff --git a/doc/todo/separate_rsync_bwlimit_options_for_upload_and_download.mdwn b/doc/todo/separate_rsync_bwlimit_options_for_upload_and_download.mdwn index 65a3a12ef..2b93ad2d6 100644 --- a/doc/todo/separate_rsync_bwlimit_options_for_upload_and_download.mdwn +++ b/doc/todo/separate_rsync_bwlimit_options_for_upload_and_download.mdwn @@ -1,2 +1,4 @@ The bandwidth for upload and download are often different. It would be useful to have different settings for upload and download limits. As it is, I have to keep changing annex-rsync-options options between uploads and downloads. + +> [[done]] --[[Joey]] |