summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--Remote/Git.hs4
-rw-r--r--Remote/Helper/Ssh.hs14
-rw-r--r--Remote/Rsync.hs31
-rw-r--r--Types/GitConfig.hs4
-rw-r--r--debian/changelog2
-rw-r--r--doc/git-annex.mdwn24
-rw-r--r--doc/todo/separate_rsync_bwlimit_options_for_upload_and_download.mdwn2
7 files changed, 60 insertions, 21 deletions
diff --git a/Remote/Git.hs b/Remote/Git.hs
index 93110745c..d714cfec5 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -310,7 +310,7 @@ copyFromRemote r key file dest _p = copyFromRemote' r key file dest
copyFromRemote' :: Remote -> Key -> AssociatedFile -> FilePath -> Annex Bool
copyFromRemote' r key file dest
| not $ Git.repoIsUrl (repo r) = guardUsable (repo r) False $ do
- let params = Ssh.rsyncParams r
+ let params = Ssh.rsyncParams r Download
u <- getUUID
-- run copy from perspective of remote
liftIO $ onLocal (repo r) $ do
@@ -409,7 +409,7 @@ copyToRemote r key file p
-- the remote's Annex, but it needs access to the current
-- Annex monad's state.
checksuccessio <- Annex.withCurrentState checksuccess
- let params = Ssh.rsyncParams r
+ let params = Ssh.rsyncParams r Upload
u <- getUUID
-- run copy from perspective of remote
liftIO $ onLocal (repo r) $ ifM (Annex.Content.inAnnex key)
diff --git a/Remote/Helper/Ssh.hs b/Remote/Helper/Ssh.hs
index 964c8355a..8de88953f 100644
--- a/Remote/Helper/Ssh.hs
+++ b/Remote/Helper/Ssh.hs
@@ -122,7 +122,7 @@ rsyncParamsRemote direct r direction key file afile = do
fields
-- Convert the ssh command into rsync command line.
let eparam = rsyncShell (Param shellcmd:shellparams)
- let o = rsyncParams r
+ let o = rsyncParams r direction
return $ if direction == Download
then o ++ rsyncopts eparam dummy (File file)
else o ++ rsyncopts eparam (File file) dummy
@@ -140,7 +140,11 @@ rsyncParamsRemote direct r direction key file afile = do
dummy = Param "dummy:"
-- --inplace to resume partial files
-rsyncParams :: Remote -> [CommandParam]
-rsyncParams r = Params "--progress --inplace" :
- map Param (remoteAnnexRsyncOptions $ gitconfig r)
-
+rsyncParams :: Remote -> Direction -> [CommandParam]
+rsyncParams r direction = Params "--progress --inplace" :
+ map Param (remoteAnnexRsyncOptions gc ++ dps)
+ where
+ dps
+ | direction == Download = remoteAnnexRsyncDownloadOptions gc
+ | otherwise = remoteAnnexRsyncUploadOptions gc
+ gc = gitconfig r
diff --git a/Remote/Rsync.hs b/Remote/Rsync.hs
index 430554ab8..e27286d5a 100644
--- a/Remote/Rsync.hs
+++ b/Remote/Rsync.hs
@@ -41,12 +41,15 @@ import Utility.Rsync
import Utility.CopyFile
import Utility.Metered
import Annex.Perms
+import Logs.Transfer
type RsyncUrl = String
data RsyncOpts = RsyncOpts
{ rsyncUrl :: RsyncUrl
, rsyncOptions :: [CommandParam]
+ , rsyncUploadOptions :: [CommandParam]
+ , rsyncDownloadOptions :: [CommandParam]
, rsyncShellEscape :: Bool
}
@@ -93,10 +96,16 @@ gen r u c gc = do
}
genRsyncOpts :: RemoteConfig -> RemoteGitConfig -> [CommandParam] -> RsyncUrl -> RsyncOpts
-genRsyncOpts c gc transport url = RsyncOpts url (transport ++ opts) escape
+genRsyncOpts c gc transport url = RsyncOpts
+ { rsyncUrl = url
+ , rsyncOptions = opts []
+ , rsyncUploadOptions = transport ++ opts (remoteAnnexRsyncUploadOptions gc)
+ , rsyncDownloadOptions = transport ++ opts (remoteAnnexRsyncDownloadOptions gc)
+ , rsyncShellEscape = M.lookup "shellescape" c /= Just "no"
+ }
where
- opts = map Param $ filter safe $ remoteAnnexRsyncOptions gc
- escape = M.lookup "shellescape" c /= Just "no"
+ opts specificopts = map Param $ filter safe $
+ remoteAnnexRsyncOptions gc ++ specificopts
safe opt
-- Don't allow user to pass --delete to rsync;
-- that could cause it to delete other keys
@@ -257,7 +266,7 @@ withRsyncScratchDir a = do
rsyncRetrieve :: RsyncOpts -> Key -> FilePath -> Maybe MeterUpdate -> Annex Bool
rsyncRetrieve o k dest callback =
- showResumable $ untilTrue (rsyncUrls o k) $ \u -> rsyncRemote o callback
+ showResumable $ untilTrue (rsyncUrls o k) $ \u -> rsyncRemote Download o callback
-- use inplace when retrieving to support resuming
[ Param "--inplace"
, Param u
@@ -272,13 +281,15 @@ showResumable a = ifM a
return False
)
-rsyncRemote :: RsyncOpts -> Maybe MeterUpdate -> [CommandParam] -> Annex Bool
-rsyncRemote o callback params = do
+rsyncRemote :: Direction -> RsyncOpts -> Maybe MeterUpdate -> [CommandParam] -> Annex Bool
+rsyncRemote direction o callback params = do
showOutput -- make way for progress bar
- liftIO $ (maybe rsync rsyncProgress callback) ps
+ liftIO $ (maybe rsync rsyncProgress callback) $
+ opts ++ [Params "--progress"] ++ params
where
- defaultParams = [Params "--progress"]
- ps = rsyncOptions o ++ defaultParams ++ params
+ opts
+ | direction == Download = rsyncDownloadOptions o
+ | otherwise = rsyncUploadOptions o
{- To send a single key is slightly tricky; need to build up a temporary
- directory structure to pass to rsync so it can create the hash
@@ -301,7 +312,7 @@ rsyncSend o callback k canrename src = withRsyncScratchDir $ \tmp -> do
else createLinkOrCopy src dest
ps <- sendParams
if ok
- then showResumable $ rsyncRemote o (Just callback) $ ps ++
+ then showResumable $ rsyncRemote Upload o (Just callback) $ ps ++
[ Param "--recursive"
, partialParams
-- tmp/ to send contents of tmp dir
diff --git a/Types/GitConfig.hs b/Types/GitConfig.hs
index e54915a68..ab3dbd7b9 100644
--- a/Types/GitConfig.hs
+++ b/Types/GitConfig.hs
@@ -115,6 +115,8 @@ data RemoteGitConfig = RemoteGitConfig
- including special remotes. -}
, remoteAnnexSshOptions :: [String]
, remoteAnnexRsyncOptions :: [String]
+ , remoteAnnexRsyncUploadOptions :: [String]
+ , remoteAnnexRsyncDownloadOptions :: [String]
, remoteAnnexRsyncTransport :: [String]
, remoteAnnexGnupgOptions :: [String]
, remoteAnnexRsyncUrl :: Maybe String
@@ -144,6 +146,8 @@ extractRemoteGitConfig r remotename = RemoteGitConfig
, remoteAnnexSshOptions = getoptions "ssh-options"
, remoteAnnexRsyncOptions = getoptions "rsync-options"
+ , remoteAnnexRsyncDownloadOptions = getoptions "rsync-download-options"
+ , remoteAnnexRsyncUploadOptions = getoptions "rsync-upload-options"
, remoteAnnexRsyncTransport = getoptions "rsync-transport"
, remoteAnnexGnupgOptions = getoptions "gnupg-options"
, remoteAnnexRsyncUrl = notempty $ getmaybe "rsyncurl"
diff --git a/debian/changelog b/debian/changelog
index 2f979aa69..fdd909c53 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -12,6 +12,8 @@ git-annex (5.20140128) UNRELEASED; urgency=medium
* sync --content: Re-pull from remotes after downloading content,
since that can take a while and other changes may be pushed in the
meantime.
+ * Added ways to configure rsync options to be used only when uploading
+ or downloading from a remote. Useful to eg limit upload bandwidth.
-- Joey Hess <joeyh@debian.org> Tue, 28 Jan 2014 13:57:19 -0400
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index fa1175fb6..ef935921b 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -1379,6 +1379,21 @@ Here are all the supported configuration settings.
to or from this remote. For example, to force ipv6, and limit
the bandwidth to 100Kbyte/s, set it to `-6 --bwlimit 100`
+* `remote.<name>.annex-rsync-upload-options`
+
+ Options to use when using rsync to upload a file to a remote.
+
+ These options are passed after other applicable rsync options,
+ so can be used to override them. For example, to limit upload bandwidth
+ to 10Kbye/s, set `--bwlimit 10`.
+
+* `remote.<name>.annex-rsync-download-options`
+
+ Options to use when using rsync to download a file from a remote.
+
+ These options are passed after other applicable rsync options,
+ so can be used to override them.
+
* `remote.<name>.annex-rsync-transport`
The remote shell to use to connect to the rsync remote. Possible
@@ -1402,11 +1417,12 @@ Here are all the supported configuration settings.
precedence over the default GnuPG configuration, which is otherwise
used.)
-* `annex.ssh-options`, `annex.rsync-options`, `annex.bup-split-options`,
- `annex.gnupg-options`
+* `annex.ssh-options`, `annex.rsync-options`,
+ `annex.rsync-upload-options`, `annex.rsync-download-options`,
+ `annex.bup-split-options`, `annex.gnupg-options`
- Default ssh, rsync, wget/curl, bup, and GnuPG options to use if a
- remote does not have specific options.
+ Default options to use if a remote does not have more specific options
+ as described above.
* `annex.web-options`
diff --git a/doc/todo/separate_rsync_bwlimit_options_for_upload_and_download.mdwn b/doc/todo/separate_rsync_bwlimit_options_for_upload_and_download.mdwn
index 65a3a12ef..2b93ad2d6 100644
--- a/doc/todo/separate_rsync_bwlimit_options_for_upload_and_download.mdwn
+++ b/doc/todo/separate_rsync_bwlimit_options_for_upload_and_download.mdwn
@@ -1,2 +1,4 @@
The bandwidth for upload and download are often different. It would be useful to have different settings for upload and download limits.
As it is, I have to keep changing annex-rsync-options options between uploads and downloads.
+
+> [[done]] --[[Joey]]