diff options
| -rw-r--r-- | Annex/Content.hs | 16 | ||||
| -rw-r--r-- | CHANGELOG | 11 | ||||
| -rw-r--r-- | CmdLine/GitAnnexShell.hs | 17 | ||||
| -rw-r--r-- | CmdLine/GitAnnexShell/Checks.hs | 23 | ||||
| -rw-r--r-- | Command/LockContent.hs | 18 | ||||
| -rw-r--r-- | Command/P2PStdIO.hs | 44 | ||||
| -rw-r--r-- | P2P/IO.hs | 10 | ||||
| -rw-r--r-- | P2P/Protocol.hs | 55 | ||||
| -rw-r--r-- | Remote/Git.hs | 7 | ||||
| -rw-r--r-- | RemoteDaemon/Transport/Tor.hs | 2 | ||||
| -rw-r--r-- | doc/design/p2p_protocol.mdwn | 172 | ||||
| -rw-r--r-- | doc/devblog/day_487__git-annex-shell_p2pstdio.mdwn | 38 | ||||
| -rw-r--r-- | doc/git-annex-shell.mdwn | 6 | ||||
| -rw-r--r-- | doc/thanks/list | 2 | ||||
| -rw-r--r-- | doc/todo/accellerate_ssh_remotes_with_git-annex-shell_mass_protocol.mdwn | 11 | ||||
| -rw-r--r-- | git-annex.cabal | 1 |
16 files changed, 379 insertions, 54 deletions
diff --git a/Annex/Content.hs b/Annex/Content.hs index 768b2a9dc..6542804f7 100644 --- a/Annex/Content.hs +++ b/Annex/Content.hs @@ -196,19 +196,19 @@ contentLockFile key = Just <$> calcRepo (gitAnnexContentLock key) {- Prevents the content from being removed while the action is running. - Uses a shared lock. - - - Does not actually check if the content is present. Use inAnnex for that. - - However, since the contentLockFile is the content file in indirect mode, - - if the content is not present, locking it will fail. - - - - If locking fails, throws an exception rather than running the action. + - If locking fails, or the content is not present, throws an exception + - rather than running the action. - - Note that, in direct mode, nothing prevents the user from directly - editing or removing the content, even while it's locked by this. -} lockContentShared :: Key -> (VerifiedCopy -> Annex a) -> Annex a -lockContentShared key a = lockContentUsing lock key $ do - u <- getUUID - withVerifiedCopy LockedCopy u (return True) a +lockContentShared key a = lockContentUsing lock key $ ifM (inAnnex key) + ( do + u <- getUUID + withVerifiedCopy LockedCopy u (return True) a + , giveup $ "failed to lock content: not present" + ) where #ifndef mingw32_HOST_OS lock contentfile Nothing = tryLockShared Nothing contentfile @@ -6,6 +6,17 @@ git-annex (6.20180228) UNRELEASED; urgency=medium * Improve SHA*E extension extraction code to not treat parts of the filename that contain punctuation or other non-alphanumeric characters as extensions. Before, such characters were filtered out. + * Fix data loss bug when the local repository uses direct mode, and a + locally modified file is dropped from a remote repsitory. The bug + caused the modified file to be counted as a copy of the original file. + (This is not a severe bug because in such a situation, dropping + from the remote and then modifying the file is allowed and has the same + end result.) + * Fix bug in content locking over tor, when the remote repository is + in direct mode, it neglected to check that the content was actually + present when locking it. This could cause git annex drop to remove + the only copy of a file when it thought the tor remote had a copy. + * git-annex-shell: Added p2pstdio mode. -- Joey Hess <id@joeyh.name> Wed, 28 Feb 2018 11:53:03 -0400 diff --git a/CmdLine/GitAnnexShell.hs b/CmdLine/GitAnnexShell.hs index 154bfeb38..3dc31e602 100644 --- a/CmdLine/GitAnnexShell.hs +++ b/CmdLine/GitAnnexShell.hs @@ -1,6 +1,6 @@ {- git-annex-shell main program - - - Copyright 2010-2012 Joey Hess <id@joeyh.name> + - Copyright 2010-2018 Joey Hess <id@joeyh.name> - - Licensed under the GNU GPL version 3 or higher. -} @@ -28,6 +28,7 @@ import qualified Command.TransferInfo import qualified Command.Commit import qualified Command.NotifyChanges import qualified Command.GCryptSetup +import qualified Command.P2PStdIO cmds_readonly :: [Command] cmds_readonly = @@ -47,8 +48,18 @@ cmds_notreadonly = , Command.GCryptSetup.cmd ] +-- Commands that can operate readonly or not; they use checkNotReadOnly. +cmds_readonly_capable :: [Command] +cmds_readonly_capable = + [ gitAnnexShellCheck Command.P2PStdIO.cmd + ] + +cmds_readonly_safe :: [Command] +cmds_readonly_safe = cmds_readonly ++ cmds_readonly_capable + cmds :: [Command] -cmds = map (adddirparam . noMessages) (cmds_readonly ++ cmds_notreadonly) +cmds = map (adddirparam . noMessages) + (cmds_readonly ++ cmds_notreadonly ++ cmds_readonly_capable) where adddirparam c = c { cmdparamdesc = "DIRECTORY " ++ cmdparamdesc c } @@ -94,7 +105,7 @@ builtins = map cmdname cmds builtin :: String -> String -> [String] -> IO () builtin cmd dir params = do - unless (cmd `elem` map cmdname cmds_readonly) + unless (cmd `elem` map cmdname cmds_readonly_safe) checkNotReadOnly checkDirectory $ Just dir let (params', fieldparams, opts) = partitionParams params diff --git a/CmdLine/GitAnnexShell/Checks.hs b/CmdLine/GitAnnexShell/Checks.hs index fcbf14b24..3409884c0 100644 --- a/CmdLine/GitAnnexShell/Checks.hs +++ b/CmdLine/GitAnnexShell/Checks.hs @@ -14,17 +14,28 @@ import Annex.Init import Utility.UserInfo import Utility.Env +limitedEnv :: String +limitedEnv = "GIT_ANNEX_SHELL_LIMITED" + checkNotLimited :: IO () -checkNotLimited = checkEnv "GIT_ANNEX_SHELL_LIMITED" +checkNotLimited = checkEnv limitedEnv + +readOnlyEnv :: String +readOnlyEnv = "GIT_ANNEX_SHELL_READONLY" checkNotReadOnly :: IO () -checkNotReadOnly = checkEnv "GIT_ANNEX_SHELL_READONLY" +checkNotReadOnly = checkEnv readOnlyEnv checkEnv :: String -> IO () -checkEnv var = getEnv var >>= \case - Nothing -> noop - Just "" -> noop - Just _ -> giveup $ "Action blocked by " ++ var +checkEnv var = checkEnvSet var >>= \case + False -> noop + True -> giveup $ "Action blocked by " ++ var + +checkEnvSet :: String -> IO Bool +checkEnvSet var = getEnv var >>= return . \case + Nothing -> False + Just "" -> False + Just _ -> True checkDirectory :: Maybe FilePath -> IO () checkDirectory mdir = do diff --git a/Command/LockContent.hs b/Command/LockContent.hs index 202ba20d1..1ed8cdf0b 100644 --- a/Command/LockContent.hs +++ b/Command/LockContent.hs @@ -22,9 +22,8 @@ cmd = noCommit $ seek :: CmdParams -> CommandSeek seek = withWords start --- First, lock the content. Then, make sure the content is actually --- present, and print out "OK". Wait for the caller to send a line before --- dropping the lock. +-- First, lock the content, then print out "OK". +-- Wait for the caller to send a line before dropping the lock. start :: [String] -> CommandStart start [ks] = do ok <- lockContentShared k (const locksuccess) @@ -34,12 +33,9 @@ start [ks] = do else exitFailure where k = fromMaybe (giveup "bad key") (file2key ks) - locksuccess = ifM (inAnnex k) - ( liftIO $ do - putStrLn contentLockedMarker - hFlush stdout - _ <- getProtocolLine stdin - return True - , return False - ) + locksuccess = liftIO $ do + putStrLn contentLockedMarker + hFlush stdout + _ <- getProtocolLine stdin + return True start _ = giveup "Specify exactly 1 key." diff --git a/Command/P2PStdIO.hs b/Command/P2PStdIO.hs new file mode 100644 index 000000000..f6e4ae0f0 --- /dev/null +++ b/Command/P2PStdIO.hs @@ -0,0 +1,44 @@ +{- git-annex command + - + - Copyright 2018 Joey Hess <id@joeyh.name> + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.P2PStdIO where + +import Command +import P2P.IO +import P2P.Annex +import qualified P2P.Protocol as P2P +import Git.Types +import qualified Annex +import Annex.UUID +import qualified CmdLine.GitAnnexShell.Checks as Checks +import qualified CmdLine.GitAnnexShell.Fields as Fields +import Utility.AuthToken +import Utility.Tmp.Dir + +cmd :: Command +cmd = noMessages $ command "p2pstdio" SectionPlumbing + "communicate in P2P protocol over stdio" + paramNothing (withParams seek) + +seek :: CmdParams -> CommandSeek +seek = withNothing start + +start :: CommandStart +start = do + servermode <- liftIO $ + Checks.checkEnvSet Checks.readOnlyEnv >>= return . \case + True -> P2P.ServeReadOnly + False -> P2P.ServeReadWrite + theiruuid <- Fields.getField Fields.remoteUUID >>= \case + Nothing -> giveup "missing remoteuuid field" + Just u -> return (toUUID u) + myuuid <- getUUID + conn <- stdioP2PConnection <$> Annex.gitRepo + let server = P2P.serveAuthed servermode myuuid + runFullProto (Serving theiruuid Nothing) conn server >>= \case + Right () -> next $ next $ return True + Left e -> giveup e @@ -10,6 +10,7 @@ module P2P.IO ( RunProto , P2PConnection(..) + , stdioP2PConnection , connectPeer , closeConnection , serveUnixSocket @@ -50,6 +51,15 @@ data P2PConnection = P2PConnection , connOhdl :: Handle } +-- P2PConnection using stdio. +stdioP2PConnection :: Git.Repo -> P2PConnection +stdioP2PConnection g = P2PConnection + { connRepo = g + , connCheckAuth = const False + , connIhdl = stdin + , connOhdl = stdout + } + -- Opens a connection to a peer. Does not authenticate with it. connectPeer :: Git.Repo -> P2PAddress -> IO P2PConnection connectPeer g (TorAnnex onionaddress onionport) = do diff --git a/P2P/Protocol.hs b/P2P/Protocol.hs index f762c3783..c750ae6ff 100644 --- a/P2P/Protocol.hs +++ b/P2P/Protocol.hs @@ -1,5 +1,7 @@ {- P2P protocol - + - See doc/design/p2p_protocol.mdwn + - - Copyright 2016 Joey Hess <id@joeyh.name> - - Licensed under the GNU GPL version 3 or higher. @@ -233,8 +235,8 @@ data LocalF c | TryLockContent Key (Bool -> Proto ()) c -- ^ Try to lock the content of a key, preventing it -- from being deleted, while running the provided protocol - -- action. If unable to lock the content, runs the protocol action - -- with False. + -- action. If unable to lock the content, or the content is not + -- present, runs the protocol action with False. | WaitRefChange (ChangedRefs -> c) -- ^ Waits for one or more git refs to change and returns them. deriving (Functor) @@ -351,10 +353,13 @@ serveAuth myuuid = serverLoop handler return ServerContinue handler _ = return ServerUnexpected +data ServerMode = ServeReadOnly | ServeReadWrite + -- | Serve the protocol, with a peer that has authenticated. -serveAuthed :: UUID -> Proto () -serveAuthed myuuid = void $ serverLoop handler +serveAuthed :: ServerMode -> UUID -> Proto () +serveAuthed servermode myuuid = void $ serverLoop handler where + readonlyerror = net $ sendMessage (ERROR "this repository is read-only; write access denied") handler (LOCKCONTENT key) = do local $ tryLockContent key $ \locked -> do sendSuccess locked @@ -367,27 +372,39 @@ serveAuthed myuuid = void $ serverLoop handler handler (CHECKPRESENT key) = do sendSuccess =<< local (checkContentPresent key) return ServerContinue - handler (REMOVE key) = do - sendSuccess =<< local (removeContent key) - return ServerContinue - handler (PUT af key) = do - have <- local $ checkContentPresent key - if have - then net $ sendMessage ALREADY_HAVE - else do - let sizer = tmpContentSize key - let storer = storeContent key af - ok <- receiveContent nullMeterUpdate sizer storer PUT_FROM - when ok $ - local $ setPresent key myuuid - return ServerContinue + handler (REMOVE key) = case servermode of + ServeReadWrite -> do + sendSuccess =<< local (removeContent key) + return ServerContinue + ServeReadOnly -> do + readonlyerror + return ServerContinue + handler (PUT af key) = case servermode of + ServeReadWrite -> do + have <- local $ checkContentPresent key + if have + then net $ sendMessage ALREADY_HAVE + else do + let sizer = tmpContentSize key + let storer = storeContent key af + ok <- receiveContent nullMeterUpdate sizer storer PUT_FROM + when ok $ + local $ setPresent key myuuid + return ServerContinue + ServeReadOnly -> do + readonlyerror + return ServerContinue handler (GET offset key af) = do void $ sendContent af key offset nullMeterUpdate -- setPresent not called because the peer may have -- requested the data but not permanently stored it. return ServerContinue handler (CONNECT service) = do - net $ relayService service + let goahead = net $ relayService service + case (servermode, service) of + (ServeReadWrite, _) -> goahead + (ServeReadOnly, UploadPack) -> goahead + (ServeReadOnly, ReceivePack) -> readonlyerror -- After connecting to git, there may be unconsumed data -- from the git processes hanging around (even if they -- exited successfully), so stop serving this connection. diff --git a/Remote/Git.hs b/Remote/Git.hs index 2cebcce4a..caa677464 100644 --- a/Remote/Git.hs +++ b/Remote/Git.hs @@ -391,11 +391,8 @@ lockKey r duc key callback -- and then run the callback in the original -- annex monad, not the remote's. onLocalFast r $ - Annex.Content.lockContentShared key $ \vc -> - ifM (Annex.Content.inAnnex key) - ( liftIO $ inorigrepo $ callback vc - , failedlock - ) + Annex.Content.lockContentShared key $ + liftIO . inorigrepo . callback , failedlock ) | Git.repoIsSsh (repo r) = do diff --git a/RemoteDaemon/Transport/Tor.hs b/RemoteDaemon/Transport/Tor.hs index 623ff03e3..133aba1ec 100644 --- a/RemoteDaemon/Transport/Tor.hs +++ b/RemoteDaemon/Transport/Tor.hs @@ -128,7 +128,7 @@ serveClient th u r q = bracket setup cleanup start authed conn theiruuid = bracket watchChangedRefs (liftIO . maybe noop stopWatchingChangedRefs) $ \crh -> do v' <- runFullProto (Serving theiruuid crh) conn $ - P2P.serveAuthed u + P2P.serveAuthed P2P.ServeReadWrite u case v' of Right () -> return () Left e -> liftIO $ debugM "remotedaemon" ("Tor connection error: " ++ e) diff --git a/doc/design/p2p_protocol.mdwn b/doc/design/p2p_protocol.mdwn new file mode 100644 index 000000000..be3c5e6cb --- /dev/null +++ b/doc/design/p2p_protocol.mdwn @@ -0,0 +1,172 @@ +The git-annex P2P protocol is a custom protocol that git-annex uses to +communicate between peers. + +There's a common line-based serialization of the protocol, but other +serializations are also possible. The line-based serialization is spoken +by [[git-annex-shell], and by git-annex over tor. + +One peer is known as the client, and is the peer that initiates the +connection. The other peer is known as the server, and is the peer that the +client connects to. It's possible for two connections to be run at the same +time between the same two peers, in different directions. + +## Authentication + +The protocol genernally starts with authentication. However, if +authentication already occurs on another layer, as is the case with +git-annex-shell, authentication will be skipped. + +The client starts by sending an authentication to the server, +along with its UUID. The AuthToken is some arbitrary token that has been +agreed upon beforehand. + + AUTH UUID AuthToken + +The server responds with either its own UUID when authentication +is successful. Or, it can fail the authentication, and close the +connection. + + AUTH_SUCCESS UUID + AUTH_FAILURE + +Note that authentication does not guarantee that the client is talking to +who they expect to be talking to. This, and encryption of the connection, +are handled at a lower level. + +## Errors + +Either the client or the server may send an error message at any +time. + +When the client sends an ERROR, the server will close the connection. + +If the server sends an ERROR in response to the client's +request, the connection will remain open, and the client can make +another request. + + ERROR this repository is read-only; write access denied + +## Binary data + +The protocol allows raw binary data to be sent. This is done +using a DATA message. In the line-based serialization, this comes +on its own line, followed by a newline and the binary data. +The Len value tells how many bytes of data to read. + + DATA 3 + foo + +Note that there is no newline after the binary data; the next protocol +message will come immediately after it. + +## Checking if content is present + +To check if a key is present on the server, the client sends: + + CHECKPRESENT Key + +The server responds with either SUCCESS or FAILURE. + +## Locking content + +To lock content on the server, preventing it from being removed, +the client sends: + + LOCKCONTENT Key + +The server responds with either SUCCESS or FAILURE. +The former indicates the content is locked. It will remain +locked until the connection is broken, or the client +sends: + + UNLOCKCONTENT Key + +The server makes no response to that. + +## Removing content + +To remove a key's content from the server, the client sends: + + REMOVE Key + +The server responds with either SUCCESS or FAILURE. + +## Storing content on the server + +To store content on the server, the client sends: + + PUT AssociatedFile Key + +Here AssociatedFile may be the name of a file in the git +repository, for information purposes only. Or it can be the +empty string. It will always have unix directory separators. + +(Note that in the line-based serialization. AssociatedFile may not contain any +spaces, since it's not the last token in the line. Use '%' to indicate +whitespace.) + +The server may respond with ALREADY-HAVE if it already +had the conent of that key. Otherwise, it responds with: + + PUT-FROM Offset + +Offset is the number of bytes into the file that the server wants +the client to start. This allows resuming transfers. + +The client then sends a DATA message with content of the file from +the offset to the end of file. + +If the server successfully receives the data and stores the content, +it replies with SUCCESS. Otherwise, FAILURE. + +## Getting content from the server + +To get content from the server, the client sends: + + GET Offset AssociatedFile Key + +The Offset is the number of bytes into the file that the client wants +the server to skip, which allows resuming transfers. +See description of AssociatedFile above. + +The server then sends a DATA message with the content of the file +from the offset to end of file. + +The client replies with SUCCESS or FAILURE. + +## Connection to services + +This is used to connect to services like git-upload-pack and +git-receive-pack that speak their own protocol. + +The client sends a message to request the connection. +Service is the name of the service, eg "git-upload-pack". + + CONNECT Service + +Both client and server may now exchange DATA messages in any order, +encapsulating the service's protocol. + +When the service exits, the server indicates this by telling the client +its exit code. + + CONNECTDONE ExitCode + +## Change notification + +The client can request to be notified when a ref in +the git repository on the server changes. + + NOTIFYCHANGE + +The server will block until at least +one of the refs changes, and send a list of changed +refs. + + CHANGED ChangedRefs + +For example: + + CHANGED refs/heads/master refs/heads/git-annex + +Some servers may not support this command. diff --git a/doc/devblog/day_487__git-annex-shell_p2pstdio.mdwn b/doc/devblog/day_487__git-annex-shell_p2pstdio.mdwn new file mode 100644 index 000000000..2475df87e --- /dev/null +++ b/doc/devblog/day_487__git-annex-shell_p2pstdio.mdwn @@ -0,0 +1,38 @@ +It was rather easy to implement `git-annex-shell p2pstdio`, the P2P +protocol implementation done for tor is quite generic and easy to adapt for +this. + +The only complication was that git-annex-shell has a readonly mode, +so the protocol server needed modifications to support that. +Well, there's also some innefficiency around unncessary verification +of transferred content in some cases, which will probably need extensions +to the P2P protocol later. + +Also wrote up some documentation of what the P2P protocol looks like, +for anyone who might want to communiate with git-annex-shell using it, +for some reason, and doesn't understand Haskell and free monads. +[[design/P2P_protocol]] + +While comparing the code of the P2P server and git-annex-shell commands, I +noticed that the P2P server didn't check inAnnex when locking content, +while `git-annex-shell lockcontent` did check inAnnex. This turned out to +be a ugly data loss bug involving direct mode repositories, where a +modified direct mode file was allowed when locking a key in the repository. +Turned out that the bug happened when locking a file over tor to drop it +locally, but also when locking a file locally in a direct mode repository +to allow dropping it from any remote. + +Very glad I noticed that, and I've changed the API to prevent that class of +bug. I feel this is not a severe data loss bug, because when a direct mode +repository is involved, dropping from somewhere and then modifying the file +in the direct mode repository can have the same effect of losing the old copy. +The bug just made data loss happen when running the same operations in the +other order. + +Next will be making git-annex use this new git-annex-shell feature +when available. + +---- + +Today's work was sponsored by Trenton Cronholm on +[Patreon](https://patreon.com/joeyh) diff --git a/doc/git-annex-shell.mdwn b/doc/git-annex-shell.mdwn index 167f54012..cf72e091b 100644 --- a/doc/git-annex-shell.mdwn +++ b/doc/git-annex-shell.mdwn @@ -90,6 +90,12 @@ first "/~/" or "/~user/" is expanded to the specified home directory. Sets up a repository as a gcrypt repository. +* p2pstdio directory + + This causes git-annex-shell to communicate using the git-annex p2p + protocol over stdio. When supported by git-annex-shell, this allows + multiple actions to be run over a single connection, improving speed. + # OPTIONS Most options are the same as in git-annex. The ones specific diff --git a/doc/thanks/list b/doc/thanks/list index 11adc13a2..5b95a8ae2 100644 --- a/doc/thanks/list +++ b/doc/thanks/list @@ -93,3 +93,5 @@ andrea rota, Riku Voipio, Ethan Aubin, Pedro Luz, +John Pellman, +Duncan Holm, diff --git a/doc/todo/accellerate_ssh_remotes_with_git-annex-shell_mass_protocol.mdwn b/doc/todo/accellerate_ssh_remotes_with_git-annex-shell_mass_protocol.mdwn index dd6be9a30..ff4b8c59d 100644 --- a/doc/todo/accellerate_ssh_remotes_with_git-annex-shell_mass_protocol.mdwn +++ b/doc/todo/accellerate_ssh_remotes_with_git-annex-shell_mass_protocol.mdwn @@ -23,7 +23,7 @@ letting git-annex-shell on the remote work that out. So, it seems better to not use sftp, and instead roll our own simple file transfer protocol. -So, "git-annex-shell -c multi" would speak a protocol over stdin/stdout +So, "git-annex-shell -c p2pstdio" would speak a protocol over stdin/stdout that essentially contains the commands inannex, lockcontent, dropkey, recvkey, and sendkey. @@ -31,3 +31,12 @@ P2P.Protocol already contains such a similar protocol, used over tor. That protocol even supports resuming interrupted transfers. It has stuff including auth that this wouldn't need, but it would be good to unify with it as much as possible. + +---- + +Implementation todos: + +* git-annex-shell p2pstdio currently always verifies content it receives. + git-annex-shell recvkey has a speed optimisation, when it's told the file + being sent is locked, it can avoid an expensive verification. +* Maybe similar for transfers in the other direction? diff --git a/git-annex.cabal b/git-annex.cabal index 994e01570..6a8aa490a 100644 --- a/git-annex.cabal +++ b/git-annex.cabal @@ -754,6 +754,7 @@ Executable git-annex Command.NotifyChanges Command.NumCopies Command.P2P + Command.P2PStdIO Command.PostReceive Command.PreCommit Command.Proxy |