diff options
367 files changed, 4969 insertions, 751 deletions
@@ -1,7 +1,28 @@ +Antoine Beaupré <anarcat@koumbit.org> anarcat <anarcat@web> +Antoine Beaupré <anarcat@koumbit.org> https://id.koumbit.net/anarcat <https://id.koumbit.net/anarcat@web> +Greg Grossmeier <greg@grossmeier.net> http://grossmeier.net/ <greg@web> +Jimmy Tang <jtang@tchpc.tcd.ie> jtang <jtang@web> +Joachim Breitner <mail@joachim-breitner.de> http://www.joachim-breitner.de/ <nomeata@web> +Joey Hess <id@joeyh.name> Joey Hess <joey@gnu.kitenet.net> +Joey Hess <id@joeyh.name> Joey Hess <joey@kitenet.net> +Joey Hess <id@joeyh.name> Joey Hess <joeyh@debian.org> +Joey Hess <id@joeyh.name> Joey Hess <joeyh@fischer.debian.org> +Joey Hess <id@joeyh.name> Joey Hess <joeyh@joeyh.name> +Joey Hess <id@joeyh.name> Joey Hess <joeyh@oberon.tam-lin.net> +Joey Hess <id@joeyh.name> Joey Hess <joeyh@oberon.underhill.private> Joey Hess <id@joeyh.name> http://joey.kitenet.net/ <joey@web> -Joey Hess <id@joeyh.name> http://joeyh.name/ <joey@web> Joey Hess <id@joeyh.name> http://joeyh.name/ <http://joeyh.name/@web> +Joey Hess <id@joeyh.name> http://joeyh.name/ <joey@web> +Joey Hess <id@joeyh.name> https://www.google.com/accounts/o8/id?id=AItOawmJfIszzreLNvCqzqzvTayA9_9L6gb9RtY <Joey@web> +Johan Kiviniemi <devel@johan.kiviniemi.name> http://johan.kiviniemi.name/ <Johan@web> +Johan Kiviniemi <devel@johan.kiviniemi.name> http://johan.kiviniemi.name/ <http://johan.kiviniemi.name/@web> +Nicolas Pouillard <nicolas.pouillard@gmail.com> http://ertai.myopenid.com/ <npouillard@web> +Peter Simons <simons@cryp.to> Peter Simons <simons@ubuntu-12.04> +Peter Simons <simons@cryp.to> http://peter-simons.myopenid.com/ <http://peter-simons.myopenid.com/@web> +Philipp Kern <pkern@debian.org> http://phil.0x539.de/ <Philipp_Kern@web> +Richard Hartmann <richih@debian.org> https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U <Richard@web> Yaroslav Halchenko <debian@onerussian.com> Yaroslav Halchenko <debian@onerussian.com> http://yarikoptic.myopenid.com/ <site-myopenid@web> Yaroslav Halchenko <debian@onerussian.com> https://www.google.com/accounts/o8/id?id=AItOawnx8kHW66N3BqmkVpgtXDlYMvr8TJ5VvfY <Yaroslav@web> -Richard Hartmann <richih@debian.org> https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U <Richard@web> +Øyvind A. Holm <sunny@sunbase.org> http://sunny256.sunbase.org/ <sunny256@web> +Øyvind A. Holm <sunny@sunbase.org> https://sunny256.wordpress.com/ <sunny256@web> @@ -5,7 +5,7 @@ - Licensed under the GNU GPL version 3 or higher. -} -{-# LANGUAGE CPP, GeneralizedNewtypeDeriving, PackageImports #-} +{-# LANGUAGE CPP, GeneralizedNewtypeDeriving, PackageImports, BangPatterns #-} module Annex ( Annex, @@ -32,6 +32,7 @@ module Annex ( getRemoteGitConfig, withCurrentState, changeDirectory, + incError, ) where import Common @@ -312,3 +313,9 @@ changeDirectory d = do liftIO $ setCurrentDirectory d r' <- liftIO $ Git.relPath r changeState $ \s -> s { repo = r' } + +incError :: Annex () +incError = changeState $ \s -> + let ! c = errcounter s + 1 + ! s' = s { errcounter = c } + in s' diff --git a/Annex/Content.hs b/Annex/Content.hs index 9d70ccee3..dc60dfe1a 100644 --- a/Annex/Content.hs +++ b/Annex/Content.hs @@ -280,17 +280,19 @@ withTmp key action = do {- Checks that there is disk space available to store a given key, - in a destination (or the annex) printing a warning if not. -} checkDiskSpace :: Maybe FilePath -> Key -> Integer -> Annex Bool -checkDiskSpace destination key alreadythere = do - reserve <- annexDiskReserve <$> Annex.getGitConfig - free <- liftIO . getDiskFree =<< dir - force <- Annex.getState Annex.force - case (free, keySize key) of - (Just have, Just need) -> do - let ok = (need + reserve <= have + alreadythere) || force - unless ok $ - needmorespace (need + reserve - have - alreadythere) - return ok - _ -> return True +checkDiskSpace destination key alreadythere = ifM (Annex.getState Annex.force) + ( return True + , do + reserve <- annexDiskReserve <$> Annex.getGitConfig + free <- liftIO . getDiskFree =<< dir + case (free, fromMaybe 1 (keySize key)) of + (Just have, need) -> do + let ok = (need + reserve <= have + alreadythere) + unless ok $ + needmorespace (need + reserve - have - alreadythere) + return ok + _ -> return True + ) where dir = maybe (fromRepo gitAnnexDir) return destination needmorespace n = @@ -498,9 +500,9 @@ getKeysPresent keyloc = do direct <- isDirect dir <- fromRepo gitAnnexObjectDir s <- getstate direct - liftIO $ traverse s direct (2 :: Int) dir + liftIO $ walk s direct (2 :: Int) dir where - traverse s direct depth dir = do + walk s direct depth dir = do contents <- catchDefaultIO [] (dirContents dir) if depth == 0 then do @@ -508,7 +510,7 @@ getKeysPresent keyloc = do let keys = mapMaybe (fileKey . takeFileName) contents' continue keys [] else do - let deeper = traverse s direct (depth - 1) + let deeper = walk s direct (depth - 1) continue [] (map deeper contents) continue keys [] = return keys continue keys (a:as) = do diff --git a/Annex/Drop.hs b/Annex/Drop.hs index 0ea815db2..973e51348 100644 --- a/Annex/Drop.hs +++ b/Annex/Drop.hs @@ -9,7 +9,7 @@ module Annex.Drop where import Common.Annex import Logs.Trust -import Config.NumCopies +import Annex.NumCopies import Types.Remote (uuid) import Types.Key (key2file) import qualified Remote diff --git a/Annex/Init.hs b/Annex/Init.hs index 50f4d8522..2cc1c1897 100644 --- a/Annex/Init.hs +++ b/Annex/Init.hs @@ -57,15 +57,15 @@ genDescription Nothing = do initialize :: Maybe String -> Annex () initialize mdescription = do + {- This will make the first commit to git, so ensure git is set up + - properly to allow commits when running it. -} + ensureCommit $ Annex.Branch.create + prepUUID initialize' u <- getUUID - {- This will make the first commit to git, so ensure git is set up - - properly to allow commits when running it. -} - ensureCommit $ do - Annex.Branch.create - describeUUID u =<< genDescription mdescription + describeUUID u =<< genDescription mdescription -- Everything except for uuid setup. initialize' :: Annex () diff --git a/Config/NumCopies.hs b/Annex/NumCopies.hs index 50dcdf684..879513927 100644 --- a/Config/NumCopies.hs +++ b/Annex/NumCopies.hs @@ -1,11 +1,11 @@ -{- git-annex numcopies configuration +{- git-annex numcopies configuration and checking - - - Copyright 2014 Joey Hess <id@joeyh.name> + - Copyright 2014-2015 Joey Hess <id@joeyh.name> - - Licensed under the GNU GPL version 3 or higher. -} -module Config.NumCopies ( +module Annex.NumCopies ( module Types.NumCopies, module Logs.NumCopies, getFileNumCopies, @@ -15,6 +15,8 @@ module Config.NumCopies ( defaultNumCopies, numCopiesCheck, numCopiesCheck', + verifyEnoughCopies, + knownCopies, ) where import Common.Annex @@ -24,6 +26,8 @@ import Logs.NumCopies import Logs.Trust import Annex.CheckAttr import qualified Remote +import Annex.UUID +import Annex.Content defaultNumCopies :: NumCopies defaultNumCopies = NumCopies 1 @@ -83,3 +87,61 @@ numCopiesCheck' :: FilePath -> (Int -> Int -> v) -> [UUID] -> Annex v numCopiesCheck' file vs have = do NumCopies needed <- getFileNumCopies file return $ length have `vs` needed + +{- Verifies that enough copies of a key exist amoung the listed remotes, + - priting an informative message if not. + -} +verifyEnoughCopies + :: String -- message to print when there are no known locations + -> Key + -> NumCopies + -> [UUID] -- repos to skip (generally untrusted remotes) + -> [UUID] -- repos that are trusted or already verified to have it + -> [Remote] -- remotes to check to see if they have it + -> Annex Bool +verifyEnoughCopies nolocmsg key need skip trusted tocheck = + helper [] [] (nub trusted) (nub tocheck) + where + helper bad missing have [] + | NumCopies (length have) >= need = return True + | otherwise = do + notEnoughCopies key need have (skip++missing) bad nolocmsg + return False + helper bad missing have (r:rs) + | NumCopies (length have) >= need = return True + | otherwise = do + let u = Remote.uuid r + let duplicate = u `elem` have + haskey <- Remote.hasKey r key + case (duplicate, haskey) of + (False, Right True) -> helper bad missing (u:have) rs + (False, Left _) -> helper (r:bad) missing have rs + (False, Right False) -> helper bad (u:missing) have rs + _ -> helper bad missing have rs + +notEnoughCopies :: Key -> NumCopies -> [UUID] -> [UUID] -> [Remote] -> String -> Annex () +notEnoughCopies key need have skip bad nolocmsg = do + showNote "unsafe" + showLongNote $ + "Could only verify the existence of " ++ + show (length have) ++ " out of " ++ show (fromNumCopies need) ++ + " necessary copies" + Remote.showTriedRemotes bad + Remote.showLocations True key (have++skip) nolocmsg + +{- Cost ordered lists of remotes that the location log indicates + - may have a key. + - + - Also returns a list of UUIDs that are trusted to have the key + - (some may not have configured remotes). If the current repository + - currently has the key, and is not untrusted, it is included in this list. + -} +knownCopies :: Key -> Annex ([Remote], [UUID]) +knownCopies key = do + (remotes, trusteduuids) <- Remote.keyPossibilitiesTrusted key + u <- getUUID + trusteduuids' <- ifM (inAnnex key <&&> (<= SemiTrusted) <$> lookupTrust u) + ( pure (u:trusteduuids) + , pure trusteduuids + ) + return (remotes, trusteduuids') diff --git a/Annex/Perms.hs b/Annex/Perms.hs index 3ae351d8c..06971173f 100644 --- a/Annex/Perms.hs +++ b/Annex/Perms.hs @@ -69,14 +69,14 @@ annexFileMode = withShared $ return . go {- Creates a directory inside the gitAnnexDir, including any parent - directories. Makes directories with appropriate permissions. -} createAnnexDirectory :: FilePath -> Annex () -createAnnexDirectory dir = traverse dir [] =<< top +createAnnexDirectory dir = walk dir [] =<< top where top = parentDir <$> fromRepo gitAnnexDir - traverse d below stop + walk d below stop | d `equalFilePath` stop = done | otherwise = ifM (liftIO $ doesDirectoryExist d) ( done - , traverse (parentDir d) (d:below) stop + , walk (parentDir d) (d:below) stop ) where done = forM_ below $ \p -> do diff --git a/Assistant.hs b/Assistant.hs index fb04f5110..265827a77 100644 --- a/Assistant.hs +++ b/Assistant.hs @@ -57,7 +57,6 @@ import Utility.LogFile #ifdef mingw32_HOST_OS import Utility.Env import Annex.Path -import Config.Files import System.Environment (getArgs) #endif diff --git a/Assistant/DaemonStatus.hs b/Assistant/DaemonStatus.hs index 1ed40595e..4c42ffdbe 100644 --- a/Assistant/DaemonStatus.hs +++ b/Assistant/DaemonStatus.hs @@ -16,6 +16,7 @@ import Assistant.Types.NetMessager import Utility.NotificationBroadcaster import Logs.Transfer import Logs.Trust +import Logs.TimeStamp import qualified Remote import qualified Types.Remote as Remote import qualified Git @@ -23,8 +24,6 @@ import qualified Git import Control.Concurrent.STM import System.Posix.Types import Data.Time.Clock.POSIX -import Data.Time -import System.Locale import qualified Data.Map as M import qualified Data.Set as S import qualified Data.Text as T @@ -125,21 +124,18 @@ readDaemonStatusFile file = parse <$> newDaemonStatus <*> readFile file where parse status = foldr parseline status . lines parseline line status - | key == "lastRunning" = parseval readtime $ \v -> + | key == "lastRunning" = parseval parsePOSIXTime $ \v -> status { lastRunning = Just v } | key == "scanComplete" = parseval readish $ \v -> status { scanComplete = v } | key == "sanityCheckRunning" = parseval readish $ \v -> status { sanityCheckRunning = v } - | key == "lastSanityCheck" = parseval readtime $ \v -> + | key == "lastSanityCheck" = parseval parsePOSIXTime $ \v -> status { lastSanityCheck = Just v } | otherwise = status -- unparsable line where (key, value) = separate (== ':') line parseval parser a = maybe status a (parser value) - readtime s = do - d <- parseTime defaultTimeLocale "%s%Qs" s - Just $ utcTimeToPOSIXSeconds d {- Checks if a time stamp was made after the daemon was lastRunning. - diff --git a/Assistant/Install.hs b/Assistant/Install.hs index 6da6d2389..00d719bec 100644 --- a/Assistant/Install.hs +++ b/Assistant/Install.hs @@ -145,10 +145,12 @@ installFileManagerHooks program = do , "Name=" ++ command , "Icon=git-annex" , unwords - [ "Exec=sh -c 'cd \"$(dirname '%U')\" &&" + [ "Exec=sh -c 'cd \"$(dirname \"$1\")\" &&" , program , command - , "--notify-start --notify-finish -- %U'" + , "--notify-start --notify-finish -- \"$1\"'" + , "false" -- this becomes $0 in sh, so unused + , "%f" ] ] #else diff --git a/Assistant/Install/AutoStart.hs b/Assistant/Install/AutoStart.hs index b27b69775..57450304b 100644 --- a/Assistant/Install/AutoStart.hs +++ b/Assistant/Install/AutoStart.hs @@ -6,6 +6,7 @@ -} {-# LANGUAGE CPP #-} +{-# OPTIONS_GHC -fno-warn-tabs #-} module Assistant.Install.AutoStart where diff --git a/Assistant/Install/Menu.hs b/Assistant/Install/Menu.hs index 32393abaf..dd21ee117 100644 --- a/Assistant/Install/Menu.hs +++ b/Assistant/Install/Menu.hs @@ -6,6 +6,7 @@ -} {-# LANGUAGE CPP #-} +{-# OPTIONS_GHC -fno-warn-tabs #-} module Assistant.Install.Menu where diff --git a/Assistant/Pairing.hs b/Assistant/Pairing.hs index b24e5fdb6..2390379e2 100644 --- a/Assistant/Pairing.hs +++ b/Assistant/Pairing.hs @@ -81,6 +81,8 @@ data PairingInProgress = PairingInProgress } deriving (Show) +data AddrClass = IPv4AddrClass | IPv6AddrClass + data SomeAddr = IPv4Addr HostAddress {- My Android build of the Network library does not currently have IPV6 - support. -} diff --git a/Assistant/Pairing/MakeRemote.hs b/Assistant/Pairing/MakeRemote.hs index 05533e270..e2e120e22 100644 --- a/Assistant/Pairing/MakeRemote.hs +++ b/Assistant/Pairing/MakeRemote.hs @@ -88,8 +88,8 @@ bestHostName msg = case remoteHostName $ pairMsgData msg of fallback = do let a = pairMsgAddr msg let sockaddr = case a of - IPv4Addr addr -> SockAddrInet (PortNum 0) addr - IPv6Addr addr -> SockAddrInet6 (PortNum 0) 0 addr 0 + IPv4Addr addr -> SockAddrInet (fromInteger 0) addr + IPv6Addr addr -> SockAddrInet6 (fromInteger 0) 0 addr 0 fromMaybe (showAddr a) <$> catchDefaultIO Nothing (fst <$> getNameInfo [] True False sockaddr) diff --git a/Assistant/Pairing/Network.hs b/Assistant/Pairing/Network.hs index 7a4ac3ffe..694dcbbcc 100644 --- a/Assistant/Pairing/Network.hs +++ b/Assistant/Pairing/Network.hs @@ -33,9 +33,9 @@ pairingPort = 55556 {- Goal: Reach all hosts on the same network segment. - Method: Use same address that avahi uses. Other broadcast addresses seem - to not be let through some routers. -} -multicastAddress :: SomeAddr -> HostName -multicastAddress (IPv4Addr _) = "224.0.0.251" -multicastAddress (IPv6Addr _) = "ff02::fb" +multicastAddress :: AddrClass -> HostName +multicastAddress IPv4AddrClass = "224.0.0.251" +multicastAddress IPv6AddrClass = "ff02::fb" {- Multicasts a message repeatedly on all interfaces, with a 2 second - delay between each transmission. The message is repeated forever @@ -62,7 +62,7 @@ multicastPairMsg repeats secret pairdata stage = go M.empty repeats sendinterface cache i = void $ tryIO $ withSocketsDo $ bracket setup cleanup use where - setup = multicastSender (multicastAddress i) pairingPort + setup = multicastSender (multicastAddress IPv4AddrClass) pairingPort cleanup (sock, _) = sClose sock -- FIXME does not work use (sock, addr) = do setInterface sock (showAddr i) diff --git a/Assistant/Threads/Committer.hs b/Assistant/Threads/Committer.hs index 717a99c96..f4af93285 100644 --- a/Assistant/Threads/Committer.hs +++ b/Assistant/Threads/Committer.hs @@ -196,7 +196,7 @@ maxCommitSize :: Int maxCommitSize = 5000 {- Decide if now is a good time to make a commit. - - Note that the list of changes has an undefined order. + - Note that the list of changes has a random order. - - Current strategy: If there have been 10 changes within the past second, - a batch activity is taking place, so wait for later. diff --git a/Assistant/Threads/MountWatcher.hs b/Assistant/Threads/MountWatcher.hs index 023af53cb..3ccdd1adc 100644 --- a/Assistant/Threads/MountWatcher.hs +++ b/Assistant/Threads/MountWatcher.hs @@ -63,11 +63,7 @@ dbusThread urlrenderer = do wasmounted <- liftIO $ swapMVar mvar nowmounted handleMounts urlrenderer wasmounted nowmounted liftIO $ forM_ mountChanged $ \matcher -> -#if MIN_VERSION_dbus(0,10,7) void $ addMatch client matcher handleevent -#else - listen client matcher handleevent -#endif , do liftAnnex $ warning "No known volume monitor available through dbus; falling back to mtab polling" diff --git a/Assistant/Threads/NetWatcher.hs b/Assistant/Threads/NetWatcher.hs index ad3a87a91..07ccdaf24 100644 --- a/Assistant/Threads/NetWatcher.hs +++ b/Assistant/Threads/NetWatcher.hs @@ -112,11 +112,7 @@ checkNetMonitor client = do -} listenNMConnections :: Client -> (Bool -> IO ()) -> IO () listenNMConnections client setconnected = -#if MIN_VERSION_dbus(0,10,7) void $ addMatch client matcher -#else - listen client matcher -#endif $ \event -> mapM_ handleevent (map dictionaryItems $ mapMaybe fromVariant $ signalBody event) where @@ -166,11 +162,7 @@ listenWicdConnections client setconnected = do | any (== wicd_disconnected) status = setconnected False | otherwise = noop match matcher a = -#if MIN_VERSION_dbus(0,10,7) void $ addMatch client matcher a -#else - listen client matcher a -#endif #endif handleConnection :: Assistant () diff --git a/Assistant/Threads/PairListener.hs b/Assistant/Threads/PairListener.hs index e4f87494c..ba2ae955c 100644 --- a/Assistant/Threads/PairListener.hs +++ b/Assistant/Threads/PairListener.hs @@ -31,7 +31,7 @@ pairListenerThread urlrenderer = namedThread "PairListener" $ do where {- Note this can crash if there's no network interface, - or only one like lo that doesn't support multicast. -} - getsock = multicastReceiver (multicastAddress $ IPv4Addr undefined) pairingPort + getsock = multicastReceiver (multicastAddress IPv4AddrClass) pairingPort go reqs cache sock = liftIO (getmsg sock []) >>= \msg -> case readish msg of Nothing -> go reqs cache sock diff --git a/Assistant/Threads/XMPPPusher.hs b/Assistant/Threads/XMPPPusher.hs index ec11b9b94..bff17356d 100644 --- a/Assistant/Threads/XMPPPusher.hs +++ b/Assistant/Threads/XMPPPusher.hs @@ -78,4 +78,5 @@ selectNextPush lastpushedto l = go [] l (Pushing clientid _) | Just clientid /= lastpushedto -> (m, rejected ++ ms) _ -> go (m:rejected) ms - go [] [] = undefined + go [] [] = error "empty push queue" + diff --git a/Assistant/Types/BranchChange.hs b/Assistant/Types/BranchChange.hs index f769657d0..4c15f133a 100644 --- a/Assistant/Types/BranchChange.hs +++ b/Assistant/Types/BranchChange.hs @@ -8,7 +8,8 @@ module Assistant.Types.BranchChange where import Control.Concurrent.MSampleVar -import Common.Annex +import Control.Applicative +import Prelude newtype BranchChangeHandle = BranchChangeHandle (MSampleVar ()) diff --git a/Assistant/XMPP.hs b/Assistant/XMPP.hs index b9ae50e27..52cd31939 100644 --- a/Assistant/XMPP.hs +++ b/Assistant/XMPP.hs @@ -22,7 +22,8 @@ import qualified Data.Map as M import Data.ByteString (ByteString) import qualified Data.ByteString as B import Data.XML.Types -import qualified "dataenc" Codec.Binary.Base64 as B64 +import qualified "sandi" Codec.Binary.Base64 as B64 +import Data.Bits.Utils {- Name of the git-annex tag, in our own XML namespace. - (Not using a namespace URL to avoid unnecessary bloat.) -} @@ -212,10 +213,10 @@ encodeExitCode (ExitFailure n) = n {- Base 64 encoding a ByteString to use as the content of a tag. -} encodeTagContent :: ByteString -> [Node] -encodeTagContent b = [NodeContent $ ContentText $ T.pack $ B64.encode $ B.unpack b] +encodeTagContent b = [NodeContent $ ContentText $ T.pack $ w82s $ B.unpack $ B64.encode b] decodeTagContent :: Element -> Maybe ByteString -decodeTagContent elt = B.pack <$> B64.decode s +decodeTagContent elt = either (const Nothing) Just (B64.decode $ B.pack $ s2w8 s) where s = T.unpack $ T.concat $ elementText elt diff --git a/Build/BundledPrograms.hs b/Build/BundledPrograms.hs index 1e826cb10..4ceee3b52 100644 --- a/Build/BundledPrograms.hs +++ b/Build/BundledPrograms.hs @@ -35,13 +35,14 @@ bundledPrograms = catMaybes #endif , Just "rsync" #ifndef darwin_HOST_OS +#ifndef mingw32_HOST_OS -- OS X has ssh installed by default. -- Linux probably has ssh, but not guaranteed. - -- On Windows, msysgit provides ssh, but not in PATH, - -- so we ship our own. + -- On Windows, msysgit provides ssh. , Just "ssh" , Just "ssh-keygen" #endif +#endif #ifndef mingw32_HOST_OS , Just "sh" #endif diff --git a/Build/Configure.hs b/Build/Configure.hs index c90231b29..55f2493a1 100644 --- a/Build/Configure.hs +++ b/Build/Configure.hs @@ -1,5 +1,7 @@ {- Checks system configuration and generates SysConfig.hs. -} +{-# OPTIONS_GHC -fno-warn-tabs #-} + module Build.Configure where import System.Directory diff --git a/Build/DesktopFile.hs b/Build/DesktopFile.hs index 6e70b0d5f..9d68ff1d9 100644 --- a/Build/DesktopFile.hs +++ b/Build/DesktopFile.hs @@ -7,6 +7,7 @@ -} {-# LANGUAGE CPP #-} +{-# OPTIONS_GHC -fno-warn-tabs #-} module Build.DesktopFile where diff --git a/Build/DistributionUpdate.hs b/Build/DistributionUpdate.hs index 37b8e04aa..da1202fe2 100644 --- a/Build/DistributionUpdate.hs +++ b/Build/DistributionUpdate.hs @@ -10,7 +10,7 @@ import Common.Annex import Types.Distribution -import Build.Version +import Build.Version (getChangelogVersion, Version) import Utility.UserInfo import Utility.Url import qualified Git.Construct diff --git a/Build/NullSoftInstaller.hs b/Build/NullSoftInstaller.hs index 42260bd3f..7bb16f3f7 100644 --- a/Build/NullSoftInstaller.hs +++ b/Build/NullSoftInstaller.hs @@ -1,7 +1,8 @@ {- Generates a NullSoft installer program for git-annex on Windows.
-
- To build the installer, git-annex should already be built by cabal,
- - and ssh and rsync, as well as cygwin libraries, already installed.
+ - and ssh and rsync etc, as well as cygwin libraries, already installed
+ - from cygwin.
-
- This uses the Haskell nsis package (cabal install nsis)
- to generate a .nsi file, which is then used to produce
@@ -11,7 +12,7 @@ - exception of git. The user needs to install git separately,
- and the installer checks for that.
-
- - Copyright 2013 Joey Hess <id@joeyh.name>
+ - Copyright 2013-2015 Joey Hess <id@joeyh.name>
-
- Licensed under the GNU GPL version 3 or higher.
-}
@@ -22,13 +23,17 @@ import Development.NSIS import System.Directory
import System.FilePath
import Control.Monad
+import Control.Applicative
import Data.String
import Data.Maybe
+import Data.Char
+import Data.List (nub, isPrefixOf)
import Utility.Tmp
import Utility.Path
import Utility.CopyFile
import Utility.SafeCommand
+import Utility.Process
import Build.BundledPrograms
main = do
@@ -37,17 +42,19 @@ main = do mustSucceed "ln" [File "dist/build/git-annex/git-annex.exe", File gitannex]
let license = tmpdir </> licensefile
mustSucceed "sh" [Param "-c", Param $ "zcat standalone/licences.gz > '" ++ license ++ "'"]
- extrabins <- forM (cygwinPrograms ++ cygwinDlls) $ \f -> do
+ extrabins <- forM (cygwinPrograms) $ \f -> do
p <- searchPath f
when (isNothing p) $
print ("unable to find in PATH", f)
return p
+ dlls <- forM (catMaybes extrabins) findCygLibs
+ dllpaths <- mapM searchPath (nub (concat dlls))
webappscript <- vbsLauncher tmpdir "git-annex-webapp" "git-annex webapp"
autostartscript <- vbsLauncher tmpdir "git-annex-autostart" "git annex assistant --autostart"
let htmlhelp = tmpdir </> "git-annex.html"
writeFile htmlhelp htmlHelpText
writeFile nsifile $ makeInstaller gitannex license htmlhelp
- (catMaybes extrabins)
+ (wrappers ++ catMaybes (extrabins ++ dllpaths))
[ webappscript, autostartscript ]
mustSucceed "makensis" [File nsifile]
removeFile nsifile -- left behind if makensis fails
@@ -85,7 +92,7 @@ uninstaller = "git-annex-uninstall.exe" gitInstallDir :: Exp FilePath
gitInstallDir = fromString "$PROGRAMFILES\\Git"
--- This intentionall has a different name than git-annex or
+-- This intentionally has a different name than git-annex or
-- git-annex-webapp, since it is itself treated as an executable file.
-- Also, on XP, the filename is displayed, not the description.
startMenuItem :: Exp FilePath
@@ -169,46 +176,6 @@ makeInstaller gitannex license htmlhelp extrabins launchers = nsis $ do cygwinPrograms :: [FilePath]
cygwinPrograms = map (\p -> p ++ ".exe") bundledPrograms
--- These are the dlls needed by Cygwin's rsync, ssh, etc.
--- TODO: Use ldd (available in cygwin) to automatically find all
--- needed libs.
-cygwinDlls :: [FilePath]
-cygwinDlls =
- [ "cygwin1.dll"
- , "cygasn1-8.dll"
- , "cygattr-1.dll"
- , "cygheimbase-1.dll"
- , "cygroken-18.dll"
- , "cygcom_err-2.dll"
- , "cygheimntlm-0.dll"
- , "cygsqlite3-0.dll"
- , "cygcrypt-0.dll"
- , "cyghx509-5.dll"
- , "cygssp-0.dll"
- , "cygcrypto-1.0.0.dll"
- , "cygiconv-2.dll"
- , "cyggcc_s-1.dll"
- , "cygintl-8.dll"
- , "cygwind-0.dll"
- , "cyggssapi-3.dll"
- , "cygkrb5-26.dll"
- , "cygz.dll"
- , "cygidn-11.dll"
- , "libcurl-4.dll"
- , "cyggnutls-26.dll"
- , "libcrypto.dll"
- , "libssl.dll"
- , "cyggcrypt-11.dll"
- , "cyggpg-error-0.dll"
- , "cygp11-kit-0.dll"
- , "cygtasn1-3.dll"
- , "cygffi-6.dll"
- , "cygbz2-1.dll"
- , "cygreadline7.dll"
- , "cygncursesw-10.dll"
- , "cygusb0.dll"
- ]
-
-- msysgit opens Program Files/Git/doc/git/html/git-annex.html
-- when git annex --help is run.
htmlHelpText :: String
@@ -221,3 +188,18 @@ htmlHelpText = unlines , "</body>"
, "</html"
]
+
+-- Find cygwin libraries used by the specified executable.
+findCygLibs :: FilePath -> IO [FilePath]
+findCygLibs p = filter iscyg . mapMaybe parse . lines <$> readProcess "ldd" [p]
+ where
+ parse l = case words (dropWhile isSpace l) of
+ (dll:"=>":_dllpath:_offset:[]) -> Just dll
+ _ -> Nothing
+ iscyg f = "cyg" `isPrefixOf` f || "lib" `isPrefixOf` f
+
+wrappers :: [FilePath]
+wrappers =
+ [ "standalone\\windows\\ssh.cmd"
+ , "standalone\\windows\\ssh-keygen.cmd"
+ ]
diff --git a/Build/TestConfig.hs b/Build/TestConfig.hs index e55641fb0..35daf1945 100644 --- a/Build/TestConfig.hs +++ b/Build/TestConfig.hs @@ -1,5 +1,7 @@ {- Tests the system and generates Build.SysConfig.hs. -} +{-# OPTIONS_GHC -fno-warn-tabs #-} + module Build.TestConfig where import Utility.Path diff --git a/Build/Version.hs b/Build/Version.hs index da9d1bbcb..44b2b975b 100644 --- a/Build/Version.hs +++ b/Build/Version.hs @@ -1,5 +1,7 @@ {- Package version determination, for configure script. -} +{-# OPTIONS_GHC -fno-warn-tabs #-} + module Build.Version where import Data.Maybe @@ -18,7 +20,7 @@ type Version = String {- Set when making an official release. (Distribution vendors should set - this too.) -} isReleaseBuild :: IO Bool -isReleaseBuild = isJust <$> catchMaybeIO (getEnv "RELEASE_BUILD") +isReleaseBuild = (== Just "1") <$> catchMaybeIO (getEnv "RELEASE_BUILD") {- Version is usually based on the major version from the changelog, - plus the date of the last commit, plus the git rev of that commit. diff --git a/Build/mdwn2man b/Build/mdwn2man index a29ce649e..7ad0d889b 100755 --- a/Build/mdwn2man +++ b/Build/mdwn2man @@ -20,6 +20,7 @@ while (<>) { s/^[ \n]+//; s/^\t/ /; s/-/\\-/g; + s/git\\-annex/git-annex/g; s/^Warning:.*//g; s/^$/.PP\n/; s/^\*\s+(.*)/.IP "$1"/; diff --git a/CmdLine/Action.hs b/CmdLine/Action.hs index 5bef833c2..2838e4ff8 100644 --- a/CmdLine/Action.hs +++ b/CmdLine/Action.hs @@ -5,8 +5,6 @@ - Licensed under the GNU GPL version 3 or higher. -} -{-# LANGUAGE BangPatterns #-} - module CmdLine.Action where import Common.Annex @@ -119,14 +117,11 @@ includeCommandAction a = account =<< tryIO go account (Right True) = return True account (Right False) = incerr account (Left err) = do - showErr err + toplevelWarning True (show err) showEndFail incerr incerr = do - Annex.changeState $ \s -> - let ! c = Annex.errcounter s + 1 - ! s' = s { Annex.errcounter = c } - in s' + Annex.incError return False {- Runs a single command action through the start, perform and cleanup diff --git a/CmdLine/Batch.hs b/CmdLine/Batch.hs new file mode 100644 index 000000000..836472eb0 --- /dev/null +++ b/CmdLine/Batch.hs @@ -0,0 +1,41 @@ +{- git-annex batch commands + - + - Copyright 2015 Joey Hess <id@joeyh.name> + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module CmdLine.Batch where + +import Common.Annex +import Command + +batchOption :: Option +batchOption = flagOption [] "batch" "enable batch mode" + +data BatchMode = Batch | NoBatch +type Batchable t = BatchMode -> t -> CommandStart + +-- A Batchable command can run in batch mode, or not. +-- In batch mode, one line at a time is read, parsed, and a reply output to +-- stdout. In non batch mode, the command's parameters are parsed and +-- a reply output for each. +batchable :: ((t -> CommandStart) -> CommandSeek) -> Batchable t -> CommandSeek +batchable seeker starter params = ifM (getOptionFlag batchOption) + ( batchloop + , seeker (starter NoBatch) params + ) + where + batchloop = do + mp <- liftIO $ catchMaybeIO getLine + case mp of + Nothing -> return () + Just p -> do + seeker (starter Batch) [p] + batchloop + +-- bad input is indicated by an empty line in batch mode. In non batch +-- mode, exit on bad input. +batchBadInput :: BatchMode -> Annex () +batchBadInput NoBatch = liftIO exitFailure +batchBadInput Batch = liftIO $ putStrLn "" diff --git a/CmdLine/GitAnnex.hs b/CmdLine/GitAnnex.hs index fde4e2d08..326dd3b2b 100644 --- a/CmdLine/GitAnnex.hs +++ b/CmdLine/GitAnnex.hs @@ -74,6 +74,7 @@ import qualified Command.Dead import qualified Command.Group import qualified Command.Wanted import qualified Command.GroupWanted +import qualified Command.Required import qualified Command.Schedule import qualified Command.Ungroup import qualified Command.Vicfg @@ -149,6 +150,7 @@ cmds = concat , Command.Group.cmd , Command.Wanted.cmd , Command.GroupWanted.cmd + , Command.Required.cmd , Command.Schedule.cmd , Command.Ungroup.cmd , Command.Vicfg.cmd diff --git a/CmdLine/Seek.hs b/CmdLine/Seek.hs index ea06fc976..3166ab83d 100644 --- a/CmdLine/Seek.hs +++ b/CmdLine/Seek.hs @@ -218,8 +218,9 @@ seekHelper a params = do ll <- inRepo $ \g -> concat <$> forM (segmentXargsOrdered params) (runSegmentPaths (\fs -> Git.Command.leaveZombie <$> a fs g)) forM_ (map fst $ filter (null . snd) $ zip params ll) $ \p -> - unlessM (isJust <$> liftIO (catchMaybeIO $ getSymbolicLinkStatus p)) $ - error $ p ++ " not found" + unlessM (isJust <$> liftIO (catchMaybeIO $ getSymbolicLinkStatus p)) $ do + toplevelWarning False (p ++ " not found") + Annex.incError return $ concat ll notSymlink :: FilePath -> IO Bool diff --git a/Command/Add.hs b/Command/Add.hs index c461c4d56..d53ba91ad 100644 --- a/Command/Add.hs +++ b/Command/Add.hs @@ -116,7 +116,10 @@ start file = ifAnnexed file addpresent add - Lockdown can fail if a file gets deleted, and Nothing will be returned. -} lockDown :: FilePath -> Annex (Maybe KeySource) -lockDown = either (\e -> showErr e >> return Nothing) (return . Just) <=< lockDown' +lockDown = either + (\e -> warning (show e) >> return Nothing) + (return . Just) + <=< lockDown' lockDown' :: FilePath -> Annex (Either IOException KeySource) lockDown' file = ifM crippledFileSystem diff --git a/Command/AddUrl.hs b/Command/AddUrl.hs index 5defc52d9..6474f2614 100644 --- a/Command/AddUrl.hs +++ b/Command/AddUrl.hs @@ -178,7 +178,7 @@ startWeb relaxed optfile pathdepth s = go $ fromMaybe bad $ parseURI urlstring pathmax <- liftIO $ fileNameLengthLimit "." let file = flip fromMaybe optfile $ truncateFilePath pathmax $ sanitizeFilePath $ - Quvi.pageTitle page ++ "." ++ Quvi.linkSuffix link + Quvi.pageTitle page ++ "." ++ fromMaybe "m" (Quvi.linkSuffix link) showStart "addurl" file next $ performQuvi relaxed urlstring (Quvi.linkUrl link) file #else diff --git a/Command/Assistant.hs b/Command/Assistant.hs index 590a2e437..97bc08c7b 100644 --- a/Command/Assistant.hs +++ b/Command/Assistant.hs @@ -20,7 +20,7 @@ import Assistant.Install import System.Environment cmd :: [Command] -cmd = [noRepo checkAutoStart $ dontCheck repoExists $ withOptions options $ +cmd = [noRepo checkNoRepoOpts $ dontCheck repoExists $ withOptions options $ notBareRepo $ command "assistant" paramNothing seek SectionCommon "automatically sync changes"] @@ -30,11 +30,15 @@ options = , Command.Watch.stopOption , autoStartOption , startDelayOption + , autoStopOption ] autoStartOption :: Option autoStartOption = flagOption [] "autostart" "start in known repositories" +autoStopOption :: Option +autoStopOption = flagOption [] "autostop" "stop in known repositories" + startDelayOption :: Option startDelayOption = fieldOption [] "startdelay" paramNumber "delay before running startup scan" @@ -43,25 +47,31 @@ seek ps = do stopdaemon <- getOptionFlag Command.Watch.stopOption foreground <- getOptionFlag Command.Watch.foregroundOption autostart <- getOptionFlag autoStartOption + autostop <- getOptionFlag autoStopOption startdelay <- getOptionField startDelayOption (pure . maybe Nothing parseDuration) - withNothing (start foreground stopdaemon autostart startdelay) ps + withNothing (start foreground stopdaemon autostart autostop startdelay) ps -start :: Bool -> Bool -> Bool -> Maybe Duration -> CommandStart -start foreground stopdaemon autostart startdelay +start :: Bool -> Bool -> Bool -> Bool -> Maybe Duration -> CommandStart +start foreground stopdaemon autostart autostop startdelay | autostart = do liftIO $ autoStart startdelay stop + | autostop = do + liftIO autoStop + stop | otherwise = do liftIO ensureInstalled ensureInitialized Command.Watch.start True foreground stopdaemon startdelay -{- Run outside a git repository. Check to see if any parameter is - - --autostart and enter autostart mode. -} -checkAutoStart :: CmdParams -> IO () -checkAutoStart _ = ifM (elem "--autostart" <$> getArgs) +{- Run outside a git repository; support autostart and autostop mode. -} +checkNoRepoOpts :: CmdParams -> IO () +checkNoRepoOpts _ = ifM (elem "--autostart" <$> getArgs) ( autoStart Nothing - , error "Not in a git repository." + , ifM (elem "--autostop" <$> getArgs) + ( autoStop + , error "Not in a git repository." + ) ) autoStart :: Maybe Duration -> IO () @@ -89,3 +99,15 @@ autoStart startdelay = do [ Param "assistant" , Param $ "--startdelay=" ++ fromDuration (fromMaybe (Duration 5) startdelay) ] + +autoStop :: IO () +autoStop = do + dirs <- liftIO readAutoStartFile + program <- programPath + forM_ dirs $ \d -> do + putStrLn $ "git-annex autostop in " ++ d + setCurrentDirectory d + ifM (boolSystem program [Param "assistant", Param "--stop"]) + ( putStrLn "ok" + , putStrLn "failed" + ) diff --git a/Command/ContentLocation.hs b/Command/ContentLocation.hs index 3f4775f57..10879f5b1 100644 --- a/Command/ContentLocation.hs +++ b/Command/ContentLocation.hs @@ -9,19 +9,20 @@ module Command.ContentLocation where import Common.Annex import Command +import CmdLine.Batch import Annex.Content cmd :: [Command] -cmd = [noCommit $ noMessages $ +cmd = [withOptions [batchOption] $ noCommit $ noMessages $ command "contentlocation" (paramRepeating paramKey) seek SectionPlumbing "looks up content for a key"] seek :: CommandSeek -seek = withKeys start +seek = batchable withKeys start -start :: Key -> CommandStart -start k = do - liftIO . maybe exitFailure putStrLn +start :: Batchable Key +start batchmode k = do + maybe (batchBadInput batchmode) (liftIO . putStrLn) =<< inAnnex' (pure True) Nothing check k stop where diff --git a/Command/Copy.hs b/Command/Copy.hs index 1b9b2aac8..5cfdabb4e 100644 --- a/Command/Copy.hs +++ b/Command/Copy.hs @@ -12,7 +12,7 @@ import Command import qualified Command.Move import qualified Remote import Annex.Wanted -import Config.NumCopies +import Annex.NumCopies cmd :: [Command] cmd = [withOptions copyOptions $ command "copy" paramPaths seek diff --git a/Command/Drop.hs b/Command/Drop.hs index 63b9ccb7f..698dd7bad 100644 --- a/Command/Drop.hs +++ b/Command/Drop.hs @@ -15,7 +15,7 @@ import Annex.UUID import Logs.Location import Logs.Trust import Logs.PreferredContent -import Config.NumCopies +import Annex.NumCopies import Annex.Content import Annex.Wanted import Annex.Notification @@ -27,7 +27,7 @@ cmd = [withOptions (dropOptions) $ command "drop" paramPaths seek SectionCommon "indicate content of files not currently wanted"] dropOptions :: [Option] -dropOptions = dropFromOption : annexedMatchingOptions ++ [autoOption] +dropOptions = dropFromOption : annexedMatchingOptions ++ [autoOption] ++ keyOptions dropFromOption :: Option dropFromOption = fieldOption ['f'] "from" paramRemote "drop content from a remote" @@ -36,23 +36,32 @@ seek :: CommandSeek seek ps = do from <- getOptionField dropFromOption Remote.byNameWithUUID auto <- getOptionFlag autoOption - withFilesInGit (whenAnnexed $ start auto from) ps + withKeyOptions auto + (startKeys auto from) + (withFilesInGit $ whenAnnexed $ start auto from) + ps start :: Bool -> Maybe Remote -> FilePath -> Key -> CommandStart -start auto from file key = checkDropAuto auto from file key $ \numcopies -> +start auto from file key = start' auto from key (Just file) + +start' :: Bool -> Maybe Remote -> Key -> AssociatedFile -> CommandStart +start' auto from key afile = checkDropAuto auto from afile key $ \numcopies -> stopUnless want $ case from of - Nothing -> startLocal (Just file) numcopies key Nothing + Nothing -> startLocal afile numcopies key Nothing Just remote -> do u <- getUUID if Remote.uuid remote == u - then startLocal (Just file) numcopies key Nothing - else startRemote (Just file) numcopies key remote + then startLocal afile numcopies key Nothing + else startRemote afile numcopies key remote where want - | auto = wantDrop False (Remote.uuid <$> from) (Just key) (Just file) + | auto = wantDrop False (Remote.uuid <$> from) (Just key) afile | otherwise = return True +startKeys :: Bool -> Maybe Remote -> Key -> CommandStart +startKeys auto from key = start' auto from key Nothing + startLocal :: AssociatedFile -> NumCopies -> Key -> Maybe Remote -> CommandStart startLocal afile numcopies key knownpresentremote = stopUnless (inAnnex key) $ do showStart' "drop" key afile @@ -72,7 +81,7 @@ performLocal key afile numcopies knownpresentremote = lockContent key $ \content (remotes, trusteduuids) <- Remote.keyPossibilitiesTrusted key let trusteduuids' = case knownpresentremote of Nothing -> trusteduuids - Just r -> nub (Remote.uuid r:trusteduuids) + Just r -> Remote.uuid r:trusteduuids untrusteduuids <- trustGet UnTrusted let tocheck = Remote.remotesWithoutUUID remotes (trusteduuids'++untrusteduuids) u <- getUUID @@ -91,17 +100,9 @@ performRemote key afile numcopies remote = do -- Filter the remote it's being dropped from out of the lists of -- places assumed to have the key, and places to check. -- When the local repo has the key, that's one additional copy, - -- as long asthe local repo is not untrusted. - (remotes, trusteduuids) <- Remote.keyPossibilitiesTrusted key - present <- inAnnex key - u <- getUUID - trusteduuids' <- if present - then ifM ((<= SemiTrusted) <$> lookupTrust u) - ( pure (u:trusteduuids) - , pure trusteduuids - ) - else pure trusteduuids - let have = filter (/= uuid) trusteduuids' + -- as long as the local repo is not untrusted. + (remotes, trusteduuids) <- knownCopies key + let have = filter (/= uuid) trusteduuids untrusteduuids <- trustGet UnTrusted let tocheck = filter (/= remote) $ Remote.remotesWithoutUUID remotes (have++untrusteduuids) @@ -131,45 +132,20 @@ cleanupRemote key remote ok = do - --force overrides and always allows dropping. -} canDrop :: UUID -> Key -> AssociatedFile -> NumCopies -> [UUID] -> [Remote] -> [UUID] -> Annex Bool -canDrop dropfrom key afile numcopies have check skip = ifM (Annex.getState Annex.force) - ( return True - , checkRequiredContent dropfrom key afile - <&&> - findCopies key numcopies skip have check - ) - -findCopies :: Key -> NumCopies -> [UUID] -> [UUID] -> [Remote] -> Annex Bool -findCopies key need skip = helper [] [] - where - helper bad missing have [] - | NumCopies (length have) >= need = return True - | otherwise = notEnoughCopies key need have (skip++missing) bad - helper bad missing have (r:rs) - | NumCopies (length have) >= need = return True - | otherwise = do - let u = Remote.uuid r - let duplicate = u `elem` have - haskey <- Remote.hasKey r key - case (duplicate, haskey) of - (False, Right True) -> helper bad missing (u:have) rs - (False, Left _) -> helper (r:bad) missing have rs - (False, Right False) -> helper bad (u:missing) have rs - _ -> helper bad missing have rs - -notEnoughCopies :: Key -> NumCopies -> [UUID] -> [UUID] -> [Remote] -> Annex Bool -notEnoughCopies key need have skip bad = do - unsafe - showLongNote $ - "Could only verify the existence of " ++ - show (length have) ++ " out of " ++ show (fromNumCopies need) ++ - " necessary copies" - Remote.showTriedRemotes bad - Remote.showLocations True key (have++skip) - "Rather than dropping this file, try using: git annex move" - hint - return False +canDrop dropfrom key afile numcopies have check skip = + ifM (Annex.getState Annex.force) + ( return True + , ifM (checkRequiredContent dropfrom key afile + <&&> verifyEnoughCopies nolocmsg key numcopies skip have check + ) + ( return True + , do + hint + return False + ) + ) where - unsafe = showNote "unsafe" + nolocmsg = "Rather than dropping this file, try using: git annex move" hint = showLongNote "(Use --force to override this check, or adjust numcopies.)" checkRequiredContent :: UUID -> Key -> AssociatedFile -> Annex Bool @@ -187,8 +163,8 @@ requiredContent = do {- In auto mode, only runs the action if there are enough - copies on other semitrusted repositories. -} -checkDropAuto :: Bool -> Maybe Remote -> FilePath -> Key -> (NumCopies -> CommandStart) -> CommandStart -checkDropAuto auto mremote file key a = go =<< getFileNumCopies file +checkDropAuto :: Bool -> Maybe Remote -> AssociatedFile -> Key -> (NumCopies -> CommandStart) -> CommandStart +checkDropAuto auto mremote afile key a = go =<< maybe getNumCopies getFileNumCopies afile where go numcopies | auto = do diff --git a/Command/DropUnused.hs b/Command/DropUnused.hs index 36ff49720..d441a4bd2 100644 --- a/Command/DropUnused.hs +++ b/Command/DropUnused.hs @@ -14,7 +14,7 @@ import qualified Command.Drop import qualified Remote import qualified Git import Command.Unused (withUnusedMaps, UnusedMaps(..), startUnused) -import Config.NumCopies +import Annex.NumCopies cmd :: [Command] cmd = [withOptions [Command.Drop.dropFromOption] $ diff --git a/Command/ExamineKey.hs b/Command/ExamineKey.hs index 00d4d3a95..05db9817a 100644 --- a/Command/ExamineKey.hs +++ b/Command/ExamineKey.hs @@ -9,21 +9,22 @@ module Command.ExamineKey where import Common.Annex import Command +import CmdLine.Batch import qualified Utility.Format import Command.Find (formatOption, getFormat, showFormatted, keyVars) import Types.Key cmd :: [Command] -cmd = [noCommit $ noMessages $ withOptions [formatOption, jsonOption] $ +cmd = [noCommit $ noMessages $ withOptions [formatOption, jsonOption, batchOption] $ command "examinekey" (paramRepeating paramKey) seek SectionPlumbing "prints information from a key"] seek :: CommandSeek seek ps = do format <- getFormat - withKeys (start format) ps + batchable withKeys (start format) ps -start :: Maybe Utility.Format.Format -> Key -> CommandStart -start format key = do +start :: Maybe Utility.Format.Format -> Batchable Key +start format _ key = do showFormatted format (key2file key) (keyVars key) stop diff --git a/Command/Fsck.hs b/Command/Fsck.hs index 54f20f5e8..8414b5b26 100644 --- a/Command/Fsck.hs +++ b/Command/Fsck.hs @@ -24,21 +24,21 @@ import Annex.Link import Logs.Location import Logs.Trust import Logs.Activity -import Config.NumCopies +import Logs.TimeStamp +import Annex.NumCopies import Annex.UUID import Utility.DataUnits import Config import Types.Key import Types.CleanupActions import Utility.HumanTime +import Utility.CopyFile import Git.FilePath import Utility.PID import qualified Database.Fsck as FsckDb import Data.Time.Clock.POSIX -import Data.Time import System.Posix.Types (EpochTime) -import System.Locale cmd :: [Command] cmd = [withOptions fsckOptions $ command "fsck" paramPaths seek @@ -75,7 +75,7 @@ seek ps = do (withFilesInGit $ whenAnnexed $ start from i) ps withFsckDb i FsckDb.closeDb - recordActivity Fsck u + void $ tryIO $ recordActivity Fsck u start :: Maybe Remote -> Incremental -> FilePath -> Key -> CommandStart start from inc file key = do @@ -111,14 +111,15 @@ performRemote key file backend numcopies remote = dispatch (Left err) = do showNote err return False - dispatch (Right True) = withtmp $ \tmpfile -> - ifM (getfile tmpfile) - ( go True (Just tmpfile) - , do + dispatch (Right True) = withtmp $ \tmpfile -> do + r <- getfile tmpfile + case r of + Nothing -> go True Nothing + Just True -> go True (Just tmpfile) + Just False -> do warning "failed to download file from remote" void $ go True Nothing return False - ) dispatch (Right False) = go False Nothing go present localcopy = check [ verifyLocationLogRemote key file remote present @@ -134,14 +135,17 @@ performRemote key file backend numcopies remote = let cleanup = liftIO $ catchIO (removeFile tmp) (const noop) cleanup cleanup `after` a tmp - getfile tmp = - ifM (Remote.retrieveKeyFileCheap remote key (Just file) tmp) - ( return True + getfile tmp = ifM (checkDiskSpace (Just tmp) key 0) + ( ifM (Remote.retrieveKeyFileCheap remote key (Just file) tmp) + ( return (Just True) , ifM (Annex.getState Annex.fast) - ( return False - , Remote.retrieveKeyFile remote key (Just file) tmp dummymeter + ( return Nothing + , Just <$> + Remote.retrieveKeyFile remote key Nothing tmp dummymeter ) ) + , return (Just False) + ) dummymeter _ = noop startKey :: Incremental -> Key -> NumCopies -> CommandStart @@ -273,7 +277,7 @@ checkKeySize key = ifM isDirect checkKeySizeRemote :: Key -> Remote -> Maybe FilePath -> Annex Bool checkKeySizeRemote _ _ Nothing = return True checkKeySizeRemote key remote (Just file) = - checkKeySizeOr (badContentRemote remote) key file + checkKeySizeOr (badContentRemote remote file) key file checkKeySizeOr :: (Key -> Annex String) -> Key -> FilePath -> Annex Bool checkKeySizeOr bad key file = case Types.Key.keySize key of @@ -318,7 +322,7 @@ checkBackend backend key mfile = go =<< isDirect checkBackendRemote :: Backend -> Key -> Remote -> Maybe FilePath -> Annex Bool checkBackendRemote backend key remote = maybe (return True) go where - go = checkBackendOr (badContentRemote remote) backend key + go file = checkBackendOr (badContentRemote remote file) backend key file checkBackendOr :: (Key -> Annex String) -> Backend -> Key -> FilePath -> Annex Bool checkBackendOr bad backend key file = @@ -380,13 +384,36 @@ badContentDirect file key = do logStatus key InfoMissing return "left in place for you to examine" -badContentRemote :: Remote -> Key -> Annex String -badContentRemote remote key = do - ok <- Remote.removeKey remote key - when ok $ +{- Bad content is dropped from the remote. We have downloaded a copy + - from the remote to a temp file already (in some cases, it's just a + - symlink to a file in the remote). To avoid any further data loss, + - that temp file is moved to the bad content directory unless + - the local annex has a copy of the content. -} +badContentRemote :: Remote -> FilePath -> Key -> Annex String +badContentRemote remote localcopy key = do + bad <- fromRepo gitAnnexBadDir + let destbad = bad </> key2file key + movedbad <- ifM (inAnnex key <||> liftIO (doesFileExist destbad)) + ( return False + , do + createAnnexDirectory (parentDir destbad) + liftIO $ catchDefaultIO False $ + ifM (isSymbolicLink <$> getSymbolicLinkStatus localcopy) + ( copyFileExternal CopyTimeStamps localcopy destbad + , do + moveFile localcopy destbad + return True + ) + ) + + dropped <- Remote.removeKey remote key + when dropped $ Remote.logStatus remote key InfoMissing - return $ (if ok then "dropped from " else "failed to drop from ") - ++ Remote.name remote + return $ case (movedbad, dropped) of + (True, True) -> "moved from " ++ Remote.name remote ++ + " to " ++ destbad + (False, True) -> "dropped from " ++ Remote.name remote + (_, False) -> "failed to drop from" ++ Remote.name remote runFsck :: Incremental -> FilePath -> Key -> Annex Bool -> CommandStart runFsck inc file key a = ifM (needFsck inc key) @@ -448,14 +475,11 @@ getStartTime u = do liftIO $ catchDefaultIO Nothing $ do timestamp <- modificationTime <$> getFileStatus f let fromstatus = Just (realToFrac timestamp) - fromfile <- readishTime <$> readFile f + fromfile <- parsePOSIXTime <$> readFile f return $ if matchingtimestamp fromfile fromstatus then Just timestamp else Nothing where - readishTime :: String -> Maybe POSIXTime - readishTime s = utcTimeToPOSIXSeconds <$> - parseTime defaultTimeLocale "%s%Qs" s matchingtimestamp fromfile fromstatus = #ifndef mingw32_HOST_OS fromfile == fromstatus diff --git a/Command/Get.hs b/Command/Get.hs index 111c69e32..380a68097 100644 --- a/Command/Get.hs +++ b/Command/Get.hs @@ -12,7 +12,7 @@ import Command import qualified Remote import Annex.Content import Annex.Transfer -import Config.NumCopies +import Annex.NumCopies import Annex.Wanted import qualified Command.Move diff --git a/Command/GroupWanted.hs b/Command/GroupWanted.hs index 859a39c1b..5cdf785d7 100644 --- a/Command/GroupWanted.hs +++ b/Command/GroupWanted.hs @@ -8,13 +8,9 @@ module Command.GroupWanted where import Common.Annex -import qualified Annex import Command import Logs.PreferredContent -import Types.Messages -import Types.Group - -import qualified Data.Map as M +import Command.Wanted (performGet, performSet) cmd :: [Command] cmd = [command "groupwanted" (paramPair paramGroup (paramOptional paramExpression)) seek @@ -24,22 +20,8 @@ seek :: CommandSeek seek = withWords start start :: [String] -> CommandStart -start (g:[]) = next $ performGet g +start (g:[]) = next $ performGet groupPreferredContentMapRaw g start (g:expr:[]) = do showStart "groupwanted" g - next $ performSet g expr + next $ performSet groupPreferredContentSet expr g start _ = error "Specify a group." - -performGet :: Group -> CommandPerform -performGet g = do - Annex.setOutput QuietOutput - m <- groupPreferredContentMapRaw - liftIO $ putStrLn $ fromMaybe "" $ M.lookup g m - next $ return True - -performSet :: Group -> String -> CommandPerform -performSet g expr = case checkPreferredContentExpression expr of - Just e -> error $ "Parse error: " ++ e - Nothing -> do - groupPreferredContentSet g expr - next $ return True diff --git a/Command/Import.hs b/Command/Import.hs index 17cb49db1..fffa301ec 100644 --- a/Command/Import.hs +++ b/Command/Import.hs @@ -9,6 +9,7 @@ module Command.Import where import Common.Annex import Command +import qualified Git import qualified Annex import qualified Command.Add import Utility.CopyFile @@ -16,6 +17,10 @@ import Backend import Remote import Types.KeySource import Types.Key +import Annex.CheckIgnore +import Annex.NumCopies +import Types.TrustLevel +import Logs.Trust cmd :: [Command] cmd = [withOptions opts $ notBareRepo $ command "import" paramPaths seek @@ -58,6 +63,10 @@ getDuplicateMode = go . catMaybes <$> mapM getflag [minBound..maxBound] seek :: CommandSeek seek ps = do mode <- getDuplicateMode + repopath <- liftIO . absPath =<< fromRepo Git.repoPath + inrepops <- liftIO $ filter (dirContains repopath) <$> mapM absPath ps + unless (null inrepops) $ do + error $ "cannot import files from inside the working tree (use git annex add instead): " ++ unwords inrepops withPathContents (start mode) ps start :: DuplicateMode -> (FilePath, FilePath) -> CommandStart @@ -75,23 +84,41 @@ start mode (srcfile, destfile) = where deletedup k = do showNote $ "duplicate of " ++ key2file k - liftIO $ removeFile srcfile - next $ return True + ifM (verifiedExisting k destfile) + ( do + liftIO $ removeFile srcfile + next $ return True + , do + warning "Could not verify that the content is still present in the annex; not removing from the import location." + stop + ) importfile = do - handleexisting =<< liftIO (catchMaybeIO $ getSymbolicLinkStatus destfile) + ignored <- not <$> Annex.getState Annex.force <&&> checkIgnored destfile + if ignored + then do + warning $ "not importing " ++ destfile ++ " which is .gitignored (use --force to override)" + stop + else do + existing <- liftIO (catchMaybeIO $ getSymbolicLinkStatus destfile) + case existing of + Nothing -> importfilechecked + (Just s) + | isDirectory s -> notoverwriting "(is a directory)" + | otherwise -> ifM (Annex.getState Annex.force) + ( do + liftIO $ nukeFile destfile + importfilechecked + , notoverwriting "(use --force to override, or a duplication option such as --deduplicate to clean up)" + ) + importfilechecked = do liftIO $ createDirectoryIfMissing True (parentDir destfile) liftIO $ if mode == Duplicate || mode == SkipDuplicates then void $ copyFileExternal CopyAllMetaData srcfile destfile else moveFile srcfile destfile Command.Add.perform destfile - handleexisting Nothing = noop - handleexisting (Just s) - | isDirectory s = notoverwriting "(is a directory)" - | otherwise = ifM (Annex.getState Annex.force) - ( liftIO $ nukeFile destfile - , notoverwriting "(use --force to override, or a duplication option such as --deduplicate to clean up)" - ) - notoverwriting why = error $ "not overwriting existing " ++ destfile ++ " " ++ why + notoverwriting why = do + warning $ "not overwriting existing " ++ destfile ++ " " ++ why + stop checkdup dupa notdupa = do backend <- chooseBackend destfile let ks = KeySource srcfile srcfile Nothing @@ -107,3 +134,14 @@ start mode (srcfile, destfile) = CleanDuplicates -> checkdup (Just deletedup) Nothing SkipDuplicates -> checkdup Nothing (Just importfile) _ -> return (Just importfile) + +verifiedExisting :: Key -> FilePath -> Annex Bool +verifiedExisting key destfile = do + -- Look up the numcopies setting for the file that it would be + -- imported to, if it were imported. + need <- getFileNumCopies destfile + + (remotes, trusteduuids) <- knownCopies key + untrusteduuids <- trustGet UnTrusted + let tocheck = Remote.remotesWithoutUUID remotes (trusteduuids++untrusteduuids) + verifyEnoughCopies [] key need trusteduuids [] tocheck diff --git a/Command/ImportFeed.hs b/Command/ImportFeed.hs index 2a278dea1..6d3a1765b 100644 --- a/Command/ImportFeed.hs +++ b/Command/ImportFeed.hs @@ -16,7 +16,9 @@ import qualified Data.Set as S import qualified Data.Map as M import Data.Time.Clock import Data.Time.Format +#if ! MIN_VERSION_time(1,5,0) import System.Locale +#endif import Common.Annex import qualified Annex @@ -196,7 +198,7 @@ performDownload opts cache todownload = case location todownload of Just link -> do let videourl = Quvi.linkUrl link checkknown videourl $ - rundownload videourl ("." ++ Quvi.linkSuffix link) $ \f -> + rundownload videourl ("." ++ fromMaybe "m" (Quvi.linkSuffix link)) $ \f -> maybeToList <$> addUrlFileQuvi (relaxedOpt opts) quviurl videourl f #else return False diff --git a/Command/Info.hs b/Command/Info.hs index b7cb3232f..1c2dd2fb2 100644 --- a/Command/Info.hs +++ b/Command/Info.hs @@ -30,7 +30,7 @@ import Types.Key import Logs.UUID import Logs.Trust import Logs.Location -import Config.NumCopies +import Annex.NumCopies import Remote import Config import Utility.Percentage diff --git a/Command/Log.hs b/Command/Log.hs index 4bc7bb89a..671c9d674 100644 --- a/Command/Log.hs +++ b/Command/Log.hs @@ -5,15 +5,19 @@ - Licensed under the GNU GPL version 3 or higher. -} +{-# LANGUAGE CPP #-} + module Command.Log where import qualified Data.Set as S import qualified Data.Map as M import qualified Data.ByteString.Lazy.Char8 as L +import Data.Char import Data.Time.Clock.POSIX import Data.Time +#if ! MIN_VERSION_time(1,5,0) import System.Locale -import Data.Char +#endif import Common.Annex import Command @@ -172,7 +176,11 @@ parseRaw l = go $ words l parseTimeStamp :: String -> POSIXTime parseTimeStamp = utcTimeToPOSIXSeconds . fromMaybe (error "bad timestamp") . +#if MIN_VERSION_time(1,5,0) + parseTimeM True defaultTimeLocale "%s" +#else parseTime defaultTimeLocale "%s" +#endif showTimeStamp :: TimeZone -> POSIXTime -> String showTimeStamp zone = show . utcToLocalTime zone . posixSecondsToUTCTime diff --git a/Command/LookupKey.hs b/Command/LookupKey.hs index 0485232ae..6e7f07049 100644 --- a/Command/LookupKey.hs +++ b/Command/LookupKey.hs @@ -9,18 +9,20 @@ module Command.LookupKey where import Common.Annex import Command +import CmdLine.Batch import Annex.CatFile import Types.Key cmd :: [Command] -cmd = [notBareRepo $ noCommit $ noMessages $ +cmd = [withOptions [batchOption] $ notBareRepo $ noCommit $ noMessages $ command "lookupkey" (paramRepeating paramFile) seek SectionPlumbing "looks up key used for file"] seek :: CommandSeek -seek = withStrings start +seek = batchable withStrings start -start :: String -> CommandStart -start file = do - liftIO . maybe exitFailure (putStrLn . key2file) =<< catKeyFile file +start :: Batchable String +start batchmode file = do + maybe (batchBadInput batchmode) (liftIO . putStrLn . key2file) + =<< catKeyFile file stop diff --git a/Command/Mirror.hs b/Command/Mirror.hs index 6c3895be1..535dc64b6 100644 --- a/Command/Mirror.hs +++ b/Command/Mirror.hs @@ -14,7 +14,7 @@ import qualified Command.Drop import qualified Command.Get import qualified Remote import Annex.Content -import Config.NumCopies +import Annex.NumCopies cmd :: [Command] cmd = [withOptions mirrorOptions $ command "mirror" paramPaths seek diff --git a/Command/NumCopies.hs b/Command/NumCopies.hs index 6c69b2166..1e710f561 100644 --- a/Command/NumCopies.hs +++ b/Command/NumCopies.hs @@ -10,7 +10,7 @@ module Command.NumCopies where import Common.Annex import qualified Annex import Command -import Config.NumCopies +import Annex.NumCopies import Types.Messages cmd :: [Command] diff --git a/Command/Required.hs b/Command/Required.hs new file mode 100644 index 000000000..3d9c59279 --- /dev/null +++ b/Command/Required.hs @@ -0,0 +1,17 @@ +{- git-annex command + - + - Copyright 2015 Joey Hess <id@joeyh.name> + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Required where + +import Command +import Logs.PreferredContent +import qualified Command.Wanted + +cmd :: [Command] +cmd = Command.Wanted.cmd' "required" "get or set required content expression" + requiredContentMapRaw + requiredContentSet diff --git a/Command/Wanted.hs b/Command/Wanted.hs index 6b87e51d8..07f5ee7c3 100644 --- a/Command/Wanted.hs +++ b/Command/Wanted.hs @@ -1,6 +1,6 @@ {- git-annex command - - - Copyright 2013 Joey Hess <id@joeyh.name> + - Copyright 2013-2015 Joey Hess <id@joeyh.name> - - Licensed under the GNU GPL version 3 or higher. -} @@ -13,39 +13,47 @@ import Command import qualified Remote import Logs.PreferredContent import Types.Messages +import Types.StandardGroups import qualified Data.Map as M cmd :: [Command] -cmd = [command "wanted" (paramPair paramRemote (paramOptional paramExpression)) seek - SectionSetup "get or set preferred content expression"] - -seek :: CommandSeek -seek = withWords start - -start :: [String] -> CommandStart -start = parse +cmd = cmd' "wanted" "get or set preferred content expression" + preferredContentMapRaw + preferredContentSet + +cmd' + :: String + -> String + -> Annex (M.Map UUID PreferredContentExpression) + -> (UUID -> PreferredContentExpression -> Annex ()) + -> [Command] +cmd' name desc getter setter = [command name pdesc seek SectionSetup desc] where - parse (name:[]) = go name performGet - parse (name:expr:[]) = go name $ \uuid -> do - showStart "wanted" name - performSet expr uuid - parse _ = error "Specify a repository." - - go name a = do - u <- Remote.nameToUUID name + pdesc = paramPair paramRemote (paramOptional paramExpression) + + seek = withWords start + + start (rname:[]) = go rname (performGet getter) + start (rname:expr:[]) = go rname $ \uuid -> do + showStart name rname + performSet setter expr uuid + start _ = error "Specify a repository." + + go rname a = do + u <- Remote.nameToUUID rname next $ a u -performGet :: UUID -> CommandPerform -performGet uuid = do +performGet :: Ord a => Annex (M.Map a PreferredContentExpression) -> a -> CommandPerform +performGet getter a = do Annex.setOutput QuietOutput - m <- preferredContentMapRaw - liftIO $ putStrLn $ fromMaybe "" $ M.lookup uuid m + m <- getter + liftIO $ putStrLn $ fromMaybe "" $ M.lookup a m next $ return True -performSet :: String -> UUID -> CommandPerform -performSet expr uuid = case checkPreferredContentExpression expr of +performSet :: Ord a => (a -> PreferredContentExpression -> Annex ()) -> String -> a -> CommandPerform +performSet setter expr a = case checkPreferredContentExpression expr of Just e -> error $ "Parse error: " ++ e Nothing -> do - preferredContentSet uuid expr + setter a expr next $ return True diff --git a/Command/WebApp.hs b/Command/WebApp.hs index 46ba556a3..e872d4be0 100644 --- a/Command/WebApp.hs +++ b/Command/WebApp.hs @@ -143,10 +143,10 @@ firstRun :: Maybe HostName -> IO () firstRun listenhost = do checkEnvironmentIO {- Without a repository, we cannot have an Annex monad, so cannot - - get a ThreadState. Using undefined is only safe because the + - get a ThreadState. This is only safe because the - webapp checks its noAnnex field before accessing the - threadstate. -} - let st = undefined + let st = error "annex state not available" {- Get a DaemonStatus without running in the Annex monad. -} dstatus <- atomically . newTMVar =<< newDaemonStatus d <- newAssistantData st dstatus diff --git a/Config/Files.hs b/Config/Files.hs index d2b2f6a51..294eae22a 100644 --- a/Config/Files.hs +++ b/Config/Files.hs @@ -5,6 +5,8 @@ - Licensed under the GNU GPL version 3 or higher. -} +{-# OPTIONS_GHC -fno-warn-tabs #-} + module Config.Files where import Common @@ -34,8 +34,7 @@ module Crypto ( ) where import qualified Data.ByteString.Lazy as L -import Data.ByteString.Lazy.UTF8 (fromString) -import Control.Applicative +import Data.ByteString.UTF8 (fromString) import qualified Data.Map as M import Control.Monad.IO.Class @@ -93,7 +92,7 @@ genSharedCipher highQuality = {- Updates an existing Cipher, re-encrypting it to add or remove keyids, - depending on whether the first component is True or False. -} updateEncryptedCipher :: [(Bool, String)] -> StorableCipher -> IO StorableCipher -updateEncryptedCipher _ SharedCipher{} = undefined +updateEncryptedCipher _ SharedCipher{} = error "Cannot update shared cipher" updateEncryptedCipher [] encipher = return encipher updateEncryptedCipher newkeys encipher@(EncryptedCipher _ variant (KeyIds ks)) = do dropKeys <- listKeyIds [ k | (False, k) <- newkeys ] diff --git a/Database/Fsck.hs b/Database/Fsck.hs index bf0c72f6d..50c08cf61 100644 --- a/Database/Fsck.hs +++ b/Database/Fsck.hs @@ -25,6 +25,7 @@ import qualified Database.Handle as H import Locations import Utility.PosixFiles import Utility.Exception +import Common import Annex import Types.Key import Types.UUID @@ -33,13 +34,6 @@ import Annex.LockFile import Database.Persist.TH import Database.Esqueleto hiding (Key) -import Control.Monad -import Control.Monad.IfElse -import Control.Monad.IO.Class (liftIO) -import System.Directory -import System.FilePath -import Data.Maybe -import Control.Applicative data FsckHandle = FsckHandle H.DbHandle UUID @@ -55,7 +49,7 @@ Fscked - - This may fail, if other fsck processes are currently running using the - database. Removing the database in that situation would lead to crashes - - or undefined behavior. + - or unknown behavior. -} newPass :: UUID -> Annex Bool newPass u = isJust <$> tryExclusiveLock (gitAnnexFsckDbLock u) go @@ -60,7 +60,7 @@ repoLocation Repo { location = Url url } = show url repoLocation Repo { location = Local { worktree = Just dir } } = dir repoLocation Repo { location = Local { gitdir = dir } } = dir repoLocation Repo { location = LocalUnknown dir } = dir -repoLocation Repo { location = Unknown } = undefined +repoLocation Repo { location = Unknown } = error "unknown repoLocation" {- Path to a repository. For non-bare, this is the worktree, for bare, - it's the gitdir, and for URL repositories, is the path on the remote @@ -70,12 +70,12 @@ repoPath Repo { location = Url u } = unEscapeString $ uriPath u repoPath Repo { location = Local { worktree = Just d } } = d repoPath Repo { location = Local { gitdir = d } } = d repoPath Repo { location = LocalUnknown dir } = dir -repoPath Repo { location = Unknown } = undefined +repoPath Repo { location = Unknown } = error "unknown repoPath" {- Path to a local repository's .git directory. -} localGitDir :: Repo -> FilePath localGitDir Repo { location = Local { gitdir = d } } = d -localGitDir _ = undefined +localGitDir _ = error "unknown localGitDir" {- Some code needs to vary between URL and normal repos, - or bare and non-bare, these functions help with that. -} diff --git a/Git/CatFile.hs b/Git/CatFile.hs index a1a0a0d28..c63a0647c 100644 --- a/Git/CatFile.hs +++ b/Git/CatFile.hs @@ -110,4 +110,4 @@ catTree h treeref = go <$> catObjectDetails h treeref parsemodefile b = let (modestr, file) = separate (== ' ') (decodeBS b) in (file, readmode modestr) - readmode = fst . fromMaybe (0, undefined) . headMaybe . readOct + readmode = fromMaybe 0 . fmap fst . headMaybe . readOct diff --git a/Git/LsFiles.hs b/Git/LsFiles.hs index c23c282d0..e80c1b288 100644 --- a/Git/LsFiles.hs +++ b/Git/LsFiles.hs @@ -181,12 +181,13 @@ parseUnmerged s | otherwise = case words metadata of (rawblobtype:rawsha:rawstage:_) -> do stage <- readish rawstage :: Maybe Int - unless (stage == 2 || stage == 3) $ - fail undefined -- skip stage 1 - blobtype <- readBlobType rawblobtype - sha <- extractSha rawsha - return $ InternalUnmerged (stage == 2) file - (Just blobtype) (Just sha) + if stage /= 2 && stage /= 3 + then Nothing + else do + blobtype <- readBlobType rawblobtype + sha <- extractSha rawsha + return $ InternalUnmerged (stage == 2) file + (Just blobtype) (Just sha) _ -> Nothing where (metadata, file) = separate (== '\t') s diff --git a/Git/LsTree.hs b/Git/LsTree.hs index 7ef951807..8294f7b93 100644 --- a/Git/LsTree.hs +++ b/Git/LsTree.hs @@ -13,10 +13,6 @@ module Git.LsTree ( parseLsTree ) where -import Numeric -import Control.Applicative -import System.Posix.Types - import Common import Git import Git.Command @@ -24,6 +20,9 @@ import Git.Sha import Git.FilePath import qualified Git.Filename +import Numeric +import System.Posix.Types + data TreeItem = TreeItem { mode :: FileMode , typeobj :: String diff --git a/Git/Remote/Remove.hs b/Git/Remote/Remove.hs index 2ffc9b358..e6c13b9ea 100644 --- a/Git/Remote/Remove.hs +++ b/Git/Remote/Remove.hs @@ -1,4 +1,4 @@ -{- git remote stuff +{- git remote removal - - Copyright 2012 Joey Hess <id@joeyh.name> - diff --git a/Git/Version.hs b/Git/Version.hs index ecd12444c..19ff945c8 100644 --- a/Git/Version.hs +++ b/Git/Version.hs @@ -5,6 +5,8 @@ - Licensed under the GNU GPL version 3 or higher. -} +{-# OPTIONS_GHC -fno-warn-tabs #-} + module Git.Version ( installed, older, @@ -15,7 +15,7 @@ import qualified Backend import Annex.Content import Annex.UUID import Logs.Trust -import Config.NumCopies +import Annex.NumCopies import Types.TrustLevel import Types.Key import Types.Group diff --git a/Logs/Difference.hs b/Logs/Difference.hs index fcebffee9..8d37a09c4 100644 --- a/Logs/Difference.hs +++ b/Logs/Difference.hs @@ -12,7 +12,6 @@ module Logs.Difference ( module Logs.Difference.Pure ) where -import Data.Monoid import Data.Time.Clock.POSIX import qualified Data.Map as M diff --git a/Logs/Difference/Pure.hs b/Logs/Difference/Pure.hs index 25f3844d6..0e68ffd3c 100644 --- a/Logs/Difference/Pure.hs +++ b/Logs/Difference/Pure.hs @@ -10,7 +10,6 @@ module Logs.Difference.Pure ( parseDifferencesLog, ) where -import Data.Monoid import qualified Data.Map as M import Common.Annex diff --git a/Logs/MapLog.hs b/Logs/MapLog.hs index 3c7eef26b..d5bb67f68 100644 --- a/Logs/MapLog.hs +++ b/Logs/MapLog.hs @@ -1,3 +1,5 @@ +{-# LANGUAGE CPP #-} + {- git-annex Map log - - This is used to store a Map, in a way that can be union merged. @@ -13,10 +15,9 @@ module Logs.MapLog where import qualified Data.Map as M import Data.Time.Clock.POSIX -import Data.Time -import System.Locale import Common +import Logs.TimeStamp data TimeStamp = Unknown | Date POSIXTime deriving (Eq, Ord, Show) @@ -42,7 +43,7 @@ parseMapLog fieldparser valueparser = M.fromListWith best . mapMaybe parse . lin parse line = do let (ts, rest) = splitword line (sf, sv) = splitword rest - date <- Date . utcTimeToPOSIXSeconds <$> parseTime defaultTimeLocale "%s%Qs" ts + date <- Date <$> parsePOSIXTime ts f <- fieldparser sf v <- valueparser sv Just (f, LogEntry date v) diff --git a/Logs/MetaData.hs b/Logs/MetaData.hs index ed4e2363e..0dc76214c 100644 --- a/Logs/MetaData.hs +++ b/Logs/MetaData.hs @@ -41,12 +41,11 @@ import qualified Annex.Branch import qualified Annex import Logs import Logs.SingleValue +import Logs.TimeStamp import qualified Data.Set as S import qualified Data.Map as M import Data.Time.Clock.POSIX -import Data.Time.Format -import System.Locale instance SingleValueSerializable MetaData where serialize = Types.MetaData.serialize @@ -86,7 +85,7 @@ getCurrentMetaData k = do ts = lastchangedval l in M.map (const ts) m lastchangedval l = S.singleton $ toMetaValue $ showts $ changed l - showts = formatTime defaultTimeLocale "%F@%H-%M-%S" . posixSecondsToUTCTime + showts = formatPOSIXTime "%F@%H-%M-%S" {- Adds in some metadata, which can override existing values, or unset - them, but otherwise leaves any existing metadata as-is. -} diff --git a/Logs/Presence/Pure.hs b/Logs/Presence/Pure.hs index 88f18435f..ec4415aad 100644 --- a/Logs/Presence/Pure.hs +++ b/Logs/Presence/Pure.hs @@ -8,11 +8,10 @@ module Logs.Presence.Pure where import Data.Time.Clock.POSIX -import Data.Time -import System.Locale import qualified Data.Map as M import Common.Annex +import Logs.TimeStamp import Utility.QuickCheck data LogLine = LogLine { @@ -29,7 +28,7 @@ parseLog :: String -> [LogLine] parseLog = mapMaybe parseline . lines where parseline l = LogLine - <$> (utcTimeToPOSIXSeconds <$> parseTime defaultTimeLocale "%s%Qs" d) + <$> parsePOSIXTime d <*> parseStatus s <*> pure rest where diff --git a/Logs/SingleValue.hs b/Logs/SingleValue.hs index dbbe996f3..213bcd217 100644 --- a/Logs/SingleValue.hs +++ b/Logs/SingleValue.hs @@ -15,11 +15,10 @@ module Logs.SingleValue where import Common.Annex import qualified Annex.Branch +import Logs.TimeStamp import qualified Data.Set as S import Data.Time.Clock.POSIX -import Data.Time -import System.Locale class SingleValueSerializable v where serialize :: v -> String @@ -42,7 +41,7 @@ parseLog = S.fromList . mapMaybe parse . lines where parse line = do let (ts, s) = splitword line - date <- utcTimeToPOSIXSeconds <$> parseTime defaultTimeLocale "%s%Qs" ts + date <- parsePOSIXTime ts v <- deserialize s Just (LogEntry date v) splitword = separate (== ' ') diff --git a/Logs/TimeStamp.hs b/Logs/TimeStamp.hs new file mode 100644 index 000000000..0891f920b --- /dev/null +++ b/Logs/TimeStamp.hs @@ -0,0 +1,30 @@ +{- log timestamp parsing + - + - Copyright 2015 Joey Hess <id@joeyh.name> + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP #-} + +module Logs.TimeStamp where + +import Data.Time.Clock.POSIX +import Data.Time +#if ! MIN_VERSION_time(1,5,0) +import System.Locale +#endif +import Control.Applicative +import Prelude + +{- Parses how POSIXTime shows itself: "1431286201.113452s" + - Also handles the format with no fractional seconds. -} +parsePOSIXTime :: String -> Maybe POSIXTime +#if MIN_VERSION_time(1,5,0) +parsePOSIXTime s = utcTimeToPOSIXSeconds <$> parseTimeM True defaultTimeLocale "%s%Qs" s +#else +parsePOSIXTime s = utcTimeToPOSIXSeconds <$> parseTime defaultTimeLocale "%s%Qs" s +#endif + +formatPOSIXTime :: String -> POSIXTime -> String +formatPOSIXTime fmt t = formatTime defaultTimeLocale fmt (posixSecondsToUTCTime t) diff --git a/Logs/Transfer.hs b/Logs/Transfer.hs index 5b5b8f8cb..078157208 100644 --- a/Logs/Transfer.hs +++ b/Logs/Transfer.hs @@ -18,11 +18,10 @@ import Utility.Percentage import Utility.QuickCheck import Utility.PID import Utility.LockFile +import Logs.TimeStamp import Data.Time.Clock import Data.Time.Clock.POSIX -import Data.Time -import System.Locale import Control.Concurrent {- Enough information to uniquely identify a transfer, used as the filename @@ -276,10 +275,6 @@ readTransferInfo mpid s = TransferInfo then Just <$> readish =<< headMaybe (drop 1 bits) else pure Nothing -- not failure -parsePOSIXTime :: String -> Maybe POSIXTime -parsePOSIXTime s = utcTimeToPOSIXSeconds - <$> parseTime defaultTimeLocale "%s%Qs" s - {- The directory holding transfer information files for a given Direction. -} transferDir :: Direction -> Git.Repo -> FilePath transferDir direction r = gitAnnexTransferDir r </> showLcDirection direction diff --git a/Logs/Transitions.hs b/Logs/Transitions.hs index d782455a6..5ccfd7e8c 100644 --- a/Logs/Transitions.hs +++ b/Logs/Transitions.hs @@ -15,11 +15,10 @@ module Logs.Transitions where import Data.Time.Clock.POSIX -import Data.Time -import System.Locale import qualified Data.Set as S import Common.Annex +import Logs.TimeStamp transitionsLog :: FilePath transitionsLog = "transitions.log" @@ -66,12 +65,13 @@ showTransitionLine :: TransitionLine -> String showTransitionLine (TransitionLine ts t) = unwords [show t, show ts] parseTransitionLine :: String -> Maybe TransitionLine -parseTransitionLine s = TransitionLine <$> pdate ds <*> readish ts +parseTransitionLine s = TransitionLine + <$> parsePOSIXTime ds + <*> readish ts where ws = words s ts = Prelude.head ws ds = unwords $ Prelude.tail ws - pdate = utcTimeToPOSIXSeconds <$$> parseTime defaultTimeLocale "%s%Qs" combineTransitions :: [Transitions] -> Transitions combineTransitions = S.unions diff --git a/Logs/UUIDBased.hs b/Logs/UUIDBased.hs index ac876e65f..5613c6fb4 100644 --- a/Logs/UUIDBased.hs +++ b/Logs/UUIDBased.hs @@ -30,12 +30,11 @@ module Logs.UUIDBased ( import qualified Data.Map as M import Data.Time.Clock.POSIX -import Data.Time -import System.Locale import Common import Types.UUID import Logs.MapLog +import Logs.TimeStamp type Log v = MapLog UUID v @@ -73,9 +72,9 @@ parseLogWithUUID parser = M.fromListWith best . mapMaybe parse . lines info | ts == Unknown = drop 1 ws | otherwise = drop 1 $ beginning ws - pdate s = case parseTime defaultTimeLocale "%s%Qs" s of + pdate s = case parsePOSIXTime s of Nothing -> Unknown - Just d -> Date $ utcTimeToPOSIXSeconds d + Just d -> Date d showLogNew :: (v -> String) -> Log v -> String showLogNew = showMapLog fromUUID diff --git a/Logs/Unused.hs b/Logs/Unused.hs index 883ab8c5b..7a12f186c 100644 --- a/Logs/Unused.hs +++ b/Logs/Unused.hs @@ -32,12 +32,12 @@ import qualified Data.Map as M import qualified Data.Set as S import Data.Time.Clock.POSIX import Data.Time -import System.Locale import Common.Annex import qualified Annex import Types.Key import Utility.Tmp +import Logs.TimeStamp -- everything that is stored in the unused log type UnusedLog = M.Map Key (Int, Maybe POSIXTime) @@ -81,7 +81,7 @@ readUnusedLog prefix = do , return M.empty ) where - parse line = case (readish sint, file2key skey, utcTimeToPOSIXSeconds <$> parseTime defaultTimeLocale "%s%Qs" ts) of + parse line = case (readish sint, file2key skey, parsePOSIXTime ts) of (Just int, Just key, mtimestamp) -> Just (key, (int, mtimestamp)) _ -> Nothing where @@ -140,6 +140,25 @@ linuxstandalone-nobuild: Build/Standalone Build/LinuxMkLibs cd tmp/git-annex.linux && find . -type l >> git-annex.MANIFEST cd tmp && tar czf git-annex-standalone-$(shell dpkg --print-architecture).tar.gz git-annex.linux +# Run this target to build git-annex-standalone*.deb +debianstandalone: dpkg-buildpackage-F +# Run this target to build git-annex-standalone*.dsc +debianstandalone-dsc: dpkg-buildpackage-S + +prep-standalone: + $(MAKE) undo-standalone + QUILT_PATCHES=debian/patches QUILT_SERIES=series.standalone-build quilt push -a + debian/create-standalone-changelog + +undo-standalone: + test -e .git + git checkout debian/changelog + quilt pop -a || true + +dpkg-buildpackage%: prep-standalone + umask 022; dpkg-buildpackage -rfakeroot $* + $(MAKE) undo-standalone + OSXAPP_DEST=tmp/build-dmg/git-annex.app OSXAPP_BASE=$(OSXAPP_DEST)/Contents/MacOS/bundle osxapp: Build/Standalone Build/OSXMkLibs diff --git a/Messages.hs b/Messages.hs index d82eb731b..f6d26db9d 100644 --- a/Messages.hs +++ b/Messages.hs @@ -20,7 +20,7 @@ module Messages ( showEndFail, showEndResult, endResult, - showErr, + toplevelWarning, warning, warningIO, indent, @@ -118,15 +118,16 @@ endResult :: Bool -> String endResult True = "ok" endResult False = "failed" -showErr :: (Show a) => a -> Annex () -showErr e = warning' $ "git-annex: " ++ show e +toplevelWarning :: Bool -> String -> Annex () +toplevelWarning makeway s = warning' makeway ("git-annex: " ++ s) warning :: String -> Annex () -warning = warning' . indent +warning = warning' True . indent -warning' :: String -> Annex () -warning' w = do - handleMessage q $ putStr "\n" +warning' :: Bool -> String -> Annex () +warning' makeway w = do + when makeway $ + handleMessage q $ putStr "\n" liftIO $ do hFlush stdout hPutStrLn stderr w @@ -282,7 +282,9 @@ showLocations separateuntrusted key exclude nolocmsg = do let uuidsskipped = filteruuids uuids (u:exclude++uuidswanted) ppuuidswanted <- prettyPrintUUIDs "wanted" uuidswanted ppuuidsskipped <- prettyPrintUUIDs "skipped" uuidsskipped - showLongNote $ message ppuuidswanted ppuuidsskipped + let msg = message ppuuidswanted ppuuidsskipped + unless (null msg) $ + showLongNote msg ignored <- filter (remoteAnnexIgnore . gitconfig) <$> remoteList unless (null ignored) $ showLongNote $ "(Note that these git remotes have annex-ignore set: " ++ unwords (map name ignored) ++ ")" diff --git a/Remote/Directory.hs b/Remote/Directory.hs index f210f557d..8b727c77e 100644 --- a/Remote/Directory.hs +++ b/Remote/Directory.hs @@ -162,9 +162,13 @@ retrieveCheap _ (UnpaddedChunks _) _ _ _ = return False retrieveCheap _ (LegacyChunks _) _ _ _ = return False #ifndef mingw32_HOST_OS retrieveCheap d NoChunks k _af f = liftIO $ catchBoolIO $ do - file <- getLocation d k - createSymbolicLink file f - return True + file <- absPath =<< getLocation d k + ifM (doesFileExist file) + ( do + createSymbolicLink file f + return True + , return False + ) #else retrieveCheap _ _ _ _ _ = return False #endif diff --git a/Remote/GCrypt.hs b/Remote/GCrypt.hs index 7685418b0..fc0c27f37 100644 --- a/Remote/GCrypt.hs +++ b/Remote/GCrypt.hs @@ -397,7 +397,7 @@ getGCryptId fast r gc | Git.repoIsLocal r || Git.repoIsLocalUnknown r = extract <$> liftIO (catchMaybeIO $ Git.Config.read r) | not fast = extract . liftM fst <$> getM (eitherToMaybe <$>) - [ Ssh.onRemote r (Git.Config.fromPipe r, return (Left undefined)) "configlist" [] [] + [ Ssh.onRemote r (Git.Config.fromPipe r, return (Left $ error "configlist failed")) "configlist" [] [] , getConfigViaRsync r gc ] | otherwise = return (Nothing, r) diff --git a/Remote/Git.hs b/Remote/Git.hs index 2807c62fb..170c6fbf6 100644 --- a/Remote/Git.hs +++ b/Remote/Git.hs @@ -200,7 +200,7 @@ tryGitConfigRead :: Git.Repo -> Annex Git.Repo tryGitConfigRead r | haveconfig r = return r -- already read | Git.repoIsSsh r = store $ do - v <- Ssh.onRemote r (pipedconfig, return (Left undefined)) "configlist" [] [] + v <- Ssh.onRemote r (pipedconfig, return (Left $ error "configlist failed")) "configlist" [] [] case v of Right r' | haveconfig r' -> return r' @@ -229,9 +229,10 @@ tryGitConfigRead r uo <- Url.getUrlOptions v <- liftIO $ withTmpFile "git-annex.tmp" $ \tmpfile h -> do hClose h - ifM (Url.downloadQuiet (Git.repoLocation r ++ "/config") tmpfile uo) + let url = Git.repoLocation r ++ "/config" + ifM (Url.downloadQuiet url tmpfile uo) ( pipedconfig "git" [Param "config", Param "--null", Param "--list", Param "--file", File tmpfile] - , return $ Left undefined + , return $ Left $ error $ "unable to load config from " ++ url ) case v of Left _ -> do @@ -450,10 +451,17 @@ copyFromRemote' r key file dest meterupdate copyFromRemoteCheap :: Remote -> Key -> AssociatedFile -> FilePath -> Annex Bool #ifndef mingw32_HOST_OS copyFromRemoteCheap r key af file - | not $ Git.repoIsUrl (repo r) = guardUsable (repo r) (return False) $ do - loc <- liftIO $ gitAnnexLocation key (repo r) $ + | not $ Git.repoIsUrl (repo r) = guardUsable (repo r) (return False) $ liftIO $ do + loc <- gitAnnexLocation key (repo r) $ fromJust $ remoteGitConfig $ gitconfig r - liftIO $ catchBoolIO $ createSymbolicLink loc file >> return True + ifM (doesFileExist loc) + ( do + absloc <- absPath loc + catchBoolIO $ do + createSymbolicLink absloc file + return True + , return False + ) | Git.repoIsSsh (repo r) = ifM (Annex.Content.preseedTmp key file) ( parallelMetered Nothing key af $ diff --git a/Remote/Helper/Chunked.hs b/Remote/Helper/Chunked.hs index 2f21ba66c..23ed3dbf8 100644 --- a/Remote/Helper/Chunked.hs +++ b/Remote/Helper/Chunked.hs @@ -72,7 +72,7 @@ chunkKeyStream basek chunksize = ChunkKeyStream $ map mk [1..] nextChunkKeyStream :: ChunkKeyStream -> (Key, ChunkKeyStream) nextChunkKeyStream (ChunkKeyStream (k:l)) = (k, ChunkKeyStream l) -nextChunkKeyStream (ChunkKeyStream []) = undefined -- stream is infinite! +nextChunkKeyStream (ChunkKeyStream []) = error "expected infinite ChunkKeyStream" takeChunkKeyStream :: ChunkCount -> ChunkKeyStream -> [Key] takeChunkKeyStream n (ChunkKeyStream l) = genericTake n l diff --git a/Remote/Helper/Encryptable.hs b/Remote/Helper/Encryptable.hs index 2c1935ba9..3395db978 100644 --- a/Remote/Helper/Encryptable.hs +++ b/Remote/Helper/Encryptable.hs @@ -20,7 +20,8 @@ module Remote.Helper.Encryptable ( ) where import qualified Data.Map as M -import qualified "dataenc" Codec.Binary.Base64 as B64 +import qualified "sandi" Codec.Binary.Base64 as B64 +import qualified Data.ByteString as B import Data.Bits.Utils import Common.Annex @@ -172,12 +173,12 @@ describeEncryption c = case extractCipher c of ] {- Not using Utility.Base64 because these "Strings" are really - - bags of bytes and that would convert to unicode and not roung-trip + - bags of bytes and that would convert to unicode and not round-trip - cleanly. -} toB64bs :: String -> String -toB64bs = B64.encode . s2w8 +toB64bs = w82s . B.unpack . B64.encode . B.pack . s2w8 fromB64bs :: String -> String -fromB64bs s = fromMaybe bad $ w82s <$> B64.decode s +fromB64bs s = either (const bad) (w82s . B.unpack) (B64.decode $ B.pack $ s2w8 s) where bad = error "bad base64 encoded data" diff --git a/Remote/Helper/Special.hs b/Remote/Helper/Special.hs index c11584bb8..483ef576e 100644 --- a/Remote/Helper/Special.hs +++ b/Remote/Helper/Special.hs @@ -199,7 +199,7 @@ specialRemote' cfg c preparestorer prepareretriever prepareremover preparecheckp readBytes $ \encb -> storer (enck k) (ByteContent encb) p - -- call retrieve-r to get chunks; decrypt them; stream to dest file + -- call retriever to get chunks; decrypt them; stream to dest file retrieveKeyFileGen k f dest p enc = safely $ prepareretriever k $ safely . go where diff --git a/Remote/S3.hs b/Remote/S3.hs index 83d35035e..21ab45674 100644 --- a/Remote/S3.hs +++ b/Remote/S3.hs @@ -28,6 +28,8 @@ import Control.Monad.Trans.Resource import Control.Monad.Catch import Data.Conduit import Data.IORef +import Data.Bits.Utils +import System.Log.Logger import Common.Annex import Types.Remote @@ -88,13 +90,7 @@ gen r u c gc = do , availability = GloballyAvailable , remotetype = remote , mkUnavailable = gen r u (M.insert "host" "!dne!" c) gc - , getInfo = includeCredsInfo c (AWS.creds u) $ catMaybes - [ Just ("bucket", fromMaybe "unknown" (getBucketName c)) - , if configIA c - then Just ("internet archive item", iaItemUrl $ fromMaybe "unknown" $ getBucketName c) - else Nothing - , Just ("partsize", maybe "unlimited" (roughSize storageUnits False) (getPartSize c)) - ] + , getInfo = includeCredsInfo c (AWS.creds u) (s3Info c) , claimUrl = Nothing , checkUrl = Nothing } @@ -102,9 +98,9 @@ gen r u c gc = do s3Setup :: Maybe UUID -> Maybe CredPair -> RemoteConfig -> Annex (RemoteConfig, UUID) s3Setup mu mcreds c = do u <- maybe (liftIO genUUID) return mu - s3Setup' u mcreds c -s3Setup' :: UUID -> Maybe CredPair -> RemoteConfig -> Annex (RemoteConfig, UUID) -s3Setup' u mcreds c = if configIA c then archiveorg else defaulthost + s3Setup' (isNothing mu) u mcreds c +s3Setup' :: Bool -> UUID -> Maybe CredPair -> RemoteConfig -> Annex (RemoteConfig, UUID) +s3Setup' new u mcreds c = if configIA c then archiveorg else defaulthost where remotename = fromJust (M.lookup "name" c) defbucket = remotename ++ "-" ++ fromUUID u @@ -124,7 +120,8 @@ s3Setup' u mcreds c = if configIA c then archiveorg else defaulthost (c', encsetup) <- encryptionSetup c c'' <- setRemoteCredPair encsetup c' (AWS.creds u) mcreds let fullconfig = c'' `M.union` defaults - genBucket fullconfig u + when new $ + genBucket fullconfig u use fullconfig archiveorg = do @@ -132,7 +129,7 @@ s3Setup' u mcreds c = if configIA c then archiveorg else defaulthost c' <- setRemoteCredPair noEncryptionUsed c (AWS.creds u) mcreds -- Ensure user enters a valid bucket name, since -- this determines the name of the archive.org item. - let validbucket = replace " " "-" $ map toLower $ + let validbucket = replace " " "-" $ fromMaybe (error "specify bucket=") $ getBucketName c' let archiveconfig = @@ -149,7 +146,7 @@ s3Setup' u mcreds c = if configIA c then archiveorg else defaulthost writeUUIDFile archiveconfig u use archiveconfig --- Sets up a http connection manager for S3 encdpoint, which allows +-- Sets up a http connection manager for S3 endpoint, which allows -- http connections to be reused across calls to the helper. prepareS3 :: Remote -> S3Info -> (S3Handle -> helper) -> Preparer helper prepareS3 r info = resourcePrepare $ const $ @@ -388,13 +385,13 @@ sendS3Handle' => S3Handle -> r -> ResourceT IO a -sendS3Handle' h = AWS.pureAws (hawscfg h) (hs3cfg h) (hmanager h) +sendS3Handle' h r = AWS.pureAws (hawscfg h) (hs3cfg h) (hmanager h) r withS3Handle :: RemoteConfig -> UUID -> S3Info -> (S3Handle -> Annex a) -> Annex a withS3Handle c u info a = do creds <- getRemoteCredPairFor "S3" c (AWS.creds u) awscreds <- liftIO $ genCredentials $ fromMaybe nocreds creds - let awscfg = AWS.Configuration AWS.Timestamp awscreds (AWS.defaultLog AWS.Error) + let awscfg = AWS.Configuration AWS.Timestamp awscreds debugMapper bracketIO (newManager httpcfg) closeManager $ \mgr -> a $ S3Handle mgr awscfg s3cfg info where @@ -450,7 +447,7 @@ extractS3Info c = do } getBucketName :: RemoteConfig -> Maybe BucketName -getBucketName = M.lookup "bucket" +getBucketName = map toLower <$$> M.lookup "bucket" getStorageClass :: RemoteConfig -> S3.StorageClass getStorageClass c = case M.lookup "storageclass" c of @@ -486,7 +483,7 @@ iaMunge = (>>= munge) where munge c | isAsciiUpper c || isAsciiLower c || isNumber c = [c] - | c `elem` "_-.\"" = [c] + | c `elem` ("_-.\"" :: String) = [c] | isSpace c = [] | otherwise = "&" ++ show (ord c) ++ ";" @@ -518,3 +515,26 @@ genCredentials (keyid, secret) = AWS.Credentials mkLocationConstraint :: AWS.Region -> S3.LocationConstraint mkLocationConstraint "US" = S3.locationUsClassic mkLocationConstraint r = r + +debugMapper :: AWS.Logger +debugMapper level t = forward "S3" (T.unpack t) + where + forward = case level of + AWS.Debug -> debugM + AWS.Info -> infoM + AWS.Warning -> warningM + AWS.Error -> errorM + +s3Info :: RemoteConfig -> [(String, String)] +s3Info c = catMaybes + [ Just ("bucket", fromMaybe "unknown" (getBucketName c)) + , Just ("endpoint", w82s (S.unpack (S3.s3Endpoint s3c))) + , Just ("port", show (S3.s3Port s3c)) + , Just ("storage class", show (getStorageClass c)) + , if configIA c + then Just ("internet archive item", iaItemUrl $ fromMaybe "unknown" $ getBucketName c) + else Nothing + , Just ("partsize", maybe "unlimited" (roughSize storageUnits False) (getPartSize c)) + ] + where + s3c = s3Configuration c diff --git a/Remote/Tahoe.hs b/Remote/Tahoe.hs index ca822d4fd..4a5216194 100644 --- a/Remote/Tahoe.hs +++ b/Remote/Tahoe.hs @@ -177,7 +177,7 @@ getSharedConvergenceSecret configdir = go (60 :: Int) v <- catchMaybeIO (readFile f) case v of Just s | "\n" `isSuffixOf` s || "\r" `isSuffixOf` s -> - return $ takeWhile (`notElem` "\n\r") s + return $ takeWhile (`notElem` ("\n\r" :: String)) s _ -> do threadDelaySeconds (Seconds 1) go (n - 1) @@ -1,4 +1,5 @@ {-# LANGUAGE NamedFieldPuns #-} +{-# OPTIONS_GHC -fno-warn-tabs #-} {- cabal setup file -} @@ -14,7 +14,6 @@ import Test.Tasty.Runners import Test.Tasty.HUnit import Test.Tasty.QuickCheck import Test.Tasty.Ingredients.Rerun -import Data.Monoid import Options.Applicative hiding (command) import qualified Data.Map as M @@ -156,6 +155,7 @@ properties = localOption (QuickCheckTests 1000) $ testGroup "QuickCheck" , testProperty "prop_read_show_TrustLevel" Types.TrustLevel.prop_read_show_TrustLevel , testProperty "prop_parse_show_TrustLog" Logs.Trust.prop_parse_show_TrustLog , testProperty "prop_hashes_stable" Utility.Hash.prop_hashes_stable + , testProperty "prop_mac_stable" Utility.Hash.prop_mac_stable , testProperty "prop_schedule_roundtrips" Utility.Scheduled.prop_schedule_roundtrips , testProperty "prop_past_sane" Utility.Scheduled.prop_past_sane , testProperty "prop_duration_roundtrips" Utility.HumanTime.prop_duration_roundtrips @@ -199,6 +199,7 @@ unitTests note = testGroup ("Unit Tests " ++ note) , testCase "fsck (bare)" test_fsck_bare , testCase "fsck (local untrusted)" test_fsck_localuntrusted , testCase "fsck (remote untrusted)" test_fsck_remoteuntrusted + , testCase "fsck --from remote" test_fsck_fromremote , testCase "migrate" test_migrate , testCase "migrate (via gitattributes)" test_migrate_via_gitattributes , testCase "unused" test_unused @@ -613,6 +614,10 @@ test_fsck_remoteuntrusted = intmpclonerepo $ do git_annex "untrust" ["origin"] @? "untrust of origin failed" fsck_should_fail "content not replicated to enough non-untrusted repositories" +test_fsck_fromremote :: Assertion +test_fsck_fromremote = intmpclonerepo $ do + git_annex "fsck" ["--from", "origin"] @? "fsck --from origin failed" + fsck_should_fail :: String -> Assertion fsck_should_fail m = not <$> git_annex "fsck" [] @? "fsck failed to fail with " ++ m diff --git a/Types/Crypto.hs b/Types/Crypto.hs index 682629d6a..79970c288 100644 --- a/Types/Crypto.hs +++ b/Types/Crypto.hs @@ -1,6 +1,6 @@ {- git-annex crypto types - - - Copyright 2011-2012 Joey Hess <id@joeyh.name> + - Copyright 2011-2015 Joey Hess <id@joeyh.name> - - Licensed under the GNU GPL version 3 or higher. -} @@ -17,9 +17,7 @@ module Types.Crypto ( calcMac, ) where -import qualified Data.ByteString.Lazy as L -import Data.Digest.Pure.SHA - +import Utility.Hash import Utility.Gpg (KeyIds(..)) -- XXX ideally, this would be a locked memory region @@ -31,13 +29,6 @@ data StorableCipher = EncryptedCipher String EncryptedCipherVariant KeyIds data EncryptedCipherVariant = Hybrid | PubKey deriving (Ord, Eq) -{- File names are (client-side) MAC'ed on special remotes. - - The chosen MAC algorithm needs to be same for all files stored on the - - remote. - -} -data Mac = HmacSha1 | HmacSha224 | HmacSha256 | HmacSha384 | HmacSha512 - deriving (Eq) - defaultMac :: Mac defaultMac = HmacSha1 @@ -57,17 +48,3 @@ readMac "HMACSHA256" = Just HmacSha256 readMac "HMACSHA384" = Just HmacSha384 readMac "HMACSHA512" = Just HmacSha512 readMac _ = Nothing - -calcMac - :: Mac -- ^ MAC - -> L.ByteString -- ^ secret key - -> L.ByteString -- ^ message - -> String -- ^ MAC'ed message, in hexadecimal -calcMac mac = case mac of - HmacSha1 -> showDigest $* hmacSha1 - HmacSha224 -> showDigest $* hmacSha224 - HmacSha256 -> showDigest $* hmacSha256 - HmacSha384 -> showDigest $* hmacSha384 - HmacSha512 -> showDigest $* hmacSha512 - where - ($*) g f x y = g $ f x y diff --git a/Types/DesktopNotify.hs b/Types/DesktopNotify.hs index 9ea51401f..e6df05ab1 100644 --- a/Types/DesktopNotify.hs +++ b/Types/DesktopNotify.hs @@ -8,6 +8,7 @@ module Types.DesktopNotify where import Data.Monoid +import Prelude data DesktopNotify = DesktopNotify { notifyStart :: Bool diff --git a/Types/Difference.hs b/Types/Difference.hs index 064703bf7..1bab3fe36 100644 --- a/Types/Difference.hs +++ b/Types/Difference.hs @@ -20,9 +20,10 @@ import Utility.PartialPrelude import qualified Git import qualified Git.Config +import qualified Data.Set as S import Data.Maybe import Data.Monoid -import qualified Data.Set as S +import Prelude -- Describes differences from the v5 repository format. -- diff --git a/Types/Remote.hs b/Types/Remote.hs index 87a964ca7..4237f5e00 100644 --- a/Types/Remote.hs +++ b/Types/Remote.hs @@ -66,7 +66,8 @@ data RemoteA a = Remote { -- (The MeterUpdate does not need to be used if it retrieves -- directly to the file, and not to an intermediate file.) retrieveKeyFile :: Key -> AssociatedFile -> FilePath -> MeterUpdate -> a Bool, - -- retrieves a key's contents to a tmp file, if it can be done cheaply + -- Retrieves a key's contents to a tmp file, if it can be done cheaply. + -- It's ok to create a symlink or hardlink. retrieveKeyFileCheap :: Key -> AssociatedFile -> FilePath -> a Bool, -- removes a key's contents (succeeds if the contents are not present) removeKey :: Key -> a Bool, diff --git a/Utility/Base64.hs b/Utility/Base64.hs index 80cc122a1..6ab3c984f 100644 --- a/Utility/Base64.hs +++ b/Utility/Base64.hs @@ -1,23 +1,22 @@ {- Simple Base64 encoding of Strings - - - Copyright 2011 Joey Hess <id@joeyh.name> + - Copyright 2011, 2015 Joey Hess <id@joeyh.name> - - License: BSD-2-clause -} module Utility.Base64 (toB64, fromB64Maybe, fromB64, prop_b64_roundtrips) where -import qualified "dataenc" Codec.Binary.Base64 as B64 -import Control.Applicative +import qualified "sandi" Codec.Binary.Base64 as B64 import Data.Maybe -import qualified Data.ByteString.Lazy as L -import Data.ByteString.Lazy.UTF8 (fromString, toString) +import Data.ByteString.UTF8 (fromString, toString) toB64 :: String -> String -toB64 = B64.encode . L.unpack . fromString +toB64 = toString . B64.encode . fromString fromB64Maybe :: String -> Maybe String -fromB64Maybe s = toString . L.pack <$> B64.decode s +fromB64Maybe s = either (const Nothing) (Just . toString) + (B64.decode $ fromString s) fromB64 :: String -> String fromB64 = fromMaybe bad . fromB64Maybe diff --git a/Utility/Bloom.hs b/Utility/Bloom.hs index aee760a1d..95ade6d32 100644 --- a/Utility/Bloom.hs +++ b/Utility/Bloom.hs @@ -27,7 +27,7 @@ import qualified Data.BloomFilter as Bloom #endif import Data.BloomFilter.Easy (safeSuggestSizing, Bloom) import Data.BloomFilter.Hash (Hashable, cheapHashes) -import Control.Monad.ST.Safe (ST) +import Control.Monad.ST (ST) #if MIN_VERSION_bloomfilter(2,0,0) diff --git a/Utility/CopyFile.hs b/Utility/CopyFile.hs index b123d006d..1c07ca110 100644 --- a/Utility/CopyFile.hs +++ b/Utility/CopyFile.hs @@ -16,7 +16,12 @@ module Utility.CopyFile ( import Common import qualified Build.SysConfig as SysConfig -data CopyMetaData = CopyTimeStamps | CopyAllMetaData +data CopyMetaData + -- Copy timestamps when possible, but no other metadata, and + -- when copying a symlink, makes a copy of its content. + = CopyTimeStamps + -- Copy all metadata when possible. + | CopyAllMetaData deriving (Eq) {- The cp command is used, because I hate reinventing the wheel, diff --git a/Utility/Data.hs b/Utility/Data.hs index 5ecd218fb..27c0a824c 100644 --- a/Utility/Data.hs +++ b/Utility/Data.hs @@ -5,6 +5,8 @@ - License: BSD-2-clause -} +{-# OPTIONS_GHC -fno-warn-tabs #-} + module Utility.Data where {- First item in the list that is not Nothing. -} diff --git a/Utility/DirWatcher.hs b/Utility/DirWatcher.hs index 3d3c14619..bde710626 100644 --- a/Utility/DirWatcher.hs +++ b/Utility/DirWatcher.hs @@ -57,7 +57,7 @@ eventsCoalesce = False #if (WITH_KQUEUE || WITH_FSEVENTS) eventsCoalesce = True #else -eventsCoalesce = undefined +eventsCoalesce = error "eventsCoalesce not defined" #endif #endif @@ -78,7 +78,7 @@ closingTracked = True #if WITH_KQUEUE closingTracked = False #else -closingTracked = undefined +closingTracked = error "closingTracked not defined" #endif #endif @@ -93,7 +93,7 @@ modifyTracked = True #if WITH_KQUEUE modifyTracked = False #else -modifyTracked = undefined +modifyTracked = error "modifyTracked not defined" #endif #endif @@ -131,7 +131,7 @@ watchDir dir prune scanevents hooks runstartup = #else type DirWatcherHandle = () watchDir :: FilePath -> Pruner -> Bool -> WatchHooks -> (IO () -> IO ()) -> IO DirWatcherHandle -watchDir = undefined +watchDir = error "watchDir not defined" #endif #endif #endif @@ -150,7 +150,7 @@ stopWatchDir = FSEvents.eventStreamDestroy #if WITH_WIN32NOTIFY stopWatchDir = Win32Notify.killWatchManager #else -stopWatchDir = undefined +stopWatchDir = error "stopWatchDir not defined" #endif #endif #endif diff --git a/Utility/Directory.hs b/Utility/Directory.hs index 85ec8bf45..7322cd85f 100644 --- a/Utility/Directory.hs +++ b/Utility/Directory.hs @@ -6,6 +6,7 @@ -} {-# LANGUAGE CPP #-} +{-# OPTIONS_GHC -fno-warn-tabs #-} module Utility.Directory where @@ -18,6 +19,7 @@ import Control.Applicative import Control.Concurrent import System.IO.Unsafe (unsafeInterleaveIO) import Data.Maybe +import Prelude #ifdef mingw32_HOST_OS import qualified System.Win32 as Win32 @@ -111,7 +113,7 @@ moveFile src dest = tryIO (rename src dest) >>= onrename -- But, mv will move into a directory if -- dest is one, which is not desired. whenM (isdir dest) rethrow - viaTmp mv dest undefined + viaTmp mv dest "" where rethrow = throwM e mv tmp _ = do diff --git a/Utility/DottedVersion.hs b/Utility/DottedVersion.hs index 67e40ff3c..ebf4c0bd1 100644 --- a/Utility/DottedVersion.hs +++ b/Utility/DottedVersion.hs @@ -5,6 +5,8 @@ - License: BSD-2-clause -} +{-# OPTIONS_GHC -fno-warn-tabs #-} + module Utility.DottedVersion where import Common diff --git a/Utility/Env.hs b/Utility/Env.hs index fdf06d807..c56f4ec23 100644 --- a/Utility/Env.hs +++ b/Utility/Env.hs @@ -6,6 +6,7 @@ -} {-# LANGUAGE CPP #-} +{-# OPTIONS_GHC -fno-warn-tabs #-} module Utility.Env where @@ -13,6 +14,7 @@ module Utility.Env where import Utility.Exception import Control.Applicative import Data.Maybe +import Prelude import qualified System.Environment as E import qualified System.SetEnv #else diff --git a/Utility/Exception.hs b/Utility/Exception.hs index ab47ae95f..9d4236c47 100644 --- a/Utility/Exception.hs +++ b/Utility/Exception.hs @@ -6,6 +6,7 @@ -} {-# LANGUAGE ScopedTypeVariables #-} +{-# OPTIONS_GHC -fno-warn-tabs #-} module Utility.Exception ( module X, diff --git a/Utility/ExternalSHA.hs b/Utility/ExternalSHA.hs index 6cef2830d..0defbaa16 100644 --- a/Utility/ExternalSHA.hs +++ b/Utility/ExternalSHA.hs @@ -8,6 +8,8 @@ - License: BSD-2-clause -} +{-# OPTIONS_GHC -fno-warn-tabs #-} + module Utility.ExternalSHA (externalSHA) where import Utility.SafeCommand @@ -18,8 +20,9 @@ import Utility.Exception import Data.List import Data.Char -import Control.Applicative import System.IO +import Control.Applicative +import Prelude externalSHA :: String -> Int -> FilePath -> IO (Either String String) externalSHA command shasize file = do diff --git a/Utility/FileMode.hs b/Utility/FileMode.hs index 5c4001ed8..201b8451c 100644 --- a/Utility/FileMode.hs +++ b/Utility/FileMode.hs @@ -124,7 +124,7 @@ withUmask _ a = a #endif combineModes :: [FileMode] -> FileMode -combineModes [] = undefined +combineModes [] = 0 combineModes [m] = m combineModes (m:ms) = foldl unionFileModes m ms @@ -151,7 +151,11 @@ setSticky f = modifyFileMode f $ addModes [stickyMode] - as writeFile. -} writeFileProtected :: FilePath -> String -> IO () -writeFileProtected file content = withUmask 0o0077 $ +writeFileProtected file content = writeFileProtected' file + (\h -> hPutStr h content) + +writeFileProtected' :: FilePath -> (Handle -> IO ()) -> IO () +writeFileProtected' file writer = withUmask 0o0077 $ withFile file WriteMode $ \h -> do void $ tryIO $ modifyFileMode file $ removeModes otherGroupModes - hPutStr h content + writer h diff --git a/Utility/FileSystemEncoding.hs b/Utility/FileSystemEncoding.hs index 139b74fe4..41c5972a0 100644 --- a/Utility/FileSystemEncoding.hs +++ b/Utility/FileSystemEncoding.hs @@ -6,6 +6,7 @@ -} {-# LANGUAGE CPP #-} +{-# OPTIONS_GHC -fno-warn-tabs #-} module Utility.FileSystemEncoding ( fileEncoding, diff --git a/Utility/FreeDesktop.hs b/Utility/FreeDesktop.hs index ee1c2f302..70332490b 100644 --- a/Utility/FreeDesktop.hs +++ b/Utility/FreeDesktop.hs @@ -10,6 +10,8 @@ - License: BSD-2-clause -} +{-# OPTIONS_GHC -fno-warn-tabs #-} + module Utility.FreeDesktop ( DesktopEntry, genDesktopEntry, @@ -39,6 +41,7 @@ import Data.List import Data.String.Utils import Data.Maybe import Control.Applicative +import Prelude type DesktopEntry = [(Key, Value)] diff --git a/Utility/Gpg.hs b/Utility/Gpg.hs index 6323d3a00..a1b782d97 100644 --- a/Utility/Gpg.hs +++ b/Utility/Gpg.hs @@ -9,14 +9,8 @@ module Utility.Gpg where -import Control.Applicative -import Control.Concurrent -import Control.Monad.IO.Class -import qualified Data.Map as M - import Common import qualified Build.SysConfig as SysConfig - #ifndef mingw32_HOST_OS import System.Posix.Types import qualified System.Posix.IO @@ -27,6 +21,10 @@ import Utility.Tmp #endif import Utility.Format (decode_c) +import Control.Concurrent +import Control.Monad.IO.Class +import qualified Data.Map as M + type KeyId = String newtype KeyIds = KeyIds { keyIds :: [KeyId] } diff --git a/Utility/Hash.hs b/Utility/Hash.hs index 9881815bd..f960a134f 100644 --- a/Utility/Hash.hs +++ b/Utility/Hash.hs @@ -9,13 +9,16 @@ module Utility.Hash ( skein256, skein512, md5, - prop_hashes_stable + prop_hashes_stable, + Mac(..), + calcMac, + prop_mac_stable, ) where import qualified Data.ByteString.Lazy as L import qualified Data.Text as T import qualified Data.Text.Encoding as T - +import qualified Data.ByteString as S import Crypto.Hash sha1 :: L.ByteString -> Digest SHA1 @@ -60,3 +63,37 @@ prop_hashes_stable = all (\(hasher, result) -> hasher foo == result) ] where foo = L.fromChunks [T.encodeUtf8 $ T.pack "foo"] + +{- File names are (client-side) MAC'ed on special remotes. + - The chosen MAC algorithm needs to be same for all files stored on the + - remote. + -} +data Mac = HmacSha1 | HmacSha224 | HmacSha256 | HmacSha384 | HmacSha512 + deriving (Eq) + +calcMac + :: Mac -- ^ MAC + -> S.ByteString -- ^ secret key + -> S.ByteString -- ^ message + -> String -- ^ MAC'ed message, in hexadecimal +calcMac mac = case mac of + HmacSha1 -> use SHA1 + HmacSha224 -> use SHA224 + HmacSha256 -> use SHA256 + HmacSha384 -> use SHA384 + HmacSha512 -> use SHA512 + where + use alg k m = show (hmacGetDigest (hmacAlg alg k m)) + +-- Check that all the MACs continue to produce the same. +prop_mac_stable :: Bool +prop_mac_stable = all (\(mac, result) -> calcMac mac key msg == result) + [ (HmacSha1, "46b4ec586117154dacd49d664e5d63fdc88efb51") + , (HmacSha224, "4c1f774863acb63b7f6e9daa9b5c543fa0d5eccf61e3ffc3698eacdd") + , (HmacSha256, "f9320baf0249169e73850cd6156ded0106e2bb6ad8cab01b7bbbebe6d1065317") + , (HmacSha384, "3d10d391bee2364df2c55cf605759373e1b5a4ca9355d8f3fe42970471eca2e422a79271a0e857a69923839015877fc6") + , (HmacSha512, "114682914c5d017dfe59fdc804118b56a3a652a0b8870759cf9e792ed7426b08197076bf7d01640b1b0684df79e4b67e37485669e8ce98dbab60445f0db94fce") + ] + where + key = T.encodeUtf8 $ T.pack "foo" + msg = T.encodeUtf8 $ T.pack "bar" diff --git a/Utility/HumanTime.hs b/Utility/HumanTime.hs index 85a9e15b6..e8fdb7c6e 100644 --- a/Utility/HumanTime.hs +++ b/Utility/HumanTime.hs @@ -20,11 +20,12 @@ import Utility.PartialPrelude import Utility.Applicative import Utility.QuickCheck +import qualified Data.Map as M import Data.Time.Clock import Data.Time.Clock.POSIX (POSIXTime) import Data.Char import Control.Applicative -import qualified Data.Map as M +import Prelude newtype Duration = Duration { durationSeconds :: Integer } deriving (Eq, Ord, Read, Show) diff --git a/Utility/LinuxMkLibs.hs b/Utility/LinuxMkLibs.hs index db64d1236..fdeb77959 100644 --- a/Utility/LinuxMkLibs.hs +++ b/Utility/LinuxMkLibs.hs @@ -7,7 +7,12 @@ module Utility.LinuxMkLibs where -import Control.Applicative +import Utility.PartialPrelude +import Utility.Directory +import Utility.Process +import Utility.Monad +import Utility.Path + import Data.Maybe import System.Directory import System.FilePath @@ -15,12 +20,8 @@ import Data.List.Utils import System.Posix.Files import Data.Char import Control.Monad.IfElse - -import Utility.PartialPrelude -import Utility.Directory -import Utility.Process -import Utility.Monad -import Utility.Path +import Control.Applicative +import Prelude {- Installs a library. If the library is a symlink to another file, - install the file it links to, and update the symlink to be relative. -} diff --git a/Utility/Metered.hs b/Utility/Metered.hs index f94b5d121..c34e931a4 100644 --- a/Utility/Metered.hs +++ b/Utility/Metered.hs @@ -144,7 +144,7 @@ defaultChunkSize :: Int defaultChunkSize = 32 * k - chunkOverhead where k = 1024 - chunkOverhead = 2 * sizeOf (undefined :: Int) -- GHC specific + chunkOverhead = 2 * sizeOf (1 :: Int) -- GHC specific data OutputHandler = OutputHandler { quietMode :: Bool diff --git a/Utility/Misc.hs b/Utility/Misc.hs index e4eccac43..45d5a0639 100644 --- a/Utility/Misc.hs +++ b/Utility/Misc.hs @@ -6,23 +6,25 @@ -} {-# LANGUAGE CPP #-} +{-# OPTIONS_GHC -fno-warn-tabs #-} module Utility.Misc where +import Utility.FileSystemEncoding +import Utility.Monad + import System.IO import Control.Monad import Foreign import Data.Char import Data.List -import Control.Applicative import System.Exit #ifndef mingw32_HOST_OS import System.Posix.Process (getAnyProcessStatus) import Utility.Exception #endif - -import Utility.FileSystemEncoding -import Utility.Monad +import Control.Applicative +import Prelude {- A version of hgetContents that is not lazy. Ensures file is - all read before it gets closed. -} diff --git a/Utility/Monad.hs b/Utility/Monad.hs index 878e0da67..ac751043c 100644 --- a/Utility/Monad.hs +++ b/Utility/Monad.hs @@ -5,6 +5,8 @@ - License: BSD-2-clause -} +{-# OPTIONS_GHC -fno-warn-tabs #-} + module Utility.Monad where import Data.Maybe diff --git a/Utility/Mounts.hsc b/Utility/Mounts.hsc index 1fb2362df..ad4adf334 100644 --- a/Utility/Mounts.hsc +++ b/Utility/Mounts.hsc @@ -26,6 +26,7 @@ import Utility.Exception import Data.Maybe import Control.Applicative #endif +import Prelude {- This is a stripped down mntent, containing only - fields available everywhere. -} diff --git a/Utility/Network.hs b/Utility/Network.hs index 7f228e155..4def3c5c5 100644 --- a/Utility/Network.hs +++ b/Utility/Network.hs @@ -5,12 +5,15 @@ - License: BSD-2-clause -} +{-# OPTIONS_GHC -fno-warn-tabs #-} + module Utility.Network where import Utility.Process import Utility.Exception import Control.Applicative +import Prelude {- Haskell lacks uname(2) bindings, except in the - Bindings.Uname addon. Rather than depend on that, diff --git a/Utility/OSX.hs b/Utility/OSX.hs index 22028e210..f6aba5096 100644 --- a/Utility/OSX.hs +++ b/Utility/OSX.hs @@ -5,6 +5,8 @@ - License: BSD-2-clause -} +{-# OPTIONS_GHC -fno-warn-tabs #-} + module Utility.OSX where import Utility.UserInfo diff --git a/Utility/PartialPrelude.hs b/Utility/PartialPrelude.hs index 6efa093fd..557955633 100644 --- a/Utility/PartialPrelude.hs +++ b/Utility/PartialPrelude.hs @@ -5,6 +5,8 @@ - them being accidentially used. -} +{-# OPTIONS_GHC -fno-warn-tabs #-} + module Utility.PartialPrelude where import qualified Data.Maybe diff --git a/Utility/Path.hs b/Utility/Path.hs index 9f0737fe8..8e3c2bddb 100644 --- a/Utility/Path.hs +++ b/Utility/Path.hs @@ -6,6 +6,7 @@ -} {-# LANGUAGE PackageImports, CPP #-} +{-# OPTIONS_GHC -fno-warn-tabs #-} module Utility.Path where @@ -16,6 +17,7 @@ import Data.List import Data.Maybe import Data.Char import Control.Applicative +import Prelude #ifdef mingw32_HOST_OS import qualified System.FilePath.Posix as Posix diff --git a/Utility/PosixFiles.hs b/Utility/PosixFiles.hs index 5a94ead01..4550bebdf 100644 --- a/Utility/PosixFiles.hs +++ b/Utility/PosixFiles.hs @@ -8,6 +8,7 @@ -} {-# LANGUAGE CPP #-} +{-# OPTIONS_GHC -fno-warn-tabs #-} module Utility.PosixFiles ( module X, diff --git a/Utility/Process.hs b/Utility/Process.hs index cbbe8a811..9f98596be 100644 --- a/Utility/Process.hs +++ b/Utility/Process.hs @@ -7,6 +7,7 @@ -} {-# LANGUAGE CPP, Rank2Types #-} +{-# OPTIONS_GHC -fno-warn-tabs #-} module Utility.Process ( module X, @@ -54,6 +55,7 @@ import qualified System.Posix.IO import Control.Applicative #endif import Data.Maybe +import Prelude import Utility.Misc import Utility.Exception diff --git a/Utility/QuickCheck.hs b/Utility/QuickCheck.hs index 54200d3f7..cd408ddc9 100644 --- a/Utility/QuickCheck.hs +++ b/Utility/QuickCheck.hs @@ -19,6 +19,7 @@ import System.Posix.Types import qualified Data.Map as M import qualified Data.Set as S import Control.Applicative +import Prelude instance (Arbitrary k, Arbitrary v, Eq k, Ord k) => Arbitrary (M.Map k v) where arbitrary = M.fromList <$> arbitrary diff --git a/Utility/Quvi.hs b/Utility/Quvi.hs index 0669e7351..0412116a1 100644 --- a/Utility/Quvi.hs +++ b/Utility/Quvi.hs @@ -6,6 +6,7 @@ -} {-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE FlexibleContexts #-} module Utility.Quvi where @@ -30,7 +31,7 @@ data Page = Page } deriving (Show) data Link = Link - { linkSuffix :: String + { linkSuffix :: Maybe String , linkUrl :: URLString } deriving (Show) @@ -43,7 +44,7 @@ instance FromJSON Page where instance FromJSON Link where parseJSON (Object v) = Link - <$> v .: "file_suffix" + <$> v .:? "file_suffix" <*> v .: "url" parseJSON _ = mzero @@ -53,7 +54,7 @@ parseEnum s = Page <$> get "QUVI_MEDIA_PROPERTY_TITLE" <*> ((:[]) <$> ( Link - <$> get "QUVI_MEDIA_STREAM_PROPERTY_CONTAINER" + <$> Just <$> (get "QUVI_MEDIA_STREAM_PROPERTY_CONTAINER") <*> get "QUVI_MEDIA_STREAM_PROPERTY_URL" ) ) diff --git a/Utility/SRV.hs b/Utility/SRV.hs index 203585a7e..b6d57dea5 100644 --- a/Utility/SRV.hs +++ b/Utility/SRV.hs @@ -25,8 +25,9 @@ import Utility.PartialPrelude import Network import Data.Function import Data.List -import Control.Applicative import Data.Maybe +import Control.Applicative +import Prelude #ifdef WITH_ADNS import ADNS.Resolver diff --git a/Utility/SafeCommand.hs b/Utility/SafeCommand.hs index f44112b82..0704e69f7 100644 --- a/Utility/SafeCommand.hs +++ b/Utility/SafeCommand.hs @@ -1,18 +1,21 @@ {- safely running shell commands - - - Copyright 2010-2013 Joey Hess <id@joeyh.name> + - Copyright 2010-2015 Joey Hess <id@joeyh.name> - - License: BSD-2-clause -} +{-# OPTIONS_GHC -fno-warn-tabs #-} + module Utility.SafeCommand where import System.Exit import Utility.Process import Data.String.Utils -import Control.Applicative import System.FilePath import Data.Char +import Control.Applicative +import Prelude {- A type for parameters passed to a shell command. A command can - be passed either some Params (multiple parameters can be included, @@ -44,23 +47,32 @@ toCommand = concatMap unwrap - if it succeeded or failed. -} boolSystem :: FilePath -> [CommandParam] -> IO Bool -boolSystem command params = boolSystemEnv command params Nothing +boolSystem command params = boolSystem' command params id -boolSystemEnv :: FilePath -> [CommandParam] -> Maybe [(String, String)] -> IO Bool -boolSystemEnv command params environ = dispatch <$> safeSystemEnv command params environ +boolSystem' :: FilePath -> [CommandParam] -> (CreateProcess -> CreateProcess) -> IO Bool +boolSystem' command params mkprocess = dispatch <$> safeSystem' command params mkprocess where dispatch ExitSuccess = True dispatch _ = False +boolSystemEnv :: FilePath -> [CommandParam] -> Maybe [(String, String)] -> IO Bool +boolSystemEnv command params environ = boolSystem' command params $ + \p -> p { env = environ } + {- Runs a system command, returning the exit status. -} safeSystem :: FilePath -> [CommandParam] -> IO ExitCode -safeSystem command params = safeSystemEnv command params Nothing +safeSystem command params = safeSystem' command params id -safeSystemEnv :: FilePath -> [CommandParam] -> Maybe [(String, String)] -> IO ExitCode -safeSystemEnv command params environ = do - (_, _, _, pid) <- createProcess (proc command $ toCommand params) - { env = environ } +safeSystem' :: FilePath -> [CommandParam] -> (CreateProcess -> CreateProcess) -> IO ExitCode +safeSystem' command params mkprocess = do + (_, _, _, pid) <- createProcess p waitForProcess pid + where + p = mkprocess $ proc command (toCommand params) + +safeSystemEnv :: FilePath -> [CommandParam] -> Maybe [(String, String)] -> IO ExitCode +safeSystemEnv command params environ = safeSystem' command params $ + \p -> p { env = environ } {- Wraps a shell command line inside sh -c, allowing it to be run in a - login shell that may not support POSIX shell, eg csh. -} diff --git a/Utility/Scheduled.hs b/Utility/Scheduled.hs index e077a1fea..b3813323d 100644 --- a/Utility/Scheduled.hs +++ b/Utility/Scheduled.hs @@ -32,7 +32,6 @@ import Utility.QuickCheck import Utility.PartialPrelude import Utility.Misc -import Control.Applicative import Data.List import Data.Time.Clock import Data.Time.LocalTime @@ -41,6 +40,8 @@ import Data.Time.Calendar.WeekDate import Data.Time.Calendar.OrdinalDate import Data.Tuple.Utils import Data.Char +import Control.Applicative +import Prelude {- Some sort of scheduled event. -} data Schedule = Schedule Recurrance ScheduledTime diff --git a/Utility/Tmp.hs b/Utility/Tmp.hs index dc5598137..de970fe56 100644 --- a/Utility/Tmp.hs +++ b/Utility/Tmp.hs @@ -6,6 +6,7 @@ -} {-# LANGUAGE CPP #-} +{-# OPTIONS_GHC -fno-warn-tabs #-} module Utility.Tmp where diff --git a/Utility/Touch.hsc b/Utility/Touch.hsc index f87bb62d6..c3318e6da 100644 --- a/Utility/Touch.hsc +++ b/Utility/Touch.hsc @@ -54,8 +54,8 @@ instance Storable TimeSpec where -- use the larger alignment of the two types in the struct alignment _ = max sec_alignment nsec_alignment where - sec_alignment = alignment (undefined::CTime) - nsec_alignment = alignment (undefined::CLong) + sec_alignment = alignment (1::CTime) + nsec_alignment = alignment (1::CLong) sizeOf _ = #{size struct timespec} peek ptr = do sec <- #{peek struct timespec, tv_sec} ptr @@ -92,7 +92,7 @@ touchBoth file atime mtime follow = -} instance Storable TimeSpec where - alignment _ = alignment (undefined::CLong) + alignment _ = alignment (1::CLong) sizeOf _ = #{size struct timeval} peek ptr = do sec <- #{peek struct timeval, tv_sec} ptr diff --git a/Utility/Url.hs b/Utility/Url.hs index 1b0c394b7..2ef1167e5 100644 --- a/Utility/Url.hs +++ b/Utility/Url.hs @@ -8,6 +8,7 @@ {-# LANGUAGE CPP #-} {-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE RankNTypes #-} +{-# LANGUAGE FlexibleContexts #-} module Utility.Url ( URLString, @@ -25,6 +26,9 @@ module Utility.Url ( ) where import Common +import Utility.Tmp +import qualified Build.SysConfig + import Network.URI import Network.HTTP.Conduit import Network.HTTP.Types @@ -32,8 +36,6 @@ import qualified Data.CaseInsensitive as CI import qualified Data.ByteString as B import qualified Data.ByteString.UTF8 as B8 -import qualified Build.SysConfig - type URLString = String type Headers = [String] @@ -122,10 +124,14 @@ getUrlInfo url uo = case parseURIRelaxed url of | Build.SysConfig.curl -> do output <- catchDefaultIO "" $ readProcess "curl" $ toCommand curlparams + let len = extractlencurl output + let good = found len Nothing case lastMaybe (lines output) of - Just ('2':_:_) -> found - (extractlencurl output) - Nothing + Just ('2':_:_) -> good + -- don't try to parse ftp status + -- codes; if curl got a length, + -- it's good + _ | "ftp" `isInfixOf` uriScheme u && isJust len -> good _ -> dne | otherwise -> dne Nothing -> dne @@ -242,8 +248,15 @@ download' quiet url file uo = do writeFile file "" go "curl" $ headerparams ++ quietopt "-s" ++ [Params "-f -L -C - -# -o"] - go cmd opts = boolSystem cmd $ - addUserAgent uo $ reqParams uo++opts++[File file, File url] + + {- Run wget in a temp directory because it has been buggy + - and overwritten files in the current directory, even though + - it was asked to write to a file elsewhere. -} + go cmd opts = withTmpDir "downloadurl" $ \tmp -> do + absfile <- absPath file + let ps = addUserAgent uo $ reqParams uo++opts++[File absfile, File url] + boolSystem' cmd ps $ \p -> p { cwd = Just tmp } + quietopt s | quiet = [Param s] | otherwise = [] diff --git a/Utility/UserInfo.hs b/Utility/UserInfo.hs index 5bf8d5c09..7e94cafa4 100644 --- a/Utility/UserInfo.hs +++ b/Utility/UserInfo.hs @@ -6,6 +6,7 @@ -} {-# LANGUAGE CPP #-} +{-# OPTIONS_GHC -fno-warn-tabs #-} module Utility.UserInfo ( myHomeDir, @@ -13,12 +14,13 @@ module Utility.UserInfo ( myUserGecos, ) where +import Utility.Env + import System.PosixCompat #ifndef mingw32_HOST_OS import Control.Applicative #endif - -import Utility.Env +import Prelude {- Current user's home directory. - diff --git a/Utility/Verifiable.hs b/Utility/Verifiable.hs index a861416e2..278c320ae 100644 --- a/Utility/Verifiable.hs +++ b/Utility/Verifiable.hs @@ -7,11 +7,12 @@ module Utility.Verifiable where -import Data.Digest.Pure.SHA -import Data.ByteString.Lazy.UTF8 (fromString) -import qualified Data.ByteString.Lazy as L +import Data.ByteString.UTF8 (fromString) +import qualified Data.ByteString as S -type Secret = L.ByteString +import Utility.Hash + +type Secret = S.ByteString type HMACDigest = String {- A value, verifiable using a HMAC digest and a secret. -} @@ -28,7 +29,7 @@ verify :: (Eq a, Show a) => Verifiable a -> Secret -> Bool verify v secret = v == mkVerifiable (verifiableVal v) secret calcDigest :: String -> Secret -> HMACDigest -calcDigest v secret = showDigest $ hmacSha1 secret $ fromString v +calcDigest v secret = calcMac HmacSha1 secret (fromString v) {- for quickcheck -} prop_verifiable_sane :: String -> String -> Bool diff --git a/Utility/WebApp.hs b/Utility/WebApp.hs index 54f2d6f2b..ce6a61c42 100644 --- a/Utility/WebApp.hs +++ b/Utility/WebApp.hs @@ -94,11 +94,7 @@ fixSockAddr addr = addr -- disable buggy sloworis attack prevention code webAppSettings :: Settings -#if MIN_VERSION_warp(2,1,0) webAppSettings = setTimeout halfhour defaultSettings -#else -webAppSettings = defaultSettings { settingsTimeout = halfhour } -#endif where halfhour = 30 * 60 @@ -155,11 +151,7 @@ lookupRequestField k req = fromMaybe "" . lookup k $ Wai.requestHeaders req {- Rather than storing a session key on disk, use a random key - that will only be valid for this run of the webapp. -} -#if MIN_VERSION_yesod(1,2,0) webAppSessionBackend :: Yesod.Yesod y => y -> IO (Maybe Yesod.SessionBackend) -#else -webAppSessionBackend :: Yesod.Yesod y => y -> IO (Maybe (Yesod.SessionBackend y)) -#endif webAppSessionBackend _ = do g <- newGenIO :: IO SystemRandom case genBytes 96 g of @@ -170,18 +162,8 @@ webAppSessionBackend _ = do where timeout = 120 * 60 -- 120 minutes use key = -#if MIN_VERSION_yesod(1,2,0) Just . Yesod.clientSessionBackend key . fst <$> Yesod.clientSessionDateCacher timeout -#else -#if MIN_VERSION_yesod(1,1,7) - Just . Yesod.clientSessionBackend2 key . fst - <$> Yesod.clientSessionDateCacher timeout -#else - return $ Just $ - Yesod.clientSessionBackend key timeout -#endif -#endif #ifdef WITH_WEBAPP_SECURE type AuthToken = SecureMem @@ -219,11 +201,7 @@ genAuthToken = do - Note that the usual Yesod error page is bypassed on error, to avoid - possibly leaking the auth token in urls on that page! -} -#if MIN_VERSION_yesod(1,2,0) checkAuthToken :: (Monad m, Yesod.MonadHandler m) => (Yesod.HandlerSite m -> AuthToken) -> m Yesod.AuthResult -#else -checkAuthToken :: forall t sub. (t -> AuthToken) -> Yesod.GHandler sub t Yesod.AuthResult -#endif checkAuthToken extractAuthToken = do webapp <- Yesod.getYesod req <- Yesod.getRequest diff --git a/Utility/Yesod.hs b/Utility/Yesod.hs index 231bb291e..a8055d34d 100644 --- a/Utility/Yesod.hs +++ b/Utility/Yesod.hs @@ -20,69 +20,37 @@ module Utility.Yesod #if ! MIN_VERSION_yesod(1,4,0) , withUrlRenderer #endif -#if ! MIN_VERSION_yesod(1,2,0) - , Html -#endif ) where -#if MIN_VERSION_yesod(1,2,0) import Yesod as Y -#else -import Yesod as Y hiding (Html) -#endif -#if MIN_VERSION_yesod_form(1,3,8) import Yesod.Form.Bootstrap3 as Y hiding (bfs) -#else -import Assistant.WebApp.Bootstrap3 as Y hiding (bfs) -#endif #ifndef __NO_TH__ import Yesod.Default.Util import Language.Haskell.TH.Syntax (Q, Exp) -#if MIN_VERSION_yesod_default(1,1,0) import Data.Default (def) import Text.Hamlet hiding (Html) #endif -#endif #if ! MIN_VERSION_yesod(1,4,0) -#if MIN_VERSION_yesod(1,2,0) import Data.Text (Text) #endif -#endif #ifndef __NO_TH__ widgetFile :: String -> Q Exp -#if ! MIN_VERSION_yesod_default(1,1,0) -widgetFile = widgetFileNoReload -#else widgetFile = widgetFileNoReload $ def { wfsHamletSettings = defaultHamletSettings { hamletNewlines = AlwaysNewlines } } -#endif hamletTemplate :: FilePath -> FilePath hamletTemplate f = globFile "hamlet" f #endif {- Lift Handler to Widget -} -#if MIN_VERSION_yesod(1,2,0) liftH :: Monad m => HandlerT site m a -> WidgetT site m a liftH = handlerToWidget -#else -liftH :: MonadLift base m => base a -> m a -liftH = lift -#endif -{- Misc new names for stuff. -} -#if ! MIN_VERSION_yesod(1,2,0) -withUrlRenderer :: forall master sub. HtmlUrl (Route master) -> GHandler sub master RepHtml -withUrlRenderer = hamletToRepHtml - -type Html = RepHtml -#else #if ! MIN_VERSION_yesod_core(1,2,20) withUrlRenderer :: MonadHandler m => ((Route (HandlerSite m) -> [(Text, Text)] -> Text) -> output) -> m output withUrlRenderer = giveUrlRenderer #endif -#endif @@ -1 +1 @@ -C:\MINGW\MSYS\1.0\BIN\SH.EXE standalone/windows/build-simple.sh +sh standalone/windows/build-simple.sh diff --git a/debian/changelog b/debian/changelog index c29058ca6..53a59875d 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,7 +1,80 @@ -git-annex (5.20150410) UNRELEASED; urgency=medium - * get, move, copy, mirror: Concurrent downloads and uploads are now supported! For example: git-annex get -J10 + +git-annex (5.20150508.2) UNRELEASED; urgency=medium + + * import: Refuse to import files that are within the work tree, as that + does not make sense and could cause data loss. + * drop: Now supports --all, --unused, and --key. + * drop: Now defaults to --all when run in a bare repository. + (Previously, did nothing when run in a bare repository.) + + -- Joey Hess <id@joeyh.name> Mon, 11 May 2015 12:45:06 -0400 + +git-annex (5.20150508.1) unstable; urgency=medium + + * Now builds cleanly using ghc 7.10 (as well as ghc back to 7.6). + * Imrovements to the git-annex-standalone.deb build process. + (Thanks, Yaroslav Halchenko) + + -- Joey Hess <id@joeyh.name> Mon, 11 May 2015 12:08:58 -0400 + +git-annex (5.20150508) unstable; urgency=medium + + * Improve behavior when a git-annex command is told to operate + on a file that doesn't exist. It will now continue to other + files specified after that on the command line, and only error out at + the end. + * S3: Enable debug logging when annex.debug or --debug is set. + * S3: git annex info will show additional information about a S3 remote + (endpoint, port, storage class) + * S3: Let git annex enableremote be used, without trying to recreate + a bucket that should already exist. + * S3: Fix incompatability with bucket names used by hS3; the aws library + cannot handle upper-case bucket names. git-annex now converts them to + lower case automatically. + * import: Check for gitignored files before moving them into the tree. + (Needs git 1.8.4 or newer.) + * import: Don't stop entire import when one file fails due to being + gitignored or conflicting with something in the work tree. + * import: Before removing a duplicate file in --deduplicate or + --clean-duplicates mode, verify that enough copies of its content still + exist. + * Improve integration with KDE's file manager to work with dolphin + version 14.12.3 while still being compatable with 4.14.2. + Thanks, silvio. + * assistant: Added --autostop to complement --autostart. + * Work around wget bug #784348 which could cause it to clobber git-annex + symlinks when downloading from ftp. + * Support checking ftp urls for file presence. + * Fix bogus failure of fsck --fast. + * fsck: Ignore error recording the fsck in the activity log, + which can happen when running fsck in a read-only repository. + Closes: #698559 + (fsck can still need to write to the repository if it find problems, + but a successful fsck can be done read-only) + * Improve quvi 0.4 output parsing to handle cases wher there is no known + filename extension. This is currently the case when using quvi with + youtube. In this case, the extension ".m" will be used. + * Dropped support for older versions of yesod, warp, and dbus than the ones + in Debian Jessie. + * Switch from the obsolete dataenc library for base64 encoding to sandi. + (Thanks, Magnus Therning) + * Debian's ghc now supports TH on arm! Adjust build dependencies + to build the webapp on arm, and enable DAV support on arm. \o/ + * Adjust some other arch specific build dependencies that are now + available on more architectures in Devian unstable. + * Windows: Remove cygwin ssh, the newer version of which has stopped + honoring the setting of HOME. Instead, copy msysgit's ssh into PATH. + Note that setting up a remote ssh server using password authentication + is known to be broken in this release on Windows. + * Windows: Roll back to an older version of rsync from cygwin. + The newer version has some dependency on a newer ssh from cygwin. + + -- Joey Hess <id@joeyh.name> Fri, 08 May 2015 13:42:30 -0400 + +git-annex (5.20150420) unstable; urgency=medium + * Fix activity log parsing, which caused the log to not retain activity from other uuids. * Union merge could fall over if there was a file in the repository @@ -9,13 +82,32 @@ git-annex (5.20150410) UNRELEASED; urgency=medium * info dir: Added information about repositories that contain files in the specified directory. * info: Added --bytes option. + * bittorrent: Fix handling of magnet links. + * When a key's size is unknown, still check the annex.diskreserve, + and avoid getting content if the disk is too full. + * Fix fsck --from a git remote in a local directory, and from + a directory special remote. + This was a reversion caused by the relative path changes in 5.20150113. + * fsck --from remote: When bad content is found in the remote, + and the local repo does not have a copy of the content, preserve + the bad content in .git/annex/bad/ to avoid further data loss. + * fsck --from remote: Avoid downloading a key if it would go over + the annex.diskreserve limit. + * required: New command, like wanted, but for required content. + * Removed dependency on haskell SHA library, + instead using cryptohash >= 0.11.0. + * Make repo init more robust. + * New debian/rules build-standalone target, which generates a + git-annex-standalone.deb that should work on many old Debian etc + systems. Thanks, Yaroslav Halchenko. * Windows: Renamed start menu file to avoid loop in some versions of Windows where the menu file is treated as a git-annex program. - * bittorrent: Fix handling of magnet links. * Windows: Fixed support of remotes on other drives. (A reversion introduced in version 5.20150113.) + * Windows: Bundled versions of rsync, wget, ssh, and gpg from + cygwin all updated. Thanks, Yury V. Zaytsev. - -- Joey Hess <id@joeyh.name> Thu, 09 Apr 2015 20:59:43 -0400 + -- Joey Hess <id@joeyh.name> Mon, 20 Apr 2015 14:44:04 -0400 git-annex (5.20150409) unstable; urgency=medium diff --git a/debian/control b/debian/control index 6cb4a6327..674651603 100644 --- a/debian/control +++ b/debian/control @@ -10,14 +10,13 @@ Build-Depends: libghc-data-default-dev, libghc-hslogger-dev, libghc-pcre-light-dev, - libghc-sha-dev, - libghc-cryptohash-dev, - libghc-dataenc-dev, + libghc-cryptohash-dev (>= 0.11.0), + libghc-sandi-dev, libghc-utf8-string-dev, libghc-aws-dev (>= 0.9.2-2~), libghc-conduit-dev, libghc-resourcet-dev, - libghc-dav-dev (>= 1.0) [amd64 i386 kfreebsd-amd64 kfreebsd-i386 powerpc], + libghc-dav-dev (>= 1.0) [amd64 armel armhf i386 kfreebsd-amd64 kfreebsd-i386 powerpc ppc64el hurd-i386], libghc-quickcheck2-dev, libghc-monad-control-dev (>= 0.3), libghc-exceptions-dev (>= 0.6), @@ -32,18 +31,20 @@ Build-Depends: libghc-edit-distance-dev, libghc-hinotify-dev [linux-any], libghc-stm-dev (>= 2.3), - libghc-dbus-dev (>= 0.10.3) [linux-any], + libghc-dbus-dev (>= 0.10.7) [linux-any], libghc-fdo-notify-dev (>= 0.3) [linux-any], - libghc-yesod-dev [i386 amd64 kfreebsd-i386 kfreebsd-amd64 powerpc], - libghc-yesod-static-dev [i386 amd64 kfreebsd-i386 kfreebsd-amd64 powerpc], - libghc-yesod-default-dev [i386 amd64 kfreebsd-amd64 powerpc], - libghc-hamlet-dev [i386 amd64 kfreebsd-i386 kfreebsd-amd64 powerpc], - libghc-shakespeare-dev [i386 amd64 kfreebsd-i386 kfreebsd-amd64 powerpc], - libghc-clientsession-dev [i386 amd64 kfreebsd-i386 kfreebsd-amd64 powerpc], - libghc-warp-dev [i386 amd64 kfreebsd-i386 kfreebsd-amd64 powerpc], - libghc-warp-tls-dev [i386 amd64 kfreebsd-i386 kfreebsd-amd64 powerpc], - libghc-wai-dev [i386 amd64 kfreebsd-i386 kfreebsd-amd64 powerpc], - libghc-wai-extra-dev [i386 amd64 kfreebsd-i386 kfreebsd-amd64 powerpc], + libghc-yesod-dev (>= 1.2.6.1) [i386 amd64 armel armhf kfreebsd-i386 kfreebsd-amd64 powerpc ppc64el], + libghc-yesod-core-dev (>= 1.2.19) [i386 amd64 armel armhf kfreebsd-i386 kfreebsd-amd64 powerpc ppc64el], + libghc-yesod-form-dev (>= 1.3.15) [i386 amd64 armel armhf kfreebsd-i386 kfreebsd-amd64 powerpc ppc64el], + libghc-yesod-static-dev (>= 1.2.4) [i386 amd64 armel armhf kfreebsd-i386 kfreebsd-amd64 powerpc ppc64el], + libghc-yesod-default-dev (>= 1.2.0) [i386 amd64 armel armhf kfreebsd-amd64 powerpc ppc64el], + libghc-hamlet-dev [i386 amd64 armel armhf kfreebsd-i386 kfreebsd-amd64 powerpc ppc64el], + libghc-shakespeare-dev [i386 amd64 armel armhf kfreebsd-i386 kfreebsd-amd64 powerpc ppc64el], + libghc-clientsession-dev [i386 amd64 armel armhf kfreebsd-i386 kfreebsd-amd64 powerpc ppc64el], + libghc-warp-dev (>= 3.0.0.5) [i386 amd64 armel armhf kfreebsd-i386 kfreebsd-amd64 powerpc ppc64el], + libghc-warp-tls-dev [i386 amd64 armel armhf kfreebsd-i386 kfreebsd-amd64 powerpc ppc64el], + libghc-wai-dev [i386 amd64 kfreebsd-i386 armel armhf kfreebsd-amd64 powerpc ppc64el], + libghc-wai-extra-dev [i386 amd64 armel armhf kfreebsd-i386 kfreebsd-amd64 powerpc ppc64el], libghc-securemem-dev, libghc-byteable-dev, libghc-dns-dev, @@ -65,13 +66,12 @@ Build-Depends: libghc-esqueleto-dev, libghc-monad-logger-dev, libghc-feed-dev (>= 0.3.9.2), - libghc-regex-tdfa-dev [!mipsel !s390], - libghc-regex-compat-dev [mipsel s390], - libghc-tasty-dev (>= 0.7) [!sparc], - libghc-tasty-hunit-dev [!sparc], - libghc-tasty-quickcheck-dev [!sparc], - libghc-tasty-rerun-dev [!sparc], - libghc-optparse-applicative-dev [!sparc], + libghc-regex-tdfa-dev, + libghc-tasty-dev (>= 0.7), + libghc-tasty-hunit-dev, + libghc-tasty-quickcheck-dev, + libghc-tasty-rerun-dev, + libghc-optparse-applicative-dev, lsof [!kfreebsd-i386 !kfreebsd-amd64 !hurd-any], ikiwiki, perlmagick, @@ -81,7 +81,6 @@ Build-Depends: curl, openssh-client, git-remote-gcrypt (>= 0.20130908-6), - llvm-3.4 [armel armhf], Maintainer: Gergely Nagy <algernon@madhouse-project.org> Standards-Version: 3.9.6 Vcs-Git: git://git.kitenet.net/git-annex diff --git a/debian/copyright b/debian/copyright index 7b9807680..322d68f04 100644 --- a/debian/copyright +++ b/debian/copyright @@ -28,10 +28,6 @@ Files: Utility/Gpg.hs Utility/DirWatcher* Copyright: © 2010-2014 Joey Hess <id@joeyh.name> License: GPL-3+ -Files: Assistant/WebApp/Bootstrap3.hs -Copyright: 2010 Michael Snoyman -License: BSD-2-clause - Files: doc/logo* */favicon.ico standalone/osx/git-annex.app/Contents/Resources/git-annex.icns standalone/android/icons/* Copyright: 2007 Henrik Nyh <http://henrik.nyh.se/> 2010 Joey Hess <id@joeyh.name> diff --git a/debian/create-standalone-changelog b/debian/create-standalone-changelog new file mode 100755 index 000000000..d972f4e30 --- /dev/null +++ b/debian/create-standalone-changelog @@ -0,0 +1,17 @@ +#!/bin/bash +# +# A little helper script to build a package with standalone git-annex +# It relies on being run within git-annex's git repository +# +set -eu + +umask 022 + +# This is the same method that the configure script uses when git-annex is +# built from git master. +ANNEX_VERSION=$(runghc Build/BuildVersion.hs) + +ANNEX_NDVERSION=$( echo ${ANNEX_VERSION} | sed -e 's,-,+git,' -e 's,$,-1~ndall+1,') + +dch --noconf -v ${ANNEX_NDVERSION} \ + --force-bad-version --force-distribution -D neurodebian "Backported fresh snapshot" diff --git a/debian/patches/series.standalone-build b/debian/patches/series.standalone-build new file mode 100644 index 000000000..9dc994164 --- /dev/null +++ b/debian/patches/series.standalone-build @@ -0,0 +1 @@ +standalone-build diff --git a/debian/patches/standalone-build b/debian/patches/standalone-build new file mode 100644 index 000000000..51bfa2f17 --- /dev/null +++ b/debian/patches/standalone-build @@ -0,0 +1,59 @@ +From: Yaroslav Halchenko <debian@onerussian.com> +Subject: Patch debian/ to provide a standalone build of git-annex + +Origin: NeuroDebian +Last-Update: 2015-04-20 + +--- a/debian/control ++++ b/debian/control +@@ -87,15 +87,14 @@ Vcs-Git: git://git.kitenet.net/git-annex + Homepage: http://git-annex.branchable.com/ + XS-Testsuite: autopkgtest + +-Package: git-annex ++Package: git-annex-standalone + Architecture: any + Section: utils +-Depends: ${misc:Depends}, ${shlibs:Depends}, +- git (>= 1:1.8.1), +- rsync, +- wget, +- curl, +- openssh-client (>= 1:5.6p1) ++Conflicts: git-annex ++Provides: git-annex ++Depends: ${misc:Depends}, ++ git, ++ openssh-client + Recommends: + lsof, + gnupg, +@@ -110,7 +109,7 @@ Suggests: + bup, + tahoe-lafs, + libnss-mdns, +-Description: manage files with git, without checking their contents into git ++Description: manage files with git, without checking their contents into git -- standalone build + git-annex allows managing files with git, without checking the file + contents into git. While that may seem paradoxical, it is useful when + dealing with files larger than git can currently easily handle, whether due +@@ -128,3 +127,7 @@ Description: manage files with git, with + noticing when files are changed, and automatically committing them + to git and transferring them to other computers. The git-annex webapp + makes it easy to set up and use git-annex this way. ++ . ++ This package provides a standalone bundle build of git-annex, which ++ should be installable on any more or less recent Debian or Ubuntu ++ release. +--- /dev/null ++++ b/debian/install +@@ -0,0 +1 @@ ++tmp/git-annex.linux usr/lib +--- /dev/null ++++ b/debian/links +@@ -0,0 +1 @@ ++/usr/lib/git-annex.linux/git-annex /usr/bin/git-annex +--- /dev/null ++++ b/debian/manpages +@@ -0,0 +1 @@ ++debian/git-annex-standalone/usr/lib/git-annex.linux/usr/share/man/man1/git-annex* diff --git a/debian/rules b/debian/rules index dab54a7b1..927c1c925 100755 --- a/debian/rules +++ b/debian/rules @@ -2,8 +2,26 @@ export CABAL=debian/cabal-wrapper +STANDALONE_BUILD=$(shell grep -qe '^Package: git-annex-standalone' debian/control \ + && echo 1 || echo 0) + # Do use the changelog's version number, rather than making one up. export RELEASE_BUILD=1 %: dh $@ + + +# Standalone build logic/helpers +ifeq ($(STANDALONE_BUILD),1) + +override_dh_auto_build: + make linuxstandalone + +override_dh_auto_install: + : # nothing to do, we just need to copy the beast, as instructed in debian/install + +override_dh_fixperms: + dh_fixperms -Xld-linux + +endif diff --git a/doc/Android/comment_6_455bcbd36c7b5eeb905cc56da4666b07._comment b/doc/Android/comment_6_455bcbd36c7b5eeb905cc56da4666b07._comment new file mode 100644 index 000000000..75a1db4c2 --- /dev/null +++ b/doc/Android/comment_6_455bcbd36c7b5eeb905cc56da4666b07._comment @@ -0,0 +1,30 @@ +[[!comment format=mdwn + username="madduck" + subject="Weirdness when run from adb shell" + date="2015-05-06T14:28:43Z" + content=""" +How is this designed to work in the contact of Androids crap permissions? + + shell@kminilte:/ $ /data/data/ga.androidterm/runshell + /system/bin/sh: /data/data/ga.androidterm/runshell: can't execute: Permission denied + + 126|shell@kminilte:/ $ /system/bin/sh /data/data/ga.androidterm/runshell + Falling back to hardcoded app location; cannot find expected files in /data/app-lib + + shell@kminilte:/sdcard/git-annex.home $ git + /system/bin/sh: git: not found + + 127|shell@kminilte:/sdcard/git-annex.home $ echo $PATH + /data/data/ga.androidterm/bin:/sbin:/vendor/bin:/system/sbin:/system/bin:/system/xbin + + shell@kminilte:/sdcard/git-annex.home $ /data/data/ga.androidterm/bin/git < + /data/data/ga.androidterm/bin/git: Permission denied + + shell@kminilte:/sdcard/git-annex.home $ ls -l /data/data/ga.androidterm/bin -d + drwx------ u0_a255 u0_a255 2015-05-05 07:58 bin + + shell@kminilte:/sdcard/git-annex.home $ id + uid=2000(shell) gid=2000(shell) groups=1003(graphics),1004(input),1007(log),1011(adb),1015(sdcard_rw),1028(sdcard_r),3001(net_bt_admin),3002(net_bt),3003(inet),3006(net_bw_stats) context=u:r:shell:s0 + + +"""]] diff --git a/doc/bugs/Can__39__t_get_content_from_S3_with_s3-aws_library.mdwn b/doc/bugs/Can__39__t_get_content_from_S3_with_s3-aws_library.mdwn new file mode 100644 index 000000000..5f7e490ca --- /dev/null +++ b/doc/bugs/Can__39__t_get_content_from_S3_with_s3-aws_library.mdwn @@ -0,0 +1,62 @@ +### Please describe the problem. +I have lots of content stored in Amazon S3. Using git-annex from before commit 911ba8d972e4e7b151385d30c198598e1a0dfaca, I am able to ``git annex get`` from S3 and files are downloaded. +Using a more recent version (eg that commit, or the current master, or release 20150409), I am unable to download the content. + +I'm not sure if my repo or remote is somehow misconfigured, or if there's something else going on here. + +--Walter + +### What steps will reproduce the problem? +Use a version of git-annex with s3-aws + +### What version of git-annex are you using? On what operating system? +Debian, versions as above + +### Please provide any additional information below. + +[[!format sh """ +# If you can, paste a complete transcript of the problem occurring here. +# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log + +git annex get . --from cloud --debug +[2015-04-19 22:23:37 BST] read: git ["--git-dir=../../../.git","--work-tree=../../..","--literal-pathspecs","show-ref","git-annex"] +[2015-04-19 22:23:37 BST] read: git ["--git-dir=../../../.git","--work-tree=../../..","--literal-pathspecs","show-ref","--hash","refs/heads/git-annex"] +[2015-04-19 22:23:37 BST] read: git ["--git-dir=../../../.git","--work-tree=../../..","--literal-pathspecs","log","refs/heads/git-annex..a51a912223d3d86f19762e387e3eae23c3024d2c","-n1","--pretty=%H"] +[2015-04-19 22:23:37 BST] chat: git ["--git-dir=../../../.git","--work-tree=../../..","--literal-pathspecs","cat-file","--batch"] +[2015-04-19 22:23:37 BST] read: git ["--git-dir=../../../.git","--work-tree=../../..","--literal-pathspecs","ls-files","--cached","-z","--","."] +get IMG_7079.JPG (from cloud...) [2015-04-19 22:23:37 BST] chat: gpg ["--batch","--no-tty","--use-agent","--quiet","--trust-model","always","--decrypt"] + +failed +get IMG_7080.JPG (from cloud...) +failed + + + +git annex info cloud +remote: cloud +description: [cloud] +uuid: be992080-b1db-11e1-8f79-1b10bb4092ef +trust: semitrusted +cost: 250.0 +type: S3 +creds: embedded in git repository (gpg encrypted) +bucket: ffffffffffffffffffffffffff +partsize: unlimited +encryption: encrypted (to gpg keys: FFFFFFFFFFFF) (hybrid mode) +chunking: none +remote annex keys: 0 +remote annex size: 0 bytes + + +git annex fsck -f cloud +fsck IMG_6876.JPG (checking cloud...) (StatusCodeException (Status {statusCode = 301, statusMessage = "Moved Permanently"}) [("x-amz-request-id","275ADF5B1B77D514"),("x-amz-id-2","flWGBHOZYEZAohygAzBIZAYd7nBGkm3HpSMfJuhgRp3txXx20yJz7S4yRlNLwCs1cHUMyWc9JbA="),("Content-Type","application/xml"),("Transfer-Encoding","chunked"),("Date","Sun, 19 Apr 2015 22:23:15 GMT"),("Server","AmazonS3")] (CJ {expose = []})) failed + + + +# End of transcript or log. +"""]] + +> I think I've made all the git-annex improvements that are going to come +> from this bug report. There's still the open bug on the aws library to better +> follow these redirects. Anyway, I think it makes sense to call this +> bug [[done]]. --[[Joey]] diff --git a/doc/bugs/Can__39__t_get_content_from_S3_with_s3-aws_library/comment_11_f30f86482d48ff8d658a4ee74d4c8586._comment b/doc/bugs/Can__39__t_get_content_from_S3_with_s3-aws_library/comment_11_f30f86482d48ff8d658a4ee74d4c8586._comment new file mode 100644 index 000000000..2f2a3f0c6 --- /dev/null +++ b/doc/bugs/Can__39__t_get_content_from_S3_with_s3-aws_library/comment_11_f30f86482d48ff8d658a4ee74d4c8586._comment @@ -0,0 +1,21 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawnSenxKyE_2Z6Wb-EBMO8FciyRywjx1ZiQ" + nickname="Walter" + subject="comment 11" + date="2015-04-23T20:29:17Z" + content=""" +I think I may have not been entirely clear previously; the file \"GPGHMACSHA1--417830f4c50a2887674917abd2c18c522853255a\" was not in the bucket, but git annex said it was. That is, the upload failed, but git annex thought it succeeded. + +Similarly, all the files I recently added we not actually uploaded, but git annex thought they were. I ``git annex fsck``ed them, which was fast because it failed to download any of them. fscking other files is slow, as it has to download them of course. + +Maybe to reproduce this, you could try: + +[[!format sh \"\"\" +git annex initremote cloud datacenter=ap-southeast-1 +git annex add file +git annex copy --to cloud file +git annex drop file +git annex enableremote could dataceter=ap-southeast-2 +git annex get file +\"\"\"]] +"""]] diff --git a/doc/bugs/Can__39__t_get_content_from_S3_with_s3-aws_library/comment_12_1ba3271a62b8ce8088c67e4741ecf385._comment b/doc/bugs/Can__39__t_get_content_from_S3_with_s3-aws_library/comment_12_1ba3271a62b8ce8088c67e4741ecf385._comment new file mode 100644 index 000000000..3b416f09b --- /dev/null +++ b/doc/bugs/Can__39__t_get_content_from_S3_with_s3-aws_library/comment_12_1ba3271a62b8ce8088c67e4741ecf385._comment @@ -0,0 +1,32 @@ +[[!comment format=mdwn + username="joey" + subject="""fully reproduced""" + date="2015-04-24T16:33:02Z" + content=""" +I was able to fully reproduce this bug! I installed the old version of +git-annex that used the S3 library, and made a remote: + + joey@darkstar:~/tmp/rrold>git annex initremote S3 type=S3 encryption=none datacenter=ap-southeast-1 + initremote S3 (checking bucket...) (creating bucket in ap-southeast-1...) ok + joey@darkstar:~/tmp/rrold>git annex move me --to S3 + move me (checking S3...) (to S3...) + ok + +Retrieval then failed using current git-annex. + +Also, a remote made with the old git-annex with datacenter=ap-southeast-2 +fails with the new git-annex. + +Hypothesis: Either the new or the old S3 library must be confusing between +ap-southeast-1/2. My guess is the old library was just creating and using +buckets in the wrong place, at least when told to use ap-southeast-*. + +--- + +I cannot reproduce anything about "the upload failed, but git annex thought it succeeded", +nor do I see any indications in comments 11 or 12 that git-annex's location +log is failing in any way. The sequence of commands in comment 11 ends with the +get failing, as it should, since the remote has been switched to a different +datacenter. I don't understand what you're seeing in comment #12 at all; +it seems to just show it getting a file successfully. +"""]] diff --git a/doc/bugs/Can__39__t_get_content_from_S3_with_s3-aws_library/comment_12_a208344e552bfe6b8d9f409560c9a515._comment b/doc/bugs/Can__39__t_get_content_from_S3_with_s3-aws_library/comment_12_a208344e552bfe6b8d9f409560c9a515._comment new file mode 100644 index 000000000..030e82d9d --- /dev/null +++ b/doc/bugs/Can__39__t_get_content_from_S3_with_s3-aws_library/comment_12_a208344e552bfe6b8d9f409560c9a515._comment @@ -0,0 +1,61 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawnSenxKyE_2Z6Wb-EBMO8FciyRywjx1ZiQ" + nickname="Walter" + subject="comment 12" + date="2015-04-23T21:03:36Z" + content=""" +For completeness, here is the output when I get a file that *is* properly in the bucket (and you could use for any further testing you need to do). + +While this may have been caused by some misconfiguration on my part (though I'm not entirely sure how that could happen, strangely it would be easier to muck up now enableremote doesn't create a new bucket), I feel the potential harm here (the location information being wrong) is quite serious. (I'm sure this point does not escape you). + +[[!format sh \"\"\" +>git annex get --force --debug file.jpg --from cloud +[2015-04-23 21:52:41 BST] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"show-ref\",\"git-annex\"] +[2015-04-23 21:52:41 BST] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"show-ref\",\"--hash\",\"refs/heads/git-annex\"] +[2015-04-23 21:52:41 BST] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"log\",\"refs/heads/git-annex..cb0f954d09e3ea28171434e0e7499c84d1722fce\",\"-n1\",\"--pretty=%H\"] +[2015-04-23 21:52:41 BST] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"log\",\"refs/heads/git-annex..573f75e01681e9bf2b513bc85e18fc250298a4d3\",\"-n1\",\"--pretty=%H\"] +[2015-04-23 21:52:41 BST] chat: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"cat-file\",\"--batch\"] +[2015-04-23 21:52:41 BST] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"ls-files\",\"--cached\",\"-z\",\"--\",\"file.jpg\"] +[2015-04-23 21:52:41 BST] chat: gpg [\"--batch\",\"--no-tty\",\"--use-agent\",\"--quiet\",\"--trust-model\",\"always\",\"--decrypt\"] +(checking cloud...) [2015-04-23 21:52:42 BST] String to sign: \"HEAD\n\n\nThu, 23 Apr 2015 20:52:42 GMT\n/BUCKET/GPGHMACSHA1--08b3dee71059819e3558ac9ef8b82ad87e2d8951\" +[2015-04-23 21:52:42 BST] Host: \"BUCKET.s3-ap-southeast-2.amazonaws.com\" +[2015-04-23 21:52:42 BST] Path: \"/GPGHMACSHA1--08b3dee71059819e3558ac9ef8b82ad87e2d8951\" +[2015-04-23 21:52:42 BST] Query string: \"\" +[2015-04-23 21:52:42 BST] Response status: Status {statusCode = 200, statusMessage = \"OK\"} +[2015-04-23 21:52:42 BST] Response header 'x-amz-id-2': 'f8bEclNud1KNHevvGPVHutG3V0TH/ixnMSuu3NBhEKRrWaUYtENbKyA5PyxCdSrz0REgq/Bgu1w=' +[2015-04-23 21:52:42 BST] Response header 'x-amz-request-id': '7A344C3C3A27308E' +[2015-04-23 21:52:42 BST] Response header 'Date': 'Thu, 23 Apr 2015 20:52:43 GMT' +[2015-04-23 21:52:42 BST] Response header 'Last-Modified': 'Fri, 31 Oct 2014 07:03:03 GMT' +[2015-04-23 21:52:42 BST] Response header 'ETag': '\"66a85b0007a52d82e5bd29192ebdb510\"' +[2015-04-23 21:52:42 BST] Response header 'Accept-Ranges': 'bytes' +[2015-04-23 21:52:42 BST] Response header 'Content-Type': '' +[2015-04-23 21:52:42 BST] Response header 'Content-Length': '46058' +[2015-04-23 21:52:42 BST] Response header 'Server': 'AmazonS3' +[2015-04-23 21:52:42 BST] Response metadata: S3: request ID=<none>, x-amz-id-2=<none> +get file.jpg (from cloud...) +[2015-04-23 21:52:42 BST] String to sign: \"GET\n\n\nThu, 23 Apr 2015 20:52:42 GMT\n/BUCKET/GPGHMACSHA1--08b3dee71059819e3558ac9ef8b82ad87e2d8951\" +[2015-04-23 21:52:42 BST] Host: \"BUCKET.s3-ap-southeast-2.amazonaws.com\" +[2015-04-23 21:52:42 BST] Path: \"/GPGHMACSHA1--08b3dee71059819e3558ac9ef8b82ad87e2d8951\" +[2015-04-23 21:52:42 BST] Query string: \"\" +[2015-04-23 21:52:43 BST] Response status: Status {statusCode = 200, statusMessage = \"OK\"} +[2015-04-23 21:52:43 BST] Response header 'x-amz-id-2': 'LRDMgQAj+F81m3UqDebJ5CoZdyM/c2tMaFUvhjn8kjqq3x2Evy7O+wgLUiwE7lqascd0yrHR+xA=' +[2015-04-23 21:52:43 BST] Response header 'x-amz-request-id': '068D946E995E7473' +[2015-04-23 21:52:43 BST] Response header 'Date': 'Thu, 23 Apr 2015 20:52:44 GMT' +[2015-04-23 21:52:43 BST] Response header 'Last-Modified': 'Fri, 31 Oct 2014 07:03:03 GMT' +[2015-04-23 21:52:43 BST] Response header 'ETag': '\"66a85b0007a52d82e5bd29192ebdb510\"' +[2015-04-23 21:52:43 BST] Response header 'Accept-Ranges': 'bytes' +[2015-04-23 21:52:43 BST] Response header 'Content-Type': '' +[2015-04-23 21:52:43 BST] Response header 'Content-Length': '46058' +[2015-04-23 21:52:43 BST] Response header 'Server': 'AmazonS3' +[2015-04-23 21:52:43 BST] Response metadata: S3: request ID=068D946E995E7473, x-amz-id-2=LRDMgQAj+F81m3UqDebJ5CoZdyM/c2tMaFUvhjn8kjqq3x2Evy7O+wgLUiwE7lqascd0yrHR+xA= +99% 22.5KB/s 0s[2015-04-23 21:52:44 BST] chat: gpg [\"--batch\",\"--no-tty\",\"--use-agent\",\"--quiet\",\"--trust-model\",\"always\",\"--batch\",\"--passphrase-fd\",\"14\",\"--decrypt\"] +ok +[2015-04-23 21:52:44 BST] chat: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"hash-object\",\"-w\",\"--stdin-paths\",\"--no-filters\"] +[2015-04-23 21:52:44 BST] feed: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"update-index\",\"-z\",\"--index-info\"] +[2015-04-23 21:52:44 BST] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"show-ref\",\"--hash\",\"refs/heads/git-annex\"] +(recording state in git...) +[2015-04-23 21:52:44 BST] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"write-tree\"] +[2015-04-23 21:52:44 BST] chat: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"commit-tree\",\"444e504a0ab73d01df08ef731e691205cfd485f5\",\"--no-gpg-sign\",\"-p\",\"refs/heads/git-annex\"] +[2015-04-23 21:52:44 BST] call: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"update-ref\",\"refs/heads/git-annex\",\"6e57ed008525cd58641c54a5ac6f07a960a7dc5c\"] +\"\"\"]] +"""]] diff --git a/doc/bugs/Can__39__t_get_content_from_S3_with_s3-aws_library/comment_14_035a933289f95c365ce2c851f6946181._comment b/doc/bugs/Can__39__t_get_content_from_S3_with_s3-aws_library/comment_14_035a933289f95c365ce2c851f6946181._comment new file mode 100644 index 000000000..96d212a39 --- /dev/null +++ b/doc/bugs/Can__39__t_get_content_from_S3_with_s3-aws_library/comment_14_035a933289f95c365ce2c851f6946181._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawnSenxKyE_2Z6Wb-EBMO8FciyRywjx1ZiQ" + nickname="Walter" + subject="comment 14" + date="2015-04-24T20:13:31Z" + content=""" +Playing around with it, I also can't reproduce it (using new or old versions of git-annex; it may be, as you allude, a problem in an old version of the s3 library). + +Anyway, I'm happy that it's working now. +"""]] diff --git a/doc/bugs/Can__39__t_get_content_from_S3_with_s3-aws_library/comment_16_2f16c63e539e51d9d1c0f85605e6d1e8._comment b/doc/bugs/Can__39__t_get_content_from_S3_with_s3-aws_library/comment_16_2f16c63e539e51d9d1c0f85605e6d1e8._comment new file mode 100644 index 000000000..e5141a30b --- /dev/null +++ b/doc/bugs/Can__39__t_get_content_from_S3_with_s3-aws_library/comment_16_2f16c63e539e51d9d1c0f85605e6d1e8._comment @@ -0,0 +1,22 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 16""" + date="2015-04-25T01:18:51Z" + content=""" +Investigating further, when I create a bucket with the AWS library +in ap-southeast-2, `s3cmd info` shows it is located there. + +When I create a bucket with hS3 in ap-southeast-2, I get this +interesting output: + + joey@darkstar:~>s3cmd info s3://s3-43302240-076c-4420-8099-f2ef0b517e5f + s3://s3-43302240-076c-4420-8099-f2ef0b517e5f/ (bucket): + Location: ap-southeast-2 + WARNING: Redirected to: s3-43302240-076c-4420-8099-f2ef0b517e5f.s3-ap-southeast-2.amazonaws.com + Expiration Rule: none + policy: none + ACL: joeyhess: FULL_CONTROL + +So, it's apparently in the datacenter I asked for when making it, +but here's a redirect again. +"""]] diff --git a/doc/bugs/Can__39__t_get_content_from_S3_with_s3-aws_library/comment_1_533c4a26200501486a9ec103e1301391._comment b/doc/bugs/Can__39__t_get_content_from_S3_with_s3-aws_library/comment_1_533c4a26200501486a9ec103e1301391._comment new file mode 100644 index 000000000..8921214a9 --- /dev/null +++ b/doc/bugs/Can__39__t_get_content_from_S3_with_s3-aws_library/comment_1_533c4a26200501486a9ec103e1301391._comment @@ -0,0 +1,18 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2015-04-21T19:23:18Z" + content=""" +So the http 301 redirect seems to be the relevant clue. + +Perhaps the old S3 library followed such redirects and the new aws library +does not. Although AFAICS, http-client should follow redirects by default. + +Are you able to store new files in this remote and get them back out? + +Which Amazon datacenter is the bucket at? + +I've added some more debugging to the S3 remote; --debug will now +have it log the http requests and responses. Perhaps this will give us +more clues to the problem. +"""]] diff --git a/doc/bugs/Can__39__t_get_content_from_S3_with_s3-aws_library/comment_2_24224679a1516e2852b43624c787f639._comment b/doc/bugs/Can__39__t_get_content_from_S3_with_s3-aws_library/comment_2_24224679a1516e2852b43624c787f639._comment new file mode 100644 index 000000000..2bafdf7ea --- /dev/null +++ b/doc/bugs/Can__39__t_get_content_from_S3_with_s3-aws_library/comment_2_24224679a1516e2852b43624c787f639._comment @@ -0,0 +1,63 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawnSenxKyE_2Z6Wb-EBMO8FciyRywjx1ZiQ" + nickname="Walter" + subject="comment 2" + date="2015-04-21T22:53:21Z" + content=""" +It's in bucket ap-southeast-2, which conflicts with the info in the log below. I'm not sure why they disagree. + +This was a new file, which I also \"uploaded\" with a ``git annex copy --to cloud test_file``, and ``git annex whereis`` says is in cloud. However, using ``s3cmd`` to retrieve it (using the info from the log below), it claims it doesn't exist (404). So, I don't think it got uploaded correctly. I don't seem to get any useful logs when forcing an upload with ``--debug`` (as in no S3-related logs, but I've included that at the very bottom). + +[[!format sh \"\"\" +> git annex get test_file +get test_file (from cloud...) + + Unable to access these remotes: cloud + + Try making some of these repositories available: + be992080-b1db-11e1-8f79-1b10bb4092ef -- [cloud] + + (Note that these git remotes have annex-ignore set: origin) +failed +git-annex: get: 1 failed +walter@kronos:~/NewPics$ git annex get test_file --debug +[2015-04-21 23:39:57 BST] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"ls-files\",\"--cached\",\"-z\",\"--\",\"test_file\"] +get test_file [2015-04-21 23:39:57 BST] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"show-ref\",\"git-annex\"] +[2015-04-21 23:39:57 BST] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"show-ref\",\"--hash\",\"refs/heads/git-annex\"] +[2015-04-21 23:39:57 BST] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"log\",\"refs/heads/git-annex..347f154f5dfa9e41dc459eda328421741e1e90a6\",\"-n1\",\"--pretty=%H\"] +[2015-04-21 23:39:57 BST] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"log\",\"refs/heads/git-annex..83793925a571a3228cc64e204598f8c54203b1f7\",\"-n1\",\"--pretty=%H\"] +[2015-04-21 23:39:57 BST] chat: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"cat-file\",\"--batch\"] +(from cloud...) [2015-04-21 23:39:57 BST] chat: gpg [\"--batch\",\"--no-tty\",\"--use-agent\",\"--quiet\",\"--trust-model\",\"always\",\"--decrypt\"] + +[2015-04-21 23:39:57 BST] String to sign: \"GET\n\n\nTue, 21 Apr 2015 22:39:57 GMT\n/BUCKETID/GPGHMACSHA1--417830f4c50a2887674917abd2c18c522853255a\" +[2015-04-21 23:39:57 BST] Host: \"BUCKETID.s3-ap-southeast-1.amazonaws.com\" +[2015-04-21 23:39:57 BST] Path: \"/GPGHMACSHA1--417830f4c50a2887674917abd2c18c522853255a\" +[2015-04-21 23:39:57 BST] Query string: \"\" +[2015-04-21 23:39:57 BST] Response status: Status {statusCode = 301, statusMessage = \"Moved Permanently\"} +[2015-04-21 23:39:57 BST] Response header 'x-amz-request-id': 'C2825FBB20ED22B4' +[2015-04-21 23:39:57 BST] Response header 'x-amz-id-2': 'I93feDTHOrPR+bwVqoMBuEEwYQAN7ZfjOq0jdIJ6ywzOPYYxTfqZg9OR+M0L+MFdilHKRJ+CEv8=' +[2015-04-21 23:39:57 BST] Response header 'Content-Type': 'application/xml' +[2015-04-21 23:39:57 BST] Response header 'Transfer-Encoding': 'chunked' +[2015-04-21 23:39:57 BST] Response header 'Date': 'Tue, 21 Apr 2015 22:39:56 GMT' +[2015-04-21 23:39:57 BST] Response header 'Server': 'AmazonS3' +[2015-04-21 23:39:57 BST] Response metadata: S3: request ID=C2825FBB20ED22B4, x-amz-id-2=I93feDTHOrPR+bwVqoMBuEEwYQAN7ZfjOq0jdIJ6ywzOPYYxTfqZg9OR+M0L+MFdilHKRJ+CEv8= + + Unable to access these remotes: cloud + + Try making some of these repositories available: + be992080-b1db-11e1-8f79-1b10bb4092ef -- [cloud] + + (Note that these git remotes have annex-ignore set: origin) +failed +git-annex: get: 1 failed + +> git annex copy --to cloud --force --debug test_file +[2015-04-21 23:47:24 BST] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"show-ref\",\"git-annex\"] +[2015-04-21 23:47:24 BST] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"show-ref\",\"--hash\",\"refs/heads/git-annex\"] +[2015-04-21 23:47:24 BST] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"log\",\"refs/heads/git-annex..347f154f5dfa9e41dc459eda328421741e1e90a6\",\"-n1\",\"--pretty=%H\"] +[2015-04-21 23:47:24 BST] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"log\",\"refs/heads/git-annex..83793925a571a3228cc64e204598f8c54203b1f7\",\"-n1\",\"--pretty=%H\"] +[2015-04-21 23:47:24 BST] chat: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"cat-file\",\"--batch\"] +[2015-04-21 23:47:24 BST] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"ls-files\",\"--cached\",\"-z\",\"--\",\"test_file\"] + +\"\"\"]] +"""]] diff --git a/doc/bugs/Can__39__t_get_content_from_S3_with_s3-aws_library/comment_3_9665a868f786afa445f5e3b0fbd20011._comment b/doc/bugs/Can__39__t_get_content_from_S3_with_s3-aws_library/comment_3_9665a868f786afa445f5e3b0fbd20011._comment new file mode 100644 index 000000000..8b92ca305 --- /dev/null +++ b/doc/bugs/Can__39__t_get_content_from_S3_with_s3-aws_library/comment_3_9665a868f786afa445f5e3b0fbd20011._comment @@ -0,0 +1,29 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 3""" + date="2015-04-23T17:00:20Z" + content=""" +ap-southeast-2 vs ap-southeast-1 must have something to do with this +problem. + +I tried making a S3 remote with datacenter=ap-southeast-2 and it didn't +have this problem, and used s3-ap-southeast-2.amazonaws.com, so doesn't +seem that git-annex is getting the mapping to endpoints wrong. + +I also tried forcing git-annex to use ap-southeast-1 instead of the right +datacenter, and I don't get a redirect, getting a file just fails in that +misconfiguration. + +So, it seems there is something special about your bucket; git-annex thinks +it's in ap-southeast-1, S3 apparently redirects to somewhere else. Sounds +like the AWS interface shows you the bucket is located in ap-southeast-2? + +This is looking more like a bug with the haskell-AWS library. If AWS can +send redirects here, it ought to follow them. It would be really helpful if +I had a way to reproduce the problem. Since your remote is encrypted +anyway, is there any chance you could generate AWS creds that I could use +to access that bucket to try to get files from it? + +(The `copy --to cloud` didn't do anything because git-annex already thinks +the file is there (--force has no effect on copy --to).) +"""]] diff --git a/doc/bugs/Can__39__t_get_content_from_S3_with_s3-aws_library/comment_4_b70a0075e61565e4501f0b5b143b222d._comment b/doc/bugs/Can__39__t_get_content_from_S3_with_s3-aws_library/comment_4_b70a0075e61565e4501f0b5b143b222d._comment new file mode 100644 index 000000000..2f862c560 --- /dev/null +++ b/doc/bugs/Can__39__t_get_content_from_S3_with_s3-aws_library/comment_4_b70a0075e61565e4501f0b5b143b222d._comment @@ -0,0 +1,9 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 4""" + date="2015-04-23T18:17:23Z" + content=""" +I agree, enableremote should not create the bucket. Made that change. Also, +`git annex info $s3remote` will show some more info about it, including its +endpoint. +"""]] diff --git a/doc/bugs/Can__39__t_get_content_from_S3_with_s3-aws_library/comment_4_d37d9b008e2e8f570af86e620ad6064f._comment b/doc/bugs/Can__39__t_get_content_from_S3_with_s3-aws_library/comment_4_d37d9b008e2e8f570af86e620ad6064f._comment new file mode 100644 index 000000000..b79f5812d --- /dev/null +++ b/doc/bugs/Can__39__t_get_content_from_S3_with_s3-aws_library/comment_4_d37d9b008e2e8f570af86e620ad6064f._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawnSenxKyE_2Z6Wb-EBMO8FciyRywjx1ZiQ" + nickname="Walter" + subject="comment 4" + date="2015-04-23T17:58:14Z" + content=""" +I've sent an email regarding this. + +One comment, it seems that it's not possible to change the configuration of an S3 remote, as ``enableremote`` causes git-annex to try to create a new bucket (which fails, as the name is already used). Otherwise, I could try to tell git-annex it has the region wrong. Perhaps an option ``use-existing-bucket`` or something would work? +"""]] diff --git a/doc/bugs/Can__39__t_get_content_from_S3_with_s3-aws_library/comment_6_7caef30897eabf04faab12f4b4a16916._comment b/doc/bugs/Can__39__t_get_content_from_S3_with_s3-aws_library/comment_6_7caef30897eabf04faab12f4b4a16916._comment new file mode 100644 index 000000000..fd55b5134 --- /dev/null +++ b/doc/bugs/Can__39__t_get_content_from_S3_with_s3-aws_library/comment_6_7caef30897eabf04faab12f4b4a16916._comment @@ -0,0 +1,29 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 6""" + date="2015-04-23T18:44:02Z" + content=""" +I reproduced the problem using the creds, and a modified version of +GetObject.hs from haskell-aws. + +I had to upgrade from aws-0.9.2 to 0.11.4 to see the 301 redirect in the +debug mesages, but both versions seemed to fail the same way, it's just the +newer version added more debugging output. + +There's an exception, which git-annex obscured: + + Response metadata: S3: request ID=5B5B9AE39E0C9729, x-amz-id-2=t6xToNbirPnwgEhTdbFr+Ncq5cGr3fMCRNq5WlFLjEk3ZJtan5aCotcsCS3GMTMgjsP/MNcOWUw= + GetObject: HeaderException {headerErrorMessage = "ETag missing"} + +I've seen this "ETag missing" before when an object was genuinely missing, but +I'm not sure what it indicates in this case. + +This was using s3-ap-southeast-1.amazonaws.com. If I use southeast-2, it +just fails with "NoSuchKey". So I think that the 1-vs-2 was a red herring; +git-annex is using the right endpoint. + +I have forwarded this bug report to: <https://github.com/aristidb/aws/issues/160> + +It might be good to get in touch with the haskell-aws maintainer and provide the +creds so they can reproduce it too. +"""]] diff --git a/doc/bugs/Can__39__t_get_content_from_S3_with_s3-aws_library/comment_7_cd943597e319c94e91b17f24346be456._comment b/doc/bugs/Can__39__t_get_content_from_S3_with_s3-aws_library/comment_7_cd943597e319c94e91b17f24346be456._comment new file mode 100644 index 000000000..171b2b371 --- /dev/null +++ b/doc/bugs/Can__39__t_get_content_from_S3_with_s3-aws_library/comment_7_cd943597e319c94e91b17f24346be456._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 7""" + date="2015-04-23T18:59:10Z" + content=""" +Hrm, I get an identical error message if I try to request a file that +is certianly not there! + +So maybe the 301 is a red herring? +"""]] diff --git a/doc/bugs/Can__39__t_get_content_from_S3_with_s3-aws_library/comment_8_13f862524d4aa503fc998ede41617942._comment b/doc/bugs/Can__39__t_get_content_from_S3_with_s3-aws_library/comment_8_13f862524d4aa503fc998ede41617942._comment new file mode 100644 index 000000000..531e8c79d --- /dev/null +++ b/doc/bugs/Can__39__t_get_content_from_S3_with_s3-aws_library/comment_8_13f862524d4aa503fc998ede41617942._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawnSenxKyE_2Z6Wb-EBMO8FciyRywjx1ZiQ" + nickname="Walter" + subject="comment 8" + date="2015-04-23T19:02:18Z" + content=""" +Ok, so I did ``git annex enableremote cloud datacenter=ap-southeast-2``, and can now get files properly. So from that point of view it now works. And I guess that provides an easy way to reproduce (just set a working S3 remote to the wrong datacenter). I'm prepared to accept that this was something I did somehow (at some point I manually moved files from one S3 bucket (actually account) to another, but it seems that git-annex would have created the bucket, so I'm not sure how the datacenter could be wrong.) + +In any case, now I'm not sure exactly which files did get uploaded properly, so will run a ``fsck``. I guess it would be good to either return an error when this happens, or follow the redirect. + +Also, I really appreciate the quick response you have to bugs! +"""]] diff --git a/doc/bugs/Can__39__t_get_content_from_S3_with_s3-aws_library/comment_8_44915e2b9e871c16861223e1e2a0827b._comment b/doc/bugs/Can__39__t_get_content_from_S3_with_s3-aws_library/comment_8_44915e2b9e871c16861223e1e2a0827b._comment new file mode 100644 index 000000000..dd6f2f7a6 --- /dev/null +++ b/doc/bugs/Can__39__t_get_content_from_S3_with_s3-aws_library/comment_8_44915e2b9e871c16861223e1e2a0827b._comment @@ -0,0 +1,13 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 8""" + date="2015-04-23T19:06:02Z" + content=""" +Glad that resolved it, but I'm still confused why I can't seem to get the +test file when I have my test program use ap-southeast-2. (But maybe those +creds you gave me just don't let me or something..) + +If you're up for some sleuthing, you could check out the git-annex branch +and look through `git blame remotes.log`. And changes that git-annex ever +made to the config of the remote will be in the history of that file. +"""]] diff --git a/doc/bugs/Can__39__t_get_content_from_S3_with_s3-aws_library/comment_9_769de1e47221dfb6c810665e3704bbb2._comment b/doc/bugs/Can__39__t_get_content_from_S3_with_s3-aws_library/comment_9_769de1e47221dfb6c810665e3704bbb2._comment new file mode 100644 index 000000000..788ccf7ed --- /dev/null +++ b/doc/bugs/Can__39__t_get_content_from_S3_with_s3-aws_library/comment_9_769de1e47221dfb6c810665e3704bbb2._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawnSenxKyE_2Z6Wb-EBMO8FciyRywjx1ZiQ" + nickname="Walter" + subject="comment 9" + date="2015-04-23T19:07:56Z" + content=""" +Is it possible to do a fast ``fsck`` on an S3 remote? Because I don't want to download all the files again, it would be nice to just have the option to check it it exists. + +I get a ``failed to download file from remote`` error when I try it. +"""]] diff --git a/doc/bugs/Data_loss_when_doing___96__git_annex_import_--force__96__.mdwn b/doc/bugs/Data_loss_when_doing___96__git_annex_import_--force__96__.mdwn new file mode 100644 index 000000000..60e04fc14 --- /dev/null +++ b/doc/bugs/Data_loss_when_doing___96__git_annex_import_--force__96__.mdwn @@ -0,0 +1,62 @@ +### Please describe the problem. + +Calling `git annex import --force file-in-working-copy` removes `file-in-working-copy` from disk. + +My workflow is: + +1) copy the CR2 files from a card to the desired directory structure using a tool of my choice, +2) import the created directory layout to git-annex + +### What version of git-annex are you using? On what operating system? + +[[!format sh """ +$ git-annex version +git-annex version: 5.20150327 +build flags: Pairing Testsuite S3 DBus DNS Feeds Quvi TDFA +key/value backends: SHA256E SHA1E SHA512E SHA224E SHA384E SKEIN256E SKEIN512E MD5E SHA256 SHA1 SHA512 SHA224 SHA384 SKEIN256 SKEIN512 MD5 WORM URL +remote types: git gcrypt S3 bup directory rsync web bittorrent tahoe glacier ddar hook external +local repository version: 5 +supported repository version: 5 +upgrade supported from repository versions: 0 1 2 4 +"""]] + +### How to reproduce: + +[[!format sh """ +jkt@svist ~/temp $ mkdir annex-add-force-data-loss +jkt@svist ~/temp $ cd annex-add-force-data-loss/ +jkt@svist ~/temp/annex-add-force-data-loss $ git init +Initialized empty Git repository in /home/jkt/temp/annex-add-force-data-loss/.git/ +jkt@svist ~/temp/annex-add-force-data-loss $ echo 1234 > foo +jkt@svist ~/temp/annex-add-force-data-loss $ ls -al +total 16K +drwxr-xr-x 3 jkt jkt 27 May 8 13:54 . +drwx------ 55 jkt jkt 8.0K May 8 13:54 .. +drwxr-xr-x 6 jkt jkt 96 May 8 13:54 .git +-rw-r--r-- 1 jkt jkt 5 May 8 13:54 foo +jkt@svist ~/temp/annex-add-force-data-loss $ git annex import --force foo +git-annex: First run: git-annex init +jkt@svist ~/temp/annex-add-force-data-loss $ git annex init +init ok +(recording state in git...) +jkt@svist ~/temp/annex-add-force-data-loss $ ls -al +total 16K +drwxr-xr-x 3 jkt jkt 27 May 8 13:54 . +drwx------ 55 jkt jkt 8.0K May 8 13:54 .. +drwxr-xr-x 8 jkt jkt 119 May 8 13:54 .git +-rw-r--r-- 1 jkt jkt 5 May 8 13:54 foo +jkt@svist ~/temp/annex-add-force-data-loss $ git annex import --force foo +import foo +git-annex: foo: rename: does not exist (No such file or directory) +failed +git-annex: import: 1 failed +jkt@svist ~/temp/annex-add-force-data-loss $ ls -al +total 12K +drwxr-xr-x 3 jkt jkt 17 May 8 13:55 . +drwx------ 55 jkt jkt 8.0K May 8 13:54 .. +drwxr-xr-x 8 jkt jkt 119 May 8 13:55 .git +"""]] +...and the file is gone :(. + +> You should use `git annex add` in this case, not import. +> I've made import refuse to run in this case. [[done]] --[[Joey]] diff --git a/doc/bugs/Git-Annex_requires_all_repositories_to_repair.mdwn b/doc/bugs/Git-Annex_requires_all_repositories_to_repair.mdwn index 7087247f5..eef705e69 100644 --- a/doc/bugs/Git-Annex_requires_all_repositories_to_repair.mdwn +++ b/doc/bugs/Git-Annex_requires_all_repositories_to_repair.mdwn @@ -1,3 +1,6 @@ I recently had my git-annex repository die and it needed to be repaired. Two of my repositories are external hard drives. When I tried to use git-annex repair, it would churn for some hours, then error because the external hard drives were not plugged in. When I brought the two hard drives home from the various places that they are (safely) stored, it all worked fine, but it would have been great if git-annex repair could somehow do what it could with what was connected and do the rest as and when the other drives are plugged in. This must only become more of a problem as git-annex is used for longer, as one may have a handful of USB keys storing a little on each. [[!taglink moreinfo]] + +> With the lack of an error message or any followup, it's hard to take +> this bug seriously, so [[done]] --[[Joey]] diff --git a/doc/bugs/Proxy_support.mdwn b/doc/bugs/Proxy_support.mdwn index ae6eaf689..2af63d706 100644 --- a/doc/bugs/Proxy_support.mdwn +++ b/doc/bugs/Proxy_support.mdwn @@ -17,3 +17,26 @@ Please provide any additional information below. I don't use networkmanager if proxy information is obtained from it. There should be a fallback to environment variables. [[!tag confirmed]] + +> Here's a patch that shows how to enable proxy support for the +> parts of git-annex that use http-client and http-conduit: +> <https://github.com/kiwiroy/git-annex/commit/18a3ceda1beb7c356541270f37ae4c0d56ada726> +> +> Other parts of git-annex use wget/curl and should already support +> the environment variables. + +>> I had a closer look at this, and it turns out that http-client +>> now enables http proxy support by default, when the environment variable +>> is set. Since version 0.4.7. See `defaultProxy`. +>> +>> git-annex uses http-client for all its http access (either directly +>> or through layers like http-conduit and DAV). +>> +>> So I don't need to change +>> git-annex at all; it just needs to be built with a new enough version +>> of http-client. This will happen for the linux autobuilders once the new +>> version reaches Debian. I've updated the OSX autobuilder's http-client +>> so it will already support proxying and anyone can built git-annex +>> using the new http-client and get proxy support. +>> +>> Calling this [[done]] as nothing remains to do in git-annex. --[[Joey]] diff --git a/doc/bugs/Repository_Information_Is_Lost.mdwn b/doc/bugs/Repository_Information_Is_Lost.mdwn index 7b11ac4cf..cbaf866c8 100644 --- a/doc/bugs/Repository_Information_Is_Lost.mdwn +++ b/doc/bugs/Repository_Information_Is_Lost.mdwn @@ -31,3 +31,9 @@ Clones of my repositories lost all track of other repositories they only seem to """]] [[!tag moreinfo]] + +> No followup for over a year, and not enough information in the intial +> report to know if there's a bug at all. It occurs to me that the user +> might have just forgotten to push the git-annex branch to wherever they +> later cloned the repo from. `git annex sync` would fix that mistake up. +> Anyway, [[done]]. --[[Joey]] diff --git a/doc/bugs/Small_archive_behaving_like_archive.mdwn b/doc/bugs/Small_archive_behaving_like_archive.mdwn index 384a75baf..947777279 100644 --- a/doc/bugs/Small_archive_behaving_like_archive.mdwn +++ b/doc/bugs/Small_archive_behaving_like_archive.mdwn @@ -31,3 +31,6 @@ Mac OSX 10.8.3 (Build 12D78) """]] [[!tag moreinfo]] + +> Since there's a plausible explanation in my comment and no followup, +> [[done]] --[[Joey]] diff --git a/doc/bugs/__34__transfers_in_progress__34___gets_the_direction_wrong.mdwn b/doc/bugs/__34__transfers_in_progress__34___gets_the_direction_wrong.mdwn new file mode 100644 index 000000000..e0b02a704 --- /dev/null +++ b/doc/bugs/__34__transfers_in_progress__34___gets_the_direction_wrong.mdwn @@ -0,0 +1,55 @@ +### Please describe the problem. + + 07:32 warp@18-volt:/mnt/zooyork/annex/somerepo (master)$ git annex info --fast + repository mode: indirect + trusted repositories: 0 + semitrusted repositories: 4 + 00000000-0000-0000-0000-000000000001 -- web + 3dca408f-ccfd-48df-a8a0-b7d517a482ef -- zooyork [here] + 9fbb4e2c-4ed9-47b4-877c-74d2eeea6296 -- [kirby] + d1b57403-2bc2-41d8-9a8e-eec0d3f31e67 -- [pucca] + untrusted repositories: 0 + transfers in progress: + downloading folder/some-large-file.mp4 from pucca + available local disk space: 232.2 gigabytes (+1 megabyte reserved) + +The "transfers in progress" section of the above "git annex info" output describes a file being downloaded from pucca, presumably to [here], which is zooyork. This does not describe the transfer correctly, when I ran the "git annex info --fast" command another git annex process was downloading folder/some-large-file.mp4 from zooyork to pucca (so, in the other direction than git annex info described). + +### What steps will reproduce the problem? + +1. Make two regular git annex repositories, add some big files so a transfer takes long enough that you can run some git annex commands while the transfer is ongoing +2. cd /repo/a ; git annex get a-big-file.mp4 --from=b +3. (in a different window), cd /repo/b ; git annex info --fast + +### What version of git-annex are you using? On what operating system? + + 07:37 warp@18-volt:/mnt/zooyork/annex$ git annex version + git-annex version: 5.20141105-g8b19598 + build flags: Assistant Webapp Webapp-secure Pairing Testsuite S3 WebDAV Inotify DBus DesktopNotify XMPP DNS Feeds Quvi TDFA CryptoHash + key/value backends: SHA256E SHA1E SHA512E SHA224E SHA384E SKEIN256E SKEIN512E SHA256 SHA1 SHA512 SHA224 SHA384 SKEIN256 SKEIN512 WORM URL + remote types: git gcrypt S3 bup directory rsync web webdav tahoe glacier ddar hook external + + 07:37 warp@18-volt:/mnt/zooyork/annex$ lsb_release -a + No LSB modules are available. + Distributor ID: Ubuntu + Description: Ubuntu 14.04 LTS + Release: 14.04 + Codename: trusty + + +### Please provide any additional information below. + +If you need more info, I'm "kuno" in the #git-annex IRC channel. + +[[!format sh """ +# If you can, paste a complete transcript of the problem occurring here. +# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log + + +# End of transcript or log. +"""]] + +> When I follow those steps, I see "uploading bigfile" in repo B, +> which is indeed the case since repo A is downloading that file from B. +> +> I suspect you're the one who got confused. [[done]] --[[Joey]] diff --git a/doc/bugs/_git-annex:_bad_url_ssh:__47____47__git__64__gitlab.com:__126____47__gitlabname__47__reponame.git.mdwn b/doc/bugs/_git-annex:_bad_url_ssh:__47____47__git__64__gitlab.com:__126____47__gitlabname__47__reponame.git.mdwn new file mode 100644 index 000000000..74e9235dd --- /dev/null +++ b/doc/bugs/_git-annex:_bad_url_ssh:__47____47__git__64__gitlab.com:__126____47__gitlabname__47__reponame.git.mdwn @@ -0,0 +1,33 @@ +### Please describe the problem. +An encrypted remote is added to a working git annex repository (mind ":~/" in the remote add command). However, after that any git annex command fails. + +### What steps will reproduce the problem? + > git remote add encrypted gcrypt::ssh://git@gitlab.com:~/gitlabname/reponame.git + > git push encrypted master + gcrypt: Repository not found: ssh://git@gitlab.com:~/gitlabname/reponame.git + gcrypt: Setting up new repository + gcrypt: Remote ID is :id:abcdefghijklmnopqrst + Counting objects: 53, done. + Compressing objects: 100% (52/52), done. + Total 53 (delta 12), reused 0 (delta 0) + gcrypt: Encrypting to: --throw-keyids --default-recipient-self + gcrypt: Requesting manifest signature + ... + To gcrypt::ssh://git@gitlab.com:~/gitlabname/reponame.git + * [new branch] master -> master + > + > git annex sync + git-annex: bad url ssh://git@gitlab.com:~/gitlabname/reponame.git + +### What version of git-annex are you using? On what operating system? +5.20150419-g900e1b6 on Mac OS X + +### Please provide any additional information below. + +[[!format sh """ +# If you can, paste a complete transcript of the problem occurring here. +# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log +- + +# End of transcript or log. +"""]] diff --git a/doc/bugs/_git-annex:_bad_url_ssh:__47____47__git__64__gitlab.com:__126____47__gitlabname__47__reponame.git/comment_1_60db0d6bfc71a62b3c1021527a8d2d60._comment b/doc/bugs/_git-annex:_bad_url_ssh:__47____47__git__64__gitlab.com:__126____47__gitlabname__47__reponame.git/comment_1_60db0d6bfc71a62b3c1021527a8d2d60._comment new file mode 100644 index 000000000..4b7b0e90a --- /dev/null +++ b/doc/bugs/_git-annex:_bad_url_ssh:__47____47__git__64__gitlab.com:__126____47__gitlabname__47__reponame.git/comment_1_60db0d6bfc71a62b3c1021527a8d2d60._comment @@ -0,0 +1,23 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2015-04-29T15:16:31Z" + content=""" +According to the git-fetch man page, the syntax to use +for this kind of url is: + + ssh://[user@]host.xz[:port]/~[user]/path/to/repo.git/ + +Your url is missing the leading slash before the `~`, and has +a : with no port specified. + + ssh://git@gitlab.com:~/gitlabname/reponame.git + +It is in fact, not a legal url. + +Now, git might accept it despite not documenting it as an accepted form, +but why wander into undefined territory when there are legal ways to write +this url that work fine? + +Does GitLab promote using these malformed urls? +"""]] diff --git a/doc/bugs/_git-annex:_bad_url_ssh:__47____47__git__64__gitlab.com:__126____47__gitlabname__47__reponame.git/comment_2_49682a91147990a578929678880bd226._comment b/doc/bugs/_git-annex:_bad_url_ssh:__47____47__git__64__gitlab.com:__126____47__gitlabname__47__reponame.git/comment_2_49682a91147990a578929678880bd226._comment new file mode 100644 index 000000000..e93708644 --- /dev/null +++ b/doc/bugs/_git-annex:_bad_url_ssh:__47____47__git__64__gitlab.com:__126____47__gitlabname__47__reponame.git/comment_2_49682a91147990a578929678880bd226._comment @@ -0,0 +1,26 @@ +[[!comment format=mdwn + username="http://rfhbuk.pip.verisignlabs.com.pip.verisignlabs.com/" + nickname="rfhb" + subject="comment 2" + date="2015-04-29T15:45:34Z" + content=""" +Thank you. I was wondering about the URL as flagged above. When I specify the URL as I did, the git commands work but when I change it to ssh://git@gitlab.com:/~/gitlabname/reponame.git, git command fails with + + > git push encrypted master + gcrypt: Development version -- Repository format MAY CHANGE + gcrypt: Repository not found: ssh://git@gitlab.com:/~/gitlabname/reponame.git + gcrypt: Setting up new repository + gcrypt: Remote ID is :id:xxxxxxxxxxxxxxxxx + Counting objects: 3, done. + Total 3 (delta 0), reused 0 (delta 0) + gcrypt: Encrypting to: --throw-keyids --default-recipient-self + gcrypt: Requesting manifest signature + .... passphase entered ... + GitLab: No such project + fatal: Could not read from remote repository. + Please make sure you have the correct access rights + and the repository exists. + error: failed to push some refs to 'gcrypt::ssh://git@gitlab.com:/~/gitlabname/reponame.git' + +So perhaps I need to investigate how to get gcrypt to work with remote git(lab) repositories. Thanks. +"""]] diff --git a/doc/bugs/_git-annex:_bad_url_ssh:__47____47__git__64__gitlab.com:__126____47__gitlabname__47__reponame.git/comment_3_bf0fa06596338444a3772d1865b17e26._comment b/doc/bugs/_git-annex:_bad_url_ssh:__47____47__git__64__gitlab.com:__126____47__gitlabname__47__reponame.git/comment_3_bf0fa06596338444a3772d1865b17e26._comment new file mode 100644 index 000000000..0d7fb9c2d --- /dev/null +++ b/doc/bugs/_git-annex:_bad_url_ssh:__47____47__git__64__gitlab.com:__126____47__gitlabname__47__reponame.git/comment_3_bf0fa06596338444a3772d1865b17e26._comment @@ -0,0 +1,7 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 3""" + date="2015-04-29T17:09:07Z" + content=""" +This works: ssh://git@gitlab.com/username/reponame.git +"""]] diff --git a/doc/bugs/addurl_+_sync_vs_addurl_+_commit.mdwn b/doc/bugs/addurl_+_sync_vs_addurl_+_commit.mdwn new file mode 100644 index 000000000..21aa96952 --- /dev/null +++ b/doc/bugs/addurl_+_sync_vs_addurl_+_commit.mdwn @@ -0,0 +1,27 @@ +### Please describe the problem. + +I think this is what happened; I need to go back and check this again (maybe I was just misreading something) but I want to get it written down first. + +I've got a git repository, and I just ran git annex init. Then I ran git annex addurl a bunch of times, followed by git annex sync. The result was apparently a repository where the files downloaded by addurl were added using the SHA256 backend rather than the URL backend. I deleted the branches and tried again, but this time after calling git annex addurl a bunch of times I did a normal git commit. This time everything looked fine; the files were all listed in as present in the web remote. + +### What steps will reproduce the problem? + +[[!format sh """ +git annex init +git annex addurl "https://archive.org/download/emularity_engine_jsmess/messnapple2e.js.gz" --file "messnapple2e.js.gz" +git annex sync +"""]] + +### What version of git-annex are you using? On what operating system? + + git-annex version: 5.20150412-g2be4834 + build flags: Assistant Webapp Webapp-secure Pairing Testsuite S3 WebDAV Inotify DBus DesktopNotify XMPP DNS Feeds Quvi TDFA TorrentParser + key/value backends: SHA256E SHA1E SHA512E SHA224E SHA384E SKEIN256E SKEIN512E MD5E SHA256 SHA1 SHA512 SHA224 SHA384 SKEIN256 SKEIN512 MD5 WORM URL + remote types: git gcrypt S3 bup directory rsync web bittorrent webdav tahoe glacier ddar hook external + local repository version: 5 + supported repository version: 5 + upgrade supported from repository versions: 0 1 2 4 +This is on Linux. + +> In the lack of any followups, I'm confident this was a case of user +> error, so closing. [[done]] --[[Joey]] diff --git a/doc/bugs/addurl_+_sync_vs_addurl_+_commit/comment_1_513d89ddd728dc9c6cc9fbf8af18afb0._comment b/doc/bugs/addurl_+_sync_vs_addurl_+_commit/comment_1_513d89ddd728dc9c6cc9fbf8af18afb0._comment new file mode 100644 index 000000000..48ce2354d --- /dev/null +++ b/doc/bugs/addurl_+_sync_vs_addurl_+_commit/comment_1_513d89ddd728dc9c6cc9fbf8af18afb0._comment @@ -0,0 +1,17 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2015-04-18T18:44:16Z" + content=""" +If you use `git annex addurl`, the file will use the SHA256 backend. + +If you use `git annex addurl --fast`, it will use the URL backend. + +**Either way**, git-annex will know that you added the file from the web, +and `git-annex whereis` will tell you the file is in the web remote, +and tell you its url. + +I don't think there's a bug here. I think you got confused by +seeing URL some of the time and SHA256 some of the time. +If you disagree, please post a transcript demonstrating your problem. +"""]] diff --git a/doc/bugs/clean-duplicates_causes_data_loss.mdwn b/doc/bugs/clean-duplicates_causes_data_loss.mdwn new file mode 100644 index 000000000..c5d545420 --- /dev/null +++ b/doc/bugs/clean-duplicates_causes_data_loss.mdwn @@ -0,0 +1,30 @@ +### Please describe the problem. + +Use of git-annex import --clean-duplicates can cause data loss, where git-annex deletes content that it doesn't actually have a copy of (i.e. there is no duplicate). + +### What steps will reproduce the problem? + +I've written a quick'n'dirty test script which goes through a bunch of combinations and tests --clean-duplicates. Given an 'origin' repo containing 'a' and 'b' content and a clone of it ('import') which doesn't contain 'a' and 'b' content. + +g-a import --clean-duplicates ~/tmp/importme (containing a, b and c) into 'import' after: + + Origin is set to trusted in import, b is dropped from within origin: + b is deleted from importme even though no annexes have copies (reasonable, as origin is set to trusted and import thinks it has the content). + + Origin is set to semitrusted in import, b is dropped within origin: + b is deleted from importme even though no annexes have copies (this is most likely one to bite people). + + Origin is set to untrusted in import, b is dropped within origin: + b is deleted from importme even though no annexes have copies and git-annex has been explicitly told to not trust information about origin :( This is really surprising behaviour! + +### What version of git-annex are you using? On what operating system? + +* 5.20150409 +* Arch Linux (git-annex-bin) + +### Please provide any additional information below. + +I can provide the script if it is wanted (coded in Perl, couple of non-core dependencies). + +> Decided to go ahead and make it check remotes like drop does, so [[done]] +> --[[Joey]] diff --git a/doc/bugs/clean-duplicates_causes_data_loss/comment_1_0c5da8b1285d16967a0423a0e259e06b._comment b/doc/bugs/clean-duplicates_causes_data_loss/comment_1_0c5da8b1285d16967a0423a0e259e06b._comment new file mode 100644 index 000000000..689cef97a --- /dev/null +++ b/doc/bugs/clean-duplicates_causes_data_loss/comment_1_0c5da8b1285d16967a0423a0e259e06b._comment @@ -0,0 +1,36 @@ +[[!comment format=mdwn + username="CandyAngel" + subject="comment 1" + date="2015-04-24T11:57:35Z" + content=""" +Command to exemplify the \"worst case\" (untrusted causing deletion): + + mkdir /tmp/ga-icd + cd /tmp/ga-icd + + git init origin + cd origin + git commit -m create + git annex init origin + echo a > a + echo b > b + git annex add . + git commit -m files + + mkdir /tmp/ga-icd/importme + echo a > a + echo b > b + echo c > c + + cd /tmp/ga-icd + git clone origin import + git annex init import + +So we now have origin (with content for 2 files), import which knows origin has content for both files and directory we want to clean up. The following causes 'b' to be lost (hope you have backups!). + + cd /tmp/ga-icd/origin + git annex drop b --force + cd /tmp/ga-icd/import + git annex untrust origin + git annex import --clean-duplicates /tmp/ga-icd/importme +"""]] diff --git a/doc/bugs/clean-duplicates_causes_data_loss/comment_2_14c5c3417f107a3619a4402f75bc0002._comment b/doc/bugs/clean-duplicates_causes_data_loss/comment_2_14c5c3417f107a3619a4402f75bc0002._comment new file mode 100644 index 000000000..c99c7699f --- /dev/null +++ b/doc/bugs/clean-duplicates_causes_data_loss/comment_2_14c5c3417f107a3619a4402f75bc0002._comment @@ -0,0 +1,17 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 2""" + date="2015-04-29T17:18:17Z" + content=""" +--clean-duplicates is not really intended to go do a full verification +that the annexed content is present. I can see how this is surprising; +it would probably be better if it were named "--clean-seen-content" +or something like that. + +But, --deduplicate has the same behavior for such files, and I can't see a +way to rename that more effectively. + +For now I have improved the git-annex-import man page to note that +annexed content that's been removed would make a file be considered a +duplicate. +"""]] diff --git a/doc/bugs/clean-duplicates_causes_data_loss/comment_3_ed3f40b5798d9e770657e9c6b8e28039._comment b/doc/bugs/clean-duplicates_causes_data_loss/comment_3_ed3f40b5798d9e770657e9c6b8e28039._comment new file mode 100644 index 000000000..f4c367d9c --- /dev/null +++ b/doc/bugs/clean-duplicates_causes_data_loss/comment_3_ed3f40b5798d9e770657e9c6b8e28039._comment @@ -0,0 +1,62 @@ +[[!comment format=mdwn + username="CandyAngel" + subject="comment 3" + date="2015-04-29T18:42:38Z" + content=""" +Well, yeah, it is quite surprising as you don't like git-annex messing about with files outside the annex, but one of the commands that does so will cause wanton destruction with barely any checks against data loss. + +Oddly, if the origin is marked as dead instead of untrusted, --clean-duplicates doesn't remove anything from /tmp/ga-icd/importme, even though 'import' still knows about the files and has enough information to delete them. Weird. + + + mkdir /tmp/ga-icd + + cd /tmp/ga-icd + + git init origin + Initialized empty Git repository in /tmp/ga-icd/origin/.git/ + + cd origin + + git commit -m create --allow-empty + [master (root-commit) 6cfdbc1] create + + git annex init origin + init origin ok + (recording state in git...) + + echo a + + echo b + + git annex add . + add a ok + add b ok + (recording state in git...) + + git commit -m files + [master 2c2ed64] files + 2 files changed, 2 insertions(+) + create mode 120000 a + create mode 120000 b + + mkdir /tmp/ga-icd/importme + + cd /tmp/ga-icd/importme + + echo a + + echo b + + echo c + + cd /tmp/ga-icd + + git clone origin import + Cloning into 'import'... + done. + + cd import + + git annex init import + init import (merging origin/git-annex into git-annex...) + ok + (recording state in git...) + + cd /tmp/ga-icd/origin + + git annex drop b --force + drop b ok + (recording state in git...) + + cd /tmp/ga-icd/import + + git annex dead origin + dead origin ok + (recording state in git...) + + git annex import --clean-duplicates /tmp/ga-icd/importme + + ls /tmp/ga-icd/import + a b + + ls /tmp/ga-icd/importme/ + a b c + +I'm think I'm just going to steer clear of it completely (and roll my own) until it is as fussy about preserving data as the rest of git-annex. + +(Also, apologies for the original name, I didn't realise it would cause any problems.) +"""]] diff --git a/doc/bugs/commitBuffer:_invalid_argument___40__invalid_character__41__2.mdwn b/doc/bugs/commitBuffer:_invalid_argument___40__invalid_character__41__2.mdwn index 7a67a7509..1be88f4a1 100644 --- a/doc/bugs/commitBuffer:_invalid_argument___40__invalid_character__41__2.mdwn +++ b/doc/bugs/commitBuffer:_invalid_argument___40__invalid_character__41__2.mdwn @@ -20,3 +20,5 @@ It will fail at entry "DR14: Verschwörungstheorien" ### What version of git-annex are you using? On what operating system? 5.20150327-g19a1a35 (standalone build) on Fedora 21 + +[[!tag moreinfo]] diff --git a/doc/bugs/corrupt_backend_upon_sync__63__.mdwn b/doc/bugs/corrupt_backend_upon_sync__63__.mdwn index b669c22ec..e99dfcc88 100644 --- a/doc/bugs/corrupt_backend_upon_sync__63__.mdwn +++ b/doc/bugs/corrupt_backend_upon_sync__63__.mdwn @@ -74,3 +74,5 @@ $ git annex get --from=laptop # End of transcript or log. """]] + +[[!tag moreinfo]] diff --git a/doc/bugs/dolphin_integration_file_is_broken.mdwn b/doc/bugs/dolphin_integration_file_is_broken.mdwn new file mode 100644 index 000000000..2f2fc1c4e --- /dev/null +++ b/doc/bugs/dolphin_integration_file_is_broken.mdwn @@ -0,0 +1,37 @@ +### Please describe the problem. + +git annex will automatically create the file + +.kde/share/kde4/services/ServiceMenus/git-annex.desktop + +However the actions created do not work because the variable used is %U (file:/// style URL) which git annex does not understand. + +According to http://standards.freedesktop.org/desktop-entry-spec/latest/ar01s06.html + +Also the escaping seems broken. The following line is one that works for me. + + Exec=sh -c 'cd "$(dirname -- "$1")" && git-annex get --notify-start --notify-finish -- "$1"' command_string_is_ignored %f + +or simply + + Exec=git-annex get --notify-start --notify-finish -- %F + +### What steps will reproduce the problem? + + +### What version of git-annex are you using? On what operating system? + +5.20141125 + +### Please provide any additional information below. + +[[!format sh """ +# If you can, paste a complete transcript of the problem occurring here. +# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log + + +# End of transcript or log. +"""]] + +> [[fixed|done]]; confirmed the new version still works on filenames with +> spaces in them. --[[Joey]] diff --git a/doc/bugs/dolphin_integration_file_is_broken/comment_1_65c4fd7547be9c16abadaf5fc9d0b793._comment b/doc/bugs/dolphin_integration_file_is_broken/comment_1_65c4fd7547be9c16abadaf5fc9d0b793._comment new file mode 100644 index 000000000..15077277e --- /dev/null +++ b/doc/bugs/dolphin_integration_file_is_broken/comment_1_65c4fd7547be9c16abadaf5fc9d0b793._comment @@ -0,0 +1,11 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2015-04-29T16:49:22Z" + content=""" +You forgot to say what version of dolphin you are having trouble with. +I've tested it working with version 4.14.2. + +I don't understand what the "command_string_is_ignored" in your example +is supposed to do, or why you'd not want to quote the filename. +"""]] diff --git a/doc/bugs/dolphin_integration_file_is_broken/comment_2_237fdaddaa80c1e4d262d724d9aed82b._comment b/doc/bugs/dolphin_integration_file_is_broken/comment_2_237fdaddaa80c1e4d262d724d9aed82b._comment new file mode 100644 index 000000000..3dcaa7c21 --- /dev/null +++ b/doc/bugs/dolphin_integration_file_is_broken/comment_2_237fdaddaa80c1e4d262d724d9aed82b._comment @@ -0,0 +1,21 @@ +[[!comment format=mdwn + username="silvio" + subject="comment 2" + date="2015-04-29T19:24:55Z" + content=""" +I currently use dolphin Version 14.12.3 but it's been broken for me forever. I always had my own fix but now it keeps getting overwritten by git-annex. + +the current line in my git-annex version is: + +sh -c 'cd \"$(dirname '%U')\" && git-annex get --notify-start --notify-finish -- %U' + +The problem is that %U gives an url style file (file:///some/path) which git-annex doesn't understand. + +Further more the quoting is broken because and i quote: + +\"Field codes (%F,%U) must not be used inside a quoted argument, the result of field code expansion inside a quoted argument is undefined.\" + +Additionally I don't think it's possible to prevent an arbitrary command injection unless you have the Field code as a single argument. + +The command_string_is_ignored is just the string that will be assigned to $0 which is not used. You could use it instead of $1 but didn't like using the command string as an argument. +"""]] diff --git a/doc/bugs/dolphin_integration_file_is_broken/comment_3_89c166c4e44380b8f354f2c6b5e42fad._comment b/doc/bugs/dolphin_integration_file_is_broken/comment_3_89c166c4e44380b8f354f2c6b5e42fad._comment new file mode 100644 index 000000000..fdbc3beb1 --- /dev/null +++ b/doc/bugs/dolphin_integration_file_is_broken/comment_3_89c166c4e44380b8f354f2c6b5e42fad._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 3""" + date="2015-04-29T19:37:42Z" + content=""" +So a much newer version than the one in Debian yet.. + +I tried your command line with the old version in Debian, and it seems to +work ok, so thanks for the improvement! +"""]] diff --git a/doc/bugs/dolphin_integration_file_is_broken/comment_3_f7624258e8c2269f3ccdab683df8bb3d._comment b/doc/bugs/dolphin_integration_file_is_broken/comment_3_f7624258e8c2269f3ccdab683df8bb3d._comment new file mode 100644 index 000000000..a6437e470 --- /dev/null +++ b/doc/bugs/dolphin_integration_file_is_broken/comment_3_f7624258e8c2269f3ccdab683df8bb3d._comment @@ -0,0 +1,7 @@ +[[!comment format=mdwn + username="silvio" + subject="comment 3" + date="2015-04-29T19:39:33Z" + content=""" +Also %U gives a list of urls and basename of a list doesn't give back anything useful. I don't see how this could possibly work for you. +"""]] diff --git a/doc/bugs/dolphin_integration_file_is_broken/comment_4_3e12372915c4c72b05db2a1c28a9daf7._comment b/doc/bugs/dolphin_integration_file_is_broken/comment_4_3e12372915c4c72b05db2a1c28a9daf7._comment new file mode 100644 index 000000000..783fd6cb1 --- /dev/null +++ b/doc/bugs/dolphin_integration_file_is_broken/comment_4_3e12372915c4c72b05db2a1c28a9daf7._comment @@ -0,0 +1,9 @@ +[[!comment format=mdwn + username="silvio" + subject="comment 4" + date="2015-04-29T19:47:21Z" + content=""" +sorry for so many posts :) + +As for not quoting the file name I think it doesn't need to be. It will still be just one argument. Exec doesn't work like sh. The standard doesn't mention anything about single quote quoting either but it seems to work and there is no good alternative. +"""]] diff --git a/doc/bugs/failure_to_find_file_that___34__should__34___exist_in_remote_is_silent.mdwn b/doc/bugs/failure_to_find_file_that___34__should__34___exist_in_remote_is_silent.mdwn index 5718e5011..6197861e1 100644 --- a/doc/bugs/failure_to_find_file_that___34__should__34___exist_in_remote_is_silent.mdwn +++ b/doc/bugs/failure_to_find_file_that___34__should__34___exist_in_remote_is_silent.mdwn @@ -35,3 +35,9 @@ My particular issue has probably existed through a few version upgrades though. # End of transcript or log. """]] + +> Since neither of us can think of a better behavior for `git annex +> get/copy > --from remote` in this case, I've been reduced to documenting +> it better. The docs now mention that --from will cause it to silently +> skip files that are not present in the specified remote. So, [[done]] +> --[[Joey]] diff --git a/doc/bugs/failure_to_return_to_indirect_mode_on_usb.mdwn b/doc/bugs/failure_to_return_to_indirect_mode_on_usb.mdwn index f2a11eea8..a61c67254 100644 --- a/doc/bugs/failure_to_return_to_indirect_mode_on_usb.mdwn +++ b/doc/bugs/failure_to_return_to_indirect_mode_on_usb.mdwn @@ -17,3 +17,7 @@ git-annex: indirect: 1 failed """]] [[!tag moreinfo]] + +> I don't like closing bug reports, but after over a year with no followup, +> and with the original bug report lacking even an error message, closing +> it seems like the best course of action. [[done]] --[[Joey]] diff --git a/doc/bugs/fsck_--from___36__remote__44___copies_symlinks_instead_of_content.mdwn b/doc/bugs/fsck_--from___36__remote__44___copies_symlinks_instead_of_content.mdwn new file mode 100644 index 000000000..732af2990 --- /dev/null +++ b/doc/bugs/fsck_--from___36__remote__44___copies_symlinks_instead_of_content.mdwn @@ -0,0 +1,80 @@ +### Please describe the problem. + + git annex fsck --from $remote + +copies the symlinks from the remote into *.git/annex/tmp*, which then fail to fsck as they don't point to content. + +In the transcript below, the 'a' file should fail fsck, but the rest pass. + +### What steps will reproduce the problem? + +See transcript + +### What version of git-annex are you using? On what operating system? + +* git-annex version: 5.20150409-g3575ee5 +* Arch Linux (git-annex-bin) + +### Please provide any additional information below. + + ## git init origin + ## cd corrupt/ + ## git annex init corrupt + ## echo a > a.txt + ## echo b > b.txt + ## echo c > c.txt + + ## git annex add . + add a.txt ok + add b.txt ok + add c.txt ok + (recording state in git...) + + ## git commit -m "add files" + [master (root-commit) 1d670a5] add files + 3 files changed, 3 insertions(+) + create mode 120000 a.txt + create mode 120000 b.txt + create mode 120000 c.txt + +("corrupting" a file not needed but here for completeness) + + ## chmod +w .git/annex/objects/41/pJ/SHA256E-s2--87428fc522803d31065e7bce3cf03fe475096631e5e07bbd7a0fde60c4cf25c7.txt/ + ## echo 'd' > .git/annex/objects/41/pJ/SHA256E-s2--87428fc522803d31065e7bce3cf03fe475096631e5e07bbd7a0fde60c4cf25c7.txt/SHA256E-s2--87428fc522803d31065e7bce3cf03fe475096631e5e07bbd7a0fde60c4cf25c7.txt + + ## cd .. + + ## git clone corrupt/ recovery/ + Cloning into 'recovery'... + done. + + ## cd recovery/ + + ## git annex init recovery + init recovery (merging origin/git-annex into git-annex...) + (recording state in git...) + ok + (recording state in git...) + + ## git annex fsck --from origin + fsck a.txt + git-annex: .git/annex/tmp/fsck24477.SHA256E-s2--87428fc522803d31065e7bce3cf03fe475096631e5e07bbd7a0fde60c4cf25c7.txt: getFileStatus: does not exist (No such file or directory) + failed + fsck b.txt + git-annex: .git/annex/tmp/fsck24477.SHA256E-s2--0263829989b6fd954f72baaf2fc64bc2e2f01d692d4de72986ea808f6e99813f.txt: getFileStatus: does not exist (No such file or directory) + failed + fsck c.txt + git-annex: .git/annex/tmp/fsck24477.SHA256E-s2--a3a5e715f0cc574a73c3f9bebb6bc24f32ffd5b67b387244c2c909da779a1478.txt: getFileStatus: does not exist (No such file or directory) + failed + (recording state in git...) + git-annex: fsck: 3 failed + + ## ls -lah .git/annex/tmp/ + total 20K + drwxr-xr-x 2 gemma users 4.0K Apr 18 17:27 .. + drwxr-xr-x 5 gemma users 4.0K Apr 18 17:27 .. + lrwxrwxrwx 1 gemma users 197 Apr 18 17:27 fsck24477.SHA256E-s2--0263829989b6fd954f72baaf2fc64bc2e2f01d692d4de72986ea808f6e99813f.txt -> ../corrupt/.git/annex/objects/x7/01/SHA256E-s2--0263829989b6fd954f72baaf2fc64bc2e2f01d692d4de72986ea808f6e99813f.txt/SHA256E-s2--0263829989b6fd954f72baaf2fc64bc2e2f01d692d4de72986ea808f6e99813f.txt + lrwxrwxrwx 1 gemma users 197 Apr 18 17:27 fsck24477.SHA256E-s2--87428fc522803d31065e7bce3cf03fe475096631e5e07bbd7a0fde60c4cf25c7.txt -> ../corrupt/.git/annex/objects/41/pJ/SHA256E-s2--87428fc522803d31065e7bce3cf03fe475096631e5e07bbd7a0fde60c4cf25c7.txt/SHA256E-s2--87428fc522803d31065e7bce3cf03fe475096631e5e07bbd7a0fde60c4cf25c7.txt + lrwxrwxrwx 1 gemma users 197 Apr 18 17:27 fsck24477.SHA256E-s2--a3a5e715f0cc574a73c3f9bebb6bc24f32ffd5b67b387244c2c909da779a1478.txt -> ../corrupt/.git/annex/objects/Vw/zz/SHA256E-s2--a3a5e715f0cc574a73c3f9bebb6bc24f32ffd5b67b387244c2c909da779a1478.txt/SHA256E-s2--a3a5e715f0cc574a73c3f9bebb6bc24f32ffd5b67b387244c2c909da779a1478.txt + +> reproduced, test cased, fixed, [[done]] --[[Joey]] diff --git a/doc/bugs/git-annex_deletes_file_when_using___34__git_annex_get__34___after___34__git_annex_addurl_--file__34__.mdwn b/doc/bugs/git-annex_deletes_file_when_using___34__git_annex_get__34___after___34__git_annex_addurl_--file__34__.mdwn new file mode 100644 index 000000000..1081b7acf --- /dev/null +++ b/doc/bugs/git-annex_deletes_file_when_using___34__git_annex_get__34___after___34__git_annex_addurl_--file__34__.mdwn @@ -0,0 +1,112 @@ +### Please describe the problem. + +When using "`git annex addurl --file`" with an ftp url, the committed +file is deleted after dropping the contents with --force (because +git-annex can't determine if the ftp server contains a valid copy) and +executing "`git annex get`". It's the "`git annex get`" command that +deletes the file. + +This does not happen when using an http url. + +### What steps will reproduce the problem? + +`git clone https://gist.github.com/sunny256/24f6c29645efd0aab4d9` + +and execute the bash script `runme`. There's more info in a long comment +there, plus various flags you can enable/disable to test under different +conditions. + +### What version of git-annex are you using? On what operating system? + +Using the newest git-annex from <https://downloads.kitenet.net/.git/> in +directory git-annex/linux/current/, 5.20150420-gb0ebb23. + +Have tested with versions way back to v5.20131221, they all behave the +same. + +Using Debian GNU/Linux 7.8 (wheezy) on x86_64 with brand new git 2.4.0. + +### Please provide any additional information below. + +[[!format sh """ +# If you can, paste a complete transcript of the problem occurring here. +# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log + +$ ./runme + +================== git init ================== +Initialized empty Git repository in /home/sunny/src/git/ga-bug/tmpdirawedsfkn/.git/ + +================== git annex init ================== +init ok +(recording state in git...) + +================== git commit, empty start commit ================== +[master (root-commit) 6d5d623] Empty startcommit + +================== git annex addurl ================== +addurl README (downloading ftp://ftp.funet.fi/pub/Linux/mirrors/debian/README ...) +--2015-05-02 03:28:59-- ftp://ftp.funet.fi/pub/Linux/mirrors/debian/README + => '.git/annex/tmp/URL--ftp&c%%ftp.funet.fi%pub%Linux%mirrors%debian%README' +Resolving ftp.funet.fi (ftp.funet.fi)... 193.166.3.2, 2001:708:10:9::20:2 +Connecting to ftp.funet.fi (ftp.funet.fi)|193.166.3.2|:21... connected. +Logging in as anonymous ... Logged in! +==> SYST ... done. ==> PWD ... done. +==> TYPE I ... done. ==> CWD (1) /pub/Linux/mirrors/debian ... done. +==> SIZE README ... 1495 +==> PASV ... done. ==> RETR README ... done. +Length: 1495 (1.5K) (unauthoritative) + +100%[================================================>] 1,495 --.-K/s in 0.01s + +2015-05-02 03:29:00 (125 KB/s) - '.git/annex/tmp/URL--ftp&c%%ftp.funet.fi%pub%Linux%mirrors%debian%README' saved [1495] + +ok +(recording state in git...) + +================== git commit, add README ================== +[master 264d597] Add README + 1 file changed, 1 insertion(+) + create mode 120000 README + +================== git annex drop --force ================== +drop README ok +(recording state in git...) + +================== git annex get ================== +get README (from web...) +--2015-05-02 03:29:00-- ftp://ftp.funet.fi/pub/Linux/mirrors/debian/README + => '.git/annex/tmp/SHA256-s1495--8822780b87a880ca9956ac108812557044618859cecb07df488df57e8134e34f' +Resolving ftp.funet.fi (ftp.funet.fi)... 193.166.3.2, 2001:708:10:9::20:2 +Connecting to ftp.funet.fi (ftp.funet.fi)|193.166.3.2|:21... connected. +Logging in as anonymous ... Logged in! +==> SYST ... done. ==> PWD ... done. +==> TYPE I ... done. ==> CWD (1) /pub/Linux/mirrors/debian ... done. +==> SIZE README ... 1495 +==> PASV ... done. ==> RETR README ... done. +Length: 1495 (1.5K) (unauthoritative) + +100%[================================================>] 1,495 --.-K/s in 0s + +2015-05-02 03:29:02 (73.1 MB/s) - '.git/annex/tmp/SHA256-s1495--8822780b87a880ca9956ac108812557044618859cecb07df488df57e8134e34f' saved [1495] + +ok +(recording state in git...) + +================== ls -l ================== +total 0 + +README is gone, should not happen + +Reached the end +$ + +# End of transcript or log. +"""]] + +> workaround in place; [[done]] --[[Joey]] + +> Also, fixed it to allow dropping the file if the ftp server seems +> to reply with a successful result (it's replying with 350, which is not +> unambiguously good, but since curl is able to get the right file length, +> the file is presumably still on the ftp server. --[[Joey]] diff --git a/doc/bugs/git-annex_deletes_file_when_using___34__git_annex_get__34___after___34__git_annex_addurl_--file__34__/comment_1_dca81d5db9a966fc992ed28bb3c2a242._comment b/doc/bugs/git-annex_deletes_file_when_using___34__git_annex_get__34___after___34__git_annex_addurl_--file__34__/comment_1_dca81d5db9a966fc992ed28bb3c2a242._comment new file mode 100644 index 000000000..65ebab1f2 --- /dev/null +++ b/doc/bugs/git-annex_deletes_file_when_using___34__git_annex_get__34___after___34__git_annex_addurl_--file__34__/comment_1_dca81d5db9a966fc992ed28bb3c2a242._comment @@ -0,0 +1,19 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2015-05-05T17:11:38Z" + content=""" +Thanks for a great bug report! + +Unfortunately, this turns out to be a bug in wget, as shown by this transcript: + + joey@darkstar:~/tmp/y>ls + README@ + joey@darkstar:~/tmp/y>wget -q --show-progress --clobber -c -O .git/annex/tmp/SHA256E-s1495--8822780b87a880ca9956ac108812557044618859cecb07df488df57e8134e34f ftp://ftp.funet.fi/pub/Linux/mirrors/debian/README --user-agent git-annex/5.20150505-gcdb212f + joey@darkstar:~/tmp/y>ls + joey@darkstar:~/tmp/y> + +I have filed a bug report on wget <http://bugs.debian.org/784348>, +and I guess I'll try to work around it in git-annex by running wget +inside an empty temp directory. +"""]] diff --git a/doc/bugs/git-annex_deletes_file_when_using___34__git_annex_get__34___after___34__git_annex_addurl_--file__34__/comment_2_c683d729fb0b3285ba9946539fe50d0d._comment b/doc/bugs/git-annex_deletes_file_when_using___34__git_annex_get__34___after___34__git_annex_addurl_--file__34__/comment_2_c683d729fb0b3285ba9946539fe50d0d._comment new file mode 100644 index 000000000..8c757b8a3 --- /dev/null +++ b/doc/bugs/git-annex_deletes_file_when_using___34__git_annex_get__34___after___34__git_annex_addurl_--file__34__/comment_2_c683d729fb0b3285ba9946539fe50d0d._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://sunny256.wordpress.com/" + nickname="sunny256" + subject="Thanks" + date="2015-05-07T14:20:49Z" + content=""" +Thank you for looking into this and coming up with a fix. Awesome. :) +"""]] diff --git a/doc/bugs/git-annex_fails_to_parse_external_remotes__39_____34__CHECKPRESENT-SUCCESS__34___response.mdwn b/doc/bugs/git-annex_fails_to_parse_external_remotes__39_____34__CHECKPRESENT-SUCCESS__34___response.mdwn new file mode 100644 index 000000000..f18d7a522 --- /dev/null +++ b/doc/bugs/git-annex_fails_to_parse_external_remotes__39_____34__CHECKPRESENT-SUCCESS__34___response.mdwn @@ -0,0 +1,56 @@ +### Please describe the problem. + +During a git annex fsck --fast --from <someexternalremote>, any CHECKPRESENT-SUCCESS responses are considered failures: + + fsck file + failed to download file from remote + failed + (recording state in git...) + git-annex: fsck: 1 failed + +It doesn't appear that the external protocol is being violated in any way; it occurs both with my special external (https://git.encryptio.com/slime/blob/refs/heads/master:/misc/git-annex-remote-slime/main.go) and with the example external remote. Making these dump their IO to stderr shows they're behaving correctly, as far as I can tell. + +`git annex testremote` passes on these remotes too, so it's not catching the issue (though it probably should.) + +These implementations used to work fine (~6mo ago? not sure); I haven't been able to get git-annex to build (cabal woes), so I can't bisect it myself. + +### What steps will reproduce the problem? + +With https://git-annex.branchable.com/special_remotes/external/example.sh/ installed as "git-annex-remote-externaltest", this script shows the issue: + +[[!format sh """ +#!/bin/sh +set -e + +[ -e "annex-test-dirs/repo/.git/annex/objects" ] && ( + find "annex-test-dirs/repo/.git/annex/objects" -type f -exec chmod 0644 {} \; + find "annex-test-dirs/repo/.git/annex/objects" -type d -exec chmod 0755 {} \; +) + +rm -rf annex-test-dirs +mkdir -p annex-test-dirs/{repo,data} + +cd annex-test-dirs/repo +git init +git annex init +MYPASSWORD=a MYLOGIN=b git annex initremote ext type=external encryption=none externaltype=externaltest directory="$(pwd)/../data" + +echo "data" > file +git annex add file +git commit -m message + +git annex copy --to ext + +# this works: +git annex fsck --from ext + +# but this incorrectly fails and marks the file "not present": +git annex fsck --fast --from ext +"""]] + +### What version of git-annex are you using? On what operating system? + +git-annex 5.20150420-gb0ebb23, from the linux standalone tarball, on linux. + +> Upgrade, this was fixed earlier this week, in [[!commit +> cfbeb1e7b74aa9759bd62b342e80869de582f10d]]; [[done]] afaik --[[Joey]] diff --git a/doc/bugs/git-annex_happily_deleted_most_of_my_files___36____35____38____33__/comment_5_fe2ba9550fc9dc2dc877965b436fe823._comment b/doc/bugs/git-annex_happily_deleted_most_of_my_files___36____35____38____33__/comment_5_fe2ba9550fc9dc2dc877965b436fe823._comment new file mode 100644 index 000000000..d5d2c35d2 --- /dev/null +++ b/doc/bugs/git-annex_happily_deleted_most_of_my_files___36____35____38____33__/comment_5_fe2ba9550fc9dc2dc877965b436fe823._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkW9u-8uqR62QBZjeTNCXsL7Ds55dAMGbA" + nickname="Horea" + subject="comment 5" + date="2015-04-17T22:15:07Z" + content=""" +I am experiencing the same thing (and have experienced it 1.5 years ago as well, right before deciding to never again use git-annex). Enough info or not is it simply unacceptable for your software to do this and you should look into it as best you can. Try to repeat the steps which he described. I for one, am getting much the same results. +"""]] diff --git a/doc/bugs/git-annex_happily_deleted_most_of_my_files___36____35____38____33__/comment_6_f36c0ab40bac68445fa0f205661c9f51._comment b/doc/bugs/git-annex_happily_deleted_most_of_my_files___36____35____38____33__/comment_6_f36c0ab40bac68445fa0f205661c9f51._comment new file mode 100644 index 000000000..d794c23f9 --- /dev/null +++ b/doc/bugs/git-annex_happily_deleted_most_of_my_files___36____35____38____33__/comment_6_f36c0ab40bac68445fa0f205661c9f51._comment @@ -0,0 +1,16 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 6""" + date="2015-04-18T18:52:20Z" + content=""" +It's approximately 1000 times easier for someone who is experiencing a +problem to paste a tracript into this bug report than it is for me to guess +what someone might be doing and reproduce their bug (or problem, or mistake +or oops, or whatever) from basically no information. + +This is why I ask for bugs to be filed with either transcripts, or the +steps that can be used to reproduce them. + +I think you'll find that if you post enough information for a bug to be +reproduced, it'll get fixed quite fast. +"""]] diff --git a/doc/bugs/git_annex_assistant_--stop_fails_if_not_run_from_inside_a_git_annex_repository.mdwn b/doc/bugs/git_annex_assistant_--stop_fails_if_not_run_from_inside_a_git_annex_repository.mdwn new file mode 100644 index 000000000..61a01d73f --- /dev/null +++ b/doc/bugs/git_annex_assistant_--stop_fails_if_not_run_from_inside_a_git_annex_repository.mdwn @@ -0,0 +1,21 @@ +I set up a systemd unit to start the assistant, which works fine (it uses --autostart). When it comes time to stop the unit, however, it calls git annex assistant --stop, which fails because the current directory doesn't have a git annex repository in it. + + git-annex version: 5.20140717 + build flags: Assistant Inotify DBus Quvi TDFA + key/value backends: SHA256E SHA1E SHA512E SHA224E SHA384E SHA256 SHA1 SHA512 SHA224 SHA384 WORM URL + remote types: git gcrypt bup directory rsync web tahoe glacier ddar hook external + +Here's my unit file: +[[!format ini """ +[Service] +ExecStart=/usr/bin/git-annex assistant --autostart +ExecStop=/usr/bin/git-annex assistant --stop + +[Install] +WantedBy=default.target +"""]] + +> This seems to overlap with [[todo/server-level_daemon__63__/]] in some way... --[[anarcat]] + +> Added --autostop and improved the docuemntation for --stop. [[done]] +> --[[Joey]] diff --git a/doc/bugs/git_annex_assistant_--stop_fails_if_not_run_from_inside_a_git_annex_repository/comment_1_a4cb3f759a1b076851c92fa8f9cca1be._comment b/doc/bugs/git_annex_assistant_--stop_fails_if_not_run_from_inside_a_git_annex_repository/comment_1_a4cb3f759a1b076851c92fa8f9cca1be._comment new file mode 100644 index 000000000..24f24455f --- /dev/null +++ b/doc/bugs/git_annex_assistant_--stop_fails_if_not_run_from_inside_a_git_annex_repository/comment_1_a4cb3f759a1b076851c92fa8f9cca1be._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2015-04-29T15:22:57Z" + content=""" +Well, --stop is intended to stop the daemon in the current git repository, +not all of them. That would be --autostop or something not implemented. + +I don't see how this is a bug. Feature request at best. +"""]] diff --git a/doc/bugs/git_annex_assistant_--stop_fails_if_not_run_from_inside_a_git_annex_repository/comment_2_953e6d1262728da6d30f4d3f1dd8fc29._comment b/doc/bugs/git_annex_assistant_--stop_fails_if_not_run_from_inside_a_git_annex_repository/comment_2_953e6d1262728da6d30f4d3f1dd8fc29._comment new file mode 100644 index 000000000..a58c1094b --- /dev/null +++ b/doc/bugs/git_annex_assistant_--stop_fails_if_not_run_from_inside_a_git_annex_repository/comment_2_953e6d1262728da6d30f4d3f1dd8fc29._comment @@ -0,0 +1,11 @@ +[[!comment format=mdwn + username="db48x" + subject="comment 2" + date="2015-04-29T22:56:47Z" + content=""" +Well, the usage just says + + --stop stop daemon + +so naturally I assumed that this is how you stop the daemon. It's cool though; systemd can just send it a SIGKILL or whatever. +"""]] diff --git a/doc/bugs/git_annex_import:_ignored_names_fatal.mdwn b/doc/bugs/git_annex_import:_ignored_names_fatal.mdwn index e96ca2acb..8981cadd8 100644 --- a/doc/bugs/git_annex_import:_ignored_names_fatal.mdwn +++ b/doc/bugs/git_annex_import:_ignored_names_fatal.mdwn @@ -36,3 +36,6 @@ Can't the ignored files just be ignored? # End of transcript or log. """]] + +> Made git-annex import check for gitignored files before +> moving them into the work tree. [[done]] --[[Joey]] diff --git a/doc/bugs/git_annex_import:_ignored_names_fatal/comment_2_99f575f424ec2f0d739fa7296f0d0086._comment b/doc/bugs/git_annex_import:_ignored_names_fatal/comment_2_99f575f424ec2f0d739fa7296f0d0086._comment new file mode 100644 index 000000000..0806cdef0 --- /dev/null +++ b/doc/bugs/git_annex_import:_ignored_names_fatal/comment_2_99f575f424ec2f0d739fa7296f0d0086._comment @@ -0,0 +1,11 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 2""" + date="2015-04-29T17:35:22Z" + content=""" +I cannot reproduce it repeating; the file is actually moved out of the +import location, so importing again won't do anything with it. You can +`git annex add --force` the file to bypass the .gitignore. + +`git annex import --force` also bypasses the problem entirely. +"""]] diff --git a/doc/bugs/git_annex_importfeed_not_working_on_youtube_playlists.mdwn b/doc/bugs/git_annex_importfeed_not_working_on_youtube_playlists.mdwn new file mode 100644 index 000000000..2306642fc --- /dev/null +++ b/doc/bugs/git_annex_importfeed_not_working_on_youtube_playlists.mdwn @@ -0,0 +1,39 @@ +### Please describe the problem. +The [podcasts page](https://git-annex.branchable.com/tips/downloading_podcasts/) mentions that you should be able to use `git annex importfeed` on youtube playlists, however, that doesn't seem to work (or I am using it incorrectly). + +For example: + + » git annex importfeed --fast "https://www.youtube.com/playlist?list=PLz8YL4HVC87X3tYVYOjOLzSasPwgUsMPT" + (checking known urls...) + importfeed https://www.youtube.com/playlist?list=PLz8YL4HVC87X3tYVYOjOLzSasPwgUsMPT + /var/folders/kb/fplmcbhs4z52p4x59cqgm4kw0000gn/T/f [ <=> ] 460.08K 1.87MB/s in 0.2s + + warning: bad feed content + ok + +### What steps will reproduce the problem? + +1. Enter a git annex repository +2. Attempt to import a youtube playlist feed (see command above) with `--fast` + +### What version of git-annex are you using? On what operating system? + + » git annex version + git-annex version: 5.20150205 + build flags: Assistant Webapp Webapp-secure Pairing Testsuite S3 WebDAV FsEvents XMPP DNS Feeds Quvi TDFA TorrentParser + key/value backends: SHA256E SHA1E SHA512E SHA224E SHA384E SKEIN256E SKEIN512E MD5E SHA256 SHA1 SHA512 SHA224 SHA384 SKEIN256 SKEIN512 MD5 WORM URL + remote types: git gcrypt S3 bup directory rsync web bittorrent webdav tahoe glacier ddar hook external + local repository version: 5 + supported repository version: 5 + upgrade supported from repository versions: 0 1 2 4 + +This is on Mac OSX 10.10.3, with git-annex installed via homebrew + +> You have to give it an url to a RSS feed containing the playlist. +> +> For example, `git annex importfeed +> 'http://gdata.youtube.com/feeds/api/playlists/PLz8ZG1e9MPlzefklz1Gv79icjywTXycR-'` +> still works despite there having been various news stories about youtube +> removing RSS functionality. +> +> I've added this example to the documentation. [[done]] --[[Joey]] diff --git a/doc/bugs/git_annex_list__47__whereis_and_uncommited_local_changes.mdwn b/doc/bugs/git_annex_list__47__whereis_and_uncommited_local_changes.mdwn index 691591519..cf7ce7e5e 100644 --- a/doc/bugs/git_annex_list__47__whereis_and_uncommited_local_changes.mdwn +++ b/doc/bugs/git_annex_list__47__whereis_and_uncommited_local_changes.mdwn @@ -28,3 +28,6 @@ git-annex 5.20140421 Linux 3.14.3 [[!tag confirmed]] + +> I think it's reasonable for whereis to show location tracking information +> which may be out of date for many reasons, and so, [[done]] --[[Joey]] diff --git a/doc/bugs/local_pair_fails_if_non-ascii_characters_present_on_annex_path/comment_2_89f25f787558c77201fa6226cc7af5f5._comment b/doc/bugs/local_pair_fails_if_non-ascii_characters_present_on_annex_path/comment_2_89f25f787558c77201fa6226cc7af5f5._comment new file mode 100644 index 000000000..d922d1722 --- /dev/null +++ b/doc/bugs/local_pair_fails_if_non-ascii_characters_present_on_annex_path/comment_2_89f25f787558c77201fa6226cc7af5f5._comment @@ -0,0 +1,14 @@ +[[!comment format=mdwn + username="http://xgm.de/oid/" + nickname="Horus" + subject="comment 2" + date="2015-05-05T17:35:37Z" + content=""" +I have the same problem without any special characters in my path (besides ~ of $HOME) +Debug Log gives: + + illegal control characters in pairing message; ignoring + [2015-05-05 19:11:40 CEST] PairListener: received \"PairMsg (Verifiable {verifiableVal = (PairReq,PairData {remoteHostName = Just \\"asaru\\", remoteUserName = \\"florian\\", remoteDirectory = \\"~/Desktop/annex\\", remoteSshPubKey = \\"ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDE8cd+cThzgRD+9RuiFhbL6UbPP+gvyNcUdrwVZoqfn2AE0niOe6XwsvqNrL4BZE50ySIo71XHyyAtRPiW3h0R8NjJo8+VFha2KL9vCXySNjq0Ib6HinfCDNUp5hI35F+LnUtUAVkhhhVqfJj4C6K3JTjXQ9J/hgiYRpNCY+2hV0+sF/e643SsyNlkUhiNxfCd4LQ5bedX6FeSCYBwteVgtZQzyByeawFpj1uajqBbDgDBLmclXDNrb4DwqavLRj+L+XxPtNqSKXSp8Q2/oypr/GQeTjmHEb8K/7qSjNHcBDAHH9fUI5lhDDhyxc4lMfap0lseSWtlwldhKjGqPnB9 florian@asaru\\n\\", pairUUID = UUID \\"0b1e8007-4a8d-4cc4-9ca1-320f4f700081\\"},IPv4Addr 347252928), verifiableDigest = \\"dff64a76c0333223cc3909f13bbdb3e1a70ddaa4\\"})\" + +I'll be very happy to provide any other help... +"""]] diff --git a/doc/bugs/man_page_has_example_lines_joined_into_a_single_line.mdwn b/doc/bugs/man_page_has_example_lines_joined_into_a_single_line.mdwn new file mode 100644 index 000000000..397666a2c --- /dev/null +++ b/doc/bugs/man_page_has_example_lines_joined_into_a_single_line.mdwn @@ -0,0 +1,18 @@ + +[[!format sh """ +# If you can, paste a complete transcript of the problem occurring here. +# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log +$> git annex help schedule | grep -A3 examples +man: can't set the locale; make sure $LC_* and $LANG are correct + Some examples: + + fsck self 30m every day at any time fsck self 1h every month at 3 AM fsck self 1h on day 1 of every month at any time fsck self 1h every week divisible by 2 at any time + +$> git annex version +git-annex version: 5.20150409+git126-ga29f683-1~ndall+1 +# about to build a fresh standlone + +# End of transcript or log. +"""]] + +> [[fixed|done]] --[[Joey]] diff --git a/doc/bugs/regression:_behavior_when_files_to_add_do_not_exist.mdwn b/doc/bugs/regression:_behavior_when_files_to_add_do_not_exist.mdwn new file mode 100644 index 000000000..9421dc66d --- /dev/null +++ b/doc/bugs/regression:_behavior_when_files_to_add_do_not_exist.mdwn @@ -0,0 +1,103 @@ +### Please describe the problem. + +When adding a list of files, where some exist and some don't, annex claims to add some of the files until it encounters the first missing file. Then it bails out, leaving files hashed but not added. + +### What steps will reproduce the problem? + +Create a file, ask annex to add the file and a non-existant file + +Expected and historic behavior: annex adds the file + +Actual behavior: annex hashes but does not add the file + +### What version of git-annex are you using? On what operating system? + +git-annex version: 5.20150420-gb0ebb23 +standalone linux amd64 + +### Please provide any additional information below. + +[[!format sh """ +# If you can, paste a complete transcript of the problem occurring here. +# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log + +$ git annex version +git-annex version: 5.20150420-gb0ebb23 +[ . . . ] + +$ git init asdf +Initialized empty Git repository in /tmp/asdf/.git/ + +$ cd asdf + +$ git annex init +init ok +(recording state in git...) + +$ touch asdf + +$ git add asdf qwer +fatal: pathspec 'qwer' did not match any files + +$ git annex add asdf qwer +add asdf ok +git-annex: qwer not found + +$ file * | sed -e 's/`.*//' +asdf: symbolic link to + +$ git status +On branch master + +Initial commit + +Untracked files: + (use "git add <file>..." to include in what will be committed) + + asdf + +nothing added to commit but untracked files present (use "git add" to track) + + +# End of transcript or log. +"""]] + +Older version of git-annex: + +[[!format sh """ + +$ git annex version +git-annex version: 5.20140412ubuntu1 +[ . . . ] + +$ git init asdf +Initialized empty Git repository in /tmp/asdf/.git/ + +$ cd asdf + +$ git annex init asdf +init asdf ok +(Recording state in git...) + +$ touch asdf + +$ git annex add asdf qwer +add asdf ok +git-annex: qwer not found +(Recording state in git...) + +$ file * | sed -e 's/`.*//' +asdf: symbolic link to + +$ git status +On branch master + +Initial commit + +Changes to be committed: + (use "git rm --cached <file>..." to unstage) + + new file: asdf +"""]] + +> [[fixed|done]] --[[Joey]] diff --git a/doc/bugs/regression:_behavior_when_files_to_add_do_not_exist/comment_1_3e444d500071779bcbfbc781b4756daf._comment b/doc/bugs/regression:_behavior_when_files_to_add_do_not_exist/comment_1_3e444d500071779bcbfbc781b4756daf._comment new file mode 100644 index 000000000..6fc7f84db --- /dev/null +++ b/doc/bugs/regression:_behavior_when_files_to_add_do_not_exist/comment_1_3e444d500071779bcbfbc781b4756daf._comment @@ -0,0 +1,30 @@ +[[!comment format=mdwn + username="clacke" + subject="Inconsistent between batches of files" + date="2015-04-22T12:56:42Z" + content=""" +If you add enough files, annex gets past the first `(Recording state in git...)` and then breaks on only the last portion, so some files are added and some are only hashed: + +[[!format sh \"\"\" +$ touch {10000..20240} 20242 + +$ git annex add {10000..20242} +[ . . . ] +add 20240 ok +(recording state in git...) +add 20242 ok +git-annex: 20241 not found + +$ file 20240 20242 | sed -e 's/`.*//' +20240: symbolic link to +20242: symbolic link to + +$ git status | tail -n 7 + new file: 20240 + +Untracked files: + (use \"git add <file>...\" to include in what will be committed) + + 20242 +\"\"\"]] +"""]] diff --git a/doc/bugs/regression:_behavior_when_files_to_add_do_not_exist/comment_2_4e2bd0704377c9bdc9fdb06a078fbc62._comment b/doc/bugs/regression:_behavior_when_files_to_add_do_not_exist/comment_2_4e2bd0704377c9bdc9fdb06a078fbc62._comment new file mode 100644 index 000000000..3e77d3a39 --- /dev/null +++ b/doc/bugs/regression:_behavior_when_files_to_add_do_not_exist/comment_2_4e2bd0704377c9bdc9fdb06a078fbc62._comment @@ -0,0 +1,9 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 2""" + date="2015-04-22T20:27:29Z" + content=""" +This behavior could stand to be improved, but it's easy to deal with: Just +git annex add the files again, listing only the ones that actually exist, +and it will proceed where it was interrupted by the error. +"""]] diff --git a/doc/bugs/regression:_behavior_when_files_to_add_do_not_exist/comment_3_9ea62129cc568b07994b7e3d517a100c._comment b/doc/bugs/regression:_behavior_when_files_to_add_do_not_exist/comment_3_9ea62129cc568b07994b7e3d517a100c._comment new file mode 100644 index 000000000..20de644e9 --- /dev/null +++ b/doc/bugs/regression:_behavior_when_files_to_add_do_not_exist/comment_3_9ea62129cc568b07994b7e3d517a100c._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 3""" + date="2015-04-29T18:03:17Z" + content=""" +Also, I'm not so sure it's a regression, at least back at version +5.20141125 it behaved the same way. +"""]] diff --git a/doc/bugs/regression:_behavior_when_files_to_add_do_not_exist/comment_4_42481bb2f6f625a9891e59ec97574164._comment b/doc/bugs/regression:_behavior_when_files_to_add_do_not_exist/comment_4_42481bb2f6f625a9891e59ec97574164._comment new file mode 100644 index 000000000..4755498c1 --- /dev/null +++ b/doc/bugs/regression:_behavior_when_files_to_add_do_not_exist/comment_4_42481bb2f6f625a9891e59ec97574164._comment @@ -0,0 +1,9 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 4""" + date="2015-04-30T18:56:31Z" + content=""" +Regression or not, it would be useful if git-annex continued past such +not-existing files to process the rest of the specified files, and only +set the error flag. +"""]] diff --git a/doc/bugs/rsync_on_windows_broken_by_upgrade.mdwn b/doc/bugs/rsync_on_windows_broken_by_upgrade.mdwn new file mode 100644 index 000000000..929770ccc --- /dev/null +++ b/doc/bugs/rsync_on_windows_broken_by_upgrade.mdwn @@ -0,0 +1,69 @@ +rsync on windows has been broken by the upgrade to a newer version of +cygwin. `rsync user@host:file file` opens the ssh connection, but hangs +up with a protocol error. Apparently it doesn't get even the protocol +version message from the server. + +Problem doesn't seem to affect the bundled ssh, just rsync. --[[Joey]] + +> Update: Apparently there are two ssh's! msysgit bundles one (did it used +> to in PATH?) and git-annex bundles one from cygwin. msysgit's ends up +> in Git/bin and git-annex's in Git/cmd. +> +> Seems that cygwin's rsync cannot use git's ssh for whatever reason. +> +> So the workaround is to +> delete Git/bin/ssh.exe and leave Git/cmd/ssh.exe. Then rsync works. +> However, this may screw up git's use of ssh or other stuff. +> +> Particularly, cygwin's ssh doesn't honor HOME anymore, instead using +> the getpwent home, which doesn't exist. +> +> Also, see +> [[webapp_fails_to_connect_to_ssh_repository___40__windows__41__]] +> which is the inverse of this bug perhaps, or at least seems related. +> +> Using 2 ssh's that try to use config from different places seems like +> a losing propisition. Need to find an rsync that works with git's ssh. +> --[[Joey]] +> +> > Update: The git bin/ directory is only in PATH when inside "git bash". +> > This bug only seems to affect using git-annex that way. The git bash +> > PATH has `bin` before `cmd`. +> > +> > Also, git seems to work ok using the newer ssh from cygwin. +> > However, that ssh tries to write to a .ssh/known_hosts +> > in a cygwin location and so doesn't remember hosts. +> > +> > What a mess. You can install any version of Linux and get rsync, ssh, +> > git that all integrate and work together. Or you can use Windows and +> > enjoy the pain(TM) --[[Joey]] + +>>> Possible fixes: +>>> +>>> * Roll the bundled ssh and rsync back to the older versions. +>>> +>>> **This works**. And, seems that the older version of ssh from cygwin +>>> looks at HOME, rather than getpwent home which the newer +>>> cygwin ssh does. +>>> +>>> * Roll the bundled rsync back, drop ssh. Rely on msysgit's bundled ssh, +>>> copying it into cmd so it's in PATH. Check: Does this combo work? +>>> +>>> **This works**! rsync 3.0.9 works ok with msysgit's bundled ssh. +>>> rsync 3.1.1 is the one that needs a newer ssh. **[[done]]** +>>> +>>> Note that this means we're using an old version of rsync +>>> from cygwin with libraries from a newer cygwin. That might prove +>>> fragile as cygwin is upgraded. +>>> +>>> * Hope that msysgit gets updated to include a newer version of ssh +>>> which works with the new rsync. +>>> +>>> (Seems reasonable as a long-term plan, assuming that the +>>> new rsync's problem with ssh is that it needs a new one, and not some +>>> special cygwin thing.) +>>> +>>> * Get rsync from somewhere else, perhaps msysgit. (Maybe also get ssh +>>> from msysgit?) +>>> * Keep the new rsync from cygwin, and build ssh from source, +>>> patching it to use HOME in preference to getpwent home. diff --git a/doc/bugs/rsync_on_windows_broken_by_upgrade/comment_1_de67cd10dd73fdc4d43eeb7ffbe67568._comment b/doc/bugs/rsync_on_windows_broken_by_upgrade/comment_1_de67cd10dd73fdc4d43eeb7ffbe67568._comment new file mode 100644 index 000000000..85c5e740f --- /dev/null +++ b/doc/bugs/rsync_on_windows_broken_by_upgrade/comment_1_de67cd10dd73fdc4d43eeb7ffbe67568._comment @@ -0,0 +1,7 @@ +[[!comment format=mdwn + username="anton" + subject="Thanks!" + date="2015-05-08T07:11:34Z" + content=""" +Thank you Joey for the great support and all your work on git-annex in general! +"""]] diff --git a/doc/bugs/ssh_fails_in_windows_nightly_build.mdwn b/doc/bugs/ssh_fails_in_windows_nightly_build.mdwn new file mode 100644 index 000000000..a40e3ea71 --- /dev/null +++ b/doc/bugs/ssh_fails_in_windows_nightly_build.mdwn @@ -0,0 +1,10 @@ +### Please describe the problem. + +After installing a nightly build from https://qa.nest-initiative.org/view/msysGit/job/msysgit-git-annex-assistant-test/, ssh fails to run when launched by the webapp. It shows a system error dialog with the message "The program can't start because msys-crypto-1.0.0.dll is missing...". The dll is present in $PROGRAMFILES/Git/bin, and ssh works when run from the command line. + +[[!format sh """ +Version: 5.20150508-g8e96a31 +Build flags: Assistant Webapp Webapp-secure Pairing Testsuite S3 WebDAV DNS Feeds Quvi TDFA TorrentParser +"""]] + +> sigh.. [[fixed|done]] now.. --[[Joey]] diff --git a/doc/bugs/ssh_portnum_bugs.mdwn b/doc/bugs/ssh_portnum_bugs.mdwn index 4f24a9945..0d7199d8e 100644 --- a/doc/bugs/ssh_portnum_bugs.mdwn +++ b/doc/bugs/ssh_portnum_bugs.mdwn @@ -13,3 +13,6 @@ Change Port 22 in /etc/ssh/sshd_config to Port 9999, restart ssh on both compute When I was experiencing this issue, I was running the default on Jessie/Wheezy. Now I'm running the latest (via auto-update and distributed binary) but don't know if this is still an issue with latest versions (I switched to 22 as a workaround). [[!tag moreinfo]] + +> Closing as it seems likely this is an old bug fixed after wheezy. +> [[done]] --[[Joey]] diff --git a/doc/bugs/ssh_portnum_bugs/comment_6_5360e3c8db03402d2b7d81c0bbe8b35e._comment b/doc/bugs/ssh_portnum_bugs/comment_6_5360e3c8db03402d2b7d81c0bbe8b35e._comment new file mode 100644 index 000000000..cfe43137c --- /dev/null +++ b/doc/bugs/ssh_portnum_bugs/comment_6_5360e3c8db03402d2b7d81c0bbe8b35e._comment @@ -0,0 +1,7 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 6""" + date="2015-05-01T18:21:09Z" + content=""" +Any chance of a followup? +"""]] diff --git a/doc/bugs/ssh_portnum_bugs/comment_7_4157337c26a4267323da78dc7609a834._comment b/doc/bugs/ssh_portnum_bugs/comment_7_4157337c26a4267323da78dc7609a834._comment new file mode 100644 index 000000000..2657443a4 --- /dev/null +++ b/doc/bugs/ssh_portnum_bugs/comment_7_4157337c26a4267323da78dc7609a834._comment @@ -0,0 +1,7 @@ +[[!comment format=mdwn + username="jamieson" + subject="Follow up" + date="2015-05-01T20:40:49Z" + content=""" +Sorry about that! please close and I'll follow up if I have any more issues. Thanks!! +"""]] diff --git a/doc/bugs/webapp_fails_to_connect_to_ssh_repository___40__windows__41__.mdwn b/doc/bugs/webapp_fails_to_connect_to_ssh_repository___40__windows__41__.mdwn new file mode 100644 index 000000000..83fec53ab --- /dev/null +++ b/doc/bugs/webapp_fails_to_connect_to_ssh_repository___40__windows__41__.mdwn @@ -0,0 +1,40 @@ +### Please describe the problem. + +I added a remote repository, and it successfully tested the ssh connection to the server. Nothing happens, however, once it comes to actually creating the remote repository (or merging with an existing one). It'll just sit there forever, never actually connecting to the server. + +A quick look in process explorer shows something of what's going on: git-annex has launched ssh, and ssh is spamming subprocesses. It's launching ssh.exe which is launching git-annex.exe (yes, on the local machine.) It exits right away, but the command line is "C:\Program Files (x86)\Git\cmd\git-annex.exe" "Please type 'yes' or 'no': ". I've no idea why it's doing that though. + +If I kill that parent ssh process, I get this error message in the transcript: + + Could not create directory '/home/db48x/.ssh'. + +This seems a bit dubious; both my local computer and the remote computer have a ~/.ssh directory. + +In order to rule out a problem with my local computer (an ancient install of Git or cygwin/msys or something, we've tried this on fresh computers which have never had git installed; we get exactly the same problem there. + +### What steps will reproduce the problem? + +Create or connect to a repository via SSH. + +### What version of git-annex are you using? On what operating system? + +Windows 7 + + Version: 5.20150420-gb0ebb23 + Build flags: Assistant Webapp Webapp-secure Pairing Testsuite S3 WebDAV DNS Feeds Quvi TDFA TorrentParser + +### Please provide any additional information below. + +While this is going on, the log has is showing that it's executing the following command: + +[[!format sh """ +[2015-04-28 22:34:16 Pacific Daylight Time] chat: ssh ["-oNumberOfPasswordPrompts=1","-p","22","db48x@eambar.db48x.net","sh -c 'mkdir -p '\"'\"'annex'\"'\"'&&cd '\"'\"'annex'\"'\"'&&if [ ! -d .git ]; then if [ -x ~/.ssh/git-annex-wrapper ]; then ~/.ssh/git-annex-wrapper git init --bare --shared; else git init --bare --shared; fi && if [ -x ~/.ssh/git-annex-wrapper ]; then ~/.ssh/git-annex-wrapper git config receive.denyNonFastforwards; else git config receive.denyNonFastforwards; fi ;fi&&mkdir -p ~/.ssh&&if [ ! -e ~/.ssh/git-annex-shell ]; then (echo '\"'\"'#!/bin/sh'\"'\"';echo '\"'\"'set -e'\"'\"';echo '\"'\"'if [ \"x$SSH_ORIGINAL_COMMAND\" != \"x\" ]; then'\"'\"';echo '\"'\"'exec git-annex-shell -c \"$SSH_ORIGINAL_COMMAND\"'\"'\"';echo '\"'\"'else'\"'\"';echo '\"'\"'exec git-annex-shell -c \"$@\"'\"'\"';echo '\"'\"'fi'\"'\"') > ~/.ssh/git-annex-shell; fi&&chmod 700 ~/.ssh/git-annex-shell&&touch ~/.ssh/authorized_keys&&chmod 600 ~/.ssh/authorized_keys&&echo '\"'\"'command=\"env GIT_ANNEX_SHELL_DIRECTORY='\"'\"'\"'\"'\"'\"'\"'\"'annex'\"'\"'\"'\"'\"'\"'\"'\"' ~/.ssh/git-annex-shell\",no-agent-forwarding,no-port-forwarding,no-X11-forwarding,no-pty ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDiPFdIMyYCBmc14f9cUZaG36Zw+NziqX9Z+xGl5GAYq16nORxVc+70Bj+A9cHoHLzTMBJnw7G/f7xJNGbKNgKUPKZohT8AQfg8lnyK8qpyzI2dJB3R0vPBMPxZCBm4IOpdB6ad3B6dUiyNtyMn1hza7GVhIFOsHfGG+K3PGtFgwOz/Zj+2zmcZIL/BHObFsba/yhQxbsjLYPI7mmNV9CLb1+XcR0z2okWvxu28lOrcIXDAdEhp1cjjjpBhwTH1F8/gJcJ6ENBa4JiGt/oEKb1b/pXLaMX6dRjc/gYoy7z0Hw7RD73hH+UtPj5TAeKwoNdaVXdqSsVI+3ql+O5PFTxt db48x@caradhras\n'\"'\"' >>~/.ssh/authorized_keys'"] +"""]] + +> This sounds like it's all down to the newer ssh from cygwin ignoring HOME +> and trying to use /home/user which doesn't work very well outside cygwin. +> +> Since git-annex has switched to not include that ssh any longer, and +> instead use the ssh that's bundled with msysgit, I think this bug is +> closed. [[done]] Upgrading to the latest daily build should fix your +> system's ssh. Please followup if I'm wrong. --[[Joey]] diff --git a/doc/bugs/webapp_fails_to_connect_to_ssh_repository___40__windows__41__/comment_1_c89c2fcd8d93bc64b80749b4207a3ebd._comment b/doc/bugs/webapp_fails_to_connect_to_ssh_repository___40__windows__41__/comment_1_c89c2fcd8d93bc64b80749b4207a3ebd._comment new file mode 100644 index 000000000..6db718bf2 --- /dev/null +++ b/doc/bugs/webapp_fails_to_connect_to_ssh_repository___40__windows__41__/comment_1_c89c2fcd8d93bc64b80749b4207a3ebd._comment @@ -0,0 +1,9 @@ +[[!comment format=mdwn + username="anton" + subject="me too" + date="2015-05-07T09:50:09Z" + content=""" +I get similar results on Windows, but I only use the command line. For some reason git-annex ignores the ssh-agent settings (SSH_AUTH_SOCK=...) and uses the wrong path for the ssh config dir -- /home/username/.ssh (that probably doesn't exist) -- instead of c:/Users/username/.ssh (or whatever it really is). Your issue is probably that ssh wrongly looks for known_hosts in /home/username/.ssh and asks whether you wan't to accept the unknown host key. + +SSH works when ran by git itself (like git clone/push/fetch), also for rsync, but seemingly not git-annex. +"""]] diff --git a/doc/bugs/webapp_fails_to_connect_to_ssh_repository___40__windows__41__/comment_2_0562eb168f7a35262a24346030c4fa9f._comment b/doc/bugs/webapp_fails_to_connect_to_ssh_repository___40__windows__41__/comment_2_0562eb168f7a35262a24346030c4fa9f._comment new file mode 100644 index 000000000..9089921ec --- /dev/null +++ b/doc/bugs/webapp_fails_to_connect_to_ssh_repository___40__windows__41__/comment_2_0562eb168f7a35262a24346030c4fa9f._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 2""" + date="2015-05-07T18:26:25Z" + content=""" +See [[rsync_on_windows_broken_by_upgrade]] which is the inverse of this +bug. +"""]] diff --git a/doc/bugs/windows_ssh_webapp_password_entry_broken.mdwn b/doc/bugs/windows_ssh_webapp_password_entry_broken.mdwn new file mode 100644 index 000000000..e734e007b --- /dev/null +++ b/doc/bugs/windows_ssh_webapp_password_entry_broken.mdwn @@ -0,0 +1,5 @@ +Recent changes to ssh on Windows have broken the webapps's support for +entering a password when adding a ssh remote. + +Using ssh on windows with an existing remote does work. So as a workaround, +set up a passwordless ssh key that can log into the ssh server. --[[Joey]] diff --git a/doc/bugs/wrong_synopsis_in_manpage_of_git_annex_pre-commit.mdwn b/doc/bugs/wrong_synopsis_in_manpage_of_git_annex_pre-commit.mdwn new file mode 100644 index 000000000..f94786b03 --- /dev/null +++ b/doc/bugs/wrong_synopsis_in_manpage_of_git_annex_pre-commit.mdwn @@ -0,0 +1,36 @@ +### Please describe the problem. + +The synopsis in the manpage of git annex pre-commit mentions +git annex instead of git annex pre-commit. + +I'll attach a patch to fix the manpage, below: + +[[!format diff """ +From f5fed6ccdcef3bcb8be07691265842d437037dec Mon Sep 17 00:00:00 2001 +From: Felix Gruber <felgru@gmx.de> +Date: Fri, 1 May 2015 02:05:32 +0200 +Subject: [PATCH] fix synopsis in manpage of git annex pre-commit + +--- + doc/git-annex-pre-commit.mdwn | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/doc/git-annex-pre-commit.mdwn b/doc/git-annex-pre-commit.mdwn +index e0f6fdb..bc1e86e 100644 +--- a/doc/git-annex-pre-commit.mdwn ++++ b/doc/git-annex-pre-commit.mdwn +@@ -4,7 +4,7 @@ git-annex pre-commit - run by git pre-commit hook + + # SYNOPSIS + +-git annex `[path ...]` ++git annex pre-commit `[path ...]` + + # DESCRIPTION + +-- +2.1.4 +"""]] + +> You know, this is a wiki, you could fix it yourself. With git push even. +> In any case [[done]] now. --[[Joey]] diff --git a/doc/design/balanced_preferred_content.mdwn b/doc/design/balanced_preferred_content.mdwn new file mode 100644 index 000000000..1f00a0339 --- /dev/null +++ b/doc/design/balanced_preferred_content.mdwn @@ -0,0 +1,66 @@ +Say we have 2 backup drives and want to fill them both evenly with files, +different files in each drive. Currently, preferred content cannot express +that entirely: + +* One way is to use a-m* and n-z*, but that's unlikely to split filenames evenly. +* Or, can let both repos take whatever files, perhaps at random, that the + other repo is not know to contain, but then repos will race and both get + the same file, or similarly if they are not communicating frequently. + +So, let's add a new expression: `balanced_amoung(group)` + +This would work by taking the list of uuids of all repositories in the +group, and sorting them, which yields a list from 0..M-1 repositories. + +To decide which repository wants key K, convert K to a number N in some +stable way and then `N mod M` yields the number of the repository that +wants it, while all the rest don't. + +(Since git-annex keys can be pretty long and not all of them are random +hashes, let's md5sum the key and then use the md5 as a number.) + +This expression is stable as long as the members of the group don't change. +I think that's stable enough to work as a preferred content expression. + +Now, you may want to be able to add a third repo and have the data be +rebalanced, with some moving to it. And that would happen. However, as this +scheme stands, it's equally likely that adding repo3 will make repo1 and +repo2 want to swap files between them. So, we'll want to add some +precautions to avoid a lof of data moving around in this case: + + ((balanced_amoung(backup) and not (copies=backup:1)) or present + +So once file lands on a backup drive, it stays there, even if more backup +drives change the balancing. + +----- + +Some limitations: + +* The item size is not taken into account. One repo could end up with a + much larger item or items and so fill up faster. And the other repo + wouldn't then notice it was full and take up some slack. +* With the complicated expression above, adding a new repo when one + is full would not necessarily result in new files going to one of the 2 + repos that still have space. Some items would end up going to the full + repo. + +These can be dealt with by noticing when a repo is full and moving some +of it's files (any will do) to other repos in its group. I don't see a way +to make preferred content express that movement though; it would need to be +a manual/scripted process. + +----- + +What if we have 5 backup repos and want each file to land in 3 of them? +There's a simple change that can support that: +`balanced_amoung(group:3)` + +This works the same as before, but rather than just `N mod M`, take +`N+I mod M` where I is [0..2] to get the list of 3 repositories that want a +key. + +This does not really avoid the limitations above, but having more repos +that want each file will reduce the chances that no repo will be able to +take a given file. In the [[iabackup]] scenario, new clients will just be +assigned until all the files reach the desired level or replication. diff --git a/doc/design/iabackup/comment_4_465c0966c96a57d189f678d4fa724aa0._comment b/doc/design/iabackup/comment_4_465c0966c96a57d189f678d4fa724aa0._comment new file mode 100644 index 000000000..15faa40b5 --- /dev/null +++ b/doc/design/iabackup/comment_4_465c0966c96a57d189f678d4fa724aa0._comment @@ -0,0 +1,13 @@ +[[!comment format=mdwn + username="https://me.yahoo.com/a/idrn495us85k6mwfdMUolYIsyp4-#cf755" + subject="It's not so bad .. only about 10PB" + date="2015-04-30T01:19:38Z" + content=""" +The good news is that web-archive (.ARC) items are not publicly browsable, and that's about half of the archive's content, so you're only looking at about 10PB to backup. + +The bad news is that unless you can work something out with archive.org (which seems unlikely; web-archive items are restricted to protect them legally), or use the old waybackup interface (which I don't think works anymore), or use the wayback machine (which last I heard only supported a few hundred connections per second) you'll only be able to back up half their data. + +Still, non-web items seem like a nice place to start. + + +"""]] diff --git a/doc/design/iabackup/comment_5_7e4d1db9c69c63e79ca13db2ad87c384._comment b/doc/design/iabackup/comment_5_7e4d1db9c69c63e79ca13db2ad87c384._comment new file mode 100644 index 000000000..4d24e8f6f --- /dev/null +++ b/doc/design/iabackup/comment_5_7e4d1db9c69c63e79ca13db2ad87c384._comment @@ -0,0 +1,7 @@ +[[!comment format=mdwn + username="db48x" + subject="14 of 21PB, actually" + date="2015-04-30T02:58:05Z" + content=""" +IA helpfully did a quick count for us: https://archive.org/details/ia-bak-census_20150304 +"""]] diff --git a/doc/devblog/day_275-276__mostly_Windows.mdwn b/doc/devblog/day_275-276__mostly_Windows.mdwn index 4b5a066e2..bcdc94696 100644 --- a/doc/devblog/day_275-276__mostly_Windows.mdwn +++ b/doc/devblog/day_275-276__mostly_Windows.mdwn @@ -2,6 +2,9 @@ Mostly working on Windows recently. Fixed handling of git repos on different drive letters. Fixed crazy start menu loop. Worked around stange msysgit version problem. +Also some more work on the `concurrentprogress` branch, making the progress +display prettier. + Added one nice new feature yesterday: `git annex info $dir` now includes a table of repositories that are storing files in the directory, with their sizes. diff --git a/doc/devblog/day_277__thanks.mdwn b/doc/devblog/day_277__thanks.mdwn new file mode 100644 index 000000000..de82769bb --- /dev/null +++ b/doc/devblog/day_277__thanks.mdwn @@ -0,0 +1,20 @@ +Recent work has included improving `fsck --from remote` (and fixing a +reversion caused by the relative path changes in January), and making +annex.diskreserve be checked in more cases. And added a `git annex +required` command for setting [[required_content]]. + +Also, I want to thank several people for their work: + +* Roy sent a patch to enable http proxy support.. despite + having only learned some haskell by "30 mins with YAHT". I investigated + that more, and no patch is actually necessary, but just a newer version + of the http-client library. +* CandyAngel has been posting lots of helpful comments on the website, + including [this tip](http://git-annex.branchable.com/forum/__34__git_annex_sync__34___synced_after_8_hours/#comment-890ca1381d800ac833ccbb8c5db175ea) + that significantly speeds up a large git repository. +* Øyvind fixed a lot of typos throughout the git-annex + documentation. +* Yaroslav has created a `git-annex-standalone.deb` package + that will work on any system where debian packages can be installed, + no matter how out of date it is (within reason), using the same + methods as the standalone tarball. diff --git a/doc/devblog/day_278__release_day.mdwn b/doc/devblog/day_278__release_day.mdwn new file mode 100644 index 000000000..22d637c89 --- /dev/null +++ b/doc/devblog/day_278__release_day.mdwn @@ -0,0 +1,25 @@ +I hope that today's git-annex release will be landing in Debian unstable +toward the end of the month. And I'm looking forward to some changes that +have been blocked by wanting to keep git-annex buildable on Debian 7. + +Yesterday I got rid of the [SHA](http://hackage.haskell.org/package/SHA/) +dependency, switching git-annex to use a newer version of cryptohash for +HMAC generation (which its author Vincent Hanquez kindly added to it when I +requested it, waay back in 2013). I'm considering using the LambdaCase +extension to clean up a lot of the code next, and there are 500+ lines of +old yesod compatability code I can eventually remove. + +These changes and others will prevent backporting to the soon to be Debian +oldstable, but the standalone tarball will still work there. And, the +git-annex-standalone.deb that can be installed on any version of Debian is +[now available from the NeuroDebian repository](http://neuro.debian.net/install_pkg.html?p=git-annex-standalone), +and its build support has been merged into the source tree. + +In the run up to the release today, I also dealt with getting the +Windows build tested and working, now that it's been updated to newer +versions of rsync, ssh, etc from Cygwin. Had to add several more dlls to +the installer. That testing also turned up a case where `git-annex init` +could fail, which got a last-minute fix. + +PS, scroll down [this 10 year of git timeline](https://www.atlassian.com/git/articles/10-years-of-git/) +and see what you find! diff --git a/doc/devblog/day_279__.mdwn b/doc/devblog/day_279__.mdwn new file mode 100644 index 000000000..25eb9a64e --- /dev/null +++ b/doc/devblog/day_279__.mdwn @@ -0,0 +1,10 @@ +Posted a design for [[design/balanced_preferred_content]]. This would let +preferred content expressions assign each file to N repositories out of a +group, selected using Math. Adding a repository could optionally be +configured to automatically rebalance the files (not very bandwidth +efficiently though). I think some have asked for a feature like this +before, so read the design and see if it would be useful for you. + +Spent a while debugging a problem with a S3 remote, which seems to have +been a misconfiguration in the end. But several improvements came out of +it to make it easier to debug S3 in the future etc. diff --git a/doc/devblog/day_280__slow_week.mdwn b/doc/devblog/day_280__slow_week.mdwn new file mode 100644 index 000000000..0c49b33bc --- /dev/null +++ b/doc/devblog/day_280__slow_week.mdwn @@ -0,0 +1,16 @@ +Reduced activity this week (didn't work on the assistant after all), +but several things got done: + +Monday: Fixed `fsck --fast --from remote` to not fail when the remote +didn't support fast copy mode. And dealt with an incompatability in S3 bucket +names; the old hS3 library supported upper-case bucket names but the new +one needs them all in lower case. + +Wednesday: Caught up on most recent backlog, made some improvements +to error handling in `import`, and improved integration with KDE's file +manager to work with newer versions. + +Today: Made `import --deduplicate/--clean-duplicates` actively +verify that enough copies of a file exist before deleting it. And, +thinking about some options for batch mode access to git-annex plumbing, +to speed up things that use it a lot. diff --git a/doc/devblog/day_281__catching_up__and_arm_autobuilder_needed.mdwn b/doc/devblog/day_281__catching_up__and_arm_autobuilder_needed.mdwn new file mode 100644 index 000000000..58755de91 --- /dev/null +++ b/doc/devblog/day_281__catching_up__and_arm_autobuilder_needed.mdwn @@ -0,0 +1,43 @@ +I've not been blogging, but have been busy this week. Backlog is down to +113 messages. + +Tuesday: I got a weird bug report where `git annex get` was deleting +a file. This turned out to be a bug in `wget ftp://...` where it would +delete a symlink that was not where it had been told to download the fie +to. I put a workaround in git-annex; wget is now run in a temp +directory. But this was a legitimate wget bug, and it's now been reported +to the wget developers and will hopefully get fixed there. + +Wednesday: Added a --batch mode for several plumbing commands +(contentlocation, examinekey, and lookupkey). This avoids startup overhead, +and so lets a lot of queries be done much faster. The implementation +should make it easy to add --batch to more plumbing commands as needed, +and could probably extend to non-plumbing commands too. + +Today: The first 5 hours involved an incompatable mess of ssh and rsync +versions on Windows. A gordian knot of brokenness and depedency hell. +I finally found a solution which involves downgrading the cygwin rsync +to an older version, and using msysgit's ssh rather than cygwin's. + +Finished up today with more post-Debian-release changes. Landed a patch to +switch from dataenc to sandi that had been waiting since 2013, and got +sandi installed on all the git-annex autobuilders. Finished up with some +prep for a release tomorrow. + +---- + +Finally, Debian has a new enough ghc that it can build template haskell +on arm! So, whenever a new version of git-annex finally gets into Debian +(I hope soon), the webapp will be available on arm for those arm laptops. +Yay! + +This also means I have the opportunity to make the standalone arm build +be done much more simply. Currently it involves qemu and a separate +companion native mode container that it has to ssh to and build stuff, +that has to have the same versions of all libraries. It's just enormously +complicated and touchy. With template haskell building support, all that +complexity can fall away. + +What I'd really like to do is get a fast-ish arm box with 2gb of ram +hosted somewhere, and use that to do the builds, in native mode. +Anyone want to help provide such a box for git-annex arm autobuilds? diff --git a/doc/devblog/day_282__release_day.mdwn b/doc/devblog/day_282__release_day.mdwn new file mode 100644 index 000000000..da648c1cd --- /dev/null +++ b/doc/devblog/day_282__release_day.mdwn @@ -0,0 +1,16 @@ +Got the release out after more struggling with ssh on windows and a last +minute fix to the quvi support. + +The downloads.kitenet.net git annex repository had accumulated 6 gb of past +builds that were not publically available. I am publishing those +[on the Internet Archive now](https://archive.org/details/git-annex-builds), +so past builds can be downloaded using git-annex in that repository in the +usual way. This worked great! :) + +I have ordered a CubieTruck with 2 gb of ram to use for the new Arm builder. +Hosting still TBD. + +Looks like git-annex is almost ready to +[be included in stackage](https://github.com/fpco/stackage/pull/519#issuecomment-98590181), +which will make building it from source much less likely to fail due to +broken libraries etc. diff --git a/doc/devblog/day_283__lazy_sunday.mdwn b/doc/devblog/day_283__lazy_sunday.mdwn new file mode 100644 index 000000000..2850e2cbb --- /dev/null +++ b/doc/devblog/day_283__lazy_sunday.mdwn @@ -0,0 +1,7 @@ +Lazy afternoon spent porting git-anenx to build under ghc 7.10. Required +rather a lot of changes to build, and even more to build cleanly after the +AMP transition. + +Unfortunately, ghc 7.10 has started warning about every line that uses tab +for indentation. I had to add additional cruft to turn those warnings off +everywhere, and cannot say I'm happy about this at all. diff --git a/doc/devblog/day_283__lazy_sunday/comment_1_fdc6da890e2dc6b86b6a5fe2ebceea4a._comment b/doc/devblog/day_283__lazy_sunday/comment_1_fdc6da890e2dc6b86b6a5fe2ebceea4a._comment new file mode 100644 index 000000000..e5d4d4c5e --- /dev/null +++ b/doc/devblog/day_283__lazy_sunday/comment_1_fdc6da890e2dc6b86b6a5fe2ebceea4a._comment @@ -0,0 +1,9 @@ +[[!comment format=mdwn + username="spwhitton" + subject="comment 1" + date="2015-05-11T09:02:10Z" + content=""" +I'd be interested to hear your reasons for preferring tabs over spaces, if it's anything more than an aesthetic preference. + +The [GHC trac](https://ghc.haskell.org/trac/ghc/ticket/9230) suggests that it was turned on by default because beginners accidentally mix tabs and spaces and get very confused. +"""]] diff --git a/doc/forum/A_git-annex_helper.mdwn b/doc/forum/A_git-annex_helper.mdwn new file mode 100644 index 000000000..23a568eeb --- /dev/null +++ b/doc/forum/A_git-annex_helper.mdwn @@ -0,0 +1,6 @@ +Hi, +I have developed a [sort-of-a-GUI](http://github.com/atrent/AdMinchiam/tree/master/ga-gui) for git-annex and I'd like you to test it if you have time, see README and [screenshots](http://github.com/atrent/AdMinchiam/blob/master/ga-gui/Screenshots/GitAnnexGUI_020.png), the idea is very simple: visually tag items then generate a script based on editable templates. + +The only problem at the moment is the very slow git-annex data acquisition; not my fault, on very large annexes, in terms of number of files, the 'git-annex list' and 'git-annex metadata' commands take AGES... + +Thank you diff --git a/doc/forum/Adding_files_with_wildcard_on_Mac_Yosemite/comment_6_703ecd8e1dfc5b5b58655e27c9db838a._comment b/doc/forum/Adding_files_with_wildcard_on_Mac_Yosemite/comment_6_703ecd8e1dfc5b5b58655e27c9db838a._comment new file mode 100644 index 000000000..30a6bd26f --- /dev/null +++ b/doc/forum/Adding_files_with_wildcard_on_Mac_Yosemite/comment_6_703ecd8e1dfc5b5b58655e27c9db838a._comment @@ -0,0 +1,9 @@ +[[!comment format=mdwn + username="clacke" + subject="regression" + date="2015-04-22T11:39:19Z" + content=""" +The fix is a regression compared to previous behavior. + +Being able to do `git annex add '*'.{JPG,jp{,e}g,AVI,avi,MOV,mov,3GP,3gp,mp4,tif,pdf,PDF,doc,pps,bmp,png,mp{,e}g,MP{,E}G,wav,WAV,nef,NEF,thm,THM,key.gz,ogg,OGG,ppt,GIF,gif,m4a}` in the root of my binary files archive to catch any newly copied files was a real feature. This worked until recently. I was just about to add a bug report stating the same problem as the OP for the OSX homebrew version, but I verified with the latest Linux standalone and there wildcards were completely turned off. +"""]] diff --git a/doc/forum/Adding_files_with_wildcard_on_Mac_Yosemite/comment_7_dbe40fef2ba65cc0f1c20f41f2daab4d._comment b/doc/forum/Adding_files_with_wildcard_on_Mac_Yosemite/comment_7_dbe40fef2ba65cc0f1c20f41f2daab4d._comment new file mode 100644 index 000000000..47b00a5f0 --- /dev/null +++ b/doc/forum/Adding_files_with_wildcard_on_Mac_Yosemite/comment_7_dbe40fef2ba65cc0f1c20f41f2daab4d._comment @@ -0,0 +1,11 @@ +[[!comment format=mdwn + username="clacke" + subject="workaround" + date="2015-04-22T13:31:10Z" + content=""" +This is a workaround to the missing functionality, generating an `--include` expression: + +[[!format sh \"\"\" +git annex add $(printf -- '--include=*.%s --or ' jp{,e}g avi mov 3gp mp4 tif pdf doc pps bmp png mp{,e}g wav nef thm key.gz ogg ppt gif) --include='*.m4a' +\"\"\"]] +"""]] diff --git a/doc/forum/Adding_files_with_wildcard_on_Mac_Yosemite/comment_8_d8620ce7b3dbb81c0d3d0b09ded1deb0._comment b/doc/forum/Adding_files_with_wildcard_on_Mac_Yosemite/comment_8_d8620ce7b3dbb81c0d3d0b09ded1deb0._comment new file mode 100644 index 000000000..ac95deb7f --- /dev/null +++ b/doc/forum/Adding_files_with_wildcard_on_Mac_Yosemite/comment_8_d8620ce7b3dbb81c0d3d0b09ded1deb0._comment @@ -0,0 +1,18 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 8""" + date="2015-04-24T17:26:01Z" + content=""" +The old behavior was accidental, and was never documented. When you use +such undocumented behaviors, you're taking the risk of bugfixes breaking +things. It's not fair to call that a "regression". If I had to worry about +every bugfix breaking users who relied on the old buggy behavior in some +way, I could just stop working now. + +It might be reasonable to add the "expand wildcards rather than letting the +shell do it" feature, as an option. Of course, it would need to be tested +for every git-annex command and problems like the one that caused this bug +to be noticed in the first place dealt with, for every git-annex command. +Using --include and --exclude, which already work seems pretty reasonable +instead. +"""]] diff --git a/doc/forum/Adding_files_with_wildcard_on_Mac_Yosemite/comment_9_6a43f52449c4a38a986772ec9d65f9d5._comment b/doc/forum/Adding_files_with_wildcard_on_Mac_Yosemite/comment_9_6a43f52449c4a38a986772ec9d65f9d5._comment new file mode 100644 index 000000000..210ece464 --- /dev/null +++ b/doc/forum/Adding_files_with_wildcard_on_Mac_Yosemite/comment_9_6a43f52449c4a38a986772ec9d65f9d5._comment @@ -0,0 +1,7 @@ +[[!comment format=mdwn + username="clacke" + subject="not regression" + date="2015-04-27T12:13:12Z" + content=""" +Fair enough. And I have got over my initial frustration with the change in behavior. The workaround works. +"""]] diff --git a/doc/forum/Cant_see_git-annex-shell_via_SSH_in_OSX/comment_1_1fafdc4ed4a0f601918361dca688aa6c._comment b/doc/forum/Cant_see_git-annex-shell_via_SSH_in_OSX/comment_1_1fafdc4ed4a0f601918361dca688aa6c._comment new file mode 100644 index 000000000..edbf2edfc --- /dev/null +++ b/doc/forum/Cant_see_git-annex-shell_via_SSH_in_OSX/comment_1_1fafdc4ed4a0f601918361dca688aa6c._comment @@ -0,0 +1,14 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2015-04-18T20:05:42Z" + content=""" +This is a common gotcha; bash sources something like .bashrc +or whatever that puts /usr/local/bin/ in your path when it's a login shell. +However, when git-annex is sshing in, there is no login shell, +and bash does not source any dotfiles, and so git-annex-shell in not in +PATH. + +The solution is probably to move or symlink it to some other directory +that is in PATH always. +"""]] diff --git a/doc/forum/Git-annex_deleted_all_my_files__44___how_can_I_recover_them__63__.mdwn b/doc/forum/Git-annex_deleted_all_my_files__44___how_can_I_recover_them__63__.mdwn new file mode 100644 index 000000000..c5a0d70fb --- /dev/null +++ b/doc/forum/Git-annex_deleted_all_my_files__44___how_can_I_recover_them__63__.mdwn @@ -0,0 +1,19 @@ +I am trying to follow this guide: + +https://git-annex.branchable.com/walkthrough/ + +I: + +* created a repository +* added a remote +* added all the files via `git annex add .` +* commited my changes +* synced my repositories + + +By the time I tied to actually get the content from repo one into repo two (via `git annex get folder`) I noticed that all of my stuff had been deleted from repo one (and not backed up by git annex anywhere else!). + +How can I get my stuff back? + +Also, why would any software which is meant for backup or archiving (or any sane software in general) delete stuff without me actually typing `rm` or `delete` into the terminal and without giving me a big warning message and a confirmation prompt? + diff --git a/doc/forum/Git-annex_deleted_all_my_files__44___how_can_I_recover_them__63__/comment_1_65b41a6eb28261e04e4fe8732f97a1f1._comment b/doc/forum/Git-annex_deleted_all_my_files__44___how_can_I_recover_them__63__/comment_1_65b41a6eb28261e04e4fe8732f97a1f1._comment new file mode 100644 index 000000000..c06b202f2 --- /dev/null +++ b/doc/forum/Git-annex_deleted_all_my_files__44___how_can_I_recover_them__63__/comment_1_65b41a6eb28261e04e4fe8732f97a1f1._comment @@ -0,0 +1,9 @@ +[[!comment format=mdwn + username="CandyAngel" + subject="comment 1" + date="2015-04-18T07:16:01Z" + content=""" +Are you sure the files are actually deleted? You don't mention using direct mode or anything, so you should be able to reset the git repository to the last commit you did (restoring the symlinks) and be back on track. + +Sounds like you missed something either in your description of the problem, or while following the tutorial. +"""]] diff --git a/doc/forum/Git-annex_deleted_all_my_files__44___how_can_I_recover_them__63__/comment_2_200a869f335909566b9ddab3032fd5a2._comment b/doc/forum/Git-annex_deleted_all_my_files__44___how_can_I_recover_them__63__/comment_2_200a869f335909566b9ddab3032fd5a2._comment new file mode 100644 index 000000000..bb55a04ed --- /dev/null +++ b/doc/forum/Git-annex_deleted_all_my_files__44___how_can_I_recover_them__63__/comment_2_200a869f335909566b9ddab3032fd5a2._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkW9u-8uqR62QBZjeTNCXsL7Ds55dAMGbA" + nickname="Horea" + subject="comment 2" + date="2015-04-28T10:15:24Z" + content=""" +Apparently what happened is that whenever you want to set up git annex, you have to create just one repository, them *move* your files to a temporary folder on all your other media, *clone* the repository to the other media, and then copy all your files back to the clone and add them to git annex. This is a horrible workflow and not mentioned anywhere. If you do the intuitive thing and start by making a bunch of repos, because all repos are created equal, git annex will happily delete all your stuff on some of them. + +Indeed, as you correctly mention, this can go horribly wrong if you use direct mode and cannot recover your files. A shame, really, that direct mode is the only mode in which you actually have write and copy access to your files, and thus, for most - including myself - the only sane way of managing your files. For the record, I was able to retrieve my files, but to this day git annex is the only software I ever used that ever managed to lose my data. +"""]] diff --git a/doc/forum/How_do_I_backup_my_data_to_blue_ray_disks__63__.mdwn b/doc/forum/How_do_I_backup_my_data_to_blue_ray_disks__63__.mdwn new file mode 100644 index 000000000..b94111e94 --- /dev/null +++ b/doc/forum/How_do_I_backup_my_data_to_blue_ray_disks__63__.mdwn @@ -0,0 +1 @@ +I have several TB of media on a Debian ZFS server. If I created a git-annex repo for the data, how hard would it be get git-annex (using bup I assume) to back up the files onto a set of Blu Ray disks? I realize that 8TB of data would take about 320 BR 25GB disks, but it seems like git annex would be great for identifying what disk(s) I needed to recover a file. I'm good at bash scripting, and use git daily. I have no experience with git-annex or bup however. Any links to information or suggestions is very appreciated. Thanks! diff --git a/doc/forum/How_do_I_backup_my_data_to_blue_ray_disks__63__/comment_1_ea2f6ca0768c55fa136606cf091471cd._comment b/doc/forum/How_do_I_backup_my_data_to_blue_ray_disks__63__/comment_1_ea2f6ca0768c55fa136606cf091471cd._comment new file mode 100644 index 000000000..d004c06a0 --- /dev/null +++ b/doc/forum/How_do_I_backup_my_data_to_blue_ray_disks__63__/comment_1_ea2f6ca0768c55fa136606cf091471cd._comment @@ -0,0 +1,25 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2015-05-01T21:36:59Z" + content=""" +There was a previous thread about using DVDS: +<http://git-annex.branchable.com/forum/Managing_a_large_number_of_files_archived_on_many_pieces_of_read-only_medium___40__E.G._DVDs__41__> + +If the bluerays are rewritable, I'd probably just slap a Real Filesystem +(ext2 not isofs) on there and put a regular git-annex repo on it. I'd probably +run git-annex with the option "-c annex.alwayscommit=false" to prevent it +making many commits to the repo on the blueray, which would rewrite parts of +this disk, perhaps too often. + +Or, to avoid any rewrites at all (except to directory metadata), +I might use a directory special remote on the blueray. + +I don't see much benefit to using bup over the directory special remote. + +If the bluerays are not rewritable, I might try making the git-annex repo +in a temporary directory on the hard disk, and then generating the ISO +from that once I've filled it up. Should work fine, I might set "remote.<name>.annex-readonly" +to true in git repos that had such a disk as a remote to let git-annex +know to not try to write to it. +"""]] diff --git a/doc/forum/How_do_I_backup_my_data_to_blue_ray_disks__63__/comment_2_d79387905af9f8dce77f9aa85f2d6a03._comment b/doc/forum/How_do_I_backup_my_data_to_blue_ray_disks__63__/comment_2_d79387905af9f8dce77f9aa85f2d6a03._comment new file mode 100644 index 000000000..4d7caf749 --- /dev/null +++ b/doc/forum/How_do_I_backup_my_data_to_blue_ray_disks__63__/comment_2_d79387905af9f8dce77f9aa85f2d6a03._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://me.yahoo.com/a/e1w.8yMTnpUh.5fOXneP92pTdy23XJPwx84uLSM-#aca7c" + nickname="Michael" + subject="re: backup my data to blue ray disks" + date="2015-05-11T21:13:19Z" + content=""" +Thank you for your response. The disks are BD-R, so they are not rewritable. My experience in the past with rewritable disks has been that they work about as well as the average TV infomercial product. Do you have any opinions or suggestions on using par2 files on the disks? +"""]] diff --git a/doc/forum/How_do_I_hide_files_not_present_in_the_local_annex__63__.mdwn b/doc/forum/How_do_I_hide_files_not_present_in_the_local_annex__63__.mdwn new file mode 100644 index 000000000..72e39cc36 --- /dev/null +++ b/doc/forum/How_do_I_hide_files_not_present_in_the_local_annex__63__.mdwn @@ -0,0 +1,2 @@ +Files only present in remotes show up as broken symlinks. That's great for knowing what files exist, but sometimes I just want to browse the files that are actually present. In this case, the many broken symlinks are just clutter. +Is there a straightforward way to switch to a view that shows only locally present files? diff --git a/doc/forum/How_do_I_hide_files_not_present_in_the_local_annex__63__/comment_1_005e9254a1239164df34ab5fbf2115a8._comment b/doc/forum/How_do_I_hide_files_not_present_in_the_local_annex__63__/comment_1_005e9254a1239164df34ab5fbf2115a8._comment new file mode 100644 index 000000000..b15296b33 --- /dev/null +++ b/doc/forum/How_do_I_hide_files_not_present_in_the_local_annex__63__/comment_1_005e9254a1239164df34ab5fbf2115a8._comment @@ -0,0 +1,27 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawlZ-6dtxJY4cP7shhvV8E6YyuV0Rak8it4" + nickname="Giovanni" + subject="direct mode: avoid broken symlinks; indirect mode: matching-options in views (not woring)" + date="2015-04-18T07:56:32Z" + content=""" +I would like to see only locally present files too, for some of my use case (e.g.: movies exported via NFS to a XBMS/Kodi media player) + +I'm using git annex (debian) version 5.20141024~bpo70+1 + +# [direct mode](http://git-annex.branchable.com/direct_mode/) + +as far as I understand, in direct mode broken symlinks (or text files placeholders for filesystems not supporting symlinks) are there by [design]( +https://git-annex.branchable.com/design/assistant/desymlink/), in particular in [partial content]( +https://git-annex.branchable.com/design/assistant/partial_content/) use case where \"there needs to be a way for the user to browse files not on the gadget and request they be transferred to it\" + +if there are no other reasons to have broken symlinks (or placeholders) I think git-annex should avoid them (at least in direct mode): if a user needs a file not listed by regular filesystem tools she can simply `git-annex find` it and `git-annex get` it: or do I miss something here? + +please consider avoiding broken symlinks or placeholders + +# indirect mode + +in indirect mode we could live with broken symlinks using views; I tried `git annex view /=* --include=\"*\" --in=here` but the resulting tree was not filtered by matching-options + +are matching-options intended to work with the view command or not? + +"""]] diff --git a/doc/forum/How_do_I_hide_files_not_present_in_the_local_annex__63__/comment_2_f376604560c36a0aa5afa4619797b396._comment b/doc/forum/How_do_I_hide_files_not_present_in_the_local_annex__63__/comment_2_f376604560c36a0aa5afa4619797b396._comment new file mode 100644 index 000000000..27197fc82 --- /dev/null +++ b/doc/forum/How_do_I_hide_files_not_present_in_the_local_annex__63__/comment_2_f376604560c36a0aa5afa4619797b396._comment @@ -0,0 +1,9 @@ +[[!comment format=mdwn + username="GiovanniBiscuolo" + subject="hide files not present in the local annex: related forum post" + date="2015-04-23T11:00:56Z" + content=""" +[How to hide broken symlinks](http://git-annex.branchable.com/forum/How_to_hide_broken_symlinks/) + +using views (but see comments) +"""]] diff --git a/doc/forum/How_to_find_deleted_files_that_I_know_have_been_backed_up__63__.mdwn b/doc/forum/How_to_find_deleted_files_that_I_know_have_been_backed_up__63__.mdwn new file mode 100644 index 000000000..998c608dd --- /dev/null +++ b/doc/forum/How_to_find_deleted_files_that_I_know_have_been_backed_up__63__.mdwn @@ -0,0 +1,9 @@ +<video src=http://s.natalian.org/2015-04-15/git-annex-issues.mp4></video> + +I removed some archived directories perhaps foolishly with `rm -rf`. How do I find the files that I've had deleted? + + + +I also have an issue where by I want one command to sync between two hardrives and [github](https://github.com/kaihendry/uploadme). Or do I have to: `git-annex move --to {foo,bar}; git-annex drop; git-annex sync`? Basically I want copies everywhere except on my laptop (X1C3). + +I also expected my git dir to be much smaller than 1.4GB after dropping everything. Thanks! diff --git a/doc/forum/How_to_find_deleted_files_that_I_know_have_been_backed_up__63__/comment_1_8332f71241335a31e270407477bd84f3._comment b/doc/forum/How_to_find_deleted_files_that_I_know_have_been_backed_up__63__/comment_1_8332f71241335a31e270407477bd84f3._comment new file mode 100644 index 000000000..d142d742c --- /dev/null +++ b/doc/forum/How_to_find_deleted_files_that_I_know_have_been_backed_up__63__/comment_1_8332f71241335a31e270407477bd84f3._comment @@ -0,0 +1,18 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2015-04-18T19:06:27Z" + content=""" +Is your titular question a git-annex question, or a generic git question? +Because I think the answer would be the same in either case. Ie, `git log` +or `git status` will tell you what changes you've made to the work tree. + +If you've deleted files from your git working tree with rm -rf, then +their content is still stored in the .git directory. This is also true +when using git-annex (unless you're using direct mode). + +The size of your .git directory might be a clue: If you've deleted +files from the working tree, you may not have dropped their content +from git annex. You can use `git annex unused` to find and clean up +such file contents. +"""]] diff --git a/doc/forum/How_to_find_deleted_files_that_I_know_have_been_backed_up__63__/comment_2_984286a35ec828f1e8dda928ea577571._comment b/doc/forum/How_to_find_deleted_files_that_I_know_have_been_backed_up__63__/comment_2_984286a35ec828f1e8dda928ea577571._comment new file mode 100644 index 000000000..15a48bfe4 --- /dev/null +++ b/doc/forum/How_to_find_deleted_files_that_I_know_have_been_backed_up__63__/comment_2_984286a35ec828f1e8dda928ea577571._comment @@ -0,0 +1,35 @@ +[[!comment format=mdwn + username="http://hendry.iki.fi/" + nickname="Kai Hendry" + subject="git annex unused doesn't help" + date="2015-04-19T13:24:50Z" + content=""" +This is a git-annex question. I want to find files that are not in my working tree but are on the backups on my two USB hard drives. + + +I see in this commit the images that I deleted: <https://github.com/kaihendry/uploadme/commit/c53dbdd9bd7879d68635a2adc81a7bc59a84c5ea> + +How can I simply just get a listing of where the copies are? + +For example I deleted IMG_4110.JPG. + +But when I run: + + X1C3:~/media/uploadme$ git-annex whereis IMG_4110.JPG + git-annex: IMG_4110.JPG not found + +I am pretty confident I have a copy of this on my external USB drives. + +Also tried another file: + + X1C3:~/media/uploadme$ git-annex whereis IMG_4558.JPG + git-annex: IMG_4558.JPG not found + X1C3:~/media/uploadme$ git-annex whereis 2014-10-05/IMG_4558.JPG + git-annex: 2014-10-05/IMG_4558.JPG not found + +So I am still in the dark how to see how git-annex tracks deleted files across my remotes. + + + + +"""]] diff --git a/doc/forum/How_to_find_deleted_files_that_I_know_have_been_backed_up__63__/comment_3_29f6f9df1ad22113e9690b0d1da36ba0._comment b/doc/forum/How_to_find_deleted_files_that_I_know_have_been_backed_up__63__/comment_3_29f6f9df1ad22113e9690b0d1da36ba0._comment new file mode 100644 index 000000000..3a39314a9 --- /dev/null +++ b/doc/forum/How_to_find_deleted_files_that_I_know_have_been_backed_up__63__/comment_3_29f6f9df1ad22113e9690b0d1da36ba0._comment @@ -0,0 +1,13 @@ +[[!comment format=mdwn + username="CandyAngel" + subject="comment 3" + date="2015-04-19T15:51:45Z" + content=""" +Have you tried checking out a commit which it was present in and then using whereis? + +Not sure if you can do: + + git annex whereis --key $KEY + +But if not, may be worth a feature request! +"""]] diff --git a/doc/forum/How_to_find_deleted_files_that_I_know_have_been_backed_up__63__/comment_4_43549b3d231f52cf53a66c477c34a708._comment b/doc/forum/How_to_find_deleted_files_that_I_know_have_been_backed_up__63__/comment_4_43549b3d231f52cf53a66c477c34a708._comment new file mode 100644 index 000000000..7d29bec3a --- /dev/null +++ b/doc/forum/How_to_find_deleted_files_that_I_know_have_been_backed_up__63__/comment_4_43549b3d231f52cf53a66c477c34a708._comment @@ -0,0 +1,15 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 4""" + date="2015-04-21T20:22:25Z" + content=""" +Yeah, you need to `git checkout` a tree from before you deleted the files, +and then you'll be able to use `git annex whereis` in there on the deleted +files. This will tell you where the files are currently located (not some historical data). + +`git annex whereis --key` is indeed an alternative approach, if you know +the key corresponding to the deleted file. You can see the keys in the git +diff, if you know where to look. + +[[internals]] will let you understand how this all really works. +"""]] diff --git a/doc/forum/How_to_find_deleted_files_that_I_know_have_been_backed_up__63__/comment_5_94aca10f84783f35d927dbefbeba263a._comment b/doc/forum/How_to_find_deleted_files_that_I_know_have_been_backed_up__63__/comment_5_94aca10f84783f35d927dbefbeba263a._comment new file mode 100644 index 000000000..77cf1c289 --- /dev/null +++ b/doc/forum/How_to_find_deleted_files_that_I_know_have_been_backed_up__63__/comment_5_94aca10f84783f35d927dbefbeba263a._comment @@ -0,0 +1,16 @@ +[[!comment format=mdwn + username="http://hendry.iki.fi/" + nickname="Kai Hendry" + subject="Two more questions" + date="2015-05-07T07:40:41Z" + content=""" +How do I undelete in my case then? I.e. not `git checkout ` to find where they are. + +`git revert c53dbdd9bd7879d68635a2adc81a7bc59a84c5ea` + + + +Assuming I make no effort to undelete or recover the files: + +The (deleted) files I have copies of on my other two hard drives. Do they detect I've deleted the files and just clean up? How does that process work? +"""]] diff --git a/doc/forum/How_to_hide_broken_symlinks/comment_3_e3606aa746f516fc771d5d9bf93d70af._comment b/doc/forum/How_to_hide_broken_symlinks/comment_3_e3606aa746f516fc771d5d9bf93d70af._comment new file mode 100644 index 000000000..2103ec35e --- /dev/null +++ b/doc/forum/How_to_hide_broken_symlinks/comment_3_e3606aa746f516fc771d5d9bf93d70af._comment @@ -0,0 +1,7 @@ +[[!comment format=mdwn + username="GiovanniBiscuolo" + subject="Related forum post" + date="2015-04-23T11:03:39Z" + content=""" +[How do I hide files not present in the local annex?](http://git-annex.branchable.com/forum/How_do_I_hide_files_not_present_in_the_local_annex__63__/) +"""]] diff --git a/doc/forum/How_to_hide_broken_symlinks/comment_4_c6e3ef3ba5f6e9e54d998bcbe3035650._comment b/doc/forum/How_to_hide_broken_symlinks/comment_4_c6e3ef3ba5f6e9e54d998bcbe3035650._comment new file mode 100644 index 000000000..206333db7 --- /dev/null +++ b/doc/forum/How_to_hide_broken_symlinks/comment_4_c6e3ef3ba5f6e9e54d998bcbe3035650._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawnVnsqEy82M-MuS2gLri-az83wSQ6lXSrc" + nickname="Jean" + subject="comment 4" + date="2015-04-26T01:19:21Z" + content=""" +If a file is dropped while in a view, how about letting the relevant symlinks become broken until the view is refreshed? +"""]] diff --git a/doc/forum/Multiple_repos_on_same_path.mdwn b/doc/forum/Multiple_repos_on_same_path.mdwn new file mode 100644 index 000000000..d90a9f0fb --- /dev/null +++ b/doc/forum/Multiple_repos_on_same_path.mdwn @@ -0,0 +1,12 @@ +I want to use git-annex to manage some data which I keep at, say, /home/my_user/data . +I have multiple external storage devices for this repo, such as /run/media/my_user/data1 e + +BUT, I also have multiple machines, e.g. my desktop and my laptop. +These all have the data under /home/my_user/data, as I like to keep my paths consistent. +So, when I connect my external media to these machines, practically they see the same repo path, though these are different repos. + +1. Should I try to create multiple repos with the same path, or should I just add one + +2. Can I expect any major issues from this set-up? + + diff --git a/doc/forum/Multiple_repos_on_same_path/comment_1_67ed823c4c40f5acf12b48eb75d7afa8._comment b/doc/forum/Multiple_repos_on_same_path/comment_1_67ed823c4c40f5acf12b48eb75d7afa8._comment new file mode 100644 index 000000000..5ab93d9ed --- /dev/null +++ b/doc/forum/Multiple_repos_on_same_path/comment_1_67ed823c4c40f5acf12b48eb75d7afa8._comment @@ -0,0 +1,32 @@ +[[!comment format=mdwn + username="https://sunny256.wordpress.com/" + nickname="sunny256" + subject="Create symlink in the root directory" + date="2015-05-12T13:04:18Z" + content=""" +My way of dealing with this is to create a symlink in the root directory that point to the root directory, like + +[[!format sh \"\"\" +$ cd / +$ sudo ln -sv . compname +[sudo] password for sunny: hunter2 +'compname' -> '.' +$ ls -l compname +lrwxrwxrwx 1 root root 1 May 12 14:52 compname -> . +$ +\"\"\"]] + +where `compname` is the hostname for the computer. Now you can create paths like + +[[!format sh \"\"\" +git remote add comp-a /comp-a/home/my_user/data/repo +git remote add comp-b /comp-b/home/my_user/data/repo +\"\"\"]] + +This is also useful in other scripts where fetching data from the right directory on the wrong computer is bad. Also, this is a cheap way for a script to determine which computer it's running on: + +[[!format sh \"\"\" +test -L \"/comp-a\" && echo Running on computer comp-a +\"\"\"]] + +"""]] diff --git a/doc/forum/Slow_transfer_speeds_on_copy_in_Windows/comment_4_aee3fc6be01bb75709451eea0decf112._comment b/doc/forum/Slow_transfer_speeds_on_copy_in_Windows/comment_4_aee3fc6be01bb75709451eea0decf112._comment new file mode 100644 index 000000000..ed2f3cc35 --- /dev/null +++ b/doc/forum/Slow_transfer_speeds_on_copy_in_Windows/comment_4_aee3fc6be01bb75709451eea0decf112._comment @@ -0,0 +1,13 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 4""" + date="2015-04-18T19:12:19Z" + content=""" +Well, git-annex bundles a copy of rsync, and the one that it includes +is the one from cygwin. + +It sounds like there's a newer version, also from cygwin, that is faster? + +I've pinged the git-annex Windows autobuilder admin to see if we can +upgrade that. +"""]] diff --git a/doc/forum/Slow_transfer_speeds_on_copy_in_Windows/comment_5_b1841fc129a9ce6d1c22840ee648f958._comment b/doc/forum/Slow_transfer_speeds_on_copy_in_Windows/comment_5_b1841fc129a9ce6d1c22840ee648f958._comment new file mode 100644 index 000000000..147ca7682 --- /dev/null +++ b/doc/forum/Slow_transfer_speeds_on_copy_in_Windows/comment_5_b1841fc129a9ce6d1c22840ee648f958._comment @@ -0,0 +1,13 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 5""" + date="2015-04-18T19:26:25Z" + content=""" +<https://www.itefix.net/cwrsync> + +Hmm, ok, so that's not the cygwin rsync and it has a gui and bundles its +own ssh that won't behave the same as the one bundled with git-annex. + +Maybe at least some of the speed improvements could be had by just +upgrading the rsync and ssh from cygwin though.. +"""]] diff --git a/doc/forum/Slow_transfer_speeds_on_copy_in_Windows/comment_6_1ce9bb47dadd2b1c500b2a20fd669907._comment b/doc/forum/Slow_transfer_speeds_on_copy_in_Windows/comment_6_1ce9bb47dadd2b1c500b2a20fd669907._comment new file mode 100644 index 000000000..43f5c3f2e --- /dev/null +++ b/doc/forum/Slow_transfer_speeds_on_copy_in_Windows/comment_6_1ce9bb47dadd2b1c500b2a20fd669907._comment @@ -0,0 +1,9 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawlh1G1u_AMJEyADqlfuzV2cePniocDyK6A" + nickname="Adam" + subject="comment 6" + date="2015-04-20T17:40:38Z" + content=""" +There are pathing issues with other applications if I add cygwin to the path or are you saying copy over cygwin's copy? I did have issues with ssh and had to copy all the files to the bin dir rather than the cmd dir to get it to work. This also screwed up my home dir, making c:\ my home. I went with it, since I'm the only user, and copied my .ssh folder to C:\... not the most secure option, but it worked. (Note: it seemed to ignore the $HOME env variable) + +"""]] diff --git a/doc/forum/Slow_transfer_speeds_on_copy_in_Windows/comment_7_4b78b200b884c4ac7c052055b3e26784._comment b/doc/forum/Slow_transfer_speeds_on_copy_in_Windows/comment_7_4b78b200b884c4ac7c052055b3e26784._comment new file mode 100644 index 000000000..a49eacd14 --- /dev/null +++ b/doc/forum/Slow_transfer_speeds_on_copy_in_Windows/comment_7_4b78b200b884c4ac7c052055b3e26784._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawlh1G1u_AMJEyADqlfuzV2cePniocDyK6A" + nickname="Adam" + subject="comment 7" + date="2015-04-20T17:43:11Z" + content=""" +I'm not using cygwin for git-annex. I'm using msysgit and git's command window. +"""]] diff --git a/doc/forum/Slow_transfer_speeds_on_copy_in_Windows/comment_8_97ef11581c5dc6eeeabb4b244bdc6c30._comment b/doc/forum/Slow_transfer_speeds_on_copy_in_Windows/comment_8_97ef11581c5dc6eeeabb4b244bdc6c30._comment new file mode 100644 index 000000000..9363c0bfa --- /dev/null +++ b/doc/forum/Slow_transfer_speeds_on_copy_in_Windows/comment_8_97ef11581c5dc6eeeabb4b244bdc6c30._comment @@ -0,0 +1,7 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + subject="comment 8" + date="2015-04-20T17:43:48Z" + content=""" +We've updated the autobuilder, so the daily build now has the most recent rsync and ssh from cygwin included in it. I'd be curious if that improves the speed any. +"""]] diff --git a/doc/forum/Symlink_points_to_old_version.mdwn b/doc/forum/Symlink_points_to_old_version.mdwn new file mode 100644 index 000000000..257a7d5aa --- /dev/null +++ b/doc/forum/Symlink_points_to_old_version.mdwn @@ -0,0 +1,13 @@ +Hello everyone, + +I have some PDF documents in a git annex repository. I appended a page to (several) of these PDF documents the following way: + +- `git annex edit file.pdf` +- Append page to file.pdf +- `git annex add file.pdf` +- `git commit` + +If I now look at file.pdf it will not have the appended page. But if do `git annex edit file.pdf` again I will get the version with the appended page. `git annex add file.pdf` and the page "disappears" again. Anyone got any tips on how to solve this "mystery"? + +All the best, +Per diff --git a/doc/forum/Using_a_glacier_remote_as_a_backup.mdwn b/doc/forum/Using_a_glacier_remote_as_a_backup.mdwn new file mode 100644 index 000000000..2baaa7f91 --- /dev/null +++ b/doc/forum/Using_a_glacier_remote_as_a_backup.mdwn @@ -0,0 +1,28 @@ +Hallo everybody, + +I have an ARM based Synology NAS and I would like to use git annex to replace the "backup" solution provided by Synology. The basic idea is that I want files in a safe place when the house burns down or they get removed by accident. + +Since I only care about the latest version and want to make really sure local programs (cifs service, photostation and so on) do not run into trouble caused by symlinks, I guess direct mode is what I want. I have been tinkering around and things seem to be working for the most part. A few questions remain: + +Assuming I have all files synced to glacier and then accidentally remove all content and try to recover with the bare repo - with metadata but without content. the situation looks like this. + + ➜ syno-archive git:(annex/direct/master) git annex status test.txt + D test.txt + ➜ syno-archive git:(annex/direct/master) + +I try to get my files back out of glacier: + + ➜ syno-archive git:(annex/direct/master) git annex get test.txt + get test.txt (from glacier...) + ok + (Recording state in git...) + ➜ syno-archive git:(annex/direct/master) + +Contrary to my expectation, text.txt did not appear on disk. + +Given the bare repo, you would one recover all content (thousands of files)? I expected "git annex get --all" to do the trick. + +PS: This is from git-annex version 5.20141125 + +regards +Andreas diff --git a/doc/forum/Using_a_glacier_remote_as_a_backup/comment_1_bb6022943ec739b1e80351c76f259e89._comment b/doc/forum/Using_a_glacier_remote_as_a_backup/comment_1_bb6022943ec739b1e80351c76f259e89._comment new file mode 100644 index 000000000..c6899e422 --- /dev/null +++ b/doc/forum/Using_a_glacier_remote_as_a_backup/comment_1_bb6022943ec739b1e80351c76f259e89._comment @@ -0,0 +1,17 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2015-05-11T16:58:01Z" + content=""" +Your status shows that test.txt is deleted. +`git annex get` does not un-delete files, it just gets the *content* +of a file (whether that file is deleted or not). + +You can use normal git commands to un-delete the file. Ie, "git checkout +text.txt". If you're using direct mode, you can't use such commands, but +can use "git annex undo" to undo a deletion. + +Normally, if you have a bare repo, you'll want to clone it to get a non-bare +repo. I suspect you did something else that resulted in your repo being in +this state were files are deleted. +"""]] diff --git a/doc/forum/Using_a_glacier_remote_as_a_backup/comment_2_41d47f0ca2e05e3296face5f89b819da._comment b/doc/forum/Using_a_glacier_remote_as_a_backup/comment_2_41d47f0ca2e05e3296face5f89b819da._comment new file mode 100644 index 000000000..66b08b2fe --- /dev/null +++ b/doc/forum/Using_a_glacier_remote_as_a_backup/comment_2_41d47f0ca2e05e3296face5f89b819da._comment @@ -0,0 +1,15 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawnwNDA50ZupMvOgpgDqzDRyu5B-mYlVwa4" + nickname="Andreas" + subject="comment 2" + date="2015-05-12T06:52:04Z" + content=""" +Actually, I just removed all annexed files to try out disaster recovery. I stumbled across the `undo` command. Unfortunately, it does not seem available on Ubuntu running version 5.20141125 shipped by vivid (`git-annex: Unknown command 'undo'`). + +Am I missing something, or how do I get a version supporting `undo`? + +thanks +Andreas + + +"""]] diff --git a/doc/forum/Where_did_my_files_go__63__.mdwn b/doc/forum/Where_did_my_files_go__63__.mdwn new file mode 100644 index 000000000..f7d811d8f --- /dev/null +++ b/doc/forum/Where_did_my_files_go__63__.mdwn @@ -0,0 +1,20 @@ +I import some files that I've seen before somewhere: + + $ git annex import --deduplicate .../download + ... + import download/What_is_The_Digital_Fiction_Factory-Conker_Media.pdf (duplicate) ok + ... + +But the resulting download directory is empty, and `list` doesn't show any of the files: + + $ git annex list --allrepos What_is_The_Digital_Fiction_Factory-Conker_Media.pdf + here + |... + |... + |... + |... + |... + |... + git-annex: What_is_The_Digital_Fiction_Factory-Conker_Media.pdf not found + +How do I find out where this file can be found? diff --git a/doc/forum/Where_did_my_files_go__63__/comment_1_3ff3ffa95eb2745ff9ec2a903e071d97._comment b/doc/forum/Where_did_my_files_go__63__/comment_1_3ff3ffa95eb2745ff9ec2a903e071d97._comment new file mode 100644 index 000000000..f0b2ae3e7 --- /dev/null +++ b/doc/forum/Where_did_my_files_go__63__/comment_1_3ff3ffa95eb2745ff9ec2a903e071d97._comment @@ -0,0 +1,9 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawnVnsqEy82M-MuS2gLri-az83wSQ6lXSrc" + nickname="Jean" + subject="comment 1" + date="2015-05-06T09:21:19Z" + content=""" +I did a dump of all known files: `$ git annex whereis > ../git-annex-whereis` +No sign of the imported file. +"""]] diff --git a/doc/forum/Where_did_my_files_go__63__/comment_2_9d902e66ca19b3332f4454f694d4a12e._comment b/doc/forum/Where_did_my_files_go__63__/comment_2_9d902e66ca19b3332f4454f694d4a12e._comment new file mode 100644 index 000000000..b1b186ab9 --- /dev/null +++ b/doc/forum/Where_did_my_files_go__63__/comment_2_9d902e66ca19b3332f4454f694d4a12e._comment @@ -0,0 +1,16 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 2""" + date="2015-05-06T18:04:35Z" + content=""" +I don't think you're using a recent version of git-annex, the "(duplicate)" +message has changed to something else a while ago. + +Also, that message is only shown if you pass the --deduplicate flag, which +you don't show in your transcript. + +That flag does what it says on the tin: If the file is a duplicate of one +git-annex has seen before, the file is deleted from the import location. +(The next version of git-annex does add an additional check that the +content of the file is present in the annex.) +"""]] diff --git a/doc/forum/Why_are_ignored_files_being_deleted__63__/comment_2_178aa574855a3bfffab4b21f90a84092._comment b/doc/forum/Why_are_ignored_files_being_deleted__63__/comment_2_178aa574855a3bfffab4b21f90a84092._comment new file mode 100644 index 000000000..d23ad6ff4 --- /dev/null +++ b/doc/forum/Why_are_ignored_files_being_deleted__63__/comment_2_178aa574855a3bfffab4b21f90a84092._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 2""" + date="2015-04-29T17:48:12Z" + content=""" +It seems this was also filed as a bug report +<http://git-annex.branchable.com/bugs/git_annex_import:_ignored_names_fatal/> so I'll deal with it there. +"""]] diff --git a/doc/forum/Why_are_we_stopping_at_a_duplicate__63__/comment_1_dd3b610032cd3091effdb3f0828f45a8._comment b/doc/forum/Why_are_we_stopping_at_a_duplicate__63__/comment_1_dd3b610032cd3091effdb3f0828f45a8._comment new file mode 100644 index 000000000..0c2a1659c --- /dev/null +++ b/doc/forum/Why_are_we_stopping_at_a_duplicate__63__/comment_1_dd3b610032cd3091effdb3f0828f45a8._comment @@ -0,0 +1,9 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2015-04-29T17:47:08Z" + content=""" +The --deduplicate option deals with duplicated file contents. This is +a filename that is already in your git repo, git annex import avoids +overwriting that with a different imported file unless you tell it --force. +"""]] diff --git a/doc/forum/__34__git_annex_sync_--content__34___with_special_remote_of_type___34__directory__34__.mdwn b/doc/forum/__34__git_annex_sync_--content__34___with_special_remote_of_type___34__directory__34__.mdwn new file mode 100644 index 000000000..bab3167e5 --- /dev/null +++ b/doc/forum/__34__git_annex_sync_--content__34___with_special_remote_of_type___34__directory__34__.mdwn @@ -0,0 +1,17 @@ +Hello, I started using the "wanted" feature of git-annex. + +I have (besides others) one local repository ("neon"), and two special remotes "ldk" (rsync) and "storage" (directory). + +"wanted" and "group" are configured as (replaced UUIDs with names): + + group "storage" = backup + wanted "storage" = standard + wanted "neon" = (exclude=pictures/* and exclude=video/*) or present + +Now, let's assume there is a file named "video/foo.mp4". It is only present in "ldk". I want it to be present in "storage", too. + +When I run "git annex sync --content" on "neon" the file "video/foo.mp4" is neither fetched to be placed in "neon" nor in "storage". + +Which command do I have to run to transfer the file "video/foo.mp4" from "ldk" to "storage" when run from "neon". + +Previously, I started with "git annex get \`git annex find --not --in storage\`" and then continued with "git annex copy \`git annex find --not --in storage\` --to storage". I was hoping that the wanted feature would simplify this. diff --git a/doc/forum/__34__git_annex_sync_--content__34___with_special_remote_of_type___34__directory__34__/comment_1_c51363e109bcc5cd1df40c5d0ec993b3._comment b/doc/forum/__34__git_annex_sync_--content__34___with_special_remote_of_type___34__directory__34__/comment_1_c51363e109bcc5cd1df40c5d0ec993b3._comment new file mode 100644 index 000000000..b7d538ae9 --- /dev/null +++ b/doc/forum/__34__git_annex_sync_--content__34___with_special_remote_of_type___34__directory__34__/comment_1_c51363e109bcc5cd1df40c5d0ec993b3._comment @@ -0,0 +1,38 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2015-04-21T20:01:26Z" + content=""" +What's going on here is that git-annex does not know that you want to +use "neon" as a repository that transfers content between "ldk" and +"storage". The configuration for "neon" only makes it want a specific +set of files, so it doesn't get other files from "ldk", and so does +not have them to send to "storage". + +The solution is to change "neon"'s preferred content settings so it wants +files that are not yet present in "storage". + +The [[transfer repository expression|preferred_content/standard_groups]] +is one way to configure preferred content settings so that a repo +will want to download files that other repos want to have. + + not (inallgroup=client and copies=client:2) and ($client) + +But that expression only works for transfer repsitories in between client +repositories. Your "storage" repo is set to be a backup repository. + +So, we need something a little bit different. + + not inallgroup=backup and include=* + +I think that would work, but I don't see a way to reconcile it with +the configuration you already have for "neon". If neon wants to +"exclude=pictures/* and exclude=video/*" then it will never get those +and so can never send them on to "storage". And if neon wants all +"present" files, then anything it does get for whatever reason +will stay in it, which is just not how a transfer repo needs to work. + +What might work better is to set up a separate repository that can talk +to "ldk" and "storage" (as well as perhaps pulling files from "neon" +when available), and make it have that preferred content expression. +"""]] diff --git a/doc/forum/__34__git_annex_sync_--content__34___with_special_remote_of_type___34__directory__34__/comment_2_be52a6d21df4732c9f83463bb5e6f612._comment b/doc/forum/__34__git_annex_sync_--content__34___with_special_remote_of_type___34__directory__34__/comment_2_be52a6d21df4732c9f83463bb5e6f612._comment new file mode 100644 index 000000000..c2566bf60 --- /dev/null +++ b/doc/forum/__34__git_annex_sync_--content__34___with_special_remote_of_type___34__directory__34__/comment_2_be52a6d21df4732c9f83463bb5e6f612._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://me.yahoo.com/a/wS4b1K9kooKjDS5nqjBuZsN4Czwko0ECcg--#2eff3" + nickname="Markus" + subject="comment 2" + date="2015-04-22T10:47:26Z" + content=""" +Thank you. I settled on using `present or (not inallgroup=backup) or (exclude=pictures/* and exclude=video/*)`, which seems to be doing what I want. The most important thing to me is that files are transferred to the backup group. Some files might be left in pictures/ or video/ on \"neon\", but I can drop those from time to time if disk space is low. +"""]] diff --git a/doc/forum/__34__git_annex_sync__34___synced_after_8_hours/comment_2_d13a0af48b8831c81276a0b2c8e25303._comment b/doc/forum/__34__git_annex_sync__34___synced_after_8_hours/comment_2_d13a0af48b8831c81276a0b2c8e25303._comment new file mode 100644 index 000000000..940922e6c --- /dev/null +++ b/doc/forum/__34__git_annex_sync__34___synced_after_8_hours/comment_2_d13a0af48b8831c81276a0b2c8e25303._comment @@ -0,0 +1,37 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawlXt6nnNs-3uw61EGYtxr_AVhJqXybwLR8" + nickname="Bruno" + subject="comment 2" + date="2015-04-15T17:51:18Z" + content=""" +@CandyAngel Thank you for your **git annex find** tips. But for git gc, it seem not working fine :) +After i have executed the **git gc**, the **git annex info** return the result after **1h 45m** + + % time git annex info + repository mode: indirect + trusted repositories: 0 + semitrusted repositories: 5 + 00000000-0000-0000-0000-000000000001 -- web + 00000000-0000-0000-0000-000000000002 -- bittorrent + 181d4dae-2131-435e-9c00-b8c7f1bfc332 -- [sbackup] + 2db1f8e7-0b29-4d61-8875-a4a4a42a79dd -- [dellcomputer] + 703df355-73a6-4487-97fd-a3a5d6ae034e -- usbhomebackup [here] + untrusted repositories: 0 + transfers in progress: none + available local disk space: 135.24 gigabytes (+1 megabyte reserved) + local annex keys: 275416 + local annex size: 780.55 gigabytes + annexed files in working tree: 265888 + size of annexed files in working tree: 751.49 gigabytes + bloom filter size: 16 mebibytes (55.1% full) + backend usage: + SHA256E: 541304 + +git annex info 83,95s user 50,68s system 2% **cpu 1:45:01,70 total** + + +Can you explain exactly the git gc or git repack parameters that you use for optimizing git annex performance ? + +Thanks + +"""]] diff --git a/doc/forum/__34__git_annex_sync__34___synced_after_8_hours/comment_3_c50b62e5a84b861117a4405c2a2f5cfb._comment b/doc/forum/__34__git_annex_sync__34___synced_after_8_hours/comment_3_c50b62e5a84b861117a4405c2a2f5cfb._comment new file mode 100644 index 000000000..20031b952 --- /dev/null +++ b/doc/forum/__34__git_annex_sync__34___synced_after_8_hours/comment_3_c50b62e5a84b861117a4405c2a2f5cfb._comment @@ -0,0 +1,16 @@ +[[!comment format=mdwn + username="CandyAngel" + subject="comment 3" + date="2015-04-16T07:41:07Z" + content=""" +*git annex info* has check every file (not sure if it traverses *.git/annex/objects* specifically or not) to get \"local annex\" information. You can improve its performance by improving directory traversal in general (different filesystem or [changing the hashing method so it isn't Xx/Yy/KEY/FILE](https://github.com/datalad/datalad/issues/32)). + +The repack/gc speeds up operations for the git side of things, like syncing (pull/push), cloning and committing. + +Here's what I used: + + git repack -ad + git gc + +This took git actions down from 1 hour+ to ~10 minutes (for a repo with 5.6 million objects). +"""]] diff --git a/doc/forum/__34__git_annex_sync__34___synced_after_8_hours/comment_4_0051e83196945b97e2f3ed14a58daaea._comment b/doc/forum/__34__git_annex_sync__34___synced_after_8_hours/comment_4_0051e83196945b97e2f3ed14a58daaea._comment new file mode 100644 index 000000000..2fc762f77 --- /dev/null +++ b/doc/forum/__34__git_annex_sync__34___synced_after_8_hours/comment_4_0051e83196945b97e2f3ed14a58daaea._comment @@ -0,0 +1,14 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawlXt6nnNs-3uw61EGYtxr_AVhJqXybwLR8" + nickname="Bruno" + subject="comment 4" + date="2015-04-16T11:47:50Z" + content=""" +Thanks @CandyAngle, + +Effectively, your tips for reduce a time for some git-annex commands if works fine, i will see in the long term if that is work perfectly + +ex:, now **git annex sync** it work in **45s** ! :) + +Thanks +"""]] diff --git a/doc/forum/__34__git_annex_sync__34___synced_after_8_hours/comment_5_26de32ea240621e23717c55866ad9764._comment b/doc/forum/__34__git_annex_sync__34___synced_after_8_hours/comment_5_26de32ea240621e23717c55866ad9764._comment new file mode 100644 index 000000000..907f32b94 --- /dev/null +++ b/doc/forum/__34__git_annex_sync__34___synced_after_8_hours/comment_5_26de32ea240621e23717c55866ad9764._comment @@ -0,0 +1,9 @@ +[[!comment format=mdwn + username="CandyAngel" + subject="comment 5" + date="2015-04-16T21:08:32Z" + content=""" +My pleasure, glad it is working for you! + +Going forward, you should run *git repack* (without -ad) every now and again to pack new objects into pack files. You can use *git count-objects -v* to find out how many unpacked objects you have. +"""]] diff --git a/doc/forum/__96__git_annex_sync__96___uses_too_much_space__63__/comment_1_29cd5e9acd78d8ac6b58fe535fee9650._comment b/doc/forum/__96__git_annex_sync__96___uses_too_much_space__63__/comment_1_29cd5e9acd78d8ac6b58fe535fee9650._comment new file mode 100644 index 000000000..bdf9861f1 --- /dev/null +++ b/doc/forum/__96__git_annex_sync__96___uses_too_much_space__63__/comment_1_29cd5e9acd78d8ac6b58fe535fee9650._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawnVnsqEy82M-MuS2gLri-az83wSQ6lXSrc" + nickname="Jean" + subject="comment 1" + date="2015-05-02T08:28:56Z" + content=""" +I tried reformatting the device with smaller blocksize and inode size, but this didn't help. +"""]] diff --git a/doc/forum/__96__git_annex_sync__96___uses_too_much_space__63__/comment_2_7a24236bc511cbfa869aaeb431a003d2._comment b/doc/forum/__96__git_annex_sync__96___uses_too_much_space__63__/comment_2_7a24236bc511cbfa869aaeb431a003d2._comment new file mode 100644 index 000000000..60e1f22af --- /dev/null +++ b/doc/forum/__96__git_annex_sync__96___uses_too_much_space__63__/comment_2_7a24236bc511cbfa869aaeb431a003d2._comment @@ -0,0 +1,18 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 2""" + date="2015-05-05T16:42:43Z" + content=""" +It's failing to pull the git repository from the remote because that +git repository is apparently larger than 487 mb. That is not usual +when using git-annex, or even when using git unless you have millions +of files in the git repository. + +Since you only have a few thousand files in the git repository, +my guess is you have committed some large files directly to git, +instead of using git-annex. So, you're seeing why git-annex exists... + +You should find the files in your git repository that are not git-annex +symlinks, and are large files. You may need to use `git filter-branch` +to remove the from your repository's history. +"""]] diff --git a/doc/forum/endless_password_prompt_loop/comment_3_eb1dce6d9af6e19cf77df63da1a68425._comment b/doc/forum/endless_password_prompt_loop/comment_3_eb1dce6d9af6e19cf77df63da1a68425._comment new file mode 100644 index 000000000..58914c58e --- /dev/null +++ b/doc/forum/endless_password_prompt_loop/comment_3_eb1dce6d9af6e19cf77df63da1a68425._comment @@ -0,0 +1,16 @@ +[[!comment format=mdwn + username="damien.courousse" + subject="same behaviour when the remote repo is not reachable" + date="2015-05-01T15:47:55Z" + content=""" +hello, + +I experience the same behaviour : once git-annex is started using the assistant, the ssh askpass window keeps asking me for a password. +The repo is not reachable at this time (the server is down). + +What's really annoying is that the askpass window is sometimes hidden behind other windows, but it keeps grabbing keyboard events: it looks like the keyboard \"does not work\" anymore. Hopefully pressing escape shuts closes askpass. + +The askpass window still pop ups from time to time even once the assistant has been closed. + +Running git-annex version 5.20141125 on Debian (amd64) on jessie. +"""]] diff --git a/doc/forum/git_annex_windows_and_rsync.mdwn b/doc/forum/git_annex_windows_and_rsync.mdwn new file mode 100644 index 000000000..c45cd8d2d --- /dev/null +++ b/doc/forum/git_annex_windows_and_rsync.mdwn @@ -0,0 +1,26 @@ +Issue with getting files from a Linux ssh/rsync repo. + +I have a centralized repo on a small linux VM that I synchronize my workstations with, most are linux machines but one is a windows one. I installed msysgit and git-annex for windows, and then run the following: + + + git clone ssh://user@IP.ADDRESS/home/user/annex annex + cd annex + git annex init 'windows' + git annex copy --from origin + + +So basically I am just trying to get a copy of the central repo onto this windows machine for starters and get: + + + rsync: connection unexpectedly closed (0 bytes received so far) [sender] + rsync error: error in rsync protocol data stream (code 12) at io.c(226) [sender=3.1.1] + rsync: connection unexpectedly closed (0 bytes received so far) [Receiver] + rsync error: error in rsync protoco rsync failed -- run git annex again to resume file transfer + l dafailed + ta stream (code 12) atcopy + + +And I get this message for every file. + + +I do have cygwin with rsync and ssh installed on this machine previously so I tried on a separate machine thinking there may be compatibility issues with no avail either. I am not sure if this is an existing issue/work in progress with Windows/git-annex or if it is something I am just experiencing. diff --git a/doc/forum/git_annex_windows_and_rsync/comment_1_33249bf910446fcf98ffb2e7e35017bf._comment b/doc/forum/git_annex_windows_and_rsync/comment_1_33249bf910446fcf98ffb2e7e35017bf._comment new file mode 100644 index 000000000..270f499e9 --- /dev/null +++ b/doc/forum/git_annex_windows_and_rsync/comment_1_33249bf910446fcf98ffb2e7e35017bf._comment @@ -0,0 +1,7 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2015-05-07T18:14:43Z" + content=""" +[[bugs/rsync_on_windows_broken_by_upgrade]] +"""]] diff --git a/doc/forum/possible_gpg_issue/comment_7_f095eadcd9f6947f64e6830acea8228e._comment b/doc/forum/possible_gpg_issue/comment_7_f095eadcd9f6947f64e6830acea8228e._comment new file mode 100644 index 000000000..c262c9b52 --- /dev/null +++ b/doc/forum/possible_gpg_issue/comment_7_f095eadcd9f6947f64e6830acea8228e._comment @@ -0,0 +1,11 @@ +[[!comment format=mdwn + username="ka7" + subject="comment 7" + date="2015-05-05T16:58:42Z" + content=""" +is on the \"SMB share\" running something special ? ..like virus-scanner, quota, backup-in-progress + +and.. smb like SAMBA or Windows ? + +in theory you can do lots of funny stuff to get a smb share: sharing a samba which is a webdav mounted via nfs on a clamFS. (*scary*) +"""]] diff --git a/doc/forum/pre-commit_hook_to_use_git_annex_for_only_large_files.mdwn b/doc/forum/pre-commit_hook_to_use_git_annex_for_only_large_files.mdwn new file mode 100644 index 000000000..2a8135d99 --- /dev/null +++ b/doc/forum/pre-commit_hook_to_use_git_annex_for_only_large_files.mdwn @@ -0,0 +1,55 @@ +I've wanted to use git-annex for the longest time, but I really only wanted to use it if the files were over a certain size, otherwise, I just want to use regular git. + +After writing this pre-commit hook, I wanted to share and get some feedback. + +This would be saved as `.git/hooks/pre-commit` + + #!/bin/sh + let MAX=1*1024*1024 # 1048576 == 1 MB + if [ ! -d '.git/annex/' ]; then + /usr/local/bin/git annex init >/dev/null 2>&1 + fi + if git rev-parse --verify HEAD >/dev/null 2>&1; then + against=HEAD + else + # Initial commit: diff against an empty tree object + against=$(/usr/local/bin/git hash-object -t tree /dev/null) + fi + /usr/local/bin/git diff-index --cached $against | \ + /usr/bin/tr '\t' ' ' | \ + /usr/bin/cut -d ' ' -f4,6- | \ + while read line; do + sha1=$(/usr/bin/cut -d ' ' -f1 <<< "$line") + if [ "$sha1" == "0000000000000000000000000000000000000000" ]; then + continue + fi + size=$(/usr/local/bin/git cat-file -s "$sha1") + if [ $size -ge $MAX ]; then + file=$(/usr/bin/cut -d ' ' -f2- <<< "$line") + /usr/local/bin/git update-index --force-remove "$file" + /usr/local/bin/git annex add "$file" + /usr/bin/killall -TERM Finder + fi + done + /usr/local/bin/git annex pre-commit . + +I also wrote an `Unlock Git Annex File.workflow` service for OS X: + + set gitAnnex to "/Applications/git-annex.app/Contents/MacOS/git-annex" + + tell application "Finder" + repeat with theItem in (get selection) + if file type of theItem is "slnk" then + set theFolder to quoted form of POSIX path of (container of theItem as alias) + set filePath to do shell script "/usr/bin/basename " & quoted form of POSIX path of (theItem as text) + set theCommand to "cd " & theFolder & "; " & gitAnnex & " unlock '" & filePath & "'" + do shell script theCommand + end if + end repeat + end tell + +Use Automator to create a new service that receives selected "files or folders" in "Finder.app". Then drag the "Run AppleScript" action to the workflow panel. The script above should be copied into the code area replacing all the default content. + +Am I all alone in wanting these types of scripts? + +- Peter diff --git a/doc/forum/pre-commit_hook_to_use_git_annex_for_only_large_files/comment_1_01db183b1f1d081066d88332e2b6166a._comment b/doc/forum/pre-commit_hook_to_use_git_annex_for_only_large_files/comment_1_01db183b1f1d081066d88332e2b6166a._comment new file mode 100644 index 000000000..98618cd51 --- /dev/null +++ b/doc/forum/pre-commit_hook_to_use_git_annex_for_only_large_files/comment_1_01db183b1f1d081066d88332e2b6166a._comment @@ -0,0 +1,15 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2015-05-05T18:46:42Z" + content=""" +The problem with this pre-commit hook is that by the time you run `git add +largfile`, it has copied it into the git repository. Your hook will prevent +it getting into a commit, so the repository will eventually garbage collect +the copy away, but this can take some time or manual work to do. + +Recent versions of `git-annex add` will look at the annex.largefiles +configuration and if the file does not match, add it to git directly. +So that's an alternate workflow, where you `git annex add` files and let +git-annex decide whether to put them in the annex or the git repository. +"""]] diff --git a/doc/forum/pre-commit_hook_to_use_git_annex_for_only_large_files/comment_2_2b76809869e0289f78f137afbdcf36c8._comment b/doc/forum/pre-commit_hook_to_use_git_annex_for_only_large_files/comment_2_2b76809869e0289f78f137afbdcf36c8._comment new file mode 100644 index 000000000..7ed0c5768 --- /dev/null +++ b/doc/forum/pre-commit_hook_to_use_git_annex_for_only_large_files/comment_2_2b76809869e0289f78f137afbdcf36c8._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 2""" + date="2015-05-05T18:56:33Z" + content=""" +For the Finder script, there's a page collecting such stuff, +[[tips/file_manager_integration]] +"""]] diff --git a/doc/forum/rsync_regression_on_Windows.mdwn b/doc/forum/rsync_regression_on_Windows.mdwn new file mode 100644 index 000000000..d0bc0f14d --- /dev/null +++ b/doc/forum/rsync_regression_on_Windows.mdwn @@ -0,0 +1,16 @@ +I'm on Windows 7 and msysgit. The rsync that comes with the git-annex installer downloaded on January 19 2015 works fine (rsync version 3.0.9, protocol version 30; git annex version 5.20150113-gcf247cf). The rsync that comes with the current git-annex installer errors out (rsync version 3.1.1 protocol version 31; git annex v ersion 5.20150420-gb0ebb23). I don't think it's a version/protocol mismatch, as I get the error against a server with the same version and protocol. + +[[!format sh """ +$ rsync -rvvp anton@myhost:wtmp/ wtmp +opening connection using: ssh -l anton myhost rsync --server --sender -vv +pre.iLsfx . wtmp/ (10 args) +rsync: connection unexpectedly closed (0 bytes received so far) [Receiver] +rsync error: error in rsync protocol data stream (code 12) at io.c(226) [Receiver=3.1.1] + +$ ssh -l anton myhost rsync --version +rsync version 3.1.1 protocol version 31 +... + +"""]] + +Bug or user error? diff --git a/doc/forum/rsync_regression_on_Windows/comment_1_0e713f8949a3e5a949489fc89c0b9078._comment b/doc/forum/rsync_regression_on_Windows/comment_1_0e713f8949a3e5a949489fc89c0b9078._comment new file mode 100644 index 000000000..5642560fd --- /dev/null +++ b/doc/forum/rsync_regression_on_Windows/comment_1_0e713f8949a3e5a949489fc89c0b9078._comment @@ -0,0 +1,7 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2015-05-07T18:16:07Z" + content=""" +[[bugs/rsync_on_windows_broken_by_upgrade]] +"""]] diff --git a/doc/git-annex-assistant.mdwn b/doc/git-annex-assistant.mdwn index 7c57729a1..5c0f6408b 100644 --- a/doc/git-annex-assistant.mdwn +++ b/doc/git-annex-assistant.mdwn @@ -34,7 +34,12 @@ For more details about the git-annex assistant, see * `--stop` - Stop a running daemon. + Stop a running daemon in the current repository. + +* `--autostop` + + The complement to --autostart; stops all running daemons in the + repositories listed in the autostart file. # SEE ALSO diff --git a/doc/git-annex-contentlocation.mdwn b/doc/git-annex-contentlocation.mdwn index 128622bc8..a090e3754 100644 --- a/doc/git-annex-contentlocation.mdwn +++ b/doc/git-annex-contentlocation.mdwn @@ -16,6 +16,17 @@ Note that in direct mode, the file will typically be in the git work tree, and while its content should correspond to the key, the file could become modified at any time after git-annex checks it. +# OPTIONS + +* `--batch` + + Enable batch mode, in which a line containing the key is read from + stdin, the filename to its content is output to stdout (with a trailing + newline), and repeat. + + Note that if a key's content is not present, an empty line is output to + stdout instead. + # SEE ALSO [[git-annex]](1) diff --git a/doc/git-annex-copy.mdwn b/doc/git-annex-copy.mdwn index ea0b89670..19d328c19 100644 --- a/doc/git-annex-copy.mdwn +++ b/doc/git-annex-copy.mdwn @@ -16,6 +16,8 @@ Copies the content of files from or to another remote. Use this option to copy the content of files from the specified remote to the local repository. + + Any files that are not available on the remote will be silently skipped. * `--to=remote` diff --git a/doc/git-annex-drop.mdwn b/doc/git-annex-drop.mdwn index 3fd13467c..c19a716e9 100644 --- a/doc/git-annex-drop.mdwn +++ b/doc/git-annex-drop.mdwn @@ -35,6 +35,21 @@ safe to do so. the last repository that is storing their content. Data loss can result from using this option. +* `--all` + + Rather than specifying a filename or path to drop, this option can be + used to drop all available versions of all files. + + This is the default behavior when running git-annex drop in a bare repository. + +* `--unused` + + Drop files found by last run of git-annex unused. + +* `--key=keyname` + + Use this option to drop a specified key. + * file matching options The [[git-annex-matching-options]](1) diff --git a/doc/git-annex-examinekey.mdwn b/doc/git-annex-examinekey.mdwn index 3a8159f66..49bc95711 100644 --- a/doc/git-annex-examinekey.mdwn +++ b/doc/git-annex-examinekey.mdwn @@ -33,6 +33,11 @@ that can be determined purely by looking at the key. Enable JSON output. This is intended to be parsed by programs that use git-annex. Each line of output is a JSON object. +* `--batch` + + Enable batch mode, in which a line containing a key is read from stdin, + the information about it is output to stdout, and repeat. + # EXAMPLES The location a key's value is stored (in indirect mode) diff --git a/doc/git-annex-get.mdwn b/doc/git-annex-get.mdwn index 516e5da85..da9c8c05a 100644 --- a/doc/git-annex-get.mdwn +++ b/doc/git-annex-get.mdwn @@ -23,7 +23,9 @@ or transferring them from some kind of key-value store. * `--from=remote` Normally git-annex will choose which remotes to get the content - from. Use this option to specify which remote to use. + from. Use this option to specify which remote to use. + + Any files that are not available on the remote will be silently skipped. * `--jobs=N` `-JN` diff --git a/doc/git-annex-import.mdwn b/doc/git-annex-import.mdwn index 43e619607..69b9a5ebf 100644 --- a/doc/git-annex-import.mdwn +++ b/doc/git-annex-import.mdwn @@ -16,8 +16,9 @@ If a directory is specified, the entire directory is imported. When importing files, there's a possibility of importing a duplicate of a file that is already known to git-annex -- its content is either -present in the local repository already, or git-annex knows of anther -repository that contains it. +present in the local repository already, or git-annex knows of another +repository that contains it, or it was present in the annex before but has +been removed now. By default, importing a duplicate of a known file will result in a new filename being added to the repository, so the duplicate file @@ -52,6 +53,12 @@ Several options can be used to adjust handling of duplicate files. Does not import any files, but any files found in the import location that are duplicates are deleted. +* `--force` + + Allow existing files to be overwritten by newly imported files. + + Also, causes .gitignore to not take effect when adding files. + * file matching options Many of the [[git-annex-matching-options]](1) diff --git a/doc/git-annex-lock.mdwn b/doc/git-annex-lock.mdwn index aa03d4ce7..21be9f5aa 100644 --- a/doc/git-annex-lock.mdwn +++ b/doc/git-annex-lock.mdwn @@ -1,6 +1,6 @@ # NAME -git-annex lock - unco unlock command +git-annex lock - undo unlock command # SYNOPSIS diff --git a/doc/git-annex-lookupkey.mdwn b/doc/git-annex-lookupkey.mdwn index 568bbdc05..154b4a753 100644 --- a/doc/git-annex-lookupkey.mdwn +++ b/doc/git-annex-lookupkey.mdwn @@ -13,6 +13,16 @@ index. The key is output to stdout. If there is no key (because the file is not present in the index, or is not a git-annex managed file), nothing is output, and it exits nonzero. +# OPTIONS + +* `--batch` + + Enable batch mode, in which a line containing the filename is read from + stdin, the key is output to stdout (with a trailing newline), and repeat. + + Note that is there is no key corresponding to the file, an empty line is + output to stdout instead. + # SEE ALSO [[git-annex]](1) diff --git a/doc/git-annex-metadata.mdwn b/doc/git-annex-metadata.mdwn index 7d613456b..c539de6ee 100644 --- a/doc/git-annex-metadata.mdwn +++ b/doc/git-annex-metadata.mdwn @@ -66,6 +66,19 @@ When run without any -s or -t parameters, displays the current metadata. Enable JSON output. This is intended to be parsed by programs that use git-annex. Each line of output is a JSON object. +* `--all` + + Specify instead of a file to get/set metadata on all known keys. + +* `--unused` + + Specify instead of a file to get/set metadata on + files found by last run of git-annex unused. + +* `--key=keyname` + + Specify instead of a file to get/set metadata of the specified key. + # EXAMPLES To set some tags on a file and also its author: diff --git a/doc/git-annex-pre-commit.mdwn b/doc/git-annex-pre-commit.mdwn index e0f6fdb2a..bc1e86e18 100644 --- a/doc/git-annex-pre-commit.mdwn +++ b/doc/git-annex-pre-commit.mdwn @@ -4,7 +4,7 @@ git-annex pre-commit - run by git pre-commit hook # SYNOPSIS -git annex `[path ...]` +git annex pre-commit `[path ...]` # DESCRIPTION diff --git a/doc/git-annex-preferred-content.mdwn b/doc/git-annex-preferred-content.mdwn index 9ea19ce09..49512f465 100644 --- a/doc/git-annex-preferred-content.mdwn +++ b/doc/git-annex-preferred-content.mdwn @@ -10,6 +10,13 @@ using `git annex vicfg` or `git annex wanted`. They are used by the `--auto` option, by `git annex sync --content`, and by the git-annex assistant. +While preferred content expresses a preference, it can be overridden +by simply using `git annex drop`. On the other hand, required content +settings are enforced; `git annex drop` will refuse to drop a file if +doing so would violate its required content settings. A repository's +required content can be configured using `git annex vicfg` or +`git annex required`. + Preferred content expressions are similar, but not identical to the [[git-annex-matching-options]](1), just without the dashes. For example: @@ -18,7 +25,7 @@ For example: The main differences are that `exclude=` and `include=` always match relative to the top of the git repository, and that there is -no equivilant to `--in`. +no equivalent to `--in`. For more details about preferred content expressions, see See <https://git-annex.branchable.com/preferred_content/> diff --git a/doc/git-annex-readpresentkey.mdwn b/doc/git-annex-readpresentkey.mdwn index 3e552e05d..6dcc2ca0b 100644 --- a/doc/git-annex-readpresentkey.mdwn +++ b/doc/git-annex-readpresentkey.mdwn @@ -9,7 +9,7 @@ git annex readpresentkey `key uuid` # DESCRIPTION This plumbing-level command reads git-annex's records about whether -the specified key's content is present in the remote with the speficied +the specified key's content is present in the remote with the specified uuid. It exits 0 if the key is recorded to be present and 1 if not. diff --git a/doc/git-annex-required.mdwn b/doc/git-annex-required.mdwn new file mode 100644 index 000000000..cde3cd6ab --- /dev/null +++ b/doc/git-annex-required.mdwn @@ -0,0 +1,29 @@ +# NAME + +git-annex required - get or set required content expression + +# SYNOPSIS + +git annex required `repository [expression]` + +# DESCRIPTION + +When run with an expression, configures the content that is required +to be held in the archive. See [[git-annex-preferred-content]](1) + +For example: + + git annex required . "include=*.mp3 or include=*.ogg" + +Without an expression, displays the current required content setting +of the repository. + +# SEE ALSO + +[[git-annex]](1) + +# AUTHOR + +Joey Hess <id@joeyh.name> + +Warning: Automatically converted into a man page by mdwn2man. Edit with care. diff --git a/doc/git-annex-schedule.mdwn b/doc/git-annex-schedule.mdwn index 78ca7a580..bffb15ec5 100644 --- a/doc/git-annex-schedule.mdwn +++ b/doc/git-annex-schedule.mdwn @@ -31,10 +31,10 @@ To schedule multiple jobs, separate them with "; ". Some examples: - fsck self 30m every day at any time - fsck self 1h every month at 3 AM - fsck self 1h on day 1 of every month at any time - fsck self 1h every week divisible by 2 at any time + fsck self 30m every day at any time + fsck self 1h every month at 3 AM + fsck self 1h on day 1 of every month at any time + fsck self 1h every week divisible by 2 at any time # SEE ALSO diff --git a/doc/git-annex-shell.mdwn b/doc/git-annex-shell.mdwn index e43d51657..4185774e7 100644 --- a/doc/git-annex-shell.mdwn +++ b/doc/git-annex-shell.mdwn @@ -86,7 +86,7 @@ to git-annex-shell are: * -- fields=val fields=val.. -- - Additional fields may be specified this way, to retain compatability with + Additional fields may be specified this way, to retain compatibility with past versions of git-annex-shell (that ignore these, but would choke on new dashed options). diff --git a/doc/git-annex-undo.mdwn b/doc/git-annex-undo.mdwn index 391b373bf..1fda4778e 100644 --- a/doc/git-annex-undo.mdwn +++ b/doc/git-annex-undo.mdwn @@ -15,7 +15,7 @@ When passed a directory, undoes the last change that was made to the contents of that directory. Running undo a second time will undo the undo, returning the working -tree to the same state it had before. In order for undoing an undo of +tree to the same state it had before. To support undoing an undo of staged changes, any staged changes are first committed by the undo command. diff --git a/doc/git-annex-watch.mdwn b/doc/git-annex-watch.mdwn index 1e844d714..07b0585c0 100644 --- a/doc/git-annex-watch.mdwn +++ b/doc/git-annex-watch.mdwn @@ -30,7 +30,7 @@ files that it does not match will instead be added with `git add`. * `--stop` - Stop a running daemon. + Stop a running daemon in the current repository. # SEE ALSO diff --git a/doc/git-annex-whereis.mdwn b/doc/git-annex-whereis.mdwn index 1e1092ba3..c99e0722c 100644 --- a/doc/git-annex-whereis.mdwn +++ b/doc/git-annex-whereis.mdwn @@ -32,6 +32,19 @@ For example: The [[git-annex-matching-options]](1) can be used to specify files to act on. +* `--key=keyname` + + Show where a particular git-annex key is located. + +* `--all` + + Show whereis information for all known keys. + +* `--unused` + + Show whereis information for files found by last run of git-annex unused. + + # SEE ALSO [[git-annex]](1) diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index 6fd10aed0..3dc54a308 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -264,8 +264,16 @@ subdirectories). * `groupwanted groupname [expression]` + Get or set groupwanted expression. + See [[git-annex-groupwanted]](1) for details. +* `required repository [expression]` + + Get or set required content expression. + + See [[git-annex-required]](1) for details. + * `schedule repository [expression]` Get or set scheduled jobs. diff --git a/doc/git-union-merge.mdwn b/doc/git-union-merge.mdwn index d0ceb3a8f..ca06d2f93 100644 --- a/doc/git-union-merge.mdwn +++ b/doc/git-union-merge.mdwn @@ -12,7 +12,7 @@ Does a union merge between two refs, storing the result in the specified newref. The union merge will always succeed, but assumes that files can be merged -simply by concacenating together lines from all the oldrefs, in any order. +simply by concatenating together lines from all the oldrefs, in any order. So, this is useful only for branches containing log-type data. Note that this does not touch the checked out working copy. It operates diff --git a/doc/install/Debian.mdwn b/doc/install/Debian.mdwn index c71d4d244..3401afded 100644 --- a/doc/install/Debian.mdwn +++ b/doc/install/Debian.mdwn @@ -2,6 +2,10 @@ sudo apt-get install git-annex +## Debian 8.0 "jessie" + + sudo apt-get install git-annex + ## Debian 7.0 "wheezy": sudo apt-get install git-annex diff --git a/doc/install/OSX/MacPorts.mdwn b/doc/install/OSX/MacPorts.mdwn index 379e42d12..dd6db8b0e 100644 --- a/doc/install/OSX/MacPorts.mdwn +++ b/doc/install/OSX/MacPorts.mdwn @@ -8,7 +8,7 @@ The version provided by Macports is too old to work with current versions of git Then execute <pre> -sudo port install git-core ossp-uuid md5sha1sum coreutils gnutls libxml2 libgsasl pkgconfig +sudo port install git ossp-uuid md5sha1sum coreutils gnutls libxml2 libgsasl pkgconfig sudo cabal update PATH=$HOME/bin:$PATH cabal install c2hs git-annex --bindir=$HOME/bin diff --git a/doc/install/Windows.mdwn b/doc/install/Windows.mdwn index 5080fc9cb..3d4b7a073 100644 --- a/doc/install/Windows.mdwn +++ b/doc/install/Windows.mdwn @@ -25,7 +25,7 @@ A daily build is also available, thanks to Yury V. Zaytsev and ## building it yourself To build git-annex from source on Windows, you need to install -the Haskell Platform, Mingw, and Cygwin. Use Cygwin to install: +the Haskell Platform and Cygwin. Use Cygwin to install these packages: gcc rsync git wget ssh gnupg Once the prerequisites are installed, run: @@ -35,5 +35,8 @@ Once the prerequisites are installed, run: cd gitannex build +Note that git from Cygwin is able to clone git-annex's git repository; +Msysgit cannot due to some characters in filenames. + (To build the git-annex installer, you also need to install the NullSoft installer system.) diff --git a/doc/metadata/comment_3_50b17af1cf75ce88c4aef59dcd971b82._comment b/doc/metadata/comment_3_50b17af1cf75ce88c4aef59dcd971b82._comment new file mode 100644 index 000000000..771b6a970 --- /dev/null +++ b/doc/metadata/comment_3_50b17af1cf75ce88c4aef59dcd971b82._comment @@ -0,0 +1,7 @@ +[[!comment format=mdwn + username="madduck" + subject="Overwriting metadata from json" + date="2015-04-30T18:42:45Z" + content=""" +I see that --json allows me to export metadata in a well-parseable format. I'd really like to be able to pipe this into a file, edit the file and then pipe it back to git-annex, causing the metadata for each file to be rewritten from the JSON input. This would make it trivial to write an external metadata editor (like vidir, vorbistagedit, git annex vicfg even… etc.) +"""]] diff --git a/doc/metadata/comment_4_237721c5e8f66f303a1828810573a23d._comment b/doc/metadata/comment_4_237721c5e8f66f303a1828810573a23d._comment new file mode 100644 index 000000000..d2c13888e --- /dev/null +++ b/doc/metadata/comment_4_237721c5e8f66f303a1828810573a23d._comment @@ -0,0 +1,15 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 4""" + date="2015-05-01T19:38:36Z" + content=""" +@madduck, you could file a todo if you want about that. + +However, I have my doubts; if the json supposed to include the full set of +metadata for the file? If so, that seems a potentially expensive interface. +If not, it would be hard to tell when metadata should be deleted, or when +multiple values are being set, vs a value being changed. + +The current interface to set metadata deals with these possibilities in a +compact and sensible way. +"""]] diff --git a/doc/metadata/comment_5_fd30444aecfc4792eb4dbfdebc230786._comment b/doc/metadata/comment_5_fd30444aecfc4792eb4dbfdebc230786._comment new file mode 100644 index 000000000..195eea278 --- /dev/null +++ b/doc/metadata/comment_5_fd30444aecfc4792eb4dbfdebc230786._comment @@ -0,0 +1,9 @@ +[[!comment format=mdwn + username="madduck" + subject="TODO written" + date="2015-05-06T14:24:56Z" + content=""" +@joeyh, yes, it would overwrite all metadata. The idea would be to export with --json, manipulate, import… + +http://git-annex.branchable.com/todo/ability_to_set_metadata_from_json/?updated +"""]] diff --git a/doc/news/version_5.20150317.mdwn b/doc/news/version_5.20150317.mdwn deleted file mode 100644 index fb889413b..000000000 --- a/doc/news/version_5.20150317.mdwn +++ /dev/null @@ -1,45 +0,0 @@ -git-annex 5.20150317 released with [[!toggle text="these changes"]] -[[!toggleable text=""" - * fsck: Incremental fsck uses sqlite to store its records, instead - of abusing the sticky bit. Existing sticky bits are ignored; - incremental fscks started by old versions won't be resumed by - this version. - * fsck: Multiple incremental fscks of different repos (including remotes) - can now be running at the same time in the same repo without it - getting confused about which files have been checked for which remotes. - * unannex: Refuse to unannex when repo is too new to have a HEAD, - since in this case there must be staged changes in the index - (if there is anything to unannex), and the unannex code path - needs to run with a clean index. - * Linux standalone: Set LOCPATH=/dev/null to work around - https://ghc.haskell.org/trac/ghc/ticket/7695 - This prevents localization from working, but git-annex - is not localized anyway. - * sync: As well as the synced/git-annex push, attempt a - git-annex:git-annex push, as long as the remote branch - is an ancestor of the local branch, to better support bare git repos. - (This used to be done, but it forgot to do it since version 4.20130909.) - * When re-execing git-annex, use current program location, rather than - ~/.config/git-annex/program, when possible. - * Submodules are now supported by git-annex! - * metadata: Fix encoding problem that led to mojibake when storing - metadata strings that contained both unicode characters and a space - (or '!') character. - * Also potentially fixes encoding problem when embedding credentials - that contain unicode characters. - * sync: Fix committing when in a direct mode repo that has no HEAD ref. - (For example, a newly checked out git submodule.) - * Added SETURIPRESENT and SETURIMISSING to external special remote protocol, - useful for things like ipfs that don't use regular urls. - * addurl: Added --raw option, which bypasses special handling of quvi, - bittorrent etc urls. - * git-annex-shell: Improve error message when the specified repository - doesn't exist or git config fails for some reason. - * fromkey --force: Skip test that the key has its content in the annex. - * fromkey: Add stdin mode. - * registerurl: New plumbing command for mass-adding urls to keys. - * remotedaemon: Fixed support for notifications of changes to gcrypt - remotes, which was never tested and didn't quite work before."""]] - -Update: The OSX build for this release was missing the webapp. An updated -build is now available fixing that problem. diff --git a/doc/news/version_5.20150327.mdwn b/doc/news/version_5.20150327.mdwn deleted file mode 100644 index 3a46aad4a..000000000 --- a/doc/news/version_5.20150327.mdwn +++ /dev/null @@ -1,22 +0,0 @@ -git-annex 5.20150327 released with [[!toggle text="these changes"]] -[[!toggleable text=""" - * readpresentkey: New plumbing command for checking location log. - * checkpresentkey: New plumbing command to check if a key can be verified - to be present on a remote. - * Added a post-update-annex hook, which is run after the git-annex branch - is updated. Needed for git update-server-info. - * migrate: --force will force migration of keys already using the - destination backend. Useful in rare cases. - * Man pages for individual commands now available, and can be - opened using "git annex help <command>" - * --auto is no longer a global option; only get, drop, and copy - accept it. (Not a behavior change unless you were passing it to a - command that ignored it.) - * Improve error message when --in @date is used and there is no - reflog for the git-annex branch. - * assistant: Committing a whole lot of files at once could overflow - command-line length limits and cause the commit to fail. This - only happened when using the assistant in an indirect mode repository. - * Work around curl bug when asked to download an empty url to a file. - * Fix bug introduced in the last release that broke git-annex sync - when git-annex was installed from the standalone tarball."""]]
\ No newline at end of file diff --git a/doc/news/version_5.20150420.mdwn b/doc/news/version_5.20150420.mdwn new file mode 100644 index 000000000..e084735c4 --- /dev/null +++ b/doc/news/version_5.20150420.mdwn @@ -0,0 +1,33 @@ +git-annex 5.20150420 released with [[!toggle text="these changes"]] +[[!toggleable text=""" + * Fix activity log parsing, which caused the log to not retain + activity from other uuids. + * Union merge could fall over if there was a file in the repository + with the same name as a git ref. Now fixed. + * info dir: Added information about repositories that + contain files in the specified directory. + * info: Added --bytes option. + * bittorrent: Fix handling of magnet links. + * When a key's size is unknown, still check the annex.diskreserve, + and avoid getting content if the disk is too full. + * Fix fsck --from a git remote in a local directory, and from + a directory special remote. + This was a reversion caused by the relative path changes in 5.20150113. + * fsck --from remote: When bad content is found in the remote, + and the local repo does not have a copy of the content, preserve + the bad content in .git/annex/bad/ to avoid further data loss. + * fsck --from remote: Avoid downloading a key if it would go over + the annex.diskreserve limit. + * required: New command, like wanted, but for required content. + * Removed dependency on haskell SHA library, + instead using cryptohash >= 0.11.0. + * Make repo init more robust. + * New debian/rules build-standalone target, which generates a + git-annex-standalone.deb that should work on many old Debian etc + systems. Thanks, Yaroslav Halchenko. + * Windows: Renamed start menu file to avoid loop in some versions + of Windows where the menu file is treated as a git-annex program. + * Windows: Fixed support of remotes on other drives. + (A reversion introduced in version 5.20150113.) + * Windows: Bundled versions of rsync, wget, ssh, and gpg from + cygwin all updated. Thanks, Yury V. Zaytsev."""]]
\ No newline at end of file diff --git a/doc/news/version_5.20150508.mdwn b/doc/news/version_5.20150508.mdwn new file mode 100644 index 000000000..0f3cb8fbb --- /dev/null +++ b/doc/news/version_5.20150508.mdwn @@ -0,0 +1,51 @@ +git-annex 5.20150508 released with [[!toggle text="these changes"]] +[[!toggleable text=""" + * Improve behavior when a git-annex command is told to operate + on a file that doesn't exist. It will now continue to other + files specified after that on the command line, and only error out at + the end. + * S3: Enable debug logging when annex.debug or --debug is set. + * S3: git annex info will show additional information about a S3 remote + (endpoint, port, storage class) + * S3: Let git annex enableremote be used, without trying to recreate + a bucket that should already exist. + * S3: Fix incompatability with bucket names used by hS3; the aws library + cannot handle upper-case bucket names. git-annex now converts them to + lower case automatically. + * import: Check for gitignored files before moving them into the tree. + (Needs git 1.8.4 or newer.) + * import: Don't stop entire import when one file fails due to being + gitignored or conflicting with something in the work tree. + * import: Before removing a duplicate file in --deduplicate or + --clean-duplicates mode, verify that enough copies of its content still + exist. + * Improve integration with KDE's file manager to work with dolphin + version 14.12.3 while still being compatable with 4.14.2. + Thanks, silvio. + * assistant: Added --autostop to complement --autostart. + * Work around wget bug #784348 which could cause it to clobber git-annex + symlinks when downloading from ftp. + * Support checking ftp urls for file presence. + * Fix bogus failure of fsck --fast. + * fsck: Ignore error recording the fsck in the activity log, + which can happen when running fsck in a read-only repository. + Closes: #[698559](http://bugs.debian.org/698559) + (fsck can still need to write to the repository if it find problems, + but a successful fsck can be done read-only) + * Improve quvi 0.4 output parsing to handle cases wher there is no known + filename extension. This is currently the case when using quvi with + youtube. In this case, the extension ".m" will be used. + * Dropped support for older versions of yesod, warp, and dbus than the ones + in Debian Jessie. + * Switch from the obsolete dataenc library for base64 encoding to sandi. + (Thanks, Magnus Therning) + * Debian's ghc now supports TH on arm! Adjust build dependencies + to build the webapp on arm, and enable DAV support on arm. \o/ + * Adjust some other arch specific build dependencies that are now + available on more architectures in Devian unstable. + * Windows: Remove cygwin ssh, the newer version of which has stopped + honoring the setting of HOME. Instead, copy msysgit's ssh into PATH. + Note that setting up a remote ssh server using password authentication + is known to be broken in this release on Windows. + * Windows: Roll back to an older version of rsync from cygwin. + The newer version has some dependency on a newer ssh from cygwin."""]]
\ No newline at end of file diff --git a/doc/preferred_content.mdwn b/doc/preferred_content.mdwn index 557305aae..d3aa4fa47 100644 --- a/doc/preferred_content.mdwn +++ b/doc/preferred_content.mdwn @@ -44,7 +44,7 @@ options in commands like this: git annex get --include='*.mp3' --and -'(' --not --largerthan=100mb -')' -The equivilant preferred content expression looks like this: +The equivalent preferred content expression looks like this: include=*.mp3 and (not largerthan=100mb) @@ -63,7 +63,7 @@ to not retain those files, like this: ### difference: no "in=" -Preferred content expressions have no direct equivilant to `--in`. +Preferred content expressions have no direct equivalent to `--in`. Often, it's best to add repositories to groups, and match against the groups in a preferred content expression. So rather than @@ -146,7 +146,7 @@ expression tuned for your needs, and every repository you put in this group and make its preferred content be "groupwanted" will use it. For example, the archive group only wants to archive 1 copy of each file, -spread amoung every repository in the group. +spread among every repository in the group. Here's how to configure a group named redundantarchive, that instead wants to contain 3 copies of each file: diff --git a/doc/privacy.mdwn b/doc/privacy.mdwn index 3ac4b1eec..5be735d1a 100644 --- a/doc/privacy.mdwn +++ b/doc/privacy.mdwn @@ -41,7 +41,7 @@ output you post, and feel free to remove identifying information. Note that the git-annex assistant *does* sanitize XMPP protocol information logged when debugging is enabled. -If you prefer not to post information publicaly, you can send a GPG +If you prefer not to post information publically, you can send a GPG encrypted mail to Joey Hess <id@joeyh.name> (gpg key ID 2512E3C7). Or you can post a public bug report, and send a followup email with private details. diff --git a/doc/related_software.mdwn b/doc/related_software.mdwn index 4b6f42d2f..a0462f541 100644 --- a/doc/related_software.mdwn +++ b/doc/related_software.mdwn @@ -4,9 +4,9 @@ designed to interoperate with it. * The [[git-annex assistant|assistant]] is included in git-annex, and extends its use cases into new territory. * [gitlab-shell](https://gitlab.com/gitlab-org/gitlab-shell) supports - git-annex. So git-annex can be used with repositries served by Gitlab, + git-annex. So git-annex can be used with repositories served by GitLab, including gitlab.com, or deploy your own. - [See Gitlab's announcment](https://about.gitlab.com/2015/02/17/gitlab-annex-solves-the-problem-of-versioning-large-binaries-with-git/) + [See GitLab's announcement](https://about.gitlab.com/2015/02/17/gitlab-annex-solves-the-problem-of-versioning-large-binaries-with-git/) * Emacs Org mode can auto-commit attached files to git-annex. * [git annex darktable integration](https://github.com/xxv/darktable-git-annex) * [Magit](http://github.com/magit/magit), an Emacs mode for Git, has diff --git a/doc/required_content.mdwn b/doc/required_content.mdwn index 91c5614a8..e17951d9d 100644 --- a/doc/required_content.mdwn +++ b/doc/required_content.mdwn @@ -6,7 +6,8 @@ archival repositories, and also require that one copy be stored offsite. The format of required content expressions is the same as [[preferred_content]] expressions. -Required content settings can be edited using `git annex vicfg`. +Required content settings can be edited using `git annex vicfg` +or set using `git annex required`. Each repository can have its own settings, and other repositories will try to honor those settings when interacting with it. diff --git a/doc/required_content/comment_2_d475f4afc84a58afe7af6d492de3ebe5._comment b/doc/required_content/comment_2_d475f4afc84a58afe7af6d492de3ebe5._comment new file mode 100644 index 000000000..4db20c013 --- /dev/null +++ b/doc/required_content/comment_2_d475f4afc84a58afe7af6d492de3ebe5._comment @@ -0,0 +1,7 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 2""" + date="2015-04-29T18:04:52Z" + content=""" +`git annex required` was recently added to the CLI. +"""]] diff --git a/doc/tips/Synology_NAS_and_git_annex.mdwn b/doc/tips/Synology_NAS_and_git_annex.mdwn index 50c604483..8a6d282c9 100644 --- a/doc/tips/Synology_NAS_and_git_annex.mdwn +++ b/doc/tips/Synology_NAS_and_git_annex.mdwn @@ -19,7 +19,7 @@ This is known to work with DSM 4.3-3810 Update 1 and git-annex standalone versio (2) Setup ssh key based authentication with the Synology for each computer you want to sync with it. You want a specific key that is used only by git-annex, for each computer. Again, many good guides online. -(3) In the Synology .ssh/authorized_keys file for your account, add (substituing your username) +(3) In the Synology .ssh/authorized_keys file for your account, add (substituting your username) [[!format sh """ command="/home/$yourusername/.ssh/git-annex-shell" """]] diff --git a/doc/tips/centralized_git_repository_tutorial.mdwn b/doc/tips/centralized_git_repository_tutorial.mdwn index 8088e7d23..e646ed0ee 100644 --- a/doc/tips/centralized_git_repository_tutorial.mdwn +++ b/doc/tips/centralized_git_repository_tutorial.mdwn @@ -1,13 +1,13 @@ The [[walkthrough]] builds up a decentralized git repository setup, but git-annex can also be used with a centralized bare repository, just like git can. This tutorial shows how to set up a centralized repository hosted on -GitHub. +GitHub on GitLab or your own git server. ## set up the repository, and make a checkout I've created a repository for technical talk videos, which you can [fork on Github](https://github.com/joeyh/techtalks). -Or make your own repository on GitHub (or elsewhere) now. +Or make your own repository on GitHub (or GitLab elsewhere) now. On your laptop, [[install]] git-annex, and clone the repository: @@ -21,12 +21,14 @@ located: init my laptop ok Let's tell git-annex that GitHub doesn't support running git-annex-shell there. + + # git config remote.origin.annex-ignore true + This means you can't store annexed file *contents* on GitHub; it would really be better to host the bare repository on your own server, which would not have this limitation. (If you want to do that, check out -[[using_gitolite_with_git-annex]].) - - # git config remote.origin.annex-ignore true +[[using_gitolite_with_git-annex]].) Or, you could use GitLab, which +*does* [support git-annex on their servers](https://about.gitlab.com/2015/02/17/gitlab-annex-solves-the-problem-of-versioning-large-binaries-with-git/). ## add files to the repository @@ -53,9 +55,9 @@ Feel free to rename the files, etc, using normal git commands: # git mv kitenet.net_~joey_screencasts_git-annex_coding_in_haskell.ogg git-annex_coding_in_haskell.ogg # git commit -m 'better filenames' -Now push your changes back to the central repository. This first time, -remember to push the git-annex branch, which is used to track the file -contents. +Now push your changes back to the central repository. As well as pushing +the master branch, remember to push the git-annex branch, which is used to +track the file contents. # git push origin master git-annex To git@github.com:joeyh/techtalks.git @@ -126,7 +128,7 @@ desired. After you use git-annex to move files around, remember to push, which will broadcast its updated location information. - # git push + # git push origin master git-annex ## take it farther diff --git a/doc/tips/downloading_podcasts.mdwn b/doc/tips/downloading_podcasts.mdwn index fb875180d..e11825420 100644 --- a/doc/tips/downloading_podcasts.mdwn +++ b/doc/tips/downloading_podcasts.mdwn @@ -75,6 +75,10 @@ If your git-annex is also built with quvi support, you can also use `git annex importfeed` on youtube playlists. It will automatically download the videos linked to by the playlist. +For this you need an rss file containing links to the videos. +For example, this url currently works: +<http://gdata.youtube.com/feeds/api/playlists/PLz8ZG1e9MPlzefklz1Gv79icjywTXycR-> + ## metadata As well as storing the urls for items imported from a feed, git-annex can diff --git a/doc/tips/file_manager_integration.mdwn b/doc/tips/file_manager_integration.mdwn index 4429b9093..cd4218ff2 100644 --- a/doc/tips/file_manager_integration.mdwn +++ b/doc/tips/file_manager_integration.mdwn @@ -24,9 +24,9 @@ Even more recent git-annex comes with built-in integration with Konqueror. This is set up by git-annex creating a `~/.kde/share/kde4/services/ServiceMenus/git-annex.desktop file. -## XFCE (Thunar) +## Xfce (Thunar) -XFCE uses the Thunar file manager, which can also be easily configured to +Xfce uses the Thunar file manager, which can also be easily configured to allow for custom actions. Just go to the "Configure custom actions..." item in the "Edit" menu, and create a custom action for get, drop, and undo with the following commands: @@ -72,12 +72,12 @@ This gives me the resulting config on disk, in `.config/Thunar/uca.xml`: <video-files/> </action> -The complete instructions on how to setup actions is [in the XFCE documentation](http://docs.xfce.org/xfce/thunar/custom-actions). +The complete instructions on how to setup actions is [in the Xfce documentation](http://docs.xfce.org/xfce/thunar/custom-actions). ## OS X (Finder) For OS X, it is possible to get context menus in Finder. Due to how OS X -deals with sym links, one needs to operate on folders if using indirect +deals with symlinks, one needs to operate on folders if using indirect mode. Direct mode operation has not been tested. 1. Open Automator and create a new Service. diff --git a/doc/tips/megaannex.mdwn b/doc/tips/megaannex.mdwn index 46bab4aef..b547aaa5c 100644 --- a/doc/tips/megaannex.mdwn +++ b/doc/tips/megaannex.mdwn @@ -1,7 +1,7 @@ megaannex 0.2.0 ========= -Hook program for gitannex to use mega.co.nz as backend +Hook program for git-annex to use mega.co.nz as backend # Requirements: diff --git a/doc/tips/using_gitolite_with_git-annex.mdwn b/doc/tips/using_gitolite_with_git-annex.mdwn index 31f34c6fb..0d20c9372 100644 --- a/doc/tips/using_gitolite_with_git-annex.mdwn +++ b/doc/tips/using_gitolite_with_git-annex.mdwn @@ -92,7 +92,7 @@ Make the ADC directory, and a "ua" subdirectory. mkdir -p /usr/local/lib/gitolite/adc/ua </pre> -Install the git-annex-shell ADC into the "ua" subdirectory from the gitolie repository. +Install the git-annex-shell ADC into the "ua" subdirectory from the gitolite repository. <pre> cd /usr/local/lib/gitolite/adc/ua/ diff --git a/doc/tips/using_the_web_as_a_special_remote.mdwn b/doc/tips/using_the_web_as_a_special_remote.mdwn index 087d2e24b..2dc3419ad 100644 --- a/doc/tips/using_the_web_as_a_special_remote.mdwn +++ b/doc/tips/using_the_web_as_a_special_remote.mdwn @@ -81,7 +81,7 @@ it is a video and download the video content for offline viewing. Later, in another clone of the repository, you can run `git annex get` on the file and it will also be downloaded with the help of quvi. This works even if the video host has transcoded or otherwise changed the video -in the meantime; the assumption is that these video files are equivilant. +in the meantime; the assumption is that these video files are equivalent. There is an `annex.quvi-options` configuration setting that can be used to pass parameters to quvi. For example, you could set `git config diff --git a/doc/todo/Add_gitlab.com_as_cloud_provider/comment_3_9d60cfa947c5ce82c69fb77961db2d25._comment b/doc/todo/Add_gitlab.com_as_cloud_provider/comment_3_9d60cfa947c5ce82c69fb77961db2d25._comment new file mode 100644 index 000000000..40b49f875 --- /dev/null +++ b/doc/todo/Add_gitlab.com_as_cloud_provider/comment_3_9d60cfa947c5ce82c69fb77961db2d25._comment @@ -0,0 +1,31 @@ +[[!comment format=mdwn + username="http://rfhbuk.pip.verisignlabs.com/" + nickname="rfhb" + subject="git-annex: bad url ssh://git@gitlab.com:~/gitlabname/reponame.git" + date="2015-04-26T12:19:37Z" + content=""" +Adding an encrypted remote works (mind \":~/\"): + + > git remote add encrypted gcrypt::ssh://git@gitlab.com:~/gitlabname/reponame.git + > git push encrypted master + gcrypt: Repository not found: ssh://git@gitlab.com:~/gitlabname/reponame.git + gcrypt: Setting up new repository + gcrypt: Remote ID is :id:abcdefghijklmnopqrst + Counting objects: 53, done. + Compressing objects: 100% (52/52), done. + Total 53 (delta 12), reused 0 (delta 0) + gcrypt: Encrypting to: --throw-keyids --default-recipient-self + gcrypt: Requesting manifest signature + ... + To gcrypt::ssh://git@gitlab.com:~/gitlabname/reponame.git + * [new branch] master -> master + > + +However, git-annex then fails: + + > git annex sync + git-annex: bad url ssh://git@gitlab.com:~/gitlabname/reponame.git + +Should the encrypted repository be configured or added in a different way? Sorry, did not find it so easy to set up. + +"""]] diff --git a/doc/todo/Add_gitlab.com_as_cloud_provider/comment_4_94c0a53da4f3bc89afd97a036d832f78._comment b/doc/todo/Add_gitlab.com_as_cloud_provider/comment_4_94c0a53da4f3bc89afd97a036d832f78._comment new file mode 100644 index 000000000..ec8d18cd1 --- /dev/null +++ b/doc/todo/Add_gitlab.com_as_cloud_provider/comment_4_94c0a53da4f3bc89afd97a036d832f78._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2015-04-29T18:05:56Z" + content=""" +Current status is that I'm waiting to see git-annex-shell on gitlab actally +work. I get an API error. I've emailed the gitlab people about this. +"""]] diff --git a/doc/todo/ability_to_set_metadata_from_json.mdwn b/doc/todo/ability_to_set_metadata_from_json.mdwn new file mode 100644 index 000000000..ee4d68823 --- /dev/null +++ b/doc/todo/ability_to_set_metadata_from_json.mdwn @@ -0,0 +1,6 @@ +I can export metadata to JSON format, which is nice as this can now be read into any other tool and manipulated. But I cannot find a way to set the metadata from JSON and so I am left to figure out what changes need to be made via the g-a interface to get to the desired state, and that is hard to get right. + +Maybe g-a metadata could grow an import-json function which would set (overwrite) the metadata for the given file(s) from JSON input. + +Thanks, +-m diff --git a/doc/todo/addurl___8211__force-torrent_option/comment_2_52c04c388b807993cecacc7f98b73cd3._comment b/doc/todo/addurl___8211__force-torrent_option/comment_2_52c04c388b807993cecacc7f98b73cd3._comment new file mode 100644 index 000000000..f6d52be65 --- /dev/null +++ b/doc/todo/addurl___8211__force-torrent_option/comment_2_52c04c388b807993cecacc7f98b73cd3._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkutSE8_3fFAETmO_E598zja4gKwYXbb8E" + nickname="Сергей" + subject="comment 2" + date="2015-04-14T19:57:52Z" + content=""" +Sure, that's even better. +"""]] diff --git a/doc/todo/command_line_interface_for_required_content_setthings.mdwn b/doc/todo/command_line_interface_for_required_content_setthings.mdwn index 1334b151a..30889f8bb 100644 --- a/doc/todo/command_line_interface_for_required_content_setthings.mdwn +++ b/doc/todo/command_line_interface_for_required_content_setthings.mdwn @@ -9,3 +9,5 @@ used feature, and vicfg can already configure it. one when it comes to that. Oh well.) --[[Joey]] + +> [[done]] --[[Joey]] diff --git a/doc/todo/enable_fsck_--fast_for_S3_remotes.mdwn b/doc/todo/enable_fsck_--fast_for_S3_remotes.mdwn new file mode 100644 index 000000000..77392b36d --- /dev/null +++ b/doc/todo/enable_fsck_--fast_for_S3_remotes.mdwn @@ -0,0 +1,30 @@ +At the moment, ``git annex fsck --fast -f S3remote`` fails as + +[[!format sh """ +> git annex fsck --fast -f S3remote +(checking cloud...) [2015-04-24 21:39:35 BST] String to sign: "HEAD\n\n\nFri, 24 Apr 2015 20:39:35 GMT\n/BUCKET/GPGHMACSHA1--6e7e880f80de44ddd845c6241198622b9102eaa1" +[2015-04-24 21:39:35 BST] Host: "BUCKET.s3-ap-southeast-2.amazonaws.com" +[2015-04-24 21:39:35 BST] Path: "/GPGHMACSHA1--6e7e880f80de44ddd845c6241198622b9102eaa1" +[2015-04-24 21:39:35 BST] Query string: "" +[2015-04-24 21:39:36 BST] Response status: Status {statusCode = 200, statusMessage = "OK"} +[2015-04-24 21:39:36 BST] Response header 'x-amz-id-2': 'D9wtD8voZZgijJRc6i8i0QasAo85REdMMf4GpRaER5+g6sDaUYtCKi42RCdYU0kBxrx4d4dM4xM=' +[2015-04-24 21:39:36 BST] Response header 'x-amz-request-id': 'DDF95C327078E584' +[2015-04-24 21:39:36 BST] Response header 'Date': 'Fri, 24 Apr 2015 20:39:37 GMT' +[2015-04-24 21:39:36 BST] Response header 'Last-Modified': 'Sun, 02 Nov 2014 05:42:48 GMT' +[2015-04-24 21:39:36 BST] Response header 'ETag': '"3bd1b766a68a305ba0495af36b353a07"' +[2015-04-24 21:39:36 BST] Response header 'Accept-Ranges': 'bytes' +[2015-04-24 21:39:36 BST] Response header 'Content-Type': '' +[2015-04-24 21:39:36 BST] Response header 'Content-Length': '775647' +[2015-04-24 21:39:36 BST] Response header 'Server': 'AmazonS3' +[2015-04-24 21:39:36 BST] Response metadata: S3: request ID=<none>, x-amz-id-2=<none> + + failed to download file from remote + +failed +"""]] + + +while ``git annex fsck -f S3remote`` works fine. But, to check for the presence of a file (which is my understanding of what ``--fast`` is for here), it shouldn't be necessary to download the file. + +> [[fixed|done]]; it was not supposed to fail at all in this case, and +> won't anymore. --[[Joey]] diff --git a/doc/todo/enable_fsck_--fast_for_S3_remotes/comment_1_b9c1da34a1d55333f864f2b7f9f4e4c7._comment b/doc/todo/enable_fsck_--fast_for_S3_remotes/comment_1_b9c1da34a1d55333f864f2b7f9f4e4c7._comment new file mode 100644 index 000000000..83e95b6df --- /dev/null +++ b/doc/todo/enable_fsck_--fast_for_S3_remotes/comment_1_b9c1da34a1d55333f864f2b7f9f4e4c7._comment @@ -0,0 +1,9 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2015-04-25T01:28:34Z" + content=""" +It's HEADing the file, you can see it in the transcript. + +Appears the error message could be better though. +"""]] diff --git a/doc/todo/enable_fsck_--fast_for_S3_remotes/comment_2_cf86f921db2a9f1f5ffad14616e3279b._comment b/doc/todo/enable_fsck_--fast_for_S3_remotes/comment_2_cf86f921db2a9f1f5ffad14616e3279b._comment new file mode 100644 index 000000000..90c39d2ae --- /dev/null +++ b/doc/todo/enable_fsck_--fast_for_S3_remotes/comment_2_cf86f921db2a9f1f5ffad14616e3279b._comment @@ -0,0 +1,84 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawnSenxKyE_2Z6Wb-EBMO8FciyRywjx1ZiQ" + nickname="Walter" + subject="Output for a full fsck" + date="2015-04-25T08:07:35Z" + content=""" +Sorry, I should have provided this output also, which is when I do a non-fast fsck. Below that is the output for a fsck in a file not in the remote. Basically, they both work. The case of a file not present with --fast also works (it gets a 404 response). But fscking a file with --fast that *is* there gets a 200 response for the HEAD, and then decides it didn't get downloaded properly (it shouldn't download it), and reports a fail. It should see the 200 response and report OK. + +I guess this should have been a bug report instead of todo. + +[[!format sh \"\"\" +> git annex fsck -f cloud file --debug +[2015-04-25 08:52:51 BST] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"show-ref\",\"git-annex\"] +[2015-04-25 08:52:51 BST] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"show-ref\",\"--hash\",\"refs/heads/git-annex\"] +[2015-04-25 08:52:51 BST] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"log\",\"refs/heads/git-annex..7f6b0b58ef362edd43fc89d8ef641e18cfebcb4a\",\"-n1\",\"--pretty=%H\"] +[2015-04-25 08:52:51 BST] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"log\",\"refs/heads/git-annex..19ca78351e854273ccb2b6a83fbaf7e2ed9b32da\",\"-n1\",\"--pretty=%H\"] +[2015-04-25 08:52:51 BST] chat: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"cat-file\",\"--batch\"] +[2015-04-25 08:52:51 BST] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"ls-files\",\"--cached\",\"-z\",\"--\",\"file\"] +[2015-04-25 08:52:51 BST] chat: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"check-attr\",\"-z\",\"--stdin\",\"annex.backend\",\"annex.numcopies\",\"--\"] +[2015-04-25 08:52:51 BST] read: git [\"--version\"] +fsck file [2015-04-25 08:52:51 BST] chat: gpg [\"--batch\",\"--no-tty\",\"--use-agent\",\"--quiet\",\"--trust-model\",\"always\",\"--decrypt\"] +(checking cloud...) [2015-04-25 08:52:51 BST] String to sign: \"HEAD\n\n\nSat, 25 Apr 2015 07:52:51 GMT\n/BUCKET/GPGHMACSHA1--6e7e880f80de44ddd845c6241198622b9102eaa1\" +[2015-04-25 08:52:51 BST] Host: \"BUCKET.s3-ap-southeast-2.amazonaws.com\" +[2015-04-25 08:52:51 BST] Path: \"/GPGHMACSHA1--6e7e880f80de44ddd845c6241198622b9102eaa1\" +[2015-04-25 08:52:51 BST] Query string: \"\" +[2015-04-25 08:52:52 BST] Response status: Status {statusCode = 200, statusMessage = \"OK\"} +[2015-04-25 08:52:52 BST] Response header 'x-amz-id-2': 'mLGNeVBzsS7BusAEsDpIyECSpmErjO0HLA/G04svlIgIwsD+K8FpquTvtuA/UoIJK5FrJV0geCE=' +[2015-04-25 08:52:52 BST] Response header 'x-amz-request-id': '2E977E4D5EC072F6' +[2015-04-25 08:52:52 BST] Response header 'Date': 'Sat, 25 Apr 2015 07:52:53 GMT' +[2015-04-25 08:52:52 BST] Response header 'Last-Modified': 'Sun, 02 Nov 2014 05:42:48 GMT' +[2015-04-25 08:52:52 BST] Response header 'ETag': '\"3bd1b766a68a305ba0495af36b353a07\"' +[2015-04-25 08:52:52 BST] Response header 'Accept-Ranges': 'bytes' +[2015-04-25 08:52:52 BST] Response header 'Content-Type': '' +[2015-04-25 08:52:52 BST] Response header 'Content-Length': '775647' +[2015-04-25 08:52:52 BST] Response header 'Server': 'AmazonS3' +[2015-04-25 08:52:52 BST] Response metadata: S3: request ID=<none>, x-amz-id-2=<none> + +[2015-04-25 08:52:52 BST] String to sign: \"GET\n\n\nSat, 25 Apr 2015 07:52:52 GMT\n/BUCKET/GPGHMACSHA1--6e7e880f80de44ddd845c6241198622b9102eaa1\" +[2015-04-25 08:52:52 BST] Host: \"BUCKET.s3-ap-southeast-2.amazonaws.com\" +[2015-04-25 08:52:52 BST] Path: \"/GPGHMACSHA1--6e7e880f80de44ddd845c6241198622b9102eaa1\" +[2015-04-25 08:52:52 BST] Query string: \"\" +[2015-04-25 08:52:53 BST] Response status: Status {statusCode = 200, statusMessage = \"OK\"} +[2015-04-25 08:52:53 BST] Response header 'x-amz-id-2': 'QufZ3GyBdogXO8nVnqmJGU5mKZ7+I4DnU95aBUhy04f4158CGAIlp8vHrnGAMDVgLnLuM2TA70A=' +[2015-04-25 08:52:53 BST] Response header 'x-amz-request-id': 'A4EBAB4DD9E11352' +[2015-04-25 08:52:53 BST] Response header 'Date': 'Sat, 25 Apr 2015 07:52:54 GMT' +[2015-04-25 08:52:53 BST] Response header 'Last-Modified': 'Sun, 02 Nov 2014 05:42:48 GMT' +[2015-04-25 08:52:53 BST] Response header 'ETag': '\"3bd1b766a68a305ba0495af36b353a07\"' +[2015-04-25 08:52:53 BST] Response header 'Accept-Ranges': 'bytes' +[2015-04-25 08:52:53 BST] Response header 'Content-Type': '' +[2015-04-25 08:52:53 BST] Response header 'Content-Length': '775647' +[2015-04-25 08:52:53 BST] Response header 'Server': 'AmazonS3' +[2015-04-25 08:52:53 BST] Response metadata: S3: request ID=A4EBAB4DD9E11352, x-amz-id-2=QufZ3GyBdogXO8nVnqmJGU5mKZ7+I4DnU95aBUhy04f4158CGAIlp8vHrnGAMDVgLnLuM2TA70A= +74% 189.4KB/s 1s[2015-04-25 08:52:56 BST] chat: gpg [\"--batch\",\"--no-tty\",\"--use-agent\",\"--quiet\",\"--trust-model\",\"always\",\"--batch\",\"--passphrase-fd\",\"15\",\"--decrypt\"] +(checksum...) +ok +\"\"\"]] + + +In contrast, here is the output for a file that isn't in the remote +[[!format sh \"\"\" +> git annex fsck -f cloud notpresent --debug +git annex fsck -f cloud notpresent --debug --numcopies 1 +[2015-04-25 09:00:34 BST] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"show-ref\",\"git-annex\"] +[2015-04-25 09:00:34 BST] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"show-ref\",\"--hash\",\"refs/heads/git-annex\"] +[2015-04-25 09:00:34 BST] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"log\",\"refs/heads/git-annex..071d29cd21384f0ca129c76442c95c705b4ddc7b\",\"-n1\",\"--pretty=%H\"] +[2015-04-25 09:00:34 BST] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"log\",\"refs/heads/git-annex..7f6b0b58ef362edd43fc89d8ef641e18cfebcb4a\",\"-n1\",\"--pretty=%H\"] +[2015-04-25 09:00:34 BST] chat: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"cat-file\",\"--batch\"] +[2015-04-25 09:00:34 BST] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"ls-files\",\"--cached\",\"-z\",\"--\",\"notpresent\"] +fsck notpresent [2015-04-25 09:00:34 BST] chat: gpg [\"--batch\",\"--no-tty\",\"--use-agent\",\"--quiet\",\"--trust-model\",\"always\",\"--decrypt\"] +(checking cloud...) [2015-04-25 09:00:35 BST] String to sign: \"HEAD\n\n\nSat, 25 Apr 2015 08:00:35 GMT\n/BUCKET/GPGHMACSHA1--e46ce4a11bc47622fb40affac818d6128bcd94bd\" +[2015-04-25 09:00:35 BST] Host: \"BUCKET.s3-ap-southeast-2.amazonaws.com\" +[2015-04-25 09:00:35 BST] Path: \"/GPGHMACSHA1--e46ce4a11bc47622fb40affac818d6128bcd94bd\" +[2015-04-25 09:00:35 BST] Query string: \"\" +[2015-04-25 09:00:35 BST] Response status: Status {statusCode = 404, statusMessage = \"Not Found\"} +[2015-04-25 09:00:35 BST] Response header 'x-amz-request-id': 'AFA9934844CD547C' +[2015-04-25 09:00:35 BST] Response header 'x-amz-id-2': 'sDLFvcFj1pBh4Dhar/nxGGneN2ZP9XXPlI7GHyzuO1XiyW94b52pypel/1uSeFouWl8dXo4xOjc=' +[2015-04-25 09:00:35 BST] Response header 'Content-Type': 'application/xml' +[2015-04-25 09:00:35 BST] Response header 'Transfer-Encoding': 'chunked' +[2015-04-25 09:00:35 BST] Response header 'Date': 'Sat, 25 Apr 2015 08:00:34 GMT' +[2015-04-25 09:00:35 BST] Response header 'Server': 'AmazonS3' +[2015-04-25 09:00:35 BST] Response metadata: S3: request ID=<none>, x-amz-id-2=<none> +ok +\"\"\"]] +"""]] diff --git a/doc/todo/enable_fsck_--fast_for_S3_remotes/comment_3_b7402508dfc7bbbd09382692aa740c39._comment b/doc/todo/enable_fsck_--fast_for_S3_remotes/comment_3_b7402508dfc7bbbd09382692aa740c39._comment new file mode 100644 index 000000000..daa24f0cc --- /dev/null +++ b/doc/todo/enable_fsck_--fast_for_S3_remotes/comment_3_b7402508dfc7bbbd09382692aa740c39._comment @@ -0,0 +1,88 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawnSenxKyE_2Z6Wb-EBMO8FciyRywjx1ZiQ" + nickname="Walter" + subject="comment 3" + date="2015-04-25T08:36:25Z" + content=""" +I also obtain the expected result if a file is thought to be present, but isn't. + +[[!format sh \"\"\" +> git annex setpresentkey `git annex lookupkey notpresent` be992080-b1db-11e1-8f79-1b10bb4092ef 1 +setpresentkey SHA256E-s37--2f9b7d77d43f49b59fb00148bc1b3d31a887ba717c988be55b9377d403a91f53 ok + +> git annex fsck --debug -f cloud --fast notpresent +[2015-04-25 09:24:25 BST] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"show-ref\",\"git-annex\"] +[2015-04-25 09:24:25 BST] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"show-ref\",\"--hash\",\"refs/heads/git-annex\"] +[2015-04-25 09:24:25 BST] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"log\",\"refs/heads/git-annex..0547dfd2d61ff9a24a08ff97cf4984bebbd4f0f1\",\"-n1\",\"--pretty=%H\"] +[2015-04-25 09:24:25 BST] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"log\",\"refs/heads/git-annex..b43028d651236ce59a3e47240bead91cdbfc37ea\",\"-n1\",\"--pretty=%H\"] +[2015-04-25 09:24:25 BST] chat: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"cat-file\",\"--batch\"] +[2015-04-25 09:24:25 BST] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"ls-files\",\"--cached\",\"-z\",\"--\",\"notpresent\"] +fsck notpresent [2015-04-25 09:24:25 BST] chat: gpg [\"--batch\",\"--no-tty\",\"--use-agent\",\"--quiet\",\"--trust-model\",\"always\",\"--decrypt\"] +(checking cloud...) [2015-04-25 09:24:25 BST] String to sign: \"HEAD\n\n\nSat, 25 Apr 2015 08:24:25 GMT\n/BUCKET/GPGHMACSHA1--e46ce4a11bc47622fb40affac818d6128bcd94bd\" +[2015-04-25 09:24:25 BST] Host: \"BUCKET.s3-ap-southeast-2.amazonaws.com\" +[2015-04-25 09:24:25 BST] Path: \"/GPGHMACSHA1--e46ce4a11bc47622fb40affac818d6128bcd94bd\" +[2015-04-25 09:24:25 BST] Query string: \"\" +[2015-04-25 09:24:25 BST] Response status: Status {statusCode = 404, statusMessage = \"Not Found\"} +[2015-04-25 09:24:25 BST] Response header 'x-amz-request-id': 'D562150974717AB1' +[2015-04-25 09:24:25 BST] Response header 'x-amz-id-2': 'Geq6BKC3Sg1rUuhgOHE7fOa5fq+L5ecShidW0ktI/ri3zNXKudhK5O5qT2qmUraJP6BCzDFuj1Q=' +[2015-04-25 09:24:25 BST] Response header 'Content-Type': 'application/xml' +[2015-04-25 09:24:25 BST] Response header 'Transfer-Encoding': 'chunked' +[2015-04-25 09:24:25 BST] Response header 'Date': 'Sat, 25 Apr 2015 08:24:24 GMT' +[2015-04-25 09:24:25 BST] Response header 'Server': 'AmazonS3' +[2015-04-25 09:24:25 BST] Response metadata: S3: request ID=<none>, x-amz-id-2=<none> +(fixing location log) + ** Based on the location log, notpresent + ** was expected to be present, but its content is missing. +failed +\"\"\"]] + + +That leaves only one case: when the file isn't thought to be in cloud, but is. For completeness + +[[!format sh \"\"\" +> git annex copy --to cloud notpresent +copy notpresent (checking cloud...) (to cloud...) +ok +(recording state in git...) +> git annex setpresentkey `git annex lookupkey notpresent` be992080-b1db-11e1-8f79-1b10bb4092ef 0 +setpresentkey SHA256E-s37--2f9b7d77d43f49b59fb00148bc1b3d31a887ba717c988be55b9377d403a91f53 ok + +> git annex fsck --debug -f cloud --fast notpresent +[2015-04-25 09:26:33 BST] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"show-ref\",\"git-annex\"] +[2015-04-25 09:26:33 BST] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"show-ref\",\"--hash\",\"refs/heads/git-annex\"] +[2015-04-25 09:26:33 BST] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"log\",\"refs/heads/git-annex..dca379d2631cd39bd205ccb7d6c192faea7c05c5\",\"-n1\",\"--pretty=%H\"] +[2015-04-25 09:26:33 BST] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"log\",\"refs/heads/git-annex..b43028d651236ce59a3e47240bead91cdbfc37ea\",\"-n1\",\"--pretty=%H\"] +[2015-04-25 09:26:33 BST] chat: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"cat-file\",\"--batch\"] +[2015-04-25 09:26:33 BST] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"ls-files\",\"--cached\",\"-z\",\"--\",\"notpresent\"] +fsck notpresent [2015-04-25 09:26:33 BST] chat: gpg [\"--batch\",\"--no-tty\",\"--use-agent\",\"--quiet\",\"--trust-model\",\"always\",\"--decrypt\"] +(checking cloud...) [2015-04-25 09:26:33 BST] String to sign: \"HEAD\n\n\nSat, 25 Apr 2015 08:26:33 GMT\n/BUCKET/GPGHMACSHA1--e46ce4a11bc47622fb40affac818d6128bcd94bd\" +[2015-04-25 09:26:33 BST] Host: \"BUCKET.s3-ap-southeast-2.amazonaws.com\" +[2015-04-25 09:26:33 BST] Path: \"/GPGHMACSHA1--e46ce4a11bc47622fb40affac818d6128bcd94bd\" +[2015-04-25 09:26:33 BST] Query string: \"\" +[2015-04-25 09:26:34 BST] Response status: Status {statusCode = 200, statusMessage = \"OK\"} +[2015-04-25 09:26:34 BST] Response header 'x-amz-id-2': '4Ti/62fBMzjW0woyrX5C++tQUw4uV97bbowjSiCkUNI6X2bAt+JCKbRYvZf/Is1QSY6SI2Aqgv4=' +[2015-04-25 09:26:34 BST] Response header 'x-amz-request-id': '9311809D4C8485FD' +[2015-04-25 09:26:34 BST] Response header 'Date': 'Sat, 25 Apr 2015 08:26:35 GMT' +[2015-04-25 09:26:34 BST] Response header 'Last-Modified': 'Sat, 25 Apr 2015 08:26:22 GMT' +[2015-04-25 09:26:34 BST] Response header 'ETag': '\"c5c3c0f720110210e73c7bf962d76390\"' +[2015-04-25 09:26:34 BST] Response header 'Accept-Ranges': 'bytes' +[2015-04-25 09:26:34 BST] Response header 'Content-Type': 'binary/octet-stream' +[2015-04-25 09:26:34 BST] Response header 'Content-Length': '99' +[2015-04-25 09:26:34 BST] Response header 'Server': 'AmazonS3' +[2015-04-25 09:26:34 BST] Response metadata: S3: request ID=<none>, x-amz-id-2=<none> + + failed to download file from remote +(fixing location log) failed +[2015-04-25 09:26:34 BST] chat: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"hash-object\",\"-w\",\"--stdin-paths\",\"--no-filters\"] +[2015-04-25 09:26:34 BST] feed: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"update-index\",\"-z\",\"--index-info\"] +[2015-04-25 09:26:34 BST] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"show-ref\",\"--hash\",\"refs/heads/git-annex\"] +(recording state in git...) +[2015-04-25 09:26:34 BST] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"write-tree\"] +[2015-04-25 09:26:34 BST] chat: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"commit-tree\",\"83cec04d148757f98565eacda236d6e9dbd48678\",\"--no-gpg-sign\",\"-p\",\"refs/heads/git-annex\"] +[2015-04-25 09:26:34 BST] call: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"update-ref\",\"refs/heads/git-annex\",\"31d4a714f6977197029faf23b099ea32a298be59\"] +git-annex: fsck: 1 failed +\"\"\"]] + +This correctly determines that the file is present, and updates the location log. But I don't understand why the message ``failed to download file from remote`` is used (which is also used when a file is present, and thought to be present). For a fast fsck it shouldn't be trying to download the file. Also, I don't think this is specific to S3, I expect any remote will have the same behaviour. + +"""]] diff --git a/doc/todo/git-annex-standalone_Debian_package.mdwn b/doc/todo/git-annex-standalone_Debian_package.mdwn index 0172e1e72..e45a72843 100644 --- a/doc/todo/git-annex-standalone_Debian_package.mdwn +++ b/doc/todo/git-annex-standalone_Debian_package.mdwn @@ -1 +1,3 @@ As proposed with a sketch in https://github.com/joeyh/git-annex/pull/39, for DataLad we would need to get recent annex on older Debian/Ubuntu releases to get our testing farm and perspective users equipped with bleeding edge annex + +> merged [[done]] --[[Joey]] diff --git a/doc/todo/make_glacier-cli_executable_path_configurable.mdwn b/doc/todo/make_glacier-cli_executable_path_configurable.mdwn new file mode 100644 index 000000000..4c4028dc0 --- /dev/null +++ b/doc/todo/make_glacier-cli_executable_path_configurable.mdwn @@ -0,0 +1,8 @@ +[glacier-cli](https://github.com/basak/glacier-cli) calls its own command `glacier` rather than `glacier-cli` or something else. This conflicts with [boto](https://github.com/boto/boto/)'s own `glacier` executable, as noted here: + +* <https://github.com/basak/glacier-cli/issues/30> +* <https://github.com/basak/glacier-cli/issues/47> + +Whilst the `glacier-cli` project should resolve this conflict, it would be good if git-annex could be made to use a configurable path for this executable, rather than just assuming that it has been installed as `glacier`. After all, its installation procedure is simply telling the user to run `ln -s`, so there's no reason why the user couldn't make the target of this command `~/bin/glacier-cli` rather than `~/bin/glacier` - it's really irrelevant what the source file inside the git repo is called. + +Of course, [`checkSaneGlacierCommand`](https://github.com/joeyh/git-annex/blob/master/Remote/Glacier.hs#L307) is still very much worth having, for safety. diff --git a/doc/todo/make_glacier-cli_executable_path_configurable/comment_1_08ab00266ad06fed9123d6a2ea0b5e6a._comment b/doc/todo/make_glacier-cli_executable_path_configurable/comment_1_08ab00266ad06fed9123d6a2ea0b5e6a._comment new file mode 100644 index 000000000..77c1bdd43 --- /dev/null +++ b/doc/todo/make_glacier-cli_executable_path_configurable/comment_1_08ab00266ad06fed9123d6a2ea0b5e6a._comment @@ -0,0 +1,7 @@ +[[!comment format=mdwn + username="basak" + subject="comment 1" + date="2015-04-24T15:48:48Z" + content=""" +Well, it's supposed to be a command line command, and I don't type `cd-cli` and `ls-cli`. So while `glacier-cli` might be fine as a project name and is fine for a name for integration, I don't think it makes sense to call it that in `/usr/bin/`, which is why I didn't. I'd prefer to have seen that boto integrate an improved `glacier` command, or for packaging to provide this one as an alternative (like `mawk` vs. `gawk` as `/usr/bin/awk`). But upstream boto considers themselves deprecated, so that's not going to happen. One of these days I'll package glacier-cli up for Debian, at which point I'll see if the boto maintainer is interested in doing something, since I don't actually believe anybody uses boto's glacier command (since it's mostly useless). +"""]] diff --git a/doc/todo/make_glacier-cli_executable_path_configurable/comment_2_d89e073643af0d80833b2d7c9752d23d._comment b/doc/todo/make_glacier-cli_executable_path_configurable/comment_2_d89e073643af0d80833b2d7c9752d23d._comment new file mode 100644 index 000000000..8032ed33b --- /dev/null +++ b/doc/todo/make_glacier-cli_executable_path_configurable/comment_2_d89e073643af0d80833b2d7c9752d23d._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://adamspiers.wordpress.com/" + nickname="adamspiers" + subject="Good point" + date="2015-04-24T15:55:29Z" + content=""" +glacier-cli would be a rather silly name to put in `/usr/bin`. How about `glcr`, as suggested [here](https://github.com/basak/glacier-cli/issues/30#issuecomment-95972840)? +"""]] diff --git a/doc/todo/make_glacier-cli_executable_path_configurable/comment_3_451c6788535e27482377cd60128c1cd6._comment b/doc/todo/make_glacier-cli_executable_path_configurable/comment_3_451c6788535e27482377cd60128c1cd6._comment new file mode 100644 index 000000000..da9033f79 --- /dev/null +++ b/doc/todo/make_glacier-cli_executable_path_configurable/comment_3_451c6788535e27482377cd60128c1cd6._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 3""" + date="2015-04-24T17:23:10Z" + content=""" +I don't want to complicate git-annex more with configurable names for +programs, and glacier is not at all special in this regard, any program +could be installed under any namee. We pick non-conflicting names to +avoid integration nightmares. Pick a name and I'll use it. +"""]] diff --git a/doc/todo/patch:_Add_names_to_.mailmap_to_remove_duplicates_from_git_shortlog.mdwn b/doc/todo/patch:_Add_names_to_.mailmap_to_remove_duplicates_from_git_shortlog.mdwn new file mode 100644 index 000000000..cb48db344 --- /dev/null +++ b/doc/todo/patch:_Add_names_to_.mailmap_to_remove_duplicates_from_git_shortlog.mdwn @@ -0,0 +1,147 @@ +Hi, +these three patches sort .mailmap to get consistent ordering, add nine +names with proper emails to it, and then there is the third one, which +contains all the joeyh changes. I put them in a separate patch in case +you have other opinions about that. + +I notice you're not a fan of GitHub pull requests, and attachments +wasn't allowed here, so I just paste a `cat 000* >all.patch` here, hope +that's ok. The branches are also available from +<https://github.com/sunny256/git-annex> as the branches "edit-mailmap" +(this version) and "edit-mailmap.wip" (the whole process) in case that's +easier. + +There will be more "useful" patches in the future, have started browsing +"Learn you a Haskell for great good", it's awesome. I'll have to get the +build working first, though. There is some dependency problem: + +[[!format hs """ + +$ make +if [ "cabal " = ./Setup ]; then ghc --make Setup; fi +cabal configure +Resolving dependencies... + +Utility/Exception.hs:25:18: + Could not find module `Control.Monad.Catch' + Perhaps you meant + Control.Monad.CatchIO (from MonadCatchIO-mtl-0.3.0.4) + Control.Monad.Cont (needs flag -package mtl-2.1.1) + Control.Monad.State (needs flag -package mtl-2.1.1) + Use -v to see a list of the files searched for. +make: *** [Build/SysConfig.hs] Error 1 + +"""]] + +Mentioning it just in case you have a quick solution. Have tried to fix +it by summoning cabal in various ways, but no luck yet. The OS used is +Debian GNU/Linux 7.8 (wheezy) on x86_64. + +[[!format sh """ + +From 20317aff9fbb8662aaeda4aa2285f92e728adc58 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?=C3=98yvind=20A=2E=20Holm?= <sunny@sunbase.org> +Date: Sat, 2 May 2015 17:22:48 +0200 +Subject: [PATCH 1/3] Filter .mailmap through "sort -u" for predictability + +--- + .mailmap | 4 ++-- + 1 file changed, 2 insertions(+), 2 deletions(-) + +diff --git a/.mailmap b/.mailmap +index 275b236..5d51042 100644 +--- a/.mailmap ++++ b/.mailmap +@@ -1,7 +1,7 @@ + Joey Hess <id@joeyh.name> http://joey.kitenet.net/ <joey@web> +-Joey Hess <id@joeyh.name> http://joeyh.name/ <joey@web> + Joey Hess <id@joeyh.name> http://joeyh.name/ <http://joeyh.name/@web> ++Joey Hess <id@joeyh.name> http://joeyh.name/ <joey@web> ++Richard Hartmann <richih@debian.org> https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U <Richard@web> + Yaroslav Halchenko <debian@onerussian.com> + Yaroslav Halchenko <debian@onerussian.com> http://yarikoptic.myopenid.com/ <site-myopenid@web> + Yaroslav Halchenko <debian@onerussian.com> https://www.google.com/accounts/o8/id?id=AItOawnx8kHW66N3BqmkVpgtXDlYMvr8TJ5VvfY <Yaroslav@web> +-Richard Hartmann <richih@debian.org> https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U <Richard@web> +-- +2.4.0 + +From b216bfdb3ab65f025e46c7fcdc86db3a3440b0af Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?=C3=98yvind=20A=2E=20Holm?= <sunny@sunbase.org> +Date: Mon, 4 May 2015 15:36:41 +0200 +Subject: [PATCH 2/3] .mailmap: Add nine more uncontroversial names + +Including only those with a proper email where there is no doubt about +which is the correct one. +--- + .mailmap | 13 +++++++++++++ + 1 file changed, 13 insertions(+) + +diff --git a/.mailmap b/.mailmap +index 5d51042..2dafcea 100644 +--- a/.mailmap ++++ b/.mailmap +@@ -1,7 +1,20 @@ ++Antoine Beaupré <anarcat@koumbit.org> anarcat <anarcat@web> ++Antoine Beaupré <anarcat@koumbit.org> https://id.koumbit.net/anarcat <https://id.koumbit.net/anarcat@web> ++Greg Grossmeier <greg@grossmeier.net> http://grossmeier.net/ <greg@web> ++Jimmy Tang <jtang@tchpc.tcd.ie> jtang <jtang@web> ++Joachim Breitner <mail@joachim-breitner.de> http://www.joachim-breitner.de/ <nomeata@web> + Joey Hess <id@joeyh.name> http://joey.kitenet.net/ <joey@web> + Joey Hess <id@joeyh.name> http://joeyh.name/ <http://joeyh.name/@web> + Joey Hess <id@joeyh.name> http://joeyh.name/ <joey@web> ++Johan Kiviniemi <devel@johan.kiviniemi.name> http://johan.kiviniemi.name/ <Johan@web> ++Johan Kiviniemi <devel@johan.kiviniemi.name> http://johan.kiviniemi.name/ <http://johan.kiviniemi.name/@web> ++Nicolas Pouillard <nicolas.pouillard@gmail.com> http://ertai.myopenid.com/ <npouillard@web> ++Peter Simons <simons@cryp.to> Peter Simons <simons@ubuntu-12.04> ++Peter Simons <simons@cryp.to> http://peter-simons.myopenid.com/ <http://peter-simons.myopenid.com/@web> ++Philipp Kern <pkern@debian.org> http://phil.0x539.de/ <Philipp_Kern@web> + Richard Hartmann <richih@debian.org> https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U <Richard@web> + Yaroslav Halchenko <debian@onerussian.com> + Yaroslav Halchenko <debian@onerussian.com> http://yarikoptic.myopenid.com/ <site-myopenid@web> + Yaroslav Halchenko <debian@onerussian.com> https://www.google.com/accounts/o8/id?id=AItOawnx8kHW66N3BqmkVpgtXDlYMvr8TJ5VvfY <Yaroslav@web> ++Øyvind A. Holm <sunny@sunbase.org> http://sunny256.sunbase.org/ <sunny256@web> ++Øyvind A. Holm <sunny@sunbase.org> https://sunny256.wordpress.com/ <sunny256@web> +-- +2.4.0 + +From b730720bf85217051b0bd6414650f3bfd5928edb Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?=C3=98yvind=20A=2E=20Holm?= <sunny@sunbase.org> +Date: Mon, 4 May 2015 15:46:29 +0200 +Subject: [PATCH 3/3] .mailmap: Add all variations for Joey Hess + +--- + .mailmap | 8 ++++++++ + 1 file changed, 8 insertions(+) + +diff --git a/.mailmap b/.mailmap +index 2dafcea..3013a39 100644 +--- a/.mailmap ++++ b/.mailmap +@@ -3,9 +3,17 @@ Antoine Beaupré <anarcat@koumbit.org> https://id.koumbit.net/anarcat <https://i + Greg Grossmeier <greg@grossmeier.net> http://grossmeier.net/ <greg@web> + Jimmy Tang <jtang@tchpc.tcd.ie> jtang <jtang@web> + Joachim Breitner <mail@joachim-breitner.de> http://www.joachim-breitner.de/ <nomeata@web> ++Joey Hess <id@joeyh.name> Joey Hess <joey@gnu.kitenet.net> ++Joey Hess <id@joeyh.name> Joey Hess <joey@kitenet.net> ++Joey Hess <id@joeyh.name> Joey Hess <joeyh@debian.org> ++Joey Hess <id@joeyh.name> Joey Hess <joeyh@fischer.debian.org> ++Joey Hess <id@joeyh.name> Joey Hess <joeyh@joeyh.name> ++Joey Hess <id@joeyh.name> Joey Hess <joeyh@oberon.tam-lin.net> ++Joey Hess <id@joeyh.name> Joey Hess <joeyh@oberon.underhill.private> + Joey Hess <id@joeyh.name> http://joey.kitenet.net/ <joey@web> + Joey Hess <id@joeyh.name> http://joeyh.name/ <http://joeyh.name/@web> + Joey Hess <id@joeyh.name> http://joeyh.name/ <joey@web> ++Joey Hess <id@joeyh.name> https://www.google.com/accounts/o8/id?id=AItOawmJfIszzreLNvCqzqzvTayA9_9L6gb9RtY <Joey@web> + Johan Kiviniemi <devel@johan.kiviniemi.name> http://johan.kiviniemi.name/ <Johan@web> + Johan Kiviniemi <devel@johan.kiviniemi.name> http://johan.kiviniemi.name/ <http://johan.kiviniemi.name/@web> + Nicolas Pouillard <nicolas.pouillard@gmail.com> http://ertai.myopenid.com/ <npouillard@web> +-- +2.4.0 + +"""]] + +> [[merged|done]] --[[Joey]] +> +> Control.Monad.Catch should be provided by the exceptions package, +> which there is a dependency on in the cabal file. However, building +> git-annex on wheezy is not supported anymore. --[[Joey]] diff --git a/doc/todo/patch:_Add_names_to_.mailmap_to_remove_duplicates_from_git_shortlog/comment_1_031c5a4afe9453a5b62004c85e7e7192._comment b/doc/todo/patch:_Add_names_to_.mailmap_to_remove_duplicates_from_git_shortlog/comment_1_031c5a4afe9453a5b62004c85e7e7192._comment new file mode 100644 index 000000000..474c61c1e --- /dev/null +++ b/doc/todo/patch:_Add_names_to_.mailmap_to_remove_duplicates_from_git_shortlog/comment_1_031c5a4afe9453a5b62004c85e7e7192._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://sunny256.wordpress.com/" + nickname="sunny256" + subject="Character encoding error" + date="2015-05-06T14:53:50Z" + content=""" +Hm, seems as there are some character encoding problem there, two names (Antoine Beaupré and mine) in the second patch. The infamous 'Ø' strikes again. Just wanted to mention it so the patch doesn't introduce errors. Maybe it's safest to fetch it from GitHub. +"""]] diff --git a/doc/todo/replace_dataenc_with_sandi.mdwn b/doc/todo/replace_dataenc_with_sandi.mdwn index a35e54efd..9ce29663b 100644 --- a/doc/todo/replace_dataenc_with_sandi.mdwn +++ b/doc/todo/replace_dataenc_with_sandi.mdwn @@ -2,3 +2,5 @@ sandi is available in jessie, but not wheezy, so this is pending EOL of wheezy support. --[[Joey]] + +> [[fixed|done]] --[[Joey]] diff --git a/doc/todo/server-level_daemon__63__.mdwn b/doc/todo/server-level_daemon__63__.mdwn index 931c9219b..544624031 100644 --- a/doc/todo/server-level_daemon__63__.mdwn +++ b/doc/todo/server-level_daemon__63__.mdwn @@ -186,3 +186,18 @@ Now this is not without problems: 3. it is Debian-specific (a proper init script would be POSIX only and/or a `.service` file) Maybe using [metainit](https://wiki.debian.org/MetaInit) would be a good idea here? --[[anarcat]] + +> This strikes me as a bad idea if the system has regular desktop etc +> users; the assistant can already start itself when they login and +> a separate user has the same problems the separate mpd user seems to; +> of separating the user from the files that effcetively belong to them. +> +> It's fine if you're doing something more specialized, but that seems +> like an unusual case and not one that git-annex should cater to. +> +> Anyway, it's about 5 lines to write a systemd service file. I've added +> `git annex assistant --autostop` that such a service can use if desired. +> +> Since I don't want to include that in git-annex and be stuck supporting +> it, and it should be easy for users to add if they need it, I think I'm +> going to call this [[done]]. --[[Joey]] diff --git a/doc/todo/smudge/comment_4_5ff4ad15865a93dc8c066220561936b2._comment b/doc/todo/smudge/comment_4_5ff4ad15865a93dc8c066220561936b2._comment new file mode 100644 index 000000000..392df5389 --- /dev/null +++ b/doc/todo/smudge/comment_4_5ff4ad15865a93dc8c066220561936b2._comment @@ -0,0 +1,9 @@ +[[!comment format=mdwn + username="tomekwi" + subject="How about the other way round?" + date="2015-04-25T09:13:31Z" + content=""" +If I understand this right, this feature should allow us to say to *git*: “Hey, from now on whenever I `git add` a `*.png` file, add it to *git-annex* instead!” + +How about saying to *git-annex*: “Hey, whenever I `git-annex add` a file which is *not* `*.png`, add it to *git* instead! Or at least leave it unadded so that I can decide later.” Is it possible now? If not, would it be reasonable to add such a feature? +"""]] diff --git a/doc/todo/smudge/comment_5_80bbc3710f9a18644571c6dd60baf4e5._comment b/doc/todo/smudge/comment_5_80bbc3710f9a18644571c6dd60baf4e5._comment new file mode 100644 index 000000000..04904dc7c --- /dev/null +++ b/doc/todo/smudge/comment_5_80bbc3710f9a18644571c6dd60baf4e5._comment @@ -0,0 +1,7 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 5""" + date="2015-04-29T15:30:50Z" + content=""" +That feature already exists (annex.largefiles). +"""]] diff --git a/doc/todo/smudge/comment_6_86f536515575a6c2ed3a89c80b2c232f._comment b/doc/todo/smudge/comment_6_86f536515575a6c2ed3a89c80b2c232f._comment new file mode 100644 index 000000000..6a15157bc --- /dev/null +++ b/doc/todo/smudge/comment_6_86f536515575a6c2ed3a89c80b2c232f._comment @@ -0,0 +1,7 @@ +[[!comment format=mdwn + username="tomekwi" + subject="comment 6" + date="2015-04-29T20:38:51Z" + content=""" +Thanks! Looks great :) +"""]] diff --git a/doc/todo/transfer_between_git-annexes.mdwn b/doc/todo/transfer_between_git-annexes.mdwn new file mode 100644 index 000000000..ccb6908c2 --- /dev/null +++ b/doc/todo/transfer_between_git-annexes.mdwn @@ -0,0 +1,20 @@ +What do you think of the ability to transfer a file between unrelated annexes? With "migrate" already taken, I would suggest "catapult" (or "teleport")! + + git annex catapult dir1/ $HOME/otherannex/somedir/ + git annex catapult dir2/thisfile.jpg $HOME/otherannex/somedir/ + +git-annex would then: + +* Get list of present files +* Copy the file to temporary space in $HOME/otherannex/.git/annex +* fsck file +* Move file to $HOME/otherannex/.git/annnex/objects +* Create symlinks/directories in $HOME/otherannex/somedir/ +* Stage symlinks +* Drop content and rm symlink + +with the usual modifiers (e.g. --fast would skip the fsck, --force to skip non-present files?). + +Reason I ask: I have a huge annex from importing the contents of a bunch of random harddrives and will eventually sort the contents into various other annexes I can put files into (personal, general family, specific people). Having git-annex guiding and checking the transfers from the sorting annex to the individual ones would be really nice. + +Not having this isn't a showstopper (I can use rsync) so no worries if you don't think it is worth it or think it is but put it on the backburner :) Would just be a nice-to-have. diff --git a/doc/todo/wishlist:___39__get__39___queue_and_schedule./comment_1_d6ab311beee2ce5f73ae325a4ae463d4._comment b/doc/todo/wishlist:___39__get__39___queue_and_schedule./comment_1_d6ab311beee2ce5f73ae325a4ae463d4._comment new file mode 100644 index 000000000..df273afb2 --- /dev/null +++ b/doc/todo/wishlist:___39__get__39___queue_and_schedule./comment_1_d6ab311beee2ce5f73ae325a4ae463d4._comment @@ -0,0 +1,11 @@ +[[!comment format=mdwn + username="anarcat" + subject="how does the queue get built now?" + date="2015-04-28T21:32:36Z" + content=""" +how does the queue get created now? is it random or is there some deterministic thing that could be abuse to prioritise some content already (say alphabetical order or some similar hack)? + +the scheduling I think i can work around with clunky cron jobs to start/stop the assistant, but the queue I feel is a different beast. would you be available for hire to implement this feature in priority? + +thanks! +"""]] diff --git a/doc/todo/wishlist:___96__git_annex_optimize__96__.mdwn b/doc/todo/wishlist:___96__git_annex_optimize__96__.mdwn new file mode 100644 index 000000000..8e6fced45 --- /dev/null +++ b/doc/todo/wishlist:___96__git_annex_optimize__96__.mdwn @@ -0,0 +1,6 @@ +Given [this tip](https://git-annex.branchable.com/forum/__34__git_annex_sync__34___synced_after_8_hours/#comment-890ca1381d800ac833ccbb8c5db175ea), [this comment](https://git-annex.branchable.com/todo/wishlist:_rsync_efficiency/#comment-870ae805efd35343edefdbed792dac04) and possibly others, it would be nice if git-annex could look at any given repo and make suggestions for improvements. As a second step, it could look at remotes as well. And as a third, maybe even change repo settings and not just make suggestions. + +Having a few old repos with terabytes of data on various disks, I would just toss this stanza into my mr templates to eventually optimize all my repos to current best practices. + + +-- Richard diff --git a/doc/todo/wishlist:_rsync_efficiency/comment_1_6e7cceb9d23f0cad3d9f839dd2a04901._comment b/doc/todo/wishlist:_rsync_efficiency/comment_1_6e7cceb9d23f0cad3d9f839dd2a04901._comment new file mode 100644 index 000000000..35f6b885b --- /dev/null +++ b/doc/todo/wishlist:_rsync_efficiency/comment_1_6e7cceb9d23f0cad3d9f839dd2a04901._comment @@ -0,0 +1,18 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2015-04-18T19:21:44Z" + content=""" +The `concurrentprogress` branch can already parallelize transfers. +It's waiting on some progress display improvements for merging. + +I imagine this would keep the pipe full more consistently. + +Although, ssh caching is perhaps even more important, since +it avoids a tcp slow start having to be done for each file +that's transferred. So make sure it's working. + +Even with those, I would not expect git-annex to be as efficient +as pure rsync of a directory. git-annex's more general design mean +that there are more moving parts.. +"""]] diff --git a/doc/todo/wishlist:_rsync_efficiency/comment_2_d87df8f1012f71248ed33f64f9ace17e._comment b/doc/todo/wishlist:_rsync_efficiency/comment_2_d87df8f1012f71248ed33f64f9ace17e._comment new file mode 100644 index 000000000..3acd6b74b --- /dev/null +++ b/doc/todo/wishlist:_rsync_efficiency/comment_2_d87df8f1012f71248ed33f64f9ace17e._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" + nickname="Richard" + subject="comment 2" + date="2015-04-20T14:37:03Z" + content=""" +I read the blog after my time abroad and chuckled about the timing of my request, yah :) + +Could git-annex reasonably detach generating the list of files to transfer from the actual transfer? That way, there is never a delay while git-annex looks for the next file. + +Richard +"""]] diff --git a/doc/transferring_data.mdwn b/doc/transferring_data.mdwn index d1ec5963f..2aab3b01f 100644 --- a/doc/transferring_data.mdwn +++ b/doc/transferring_data.mdwn @@ -9,7 +9,7 @@ If a data transfer is interrupted, git-annex retains the partial transfer to allow it to be automatically resumed later. It's equally easy to transfer a single file to or from a repository, -or to launch a retrievel of a massive pile of files from whatever +or to launch a retrieval of a massive pile of files from whatever repositories they are scattered amongst. git-annex automatically uses whatever remotes are currently accessible, diff --git a/doc/trust.mdwn b/doc/trust.mdwn index 1fd47fd1d..a33c6dd42 100644 --- a/doc/trust.mdwn +++ b/doc/trust.mdwn @@ -45,7 +45,7 @@ archival drive, from which you rarely or never remove content. Deciding when it makes sense to trust the tracking info is up to you. One way to handle this is just to use `--force` when a command cannot -access a remote you trust. Or to use `--trust` to specify a repisitory to +access a remote you trust. Or to use `--trust` to specify a repository to trust temporarily. To configure a repository as fully and permanently trusted, @@ -55,5 +55,5 @@ use the `git annex trust` command. This is used to indicate that you have no trust that the repository exists at all. It's appropriate to use when a drive has been lost, -or a directory irretrevably deleted. It will make git-annex avoid +or a directory irretrievably deleted. It will make git-annex avoid even showing the repository as a place where data might still reside. diff --git a/doc/trust/comment_1_305e4e7c6b75db29212b758e8504d8c9._comment b/doc/trust/comment_1_305e4e7c6b75db29212b758e8504d8c9._comment new file mode 100644 index 000000000..11c6ce133 --- /dev/null +++ b/doc/trust/comment_1_305e4e7c6b75db29212b758e8504d8c9._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawlJl0OCe6AJEnIFIcg-t5Rhk-lI_Y-tWUs" + nickname="Michael" + subject="default trust for hosts" + date="2015-05-01T21:41:05Z" + content=""" +Is it possible to set a default trust per host (e.g. in `~/.gitconfig`)? + +I have one server that does its own backups, and a client that I'd like to keep thin across multiple repositories. +"""]] diff --git a/doc/trust/comment_2_2262eaa830306d3dc75999bc0433b6a8._comment b/doc/trust/comment_2_2262eaa830306d3dc75999bc0433b6a8._comment new file mode 100644 index 000000000..b2136f43e --- /dev/null +++ b/doc/trust/comment_2_2262eaa830306d3dc75999bc0433b6a8._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 2""" + date="2015-05-05T18:07:02Z" + content=""" +You can use `remote.<name>.annex-trustlevel` as documented in the git-annex +man page. +"""]] diff --git a/doc/upgrades.mdwn b/doc/upgrades.mdwn index 0b43a972d..6cf2784e7 100644 --- a/doc/upgrades.mdwn +++ b/doc/upgrades.mdwn @@ -1,9 +1,9 @@ -Occasionally improvments are made to how git-annex stores its data, +Occasionally improvements are made to how git-annex stores its data, that require an upgrade process to convert repositories made with an older version to be used by a newer version. It's annoying, it should happen rarely, but sometimes, it's worth it. -There's a committment that git-annex will always support upgrades from all +There's a commitment that git-annex will always support upgrades from all past versions. After all, you may have offline drives from an earlier git-annex, and might want to use them with a newer git-annex. diff --git a/git-annex.cabal b/git-annex.cabal index 4340deaff..a67cc8ea5 100644 --- a/git-annex.cabal +++ b/git-annex.cabal @@ -1,5 +1,5 @@ Name: git-annex -Version: 5.20150409 +Version: 5.20150508.1 Cabal-Version: >= 1.8 License: GPL-3 Maintainer: Joey Hess <id@joeyh.name> @@ -101,21 +101,31 @@ Flag network-uri Description: Get Network.URI from the network-uri package Default: True +Flag new-time + Description: Build with new version of time and without old-locale + Default: True + Executable git-annex Main-Is: git-annex.hs - Build-Depends: MissingH, hslogger, directory, filepath, - containers (>= 0.5.0.0), utf8-string, mtl (>= 2), - bytestring, old-locale, time, dataenc, SHA, process, json, - base (>= 4.5 && < 4.9), monad-control, exceptions (>= 0.6), transformers, - IfElse, text, QuickCheck >= 2.1, bloomfilter, edit-distance, - SafeSemaphore, uuid, random, dlist, unix-compat, async, stm (>= 2.3), - data-default, case-insensitive, http-conduit, http-types, - cryptohash (>= 0.10.0), + Build-Depends: + base (>= 4.5 && < 4.9), + cryptohash (>= 0.11.0), + containers (>= 0.5.0.0), + exceptions (>= 0.6), + QuickCheck (>= 2.1), + stm (>= 2.3), + mtl (>= 2), + process, data-default, case-insensitive, uuid, random, dlist, + unix-compat, SafeSemaphore, async, directory, filepath, IfElse, + MissingH, hslogger, monad-logger, + utf8-string, bytestring, text, sandi, json, + monad-control, transformers, + bloomfilter, edit-distance, + resourcet, http-conduit, http-types, esqueleto, persistent-sqlite, persistent, persistent-template, - monad-logger, resourcet, ascii-progress (<= 0.2.1.2), terminal-size CC-Options: -Wall - GHC-Options: -Wall + GHC-Options: -Wall -fno-warn-tabs Extensions: PackageImports -- Some things don't work with the non-threaded RTS. GHC-Options: -threaded @@ -125,6 +135,11 @@ Executable git-annex else Build-Depends: network (< 2.6), network (>= 2.0) + if flag(new-time) + Build-Depends: time (>= 1.5) + else + Build-Depends: time, old-locale + if flag(Production) GHC-Options: -O2 @@ -184,7 +199,7 @@ Executable git-annex if (os(linux)) if flag(Dbus) - Build-Depends: dbus (>= 0.10.3) + Build-Depends: dbus (>= 0.10.7) CPP-Options: -DWITH_DBUS if flag(DesktopNotify) @@ -200,12 +215,20 @@ Executable git-annex if flag(Webapp) Build-Depends: - yesod, yesod-default, yesod-static, yesod-form, yesod-core, - wai, wai-extra, warp, warp-tls, + yesod (>= 1.2.6), + yesod-default (>= 1.2.0), + yesod-static (>= 1.2.4), + yesod-form (>= 1.3.15), + yesod-core (>= 1.2.19), + path-pieces (>= 0.1.4), + warp (>= 3.0.0.5), + warp-tls, + wai, wai-extra, blaze-builder, crypto-api, hamlet, clientsession, - template-haskell, aeson, path-pieces, + template-haskell, aeson, shakespeare CPP-Options: -DWITH_WEBAPP + if flag(Webapp) && flag (Webapp-secure) Build-Depends: warp-tls (>= 1.4), securemem, byteable CPP-Options: -DWITH_WEBAPP_SECURE diff --git a/standalone/android/cabal.config b/standalone/android/cabal.config index 4eecbfeaf..de7d72dc0 100644 --- a/standalone/android/cabal.config +++ b/standalone/android/cabal.config @@ -6,7 +6,6 @@ constraints: Crypto ==4.2.5.1, MissingH ==1.2.1.0, MonadRandom ==0.1.13, QuickCheck ==2.7.6, - SHA ==1.6.1, SafeSemaphore ==0.10.1, aeson ==0.7.0.6, ansi-terminal ==0.6.1.1, @@ -55,7 +54,7 @@ constraints: Crypto ==4.2.5.1, data-default-instances-containers ==0.0.1, data-default-instances-dlist ==0.0.1, data-default-instances-old-locale ==0.0.1, - dataenc ==0.14.0.7, + sandi ==0.3.0.1, dbus ==0.10.8, distributive ==0.4.4, dlist ==0.7.0.1, @@ -82,9 +81,9 @@ constraints: Crypto ==4.2.5.1, hinotify ==0.3.5, hjsmin ==0.1.4.7, hslogger ==1.2.1, - http-client ==0.3.8.2, + http-client ==0.4.11.1, http-client-tls ==0.2.2, - http-conduit ==2.1.2.3, + http-conduit ==2.1.5, http-date ==0.0.2, http-types ==0.8.5, hxt ==9.3.1.4, diff --git a/standalone/linux/cabal.config b/standalone/linux/cabal.config index 4eecbfeaf..f39dee91f 100644 --- a/standalone/linux/cabal.config +++ b/standalone/linux/cabal.config @@ -6,7 +6,6 @@ constraints: Crypto ==4.2.5.1, MissingH ==1.2.1.0, MonadRandom ==0.1.13, QuickCheck ==2.7.6, - SHA ==1.6.1, SafeSemaphore ==0.10.1, aeson ==0.7.0.6, ansi-terminal ==0.6.1.1, @@ -82,9 +81,9 @@ constraints: Crypto ==4.2.5.1, hinotify ==0.3.5, hjsmin ==0.1.4.7, hslogger ==1.2.1, - http-client ==0.3.8.2, + http-client ==0.4.11.1, http-client-tls ==0.2.2, - http-conduit ==2.1.2.3, + http-conduit ==2.1.5, http-date ==0.0.2, http-types ==0.8.5, hxt ==9.3.1.4, diff --git a/standalone/linux/skel/runshell b/standalone/linux/skel/runshell index f6db1da13..6d846104c 100755 --- a/standalone/linux/skel/runshell +++ b/standalone/linux/skel/runshell @@ -31,16 +31,18 @@ cd "$orig" # assistant. if [ ! -e "$HOME/.ssh/git-annex-shell" ]; then mkdir "$HOME/.ssh" >/dev/null 2>&1 || true - ( - echo "#!/bin/sh" - echo "set -e" - echo "if [ \"x\$SSH_ORIGINAL_COMMAND\" != \"x\" ]; then" - echo "exec $base/runshell git-annex-shell -c \"\$SSH_ORIGINAL_COMMAND\"" - echo "else" - echo "exec $base/runshell git-annex-shell -c \"\$@\"" - echo "fi" - ) > "$HOME/.ssh/git-annex-shell" - chmod +x "$HOME/.ssh/git-annex-shell" + if [ -e "$HOME/.ssh" ]; then + ( + echo "#!/bin/sh" + echo "set -e" + echo "if [ \"x\$SSH_ORIGINAL_COMMAND\" != \"x\" ]; then" + echo "exec $base/runshell git-annex-shell -c \"\$SSH_ORIGINAL_COMMAND\"" + echo "else" + echo "exec $base/runshell git-annex-shell -c \"\$@\"" + echo "fi" + ) > "$HOME/.ssh/git-annex-shell" + chmod +x "$HOME/.ssh/git-annex-shell" + fi fi # And this shim is used by the webapp when adding a remote ssh server. diff --git a/standalone/osx/git-annex.app/Contents/MacOS/runshell b/standalone/osx/git-annex.app/Contents/MacOS/runshell index c5d689c6e..534e9707e 100755 --- a/standalone/osx/git-annex.app/Contents/MacOS/runshell +++ b/standalone/osx/git-annex.app/Contents/MacOS/runshell @@ -33,16 +33,18 @@ cd "$orig" # assistant. if [ ! -e "$HOME/.ssh/git-annex-shell" ]; then mkdir "$HOME/.ssh" >/dev/null 2>&1 || true - ( - echo "#!/bin/sh" - echo "set -e" - echo "if [ \"x\$SSH_ORIGINAL_COMMAND\" != \"x\" ]; then" - echo "exec $base/runshell git-annex-shell -c \"\$SSH_ORIGINAL_COMMAND\"" - echo "else" - echo "exec $base/runshell git-annex-shell -c \"\$@\"" - echo "fi" - ) > "$HOME/.ssh/git-annex-shell" - chmod +x "$HOME/.ssh/git-annex-shell" + if [ -e "$HOME/.ssh" ]; then + ( + echo "#!/bin/sh" + echo "set -e" + echo "if [ \"x\$SSH_ORIGINAL_COMMAND\" != \"x\" ]; then" + echo "exec $base/runshell git-annex-shell -c \"\$SSH_ORIGINAL_COMMAND\"" + echo "else" + echo "exec $base/runshell git-annex-shell -c \"\$@\"" + echo "fi" + ) > "$HOME/.ssh/git-annex-shell" + chmod +x "$HOME/.ssh/git-annex-shell" + fi fi # And this shim is used by the webapp when adding a remote ssh server. diff --git a/standalone/windows/build-simple.sh b/standalone/windows/build-simple.sh index ade21ce6d..9b26e5189 100755 --- a/standalone/windows/build-simple.sh +++ b/standalone/windows/build-simple.sh @@ -4,10 +4,10 @@ set -e set -x -# Path to the Haskell Platform. -HP="/c/Program Files/Haskell Platform/2013.2.0.0" +# Path to the Haskell Platform. (As mingw sh sees it) +HP="/c/Program Files/Haskell Platform/2014.2.0.0" -PATH="$HP/bin:$HP/lib/extralibs/bin:$PATH" +PATH="$HP/bin:$HP/mingw/bin:$HP/lib/extralibs/bin:/c/Program Files/Git/cmd:/c/Program Files/NSIS:$PATH" # Run a command with the cygwin environment available. # However, programs not from cygwin are preferred. @@ -21,8 +21,10 @@ withcygpreferred () { # Install haskell dependencies. # cabal install is not run in cygwin, because we don't want configure scripts # for haskell libraries to link them with the cygwin library. -cabal update || true -cabal install --only-dependencies || true +if ! cabal install --only-dependencies; then + cabal update || true + cabal install --only-dependencies +fi # Build git-annex if [ ! -e "dist/setup-config" ]; then diff --git a/standalone/windows/build.sh b/standalone/windows/build.sh index 56323d8a3..7cbfb342d 100755 --- a/standalone/windows/build.sh +++ b/standalone/windows/build.sh @@ -16,9 +16,6 @@ PATH="/c/Program Files (x86)/NSIS:/c/msysgit/cmd:/c/msysgit/bin:$PATH" withcyg () { PATH="$PATH:/c/cygwin/bin" "$@" } -withcygpreferred () { - PATH="/c/cygwin/bin:$PATH" "$@" -} # This tells git-annex where to upgrade itself from. UPGRADE_LOCATION=http://downloads.kitenet.net/git-annex/windows/current/git-annex-installer.exe @@ -26,8 +23,8 @@ export UPGRADE_LOCATION # This can be used to force git-annex to build supporting a particular # version of git, instead of the version installed at build time. -FORCE_GIT_VERSION=1.9.5 -export FORCE_GIT_VERSION +#FORCE_GIT_VERSION=1.9.5 +#export FORCE_GIT_VERSION # Uncomment to get rid of cabal installed libraries. #rm -rf /c/Users/jenkins/AppData/Roaming/cabal /c/Users/jenkins/AppData/Roaming/ghc @@ -70,7 +67,12 @@ cabal install nsis ghc -fforce-recomp --make Build/NullSoftInstaller.hs # Want to include cygwin programs in bundle, not others, since # it includes the cygwin libs that go with them. -withcygpreferred Build/NullSoftInstaller.exe +# Currently need an older version of rsync than the one from cygwin. +if [ ! -e rsync.exe ]; then + withcyg wget https://downloads.kitenet.net/git-annex/windows/assets/rsync.exe + withcyg chmod +x rsync.exe +fi +PATH=".:/c/cygwin/bin:$PATH" Build/NullSoftInstaller.exe rm -f last-incremental-failed diff --git a/standalone/windows/ssh-keygen.cmd b/standalone/windows/ssh-keygen.cmd new file mode 100644 index 000000000..c4ec3cdb4 --- /dev/null +++ b/standalone/windows/ssh-keygen.cmd @@ -0,0 +1,31 @@ +@rem Do not use "echo off" to not affect any child calls.
+
+@rem Enable extensions, the `verify other 2>nul` is a trick from the setlocal help
+@verify other 2>nul
+@setlocal enableDelayedExpansion
+@if errorlevel 1 (
+ @echo Unable to enable delayed expansion. Immediate expansion will be used.
+ @goto fallback
+)
+
+@rem Get the absolute path to the parent directory, which is assumed to be the
+@rem Git installation root.
+@for /F "delims=" %%I in ("%~dp0..") do @set git_install_root=%%~fI
+@set PATH=!git_install_root!\bin;!git_install_root!\mingw\bin;!PATH!
+
+ssh-keygen %*
+@goto end
+
+:fallback
+@rem The above script again with immediate expansion, in case delayed expansion
+@rem is unavailable.
+@for /F "delims=" %%I in ("%~dp0..") do @set git_install_root=%%~fI
+@set PATH=%git_install_root%\bin;%git_install_root%\mingw\bin;%PATH%
+
+@if not exist "%HOME%" @set HOME=%HOMEDRIVE%%HOMEPATH%
+@if not exist "%HOME%" @set HOME=%USERPROFILE%
+
+ssh-keygen %*
+
+:end
+@rem End of script
diff --git a/standalone/windows/ssh.cmd b/standalone/windows/ssh.cmd new file mode 100644 index 000000000..32aa60a4d --- /dev/null +++ b/standalone/windows/ssh.cmd @@ -0,0 +1,31 @@ +@rem Do not use "echo off" to not affect any child calls.
+
+@rem Enable extensions, the `verify other 2>nul` is a trick from the setlocal help
+@verify other 2>nul
+@setlocal enableDelayedExpansion
+@if errorlevel 1 (
+ @echo Unable to enable delayed expansion. Immediate expansion will be used.
+ @goto fallback
+)
+
+@rem Get the absolute path to the parent directory, which is assumed to be the
+@rem Git installation root.
+@for /F "delims=" %%I in ("%~dp0..") do @set git_install_root=%%~fI
+@set PATH=!git_install_root!\bin;!git_install_root!\mingw\bin;!PATH!
+
+ssh %*
+@goto end
+
+:fallback
+@rem The above script again with immediate expansion, in case delayed expansion
+@rem is unavailable.
+@for /F "delims=" %%I in ("%~dp0..") do @set git_install_root=%%~fI
+@set PATH=%git_install_root%\bin;%git_install_root%\mingw\bin;%PATH%
+
+@if not exist "%HOME%" @set HOME=%HOMEDRIVE%%HOMEPATH%
+@if not exist "%HOME%" @set HOME=%USERPROFILE%
+
+ssh %*
+
+:end
+@rem End of script
|