Merge branch 'master' into tor

35 files changed, 1415 insertions, 15 deletions

diff --git a/Assistant/Fsck.hs b/Assistant/Fsck.hs
new file mode 100644
index 000000000..9d8848ba9
--- /dev/null
+++ b/Assistant/Fsck.hs
@@ -0,0 +1,50 @@
+{- git-annex assistant fscking
+ -
+ - Copyright 2013 Joey Hess <id@joeyh.name>
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Assistant.Fsck where
+
+import Assistant.Common
+import Types.ScheduledActivity
+import qualified Types.Remote as Remote
+import Annex.UUID
+import Assistant.Alert
+import Assistant.Types.UrlRenderer
+import Logs.Schedule
+import qualified Annex
+
+import qualified Data.Set as S
+
+{- Displays a nudge in the webapp if a fsck is not configured for
+ - the specified remote, or for the local repository. -}
+fsckNudge :: UrlRenderer -> Maybe Remote -> Assistant ()
+fsckNudge urlrenderer mr
+	| maybe True fsckableRemote mr =
+		whenM (liftAnnex $ annexFsckNudge <$> Annex.getGitConfig) $
+			unlessM (liftAnnex $ checkFscked mr) $
+				notFsckedNudge urlrenderer mr
+	| otherwise = noop
+
+fsckableRemote :: Remote -> Bool
+fsckableRemote = isJust . Remote.remoteFsck
+
+{- Checks if the remote, or the local repository, has a fsck scheduled.
+ - Only looks at fscks configured to run via the local repository, not
+ - other repositories. -}
+checkFscked :: Maybe Remote -> Annex Bool
+checkFscked mr = any wanted . S.toList <$> (scheduleGet =<< getUUID)
+  where
+	wanted = case mr of
+		Nothing -> isSelfFsck
+		Just r -> flip isFsckOf (Remote.uuid r)
+
+isSelfFsck :: ScheduledActivity -> Bool
+isSelfFsck (ScheduledSelfFsck _ _) = True
+isSelfFsck _ = False
+
+isFsckOf :: ScheduledActivity -> UUID -> Bool
+isFsckOf (ScheduledRemoteFsck u _ _) u' = u == u'
+isFsckOf _ _ = False
diff --git a/Assistant/Gpg.hs b/Assistant/Gpg.hs
new file mode 100644
index 000000000..34d00a384
--- /dev/null
+++ b/Assistant/Gpg.hs
@@ -0,0 +1,36 @@
+{- git-annex assistant gpg stuff
+ -
+ - Copyright 2013 Joey Hess <id@joeyh.name>
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Assistant.Gpg where
+
+import Utility.Gpg
+import Utility.UserInfo
+import Types.Remote (RemoteConfigKey)
+
+import qualified Data.Map as M
+import Control.Applicative
+import Prelude
+
+{- Generates a gpg user id that is not used by any existing secret key -}
+newUserId :: GpgCmd -> IO UserId
+newUserId cmd = do
+	oldkeys <- secretKeys cmd
+	username <- either (const "unknown") id <$> myUserName
+	let basekeyname = username ++ "'s git-annex encryption key"
+	return $ Prelude.head $ filter (\n -> M.null $ M.filter (== n) oldkeys)
+		( basekeyname
+		: map (\n -> basekeyname ++ show n) ([2..] :: [Int])
+		)
+
+data EnableEncryption = HybridEncryption | SharedEncryption | NoEncryption
+	deriving (Eq)
+
+{- Generates Remote configuration for encryption. -}
+configureEncryption :: EnableEncryption -> (RemoteConfigKey, String)
+configureEncryption SharedEncryption = ("encryption", "shared")
+configureEncryption NoEncryption = ("encryption", "none")
+configureEncryption HybridEncryption = ("encryption", "hybrid")
diff --git a/Assistant/Repair.hs b/Assistant/Repair.hs
new file mode 100644
index 000000000..29bdc44f1
--- /dev/null
+++ b/Assistant/Repair.hs
@@ -0,0 +1,159 @@
+{- git-annex assistant repository repair
+ -
+ - Copyright 2013 Joey Hess <id@joeyh.name>
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+{-# LANGUAGE CPP #-}
+
+module Assistant.Repair where
+
+import Assistant.Common
+import Command.Repair (repairAnnexBranch, trackingOrSyncBranch)
+import Git.Fsck (FsckResults, foundBroken)
+import Git.Repair (runRepairOf)
+import qualified Git
+import qualified Remote
+import qualified Types.Remote as Remote
+import Logs.FsckResults
+import Annex.UUID
+import Utility.Batch
+import Annex.Path
+import Assistant.Sync
+import Assistant.Alert
+import Assistant.DaemonStatus
+import Assistant.Types.UrlRenderer
+#ifdef WITH_WEBAPP
+import Assistant.WebApp.Types
+import qualified Data.Text as T
+#endif
+import qualified Utility.Lsof as Lsof
+import Utility.ThreadScheduler
+
+import Control.Concurrent.Async
+
+{- When the FsckResults require a repair, tries to do a non-destructive
+ - repair. If that fails, pops up an alert. -}
+repairWhenNecessary :: UrlRenderer -> UUID -> Maybe Remote -> FsckResults -> Assistant Bool
+repairWhenNecessary urlrenderer u mrmt fsckresults
+	| foundBroken fsckresults = do
+		liftAnnex $ writeFsckResults u fsckresults
+		repodesc <- liftAnnex $ Remote.prettyUUID u
+		ok <- alertWhile (repairingAlert repodesc)
+			(runRepair u mrmt False)
+#ifdef WITH_WEBAPP
+		unless ok $ do
+			button <- mkAlertButton True (T.pack "Click Here") urlrenderer $
+				RepairRepositoryR u
+			void $ addAlert $ brokenRepositoryAlert [button]
+#endif
+		return ok
+	| otherwise = return False
+
+runRepair :: UUID -> Maybe Remote -> Bool -> Assistant Bool
+runRepair u mrmt destructiverepair = do
+	fsckresults <- liftAnnex $ readFsckResults u
+	myu <- liftAnnex getUUID
+	ok <- if u == myu
+		then localrepair fsckresults
+		else remoterepair fsckresults
+	liftAnnex $ clearFsckResults u
+	debug [ "Repaired", show u, show ok ]
+
+	return ok
+  where
+	localrepair fsckresults = do
+		-- Stop the watcher from running while running repairs.
+		changeSyncable Nothing False
+
+		-- This intentionally runs the repair inside the Annex
+		-- monad, which is not strictly necessary, but keeps
+		-- other threads that might be trying to use the Annex
+		-- from running until it completes.
+		ok <- liftAnnex $ repair fsckresults Nothing
+
+		-- Run a background fast fsck if a destructive repair had
+		-- to be done, to ensure that the git-annex branch
+		-- reflects the current state of the repo.
+		when destructiverepair $
+			backgroundfsck [ Param "--fast" ]
+
+		-- Start the watcher running again. This also triggers it to
+		-- do a startup scan, which is especially important if the
+		-- git repo repair removed files from the index file. Those
+		-- files will be seen as new, and re-added to the repository.
+		when (ok || destructiverepair) $
+			changeSyncable Nothing True
+
+		return ok
+
+	remoterepair fsckresults = case Remote.repairRepo =<< mrmt of
+		Nothing -> return False
+		Just mkrepair -> do
+			thisrepopath <- liftIO . absPath
+				=<< liftAnnex (fromRepo Git.repoPath)
+			a <- liftAnnex $ mkrepair $
+				repair fsckresults (Just thisrepopath)
+			liftIO $ catchBoolIO a
+
+	repair fsckresults referencerepo = do
+		(ok, modifiedbranches) <- inRepo $
+			runRepairOf fsckresults trackingOrSyncBranch destructiverepair referencerepo
+		when destructiverepair $
+			repairAnnexBranch modifiedbranches
+		return ok
+	
+	backgroundfsck params = liftIO $ void $ async $ do
+		program <- programPath
+		batchCommand program (Param "fsck" : params)
+
+{- Detect when a git lock file exists and has no git process currently
+ - writing to it. This strongly suggests it is a stale lock file.
+ -
+ - However, this could be on a network filesystem. Which is not very safe
+ - anyway (the assistant relies on being able to check when files have
+ - no writers to know when to commit them). Also, a few lock-file-ish
+ - things used by git are not kept open, particularly MERGE_HEAD.
+ -
+ - So, just in case, when the lock file appears stale, we delay for one
+ - minute, and check its size. If the size changed, delay for another
+ - minute, and so on. This will at work to detect when another machine
+ - is writing out a new index file, since git does so by writing the
+ - new content to index.lock.
+ -
+ - Returns true if locks were cleaned up.
+ -}
+repairStaleGitLocks :: Git.Repo -> Assistant Bool
+repairStaleGitLocks r = do
+	lockfiles <- liftIO $ filter islock <$> findgitfiles r
+	repairStaleLocks lockfiles
+	return $ not $ null lockfiles
+  where
+	findgitfiles = dirContentsRecursiveSkipping (== dropTrailingPathSeparator annexDir) True . Git.localGitDir
+	islock f
+		| "gc.pid" `isInfixOf` f = False
+		| ".lock" `isSuffixOf` f = True
+		| takeFileName f == "MERGE_HEAD" = True
+		| otherwise = False
+
+repairStaleLocks :: [FilePath] -> Assistant ()
+repairStaleLocks lockfiles = go =<< getsizes
+  where
+	getsize lf = catchMaybeIO $ (\s -> (lf, s)) <$> getFileSize lf
+	getsizes = liftIO $ catMaybes <$> mapM getsize lockfiles
+	go [] = return ()
+	go l = ifM (liftIO $ null <$> Lsof.query ("--" : map fst l))
+		( do
+			waitforit "to check stale git lock file"
+			l' <- getsizes
+			if l' == l
+				then liftIO $ mapM_ nukeFile (map fst l)
+				else go l'
+		, do
+			waitforit "for git lock file writer"
+			go =<< getsizes
+		)
+	waitforit why = do
+		notice ["Waiting for 60 seconds", why]
+		liftIO $ threadDelaySeconds $ Seconds 60
diff --git a/Assistant/Restart.hs b/Assistant/Restart.hs
new file mode 100644
index 000000000..be1b21392
--- /dev/null
+++ b/Assistant/Restart.hs
@@ -0,0 +1,117 @@
+{- git-annex assistant restarting
+ -
+ - Copyright 2013 Joey Hess <id@joeyh.name>
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+{-# LANGUAGE CPP #-}
+
+module Assistant.Restart where
+
+import Assistant.Common
+import Assistant.Threads.Watcher
+import Assistant.DaemonStatus
+import Assistant.NamedThread
+import Utility.ThreadScheduler
+import Utility.NotificationBroadcaster
+import Utility.Url
+import Utility.PID
+import qualified Git.Construct
+import qualified Git.Config
+import qualified Annex
+import qualified Git
+import Annex.Path
+
+import Control.Concurrent
+#ifndef mingw32_HOST_OS
+import System.Posix (signalProcess, sigTERM)
+#else
+import Utility.WinProcess
+#endif
+import Network.URI
+
+{- Before the assistant can be restarted, have to remove our 
+ - gitAnnexUrlFile and our gitAnnexPidFile. Pausing the watcher is also
+ - a good idea, to avoid fighting when two assistants are running in the
+ - same repo.
+ -}
+prepRestart :: Assistant ()
+prepRestart = do
+	liftIO . maybe noop (`throwTo` PauseWatcher) =<< namedThreadId watchThread
+	liftIO . nukeFile =<< liftAnnex (fromRepo gitAnnexUrlFile)
+	liftIO . nukeFile =<< liftAnnex (fromRepo gitAnnexPidFile)
+
+{- To finish a restart, send a global redirect to the new url
+ - to any web browsers that are displaying the webapp.
+ -
+ - Wait for browser to update before terminating this process. -}
+postRestart :: URLString -> Assistant ()
+postRestart url = do
+	modifyDaemonStatus_ $ \status -> status { globalRedirUrl = Just url }
+	liftIO . sendNotification . globalRedirNotifier =<< getDaemonStatus
+	void $ liftIO $ forkIO $ do
+		threadDelaySeconds (Seconds 120)
+		terminateSelf
+
+terminateSelf :: IO ()
+terminateSelf =
+#ifndef mingw32_HOST_OS
+		signalProcess sigTERM =<< getPID
+#else
+		terminatePID =<< getPID
+#endif
+
+runRestart :: Assistant URLString
+runRestart = liftIO . newAssistantUrl
+	=<< liftAnnex (Git.repoLocation <$> Annex.gitRepo)
+
+{- Starts up the assistant in the repository, and waits for it to create
+ - a gitAnnexUrlFile. Waits for the assistant to be up and listening for
+ - connections by testing the url. -}
+newAssistantUrl :: FilePath -> IO URLString
+newAssistantUrl repo = do
+	startAssistant repo
+	geturl
+  where
+	geturl = do
+		r <- Git.Config.read =<< Git.Construct.fromPath repo
+		waiturl $ gitAnnexUrlFile r
+	waiturl urlfile = do
+		v <- tryIO $ readFile urlfile
+		case v of
+			Left _ -> delayed $ waiturl urlfile
+			Right url -> ifM (assistantListening url)
+				( return url
+				, delayed $ waiturl urlfile
+				)
+	delayed a = do
+		threadDelay 100000 -- 1/10th of a second
+		a
+
+{- Checks if the assistant is listening on an url.
+ -
+ - Always checks http, because https with self-signed cert is problematic.
+ - warp-tls listens to http, in order to show an error page, so this works.
+ -}
+assistantListening :: URLString -> IO Bool
+assistantListening url = catchBoolIO $ exists url' def
+  where
+	url' = case parseURI url of
+		Nothing -> url
+		Just uri -> show $ uri
+			{ uriScheme = "http:"
+			}
+
+{- Does not wait for assistant to be listening for web connections. 
+ -
+ - On windows, the assistant does not daemonize, which is why the forkIO is
+ - done.
+ -}
+startAssistant :: FilePath -> IO ()
+startAssistant repo = void $ forkIO $ do
+	program <- programPath
+	(_, _, _, pid) <- 
+		createProcess $
+			(proc program ["assistant"]) { cwd = Just repo }
+	void $ checkSuccessProcess pid
diff --git a/Assistant/Upgrade.hs b/Assistant/Upgrade.hs
new file mode 100644
index 000000000..afbb61924
--- /dev/null
+++ b/Assistant/Upgrade.hs
@@ -0,0 +1,361 @@
+{- git-annex assistant upgrading
+ -
+ - Copyright 2013 Joey Hess <id@joeyh.name>
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+{-# LANGUAGE CPP #-}
+
+module Assistant.Upgrade where
+
+import Assistant.Common
+import Assistant.Restart
+import qualified Annex
+import Assistant.Alert
+import Assistant.DaemonStatus
+import Utility.Env
+import Types.Distribution
+import Types.Transfer
+import Logs.Web
+import Logs.Presence
+import Logs.Location
+import Annex.Content
+import Annex.UUID
+import qualified Backend
+import qualified Types.Backend
+import qualified Types.Key
+import Assistant.TransferQueue
+import Assistant.TransferSlots
+import Remote (remoteFromUUID)
+import Annex.Path
+import Config.Files
+import Utility.ThreadScheduler
+import Utility.Tmp
+import Utility.UserInfo
+import Utility.Gpg
+import Utility.FileMode
+import qualified Utility.Lsof as Lsof
+import qualified Build.SysConfig
+import qualified Utility.Url as Url
+import qualified Annex.Url as Url
+
+import qualified Data.Map as M
+import Data.Tuple.Utils
+
+{- Upgrade without interaction in the webapp. -}
+unattendedUpgrade :: Assistant ()
+unattendedUpgrade = do
+	prepUpgrade
+	url <- runRestart
+	postUpgrade url
+
+prepUpgrade :: Assistant ()
+prepUpgrade = do
+	void $ addAlert upgradingAlert
+	liftIO $ setEnv upgradedEnv "1" True
+	prepRestart
+
+postUpgrade :: URLString -> Assistant ()
+postUpgrade = postRestart
+
+autoUpgradeEnabled :: Assistant Bool
+autoUpgradeEnabled = liftAnnex $ (==) AutoUpgrade . annexAutoUpgrade <$> Annex.getGitConfig
+
+checkSuccessfulUpgrade :: IO Bool
+checkSuccessfulUpgrade = isJust <$> getEnv upgradedEnv
+
+upgradedEnv :: String
+upgradedEnv = "GIT_ANNEX_UPGRADED"
+
+{- Start downloading the distribution key from the web.
+ - Install a hook that will be run once the download is complete,
+ - and finishes the upgrade.
+ -
+ - Creates the destination directory where the upgrade will be installed
+ - early, in order to check if another upgrade has happened (or is
+ - happending). On failure, the directory is removed.
+ -}
+startDistributionDownload :: GitAnnexDistribution -> Assistant ()
+startDistributionDownload d = go =<< liftIO . newVersionLocation d =<< liftIO oldVersionLocation
+  where
+	go Nothing = debug ["Skipping redundant upgrade"]
+	go (Just dest) = do
+		liftAnnex $ setUrlPresent webUUID k u
+		hook <- asIO1 $ distributionDownloadComplete d dest cleanup
+		modifyDaemonStatus_ $ \s -> s
+			{ transferHook = M.insert k hook (transferHook s) }
+		maybe noop (queueTransfer "upgrade" Next (Just f) t)
+			=<< liftAnnex (remoteFromUUID webUUID)
+		startTransfer t
+	k = distributionKey d
+	u = distributionUrl d
+	f = takeFileName u ++ " (for upgrade)"
+	t = Transfer
+		{ transferDirection = Download
+		, transferUUID = webUUID
+		, transferKey = k
+		}
+	cleanup = liftAnnex $ do
+		lockContentForRemoval k removeAnnex
+		setUrlMissing webUUID k u
+		logStatus k InfoMissing
+
+{- Called once the download is done.
+ - Passed an action that can be used to clean up the downloaded file.
+ -
+ - Verifies the content of the downloaded key.
+ -}
+distributionDownloadComplete :: GitAnnexDistribution -> FilePath -> Assistant () -> Transfer -> Assistant ()
+distributionDownloadComplete d dest cleanup t 
+	| transferDirection t == Download = do
+		debug ["finished downloading git-annex distribution"]
+		maybe (failedupgrade "bad download") go
+			=<< liftAnnex (withObjectLoc k fsckit (getM fsckit))
+	| otherwise = cleanup
+  where
+	k = distributionKey d
+	fsckit f = case Backend.maybeLookupBackendName (Types.Key.keyBackendName k) of
+		Nothing -> return $ Just f
+		Just b -> case Types.Backend.verifyKeyContent b of
+			Nothing -> return $ Just f
+			Just verifier -> ifM (verifier k f)
+				( return $ Just f
+				, return Nothing
+				)
+	go f = do
+		ua <- asIO $ upgradeToDistribution dest cleanup f
+		fa <- asIO1 failedupgrade
+		liftIO $ ua `catchNonAsync` (fa . show)
+	failedupgrade msg = do
+		void $ addAlert $ upgradeFailedAlert msg
+		cleanup
+		liftIO $ void $ tryIO $ removeDirectoryRecursive dest
+
+{- The upgrade method varies by OS.
+ -
+ - In general, find where the distribution was installed before,
+ - and unpack the new distribution next to it (in a versioned directory).
+ - Then update the programFile to point to the new version.
+ -}
+upgradeToDistribution :: FilePath -> Assistant () -> FilePath -> Assistant ()
+upgradeToDistribution newdir cleanup distributionfile = do
+	liftIO $ createDirectoryIfMissing True newdir
+	(program, deleteold) <- unpack
+	changeprogram program
+	cleanup
+	prepUpgrade
+	url <- runRestart
+	{- At this point, the new assistant is fully running, so
+	 - it's safe to delete the old version. -}
+	liftIO $ void $ tryIO deleteold
+	postUpgrade url
+  where
+	changeprogram program = liftIO $ do
+		unlessM (boolSystem program [Param "version"]) $
+			giveup "New git-annex program failed to run! Not using."
+		pf <- programFile
+		liftIO $ writeFile pf program
+	
+#ifdef darwin_HOST_OS
+	{- OS X uses a dmg, so mount it, and copy the contents into place. -}
+	unpack = liftIO $ do
+		olddir <- oldVersionLocation
+		withTmpDirIn (parentDir newdir) "git-annex.upgrade" $ \tmpdir -> do
+			void $ boolSystem "hdiutil"
+				[ Param "attach", File distributionfile
+				, Param "-mountpoint", File tmpdir
+				]
+			void $ boolSystem "cp"
+				[ Param "-R"
+				, File $ tmpdir </> installBase </> "Contents"
+				, File $ newdir
+				]
+			void $ boolSystem "hdiutil"
+				[ Param "eject"
+				, File tmpdir
+				]
+			sanitycheck newdir
+		let deleteold = do
+			deleteFromManifest $ olddir </> "Contents" </> "MacOS"
+			makeorigsymlink olddir
+		return (newdir </> "Contents" </> "MacOS" </> "git-annex", deleteold)
+#else
+	{- Linux uses a tarball (so could other POSIX systems), so
+	 - untar it (into a temp directory) and move the directory
+	 - into place. -}
+	unpack = liftIO $ do
+		olddir <- oldVersionLocation
+		withTmpDirIn (parentDir newdir) "git-annex.upgrade" $ \tmpdir -> do
+			let tarball = tmpdir </> "tar"
+			-- Cannot rely on filename extension, and this also
+			-- avoids problems if tar doesn't support transparent
+			-- decompression.
+			void $ boolSystem "sh"
+				[ Param "-c"
+				, Param $ "zcat < " ++ shellEscape distributionfile ++
+					" > " ++ shellEscape tarball
+				]
+			tarok <- boolSystem "tar"
+				[ Param "xf"
+				, Param tarball
+				, Param "--directory", File tmpdir
+				]
+			unless tarok $
+				error $ "failed to untar " ++ distributionfile
+			sanitycheck $ tmpdir </> installBase
+			installby rename newdir (tmpdir </> installBase)
+		let deleteold = do
+			deleteFromManifest olddir
+			makeorigsymlink olddir
+		return (newdir </> "git-annex", deleteold)
+	installby a dstdir srcdir =
+		mapM_ (\x -> a x (dstdir </> takeFileName x))
+			=<< dirContents srcdir
+#endif
+	sanitycheck dir = 
+		unlessM (doesDirectoryExist dir) $
+			error $ "did not find " ++ dir ++ " in " ++ distributionfile
+	makeorigsymlink olddir = do
+		let origdir = parentDir olddir </> installBase
+		nukeFile origdir
+		createSymbolicLink newdir origdir
+
+{- Finds where the old version was installed. -}
+oldVersionLocation :: IO FilePath
+oldVersionLocation = do
+	pdir <- parentDir <$> readProgramFile
+#ifdef darwin_HOST_OS
+	let dirs = splitDirectories pdir
+	{- It will probably be deep inside a git-annex.app directory. -}
+	let olddir = case findIndex ("git-annex.app" `isPrefixOf`) dirs of
+		Nothing -> pdir
+		Just i -> joinPath (take (i + 1) dirs)
+#else
+	let olddir = pdir
+#endif
+	when (null olddir) $
+		error $ "Cannot find old distribution bundle; not upgrading. (Looked in " ++ pdir ++ ")"
+	return olddir
+
+{- Finds a place to install the new version.
+ - Generally, put it in the parent directory of where the old version was
+ - installed, and use a version number in the directory name.
+ - If unable to write to there, instead put it in the home directory.
+ -
+ - The directory is created. If it already exists, returns Nothing.
+ -}
+newVersionLocation :: GitAnnexDistribution -> FilePath -> IO (Maybe FilePath)
+newVersionLocation d olddir = 
+	trymkdir newloc $ do
+		home <- myHomeDir
+		trymkdir (home </> s) $
+			return Nothing
+  where
+	s = installBase ++ "." ++ distributionVersion d
+	topdir = parentDir olddir
+	newloc = topdir </> s
+	trymkdir dir fallback =
+		(createDirectory dir >> return (Just dir))
+			`catchIO` const fallback
+
+installBase :: String
+installBase = "git-annex." ++
+#ifdef linux_HOST_OS
+	"linux"
+#else
+#ifdef darwin_HOST_OS
+	"app"
+#else
+	"dir"
+#endif
+#endif
+
+deleteFromManifest :: FilePath -> IO ()
+deleteFromManifest dir = do
+	fs <- map (dir </>) . lines <$> catchDefaultIO "" (readFile manifest)
+	mapM_ nukeFile fs
+	nukeFile manifest
+	removeEmptyRecursive dir
+  where
+	manifest = dir </> "git-annex.MANIFEST"
+
+removeEmptyRecursive :: FilePath -> IO ()
+removeEmptyRecursive dir = do
+	mapM_ removeEmptyRecursive =<< dirContents dir
+	void $ tryIO $ removeDirectory dir
+
+{- This is a file that the UpgradeWatcher can watch for modifications to
+ - detect when git-annex has been upgraded.
+ -}
+upgradeFlagFile :: IO FilePath
+upgradeFlagFile = programPath
+
+{- Sanity check to see if an upgrade is complete and the program is ready
+ - to be run. -}
+upgradeSanityCheck :: IO Bool
+upgradeSanityCheck = ifM usingDistribution
+	( doesFileExist =<< programFile
+	, do
+		-- Ensure that the program is present, and has no writers,
+		-- and can be run. This should handle distribution
+		-- upgrades, manual upgrades, etc.
+		program <- programPath
+		untilM (doesFileExist program <&&> nowriter program) $
+			threadDelaySeconds (Seconds 60)
+		boolSystem program [Param "version"]
+	)
+  where
+	nowriter f = null
+		. filter (`elem` [Lsof.OpenReadWrite, Lsof.OpenWriteOnly])
+		. map snd3
+		<$> Lsof.query [f]
+
+usingDistribution :: IO Bool
+usingDistribution = isJust <$> getEnv "GIT_ANNEX_STANDLONE_ENV"
+
+downloadDistributionInfo :: Assistant (Maybe GitAnnexDistribution)
+downloadDistributionInfo = do
+	uo <- liftAnnex Url.getUrlOptions
+	gpgcmd <- liftAnnex $ gpgCmd <$> Annex.getGitConfig
+	liftIO $ withTmpDir "git-annex.tmp" $ \tmpdir -> do
+		let infof = tmpdir </> "info"
+		let sigf = infof ++ ".sig"
+		ifM (Url.downloadQuiet distributionInfoUrl infof uo
+			<&&> Url.downloadQuiet distributionInfoSigUrl sigf uo
+			<&&> verifyDistributionSig gpgcmd sigf)
+			( readish <$> readFileStrict infof
+			, return Nothing
+			)
+
+distributionInfoUrl :: String
+distributionInfoUrl = fromJust Build.SysConfig.upgradelocation ++ ".info"
+
+distributionInfoSigUrl :: String
+distributionInfoSigUrl = distributionInfoUrl ++ ".sig"
+
+{- Verifies that a file from the git-annex distribution has a valid
+ - signature. Pass the detached .sig file; the file to be verified should
+ - be located next to it.
+ -
+ - The gpg keyring used to verify the signature is located in
+ - trustedkeys.gpg, next to the git-annex program.
+ -}
+verifyDistributionSig :: GpgCmd -> FilePath -> IO Bool
+verifyDistributionSig gpgcmd sig = do
+	p <- readProgramFile
+	if isAbsolute p
+		then withUmask 0o0077 $ withTmpDir "git-annex-gpg.tmp" $ \gpgtmp -> do
+			let trustedkeys = takeDirectory p </> "trustedkeys.gpg"
+			boolGpgCmd gpgcmd
+				[ Param "--no-default-keyring"
+				, Param "--no-auto-check-trustdb"
+				, Param "--no-options"
+				, Param "--homedir"
+				, File gpgtmp
+				, Param "--keyring"
+				, File trustedkeys
+				, Param "--verify"
+				, File sig
+				]
+		else return False
diff --git a/CHANGELOG b/CHANGELOG
index cc20ec751..0fb741ee8 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -6,8 +6,12 @@ git-annex (6.20161119) UNRELEASED; urgency=medium
     hidden service.
   * remotedaemon: Fork to background by default. Added --foreground switch
     to enable old behavior.
+  * addurl: Fix bug in checking annex.largefiles expressions using
+    largerthan, mimetype, and smallerthan; the first two always failed
+    to match, and the latter always matched.
+  * Relicense 5 source files that are not part of the webapp from AGPL to GPL.
 
- -- Joey Hess <id@joeyh.name>  Sun, 20 Nov 2016 14:10:15 -0400
+ -- Joey Hess <id@joeyh.name>  Mon, 21 Nov 2016 11:27:50 -0400
 
 git-annex (6.20161118) unstable; urgency=medium
 
diff --git a/Command/AddUrl.hs b/Command/AddUrl.hs
index e32ceb568..e49d2727c 100644
--- a/Command/AddUrl.hs
+++ b/Command/AddUrl.hs
@@ -340,13 +340,18 @@ cleanup :: UUID -> URLString -> FilePath -> Key -> Maybe FilePath -> Annex ()
 cleanup u url file key mtmp = case mtmp of
 	Nothing -> go
 	Just tmp -> do
+		-- Move to final location for large file check.
+		liftIO $ renameFile tmp file
 		largematcher <- largeFilesMatcher
-		ifM (checkFileMatcher largematcher file)
-			( go
-			, do
-				liftIO $ renameFile tmp file
-				void $ Command.Add.addSmall file
-			)
+		large <- checkFileMatcher largematcher file
+		if large
+			then do
+				-- Move back to tmp because addAnnexedFile
+				-- needs the file in a different location
+				-- than the work tree file.
+				liftIO $ renameFile file tmp
+				go
+			else void $ Command.Add.addSmall file
   where
 	go = do
 		maybeShowJSON $ JSONChunk [("key", key2file key)]
diff --git a/Command/ReKey.hs b/Command/ReKey.hs
index 51f9f6fe1..33734ebe7 100644
--- a/Command/ReKey.hs
+++ b/Command/ReKey.hs
@@ -44,7 +44,7 @@ perform :: FilePath -> Key -> Key -> CommandPerform
 perform file oldkey newkey = do
 	ifM (inAnnex oldkey) 
 		( unlessM (linkKey file oldkey newkey) $
-			error "failed"
+			giveup "failed"
 		, unlessM (Annex.getState Annex.force) $
 			giveup $ file ++ " is not available (use --force to override)"
 		)
diff --git a/Command/Schedule.hs b/Command/Schedule.hs
index 5cc8b37bf..c9d4f915f 100644
--- a/Command/Schedule.hs
+++ b/Command/Schedule.hs
@@ -29,7 +29,7 @@ start = parse
   where
 	parse (name:[]) = go name performGet
 	parse (name:expr:[]) = go name $ \uuid -> do
-		showStart "schedile" name
+		showStart "schedule" name
 		performSet expr uuid
 	parse _ = giveup "Specify a repository."
 
diff --git a/doc/bugs/Assistant_drops_files_from_remote_repos_that_it_shouldn__39__t/comment_9_c46cdba62da4f5ccfdc42dfc33aec600._comment b/doc/bugs/Assistant_drops_files_from_remote_repos_that_it_shouldn__39__t/comment_9_c46cdba62da4f5ccfdc42dfc33aec600._comment
new file mode 100644
index 000000000..a6a2397e7
--- /dev/null
+++ b/doc/bugs/Assistant_drops_files_from_remote_repos_that_it_shouldn__39__t/comment_9_c46cdba62da4f5ccfdc42dfc33aec600._comment
@@ -0,0 +1,34 @@
+[[!comment format=mdwn
+ username="boh"
+ avatar="http://cdn.libravatar.org/avatar/e7fa2d1c5d95e323fe48887f7f827b1f"
+ subject="comment 9"
+ date="2016-11-27T12:23:20Z"
+ content="""
+Seems as if the problem still exists in 6.20161118 (Debian).
+
+I have three repositories (among others), `jolla`, `sts-3xx`, and `here`.  `jolla` and `here` are in group `manual`, `sts-3xx` is `backup`; `here` and `sts-3xx` have assistants running, `jolla` not.  `jolla` and `sts-3xx` have slightly older versions of git-annex installed.
+
+Now, when I copy a file from `here` to `jolla` like this
+
+    git annex copy real_programmers.png -t jolla
+
+the file is subsequently dropped by the assistant:
+
+```
+drop real_programmers.png (locking jolla...) [2016-11-27 13:00:02.667376556] chat: ssh [\"-S\",\".git/annex/ssh/jolla\",\"-o\",\"ControlMaster
+=auto\",\"-o\",\"ControlPersist=yes\",\"-F\",\".git/annex/ssh.config\",\"-T\",\"jolla\",\"git-annex-shell 'lockcontent' '/~/Music/media/' '--debug' '
+SHA256E-s84499--ff98a733cc0122858fb11433c720e2d038fec190a3d36380d0e7e8dab468f883.png' --uuid 5298e3ce-1106-4d5e-b052-0aee4b27a344\"]
+(locking sts-3xx...) [2016-11-27 13:00:03.252473676] chat: ssh [..., \"git-annex-shell 'lockcontent' '/backups/exot/media/' '--debug' 'SHA256E-s84499--ff98a733cc0122858fb11433c720e2d038fec190a3d 36380d0e7e8dab468f883.png' --uuid 1fec6253-171d-4f86-885b-e233be2d65ec\"]
+(lockcontent failed) [2016-11-27 13:00:03.486158016] process done ExitFailure 1
+(checking sts-3xx...) [2016-11-27 13:00:03.487047149] call: ssh [..., \"git-annex-shell 'inannex' '/backups/exot/media/' '--debug' 'SHA256E-s84499--ff98a733cc0122858fb11433c720e2d038fec190a3d363 80d0e7e8dab468f883.png' --uuid 1fec6253-171d-4f86-885b-e233be2d65ec\"]
+[2016-11-27 13:00:03.76435136] process done ExitSuccess
+[2016-11-27 13:00:03.764705754] Dropping from here proof: Just (SafeDropProof (NumCopies 2) [RecentlyVerifiedCopy UUID \"1fec6253-171d-4 f86-885b-e233be2d65ec\",LockedCopy UUID \"5298e3ce-1106-4d5e-b052-0aee4b27a344\"] (Just (ContentRemovalLock (Key {keyName = \"ff98a733cc012 2858fb11433c720e2d038fec190a3d36380d0e7e8dab468f883.png\", keyBackendName = \"SHA256E\", keySize = Just 84499, keyMtime = Nothing, keyChun kSize = Nothing, keyChunkNum = Nothing}))))
+[2016-11-27 13:00:04.24333081] process done ExitFailure 1
+ok
+[2016-11-27 13:00:04.251232455] dropped real_programmers.png (from here) (copies now 4) : drop wanted after Upload UUID \"5298e3ce-1106- 4d5e-b052-0aee4b27a344\" real_programmers.png Just 84499
+```
+
+However, I failed to reproduce the problem by replicating my setup with fresh repositories …
+
+Please let me know if you need more information, and *so* many thanks for git-annex!
+"""]]
diff --git a/doc/bugs/Build_with_aws_head_fails.mdwn b/doc/bugs/Build_with_aws_head_fails.mdwn
new file mode 100644
index 000000000..9937d6c35
--- /dev/null
+++ b/doc/bugs/Build_with_aws_head_fails.mdwn
@@ -0,0 +1,48 @@
+### Please describe the problem.
+https://github.com/aristidb/aws/issues/206 was recently resolved in https://github.com/aristidb/aws/pull/213.
+
+A newer version will be tagged imminently according to https://github.com/aristidb/aws/issues/206#issuecomment-260214736.
+
+With the http-conduit (<2.2.0) constraint removed from git-annex.cabal, and the aws dependency set to use aws head (currently c8806dc), the git-annex build fails. 
+
+### What steps will reproduce the problem?
+
+Remove the http-conduit (<2.2.0) constraint and attempt to build git-annex with aws head.
+
+### What version of git-annex are you using? On what operating system?
+
+macOS 10.11, git-annex 6.20161118.
+
+### Please provide any additional information below.
+Full build log: https://gist.github.com/ilovezfs/15bcd8f1086b3d825beff58140e04eec
+[[!format sh """
+[ 90 of 542] Compiling Types.Crypto     ( Types/Crypto.hs, dist/dist-sandbox-6b15e8f0/build/git-annex/git-annex-tmp/Types/Crypto.o )
+[ 91 of 542] Compiling Utility.Metered  ( Utility/Metered.hs, dist/dist-sandbox-6b15e8f0/build/git-annex/git-annex-tmp/Utility/Metered.o )
+[ 92 of 542] Compiling Messages.JSON    ( Messages/JSON.hs, dist/dist-sandbox-6b15e8f0/build/git-annex/git-annex-tmp/Messages/JSON.o )
+[ 93 of 542] Compiling Utility.Url      ( Utility/Url.hs, dist/dist-sandbox-6b15e8f0/build/git-annex/git-annex-tmp/Utility/Url.o )
+
+Utility/Url.hs:354:34: error:
+    • The constructor ‘StatusCodeException’ should have 2 arguments, but has been given 3
+    • In the pattern: StatusCodeException s _ _
+      In an equation for ‘matchStatusCodeException’:
+          matchStatusCodeException want e@(StatusCodeException s _ _)
+            | want s = Just e
+            | otherwise = Nothing
+
+Utility/Url.hs:354:34: error:
+    • Couldn't match expected type ‘HttpException’
+                  with actual type ‘HttpExceptionContent’
+    • In the pattern: StatusCodeException s _ _
+      In an equation for ‘matchStatusCodeException’:
+          matchStatusCodeException want e@(StatusCodeException s _ _)
+            | want s = Just e
+            | otherwise = Nothing
+cabal: Leaving directory '.'
+cabal: Error: some packages failed to install:
+git-annex-6.20161118 failed during the building phase. The exception was:
+ExitFailure 1
+"""]]
+
+### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
+Yes :)
+
diff --git a/doc/bugs/Inconsistent_results_between_git-annex-fsck_and_git-annex-whereis.mdwn b/doc/bugs/Inconsistent_results_between_git-annex-fsck_and_git-annex-whereis.mdwn
new file mode 100644
index 000000000..c9037b575
--- /dev/null
+++ b/doc/bugs/Inconsistent_results_between_git-annex-fsck_and_git-annex-whereis.mdwn
@@ -0,0 +1,53 @@
+### Please describe the problem.
+
+I'm seeing some inconsistent results between runs of `git annex fsck` and `git annex whereis` that I'm not able to explain. When I run `git annex fsck`, it reports a few keys that only have 1 copy, and advises me to make more copies. If I run `git annex whereis --key <key>`, git annex confirms that it only knows about 1 copy of this key. If I then use `git log --stat -S'<key>'` to find the actual file that it refers to, and run `git annex whereis <file>`, git annex report 9 copies of this file. Checking on remotes shows that these files do exist on the remote, so why does `git annex fsck` and `git annex whereis` mis-report the number of copies when querying for the key - but not for the actual filename? Additionally, `git annex find --lackingcopies 1` doesn't return any results, but should if there are actually files with not enough copies?
+
+
+### What steps will reproduce the problem?
+
+
+### What version of git-annex are you using? On what operating system?
+
+5.20151208-1build1 on Ubuntu Xenial, one remote running 5.20141024~bpo70+1 on Debian Wheezy
+
+### Please provide any additional information below.
+
+[[!format sh """
+# If you can, paste a complete transcript of the problem occurring here.
+# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log
+
+[william@hactar ~/Pictures/Photo Library]$ git annex whereis SHA256E-s1071765--dbaa7f32ee44c28d6a1f0c8095e8dfd8b4ec433b144085d5097425303a510ea9
+git-annex: SHA256E-s1071765--dbaa7f32ee44c28d6a1f0c8095e8dfd8b4ec433b144085d5097425303a510ea9 not found
+git-annex: whereis: 1 failed
+[william@hactar ~/Pictures/Photo Library]$ git annex whereis --key SHA256E-s1071765--dbaa7f32ee44c28d6a1f0c8095e8dfd8b4ec433b144085d5097425303a510ea9
+whereis SHA256E-s1071765--dbaa7f32ee44c28d6a1f0c8095e8dfd8b4ec433b144085d5097425303a510ea9 (1 copy) 
+        7691934f-2542-4103-9122-2db4e6cfc887 -- hactar [here]
+ok
+[william@hactar ~/Pictures/Photo Library]$ git annex fsck --key SHA256E-s1071765--dbaa7f32ee44c28d6a1f0c8095e8dfd8b4ec433b144085d5097425303a510ea9
+fsck SHA256E-s1071765--dbaa7f32ee44c28d6a1f0c8095e8dfd8b4ec433b144085d5097425303a510ea9 
+  Only 1 of 3 trustworthy copies exist of SHA256E-s1071765--dbaa7f32ee44c28d6a1f0c8095e8dfd8b4ec433b144085d5097425303a510ea9
+  Back it up with git-annex copy.
+failed
+(recording state in git...)
+git-annex: fsck: 1 failed
+[william@hactar ~/Pictures/Photo Library]$ git log --stat -S'SHA256E-s1071765--dbaa7f32ee44c28d6a1f0c8095e8dfd8b4ec433b144085d5097425303a510ea9'
+[william@hactar ~/Pictures/Photo Library]$ git annex whereis 2009/05/05/P1040890.JPG
+whereis 2009/05/05/P1040890.JPG (9 copies) 
+        0e825a69-1927-4f62-b731-6f3e98bba998 -- william@marvin:/media/backup/annex/photos [marvin]
+        1b728ab5-1e32-45a6-bc11-2a4bfdc9d6ab -- backup1
+        5c0caa42-b489-467b-a612-9590fa9d5a94 -- backup2
+        7691934f-2542-4103-9122-2db4e6cfc887 -- hactar [here]
+        894b2216-72e0-40e1-8765-1386e1e9e4b4 -- backup3
+        96f19fa8-d385-4e8b-b000-61ee15993a70 -- backup3
+        a862b121-d794-4af4-bb56-21adfe8962f2 -- S3
+        b083f8ae-42fb-41f0-a2a3-4e7c9f93aadb -- [guide]
+        bf021ce9-465b-4419-86e7-bddfd208fca4 -- git@newzaphod:~/repositories/annex/photos.git [zaphod]
+ok
+
+
+# End of transcript or log.
+"""]]
+
+### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
+
+I trust Git Annex to keep hundreds of GB of data safe, and it has never failed me - despite my best efforts
diff --git a/doc/bugs/Nearline_bucket_stopped_working___40__can__39__t_even_HEAD_files__41__/comment_2_c227071f23a96ed9928f128e7f77e503._comment b/doc/bugs/Nearline_bucket_stopped_working___40__can__39__t_even_HEAD_files__41__/comment_2_c227071f23a96ed9928f128e7f77e503._comment
new file mode 100644
index 000000000..820e6b040
--- /dev/null
+++ b/doc/bugs/Nearline_bucket_stopped_working___40__can__39__t_even_HEAD_files__41__/comment_2_c227071f23a96ed9928f128e7f77e503._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="justin.lebar@7a36fcafc322d9a381e89f08ab6289033c6dde91"
+ nickname="justin.lebar"
+ avatar="http://cdn.libravatar.org/avatar/9fca4b61a1ab555f231851e7543f9a3e"
+ subject="comment 2"
+ date="2016-11-20T03:47:23Z"
+ content="""
+Thanks for your reply, Joey.  Sorry for the delay getting back to this -- I didn't realize I hadn't enabled notifications on the thread.
+
+The GCS docs suggest that 400 errors should be accompanied by an explanation in the reply body.
+
+> Error responses usually include a JSON document in the response body, which contains information about the error.
+
+https://cloud.google.com/storage/docs/json_api/v1/status-codes
+
+Do you think we're not getting an http response body here, or that it's not being printed out?
+"""]]
diff --git a/doc/bugs/Single_space_in_file_name_causes_git_annex_add_to_fail/comment_2_13c242250d1509d933b8f0bcb7b67302._comment b/doc/bugs/Single_space_in_file_name_causes_git_annex_add_to_fail/comment_2_13c242250d1509d933b8f0bcb7b67302._comment
new file mode 100644
index 000000000..bf41d40a5
--- /dev/null
+++ b/doc/bugs/Single_space_in_file_name_causes_git_annex_add_to_fail/comment_2_13c242250d1509d933b8f0bcb7b67302._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="https://launchpad.net/~stephane-gourichon-lpad"
+ nickname="stephane-gourichon-lpad"
+ avatar="http://cdn.libravatar.org/avatar/02d4a0af59175f9123720b4481d55a769ba954e20f6dd9b2792217d9fa0c6089"
+ subject="Known bug, fixed."
+ date="2016-11-23T18:04:27Z"
+ content="""
+This is a known bug introduced in 6.20161012 and fixed in 6.20161031.
+
+Solution is: just update your copy of git-annex. At this time most recent is 6.20161119 .
+
+For more details, see changelog at https://github.com/joeyh/git-annex/blob/master/CHANGELOG#L53
+
+
+
+"""]]
diff --git a/doc/bugs/add_fails_with_v6_repo_when_four_levels_deep/comment_2_b3998823aca4266089dcbcf325d8f8c1._comment b/doc/bugs/add_fails_with_v6_repo_when_four_levels_deep/comment_2_b3998823aca4266089dcbcf325d8f8c1._comment
new file mode 100644
index 000000000..1046fb066
--- /dev/null
+++ b/doc/bugs/add_fails_with_v6_repo_when_four_levels_deep/comment_2_b3998823aca4266089dcbcf325d8f8c1._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="t.z.mates"
+ avatar="http://cdn.libravatar.org/avatar/90f15fad216078fd08d62cc676487925"
+ subject="comment 2"
+ date="2016-11-19T04:42:25Z"
+ content="""
+Thanks for looking into it; I just checked again, and even on the newest version (6.20161118 binary), I'm still experiencing the behavior. However, I checked on an older OpenSuse box I have, and there it works (6.20161031 from OpenSuse repo).
+
+Since my two machines experiencing the problem are both running arch, it seems it's somehow related to that distro. I've checked both installing via the binary (from kitenet) and from the arch community repo, but both produce the same behavior. Further, the OpenSuse install has the same build flags as the binaries, so that doesn't seem to be it. Are there any other diagnostics I can run?
+
+This particular problem isn't very troublesome (it doesn't seem to have any material impact aside from error messages); however, I also occasionally experience a more serious bug. Namely, when certain (seemingly random) files are added to the repo locked, their content disappears and the symlink is broken (this is the other problem I alluded to in the description). I suspect that problem is related to this one though, since it also only affects my arch machines. I haven't yet submitted a report for that bug yet, though, since I can't reliably replicate it. 
+"""]]
diff --git a/doc/bugs/adds_file_destined_for_annex_into_git_in___39__addurl__39__.mdwn b/doc/bugs/adds_file_destined_for_annex_into_git_in___39__addurl__39__.mdwn
new file mode 100644
index 000000000..a5cdfd6d6
--- /dev/null
+++ b/doc/bugs/adds_file_destined_for_annex_into_git_in___39__addurl__39__.mdwn
@@ -0,0 +1,41 @@
+### Please describe the problem.
+
+When addurl'ing a big file with .gitattributes configured to add only some files directly into git (and 'git annex add' operating correctly), addurl adds large files straight into git.
+
+### What version of git-annex are you using? On what operating system?
+
+git-annex version: 6.20161018+gitgf3c366a-1~ndall+1
+
+
+### Please provide any additional information below.
+
+[[!format sh """
+$> cat .gitattributes 
+* annex.backend=MD5E
+* annex.largefiles=(largerthan=100kb)
+*.json annex.largefiles=nothing
+*.txt annex.largefiles=nothing
+*.tsv annex.largefiles=nothing
+*.nii.gz annex.largefiles=(largerthan=0kb)
+*.tgz annex.largefiles=(largerthan=0kb)
+*.tar.gz annex.largefiles=(largerthan=0kb)
+*.gz annex.largefiles=(largerthan=0kb)
+
+$> git annex addurl http://fcp-indi.s3.amazonaws.com/data/Projects/HBNSSI/RawDataTars/sub-0031121_baseline.tar.gz\?versionId\=7FvexHgyazWF.dUo238FA7XRiK0FWQDw.
+addurl fcp_indi.s3.amazonaws.com_data_Projects_HBNSSI_RawDataTars_sub_0031121_baseline.tar.gz_versionId_7FvexHgyazWF.dUo238FA7XRiK0FWQDw. (downloading http://fcp-indi.s3.amazonaws.com/data/Projects/HBNSSI/RawDataTars/sub-0031121_baseline.tar.gz?versionId=7FvexHgyazWF.dUo238FA7XRiK0FWQDw. ...) 
+/mnt/btrfs/datasets/datalad/crawl-misc/indi/ 100%[==============================================================================================>] 195.44M  21.2MB/s    in 12s     
+(non-large file; adding content to git repository) ok
+(recording state in git...)
+cached/staged changes:                                                                                                                                                              
+ \u2026r.gz_versionId_7FvexHgyazWF.dUo238FA7XRiK0FWQDw. | Bin 0 -> 204937338 bytes
+
+$> ls -l fcp_indi.s3.amazonaws.com_data_Projects_HBNSSI_RawDataTars_sub_0031121_baseline.tar.gz_versionId_7FvexHgyazWF.dUo238FA7XRiK0FWQDw. 
+-rw------- 1 yoh datalad 204937338 Oct 25 17:30 fcp_indi.s3.amazonaws.com_data_Projects_HBNSSI_RawDataTars_sub_0031121_baseline.tar.gz_versionId_7FvexHgyazWF.dUo238FA7XRiK0FWQDw.
+cached/staged changes:                                                                                                                                                              
+ \u2026r.gz_versionId_7FvexHgyazWF.dUo238FA7XRiK0FWQDw. | Bin 0 -> 204937338 bytes
+
+"""]]
+
+[[!meta author=yoh]]
+
+> [[fixed|done]] --[[Joey]]
diff --git a/doc/bugs/adds_file_destined_for_annex_into_git_in___39__addurl__39__/comment_1_d598317883753baf02175a3bf866e08a._comment b/doc/bugs/adds_file_destined_for_annex_into_git_in___39__addurl__39__/comment_1_d598317883753baf02175a3bf866e08a._comment
new file mode 100644
index 000000000..e03e574f3
--- /dev/null
+++ b/doc/bugs/adds_file_destined_for_annex_into_git_in___39__addurl__39__/comment_1_d598317883753baf02175a3bf866e08a._comment
@@ -0,0 +1,20 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-11-21T15:12:54Z"
+ content="""
+It's sufficient to have "* annex.largefiles=(largerthan=100kb)"
+in .gitattributes.
+
+Even "* annex.largefiles=(largerthan=0kb)" will reproduce it.
+
+Ok, I see why.. It's running the largefile matcher on the destination file
+before it renames the temp file to it!
+
+Seems to have been broken this way ever since addurl got largefiles
+support. Testing didn't catch it because it only affects largefiles
+expressions that need to examine the file.
+
+Fixed in git. Audited other checkFileMatcher calls for this problem;
+the rest are ok.
+"""]]
diff --git a/doc/bugs/addurl_pathdepth_description_misleading/comment_4_2a9eb14a8c6d06747bb5dda7ff179ec7._comment b/doc/bugs/addurl_pathdepth_description_misleading/comment_4_2a9eb14a8c6d06747bb5dda7ff179ec7._comment
new file mode 100644
index 000000000..0cafc2a4d
--- /dev/null
+++ b/doc/bugs/addurl_pathdepth_description_misleading/comment_4_2a9eb14a8c6d06747bb5dda7ff179ec7._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="CandyAngel"
+ avatar="http://cdn.libravatar.org/avatar/15c0aade8bec5bf004f939dd73cf9ed8"
+ subject="comment 4"
+ date="2016-11-25T20:27:07Z"
+ content="""
+I really don't know what to say. I can't even figure out which computer I updated git-annex on to test if it was still happening.. let alone reproduce it anymore. It does work fine.
+
+I'm so sorry to bother you with this, I've done something stupid! This is exactly why you ask for a transcript of bugs occurring. (Feel free to use this as an example for why you ask for them, so some good can come of it at least..).
+"""]]
diff --git a/doc/bugs/cabal_constraints_for_aws_and_esqueleto/comment_1_d91e44573ef4a0ec6e7098cb4cd360f5._comment b/doc/bugs/cabal_constraints_for_aws_and_esqueleto/comment_1_d91e44573ef4a0ec6e7098cb4cd360f5._comment
new file mode 100644
index 000000000..327b63469
--- /dev/null
+++ b/doc/bugs/cabal_constraints_for_aws_and_esqueleto/comment_1_d91e44573ef4a0ec6e7098cb4cd360f5._comment
@@ -0,0 +1,30 @@
+[[!comment format=mdwn
+ username="https://launchpad.net/~felixonmars"
+ nickname="felixonmars"
+ avatar="http://cdn.libravatar.org/avatar/17284a3bb2e4ad9d3be8fab31f49865be9c1dc22143c728de731fe800a335d38"
+ subject="comment 1"
+ date="2016-11-28T04:17:12Z"
+ content="""
+aws has merged a PR to support http-conduit 2.2, but git-annex itself doesn't build with the new component yet:
+
+```
+[ 95 of 544] Compiling Utility.Url      ( Utility/Url.hs, dist/build/git-annex/git-annex-tmp/Utility/Url.o )
+
+Utility/Url.hs:354:34: error:
+    * The constructor `StatusCodeException' should have 2 arguments, but has been given 3
+    * In the pattern: StatusCodeException s _ _
+      In an equation for `matchStatusCodeException':
+          matchStatusCodeException want e@(StatusCodeException s _ _)
+            | want s = Just e
+            | otherwise = Nothing
+
+Utility/Url.hs:354:34: error:
+    * Couldn't match expected type `HttpException'
+                  with actual type `HttpExceptionContent'
+    * In the pattern: StatusCodeException s _ _
+      In an equation for `matchStatusCodeException':
+          matchStatusCodeException want e@(StatusCodeException s _ _)
+            | want s = Just e
+            | otherwise = Nothing
+```
+"""]]
diff --git a/doc/bugs/git-annex_fromkey_barfs_on_utf-8_input.mdwn b/doc/bugs/git-annex_fromkey_barfs_on_utf-8_input.mdwn
new file mode 100644
index 000000000..c1f71789b
--- /dev/null
+++ b/doc/bugs/git-annex_fromkey_barfs_on_utf-8_input.mdwn
@@ -0,0 +1,34 @@
+### Please describe the problem.
+
+I'm sending a stream of keys and filenames to git-annex fromkey on stdin, and it errors out with "git-annex: <stdin>: hGetContents: invalid argument (invalid byte sequence)". On the other hand yipdw tried to reproduce this and it worked fine for him, so I must be doing something wrong.
+
+I have LANG=en_US.UTF-8 set in my environment, if that matters.
+
+### What steps will reproduce the problem?
+
+[[!format sh """
+echo "MD5-s3263532--0b4d070eff7baa8ef314ca330aecb71f é" | git-annex fromkey
+"""]]
+
+### What version of git-annex are you using? On what operating system?
+
+[[!format sh """
+git-annex version: 6.20161118-g0a34f08
+build flags: Assistant Webapp Pairing Testsuite S3(multipartupload)(storageclasses) WebDAV Inotify DBus DesktopNotify XMPP ConcurrentOutput TorrentParser MagicMime Feeds Quvi
+key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 SHA1E SHA1 MD5E MD5 WORM URL
+remote types: git gcrypt S3 bup directory rsync web bittorrent webdav tahoe glacier ddar hook external
+local repository version: 5
+supported repository versions: 3 5 6
+upgrade supported from repository versions: 0 1 2 3 4 5
+operating system: linux x86_64
+"""]]
+
+### Please provide any additional information below.
+
+Note that this is indeed valid utf-8:
+
+[[!format sh """
+ db48x  ~  projects  IA.BAK-server  echo "é" | hexdump -C
+00000000  c3 a9 0a                                          |...|
+00000003
+"""]]
diff --git a/doc/bugs/git_annex_init_failed_due_to_unsupported_ssh_option/comment_2_32e142afd9fe65843d53883ba2ae48cb._comment b/doc/bugs/git_annex_init_failed_due_to_unsupported_ssh_option/comment_2_32e142afd9fe65843d53883ba2ae48cb._comment
new file mode 100644
index 000000000..42550d51f
--- /dev/null
+++ b/doc/bugs/git_annex_init_failed_due_to_unsupported_ssh_option/comment_2_32e142afd9fe65843d53883ba2ae48cb._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="scottgorlin@a32946b2aad278883c1690a0753241583a9855b9"
+ nickname="scottgorlin"
+ avatar="http://cdn.libravatar.org/avatar/2dd1fc8add62bbf4ffefac081b322563"
+ subject="IgnoreUnknown Include considered harmful?"
+ date="2016-11-23T20:07:45Z"
+ content="""
+As noted, include appears to not work on a mac at the moment.  This means git-annex silently ignores the included configs, which may be required to ssh to the remotes of interest.  This is happening to me.
+
+My understanding is that ssh aliases are the recommended way of juggling multiple private keys amongst multiple hosts, so it is a required part of many git workflows.  In this particular case, I have set up git annex on a NAS which does not allow multiple ssh users (QNAP) and the authentication is done only via key identity, not username.  Thus, host aliases are necessary.
+
+If one config can't include another, I would prefer an early failure indicating a problem with the config file, or better, a solution where git-annex doesn't require a config.  In this scenario, git fetch remote_name and git annex copy --to remotename do not resolve to the same alias definitions (the latter is missing because of the ignored config!).
+
+I got my setup to work only by finding and manually editing <repo>/.git/annex/ssh_config, which to my knowledge is undocumented (ie when is it written? do any commands change it?); manual mucking around inside .git to me is not a good practice, and for now I have two different alias's defined (in repo and in ~/.ssh/config)
+
+
+"""]]
diff --git a/doc/bugs/regression_due_to_usage_of_ssh_7.3___34__include__34___feature/comment_1_45003ab569c4649ca29c07877a83af29._comment b/doc/bugs/regression_due_to_usage_of_ssh_7.3___34__include__34___feature/comment_1_45003ab569c4649ca29c07877a83af29._comment
new file mode 100644
index 000000000..566926a32
--- /dev/null
+++ b/doc/bugs/regression_due_to_usage_of_ssh_7.3___34__include__34___feature/comment_1_45003ab569c4649ca29c07877a83af29._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="palday@91f366b5178879146d2b6e1e53bfa21389ee89a8"
+ nickname="palday"
+ avatar="http://cdn.libravatar.org/avatar/077a63af75ddba159980fbf88690f401"
+ subject="Temporary workaround until the brew formula is updated"
+ date="2016-11-29T02:17:52Z"
+ content="""
+The homebrew formula doesn't yet this fix, but you can get around the problem in the meantime by getting a newer SSH via homebrew:
+
+```
+brew install homebrew/dupes/openssh
+```
+
+You can then choose to keep that or get rid of it when the formula for git annex is later updated.
+"""]]
diff --git a/doc/devblog/day_428-429__git_push_to_hiddden_service.mdwn b/doc/devblog/day_428-429__git_push_to_hiddden_service.mdwn
new file mode 100644
index 000000000..ec1bf12fd
--- /dev/null
+++ b/doc/devblog/day_428-429__git_push_to_hiddden_service.mdwn
@@ -0,0 +1,31 @@
+The `tor` branch is coming along nicely.
+
+This weekend, I continued working on the P2P protocol, implementing
+it for network sockets, and extending it to support connecting up
+git-send-pack/git-receive-pack. 
+
+There was a bit of a detour when I split the Free monad into two separate
+ones, one for Net operations and the other for Local filesystem operations.
+
+This weekend's work was sponsored by Thomas Hochstein on Patreon.
+
+----
+
+Today, implemented a `git-remote-tor-annex` command that git will
+use for tor-annex:: urls,  and made `git annex remotedaemon`
+serve the tor hidden service.
+
+Now I have git push/pull working to the hidden service, for example:
+
+	git pull tor-annex::eeaytkuhaupbarfi.onion:47651
+
+That works very well, but does not yet check that the user is authorized
+to use the repo, beyond knowing the onion address. And currently
+it only works in git-annex repos; with some tweaks it should
+also work in plain git repos.
+
+Next, I need to teach git-annex how to access tor-annex remotes.
+And after that, an interface in the webapp for setting them up and
+connecting them together.
+
+Today's work was sponsored by Josh Taylor on Patreon.
diff --git a/doc/forum/Git-annex_link_to_different_file_names.mdwn b/doc/forum/Git-annex_link_to_different_file_names.mdwn
new file mode 100644
index 000000000..a7a7c2727
--- /dev/null
+++ b/doc/forum/Git-annex_link_to_different_file_names.mdwn
@@ -0,0 +1,41 @@
+This is a recreation of a stackexchange question, in case the community here is more knowledgeable.
+
+Link to stackexchange question : http://unix.stackexchange.com/questions/325753/git-annex-link-to-different-file-names
+
+Content :
+"Maybe this is just a crazy use case that doesn't work, but I was wondering if there's a way to build a file's history from files with different file names. I'm exploring this idea because I'd like to have a git-annex system but I can't force my coworkers to adapt.
+
+Here's what I have in mind :
+
+    Folder 1, managed by coworkers (On a shared disk) :
+
+        drawing_shop_12_nov_2015.pdf
+        drawing_shop_13_nov_2015.pdf
+        drawing_asbuilt_14_nov_2015.pdf
+        drawing_asbuilt_rev1_15_nov_2015.pdf
+
+And
+
+    Git-annex, managed by me :
+
+        drawing.pdf
+
+    (with a shop branch and a asbuilt branch)
+
+The git-annex's drawing.pdf would have an history like this :
+
+    [shop]
+    |
+    Commit A "Initial shop drawing"
+    |
+    Commit B "Add corrections from Wizzbasket"
+     \
+      |
+      [asbuilt]
+      Commit C "Reflect as built"
+      |
+      Commit D "Change dweezelbox block for simplicity"
+
+But somehow the "managed by coworkers" repo would be a direct mode repo with Commit A pointing to drawing_shop_12_nov_2015.pdf, Commit B to drawing_shop_13_nov_2015.pdf etc.
+
+Can this be done?"
diff --git a/doc/forum/Odd_Hybrid_Symlinks_To_Content.mdwn b/doc/forum/Odd_Hybrid_Symlinks_To_Content.mdwn
new file mode 100644
index 000000000..5e3b4d8b4
--- /dev/null
+++ b/doc/forum/Odd_Hybrid_Symlinks_To_Content.mdwn
@@ -0,0 +1,27 @@
+I've somehow managed to get my indirect repository to symlink to literal content instead of object files. 
+
+By this I mean literally the symlink is pointing at the contents of the file as the filename.
+
+So if I have a blah.txt file with this content:
+
+* First line
+* second line
+
+And I ls -al to view the symlink pointer, it shows up as this:
+
+* blah.txt -> First line?second line
+
+It literally has the contents of the file as the destination filename.
+
+I've tried a couple things I could think of to re-symlink the files, but they don't seem to do anything as they think everything is fine:
+
+* git annex indirect //returns nothing
+* git annex lock blah.txt //returns nothing
+* git annex fix blah.txt //returns nothing
+* git annex fsck //returns nothing
+
+I'm actually able to find several of these files hanging around by searching for all symlinks that don't point to something in the .git directory.
+
+Is there a way for me to replace the symlinks with correct symlinks to the objects in .git/annex? Can it even figure out which ones it was supposed to point to if the symlinks are messed up (are filenames -> content hashes stored anywhere else)?
+
+Else I might have to go do some manual rebasing and history editing to try to undo the bad commits manually. I've synced this repo to another direct repo so I'll need to figure out how to manually fix that repo too (using proxy).  From what I can tell the annex/direct/master seems to be same as master and synced/master branches? Is there an [[internals]] page for direct branches besides [[direct_mode]] so I know what should be fixed where?
diff --git a/doc/forum/Preserving_Directories_in_Metadata_Views.mdwn b/doc/forum/Preserving_Directories_in_Metadata_Views.mdwn
new file mode 100644
index 000000000..dfc45cb4b
--- /dev/null
+++ b/doc/forum/Preserving_Directories_in_Metadata_Views.mdwn
@@ -0,0 +1,47 @@
+I want to use metadata views to sort files into top-level directories based on a tag, but then preserve the directory structure underneath that. I'm having trouble with this.
+
+Say I have an annex at `~/annex` with a structure like this:
+
+    $ tree
+    .
+    ├── foo
+    │   └── bar
+    │       ├── one.txt
+    │       ├── three.txt
+    │       └── two.txt
+    └── waldo
+        └── fred
+            ├── a.txt
+            ├── b.txt
+            └── c.txt
+
+I tag some of the files with `blah`:
+
+    $ git annex metadata -t blah foo/bar/*
+
+Now I want to change my view to only see those files with a certain tag, but I want to maintain their directory structure, ie I want to end up with something like this:
+
+    $ tree
+    .
+    ├── blah
+    │   └── foo
+    │       └── bar
+    │           ├── one.txt
+    │           ├── three.txt
+    │           └── two.txt
+
+If I do `git annex view blah` I see the files `one.txt`, `two.txt` and `three.txt` but they are in the top level of `~/annex`. The `foo` and `bar` directories are not present.
+
+If I do `git annex view blah "/=*"` then the files I present under the `foo` directory, but the `bar` subdirectory is not there.
+
+It would also be fine if I could just hide the files that did not have the `blah` tag, so that I ended up with this:
+
+    $ tree
+    .
+    ├── foo
+    │   └── bar
+    │       ├── one.txt
+    │       ├── three.txt
+    │       └── two.txt
+
+Is something like this possible?
diff --git a/doc/forum/two-way_assistant_sync_with_ssh_special_remote.mdwn b/doc/forum/two-way_assistant_sync_with_ssh_special_remote.mdwn
new file mode 100644
index 000000000..ca04e442c
--- /dev/null
+++ b/doc/forum/two-way_assistant_sync_with_ssh_special_remote.mdwn
@@ -0,0 +1,32 @@
+I am attempting to set up automatic two-way synchronization between my laptop and a server via ssh by running assistant on both machines. I want to have both machines be non-bare and unlocked.
+
+On the rhel server:
+
+    $ mkdir ~/annex
+    $ cd ~/annex
+    $ git init
+    $ git annex init u --version=6
+    $ echo This is test file 1. >testfile1.txt
+    $ git annex add testfile1.txt
+    $ git annex sync
+    $ git remote add ml2 ssh://laptop/Users/username/annex
+    $ git annex adjust --unlock
+    $ git annex wanted . standard
+    $ git annex group . client
+
+On my mac laptop:
+
+    $ cd ~/
+    $ git clone ssh://server/home/username/annex
+    $ cd annex
+    $ git annex init ml2 --version=6
+    $ git annex sync
+    $ git annex adjust --unlock
+    $ git annex wanted . standard
+    $ git annex group . client
+
+Everything seems to work when I manually sync. But when I run
+
+    $ git annex assistant
+
+on both machines, I only get one-way automatic synchronization. Changes on the laptop are immediately propagated to the server. But changes on the server do not show up on the laptop until I manually sync. What am I doing wrong?
diff --git a/doc/forum/vanilla_git_repo_as_special_remote__63__/comment_2_6314256da98966f4c7d02aa0d6bf94ff._comment b/doc/forum/vanilla_git_repo_as_special_remote__63__/comment_2_6314256da98966f4c7d02aa0d6bf94ff._comment
new file mode 100644
index 000000000..f46355403
--- /dev/null
+++ b/doc/forum/vanilla_git_repo_as_special_remote__63__/comment_2_6314256da98966f4c7d02aa0d6bf94ff._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="neocryptek@659edac901ffbc8e541a974f8f18987eeafc63bd"
+ nickname="neocryptek"
+ avatar="http://cdn.libravatar.org/avatar/d9bfdefa9b503f1ac4844a686618374e"
+ subject="comment 2"
+ date="2016-11-21T22:26:52Z"
+ content="""
+Right, though bup also requires installation on the server. I'm looking for a way to store content into a vanilla git repo (as I don't have permission to install anything custom on the server).
+
+Since I want to store the content outside of git annex, it feels like a special remote. Though ideally it would have human readable files like:
+
+* <https://git-annex.branchable.com/todo/dumb__44___unsafe__44___human-readable_backend/>
+
+But since it's git and not just a normal (single version) filesystem, it could dedupe and save previous versions. Is there an easy way to hook git up safely to the external remote protocol:
+
+* [[special_remotes/external]]
+"""]]
diff --git a/doc/git-annex-rekey.mdwn b/doc/git-annex-rekey.mdwn
index 7dbe6ae96..4ec0b49e8 100644
--- a/doc/git-annex-rekey.mdwn
+++ b/doc/git-annex-rekey.mdwn
@@ -20,8 +20,6 @@ Multiple pairs of file and key can be given in a single command line.
   Allow rekeying of even files whose content is not currently available.
   Use with caution.
 
-# OPTIONS
-
 # SEE ALSO
 
 [[git-annex]](1)
diff --git a/doc/install/OSX.mdwn b/doc/install/OSX.mdwn
index 10fc8bad4..082fd517d 100644
--- a/doc/install/OSX.mdwn
+++ b/doc/install/OSX.mdwn
@@ -18,7 +18,7 @@ several more. Handy if you don't otherwise have git installed.
 
 ## autobuilds
 
-Thanks to Dartmouth for hosting the autobuilder.
+Thanks to Dartmouth College for hosting the autobuilder.
 
 * [autobuild of git-annex.dmg](https://downloads.kitenet.net/git-annex/autobuild/x86_64-apple-yosemite/git-annex.dmg) ([build logs](https://downloads.kitenet.net/git-annex/autobuild/x86_64-apple-yosemite/))
 
diff --git a/doc/install/Windows.mdwn b/doc/install/Windows.mdwn
index 7ea667a10..c9b60d642 100644
--- a/doc/install/Windows.mdwn
+++ b/doc/install/Windows.mdwn
@@ -1,8 +1,12 @@
 git-annex now does Windows!
 
-* First, [install Git for Windows](http://git-scm.com/downloads)  
+* First, [install Git for Windows](http://git-scm.com/downloads)
+
   Important: **Get the 32 bit version not the 64 bit version.**  
-  (Note that msysgit is no longer supported.)
+  If you installed the 64 bit version of git, then parts of git-annex will
+  still run, however, some features, including tools like rsync, will
+  not work.
+
 * Then, [install git-annex](https://downloads.kitenet.net/git-annex/windows/current/)
 
 This port is now in reasonably good shape for command-line use of
@@ -18,7 +22,7 @@ important thing is that it should end with "All tests passed".
 ## autobuilds
 
 A daily build is also available, thanks to Yury V. Zaytsev and
-[NEST](http://nest-initiative.org/).
+Dartmouth College.
 
 * [download](https://downloads.kitenet.net/git-annex/autobuild/windows/) ([build logs](https://qa.nest-initiative.org/view/msysGit/job/msysgit-git-annex-assistant-test/))
 
diff --git a/doc/special_remotes/rsync/comment_14_2261b1b7441eff9e28ec8e1f98d77980._comment b/doc/special_remotes/rsync/comment_14_2261b1b7441eff9e28ec8e1f98d77980._comment
new file mode 100644
index 000000000..4d6f0cfc2
--- /dev/null
+++ b/doc/special_remotes/rsync/comment_14_2261b1b7441eff9e28ec8e1f98d77980._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="davidriod@e75b369a4b1cced29c14354bce7493c61f00b1c7"
+ nickname="davidriod"
+ avatar="http://cdn.libravatar.org/avatar/d6e327bd88b88802d6f0c20c83f682a2"
+ subject="Sharing rsync special remote between repository"
+ date="2016-11-24T19:23:42Z"
+ content="""
+I was wondering if it is possible to share a rsync special remote between repository which are not parented in any way. The use case would be that even if these repositories are not related at all they still may contains the same binary file. It would be useful to have a single rsync remote in order to reduce space usage. I think it could work as the object names are based on their checksum, but I wonder if anyone has already try that ?
+"""]]
diff --git a/doc/tips/How_to_retroactively_annex_a_file_already_in_a_git_repo/comment_8_834410421ccede5194bd8fbaccea8d1a._comment b/doc/tips/How_to_retroactively_annex_a_file_already_in_a_git_repo/comment_8_834410421ccede5194bd8fbaccea8d1a._comment
new file mode 100644
index 000000000..2c36962aa
--- /dev/null
+++ b/doc/tips/How_to_retroactively_annex_a_file_already_in_a_git_repo/comment_8_834410421ccede5194bd8fbaccea8d1a._comment
@@ -0,0 +1,82 @@
+[[!comment format=mdwn
+ username="StephaneGourichon"
+ avatar="http://cdn.libravatar.org/avatar/8cea01af2c7a8bf529d0a3d918ed4abf"
+ subject="Walkthrough of a prudent retroactive annex."
+ date="2016-11-24T11:27:59Z"
+ content="""
+Been using the one-liner.  Despite the warning, I'm not dead yet.
+
+There's much more to do than the one-liner.
+
+This post offers instructions.
+
+# First simple try: slow
+
+Was slow (estimated >600s for 189 commits).
+
+# In tmpfs: about 6 times faster
+
+I have cloned repository into /run/user/1000/rewrite-git, which is a tmpfs mount point. (Machine has plenty of RAM.)
+
+There I also did `git annex init`, git-annex found its state branches.
+
+On second try I also did 
+
+    git checkout -t remotes/origin/synced/master
+
+So that filter-branch would clean that, too.
+
+There, `filter-branch` operation finished in 90s first try, 149s second try.
+
+`.git/objects` wasn't smaller.
+
+# Practicing reduction on clone
+
+This produced no visible benefit:
+
+time git gc --aggressive
+time git repack -a -d 
+
+Even cloning and retrying on clone. Oh, but I should have done `git clone file:///path` as said on git-filter-branch man page's section titled \"CHECKLIST FOR SHRINKING A REPOSITORY\" 
+
+This (as seen on https://rtyley.github.io/bfg-repo-cleaner/ ) was efficient:
+
+    git reflog expire --expire=now --all && git gc --prune=now --aggressive
+
+`.git/objects` shrunk from 148M to 58M
+
+All this was on a clone of the repo in tmpfs.
+
+# Propagating cleaned up branches to origin
+
+This confirmed that filter-branch did not change last tree:
+
+    git diff remotes/origin/master..master
+    git diff remotes/origin/synced/master synced/master
+
+This, expectedly, was refused:
+
+    git push origin master
+    git push origin synced/master
+
+On origin, I checked out the hash of current master, then on tmpfs clone
+
+    git push -f origin master
+    git push -f origin synced/master
+
+Looks good.
+
+I'm not doing the aggressive shrink now, because of the \"two orders of magnitude more caution than normal filter-branch\" recommended by arand.
+
+# Now what? Check if precious not broken
+
+I'm planning to do the same operation on the other repos, then :
+
+* if everything seems right, 
+* if `git annex sync` works between all those fellows
+* etc, 
+* then I would perform the reflog expire, gc prune on some then all of them, etc.
+
+Joey, does this seem okay?  Any comment?
+
+"""]]
diff --git a/doc/tips/using_Google_Cloud_Storage/comment_8_1b4eb7e0f44865cd5ff0f8ef507d99c1._comment b/doc/tips/using_Google_Cloud_Storage/comment_8_1b4eb7e0f44865cd5ff0f8ef507d99c1._comment
new file mode 100644
index 000000000..1a71f7726
--- /dev/null
+++ b/doc/tips/using_Google_Cloud_Storage/comment_8_1b4eb7e0f44865cd5ff0f8ef507d99c1._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="scottgorlin@a32946b2aad278883c1690a0753241583a9855b9"
+ nickname="scottgorlin"
+ avatar="http://cdn.libravatar.org/avatar/2dd1fc8add62bbf4ffefac081b322563"
+ subject="Coldline"
+ date="2016-11-21T00:49:23Z"
+ content="""
+Wanted to add that \"storageclass=COLDLINE\" appears to work seamlessly, both from my mac and arm NAS.  As far as I can tell, this appears to be a no-brainer vs glacier - builtin git annex client, simpler/cheaper billing, and no 4 hour delay!
+"""]]
diff --git a/doc/todo/renameremote.mdwn b/doc/todo/renameremote.mdwn
new file mode 100644
index 000000000..3a92bf507
--- /dev/null
+++ b/doc/todo/renameremote.mdwn
@@ -0,0 +1,24 @@
+Sometimes a name has been used for a special remote, and you want to change
+the name. A common reason is that the special remote has become dead, and
+you want to reuse the name for a new special remote. 
+
+Initremote prevents reusing a name when the old one exists, even if the old
+one is dead. And that makes sense in general, because a dead remote can
+come back sometimes, and that would leave the repo with two special remotes
+with the same name, and so enableremote would need to be run with a uuid
+instead of a name to specify which one to enable, which is not a desirable
+state of affairs.
+
+So, add `git annex renameremote oldname newname`. This could also do a `git
+remote rename`, or equivilant. (`git remote rename` gets confused by special
+remotes not having a fetch url and fails; this can be worked around by
+manually renaming the stanza in git config.)
+
+Implementing that would need a way to remove the old name from remote.log.
+We can't remove lines from union merged files, but what we could do is
+add a new line like:
+
+	- name=oldname timestamp=<latest>
+
+And in parsing remote.log, if the UUID is "-", don't include the
+remote with that name in the the resulting map.