Merge branch 'master' into smudge

15 files changed, 583 insertions, 15 deletions

diff --git a/Utility/FileMode.hs b/Utility/FileMode.hs
index 1e9b63483..efef5fa25 100644
--- a/Utility/FileMode.hs
+++ b/Utility/FileMode.hs
@@ -27,12 +27,24 @@ import Utility.Exception
 
 {- Applies a conversion function to a file's mode. -}
 modifyFileMode :: FilePath -> (FileMode -> FileMode) -> IO ()
-modifyFileMode f convert = do
+modifyFileMode f convert = void $ modifyFileMode' f convert
+
+modifyFileMode' :: FilePath -> (FileMode -> FileMode) -> IO FileMode
+modifyFileMode' f convert = do
 	s <- getFileStatus f
 	let old = fileMode s
 	let new = convert old
 	when (new /= old) $
 		setFileMode f new
+	return old
+
+{- Runs an action after changing a file's mode, then restores the old mode. -}
+withModifiedFileMode :: FilePath -> (FileMode -> FileMode) -> IO a -> IO a
+withModifiedFileMode file convert a = bracket setup cleanup go
+  where
+	setup = modifyFileMode' file convert
+	cleanup oldmode = modifyFileMode file (const oldmode)
+	go _ = a
 
 {- Adds the specified FileModes to the input mode, leaving the rest
  - unchanged. -}
diff --git a/Utility/Tmp.hs b/Utility/Tmp.hs
index 7e4db1101..7610f6cc8 100644
--- a/Utility/Tmp.hs
+++ b/Utility/Tmp.hs
@@ -15,6 +15,9 @@ import System.Directory
 import Control.Monad.IfElse
 import System.FilePath
 import Control.Monad.IO.Class
+#ifndef mingw32_HOST_OS
+import System.Posix.Temp (mkdtemp)
+#endif
 
 import Utility.Exception
 import Utility.FileSystemEncoding
@@ -64,25 +67,22 @@ withTmpFileIn tmpdir template a = bracket create remove use
  - directory and all its contents. -}
 withTmpDir :: (MonadMask m, MonadIO m) => Template -> (FilePath -> m a) -> m a
 withTmpDir template a = do
-	tmpdir <- liftIO $ catchDefaultIO "." getTemporaryDirectory
-	withTmpDirIn tmpdir template a
+	topleveltmpdir <- liftIO $ catchDefaultIO "." getTemporaryDirectory
+#ifndef mingw32_HOST_OS
+	-- Use mkdtemp to create a temp directory securely in /tmp.
+	bracket
+		(liftIO $ mkdtemp $ topleveltmpdir </> template)
+		removeTmpDir
+		a
+#else
+	withTmpDirIn topleveltmpdir template a
+#endif
 
 {- Runs an action with a tmp directory located within a specified directory,
  - then removes the tmp directory and all its contents. -}
 withTmpDirIn :: (MonadMask m, MonadIO m) => FilePath -> Template -> (FilePath -> m a) -> m a
-withTmpDirIn tmpdir template = bracketIO create remove
+withTmpDirIn tmpdir template = bracketIO create removeTmpDir
   where
-	remove d = whenM (doesDirectoryExist d) $ do
-#if mingw32_HOST_OS
-		-- Windows will often refuse to delete a file
-		-- after a process has just written to it and exited.
-		-- Because it's crap, presumably. So, ignore failure
-		-- to delete the temp directory.
-		_ <- tryIO $ removeDirectoryRecursive d
-		return ()
-#else
-		removeDirectoryRecursive d
-#endif
 	create = do
 		createDirectoryIfMissing True tmpdir
 		makenewdir (tmpdir </> template) (0 :: Int)
@@ -92,6 +92,21 @@ withTmpDirIn tmpdir template = bracketIO create remove
 			createDirectory dir
 			return dir
 
+{- Deletes the entire contents of the the temporary directory, if it
+ - exists. -}
+removeTmpDir :: MonadIO m => FilePath -> m ()
+removeTmpDir tmpdir = liftIO $ whenM (doesDirectoryExist tmpdir) $ do
+#if mingw32_HOST_OS
+	-- Windows will often refuse to delete a file
+	-- after a process has just written to it and exited.
+	-- Because it's crap, presumably. So, ignore failure
+	-- to delete the temp directory.
+	_ <- tryIO $ removeDirectoryRecursive tmpdir
+	return ()
+#else
+	removeDirectoryRecursive tmpdir
+#endif
+
 {- It's not safe to use a FilePath of an existing file as the template
  - for openTempFile, because if the FilePath is really long, the tmpfile
  - will be longer, and may exceed the maximum filename length.
diff --git a/debian/changelog b/debian/changelog
index 838b2d39c..f323b4b5a 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -24,6 +24,9 @@ git-annex (5.20151209) UNRELEASED; urgency=medium
     The empty file it was sending tickled bugs in some php WebDAV server.
   * fsck: Failed to honor annex.diskreserve when checking a remote.
   * Debian: Build depend on concurrent-output.
+  * Fix insecure temporary permissions when git-annex repair is used in
+    in a corrupted git repository.
+  * Fix potential denial of service attack when creating temp dirs.
 
  -- Joey Hess <id@joeyh.name>  Thu, 10 Dec 2015 11:39:34 -0400
 
diff --git a/doc/bugs/Low_disk_space_corrupts_state.mdwn b/doc/bugs/Low_disk_space_corrupts_state.mdwn
new file mode 100644
index 000000000..ecdfc48a2
--- /dev/null
+++ b/doc/bugs/Low_disk_space_corrupts_state.mdwn
@@ -0,0 +1,61 @@
+### Please describe the problem.
+
+When there are low disk space left, changes in the annex repo are only semi updated, leaving the dir and git annex state out of sync, leaving error messages with "invalid objects" and "fatal: git-write-tree: error building trees".
+
+I ignored these errors, and kept on trying to copy over all the files to a remote disk, since I wanted a backup, which resultet in symlinks pointing to files which aren't there.
+
+Maybe git-annex should stop if it sees that it's not enough disk space to perform a certain operation? E.g. cache space needed for syncing.
+
+### What steps will reproduce the problem?
+
+I'm not certain about what command that created the issue, but i ran various commands:
+
+- `git annex sync EXTERNALREPO`
+- `git annex copy --to EXTERNALREPO`
+- `git annex unlock Videos/`
+- `git annex lock Videos/`
+
+My disk had 280MB left, and the repo were at a few GBs. Somewhere in between these commands, git-annex started producing a lot of error messages for various files.
+
+### What version of git-annex are you using? On what operating system?
+
+git-annex version: 5.20140717 (Fedora 23)
+
+### Please provide any additional information below.
+
+Unfortunately, I don't have the full transcript of my badly behaved commands. Instead I add some of the error messages:
+
+[[!format sh """
+# If you can, paste a complete transcript of the problem occurring here.
+# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log
+
+
+    $ git annex lock Videos/
+    error: invalid object 100644 56715f46d9256bdcef0cd387364818e597dc9f41 for '003/a56/SHA256-s23430--26dc3b33a5101e4ead217241a371b17f15e7a2f37bbcaedd0d35a0a1aa4eb9b0.log.cnk'
+    fatal: git-write-tree: error building trees
+    git-annex: failed to read sha from git write-tree
+    
+    $ git annex sync
+    error: refs/remotes/remote_annex/synced/master does not point to a valid object!
+    error: refs/remotes/home/synced/master does not point to a valid object!
+    error: refs/remotes/work_annex/synced/master does not point to a valid object!
+    error: refs/remotes/home_annex/synced/master does not point to a valid object!
+    error: refs/remotes/home/synced/master does not point to a valid object!
+    error: refs/remotes/work_annex/synced/master does not point to a valid object!
+    error: Could not read bf3d6640fa32460032926ae6...
+    fatal: revision walk setup failed
+    error: Could not read 8cdec1808723971eaf30e32...
+    fatal: revision walk setup failed
+    (merging work/git-annex into git-annex...)
+    fatal: unable to read tree 21e0681190de239f41d...
+    (Recording state in git...)
+    error: invalid object 100644 56715f36d9256bdeff... for '003/a56/SHA256-s23430--2dc3b411e5ead...log.cnk'
+    fatal: git-write-tree: error building trees
+    git-annex: failed to read sha from git write-tree
+
+# End of transcript or log.
+"""]]
+
+### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
+
+It worked as a charm until my disk got full! (Maybe it is better to split it up in various, smaller repos, and sync them individually?)
diff --git a/doc/bugs/Low_disk_space_corrupts_state/comment_1_d8235498e2206c3b5284455e5c4182b9._comment b/doc/bugs/Low_disk_space_corrupts_state/comment_1_d8235498e2206c3b5284455e5c4182b9._comment
new file mode 100644
index 000000000..334a0fd94
--- /dev/null
+++ b/doc/bugs/Low_disk_space_corrupts_state/comment_1_d8235498e2206c3b5284455e5c4182b9._comment
@@ -0,0 +1,20 @@
+[[!comment format=mdwn
+ username="joakim.hovlandsvag@ad788ffa13d1ccbf03f2c485653900f8baa33950"
+ nickname="joakim.hovlandsvag"
+ subject="Repairing ignores disk space issues too"
+ date="2015-12-13T09:13:24Z"
+ content="""
+It seems that `git annex repair` also ignores such issues:
+
+    $ git annex repair
+    Running git fsck ...
+    Initialized empty Git repository in /tmp/tmprepo.0/.git/
+    Trying to recover missing objects from remote home.
+    Unpacking all pack files.
+    error: file write error (No space left on device)
+    fatal: unable to write sha1 file
+    Successfully recovered repository!
+    You should run \"git fsck\" to make sure, but it looks like everything was recovered ok.
+    ok
+
+"""]]
diff --git a/doc/bugs/STANDARD__95__IA_for_S3_remote_not_working/comment_5_ebb7cc0e1c125765611014049d403d61._comment b/doc/bugs/STANDARD__95__IA_for_S3_remote_not_working/comment_5_ebb7cc0e1c125765611014049d403d61._comment
new file mode 100644
index 000000000..73feda9a9
--- /dev/null
+++ b/doc/bugs/STANDARD__95__IA_for_S3_remote_not_working/comment_5_ebb7cc0e1c125765611014049d403d61._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="wsha.code+ga@b38779424f41c5701bbe5937340be43ff1474b2d"
+ subject="comment 5"
+ date="2015-12-12T05:28:51Z"
+ content="""
+Thanks for the clarification. I was referring to the Linux build. I might have been wrong about aws 0.13 being released. It is listed in the NixOS and Hackage package repos, but I don't see a 0.13.0 tag on the project's GitHub page, so maybe those other repos are packaging the nightly build of the master branch as version 0.13.0 while the final 0.13.0 hasn't been frozen yet. I will wait for the Debian repo to get updated (I could build it by hand but I like to rely on the package manager as I will otherwise forget to update the package in the future). It looks like Debian unstable lags the git repo tags by 1-2 months.
+"""]]
diff --git a/doc/bugs/git_annex_sync_deleted_a_bunch_of_files___40__not_expected__41__/comment_8_7b89c224ab6f79d406785fe751c241b6._comment b/doc/bugs/git_annex_sync_deleted_a_bunch_of_files___40__not_expected__41__/comment_8_7b89c224ab6f79d406785fe751c241b6._comment
new file mode 100644
index 000000000..943b9718e
--- /dev/null
+++ b/doc/bugs/git_annex_sync_deleted_a_bunch_of_files___40__not_expected__41__/comment_8_7b89c224ab6f79d406785fe751c241b6._comment
@@ -0,0 +1,77 @@
+[[!comment format=mdwn
+ username="yminus"
+ subject="comment 8"
+ date="2015-12-13T22:55:15Z"
+ content="""
+I think the problem in my case is that I had special characters in some file names which fat does not support.
+
+I tried to recover the git annex repo in direct mode using these steps:
+<pre>
+git push --force n900 master 
+git checkout synced/master
+git push --force n900 synced/master 
+git checkout master
+git annex unlock flac/Type_O_Negative/2003_Life_Is_Killing_Me/
+mv flac/Type_O_Negative/2003_Life_Is_Killing_Me/03.Less_Than_Zero_\(\<0\).flac flac/Type_O_Negative/2003_Life_Is_Killing_Me/03.Less_Than_Zero.flac
+git annex add flac/
+git commit -m \"Work around broken file systems\"
+git push --force n900 master 
+git checkout synced/master
+git push --force n900 synced/master 
+git checkout master
+git annex sync n900
+</pre>
+
+But now the last git annex sync creates a merge commit which only contains \"variants\" of the renamed files:
+
+<pre>
+    Merge remote-tracking branch 'refs/remotes/origin/synced/master' into annex/direct/master
+    
+    # Conflicts:
+    #	flac/Type_O_Negative/2003_Life_Is_Killing_Me/03.Less_Than_Zero.flac
+    #	mp3/Type_O_Negative/2003_Life_Is_Killing_Me/03.Less_Than_Zero.mp3
+
+ flac/Type_O_Negative/2003_Life_Is_Killing_Me/03.Less_Than_Zero.variant-6467.flac 
+index 0000000,c33ca87..c33ca87
+mode 000000,120000..120000
+
+ mp3/Type_O_Negative/2003_Life_Is_Killing_Me/03.Less_Than_Zero.variant-562b.mp3 
+index 0000000,a247fc6..a247fc6
+mode 000000,120000..120000
+</pre>
+
+So the repo on n900 still does not contain all the files added since the last sync (git annex get fails for those files). At least now the sync does not delete files in my laptop repo any more.
+
+This is the current state:
+
+<pre>
+*   44dd9a8 (n900/annex/direct/master) Merge remote-tracking branch 'refs/remotes/origin/synced/master' into annex/direct/master
+|\  
+| *   21df034 (HEAD -> master, tag: before_syncing_n900, nas/synced/master, n900/synced/master, n900/master, synced/master) Merge remote-tracking branch 'refs/remotes/nas/master'
+| |\  
+| | * 4f61c44 (nas/master) Work around broken file systems
+| | * 85ab30f ADDED FILES
+| * | 92bc06e Work around broken file systmes (mp3)
+| |/  
+* |   a945a24 merge refs/heads/synced/master
+|\ \  
+| |/  
+| * 1236008 (origin/synced/master, origin/master) ADDED FILES
+| * 17c4f54 ADDED FILES
+| *   364d525 Merge remote-tracking branch 'refs/remotes/origin/master'
+| |\  
+| | * c18f170 ADDED FILES
+| | * 9dd5668 ADDED FILES
+| * | c3280fc ADDED FILES
+| * | 2babe80 ADDED FILES
+| * | b964e29 ADDED FILES
+| * | 03f3bd1 ADDED FILES
+| * | 010a469 ADDED FILES
+| * | 8acf199 ADDED FILES
+| * |   f2477bc Merge remote-tracking branch 'refs/remotes/origin/master'
+| |\ \  
+| | |/  
+| | * 121ffd1 ADDED FILES
+* | | dc88b8a git-annex in lars@lars-laptop:/run/media/lars/Nokia N900/.sounds/Musik
+</pre>
+"""]]
diff --git a/doc/devblog/day_339_smudging_out_direct_mode/comment_6_ede48107675edfe40d5fdd3377553aa4._comment b/doc/devblog/day_339_smudging_out_direct_mode/comment_6_ede48107675edfe40d5fdd3377553aa4._comment
new file mode 100644
index 000000000..d52b054e3
--- /dev/null
+++ b/doc/devblog/day_339_smudging_out_direct_mode/comment_6_ede48107675edfe40d5fdd3377553aa4._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="https://openid.stackexchange.com/user/27ceb3c5-0762-42b8-8f8a-ed21c284748f"
+ nickname="g"
+ subject="comment 6"
+ date="2015-12-11T22:58:22Z"
+ content="""
+Thanks for the quick reply (and all your work on this!)
+
+Interesting, that change to git does sound like it should be relatively small compared to the workarounds needed. But in any case, glad to hear you're thinking about the issue.
+
+Also curious what your thoughts are on the performance issues you had identified previously with using smudge/clean on larger repos. Do the changes in git 2.5 address all your concerns? Or are there still some cases where this will potentially result in significant slow-down?
+"""]]
diff --git a/doc/forum/Can__39__t_upload_data_to_glacier_remote.mdwn b/doc/forum/Can__39__t_upload_data_to_glacier_remote.mdwn
new file mode 100644
index 000000000..fbf95df7b
--- /dev/null
+++ b/doc/forum/Can__39__t_upload_data_to_glacier_remote.mdwn
@@ -0,0 +1,55 @@
+I'm trying to follow the directions on [this tips page](https://git-annex.branchable.com/tips/using_Amazon_Glacier/) to easily back up some large home videos to Glacier. I followed the steps and everything worked fine until the `git annex copy`, at which point it claimed it was successful but had uploaded 0 bytes, as well as dumping the usage message for `glacier-cli` at the terminal (without any error):
+
+    Emily $ git annex copy --to glacier README
+    copy README (gpg) (checking glacier...) (to glacier...) 
+    100%          0.0 B/s 0s
+    glacier <command> [args]
+
+        Commands
+            vaults    - Operations with vaults
+            jobs      - Operations with jobs
+            upload    - Upload files to a vault. If the vault doesn't exits, it is
+                        created
+
+        Common args:
+            --access_key - Your AWS Access Key ID.  If not supplied, boto will
+                           use the value of the environment variable
+                           AWS_ACCESS_KEY_ID
+            --secret_key - Your AWS Secret Access Key.  If not supplied, boto
+                           will use the value of the environment variable
+                           AWS_SECRET_ACCESS_KEY
+            --region     - AWS region to use. Possible values: us-east-1, us-west-1,
+                           us-west-2, ap-northeast-1, eu-west-1.
+                           Default: us-east-1
+
+        Vaults operations:
+
+            List vaults:
+                glacier vaults 
+
+        Jobs operations:
+
+            List jobs:
+                glacier jobs <vault name>
+
+        Uploading files:
+
+            glacier upload <vault name> <files>
+
+            Examples : 
+                glacier upload pics *.jpg
+                glacier upload pics a.jpg b.jpg
+
+    ok                      
+    (Recording state in git...)
+
+Doing a `glacier vaults` also does not show any new vaults, and getting the usage message is obviously not normal.
+
+I tried doing a manual upload to a vault I already had sitting around from some years ago called `TVault`, and that looked to work fine:
+
+    Emily $ glacier upload TVault README 
+    Uploading README to TVault... done. Vault returned ArchiveID [omitted]
+
+(The update date hasn't updated on the management console yet, but I understand that may take up to a day.)
+
+Does anyone know what's going on, or is there at least a way to get a useful error message to output?
diff --git a/doc/forum/Can__39__t_upload_data_to_glacier_remote/comment_1_cb18d5d63663f73a343c8972faadc83a._comment b/doc/forum/Can__39__t_upload_data_to_glacier_remote/comment_1_cb18d5d63663f73a343c8972faadc83a._comment
new file mode 100644
index 000000000..5c18f1927
--- /dev/null
+++ b/doc/forum/Can__39__t_upload_data_to_glacier_remote/comment_1_cb18d5d63663f73a343c8972faadc83a._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="basak"
+ subject="comment 1"
+ date="2015-12-13T21:35:59Z"
+ content="""
+Sounds like you're running the boto glacier command rather than the glacier-cli tool. Unfortunately they have the same name (my intention was to deprecate the boto tool but boto upstream deprecated boto entirely instead). The current plan is to rename glacier-cli to glcr, but I haven't got round to doing that yet.
+
+Try making sure that \"glacier\" runs the glacier-cli tool rather than boto's glacier.
+"""]]
diff --git a/doc/forum/Can__39__t_upload_data_to_glacier_remote/comment_2_9b6a397d1c85da87481d36b8a4bd1ba6._comment b/doc/forum/Can__39__t_upload_data_to_glacier_remote/comment_2_9b6a397d1c85da87481d36b8a4bd1ba6._comment
new file mode 100644
index 000000000..321d4cc37
--- /dev/null
+++ b/doc/forum/Can__39__t_upload_data_to_glacier_remote/comment_2_9b6a397d1c85da87481d36b8a4bd1ba6._comment
@@ -0,0 +1,252 @@
+[[!comment format=mdwn
+ username="scorchgeek"
+ subject="Indeed"
+ date="2015-12-13T22:30:47Z"
+ content="""
+Nice catch! I couldn't find any install option for `glacier-cli`, so I just added a symlink to the source distribution and figured that would take care of it.
+
+    Emily $ ls /usr/bin -l | grep glacier
+    lrwxrwxrwx 1 root   root          38 Dec 11 16:19 glacier -> /home/soren/bin/glacier-cli/glacier.py
+
+But:
+
+    Emily $ which glacier
+    /usr/local/bin/glacier
+
+Oops. I changed the location of the symlink to somewhere earlier in the path, uninited and reinited the repository, and it uploaded the README.
+
+Unfortunately, now I'm getting a different error, some kind of Unicode issue, when I try to upload one of the files I actually want to back up:
+
+    Emily $ git annex add family-videos/Tape01.mpg
+    add family-videos/Tape01.mpg ok
+    (Recording state in git...)
+
+    Emily $ git commit
+    [master 5af0257] add tape01 as test
+    1 file changed, 1 insertion(+)
+    create mode 120000 family-videos/Tape01.mpg
+
+    Emily $ git annex copy --to glacier
+    copy family-videos/Tape01.mpg (gpg) (checking glacier...) (to glacier...) 
+    0%        8.1MB/s 11m32sTraceback (most recent call last):
+      File \"/home/soren/bin/glacier\", line 730, in <module>
+        App().main()
+      File \"/home/soren/bin/glacier\", line 716, in main
+        self.args.func()
+      File \"/home/soren/bin/glacier\", line 498, in archive_upload
+        file_obj=self.args.file, description=name)
+      File \"/usr/local/lib/python2.7/dist-packages/boto/glacier/vault.py\", line 177, in create_archive_from_file
+        writer.write(data)
+      File \"/usr/local/lib/python2.7/dist-packages/boto/glacier/writer.py\", line 219, in write
+        self.partitioner.write(data)
+      [...many more things in boto]
+      File \"/usr/lib/python2.7/httplib.py\", line 979, in request
+        self._send_request(method, url, body, headers)
+      File \"/usr/lib/python2.7/httplib.py\", line 1013, in _send_request
+        self.endheaders(body)
+      File \"/usr/lib/python2.7/httplib.py\", line 975, in endheaders
+        self._send_output(message_body)
+      File \"/usr/lib/python2.7/httplib.py\", line 833, in _send_output
+        msg += message_body
+    UnicodeDecodeError: 'ascii' codec can't decode byte 0x8c in position 0: ordinal not in range(128)
+    gpg: [stdout]: write error: Broken pipe
+    gpg: DBG: deflate: iobuf_write failed
+    gpg: build_packet failed: file write error
+    gpg: [stdout]: write error: Broken pipe
+    gpg: iobuf_flush failed on close: file write error
+    gpg: symmetric encryption of `[stdin]' failed: file write error
+    git-annex: fd:14: hPutBuf: resource vanished (Broken pipe)
+    failed
+    git-annex: copy: 1 failed
+
+Here's my complete session setting up the repository, in case it contains other useful information:
+
+    Emily 1020 [/media/backup/Videos]$ git init
+    Initialized empty Git repository in /media/backup/Videos/.git/
+    Emily 1021 [/media/backup/Videos](.)$ git remote add origin https://[myusername]@bitbucket.org/[myusername]/[myreponame].git
+    Emily 1022 [/media/backup/Videos](.)$ git push
+    fatal: The current branch master has no upstream branch.
+    To push the current branch and set the remote as upstream, use
+
+        git push --set-upstream origin master
+
+    Emily 1023 [/media/backup/Videos](.)$ git push -u origin master
+    error: src refspec master does not match any.
+    error: failed to push some refs to [URL]
+    Emily 1024 [/media/backup/Videos](.)$ git add README
+    Emily 1025 [/media/backup/Videos](@)$ ls
+    family-videos  README
+    Emily 1025 [/media/backup/Videos](@)$ git push -u origin master^C
+    Emily 1025 [/media/backup/Videos](@)$ git commit -am \"test commit\"
+    [master (root-commit) 03fb2a7] test commit
+     1 file changed, 1 insertion(+)
+     create mode 100644 README
+    Emily 1026 [/media/backup/Videos](.)$ git push -u origin master
+    Counting objects: 3, done.
+    Writing objects: 100% (3/3), 223 bytes | 0 bytes/s, done.
+    Total 3 (delta 0), reused 0 (delta 0)
+    remote:
+    remote: We're changing our IP addresses on 15 December 2015 at 00:00 UTC.
+    remote: Please make sure your firewalls are up to date:
+    remote: https://blog.bitbucket.org/?p=2677
+    To https://[myusername]@bitbucket.org/[myusername]/[myreponame].git
+     * [new branch]      master -> master
+    Branch master set up to track remote branch master from origin.
+    Emily 1027 [/media/backup/Videos](.)$ git annex init
+    init  ok
+    (Recording state in git...)
+    Emily 1028 [/media/backup/Videos](.)$ git annex --fast initremote glacier type=glacier encryption=hybrid keyid=[my GPG key ID]
+
+      Remote origin not usable by git-annex; setting annex-ignore
+    initremote glacier (encryption setup) (hybrid cipher with gpg key [hybrid key ID]) ok
+    (Recording state in git...)
+    Emily 1029 [/media/backup/Videos](.)$ git annex enableremote glacier
+    enableremote glacier (gpg) (encryption update) (hybrid cipher with gpg key [hybrid key ID]) ok
+    (Recording state in git...)
+    Emily 1030 [/media/backup/Videos](.)$ git annex copy README --to glacier
+    Emily 1031 [/media/backup/Videos](.)$ git annex sync
+    commit  ok
+    pull origin 
+    ok
+    push origin 
+    Counting objects: 13, done.
+    Delta compression using up to 4 threads.
+    Compressing objects: 100% (10/10), done.
+    Writing objects: 100% (12/12), 2.15 KiB | 0 bytes/s, done.
+    Total 12 (delta 3), reused 0 (delta 0)
+    remote: 
+    remote: We're changing our IP addresses on 15 December 2015 at 00:00 UTC.
+    remote: Please make sure your firewalls are up to date:
+    remote: https://blog.bitbucket.org/?p=2677
+    To [repository URL]
+     * [new branch]      git-annex -> synced/git-annex
+     * [new branch]      master -> synced/master
+    ok
+    Emily 1032 [/media/backup/Videos](.)$ git annex status
+    ? family-videos/Tape01.mpg
+    ? family-videos/Tape02.mpg
+    ? family-videos/Tape03.mpg
+    ? family-videos/Tape04.mpg
+    ? family-videos/Tape05.mpg
+    ? family-videos/Tape06.mpg
+    ? family-videos/Tape07.mpg
+    ? family-videos/Tape08.mpg
+    ? family-videos/Tape09.mpg
+    ? family-videos/Tape10.mpg
+    ? family-videos/Tape11.mpg
+    ? family-videos/Tape12.mpg
+    ? family-videos/Tape13.mpg
+    ? family-videos/Tape14.mpg
+    ? family-videos/Tape15.mpg
+    ? family-videos/Tape16.mpg
+    Emily 1033 [/media/backup/Videos](.)$ gitk
+    Emily 1034 [/media/backup/Videos](.)$ git annex copy README --to glacier^C
+    Emily 1034 [/media/backup/Videos](.)$ git annex add family-videos/Tap
+    ...01.mpg  ...04.mpg  ...07.mpg  ...10.mpg  ...13.mpg  ...16.mpg  
+    ...02.mpg  ...05.mpg  ...08.mpg  ...11.mpg  ...14.mpg  
+    ...03.mpg  ...06.mpg  ...09.mpg  ...12.mpg  ...15.mpg  
+    Emily 1034 [/media/backup/Videos](.)$ git annex add family-videos/Tape01.mpg
+    add family-videos/Tape01.mpg ok
+    (Recording state in git...)
+    Emily 1035 [/media/backup/Videos](@)$ git commit
+    [master 5af0257] add tape01 as test
+     1 file changed, 1 insertion(+)
+     create mode 120000 family-videos/Tape01.mpg
+    Emily 1036 [/media/backup/Videos](+)$ git annex copy --to glacier
+    copy family-videos/Tape01.mpg (gpg) (checking glacier...) (to glacier...) 
+    0%        8.1MB/s 11m32sTraceback (most recent call last):
+      File \"/home/soren/bin/glacier\", line 730, in <module>
+        App().main()
+      File \"/home/soren/bin/glacier\", line 716, in main
+        self.args.func()
+      File \"/home/soren/bin/glacier\", line 498, in archive_upload
+        file_obj=self.args.file, description=name)
+      File \"/usr/local/lib/python2.7/dist-packages/boto/glacier/vault.py\", line 177, in create_archive_from_file
+        writer.write(data)
+      File \"/usr/local/lib/python2.7/dist-packages/boto/glacier/writer.py\", line 219, in write
+        self.partitioner.write(data)
+      File \"/usr/local/lib/python2.7/dist-packages/boto/glacier/writer.py\", line 61, in write
+        self._send_part()
+      File \"/usr/local/lib/python2.7/dist-packages/boto/glacier/writer.py\", line 75, in _send_part
+        self.send_fn(part)
+      File \"/usr/local/lib/python2.7/dist-packages/boto/glacier/writer.py\", line 222, in _upload_part
+        self.uploader.upload_part(self.next_part_index, part_data)
+      File \"/usr/local/lib/python2.7/dist-packages/boto/glacier/writer.py\", line 129, in upload_part
+        content_range, part_data)
+      File \"/usr/local/lib/python2.7/dist-packages/boto/glacier/layer1.py\", line 1279, in upload_part
+        response_headers=response_headers)
+      File \"/usr/local/lib/python2.7/dist-packages/boto/glacier/layer1.py\", line 114, in make_request
+        data=data)
+      File \"/usr/local/lib/python2.7/dist-packages/boto/connection.py\", line 1071, in make_request
+        retry_handler=retry_handler)
+      File \"/usr/local/lib/python2.7/dist-packages/boto/connection.py\", line 943, in _mexe
+        request.body, request.headers)
+      File \"/usr/lib/python2.7/httplib.py\", line 979, in request
+        self._send_request(method, url, body, headers)
+      File \"/usr/lib/python2.7/httplib.py\", line 1013, in _send_request
+        self.endheaders(body)
+      File \"/usr/lib/python2.7/httplib.py\", line 975, in endheaders
+        self._send_output(message_body)
+      File \"/usr/lib/python2.7/httplib.py\", line 833, in _send_output
+        msg += message_body
+    UnicodeDecodeError: 'ascii' codec can't decode byte 0x8c in position 0: ordinal not in range(128)
+    gpg: [stdout]: write error: Broken pipe
+    gpg: DBG: deflate: iobuf_write failed
+    gpg: build_packet failed: file write error
+    gpg: [stdout]: write error: Broken pipe
+    gpg: iobuf_flush failed on close: file write error
+    gpg: symmetric encryption of `[stdin]' failed: file write error
+    git-annex: fd:15: hPutBuf: resource vanished (Broken pipe)
+    failed
+    git-annex: copy: 1 failed
+    Emily 1037 [/media/backup/Videos](+)$ git annex copy --to glacier
+    copy family-videos/Tape01.mpg (gpg) (checking glacier...) (to glacier...) 
+    0%        8.1MB/s 11m32sTraceback (most recent call last):
+      File \"/home/soren/bin/glacier\", line 730, in <module>
+        App().main()
+      File \"/home/soren/bin/glacier\", line 716, in main
+        self.args.func()
+      File \"/home/soren/bin/glacier\", line 498, in archive_upload
+        file_obj=self.args.file, description=name)
+      File \"/usr/local/lib/python2.7/dist-packages/boto/glacier/vault.py\", line 177, in create_archive_from_file
+        writer.write(data)
+      File \"/usr/local/lib/python2.7/dist-packages/boto/glacier/writer.py\", line 219, in write
+        self.partitioner.write(data)
+      File \"/usr/local/lib/python2.7/dist-packages/boto/glacier/writer.py\", line 61, in write
+        self._send_part()
+      File \"/usr/local/lib/python2.7/dist-packages/boto/glacier/writer.py\", line 75, in _send_part
+        self.send_fn(part)
+      File \"/usr/local/lib/python2.7/dist-packages/boto/glacier/writer.py\", line 222, in _upload_part
+        self.uploader.upload_part(self.next_part_index, part_data)
+      File \"/usr/local/lib/python2.7/dist-packages/boto/glacier/writer.py\", line 129, in upload_part
+        content_range, part_data)
+      File \"/usr/local/lib/python2.7/dist-packages/boto/glacier/layer1.py\", line 1279, in upload_part
+        response_headers=response_headers)
+      File \"/usr/local/lib/python2.7/dist-packages/boto/glacier/layer1.py\", line 114, in make_request
+        data=data)
+      File \"/usr/local/lib/python2.7/dist-packages/boto/connection.py\", line 1071, in make_request
+        retry_handler=retry_handler)
+      File \"/usr/local/lib/python2.7/dist-packages/boto/connection.py\", line 943, in _mexe
+        request.body, request.headers)
+      File \"/usr/lib/python2.7/httplib.py\", line 979, in request
+        self._send_request(method, url, body, headers)
+      File \"/usr/lib/python2.7/httplib.py\", line 1013, in _send_request
+        self.endheaders(body)
+      File \"/usr/lib/python2.7/httplib.py\", line 975, in endheaders
+        self._send_output(message_body)
+      File \"/usr/lib/python2.7/httplib.py\", line 833, in _send_output
+        msg += message_body
+    UnicodeDecodeError: 'ascii' codec can't decode byte 0x8c in position 0: ordinal not in range(128)
+    gpg: [stdout]: write error: Broken pipe
+    gpg: DBG: deflate: iobuf_write failed
+    gpg: build_packet failed: file write error
+    gpg: [stdout]: write error: Broken pipe
+    gpg: iobuf_flush failed on close: file write error
+    gpg: symmetric encryption of `[stdin]' failed: file write error
+    git-annex: fd:14: hPutBuf: resource vanished (Broken pipe)
+    failed                  
+    git-annex: copy: 1 failed
+    Emily 1037 [/media/backup/Videos](+)$ 
+
+Thanks a lot for your help!
+"""]]
diff --git a/doc/forum/Future_proofing___47___disaster_recovery_with_an_encrypted_special_remote/comment_2_9a9da0ad1538ba59e76a5c4f9fc7fcf7._comment b/doc/forum/Future_proofing___47___disaster_recovery_with_an_encrypted_special_remote/comment_2_9a9da0ad1538ba59e76a5c4f9fc7fcf7._comment
new file mode 100644
index 000000000..ff35541ec
--- /dev/null
+++ b/doc/forum/Future_proofing___47___disaster_recovery_with_an_encrypted_special_remote/comment_2_9a9da0ad1538ba59e76a5c4f9fc7fcf7._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="wsha.code+ga@b38779424f41c5701bbe5937340be43ff1474b2d"
+ subject="comment 2"
+ date="2015-12-12T10:42:46Z"
+ content="""
+Thanks for the response.
+
+With `pubkey` encryption, I am able to decrypt the remote's files normally using just `gpg --decrypt` with the public key used to encrypt them in my keyring. One thing I don't understand about `pubkey` encryption: what is in the `cipher=` entry in `remote.log` after the 256 bytes representing the HMAC cipher? The `gpg` key pair is used for encryption, so there is no encryption cipher to put in `remote.log` after the HMAC cipher, I would have thought.
+
+I understand the basic encryption set up, but I don't know how to use `gpg` to work with a raw cipher or raw encrypted text. For example with `pubkey` encryption, my interpretation is that the `cipher=` entry in `remote.log` is encrypted with the public key, but I can't just pass teh entry to `gpg --decrypt` because `gpg` expects the encrypted input to be formatted in a way that specifies which key to use to decrypt. Otherwise it says `gpg: no valid OpenPGP data found.` Similarly for `shared` encryption, I don't see how to pass the second half of the `remote.log` `cipher=` entry to `gpg` to decrypt the remote's files.
+
+I am also having trouble generating the special remote's keys. I created a `directory` remote with `shared` encryption (so that the `cipher=` entry in `remote.log` would not be encrypted). Then I added one file called `tmp.txt` which was stored in the annex as `SHA256E-s9--9c73fdec185d79405f58fc8b4e0ac22fa5ed2de7b7611a61b37606c905509650.txt` I did `git checkout git-annex -- remote.log` and then tried the following:
+
+        echo -n \"SHA256E-s9--9c73fdec185d79405f58fc8b4e0ac22fa5ed2de7b7611a61b37606c905509650.txt\" | openssl dgst -sha1 -hmac $(echo -n $(grep -oP 'cipher\=.*? ' remote.log | sed 's/cipher=//') | base64 -d | xxd -p -l 256 -c 256) -macopt hexkey:string`
+
+but the output does not match the GPGHMACSHA1 file name in the remote, and I don't understand why. I tried other variations as well (dropping the `.txt` or the SHA256E-s9--` prefix) but they did not work either.
+
+"""]]
diff --git a/doc/forum/Future_proofing___47___disaster_recovery_with_an_encrypted_special_remote/comment_3_27901f1c2c5de211f697972d9b21c104._comment b/doc/forum/Future_proofing___47___disaster_recovery_with_an_encrypted_special_remote/comment_3_27901f1c2c5de211f697972d9b21c104._comment
new file mode 100644
index 000000000..fe0187145
--- /dev/null
+++ b/doc/forum/Future_proofing___47___disaster_recovery_with_an_encrypted_special_remote/comment_3_27901f1c2c5de211f697972d9b21c104._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="wsha.code+ga@b38779424f41c5701bbe5937340be43ff1474b2d"
+ subject="comment 3"
+ date="2015-12-14T12:03:22Z"
+ content="""
+Small update: I have gotten the HMAC and decryption steps to work for shared encryption. I didn't figure out what was wrong with my HMAC command above yet, but I was able to reproduce the special remote's keys in Python using the `hmac` and `hashlib` libraries. I was trying to hash the correct string in the comment above (the full key with the `SHA...` prefix and the file extension suffix). For decryption, I used `gpg` on the encrypted file in the special remote and passed it the cipher from `remote.log` with the first 256 bytes removed as the passphrase (in the format returned by `base64.b64decode()` in Python).
+
+I still need to figure out how to decrypt the ciphers for `pubkey` and `hybrid`.
+
+I will try to put together a tip with the steps needed to reproduce special remote keys and to decrypt special remote files using only command line tools after that (assuming I can translate the Python steps back to command line tools; otherwise part of the steps will be in Python).
+"""]]
diff --git a/doc/forum/avoid_rehashing_when_converting_existing_backups_into_new_remotes/comment_1_7d0b479244fefd933193d921adcd897c._comment b/doc/forum/avoid_rehashing_when_converting_existing_backups_into_new_remotes/comment_1_7d0b479244fefd933193d921adcd897c._comment
new file mode 100644
index 000000000..538a070ed
--- /dev/null
+++ b/doc/forum/avoid_rehashing_when_converting_existing_backups_into_new_remotes/comment_1_7d0b479244fefd933193d921adcd897c._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://me.yahoo.com/a/ZF7p46cPmpWtb9zvA8iTitPmiQ--#eb014"
+ nickname="Jason"
+ subject="I'm also having this issue"
+ date="2015-12-14T03:17:58Z"
+ content="""
+I have the exact same issue: two copies of photos and videos that I have been using rsync to maintain. Converted one disk to git-annex, cloned the repo and moved .git/ into the second but can't seem to find a fast way to move the files to the correct hash id in the annex dir and create the symlinks. I'm thinking it might be best to just do a `git init; git annex init; git annex add .; git commit` then add the remotes in both directions and \"sync\".
+"""]]
diff --git a/doc/forum/avoid_rehashing_when_converting_existing_backups_into_new_remotes/comment_2_d42222ef94f4d409ac40e523f8211ffc._comment b/doc/forum/avoid_rehashing_when_converting_existing_backups_into_new_remotes/comment_2_d42222ef94f4d409ac40e523f8211ffc._comment
new file mode 100644
index 000000000..bfd8066ac
--- /dev/null
+++ b/doc/forum/avoid_rehashing_when_converting_existing_backups_into_new_remotes/comment_2_d42222ef94f4d409ac40e523f8211ffc._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://me.yahoo.com/a/ZF7p46cPmpWtb9zvA8iTitPmiQ--#eb014"
+ nickname="Jason"
+ subject="It worked"
+ date="2015-12-14T04:46:46Z"
+ content="""
+The above process just completed. Of course it took a while to hash and create symlinks for the 20K+ files. The initial sync caused an expected \"Warning: no common commits\" however since the symlinks all pointed at the exact same hashed filenames there was nothing to merge. The sync did take several minutes, but it was much shorter than copying 120GB. I hope this helps someone else in the future.
+"""]]