diff options
35 files changed, 463 insertions, 71 deletions
diff --git a/CmdLine/Batch.hs b/CmdLine/Batch.hs index 57823b67b..ce9127975 100644 --- a/CmdLine/Batch.hs +++ b/CmdLine/Batch.hs @@ -12,15 +12,13 @@ import Command data BatchMode = Batch | NoBatch -batchOption :: Parser BatchMode -batchOption = flag NoBatch Batch +parseBatchOption :: Parser BatchMode +parseBatchOption = flag NoBatch Batch ( long "batch" <> help "enable batch mode" ) -type Batchable t = BatchMode -> t -> CommandStart - --- A Batchable command can run in batch mode, or not. +-- A batchable command can run in batch mode, or not. -- In batch mode, one line at a time is read, parsed, and a reply output to -- stdout. In non batch mode, the command's parameters are parsed and -- a reply output for each. @@ -29,7 +27,7 @@ batchable handler parser paramdesc = batchseeker <$> batchparser where batchparser = (,,) <$> parser - <*> batchOption + <*> parseBatchOption <*> cmdParams paramdesc batchseeker (opts, NoBatch, params) = mapM_ (go NoBatch opts) params @@ -52,3 +50,13 @@ batchable handler parser paramdesc = batchseeker <$> batchparser batchBadInput :: BatchMode -> Annex () batchBadInput NoBatch = liftIO exitFailure batchBadInput Batch = liftIO $ putStrLn "" + +-- Reads lines of batch mode input and passes to the action to handle. +batchSeek :: (String -> Annex ()) -> Annex () +batchSeek a = do + mp <- liftIO $ catchMaybeIO getLine + case mp of + Nothing -> return () + Just p -> do + a p + batchSeek a diff --git a/Command/AddUrl.hs b/Command/AddUrl.hs index af2e04a62..de83d8c9b 100644 --- a/Command/AddUrl.hs +++ b/Command/AddUrl.hs @@ -32,6 +32,7 @@ import Annex.Content.Direct import Annex.FileMatcher import Logs.Location import Utility.Metered +import CmdLine.Batch import qualified Annex.Transfer as Transfer #ifdef WITH_QUVI import Annex.Quvi @@ -51,6 +52,7 @@ data AddUrlOptions = AddUrlOptions , suffixOption :: Maybe String , relaxedOption :: Bool , rawOption :: Bool + , batchOption :: BatchMode } optParser :: CmdParamsDesc -> Parser AddUrlOptions @@ -74,6 +76,7 @@ optParser desc = AddUrlOptions )) <*> parseRelaxedOption <*> parseRawOption + <*> parseBatchOption parseRelaxedOption :: Parser Bool parseRelaxedOption = switch @@ -88,8 +91,13 @@ parseRawOption = switch ) seek :: AddUrlOptions -> CommandSeek -seek o = allowConcurrentOutput $ - forM_ (addUrls o) $ \u -> do +seek o = allowConcurrentOutput $ do + forM_ (addUrls o) go + case batchOption o of + Batch -> batchSeek go + NoBatch -> noop + where + go u = do r <- Remote.claimingUrl u if Remote.uuid r == webUUID || rawOption o then void $ commandAction $ startWeb o u diff --git a/Command/Status.hs b/Command/Status.hs index 35195fec6..3feea7cb4 100644 --- a/Command/Status.hs +++ b/Command/Status.hs @@ -46,10 +46,19 @@ displayStatus s = do unlessM (showFullJSON [("status", [c]), ("file", f)]) $ liftIO $ putStrLn $ [c] ++ " " ++ f --- Git thinks that present direct mode files are typechanged; --- check their content to see if they are modified or not. +-- Git thinks that present direct mode files are typechanged. +-- (On crippled filesystems, git instead thinks they're modified.) +-- Check their content to see if they are modified or not. statusDirect :: Status -> Annex (Maybe Status) -statusDirect (TypeChanged t) = do +statusDirect (TypeChanged t) = statusDirect' t +statusDirect s@(Modified t) = ifM crippledFileSystem + ( statusDirect' t + , pure (Just s) + ) +statusDirect s = pure (Just s) + +statusDirect' :: TopFilePath -> Annex (Maybe Status) +statusDirect' t = do absf <- fromRepo $ fromTopFilePath t f <- liftIO $ relPathCwdToFile absf v <- liftIO (catchMaybeIO $ getFileStatus f) @@ -65,7 +74,6 @@ statusDirect (TypeChanged t) = do , return $ Just $ Modified t ) checkkey f _ Nothing = Just <$> checkNew f t -statusDirect s = pure (Just s) checkNew :: FilePath -> TopFilePath -> Annex Status checkNew f t = ifM (isJust <$> catObjectDetails (Git.Ref.fileRef f)) diff --git a/Types/DeferredParse.hs b/Types/DeferredParse.hs index 983ba3f5c..7445615f6 100644 --- a/Types/DeferredParse.hs +++ b/Types/DeferredParse.hs @@ -10,7 +10,6 @@ module Types.DeferredParse where import Annex -import Common import Options.Applicative diff --git a/Utility/LockFile/PidLock.hs b/Utility/LockFile/PidLock.hs index 086e771aa..53eb5a54f 100644 --- a/Utility/LockFile/PidLock.hs +++ b/Utility/LockFile/PidLock.hs @@ -31,11 +31,12 @@ import System.IO import System.Posix import Data.Maybe import Data.List -import Control.Applicative import Network.BSD import System.FilePath import Data.Hash.MD5 import System.Directory +import Control.Applicative +import Prelude type LockFile = FilePath diff --git a/debian/changelog b/debian/changelog index f323b4b5a..0488c2eb1 100644 --- a/debian/changelog +++ b/debian/changelog @@ -17,7 +17,17 @@ git-annex (6.20151225) unstable; urgency=medium -- Joey Hess <id@joeyh.name> Tue, 08 Dec 2015 11:14:03 -0400 -git-annex (5.20151209) UNRELEASED; urgency=medium +git-annex (5.20151219) UNRELEASED; urgency=medium + + * status: On crippled filesystems, was displaying M for all annexed files + that were present. Probably caused by a change to what git status + displays in this situation. Fixed by treating files git thinks are + modified the same as typechanged files. + * addurl: Added --batch option. + + -- Joey Hess <id@joeyh.name> Sat, 19 Dec 2015 13:31:17 -0400 + +git-annex (5.20151218) unstable; urgency=medium * Add S3 features to git-annex version output. * webdav: When testing the WebDAV server, send a file with content. @@ -28,7 +38,7 @@ git-annex (5.20151209) UNRELEASED; urgency=medium in a corrupted git repository. * Fix potential denial of service attack when creating temp dirs. - -- Joey Hess <id@joeyh.name> Thu, 10 Dec 2015 11:39:34 -0400 + -- Joey Hess <id@joeyh.name> Fri, 18 Dec 2015 12:09:33 -0400 git-annex (5.20151208) unstable; urgency=medium diff --git a/doc/bugs/Mtime_of_objects_reset_when_synchronized_to_a_different_repository/comment_2_1b17514fb769fa5668f1b77e09ff3de1._comment b/doc/bugs/Mtime_of_objects_reset_when_synchronized_to_a_different_repository/comment_2_1b17514fb769fa5668f1b77e09ff3de1._comment new file mode 100644 index 000000000..b70082bdf --- /dev/null +++ b/doc/bugs/Mtime_of_objects_reset_when_synchronized_to_a_different_repository/comment_2_1b17514fb769fa5668f1b77e09ff3de1._comment @@ -0,0 +1,7 @@ +[[!comment format=mdwn + username="https://me.yahoo.com/a/EbvxpTI_xP9Aod7Mg4cwGhgjrCrdM5s-#7c0f4" + subject="comment 2" + date="2015-12-21T15:46:53Z" + content=""" +FWIW well -- git-annex does at least a bit to maintain mtime. E.g. when I 'annex add' a file, the symlink's mtime becomes the one of the original file. Unfortunately it is not the case though whenever I acquire the load for that key, which possibly gets correct mtime again, but then symlink doesn't get mtime adjusted to match to the one of the key. +"""]] diff --git a/doc/bugs/SmartGit__39__s_Cherry-Pick_function_throws_error_with_git-annex.mdwn b/doc/bugs/SmartGit__39__s_Cherry-Pick_function_throws_error_with_git-annex.mdwn new file mode 100644 index 000000000..9bad58b80 --- /dev/null +++ b/doc/bugs/SmartGit__39__s_Cherry-Pick_function_throws_error_with_git-annex.mdwn @@ -0,0 +1,43 @@ +### Please describe the problem. + +In SmartGit, doing a cherry-pick invokes the following 2 commands. + +``git cherry-pick --no-commit <commit-id>`` +``git commit "--author='<name and email>'" "--date='<date-time>'" --file=/var/folders/bw/jh4pq43918gfd_r7qxzkkt_40000gq/T/smartgit3129597356340000806tmp/commit7268815197345344355.tmp`` + +The error is _git 'annex' is not a git command_. + + +### What steps will reproduce the problem? + +``git-annex init <some-name>`` + +Do some commits in 2 branches. +Do a cherry-pick in SmartGit. + + +### What version of git-annex are you using? On what operating system? + +PLEASE provide instructions here on how to get the git-annex version. +(It is ``git-annex version``. And why is ``git-annex | less`` blanking out when I press any key?) + +git-annex: 5.20151019 + +OS: Mac OS El Capitan. + + +### Please provide any additional information below. + +[[!format sh """ +# If you can, paste a complete transcript of the problem occurring here. +# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log + + +# End of transcript or log. +"""]] + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) + +Yes. Lots. Am an expert at it now. + +This can either be a problem with SmartGit, or a problem with the hacks that git-annex puts into a standard git repo. diff --git a/doc/bugs/SmartGit__39__s_Cherry-Pick_function_throws_error_with_git-annex/comment_1_6fd91f615e8d5212674a9748d1b4135b._comment b/doc/bugs/SmartGit__39__s_Cherry-Pick_function_throws_error_with_git-annex/comment_1_6fd91f615e8d5212674a9748d1b4135b._comment new file mode 100644 index 000000000..541eefc41 --- /dev/null +++ b/doc/bugs/SmartGit__39__s_Cherry-Pick_function_throws_error_with_git-annex/comment_1_6fd91f615e8d5212674a9748d1b4135b._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2015-12-19T17:51:55Z" + content=""" +`git-annex init` sets up .git/hooks/pre-commit to run `git annex pre-commit`. +It looks like when git commit is run, the git-annex program is not in PATH. +This probably has something to do with the way you installed git-annex. + +(Why is git-annex | less blanking out? Because git-annex outputs usage +messages to stderr, and less pages stdout. Use git-annex 2>&1 |less) +"""]] diff --git a/doc/bugs/SmartGit__39__s_Cherry-Pick_function_throws_error_with_git-annex/comment_2_3dbaf39065c2f0941f8f7092c27fd4cb._comment b/doc/bugs/SmartGit__39__s_Cherry-Pick_function_throws_error_with_git-annex/comment_2_3dbaf39065c2f0941f8f7092c27fd4cb._comment new file mode 100644 index 000000000..7f401acd6 --- /dev/null +++ b/doc/bugs/SmartGit__39__s_Cherry-Pick_function_throws_error_with_git-annex/comment_2_3dbaf39065c2f0941f8f7092c27fd4cb._comment @@ -0,0 +1,16 @@ +[[!comment format=mdwn + username="jhannwong@c9c7a67b5632a4bbc0c959cfeb3d340e02f28565" + nickname="jhannwong" + subject="A problem with SmartGit then?" + date="2015-12-21T03:21:58Z" + content=""" +> git-annex init sets up .git/hooks/pre-commit to run git annex pre-commit. It looks like when git commit is run, the git-annex program is not in PATH. This probably has something to do with the way you installed git-annex. + +``which git-annex`` shows ``/usr/local/bin/git-annex``. I think I installed via Homebrew. + +Seems to be a problem with SmartGit then. The pre-commit hooks work just fine in Terminal sessions. + +> (Why is git-annex | less blanking out? Because git-annex outputs usage messages to stderr, and less pages stdout. Use git-annex 2>&1 |less) + +Ah. Oops. +"""]] diff --git a/doc/bugs/git-annex_cannot_connect_to_freenet_cloud___40__webdav__41__/comment_3_b2ef283a3933e22542e05ce9a1acef7d._comment b/doc/bugs/git-annex_cannot_connect_to_freenet_cloud___40__webdav__41__/comment_3_b2ef283a3933e22542e05ce9a1acef7d._comment new file mode 100644 index 000000000..3403a59fc --- /dev/null +++ b/doc/bugs/git-annex_cannot_connect_to_freenet_cloud___40__webdav__41__/comment_3_b2ef283a3933e22542e05ce9a1acef7d._comment @@ -0,0 +1,38 @@ +[[!comment format=mdwn + username="sts" + subject="comment 3" + date="2015-12-20T19:49:26Z" + content=""" +hey joey! + +thanks, it (kinda) worked :). At least the initialization works now pretty fine. But uploading files does not work as expected. It does upload some data, but does not move the data to the corresponding folder (from 'tmp' to $hash): + + git annex copy --debug --to webdav 1354580391258.jpg + [2015-12-20 20:31:36.985579] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"show-ref\",\"git-annex\"] + [2015-12-20 20:31:36.990098] process done ExitSuccess + [2015-12-20 20:31:36.990584] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"show-ref\",\"--hash\",\"refs/heads/git-annex\"] + [2015-12-20 20:31:36.993635] process done ExitSuccess + [2015-12-20 20:31:36.994311] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"log\",\"refs/heads/git-annex..2ff0c487bc1c92c11a45131b55f6fb2ba034071d\",\"-n1\",\"--pretty=%H\"] + [2015-12-20 20:31:36.998675] process done ExitSuccess + [2015-12-20 20:31:36.999747] chat: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"cat-file\",\"--batch\"] + [2015-12-20 20:31:37.003483] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"ls-files\",\"--cached\",\"-z\",\"--\",\"1354580391258.jpg\"] + copy 1354580391258.jpg (checking webdav...) (to webdav...) + 100% 564.6KB/s 0s + DAV failure: 409 \"Conflict\" + failed + git-annex: copy: 1 failed + +In the 'tmp'-folder on the server I can find the file ('SHA256E-s1156230--...'), but this file is not 100% identical to the one I uploaded. Well, it seems that the first byte is missing ;) at least if I compare the files via vimdiff: + + 0000000: d8ff e000 104a 4649 4600 0101 0100 4800 .....JFIF.....H. | 0000000: ffd8 ffe0 0010 4a46 4946 0001 0101 0048 ......JFIF.....H + 0000010: 4800 00ff db00 4300 0101 0101 0101 0101 H.....C......... | 0000010: 0048 0000 ffdb 0043 0001 0101 0101 0101 .H.....C........ + 0000020: 0101 0101 0102 0203 0202 0202 0204 0303 ................ | 0000020: 0101 0101 0101 0202 0302 0202 0202 0403 ................ + 0000030: 0203 0504 0505 0504 0404 0506 0706 0505 ................ | 0000030: 0302 0305 0405 0505 0404 0405 0607 0605 ................ + 0000040: 0706 0404 0609 0607 0808 0808 0805 0609 ................ | 0000040: 0507 0604 0406 0906 0708 0808 0808 0506 ................ + 0000050: 0a09 080a 0708 0808 ffdb 0043 0101 0101 ...........C.... | 0000050: 090a 0908 0a07 0808 08ff db00 4301 0101 ............C... + 0000060: 0202 0204 0202 0408 0504 0508 0808 0808 ................ | 0000060: 0102 0202 0402 0204 0805 0405 0808 0808 ................ + +As you can see, on the left side (aka 'the uploaded file') the first two hexadecimal values are missing in comparison to the right side (the source), so the first byte is missing. + +I guess in the end it is still the same problem, right :)? +"""]] diff --git a/doc/bugs/git-annex_creates_many_zombies/comment_2_4f08b8a1bf230d090848cc2f7c2d82f7._comment b/doc/bugs/git-annex_creates_many_zombies/comment_2_4f08b8a1bf230d090848cc2f7c2d82f7._comment new file mode 100644 index 000000000..1b81252e1 --- /dev/null +++ b/doc/bugs/git-annex_creates_many_zombies/comment_2_4f08b8a1bf230d090848cc2f7c2d82f7._comment @@ -0,0 +1,9 @@ +[[!comment format=mdwn + username="SamuelTardieu" + subject="comment 2" + date="2015-12-17T11:15:55Z" + content=""" +Indeed. + +I've migrated the Gentoo server from OpenRC to systemd so that I can use systemd-nspawn instead of docker for git-annex. +"""]] diff --git a/doc/design/assistant/polls/prioritizing_special_remotes.mdwn b/doc/design/assistant/polls/prioritizing_special_remotes.mdwn index 0da8623c4..36aa2b83f 100644 --- a/doc/design/assistant/polls/prioritizing_special_remotes.mdwn +++ b/doc/design/assistant/polls/prioritizing_special_remotes.mdwn @@ -6,7 +6,7 @@ locally paired systems, and remote servers with rsync. Help me prioritize my work: What special remote would you most like to use with the git-annex assistant? -[[!poll open=yes 18 "Amazon S3 (done)" 13 "Amazon Glacier (done)" 10 "Box.com (done)" 74 "My phone (or MP3 player)" 27 "Tahoe-LAFS" 16 "OpenStack SWIFT" 37 "Google Drive"]] +[[!poll open=yes 18 "Amazon S3 (done)" 13 "Amazon Glacier (done)" 10 "Box.com (done)" 75 "My phone (or MP3 player)" 27 "Tahoe-LAFS" 16 "OpenStack SWIFT" 37 "Google Drive"]] This poll is ordered with the options I consider easiest to build listed first. Mostly because git-annex already supports them and they diff --git a/doc/devblog/day_339_smudging_out_direct_mode/comment_7_ceb1d2a0e5bbc73590b80ff69292102d._comment b/doc/devblog/day_339_smudging_out_direct_mode/comment_7_ceb1d2a0e5bbc73590b80ff69292102d._comment new file mode 100644 index 000000000..7de202eac --- /dev/null +++ b/doc/devblog/day_339_smudging_out_direct_mode/comment_7_ceb1d2a0e5bbc73590b80ff69292102d._comment @@ -0,0 +1,15 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 7""" + date="2015-12-19T18:21:04Z" + content=""" +I'm still not entirely happy with the smudge/clean interface's performance. +At least git doesn't fall over if the clean filter declines to read all the +content of the large file on stdin anymore, which was the main thing +preventing an optimised use of it. Still, git has to run the clean filter once +per file, which is suboptimal. And, the smudge filter can't modify the file in +the work tree, but instead has to pass the whole file content back to git +for writing, which is going to result in a lot of unnecessary context +switches and slowdown. Especially in git-annex's case, where all it really +needs to do is make a hard link to the content. +"""]] diff --git a/doc/devblog/day_346-347__nearly_ready_to_merge.mdwn b/doc/devblog/day_346-347__nearly_ready_to_merge.mdwn new file mode 100644 index 000000000..944e148c5 --- /dev/null +++ b/doc/devblog/day_346-347__nearly_ready_to_merge.mdwn @@ -0,0 +1,19 @@ +Two more days working on v6 and the `smudge` branch is almost ready to be +merged. The test suite is passing again for v5 repos, and is almost +passing for v6 repos. Also I decided to make `git annex init` create v5 +repos for now, so `git annex init --version=6` or a `git annex upgrade` +is needed to get a v6 repo. So while I still have plenty of todo items for +v6 repos, they are working reasonably well and almost ready for early +adopters. + +The only real blocker to merging it is that the database stuff used by v6 +is not optimised yet and probably slow, and even in v5 repos it will query +the database. I hope to find an optimisation that avoids all database +overhead unless unlocked files are used in a v6 repo. + +I'll probably make one more release before that is merged though. Yesterday +I fixed a small security hole in `git annex repair`, which could expose the +contents of an otherwise not world-writable repository to local users. + +BTW, the [2015 git-annex user survey](http://git-annex-survey.branchable.com/polls/2015/) +closes in two weeks, please go fill it out if you haven't yet done so! diff --git a/doc/forum/All_repos_on_same_filesystem.mdwn b/doc/forum/All_repos_on_same_filesystem.mdwn new file mode 100644 index 000000000..ee61ab725 --- /dev/null +++ b/doc/forum/All_repos_on_same_filesystem.mdwn @@ -0,0 +1,14 @@ +Hi, + I am looking to find out the best way to use git annex, when all repos will live on the same filesystem, using a central repo. + +What I have so far is, after creating the main repo (mainrepo). + +Create clones via: git clone -shared mainrepo clonerepo + +Then use "git annex add" and "git add". When it comes to making the data accessible to the mainrepo, it is a little unclear to me. +There is a lot of disjoint information regarding pull/pushing content and which directions use hardlinks vs copies etc. So I +was hoping that someone could fill me in on best practices. + +Thanks in advance! + +Pete diff --git a/doc/forum/All_repos_on_same_filesystem/comment_1_a6666a89f62dc7e2d40e028782e5a25a._comment b/doc/forum/All_repos_on_same_filesystem/comment_1_a6666a89f62dc7e2d40e028782e5a25a._comment new file mode 100644 index 000000000..9074c4e16 --- /dev/null +++ b/doc/forum/All_repos_on_same_filesystem/comment_1_a6666a89f62dc7e2d40e028782e5a25a._comment @@ -0,0 +1,14 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2015-12-19T17:55:40Z" + content=""" +When you clone a repository with --shared, all git-annex commands +that transfer data into or out of the repository will use hardlinks +whenever possible. + +(But, before version 5.20150916, it didn't manage to use them all the +time.) + +To get started, just use `git annex sync --content` +"""]] diff --git a/doc/forum/Can__39__t_upload_data_to_glacier_remote/comment_3_6625da96f8cdc8b367b4c2bc275f2aee._comment b/doc/forum/Can__39__t_upload_data_to_glacier_remote/comment_3_6625da96f8cdc8b367b4c2bc275f2aee._comment new file mode 100644 index 000000000..3804eacf5 --- /dev/null +++ b/doc/forum/Can__39__t_upload_data_to_glacier_remote/comment_3_6625da96f8cdc8b367b4c2bc275f2aee._comment @@ -0,0 +1,14 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 3""" + date="2015-12-19T18:28:42Z" + content=""" +Note that, since version 5.20150219, git-annex probes to see if the +"glacier" program in PATH is the one from boto, and fails with a nicer +error message. + +The UnicodeDecodeError is mentioned in the thread for +[[special_remotes/glacier]] too. There is a workaround posted in that +thread. It would probably be good to nudge the boto maintainers to apply +the fix, which has been available for at least 3 months now. +"""]] diff --git a/doc/forum/Future_proofing___47___disaster_recovery_with_an_encrypted_special_remote/comment_4_56994d9252e8651b3f33044f5c8599b3._comment b/doc/forum/Future_proofing___47___disaster_recovery_with_an_encrypted_special_remote/comment_4_56994d9252e8651b3f33044f5c8599b3._comment new file mode 100644 index 000000000..fbfa254ff --- /dev/null +++ b/doc/forum/Future_proofing___47___disaster_recovery_with_an_encrypted_special_remote/comment_4_56994d9252e8651b3f33044f5c8599b3._comment @@ -0,0 +1,7 @@ +[[!comment format=mdwn + username="wsha.code+ga@b38779424f41c5701bbe5937340be43ff1474b2d" + subject="comment 4" + date="2015-12-16T11:21:48Z" + content=""" +Decrypting the cipher for `hybrid` and `pubkey` remotes is actually pretty straightforward: `echo -n <cipher_entry> | base64 -d | gpg --decrypt`. I think with that I have enough to put together a short summary of decrypting by hand the contents of `hybrid`, `pubkey`, and `shared` special remotes. +"""]] diff --git a/doc/forum/Future_proofing___47___disaster_recovery_with_an_encrypted_special_remote/comment_5_5aecb8c58d501e929e746d5536032c81._comment b/doc/forum/Future_proofing___47___disaster_recovery_with_an_encrypted_special_remote/comment_5_5aecb8c58d501e929e746d5536032c81._comment new file mode 100644 index 000000000..8c3478d97 --- /dev/null +++ b/doc/forum/Future_proofing___47___disaster_recovery_with_an_encrypted_special_remote/comment_5_5aecb8c58d501e929e746d5536032c81._comment @@ -0,0 +1,13 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 5""" + date="2015-12-19T17:59:38Z" + content=""" +When using `pubkey`, the second 256 bytes of ciphertext are currently +not used for anything. + +For `hybrid` and `shared`, the 256 bytes of ciphertext are used +as a symmetric cipher. So the gpg option to use for both encrypting +and decrypting is --symmetric. gpg then prompts for a passphrase, +and the ciphertext is what's used. +"""]] diff --git a/doc/forum/Git_annex_status_always_showing_annexed_files_as_modified/comment_5_5f2f2275ee9de47273671a8009217fb9._comment b/doc/forum/Git_annex_status_always_showing_annexed_files_as_modified/comment_5_5f2f2275ee9de47273671a8009217fb9._comment new file mode 100644 index 000000000..b266f442f --- /dev/null +++ b/doc/forum/Git_annex_status_always_showing_annexed_files_as_modified/comment_5_5f2f2275ee9de47273671a8009217fb9._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="neocryptek@659edac901ffbc8e541a974f8f18987eeafc63bd" + nickname="neocryptek" + subject="comment 5" + date="2015-12-18T18:09:54Z" + content=""" +Aloukian were you able to get anything to help? Nothing I've tried has helped. + +Joey any more ideas (besides ditching windows :) )? +"""]] diff --git a/doc/forum/Git_annex_status_always_showing_annexed_files_as_modified/comment_6_54155343eef8c6f95574dffb3910fc61._comment b/doc/forum/Git_annex_status_always_showing_annexed_files_as_modified/comment_6_54155343eef8c6f95574dffb3910fc61._comment new file mode 100644 index 000000000..6f179bcc1 --- /dev/null +++ b/doc/forum/Git_annex_status_always_showing_annexed_files_as_modified/comment_6_54155343eef8c6f95574dffb3910fc61._comment @@ -0,0 +1,46 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 6""" + date="2015-12-19T16:30:55Z" + content=""" +Sorry I haven't had a chance to think about this problem any more. +The v6 repository format I've been working on should eliminate class problem +anyway. It will let git itself be able to tell if annexed files are modified +or not. + +But there is something you can try to debug what's going on. + +0. Run the git bash shell and cd to your repository's directory. +1. `cat .git/annex/sentinal.cache` and paste its content. +2. `stat .git/annex/sentinal --terse` and paste the output of that. +2. Pick one of the files wrongly showing as modified. Run `stat $file --terse` + and paste the output of that. +3. And then this command should output the inode cache for the file. + Be sure to replace $file with the name of the file: + `cat $(git annex find $file --format='.git/annex/objects/${hashdirlower}${key}/${key}.cache`) + +But hmm, as I was running windows to get these instructions, I seem to have +reproduced the problem myself! In my case: + +* The sentinal.cache contained data matching the stat of the sentinal file. +* The annexed file's mtime and size (and even inode) matched the cached + values. +* `git annex status` showed the file as modifed; `git annex sync` found + nothing to commit and didn't change that. + +Some more debugging and.. It seems this is not a horrible windows-specific +time zone problem. Thank goodness. Instead, what's going on is that `git -c +core.bare=false status` does not show these files as typechanged, but as +modified instead. Since `git annex status` only has special case handling +for typechanged files, it just passes the M through and displays it. + +So, this is only a display problem, and thus nothing to worry about really. +Ie, the rest of git-annex's behavior should not be impacted at all. + +It's not windows specific.. Same happens on FAT on Linux. I think git's +behavior probably changed since an earlier version; I'm pretty sure its +status showed typechanged before. Anyway, I've fixed the status display, +on these systems it will now treat files git says are modified the same +as typechanged, and so will use git-annex's inode cache info to diplay +an accurate status for them. +"""]] diff --git a/doc/forum/Manipulate_a_git-annex_repo_from_an_Android_app/comment_2_20ee5f900a2ff6154c0d90263579723e._comment b/doc/forum/Manipulate_a_git-annex_repo_from_an_Android_app/comment_2_20ee5f900a2ff6154c0d90263579723e._comment new file mode 100644 index 000000000..e95d090e5 --- /dev/null +++ b/doc/forum/Manipulate_a_git-annex_repo_from_an_Android_app/comment_2_20ee5f900a2ff6154c0d90263579723e._comment @@ -0,0 +1,13 @@ +[[!comment format=mdwn + username="cbaines" + subject="comment 2" + date="2015-12-21T12:40:52Z" + content=""" +I am also interested in this. + +It might be better to use intents [1], such that git-annex on Android can provide an interface for any app to ask it to perform actions on repositories. + +Joey, is there any kind of api in the current Android app? + +1: https://developer.android.com/reference/android/content/Intent.html +"""]] diff --git a/doc/forum/Purge_whereis.mdwn b/doc/forum/Purge_whereis.mdwn new file mode 100644 index 000000000..d5b614898 --- /dev/null +++ b/doc/forum/Purge_whereis.mdwn @@ -0,0 +1,9 @@ +Hey there, + +I am playing around with git-annex to handle my files and backup's. Sometimes, I wish to delete a file entirely from my local repo and any other repo that contains the file. + +First, I drop the file from my local repo and run git-annex sync --content. Then I run git annex whereis and drop the file from other repo's. Subsequently, I run git annex whereis to ensure that the file has been dropped and does not exist in any repo. + +However, git annex still tracks the file, but I just have 0 copies of it. How do I remove/delete/disable tracking of a single file or multiple files? + +The reason why I have deleted the file is that I don't want it in any of my repo's any longer. diff --git a/doc/forum/Purge_whereis/comment_1_89d2446e3c9a58ab7b2a85ff941f7c8d._comment b/doc/forum/Purge_whereis/comment_1_89d2446e3c9a58ab7b2a85ff941f7c8d._comment new file mode 100644 index 000000000..848d1b75b --- /dev/null +++ b/doc/forum/Purge_whereis/comment_1_89d2446e3c9a58ab7b2a85ff941f7c8d._comment @@ -0,0 +1,9 @@ +[[!comment format=mdwn + username="cbaines" + subject="comment 1" + date="2015-12-21T12:32:55Z" + content=""" +If you want to stop tracking a file, just delete it, and then run git annex sync. + +There are some other commands (and settings in the assistant) for managing unused content. That is, files in the annex that are no longer referenced/used. +"""]] diff --git a/doc/forum/Purge_whereis/comment_2_0307ce47de9b719adc2096600d94dff6._comment b/doc/forum/Purge_whereis/comment_2_0307ce47de9b719adc2096600d94dff6._comment new file mode 100644 index 000000000..66e7bd6f1 --- /dev/null +++ b/doc/forum/Purge_whereis/comment_2_0307ce47de9b719adc2096600d94dff6._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="frost.kristian@75a6b6a25121f985cd8708f98c691d41716ac720" + nickname="frost.kristian" + subject="comment 2" + date="2015-12-21T13:08:49Z" + content=""" +It seems I have to enable syncing from the webapp. I thought doing git annex sync would do the same? +"""]] diff --git a/doc/forum/Purge_whereis/comment_3_e87b53ebbf8053bb17caeb2aed871212._comment b/doc/forum/Purge_whereis/comment_3_e87b53ebbf8053bb17caeb2aed871212._comment new file mode 100644 index 000000000..bdfe711f2 --- /dev/null +++ b/doc/forum/Purge_whereis/comment_3_e87b53ebbf8053bb17caeb2aed871212._comment @@ -0,0 +1,7 @@ +[[!comment format=mdwn + username="cbaines" + subject="comment 3" + date="2015-12-21T13:33:34Z" + content=""" +I just did a quick test, running git annex sync does commit the removal of files for me. +"""]] diff --git a/doc/forum/Reusing_existing_annex.uuid.mdwn b/doc/forum/Reusing_existing_annex.uuid.mdwn new file mode 100644 index 000000000..2d438e8a2 --- /dev/null +++ b/doc/forum/Reusing_existing_annex.uuid.mdwn @@ -0,0 +1,27 @@ +I recently replaced a disk that was failing with a new disk. Rather than just clone and create a new remote, I decided I'd reuse the existing anex.uuid, since I wanted to mount the replacement disk in the same spot/using the same name as before. The annex.uuid was the one I saw [here](http://git-annex.branchable.com/forum/Truly_purging_dead_repositories/#comment-b89d6a7ab50180c901f53761f8a1a99d). + +I have multiple disks that I store full copies of each of my repositories on; several are just regular git-annex remotes, and some are gcrypt remotes. All of them are local disks. + +When I ran a `git annex sync` in the repo with the reused UUID, the repo, and subsequently the other non-encrypted repos that I synced to, have lost knowledge of the encrypted repos, e.g. when I run `git-annex info`, I do not see the encrypted repos in that list. Even more strangely, this only happened in 2 out of 7 repos; 5 of the repos retained the knowledge of the gcrypt repos. + +The steps I used to recreate all of the repos was: + +``` +mkdir resued-uuid +cd reused-uuid +git init +git config annex.uuid xxxxxxxxxxx-xxxxxxxx-xxxxxxxxx +git annex init "My reused uuid repo" +git annex fsck (as directed to run in the linked comment, but this didn't do anything because there were no files in the repo and the repo did not have other repos' remotes added to it) +git remote add existing-repo ~/some/repo/path +git annex sync +git annex sync --content (all content copied over; subsequent fscks' don't reveal any damaged files) +``` + +I'm curious as to any insight as to why this might have happened and what went wrong. + +How can I add existing gcrypt remotes to the repos that are missing them. + +Does git annex copy some part of .git/config around to different repos? + +Can I just copy part of a .git/config with the grcrypt remotes listed there and sync them up? diff --git a/doc/forum/Reusing_existing_annex.uuid/comment_1_dc3d6efbf5c076bedd0c7a75ff5314bc._comment b/doc/forum/Reusing_existing_annex.uuid/comment_1_dc3d6efbf5c076bedd0c7a75ff5314bc._comment new file mode 100644 index 000000000..5347a42e8 --- /dev/null +++ b/doc/forum/Reusing_existing_annex.uuid/comment_1_dc3d6efbf5c076bedd0c7a75ff5314bc._comment @@ -0,0 +1,11 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2015-12-19T17:48:48Z" + content=""" +Take a look at remote.log in the git-annex branch; it should list all your +gcrypt and other special remotes. If some seem to be missing, you could +examine the git log of that file and see what happened. + +I don't see how forcing reuse of a uuid could lead to this. +"""]] diff --git a/doc/forum/Reusing_existing_annex.uuid/comment_2_cdb21c9c47c8a116e82062c9354d9339._comment b/doc/forum/Reusing_existing_annex.uuid/comment_2_cdb21c9c47c8a116e82062c9354d9339._comment new file mode 100644 index 000000000..e100e5952 --- /dev/null +++ b/doc/forum/Reusing_existing_annex.uuid/comment_2_cdb21c9c47c8a116e82062c9354d9339._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="https://openid.stackexchange.com/user/e65e6d0e-58ba-41de-84cc-1f2ba54cf574" + nickname="Mica" + subject="comment 2" + date="2015-12-21T06:02:21Z" + content=""" +The gcrypt remotes are present in the remotes.log file, but I can't seem to push or sync to them. + +How do I get them added back to the repo such that using `git annex sync` syncs the content? +"""]] diff --git a/doc/forum/avoid_rehashing_when_converting_existing_backups_into_new_remotes/comment_3_dc93ae2a58de5620c440531f43f1237d._comment b/doc/forum/avoid_rehashing_when_converting_existing_backups_into_new_remotes/comment_3_dc93ae2a58de5620c440531f43f1237d._comment new file mode 100644 index 000000000..539f574f6 --- /dev/null +++ b/doc/forum/avoid_rehashing_when_converting_existing_backups_into_new_remotes/comment_3_dc93ae2a58de5620c440531f43f1237d._comment @@ -0,0 +1,25 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 3""" + date="2015-12-19T18:09:26Z" + content=""" +I'd say that method, or any similar set of steps, is the typical way to +handle this. + +Sure, everything gets hashed twice. This is unlikely to waste enough time +to make it worthwhile to develop a hack that only hashes once. + +If you really want to develop such a hack, the plumbing command that you +can use to make it happen is `git annex setkey`. So, you'd add all the +files to the first repository, and then use `git-annex find +--format="${key} ${file}"` to list all the files and the keys that resulted +from hashing them. Then in the second repository, you'd use that list to +run `git annex setkey` and force the files into the annex without +hashing them. + +This will probably turn out to be slower than just re-hashing the files +would be, since you'll have to run `git annex setkey` once per file. +Adding a `--batch` option that reads from stdin would probably be called +for to get it fast enough to bother with. Although passing `-c +annex.alwayscommit=false` might speed it up enough. +"""]] diff --git a/doc/git-annex-addurl.mdwn b/doc/git-annex-addurl.mdwn index 0f5a763bc..eecc2c2bd 100644 --- a/doc/git-annex-addurl.mdwn +++ b/doc/git-annex-addurl.mdwn @@ -70,6 +70,11 @@ be used to get better filenames. Enables parallel downloads when multiple urls are being added. For example: `-J4` +* `--batch` + + Enables batch mode, in which lines containing urls to add are read from + stdin. + # SEE ALSO [[git-annex]](1) diff --git a/doc/news/version_5.20151019.mdwn b/doc/news/version_5.20151019.mdwn deleted file mode 100644 index ff42a07ce..000000000 --- a/doc/news/version_5.20151019.mdwn +++ /dev/null @@ -1,53 +0,0 @@ -git-annex 5.20151019 released with [[!toggle text="these changes"]] -[[!toggleable text=""" - * Fix a longstanding, but unlikely to occur bug, where dropping - a file from a remote could race with other drops of the same file, - and result in all copies of its content being lost. - * git-annex-shell: Added lockcontent command, to prevent dropping of - a key's content. This is necessary due to the above bugfix. - * In some cases, the above bugfix changes what git-annex allows you to - drop: - - When a file is present in several special remotes, - but not in any accessible git repositories, dropping it from one of - the special remotes will now fail. Instead, the file has to be - moved from one of the special remotes to the git repository, and can - then safely be dropped from the git repository. - - If a git remote has too old a version of git-annex-shell installed, - git-annex won't trust it to hold onto a copy of a file when dropping - that file from the local git repository. - * Changed drop ordering when using git annex sync --content or the - assistant, to drop from remotes first and from the local repo last. - This works better with the behavior changes to drop in many cases. - * Do verification of checksums of annex objects downloaded from remotes. - * When annex objects are received into git repositories from other git - repos, their checksums are verified then too. - * To get the old, faster, behavior of not verifying checksums, set - annex.verify=false, or remote.<name>.annex-verify=false. - * setkey, rekey: These commands also now verify that the provided file - matches the expected checksum of the key, unless annex.verify=false. - * reinject: Already verified content; this can now be disabled by - setting annex.verify=false. - * sync, merge, assistant: When git merge failed for a reason other - than a conflicted merge, such as a crippled filesystem not allowing - particular characters in filenames, git-annex would make a merge commit - that could omit such files or otherwise be bad. Fixed by aborting the - whole merge process when git merge fails for any reason other than a - merge conflict. - * Allow building with S3 disabled again. - * Ported disk free space checking code to work on Solaris. - * Windows webapp: Fix support for entering password when setting - up a ssh remote. - * copy --auto was checking the wrong repo's preferred content. - (--from was checking what --to should, and vice-versa.) - Fixed this bug, which was introduced in version 5.20150727. - * Avoid unncessary write to the location log when a file is unlocked - and then added back with unchanged content. - * S3: Fix support for using https. - * Avoid displaying network transport warning when a ssh remote - does not yet have an annex.uuid set. - * Debian: Add torrent library to build-depends as it's packaged now, - and stop recommending bittornado | bittorrent. - * Debian: Remove build dependency on transformers library, as it is now - included in ghc. - * Debian: Remove menu file, since a desktop file is provided and - lintian says there can be only one."""]]
\ No newline at end of file diff --git a/doc/news/version_5.20151218.mdwn b/doc/news/version_5.20151218.mdwn new file mode 100644 index 000000000..a1543742b --- /dev/null +++ b/doc/news/version_5.20151218.mdwn @@ -0,0 +1,10 @@ +git-annex 5.20151218 released with [[!toggle text="these changes"]] +[[!toggleable text=""" + * Add S3 features to git-annex version output. + * webdav: When testing the WebDAV server, send a file with content. + The empty file it was sending tickled bugs in some php WebDAV server. + * fsck: Failed to honor annex.diskreserve when checking a remote. + * Debian: Build depend on concurrent-output. + * Fix insecure temporary permissions when git-annex repair is used in + in a corrupted git repository. + * Fix potential denial of service attack when creating temp dirs."""]]
\ No newline at end of file diff --git a/git-annex.cabal b/git-annex.cabal index 003a3cb09..4794d38e5 100644 --- a/git-annex.cabal +++ b/git-annex.cabal @@ -1,5 +1,5 @@ Name: git-annex -Version: 5.20151208 +Version: 5.20151218 Cabal-Version: >= 1.8 License: GPL-3 Maintainer: Joey Hess <id@joeyh.name> |