From f41643e02d3b4fd6dff8de2cf7e434ceaeed9daf Mon Sep 17 00:00:00 2001 From: ka7 Date: Wed, 24 Oct 2012 21:10:55 +0000 Subject: --- doc/forum/recover_deleted_files___63__.mdwn | 66 +++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 doc/forum/recover_deleted_files___63__.mdwn diff --git a/doc/forum/recover_deleted_files___63__.mdwn b/doc/forum/recover_deleted_files___63__.mdwn new file mode 100644 index 000000000..fcc567e16 --- /dev/null +++ b/doc/forum/recover_deleted_files___63__.mdwn @@ -0,0 +1,66 @@ +hi, + +i think of use git-annex as the backbone of a archival systems. at first point no distributed storage, just 1 node. +but now i run into the topic below ( deleted the "named" symlink of the "object" -- how to recover ?) + +maybe someone can enlighten me... + +thanks, +.ka + +// about the version. ( debian-squeeze, bpo ) + +$ git-annex version +git-annex version: 3.20120629~bpo60+2 +local repository version: 3 +default repository version: 3 +supported repository versions: 3 +upgrade supported from repository versions: 0 1 2 + +// building up a testcase. + +$ git init +Initialized empty Git repository in ...test2/.git/ + +$ git annex init +init ok +(Recording state in git...) + +$ echo "aaa" > 1.txt + +$ echo "bbb" > 2.txt + +$ git-annex add . +add 1.txt (checksum...) ok +add 2.txt (checksum...) ok +(Recording state in git...) + +$ git commit -a -m "added 2 files" +fatal: No HEAD commit to compare with (yet) +fatal: No HEAD commit to compare with (yet) +[master (root-commit) fc2a5d7] added 2 files + Committer: userhere user +Your name and email address were configured automatically based +on your username and hostname. Please check that they are accurate. +... + 2 files changed, 2 insertions(+), 0 deletions(-) + create mode 120000 1.txt + create mode 120000 2.txt + +// ok, so far standard. i have now 2 files - lets delete one. + +$ rm 2.txt +$ ls -l +lrwxrwxrwx 1 xp xp 176 24. Okt 22:55 1.txt -> .git/annex/objects/Z6/7q/SHA256-s4--17e682f060b5f8e47ea04c5c4855908b0a5ad612022260fe50e11ecb0cc0ab76/SHA256-s4--17e682f060b5f8e47ea04c5c4855908b0a5ad612022260fe50e11ecb0cc0ab76 + +// eek, delete of 2.txt was a bad idea (it was just the symlink) -- try to recover... + +$ git-annex fix +$ git-annex fsck +fsck 1.txt (checksum...) ok +$ ls +1.txt + +// still not here.. how to recover the link to 2.txt ??? +// i still see the content of the file in the object folter +// if I want to use git-annex as the backend of a archival system, this is important. -- cgit v1.2.3 From 0c19e89b64b2a53649eefa0e5983da5a3c290281 Mon Sep 17 00:00:00 2001 From: ka7 Date: Wed, 24 Oct 2012 21:13:12 +0000 Subject: --- doc/forum/recover_deleted_files___63__.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/forum/recover_deleted_files___63__.mdwn b/doc/forum/recover_deleted_files___63__.mdwn index fcc567e16..7bec36968 100644 --- a/doc/forum/recover_deleted_files___63__.mdwn +++ b/doc/forum/recover_deleted_files___63__.mdwn @@ -62,5 +62,5 @@ $ ls 1.txt // still not here.. how to recover the link to 2.txt ??? -// i still see the content of the file in the object folter +// i still see the content of the file in the object folder // if I want to use git-annex as the backend of a archival system, this is important. -- cgit v1.2.3 From a3e6810cfb590f65d00ff3ba04f1aa6933e0e7af Mon Sep 17 00:00:00 2001 From: "http://www.openid.albertlash.com/openid/" Date: Wed, 24 Oct 2012 22:00:31 +0000 Subject: Added a comment: It works! --- .../comment_6_55a4a3616ea59654da1c2f9902561e3b._comment | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 doc/forum/Managing_a_large_number_of_files_archived_on_many_pieces_of_read-only_medium___40__E.G._DVDs__41__/comment_6_55a4a3616ea59654da1c2f9902561e3b._comment diff --git a/doc/forum/Managing_a_large_number_of_files_archived_on_many_pieces_of_read-only_medium___40__E.G._DVDs__41__/comment_6_55a4a3616ea59654da1c2f9902561e3b._comment b/doc/forum/Managing_a_large_number_of_files_archived_on_many_pieces_of_read-only_medium___40__E.G._DVDs__41__/comment_6_55a4a3616ea59654da1c2f9902561e3b._comment new file mode 100644 index 000000000..ab971dab5 --- /dev/null +++ b/doc/forum/Managing_a_large_number_of_files_archived_on_many_pieces_of_read-only_medium___40__E.G._DVDs__41__/comment_6_55a4a3616ea59654da1c2f9902561e3b._comment @@ -0,0 +1,13 @@ +[[!comment format=mdwn + username="http://www.openid.albertlash.com/openid/" + ip="74.96.185.87" + subject="It works!" + date="2012-10-24T22:00:31Z" + content=""" +This works great! I first tried it with WORM, no-go. I can see why the SHA backends are so powerful, they appear to circumvent the commits which git usually uses for merging. When I first do the merge, it reports this: + +warning: no common commits + +Compared to how I've managed CD/DVD backups in the past, this is a quantum leap forward, and I don't find it convoluted in comparison. Yes, there is dar, but I prefer this method. In my case, its the perfect solution for original files, which in generally are treated as immutable, and not accessed very often. They are usually large, too! I'm using them for digital pictures. + +"""]] -- cgit v1.2.3 From 6e172e9d0de0485124547d44d53276db24b6e156 Mon Sep 17 00:00:00 2001 From: "http://joeyh.name/" Date: Wed, 24 Oct 2012 22:44:41 +0000 Subject: Added a comment --- .../comment_3_46b16dcd0fce07036cd8ed6ed9d2b055._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/design/assistant/blog/day_112__and_now_for_something_completely_different/comment_3_46b16dcd0fce07036cd8ed6ed9d2b055._comment diff --git a/doc/design/assistant/blog/day_112__and_now_for_something_completely_different/comment_3_46b16dcd0fce07036cd8ed6ed9d2b055._comment b/doc/design/assistant/blog/day_112__and_now_for_something_completely_different/comment_3_46b16dcd0fce07036cd8ed6ed9d2b055._comment new file mode 100644 index 000000000..5c6c161ff --- /dev/null +++ b/doc/design/assistant/blog/day_112__and_now_for_something_completely_different/comment_3_46b16dcd0fce07036cd8ed6ed9d2b055._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.154.0.23" + subject="comment 3" + date="2012-10-24T22:44:41Z" + content=""" +Google talk does not seem to support PEP. At least, I'm sending something that prosody accepts and that looks like the example at , and it replies with an IQ Error. +"""]] -- cgit v1.2.3 From a872a0bfc5a1754410f513f518a6fa98ca3ff822 Mon Sep 17 00:00:00 2001 From: "http://ciffer.net/~svend/" Date: Wed, 24 Oct 2012 22:50:05 +0000 Subject: Added a comment: git checkout --- .../comment_1_d7abb7c45c6ec2723a04f153ed215453._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/forum/recover_deleted_files___63__/comment_1_d7abb7c45c6ec2723a04f153ed215453._comment diff --git a/doc/forum/recover_deleted_files___63__/comment_1_d7abb7c45c6ec2723a04f153ed215453._comment b/doc/forum/recover_deleted_files___63__/comment_1_d7abb7c45c6ec2723a04f153ed215453._comment new file mode 100644 index 000000000..646e7ad54 --- /dev/null +++ b/doc/forum/recover_deleted_files___63__/comment_1_d7abb7c45c6ec2723a04f153ed215453._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://ciffer.net/~svend/" + ip="2001:1938:81:1ff::2" + subject="git checkout" + date="2012-10-24T22:50:05Z" + content=""" +You can use `git checkout -- file` to restore the link. `git status` will display information for restoring the file. +"""]] -- cgit v1.2.3 From d30fc1832a4cbd0a7467c9f3852116271e1ce54d Mon Sep 17 00:00:00 2001 From: Steve Date: Wed, 24 Oct 2012 23:26:54 +0000 Subject: Added a comment: web and hook special remotes --- .../comment_7_92a2af3e0e328bb48bcc67a69187ee57._comment | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 doc/forum/Managing_a_large_number_of_files_archived_on_many_pieces_of_read-only_medium___40__E.G._DVDs__41__/comment_7_92a2af3e0e328bb48bcc67a69187ee57._comment diff --git a/doc/forum/Managing_a_large_number_of_files_archived_on_many_pieces_of_read-only_medium___40__E.G._DVDs__41__/comment_7_92a2af3e0e328bb48bcc67a69187ee57._comment b/doc/forum/Managing_a_large_number_of_files_archived_on_many_pieces_of_read-only_medium___40__E.G._DVDs__41__/comment_7_92a2af3e0e328bb48bcc67a69187ee57._comment new file mode 100644 index 000000000..736fc6ce2 --- /dev/null +++ b/doc/forum/Managing_a_large_number_of_files_archived_on_many_pieces_of_read-only_medium___40__E.G._DVDs__41__/comment_7_92a2af3e0e328bb48bcc67a69187ee57._comment @@ -0,0 +1,13 @@ +[[!comment format=mdwn + username="Steve" + ip="92.104.175.136" + subject="web and hook special remotes" + date="2012-10-24T23:26:53Z" + content=""" +Hi Joey, + +Thanks for the advice. I had thought of the web special remote; but as you may have noticed from my example, I don't use automount so my DVDs and CDs all get mounted in the same place. (/mnt/cdrom) so the web special remote won't work for me. + +I'll try to play around with the hook special remote this weekend. I had a thought it might be interesting to have it search for the DVDs in some common places or even by parsing the mounted file systems, and allow an override or augmentation through git config. + +"""]] -- cgit v1.2.3 From 48fd3838b22479860fd9af99668e4a49d0e4f521 Mon Sep 17 00:00:00 2001 From: Steve Date: Wed, 24 Oct 2012 23:52:30 +0000 Subject: Added a comment: no need to merge --- ...ent_8_f6e39e71882d55cdc061166aea3e2bd3._comment | 26 ++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 doc/forum/Managing_a_large_number_of_files_archived_on_many_pieces_of_read-only_medium___40__E.G._DVDs__41__/comment_8_f6e39e71882d55cdc061166aea3e2bd3._comment diff --git a/doc/forum/Managing_a_large_number_of_files_archived_on_many_pieces_of_read-only_medium___40__E.G._DVDs__41__/comment_8_f6e39e71882d55cdc061166aea3e2bd3._comment b/doc/forum/Managing_a_large_number_of_files_archived_on_many_pieces_of_read-only_medium___40__E.G._DVDs__41__/comment_8_f6e39e71882d55cdc061166aea3e2bd3._comment new file mode 100644 index 000000000..84705d4cd --- /dev/null +++ b/doc/forum/Managing_a_large_number_of_files_archived_on_many_pieces_of_read-only_medium___40__E.G._DVDs__41__/comment_8_f6e39e71882d55cdc061166aea3e2bd3._comment @@ -0,0 +1,26 @@ +[[!comment format=mdwn + username="Steve" + ip="92.104.175.136" + subject="no need to merge" + date="2012-10-24T23:52:30Z" + content=""" +Albert, + +Thanks for feedback! I'm glad that somebody else found the method I worked out useful. As I'm going to try and turn it into a proper special remote, let me know if there is any particular use case or feature you'd like me to address. + +Note that in my testing, I found that you don't actually need to merge the DVD's branch into the local branch you are using for git annex to be able to find the files on it that are identical to files in your local branch. + +I haven't played around with cloning the repo, but I will try that this weekend. I'm thinking it *might* be necessary to create local branches from the DVD remotes so that they'll get carried along when you clone the repo. + +As far as the repos on the DVD's not having a shared ancestry with main repo, that was a conscious choice that I made. I wanted to add as little extra data to the DVDs as possible since I usually fill them to the brim anyway. I didn't feel that it would be beneficial for the DVD's to know about the history of the main repo and other files that they don't contain. Furthermore, besides all the links and history, you'd be replicating all the files in the main repo that aren't annexed. + +If you want to avoid the error, but still have a local branch for the DVD repos you should be able to do something like the following: + +WARNING: these commands are untested! +
+git checkout -b disc1 disc1/master
+git checkout -b disc2 disc2/master
+
+ +Working from the original example, you should then get local branches for the DVDs that don't have a common ancestor with your master local repo. I haven't actually tested that though. Testing will have to wait for this weekend. +"""]] -- cgit v1.2.3 From 727c089aca026ba90a3db07b22975edf441b9fe5 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 24 Oct 2012 20:05:45 -0400 Subject: add xmpp page --- doc/design/assistant.mdwn | 2 +- doc/design/assistant/cloud.mdwn | 29 ++-------------- doc/design/assistant/pairing.mdwn | 1 + doc/design/assistant/xmpp.mdwn | 70 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 74 insertions(+), 28 deletions(-) create mode 100644 doc/design/assistant/xmpp.mdwn diff --git a/doc/design/assistant.mdwn b/doc/design/assistant.mdwn index 77e96123d..5a34d4e77 100644 --- a/doc/design/assistant.mdwn +++ b/doc/design/assistant.mdwn @@ -11,7 +11,7 @@ and use cases to add. Feel free to chip in with comments! --[[Joey]] We are, approximately, here: -* Month 4 "cloud": [[!traillink cloud]] [[!traillink transfer_control]] +* Month 4 "cloud": [[!traillink cloud]] [[!traillink xmpp]] [[!traillink transfer_control]] * Month 5 user-driven features (see [[polls]]) * Months 6-7 "9k bonus round": [[!traillink Android]] [[!traillink partial_content]] [[!traillink leftovers]] * Months 8-11: more user-driven features and polishing (see remaining TODO items in all pages above) diff --git a/doc/design/assistant/cloud.mdwn b/doc/design/assistant/cloud.mdwn index b815c5d2d..18a1b3e9a 100644 --- a/doc/design/assistant/cloud.mdwn +++ b/doc/design/assistant/cloud.mdwn @@ -30,7 +30,7 @@ been a change to Alice's git repo. Then he needs to pull from Alice's git repo, or some other repo in the cloud she pushed to. Once both steps are done, the assistant will transfer the file from the cloud to Bob. -* dvcs-autosync uses jabber; all repos need to have the same jabber account +* dvcs-autosync uses xmppp; all repos need to have the same xmpp account configured, and send self-messages. An alternative would be to have different accounts that join a channel or message each other. Still needs account configuration. @@ -44,32 +44,7 @@ the assistant will transfer the file from the cloud to Bob. * pubsubhubbub does not seem like an option; its hubs want to pull down a feed over http. -### jabber TODO - -* test with big servers, eg google chat -* Prevent idle disconnection. Probably means sending or receiving pings, - but would prefer to avoid eg pinging every 60 seconds as some clients do. -* Make the git-annex clients invisible, so a user can use their regular - account without always seeming to be present when git-annex is logged in. - See - -### jabber security - -Any data git-annex sends over this XMPP will be visible to the XMPP -account's buddies, to the XMPP server, and quite likely to other interested -parties. So it's important to consider the security exposure of using it. - -If git-annex sends only a single bit notification, this lets attackers know -when the user is active and changing files. Although the assistant's other -syncing activities can somewhat mask this. - -As soon as git-annex does anything unlike any other client, an attacker can -see how many clients are connected for a user, and fingerprint the ones -running git-annex, and determine how many clients are running git-annex. - -If git-annex sent the UUID of the remote it pushed to, this would let -attackers determine how many different remotes are being used, -and map some of the connections between clients and remotes. +See [[xmpp]] for design of git-annex's use of xmpp for notifications. ## storing git repos in the cloud diff --git a/doc/design/assistant/pairing.mdwn b/doc/design/assistant/pairing.mdwn index d09c644ee..30b42a2cc 100644 --- a/doc/design/assistant/pairing.mdwn +++ b/doc/design/assistant/pairing.mdwn @@ -81,3 +81,4 @@ is escaped before going to the browser. It should be possible for third parties to tell when pairing is done, but it's actually rather hard since they don't necessarily share the secret. +* Pairing over XMPP. diff --git a/doc/design/assistant/xmpp.mdwn b/doc/design/assistant/xmpp.mdwn new file mode 100644 index 000000000..6d5384e43 --- /dev/null +++ b/doc/design/assistant/xmpp.mdwn @@ -0,0 +1,70 @@ +The git-annex assistant uses XMPP to communicate between peers that +cannot directly talk to one-another. A typical scenario is two users +who share a repository, that is stored in the [[cloud]]. + +### TODO + +* test with big servers, eg google chat +* Prevent idle disconnection. Probably means sending or receiving pings, + but would prefer to avoid eg pinging every 60 seconds as some clients do. +* Make the git-annex clients invisible, so a user can use their regular + account without always seeming to be present when git-annex is logged in. + See +* webapp configuration +* After pulling from a remote, may need to scan for transfers, which + could involve other remotes (ie, S3). Since the remote client is not able to + talk to us directly, it won't be able to upload any new files to us. + Need a fast way to find new files, and get them transferring. The expensive + transfer scan may be needed to get fully in sync, but is too expensive to + run every time this happens. + +## design goals + +1. Avoid user-visible messages. dvcs-autosync uses XMPP similarly, but + sends user-visible messages. Avoiding user-visible messages lets + the user configure git-annex to use his existing XMPP account + (eg, Google Talk). + +2. Send notifications to buddies. dvcs-autosync sends only self-messages, + but that requires every node have the same XMPP account configured. + git-annex should support that mode, but it should also send notifications + to a user's buddies. (This will also allow for using XMPP for pairing + in the future.) + +3. Don't make account appear active. Just because git-annex is being an XMPP + client, it doesn't mean that it wants to get chat messages, or make the + user appear active when he's not using his chat program. + +## protocol + +To avoid relying on XMPP extensions, git-annex communicates +using presence messages. These always mark it as extended away. +To this, it adds its own tag as [extended content](http://xmpp.org/rfcs/rfc6121.html#presence-extended). +The xml namespace is "git-annex" (not an URL because I hate wasting bandwidth). + +To indicate it's pushed changes to a git repo, a client uses: + + + +The push attribute can be repeated when the push was sent to multiple repos. + +### security + +Data git-annex sends over XMPP will be visible to the XMPP +account's buddies, to the XMPP server, and quite likely to other interested +parties. So it's important to consider the security exposure of using it. + +Even if git-annex sends only a single bit notification, this lets attackers +know when the user is active and changing files. Although the assistant's other +syncing activities can somewhat mask this. + +As soon as git-annex does anything unlike any other client, an attacker can +see how many clients are connected for a user, and fingerprint the ones +running git-annex, and determine how many clients are running git-annex. + +If git-annex sent the UUID of the remote it pushed to, this would let +attackers determine how many different remotes are being used, +and map some of the connections between clients and remotes. + +An attacker could replay push notification messages, reusing UUIDs it's +observed. This would make clients pull repeatedly, perhaps as a DOS. -- cgit v1.2.3 From 5c84739e291369ad5f489fb3120658af93ae4a42 Mon Sep 17 00:00:00 2001 From: Steve Date: Thu, 25 Oct 2012 00:34:06 +0000 Subject: --- ...m_and_builds_a_broken_git-annex_executable.mdwn | 50 ++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 doc/bugs/configure_mistakes_hashalot_bins_for_sha__63____63____63__sum_and_builds_a_broken_git-annex_executable.mdwn diff --git a/doc/bugs/configure_mistakes_hashalot_bins_for_sha__63____63____63__sum_and_builds_a_broken_git-annex_executable.mdwn b/doc/bugs/configure_mistakes_hashalot_bins_for_sha__63____63____63__sum_and_builds_a_broken_git-annex_executable.mdwn new file mode 100644 index 000000000..c7ed04c65 --- /dev/null +++ b/doc/bugs/configure_mistakes_hashalot_bins_for_sha__63____63____63__sum_and_builds_a_broken_git-annex_executable.mdwn @@ -0,0 +1,50 @@ +git-annex's configure step finds hashalot's /usr/sbin/sha256, /usr/sbin/sha384, and /usr/sbin/sha512 executables and mistakes them for sha256sum, sha384sum, and sha512sum and prefers them over the correct executables. Hashalot is not compatible, but the build does not fail producing a broken git-annex executable which tries to use hashalot's programs instead of the appropriate shaXXXsum program and is non-functional. + +Hashalot can be found at: http://www.paranoiacs.org/~sluskyb/hacks/hashalot/ + + +What steps will reproduce the problem? + +Compile with hashalot's programs in the path. + + +What is the expected output? What do you see instead? + +Expect to see configure output: + +[...] +
+  checking sha1... sha1sum
+  checking sha512... sha512sum
+  checking sha224... sha224sum
+  checking sha384... sha384sum
+  checking sha256... sha256sum
+
+[...] + + +Instead I see configure output: + +[...] +
+  checking sha1... sha1sum
+  checking sha512... sha512
+  checking sha224... sha224sum
+  checking sha384... sha384
+  checking sha256... sha256
+
+[...] + + +What version of git-annex are you using? On what operating system? + +I am using 3.20120605, but have checked out the latest GIT and confirmed the bug is still there. + + +Please provide any additional information below. + +This is not a runtime bug, only compile time. Uninstalling Hashalot or simply removing it from the PATH is enough to work around this bug. The bug is, however, frustrating because at first glance there appears to be no problem. However any functions of git-annex which use the affected SHA hash functions will fail with the resulting executable and the failure gives no clear indication of why. + +I found this bug on Gentoo when I installed git-annex on a system which already had hashalot installed. In the case of Gentoo, git-annex is compiled with hashalot's executables in the path, but normal users don't have /usr/sbin/ in their path so git-annex just fails to find the executable. If you put hashalot in the path, then git annex still fails to work as hashalot is not a replacement for sha1sum and friends. + +It may be enough to just prefer sha???sum over sha??? if they both exist. -- cgit v1.2.3 From 6ea651b4455f78461f3f2740e26e3936c8d43a92 Mon Sep 17 00:00:00 2001 From: "http://joeyh.name/" Date: Thu, 25 Oct 2012 03:29:20 +0000 Subject: Added a comment: it's a git repository... --- .../comment_2_8ea2acaa30d3ee7e9f75310f4ec859b2._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/forum/recover_deleted_files___63__/comment_2_8ea2acaa30d3ee7e9f75310f4ec859b2._comment diff --git a/doc/forum/recover_deleted_files___63__/comment_2_8ea2acaa30d3ee7e9f75310f4ec859b2._comment b/doc/forum/recover_deleted_files___63__/comment_2_8ea2acaa30d3ee7e9f75310f4ec859b2._comment new file mode 100644 index 000000000..c99461ae0 --- /dev/null +++ b/doc/forum/recover_deleted_files___63__/comment_2_8ea2acaa30d3ee7e9f75310f4ec859b2._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.152.108.138" + subject="it's a git repository..." + date="2012-10-25T03:29:20Z" + content=""" +So any git stuff can be used. If you deleted a file, committed it, and want to undo that, you can `git log --stat` to find the commit, and `git revert` it. If you deleted a file, haven't committed yet, and want one more look at it, you can `git stash` to get it back, and `git stash apply` to re-stash the change. Or yeah, just `git checkout` to get back a deleted file you have not committed yet. +"""]] -- cgit v1.2.3 From 53a057a1cb6661b62cd992efe0ad885bcac84cba Mon Sep 17 00:00:00 2001 From: "http://joeyh.name/" Date: Thu, 25 Oct 2012 03:33:29 +0000 Subject: Added a comment --- .../comment_9_6c45a6264d69e22800c329a0f8a2d470._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/forum/Managing_a_large_number_of_files_archived_on_many_pieces_of_read-only_medium___40__E.G._DVDs__41__/comment_9_6c45a6264d69e22800c329a0f8a2d470._comment diff --git a/doc/forum/Managing_a_large_number_of_files_archived_on_many_pieces_of_read-only_medium___40__E.G._DVDs__41__/comment_9_6c45a6264d69e22800c329a0f8a2d470._comment b/doc/forum/Managing_a_large_number_of_files_archived_on_many_pieces_of_read-only_medium___40__E.G._DVDs__41__/comment_9_6c45a6264d69e22800c329a0f8a2d470._comment new file mode 100644 index 000000000..5a3f2885c --- /dev/null +++ b/doc/forum/Managing_a_large_number_of_files_archived_on_many_pieces_of_read-only_medium___40__E.G._DVDs__41__/comment_9_6c45a6264d69e22800c329a0f8a2d470._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.152.108.138" + subject="comment 9" + date="2012-10-25T03:33:29Z" + content=""" +@Steve, it seems to me you could still use the web special remote, just pointing it at an url that goes through a symlink to the mount point. +"""]] -- cgit v1.2.3 From be3ba969eca03384fcf93b1da59b186c4932dcd6 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 25 Oct 2012 00:05:12 -0400 Subject: configure: Check that checksum programs produce correct checksums. + bitter rant So, it might be called sha1sum, or on some other OS, it might be called sha1. It might be hidden away off of PATH on that OS. That's just expected insanity; UNIX has been this way since 1980's. And these days, nobody even gives the flying flip about standards that we briefly did in the 90's after the first round of unix wars. But it's the 2010's now, and we've certainly learned something. So, let's make it so sometimes sha1 is a crazy program that wants to run as root so it can lock memory while prompting for a passphrase, and outputting binary garbage. Yes, that'd be wise. Let's package that in major Linux distros, too, so users can stumble over it. --- Build/Configure.hs | 30 ++++++++++++++++------ Build/TestConfig.hs | 14 +++++----- debian/changelog | 1 + ...m_and_builds_a_broken_git-annex_executable.mdwn | 7 +++++ 4 files changed, 37 insertions(+), 15 deletions(-) diff --git a/Build/Configure.hs b/Build/Configure.hs index 7fb195ad4..96582f923 100644 --- a/Build/Configure.hs +++ b/Build/Configure.hs @@ -19,7 +19,7 @@ tests = , testCp "cp_a" "-a" , testCp "cp_p" "-p" , testCp "cp_reflink_auto" "--reflink=auto" - , TestCase "uuid generator" $ selectCmd "uuid" ["uuid -m", "uuid", "uuidgen"] "" + , TestCase "uuid generator" $ selectCmd "uuid" [("uuid -m", ""), ("uuid", ""), ("uuidgen", "")] , TestCase "xargs -0" $ requireCmd "xargs_0" "xargs -0 /dev/null" , TestCase "curl" $ testCmd "curl" "curl --version >/dev/null" @@ -28,19 +28,33 @@ tests = , TestCase "gpg" $ testCmd "gpg" "gpg --version >/dev/null" , TestCase "lsof" $ testCmd "lsof" "lsof -v >/dev/null 2>&1" , TestCase "ssh connection caching" getSshConnectionCaching - ] ++ shaTestCases [1, 256, 512, 224, 384] + ] ++ shaTestCases + [ (1, "da39a3ee5e6b4b0d3255bfef95601890afd80709") + , (256, "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855") + , (512, "cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e") + , (224, "d14a028c2a3a2bc9476102bb288234c415a2b01f828ea62ac5b3e42f") + , (384, "38b060a751ac96384cd9327eb1b1e36a21fdb71114be07434c0cc7bf63f6e1da274edebfe76f65fbd51ad2f14898b95b") + ] -shaTestCases :: [Int] -> [TestCase] +{- shaNsum are the program names used by coreutils. + - On some systems, shaN is used instead, but on other + - systems, it might be "hashalot", which does not produce + - usable checksums. Only accept programs that produce + - known-good hashes. -} +shaTestCases :: [(Int, String)] -> [TestCase] shaTestCases l = map make l where - make n = TestCase key $ maybeSelectCmd key (shacmds n) " [x, osxpath x]) $ - map (\x -> "sha" ++ show n ++ x) ["", "sum"] - -- Max OSX puts GNU tools outside PATH, so look in - -- the location it uses, and remember where to run them - -- from. + map (\x -> "sha" ++ show n ++ x) ["sum", ""] + {- Max OSX puts GNU tools outside PATH, so look in + - the location it uses, and remember where to run them + - from. -} osxpath = "/opt/local/libexec/gnubin" tmpDir :: String diff --git a/Build/TestConfig.hs b/Build/TestConfig.hs index 0cc2019cf..eb920c13f 100644 --- a/Build/TestConfig.hs +++ b/Build/TestConfig.hs @@ -75,7 +75,7 @@ testCmd k cmdline = do {- Ensures that one of a set of commands is available by running each in - turn. The Config is set to the first one found. -} -selectCmd :: ConfigKey -> [String] -> String -> Test +selectCmd :: ConfigKey -> [(String, String)] -> Test selectCmd k = searchCmd (return . Config k . StringConfig) (\cmds -> do @@ -83,17 +83,17 @@ selectCmd k = searchCmd error $ "* need one of these commands, but none are available: " ++ show cmds ) -maybeSelectCmd :: ConfigKey -> [String] -> String -> Test +maybeSelectCmd :: ConfigKey -> [(String, String)] -> Test maybeSelectCmd k = searchCmd (return . Config k . MaybeStringConfig . Just) (\_ -> return $ Config k $ MaybeStringConfig Nothing) -searchCmd :: (String -> Test) -> ([String] -> Test) -> [String] -> String -> Test -searchCmd success failure cmds param = search cmds +searchCmd :: (String -> Test) -> ([String] -> Test) -> [(String, String)] -> Test +searchCmd success failure cmdsparams = search cmdsparams where - search [] = failure cmds - search (c:cs) = do - ret <- system $ quiet c ++ " " ++ param + search [] = failure $ fst $ unzip cmdsparams + search ((c, params):cs) = do + ret <- system $ quiet $ c ++ " " ++ params if ret == ExitSuccess then success c else search cs diff --git a/debian/changelog b/debian/changelog index 128adefef..246c0badc 100644 --- a/debian/changelog +++ b/debian/changelog @@ -21,6 +21,7 @@ git-annex (3.20121018) UNRELEASED; urgency=low does not accept that. * bugfix: Don't fail transferring content from read-only repos. Closes: #691341 + * configure: Check that checksum programs produce correct checksums. -- Joey Hess Wed, 17 Oct 2012 14:24:10 -0400 diff --git a/doc/bugs/configure_mistakes_hashalot_bins_for_sha__63____63____63__sum_and_builds_a_broken_git-annex_executable.mdwn b/doc/bugs/configure_mistakes_hashalot_bins_for_sha__63____63____63__sum_and_builds_a_broken_git-annex_executable.mdwn index c7ed04c65..46dfa4f70 100644 --- a/doc/bugs/configure_mistakes_hashalot_bins_for_sha__63____63____63__sum_and_builds_a_broken_git-annex_executable.mdwn +++ b/doc/bugs/configure_mistakes_hashalot_bins_for_sha__63____63____63__sum_and_builds_a_broken_git-annex_executable.mdwn @@ -48,3 +48,10 @@ This is not a runtime bug, only compile time. Uninstalling Hashalot or simply r I found this bug on Gentoo when I installed git-annex on a system which already had hashalot installed. In the case of Gentoo, git-annex is compiled with hashalot's executables in the path, but normal users don't have /usr/sbin/ in their path so git-annex just fails to find the executable. If you put hashalot in the path, then git annex still fails to work as hashalot is not a replacement for sha1sum and friends. It may be enough to just prefer sha???sum over sha??? if they both exist. + +> Grr. There is no consistency across unixes as to the names of these +> programs and now something is installing shaN commands that don't +> generate a checksum?! +> +> Ok, fine, configure now checks that the program it finds outputs a known +> good checksum. [[done]] --[[Joey]] -- cgit v1.2.3 From 0f471da96e115166d42087fa8fa57322d63ef765 Mon Sep 17 00:00:00 2001 From: "137.132.250.13" <137.132.250.13@web> Date: Thu, 25 Oct 2012 05:47:30 +0000 Subject: poll vote (Google Drive) --- doc/design/assistant/polls/prioritizing_special_remotes.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/design/assistant/polls/prioritizing_special_remotes.mdwn b/doc/design/assistant/polls/prioritizing_special_remotes.mdwn index 2f5759cc6..b5dc4d072 100644 --- a/doc/design/assistant/polls/prioritizing_special_remotes.mdwn +++ b/doc/design/assistant/polls/prioritizing_special_remotes.mdwn @@ -6,7 +6,7 @@ locally paired systems, and remote servers with rsync. Help me prioritize my work: What special remote would you most like to use with the git-annex assistant? -[[!poll open=yes 15 "Amazon S3 (done)" 9 "Amazon Glacier" 7 "Box.com" 57 "My phone (or MP3 player)" 15 "Tahoe-LAFS" 5 "OpenStack SWIFT" 20 "Google Drive"]] +[[!poll open=yes 15 "Amazon S3 (done)" 9 "Amazon Glacier" 7 "Box.com" 57 "My phone (or MP3 player)" 15 "Tahoe-LAFS" 5 "OpenStack SWIFT" 21 "Google Drive"]] This poll is ordered with the options I consider easiest to build listed first. Mostly because git-annex already supports them and they -- cgit v1.2.3 From 0285265e45ff28e914ed2d20126c4c62b89dc2d5 Mon Sep 17 00:00:00 2001 From: "http://lj.rossia.org/users/imz/" Date: Thu, 25 Oct 2012 09:31:26 +0000 Subject: Mark up the link so that someone interested can follow it. --- ...a__63____63____63__sum_and_builds_a_broken_git-annex_executable.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/bugs/configure_mistakes_hashalot_bins_for_sha__63____63____63__sum_and_builds_a_broken_git-annex_executable.mdwn b/doc/bugs/configure_mistakes_hashalot_bins_for_sha__63____63____63__sum_and_builds_a_broken_git-annex_executable.mdwn index 46dfa4f70..b71f39e5a 100644 --- a/doc/bugs/configure_mistakes_hashalot_bins_for_sha__63____63____63__sum_and_builds_a_broken_git-annex_executable.mdwn +++ b/doc/bugs/configure_mistakes_hashalot_bins_for_sha__63____63____63__sum_and_builds_a_broken_git-annex_executable.mdwn @@ -1,6 +1,6 @@ git-annex's configure step finds hashalot's /usr/sbin/sha256, /usr/sbin/sha384, and /usr/sbin/sha512 executables and mistakes them for sha256sum, sha384sum, and sha512sum and prefers them over the correct executables. Hashalot is not compatible, but the build does not fail producing a broken git-annex executable which tries to use hashalot's programs instead of the appropriate shaXXXsum program and is non-functional. -Hashalot can be found at: http://www.paranoiacs.org/~sluskyb/hacks/hashalot/ +Hashalot can be found at: What steps will reproduce the problem? -- cgit v1.2.3 From c3f63f7e55b7d5b014de1341c18d0088350c2a3b Mon Sep 17 00:00:00 2001 From: "http://adamspiers.myopenid.com/" Date: Thu, 25 Oct 2012 11:44:15 +0000 Subject: explain relationship of pairing to git remotes --- doc/assistant/pairing_walkthrough.mdwn | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/doc/assistant/pairing_walkthrough.mdwn b/doc/assistant/pairing_walkthrough.mdwn index 78bd91b16..4c0b7f2df 100644 --- a/doc/assistant/pairing_walkthrough.mdwn +++ b/doc/assistant/pairing_walkthrough.mdwn @@ -48,3 +48,10 @@ git annex assistant makes that easy too. And also, you can pair with as many other computers as you like, not just one! + +## What does pairing actually do behind the scenes? + +It ensures that both repositories have correctly configured +[[remotes|walkthrough/adding_a_remote]] pointing to each other. +If you have already configured this manually, you do not need to +perform pairing. -- cgit v1.2.3 From 44e9ba2471a39992d414f33509753aae7d13a7fb Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawniayrgSdVLUc3c6bf93VbO-_HT4hzxmyo" Date: Thu, 25 Oct 2012 16:08:44 +0000 Subject: --- ...annex:_getUserEntryForID:_failed___40__Success__41__.mdwn | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 doc/bugs/git-annex:_getUserEntryForID:_failed___40__Success__41__.mdwn diff --git a/doc/bugs/git-annex:_getUserEntryForID:_failed___40__Success__41__.mdwn b/doc/bugs/git-annex:_getUserEntryForID:_failed___40__Success__41__.mdwn new file mode 100644 index 000000000..a929048f6 --- /dev/null +++ b/doc/bugs/git-annex:_getUserEntryForID:_failed___40__Success__41__.mdwn @@ -0,0 +1,12 @@ +What steps will reproduce the problem? + Start "./git-annex-webapp" + +What is the expected output? What do you see instead? + The webapp should start, but I get the error "git-annex: getUserEntryForID: failed (Success)" + +What version of git-annex are you using? On what operating system? + 3.20121017 on "Ubuntu 10.04.4 LTS" 32-Bit + +Please provide any additional information below. + + -- cgit v1.2.3