diff options
29 files changed, 195 insertions, 16 deletions
@@ -3,20 +3,29 @@ mans=git-annex.1 git-annex-shell.1 sources=Build/SysConfig.hs Utility/Touch.hs Utility/Mounts.hs all=$(bins) $(mans) docs +CFLAGS=-Wall + OS:=$(shell uname | sed 's/[-_].*//') ifeq ($(OS),Linux) -BASEFLAGS_OPTS+=-DWITH_INOTIFY -DWITH_DBUS +BASEFLAGS_OPTS=-DWITH_INOTIFY -DWITH_DBUS clibs=Utility/libdiskfree.o Utility/libmounts.o else -BASEFLAGS_OPTS+=-DWITH_KQUEUE +# BSD system +BASEFLAGS_OPTS=-DWITH_KQUEUE clibs=Utility/libdiskfree.o Utility/libmounts.o Utility/libkqueue.o +ifeq ($(OS),Darwin) +# Ensure OSX compiler builds for 32 bit when using 32 bit ghc +GHCARCH:=$(shell ghc -e 'print System.Info.arch') +ifeq ($(GHCARCH),i386) +CFLAGS=-Wall -m32 +endif +endif endif PREFIX=/usr IGNORE=-ignore-package monads-fd -ignore-package monads-tf BASEFLAGS=-threaded -Wall $(IGNORE) -outputdir tmp -IUtility -DWITH_ASSISTANT -DWITH_S3 $(BASEFLAGS_OPTS) GHCFLAGS=-O2 $(BASEFLAGS) -CFLAGS=-Wall ifdef PROFILE GHCFLAGS=-prof -auto-all -rtsopts -caf-all -fforce-recomp $(BASEFLAGS) diff --git a/doc/bugs/fat_support/comment_6_a3b6000330c9c376611c228d746a1d55._comment b/doc/bugs/fat_support/comment_6_a3b6000330c9c376611c228d746a1d55._comment new file mode 100644 index 000000000..c7defd13e --- /dev/null +++ b/doc/bugs/fat_support/comment_6_a3b6000330c9c376611c228d746a1d55._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkZRoTRyW3tox-FD2DQWxskgI6_tkEtHL4" + nickname="Ben" + subject="comment 6" + date="2012-07-23T16:11:52Z" + content=""" +The above would work fine for me but the files in my annex (e.g. .git/annex/objects/xx/yy/blah.ogg) don't have extensions like that, so my media player doesn't recognize them as media files. How do I get the files under \"objects\" to keep the extensions of the original files like in Joey's example? +"""]] diff --git a/doc/bugs/fat_support/comment_7_a0ac7f2c44efc8116940c7b94b35e9d0._comment b/doc/bugs/fat_support/comment_7_a0ac7f2c44efc8116940c7b94b35e9d0._comment new file mode 100644 index 000000000..11668615e --- /dev/null +++ b/doc/bugs/fat_support/comment_7_a0ac7f2c44efc8116940c7b94b35e9d0._comment @@ -0,0 +1,7 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + subject="comment 7" + date="2012-07-24T14:51:50Z" + content=""" +You can get the extensions by migrating to the SHA1E (or SHA256E) backend. +"""]] diff --git a/doc/bugs/subtle_build_issue_on_OSX_10.7_and_Haskell_Platform___40__if_you_have_the_32bit_version_installed__41__.mdwn b/doc/bugs/subtle_build_issue_on_OSX_10.7_and_Haskell_Platform___40__if_you_have_the_32bit_version_installed__41__.mdwn index 0414823f8..3a50256db 100644 --- a/doc/bugs/subtle_build_issue_on_OSX_10.7_and_Haskell_Platform___40__if_you_have_the_32bit_version_installed__41__.mdwn +++ b/doc/bugs/subtle_build_issue_on_OSX_10.7_and_Haskell_Platform___40__if_you_have_the_32bit_version_installed__41__.mdwn @@ -46,3 +46,5 @@ Or just telling users to use the 64bit version of the haskell platform? It may also be possible to get osx's c compiler to output a universal binary to give you everything, but that be going down the _being too platform specific route_. + +> [[done]], it'll detect this and force -m32. --[[Joey]] diff --git a/doc/design/assistant/blog/day_40__dbus/comment_2_6799f2baf6a6ce14b1fa76a8402840c0._comment b/doc/design/assistant/blog/day_40__dbus/comment_2_6799f2baf6a6ce14b1fa76a8402840c0._comment new file mode 100644 index 000000000..832be854a --- /dev/null +++ b/doc/design/assistant/blog/day_40__dbus/comment_2_6799f2baf6a6ce14b1fa76a8402840c0._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="hamish" + ip="203.0.139.24" + subject="dbus vs polling " + date="2012-07-22T07:13:37Z" + content=""" +I, too, am running a dbus but like to hand mount my filesystems. However, I'd imagine that I am both a minority and that my minority could like the extra control, so perhaps even a \"re-read the mtab /now/\" command that can be manually run after something is manually mounted would suffice + +Is it not possible to use inotify on the mtab? +"""]] diff --git a/doc/design/assistant/blog/day_40__dbus/comment_3_fa1d7444bdafcb990cacf2ace7ee6ef1._comment b/doc/design/assistant/blog/day_40__dbus/comment_3_fa1d7444bdafcb990cacf2ace7ee6ef1._comment new file mode 100644 index 000000000..a372670b8 --- /dev/null +++ b/doc/design/assistant/blog/day_40__dbus/comment_3_fa1d7444bdafcb990cacf2ace7ee6ef1._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.154.4.169" + subject="comment 3" + date="2012-07-22T16:03:52Z" + content=""" +How did I not think about using my favorite hammer on this problem too? But, no, /proc/mounts cannot be watched with inotify it seems, and of course the BSDs don't seem to have a file at all. + +I think the dbus stuff is sorted out for manual users, see later blog entries. +"""]] diff --git a/doc/design/assistant/blog/day_42__the_answer.mdwn b/doc/design/assistant/blog/day_42__the_answer.mdwn new file mode 100644 index 000000000..fd7c4ebb5 --- /dev/null +++ b/doc/design/assistant/blog/day_42__the_answer.mdwn @@ -0,0 +1,27 @@ +Made the MountWatcher update state for remotes located in a drive that +gets mounted. This was tricky code. First I had to make remotes declare +when they're located in a local directory. Then it has to rescan git +configs of git remotes (because the git repo mounted at a mount point may +change), and update all the state that a newly available remote can affect. + +And it works: I plug in a drive containing one of my git remotes, and the +assistant automatically notices it and syncs the git repositories. + +--- + +But, data isn't transferred yet. When a disconnected remote becomes +connected, keys should be transferred in both directions to get back into +sync. + +To that end, added Yet Another Thread; the TransferScanner thread +will scan newly available remotes to find keys, and queue low priority +transfers to get them fully in sync. + +(Later, this will probably also be used for network remotes that become +available when moving between networks. I think network-manager sends +dbus events it could use..) + +This new thread is missing a crucial peice, it doesn't yet have a way to +find the keys that need to be transferred. Doing that efficiently (without +scanning the whole git working copy) is Hard. I'm considering design +possibilities.. diff --git a/doc/design/assistant/syncing.mdwn b/doc/design/assistant/syncing.mdwn index 5f00cf606..cc23f786f 100644 --- a/doc/design/assistant/syncing.mdwn +++ b/doc/design/assistant/syncing.mdwn @@ -3,20 +3,59 @@ all the other git clones, at both the git level and the key/value level. ## immediate action items -* At startup, and possibly periodically, look for files we have that - location tracking indicates remotes do not, and enqueue Uploads for - them. Also, enqueue Downloads for any files we're missing. +* At startup, and possibly periodically, or when the network connection + changes, or some heuristic suggests that a remote was disconnected from + us for a while, queue remotes for processing by the TransferScanner, + to queue Transfers of files it or we're missing. * After git sync, identify content that we don't have that is now available - on remotes, and transfer. But first, need to ensure that when a remote + on remotes, and transfer. (Needed when we have a uni-directional connection + to a remote, so it won't be uploading content to us.) + But first, need to ensure that when a remote receives content, and updates its location log, it syncs that update out. -* When MountWatcher detects a newly mounted drive, rescan git remotes - in order to get ones on the drive, and do a git sync and file transfers - to sync any repositories on it. + +## TransferScanner + +The TransferScanner thread needs to find keys that need to be Uploaded +to a remote, or Downloaded from it. + +How to find the keys to transfer? I'd like to avoid potentially +expensive traversals of the whole git working copy if I can. + +One way would be to do a git diff between the (unmerged) git-annex branches +of the git repo, and its remote. Parse that for lines that add a key to +either, and queue transfers. That should work fairly efficiently when the +remote is a git repository. Indeed, git-annex already does such a diff +when it's doing a union merge of data into the git-annex branch. It +might even be possible to have the union merge and scan use the same +git diff data. + +But that approach has several problems: + +1. The list of keys it would generate wouldn't have associated git + filenames, so the UI couldn't show the user what files were being + transferred. +2. Worse, without filenames, any later features to exclude + files/directories from being transferred wouldn't work. +3. Looking at a git diff of the git-annex branches would find keys + that were added to either side while the two repos were disconnected. + But if the two repos' keys were not fully in sync before they + disconnected (which is quite possible; transfers could be incomplete), + the diff would not show those older out of sync keys. + +The remote could also be a special remote. In this case, I have to either +traverse the git working copy, or perhaps traverse the whole git-annex +branch (which would have the same problems with filesnames not being +available). + +If a traversal is done, should check all remotes, not just +one. Probably worth handling the case where a remote is connected +while in the middle of such a scan, so part of the scan needs to be +redone to check it. ## longer-term TODO -* Test MountWatcher on Gnome (should work ok) and LXDE (dunno). +* Test MountWatcher on LXDE. * git-annex needs a simple speed control knob, which can be plumbed through to, at least, rsync. A good job for an hour in an airport somewhere. diff --git a/doc/design/assistant/syncing/comment_1_c70156174ff19b503978d623bd2df36f._comment b/doc/design/assistant/syncing/comment_1_c70156174ff19b503978d623bd2df36f._comment new file mode 100644 index 000000000..019490e61 --- /dev/null +++ b/doc/design/assistant/syncing/comment_1_c70156174ff19b503978d623bd2df36f._comment @@ -0,0 +1,19 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawk4YX0PWICfWGRLuncCPufMPDctT7KAYJA" + nickname="betabrain" + subject="selective data syncing" + date="2012-07-24T15:27:08Z" + content=""" +How will the assistant know which files' data to distribute between the repos? + +I'm using git-annex and it's numcopies attribute to maintain a redundant archive spread over different computers and usb drives. Not all drives should get a copy of everything, e.g. the usb drive I take to work should not automatically get a copy of family pictures. + +How about .gitattributes? + +* \* annex.auto-sync-data = false # don't automatically sync the data +* archive/ annex.auto-push-repos = NAS # everything added to archive/ in any repo goes automatically to the NAS remote. +* work/ annex.auto-synced-repos = LAPTOP WORKUSB # everything added to work/ in LAPTOP or WORKUSB gets synced to WORKUSB and LAPTOP +* work/ annex.auto-push-repos = LAPTOP WORKUSB # stuff added to work/ anywhere gets synced to LAPTOP and WORKUSB +* important/ annex.auto-sync-data = true # push data to all repos +* webserver_logs/ annex.remote.WEBSERVER.auto-push-repos = S3 # only the assistant running in WEBSERVER pushes webserver_logs/ to S3 remote +"""]] diff --git a/doc/install.mdwn b/doc/install.mdwn index 4eb7b179b..54d6ecb6b 100644 --- a/doc/install.mdwn +++ b/doc/install.mdwn @@ -9,8 +9,8 @@ * [[ArchLinux]] * [[NixOS]] * [[Gentoo]] -* Windows: [[sorry, not possible yet|todo/windows_support]] * [[ScientificLinux5]] - This should cover RHEL5 clones such as CentOS5 and so on +* Windows: [[sorry, not possible yet|todo/windows_support]] ## Using cabal diff --git a/doc/install/Fedora.mdwn b/doc/install/Fedora.mdwn index 73e9f9a5d..c1905e01b 100644 --- a/doc/install/Fedora.mdwn +++ b/doc/install/Fedora.mdwn @@ -1,4 +1,10 @@ -Installation recipe for Fedora 14 thruough 17. +git-annex is recently finding its way into Fedora. + +* [Status of getting a Fedora package](https://bugzilla.redhat.com/show_bug.cgi?id=662259) +* [Koji build for F17](http://koji.fedoraproject.org/koji/buildinfo?buildID=328654) +* [Koji build for F16](http://koji.fedoraproject.org/koji/buildinfo?buildID=328656) + +Installation recipe for Fedora 14 thruough 15. <pre> sudo yum install ghc cabal-install pcre-devel @@ -15,6 +21,3 @@ cabal install --bindir=$HOME/bin Note: You can't just use `cabal install git-annex`, because Fedora does not yet ship ghc 7.4. -* [Status of getting a Fedora package](https://bugzilla.redhat.com/show_bug.cgi?id=662259)a -* [Koji build for F17](http://koji.fedoraproject.org/koji/buildinfo?buildID=328654) -* [Koji build for F16](http://koji.fedoraproject.org/koji/buildinfo?buildID=328656) diff --git a/doc/install/OSX/comment_2_25552ff2942048fafe97d653757f1ad6._comment b/doc/install/OSX/comment_2_25552ff2942048fafe97d653757f1ad6._comment new file mode 100644 index 000000000..62ee10910 --- /dev/null +++ b/doc/install/OSX/comment_2_25552ff2942048fafe97d653757f1ad6._comment @@ -0,0 +1,7 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + subject="comment 2" + date="2012-07-24T15:09:29Z" + content=""" +I've moved some outdated comments about installing on OSX to [[old_comments]]. +"""]] diff --git a/doc/install/OSX/old_comments.mdwn b/doc/install/OSX/old_comments.mdwn new file mode 100644 index 000000000..ccb6785fd --- /dev/null +++ b/doc/install/OSX/old_comments.mdwn @@ -0,0 +1 @@ +Moved a bunch of outdated comments here, AFAIK all these issues are fixed. diff --git a/doc/install/OSX/comment_10_798000aab19af2944b6e44dbc550c6fe._comment b/doc/install/OSX/old_comments/comment_10_798000aab19af2944b6e44dbc550c6fe._comment index 675a90eee..675a90eee 100644 --- a/doc/install/OSX/comment_10_798000aab19af2944b6e44dbc550c6fe._comment +++ b/doc/install/OSX/old_comments/comment_10_798000aab19af2944b6e44dbc550c6fe._comment diff --git a/doc/install/OSX/comment_11_707a1a27a15b2de8dfc8d1a30420ab4c._comment b/doc/install/OSX/old_comments/comment_11_707a1a27a15b2de8dfc8d1a30420ab4c._comment index 69a4f9128..69a4f9128 100644 --- a/doc/install/OSX/comment_11_707a1a27a15b2de8dfc8d1a30420ab4c._comment +++ b/doc/install/OSX/old_comments/comment_11_707a1a27a15b2de8dfc8d1a30420ab4c._comment diff --git a/doc/install/OSX/comment_12_60d13f2c8e008af1041bea565a392c83._comment b/doc/install/OSX/old_comments/comment_12_60d13f2c8e008af1041bea565a392c83._comment index e2e85aaa9..e2e85aaa9 100644 --- a/doc/install/OSX/comment_12_60d13f2c8e008af1041bea565a392c83._comment +++ b/doc/install/OSX/old_comments/comment_12_60d13f2c8e008af1041bea565a392c83._comment diff --git a/doc/install/OSX/comment_13_a6f48c87c2d6eabe379d6e10a6cac453._comment b/doc/install/OSX/old_comments/comment_13_a6f48c87c2d6eabe379d6e10a6cac453._comment index e5ce62b13..e5ce62b13 100644 --- a/doc/install/OSX/comment_13_a6f48c87c2d6eabe379d6e10a6cac453._comment +++ b/doc/install/OSX/old_comments/comment_13_a6f48c87c2d6eabe379d6e10a6cac453._comment diff --git a/doc/install/OSX/comment_14_6ef2ddb7b11ce6ad54578ae118ed346e._comment b/doc/install/OSX/old_comments/comment_14_6ef2ddb7b11ce6ad54578ae118ed346e._comment index 35e0bb6ed..35e0bb6ed 100644 --- a/doc/install/OSX/comment_14_6ef2ddb7b11ce6ad54578ae118ed346e._comment +++ b/doc/install/OSX/old_comments/comment_14_6ef2ddb7b11ce6ad54578ae118ed346e._comment diff --git a/doc/install/OSX/comment_15_6fd1fad5b6d9f36620e5a0e99edd2f89._comment b/doc/install/OSX/old_comments/comment_15_6fd1fad5b6d9f36620e5a0e99edd2f89._comment index 0005328c4..0005328c4 100644 --- a/doc/install/OSX/comment_15_6fd1fad5b6d9f36620e5a0e99edd2f89._comment +++ b/doc/install/OSX/old_comments/comment_15_6fd1fad5b6d9f36620e5a0e99edd2f89._comment diff --git a/doc/install/OSX/old_comments/comment_16_af6fe3540032cdf4400478de87771058._comment b/doc/install/OSX/old_comments/comment_16_af6fe3540032cdf4400478de87771058._comment new file mode 100644 index 000000000..5da4b22c6 --- /dev/null +++ b/doc/install/OSX/old_comments/comment_16_af6fe3540032cdf4400478de87771058._comment @@ -0,0 +1,30 @@ +[[!comment format=mdwn + username="https://a-or-b.myopenid.com/" + ip="203.45.2.230" + subject="Compiling git-annex on OSX (with 32 bit Haskell)" + date="2012-07-24T03:26:45Z" + content=""" +I came across an issue when following the instructions here: + <http://git-annex.branchable.com/install/OSX/> + +I'm compiling the 'assistant' branch (522f568450a005ae81b24f63bb37e75320b51219). + + +The pre-compiled version of Haskell for OSX recommends the 32 bit installer, however git-annex compiles + +> Utility/libdiskfree.o Utility/libkqueue.o Utility/libmounts.o + +as 64 bit. The 'make' command fails on linking 32- and 64-bit code. + +So... I made a small change to the Makefile + +> CFLAGS=-Wall + +becomes + +> CFLAGS=-Wall -m32 + +I don't know if there is an easy way to programmatically check for this, or even if you'd want to spend time doing it, but it might help someone else out. + +<https://gist.github.com/3167798> +"""]] diff --git a/doc/install/OSX/old_comments/comment_17_8d3a0596db67108041728b20f2790f31._comment b/doc/install/OSX/old_comments/comment_17_8d3a0596db67108041728b20f2790f31._comment new file mode 100644 index 000000000..782f2fd75 --- /dev/null +++ b/doc/install/OSX/old_comments/comment_17_8d3a0596db67108041728b20f2790f31._comment @@ -0,0 +1,7 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + subject="comment 17" + date="2012-07-24T15:03:49Z" + content=""" +The instructions say to use cabal for a reason -- it's more likely to work. But I have made the Makefile detect the mismatched GHC and C compiler and force the C compiler to 32 bit. +"""]] diff --git a/doc/install/OSX/comment_2_0327c64b15249596add635d26f4ce67f._comment b/doc/install/OSX/old_comments/comment_2_0327c64b15249596add635d26f4ce67f._comment index 5768d8b93..5768d8b93 100644 --- a/doc/install/OSX/comment_2_0327c64b15249596add635d26f4ce67f._comment +++ b/doc/install/OSX/old_comments/comment_2_0327c64b15249596add635d26f4ce67f._comment diff --git a/doc/install/OSX/comment_3_47c682a779812dda77601c24a619923c._comment b/doc/install/OSX/old_comments/comment_3_47c682a779812dda77601c24a619923c._comment index 251375cbe..251375cbe 100644 --- a/doc/install/OSX/comment_3_47c682a779812dda77601c24a619923c._comment +++ b/doc/install/OSX/old_comments/comment_3_47c682a779812dda77601c24a619923c._comment diff --git a/doc/install/OSX/comment_4_e6109a964064a2a799768a370e57801d._comment b/doc/install/OSX/old_comments/comment_4_e6109a964064a2a799768a370e57801d._comment index be3ba2be4..be3ba2be4 100644 --- a/doc/install/OSX/comment_4_e6109a964064a2a799768a370e57801d._comment +++ b/doc/install/OSX/old_comments/comment_4_e6109a964064a2a799768a370e57801d._comment diff --git a/doc/install/OSX/comment_5_50777853f808d57b957f8ce9a0f84b3d._comment b/doc/install/OSX/old_comments/comment_5_50777853f808d57b957f8ce9a0f84b3d._comment index eca176178..eca176178 100644 --- a/doc/install/OSX/comment_5_50777853f808d57b957f8ce9a0f84b3d._comment +++ b/doc/install/OSX/old_comments/comment_5_50777853f808d57b957f8ce9a0f84b3d._comment diff --git a/doc/install/OSX/comment_6_18a8df794aa0ddd294dbf17d3d4c7fe2._comment b/doc/install/OSX/old_comments/comment_6_18a8df794aa0ddd294dbf17d3d4c7fe2._comment index 5cb813776..5cb813776 100644 --- a/doc/install/OSX/comment_6_18a8df794aa0ddd294dbf17d3d4c7fe2._comment +++ b/doc/install/OSX/old_comments/comment_6_18a8df794aa0ddd294dbf17d3d4c7fe2._comment diff --git a/doc/install/OSX/comment_7_2ce7acab15403d3f993cec94ec7f3bc6._comment b/doc/install/OSX/old_comments/comment_7_2ce7acab15403d3f993cec94ec7f3bc6._comment index 32093ee51..32093ee51 100644 --- a/doc/install/OSX/comment_7_2ce7acab15403d3f993cec94ec7f3bc6._comment +++ b/doc/install/OSX/old_comments/comment_7_2ce7acab15403d3f993cec94ec7f3bc6._comment diff --git a/doc/install/OSX/comment_8_a93ad4b67c5df4243268bcf32562f6be._comment b/doc/install/OSX/old_comments/comment_8_a93ad4b67c5df4243268bcf32562f6be._comment index cd128a6f1..cd128a6f1 100644 --- a/doc/install/OSX/comment_8_a93ad4b67c5df4243268bcf32562f6be._comment +++ b/doc/install/OSX/old_comments/comment_8_a93ad4b67c5df4243268bcf32562f6be._comment diff --git a/doc/install/OSX/comment_9_ae3ed5345bc84f57e44251d2e6c39342._comment b/doc/install/OSX/old_comments/comment_9_ae3ed5345bc84f57e44251d2e6c39342._comment index 70fbc6c3c..70fbc6c3c 100644 --- a/doc/install/OSX/comment_9_ae3ed5345bc84f57e44251d2e6c39342._comment +++ b/doc/install/OSX/old_comments/comment_9_ae3ed5345bc84f57e44251d2e6c39342._comment |