diff options
20 files changed, 213 insertions, 15 deletions
diff --git a/BuildFlags.hs b/BuildFlags.hs index a0f0ac298..2e8d05ea3 100644 --- a/BuildFlags.hs +++ b/BuildFlags.hs @@ -83,8 +83,11 @@ buildFlags = filter (not . null) #endif #ifdef WITH_TORRENTPARSER , "TorrentParser" +#endif +#ifdef WITH_DATABASE + , "Database" #else - +#warning Building without Database support #endif #ifdef WITH_EKG , "EKG" diff --git a/Command/Fsck.hs b/Command/Fsck.hs index 0e0c49d78..a201c4519 100644 --- a/Command/Fsck.hs +++ b/Command/Fsck.hs @@ -30,12 +30,15 @@ import Annex.UUID import Utility.DataUnits import Config import Types.Key -import Types.CleanupActions import Utility.HumanTime import Utility.CopyFile import Git.FilePath import Utility.PID + +#ifdef WITH_DATABASE import qualified Database.Fsck as FsckDb +import Types.CleanupActions +#endif import Data.Time.Clock.POSIX import System.Posix.Types (EpochTime) @@ -91,7 +94,7 @@ seek o = do (\k -> startKey i k =<< getNumCopies) (withFilesInGit $ whenAnnexed $ start from i) (fsckFiles o) - withFsckDb i FsckDb.closeDb + cleanupIncremental i void $ tryIO $ recordActivity Fsck u start :: Maybe Remote -> Incremental -> FilePath -> Key -> CommandStart @@ -454,16 +457,24 @@ runFsck inc file key a = ifM (needFsck inc key) {- Check if a key needs to be fscked, with support for incremental fscks. -} needFsck :: Incremental -> Key -> Annex Bool +#ifdef WITH_DATABASE needFsck (ContIncremental h) key = liftIO $ not <$> FsckDb.inDb h key +#endif needFsck _ _ = return True +#ifdef WITH_DATABASE withFsckDb :: Incremental -> (FsckDb.FsckHandle -> Annex ()) -> Annex () withFsckDb (ContIncremental h) a = a h withFsckDb (StartIncremental h) a = a h withFsckDb NonIncremental _ = noop +#endif recordFsckTime :: Incremental -> Key -> Annex () +#ifdef WITH_DATABASE recordFsckTime inc key = withFsckDb inc $ \h -> liftIO $ FsckDb.addDb h key +#else +recordFsckTime _ _ = return () +#endif {- Records the start time of an incremental fsck. - @@ -512,10 +523,16 @@ getStartTime u = do fromfile >= fromstatus #endif -data Incremental = StartIncremental FsckDb.FsckHandle | ContIncremental FsckDb.FsckHandle | NonIncremental +data Incremental + = NonIncremental +#ifdef WITH_DATABASE + | StartIncremental FsckDb.FsckHandle + | ContIncremental FsckDb.FsckHandle +#endif prepIncremental :: UUID -> Maybe IncrementalOpt -> Annex Incremental prepIncremental _ Nothing = pure NonIncremental +#ifdef WITH_DATABASE prepIncremental u (Just StartIncrementalO) = do recordStartTime u ifM (FsckDb.newPass u) @@ -537,3 +554,13 @@ prepIncremental u (Just (ScheduleIncrementalO delta)) = do prepIncremental u $ Just $ case started of Nothing -> StartIncrementalO Just _ -> MoreIncrementalO +#else +prepIncremental _ _ = error "This git-annex was not built with database support; incremental fsck not supported" +#endif + +cleanupIncremental :: Incremental -> Annex () +#ifdef WITH_DATABASE +cleanupIncremental i = withFsckDb i FsckDb.closeDb +#else +cleanupIncremental _ = return () +#endif diff --git a/debian/changelog b/debian/changelog index a3a261c9a..2c3b509e5 100644 --- a/debian/changelog +++ b/debian/changelog @@ -31,6 +31,8 @@ git-annex (5.20150714) UNRELEASED; urgency=medium * Adjust debian build deps: The webapp can now build on arm64, s390x and hurd-i386. WebDAV support is also available on those architectures. * Debian package now maintained by Richard Hartmann. + * Support building without persistent database on for systems that + lack TH. This removes support for incremental fsck. -- Joey Hess <id@joeyh.name> Fri, 10 Jul 2015 16:36:42 -0400 diff --git a/debian/control b/debian/control index e26e1dc57..860724667 100644 --- a/debian/control +++ b/debian/control @@ -43,6 +43,10 @@ Build-Depends: libghc-wai-dev [i386 amd64 armel armhf armhf kfreebsd-i386 kfreebsd-amd64 powerpc ppc64el s390x hurd-i386], libghc-wai-extra-dev [i386 amd64 armel armhf armhf kfreebsd-i386 kfreebsd-amd64 powerpc ppc64el s390x hurd-i386], libghc-dav-dev (>= 1.0) [i386 amd64 arm64 armel armhf kfreebsd-i386 kfreebsd-amd64 powerpc ppc64el s390x hurd-i386], + libghc-persistent-dev [i386 amd64 arm64 armel armhf kfreebsd-i386 kfreebsd-amd64 powerpc ppc64el s390x hurd-i386], + libghc-persistent-template-dev [i386 amd64 arm64 armel armhf kfreebsd-i386 kfreebsd-amd64 powerpc ppc64el s390x hurd-i386], + libghc-persistent-sqlite-dev [i386 amd64 arm64 armel armhf kfreebsd-i386 kfreebsd-amd64 powerpc ppc64el s390x hurd-i386], + libghc-esqueleto-dev [i386 amd64 arm64 armel armhf kfreebsd-i386 kfreebsd-amd64 powerpc ppc64el s390x hurd-i386], libghc-securemem-dev, libghc-byteable-dev, libghc-dns-dev, @@ -58,10 +62,6 @@ Build-Depends: libghc-gnutls-dev (>= 0.1.4), libghc-xml-types-dev, libghc-async-dev, - libghc-persistent-dev, - libghc-persistent-template-dev, - libghc-persistent-sqlite-dev, - libghc-esqueleto-dev, libghc-monad-logger-dev, libghc-feed-dev (>= 0.3.9.2), libghc-regex-tdfa-dev, diff --git a/doc/bugs/Default_startup_command/comment_1_2d63fb301586c9a97ceff1d1b9fafbee._comment b/doc/bugs/Default_startup_command/comment_1_2d63fb301586c9a97ceff1d1b9fafbee._comment new file mode 100644 index 000000000..47487f01a --- /dev/null +++ b/doc/bugs/Default_startup_command/comment_1_2d63fb301586c9a97ceff1d1b9fafbee._comment @@ -0,0 +1,29 @@ +[[!comment format=mdwn + username="git-annex@53a027ebda399714df9272a135f22ac774f3c9f1" + nickname="git-annex" + subject="i can second this" + date="2015-07-26T10:29:32Z" + content=""" +I have the same behaviour on CM11 4.4.4 on a Moto G as well. +When i installed git annex i would be greeted by the same message, but then could start the webapp from the menu successfully. +i then proceeded to setup my repo, which synced flawlessly, if i had started the webapp as stated above. +however my files were never automatically synced when just starting the app (terminal window). + +Therefore i resorted to \"cd\" to my repo and \"git annex add *\" and \"git annex sync --content\" every time i wanted to sync. + +i tried changing the startup command to \"git annex assistant --autostart\", \"git annex assistant --autostart --foreground\", \"cd ../DCIM;git annex sync --content\", \"echo test\" but nothing seems to start (not quite sure how to check for that). + +for my shell i had the default/data/data/ga.androidterm/lib/lib.start.so and /data/data/ga.androidterm/runshell if that makes any difference? + +not sure if that's related +when trying to manually invoke \"git annex --assistant [--foreground]\" i get: + + git annex autostart in /sdcard/DCIM + usage: ionice <pid> [none|rt|be|idle] [prio] + failed + +and back to prompt. Not sure if this is CM(11) only? + +my git annex version is: 5.20150527-gfc92f13 for android 4.3 and 4.4. if you need more information i'll be happy to provide it, to get this out of the way :D + +"""]] diff --git a/doc/bugs/Default_startup_command/comment_2_296de7f38161ebc21da954d46c924c98._comment b/doc/bugs/Default_startup_command/comment_2_296de7f38161ebc21da954d46c924c98._comment new file mode 100644 index 000000000..af5cfc44d --- /dev/null +++ b/doc/bugs/Default_startup_command/comment_2_296de7f38161ebc21da954d46c924c98._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="git-annex@53a027ebda399714df9272a135f22ac774f3c9f1" + nickname="git-annex" + subject="small correction to the above" + date="2015-07-26T10:33:28Z" + content=""" +i can not start the webapp via the menu (firefox opens but cant connect to the webapp) but if i do git annex webapp manually in the terminal it works as expected +"""]] diff --git a/doc/bugs/OSX:_Assistant_leaves_repo_in_inconsistent_state/comment_5_4343ee35e4091fc50268f9fc611f5148._comment b/doc/bugs/OSX:_Assistant_leaves_repo_in_inconsistent_state/comment_5_4343ee35e4091fc50268f9fc611f5148._comment new file mode 100644 index 000000000..0ec137081 --- /dev/null +++ b/doc/bugs/OSX:_Assistant_leaves_repo_in_inconsistent_state/comment_5_4343ee35e4091fc50268f9fc611f5148._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="Marco" + subject="How to debug?" + date="2015-07-25T18:10:08Z" + content=""" +I want to give the two assistants one more try with the newest version. + +But I want the assistant not to try a repair once a failure is detected. I want to check the state of the repository to get a clue what goes wrong. +As I understood the code there is no configuration to prevent the repair. Am I wrong? If not I could try to introduce one if it makes sense. +"""]] diff --git a/doc/bugs/metadata_view_does_not_vpop_to_original_view/comment_8_ef3e8e1c47984fc674e58646052205ef._comment b/doc/bugs/metadata_view_does_not_vpop_to_original_view/comment_8_ef3e8e1c47984fc674e58646052205ef._comment new file mode 100644 index 000000000..df5515ff1 --- /dev/null +++ b/doc/bugs/metadata_view_does_not_vpop_to_original_view/comment_8_ef3e8e1c47984fc674e58646052205ef._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="frederik@ffbea6a549cb3f460d110386c0f634c1ddc6a68a" + nickname="frederik" + subject="Any more info needed" + date="2015-07-27T13:47:11Z" + content=""" +Hi Joey, while I was able to recover from the issue mentioned above, I have yet to try using the metadata views again out of uncertainty about the root cause of the problem I encountered. Is there anything more I could provide to determine what caused this problem and how to avoid it next time I try? +"""]] diff --git a/doc/bugs/rsync_remote_with_-J2_fails.mdwn b/doc/bugs/rsync_remote_with_-J2_fails.mdwn new file mode 100644 index 000000000..93b317694 --- /dev/null +++ b/doc/bugs/rsync_remote_with_-J2_fails.mdwn @@ -0,0 +1,55 @@ +### Please describe the problem. + +Trying to `git annex copy` to an rsync special remote with the -J flag fails with a lot of errors. (And is slow without it). + +### What steps will reproduce the problem? + +$ git annex copy --to rsync-remote -J2 + +### What version of git-annex are you using? On what operating system? + +5.20150710-g8fd7052 on Ubuntu. + +### Please provide any additional information below. + +[[!format sh """ +$ git annex copy --to freenas-rsync -J2 +E: file has vanished: "/media/depot/annex/.git/annex/tmp/rsynctmp/29637/caa/2d1/SHA256E-s4244--4ec1ea09c7605a589a9bf6cd927e96737c512d17d8053cbfaf0dcd364702400c.txt/SHA256E-s4244--4ec1ea09c7605a589a9bf6cd927e96737c512d17d8053cbfaf0dcd364702400c.txt" +E: rsync warning: some files vanished before they could be transferred (code 24) at main.c(1183) [sender=3.1.1] +E: rsync: change_dir "/media/depot/annex//.git/annex/tmp/rsynctmp/29637" failed: No such file or directory (2) +E: rsync error: some files/attrs were not transferred (see previous errors) (code 23) at main.c(1183) [sender=3.1.1] + .git/annex/tmp/rsynctmp/29637/c17: removeDirectory: does not exist (No such file or directory) + .git/annex/tmp/rsynctmp/29637: getDirectoryContents: does not exist (No such file or directory) + .git/annex/tmp/rsynctmp/29637/67e: removeLink: does not exist (No such file or directory) + .git/annex/tmp/rsynctmp/29637: getDirectoryContents: does not exist (No such file or directory) + .git/annex/tmp/rsynctmp/29637: getDirectoryContents: does not exist (No such file or directory) + .git/annex/tmp/rsynctmp/29637/49e/833: removeLink: inappropriate type (Is a directory) +^C + +J1 hangs a long time: +$ git annex copy --to freenas-rsync -J1 +^C + +Works without J flag: +$ git annex copy --to freenas-rsync +copy GarageBand/._.DS_Store (checking freenas-rsync...) ok +copy GarageBand/._My Song.band (checking freenas-rsync...) ok +copy GarageBand/._code monkey.band (checking freenas-rsync...) ok +copy GarageBand/._first of may.band (checking freenas-rsync...) ok + +Version: +$ git annex version +git-annex version: 5.20150710-g8fd7052 +build flags: Assistant Webapp Webapp-secure Pairing Testsuite S3 WebDAV Inotify DBus DesktopNotify XMPP DNS Feeds Quvi TDFA +key/value backends: SHA256E SHA1E SHA512E SHA224E SHA384E SKEIN256E SKEIN512E MD5E SHA256 SHA1 SHA512 SHA224 SHA384 SKEIN256 SKEIN512 MD5 WORM URL +remote types: git gcrypt S3 bup directory rsync web bittorrent webdav tahoe glacier ddar hook external +local repository version: 5 +supported repository version: 5 +upgrade supported from repository versions: 0 1 2 4 + +Repo config: +[remote "freenas-rsync"] + annex-rsyncurl = freenas:/mnt/tank/home/jnewsome/annex-rsync-backend + annex-uuid = f05581cc-7236-41ed-9db8-49424f863307 + +"""]] diff --git a/doc/bugs/s3_InternalIOException__63__/comment_5_134d1d0ad9f59a2b7498d1ed335cf91a._comment b/doc/bugs/s3_InternalIOException__63__/comment_5_134d1d0ad9f59a2b7498d1ed335cf91a._comment new file mode 100644 index 000000000..59f6c1bd0 --- /dev/null +++ b/doc/bugs/s3_InternalIOException__63__/comment_5_134d1d0ad9f59a2b7498d1ed335cf91a._comment @@ -0,0 +1,9 @@ +[[!comment format=mdwn + username="anarcat" + subject="comment 5" + date="2015-07-23T19:44:10Z" + content=""" +any update of this? + +i still can't upload those files, while *other* files get uploaded to S3 fine. really strange. how can i reset the state of this? +"""]] diff --git a/doc/forum/Exclude_specific_files_from_a_special_remote/comment_2_671790686bd4724aadcadd8d8aad2c98._comment b/doc/forum/Exclude_specific_files_from_a_special_remote/comment_2_671790686bd4724aadcadd8d8aad2c98._comment new file mode 100644 index 000000000..8ad63b91b --- /dev/null +++ b/doc/forum/Exclude_specific_files_from_a_special_remote/comment_2_671790686bd4724aadcadd8d8aad2c98._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="Oliv5@43a03b767570a41ba84b9e5c72f3c9816adb22e2" + nickname="Oliv5" + subject="comment 2" + date="2015-07-23T19:07:58Z" + content=""" +Yes, this is exactly what I need. It works well indeed. Thks. I didnt understand how this worked before. +"""]] diff --git a/doc/forum/Full_restore_from_an_encrypted_special_remote.mdwn b/doc/forum/Full_restore_from_an_encrypted_special_remote.mdwn new file mode 100644 index 000000000..d724929b0 --- /dev/null +++ b/doc/forum/Full_restore_from_an_encrypted_special_remote.mdwn @@ -0,0 +1,3 @@ +If I lose all my git-annex repositories is it possible to restore the content of a repo from a special remote in which encryption was enabled? I know git-annex isn't designed to be a backup solution but more for the archive and/or nomad use case. I do like the idea of being able to restore the files from an S3 special remote though. I imagine part of the answer to this question has to do with what type of encryption is used (hybrid and shared key rely on information stored in the repo but pubkey doesn't rely on the repo at all) and would probably be easier if there were an option to disable hashing the filenames of the files (less secure, but maybe ok in some cases). + +Thoughts? diff --git a/doc/forum/Windows_-_You_don__39__t_have_access/comment_6_7656eb72bb9e39db59092557c57a2489._comment b/doc/forum/Windows_-_You_don__39__t_have_access/comment_6_7656eb72bb9e39db59092557c57a2489._comment new file mode 100644 index 000000000..c78501a87 --- /dev/null +++ b/doc/forum/Windows_-_You_don__39__t_have_access/comment_6_7656eb72bb9e39db59092557c57a2489._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="fusionx86@2cab672ef75a8502e153ceb90177dde38985dba9" + nickname="fusionx86" + subject="comment 6" + date="2015-07-24T18:10:20Z" + content=""" +Any other ideas on this problem? I was planning to use annex for large/binary files, but we have a lot of developers that would need to run it on Windows as well and I'm not sure how to roll it out when I can't get it working myself on a Windows build server. I think I saw somewhere that you have tested it on WinXP. Any chance you could download an evaluation copy of a later version (Win7/8.1/2008R2/2012R2) and try again? Annex is a great tool and would like to incorporate it into our SCM and deployment workflow if possible. Thanks. +"""]] diff --git a/doc/forum/git-annex_does_not_protect_files_on_NTFS-Fuse.mdwn b/doc/forum/git-annex_does_not_protect_files_on_NTFS-Fuse.mdwn new file mode 100644 index 000000000..e8084f6ff --- /dev/null +++ b/doc/forum/git-annex_does_not_protect_files_on_NTFS-Fuse.mdwn @@ -0,0 +1,8 @@ +I've read that git-annex probes the host filesystem to determine whether it has the necessary features for an indirect annex. Indirect annexes are supposed to protect annexed files from accidental editing: + + # echo oops > my_cool_big_file + bash: my_cool_big_file: Permission denied + +I have an NTFS drive to share files between my Windows and Linux systems (dual boot). On Linux fuse sets the file permissions to rwx for the user and nothing for the rest, and this cannot be changed. Files in the annex can be modified as in the above example. + +If git-annex detects whether a fs can handle indirect annexes or not, I suggest checking for this case if possible. diff --git a/doc/git-annex-importfeed.mdwn b/doc/git-annex-importfeed.mdwn index 26401f4e5..241d369af 100644 --- a/doc/git-annex-importfeed.mdwn +++ b/doc/git-annex-importfeed.mdwn @@ -15,7 +15,7 @@ them. When quvi is installed, links in the feed are tested to see if they are on a video hosting site, and the video is downloaded. This allows -importing e.g., youtube playlists. +importing e.g., YouTube playlists. To make the import process add metadata to the imported files from the feed, `git config annex.genmetadata true` diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index 73894c0d8..a78072808 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -1059,7 +1059,7 @@ Here are all the supported configuration settings. * `remote.<name>.annex-rsync-options` Options to use when using rsync - to or from this remote. For example, to force ipv6, and limit + to or from this remote. For example, to force IPv6, and limit the bandwidth to 100Kbyte/s, set it to `-6 --bwlimit 100` * `remote.<name>.annex-rsync-upload-options` @@ -1110,7 +1110,7 @@ Here are all the supported configuration settings. * `annex.web-options` Options to pass when running wget or curl. - For example, to force ipv4 only, set it to "-4" + For example, to force IPv4 only, set it to "-4" * `annex.quvi-options` diff --git a/doc/install/ArchLinux/comment_7_97d611f1ae6d4fbe91f84f9fe739f368._comment b/doc/install/ArchLinux/comment_7_97d611f1ae6d4fbe91f84f9fe739f368._comment new file mode 100644 index 000000000..31f153b02 --- /dev/null +++ b/doc/install/ArchLinux/comment_7_97d611f1ae6d4fbe91f84f9fe739f368._comment @@ -0,0 +1,13 @@ +[[!comment format=mdwn + username="dropped" + subject="Correct way to install git-annex directly through cabal." + date="2015-07-27T12:01:34Z" + content=""" +The effective way to install git-annex and resolve dependencies: + + $ sudo pacman -S gsasl git rsync curl wget gnupg openssh cabal-install + $ cabal update + $ cabal install c2hs + $ cabal install git-annex --bindir=$HOME/bin + $ sudo cp ~/bin/git-annex /usr/lib/git-core +"""]] diff --git a/doc/preferred_content.mdwn b/doc/preferred_content.mdwn index 33d327e9f..e6244bde5 100644 --- a/doc/preferred_content.mdwn +++ b/doc/preferred_content.mdwn @@ -98,7 +98,7 @@ below to get the full story. copies, on remotes in the specified group. For example, `copies=archive:2` - Preferred content expressions have no equivilant to the `--in` + Preferred content expressions have no equivalent to the `--in` option, but groups can accomplish similar things. You can add repositories to groups, and match against the groups in a preferred content expression. So rather than `--in=usbdrive`, diff --git a/doc/publicrepos.mdwn b/doc/publicrepos.mdwn index 891c52fd7..e54e0b99a 100644 --- a/doc/publicrepos.mdwn +++ b/doc/publicrepos.mdwn @@ -30,8 +30,8 @@ the public repositories that you can clone to try out git-annex. * [ccc-media-annex](https://github.com/ntnn/ccc-media-annex) `git clone https://github.com/ntnn/ccc-media-annex` - git-annex repository using [the CDN](http://media.ccc.de/) of the german [CCC](http://www.ccc.de/). - Contains a lot of talks (mostly in german) held on events from the CCC as well as other stuff. + git-annex repository using [the CDN](http://media.ccc.de/) of the German [CCC](http://www.ccc.de/). + Contains a lot of talks (mostly in German) held on events from the CCC as well as other stuff. This is a wiki -- add your own public repository to the list! See [[tips/centralized_git_repository_tutorial]]. diff --git a/git-annex.cabal b/git-annex.cabal index 5dde20bfe..13c0760e2 100644 --- a/git-annex.cabal +++ b/git-annex.cabal @@ -105,6 +105,10 @@ Flag network-uri Description: Get Network.URI from the network-uri package Default: True +Flag Database + Description: Enable building with persistent for database use (disable to build on platforms not supporting TH) + Default: True + Executable git-annex Main-Is: git-annex.hs Build-Depends: @@ -123,7 +127,6 @@ Executable git-annex monad-control, transformers, bloomfilter, edit-distance, resourcet, http-conduit, http-types, - esqueleto, persistent-sqlite, persistent, persistent-template, time, old-locale CC-Options: -Wall GHC-Options: -Wall -fno-warn-tabs @@ -262,6 +265,10 @@ Executable git-annex Build-Depends: torrent (>= 10000.0.0) CPP-Options: -DWITH_TORRENTPARSER + if flag(Database) + Build-Depends: esqueleto, persistent-sqlite, persistent, persistent-template + CPP-Options: -DWITH_DATABASE + if flag(AsciiProgress) Build-Depends: ascii-progress (<= 0.2.1.2), terminal-size CPP-Options: -DWITH_ASCIIPROGRESS |