diff options
Diffstat (limited to 'doc')
| -rw-r--r-- | doc/design/external_special_remote_protocol.mdwn | 22 | ||||
| -rw-r--r-- | doc/devblog/day_238__extending_addurl_further.mdwn | 67 | ||||
| -rw-r--r-- | doc/related_software.mdwn | 4 | ||||
| -rw-r--r-- | doc/special_remotes/external.mdwn | 8 | ||||
| -rwxr-xr-x | doc/special_remotes/external/git-annex-remote-torrent | 191 |
5 files changed, 281 insertions, 11 deletions
diff --git a/doc/design/external_special_remote_protocol.mdwn b/doc/design/external_special_remote_protocol.mdwn index 072c5a1a2..cc3bfce96 100644 --- a/doc/design/external_special_remote_protocol.mdwn +++ b/doc/design/external_special_remote_protocol.mdwn @@ -131,10 +131,8 @@ replying with `UNSUPPORTED-REQUEST` is acceptable. If not, it can send `CLAIMURL-FAILURE`. * `CHECKURL Url` Asks the remote to check if the url's content can currently be downloaded - (without downloading it). If the url is not accessible, send - `CHECKURL-FAILURE`. If the url is accessible and the size is known, - send the size in `CHECKURL-SIZE`. If the url is accessible, but the size - is unknown, send `CHECKURL-SIZEUNKOWN`. + (without downloading it). The remote replies with one of `CHECKURL-FAILURE`, + `CHECKURL-CONTENTS`, or `CHECKURL-MULTI`. More optional requests may be added, without changing the protocol version, so if an unknown request is seen, reply with `UNSUPPORTED-REQUEST`. @@ -181,12 +179,18 @@ while it's handling a request. Indicates that the CLAIMURL url will be handled by this remote. * `CLAIMURL-FAILURE` Indicates that the CLAIMURL url wil not be handled by this remote. -* `CHECKURL-SIZE Size` +* `CHECKURL-CONTENTS Size|UNKNOWN Filename` + Indicates that the requested url has been verified to exist. + The Size is the size in bytes, or use "UNKNOWN" if the size could not be + determined. + The Filename can be empty (in which case a default is used), + or can specify a filename that is suggested to be used for this url. +* `CHECKURL-MULTI Url Size|UNKNOWN Filename ...` Indicates that the requested url has been verified to exist, - and its size is known. The size is in bytes. -* `CHECKURL-SIZEUNKNOWN` - Indicates that the requested url has been verified to exist, - but its size could not be determined. + and contains multiple files, which can each be accessed using + their own url. + Note that since a list is returned, neither the Url nor the Filename + can contain spaces. * `CHECKURL-FAILURE` Indicates that the requested url could not be accessed. * `UNSUPPORTED-REQUEST` diff --git a/doc/devblog/day_238__extending_addurl_further.mdwn b/doc/devblog/day_238__extending_addurl_further.mdwn new file mode 100644 index 000000000..f58bee6a5 --- /dev/null +++ b/doc/devblog/day_238__extending_addurl_further.mdwn @@ -0,0 +1,67 @@ +Some more work on the interface that lets remotes claim urls for `git annex +addurl`. Added support for remotes suggesting a filename to use when +adding an url. Also, added support for urls that result in multiple files +when downloaded. The obvious use case for that is an url to a torrent that +contains multiple files. + +Then, got `git annex importfeed` to also check if a remote claims an url. + +Finally, I put together a quick demo external remote using this new +interface. [[special_remotes/external/git-annex-remote-torrent]] +adds support for torrent files to git-annex, using aria2c to download them. +It supports multi-file torrents, but not magnet links. (I'll probably +rewrite this more robustly and efficiently in haskell sometime soon.) + +Here's a demo: + +<pre> +# git annex initremote torrent type=external encryption=none externaltype=torrent +initremote torrent ok +(Recording state in git...) +# ls +# git annex addurl --fast file:///home/joey/my.torrent + % Total % Received % Xferd Average Speed Time Time Time Current + Dload Upload Total Spent Left Speed +100 198 100 198 0 0 3946k 0 --:--:-- --:--:-- --:--:-- 3946k +addurl _home_joey_my.torrent/bar (using torrent) ok +addurl _home_joey_my.torrent/baz (using torrent) ok +addurl _home_joey_my.torrent/foo (using torrent) ok +(Recording state in git...) +# ls _home_joey_my.torrent/ +bar@ baz@ foo@ +# git annex get _home_joey_my.torrent/baz +get _home_joey_my.torrent/baz (from torrent...) + % Total % Received % Xferd Average Speed Time Time Time Current + Dload Upload Total Spent Left Speed + 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-100 198 100 198 0 0 3580k 0 --:--:-- --:--:-- --:--:-- 3580k + +12/11 18:14:56 [NOTICE] IPv4 DHT: listening on UDP port 6946 + +12/11 18:14:56 [NOTICE] IPv4 BitTorrent: listening on TCP port 6961 + +12/11 18:14:56 [NOTICE] IPv6 BitTorrent: listening on TCP port 6961 + +12/11 18:14:56 [NOTICE] Seeding is over. +12/11 18:14:57 [NOTICE] Download complete: /home/joey/tmp/tmp.Le89hJSXyh/tor + +12/11 18:14:57 [NOTICE] Your share ratio was 0.0, uploaded/downloaded=0B/0B + +Download Results: +gid |stat|avg speed |path/URI +======+====+===========+======================================================= +71f6b6|OK | 0B/s|/home/joey/tmp/tmp.Le89hJSXyh/tor/baz + +Status Legend: +(OK):download completed. +ok +(Recording state in git...) +# git annex find +_home_joey_my.torrent/baz +# git annex whereis _home_joey_my.torrent/baz +whereis _home_joey_my.torrent/baz (2 copies) + 1878241d-ee49-446d-8cce-041c46442d94 -- [torrent] + 52412020-2bb3-4aa4-ae16-0da22ba48875 -- joey@darkstar:~/tmp/repo [here] + + torrent: file:///home/joey/my.torrent#2 +ok +</pre> diff --git a/doc/related_software.mdwn b/doc/related_software.mdwn index 1e5c0f6b0..f2b8e601b 100644 --- a/doc/related_software.mdwn +++ b/doc/related_software.mdwn @@ -15,3 +15,7 @@ designed to interoperate with it. [an extension](https://github.com/magit/magit-annex) for git annex. * [DataLad](http://datalad.org/) uses git-annex to provide access to scientific data available from various sources. +* The [Baobáxia](https://github.com/RedeMocambos/baobaxia) project + built by the Brazilian [Mocambos network](http://www.mocambos.net/) + is [using git-annex to connect isolated communities](http://www.modspil.dk/itpolitik/baob_xia.html). + Repositories sync over satellite internet and/or sneakernet. diff --git a/doc/special_remotes/external.mdwn b/doc/special_remotes/external.mdwn index d96999693..010f2a17b 100644 --- a/doc/special_remotes/external.mdwn +++ b/doc/special_remotes/external.mdwn @@ -14,14 +14,18 @@ It's not hard! * Install it in PATH. * When the user runs `git annex initremote foo type=external externaltype=$bar`, it will use your program. +* See [[design/external_special_remote_protocol]] for what the program + needs to do. There's an example at the end of this page. * If things don't seem to work, pass `--debug` and you'll see, amoung other things, a transcript of git-annex's communication with your program. * If you build a new special remote, please add it to the list of [[special_remotes]]. +Here's an example of using an external special remote to add torrent +support to git-annex: [[external/git-annex-remote-torrent]] + Here's a simple shell script example, which can easily be adapted to run whatever commands you need. Or better, re-written in some better -language of your choice. See [[design/external_special_remote_protocol]] -for the details. +language of your choice. [[!inline pages="special_remotes/external/example.sh" feeds=no]] diff --git a/doc/special_remotes/external/git-annex-remote-torrent b/doc/special_remotes/external/git-annex-remote-torrent new file mode 100755 index 000000000..4f5e62a90 --- /dev/null +++ b/doc/special_remotes/external/git-annex-remote-torrent @@ -0,0 +1,191 @@ +#!/bin/sh +# This is a demo git-annex external special remote program, +# which adds basic torrent download support to git-annex. +# +# Uses aria2c. Also needs the original bittorrent (or bittornado) for the +# btshowmetainfo command. +# +# Install in PATH as git-annex-remote-torrent +# +# Enable remote by running: +# git annex initremote torrent type=external encryption=none externaltype=torrent +# git annex untrust torrent +# +# Copyright 2014 Joey Hess; licenced under the GNU GPL version 3 or higher. + +set -e + +# This program speaks a line-based protocol on stdin and stdout. +# When running any commands, their stdout should be redirected to stderr +# (or /dev/null) to avoid messing up the protocol. +runcmd () { + "$@" >&2 +} + +# Gets a VALUE response and stores it in $RET +getvalue () { + read resp + # Tricky POSIX shell code to split first word of the resp, + # preserving all other whitespace + case "${resp%% *}" in + VALUE) + RET="$(echo "$resp" | sed 's/^VALUE \?//')" + ;; + *) + RET="" + ;; + esac +} + +# Get a list of all known torrent urls for a key, +# storing it in a temp file. +geturls () { + key="$1" + tmp="$2" + + echo GETURLS "$key" + getvalue + while [ -n "$RET" ]; do + if istorrent "$RET"; then + echo "$RET" >> "$tmp" + fi + getvalue + done +} + +# Does the url end in .torrent? +# Note that we use #N on the url to indicate which file +# from a multi-file torrent is wanted. +istorrent () { + echo "$1" | egrep -q "\.torrent(#.*)?$" +} + +# Download a single file from a torrent. +# +# Note: Does not support resuming interrupted transfers. +# Note: Does not feed progress info back to git-annex, and since +# the destination file is only populated at the end, git-annex will fail +# to display a progress bar for this download. +downloadtorrent () { + torrent="$1" + n="$2" + dest="$3" + + tmpdir="$(mktemp -d)" + if ! runcmd aria2c --select-file="$n" "$torrent" -d "$tmpdir"; then + false + fi + + # aria2c will create part of the directory structure + # contained in the torrent. It may download parts of other files + # in addition to the one we asked for. So, we need to find + # out the filename we want, and look for it. + wantdir="$(btshowmetainfo "$torrent" | grep "^directory name: " | sed "s/^directory name: //")" + wantfile="$(btshowmetainfo "$tmp" | grep '^ ' | sed 's/^ //' | head -n "$n" | tail -n 1 | sed 's/ ([0-9]*)$//')" + if [ -e "$tmpdir/$wantdir/$wantfile" ]; then + mv "$tmpdir/$wantdir/$wantfile" "$dest" + rm -rf "$tmpdir" + else + rm -rf "$tmpdir" + false + fi +} + +# This has to come first, to get the protocol started. +echo VERSION 1 + +while read line; do + set -- $line + case "$1" in + INITREMOTE) + echo INITREMOTE-SUCCESS + ;; + PREPARE) + echo PREPARE-SUCCESS + ;; + CLAIMURL) + url="$2" + if istorrent "$url"; then + echo CLAIMURL-SUCCESS + else + echo CLAIMURL-FAILURE + fi + ;; + CHECKURL) + url="$2" + # List contents of torrent. + tmp=$(mktemp) + if ! runcmd curl -o "$tmp" "$url"; then + echo CHECKURL-FAILURE + else + oldIFS="$IFS" + IFS=" +" + printf "CHECKURL-MULTI" + n=0 + for l in $(btshowmetainfo "$tmp" | grep '^ ' | sed 's/^ //'); do + # Note that the file cannot contain spaces. + file="$(echo "$l" | sed 's/ ([0-9]*)$//' | sed 's/ /_/g')" + size="$(echo "$l" | sed 's/.* (\([0-9]*\))$/\1/')" + n=$(expr $n + 1) + printf " $url#$n $size $file" + done + printf "\n" + IFS="$oldIFS" + fi + rm -f "$tmp" + ;; + TRANSFER) + key="$3" + file="$4" + case "$2" in + STORE) + runcmd echo "upload not supported" + echo TRANSFER-FAILURE STORE "$key" + ;; + RETRIEVE) + urltmp=$(mktemp) + geturls "$key" "$urltmp" + url="$(head "$urltmp")" || true + rm -f "$urltmp" + if [ -z "$url" ]; then + echo TRANSFER-FAILURE RETRIEVE "$key" "no known torrent urls for this key" + else + tmp=$(mktemp) + if ! runcmd curl -o "$tmp" "$url"; then + echo TRANSFER-FAILURE RETRIEVE "$key" "failed downloading torrent file from $url" + else + filenum="$(echo "$url" | sed 's/.*#\(\d*\)/\1/')" + if downloadtorrent "$tmp" "$filenum" "$file"; then + echo TRANSFER-SUCCESS RETRIEVE "$key" + else + echo TRANSFER-FAILURE RETRIEVE "$key" "failed to download torrent contents from $url" + fi + fi + rm -f "$tmp" + fi + ;; + esac + ;; + CHECKPRESENT) + key="$2" + # Let's just assume that torrents are always present + # for simplicity. + echo CHECKPRESENT-SUCCESS "$key" + ;; + REMOVE) + key="$2" + # Remove all torrent urls for the key. + tmp=$(mktemp) + geturls "$key" "$tmp" + for url in $(cat "$tmp"); do + echo SETURLMISSING "$key" "$url" + done + rm -f "$tmp" + echo REMOVE-SUCCESS "$key" + ;; + *) + echo UNSUPPORTED-REQUEST + ;; + esac +done |