diff options
| -rw-r--r-- | Makefile | 3 | ||||
| -rw-r--r-- | performance-test/.gitignore | 1 | ||||
| -rw-r--r-- | performance-test/Makefile | 7 | ||||
| -rw-r--r-- | performance-test/Makefile.local | 32 | ||||
| -rw-r--r-- | performance-test/README | 49 | ||||
| -rwxr-xr-x | performance-test/basic | 15 | ||||
| -rw-r--r-- | performance-test/download/.gitignore | 2 | ||||
| -rw-r--r-- | performance-test/download/notmuch-email-corpus-0.2.tar.xz.asc | 9 | ||||
| -rwxr-xr-x | performance-test/notmuch-perf-test | 25 | ||||
| -rw-r--r-- | performance-test/perf-test-lib.sh | 67 | ||||
| -rw-r--r-- | performance-test/version.sh | 3 |
11 files changed, 212 insertions, 1 deletions
@@ -3,7 +3,8 @@ all: # List all subdirectories here. Each contains its own Makefile.local -subdirs = compat completion emacs lib man parse-time-string util test +subdirs := compat completion emacs lib man parse-time-string +subdirs += performance-test util test # We make all targets depend on the Makefiles themselves. global_deps = Makefile Makefile.config Makefile.local \ diff --git a/performance-test/.gitignore b/performance-test/.gitignore new file mode 100644 index 00000000..53f26973 --- /dev/null +++ b/performance-test/.gitignore @@ -0,0 +1 @@ +tmp.*/ diff --git a/performance-test/Makefile b/performance-test/Makefile new file mode 100644 index 00000000..de492a7c --- /dev/null +++ b/performance-test/Makefile @@ -0,0 +1,7 @@ +# See Makefile.local for the list of files to be compiled in this +# directory. +all: + $(MAKE) -C .. all + +.DEFAULT: + $(MAKE) -C .. $@ diff --git a/performance-test/Makefile.local b/performance-test/Makefile.local new file mode 100644 index 00000000..5d2acbd7 --- /dev/null +++ b/performance-test/Makefile.local @@ -0,0 +1,32 @@ +# -*- makefile -*- + +dir := performance-test + +include $(dir)/version.sh + +CORPUS_NAME := notmuch-email-corpus-$(PERFTEST_VERSION).tar.xz +TXZFILE := ${dir}/download/${CORPUS_NAME} +SIGFILE := ${TXZFILE}.asc +TEST_SCRIPT := ${dir}/notmuch-perf-test +DEFAULT_URL := http://notmuchmail.org/releases/${CORPUS_NAME} + +perf-test: setup-perf-test all + $(TEST_SCRIPT) $(OPTIONS) + +.PHONY: download-corpus setup-perf-test + +# Note that this intentionally does not depend on download-corpus. +setup-perf-test: $(TXZFILE) + gpg --verify $(SIGFILE) + +$(TXZFILE): + @printf "\nPlease download ${TXZFILE} using:\n\n" + @printf "\t%% make download-corpus\n\n" + @echo or see http://notmuchmail.org/corpus for download locations + @echo + @false + +download-corpus: + wget -O ${TXZFILE} ${DEFAULT_URL} + +CLEAN := $(CLEAN) $(dir)/tmp.* diff --git a/performance-test/README b/performance-test/README new file mode 100644 index 00000000..630cf136 --- /dev/null +++ b/performance-test/README @@ -0,0 +1,49 @@ +Pre-requisites +-------------- + +In addition to having notmuch, you need: + +- gpg +- gnu tar +- gnu time +- xz. Some speedup can be gotten by installing "pixz", but this is + probably only worthwhile if you are debugging the tests. + +Getting set up to run tests: +---------------------------- + +First, you need to get the corpus. + +It should work to run + + % make download-corpus + +In case that fails or is too slow, check + + http://notmuchmail.org/corpus + +for a list of mirrors. + +Running tests +------------- + +The easiest way to run performance tests is to say "make perf-test", (or +simply run the notmuch-perf-test script). Either command will run all +available performance tests. + +Alternately, you can run a specific subset of tests by simply invoking +one of the executable scripts in this directory, (such as ./basic). + +Writing tests +------------- + +Have a look at "basic" for an example. + +add_email_corpus takes arguments "--small" and "--medium" for when you +want smaller subsets of the corpus to check. + +time_done does the cleanup; comment it out or define "$debug" to leave +the temporary files around. + +Currently there is no option processing (e.g. --debug) in the +performance tests. diff --git a/performance-test/basic b/performance-test/basic new file mode 100755 index 00000000..9d015eec --- /dev/null +++ b/performance-test/basic @@ -0,0 +1,15 @@ +#!/bin/bash + +. ./perf-test-lib.sh + +add_email_corpus + +print_header + +time_run 'initial notmuch new' 'notmuch new' +time_run 'second notmuch new' 'notmuch new' +time_run 'dump *' 'notmuch dump > tags.out' +time_run 'restore *' 'notmuch restore < tags.out' +time_run 'tag * +new_tag' "notmuch tag +new_tag '*'" + +time_done diff --git a/performance-test/download/.gitignore b/performance-test/download/.gitignore new file mode 100644 index 00000000..7b092346 --- /dev/null +++ b/performance-test/download/.gitignore @@ -0,0 +1,2 @@ +*.tar.gz +*.tar.xz diff --git a/performance-test/download/notmuch-email-corpus-0.2.tar.xz.asc b/performance-test/download/notmuch-email-corpus-0.2.tar.xz.asc new file mode 100644 index 00000000..c8b4b3d8 --- /dev/null +++ b/performance-test/download/notmuch-email-corpus-0.2.tar.xz.asc @@ -0,0 +1,9 @@ +-----BEGIN PGP SIGNATURE----- +Version: GnuPG v1.4.12 (GNU/Linux) + +iJwEAAECAAYFAlCsvx0ACgkQTiiN/0Um85kZAwP9GgOQ22jK8mr5X4pT/mB8EjSH +QbndlxxbRrP0ChTqjBQoD3IsTHjNL7W572BfXb/MNo94R/iIQ7yTHCDVNuwBhvKd +7qgIuW2FUS1uTfJRP5KBNf8JPuin+6wqGe8/+y/iOs+XJSdiYg1ElS49Ntnpg0yl +btImgEcxTxQ2qfzDS1g= +=iuZR +-----END PGP SIGNATURE----- diff --git a/performance-test/notmuch-perf-test b/performance-test/notmuch-perf-test new file mode 100755 index 00000000..1bea3452 --- /dev/null +++ b/performance-test/notmuch-perf-test @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + +# Run tests +# +# Copyright (c) 2005 Junio C Hamano +# +# Adapted from a Makefile to a shell script by Carl Worth (2010) + +if [ ${BASH_VERSINFO[0]} -lt 4 ]; then + echo "Error: The notmuch test suite requires a bash version >= 4.0" + echo "due to use of associative arrays within the test suite." + echo "Please try again with a newer bash (or help us fix the" + echo "test suite to be more portable). Thanks." + exit 1 +fi + +cd $(dirname "$0") + +TESTS=" + basic +" + +for test in $TESTS; do + ./$test "$@" +done diff --git a/performance-test/perf-test-lib.sh b/performance-test/perf-test-lib.sh new file mode 100644 index 00000000..80ec1eb0 --- /dev/null +++ b/performance-test/perf-test-lib.sh @@ -0,0 +1,67 @@ +. ./version.sh + +. ../test/test-lib-common.sh + +set -e + +if ! test -x ../notmuch +then + echo >&2 'You do not seem to have built notmuch yet.' + exit 1 +fi + +add_email_corpus () +{ + rm -rf ${MAIL_DIR} + + case "$1" in + --small) + arg="mail/enron/bailey-s" + ;; + --medium) + arg="mail/notmuch-archive" + ;; + *) + arg=mail + esac + + if command -v pixz > /dev/null; then + XZ=pixz + else + XZ=xz + fi + + printf "Unpacking corpus\n" + tar --checkpoint=.5000 --extract --strip-components=1 \ + --directory ${TMP_DIRECTORY} \ + --use-compress-program ${XZ} \ + --file ../download/notmuch-email-corpus-${PERFTEST_VERSION}.tar.xz \ + notmuch-email-corpus/"$arg" + + printf "\n" +} + +print_header () { + printf "[v%4s] Wall(s)\tUsr(s)\tSys(s)\tRes(K)\tIn\tOut\n" \ + ${PERFTEST_VERSION} +} + +time_run () { + printf "%-22s" "$1" + if test "$verbose" != "t"; then exec 4>test.output 3>&4; fi + if ! eval >&3 "/usr/bin/time -f '%e\t%U\t%S\t%M\t%I\t%O' $2" ; then + test_failure=$(($test_failure + 1)) + fi +} + +time_done () { + if [ "$test_failure" = "0" ]; then + rm -rf "$remove_tmp" + exit 0 + else + exit 1 + fi +} + +cd -P "$test" || error "Cannot setup test environment" +test_failure=0 diff --git a/performance-test/version.sh b/performance-test/version.sh new file mode 100644 index 00000000..d9270b18 --- /dev/null +++ b/performance-test/version.sh @@ -0,0 +1,3 @@ +# this should be both a valid Makefile fragment and valid POSIX(ish) shell. + +PERFTEST_VERSION=0.2 |