make testing on local servers hermetic

I noticed while fixing issue #2702 that the fish program being tested
was sourcing config.fish files outside of the current build. This also
happens when Travis CI runs the tests but isn't an issue there because
of how Travis is configured to execute the tests.

I also noticed that running `make test` was polluting my personal fish
history; which will become a bigger problem if and when the fishd universal
var file is moved from $XDG_CONFIG_HOME to $XDG_DATA_HOME.

This change makes it possible for an individual to run the tests on
their local machine secure in the knowledge that only the config.fish and
related files from their git repository will be used and doing so won't
pollute their personal fish history.

Resolves #469

15 files changed, 155 insertions, 142 deletions

diff --git a/.gitignore b/.gitignore
index e95e3b90..2971b0c5 100644
--- a/.gitignore
+++ b/.gitignore
@@ -31,8 +31,7 @@ share/man/
 toc.txt
 user_doc/
 xcuserdata
-tests/*tmp.*
-tests/foo.txt
+test/
 FISH-BUILD-VERSION-FILE
 version
 messages.pot
@@ -40,4 +39,3 @@ lexicon.txt
 lexicon_filter
 lexicon.log
 DerivedData/
-
diff --git a/.travis.yml b/.travis.yml
index 29524239..54c047db 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -47,7 +47,7 @@ script:
   - ./configure --prefix=$HOME/prefix || cat config.log
   - make -j2
   - make install
-  - make test SHOW_INTERACTIVE_LOG=1
+  - make test DESTDIR=$HOME/prefix/ SHOW_INTERACTIVE_LOG=1
 
 notifications:
   # Some items are encrypted so that notifications from other repositories
diff --git a/Makefile.in b/Makefile.in
index 1f2265da..88155b2d 100644
--- a/Makefile.in
+++ b/Makefile.in
@@ -23,6 +23,11 @@
 # applications, install them, and recalculate dependencies.
 #
 
+# This is the default value for SHELL but I like to be explicit about such
+# things. Especially in a project like fish where someone might otherwise
+# think fish will be used to execute make recipes.
+SHELL = /bin/sh
+
 # Used by docdir
 PACKAGE_TARNAME = @PACKAGE_TARNAME@
 
@@ -113,7 +118,6 @@ BUILTIN_FILES := src/builtin_set.cpp src/builtin_commandline.cpp	\
 #
 # All objects that the system needs to build fish_tests
 #
-
 FISH_TESTS_OBJS := $(FISH_OBJS) obj/fish_tests.o
 
 
@@ -291,12 +295,29 @@ doc/refman.pdf: doc
 	mv refman.pdf ..;
 	rm -r doc/latex;
 
+# Prep the environment for running the unit tests. When specifying DESTDIR on
+# the command line (e.g., `make DESTDIR=/usr/local/`) you must have previously
+# installed fish using the same prefix; e.g., `./configure --prefix=/usr/local`
+# followed by `make install`.
+test-prep:
+	rm -rf test
+	mkdir test test/data test/home test/temp
+ifdef DESTDIR
+	ln -s $(DESTDIR) test/root
+else
+	mkdir test/root
+endif
+.PHONY: test-prep
 
+# The test target runs both the low level code tests and the high level script
+# tests.
 #
-# This target runs both the low level code tests and the high level script tests.
-#
-
-test: test_low_level test_high_level
+# Note that doing `make DESTDIR=/some/path/ test` overrides this assignment.
+test: DESTDIR = $(PWD)/test/root/
+test: prefix = .
+test: test-prep install-force test_low_level test_high_level
+	@rm -f /tmp/file_truncation_test.txt /tmp/tee_test.txt /tmp/fish_foo.txt
+	@rm -rf /tmp/is_potential_path_test
 .PHONY: test
 
 # We want the various tests to run serially so their output doesn't mix
@@ -319,12 +340,12 @@ test_low_level: fish_tests $(call filter_up_to,test_low_level,$(active_test_goal
 test_high_level: test_fishscript test_interactive
 .PHONY: test_high_level
 
-test_fishscript: $(PROGRAMS) $(call filter_up_to,test_fishscript,$(active_test_goals))
-	cd tests && ../fish test.fish
+test_fishscript: $(call filter_up_to,test_fishscript,$(active_test_goals))
+	cd tests; ../test/root/bin/fish test.fish
 .PHONY: test_fishscript
 
-test_interactive: $(PROGRAMS) $(call filter_up_to,test_interactive,$(active_test_goals))
-	cd tests && ../fish interactive.fish
+test_interactive: $(call filter_up_to,test_interactive,$(active_test_goals))
+	cd tests; ../test/root/bin/fish interactive.fish
 .PHONY: test_interactive
 
 #
@@ -582,7 +603,6 @@ check-legacy-binaries:
 	@true;
 .PHONY: check-legacy-binaries
 
-
 #
 # This check makes sure that the install-sh script is executable. The
 # darcs repo doesn't preserve the executable bit, so this needs to be
@@ -593,12 +613,30 @@ install-sh:
 	if test -x $@; then true; else chmod 755 $@; fi
 .PHONY: install-sh
 
-
 #
 # Try to install after checking for incompatible installed versions.
 #
 
 install: all install-sh check-uninstall install-force check-legacy-binaries
+	@echo fish is now installed on your system.
+	@echo To run fish, type \'fish\' in your terminal.
+	@echo
+	@if type chsh >/dev/null 2>&1; then \
+		echo To use fish as your login shell:; \
+		grep -q -- "$(DESTDIR)$(bindir)/fish" /etc/shells || echo \* add the line \'$(DESTDIR)$(bindir)/fish\' to the file \'/etc/shells\'.; \
+		echo \* use the command \'chsh -s $(DESTDIR)$(bindir)/fish\'.; \
+		echo; \
+	fi;
+	@if type chcon >/dev/null 2>&1; then \
+		echo If you have SELinux enabled, you may need to manually update the security policy:; \
+		echo \* use the command \'chcon -t shell_exec_t $(DESTDIR)$(bindir)/fish\'.; \
+		echo; \
+	fi;
+	@echo To set your colors, run \'fish_config\'
+	@echo To scan your man pages for completions, run \'fish_update_completions\'
+	@echo To autocomplete command suggestions press Ctrl + F or right arrow key.
+	@echo
+	@echo Have fun!
 .PHONY: install
 
 #
@@ -611,11 +649,10 @@ xcode-install:
 .PHONY: xcode-install
 
 #
-# Force installation, even in presense of incompatible previous
-# version. This may fail.
-# These 'true' lines are to prevent installs from failing for (e.g.) missing man pages.
+# Force installation, even in presense of incompatible previous version. This
+# may fail. These 'true' lines are to prevent installs from failing for (e.g.)
+# missing man pages.
 #
-
 install-force: all install-translations
 	$(INSTALL) -m 755 -d $(DESTDIR)$(bindir)
 	for i in $(PROGRAMS); do\
@@ -685,25 +722,6 @@ install-force: all install-translations
 		$(INSTALL) -m 644 $$i $(DESTDIR)$(mandir)/man1/; \
 		true; \
 	done;
-	@echo fish is now installed on your system.
-	@echo To run fish, type \'fish\' in your terminal.
-	@echo
-	@if type chsh >/dev/null 2>&1; then \
-		echo To use fish as your login shell:; \
-		grep -q -- "$(DESTDIR)$(bindir)/fish" /etc/shells || echo \* add the line \'$(DESTDIR)$(bindir)/fish\' to the file \'/etc/shells\'.; \
-		echo \* use the command \'chsh -s $(DESTDIR)$(bindir)/fish\'.; \
-		echo; \
-	fi;
-	@if type chcon >/dev/null 2>&1; then \
-		echo If you have SELinux enabled, you may need to manually update the security policy:; \
-		echo \* use the command \'chcon -t shell_exec_t $(DESTDIR)$(bindir)/fish\'.; \
-		echo; \
-	fi;
-	@echo To set your colors, run \'fish_config\'
-	@echo To scan your man pages for completions, run \'fish_update_completions\'
-	@echo To autocomplete command suggestions press Ctrl + F or right arrow key.
-	@echo
-	@echo Have fun!
 .PHONY: install-force
 
 
@@ -781,7 +799,7 @@ uninstall-translations:
 #
 obj/%.o: src/%.cpp | obj
 	$(CXX) $(CXXFLAGS) -c $< -o $@
-	
+
 #
 # obj directory
 #
@@ -801,27 +819,21 @@ $(PCRE2_LIB): $(PCRE2_H)
 $(PCRE2_H):
 	(cd $(PCRE2_DIR) && ./config.status --enable-maintainer-mode)
 
-
 #
 # Build the fish_tests program.
 #
-
 fish_tests: $(FISH_TESTS_OBJS) $(EXTRA_PCRE2)
 	$(CXX) $(CXXFLAGS) $(LDFLAGS_FISH) $(FISH_TESTS_OBJS) $(LIBS) -o $@
 
-
 #
 # Build the fish_indent program.
 #
-
 fish_indent: $(FISH_INDENT_OBJS) $(EXTRA_PCRE2)
 	$(CXX) $(CXXFLAGS) $(LDFLAGS) $(FISH_INDENT_OBJS) $(LIBS) -o $@
 
-
 #
 # Neat little program to show output from terminal
 #
-
 key_reader: $(FISH_OBJS) $(EXTRA_PCRE2) obj/key_reader.o
 	$(CXX) $(CXXFLAGS) $(LDFLAGS_FISH) $^ $(LIBS) -o $@
 
@@ -860,41 +872,34 @@ _iwyu: clean $(PROGRAMS)
 # Cleanup targets
 #
 
-#
-# distclean should restore the tree to the state right after extracting a tarball.
-#
-
+# Restore the source tree to the state right after extracting a tarball.
 distclean: clean
-	$(MAKE) -C $(PCRE2_DIR) distclean
+	$(MAKE) -C $(PCRE2_DIR) distclean || true
 	rm -f config.status config.log config.h Makefile
 .PHONY: distclean
 
 
+# Remove everything built by the Makefile, but not things that are created by
+# the configure script.
 #
-# clean removes everything built by the makefile, but not things that
-# are created by the configure script.
-#
-
-# Don't delete the docs unless we have Doxygen installed
-# We provide pre-built docs in the tarball, and if they get
-# deleted we won't be able to regenerate them
-
+# Don't delete the docs unless we have Doxygen installed We provide pre-built
+# docs in the tarball, and if they get deleted we won't be able to regenerate
+# them.
 clean:
-ifdef EXTRA_PCRE2
-	$(MAKE) -C $(PCRE2_DIR) clean
-endif
-	rm -f obj/*.o *.o doc.h doc.tmp doc_src/*.doxygen doc_src/*.cpp doc_src/*.o doc_src/commands.hdr
+	$(MAKE) -C $(PCRE2_DIR) clean || true
+	rm -f obj/*.o *.o doc.h doc.tmp
+	rm -f doc_src/*.doxygen doc_src/*.cpp doc_src/*.o doc_src/commands.hdr
 	rm -f tests/tmp.err tests/tmp.out tests/tmp.status tests/foo.txt
 	rm -f $(PROGRAMS) fish_tests key_reader
 	rm -f command_list.txt command_list_toc.txt toc.txt
 	rm -f doc_src/index.hdr doc_src/commands.hdr
 	rm -f lexicon_filter lexicon.txt lexicon.log
-	rm -f FISH-BUILD-VERSION-FILE
+	rm -f FISH-BUILD-VERSION-FILE fish.pc
 	if test "$(HAVE_DOXYGEN)" = 1; then \
 		rm -rf doc user_doc share/man; \
 	fi
 	rm -f po/*.gmo
-	rm -rf obj
+	rm -rf obj build test
 .PHONY: clean
 
 
diff --git a/README.md b/README.md
index 172325ec..c9636dd6 100644
--- a/README.md
+++ b/README.md
@@ -52,6 +52,31 @@ On RedHat, CentOS, or Amazon EC2:
 
     sudo yum install ncurses-devel
 
+## Testing
+
+### Travis CI Build and Test
+
+You can have the Travis continuous integration tool automatically build and test your changes. This requires you to fork the project on GitHub or have pushed your local fish-shell repository to GitHub.
+
+Login to [Travis CI](https://travis-ci.org/) with your GitHub account and enable your fish-shell clone. To reach that page click the plus-sign to the right of "My Repositories" on the main page for your account or go to your [profile page](https://travis-ci.org/profile/). After you do that every time you push changes to GitHub Travis will automatically build and test those changes. You'll receive an email when the tests are complete telling you whether or not any tests failed. This helps avoid being embarrassed by making a pull-request only to find you introduced a bug or failed to update a unit test. This also ensures that even if you can build and run fish on your system that it can also be built and run on other types of systems.
+
+You'll find the configuration used to control Travis in the `.travis.yml` file.
+
+### Running the Tests On Your Local Server
+
+You should not build and install fish using the instructions above after
+making changs until you've run the tests. You may or may not need to create an
+appropriate `Makefile` by running the following one time:
+
+    autoconf
+    ./configure
+
+To run the unit tests:
+
+    make test
+
+Note: These instructions will work on Mac OS X as well as Linux but do require that you've used something like [Homebrew](http://brew.sh/) to install autoconf and related tools.
+
 ## Runtime Dependencies
 
 fish requires a curses implementation, such as ncurses, to run.
diff --git a/tests/expansion.in b/tests/expansion.in
index 2b6d9376..6439871f 100644
--- a/tests/expansion.in
+++ b/tests/expansion.in
@@ -94,7 +94,7 @@ set tmpdir $PWD
 cd $saved
 mkdir $tmpdir/realhome
 ln -s $tmpdir/realhome $tmpdir/linkhome
-set expandedtilde (env HOME=$tmpdir/linkhome ../fish -c 'echo ~')
+set expandedtilde (env HOME=$tmpdir/linkhome ../test/root/bin/fish -c 'echo ~')
 if test $expandedtilde != $tmpdir/realhome
 	echo '~ expands to' $expandedtilde ' - expected ' $tmpdir/realhome
 end
diff --git a/tests/indent.in b/tests/indent.in
index 226db14e..83ef915b 100644
--- a/tests/indent.in
+++ b/tests/indent.in
@@ -5,7 +5,7 @@ echo hi
 
 
 end | cat | cat | begin ; echo hi ; end | begin ; begin ; echo hi ; end ; end arg
-' | ../fish_indent
+' | ../test/root/bin/fish_indent
 
 echo \nTest2
 echo -n '
@@ -18,7 +18,7 @@ switch aloha
       echo hi
 
 end
-' | ../fish_indent
+' | ../test/root/bin/fish_indent
 
 echo \nTest3
 echo -n '
@@ -32,8 +32,8 @@ function hello_world
    echo hello;
 
    echo hello
-                  end 
-' | ../fish_indent
+                  end
+' | ../test/root/bin/fish_indent
 
 echo \nTest4
 echo -n '
@@ -53,7 +53,7 @@ switch foo #abc
 qqq
    case "*"
        echo sup
-end' | ../fish_indent
+end' | ../test/root/bin/fish_indent
 
 echo \nTest5
 echo -n '
@@ -65,7 +65,7 @@ switch beta
        echo delta
 end
 end
-' | ../fish_indent -i
+' | ../test/root/bin/fish_indent -i
 
 echo \nTest6
 # Test errors
@@ -75,11 +75,11 @@ echo hi
 else
 echo bye
 end; echo alpha "
-' | ../fish_indent
+' | ../test/root/bin/fish_indent
 
 echo \nTest7
 # issue 1665
 echo -n '
 if begin ; false; end; echo hi ; end
 while begin ; false; end; echo hi ; end
-' | ../fish_indent
+' | ../test/root/bin/fish_indent
diff --git a/tests/interactive.expect.rc b/tests/interactive.expect.rc
index 1949d328..1129c176 100644
--- a/tests/interactive.expect.rc
+++ b/tests/interactive.expect.rc
@@ -3,7 +3,7 @@
 log_user 0
 log_file -noappend interactive.tmp.log
 
-set fish ../fish
+set fish ../test/root/bin/fish
 
 set timeout 5
 
diff --git a/tests/interactive.fish b/tests/interactive.fish
index 9f5e979c..9e0d344f 100755..100644
--- a/tests/interactive.fish
+++ b/tests/interactive.fish
@@ -1,6 +1,8 @@
-#!/usr/local/bin/fish
-#
 # Interactive tests using `expect`
+#
+# There is no shebang line because you shouldn't be running this by hand. You
+# should be running it via `make test` to ensure the environment is properly
+# setup.
 
 # Change to directory containing this script
 cd (dirname (status -f))
@@ -13,6 +15,7 @@ else
 end
 
 source test_util.fish (status -f) $argv; or exit
+cat interactive.config >> $XDG_CONFIG_HOME/fish/config.fish
 
 say -o cyan "Testing interactive functionality"
 if not type -q expect
@@ -21,17 +24,9 @@ if not type -q expect
 end
 
 function test_file
-    rm -Rf tmp.interactive.config; or die "Couldn't remove tmp.interactive.config"
-    mkdir -p tmp.interactive.config/fish; or die "Couldn't create tmp.interactive.config/fish"
-    cat $XDG_CONFIG_HOME/fish/config.fish interactive.config > tmp.interactive.config/fish/config.fish
-    or die "Couldn't create tmp.interactive.config/fish/config.fish"
-
     set -l file $argv[1]
-
     echo -n "Testing file $file ... "
-
     begin
-        set -lx XDG_CONFIG_HOME $PWD/tmp.interactive.config
         set -lx TERM dumb
         expect -n -c 'source interactive.expect.rc' -f $file >$file.tmp.out ^$file.tmp.err
     end
diff --git a/tests/test.fish b/tests/test.fish
index 588a8f35..0af92413 100755..100644
--- a/tests/test.fish
+++ b/tests/test.fish
@@ -1,6 +1,8 @@
-#!/usr/local/bin/fish
-#
 # Fishscript tests
+#
+# There is no shebang line because you shouldn't be running this by hand. You
+# should be running it via `make test` to ensure the environment is properly
+# setup.
 
 # Change to directory containing this script
 cd (dirname (status -f))
@@ -22,7 +24,7 @@ function test_file
 
     echo -n "Testing file $file ... "
 
-    ../fish <$file >$base.tmp.out ^$base.tmp.err
+    ../test/root/bin/fish <$file >$base.tmp.out ^$base.tmp.err
     set -l tmp_status $status
     set -l res ok
 
diff --git a/tests/test1.in b/tests/test1.in
index cfea247a..d52d558e 100644
--- a/tests/test1.in
+++ b/tests/test1.in
@@ -33,12 +33,12 @@ end
 # Simple alias tests
 
 function foo
-    echo >foo.txt $argv
+    echo >../test/temp/fish_foo.txt $argv
 end
 
 foo hello
 
-cat foo.txt |read foo
+cat ../test/temp/fish_foo.txt |read foo
 
 if test $foo = hello;
   echo Test 2 pass
@@ -102,7 +102,7 @@ echo Test 5 $sta
 # Verify that we can turn stderr into stdout and then pipe it.
 # Note that the order here seems unspecified - 'errput' appears before 'output', why?
 echo Test redirections
-begin ; echo output ; echo errput 1>&2  ; end 2>&1 | tee /tmp/tee_test.txt ; cat /tmp/tee_test.txt
+begin ; echo output ; echo errput 1>&2  ; end 2>&1 | tee ../test/temp/tee_test.txt ; cat ../test/temp/tee_test.txt
 
 # Verify that we can pipe something other than stdout
 # The first line should be printed, since we output to stdout but pipe stderr to /dev/null
@@ -148,7 +148,7 @@ echo "/bin/echo pipe 12 <&12 12<&-" | source 12<&0
 
 # Make sure while loops don't run forever with no-exec (#1543)
 echo "Checking for infinite loops in no-execute"
-echo "while true; end" | ../fish --no-execute
+echo "while true; end" | ../test/root/bin/fish --no-execute
 
 # Comments allowed in between lines (#1987)
 echo before comment \
diff --git a/tests/test3.in b/tests/test3.in
index ee1b8575..b06ef4ab 100644
--- a/tests/test3.in
+++ b/tests/test3.in
@@ -96,7 +96,7 @@ echo Test 7 $res
 set -e t8
 if true
 	set -lx t8 foo
-	if test (../fish -c "echo $t8") = foo
+	if test (../test/root/bin/fish -c "echo $t8") = foo
 		echo Test 8 pass
 	else
 		echo Test 8 fail
@@ -105,7 +105,7 @@ end
 
 # Test if exported variables go out of scope
 
-if test (../fish -c "echo $t8")
+if test (../test/root/bin/fish -c "echo $t8")
 	echo Test 9 fail
 else
 	echo Test 9 pass
@@ -142,7 +142,7 @@ set -ge __fish_test_universal_variables_variable_foo
 set -Ue __fish_test_universal_variables_variable_foo
 set -Ux __fish_test_universal_variables_variable_foo bar
 set __fish_test_universal_variables_variable_foo baz
-if test (/bin/sh -c 'echo $__fish_test_universal_variables_variable_foo') = baz -a (../fish -c 'echo $__fish_test_universal_variables_variable_foo') = baz
+if test (/bin/sh -c 'echo $__fish_test_universal_variables_variable_foo') = baz -a (../test/root/bin/fish -c 'echo $__fish_test_universal_variables_variable_foo') = baz
   echo Test 12 pass
 else
   echo Test 12 fail
@@ -155,7 +155,7 @@ env | __fish_sgrep __fish_test_universal_variables_variable_foo
 
 set -Ux __fish_test_universal_variables_variable_foo bar
 set -U __fish_test_universal_variables_variable_foo baz
-if test (/bin/sh -c 'echo $__fish_test_universal_variables_variable_foo') = baz -a (../fish -c 'echo $__fish_test_universal_variables_variable_foo') = baz
+if test (/bin/sh -c 'echo $__fish_test_universal_variables_variable_foo') = baz -a (../test/root/bin/fish -c 'echo $__fish_test_universal_variables_variable_foo') = baz
   echo Test 13 pass
 else
   echo Test 13 fail
@@ -163,7 +163,7 @@ end
 
 set -Ux __fish_test_universal_variables_variable_foo bar
 set -u __fish_test_universal_variables_variable_foo bar
-if test (/bin/sh -c 'echo $__fish_test_universal_variables_variable_foo') = '' -a (../fish -c 'echo $__fish_test_universal_variables_variable_foo') = bar
+if test (/bin/sh -c 'echo $__fish_test_universal_variables_variable_foo') = '' -a (../test/root/bin/fish -c 'echo $__fish_test_universal_variables_variable_foo') = bar
   echo Test 14 pass
 else
   echo Test 14 fail
@@ -171,7 +171,7 @@ end
 
 set -Ux __fish_test_universal_variables_variable_foo bar
 set -Uu __fish_test_universal_variables_variable_foo baz
-if test (/bin/sh -c 'echo $__fish_test_universal_variables_variable_foo') = '' -a (../fish -c 'echo $__fish_test_universal_variables_variable_foo') = baz
+if test (/bin/sh -c 'echo $__fish_test_universal_variables_variable_foo') = '' -a (../test/root/bin/fish -c 'echo $__fish_test_universal_variables_variable_foo') = baz
   echo Test 15 pass
 else
   echo Test 15 fail
@@ -242,35 +242,35 @@ set -eU __fish_test_universal_variables_variable_foo
 # Test behavior of universals on startup (#1526)
 echo Testing Universal Startup
 set -U testu 0
-../fish -c 'set -U testu 1'
+../test/root/bin/fish -c 'set -U testu 1'
 echo $testu
-../fish -c 'echo $testu'
+../test/root/bin/fish -c 'echo $testu'
 
-../fish -c 'set -U testu 2'
+../test/root/bin/fish -c 'set -U testu 2'
 echo $testu
-../fish -c 'echo $testu'
+../test/root/bin/fish -c 'echo $testu'
 
-../fish -c 'set -e testu';
+../test/root/bin/fish -c 'set -e testu';
 echo Missing: $testu
-../fish -c 'echo Missing: $testu'
+../test/root/bin/fish -c 'echo Missing: $testu'
 
 # test SHLVL
 # use a subshell to ensure a clean slate
-env SHLVL= ../fish -c 'echo SHLVL: $SHLVL; ../fish -c \'echo SHLVL: $SHLVL\''
+env SHLVL= ../test/root/bin/fish -c 'echo SHLVL: $SHLVL; ../test/root/bin/fish -c \'echo SHLVL: $SHLVL\''
 # exec should decrement SHLVL
-env SHLVL= ../fish -c 'echo SHLVL: $SHLVL; exec ../fish -c \'echo SHLVL: $SHLVL\''
+env SHLVL= ../test/root/bin/fish -c 'echo SHLVL: $SHLVL; exec ../test/root/bin/fish -c \'echo SHLVL: $SHLVL\''
 # garbage SHLVLs should be treated as garbage
-env SHLVL=3foo ../fish -c 'echo SHLVL: $SHLVL'
+env SHLVL=3foo ../test/root/bin/fish -c 'echo SHLVL: $SHLVL'
 # whitespace is allowed though (for bash compatibility)
-env SHLVL="3  " ../fish -c 'echo SHLVL: $SHLVL'
-env SHLVL="  3" ../fish -c 'echo SHLVL: $SHLVL'
+env SHLVL="3  " ../test/root/bin/fish -c 'echo SHLVL: $SHLVL'
+env SHLVL="  3" ../test/root/bin/fish -c 'echo SHLVL: $SHLVL'
 
 # Test transformation of inherited variables
-env DISPLAY="localhost:0.0" ../fish -c 'echo Elements in DISPLAY: (count $DISPLAY)'
+env DISPLAY="localhost:0.0" ../test/root/bin/fish -c 'echo Elements in DISPLAY: (count $DISPLAY)'
 # We can't use PATH for this because the global configuration will modify PATH
 # based on /etc/paths and /etc/paths.d.
 # Exported arrays should use record separator, with a few exceptions. So we can use an arbitrary variable for this.
-env FOO=one\x1etwo\x1ethree\x1efour ../fish -c 'echo Elements in FOO: (count $FOO)'
+env FOO=one\x1etwo\x1ethree\x1efour ../test/root/bin/fish -c 'echo Elements in FOO: (count $FOO)'
 # some must use colon separators!
 set -lx MANPATH man1 man2 man3 ; env | grep MANPATH
 
diff --git a/tests/test5.in b/tests/test5.in
index 1afc0cc7..3f99fa00 100644
--- a/tests/test5.in
+++ b/tests/test5.in
@@ -27,11 +27,11 @@ end
 # Verify that we can do wildcard expansion when we
 # don't have read access to some path components
 # See #2099
-set -l where /tmp/fish_wildcard_permissions_test/noaccess/yesaccess
+set -l where ../test/temp/fish_wildcard_permissions_test/noaccess/yesaccess
 mkdir -p $where
 chmod 300 (dirname $where) # no read permissions
 mkdir -p $where
 touch $where/alpha.txt $where/beta.txt $where/delta.txt
 echo $where/*
 chmod 700 (dirname $where) # so we can delete it
-rm -rf /tmp/fish_wildcard_permissions_test
+rm -rf ../test/temp/fish_wildcard_permissions_test
diff --git a/tests/test5.out b/tests/test5.out
index bdee60aa..70f1ec81 100644
--- a/tests/test5.out
+++ b/tests/test5.out
@@ -1,4 +1,4 @@
 Test 1 pass
 Test 2 pass
 Test 3 pass
-/tmp/fish_wildcard_permissions_test/noaccess/yesaccess/alpha.txt /tmp/fish_wildcard_permissions_test/noaccess/yesaccess/beta.txt /tmp/fish_wildcard_permissions_test/noaccess/yesaccess/delta.txt
+../test/temp/fish_wildcard_permissions_test/noaccess/yesaccess/alpha.txt ../test/temp/fish_wildcard_permissions_test/noaccess/yesaccess/beta.txt ../test/temp/fish_wildcard_permissions_test/noaccess/yesaccess/delta.txt
diff --git a/tests/test9.in b/tests/test9.in
index 1199609b..0643354f 100644
--- a/tests/test9.in
+++ b/tests/test9.in
@@ -2,10 +2,10 @@
 # ensure that builtins that produce no output can still truncate files
 # (bug PCA almost reintroduced!)
 echo "Testing that builtins can truncate files"
-echo abc > /tmp/file_truncation_test.txt
-cat /tmp/file_truncation_test.txt
-echo -n > /tmp/file_truncation_test.txt
-cat /tmp/file_truncation_test.txt
+echo abc > ../test/temp/file_truncation_test.txt
+cat ../test/temp/file_truncation_test.txt
+echo -n > ../test/temp/file_truncation_test.txt
+cat ../test/temp/file_truncation_test.txt
 
 # Test events. 
 
@@ -120,7 +120,7 @@ diff -q (__fish_print_help psub | psub) (psub -hs banana | psub)
 
 # Test support for unbalanced blocks
 function try_unbalanced_block
-    ../fish -c "echo $argv | source " 2>&1 | grep "Missing end" 1>&2
+    ../test/root/bin/fish -c "echo $argv | source " 2>&1 | grep "Missing end" 1>&2
 end
 try_unbalanced_block 'begin'
 try_unbalanced_block 'while true'
diff --git a/tests/test_util.fish b/tests/test_util.fish
index 22744b34..7e8f1fd6 100644
--- a/tests/test_util.fish
+++ b/tests/test_util.fish
@@ -19,8 +19,8 @@ function die
     exit 1
 end
 
-# check if we're running in the test environment
-# if not, set it up and rerun fish with exec.
+# Check if we're running in the test environment.
+# If not, set it up and rerun fish with exec.
 # The test is whether the special var __fish_is_running_tests
 # exists and contains the same value as XDG_CONFIG_HOME. It checks
 # the value and not just the presence because we're going to delete
@@ -37,19 +37,21 @@ if not set -q __fish_is_running_tests
     end
     set -l IFS # clear IFS so cmd substitution doesn't split
     cd (dirname $script); or die
-    set -xl XDG_CONFIG_HOME (printf '%s/tmp.config/%s' $PWD %self); or die
-    if test -d $XDG_CONFIG_HOME
-        # if the dir already exists, we've reused a pid
-        # this would be surprising to reuse a fish pid that failed in the past,
-        # but clear it anyway
-        rm -r $XDG_CONFIG_HOME; or die
-    end
+
+    set -lx XDG_DATA_HOME ../test/data
+    rm -rf $XDG_DATA_HOME/fish
+    mkdir -p $XDG_DATA_HOME/fish; or die
+
+    set -lx XDG_CONFIG_HOME ../test/home
+    rm -rf $XDG_CONFIG_HOME/fish
     mkdir -p $XDG_CONFIG_HOME/fish; or die
     ln -s $PWD/test_functions $XDG_CONFIG_HOME/fish/functions; or die
+
     set -l escaped_parent (dirname $PWD | sed -e 's/[\'\\\\]/\\\\&/g'); or die
     set -l escaped_config (printf '%s/fish' $XDG_CONFIG_HOME | sed -e 's/[\'\\\\]/\\\\&/g'); or die
     printf 'set fish_function_path \'%s/functions\' \'%s/share/functions\'\n' $escaped_config $escaped_parent > $XDG_CONFIG_HOME/fish/config.fish; or die
     set -xl __fish_is_running_tests $XDG_CONFIG_HOME
+
     # set locale information to be consistent
     set -lx LANG C
     set -lx LC_ALL ''
@@ -57,27 +59,13 @@ if not set -q __fish_is_running_tests
         set -lx LC_$var ''
     end
     set -lx LC_CTYPE en_US.UTF-8
-    exec ../fish $script $args_for_test_script
+    exec ../test/root/bin/fish $script $args_for_test_script
     die 'exec failed'
 else if test "$__fish_is_running_tests" != "$XDG_CONFIG_HOME"
     echo 'Something went wrong with the test runner.' >&2
     echo "__fish_is_running_tests: $__fish_is_running_tests" >&2
     echo "XDG_CONFIG_HOME: $XDG_CONFIG_HOME" >&2
     exit 10
-else
-    # we're running tests with a temporary config directory
-    function test_util_on_exit --on-process-exit %self -V __fish_is_running_tests
-        if not set -q __fish_test_keep_tmp_config
-            # remove the temporary config directory
-            # unfortunately if this fails we can't alter the exit status of fish
-            if not rm -r "$__fish_is_running_tests"
-                echo "error: Couldn't remove temporary config directory '$__fish_is_running_tests'" >&2
-            end
-        end
-    end
-    # unset __fish_is_running_tests so any children that source
-    # test_util.fish will set up a new test environment.
-    set -e __fish_is_running_tests
 end
 
 set -l suppress_color