make testing on local servers hermetic

I noticed while fixing issue #2702 that the fish program being tested
was sourcing config.fish files outside of the current build. This also
happens when Travis CI runs the tests but isn't an issue there because
of how Travis is configured to execute the tests.

I also noticed that running `make test` was polluting my personal fish
history; which will become a bigger problem if and when the fishd universal
var file is moved from $XDG_CONFIG_HOME to $XDG_DATA_HOME.

This change makes it possible for an individual to run the tests on
their local machine secure in the knowledge that only the config.fish and
related files from their git repository will be used and doing so won't
pollute their personal fish history.

Resolves #469

11 files changed, 63 insertions, 78 deletions

diff --git a/tests/expansion.in b/tests/expansion.in
index 2b6d9376..6439871f 100644
--- a/tests/expansion.in
+++ b/tests/expansion.in
@@ -94,7 +94,7 @@ set tmpdir $PWD
 cd $saved
 mkdir $tmpdir/realhome
 ln -s $tmpdir/realhome $tmpdir/linkhome
-set expandedtilde (env HOME=$tmpdir/linkhome ../fish -c 'echo ~')
+set expandedtilde (env HOME=$tmpdir/linkhome ../test/root/bin/fish -c 'echo ~')
 if test $expandedtilde != $tmpdir/realhome
 	echo '~ expands to' $expandedtilde ' - expected ' $tmpdir/realhome
 end
diff --git a/tests/indent.in b/tests/indent.in
index 226db14e..83ef915b 100644
--- a/tests/indent.in
+++ b/tests/indent.in
@@ -5,7 +5,7 @@ echo hi
 
 
 end | cat | cat | begin ; echo hi ; end | begin ; begin ; echo hi ; end ; end arg
-' | ../fish_indent
+' | ../test/root/bin/fish_indent
 
 echo \nTest2
 echo -n '
@@ -18,7 +18,7 @@ switch aloha
       echo hi
 
 end
-' | ../fish_indent
+' | ../test/root/bin/fish_indent
 
 echo \nTest3
 echo -n '
@@ -32,8 +32,8 @@ function hello_world
    echo hello;
 
    echo hello
-                  end 
-' | ../fish_indent
+                  end
+' | ../test/root/bin/fish_indent
 
 echo \nTest4
 echo -n '
@@ -53,7 +53,7 @@ switch foo #abc
 qqq
    case "*"
        echo sup
-end' | ../fish_indent
+end' | ../test/root/bin/fish_indent
 
 echo \nTest5
 echo -n '
@@ -65,7 +65,7 @@ switch beta
        echo delta
 end
 end
-' | ../fish_indent -i
+' | ../test/root/bin/fish_indent -i
 
 echo \nTest6
 # Test errors
@@ -75,11 +75,11 @@ echo hi
 else
 echo bye
 end; echo alpha "
-' | ../fish_indent
+' | ../test/root/bin/fish_indent
 
 echo \nTest7
 # issue 1665
 echo -n '
 if begin ; false; end; echo hi ; end
 while begin ; false; end; echo hi ; end
-' | ../fish_indent
+' | ../test/root/bin/fish_indent
diff --git a/tests/interactive.expect.rc b/tests/interactive.expect.rc
index 1949d328..1129c176 100644
--- a/tests/interactive.expect.rc
+++ b/tests/interactive.expect.rc
@@ -3,7 +3,7 @@
 log_user 0
 log_file -noappend interactive.tmp.log
 
-set fish ../fish
+set fish ../test/root/bin/fish
 
 set timeout 5
 
diff --git a/tests/interactive.fish b/tests/interactive.fish
index 9f5e979c..9e0d344f 100755..100644
--- a/tests/interactive.fish
+++ b/tests/interactive.fish
@@ -1,6 +1,8 @@
-#!/usr/local/bin/fish
-#
 # Interactive tests using `expect`
+#
+# There is no shebang line because you shouldn't be running this by hand. You
+# should be running it via `make test` to ensure the environment is properly
+# setup.
 
 # Change to directory containing this script
 cd (dirname (status -f))
@@ -13,6 +15,7 @@ else
 end
 
 source test_util.fish (status -f) $argv; or exit
+cat interactive.config >> $XDG_CONFIG_HOME/fish/config.fish
 
 say -o cyan "Testing interactive functionality"
 if not type -q expect
@@ -21,17 +24,9 @@ if not type -q expect
 end
 
 function test_file
-    rm -Rf tmp.interactive.config; or die "Couldn't remove tmp.interactive.config"
-    mkdir -p tmp.interactive.config/fish; or die "Couldn't create tmp.interactive.config/fish"
-    cat $XDG_CONFIG_HOME/fish/config.fish interactive.config > tmp.interactive.config/fish/config.fish
-    or die "Couldn't create tmp.interactive.config/fish/config.fish"
-
     set -l file $argv[1]
-
     echo -n "Testing file $file ... "
-
     begin
-        set -lx XDG_CONFIG_HOME $PWD/tmp.interactive.config
         set -lx TERM dumb
         expect -n -c 'source interactive.expect.rc' -f $file >$file.tmp.out ^$file.tmp.err
     end
diff --git a/tests/test.fish b/tests/test.fish
index 588a8f35..0af92413 100755..100644
--- a/tests/test.fish
+++ b/tests/test.fish
@@ -1,6 +1,8 @@
-#!/usr/local/bin/fish
-#
 # Fishscript tests
+#
+# There is no shebang line because you shouldn't be running this by hand. You
+# should be running it via `make test` to ensure the environment is properly
+# setup.
 
 # Change to directory containing this script
 cd (dirname (status -f))
@@ -22,7 +24,7 @@ function test_file
 
     echo -n "Testing file $file ... "
 
-    ../fish <$file >$base.tmp.out ^$base.tmp.err
+    ../test/root/bin/fish <$file >$base.tmp.out ^$base.tmp.err
     set -l tmp_status $status
     set -l res ok
 
diff --git a/tests/test1.in b/tests/test1.in
index cfea247a..d52d558e 100644
--- a/tests/test1.in
+++ b/tests/test1.in
@@ -33,12 +33,12 @@ end
 # Simple alias tests
 
 function foo
-    echo >foo.txt $argv
+    echo >../test/temp/fish_foo.txt $argv
 end
 
 foo hello
 
-cat foo.txt |read foo
+cat ../test/temp/fish_foo.txt |read foo
 
 if test $foo = hello;
   echo Test 2 pass
@@ -102,7 +102,7 @@ echo Test 5 $sta
 # Verify that we can turn stderr into stdout and then pipe it.
 # Note that the order here seems unspecified - 'errput' appears before 'output', why?
 echo Test redirections
-begin ; echo output ; echo errput 1>&2  ; end 2>&1 | tee /tmp/tee_test.txt ; cat /tmp/tee_test.txt
+begin ; echo output ; echo errput 1>&2  ; end 2>&1 | tee ../test/temp/tee_test.txt ; cat ../test/temp/tee_test.txt
 
 # Verify that we can pipe something other than stdout
 # The first line should be printed, since we output to stdout but pipe stderr to /dev/null
@@ -148,7 +148,7 @@ echo "/bin/echo pipe 12 <&12 12<&-" | source 12<&0
 
 # Make sure while loops don't run forever with no-exec (#1543)
 echo "Checking for infinite loops in no-execute"
-echo "while true; end" | ../fish --no-execute
+echo "while true; end" | ../test/root/bin/fish --no-execute
 
 # Comments allowed in between lines (#1987)
 echo before comment \
diff --git a/tests/test3.in b/tests/test3.in
index ee1b8575..b06ef4ab 100644
--- a/tests/test3.in
+++ b/tests/test3.in
@@ -96,7 +96,7 @@ echo Test 7 $res
 set -e t8
 if true
 	set -lx t8 foo
-	if test (../fish -c "echo $t8") = foo
+	if test (../test/root/bin/fish -c "echo $t8") = foo
 		echo Test 8 pass
 	else
 		echo Test 8 fail
@@ -105,7 +105,7 @@ end
 
 # Test if exported variables go out of scope
 
-if test (../fish -c "echo $t8")
+if test (../test/root/bin/fish -c "echo $t8")
 	echo Test 9 fail
 else
 	echo Test 9 pass
@@ -142,7 +142,7 @@ set -ge __fish_test_universal_variables_variable_foo
 set -Ue __fish_test_universal_variables_variable_foo
 set -Ux __fish_test_universal_variables_variable_foo bar
 set __fish_test_universal_variables_variable_foo baz
-if test (/bin/sh -c 'echo $__fish_test_universal_variables_variable_foo') = baz -a (../fish -c 'echo $__fish_test_universal_variables_variable_foo') = baz
+if test (/bin/sh -c 'echo $__fish_test_universal_variables_variable_foo') = baz -a (../test/root/bin/fish -c 'echo $__fish_test_universal_variables_variable_foo') = baz
   echo Test 12 pass
 else
   echo Test 12 fail
@@ -155,7 +155,7 @@ env | __fish_sgrep __fish_test_universal_variables_variable_foo
 
 set -Ux __fish_test_universal_variables_variable_foo bar
 set -U __fish_test_universal_variables_variable_foo baz
-if test (/bin/sh -c 'echo $__fish_test_universal_variables_variable_foo') = baz -a (../fish -c 'echo $__fish_test_universal_variables_variable_foo') = baz
+if test (/bin/sh -c 'echo $__fish_test_universal_variables_variable_foo') = baz -a (../test/root/bin/fish -c 'echo $__fish_test_universal_variables_variable_foo') = baz
   echo Test 13 pass
 else
   echo Test 13 fail
@@ -163,7 +163,7 @@ end
 
 set -Ux __fish_test_universal_variables_variable_foo bar
 set -u __fish_test_universal_variables_variable_foo bar
-if test (/bin/sh -c 'echo $__fish_test_universal_variables_variable_foo') = '' -a (../fish -c 'echo $__fish_test_universal_variables_variable_foo') = bar
+if test (/bin/sh -c 'echo $__fish_test_universal_variables_variable_foo') = '' -a (../test/root/bin/fish -c 'echo $__fish_test_universal_variables_variable_foo') = bar
   echo Test 14 pass
 else
   echo Test 14 fail
@@ -171,7 +171,7 @@ end
 
 set -Ux __fish_test_universal_variables_variable_foo bar
 set -Uu __fish_test_universal_variables_variable_foo baz
-if test (/bin/sh -c 'echo $__fish_test_universal_variables_variable_foo') = '' -a (../fish -c 'echo $__fish_test_universal_variables_variable_foo') = baz
+if test (/bin/sh -c 'echo $__fish_test_universal_variables_variable_foo') = '' -a (../test/root/bin/fish -c 'echo $__fish_test_universal_variables_variable_foo') = baz
   echo Test 15 pass
 else
   echo Test 15 fail
@@ -242,35 +242,35 @@ set -eU __fish_test_universal_variables_variable_foo
 # Test behavior of universals on startup (#1526)
 echo Testing Universal Startup
 set -U testu 0
-../fish -c 'set -U testu 1'
+../test/root/bin/fish -c 'set -U testu 1'
 echo $testu
-../fish -c 'echo $testu'
+../test/root/bin/fish -c 'echo $testu'
 
-../fish -c 'set -U testu 2'
+../test/root/bin/fish -c 'set -U testu 2'
 echo $testu
-../fish -c 'echo $testu'
+../test/root/bin/fish -c 'echo $testu'
 
-../fish -c 'set -e testu';
+../test/root/bin/fish -c 'set -e testu';
 echo Missing: $testu
-../fish -c 'echo Missing: $testu'
+../test/root/bin/fish -c 'echo Missing: $testu'
 
 # test SHLVL
 # use a subshell to ensure a clean slate
-env SHLVL= ../fish -c 'echo SHLVL: $SHLVL; ../fish -c \'echo SHLVL: $SHLVL\''
+env SHLVL= ../test/root/bin/fish -c 'echo SHLVL: $SHLVL; ../test/root/bin/fish -c \'echo SHLVL: $SHLVL\''
 # exec should decrement SHLVL
-env SHLVL= ../fish -c 'echo SHLVL: $SHLVL; exec ../fish -c \'echo SHLVL: $SHLVL\''
+env SHLVL= ../test/root/bin/fish -c 'echo SHLVL: $SHLVL; exec ../test/root/bin/fish -c \'echo SHLVL: $SHLVL\''
 # garbage SHLVLs should be treated as garbage
-env SHLVL=3foo ../fish -c 'echo SHLVL: $SHLVL'
+env SHLVL=3foo ../test/root/bin/fish -c 'echo SHLVL: $SHLVL'
 # whitespace is allowed though (for bash compatibility)
-env SHLVL="3  " ../fish -c 'echo SHLVL: $SHLVL'
-env SHLVL="  3" ../fish -c 'echo SHLVL: $SHLVL'
+env SHLVL="3  " ../test/root/bin/fish -c 'echo SHLVL: $SHLVL'
+env SHLVL="  3" ../test/root/bin/fish -c 'echo SHLVL: $SHLVL'
 
 # Test transformation of inherited variables
-env DISPLAY="localhost:0.0" ../fish -c 'echo Elements in DISPLAY: (count $DISPLAY)'
+env DISPLAY="localhost:0.0" ../test/root/bin/fish -c 'echo Elements in DISPLAY: (count $DISPLAY)'
 # We can't use PATH for this because the global configuration will modify PATH
 # based on /etc/paths and /etc/paths.d.
 # Exported arrays should use record separator, with a few exceptions. So we can use an arbitrary variable for this.
-env FOO=one\x1etwo\x1ethree\x1efour ../fish -c 'echo Elements in FOO: (count $FOO)'
+env FOO=one\x1etwo\x1ethree\x1efour ../test/root/bin/fish -c 'echo Elements in FOO: (count $FOO)'
 # some must use colon separators!
 set -lx MANPATH man1 man2 man3 ; env | grep MANPATH
 
diff --git a/tests/test5.in b/tests/test5.in
index 1afc0cc7..3f99fa00 100644
--- a/tests/test5.in
+++ b/tests/test5.in
@@ -27,11 +27,11 @@ end
 # Verify that we can do wildcard expansion when we
 # don't have read access to some path components
 # See #2099
-set -l where /tmp/fish_wildcard_permissions_test/noaccess/yesaccess
+set -l where ../test/temp/fish_wildcard_permissions_test/noaccess/yesaccess
 mkdir -p $where
 chmod 300 (dirname $where) # no read permissions
 mkdir -p $where
 touch $where/alpha.txt $where/beta.txt $where/delta.txt
 echo $where/*
 chmod 700 (dirname $where) # so we can delete it
-rm -rf /tmp/fish_wildcard_permissions_test
+rm -rf ../test/temp/fish_wildcard_permissions_test
diff --git a/tests/test5.out b/tests/test5.out
index bdee60aa..70f1ec81 100644
--- a/tests/test5.out
+++ b/tests/test5.out
@@ -1,4 +1,4 @@
 Test 1 pass
 Test 2 pass
 Test 3 pass
-/tmp/fish_wildcard_permissions_test/noaccess/yesaccess/alpha.txt /tmp/fish_wildcard_permissions_test/noaccess/yesaccess/beta.txt /tmp/fish_wildcard_permissions_test/noaccess/yesaccess/delta.txt
+../test/temp/fish_wildcard_permissions_test/noaccess/yesaccess/alpha.txt ../test/temp/fish_wildcard_permissions_test/noaccess/yesaccess/beta.txt ../test/temp/fish_wildcard_permissions_test/noaccess/yesaccess/delta.txt
diff --git a/tests/test9.in b/tests/test9.in
index 1199609b..0643354f 100644
--- a/tests/test9.in
+++ b/tests/test9.in
@@ -2,10 +2,10 @@
 # ensure that builtins that produce no output can still truncate files
 # (bug PCA almost reintroduced!)
 echo "Testing that builtins can truncate files"
-echo abc > /tmp/file_truncation_test.txt
-cat /tmp/file_truncation_test.txt
-echo -n > /tmp/file_truncation_test.txt
-cat /tmp/file_truncation_test.txt
+echo abc > ../test/temp/file_truncation_test.txt
+cat ../test/temp/file_truncation_test.txt
+echo -n > ../test/temp/file_truncation_test.txt
+cat ../test/temp/file_truncation_test.txt
 
 # Test events. 
 
@@ -120,7 +120,7 @@ diff -q (__fish_print_help psub | psub) (psub -hs banana | psub)
 
 # Test support for unbalanced blocks
 function try_unbalanced_block
-    ../fish -c "echo $argv | source " 2>&1 | grep "Missing end" 1>&2
+    ../test/root/bin/fish -c "echo $argv | source " 2>&1 | grep "Missing end" 1>&2
 end
 try_unbalanced_block 'begin'
 try_unbalanced_block 'while true'
diff --git a/tests/test_util.fish b/tests/test_util.fish
index 22744b34..7e8f1fd6 100644
--- a/tests/test_util.fish
+++ b/tests/test_util.fish
@@ -19,8 +19,8 @@ function die
     exit 1
 end
 
-# check if we're running in the test environment
-# if not, set it up and rerun fish with exec.
+# Check if we're running in the test environment.
+# If not, set it up and rerun fish with exec.
 # The test is whether the special var __fish_is_running_tests
 # exists and contains the same value as XDG_CONFIG_HOME. It checks
 # the value and not just the presence because we're going to delete
@@ -37,19 +37,21 @@ if not set -q __fish_is_running_tests
     end
     set -l IFS # clear IFS so cmd substitution doesn't split
     cd (dirname $script); or die
-    set -xl XDG_CONFIG_HOME (printf '%s/tmp.config/%s' $PWD %self); or die
-    if test -d $XDG_CONFIG_HOME
-        # if the dir already exists, we've reused a pid
-        # this would be surprising to reuse a fish pid that failed in the past,
-        # but clear it anyway
-        rm -r $XDG_CONFIG_HOME; or die
-    end
+
+    set -lx XDG_DATA_HOME ../test/data
+    rm -rf $XDG_DATA_HOME/fish
+    mkdir -p $XDG_DATA_HOME/fish; or die
+
+    set -lx XDG_CONFIG_HOME ../test/home
+    rm -rf $XDG_CONFIG_HOME/fish
     mkdir -p $XDG_CONFIG_HOME/fish; or die
     ln -s $PWD/test_functions $XDG_CONFIG_HOME/fish/functions; or die
+
     set -l escaped_parent (dirname $PWD | sed -e 's/[\'\\\\]/\\\\&/g'); or die
     set -l escaped_config (printf '%s/fish' $XDG_CONFIG_HOME | sed -e 's/[\'\\\\]/\\\\&/g'); or die
     printf 'set fish_function_path \'%s/functions\' \'%s/share/functions\'\n' $escaped_config $escaped_parent > $XDG_CONFIG_HOME/fish/config.fish; or die
     set -xl __fish_is_running_tests $XDG_CONFIG_HOME
+
     # set locale information to be consistent
     set -lx LANG C
     set -lx LC_ALL ''
@@ -57,27 +59,13 @@ if not set -q __fish_is_running_tests
         set -lx LC_$var ''
     end
     set -lx LC_CTYPE en_US.UTF-8
-    exec ../fish $script $args_for_test_script
+    exec ../test/root/bin/fish $script $args_for_test_script
     die 'exec failed'
 else if test "$__fish_is_running_tests" != "$XDG_CONFIG_HOME"
     echo 'Something went wrong with the test runner.' >&2
     echo "__fish_is_running_tests: $__fish_is_running_tests" >&2
     echo "XDG_CONFIG_HOME: $XDG_CONFIG_HOME" >&2
     exit 10
-else
-    # we're running tests with a temporary config directory
-    function test_util_on_exit --on-process-exit %self -V __fish_is_running_tests
-        if not set -q __fish_test_keep_tmp_config
-            # remove the temporary config directory
-            # unfortunately if this fails we can't alter the exit status of fish
-            if not rm -r "$__fish_is_running_tests"
-                echo "error: Couldn't remove temporary config directory '$__fish_is_running_tests'" >&2
-            end
-        end
-    end
-    # unset __fish_is_running_tests so any children that source
-    # test_util.fish will set up a new test environment.
-    set -e __fish_is_running_tests
 end
 
 set -l suppress_color