Merge branch 'master' into iwyu

102 files changed, 65413 insertions, 0 deletions

diff --git a/src/autoload.cpp b/src/autoload.cpp
new file mode 100644
index 00000000..0028b184
--- /dev/null
+++ b/src/autoload.cpp
@@ -0,0 +1,382 @@
+/** \file autoload.cpp
+
+The classes responsible for autoloading functions and completions.
+*/
+
+#include "config.h" // IWYU pragma: keep
+#include "autoload.h"
+#include "wutil.h"
+#include "common.h"
+#include "signal.h" // IWYU pragma: keep - needed for CHECK_BLOCK
+#include "env.h"
+#include "exec.h"
+#include <assert.h>
+#include <errno.h>
+#include <sys/stat.h>
+#include <unistd.h>
+#include <wchar.h>
+#include <string>
+#include <utility>
+#include <vector>
+#include <algorithm>
+
+/* The time before we'll recheck an autoloaded file */
+static const int kAutoloadStalenessInterval = 15;
+
+file_access_attempt_t access_file(const wcstring &path, int mode)
+{
+    //printf("Touch %ls\n", path.c_str());
+    file_access_attempt_t result = {};
+    struct stat statbuf;
+    if (wstat(path, &statbuf))
+    {
+        result.error = errno;
+    }
+    else
+    {
+        result.mod_time = statbuf.st_mtime;
+        if (waccess(path, mode))
+        {
+            result.error = errno;
+        }
+        else
+        {
+            result.accessible = true;
+        }
+    }
+
+    // Note that we record the last checked time after the call, on the assumption that in a slow filesystem, the lag comes before the kernel check, not after.
+    result.stale = false;
+    result.last_checked = time(NULL);
+    return result;
+}
+
+autoload_t::autoload_t(const wcstring &env_var_name_var, const builtin_script_t * const scripts, size_t script_count) :
+    lock(),
+    env_var_name(env_var_name_var),
+    builtin_scripts(scripts),
+    builtin_script_count(script_count)
+{
+    pthread_mutex_init(&lock, NULL);
+}
+
+autoload_t::~autoload_t()
+{
+    pthread_mutex_destroy(&lock);
+}
+
+void autoload_t::node_was_evicted(autoload_function_t *node)
+{
+    // This should only ever happen on the main thread
+    ASSERT_IS_MAIN_THREAD();
+
+    // Tell ourselves that the command was removed if it was loaded
+    if (node->is_loaded)
+        this->command_removed(node->key);
+    delete node;
+}
+
+int autoload_t::unload(const wcstring &cmd)
+{
+    return this->evict_node(cmd);
+}
+
+int autoload_t::load(const wcstring &cmd, bool reload)
+{
+    int res;
+    CHECK_BLOCK(0);
+    ASSERT_IS_MAIN_THREAD();
+
+    env_var_t path_var = env_get_string(env_var_name);
+
+    /*
+      Do we know where to look?
+    */
+    if (path_var.empty())
+        return 0;
+
+    /* Check if the lookup path has changed. If so, drop all loaded files. path_var may only be inspected on the main thread. */
+    if (path_var != this->last_path)
+    {
+        this->last_path = path_var;
+        this->last_path_tokenized.clear();
+        tokenize_variable_array(this->last_path, this->last_path_tokenized);
+
+        scoped_lock locker(lock);
+        this->evict_all_nodes();
+    }
+
+    /* Mark that we're loading this. Hang onto the iterator for fast erasing later. Note that std::set has guarantees about not invalidating iterators, so this is safe to do across the callouts below.  */
+    typedef std::set<wcstring>::iterator set_iterator_t;
+    std::pair<set_iterator_t, bool> insert_result = is_loading_set.insert(cmd);
+    set_iterator_t where = insert_result.first;
+    bool inserted = insert_result.second;
+
+    /** Warn and fail on infinite recursion. It's OK to do this because this function is only called on the main thread. */
+    if (! inserted)
+    {
+        /* We failed to insert */
+        debug(0,
+              _(L"Could not autoload item '%ls', it is already being autoloaded. "
+                L"This is a circular dependency in the autoloading scripts, please remove it."),
+              cmd.c_str());
+        return 1;
+    }
+    /* Try loading it */
+    res = this->locate_file_and_maybe_load_it(cmd, true, reload, this->last_path_tokenized);
+
+    /* Clean up */
+    is_loading_set.erase(where);
+
+    return res;
+}
+
+bool autoload_t::can_load(const wcstring &cmd, const env_vars_snapshot_t &vars)
+{
+    const env_var_t path_var = vars.get(env_var_name);
+    if (path_var.missing_or_empty())
+        return false;
+
+    std::vector<wcstring> path_list;
+    tokenize_variable_array(path_var, path_list);
+    return this->locate_file_and_maybe_load_it(cmd, false, false, path_list);
+}
+
+static bool script_name_precedes_script_name(const builtin_script_t &script1, const builtin_script_t &script2)
+{
+    return wcscmp(script1.name, script2.name) < 0;
+}
+
+void autoload_t::unload_all(void)
+{
+    scoped_lock locker(lock);
+    this->evict_all_nodes();
+}
+
+/** Check whether the given command is loaded. */
+bool autoload_t::has_tried_loading(const wcstring &cmd)
+{
+    scoped_lock locker(lock);
+    autoload_function_t * func = this->get_node(cmd);
+    return func != NULL;
+}
+
+static bool is_stale(const autoload_function_t *func)
+{
+    /** Return whether this function is stale. Internalized functions can never be stale. */
+    return ! func->is_internalized && time(NULL) - func->access.last_checked > kAutoloadStalenessInterval;
+}
+
+autoload_function_t *autoload_t::get_autoloaded_function_with_creation(const wcstring &cmd, bool allow_eviction)
+{
+    ASSERT_IS_LOCKED(lock);
+    autoload_function_t *func = this->get_node(cmd);
+    if (! func)
+    {
+        func = new autoload_function_t(cmd);
+        if (allow_eviction)
+        {
+            this->add_node(func);
+        }
+        else
+        {
+            this->add_node_without_eviction(func);
+        }
+    }
+    return func;
+}
+
+/**
+   This internal helper function does all the real work. By using two
+   functions, the internal function can return on various places in
+   the code, and the caller can take care of various cleanup work.
+
+     cmd: the command name ('grep')
+     really_load: whether to actually parse it as a function, or just check it it exists
+     reload: whether to reload it if it's already loaded
+     path_list: the set of paths to check
+
+     Result: if really_load is true, returns whether the function was loaded. Otherwise returns whether the function existed.
+*/
+bool autoload_t::locate_file_and_maybe_load_it(const wcstring &cmd, bool really_load, bool reload, const wcstring_list_t &path_list)
+{
+    /* Note that we are NOT locked in this function! */
+    bool reloaded = 0;
+
+    /* Try using a cached function. If we really want the function to be loaded, require that it be really loaded. If we're not reloading, allow stale functions. */
+    {
+        bool allow_stale_functions = ! reload;
+
+        /* Take a lock */
+        scoped_lock locker(lock);
+
+        /* Get the function */
+        autoload_function_t * func = this->get_node(cmd);
+
+        /* Determine if we can use this cached function */
+        bool use_cached;
+        if (! func)
+        {
+            /* Can't use a function that doesn't exist */
+            use_cached = false;
+        }
+        else if (really_load && ! func->is_placeholder && ! func->is_loaded)
+        {
+            /* Can't use an unloaded function */
+            use_cached = false;
+        }
+        else if (! allow_stale_functions && is_stale(func))
+        {
+            /* Can't use a stale function */
+            use_cached = false;
+        }
+        else
+        {
+            /* I guess we can use it */
+            use_cached = true;
+        }
+
+        /* If we can use this function, return whether we were able to access it */
+        if (use_cached)
+        {
+            assert(func != NULL);
+            return func->is_internalized || func->access.accessible;
+        }
+    }
+    /* The source of the script will end up here */
+    wcstring script_source;
+    bool has_script_source = false;
+
+    /* Whether we found an accessible file */
+    bool found_file = false;
+
+    /* Look for built-in scripts via a binary search */
+    const builtin_script_t *matching_builtin_script = NULL;
+    if (builtin_script_count > 0)
+    {
+        const builtin_script_t test_script = {cmd.c_str(), NULL};
+        const builtin_script_t *array_end = builtin_scripts + builtin_script_count;
+        const builtin_script_t *found = std::lower_bound(builtin_scripts, array_end, test_script, script_name_precedes_script_name);
+        if (found != array_end && ! wcscmp(found->name, test_script.name))
+        {
+            /* We found it */
+            matching_builtin_script = found;
+        }
+    }
+    if (matching_builtin_script)
+    {
+        has_script_source = true;
+        script_source = str2wcstring(matching_builtin_script->def);
+
+        /* Make a node representing this function */
+        scoped_lock locker(lock);
+        autoload_function_t *func = this->get_autoloaded_function_with_creation(cmd, really_load);
+
+        /* This function is internalized */
+        func->is_internalized = true;
+
+        /* It's a fiction to say the script is loaded at this point, but we're definitely going to load it down below. */
+        if (really_load) func->is_loaded = true;
+    }
+
+    if (! has_script_source)
+    {
+        /* Iterate over path searching for suitable completion files */
+        for (size_t i=0; i<path_list.size(); i++)
+        {
+            wcstring next = path_list.at(i);
+            wcstring path = next + L"/" + cmd + L".fish";
+
+            const file_access_attempt_t access = access_file(path, R_OK);
+            if (access.accessible)
+            {
+                /* Found it! */
+                found_file = true;
+
+                /* Now we're actually going to take the lock. */
+                scoped_lock locker(lock);
+                autoload_function_t *func = this->get_node(cmd);
+
+                /* Generate the source if we need to load it */
+                bool need_to_load_function = really_load && (func == NULL || func->access.mod_time != access.mod_time || ! func->is_loaded);
+                if (need_to_load_function)
+                {
+
+                    /* Generate the script source */
+                    wcstring esc = escape_string(path, 1);
+                    script_source = L"source " + esc;
+                    has_script_source = true;
+
+                    /* Remove any loaded command because we are going to reload it. Note that this will deadlock if command_removed calls back into us. */
+                    if (func && func->is_loaded)
+                    {
+                        command_removed(cmd);
+                        func->is_placeholder = false;
+                    }
+
+                    /* Mark that we're reloading it */
+                    reloaded = true;
+                }
+
+                /* Create the function if we haven't yet. This does not load it. Do not trigger eviction unless we are actually loading, because we don't want to evict off of the main thread. */
+                if (! func)
+                {
+                    func = get_autoloaded_function_with_creation(cmd, really_load);
+                }
+
+                /* It's a fiction to say the script is loaded at this point, but we're definitely going to load it down below. */
+                if (need_to_load_function) func->is_loaded = true;
+
+                /* Unconditionally record our access time */
+                func->access = access;
+
+                break;
+            }
+        }
+
+        /*
+          If no file or builtin script was found we insert a placeholder function.
+          Later we only research if the current time is at least five seconds later.
+          This way, the files won't be searched over and over again.
+        */
+        if (! found_file && ! has_script_source)
+        {
+            scoped_lock locker(lock);
+            /* Generate a placeholder */
+            autoload_function_t *func = this->get_node(cmd);
+            if (! func)
+            {
+                func = new autoload_function_t(cmd);
+                func->is_placeholder = true;
+                if (really_load)
+                {
+                    this->add_node(func);
+                }
+                else
+                {
+                    this->add_node_without_eviction(func);
+                }
+            }
+            func->access.last_checked = time(NULL);
+        }
+    }
+
+    /* If we have a script, either built-in or a file source, then run it */
+    if (really_load && has_script_source)
+    {
+        if (exec_subshell(script_source, false /* do not apply exit status */) == -1)
+        {
+            /* Do nothing on failure */
+        }
+
+    }
+
+    if (really_load)
+    {
+        return reloaded;
+    }
+    else
+    {
+        return found_file || has_script_source;
+    }
+}
diff --git a/src/autoload.h b/src/autoload.h
new file mode 100644
index 00000000..610efed0
--- /dev/null
+++ b/src/autoload.h
@@ -0,0 +1,135 @@
+/** \file autoload.h
+
+    The classes responsible for autoloading functions and completions.
+*/
+
+#ifndef FISH_AUTOLOAD_H
+#define FISH_AUTOLOAD_H
+
+#include <pthread.h>
+#include <stddef.h>
+#include <time.h>
+#include <set>
+#include "common.h"
+#include "lru.h"
+
+/** A struct responsible for recording an attempt to access a file. */
+struct file_access_attempt_t
+{
+    time_t mod_time; /** The modification time of the file */
+    time_t last_checked; /** When we last checked the file */
+    bool accessible; /** Whether we believe we could access this file */
+    bool stale; /** Whether the access attempt is stale */
+    int error; /** If we could not access the file, the error code */
+};
+
+file_access_attempt_t access_file(const wcstring &path, int mode);
+
+struct autoload_function_t : public lru_node_t
+{
+    autoload_function_t(const wcstring &key) : lru_node_t(key), access(), is_loaded(false), is_placeholder(false), is_internalized(false) { }
+    file_access_attempt_t access; /** The last access attempt */
+    bool is_loaded; /** Whether we have actually loaded this function */
+    bool is_placeholder; /** Whether we are a placeholder that stands in for "no such function". If this is true, then is_loaded must be false. */
+    bool is_internalized; /** Whether this function came from a builtin "internalized" script */
+};
+
+struct builtin_script_t
+{
+    const wchar_t *name;
+    const char *def;
+};
+
+class env_vars_snapshot_t;
+
+/**
+  A class that represents a path from which we can autoload, and the autoloaded contents.
+ */
+class autoload_t : private lru_cache_t<autoload_function_t>
+{
+private:
+
+    /** Lock for thread safety */
+    pthread_mutex_t lock;
+
+    /** The environment variable name */
+    const wcstring env_var_name;
+
+    /** Builtin script array */
+    const struct builtin_script_t *const builtin_scripts;
+
+    /** Builtin script count */
+    const size_t builtin_script_count;
+
+    /** The path from which we most recently autoloaded */
+    wcstring last_path;
+
+    /** That path, tokenized (split on separators) */
+    wcstring_list_t last_path_tokenized;
+
+    /**
+       A table containing all the files that are currently being
+       loaded. This is here to help prevent recursion.
+    */
+    std::set<wcstring> is_loading_set;
+
+    void remove_all_functions(void)
+    {
+        this->evict_all_nodes();
+    }
+
+    bool locate_file_and_maybe_load_it(const wcstring &cmd, bool really_load, bool reload, const wcstring_list_t &path_list);
+
+    virtual void node_was_evicted(autoload_function_t *node);
+
+    autoload_function_t *get_autoloaded_function_with_creation(const wcstring &cmd, bool allow_eviction);
+
+protected:
+    /** Overridable callback for when a command is removed */
+    virtual void command_removed(const wcstring &cmd) { }
+
+public:
+
+    /** Create an autoload_t for the given environment variable name */
+    autoload_t(const wcstring &env_var_name_var, const builtin_script_t *scripts, size_t script_count);
+
+    /** Destructor */
+    virtual ~autoload_t();
+
+    /**
+       Autoload the specified file, if it exists in the specified path. Do
+       not load it multiple times unless its timestamp changes or
+       parse_util_unload is called.
+
+       Autoloading one file may unload another.
+
+       \param cmd the filename to search for. The suffix '.fish' is always added to this name
+       \param on_unload a callback function to run if a suitable file is found, which has not already been run. unload will also be called for old files which are unloaded.
+       \param reload wheter to recheck file timestamps on already loaded files
+    */
+    int load(const wcstring &cmd, bool reload);
+
+    /** Check whether we have tried loading the given command. Does not do any I/O. */
+    bool has_tried_loading(const wcstring &cmd);
+
+    /**
+       Tell the autoloader that the specified file, in the specified path,
+       is no longer loaded.
+
+       \param cmd the filename to search for. The suffix '.fish' is always added to this name
+       \param on_unload a callback function which will be called before (re)loading a file, may be used to unload the previous file.
+       \return non-zero if the file was removed, zero if the file had not yet been loaded
+    */
+    int unload(const wcstring &cmd);
+
+    /**
+       Unloads all files.
+    */
+    void unload_all();
+
+    /** Check whether the given command could be loaded, but do not load it. */
+    bool can_load(const wcstring &cmd, const env_vars_snapshot_t &vars);
+
+};
+
+#endif
diff --git a/src/builtin.cpp b/src/builtin.cpp
new file mode 100644
index 00000000..6a88ef5b
--- /dev/null
+++ b/src/builtin.cpp
@@ -0,0 +1,4276 @@
+/** \file builtin.c
+	Functions for executing builtin functions.
+
+	How to add a new builtin function:
+
+	1). Create a function in builtin.c with the following signature:
+
+	<tt>static int builtin_NAME( parser_t &parser, wchar_t ** args )</tt>
+
+	where NAME is the name of the builtin, and args is a zero-terminated list of arguments.
+
+	2). Add a line like { L"NAME", &builtin_NAME, N_(L"Bla bla bla") }, to the builtin_data_t variable. The description is used by the completion system. Note that this array is sorted!
+
+	3). Create a file doc_src/NAME.txt, containing the manual for the builtin in Doxygen-format. Check the other builtin manuals for proper syntax.
+
+	4). Use 'git add doc_src/NAME.txt' to start tracking changes to the documentation file.
+
+*/
+
+#include "config.h" // IWYU pragma: keep
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <wchar.h>
+#include <unistd.h>
+#include <errno.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <string.h>
+#include <signal.h>
+#include <wctype.h>
+#include <time.h>
+#include <stack>
+#include <assert.h>
+#include <algorithm>
+#include <map>
+#include <string>
+#include <utility>
+
+#include "fallback.h" // IWYU pragma: keep
+
+#include "wutil.h"
+#include "builtin.h"
+#include "function.h"
+#include "complete.h"
+#include "proc.h"
+#include "parser.h"
+#include "reader.h"
+#include "env.h"
+#include "wgetopt.h"
+#include "tokenizer.h"
+#include "input.h"
+#include "intern.h"
+#include "event.h"
+#include "signal.h"
+#include "exec.h"
+#include "highlight.h"
+#include "parse_util.h"
+#include "parser_keywords.h"
+#include "expand.h"
+#include "path.h"
+#include "history.h"
+#include "parse_tree.h"
+#include "parse_constants.h"
+#include "wcstringutil.h"
+
+/**
+   The default prompt for the read command
+*/
+#define DEFAULT_READ_PROMPT L"set_color green; echo -n read; set_color normal; echo -n \"> \""
+
+/**
+   The mode name to pass to history and input
+*/
+
+#define READ_MODE_NAME L"fish_read"
+
+/**
+   The send stuff to foreground message
+*/
+#define FG_MSG _( L"Send job %d, '%ls' to foreground\n" )
+
+/**
+   Datastructure to describe a builtin.
+*/
+struct builtin_data_t
+{
+    /**
+       Name of the builtin
+    */
+    const wchar_t *name;
+    /**
+       Function pointer tothe builtin implementation
+    */
+    int (*func)(parser_t &parser, wchar_t **argv);
+    /**
+       Description of what the builtin does
+    */
+    const wchar_t *desc;
+
+    bool operator<(const wcstring &) const;
+    bool operator<(const builtin_data_t *) const;
+};
+
+bool builtin_data_t::operator<(const wcstring &other) const
+{
+    return wcscmp(this->name, other.c_str()) < 0;
+}
+
+bool builtin_data_t::operator<(const builtin_data_t *other) const
+{
+    return wcscmp(this->name, other->name) < 0;
+}
+
+int builtin_out_redirect;
+int builtin_err_redirect;
+
+/* Buffers for storing the output of builtin functions */
+wcstring stdout_buffer, stderr_buffer;
+
+const wcstring &get_stdout_buffer()
+{
+    ASSERT_IS_MAIN_THREAD();
+    return stdout_buffer;
+}
+
+const wcstring &get_stderr_buffer()
+{
+    ASSERT_IS_MAIN_THREAD();
+    return stderr_buffer;
+}
+
+void builtin_show_error(const wcstring &err)
+{
+    ASSERT_IS_MAIN_THREAD();
+    stderr_buffer.append(err);
+}
+
+/**
+   Stack containing builtin I/O for recursive builtin calls.
+*/
+struct io_stack_elem_t
+{
+    int in;
+    wcstring out;
+    wcstring err;
+};
+static std::stack<io_stack_elem_t, std::vector<io_stack_elem_t> > io_stack;
+
+/**
+   The file from which builtin functions should attempt to read, use
+   instead of stdin.
+*/
+static int builtin_stdin;
+
+/**
+   The underlying IO redirections behind the current builtin. This
+   should normally not be used - sb_out and friends are already
+   configured to handle everything.
+*/
+static const io_chain_t *real_io;
+
+/**
+   Counts the number of non null pointers in the specified array
+*/
+static int builtin_count_args(const wchar_t * const * argv)
+{
+    int argc = 1;
+    while (argv[argc] != NULL)
+    {
+        argc++;
+    }
+    return argc;
+}
+
+/**
+	This function works like wperror, but it prints its result into
+	the sb_err string instead of to stderr. Used by the builtin
+	commands.
+*/
+
+static void builtin_wperror(const wchar_t *s)
+{
+    if (s != 0)
+    {
+        stderr_buffer.append(s);
+        stderr_buffer.append(L": ");
+    }
+    char *err = strerror(errno);
+    if (err)
+    {
+        const wcstring werr = str2wcstring(err);
+        stderr_buffer.append(werr);
+        stderr_buffer.push_back(L'\n');
+    }
+}
+
+/**
+   Count the number of times the specified character occurs in the specified string
+*/
+static int count_char(const wchar_t *str, wchar_t c)
+{
+    int res = 0;
+    for (; *str; str++)
+    {
+        res += (*str==c);
+    }
+    return res;
+}
+
+wcstring builtin_help_get(parser_t &parser, const wchar_t *name)
+{
+    /* This won't ever work if no_exec is set */
+    if (no_exec)
+        return wcstring();
+
+    wcstring_list_t lst;
+    wcstring out;
+    const wcstring name_esc = escape_string(name, 1);
+    wcstring cmd = format_string(L"__fish_print_help %ls", name_esc.c_str());
+    if (!builtin_out_redirect && isatty(1))
+    {
+        // since we're using a subshell, __fish_print_help can't tell we're in
+        // a terminal. Tell it ourselves.
+        int cols = common_get_width();
+        cmd = format_string(L"__fish_print_help --tty-width %d %ls", cols, name_esc.c_str());
+    }
+    if (exec_subshell(cmd, lst, false /* don't apply exit status */) >= 0)
+    {
+        for (size_t i=0; i<lst.size(); i++)
+        {
+            out.append(lst.at(i));
+            out.push_back(L'\n');
+        }
+    }
+    return out;
+}
+
+/**
+   Print help for the specified builtin. If \c b is sb_err, also print
+   the line information
+
+   If \c b is the buffer representing standard error, and the help
+   message is about to be printed to an interactive screen, it may be
+   shortened to fit the screen.
+
+
+*/
+
+static void builtin_print_help(parser_t &parser, const wchar_t *cmd, wcstring &b)
+{
+    if (&b == &stderr_buffer)
+    {
+        stderr_buffer.append(parser.current_line());
+    }
+
+    const wcstring h = builtin_help_get(parser, cmd);
+
+    if (!h.size())
+        return;
+
+    wchar_t *str = wcsdup(h.c_str());
+    if (str)
+    {
+        bool is_short = false;
+        if (&b == &stderr_buffer)
+        {
+
+            /*
+              Interactive mode help to screen - only print synopsis if
+              the rest won't fit
+            */
+
+            int screen_height, lines;
+
+            screen_height = common_get_height();
+            lines = count_char(str, L'\n');
+            if (!get_is_interactive() || (lines > 2*screen_height/3))
+            {
+                wchar_t *pos;
+                int cut=0;
+                int i;
+
+                is_short = true;
+
+                /*
+                  First move down 4 lines
+                */
+
+                pos = str;
+                for (i=0; (i<4) && pos && *pos; i++)
+                {
+                    pos = wcschr(pos+1, L'\n');
+                }
+
+                if (pos && *pos)
+                {
+
+                    /*
+                       Then find the next empty line
+                    */
+                    for (; *pos; pos++)
+                    {
+                        if (*pos == L'\n')
+                        {
+                            wchar_t *pos2;
+                            int is_empty = 1;
+
+                            for (pos2 = pos+1; *pos2; pos2++)
+                            {
+                                if (*pos2 == L'\n')
+                                    break;
+
+                                if (*pos2 != L'\t' && *pos2 !=L' ')
+                                {
+                                    is_empty = 0;
+                                    break;
+                                }
+                            }
+                            if (is_empty)
+                            {
+                                /*
+                                  And cut it
+                                */
+                                *(pos2+1)=L'\0';
+                                cut = 1;
+                                break;
+                            }
+                        }
+                    }
+                }
+
+                /*
+                  We did not find a good place to cut message to
+                  shorten it - so we make sure we don't print
+                  anything.
+                */
+                if (!cut)
+                {
+                    *str = 0;
+                }
+
+            }
+        }
+
+        b.append(str);
+        if (is_short)
+        {
+            append_format(b, _(L"%ls: Type 'help %ls' for related documentation\n\n"), cmd, cmd);
+        }
+
+        free(str);
+    }
+}
+
+/**
+   Perform error reporting for encounter with unknown option
+*/
+static void builtin_unknown_option(parser_t &parser, const wchar_t *cmd, const wchar_t *opt)
+{
+    append_format(stderr_buffer, BUILTIN_ERR_UNKNOWN, cmd, opt);
+    builtin_print_help(parser, cmd, stderr_buffer);
+}
+
+/**
+   Perform error reporting for encounter with missing argument
+*/
+static void builtin_missing_argument(parser_t &parser, const wchar_t *cmd, const wchar_t *opt)
+{
+    append_format(stderr_buffer, BUILTIN_ERR_MISSING, cmd, opt);
+    builtin_print_help(parser, cmd, stderr_buffer);
+}
+
+/*
+  Here follows the definition of all builtin commands. The function
+  names are all on the form builtin_NAME where NAME is the name of the
+  builtin. so the function name for the builtin 'fg' is
+  'builtin_fg'.
+
+  A few builtins, including 'while', 'command' and 'builtin' are not
+  defined here as they are handled directly by the parser. (They are
+  not parsed as commands, instead they only alter the parser state)
+
+  The builtins 'break' and 'continue' are so closely related that they
+  share the same implementation, namely 'builtin_break_continue.
+
+  Several other builtins, including jobs, ulimit and set are so big
+  that they have been given their own file. These files are all named
+  'builtin_NAME.c', where NAME is the name of the builtin. These files
+  are included directly below.
+
+*/
+
+
+#include "builtin_set.cpp"
+#include "builtin_commandline.cpp"
+#include "builtin_complete.cpp"
+#include "builtin_ulimit.cpp"
+#include "builtin_jobs.cpp"
+#include "builtin_set_color.cpp"
+#include "builtin_printf.cpp"
+
+/* builtin_test lives in builtin_test.cpp */
+int builtin_test(parser_t &parser, wchar_t **argv);
+
+/**
+   List a single key binding.
+   Returns false if no binding with that sequence and mode exists.
+ */
+static bool builtin_bind_list_one(const wcstring& seq, const wcstring& bind_mode)
+{
+    std::vector<wcstring> ecmds;
+    wcstring sets_mode;
+
+    if (!input_mapping_get(seq, bind_mode, &ecmds, &sets_mode))
+    {
+        return false;
+    }
+
+    stdout_buffer.append(L"bind");
+
+    // Append the mode flags if applicable
+    if (bind_mode != DEFAULT_BIND_MODE)
+    {
+        const wcstring emode = escape_string(bind_mode, ESCAPE_ALL);
+        stdout_buffer.append(L" -M ");
+        stdout_buffer.append(emode);
+    }
+    if (sets_mode != bind_mode)
+    {
+        const wcstring esets_mode = escape_string(sets_mode, ESCAPE_ALL);
+        stdout_buffer.append(L" -m ");
+        stdout_buffer.append(esets_mode);
+    }
+
+    // Append the name
+    wcstring tname;
+    if (input_terminfo_get_name(seq, &tname))
+    {
+        // Note that we show -k here because we have an input key name
+        append_format(stdout_buffer, L" -k %ls", tname.c_str());
+    }
+    else
+    {
+        // No key name, so no -k; we show the escape sequence directly
+        const wcstring eseq = escape_string(seq, ESCAPE_ALL);
+        append_format(stdout_buffer, L" %ls", eseq.c_str());
+    }
+
+    // Now show the list of commands
+    for (size_t i = 0; i < ecmds.size(); i++)
+    {
+        const wcstring &ecmd = ecmds.at(i);
+        const wcstring escaped_ecmd = escape_string(ecmd, ESCAPE_ALL);
+        stdout_buffer.push_back(' ');
+        stdout_buffer.append(escaped_ecmd);
+    }
+    stdout_buffer.push_back(L'\n');
+
+    return true;
+}
+
+/**
+   List all current key bindings
+ */
+static void builtin_bind_list(const wchar_t *bind_mode)
+{
+    const std::vector<input_mapping_name_t> lst = input_mapping_get_names();
+
+    for (std::vector<input_mapping_name_t>::const_iterator it = lst.begin(), end = lst.end();
+         it != end;
+         ++it)
+    {
+        if (bind_mode != NULL && bind_mode != it->mode)
+        {
+            continue;
+        }
+
+        builtin_bind_list_one(it->seq, it->mode);
+    }
+}
+
+/**
+   Print terminfo key binding names to string buffer used for standard output.
+
+   \param all if set, all terminfo key binding names will be
+   printed. If not set, only ones that are defined for this terminal
+   are printed.
+ */
+static void builtin_bind_key_names(int all)
+{
+    const wcstring_list_t names = input_terminfo_get_names(!all);
+    for (size_t i=0; i<names.size(); i++)
+    {
+        const wcstring &name = names.at(i);
+
+        append_format(stdout_buffer, L"%ls\n", name.c_str());
+    }
+}
+
+/**
+   Print all the special key binding functions to string buffer used for standard output.
+
+ */
+static void builtin_bind_function_names()
+{
+    wcstring_list_t names = input_function_get_names();
+
+    for (size_t i=0; i<names.size(); i++)
+    {
+        const wchar_t *seq = names.at(i).c_str();
+        append_format(stdout_buffer, L"%ls\n", seq);
+    }
+}
+
+// Wraps input_terminfo_get_sequence(), appending the correct error messages as needed.
+static int get_terminfo_sequence(const wchar_t *seq, wcstring *out_seq)
+{
+    if (input_terminfo_get_sequence(seq, out_seq))
+    {
+        return 1;
+    }
+    wcstring eseq = escape_string(seq, 0);
+    switch (errno)
+    {
+        case ENOENT:
+        {
+            append_format(stderr_buffer, _(L"%ls: No key with name '%ls' found\n"), L"bind", eseq.c_str());
+            break;
+        }
+
+        case EILSEQ:
+        {
+            append_format(stderr_buffer, _(L"%ls: Key with name '%ls' does not have any mapping\n"), L"bind", eseq.c_str());
+            break;
+        }
+
+        default:
+        {
+            append_format(stderr_buffer, _(L"%ls: Unknown error trying to bind to key named '%ls'\n"), L"bind", eseq.c_str());
+            break;
+        }
+    }
+    return 0;
+}
+
+/**
+   Add specified key binding.
+ */
+static int builtin_bind_add(const wchar_t *seq, const wchar_t **cmds, size_t cmds_len,
+                            const wchar_t *mode, const wchar_t *sets_mode, int terminfo)
+{
+
+    if (terminfo)
+    {
+        wcstring seq2;
+        if (get_terminfo_sequence(seq, &seq2))
+        {
+            input_mapping_add(seq2.c_str(), cmds, cmds_len, mode, sets_mode);
+        }
+        else
+        {
+            return 1;
+        }
+
+    }
+    else
+    {
+        input_mapping_add(seq, cmds, cmds_len, mode, sets_mode);
+    }
+
+    return 0;
+
+}
+
+/**
+   Erase specified key bindings
+
+   \param seq an array of all key bindings to erase
+   \param all if specified, _all_ key bindings will be erased
+   \param mode if specified, only bindings from that mode will be erased. If not given and \c all is \c false, \c DEFAULT_BIND_MODE will be used.
+ */
+static int builtin_bind_erase(wchar_t **seq, int all, const wchar_t *mode, int use_terminfo)
+{
+    if (all)
+    {
+        const std::vector<input_mapping_name_t> lst = input_mapping_get_names();
+        for (std::vector<input_mapping_name_t>::const_iterator it = lst.begin(), end = lst.end();
+             it != end;
+             ++it)
+        {
+            if (mode == NULL || mode == it->mode)
+            {
+                input_mapping_erase(it->seq, it->mode);
+            }
+        }
+
+        return 0;
+    }
+    else
+    {
+        int res = 0;
+
+        if (mode == NULL) mode = DEFAULT_BIND_MODE;
+
+        while (*seq)
+        {
+            if (use_terminfo)
+            {
+                wcstring seq2;
+                if (get_terminfo_sequence(*seq++, &seq2))
+                {
+                    input_mapping_erase(seq2, mode);
+                }
+                else
+                {
+                    res = 1;
+                }
+            }
+            else
+            {
+                input_mapping_erase(*seq++, mode);
+            }
+        }
+
+        return res;
+    }
+}
+
+
+/**
+   The bind builtin, used for setting character sequences
+*/
+static int builtin_bind(parser_t &parser, wchar_t **argv)
+{
+    wgetopter_t w;
+    enum
+    {
+        BIND_INSERT,
+        BIND_ERASE,
+        BIND_KEY_NAMES,
+        BIND_FUNCTION_NAMES
+    };
+
+    int argc=builtin_count_args(argv);
+    int mode = BIND_INSERT;
+    int res = STATUS_BUILTIN_OK;
+    int all = 0;
+
+    const wchar_t *bind_mode = DEFAULT_BIND_MODE;
+    bool bind_mode_given = false;
+    const wchar_t *sets_bind_mode = DEFAULT_BIND_MODE;
+    bool sets_bind_mode_given = false;
+
+    int use_terminfo = 0;
+
+    w.woptind=0;
+
+    static const struct woption long_options[] =
+    {
+        { L"all", no_argument, 0, 'a' },
+        { L"erase", no_argument, 0, 'e' },
+        { L"function-names", no_argument, 0, 'f' },
+        { L"help", no_argument, 0, 'h' },
+        { L"key", no_argument, 0, 'k' },
+        { L"key-names", no_argument, 0, 'K' },
+        { L"mode", required_argument, 0, 'M' },
+        { L"sets-mode", required_argument, 0, 'm' },
+        { 0, 0, 0, 0 }
+    };
+
+    while (1)
+    {
+        int opt_index = 0;
+        int opt = w.wgetopt_long(argc,
+                                 argv,
+                                 L"aehkKfM:m:",
+                                 long_options,
+                                 &opt_index);
+
+        if (opt == -1)
+            break;
+
+        switch (opt)
+        {
+            case 0:
+                if (long_options[opt_index].flag != 0)
+                    break;
+                append_format(stderr_buffer,
+                              BUILTIN_ERR_UNKNOWN,
+                              argv[0],
+                              long_options[opt_index].name);
+                builtin_print_help(parser, argv[0], stderr_buffer);
+
+                return STATUS_BUILTIN_ERROR;
+
+            case 'a':
+                all = 1;
+                break;
+
+            case 'e':
+                mode = BIND_ERASE;
+                break;
+
+            case 'h':
+                builtin_print_help(parser, argv[0], stdout_buffer);
+                return STATUS_BUILTIN_OK;
+
+            case 'k':
+                use_terminfo = 1;
+                break;
+
+            case 'K':
+                mode = BIND_KEY_NAMES;
+                break;
+
+            case 'f':
+                mode = BIND_FUNCTION_NAMES;
+                break;
+
+            case 'M':
+                bind_mode = w.woptarg;
+                bind_mode_given = true;
+                break;
+
+            case 'm':
+                sets_bind_mode = w.woptarg;
+                sets_bind_mode_given = true;
+                break;
+
+            case '?':
+                builtin_unknown_option(parser, argv[0], argv[w.woptind-1]);
+                return STATUS_BUILTIN_ERROR;
+
+
+        }
+    }
+
+    /*
+     * if mode is given, but not new mode, default to new mode to mode
+     */
+    if (bind_mode_given && !sets_bind_mode_given)
+    {
+        sets_bind_mode = bind_mode;
+    }
+
+    switch (mode)
+    {
+
+        case BIND_ERASE:
+        {
+            if (builtin_bind_erase(&argv[w.woptind], all, bind_mode_given ? bind_mode : NULL, use_terminfo))
+            {
+                res = STATUS_BUILTIN_ERROR;
+            }
+            break;
+        }
+
+        case BIND_INSERT:
+        {
+            switch (argc-w.woptind)
+            {
+                case 0:
+                {
+                    builtin_bind_list(bind_mode_given ? bind_mode : NULL);
+                    break;
+                }
+
+                case 1:
+                {
+                    wcstring seq;
+                    if (use_terminfo)
+                    {
+                        if (!get_terminfo_sequence(argv[w.woptind], &seq))
+                        {
+                            res = STATUS_BUILTIN_ERROR;
+                            // get_terminfo_sequence already printed the error
+                            break;
+                        }
+                    }
+                    else
+                    {
+                        seq = argv[w.woptind];
+                    }
+                    if (!builtin_bind_list_one(seq, bind_mode))
+                    {
+                        res = STATUS_BUILTIN_ERROR;
+                        wcstring eseq = escape_string(argv[w.woptind], 0);
+                        if (use_terminfo)
+                        {
+                            append_format(stderr_buffer, _(L"%ls: No binding found for key '%ls'\n"), argv[0], eseq.c_str());
+                        }
+                        else
+                        {
+                            append_format(stderr_buffer, _(L"%ls: No binding found for sequence '%ls'\n"), argv[0], eseq.c_str());
+                        }
+                    }
+                    break;
+                }
+
+                default:
+                {
+                    if (builtin_bind_add(argv[w.woptind], (const wchar_t **)argv + (w.woptind + 1), argc - (w.woptind + 1), bind_mode, sets_bind_mode, use_terminfo))
+                    {
+                        res = STATUS_BUILTIN_ERROR;
+                    }
+                    break;
+                }
+
+            }
+            break;
+        }
+
+        case BIND_KEY_NAMES:
+        {
+            builtin_bind_key_names(all);
+            break;
+        }
+
+
+        case BIND_FUNCTION_NAMES:
+        {
+            builtin_bind_function_names();
+            break;
+        }
+
+
+        default:
+        {
+            res = STATUS_BUILTIN_ERROR;
+            append_format(stderr_buffer, _(L"%ls: Invalid state\n"), argv[0]);
+            break;
+        }
+    }
+
+    return res;
+}
+
+
+/**
+   The block builtin, used for temporarily blocking events
+*/
+static int builtin_block(parser_t &parser, wchar_t **argv)
+{
+    wgetopter_t w;
+    enum
+    {
+        UNSET,
+        GLOBAL,
+        LOCAL,
+    }
+    ;
+
+    int scope=UNSET;
+    int erase = 0;
+    int argc=builtin_count_args(argv);
+
+    w.woptind=0;
+
+    static const struct woption
+            long_options[] =
+    {
+        {
+            L"erase", no_argument, 0, 'e'
+        }
+        ,
+        {
+            L"local", no_argument, 0, 'l'
+        }
+        ,
+        {
+            L"global", no_argument, 0, 'g'
+        }
+        ,
+        {
+            L"help", no_argument, 0, 'h'
+        }
+        ,
+        {
+            0, 0, 0, 0
+        }
+    }
+    ;
+
+    while (1)
+    {
+        int opt_index = 0;
+
+        int opt = w.wgetopt_long(argc,
+                                 argv,
+                                 L"elgh",
+                                 long_options,
+                                 &opt_index);
+        if (opt == -1)
+            break;
+
+        switch (opt)
+        {
+            case 0:
+                if (long_options[opt_index].flag != 0)
+                    break;
+                append_format(stderr_buffer,
+                              BUILTIN_ERR_UNKNOWN,
+                              argv[0],
+                              long_options[opt_index].name);
+                builtin_print_help(parser, argv[0], stderr_buffer);
+
+                return STATUS_BUILTIN_ERROR;
+            case 'h':
+                builtin_print_help(parser, argv[0], stdout_buffer);
+                return STATUS_BUILTIN_OK;
+
+            case 'g':
+                scope = GLOBAL;
+                break;
+
+            case 'l':
+                scope = LOCAL;
+                break;
+
+            case 'e':
+                erase = 1;
+                break;
+
+            case '?':
+                builtin_unknown_option(parser, argv[0], argv[w.woptind-1]);
+                return STATUS_BUILTIN_ERROR;
+
+        }
+
+    }
+
+    if (erase)
+    {
+        if (scope != UNSET)
+        {
+            append_format(stderr_buffer, _(L"%ls: Can not specify scope when removing block\n"), argv[0]);
+            return STATUS_BUILTIN_ERROR;
+        }
+
+        if (parser.global_event_blocks.empty())
+        {
+            append_format(stderr_buffer, _(L"%ls: No blocks defined\n"), argv[0]);
+            return STATUS_BUILTIN_ERROR;
+        }
+        parser.global_event_blocks.pop_front();
+    }
+    else
+    {
+        size_t block_idx = 0;
+        block_t *block = parser.block_at_index(block_idx);
+
+        event_blockage_t eb = {};
+        eb.typemask = (1<<EVENT_ANY);
+
+        switch (scope)
+        {
+            case LOCAL:
+            {
+                // If this is the outermost block, then we're global
+                if (block_idx + 1 >= parser.block_count())
+                {
+                    block = NULL;
+                }
+                break;
+            }
+            case GLOBAL:
+            {
+                block=NULL;
+            }
+            case UNSET:
+            {
+                while (block != NULL && block->type() != FUNCTION_CALL && block->type() != FUNCTION_CALL_NO_SHADOW)
+                {
+                    // Set it in function scope
+                    block = parser.block_at_index(++block_idx);
+                }
+            }
+        }
+        if (block)
+        {
+            block->event_blocks.push_front(eb);
+        }
+        else
+        {
+            parser.global_event_blocks.push_front(eb);
+        }
+    }
+
+    return STATUS_BUILTIN_OK;
+
+}
+
+/**
+   The builtin builtin, used for giving builtins precedence over
+   functions. Mostly handled by the parser. All this code does is some
+   additional operational modes, such as printing a list of all
+   builtins, printing help, etc.
+*/
+static int builtin_builtin(parser_t &parser, wchar_t **argv)
+{
+    int argc=builtin_count_args(argv);
+    int list=0;
+    wgetopter_t w;
+
+    static const struct woption
+            long_options[] =
+    {
+        {
+            L"names", no_argument, 0, 'n'
+        }
+        ,
+        {
+            L"help", no_argument, 0, 'h'
+        }
+        ,
+        {
+            0, 0, 0, 0
+        }
+    }
+    ;
+
+    while (1)
+    {
+        int opt_index = 0;
+
+        int opt = w.wgetopt_long(argc,
+                                 argv,
+                                 L"nh",
+                                 long_options,
+                                 &opt_index);
+        if (opt == -1)
+            break;
+
+        switch (opt)
+        {
+            case 0:
+                if (long_options[opt_index].flag != 0)
+                    break;
+                append_format(stderr_buffer,
+                              BUILTIN_ERR_UNKNOWN,
+                              argv[0],
+                              long_options[opt_index].name);
+                builtin_print_help(parser, argv[0], stderr_buffer);
+
+
+                return STATUS_BUILTIN_ERROR;
+            case 'h':
+                builtin_print_help(parser, argv[0], stdout_buffer);
+                return STATUS_BUILTIN_OK;
+
+            case 'n':
+                list=1;
+                break;
+
+            case '?':
+                builtin_unknown_option(parser, argv[0], argv[w.woptind-1]);
+                return STATUS_BUILTIN_ERROR;
+
+        }
+
+    }
+
+    if (list)
+    {
+        wcstring_list_t names = builtin_get_names();
+        sort(names.begin(), names.end());
+
+        for (size_t i=0; i<names.size(); i++)
+        {
+            const wchar_t *el = names.at(i).c_str();
+
+            stdout_buffer.append(el);
+            stdout_buffer.append(L"\n");
+        }
+    }
+    return STATUS_BUILTIN_OK;
+}
+
+/**
+   Implementation of the builtin emit command, used to create events.
+ */
+static int builtin_emit(parser_t &parser, wchar_t **argv)
+{
+    wgetopter_t w;
+    int argc=builtin_count_args(argv);
+
+    static const struct woption
+            long_options[] =
+    {
+        {
+            L"help", no_argument, 0, 'h'
+        }
+        ,
+        {
+            0, 0, 0, 0
+        }
+    }
+    ;
+
+    while (1)
+    {
+        int opt_index = 0;
+
+        int opt = w.wgetopt_long(argc,
+                                 argv,
+                                 L"h",
+                                 long_options,
+                                 &opt_index);
+        if (opt == -1)
+            break;
+
+        switch (opt)
+        {
+            case 0:
+                if (long_options[opt_index].flag != 0)
+                    break;
+                append_format(stderr_buffer,
+                              BUILTIN_ERR_UNKNOWN,
+                              argv[0],
+                              long_options[opt_index].name);
+                builtin_print_help(parser, argv[0], stderr_buffer);
+                return STATUS_BUILTIN_ERROR;
+
+            case 'h':
+                builtin_print_help(parser, argv[0], stdout_buffer);
+                return STATUS_BUILTIN_OK;
+
+            case '?':
+                builtin_unknown_option(parser, argv[0], argv[w.woptind-1]);
+                return STATUS_BUILTIN_ERROR;
+
+        }
+
+    }
+
+    if (!argv[w.woptind])
+    {
+        append_format(stderr_buffer, L"%ls: expected event name\n", argv[0]);
+        return STATUS_BUILTIN_ERROR;
+    }
+    const wchar_t *eventname = argv[w.woptind];
+    wcstring_list_t args(argv + w.woptind + 1, argv + argc);
+    event_fire_generic(eventname, &args);
+
+    return STATUS_BUILTIN_OK;
+}
+
+
+/**
+   Implementation of the builtin 'command'. Actual command running is handled by
+   the parser, this just processes the flags.
+*/
+static int builtin_command(parser_t &parser, wchar_t **argv)
+{
+    wgetopter_t w;
+    int argc=builtin_count_args(argv);
+    int print_path=0;
+
+    w.woptind=0;
+
+    static const struct woption
+            long_options[] =
+    {
+        { L"search", no_argument, 0, 's' },
+        { L"help", no_argument, 0, 'h' },
+        { 0, 0, 0, 0 }
+    };
+
+    while (1)
+    {
+        int opt_index = 0;
+
+        int opt = w.wgetopt_long(argc,
+                                 argv,
+                                 L"svh",
+                                 long_options,
+                                 &opt_index);
+        if (opt == -1)
+            break;
+
+        switch (opt)
+        {
+            case 0:
+                if (long_options[opt_index].flag != 0)
+                    break;
+                append_format(stderr_buffer,
+                              BUILTIN_ERR_UNKNOWN,
+                              argv[0],
+                              long_options[opt_index].name);
+                builtin_print_help(parser, argv[0], stderr_buffer);
+                return STATUS_BUILTIN_ERROR;
+
+            case 'h':
+                builtin_print_help(parser, argv[0], stdout_buffer);
+                return STATUS_BUILTIN_OK;
+
+            case 's':
+            case 'v':
+                print_path=1;
+                break;
+
+            case '?':
+                builtin_unknown_option(parser, argv[0], argv[w.woptind-1]);
+                return STATUS_BUILTIN_ERROR;
+
+        }
+
+    }
+
+    if (!print_path)
+    {
+        builtin_print_help(parser, argv[0], stdout_buffer);
+        return STATUS_BUILTIN_ERROR;
+    }
+
+    int found=0;
+
+    for (int idx = w.woptind; argv[idx]; ++idx)
+    {
+        const wchar_t *command_name = argv[idx];
+        wcstring path;
+        if (path_get_path(command_name, &path))
+        {
+            append_format(stdout_buffer, L"%ls\n", path.c_str());
+            ++found;
+        }
+    }
+    return found ? STATUS_BUILTIN_OK : STATUS_BUILTIN_ERROR;
+}
+
+/**
+   A generic bultin that only supports showing a help message. This is
+   only a placeholder that prints the help message. Useful for
+   commands that live in the parser.
+*/
+static int builtin_generic(parser_t &parser, wchar_t **argv)
+{
+    wgetopter_t w;
+    int argc=builtin_count_args(argv);
+
+    /* Hackish - if we have no arguments other than the command, we are a "naked invocation" and we just print help */
+    if (argc == 1)
+    {
+        builtin_print_help(parser, argv[0], stdout_buffer);
+        return STATUS_BUILTIN_ERROR;
+    }
+
+    static const struct woption
+            long_options[] =
+    {
+        { L"help", no_argument, 0, 'h' },
+        { 0, 0, 0, 0 }
+    };
+
+    while (1)
+    {
+        int opt_index = 0;
+
+        int opt = w.wgetopt_long(argc,
+                                 argv,
+                                 L"h",
+                                 long_options,
+                                 &opt_index);
+        if (opt == -1)
+            break;
+
+        switch (opt)
+        {
+            case 0:
+                if (long_options[opt_index].flag != 0)
+                    break;
+                append_format(stderr_buffer,
+                              BUILTIN_ERR_UNKNOWN,
+                              argv[0],
+                              long_options[opt_index].name);
+                builtin_print_help(parser, argv[0], stderr_buffer);
+                return STATUS_BUILTIN_ERROR;
+
+            case 'h':
+                builtin_print_help(parser, argv[0], stdout_buffer);
+                return STATUS_BUILTIN_OK;
+
+            case '?':
+                builtin_unknown_option(parser, argv[0], argv[w.woptind-1]);
+                return STATUS_BUILTIN_ERROR;
+
+        }
+
+    }
+    return STATUS_BUILTIN_ERROR;
+}
+
+/**
+   Output a definition of the specified function to the specified
+   string. Used by the functions builtin.
+*/
+static void functions_def(const wcstring &name, wcstring &out)
+{
+    CHECK(! name.empty(),);
+
+    wcstring desc, def;
+    function_get_desc(name, &desc);
+    function_get_definition(name, &def);
+
+    event_t search(EVENT_ANY);
+
+    search.function_name = name;
+
+    std::vector<event_t *> ev;
+    event_get(search, &ev);
+
+    out.append(L"function ");
+
+    /* Typically we prefer to specify the function name first, e.g. "function foo --description bar"
+       But If the function name starts with a -, we'll need to output it after all the options. */
+    bool defer_function_name = (name.at(0) == L'-');
+    if (! defer_function_name)
+    {
+        out.append(escape_string(name, true));
+    }
+
+    if (! desc.empty())
+    {
+        wcstring esc_desc = escape_string(desc, true);
+        out.append(L" --description ");
+        out.append(esc_desc);
+    }
+
+    if (!function_get_shadows(name))
+    {
+        out.append(L" --no-scope-shadowing");
+    }
+
+    for (size_t i=0; i<ev.size(); i++)
+    {
+        event_t *next = ev.at(i);
+        switch (next->type)
+        {
+            case EVENT_SIGNAL:
+            {
+                append_format(out, L" --on-signal %ls", sig2wcs(next->param1.signal));
+                break;
+            }
+
+            case EVENT_VARIABLE:
+            {
+                append_format(out, L" --on-variable %ls", next->str_param1.c_str());
+                break;
+            }
+
+            case EVENT_EXIT:
+            {
+                if (next->param1.pid > 0)
+                    append_format(out, L" --on-process-exit %d", next->param1.pid);
+                else
+                    append_format(out, L" --on-job-exit %d", -next->param1.pid);
+                break;
+            }
+
+            case EVENT_JOB_ID:
+            {
+                const job_t *j = job_get(next->param1.job_id);
+                if (j)
+                    append_format(out, L" --on-job-exit %d", j->pgid);
+                break;
+            }
+
+            case EVENT_GENERIC:
+            {
+                append_format(out, L" --on-event %ls", next->str_param1.c_str());
+                break;
+            }
+
+        }
+
+    }
+
+
+    wcstring_list_t named = function_get_named_arguments(name);
+    if (! named.empty())
+    {
+        append_format(out, L" --argument");
+        for (size_t i=0; i < named.size(); i++)
+        {
+            append_format(out, L" %ls", named.at(i).c_str());
+        }
+    }
+
+    /* Output the function name if we deferred it */
+    if (defer_function_name)
+    {
+        out.append(L" -- ");
+        out.append(escape_string(name, true));
+    }
+
+    /* Output any inherited variables as `set -l` lines */
+    std::map<wcstring,env_var_t> inherit_vars = function_get_inherit_vars(name);
+    for (std::map<wcstring,env_var_t>::const_iterator it = inherit_vars.begin(), end = inherit_vars.end(); it != end; ++it)
+    {
+        wcstring_list_t lst;
+        if (!it->second.missing())
+        {
+            tokenize_variable_array(it->second, lst);
+        }
+
+        /* This forced tab is crummy, but we don't know what indentation style the function uses */
+        append_format(out, L"\n\tset -l %ls", it->first.c_str());
+        for (wcstring_list_t::const_iterator arg_it = lst.begin(), arg_end = lst.end(); arg_it != arg_end; ++arg_it)
+        {
+            wcstring earg = escape_string(*arg_it, ESCAPE_ALL);
+            out.push_back(L' ');
+            out.append(earg);
+        }
+    }
+
+    /* This forced tab is sort of crummy - not all functions start with a tab */
+    append_format(out, L"\n\t%ls", def.c_str());
+
+    /* Append a newline before the 'end', unless there already is one there */
+    if (! string_suffixes_string(L"\n", def))
+    {
+        out.push_back(L'\n');
+    }
+    out.append(L"end\n");
+}
+
+
+/**
+   The functions builtin, used for listing and erasing functions.
+*/
+static int builtin_functions(parser_t &parser, wchar_t **argv)
+{
+    wgetopter_t w;
+    int i;
+    int erase=0;
+    wchar_t *desc=0;
+
+    int argc=builtin_count_args(argv);
+    int list=0;
+    int show_hidden=0;
+    int res = STATUS_BUILTIN_OK;
+    int query = 0;
+    int copy = 0;
+
+    static const struct woption
+            long_options[] =
+    {
+        {
+            L"erase", no_argument, 0, 'e'
+        }
+        ,
+        {
+            L"description", required_argument, 0, 'd'
+        }
+        ,
+        {
+            L"names", no_argument, 0, 'n'
+        }
+        ,
+        {
+            L"all", no_argument, 0, 'a'
+        }
+        ,
+        {
+            L"help", no_argument, 0, 'h'
+        }
+        ,
+        {
+            L"query", no_argument, 0, 'q'
+        }
+        ,
+        {
+            L"copy", no_argument, 0, 'c'
+        }
+        ,
+        {
+            0, 0, 0, 0
+        }
+    }
+    ;
+
+    while (1)
+    {
+        int opt_index = 0;
+
+        int opt = w.wgetopt_long(argc,
+                                 argv,
+                                 L"ed:nahqc",
+                                 long_options,
+                                 &opt_index);
+        if (opt == -1)
+            break;
+
+        switch (opt)
+        {
+            case 0:
+                if (long_options[opt_index].flag != 0)
+                    break;
+                append_format(stderr_buffer,
+                              BUILTIN_ERR_UNKNOWN,
+                              argv[0],
+                              long_options[opt_index].name);
+                builtin_print_help(parser, argv[0], stderr_buffer);
+
+
+                return STATUS_BUILTIN_ERROR;
+
+            case 'e':
+                erase=1;
+                break;
+
+            case 'd':
+                desc=w.woptarg;
+                break;
+
+            case 'n':
+                list=1;
+                break;
+
+            case 'a':
+                show_hidden=1;
+                break;
+
+            case 'h':
+                builtin_print_help(parser, argv[0], stdout_buffer);
+                return STATUS_BUILTIN_OK;
+
+            case 'q':
+                query = 1;
+                break;
+
+            case 'c':
+                copy = 1;
+                break;
+
+            case '?':
+                builtin_unknown_option(parser, argv[0], argv[w.woptind-1]);
+                return STATUS_BUILTIN_ERROR;
+
+        }
+
+    }
+
+    /*
+      Erase, desc, query, copy and list are mutually exclusive
+    */
+    if ((erase + (!!desc) + list + query + copy) > 1)
+    {
+        append_format(stderr_buffer,
+                      _(L"%ls: Invalid combination of options\n"),
+                      argv[0]);
+
+        builtin_print_help(parser, argv[0], stderr_buffer);
+
+        return STATUS_BUILTIN_ERROR;
+    }
+
+    if (erase)
+    {
+        int i;
+        for (i=w.woptind; i<argc; i++)
+            function_remove(argv[i]);
+        return STATUS_BUILTIN_OK;
+    }
+    else if (desc)
+    {
+        wchar_t *func;
+
+        if (argc-w.woptind != 1)
+        {
+            append_format(stderr_buffer,
+                          _(L"%ls: Expected exactly one function name\n"),
+                          argv[0]);
+            builtin_print_help(parser, argv[0], stderr_buffer);
+
+            return STATUS_BUILTIN_ERROR;
+        }
+        func = argv[w.woptind];
+        if (!function_exists(func))
+        {
+            append_format(stderr_buffer,
+                          _(L"%ls: Function '%ls' does not exist\n"),
+                          argv[0],
+                          func);
+
+            builtin_print_help(parser, argv[0], stderr_buffer);
+
+            return STATUS_BUILTIN_ERROR;
+        }
+
+        function_set_desc(func, desc);
+
+        return STATUS_BUILTIN_OK;
+    }
+    else if (list || (argc==w.woptind))
+    {
+        int is_screen = !builtin_out_redirect && isatty(1);
+        size_t i;
+        wcstring_list_t names = function_get_names(show_hidden);
+        std::sort(names.begin(), names.end());
+        if (is_screen)
+        {
+            wcstring buff;
+
+            for (i=0; i<names.size(); i++)
+            {
+                buff.append(names.at(i));
+                buff.append(L", ");
+            }
+
+            write_screen(buff, stdout_buffer);
+        }
+        else
+        {
+            for (i=0; i<names.size(); i++)
+            {
+                stdout_buffer.append(names.at(i).c_str());
+                stdout_buffer.append(L"\n");
+            }
+        }
+
+        return STATUS_BUILTIN_OK;
+    }
+    else if (copy)
+    {
+        wcstring current_func;
+        wcstring new_func;
+
+        if (argc-w.woptind != 2)
+        {
+            append_format(stderr_buffer,
+                          _(L"%ls: Expected exactly two names (current function name, and new function name)\n"),
+                          argv[0]);
+            builtin_print_help(parser, argv[0], stderr_buffer);
+
+            return STATUS_BUILTIN_ERROR;
+        }
+        current_func = argv[w.woptind];
+        new_func = argv[w.woptind+1];
+
+        if (!function_exists(current_func))
+        {
+            append_format(stderr_buffer,
+                          _(L"%ls: Function '%ls' does not exist\n"),
+                          argv[0],
+                          current_func.c_str());
+            builtin_print_help(parser, argv[0], stderr_buffer);
+
+            return STATUS_BUILTIN_ERROR;
+        }
+
+        if ((wcsfuncname(new_func) != 0) || parser_keywords_is_reserved(new_func))
+        {
+            append_format(stderr_buffer,
+                          _(L"%ls: Illegal function name '%ls'\n"),
+                          argv[0],
+                          new_func.c_str());
+            builtin_print_help(parser, argv[0], stderr_buffer);
+            return STATUS_BUILTIN_ERROR;
+        }
+
+        // keep things simple: don't allow existing names to be copy targets.
+        if (function_exists(new_func))
+        {
+            append_format(stderr_buffer,
+                          _(L"%ls: Function '%ls' already exists. Cannot create copy '%ls'\n"),
+                          argv[0],
+                          new_func.c_str(),
+                          current_func.c_str());
+            builtin_print_help(parser, argv[0], stderr_buffer);
+
+            return STATUS_BUILTIN_ERROR;
+        }
+
+        if (function_copy(current_func, new_func))
+            return STATUS_BUILTIN_OK;
+        return STATUS_BUILTIN_ERROR;
+    }
+
+    for (i=w.woptind; i<argc; i++)
+    {
+        if (!function_exists(argv[i]))
+            res++;
+        else
+        {
+            if (!query)
+            {
+                if (i != w.woptind)
+                    stdout_buffer.append(L"\n");
+
+                functions_def(argv[i], stdout_buffer);
+            }
+        }
+    }
+
+    return res;
+}
+
+static unsigned int builtin_echo_digit(wchar_t wc, unsigned int base)
+{
+    // base must be hex or octal
+    assert(base == 8 || base == 16);
+    switch (wc)
+    {
+        case L'0':
+            return 0;
+        case L'1':
+            return 1;
+        case L'2':
+            return 2;
+        case L'3':
+            return 3;
+        case L'4':
+            return 4;
+        case L'5':
+            return 5;
+        case L'6':
+            return 6;
+        case L'7':
+            return 7;
+    }
+
+    if (base == 16) switch (wc)
+        {
+            case L'8':
+                return 8;
+            case L'9':
+                return 9;
+            case L'a':
+            case L'A':
+                return 10;
+            case L'b':
+            case L'B':
+                return 11;
+            case L'c':
+            case L'C':
+                return 12;
+            case L'd':
+            case L'D':
+                return 13;
+            case L'e':
+            case L'E':
+                return 14;
+            case L'f':
+            case L'F':
+                return 15;
+        }
+    return UINT_MAX;
+}
+
+/* Parse a numeric escape sequence in str, returning whether we succeeded.
+   Also return the number of characters consumed and the resulting value.
+   Supported escape sequences:
+
+   \0nnn: octal value, zero to three digits
+   \nnn: octal value, one to three digits
+   \xhh: hex value, one to two digits
+*/
+static bool builtin_echo_parse_numeric_sequence(const wchar_t *str, size_t *consumed, unsigned char *out_val)
+{
+    bool success = false;
+    unsigned char val = 0; //resulting character
+    unsigned int start = 0; //the first character of the numeric part of the sequence
+
+    unsigned int base = 0, max_digits = 0;
+    if (builtin_echo_digit(str[0], 8) != UINT_MAX)
+    {
+        // Octal escape
+        base = 8;
+
+        // If the first digit is a 0, we allow four digits (including that zero)
+        // Otherwise we allow 3.
+        max_digits = (str[0] == L'0' ? 4 : 3);
+    }
+    else if (str[0] == L'x')
+    {
+        // Hex escape
+        base = 16;
+        max_digits = 2;
+
+        // Skip the x
+        start = 1;
+    }
+
+    if (base != 0)
+    {
+        unsigned int idx;
+        for (idx = start; idx < start + max_digits; idx++)
+        {
+            unsigned int digit = builtin_echo_digit(str[idx], base);
+            if (digit == UINT_MAX) break;
+            val = val * base + digit;
+        }
+
+        // We succeeded if we consumed at least one digit
+        if (idx > start)
+        {
+            *consumed = idx;
+            *out_val = val;
+            success = true;
+        }
+    }
+    return success;
+}
+
+/** The echo builtin.
+    bash only respects -n if it's the first argument. We'll do the same.
+    We also support a new option -s to mean "no spaces"
+*/
+
+static int builtin_echo(parser_t &parser, wchar_t **argv)
+{
+    /* Skip first arg */
+    if (! *argv++)
+        return STATUS_BUILTIN_ERROR;
+
+    /* Process options. Options must come at the beginning - the first non-option kicks us out. */
+    bool print_newline = true, print_spaces = true, interpret_special_chars = false;
+    size_t option_idx = 0;
+    for (option_idx = 0; argv[option_idx] != NULL; option_idx++)
+    {
+        const wchar_t *arg = argv[option_idx];
+        assert(arg != NULL);
+        bool arg_is_valid_option = false;
+        if (arg[0] == L'-')
+        {
+            // We have a leading dash. Ensure that every subseqnent character is a valid option.
+            size_t i = 1;
+            while (arg[i] != L'\0' && wcschr(L"nesE", arg[i]) != NULL)
+            {
+                i++;
+            }
+            // We must have at least two characters to be a valid option, and have consumed the whole string
+            arg_is_valid_option = (i >= 2 && arg[i] == L'\0');
+        }
+        
+        if (! arg_is_valid_option)
+        {
+            // This argument is not an option, so there are no more options
+            break;
+        }
+        
+        // Ok, we are sure the argument is an option. Parse it.
+        assert(arg_is_valid_option);
+        for (size_t i=1; arg[i] != L'\0'; i++)
+        {
+            switch (arg[i])
+            {
+                case L'n':
+                    print_newline = false;
+                    break;
+                case L'e':
+                    interpret_special_chars = true;
+                    break;
+                case L's':
+                    print_spaces = false;
+                    break;
+                case L'E':
+                    interpret_special_chars = false;
+                    break;
+                default:
+                    assert(0 && "Unexpected character in builtin_echo argument");
+                    break;
+            }
+        }
+    }
+
+    /* The special character \c can be used to indicate no more output */
+    bool continue_output = true;
+    
+    /* Skip over the options */
+    const wchar_t * const *args_to_echo = argv + option_idx;
+    for (size_t idx = 0; continue_output && args_to_echo[idx] != NULL; idx++)
+    {
+        if (print_spaces && idx > 0)
+        {
+            stdout_buffer.push_back(' ');
+        }
+
+        const wchar_t *str = args_to_echo[idx];
+        for (size_t j=0; continue_output && str[j]; j++)
+        {
+            if (! interpret_special_chars || str[j] != L'\\')
+            {
+                /* Not an escape */
+                stdout_buffer.push_back(str[j]);
+            }
+            else
+            {
+                /* Most escapes consume one character in addition to the backslash; the numeric sequences may consume more, while an unrecognized escape sequence consumes none. */
+                wchar_t wc;
+                size_t consumed = 1;
+                switch (str[j+1])
+                {
+                    case L'a':
+                        wc = L'\a';
+                        break;
+                    case L'b':
+                        wc = L'\b';
+                        break;
+                    case L'e':
+                        wc = L'\x1B';
+                        break;
+                    case L'f':
+                        wc = L'\f';
+                        break;
+                    case L'n':
+                        wc = L'\n';
+                        break;
+                    case L'r':
+                        wc = L'\r';
+                        break;
+                    case L't':
+                        wc = L'\t';
+                        break;
+                    case L'v':
+                        wc = L'\v';
+                        break;
+                    case L'\\':
+                        wc = L'\\';
+                        break;
+
+                    case L'c':
+                        wc = 0;
+                        continue_output = false;
+                        break;
+
+                    default:
+                    {
+                        /* Octal and hex escape sequences */
+                        unsigned char narrow_val = 0;
+                        if (builtin_echo_parse_numeric_sequence(str + j + 1, &consumed, &narrow_val))
+                        {
+                            /* Here consumed must have been set to something. The narrow_val is a literal byte that we want to output (#1894) */
+                            wc = ENCODE_DIRECT_BASE + narrow_val % 256;
+                        }
+                        else
+                        {
+                            /* Not a recognized escape. We consume only the backslash. */
+                            wc = L'\\';
+                            consumed = 0;
+                        }
+                        break;
+                    }
+                }
+
+                /* Skip over characters that were part of this escape sequence (but not the backslash, which will be handled by the loop increment */
+                j += consumed;
+
+                if (continue_output)
+                {
+                    stdout_buffer.push_back(wc);
+                }
+            }
+        }
+    }
+    if (print_newline && continue_output)
+    {
+        stdout_buffer.push_back('\n');
+    }
+    return STATUS_BUILTIN_OK;
+}
+
+/** The pwd builtin. We don't respect -P to resolve symbolic links because we try to always resolve them. */
+static int builtin_pwd(parser_t &parser, wchar_t **argv)
+{
+    wchar_t dir_path[4096];
+    wchar_t *res = wgetcwd(dir_path, 4096);
+    if (res == NULL)
+    {
+        return STATUS_BUILTIN_ERROR;
+    }
+    else
+    {
+        stdout_buffer.append(dir_path);
+        stdout_buffer.push_back(L'\n');
+        return STATUS_BUILTIN_OK;
+    }
+}
+
+/** Adds a function to the function set. It calls into function.cpp to perform any heavy lifting. */
+int define_function(parser_t &parser, const wcstring_list_t &c_args, const wcstring &contents, int definition_line_offset, wcstring *out_err)
+{
+    wgetopter_t w;
+    assert(out_err != NULL);
+
+    /* wgetopt expects 'function' as the first argument. Make a new wcstring_list with that property. */
+    wcstring_list_t args;
+    args.push_back(L"function");
+    args.insert(args.end(), c_args.begin(), c_args.end());
+
+    /* Hackish const_cast matches the one in builtin_run */
+    const null_terminated_array_t<wchar_t> argv_array(args);
+    wchar_t **argv = const_cast<wchar_t **>(argv_array.get());
+
+    int argc = builtin_count_args(argv);
+    int res=STATUS_BUILTIN_OK;
+    wchar_t *desc=0;
+    std::vector<event_t> events;
+    
+    bool has_named_arguments = false;
+    wcstring_list_t named_arguments;
+    wcstring_list_t inherit_vars;
+
+    bool shadows = true;
+
+    wcstring_list_t wrap_targets;
+    
+    /* If -a/--argument-names is specified before the function name,
+       then the function name is the last positional, e.g. `function -a arg1 arg2 name`.
+       If it is specified after the function name (or not specified at all) then the
+       function name is the first positional. This is the common case. */
+    bool name_is_first_positional = true;
+    wcstring_list_t positionals;
+    
+    const struct woption long_options[] =
+    {
+        { L"description", required_argument, 0, 'd' },
+        { L"on-signal", required_argument, 0, 's' },
+        { L"on-job-exit", required_argument, 0, 'j' },
+        { L"on-process-exit", required_argument, 0, 'p' },
+        { L"on-variable", required_argument, 0, 'v' },
+        { L"on-event", required_argument, 0, 'e' },
+        { L"wraps", required_argument, 0, 'w' },
+        { L"help", no_argument, 0, 'h' },
+        { L"argument-names", no_argument, 0, 'a' },
+        { L"no-scope-shadowing", no_argument, 0, 'S' },
+        { L"inherit-variable", required_argument, 0, 'V' },
+        { 0, 0, 0, 0 }
+    };
+
+    while (1 && (!res))
+    {
+        int opt_index = 0;
+
+        // The leading - here specifies RETURN_IN_ORDER
+        int opt = w.wgetopt_long(argc,
+                                 argv,
+                                 L"-d:s:j:p:v:e:haSV:",
+                                 long_options,
+                                 &opt_index);
+        if (opt == -1)
+            break;
+
+        switch (opt)
+        {
+            case 0:
+                if (long_options[opt_index].flag != 0)
+                    break;
+
+
+
+                append_format(*out_err,
+                              BUILTIN_ERR_UNKNOWN,
+                              argv[0],
+                              long_options[opt_index].name);
+
+                res = 1;
+                break;
+
+            case 'd':
+                desc=w.woptarg;
+                break;
+
+            case 's':
+            {
+                int sig = wcs2sig(w.woptarg);
+
+                if (sig < 0)
+                {
+                    append_format(*out_err,
+                                  _(L"%ls: Unknown signal '%ls'\n"),
+                                  argv[0],
+                                  w.woptarg);
+                    res=1;
+                    break;
+                }
+                events.push_back(event_t::signal_event(sig));
+                break;
+            }
+
+            case 'v':
+            {
+                if (wcsvarname(w.woptarg))
+                {
+                    append_format(*out_err,
+                                  _(L"%ls: Invalid variable name '%ls'\n"),
+                                  argv[0],
+                                  w.woptarg);
+                    res=STATUS_BUILTIN_ERROR;
+                    break;
+                }
+
+                events.push_back(event_t::variable_event(w.woptarg));
+                break;
+            }
+
+
+            case 'e':
+            {
+                events.push_back(event_t::generic_event(w.woptarg));
+                break;
+            }
+
+            case 'j':
+            case 'p':
+            {
+                pid_t pid;
+                wchar_t *end;
+                event_t e(EVENT_ANY);
+
+                if ((opt == 'j') &&
+                        (wcscasecmp(w.woptarg, L"caller") == 0))
+                {
+                    int job_id = -1;
+
+                    if (is_subshell)
+                    {
+                        size_t block_idx = 0;
+
+                        /* Find the outermost substitution block */
+                        for (block_idx = 0; ; block_idx++)
+                        {
+                            const block_t *b = parser.block_at_index(block_idx);
+                            if (b == NULL || b->type() == SUBST)
+                                break;
+                        }
+
+                        /* Go one step beyond that, to get to the caller */
+                        const block_t *caller_block = parser.block_at_index(block_idx + 1);
+                        if (caller_block != NULL && caller_block->job != NULL)
+                        {
+                            job_id = caller_block->job->job_id;
+                        }
+                    }
+
+                    if (job_id == -1)
+                    {
+                        append_format(*out_err,
+                                      _(L"%ls: Cannot find calling job for event handler\n"),
+                                      argv[0]);
+                        res=1;
+                    }
+                    else
+                    {
+                        e.type = EVENT_JOB_ID;
+                        e.param1.job_id = job_id;
+                    }
+
+                }
+                else
+                {
+                    errno = 0;
+                    pid = fish_wcstoi(w.woptarg, &end, 10);
+                    if (errno || !end || *end)
+                    {
+                        append_format(*out_err,
+                                      _(L"%ls: Invalid process id %ls\n"),
+                                      argv[0],
+                                      w.woptarg);
+                        res=1;
+                        break;
+                    }
+
+
+                    e.type = EVENT_EXIT;
+                    e.param1.pid = (opt=='j'?-1:1)*abs(pid);
+                }
+                if (res)
+                {
+                    /* nothing */
+                }
+                else
+                {
+                    events.push_back(e);
+                }
+                break;
+            }
+
+            case 'a':
+                has_named_arguments = true;
+                /* The function name is the first positional unless -a comes before all positionals */
+                name_is_first_positional = ! positionals.empty();
+                break;
+
+            case 'S':
+                shadows = 0;
+                break;
+
+            case 'w':
+                wrap_targets.push_back(w.woptarg);
+                break;
+
+            case 'V':
+            {
+                if (wcsvarname(w.woptarg))
+                {
+                    append_format(*out_err, _(L"%ls: Invalid variable name '%ls'\n"), argv[0], w.woptarg);
+                    res = STATUS_BUILTIN_ERROR;
+                    break;
+                }
+
+                inherit_vars.push_back(w.woptarg);
+                break;
+            }
+
+            case 'h':
+                builtin_print_help(parser, argv[0], stdout_buffer);
+                return STATUS_BUILTIN_OK;
+                
+            case 1:
+                assert(w.woptarg != NULL);
+                positionals.push_back(w.woptarg);
+                break;
+
+            case '?':
+                builtin_unknown_option(parser, argv[0], argv[w.woptind-1]);
+                res = 1;
+                break;
+
+        }
+
+    }
+
+    if (!res)
+    {
+        /* Determine the function name, and remove it from the list of positionals */
+        wcstring function_name;
+        bool name_is_missing = positionals.empty();
+        if (! name_is_missing)
+        {
+            if (name_is_first_positional)
+            {
+                function_name = positionals.front();
+                positionals.erase(positionals.begin());
+            }
+            else
+            {
+                function_name = positionals.back();
+                positionals.erase(positionals.end() - 1);
+            }
+        }
+        
+        if (name_is_missing)
+        {
+            append_format(*out_err,
+                          _(L"%ls: Expected function name\n"),
+                          argv[0]);
+            res=1;
+        }
+        else if (wcsfuncname(function_name))
+        {
+            append_format(*out_err,
+                          _(L"%ls: Illegal function name '%ls'\n"),
+                          argv[0],
+                          function_name.c_str());
+
+            res=1;
+        }
+        else if (parser_keywords_is_reserved(function_name))
+        {
+
+            append_format(*out_err,
+                          _(L"%ls: The name '%ls' is reserved,\nand can not be used as a function name\n"),
+                          argv[0],
+                          function_name.c_str());
+
+            res=1;
+        }
+        else if (function_name.empty())
+        {
+            append_format(*out_err, _(L"%ls: No function name given\n"), argv[0]);
+            res=1;
+        }
+        else
+        {
+            if (has_named_arguments)
+            {
+                /* All remaining positionals are named arguments */
+                named_arguments.swap(positionals);
+                for (size_t i=0; i < named_arguments.size(); i++)
+                {
+                    if (wcsvarname(named_arguments.at(i)))
+                    {
+                        append_format(*out_err,
+                                      _(L"%ls: Invalid variable name '%ls'\n"),
+                                      argv[0],
+                                      named_arguments.at(i).c_str());
+                        res = STATUS_BUILTIN_ERROR;
+                        break;
+                    }
+                }
+            }
+            else if (! positionals.empty())
+            {
+                // +1 because we already got the function name
+                append_format(*out_err,
+                              _(L"%ls: Expected one argument, got %d\n"),
+                              argv[0],
+                              positionals.size() + 1);
+                res=1;
+            }
+        }
+
+        /* Here we actually define the function! */
+        function_data_t d;
+
+        d.name = function_name;
+        if (desc)
+            d.description = desc;
+        d.events.swap(events);
+        d.shadows = shadows;
+        d.named_arguments.swap(named_arguments);
+        d.inherit_vars.swap(inherit_vars);
+
+        for (size_t i=0; i<d.events.size(); i++)
+        {
+            event_t &e = d.events.at(i);
+            e.function_name = d.name;
+        }
+
+        d.definition = contents.c_str();
+
+        function_add(d, parser, definition_line_offset);
+
+        // Handle wrap targets
+        for (size_t w=0; w < wrap_targets.size(); w++)
+        {
+            complete_add_wrapper(function_name, wrap_targets.at(w));
+        }
+    }
+
+    return res;
+}
+
+/**
+   The random builtin. For generating random numbers.
+*/
+static int builtin_random(parser_t &parser, wchar_t **argv)
+{
+    static int seeded=0;
+    static struct drand48_data seed_buffer;
+
+    int argc = builtin_count_args(argv);
+
+    wgetopter_t w;
+
+    static const struct woption
+            long_options[] =
+    {
+        {
+            L"help", no_argument, 0, 'h'
+        }
+        ,
+        {
+            0, 0, 0, 0
+        }
+    }
+    ;
+
+    while (1)
+    {
+        int opt_index = 0;
+
+        int opt = w.wgetopt_long(argc,
+                                 argv,
+                                 L"h",
+                                 long_options,
+                                 &opt_index);
+        if (opt == -1)
+            break;
+
+        switch (opt)
+        {
+            case 0:
+                if (long_options[opt_index].flag != 0)
+                    break;
+                append_format(stderr_buffer,
+                              BUILTIN_ERR_UNKNOWN,
+                              argv[0],
+                              long_options[opt_index].name);
+                builtin_print_help(parser, argv[0], stderr_buffer);
+
+                return STATUS_BUILTIN_ERROR;
+
+            case 'h':
+                builtin_print_help(parser, argv[0], stdout_buffer);
+                break;
+
+            case '?':
+                builtin_unknown_option(parser, argv[0], argv[w.woptind-1]);
+                return STATUS_BUILTIN_ERROR;
+
+        }
+
+    }
+
+    switch (argc-w.woptind)
+    {
+
+        case 0:
+        {
+            long res;
+
+            if (!seeded)
+            {
+                seeded=1;
+                srand48_r(time(0), &seed_buffer);
+            }
+            lrand48_r(&seed_buffer, &res);
+
+            append_format(stdout_buffer, L"%ld\n", labs(res%32767));
+            break;
+        }
+
+        case 1:
+        {
+            long foo;
+            wchar_t *end=0;
+
+            errno=0;
+            foo = wcstol(argv[w.woptind], &end, 10);
+            if (errno || *end)
+            {
+                append_format(stderr_buffer,
+                              _(L"%ls: Seed value '%ls' is not a valid number\n"),
+                              argv[0],
+                              argv[w.woptind]);
+
+                return STATUS_BUILTIN_ERROR;
+            }
+            seeded=1;
+            srand48_r(foo, &seed_buffer);
+            break;
+        }
+
+        default:
+        {
+            append_format(stderr_buffer,
+                          _(L"%ls: Expected zero or one argument, got %d\n"),
+                          argv[0],
+                          argc-w.woptind);
+            builtin_print_help(parser, argv[0], stderr_buffer);
+            return STATUS_BUILTIN_ERROR;
+        }
+    }
+    return STATUS_BUILTIN_OK;
+}
+
+
+/**
+   The read builtin. Reads from stdin and stores the values in environment variables.
+*/
+static int builtin_read(parser_t &parser, wchar_t **argv)
+{
+    wgetopter_t w;
+    wcstring buff;
+    int i, argc = builtin_count_args(argv);
+    int place = ENV_USER;
+    const wchar_t *prompt = DEFAULT_READ_PROMPT;
+    const wchar_t *right_prompt = L"";
+    const wchar_t *commandline = L"";
+    int exit_res=STATUS_BUILTIN_OK;
+    const wchar_t *mode_name = READ_MODE_NAME;
+    int nchars=0;
+    wchar_t *end;
+    int shell = 0;
+    int array = 0;
+    bool split_null = false;
+
+    while (1)
+    {
+        static const struct woption
+                long_options[] =
+        {
+            {
+                L"export", no_argument, 0, 'x'
+            }
+            ,
+            {
+                L"global", no_argument, 0, 'g'
+            }
+            ,
+            {
+                L"local", no_argument, 0, 'l'
+            }
+            ,
+            {
+                L"universal", no_argument, 0, 'U'
+            }
+            ,
+            {
+                L"unexport", no_argument, 0, 'u'
+            }
+            ,
+            {
+                L"prompt", required_argument, 0, 'p'
+            }
+            ,
+            {
+                L"right-prompt", required_argument, 0, 'R'
+            }
+            ,
+            {
+                L"command", required_argument, 0, 'c'
+            }
+            ,
+            {
+                L"mode-name", required_argument, 0, 'm'
+            }
+            ,
+            {
+                L"nchars", required_argument, 0, 'n'
+            }
+            ,
+            {
+                L"shell", no_argument, 0, 's'
+            }
+            ,
+            {
+                L"array", no_argument, 0, 'a'
+            }
+            ,
+            {
+                L"null", no_argument, 0, 'z'
+            }
+            ,
+            {
+                L"help", no_argument, 0, 'h'
+            }
+            ,
+            {
+                0, 0, 0, 0
+            }
+        }
+        ;
+
+        int opt_index = 0;
+
+        int opt = w.wgetopt_long(argc,
+                                 argv,
+                                 L"xglUup:R:c:hm:n:saz",
+                                 long_options,
+                                 &opt_index);
+        if (opt == -1)
+            break;
+
+        switch (opt)
+        {
+            case 0:
+                if (long_options[opt_index].flag != 0)
+                    break;
+                append_format(stderr_buffer,
+                              BUILTIN_ERR_UNKNOWN,
+                              argv[0],
+                              long_options[opt_index].name);
+                builtin_print_help(parser, argv[0], stderr_buffer);
+
+                return STATUS_BUILTIN_ERROR;
+
+            case L'x':
+                place |= ENV_EXPORT;
+                break;
+
+            case L'g':
+                place |= ENV_GLOBAL;
+                break;
+
+            case L'l':
+                place |= ENV_LOCAL;
+                break;
+
+            case L'U':
+                place |= ENV_UNIVERSAL;
+                break;
+
+            case L'u':
+                place |= ENV_UNEXPORT;
+                break;
+
+            case L'p':
+                prompt = w.woptarg;
+                break;
+
+            case L'R':
+                right_prompt = w.woptarg;
+                break;
+
+            case L'c':
+                commandline = w.woptarg;
+                break;
+
+            case L'm':
+                mode_name = w.woptarg;
+                break;
+
+            case L'n':
+                errno = 0;
+                nchars = fish_wcstoi(w.woptarg, &end, 10);
+                if (errno || *end != 0)
+                {
+                    switch (errno)
+                    {
+                        case ERANGE:
+                            append_format(stderr_buffer,
+                                    _(L"%ls: Argument '%ls' is out of range\n"),
+                                    argv[0],
+                                    w.woptarg);
+                            builtin_print_help(parser, argv[0], stderr_buffer);
+                            return STATUS_BUILTIN_ERROR;
+
+                        default:
+                            append_format(stderr_buffer,
+                                    _(L"%ls: Argument '%ls' must be an integer\n"),
+                                    argv[0],
+                                    w.woptarg);
+                            builtin_print_help(parser, argv[0], stderr_buffer);
+                            return STATUS_BUILTIN_ERROR;
+                    }
+                }
+                break;
+
+            case 's':
+                shell = 1;
+                break;
+
+            case 'a':
+                array = 1;
+                break;
+
+            case L'z':
+                split_null = true;
+                break;
+
+            case 'h':
+                builtin_print_help(parser, argv[0], stdout_buffer);
+                return STATUS_BUILTIN_OK;
+
+            case L'?':
+                builtin_unknown_option(parser, argv[0], argv[w.woptind-1]);
+                return STATUS_BUILTIN_ERROR;
+        }
+
+    }
+
+    if ((place & ENV_UNEXPORT) && (place & ENV_EXPORT))
+    {
+        append_format(stderr_buffer,
+                      BUILTIN_ERR_EXPUNEXP,
+                      argv[0]);
+
+
+        builtin_print_help(parser, argv[0], stderr_buffer);
+        return STATUS_BUILTIN_ERROR;
+    }
+
+    if ((place&ENV_LOCAL?1:0) + (place & ENV_GLOBAL?1:0) + (place & ENV_UNIVERSAL?1:0) > 1)
+    {
+        append_format(stderr_buffer,
+                      BUILTIN_ERR_GLOCAL,
+                      argv[0]);
+        builtin_print_help(parser, argv[0], stderr_buffer);
+
+        return STATUS_BUILTIN_ERROR;
+    }
+
+    if (array && w.woptind+1 != argc)
+    {
+        append_format(stderr_buffer, _(L"%ls: --array option requires a single variable name.\n"), argv[0]);
+        builtin_print_help(parser, argv[0], stderr_buffer);
+
+        return STATUS_BUILTIN_ERROR;
+    }
+
+    /*
+      Verify all variable names
+    */
+    for (i=w.woptind; i<argc; i++)
+    {
+        wchar_t *src;
+
+        if (!wcslen(argv[i]))
+        {
+            append_format(stderr_buffer, BUILTIN_ERR_VARNAME_ZERO, argv[0]);
+            return STATUS_BUILTIN_ERROR;
+        }
+
+        for (src=argv[i]; *src; src++)
+        {
+            if ((!iswalnum(*src)) && (*src != L'_'))
+            {
+                append_format(stderr_buffer, BUILTIN_ERR_VARCHAR, argv[0], *src);
+                builtin_print_help(parser, argv[0], stderr_buffer);
+                return STATUS_BUILTIN_ERROR;
+            }
+        }
+
+    }
+
+    /*
+      The call to reader_readline may change woptind, so we save it away here
+    */
+    i=w.woptind;
+
+    /*
+      Check if we should read interactively using \c reader_readline()
+    */
+    if (isatty(0) && builtin_stdin == 0 && !split_null)
+    {
+        const wchar_t *line;
+
+        reader_push(mode_name);
+        reader_set_left_prompt(prompt);
+        reader_set_right_prompt(right_prompt);
+        if (shell)
+        {
+            reader_set_complete_function(&complete);
+            reader_set_highlight_function(&highlight_shell);
+            reader_set_test_function(&reader_shell_test);
+        }
+        /* No autosuggestions or abbreviations in builtin_read */
+        reader_set_allow_autosuggesting(false);
+        reader_set_expand_abbreviations(false);
+        reader_set_exit_on_interrupt(true);
+
+        reader_set_buffer(commandline, wcslen(commandline));
+        proc_push_interactive(1);
+
+        event_fire_generic(L"fish_prompt");
+        line = reader_readline(nchars);
+        proc_pop_interactive();
+        if (line)
+        {
+            if (0 < nchars && nchars < wcslen(line))
+            {
+                // line may be longer than nchars if a keybinding used `commandline -i`
+                // note: we're deliberately throwing away the tail of the commandline.
+                // It shouldn't be unread because it was produced with `commandline -i`,
+                // not typed.
+                buff = wcstring(line, nchars);
+            }
+            else
+            {
+                buff = wcstring(line);
+            }
+        }
+        else
+        {
+            exit_res = STATUS_BUILTIN_ERROR;
+        }
+        reader_pop();
+    }
+    else
+    {
+        int eof=0;
+
+        buff.clear();
+
+        while (1)
+        {
+            int finished=0;
+
+            wchar_t res=0;
+            mbstate_t state = {};
+
+            while (!finished)
+            {
+                char b;
+                if (read_blocked(builtin_stdin, &b, 1) <= 0)
+                {
+                    eof=1;
+                    break;
+                }
+
+                size_t sz = mbrtowc(&res, &b, 1, &state);
+
+                switch (sz)
+                {
+                    case (size_t)(-1):
+                        memset(&state, '\0', sizeof(state));
+                        break;
+
+                    case (size_t)(-2):
+                        break;
+                    case 0:
+                        finished = 1;
+                        break;
+
+                    default:
+                        finished=1;
+                        break;
+
+                }
+            }
+
+            if (eof)
+                break;
+
+            if (!split_null && res == L'\n')
+                break;
+
+            if (split_null && res == L'\0')
+                break;
+
+            buff.push_back(res);
+
+            if (0 < nchars && (size_t)nchars <= buff.size())
+            {
+                break;
+            }
+        }
+
+        if (buff.empty() && eof)
+        {
+            exit_res = 1;
+        }
+    }
+
+    if (i != argc && !exit_res)
+    {
+        env_var_t ifs = env_get_string(L"IFS");
+        if (ifs.missing_or_empty())
+        {
+            /* Every character is a separate token */
+            size_t bufflen = buff.size();
+            if (array)
+            {
+                if (bufflen > 0)
+                {
+                    wcstring chars(bufflen+(bufflen-1), ARRAY_SEP);
+                    wcstring::iterator out = chars.begin();
+                    for (wcstring::const_iterator it = buff.begin(), end = buff.end(); it != end; ++it)
+                    {
+                        *out = *it;
+                        out += 2;
+                    }
+                    env_set(argv[i], chars.c_str(), place);
+                }
+                else
+                {
+                    env_set(argv[i], NULL, place);
+                }
+            }
+            else
+            {
+                size_t j = 0;
+                for (; i+1 < argc; ++i)
+                {
+                    if (j < bufflen)
+                    {
+                        wchar_t buffer[2] = {buff[j++], 0};
+                        env_set(argv[i], buffer, place);
+                    }
+                    else
+                    {
+                        env_set(argv[i], L"", place);
+                    }
+                }
+                if (i < argc) env_set(argv[i], &buff[j], place);
+            }
+        }
+        else if (array)
+        {
+            wcstring tokens;
+            tokens.reserve(buff.size());
+            bool empty = true;
+
+            for (wcstring_range loc = wcstring_tok(buff, ifs); loc.first != wcstring::npos; loc = wcstring_tok(buff, ifs, loc))
+            {
+                if (!empty) tokens.push_back(ARRAY_SEP);
+                tokens.append(buff, loc.first, loc.second);
+                empty = false;
+            }
+            env_set(argv[i], empty ? NULL : tokens.c_str(), place);
+        }
+        else
+        {
+            wcstring_range loc = wcstring_range(0,0);
+
+            while (i<argc)
+            {
+                loc = wcstring_tok(buff, (i+1<argc) ? ifs : L"", loc);
+                env_set(argv[i], loc.first == wcstring::npos ? L"" : &buff.c_str()[loc.first], place);
+
+                ++i;
+            }
+
+        }
+    }
+
+    return exit_res;
+}
+
+/**
+   The status builtin. Gives various status information on fish.
+*/
+static int builtin_status(parser_t &parser, wchar_t **argv)
+{
+    wgetopter_t w;
+    enum
+    {
+        NORMAL,
+        IS_SUBST,
+        IS_BLOCK,
+        IS_INTERACTIVE,
+        IS_LOGIN,
+        IS_FULL_JOB_CONTROL,
+        IS_INTERACTIVE_JOB_CONTROL,
+        IS_NO_JOB_CONTROL,
+        STACK_TRACE,
+        DONE,
+        CURRENT_FILENAME,
+        CURRENT_LINE_NUMBER
+    }
+    ;
+
+    int mode = NORMAL;
+
+    int argc = builtin_count_args(argv);
+    int res=STATUS_BUILTIN_OK;
+
+
+    const struct woption
+            long_options[] =
+    {
+        {
+            L"help", no_argument, 0, 'h'
+        }
+        ,
+        {
+            L"is-command-substitution", no_argument, 0, 'c'
+        }
+        ,
+        {
+            L"is-block", no_argument, 0, 'b'
+        }
+        ,
+        {
+            L"is-interactive", no_argument, 0, 'i'
+        }
+        ,
+        {
+            L"is-login", no_argument, 0, 'l'
+        }
+        ,
+        {
+            L"is-full-job-control", no_argument, &mode, IS_FULL_JOB_CONTROL
+        }
+        ,
+        {
+            L"is-interactive-job-control", no_argument, &mode, IS_INTERACTIVE_JOB_CONTROL
+        }
+        ,
+        {
+            L"is-no-job-control", no_argument, &mode, IS_NO_JOB_CONTROL
+        }
+        ,
+        {
+            L"current-filename", no_argument, 0, 'f'
+        }
+        ,
+        {
+            L"current-line-number", no_argument, 0, 'n'
+        }
+        ,
+        {
+            L"job-control", required_argument, 0, 'j'
+        }
+        ,
+        {
+            L"print-stack-trace", no_argument, 0, 't'
+        }
+        ,
+        {
+            0, 0, 0, 0
+        }
+    }
+    ;
+
+    while (1)
+    {
+        int opt_index = 0;
+
+        int opt = w.wgetopt_long(argc,
+                                 argv,
+                                 L":cbilfnhj:t",
+                                 long_options,
+                                 &opt_index);
+        if (opt == -1)
+            break;
+
+        switch (opt)
+        {
+            case 0:
+                if (long_options[opt_index].flag != 0)
+                    break;
+                append_format(stderr_buffer,
+                              BUILTIN_ERR_UNKNOWN,
+                              argv[0],
+                              long_options[opt_index].name);
+                builtin_print_help(parser, argv[0], stderr_buffer);
+                return STATUS_BUILTIN_ERROR;
+
+            case 'c':
+                mode = IS_SUBST;
+                break;
+
+            case 'b':
+                mode = IS_BLOCK;
+                break;
+
+            case 'i':
+                mode = IS_INTERACTIVE;
+                break;
+
+            case 'l':
+                mode = IS_LOGIN;
+                break;
+
+            case 'f':
+                mode = CURRENT_FILENAME;
+                break;
+
+            case 'n':
+                mode = CURRENT_LINE_NUMBER;
+                break;
+
+            case 'h':
+                builtin_print_help(parser, argv[0], stdout_buffer);
+                return STATUS_BUILTIN_OK;
+
+            case 'j':
+                if (wcscmp(w.woptarg, L"full") == 0)
+                    job_control_mode = JOB_CONTROL_ALL;
+                else if (wcscmp(w.woptarg, L"interactive") == 0)
+                    job_control_mode = JOB_CONTROL_INTERACTIVE;
+                else if (wcscmp(w.woptarg, L"none") == 0)
+                    job_control_mode = JOB_CONTROL_NONE;
+                else
+                {
+                    append_format(stderr_buffer,
+                                  L"%ls: Invalid job control mode '%ls'\n",
+                                  L"status", w.woptarg);
+                    res = 1;
+                }
+                mode = DONE;
+                break;
+
+            case 't':
+                mode = STACK_TRACE;
+                break;
+
+
+            case ':':
+                builtin_missing_argument(parser, argv[0], argv[w.woptind-1]);
+                return STATUS_BUILTIN_ERROR;
+
+            case '?':
+                builtin_unknown_option(parser, argv[0], argv[w.woptind-1]);
+                return STATUS_BUILTIN_ERROR;
+
+        }
+
+    }
+
+    if (!res)
+    {
+
+        switch (mode)
+        {
+            case CURRENT_FILENAME:
+            {
+                const wchar_t *fn = parser.current_filename();
+
+                if (!fn)
+                    fn = _(L"Standard input");
+
+                append_format(stdout_buffer, L"%ls\n", fn);
+
+                break;
+            }
+
+            case CURRENT_LINE_NUMBER:
+            {
+                append_format(stdout_buffer, L"%d\n", parser.get_lineno());
+                break;
+            }
+
+            case IS_INTERACTIVE:
+                return !is_interactive_session;
+
+            case IS_SUBST:
+                return !is_subshell;
+
+            case IS_BLOCK:
+                return !is_block;
+
+            case IS_LOGIN:
+                return !is_login;
+
+            case IS_FULL_JOB_CONTROL:
+                return job_control_mode != JOB_CONTROL_ALL;
+
+            case IS_INTERACTIVE_JOB_CONTROL:
+                return job_control_mode != JOB_CONTROL_INTERACTIVE;
+
+            case IS_NO_JOB_CONTROL:
+                return job_control_mode != JOB_CONTROL_NONE;
+
+            case STACK_TRACE:
+            {
+                parser.stack_trace(0, stdout_buffer);
+                break;
+            }
+
+            case NORMAL:
+            {
+                if (is_login)
+                    append_format(stdout_buffer, _(L"This is a login shell\n"));
+                else
+                    append_format(stdout_buffer, _(L"This is not a login shell\n"));
+
+                append_format(stdout_buffer, _(L"Job control: %ls\n"),
+                              job_control_mode==JOB_CONTROL_INTERACTIVE?_(L"Only on interactive jobs"):
+                              (job_control_mode==JOB_CONTROL_NONE ? _(L"Never") : _(L"Always")));
+
+                parser.stack_trace(0, stdout_buffer);
+                break;
+            }
+        }
+    }
+
+    return res;
+}
+
+
+/**
+   The exit builtin. Calls reader_exit to exit and returns the value specified.
+*/
+static int builtin_exit(parser_t &parser, wchar_t **argv)
+{
+    int argc = builtin_count_args(argv);
+
+    long ec=0;
+    switch (argc)
+    {
+        case 1:
+        {
+            ec = proc_get_last_status();
+            break;
+        }
+
+        case 2:
+        {
+            wchar_t *end;
+            errno = 0;
+            ec = wcstol(argv[1],&end,10);
+            if (errno || *end != 0)
+            {
+                append_format(stderr_buffer,
+                              _(L"%ls: Argument '%ls' must be an integer\n"),
+                              argv[0],
+                              argv[1]);
+                builtin_print_help(parser, argv[0], stderr_buffer);
+                return STATUS_BUILTIN_ERROR;
+            }
+            break;
+        }
+
+        default:
+        {
+            append_format(stderr_buffer,
+                          BUILTIN_ERR_TOO_MANY_ARGUMENTS,
+                          argv[0]);
+
+            builtin_print_help(parser, argv[0], stderr_buffer);
+            return STATUS_BUILTIN_ERROR;
+        }
+
+    }
+    reader_exit(1, 0);
+    return (int)ec;
+}
+
+/**
+   The cd builtin. Changes the current directory to the one specified
+   or to $HOME if none is specified. The directory can be relative to
+   any directory in the CDPATH variable.
+*/
+static int builtin_cd(parser_t &parser, wchar_t **argv)
+{
+    env_var_t dir_in;
+    wcstring dir;
+    int res=STATUS_BUILTIN_OK;
+
+
+    if (argv[1] == NULL)
+    {
+        dir_in = env_get_string(L"HOME");
+        if (dir_in.missing_or_empty())
+        {
+            append_format(stderr_buffer,
+                          _(L"%ls: Could not find home directory\n"),
+                          argv[0]);
+        }
+    }
+    else
+    {
+        dir_in = argv[1];
+    }
+
+    bool got_cd_path = false;
+    if (! dir_in.missing())
+    {
+        got_cd_path = path_get_cdpath(dir_in, &dir);
+    }
+
+    if (!got_cd_path)
+    {
+        if (errno == ENOTDIR)
+        {
+            append_format(stderr_buffer,
+                          _(L"%ls: '%ls' is not a directory\n"),
+                          argv[0],
+                          dir_in.c_str());
+        }
+        else if (errno == ENOENT)
+        {
+            append_format(stderr_buffer,
+                          _(L"%ls: The directory '%ls' does not exist\n"),
+                          argv[0],
+                          dir_in.c_str());
+        }
+        else if (errno == EROTTEN)
+        {
+            append_format(stderr_buffer,
+                          _(L"%ls: '%ls' is a rotten symlink\n"),
+                          argv[0],
+                          dir_in.c_str());
+
+        }
+        else
+        {
+            append_format(stderr_buffer,
+                          _(L"%ls: Unknown error trying to locate directory '%ls'\n"),
+                          argv[0],
+                          dir_in.c_str());
+
+        }
+
+
+        if (!get_is_interactive())
+        {
+            stderr_buffer.append(parser.current_line());
+        }
+
+        res = 1;
+    }
+    else if (wchdir(dir) != 0)
+    {
+        struct stat buffer;
+        int status;
+
+        status = wstat(dir, &buffer);
+        if (!status && S_ISDIR(buffer.st_mode))
+        {
+            append_format(stderr_buffer,
+                          _(L"%ls: Permission denied: '%ls'\n"),
+                          argv[0],
+                          dir.c_str());
+
+        }
+        else
+        {
+
+            append_format(stderr_buffer,
+                          _(L"%ls: '%ls' is not a directory\n"),
+                          argv[0],
+                          dir.c_str());
+        }
+
+        if (!get_is_interactive())
+        {
+            stderr_buffer.append(parser.current_line());
+        }
+
+        res = 1;
+    }
+    else if (!env_set_pwd())
+    {
+        res=1;
+        append_format(stderr_buffer, _(L"%ls: Could not set PWD variable\n"), argv[0]);
+    }
+
+    return res;
+}
+
+/**
+   Implementation of the builtin count command, used to count the
+   number of arguments sent to it.
+ */
+static int builtin_count(parser_t &parser, wchar_t ** argv)
+{
+    int argc;
+    argc = builtin_count_args(argv);
+    append_format(stdout_buffer, L"%d\n", argc-1);
+    return !(argc-1);
+}
+
+/**
+   Implementation of the builtin contains command, used to check if a
+   specified string is part of a list.
+ */
+static int builtin_contains(parser_t &parser, wchar_t ** argv)
+{
+    wgetopter_t w;
+    int argc;
+    argc = builtin_count_args(argv);
+    wchar_t *needle;
+    bool should_output_index = false;
+
+    const struct woption long_options[] =
+    {
+        { L"help", no_argument, 0, 'h' } ,
+        { L"index", no_argument, 0, 'i' },
+        { 0, 0, 0, 0 }
+    };
+
+    while (1)
+    {
+        int opt_index = 0;
+
+        int opt = w.wgetopt_long(argc,
+                                 argv,
+                                 L"+hi",
+                                 long_options,
+                                 &opt_index);
+        if (opt == -1)
+            break;
+
+        switch (opt)
+        {
+            case 0:
+                assert(opt_index >= 0 && (size_t)opt_index < sizeof long_options / sizeof *long_options);
+                if (long_options[opt_index].flag != 0)
+                    break;
+                append_format(stderr_buffer,
+                              BUILTIN_ERR_UNKNOWN,
+                              argv[0],
+                              long_options[opt_index].name);
+                builtin_print_help(parser, argv[0], stderr_buffer);
+                return STATUS_BUILTIN_ERROR;
+
+
+            case 'h':
+                builtin_print_help(parser, argv[0], stdout_buffer);
+                return STATUS_BUILTIN_OK;
+
+
+            case ':':
+                builtin_missing_argument(parser, argv[0], argv[w.woptind-1]);
+                return STATUS_BUILTIN_ERROR;
+
+            case '?':
+                builtin_unknown_option(parser, argv[0], argv[w.woptind-1]);
+                return STATUS_BUILTIN_ERROR;
+
+            case 'i':
+                should_output_index = true;
+                break;
+        }
+
+    }
+
+    needle = argv[w.woptind];
+    if (!needle)
+    {
+        append_format(stderr_buffer, _(L"%ls: Key not specified\n"), argv[0]);
+    }
+
+
+    for (int i=w.woptind+1; i<argc; i++)
+    {
+
+        if (!wcscmp(needle, argv[i]))
+        {
+            if (should_output_index) append_format(stdout_buffer, L"%d\n", i-w.woptind);
+            return 0;
+        }
+    }
+    return 1;
+
+}
+
+
+/**
+   The  . (dot) builtin, sometimes called source. Evaluates the contents of a file.
+*/
+static int builtin_source(parser_t &parser, wchar_t ** argv)
+{
+    ASSERT_IS_MAIN_THREAD();
+    int fd;
+    int res = STATUS_BUILTIN_OK;
+    struct stat buf;
+    int argc;
+
+    argc = builtin_count_args(argv);
+
+    const wchar_t *fn, *fn_intern;
+
+    if (argc < 2 || (wcscmp(argv[1], L"-") == 0))
+    {
+        fn = L"-";
+        fn_intern = fn;
+        fd = dup(builtin_stdin);
+    }
+    else
+    {
+
+        if ((fd = wopen_cloexec(argv[1], O_RDONLY)) == -1)
+        {
+            append_format(stderr_buffer, _(L"%ls: Error encountered while sourcing file '%ls':\n"), argv[0], argv[1]);
+            builtin_wperror(L"source");
+            return STATUS_BUILTIN_ERROR;
+        }
+
+        if (fstat(fd, &buf) == -1)
+        {
+            close(fd);
+            append_format(stderr_buffer, _(L"%ls: Error encountered while sourcing file '%ls':\n"), argv[0], argv[1]);
+            builtin_wperror(L"source");
+            return STATUS_BUILTIN_ERROR;
+        }
+
+        if (!S_ISREG(buf.st_mode))
+        {
+            close(fd);
+            append_format(stderr_buffer, _(L"%ls: '%ls' is not a file\n"), argv[0], argv[1]);
+            return STATUS_BUILTIN_ERROR;
+        }
+
+        fn_intern = intern(argv[1]);
+    }
+
+    parser.push_block(new source_block_t(fn_intern));
+    reader_push_current_filename(fn_intern);
+
+    parse_util_set_argv((argc>2)?(argv+2):(argv+1), wcstring_list_t());
+
+    res = reader_read(fd, real_io ? *real_io : io_chain_t());
+
+    parser.pop_block();
+
+    if (res)
+    {
+        append_format(stderr_buffer,
+                      _(L"%ls: Error while reading file '%ls'\n"),
+                      argv[0],
+                      fn_intern == intern_static(L"-") ? L"<stdin>" : fn_intern);
+    }
+    else
+    {
+        res = proc_get_last_status();
+    }
+
+    /*
+      Do not close fd after calling reader_read. reader_read
+      automatically closes it before calling eval.
+    */
+
+    reader_pop_current_filename();
+
+    return res;
+}
+
+/**
+   Make the specified job the first job of the job list. Moving jobs
+   around in the list makes the list reflect the order in which the
+   jobs were used.
+*/
+static void make_first(job_t *j)
+{
+    job_promote(j);
+}
+
+
+/**
+   Builtin for putting a job in the foreground
+*/
+static int builtin_fg(parser_t &parser, wchar_t **argv)
+{
+    job_t *j=NULL;
+
+    if (argv[1] == 0)
+    {
+        /*
+          Select last constructed job (I.e. first job in the job que)
+          that is possible to put in the foreground
+        */
+
+        job_iterator_t jobs;
+        while ((j = jobs.next()))
+        {
+            if (job_get_flag(j, JOB_CONSTRUCTED) && (!job_is_completed(j)) &&
+                    ((job_is_stopped(j) || (!job_get_flag(j, JOB_FOREGROUND))) && job_get_flag(j, JOB_CONTROL)))
+            {
+                break;
+            }
+        }
+        if (!j)
+        {
+            append_format(stderr_buffer,
+                          _(L"%ls: There are no suitable jobs\n"),
+                          argv[0]);
+        }
+    }
+    else if (argv[2] != 0)
+    {
+        /*
+          Specifying what more than one job to put to the foreground
+          is a syntax error, we still try to locate the job argv[1],
+          since we want to know if this is an ambigous job
+          specification or if this is an malformed job id
+        */
+        wchar_t *endptr;
+        int pid;
+        int found_job = 0;
+
+        errno = 0;
+        pid = fish_wcstoi(argv[1], &endptr, 10);
+        if (!(*endptr || errno))
+        {
+            j = job_get_from_pid(pid);
+            if (j)
+                found_job = 1;
+        }
+
+        if (found_job)
+        {
+            append_format(stderr_buffer,
+                          _(L"%ls: Ambiguous job\n"),
+                          argv[0]);
+        }
+        else
+        {
+            append_format(stderr_buffer,
+                          _(L"%ls: '%ls' is not a job\n"),
+                          argv[0],
+                          argv[1]);
+        }
+
+        builtin_print_help(parser, argv[0], stderr_buffer);
+
+        j=0;
+
+    }
+    else
+    {
+        wchar_t *end;
+        int pid;
+        errno = 0;
+        pid = abs(fish_wcstoi(argv[1], &end, 10));
+
+        if (*end || errno)
+        {
+            append_format(stderr_buffer,
+                          BUILTIN_ERR_NOT_NUMBER,
+                          argv[0],
+                          argv[1]);
+            builtin_print_help(parser, argv[0], stderr_buffer);
+        }
+        else
+        {
+            j = job_get_from_pid(pid);
+            if (!j || !job_get_flag(j, JOB_CONSTRUCTED) || job_is_completed(j))
+            {
+                append_format(stderr_buffer,
+                              _(L"%ls: No suitable job: %d\n"),
+                              argv[0],
+                              pid);
+                builtin_print_help(parser, argv[0], stderr_buffer);
+                j=0;
+            }
+            else if (!job_get_flag(j, JOB_CONTROL))
+            {
+                append_format(stderr_buffer,
+                              _(L"%ls: Can't put job %d, '%ls' to foreground because it is not under job control\n"),
+                              argv[0],
+                              pid,
+                              j->command_wcstr());
+                builtin_print_help(parser, argv[0], stderr_buffer);
+                j=0;
+            }
+        }
+    }
+
+    if (j)
+    {
+        if (builtin_err_redirect)
+        {
+            append_format(stderr_buffer,
+                          FG_MSG,
+                          j->job_id,
+                          j->command_wcstr());
+        }
+        else
+        {
+            /*
+              If we aren't redirecting, send output to real stderr,
+              since stuff in sb_err won't get printed until the
+              command finishes.
+            */
+            fwprintf(stderr,
+                     FG_MSG,
+                     j->job_id,
+                     j->command_wcstr());
+        }
+
+        const wcstring ft = tok_first(j->command_wcstr());
+        if (! ft.empty())
+            env_set(L"_", ft.c_str(), ENV_EXPORT);
+        reader_write_title(j->command());
+
+        make_first(j);
+        job_set_flag(j, JOB_FOREGROUND, 1);
+
+        job_continue(j, job_is_stopped(j));
+    }
+    return j != 0;
+}
+
+/**
+   Helper function for builtin_bg()
+*/
+static int send_to_bg(parser_t &parser, job_t *j, const wchar_t *name)
+{
+    if (j == 0)
+    {
+        append_format(stderr_buffer,
+                      _(L"%ls: Unknown job '%ls'\n"),
+                      L"bg",
+                      name);
+        builtin_print_help(parser, L"bg", stderr_buffer);
+        return STATUS_BUILTIN_ERROR;
+    }
+    else if (!job_get_flag(j, JOB_CONTROL))
+    {
+        append_format(stderr_buffer,
+                      _(L"%ls: Can't put job %d, '%ls' to background because it is not under job control\n"),
+                      L"bg",
+                      j->job_id,
+                      j->command_wcstr());
+        builtin_print_help(parser, L"bg", stderr_buffer);
+        return STATUS_BUILTIN_ERROR;
+    }
+    else
+    {
+        append_format(stderr_buffer,
+                      _(L"Send job %d '%ls' to background\n"),
+                      j->job_id,
+                      j->command_wcstr());
+    }
+    make_first(j);
+    job_set_flag(j, JOB_FOREGROUND, 0);
+    job_continue(j, job_is_stopped(j));
+    return STATUS_BUILTIN_OK;
+}
+
+
+/**
+   Builtin for putting a job in the background
+*/
+static int builtin_bg(parser_t &parser, wchar_t **argv)
+{
+    int res = STATUS_BUILTIN_OK;
+
+    if (argv[1] == 0)
+    {
+        job_t *j;
+        job_iterator_t jobs;
+        while ((j = jobs.next()))
+        {
+            if (job_is_stopped(j) && job_get_flag(j, JOB_CONTROL) && (!job_is_completed(j)))
+            {
+                break;
+            }
+        }
+
+        if (!j)
+        {
+            append_format(stderr_buffer,
+                          _(L"%ls: There are no suitable jobs\n"),
+                          argv[0]);
+            res = 1;
+        }
+        else
+        {
+            res = send_to_bg(parser, j, _(L"(default)"));
+        }
+    }
+    else
+    {
+        wchar_t *end;
+        int i;
+        int pid;
+        int err = 0;
+
+        for (i=1; argv[i]; i++)
+        {
+            errno=0;
+            pid = fish_wcstoi(argv[i], &end, 10);
+            if (errno || pid < 0 || *end || !job_get_from_pid(pid))
+            {
+                append_format(stderr_buffer,
+                              _(L"%ls: '%ls' is not a job\n"),
+                              argv[0],
+                              argv[i]);
+                err = 1;
+                break;
+            }
+        }
+
+        if (!err)
+        {
+            for (i=1; !res && argv[i]; i++)
+            {
+                pid = fish_wcstoi(argv[i], 0, 10);
+                res |= send_to_bg(parser, job_get_from_pid(pid), *argv);
+            }
+        }
+    }
+
+    return res;
+}
+
+
+/**
+   This function handles both the 'continue' and the 'break' builtins
+   that are used for loop control.
+*/
+static int builtin_break_continue(parser_t &parser, wchar_t **argv)
+{
+    int is_break = (wcscmp(argv[0],L"break")==0);
+    int argc = builtin_count_args(argv);
+
+
+    if (argc != 1)
+    {
+        append_format(stderr_buffer,
+                      BUILTIN_ERR_UNKNOWN,
+                      argv[0],
+                      argv[1]);
+
+        builtin_print_help(parser, argv[0], stderr_buffer);
+        return STATUS_BUILTIN_ERROR;
+    }
+
+    /* Find the index of the enclosing for or while loop. Recall that incrementing loop_idx goes 'up' to outer blocks */
+    size_t loop_idx;
+    for (loop_idx = 0; loop_idx < parser.block_count(); loop_idx++)
+    {
+        const block_t *b = parser.block_at_index(loop_idx);
+        if (b->type() == WHILE || b->type() == FOR)
+            break;
+    }
+
+    if (loop_idx >= parser.block_count())
+    {
+        append_format(stderr_buffer,
+                      _(L"%ls: Not inside of loop\n"),
+                      argv[0]);
+        builtin_print_help(parser, argv[0], stderr_buffer);
+        return STATUS_BUILTIN_ERROR;
+    }
+
+    /* Skip blocks interior to the loop  */
+    size_t block_idx = loop_idx;
+    while (block_idx--)
+    {
+        parser.block_at_index(block_idx)->skip = true;
+    }
+
+    /* Skip the loop itself */
+    block_t *loop_block = parser.block_at_index(loop_idx);
+    loop_block->skip = true;
+    loop_block->loop_status = is_break ? LOOP_BREAK : LOOP_CONTINUE;
+    return STATUS_BUILTIN_OK;
+}
+
+/**
+   Implementation of the builtin breakpoint command, used to launch the
+   interactive debugger.
+ */
+
+static int builtin_breakpoint(parser_t &parser, wchar_t **argv)
+{
+    parser.push_block(new breakpoint_block_t());
+
+    reader_read(STDIN_FILENO, real_io ? *real_io : io_chain_t());
+
+    parser.pop_block();
+
+    return proc_get_last_status();
+}
+
+
+/**
+   Function for handling the \c return builtin
+*/
+static int builtin_return(parser_t &parser, wchar_t **argv)
+{
+    int argc = builtin_count_args(argv);
+    int status = proc_get_last_status();
+
+    switch (argc)
+    {
+        case 1:
+            break;
+        case 2:
+        {
+            wchar_t *end;
+            errno = 0;
+            status = fish_wcstoi(argv[1],&end,10);
+            if (errno || *end != 0)
+            {
+                append_format(stderr_buffer,
+                              _(L"%ls: Argument '%ls' must be an integer\n"),
+                              argv[0],
+                              argv[1]);
+                builtin_print_help(parser, argv[0], stderr_buffer);
+                return STATUS_BUILTIN_ERROR;
+            }
+            break;
+        }
+        default:
+            append_format(stderr_buffer,
+                          _(L"%ls: Too many arguments\n"),
+                          argv[0]);
+            builtin_print_help(parser, argv[0], stderr_buffer);
+            return STATUS_BUILTIN_ERROR;
+    }
+
+    /* Find the function block */
+    size_t function_block_idx;
+    for (function_block_idx = 0; function_block_idx < parser.block_count(); function_block_idx++)
+    {
+        const block_t *b = parser.block_at_index(function_block_idx);
+        if (b->type() == FUNCTION_CALL || b->type() == FUNCTION_CALL_NO_SHADOW)
+            break;
+    }
+
+    if (function_block_idx >= parser.block_count())
+    {
+        append_format(stderr_buffer,
+                      _(L"%ls: Not inside of function\n"),
+                      argv[0]);
+        builtin_print_help(parser, argv[0], stderr_buffer);
+        return STATUS_BUILTIN_ERROR;
+    }
+
+    /* Skip everything up to (and then including) the function block */
+    for (size_t i=0; i < function_block_idx; i++)
+    {
+        block_t *b = parser.block_at_index(i);
+        b->skip = true;
+    }
+    parser.block_at_index(function_block_idx)->skip = true;
+    return status;
+}
+
+/**
+   History of commands executed by user
+*/
+static int builtin_history(parser_t &parser, wchar_t **argv)
+{
+    int argc = builtin_count_args(argv);
+
+    bool search_history = false;
+    bool delete_item = false;
+    bool search_prefix = false;
+    bool save_history = false;
+    bool clear_history = false;
+    bool merge_history = false;
+
+    static const struct woption long_options[] =
+    {
+        { L"prefix", no_argument, 0, 'p' },
+        { L"delete", no_argument, 0, 'd' },
+        { L"search", no_argument, 0, 's' },
+        { L"contains", no_argument, 0, 'c' },
+        { L"save", no_argument, 0, 'v' },
+        { L"clear", no_argument, 0, 'l' },
+        { L"merge", no_argument, 0, 'm' },
+        { L"help", no_argument, 0, 'h' },
+        { 0, 0, 0, 0 }
+    };
+
+    int opt = 0;
+    int opt_index = 0;
+
+    wgetopter_t w;
+    history_t *history = reader_get_history();
+
+    /* Use the default history if we have none (which happens if invoked non-interactively, e.g. from webconfig.py */
+    if (! history)
+        history = &history_t::history_with_name(L"fish");
+
+    while ((opt = w.wgetopt_long_only(argc, argv, L"pdscvl", long_options, &opt_index)) != EOF)
+    {
+        switch (opt)
+        {
+            case 'p':
+                search_prefix = true;
+                break;
+            case 'd':
+                delete_item = true;
+                break;
+            case 's':
+                search_history = true;
+                break;
+            case 'c':
+                break;
+            case 'v':
+                save_history = true;
+                break;
+            case 'l':
+                clear_history = true;
+                break;
+            case 'm':
+                merge_history = true;
+                break;
+            case 'h':
+                builtin_print_help(parser, argv[0], stdout_buffer);
+                return STATUS_BUILTIN_OK;
+                break;
+            case '?':
+                append_format(stderr_buffer, BUILTIN_ERR_UNKNOWN, argv[0], argv[w.woptind-1]);
+                return STATUS_BUILTIN_ERROR;
+                break;
+            default:
+                append_format(stderr_buffer, BUILTIN_ERR_UNKNOWN, argv[0], argv[w.woptind-1]);
+                return STATUS_BUILTIN_ERROR;
+        }
+    }
+
+    /* Everything after is an argument */
+    const wcstring_list_t args(argv + w.woptind, argv + argc);
+
+    if (argc == 1)
+    {
+        wcstring full_history;
+        history->get_string_representation(&full_history, wcstring(L"\n"));
+        stdout_buffer.append(full_history);
+        stdout_buffer.push_back('\n');
+        return STATUS_BUILTIN_OK;
+    }
+
+    if (merge_history)
+    {
+        history->incorporate_external_changes();
+    }
+
+    if (search_history)
+    {
+        int res = STATUS_BUILTIN_ERROR;
+        for (wcstring_list_t::const_iterator iter = args.begin(); iter != args.end(); ++iter)
+        {
+            const wcstring &search_string = *iter;
+            if (search_string.empty())
+            {
+                append_format(stderr_buffer, BUILTIN_ERR_COMBO2, argv[0], L"Use --search with either --contains or --prefix");
+                return res;
+            }
+
+            history_search_t searcher = history_search_t(*history, search_string, search_prefix?HISTORY_SEARCH_TYPE_PREFIX:HISTORY_SEARCH_TYPE_CONTAINS);
+            while (searcher.go_backwards())
+            {
+                stdout_buffer.append(searcher.current_string());
+                stdout_buffer.append(L"\n");
+                res = STATUS_BUILTIN_OK;
+            }
+        }
+        return res;
+    }
+
+    if (delete_item)
+    {
+        for (wcstring_list_t::const_iterator iter = args.begin(); iter != args.end(); ++iter)
+        {
+            wcstring delete_string = *iter;
+            if (delete_string[0] == '"' && delete_string[delete_string.length() - 1] == '"')
+                delete_string = delete_string.substr(1, delete_string.length() - 2);
+
+            history->remove(delete_string);
+        }
+        return STATUS_BUILTIN_OK;
+    }
+
+    if (save_history)
+    {
+        history->save();
+        return STATUS_BUILTIN_OK;
+    }
+
+    if (clear_history)
+    {
+        history->clear();
+        history->save();
+        return STATUS_BUILTIN_OK;
+    }
+
+    return STATUS_BUILTIN_ERROR;
+}
+
+int builtin_parse(parser_t &parser, wchar_t **argv)
+{
+    struct sigaction act;
+    sigemptyset(& act.sa_mask);
+    act.sa_flags=0;
+    act.sa_handler=SIG_DFL;
+    sigaction(SIGINT, &act, 0);
+
+    std::vector<char> txt;
+    for (;;)
+    {
+        char buff[256];
+        ssize_t amt = read_loop(builtin_stdin, buff, sizeof buff);
+        if (amt <= 0) break;
+        txt.insert(txt.end(), buff, buff + amt);
+    }
+    if (! txt.empty())
+    {
+        const wcstring src = str2wcstring(&txt.at(0), txt.size());
+        parse_node_tree_t parse_tree;
+        parse_error_list_t errors;
+        bool success = parse_tree_from_string(src, parse_flag_include_comments, &parse_tree, &errors);
+        if (! success)
+        {
+            stdout_buffer.append(L"Parsing failed:\n");
+            for (size_t i=0; i < errors.size(); i++)
+            {
+                stdout_buffer.append(errors.at(i).describe(src));
+                stdout_buffer.push_back(L'\n');
+            }
+
+            stdout_buffer.append(L"(Reparsed with continue after error)\n");
+            parse_tree.clear();
+            errors.clear();
+            parse_tree_from_string(src, parse_flag_continue_after_error | parse_flag_include_comments, &parse_tree, &errors);
+        }
+        const wcstring dump = parse_dump_tree(parse_tree, src);
+        stdout_buffer.append(dump);
+    }
+    return STATUS_BUILTIN_OK;
+}
+
+int builtin_true(parser_t &parser, wchar_t **argv)
+{
+    return STATUS_BUILTIN_OK;
+}
+
+int builtin_false(parser_t &parser, wchar_t **argv)
+{
+    return STATUS_BUILTIN_ERROR;
+}
+
+/*
+  END OF BUILTIN COMMANDS
+  Below are functions for handling the builtin commands.
+  THESE MUST BE SORTED BY NAME! Completion lookup uses binary search.
+*/
+
+/**
+   Data about all the builtin commands in fish.
+   Functions that are bound to builtin_generic are handled directly by the parser.
+   NOTE: These must be kept in sorted order!
+*/
+static const builtin_data_t builtin_datas[]=
+{
+    { 		L"[",  &builtin_test, N_(L"Test a condition")   },
+#if 0
+    { 		L"__fish_parse",  &builtin_parse, N_(L"Try out the new parser")  },
+#endif
+    { 		L"and",  &builtin_generic, N_(L"Execute command if previous command suceeded")  },
+    { 		L"begin",  &builtin_generic, N_(L"Create a block of code")   },
+    { 		L"bg",  &builtin_bg, N_(L"Send job to background")   },
+    { 		L"bind",  &builtin_bind, N_(L"Handle fish key bindings")  },
+    { 		L"block",  &builtin_block, N_(L"Temporarily block delivery of events") },
+    { 		L"break",  &builtin_break_continue, N_(L"Stop the innermost loop")   },
+    { 		L"breakpoint",  &builtin_breakpoint, N_(L"Temporarily halt execution of a script and launch an interactive debug prompt")   },
+    { 		L"builtin",  &builtin_builtin, N_(L"Run a builtin command instead of a function") },
+    { 		L"case",  &builtin_generic, N_(L"Conditionally execute a block of commands")   },
+    { 		L"cd",  &builtin_cd, N_(L"Change working directory")   },
+    { 		L"command",   &builtin_command, N_(L"Run a program instead of a function or builtin")   },
+    { 		L"commandline",  &builtin_commandline, N_(L"Set or get the commandline")   },
+    { 		L"complete",  &builtin_complete, N_(L"Edit command specific completions")   },
+    { 		L"contains",  &builtin_contains, N_(L"Search for a specified string in a list")   },
+    { 		L"continue",  &builtin_break_continue, N_(L"Skip the rest of the current lap of the innermost loop")   },
+    { 		L"count",  &builtin_count, N_(L"Count the number of arguments")   },
+    {       L"echo",  &builtin_echo, N_(L"Print arguments") },
+    { 		L"else",  &builtin_generic, N_(L"Evaluate block if condition is false")   },
+    { 		L"emit",  &builtin_emit, N_(L"Emit an event") },
+    { 		L"end",  &builtin_generic, N_(L"End a block of commands")   },
+    { 		L"exec",  &builtin_generic, N_(L"Run command in current process")  },
+    { 		L"exit",  &builtin_exit, N_(L"Exit the shell") },
+    { 		L"false",  &builtin_false, N_(L"Return an unsuccessful result") },
+    { 		L"fg",  &builtin_fg, N_(L"Send job to foreground")   },
+    { 		L"for",  &builtin_generic, N_(L"Perform a set of commands multiple times")   },
+    { 		L"function",  &builtin_generic, N_(L"Define a new function")   },
+    { 		L"functions",  &builtin_functions, N_(L"List or remove functions")   },
+    { 		L"history",  &builtin_history, N_(L"History of commands executed by user")   },
+    { 		L"if",  &builtin_generic, N_(L"Evaluate block if condition is true")   },
+    { 		L"jobs",  &builtin_jobs, N_(L"Print currently running jobs")   },
+    { 		L"not",  &builtin_generic, N_(L"Negate exit status of job")  },
+    { 		L"or",  &builtin_generic, N_(L"Execute command if previous command failed")  },
+    { 		L"printf",  &builtin_printf, N_(L"Prints formatted text")  },
+    { 		L"pwd",  &builtin_pwd, N_(L"Print the working directory")  },
+    { 		L"random",  &builtin_random, N_(L"Generate random number")  },
+    { 		L"read",  &builtin_read, N_(L"Read a line of input into variables")   },
+    { 		L"return",  &builtin_return, N_(L"Stop the currently evaluated function")   },
+    { 		L"set",  &builtin_set, N_(L"Handle environment variables")   },
+    { 		L"set_color",  &builtin_set_color, N_(L"Set the terminal color")   },
+    { 		L"source",  &builtin_source, N_(L"Evaluate contents of file")   },
+    { 		L"status",  &builtin_status, N_(L"Return status information about fish")  },
+    { 		L"switch",  &builtin_generic, N_(L"Conditionally execute a block of commands")   },
+    { 		L"test",  &builtin_test, N_(L"Test a condition")   },
+    { 		L"true",  &builtin_true, N_(L"Return a successful result") },
+    { 		L"ulimit",  &builtin_ulimit, N_(L"Set or get the shells resource usage limits")  },
+    { 		L"while",  &builtin_generic, N_(L"Perform a command multiple times")   }
+};
+
+#define BUILTIN_COUNT (sizeof builtin_datas / sizeof *builtin_datas)
+
+static const builtin_data_t *builtin_lookup(const wcstring &name)
+{
+    const builtin_data_t *array_end = builtin_datas + BUILTIN_COUNT;
+    const builtin_data_t *found = std::lower_bound(builtin_datas, array_end, name);
+    if (found != array_end && name == found->name)
+    {
+        return found;
+    }
+    else
+    {
+        return NULL;
+    }
+}
+
+void builtin_init()
+{
+    for (size_t i=0; i < BUILTIN_COUNT; i++)
+    {
+        intern_static(builtin_datas[i].name);
+    }
+}
+
+void builtin_destroy()
+{
+}
+
+int builtin_exists(const wcstring &cmd)
+{
+    return !!builtin_lookup(cmd);
+}
+
+/**
+   Return true if the specified builtin should handle it's own help,
+   false otherwise.
+*/
+static int internal_help(const wchar_t *cmd)
+{
+    CHECK(cmd, 0);
+    return contains(cmd, L"for", L"while", L"function",
+                    L"if", L"end", L"switch", L"case", L"count", L"printf");
+}
+
+
+int builtin_run(parser_t &parser, const wchar_t * const *argv, const io_chain_t &io)
+{
+    int (*cmd)(parser_t &parser, const wchar_t * const *argv)=0;
+    
+    scoped_push<const io_chain_t*> set_real_io(&real_io, &io);
+
+    CHECK(argv, STATUS_BUILTIN_ERROR);
+    CHECK(argv[0], STATUS_BUILTIN_ERROR);
+
+    const builtin_data_t *data = builtin_lookup(argv[0]);
+    cmd = (int (*)(parser_t &parser, const wchar_t * const*))(data ? data->func : NULL);
+
+    if (argv[1] != 0 && !internal_help(argv[0]))
+    {
+        if (argv[2] == 0 && (parse_util_argument_is_help(argv[1], 0)))
+        {
+            builtin_print_help(parser, argv[0], stdout_buffer);
+            return STATUS_BUILTIN_OK;
+        }
+    }
+
+    if (data != NULL)
+    {
+        int status;
+
+        status = cmd(parser, argv);
+        return status;
+
+    }
+    else
+    {
+        debug(0, UNKNOWN_BUILTIN_ERR_MSG, argv[0]);
+    }
+    return STATUS_BUILTIN_ERROR;
+}
+
+
+wcstring_list_t builtin_get_names(void)
+{
+    wcstring_list_t result;
+    result.reserve(BUILTIN_COUNT);
+    for (size_t i=0; i < BUILTIN_COUNT; i++)
+    {
+        result.push_back(builtin_datas[i].name);
+    }
+    return result;
+}
+
+void builtin_get_names(std::vector<completion_t> &list)
+{
+    for (size_t i=0; i < BUILTIN_COUNT; i++)
+    {
+        append_completion(list, builtin_datas[i].name);
+    }
+}
+
+wcstring builtin_get_desc(const wcstring &name)
+{
+    wcstring result;
+    const builtin_data_t *builtin = builtin_lookup(name);
+    if (builtin)
+    {
+        result = _(builtin->desc);
+    }
+    return result;
+}
+
+void builtin_push_io(parser_t &parser, int in)
+{
+    ASSERT_IS_MAIN_THREAD();
+    if (builtin_stdin != -1)
+    {
+        struct io_stack_elem_t elem = {builtin_stdin, stdout_buffer, stderr_buffer};
+        io_stack.push(elem);
+    }
+    builtin_stdin = in;
+    stdout_buffer.clear();
+    stderr_buffer.clear();
+}
+
+void builtin_pop_io(parser_t &parser)
+{
+    ASSERT_IS_MAIN_THREAD();
+    builtin_stdin = 0;
+    if (! io_stack.empty())
+    {
+        struct io_stack_elem_t &elem = io_stack.top();
+        stderr_buffer = elem.err;
+        stdout_buffer = elem.out;
+        builtin_stdin = elem.in;
+        io_stack.pop();
+    }
+    else
+    {
+        stdout_buffer.clear();
+        stderr_buffer.clear();
+        builtin_stdin = 0;
+    }
+}
diff --git a/src/builtin.h b/src/builtin.h
new file mode 100644
index 00000000..ca0f4f30
--- /dev/null
+++ b/src/builtin.h
@@ -0,0 +1,199 @@
+/** \file builtin.h
+  Prototypes for functions for executing builtin functions.
+*/
+
+#ifndef FISH_BUILTIN_H
+#define FISH_BUILTIN_H
+
+#include <stddef.h>                     // for size_t
+#include <vector>                       // for vector
+
+#include "io.h"
+#include "common.h"
+
+class completion_t;
+class parser_t;
+
+enum
+{
+    COMMAND_NOT_BUILTIN,
+    BUILTIN_REGULAR,
+    BUILTIN_FUNCTION
+}
+;
+
+/**
+   Error message on missing argument
+*/
+#define BUILTIN_ERR_MISSING _( L"%ls: Expected argument\n" )
+
+/**
+   Error message on invalid combination of options
+*/
+#define BUILTIN_ERR_COMBO _( L"%ls: Invalid combination of options\n" )
+
+/**
+   Error message on invalid combination of options
+*/
+#define BUILTIN_ERR_COMBO2 _( L"%ls: Invalid combination of options,\n%ls\n" )
+
+/**
+   Error message on multiple scope levels for variables
+*/
+#define BUILTIN_ERR_GLOCAL _( L"%ls: Variable scope can only be one of universal, global and local\n" )
+
+/**
+   Error message for specifying both export and unexport to set/read
+*/
+#define BUILTIN_ERR_EXPUNEXP _( L"%ls: Variable can't be both exported and unexported\n" )
+
+/**
+   Error message for unknown switch
+*/
+#define BUILTIN_ERR_UNKNOWN  _( L"%ls: Unknown option '%ls'\n" )
+
+/**
+   Error message for invalid character in variable name
+*/
+#define BUILTIN_ERR_VARCHAR _( L"%ls: Invalid character '%lc' in variable name. Only alphanumerical characters and underscores are valid in a variable name.\n" )
+
+/**
+   Error message for invalid (empty) variable name
+*/
+#define BUILTIN_ERR_VARNAME_ZERO _( L"%ls: Variable name can not be the empty string\n" )
+
+/**
+   Error message when second argument to for isn't 'in'
+*/
+#define BUILTIN_FOR_ERR_IN _( L"%ls: Second argument must be 'in'\n" )
+
+/**
+   Error message for insufficient number of arguments
+*/
+#define BUILTIN_FOR_ERR_COUNT _( L"%ls: Expected at least two arguments, got %d\n")
+
+#define BUILTIN_FOR_ERR_NAME _( L"%ls: '%ls' is not a valid variable name\n" )
+
+/** Error messages for 'else if' */
+#define BUILTIN_ELSEIF_ERR_COUNT _( L"%ls: can only take 'if' and then another command as an argument\n")
+#define BUILTIN_ELSEIF_ERR_ARGUMENT _( L"%ls: any second argument must be 'if'\n")
+
+/**
+   Error message when too many arguments are supplied to a builtin
+*/
+#define BUILTIN_ERR_TOO_MANY_ARGUMENTS _( L"%ls: Too many arguments\n" )
+
+/**
+   Error message when block types mismatch in the end builtin, e.g. 'begin; end for'
+*/
+#define BUILTIN_END_BLOCK_MISMATCH _( L"%ls: Block mismatch: '%ls' vs. '%ls'\n" )
+
+/**
+   Error message for unknown block type in the end builtin, e.g. 'begin; end beggin'
+*/
+#define BUILTIN_END_BLOCK_UNKNOWN _( L"%ls: Unknown block type '%ls'\n" )
+
+#define BUILTIN_ERR_NOT_NUMBER _( L"%ls: Argument '%ls' is not a number\n" )
+
+/** Get the string used to represent stdout and stderr */
+const wcstring &get_stdout_buffer();
+const wcstring &get_stderr_buffer();
+
+/** Output an error */
+void builtin_show_error(const wcstring &err);
+
+/**
+   Kludge. Tells builtins if output is to screen
+*/
+extern int builtin_out_redirect;
+
+/**
+   Kludge. Tells builtins if error is to screen
+*/
+extern int builtin_err_redirect;
+
+
+/**
+   Initialize builtin data.
+*/
+void builtin_init();
+
+/**
+   Destroy builtin data.
+*/
+void builtin_destroy();
+
+/**
+  Is there a builtin command with the given name?
+*/
+int builtin_exists(const wcstring &cmd);
+
+/**
+  Execute a builtin command
+
+  \param parser The parser being used
+  \param argv Array containing the command and parameters
+  of the builtin.  The list is terminated by a
+  null pointer. This syntax resembles the syntax
+  for exec.
+  \param io the io redirections to perform on this builtin.
+
+  \return the exit status of the builtin command
+*/
+int builtin_run(parser_t &parser, const wchar_t * const *argv, const io_chain_t &io);
+
+/** Returns a list of all builtin names */
+wcstring_list_t builtin_get_names(void);
+
+/** Insert all builtin names into list. */
+void builtin_get_names(std::vector<completion_t> &list);
+
+/**
+   Pushes a new set of input/output to the stack. The new stdin is supplied, a new set of output strings is created.
+*/
+void builtin_push_io(parser_t &parser, int stdin_fd);
+
+/**
+   Pops a set of input/output from the stack. The output strings are destroued, but the input file is not closed.
+*/
+void builtin_pop_io(parser_t &parser);
+
+
+/**
+   Return a one-line description of the specified builtin.
+*/
+wcstring builtin_get_desc(const wcstring &b);
+
+
+
+/** Support for setting and removing transient command lines.
+    This is used by 'complete -C' in order to make
+    the commandline builtin operate on the string to complete instead
+    of operating on whatever is to be completed. It's also used by
+    completion wrappers, to allow a command to appear as the command
+    being wrapped for the purposes of completion.
+    
+    Instantiating an instance of builtin_commandline_scoped_transient_t
+    pushes the command as the new transient commandline. The destructor removes it.
+    It will assert if construction/destruction does not happen in a stack-like (LIFO) order.
+*/
+class builtin_commandline_scoped_transient_t
+{
+    size_t token;
+    public:
+    builtin_commandline_scoped_transient_t(const wcstring &cmd);
+    ~builtin_commandline_scoped_transient_t();
+};
+
+
+/**
+   Run the __fish_print_help function to obtain the help information
+   for the specified command.
+*/
+wcstring builtin_help_get(parser_t &parser, const wchar_t *cmd);
+
+/** Defines a function, like builtin_function. Returns 0 on success. args should NOT contain 'function' as the first argument. */
+int define_function(parser_t &parser, const wcstring_list_t &args, const wcstring &contents, int definition_line_offset, wcstring *out_err);
+
+
+#endif
diff --git a/src/builtin_commandline.cpp b/src/builtin_commandline.cpp
new file mode 100644
index 00000000..23aaaa52
--- /dev/null
+++ b/src/builtin_commandline.cpp
@@ -0,0 +1,674 @@
+/** \file builtin_commandline.c Functions defining the commandline builtin
+
+Functions used for implementing the commandline builtin.
+
+*/
+#include "config.h"
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <wchar.h>
+#include <wctype.h>
+#include <sys/types.h>
+#include <termios.h>
+#include <signal.h>
+
+#include "fallback.h"
+#include "util.h"
+
+#include "wutil.h"
+#include "builtin.h"
+#include "common.h"
+#include "wgetopt.h"
+#include "reader.h"
+#include "proc.h"
+#include "parser.h"
+#include "tokenizer.h"
+#include "input_common.h"
+#include "input.h"
+
+#include "parse_util.h"
+
+/**
+   Which part of the comandbuffer are we operating on
+*/
+enum
+{
+    STRING_MODE=1, /**< Operate on entire buffer */
+    JOB_MODE, /**< Operate on job under cursor */
+    PROCESS_MODE, /**< Operate on process under cursor */
+    TOKEN_MODE /**< Operate on token under cursor */
+}
+;
+
+/**
+   For text insertion, how should it be done
+*/
+enum
+{
+    REPLACE_MODE=1, /**< Replace current text */
+    INSERT_MODE, /**< Insert at cursor position */
+    APPEND_MODE /**< Insert at end of current token/command/buffer */
+}
+;
+
+/**
+   Pointer to what the commandline builtin considers to be the current
+   contents of the command line buffer.
+ */
+static const wchar_t *current_buffer=0;
+/**
+   What the commandline builtin considers to be the current cursor
+   position.
+ */
+static size_t current_cursor_pos = (size_t)(-1);
+
+/**
+   Returns the current commandline buffer.
+*/
+static const wchar_t *get_buffer()
+{
+    return current_buffer;
+}
+
+/**
+   Returns the position of the cursor
+*/
+static size_t get_cursor_pos()
+{
+    return current_cursor_pos;
+}
+
+static pthread_mutex_t transient_commandline_lock = PTHREAD_MUTEX_INITIALIZER;
+static wcstring_list_t *get_transient_stack()
+{
+    ASSERT_IS_MAIN_THREAD();
+    ASSERT_IS_LOCKED(transient_commandline_lock);
+    // A pointer is a little more efficient than an object as a static because we can elide the thread-safe initialization
+    static wcstring_list_t *result = NULL;
+    if (! result)
+    {
+        result = new wcstring_list_t();
+    }
+    return result;
+}
+
+static bool get_top_transient(wcstring *out_result)
+{
+    ASSERT_IS_MAIN_THREAD();
+    bool result = false;
+    scoped_lock locker(transient_commandline_lock);
+    const wcstring_list_t *stack = get_transient_stack();
+    if (! stack->empty())
+    {
+        out_result->assign(stack->back());
+        result = true;
+    }
+    return result;
+}
+
+builtin_commandline_scoped_transient_t::builtin_commandline_scoped_transient_t(const wcstring &cmd)
+{
+    ASSERT_IS_MAIN_THREAD();
+    scoped_lock locker(transient_commandline_lock);
+    wcstring_list_t *stack = get_transient_stack();
+    stack->push_back(cmd);
+    this->token = stack->size();
+}
+
+builtin_commandline_scoped_transient_t::~builtin_commandline_scoped_transient_t()
+{
+    ASSERT_IS_MAIN_THREAD();
+    scoped_lock locker(transient_commandline_lock);
+    wcstring_list_t *stack = get_transient_stack();
+    assert(this->token == stack->size());
+    stack->pop_back();
+}
+
+/**
+   Replace/append/insert the selection with/at/after the specified string.
+
+   \param begin beginning of selection
+   \param end end of selection
+   \param insert the string to insert
+   \param append_mode can be one of REPLACE_MODE, INSERT_MODE or APPEND_MODE, affects the way the test update is performed
+*/
+static void replace_part(const wchar_t *begin,
+                         const wchar_t *end,
+                         const wchar_t *insert,
+                         int append_mode)
+{
+    const wchar_t *buff = get_buffer();
+    size_t out_pos = get_cursor_pos();
+
+    wcstring out;
+
+    out.append(buff, begin - buff);
+
+    switch (append_mode)
+    {
+        case REPLACE_MODE:
+        {
+
+            out.append(insert);
+            out_pos = wcslen(insert) + (begin-buff);
+            break;
+
+        }
+        case APPEND_MODE:
+        {
+            out.append(begin, end-begin);
+            out.append(insert);
+            break;
+        }
+        case INSERT_MODE:
+        {
+            long cursor = get_cursor_pos() -(begin-buff);
+            out.append(begin, cursor);
+            out.append(insert);
+            out.append(begin+cursor, end-begin-cursor);
+            out_pos +=  wcslen(insert);
+            break;
+        }
+    }
+    out.append(end);
+    reader_set_buffer(out, out_pos);
+}
+
+/**
+   Output the specified selection.
+
+   \param begin start of selection
+   \param end  end of selection
+   \param cut_at_cursor whether printing should stop at the surrent cursor position
+   \param tokenize whether the string should be tokenized, printing one string token on every line and skipping non-string tokens
+*/
+static void write_part(const wchar_t *begin,
+                       const wchar_t *end,
+                       int cut_at_cursor,
+                       int tokenize)
+{
+    size_t pos = get_cursor_pos()-(begin-get_buffer());
+
+    if (tokenize)
+    {
+        wchar_t *buff = wcsndup(begin, end-begin);
+//    fwprintf( stderr, L"Subshell: %ls, end char %lc\n", buff, *end );
+        wcstring out;
+        tokenizer_t tok(buff, TOK_ACCEPT_UNFINISHED);
+        for (; tok_has_next(&tok); tok_next(&tok))
+        {
+            if ((cut_at_cursor) &&
+                    (tok_get_pos(&tok)+wcslen(tok_last(&tok)) >= pos))
+                break;
+
+            switch (tok_last_type(&tok))
+            {
+                case TOK_STRING:
+                {
+                    wcstring tmp = tok_last(&tok);
+                    unescape_string_in_place(&tmp, UNESCAPE_INCOMPLETE);
+                    out.append(tmp);
+                    out.push_back(L'\n');
+                    break;
+                }
+
+                default:
+                {
+                    break;
+                }
+            }
+        }
+
+        stdout_buffer.append(out);
+
+        free(buff);
+    }
+    else
+    {
+        if (cut_at_cursor)
+        {
+            end = begin+pos;
+        }
+
+//    debug( 0, L"woot2 %ls -> %ls", buff, esc );
+        wcstring tmp = wcstring(begin, end - begin);
+        unescape_string_in_place(&tmp, UNESCAPE_INCOMPLETE);
+        stdout_buffer.append(tmp);
+        stdout_buffer.append(L"\n");
+
+    }
+}
+
+
+/**
+   The commandline builtin. It is used for specifying a new value for
+   the commandline.
+*/
+static int builtin_commandline(parser_t &parser, wchar_t **argv)
+{
+    wgetopter_t w;
+    int buffer_part=0;
+    int cut_at_cursor=0;
+
+    int argc = builtin_count_args(argv);
+    int append_mode=0;
+
+    int function_mode = 0;
+    int selection_mode = 0;
+
+    int tokenize = 0;
+
+    int cursor_mode = 0;
+    int line_mode = 0;
+    int search_mode = 0;
+    int paging_mode = 0;
+    const wchar_t *begin = NULL, *end = NULL;
+    
+    scoped_push<const wchar_t *> saved_current_buffer(&current_buffer);
+    scoped_push<size_t> saved_current_cursor_pos(&current_cursor_pos);
+    
+    wcstring transient_commandline;
+    if (get_top_transient(&transient_commandline))
+    {
+        current_buffer = transient_commandline.c_str();
+        current_cursor_pos = transient_commandline.size();
+    }
+    else
+    {
+        current_buffer = reader_get_buffer();
+        current_cursor_pos = reader_get_cursor_pos();
+    }
+
+    if (!get_buffer())
+    {
+        if (is_interactive_session)
+        {
+            /*
+              Prompt change requested while we don't have
+              a prompt, most probably while reading the
+              init files. Just ignore it.
+            */
+            return 1;
+        }
+
+        stderr_buffer.append(argv[0]);
+        stderr_buffer.append(L": Can not set commandline in non-interactive mode\n");
+        builtin_print_help(parser, argv[0], stderr_buffer);
+        return 1;
+    }
+
+    w.woptind=0;
+
+    while (1)
+    {
+        static const struct woption
+                long_options[] =
+        {
+            { L"append", no_argument, 0, 'a' },
+            { L"insert", no_argument, 0, 'i' },
+            { L"replace", no_argument, 0, 'r' },
+            { L"current-job", no_argument, 0, 'j' },
+            { L"current-process", no_argument, 0, 'p' },
+            { L"current-token", no_argument, 0, 't' },
+            { L"current-buffer", no_argument, 0, 'b' },
+            { L"cut-at-cursor", no_argument, 0, 'c' },
+            { L"function", no_argument, 0, 'f' },
+            { L"tokenize", no_argument, 0, 'o' },
+            { L"help", no_argument, 0, 'h' },
+            { L"input", required_argument, 0, 'I' },
+            { L"cursor", no_argument, 0, 'C' },
+            { L"line", no_argument, 0, 'L' },
+            { L"search-mode", no_argument, 0, 'S' },
+            { L"selection", no_argument, 0, 's' },
+            { L"paging-mode", no_argument, 0, 'P' },
+            { 0, 0, 0, 0 }
+        };
+
+        int opt_index = 0;
+
+        int opt = w.wgetopt_long(argc,
+                                 argv,
+                                 L"abijpctwforhI:CLSsP",
+                                 long_options,
+                                 &opt_index);
+        if (opt == -1)
+            break;
+
+        switch (opt)
+        {
+            case 0:
+                if (long_options[opt_index].flag != 0)
+                    break;
+                append_format(stderr_buffer,
+                              BUILTIN_ERR_UNKNOWN,
+                              argv[0],
+                              long_options[opt_index].name);
+                builtin_print_help(parser, argv[0], stderr_buffer);
+
+                return 1;
+
+            case L'a':
+                append_mode = APPEND_MODE;
+                break;
+
+            case L'b':
+                buffer_part = STRING_MODE;
+                break;
+
+
+            case L'i':
+                append_mode = INSERT_MODE;
+                break;
+
+            case L'r':
+                append_mode = REPLACE_MODE;
+                break;
+
+            case 'c':
+                cut_at_cursor=1;
+                break;
+
+            case 't':
+                buffer_part = TOKEN_MODE;
+                break;
+
+            case 'j':
+                buffer_part = JOB_MODE;
+                break;
+
+            case 'p':
+                buffer_part = PROCESS_MODE;
+                break;
+
+            case 'f':
+                function_mode=1;
+                break;
+
+            case 'o':
+                tokenize=1;
+                break;
+
+            case 'I':
+                current_buffer = w.woptarg;
+                current_cursor_pos = wcslen(w.woptarg);
+                break;
+
+            case 'C':
+                cursor_mode = 1;
+                break;
+
+            case 'L':
+                line_mode = 1;
+                break;
+
+            case 'S':
+                search_mode = 1;
+                break;
+
+            case 's':
+                selection_mode = 1;
+                break;
+
+            case 'P':
+                paging_mode = 1;
+                break;
+
+            case 'h':
+                builtin_print_help(parser, argv[0], stdout_buffer);
+                return 0;
+
+            case L'?':
+                builtin_unknown_option(parser, argv[0], argv[w.woptind-1]);
+                return 1;
+        }
+    }
+
+    if (function_mode)
+    {
+        int i;
+
+        /*
+          Check for invalid switch combinations
+        */
+        if (buffer_part || cut_at_cursor || append_mode || tokenize || cursor_mode || line_mode || search_mode || paging_mode)
+        {
+            append_format(stderr_buffer,
+                          BUILTIN_ERR_COMBO,
+                          argv[0]);
+
+            builtin_print_help(parser, argv[0], stderr_buffer);
+            return 1;
+        }
+
+
+        if (argc == w.woptind)
+        {
+            append_format(stderr_buffer,
+                          BUILTIN_ERR_MISSING,
+                          argv[0]);
+
+            builtin_print_help(parser, argv[0], stderr_buffer);
+            return 1;
+        }
+        for (i=w.woptind; i<argc; i++)
+        {
+            wchar_t c = input_function_get_code(argv[i]);
+            if (c != (wchar_t)(-1))
+            {
+                /*
+                  input_unreadch inserts the specified keypress or
+                  readline function at the back of the queue of unused
+                  keypresses
+                */
+                input_queue_ch(c);
+            }
+            else
+            {
+                append_format(stderr_buffer,
+                              _(L"%ls: Unknown input function '%ls'\n"),
+                              argv[0],
+                              argv[i]);
+                builtin_print_help(parser, argv[0], stderr_buffer);
+                return 1;
+            }
+        }
+
+        return 0;
+    }
+
+    if (selection_mode)
+    {
+        size_t start, len;
+        const wchar_t *buffer = reader_get_buffer();
+        if (reader_get_selection(&start, &len))
+        {
+            stdout_buffer.append(buffer + start, len);
+        }
+        return 0;
+    }
+
+    /*
+      Check for invalid switch combinations
+    */
+    if ((search_mode || line_mode || cursor_mode || paging_mode) && (argc-w.woptind > 1))
+    {
+
+        append_format(stderr_buffer,
+                      argv[0],
+                      L": Too many arguments\n",
+                      NULL);
+        builtin_print_help(parser, argv[0], stderr_buffer);
+        return 1;
+    }
+
+    if ((buffer_part || tokenize || cut_at_cursor) && (cursor_mode || line_mode || search_mode || paging_mode))
+    {
+        append_format(stderr_buffer,
+                      BUILTIN_ERR_COMBO,
+                      argv[0]);
+
+        builtin_print_help(parser, argv[0], stderr_buffer);
+        return 1;
+    }
+
+
+    if ((tokenize || cut_at_cursor) && (argc-w.woptind))
+    {
+        append_format(stderr_buffer,
+                      BUILTIN_ERR_COMBO2,
+                      argv[0],
+                      L"--cut-at-cursor and --tokenize can not be used when setting the commandline");
+
+
+        builtin_print_help(parser, argv[0], stderr_buffer);
+        return 1;
+    }
+
+    if (append_mode && !(argc-w.woptind))
+    {
+        append_format(stderr_buffer,
+                      BUILTIN_ERR_COMBO2,
+                      argv[0],
+                      L"insertion mode switches can not be used when not in insertion mode");
+
+        builtin_print_help(parser, argv[0], stderr_buffer);
+        return 1;
+    }
+
+    /*
+      Set default modes
+    */
+    if (!append_mode)
+    {
+        append_mode = REPLACE_MODE;
+    }
+
+    if (!buffer_part)
+    {
+        buffer_part = STRING_MODE;
+    }
+
+    if (cursor_mode)
+    {
+        if (argc-w.woptind)
+        {
+            wchar_t *endptr;
+            long new_pos;
+            errno = 0;
+
+            new_pos = wcstol(argv[w.woptind], &endptr, 10);
+            if (*endptr || errno)
+            {
+                append_format(stderr_buffer,
+                              BUILTIN_ERR_NOT_NUMBER,
+                              argv[0],
+                              argv[w.woptind]);
+                builtin_print_help(parser, argv[0], stderr_buffer);
+            }
+
+            current_buffer = reader_get_buffer();
+            new_pos = maxi(0L, mini(new_pos, (long)wcslen(current_buffer)));
+            reader_set_buffer(current_buffer, (size_t)new_pos);
+            return 0;
+        }
+        else
+        {
+            append_format(stdout_buffer, L"%lu\n", (unsigned long)reader_get_cursor_pos());
+            return 0;
+        }
+
+    }
+
+    if (line_mode)
+    {
+        size_t pos = reader_get_cursor_pos();
+        const wchar_t *buff = reader_get_buffer();
+        append_format(stdout_buffer, L"%lu\n", (unsigned long)parse_util_lineno(buff, pos));
+        return 0;
+
+    }
+
+    if (search_mode)
+    {
+        return ! reader_search_mode();
+    }
+
+    if (paging_mode)
+    {
+        return ! reader_has_pager_contents();
+    }
+
+
+    switch (buffer_part)
+    {
+        case STRING_MODE:
+        {
+            begin = get_buffer();
+            end = begin+wcslen(begin);
+            break;
+        }
+
+        case PROCESS_MODE:
+        {
+            parse_util_process_extent(get_buffer(),
+                                      get_cursor_pos(),
+                                      &begin,
+                                      &end);
+            break;
+        }
+
+        case JOB_MODE:
+        {
+            parse_util_job_extent(get_buffer(),
+                                  get_cursor_pos(),
+                                  &begin,
+                                  &end);
+            break;
+        }
+
+        case TOKEN_MODE:
+        {
+            parse_util_token_extent(get_buffer(),
+                                    get_cursor_pos(),
+                                    &begin,
+                                    &end,
+                                    0, 0);
+            break;
+        }
+
+    }
+
+    switch (argc-w.woptind)
+    {
+        case 0:
+        {
+            write_part(begin, end, cut_at_cursor, tokenize);
+            break;
+        }
+
+        case 1:
+        {
+            replace_part(begin, end, argv[w.woptind], append_mode);
+            break;
+        }
+
+        default:
+        {
+            wcstring sb = argv[w.woptind];
+            int i;
+
+            for (i=w.woptind+1; i<argc; i++)
+            {
+                sb.push_back(L'\n');
+                sb.append(argv[i]);
+            }
+
+            replace_part(begin, end, sb.c_str(), append_mode);
+
+            break;
+        }
+    }
+
+    return 0;
+}
diff --git a/src/builtin_complete.cpp b/src/builtin_complete.cpp
new file mode 100644
index 00000000..6c4ccbcb
--- /dev/null
+++ b/src/builtin_complete.cpp
@@ -0,0 +1,607 @@
+/** \file builtin_complete.c Functions defining the complete builtin
+
+Functions used for implementing the complete builtin.
+
+*/
+#include "config.h"
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <wchar.h>
+#include <wctype.h>
+#include <sys/types.h>
+#include <termios.h>
+#include <signal.h>
+
+#include "fallback.h"
+#include "util.h"
+
+#include "wutil.h"
+#include "builtin.h"
+#include "common.h"
+#include "complete.h"
+#include "wgetopt.h"
+#include "parser.h"
+#include "reader.h"
+
+/*
+  builtin_complete_* are a set of rather silly looping functions that
+  make sure that all the proper combinations of complete_add or
+  complete_remove get called. This is needed since complete allows you
+  to specify multiple switches on a single commandline, like 'complete
+  -s a -s b -s c', but the complete_add function only accepts one
+  short switch and one long switch.
+*/
+
+/**
+   Silly function
+*/
+static void  builtin_complete_add2(const wchar_t *cmd,
+                                   int cmd_type,
+                                   const wchar_t *short_opt,
+                                   const wcstring_list_t &gnu_opt,
+                                   const wcstring_list_t &old_opt,
+                                   int result_mode,
+                                   const wchar_t *condition,
+                                   const wchar_t *comp,
+                                   const wchar_t *desc,
+                                   int flags)
+{
+    size_t i;
+    const wchar_t *s;
+
+    for (s=short_opt; *s; s++)
+    {
+        complete_add(cmd,
+                     cmd_type,
+                     *s,
+                     0,
+                     0,
+                     result_mode,
+                     condition,
+                     comp,
+                     desc,
+                     flags);
+    }
+
+    for (i=0; i<gnu_opt.size(); i++)
+    {
+        complete_add(cmd,
+                     cmd_type,
+                     0,
+                     gnu_opt.at(i).c_str(),
+                     0,
+                     result_mode,
+                     condition,
+                     comp,
+                     desc,
+                     flags);
+    }
+
+    for (i=0; i<old_opt.size(); i++)
+    {
+        complete_add(cmd,
+                     cmd_type,
+                     0,
+                     old_opt.at(i).c_str(),
+                     1,
+                     result_mode,
+                     condition,
+                     comp,
+                     desc,
+                     flags);
+    }
+
+    if (old_opt.empty() && gnu_opt.empty() && wcslen(short_opt) == 0)
+    {
+        complete_add(cmd,
+                     cmd_type,
+                     0,
+                     0,
+                     0,
+                     result_mode,
+                     condition,
+                     comp,
+                     desc,
+                     flags);
+    }
+}
+
+/**
+   Silly function
+*/
+static void  builtin_complete_add(const wcstring_list_t &cmd,
+                                  const wcstring_list_t &path,
+                                  const wchar_t *short_opt,
+                                  wcstring_list_t &gnu_opt,
+                                  wcstring_list_t &old_opt,
+                                  int result_mode,
+                                  int authoritative,
+                                  const wchar_t *condition,
+                                  const wchar_t *comp,
+                                  const wchar_t *desc,
+                                  int flags)
+{
+    for (size_t i=0; i<cmd.size(); i++)
+    {
+        builtin_complete_add2(cmd.at(i).c_str(),
+                              COMMAND,
+                              short_opt,
+                              gnu_opt,
+                              old_opt,
+                              result_mode,
+                              condition,
+                              comp,
+                              desc,
+                              flags);
+
+        if (authoritative != -1)
+        {
+            complete_set_authoritative(cmd.at(i).c_str(),
+                                       COMMAND,
+                                       authoritative);
+        }
+
+    }
+
+    for (size_t i=0; i<path.size(); i++)
+    {
+        builtin_complete_add2(path.at(i).c_str(),
+                              PATH,
+                              short_opt,
+                              gnu_opt,
+                              old_opt,
+                              result_mode,
+                              condition,
+                              comp,
+                              desc,
+                              flags);
+
+        if (authoritative != -1)
+        {
+            complete_set_authoritative(path.at(i).c_str(),
+                                       PATH,
+                                       authoritative);
+        }
+
+    }
+}
+
+/**
+   Silly function
+*/
+static void builtin_complete_remove3(const wchar_t *cmd,
+                                     int cmd_type,
+                                     wchar_t short_opt,
+                                     const wcstring_list_t &long_opt,
+                                     int long_mode)
+{
+    for (size_t i=0; i<long_opt.size(); i++)
+    {
+        complete_remove(cmd,
+                        cmd_type,
+                        short_opt,
+                        long_opt.at(i).c_str(),
+                        long_mode);
+    }
+}
+
+/**
+   Silly function
+*/
+static void  builtin_complete_remove2(const wchar_t *cmd,
+                                      int cmd_type,
+                                      const wchar_t *short_opt,
+                                      const wcstring_list_t &gnu_opt,
+                                      const wcstring_list_t &old_opt)
+{
+    const wchar_t *s = (wchar_t *)short_opt;
+    if (*s)
+    {
+        for (; *s; s++)
+        {
+            if (old_opt.empty() && gnu_opt.empty())
+            {
+                complete_remove(cmd,
+                                cmd_type,
+                                *s,
+                                0,
+                                0);
+
+            }
+            else
+            {
+                builtin_complete_remove3(cmd,
+                                         cmd_type,
+                                         *s,
+                                         gnu_opt,
+                                         0);
+                builtin_complete_remove3(cmd,
+                                         cmd_type,
+                                         *s,
+                                         old_opt,
+                                         1);
+            }
+        }
+    }
+    else if (gnu_opt.empty() && old_opt.empty())
+    {
+        complete_remove(cmd,
+                        cmd_type,
+                        0,
+                        0,
+                        0);
+    }
+    else
+    {
+        builtin_complete_remove3(cmd,
+                                 cmd_type,
+                                 0,
+                                 gnu_opt,
+                                 0);
+        builtin_complete_remove3(cmd,
+                                 cmd_type,
+                                 0,
+                                 old_opt,
+                                 1);
+
+    }
+
+
+}
+
+/**
+   Silly function
+*/
+static void  builtin_complete_remove(const wcstring_list_t &cmd,
+                                     const wcstring_list_t &path,
+                                     const wchar_t *short_opt,
+                                     const wcstring_list_t &gnu_opt,
+                                     const wcstring_list_t &old_opt)
+{
+    for (size_t i=0; i<cmd.size(); i++)
+    {
+        builtin_complete_remove2(cmd.at(i).c_str(),
+                                 COMMAND,
+                                 short_opt,
+                                 gnu_opt,
+                                 old_opt);
+    }
+
+    for (size_t i=0; i<path.size(); i++)
+    {
+        builtin_complete_remove2(path.at(i).c_str(),
+                                 PATH,
+                                 short_opt,
+                                 gnu_opt,
+                                 old_opt);
+    }
+
+}
+
+/**
+   The complete builtin. Used for specifying programmable
+   tab-completions. Calls the functions in complete.c for any heavy
+   lifting. Defined in builtin_complete.c
+*/
+static int builtin_complete(parser_t &parser, wchar_t **argv)
+{
+    ASSERT_IS_MAIN_THREAD();
+    wgetopter_t w;
+    bool res=false;
+    int argc=0;
+    int result_mode=SHARED;
+    int remove = 0;
+    int authoritative = -1;
+
+    wcstring short_opt;
+    wcstring_list_t gnu_opt, old_opt;
+    const wchar_t *comp=L"", *desc=L"", *condition=L"";
+
+    bool do_complete = false;
+    wcstring do_complete_param;
+
+    wcstring_list_t cmd;
+    wcstring_list_t path;
+    wcstring_list_t wrap_targets;
+
+    static int recursion_level=0;
+
+    argc = builtin_count_args(argv);
+
+    w.woptind=0;
+
+    while (! res)
+    {
+        static const struct woption
+                long_options[] =
+        {
+            { L"exclusive", no_argument, 0, 'x' },
+            { L"no-files", no_argument, 0, 'f' },
+            { L"require-parameter", no_argument, 0, 'r' },
+            { L"path", required_argument, 0, 'p' },
+            { L"command", required_argument, 0, 'c' },
+            { L"short-option", required_argument, 0, 's' },
+            { L"long-option", required_argument, 0, 'l' },
+            { L"old-option", required_argument, 0, 'o' },
+            { L"description", required_argument, 0, 'd' },
+            { L"arguments", required_argument, 0, 'a' },
+            { L"erase", no_argument, 0, 'e' },
+            { L"unauthoritative", no_argument, 0, 'u' },
+            { L"authoritative", no_argument, 0, 'A' },
+            { L"condition", required_argument, 0, 'n' },
+            { L"wraps", required_argument, 0, 'w' },
+            { L"do-complete", optional_argument, 0, 'C' },
+            { L"help", no_argument, 0, 'h' },
+            { 0, 0, 0, 0 }
+        };
+
+        int opt_index = 0;
+
+        int opt = w.wgetopt_long(argc,
+                                 argv,
+                                 L"a:c:p:s:l:o:d:frxeuAn:C::w:h",
+                                 long_options,
+                                 &opt_index);
+        if (opt == -1)
+            break;
+
+        switch (opt)
+        {
+            case 0:
+                if (long_options[opt_index].flag != 0)
+                    break;
+                append_format(stderr_buffer,
+                              BUILTIN_ERR_UNKNOWN,
+                              argv[0],
+                              long_options[opt_index].name);
+                builtin_print_help(parser, argv[0], stderr_buffer);
+
+
+                res = true;
+                break;
+
+            case 'x':
+                result_mode |= EXCLUSIVE;
+                break;
+
+            case 'f':
+                result_mode |= NO_FILES;
+                break;
+
+            case 'r':
+                result_mode |= NO_COMMON;
+                break;
+
+            case 'p':
+            case 'c':
+            {
+                wcstring tmp;
+                if (unescape_string(w.woptarg, &tmp, UNESCAPE_SPECIAL))
+                {
+                    if (opt=='p')
+                        path.push_back(tmp);
+                    else
+                        cmd.push_back(tmp);
+                }
+                else
+                {
+                    append_format(stderr_buffer, L"%ls: Invalid token '%ls'\n", argv[0], w.woptarg);
+                    res = true;
+                }
+                break;
+            }
+
+            case 'd':
+                desc = w.woptarg;
+                break;
+
+            case 'u':
+                authoritative=0;
+                break;
+
+            case 'A':
+                authoritative=1;
+                break;
+
+            case 's':
+                short_opt.append(w.woptarg);
+                break;
+
+            case 'l':
+                gnu_opt.push_back(w.woptarg);
+                break;
+
+            case 'o':
+                old_opt.push_back(w.woptarg);
+                break;
+
+            case 'a':
+                comp = w.woptarg;
+                break;
+
+            case 'e':
+                remove = 1;
+                break;
+
+            case 'n':
+                condition = w.woptarg;
+                break;
+                
+            case 'w':
+                wrap_targets.push_back(w.woptarg);
+                break;
+
+            case 'C':
+                do_complete = true;
+                do_complete_param = w.woptarg ? w.woptarg : reader_get_buffer();
+                break;
+
+            case 'h':
+                builtin_print_help(parser, argv[0], stdout_buffer);
+                return 0;
+
+            case '?':
+                builtin_unknown_option(parser, argv[0], argv[w.woptind-1]);
+                res = true;
+                break;
+
+        }
+
+    }
+
+    if (!res)
+    {
+        if (condition && wcslen(condition))
+        {
+            const wcstring condition_string = condition;
+            parse_error_list_t errors;
+            if (parse_util_detect_errors(condition_string, &errors, false /* do not accept incomplete */))
+            {
+                append_format(stderr_buffer,
+                              L"%ls: Condition '%ls' contained a syntax error",
+                              argv[0],
+                              condition);
+                for (size_t i=0; i < errors.size(); i++)
+                {
+                    append_format(stderr_buffer, L"\n%s: ", argv[0]);
+                    stderr_buffer.append(errors.at(i).describe(condition_string));
+                }
+                res = true;
+            }
+        }
+    }
+
+    if (!res)
+    {
+        if (comp && wcslen(comp))
+        {
+            wcstring prefix;
+            if (argv[0])
+            {
+                prefix.append(argv[0]);
+                prefix.append(L": ");
+            }
+
+            wcstring err_text;
+            if (parser.detect_errors_in_argument_list(comp, &err_text, prefix.c_str()))
+            {
+                append_format(stderr_buffer,
+                              L"%ls: Completion '%ls' contained a syntax error\n",
+                              argv[0],
+                              comp);
+                stderr_buffer.append(err_text);
+                stderr_buffer.push_back(L'\n');
+                res = true;
+            }
+        }
+    }
+
+    if (!res)
+    {
+        if (do_complete)
+        {
+            const wchar_t *token;
+
+            parse_util_token_extent(do_complete_param.c_str(), do_complete_param.size(), &token, 0, 0, 0);
+            
+            /* Create a scoped transient command line, so that bulitin_commandline will see our argument, not the reader buffer */
+            builtin_commandline_scoped_transient_t temp_buffer(do_complete_param);
+
+            if (recursion_level < 1)
+            {
+                recursion_level++;
+
+                std::vector<completion_t> comp;
+                complete(do_complete_param, comp, COMPLETION_REQUEST_DEFAULT);
+
+                for (size_t i=0; i< comp.size() ; i++)
+                {
+                    const completion_t &next =  comp.at(i);
+
+                    /* Make a fake commandline, and then apply the completion to it.  */
+                    const wcstring faux_cmdline = token;
+                    size_t tmp_cursor = faux_cmdline.size();
+                    wcstring faux_cmdline_with_completion = completion_apply_to_command_line(next.completion, next.flags, faux_cmdline, &tmp_cursor, false);
+
+                    /* completion_apply_to_command_line will append a space unless COMPLETE_NO_SPACE is set. We don't want to set COMPLETE_NO_SPACE because that won't close quotes. What we want is to close the quote, but not append the space. So we just look for the space and clear it. */
+                    if (!(next.flags & COMPLETE_NO_SPACE) && string_suffixes_string(L" ", faux_cmdline_with_completion))
+                    {
+                        faux_cmdline_with_completion.resize(faux_cmdline_with_completion.size() - 1);
+                    }
+
+                    /* The input data is meant to be something like you would have on the command line, e.g. includes backslashes. The output should be raw, i.e. unescaped. So we need to unescape the command line. See #1127 */
+                    unescape_string_in_place(&faux_cmdline_with_completion, UNESCAPE_DEFAULT);
+                    stdout_buffer.append(faux_cmdline_with_completion);
+
+                    /* Append any description */
+                    if (! next.description.empty())
+                    {
+                        stdout_buffer.push_back(L'\t');
+                        stdout_buffer.append(next.description);
+                    }
+                    stdout_buffer.push_back(L'\n');
+                }
+
+                recursion_level--;
+            }
+        }
+        else if (w.woptind != argc)
+        {
+            append_format(stderr_buffer,
+                          _(L"%ls: Too many arguments\n"),
+                          argv[0]);
+            builtin_print_help(parser, argv[0], stderr_buffer);
+
+            res = true;
+        }
+        else if (cmd.empty() && path.empty())
+        {
+            /* No arguments specified, meaning we print the definitions of
+             * all specified completions to stdout.*/
+            complete_print(stdout_buffer);
+        }
+        else
+        {
+            int flags = COMPLETE_AUTO_SPACE;
+        
+            if (remove)
+            {
+                builtin_complete_remove(cmd,
+                                        path,
+                                        short_opt.c_str(),
+                                        gnu_opt,
+                                        old_opt);
+                
+            }
+            else
+            {
+                builtin_complete_add(cmd,
+                                     path,
+                                     short_opt.c_str(),
+                                     gnu_opt,
+                                     old_opt,
+                                     result_mode,
+                                     authoritative,
+                                     condition,
+                                     comp,
+                                     desc,
+                                     flags);
+            }
+            
+            // Handle wrap targets (probably empty)
+            // We only wrap commands, not paths
+            for (size_t w=0; w < wrap_targets.size(); w++)
+            {
+                const wcstring &wrap_target = wrap_targets.at(w);
+                for (size_t i=0; i < cmd.size(); i++)
+                {
+                    
+                    (remove ? complete_remove_wrapper : complete_add_wrapper)(cmd.at(i), wrap_target);
+                }
+            }
+        }
+    }
+
+    return res ? 1 : 0;
+}
diff --git a/src/builtin_jobs.cpp b/src/builtin_jobs.cpp
new file mode 100644
index 00000000..32079287
--- /dev/null
+++ b/src/builtin_jobs.cpp
@@ -0,0 +1,352 @@
+/** \file builtin_jobs.c
+  Functions for executing the jobs builtin.
+*/
+#include "config.h"
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <wchar.h>
+#include <unistd.h>
+#include <termios.h>
+#include <errno.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <string.h>
+#include <wctype.h>
+
+#include "fallback.h"
+#include "util.h"
+
+#include "wutil.h"
+#include "builtin.h"
+#include "proc.h"
+#include "parser.h"
+#include "common.h"
+#include "wgetopt.h"
+
+
+/**
+   Print modes for the jobs builtin
+*/
+enum
+{
+    JOBS_DEFAULT, /**< Print lots of general info */
+    JOBS_PRINT_PID, /**< Print pid of each process in job */
+    JOBS_PRINT_COMMAND, /**< Print command name of each process in job */
+    JOBS_PRINT_GROUP, /**< Print group id of job */
+}
+;
+
+
+
+#ifdef HAVE__PROC_SELF_STAT
+/**
+   Calculates the cpu usage (in percent) of the specified job.
+*/
+static int cpu_use(const job_t *j)
+{
+    double u=0;
+    process_t *p;
+
+    for (p=j->first_process; p; p=p->next)
+    {
+        struct timeval t;
+        int jiffies;
+        gettimeofday(&t, 0);
+        jiffies = proc_get_jiffies(p);
+
+        double t1 = 1000000.0*p->last_time.tv_sec+p->last_time.tv_usec;
+        double t2 = 1000000.0*t.tv_sec+t.tv_usec;
+
+        /*    fwprintf( stderr, L"t1 %f t2 %f p1 %d p2 %d\n",
+          t1, t2, jiffies, p->last_jiffies );
+        */
+
+        u += ((double)(jiffies-p->last_jiffies))/(t2-t1);
+    }
+    return u*1000000;
+}
+#endif
+
+/**
+   Print information about the specified job
+*/
+static void builtin_jobs_print(const job_t *j, int mode, int header)
+{
+    process_t *p;
+    switch (mode)
+    {
+        case JOBS_DEFAULT:
+        {
+
+            if (header)
+            {
+                /*
+                  Print table header before first job
+                */
+                stdout_buffer.append(_(L"Job\tGroup\t"));
+#ifdef HAVE__PROC_SELF_STAT
+                stdout_buffer.append(_(L"CPU\t"));
+#endif
+                stdout_buffer.append(_(L"State\tCommand\n"));
+            }
+
+            append_format(stdout_buffer, L"%d\t%d\t", j->job_id, j->pgid);
+
+#ifdef HAVE__PROC_SELF_STAT
+            append_format(stdout_buffer, L"%d%%\t", cpu_use(j));
+#endif
+            stdout_buffer.append(job_is_stopped(j)?_(L"stopped"):_(L"running"));
+            stdout_buffer.append(L"\t");
+            stdout_buffer.append(j->command_wcstr());
+            stdout_buffer.append(L"\n");
+            break;
+        }
+
+        case JOBS_PRINT_GROUP:
+        {
+            if (header)
+            {
+                /*
+                  Print table header before first job
+                */
+                stdout_buffer.append(_(L"Group\n"));
+            }
+            append_format(stdout_buffer, L"%d\n", j->pgid);
+            break;
+        }
+
+        case JOBS_PRINT_PID:
+        {
+            if (header)
+            {
+                /*
+                  Print table header before first job
+                */
+                stdout_buffer.append(_(L"Process\n"));
+            }
+
+            for (p=j->first_process; p; p=p->next)
+            {
+                append_format(stdout_buffer, L"%d\n", p->pid);
+            }
+            break;
+        }
+
+        case JOBS_PRINT_COMMAND:
+        {
+            if (header)
+            {
+                /*
+                  Print table header before first job
+                */
+                stdout_buffer.append(_(L"Command\n"));
+            }
+
+            for (p=j->first_process; p; p=p->next)
+            {
+                append_format(stdout_buffer, L"%ls\n", p->argv0());
+            }
+            break;
+        }
+    }
+
+}
+
+
+
+/**
+   The jobs builtin. Used fopr printing running jobs. Defined in builtin_jobs.c.
+*/
+static int builtin_jobs(parser_t &parser, wchar_t **argv)
+{
+    wgetopter_t w;
+    int argc=0;
+    int found=0;
+    int mode=JOBS_DEFAULT;
+    int print_last = 0;
+
+    argc = builtin_count_args(argv);
+    w.woptind=0;
+
+    while (1)
+    {
+        static const struct woption
+                long_options[] =
+        {
+            {
+                L"pid", no_argument, 0, 'p'
+            }
+            ,
+            {
+                L"command", no_argument, 0, 'c'
+            }
+            ,
+            {
+                L"group", no_argument, 0, 'g'
+            }
+            ,
+            {
+                L"last", no_argument, 0, 'l'
+            }
+            ,
+            {
+                L"help", no_argument, 0, 'h'
+            }
+            ,
+            {
+                0, 0, 0, 0
+            }
+        }
+        ;
+
+        int opt_index = 0;
+
+        int opt = w.wgetopt_long(argc,
+                                 argv,
+                                 L"pclgh",
+                                 long_options,
+                                 &opt_index);
+        if (opt == -1)
+            break;
+
+        switch (opt)
+        {
+            case 0:
+                if (long_options[opt_index].flag != 0)
+                    break;
+                append_format(stderr_buffer,
+                              BUILTIN_ERR_UNKNOWN,
+                              argv[0],
+                              long_options[opt_index].name);
+
+                builtin_print_help(parser, argv[0], stderr_buffer);
+
+
+                return 1;
+
+
+            case 'p':
+                mode=JOBS_PRINT_PID;
+                break;
+
+            case 'c':
+                mode=JOBS_PRINT_COMMAND;
+                break;
+
+            case 'g':
+                mode=JOBS_PRINT_GROUP;
+                break;
+
+            case 'l':
+            {
+                print_last = 1;
+                break;
+            }
+
+            case 'h':
+                builtin_print_help(parser, argv[0], stdout_buffer);
+                return 0;
+
+            case '?':
+                builtin_unknown_option(parser, argv[0], argv[w.woptind-1]);
+                return 1;
+
+        }
+    }
+
+
+    /*
+      Do not babble if not interactive
+    */
+    if (builtin_out_redirect)
+    {
+        found=1;
+    }
+
+    if (print_last)
+    {
+        /*
+          Ignore unconstructed jobs, i.e. ourself.
+        */
+        job_iterator_t jobs;
+        const job_t *j;
+        while ((j = jobs.next()))
+        {
+
+            if ((j->flags & JOB_CONSTRUCTED) && !job_is_completed(j))
+            {
+                builtin_jobs_print(j, mode, !found);
+                return 0;
+            }
+        }
+
+    }
+    else
+    {
+        if (w.woptind < argc)
+        {
+            int i;
+
+            found = 1;
+
+            for (i=w.woptind; i<argc; i++)
+            {
+                int pid;
+                wchar_t *end;
+                errno=0;
+                pid=fish_wcstoi(argv[i], &end, 10);
+                if (errno || *end)
+                {
+                    append_format(stderr_buffer,
+                                  _(L"%ls: '%ls' is not a job\n"),
+                                  argv[0],
+                                  argv[i]);
+                    return 1;
+                }
+
+                const job_t *j = job_get_from_pid(pid);
+
+                if (j && !job_is_completed(j))
+                {
+                    builtin_jobs_print(j, mode, !found);
+                }
+                else
+                {
+                    append_format(stderr_buffer,
+                                  _(L"%ls: No suitable job: %d\n"),
+                                  argv[0],
+                                  pid);
+                    return 1;
+                }
+            }
+        }
+        else
+        {
+            job_iterator_t jobs;
+            const job_t *j;
+            while ((j = jobs.next()))
+            {
+                /*
+                  Ignore unconstructed jobs, i.e. ourself.
+                */
+                if ((j->flags & JOB_CONSTRUCTED) && !job_is_completed(j))
+                {
+                    builtin_jobs_print(j, mode, !found);
+                    found = 1;
+                }
+            }
+        }
+    }
+
+    if (!found)
+    {
+        append_format(stdout_buffer,
+                      _(L"%ls: There are no jobs\n"),
+                      argv[0]);
+        return 1;
+    }
+
+    return 0;
+}
+
diff --git a/src/builtin_printf.cpp b/src/builtin_printf.cpp
new file mode 100644
index 00000000..916166b0
--- /dev/null
+++ b/src/builtin_printf.cpp
@@ -0,0 +1,787 @@
+/* printf - format and print data
+   Copyright (C) 1990-2007 Free Software Foundation, Inc.
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 2, or (at your option)
+   any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software Foundation,
+   Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.  */
+
+/* Usage: printf format [argument...]
+
+   A front end to the printf function that lets it be used from the shell.
+
+   Backslash escapes:
+
+   \" = double quote
+   \\ = backslash
+   \a = alert (bell)
+   \b = backspace
+   \c = produce no further output
+   \e = escape
+   \f = form feed
+   \n = new line
+   \r = carriage return
+   \t = horizontal tab
+   \v = vertical tab
+   \ooo = octal number (ooo is 1 to 3 digits)
+   \xhh = hexadecimal number (hhh is 1 to 2 digits)
+   \uhhhh = 16-bit Unicode character (hhhh is 4 digits)
+   \Uhhhhhhhh = 32-bit Unicode character (hhhhhhhh is 8 digits)
+
+   Additional directive:
+
+   %b = print an argument string, interpreting backslash escapes,
+     except that octal escapes are of the form \0 or \0ooo.
+
+   The `format' argument is re-used as many times as necessary
+   to convert all of the given arguments.
+
+   David MacKenzie <djm@gnu.ai.mit.edu> */
+
+/* This file has been imported from source code of printf command in GNU Coreutils version 6.9 */
+
+#include <stdio.h>
+#include <sys/types.h>
+#include <inttypes.h>
+
+#include "common.h"
+
+struct builtin_printf_state_t
+{
+    /* The status of the operation */
+    int exit_code;
+
+    /* Whether we should stop outputting. This gets set in the case of an error, and also with the \c escape. */
+    bool early_exit;
+
+    builtin_printf_state_t() : exit_code(0), early_exit(false)
+    {
+    }
+
+    void verify_numeric(const wchar_t *s, const wchar_t *end, int errcode);
+
+    void print_direc(const wchar_t *start, size_t length, wchar_t conversion,
+                     bool have_field_width, int field_width,
+                     bool have_precision, int precision,
+                     wchar_t const *argument);
+
+    int print_formatted(const wchar_t *format, int argc, wchar_t **argv);
+
+    void fatal_error(const wchar_t *format, ...);
+
+    long print_esc(const wchar_t *escstart, bool octal_0);
+    void print_esc_string(const wchar_t *str);
+    void print_esc_char(wchar_t c);
+
+    void append_output(wchar_t c);
+    void append_output(const wchar_t *c);
+    void append_format_output(const wchar_t *fmt, ...);
+};
+
+static bool is_octal_digit(wchar_t c)
+{
+    return c != L'\0' && wcschr(L"01234567", c) != NULL;
+}
+
+static bool is_hex_digit(wchar_t c)
+{
+    return c != L'\0' && wcschr(L"0123456789ABCDEFabcdef", c) != NULL;
+}
+
+static int hex_to_bin(const wchar_t &c)
+{
+    switch (c)
+    {
+        case L'0':
+            return 0;
+        case L'1':
+            return 1;
+        case L'2':
+            return 2;
+        case L'3':
+            return 3;
+        case L'4':
+            return 4;
+        case L'5':
+            return 5;
+        case L'6':
+            return 6;
+        case L'7':
+            return 7;
+        case L'8':
+            return 8;
+        case L'9':
+            return 9;
+        case L'a':
+        case L'A':
+            return 10;
+        case L'b':
+        case L'B':
+            return 11;
+        case L'c':
+        case L'C':
+            return 12;
+        case L'd':
+        case L'D':
+            return 13;
+        case L'e':
+        case L'E':
+            return 14;
+        case L'f':
+        case L'F':
+            return 15;
+        default:
+            return -1;
+    }
+}
+
+static int octal_to_bin(wchar_t c)
+{
+    switch (c)
+    {
+        case L'0':
+            return 0;
+        case L'1':
+            return 1;
+        case L'2':
+            return 2;
+        case L'3':
+            return 3;
+        case L'4':
+            return 4;
+        case L'5':
+            return 5;
+        case L'6':
+            return 6;
+        case L'7':
+            return 7;
+        default:
+            return -1;
+    }
+}
+
+/* This message appears in N_() here rather than just in _() below because
+   the sole use would have been in a #define.  */
+static wchar_t const *const cfcc_msg =
+    N_(L"warning: %ls: character(s) following character constant have been ignored");
+
+double C_STRTOD(wchar_t const *nptr, wchar_t **endptr)
+{
+    double r;
+
+    const wcstring saved_locale = wsetlocale(LC_NUMERIC, NULL);
+
+    if (!saved_locale.empty())
+    {
+        wsetlocale(LC_NUMERIC, L"C");
+    }
+
+    r = wcstod(nptr, endptr);
+
+    if (!saved_locale.empty())
+    {
+        wsetlocale(LC_NUMERIC, saved_locale.c_str());
+    }
+
+    return r;
+}
+
+void builtin_printf_state_t::fatal_error(const wchar_t *fmt, ...)
+{
+    // Don't error twice
+    if (early_exit)
+        return;
+
+    va_list va;
+    va_start(va, fmt);
+    wcstring errstr = vformat_string(fmt, va);
+    va_end(va);
+    stderr_buffer.append(errstr);
+    if (! string_suffixes_string(L"\n", errstr))
+        stderr_buffer.push_back(L'\n');
+
+    this->exit_code = STATUS_BUILTIN_ERROR;
+    this->early_exit = true;
+}
+
+void builtin_printf_state_t::append_output(wchar_t c)
+{
+    // Don't output if we're done
+    if (early_exit)
+        return;
+
+    stdout_buffer.push_back(c);
+}
+
+void builtin_printf_state_t::append_output(const wchar_t *c)
+{
+    // Don't output if we're done
+    if (early_exit)
+        return;
+
+    stdout_buffer.append(c);
+}
+
+void builtin_printf_state_t::append_format_output(const wchar_t *fmt, ...)
+{
+    // Don't output if we're done
+    if (early_exit)
+        return;
+
+    va_list va;
+    va_start(va, fmt);
+    append_formatv(stdout_buffer, fmt, va);
+    va_end(va);
+}
+
+
+void builtin_printf_state_t::verify_numeric(const wchar_t *s, const wchar_t *end, int errcode)
+{
+    if (errcode != 0)
+    {
+        this->fatal_error(L"%ls: %s", s, strerror(errcode));
+    }
+    else if (*end)
+    {
+        if (s == end)
+            this->fatal_error(_(L"%ls: expected a numeric value"), s);
+        else
+            this->fatal_error(_(L"%ls: value not completely converted"), s);
+    }
+}
+
+template<typename T>
+static T raw_string_to_scalar_type(const wchar_t *s, wchar_t ** end);
+
+// we use wcstoll instead of wcstoimax because FreeBSD 8 has busted wcstoumax and wcstoimax - see #626
+template<>
+intmax_t raw_string_to_scalar_type(const wchar_t *s, wchar_t ** end)
+{
+    return wcstoll(s, end, 0);
+}
+
+template<>
+uintmax_t raw_string_to_scalar_type(const wchar_t *s, wchar_t ** end)
+{
+    return wcstoull(s, end, 0);
+}
+
+template<>
+long double raw_string_to_scalar_type(const wchar_t *s, wchar_t ** end)
+{
+    return C_STRTOD(s, end);
+}
+
+template<typename T>
+static T string_to_scalar_type(const wchar_t *s, builtin_printf_state_t *state)
+{
+    T val;
+    if (*s == L'\"' || *s == L'\'')
+    {
+        wchar_t ch = *++s;
+        val = ch;
+    }
+    else
+    {
+        wchar_t *end = NULL;
+        errno = 0;
+        val = raw_string_to_scalar_type<T>(s, &end);
+        state->verify_numeric(s, end, errno);
+    }
+    return val;
+}
+
+/* Output a single-character \ escape.  */
+
+void builtin_printf_state_t::print_esc_char(wchar_t c)
+{
+    switch (c)
+    {
+        case L'a':			/* Alert. */
+            this->append_output(L'\a');
+            break;
+        case L'b':			/* Backspace. */
+            this->append_output(L'\b');
+            break;
+        case L'c':			/* Cancel the rest of the output. */
+            this->early_exit = true;
+            break;
+        case L'e':			/* Escape */
+            this->append_output(L'\x1B');
+            break;
+        case L'f':			/* Form feed. */
+            this->append_output(L'\f');
+            break;
+        case L'n':			/* New line. */
+            this->append_output(L'\n');
+            break;
+        case L'r':			/* Carriage return. */
+            this->append_output(L'\r');
+            break;
+        case L't':			/* Horizontal tab. */
+            this->append_output(L'\t');
+            break;
+        case L'v':			/* Vertical tab. */
+            this->append_output(L'\v');
+            break;
+        default:
+            this->append_output(c);
+            break;
+    }
+}
+
+/* Print a \ escape sequence starting at ESCSTART.
+   Return the number of characters in the escape sequence
+   besides the backslash.
+   If OCTAL_0 is nonzero, octal escapes are of the form \0ooo, where o
+   is an octal digit; otherwise they are of the form \ooo.  */
+long builtin_printf_state_t::print_esc(const wchar_t *escstart, bool octal_0)
+{
+    const wchar_t *p = escstart + 1;
+    int esc_value = 0;		/* Value of \nnn escape. */
+    int esc_length;		/* Length of \nnn escape. */
+
+    if (*p == L'x')
+    {
+        /* A hexadecimal \xhh escape sequence must have 1 or 2 hex. digits.  */
+        for (esc_length = 0, ++p; esc_length < 2 && is_hex_digit(*p); ++esc_length, ++p)
+            esc_value = esc_value * 16 + hex_to_bin(*p);
+        if (esc_length == 0)
+            this->fatal_error(_(L"missing hexadecimal number in escape"));
+        this->append_output(ENCODE_DIRECT_BASE + esc_value % 256);
+    }
+    else if (is_octal_digit(*p))
+    {
+        /* Parse \0ooo (if octal_0 && *p == L'0') or \ooo (otherwise).
+        Allow \ooo if octal_0 && *p != L'0'; this is an undocumented
+        extension to POSIX that is compatible with Bash 2.05b.  */
+        /* Wrap mod 256, which matches historic behavior */
+        for (esc_length = 0, p += octal_0 && *p == L'0'; esc_length < 3 && is_octal_digit(*p); ++esc_length, ++p)
+            esc_value = esc_value * 8 + octal_to_bin(*p);
+        this->append_output(ENCODE_DIRECT_BASE + esc_value % 256);
+    }
+    else if (*p && wcschr(L"\"\\abcefnrtv", *p))
+    {
+        print_esc_char(*p++);
+    }
+    else if (*p == L'u' || *p == L'U')
+    {
+        wchar_t esc_char = *p;
+        p++;
+        uint32_t uni_value = 0;
+        for (size_t esc_length = 0; esc_length < (esc_char == L'u' ? 4 : 8); esc_length++)
+        {
+            if (! is_hex_digit(*p))
+            {
+                /* Escape sequence must be done. Complain if we didn't get anything */
+                if (esc_length == 0)
+                {
+                    this->fatal_error(_(L"Missing hexadecimal number in Unicode escape"));
+                }
+                break;
+            }
+            uni_value = uni_value * 16 + hex_to_bin(*p);
+            p++;
+        }
+
+        /* PCA GNU printf respects the limitations described in ISO N717, about which universal characters "shall not" be specified. I believe this limitation is for the benefit of compilers; I see no reason to impose it in builtin_printf.
+
+          If __STDC_ISO_10646__ is defined, then it means wchar_t can and does hold Unicode code points, so just use that. If not defined, use the %lc printf conversion; this probably won't do anything good if your wide character set is not Unicode, but such platforms are exceedingly rare.
+        */
+        if (uni_value > 0x10FFFF)
+        {
+            this->fatal_error(_(L"Unicode character out of range: \\%c%0*x"), esc_char, (esc_char == L'u' ? 4 : 8), uni_value);
+        }
+        else
+        {
+#if defined(__STDC_ISO_10646__)
+            this->append_output(uni_value);
+#else
+            this->append_format_output(L"%lc", uni_value);
+#endif
+        }
+    }
+    else
+    {
+        this->append_output(L'\\');
+        if (*p)
+        {
+            this->append_output(*p);
+            p++;
+        }
+    }
+    return p - escstart - 1;
+}
+
+/* Print string STR, evaluating \ escapes. */
+
+void builtin_printf_state_t::print_esc_string(const wchar_t *str)
+{
+    for (; *str; str++)
+        if (*str == L'\\')
+            str += print_esc(str, true);
+        else
+            this->append_output(*str);
+}
+
+/* Evaluate a printf conversion specification.  START is the start of
+   the directive, LENGTH is its length, and CONVERSION specifies the
+   type of conversion.  LENGTH does not include any length modifier or
+   the conversion specifier itself.  FIELD_WIDTH and PRECISION are the
+   field width and precision for '*' values, if HAVE_FIELD_WIDTH and
+   HAVE_PRECISION are true, respectively.  ARGUMENT is the argument to
+   be formatted.  */
+
+void builtin_printf_state_t::print_direc(const wchar_t *start, size_t length, wchar_t conversion,
+        bool have_field_width, int field_width,
+        bool have_precision, int precision,
+        wchar_t const *argument)
+{
+    // Start with everything except the conversion specifier
+    wcstring fmt(start, length);
+
+    /*  Create a copy of the % directive, with an intmax_t-wide width modifier substituted for any existing integer length modifier. */
+    switch (conversion)
+    {
+        case L'd':
+        case L'i':
+        case L'u':
+            fmt.append(L"ll");
+            break;
+        case L'a':
+        case L'e':
+        case L'f':
+        case L'g':
+        case L'A':
+        case L'E':
+        case L'F':
+        case L'G':
+            fmt.append(L"L");
+            break;
+        case L's':
+        case L'c':
+            fmt.append(L"l");
+            break;
+        default:
+            break;
+    }
+
+    // Append the conversion itself
+    fmt.push_back(conversion);
+
+    switch (conversion)
+    {
+        case L'd':
+        case L'i':
+        {
+            intmax_t arg = string_to_scalar_type<intmax_t>(argument, this);
+            if (! have_field_width)
+            {
+                if (! have_precision)
+                    this->append_format_output(fmt.c_str(), arg);
+                else
+                    this->append_format_output(fmt.c_str(), precision, arg);
+            }
+            else
+            {
+                if (! have_precision)
+                    this->append_format_output(fmt.c_str(), field_width, arg);
+                else
+                    this->append_format_output(fmt.c_str(), field_width, precision, arg);
+            }
+        }
+        break;
+
+        case L'o':
+        case L'u':
+        case L'x':
+        case L'X':
+        {
+            uintmax_t arg = string_to_scalar_type<uintmax_t>(argument, this);
+            if (!have_field_width)
+            {
+                if (!have_precision)
+                    this->append_format_output(fmt.c_str(), arg);
+                else
+                    this->append_format_output(fmt.c_str(), precision, arg);
+            }
+            else
+            {
+                if (!have_precision)
+                    this->append_format_output(fmt.c_str(), field_width, arg);
+                else
+                    this->append_format_output(fmt.c_str(), field_width, precision, arg);
+            }
+        }
+        break;
+
+        case L'a':
+        case L'A':
+        case L'e':
+        case L'E':
+        case L'f':
+        case L'F':
+        case L'g':
+        case L'G':
+        {
+            long double arg = string_to_scalar_type<long double>(argument, this);
+            if (!have_field_width)
+            {
+                if (!have_precision)
+                    this->append_format_output(fmt.c_str(), arg);
+                else
+                    this->append_format_output(fmt.c_str(), precision, arg);
+            }
+            else
+            {
+                if (!have_precision)
+                    this->append_format_output(fmt.c_str(), field_width, arg);
+                else
+                    this->append_format_output(fmt.c_str(), field_width, precision, arg);
+            }
+        }
+        break;
+
+        case L'c':
+            if (!have_field_width)
+                this->append_format_output(fmt.c_str(), *argument);
+            else
+                this->append_format_output(fmt.c_str(), field_width, *argument);
+            break;
+        case L's':
+            if (!have_field_width)
+            {
+                if (!have_precision)
+                {
+                    this->append_format_output(fmt.c_str(), argument);
+                }
+                else
+                    this->append_format_output(fmt.c_str(), precision, argument);
+            }
+            else
+            {
+                if (!have_precision)
+                    this->append_format_output(fmt.c_str(), field_width, argument);
+                else
+                    this->append_format_output(fmt.c_str(), field_width, precision, argument);
+            }
+            break;
+    }
+}
+
+/* For each character in str, set the corresponding boolean in the array to the given flag */
+static inline void modify_allowed_format_specifiers(bool ok[UCHAR_MAX + 1], const char *str, bool flag)
+{
+    for (const char *c  = str; *c != '\0'; c++)
+    {
+        unsigned char idx = static_cast<unsigned char>(*c);
+        ok[idx] = flag;
+    }
+}
+
+/* Print the text in FORMAT, using ARGV (with ARGC elements) for
+   arguments to any `%' directives.
+   Return the number of elements of ARGV used.  */
+
+int builtin_printf_state_t::print_formatted(const wchar_t *format, int argc, wchar_t **argv)
+{
+    int save_argc = argc;		/* Preserve original value.  */
+    const wchar_t *f;		/* Pointer into `format'.  */
+    const wchar_t *direc_start;	/* Start of % directive.  */
+    size_t direc_length;		/* Length of % directive.  */
+    bool have_field_width;	/* True if FIELD_WIDTH is valid.  */
+    int field_width = 0;		/* Arg to first '*'.  */
+    bool have_precision;		/* True if PRECISION is valid.  */
+    int precision = 0;		/* Arg to second '*'.  */
+    bool ok[UCHAR_MAX + 1] = { };	/* ok['x'] is true if %x is allowed.  */
+
+    for (f = format; *f != L'\0'; ++f)
+    {
+        switch (*f)
+        {
+            case L'%':
+                direc_start = f++;
+                direc_length = 1;
+                have_field_width = have_precision = false;
+                if (*f == L'%')
+                {
+                    this->append_output(L'%');
+                    break;
+                }
+                if (*f == L'b')
+                {
+                    /* FIXME: Field width and precision are not supported
+                    for %b, even though POSIX requires it.  */
+                    if (argc > 0)
+                    {
+                        print_esc_string(*argv);
+                        ++argv;
+                        --argc;
+                    }
+                    break;
+                }
+
+                modify_allowed_format_specifiers(ok, "aAcdeEfFgGiosuxX", true);
+
+                for (;; f++, direc_length++)
+                {
+                    switch (*f)
+                    {
+                        case L'I':
+                        case L'\'':
+                            modify_allowed_format_specifiers(ok, "aAceEosxX", false);
+                            break;
+                        case '-':
+                        case '+':
+                        case ' ':
+                            break;
+                        case L'#':
+                            modify_allowed_format_specifiers(ok, "cdisu", false);
+                            break;
+                        case '0':
+                            modify_allowed_format_specifiers(ok, "cs", false);
+                            break;
+                        default:
+                            goto no_more_flag_characters;
+                    }
+                }
+no_more_flag_characters:
+                ;
+
+                if (*f == L'*')
+                {
+                    ++f;
+                    ++direc_length;
+                    if (argc > 0)
+                    {
+                        intmax_t width = string_to_scalar_type<intmax_t>(*argv, this);
+                        if (INT_MIN <= width && width <= INT_MAX)
+                            field_width = static_cast<int>(width);
+                        else
+                            this->fatal_error(_(L"invalid field width: %ls"), *argv);
+                        ++argv;
+                        --argc;
+                    }
+                    else
+                    {
+                        field_width = 0;
+                    }
+                    have_field_width = true;
+                }
+                else
+                {
+                    while (iswdigit(*f))
+                    {
+                        ++f;
+                        ++direc_length;
+                    }
+                }
+                if (*f == L'.')
+                {
+                    ++f;
+                    ++direc_length;
+                    modify_allowed_format_specifiers(ok, "c", false);
+                    if (*f == L'*')
+                    {
+                        ++f;
+                        ++direc_length;
+                        if (argc > 0)
+                        {
+                            intmax_t prec = string_to_scalar_type<intmax_t>(*argv, this);
+                            if (prec < 0)
+                            {
+                                /* A negative precision is taken as if the
+                                precision were omitted, so -1 is safe
+                                here even if prec < INT_MIN.  */
+                                precision = -1;
+                            }
+                            else if (INT_MAX < prec)
+                                this->fatal_error(_(L"invalid precision: %ls"), *argv);
+                            else
+                            {
+                                precision = static_cast<int>(prec);
+                            }
+                            ++argv;
+                            --argc;
+                        }
+                        else
+                        {
+                            precision = 0;
+                        }
+                        have_precision = true;
+                    }
+                    else
+                    {
+                        while (iswdigit(*f))
+                        {
+                            ++f;
+                            ++direc_length;
+                        }
+                    }
+                }
+
+                while (*f == L'l' || *f == L'L' || *f == L'h' || *f == L'j' || *f == L't' || *f == L'z')
+                    ++f;
+
+                {
+                    wchar_t conversion = *f;
+                    if (conversion > 0xFF || ! ok[conversion])
+                    {
+                        this->fatal_error(_(L"%.*ls: invalid conversion specification"), (int)(f + 1 - direc_start), direc_start);
+                        return 0;
+                    }
+                }
+
+                print_direc(direc_start, direc_length, *f,
+                            have_field_width, field_width,
+                            have_precision, precision,
+                            (argc <= 0 ? L"" : (argc--, *argv++)));
+                break;
+
+            case L'\\':
+                f += print_esc(f, false);
+                break;
+
+            default:
+                this->append_output(*f);
+        }
+    }
+    return save_argc - argc;
+}
+
+static int builtin_printf(parser_t &parser, wchar_t **argv)
+{
+    builtin_printf_state_t state;
+
+    wchar_t *format;
+    int args_used;
+    int argc = builtin_count_args(argv);
+
+    if (argc <= 1)
+    {
+        state.fatal_error(_(L"printf: not enough arguments"));
+        return STATUS_BUILTIN_ERROR;
+    }
+
+    format = argv[1];
+    argc -= 2;
+    argv += 2;
+
+    do
+    {
+        args_used = state.print_formatted(format, argc, argv);
+        argc -= args_used;
+        argv += args_used;
+    }
+    while (args_used > 0 && argc > 0 && ! state.early_exit);
+    return state.exit_code;
+}
diff --git a/src/builtin_set.cpp b/src/builtin_set.cpp
new file mode 100644
index 00000000..c7bde4fb
--- /dev/null
+++ b/src/builtin_set.cpp
@@ -0,0 +1,828 @@
+/** \file builtin_set.c Functions defining the set builtin
+
+Functions used for implementing the set builtin.
+
+*/
+#include "config.h"
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <wchar.h>
+#include <wctype.h>
+#include <sys/types.h>
+#include <termios.h>
+#include <signal.h>
+#include <vector>
+#include <algorithm>
+#include "fallback.h"
+#include "util.h"
+
+#include "wutil.h"
+#include "builtin.h"
+#include "env.h"
+#include "expand.h"
+#include "common.h"
+#include "wgetopt.h"
+#include "proc.h"
+#include "parser.h"
+
+/* We know about these buffers */
+extern wcstring stdout_buffer, stderr_buffer;
+
+/**
+   Error message for invalid path operations
+*/
+#define BUILTIN_SET_PATH_ERROR L"%ls: Warning: path component %ls may not be valid in %ls.\n"
+
+/**
+   Hint for invalid path operation with a colon
+*/
+#define BUILTIN_SET_PATH_HINT L"%ls: Did you mean 'set %ls $%ls %ls'?\n"
+
+/**
+   Error for mismatch between index count and elements
+*/
+#define BUILTIN_SET_ARG_COUNT L"%ls: The number of variable indexes does not match the number of values\n"
+
+/**
+   Test if the specified variable should be subject to path validation
+*/
+static int is_path_variable(const wchar_t *env)
+{
+    return contains(env, L"PATH", L"CDPATH");
+}
+
+/**
+   Call env_set. If this is a path variable, e.g. PATH, validate the
+   elements. On error, print a description of the problem to stderr.
+*/
+static int my_env_set(const wchar_t *key, const wcstring_list_t &val, int scope)
+{
+    size_t i;
+    int retcode = 0;
+    const wchar_t *val_str=NULL;
+
+    if (is_path_variable(key))
+    {
+        /* Fix for https://github.com/fish-shell/fish-shell/issues/199 . Return success if any path setting succeeds. */
+        bool any_success = false;
+
+        /* Don't bother validating (or complaining about) values that are already present */
+        wcstring_list_t existing_values;
+        const env_var_t existing_variable = env_get_string(key, scope);
+        if (! existing_variable.missing_or_empty())
+            tokenize_variable_array(existing_variable, existing_values);
+
+        for (i=0; i< val.size() ; i++)
+        {
+            const wcstring &dir = val.at(i);
+            if (list_contains_string(existing_values, dir))
+            {
+                any_success = true;
+                continue;
+            }
+
+            bool show_perror = false;
+            int show_hint = 0;
+            bool error = false;
+
+            struct stat buff;
+            if (wstat(dir, &buff))
+            {
+                error = true;
+                show_perror = true;
+            }
+
+            if (!(S_ISDIR(buff.st_mode)))
+            {
+                error = true;
+            }
+
+            if (!error)
+            {
+                any_success = true;
+            }
+            else
+            {
+                append_format(stderr_buffer, _(BUILTIN_SET_PATH_ERROR), L"set", dir.c_str(), key);
+                const wchar_t *colon = wcschr(dir.c_str(), L':');
+
+                if (colon && *(colon+1))
+                {
+                    show_hint = 1;
+                }
+
+            }
+
+            if (show_perror)
+            {
+                builtin_wperror(L"set");
+            }
+
+            if (show_hint)
+            {
+                append_format(stderr_buffer, _(BUILTIN_SET_PATH_HINT), L"set", key, key, wcschr(dir.c_str(), L':')+1);
+            }
+
+        }
+
+        /* Fail at setting the path if we tried to set it to something non-empty, but it wound up empty. */
+        if (! val.empty() && ! any_success)
+        {
+            return 1;
+        }
+
+    }
+
+    wcstring sb;
+    if (! val.empty())
+    {
+        for (i=0; i< val.size() ; i++)
+        {
+            sb.append(val[i]);
+            if (i<val.size() - 1)
+            {
+                sb.append(ARRAY_SEP_STR);
+            }
+        }
+        val_str = sb.c_str();
+    }
+
+    switch (env_set(key, val_str, scope | ENV_USER))
+    {
+        case ENV_PERM:
+        {
+            append_format(stderr_buffer, _(L"%ls: Tried to change the read-only variable '%ls'\n"), L"set", key);
+            retcode=1;
+            break;
+        }
+
+        case ENV_SCOPE:
+        {
+            append_format(stderr_buffer, _(L"%ls: Tried to set the special variable '%ls' with the wrong scope\n"), L"set", key);
+            retcode=1;
+            break;
+        }
+
+        case ENV_INVALID:
+        {
+            append_format(stderr_buffer, _(L"%ls: Tried to set the special variable '%ls' to an invalid value\n"), L"set", key);
+            retcode=1;
+            break;
+        }
+    }
+
+    return retcode;
+}
+
+
+
+/**
+  Extract indexes from a destination argument of the form name[index1 index2...]
+
+  \param indexes the list to insert the new indexes into
+  \param src the source string to parse
+  \param name the name of the element. Return null if the name in \c src does not match this name
+  \param var_count the number of elements in the array to parse.
+
+  \return the total number of indexes parsed, or -1 on error
+*/
+static int parse_index(std::vector<long> &indexes,
+                       const wchar_t *src,
+                       const wchar_t *name,
+                       size_t var_count)
+{
+    size_t len;
+
+    int count = 0;
+    const wchar_t *src_orig = src;
+
+    if (src == 0)
+    {
+        return 0;
+    }
+
+    while (*src != L'\0' && (iswalnum(*src) || *src == L'_'))
+    {
+        src++;
+    }
+
+    if (*src != L'[')
+    {
+        append_format(stderr_buffer, _(BUILTIN_SET_ARG_COUNT), L"set");
+        return 0;
+    }
+
+    len = src-src_orig;
+
+    if ((wcsncmp(src_orig, name, len)!=0) || (wcslen(name) != (len)))
+    {
+        append_format(stderr_buffer,
+                      _(L"%ls: Multiple variable names specified in single call (%ls and %.*ls)\n"),
+                      L"set",
+                      name,
+                      len,
+                      src_orig);
+        return 0;
+    }
+
+    src++;
+
+    while (iswspace(*src))
+    {
+        src++;
+    }
+
+    while (*src != L']')
+    {
+        wchar_t *end;
+
+        long l_ind;
+
+        errno = 0;
+
+        l_ind = wcstol(src, &end, 10);
+
+        if (end==src || errno)
+        {
+            append_format(stderr_buffer, _(L"%ls: Invalid index starting at '%ls'\n"), L"set", src);
+            return 0;
+        }
+
+        if (l_ind < 0)
+        {
+            l_ind = var_count+l_ind+1;
+        }
+
+        src = end;
+        if (*src==L'.' && *(src+1)==L'.')
+        {
+            src+=2;
+            long l_ind2 = wcstol(src, &end, 10);
+            if (end==src || errno)
+            {
+                return 1;
+            }
+            src = end;
+
+            if (l_ind2 < 0)
+            {
+                l_ind2 = var_count+l_ind2+1;
+            }
+            int direction = l_ind2<l_ind ? -1 : 1 ;
+            for (long jjj = l_ind; jjj*direction <= l_ind2*direction; jjj+=direction)
+            {
+                // debug(0, L"Expand range [set]: %i\n", jjj);
+                indexes.push_back(jjj);
+                count++;
+            }
+        }
+        else
+        {
+            indexes.push_back(l_ind);
+            count++;
+        }
+        while (iswspace(*src)) src++;
+    }
+
+    return count;
+}
+
+static int update_values(wcstring_list_t &list,
+                         std::vector<long> &indexes,
+                         wcstring_list_t &values)
+{
+    size_t i;
+
+    /* Replace values where needed */
+    for (i = 0; i < indexes.size(); i++)
+    {
+        /*
+          The '- 1' below is because the indices in fish are
+          one-based, but the vector uses zero-based indices
+        */
+        long ind = indexes[i] - 1;
+        const wcstring newv = values[ i ];
+        if (ind < 0)
+        {
+            return 1;
+        }
+        if ((size_t)ind >= list.size())
+        {
+            list.resize(ind+1);
+        }
+
+//    free((void *) al_get(list, ind));
+        list[ ind ] = newv;
+    }
+
+    return 0;
+}
+
+/**
+   Erase from a list of wcstring values at specified indexes
+*/
+static void erase_values(wcstring_list_t &list, const std::vector<long> &indexes)
+{
+    // Make a set of indexes.
+    // This both sorts them into ascending order and removes duplicates.
+    const std::set<long> indexes_set(indexes.begin(), indexes.end());
+
+    // Now walk the set backwards, so we encounter larger indexes first, and remove elements at the given (1-based) indexes.
+    std::set<long>::const_reverse_iterator iter;
+    for (iter = indexes_set.rbegin(); iter != indexes_set.rend(); ++iter)
+    {
+        long val = *iter;
+        if (val > 0 && (size_t)val <= list.size())
+        {
+            // One-based indexing!
+            list.erase(list.begin() + val - 1);
+        }
+    }
+}
+
+
+/**
+   Print the names of all environment variables in the scope, with or without shortening,
+   with or without values, with or without escaping
+*/
+static void print_variables(int include_values, int esc, bool shorten_ok, int scope)
+{
+    wcstring_list_t names = env_get_names(scope);
+    sort(names.begin(), names.end());
+
+    for (size_t i = 0; i < names.size(); i++)
+    {
+        const wcstring key = names.at(i);
+        const wcstring e_key = escape_string(key, 0);
+
+        stdout_buffer.append(e_key);
+
+        if (include_values)
+        {
+            env_var_t value = env_get_string(key, scope);
+            if (!value.missing())
+            {
+                int shorten = 0;
+
+                if (shorten_ok && value.length() > 64)
+                {
+                    shorten = 1;
+                    value.resize(60);
+                }
+
+                wcstring e_value = esc ? expand_escape_variable(value) : value;
+
+                stdout_buffer.append(L" ");
+                stdout_buffer.append(e_value);
+
+                if (shorten)
+                {
+                    stdout_buffer.push_back(ellipsis_char);
+                }
+
+            }
+        }
+
+        stdout_buffer.append(L"\n");
+    }
+}
+
+
+
+/**
+   The set builtin. Creates, updates and erases environment variables
+   and environemnt variable arrays.
+*/
+static int builtin_set(parser_t &parser, wchar_t **argv)
+{
+    wgetopter_t w;
+    /** Variables used for parsing the argument list */
+    const struct woption long_options[] =
+    {
+        { L"export", no_argument, 0, 'x' },
+        { L"global", no_argument, 0, 'g' },
+        { L"local", no_argument, 0, 'l' },
+        { L"erase", no_argument, 0, 'e' },
+        { L"names", no_argument, 0, 'n' },
+        { L"unexport", no_argument, 0, 'u' },
+        { L"universal", no_argument, 0, 'U' },
+        { L"long", no_argument, 0, 'L' },
+        { L"query", no_argument, 0, 'q' },
+        { L"help", no_argument, 0, 'h' },
+        { 0, 0, 0, 0 }
+    } ;
+
+    const wchar_t *short_options = L"+xglenuULqh";
+
+    int argc = builtin_count_args(argv);
+
+    /*
+      Flags to set the work mode
+    */
+    int local = 0, global = 0, exportv = 0;
+    int erase = 0, list = 0, unexport=0;
+    int universal = 0, query=0;
+    bool shorten_ok = true;
+    bool preserve_incoming_failure_exit_status = true;
+    const int incoming_exit_status = proc_get_last_status();
+
+    /*
+      Variables used for performing the actual work
+    */
+    wchar_t *dest = 0;
+    int retcode=0;
+    int scope;
+    int slice=0;
+    int i;
+
+    const wchar_t *bad_char = NULL;
+
+
+    /* Parse options to obtain the requested operation and the modifiers */
+    w.woptind = 0;
+    while (1)
+    {
+        int c = w.wgetopt_long(argc, argv, short_options, long_options, 0);
+
+        if (c == -1)
+        {
+            break;
+        }
+
+        switch (c)
+        {
+            case 0:
+                break;
+
+            case 'e':
+                erase = 1;
+                preserve_incoming_failure_exit_status = false;
+                break;
+
+            case 'n':
+                list = 1;
+                preserve_incoming_failure_exit_status = false;
+                break;
+
+            case 'x':
+                exportv = 1;
+                break;
+
+            case 'l':
+                local = 1;
+                break;
+
+            case 'g':
+                global = 1;
+                break;
+
+            case 'u':
+                unexport = 1;
+                break;
+
+            case 'U':
+                universal = 1;
+                break;
+
+            case 'L':
+                shorten_ok = false;
+                break;
+
+            case 'q':
+                query = 1;
+                preserve_incoming_failure_exit_status = false;
+                break;
+
+            case 'h':
+                builtin_print_help(parser, argv[0], stdout_buffer);
+                return 0;
+
+            case '?':
+                builtin_unknown_option(parser, argv[0], argv[w.woptind-1]);
+                return 1;
+
+            default:
+                break;
+        }
+    }
+
+    /*
+      Ok, all arguments have been parsed, let's validate them
+    */
+
+    /*
+      If we are checking the existance of a variable (-q) we can not
+      also specify scope
+    */
+
+    if (query && (erase || list))
+    {
+        append_format(stderr_buffer,
+                      BUILTIN_ERR_COMBO,
+                      argv[0]);
+
+        builtin_print_help(parser, argv[0], stderr_buffer);
+        return 1;
+    }
+
+
+    /* We can't both list and erase variables */
+    if (erase && list)
+    {
+        append_format(stderr_buffer,
+                      BUILTIN_ERR_COMBO,
+                      argv[0]);
+
+        builtin_print_help(parser, argv[0], stderr_buffer);
+        return 1;
+    }
+
+    /*
+      Variables can only have one scope
+    */
+    if (local + global + universal > 1)
+    {
+        append_format(stderr_buffer,
+                      BUILTIN_ERR_GLOCAL,
+                      argv[0]);
+        builtin_print_help(parser, argv[0], stderr_buffer);
+        return 1;
+    }
+
+    /*
+      Variables can only have one export status
+    */
+    if (exportv && unexport)
+    {
+        append_format(stderr_buffer,
+                      BUILTIN_ERR_EXPUNEXP,
+                      argv[0]);
+        builtin_print_help(parser, argv[0], stderr_buffer);
+        return 1;
+    }
+
+    /*
+      Calculate the scope value for variable assignement
+    */
+    scope = (local ? ENV_LOCAL : 0) | (global ? ENV_GLOBAL : 0) | (exportv ? ENV_EXPORT : 0) | (unexport ? ENV_UNEXPORT : 0) | (universal ? ENV_UNIVERSAL:0) | ENV_USER;
+
+    if (query)
+    {
+        /*
+          Query mode. Return the number of variables that do not exist
+          out of the specified variables.
+        */
+        int i;
+        for (i=w.woptind; i<argc; i++)
+        {
+            wchar_t *arg = argv[i];
+            int slice=0;
+
+            if (!(dest = wcsdup(arg)))
+            {
+                DIE_MEM();
+            }
+
+            if (wcschr(dest, L'['))
+            {
+                slice = 1;
+                *wcschr(dest, L'[')=0;
+            }
+
+            if (slice)
+            {
+                std::vector<long> indexes;
+                wcstring_list_t result;
+                size_t j;
+
+                env_var_t dest_str = env_get_string(dest, scope);
+                if (! dest_str.missing())
+                    tokenize_variable_array(dest_str, result);
+
+                if (!parse_index(indexes, arg, dest, result.size()))
+                {
+                    builtin_print_help(parser, argv[0], stderr_buffer);
+                    retcode = 1;
+                    break;
+                }
+                for (j=0; j < indexes.size() ; j++)
+                {
+                    long idx = indexes[j];
+                    if (idx < 1 || (size_t)idx > result.size())
+                    {
+                        retcode++;
+                    }
+                }
+            }
+            else
+            {
+                if (!env_exist(arg, scope))
+                {
+                    retcode++;
+                }
+            }
+
+            free(dest);
+
+        }
+        return retcode;
+    }
+
+    if (list)
+    {
+        /* Maybe we should issue an error if there are any other arguments? */
+        print_variables(0, 0, shorten_ok, scope);
+        return 0;
+    }
+
+    if (w.woptind == argc)
+    {
+        /*
+          Print values of variables
+        */
+
+        if (erase)
+        {
+            append_format(stderr_buffer,
+                          _(L"%ls: Erase needs a variable name\n"),
+                          argv[0]);
+
+            builtin_print_help(parser, argv[0], stderr_buffer);
+            retcode = 1;
+        }
+        else
+        {
+            print_variables(1, 1, shorten_ok, scope);
+        }
+
+        return retcode;
+    }
+
+    if (!(dest = wcsdup(argv[w.woptind])))
+    {
+        DIE_MEM();
+    }
+
+    if (wcschr(dest, L'['))
+    {
+        slice = 1;
+        *wcschr(dest, L'[')=0;
+    }
+
+    if (!wcslen(dest))
+    {
+        free(dest);
+        append_format(stderr_buffer, BUILTIN_ERR_VARNAME_ZERO, argv[0]);
+        builtin_print_help(parser, argv[0], stderr_buffer);
+        return 1;
+    }
+
+    if ((bad_char = wcsvarname(dest)))
+    {
+        append_format(stderr_buffer, BUILTIN_ERR_VARCHAR, argv[0], *bad_char);
+        builtin_print_help(parser, argv[0], stderr_buffer);
+        free(dest);
+        return 1;
+    }
+
+    /*
+      set assignment can work in two modes, either using slices or
+      using the whole array. We detect which mode is used here.
+    */
+
+    if (slice)
+    {
+
+        /*
+          Slice mode
+        */
+        std::vector<long> indexes;
+        wcstring_list_t result;
+
+        const env_var_t dest_str = env_get_string(dest, scope);
+        if (! dest_str.missing())
+        {
+            tokenize_variable_array(dest_str, result);
+        }
+        else if (erase)
+        {
+            retcode = 1;
+        }
+
+        if (!retcode)
+        {
+            for (; w.woptind<argc; w.woptind++)
+            {
+                if (!parse_index(indexes, argv[w.woptind], dest, result.size()))
+                {
+                    builtin_print_help(parser, argv[0], stderr_buffer);
+                    retcode = 1;
+                    break;
+                }
+
+                size_t idx_count = indexes.size();
+                size_t val_count = argc-w.woptind-1;
+
+                if (!erase)
+                {
+                    if (val_count < idx_count)
+                    {
+                        append_format(stderr_buffer, _(BUILTIN_SET_ARG_COUNT), argv[0]);
+                        builtin_print_help(parser, argv[0], stderr_buffer);
+                        retcode=1;
+                        break;
+                    }
+                    if (val_count == idx_count)
+                    {
+                        w.woptind++;
+                        break;
+                    }
+                }
+            }
+        }
+
+        if (!retcode)
+        {
+            /*
+              Slice indexes have been calculated, do the actual work
+            */
+
+            if (erase)
+            {
+                erase_values(result, indexes);
+                my_env_set(dest, result, scope);
+            }
+            else
+            {
+                wcstring_list_t value;
+
+                while (w.woptind < argc)
+                {
+                    value.push_back(argv[w.woptind++]);
+                }
+
+                if (update_values(result,
+                                  indexes,
+                                  value))
+                {
+                    append_format(stderr_buffer, L"%ls: ", argv[0]);
+                    append_format(stderr_buffer, ARRAY_BOUNDS_ERR);
+                    stderr_buffer.push_back(L'\n');
+                }
+
+                my_env_set(dest, result, scope);
+
+
+            }
+        }
+    }
+    else
+    {
+        w.woptind++;
+
+        /*
+          No slicing
+        */
+        if (erase)
+        {
+            if (w.woptind != argc)
+            {
+                append_format(stderr_buffer,
+                              _(L"%ls: Values cannot be specfied with erase\n"),
+                              argv[0]);
+                builtin_print_help(parser, argv[0], stderr_buffer);
+                retcode=1;
+            }
+            else
+            {
+                retcode = env_remove(dest, scope);
+            }
+        }
+        else
+        {
+            wcstring_list_t val;
+            for (i=w.woptind; i<argc; i++)
+                val.push_back(argv[i]);
+            retcode = my_env_set(dest, val, scope);
+        }
+    }
+
+    /* Check if we are setting variables above the effective scope.
+       See https://github.com/fish-shell/fish-shell/issues/806
+     */
+
+    env_var_t global_dest = env_get_string(dest, ENV_GLOBAL);
+    if (universal && ! global_dest.missing())
+    {
+        append_format(stderr_buffer, _(L"%ls: Warning: universal scope selected, but a global variable '%ls' exists.\n"), L"set", dest);
+    }
+
+    free(dest);
+
+    if (retcode == STATUS_BUILTIN_OK && preserve_incoming_failure_exit_status)
+        retcode = incoming_exit_status;
+    return retcode;
+
+}
+
diff --git a/src/builtin_set_color.cpp b/src/builtin_set_color.cpp
new file mode 100644
index 00000000..7f400543
--- /dev/null
+++ b/src/builtin_set_color.cpp
@@ -0,0 +1,244 @@
+/** \file builtin_set_color.cpp Functions defining the set_color builtin
+
+Functions used for implementing the set_color builtin.
+
+*/
+#include "config.h"
+
+#include "builtin.h"
+#include "color.h"
+#include "output.h"
+
+#if HAVE_NCURSES_H
+#include <ncurses.h>
+#elif HAVE_NCURSES_CURSES_H
+#include <ncurses/curses.h>
+#else
+#include <curses.h>
+#endif
+
+#if HAVE_TERM_H
+#include <term.h>
+#elif HAVE_NCURSES_TERM_H
+#include <ncurses/term.h>
+#endif
+
+
+/* We know about these buffers */
+extern wcstring stdout_buffer, stderr_buffer;
+
+/**
+   Error message for invalid path operations
+*/
+#define BUILTIN_SET_PATH_ERROR L"%ls: Warning: path component %ls may not be valid in %ls.\n"
+
+/**
+   Hint for invalid path operation with a colon
+*/
+#define BUILTIN_SET_PATH_HINT L"%ls: Did you mean 'set %ls $%ls %ls'?\n"
+
+/**
+   Error for mismatch between index count and elements
+*/
+#define BUILTIN_SET_ARG_COUNT L"%ls: The number of variable indexes does not match the number of values\n"
+
+static void print_colors(void)
+{
+    const wcstring_list_t result = rgb_color_t::named_color_names();
+    size_t i;
+    for (i=0; i < result.size(); i++)
+    {
+        stdout_buffer.append(result.at(i));
+        stdout_buffer.push_back(L'\n');
+    }
+}
+
+/* function we set as the output writer */
+static std::string builtin_set_color_output;
+static int set_color_builtin_outputter(char c)
+{
+    ASSERT_IS_MAIN_THREAD();
+    builtin_set_color_output.push_back(c);
+    return 0;
+}
+
+/**
+   set_color builtin
+*/
+static int builtin_set_color(parser_t &parser, wchar_t **argv)
+{
+    wgetopter_t w;
+    /** Variables used for parsing the argument list */
+    const struct woption long_options[] =
+    {
+        { L"background", required_argument, 0, 'b'},
+        { L"help", no_argument, 0, 'h' },
+        { L"bold", no_argument, 0, 'o' },
+        { L"underline", no_argument, 0, 'u' },
+        { L"version", no_argument, 0, 'v' },
+        { L"print-colors", no_argument, 0, 'c' },
+        { 0, 0, 0, 0 }
+    };
+
+    const wchar_t *short_options = L"b:hvocu";
+
+    int argc = builtin_count_args(argv);
+
+    /* Some code passes variables to set_color that don't exist, like $fish_user_whatever. As a hack, quietly return failure. */
+    if (argc <= 1)
+    {
+        return EXIT_FAILURE;
+    }
+
+    const wchar_t *bgcolor = NULL;
+    bool bold = false, underline=false;
+    int errret;
+
+    /* Parse options to obtain the requested operation and the modifiers */
+    w.woptind = 0;
+    while (1)
+    {
+        int c = w.wgetopt_long(argc, argv, short_options, long_options, 0);
+
+        if (c == -1)
+        {
+            break;
+        }
+
+        switch (c)
+        {
+            case 0:
+                break;
+
+            case 'b':
+                bgcolor = w.woptarg;
+                break;
+
+            case 'h':
+                builtin_print_help(parser, argv[0], stdout_buffer);
+                return STATUS_BUILTIN_OK;
+
+            case 'o':
+                bold = true;
+                break;
+
+            case 'u':
+                underline = true;
+                break;
+
+            case 'c':
+                print_colors();
+                return STATUS_BUILTIN_OK;
+
+            case '?':
+                return STATUS_BUILTIN_ERROR;
+        }
+    }
+
+    /* Remaining arguments are foreground color */
+    std::vector<rgb_color_t> fgcolors;
+    for (; w.woptind < argc; w.woptind++)
+    {
+        rgb_color_t fg = rgb_color_t(argv[w.woptind]);
+        if (fg.is_none() || fg.is_ignore())
+        {
+            append_format(stderr_buffer, _(L"%ls: Unknown color '%ls'\n"), argv[0], argv[w.woptind]);
+            return STATUS_BUILTIN_ERROR;
+        }
+        fgcolors.push_back(fg);
+    }
+
+    if (fgcolors.empty() && bgcolor == NULL && !bold && !underline)
+    {
+        append_format(stderr_buffer,
+                      _(L"%ls: Expected an argument\n"),
+                      argv[0]);
+        return STATUS_BUILTIN_ERROR;
+    }
+    
+    // #1323: We may have multiple foreground colors. Choose the best one.
+    // If we had no foreground coor, we'll get none(); if we have at least one we expect not-none
+    const rgb_color_t fg = best_color(fgcolors, output_get_color_support());
+    assert(fgcolors.empty() || !(fg.is_none() || fg.is_ignore()));
+
+    const rgb_color_t bg = rgb_color_t(bgcolor ? bgcolor : L"");
+    if (bgcolor && (bg.is_none() || bg.is_ignore()))
+    {
+        append_format(stderr_buffer, _(L"%ls: Unknown color '%ls'\n"), argv[0], bgcolor);
+        return STATUS_BUILTIN_ERROR;
+    }
+
+    /* Make sure that the term exists */
+    if (cur_term == NULL && setupterm(0, STDOUT_FILENO, &errret) == ERR)
+    {
+        append_format(stderr_buffer, _(L"%ls: Could not set up terminal\n"), argv[0]);
+        return STATUS_BUILTIN_ERROR;
+    }
+
+    /*
+       Test if we have at least basic support for setting fonts, colors
+       and related bits - otherwise just give up...
+    */
+    if (! exit_attribute_mode)
+    {
+        return STATUS_BUILTIN_ERROR;
+    }
+
+    /* Save old output function so we can restore it */
+    int (* const saved_writer_func)(char) = output_get_writer();
+
+    /* Set our output function, which writes to a std::string */
+    builtin_set_color_output.clear();
+    output_set_writer(set_color_builtin_outputter);
+
+    if (bold)
+    {
+        if (enter_bold_mode)
+            writembs(tparm(enter_bold_mode));
+    }
+
+    if (underline)
+    {
+        if (enter_underline_mode)
+            writembs(enter_underline_mode);
+    }
+
+    if (bgcolor != NULL)
+    {
+        if (bg.is_normal())
+        {
+            write_color(rgb_color_t::black(), false /* not is_fg */);
+            writembs(tparm(exit_attribute_mode));
+        }
+    }
+
+    if (! fg.is_none())
+    {
+        if (fg.is_normal() || fg.is_reset())
+        {
+            write_color(rgb_color_t::black(), true /* is_fg */);
+            writembs(tparm(exit_attribute_mode));
+        }
+        else
+        {
+            write_color(fg, true /* is_fg */);
+        }
+    }
+
+    if (bgcolor != NULL)
+    {
+        if (! bg.is_normal() && ! bg.is_reset())
+        {
+            write_color(bg, false /* not is_fg */);
+        }
+    }
+
+    /* Restore saved writer function */
+    output_set_writer(saved_writer_func);
+
+    /* Output the collected string */
+    stdout_buffer.append(str2wcstring(builtin_set_color_output));
+    builtin_set_color_output.clear();
+
+    return STATUS_BUILTIN_OK;
+}
diff --git a/src/builtin_test.cpp b/src/builtin_test.cpp
new file mode 100644
index 00000000..84d5fa9c
--- /dev/null
+++ b/src/builtin_test.cpp
@@ -0,0 +1,972 @@
+/** \file builtin_test.cpp Functions defining the test builtin
+
+Functions used for implementing the test builtin.
+Implemented from scratch (yes, really) by way of IEEE 1003.1 as reference.
+
+*/
+
+#include "config.h"
+#include "common.h"
+#include "builtin.h"
+#include "wutil.h"
+#include "proc.h"
+#include <unistd.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <memory>
+
+
+enum
+{
+    BUILTIN_TEST_SUCCESS = STATUS_BUILTIN_OK,
+    BUILTIN_TEST_FAIL = STATUS_BUILTIN_ERROR
+};
+
+
+int builtin_test(parser_t &parser, wchar_t **argv);
+
+namespace test_expressions
+{
+
+enum token_t
+{
+    test_unknown,               // arbitrary string
+
+    test_bang,                  // "!", inverts sense
+
+    test_filetype_b,            // "-b", for block special files
+    test_filetype_c,            // "-c", for character special files
+    test_filetype_d,            // "-d", for directories
+    test_filetype_e,            // "-e", for files that exist
+    test_filetype_f,            // "-f", for for regular files
+    test_filetype_G,            // "-G", for check effective group id
+    test_filetype_g,            // "-g", for set-group-id
+    test_filetype_h,            // "-h", for symbolic links
+    test_filetype_L,            // "-L", same as -h
+    test_filetype_O,            // "-O", for check effective user id
+    test_filetype_p,            // "-p", for FIFO
+    test_filetype_S,            // "-S", socket
+
+    test_filesize_s,            // "-s", size greater than zero
+
+    test_filedesc_t,            // "-t", whether the fd is associated with a terminal
+
+    test_fileperm_r,            // "-r", read permission
+    test_fileperm_u,            // "-u", whether file is setuid
+    test_fileperm_w,            // "-w", whether file write permission is allowed
+    test_fileperm_x,            // "-x", whether file execute/search is allowed
+
+    test_string_n,              // "-n", non-empty string
+    test_string_z,              // "-z", true if length of string is 0
+    test_string_equal,          // "=", true if strings are identical
+    test_string_not_equal,      // "!=", true if strings are not identical
+
+    test_number_equal,          // "-eq", true if numbers are equal
+    test_number_not_equal,      // "-ne", true if numbers are not equal
+    test_number_greater,        // "-gt", true if first number is larger than second
+    test_number_greater_equal,  // "-ge", true if first number is at least second
+    test_number_lesser,         // "-lt", true if first number is smaller than second
+    test_number_lesser_equal,   // "-le", true if first number is at most second
+
+    test_combine_and,            // "-a", true if left and right are both true
+    test_combine_or,             // "-o", true if either left or right is true
+
+    test_paren_open,             // "(", open paren
+    test_paren_close,             // ")", close paren
+};
+
+static bool binary_primary_evaluate(test_expressions::token_t token, const wcstring &left, const wcstring &right, wcstring_list_t &errors);
+static bool unary_primary_evaluate(test_expressions::token_t token, const wcstring &arg, wcstring_list_t &errors);
+
+
+enum
+{
+    UNARY_PRIMARY = 1 << 0,
+    BINARY_PRIMARY = 1 << 1
+};
+
+static const struct token_info_t
+{
+    token_t tok;
+    const wchar_t *string;
+    unsigned int flags;
+} token_infos[] =
+{
+    {test_unknown, L"", 0},
+    {test_bang, L"!", 0},
+    {test_filetype_b, L"-b", UNARY_PRIMARY},
+    {test_filetype_c, L"-c", UNARY_PRIMARY},
+    {test_filetype_d, L"-d", UNARY_PRIMARY},
+    {test_filetype_e, L"-e", UNARY_PRIMARY},
+    {test_filetype_f, L"-f", UNARY_PRIMARY},
+    {test_filetype_G, L"-G", UNARY_PRIMARY},
+    {test_filetype_g, L"-g", UNARY_PRIMARY},
+    {test_filetype_h, L"-h", UNARY_PRIMARY},
+    {test_filetype_L, L"-L", UNARY_PRIMARY},
+    {test_filetype_O, L"-O", UNARY_PRIMARY},
+    {test_filetype_p, L"-p", UNARY_PRIMARY},
+    {test_filetype_S, L"-S", UNARY_PRIMARY},
+    {test_filesize_s, L"-s", UNARY_PRIMARY},
+    {test_filedesc_t, L"-t", UNARY_PRIMARY},
+    {test_fileperm_r, L"-r", UNARY_PRIMARY},
+    {test_fileperm_u, L"-u", UNARY_PRIMARY},
+    {test_fileperm_w, L"-w", UNARY_PRIMARY},
+    {test_fileperm_x, L"-x", UNARY_PRIMARY},
+    {test_string_n, L"-n", UNARY_PRIMARY},
+    {test_string_z, L"-z", UNARY_PRIMARY},
+    {test_string_equal, L"=", BINARY_PRIMARY},
+    {test_string_not_equal, L"!=", BINARY_PRIMARY},
+    {test_number_equal, L"-eq", BINARY_PRIMARY},
+    {test_number_not_equal, L"-ne", BINARY_PRIMARY},
+    {test_number_greater, L"-gt", BINARY_PRIMARY},
+    {test_number_greater_equal, L"-ge", BINARY_PRIMARY},
+    {test_number_lesser, L"-lt", BINARY_PRIMARY},
+    {test_number_lesser_equal, L"-le", BINARY_PRIMARY},
+    {test_combine_and, L"-a", 0},
+    {test_combine_or, L"-o", 0},
+    {test_paren_open, L"(", 0},
+    {test_paren_close, L")", 0}
+};
+
+const token_info_t *token_for_string(const wcstring &str)
+{
+    for (size_t i=0; i < sizeof token_infos / sizeof *token_infos; i++)
+    {
+        if (str == token_infos[i].string)
+        {
+            return &token_infos[i];
+        }
+    }
+    return &token_infos[0]; //unknown
+}
+
+
+/* Grammar.
+
+    <expr> = <combining_expr>
+
+    <combining_expr> = <unary_expr> and/or <combining_expr> |
+                       <unary_expr>
+
+    <unary_expr> = bang <unary_expr> |
+                  <primary>
+
+    <primary> = <unary_primary> arg |
+                arg <binary_primary> arg |
+                '(' <expr> ')'
+
+*/
+
+class expression;
+class test_parser
+{
+private:
+    wcstring_list_t strings;
+    wcstring_list_t errors;
+
+    expression *error(const wchar_t *fmt, ...);
+    void add_error(const wchar_t *fmt, ...);
+
+    const wcstring &arg(unsigned int idx)
+    {
+        return strings.at(idx);
+    }
+
+public:
+    test_parser(const wcstring_list_t &val) : strings(val)
+    { }
+
+    expression *parse_expression(unsigned int start, unsigned int end);
+    expression *parse_3_arg_expression(unsigned int start, unsigned int end);
+    expression *parse_4_arg_expression(unsigned int start, unsigned int end);
+    expression *parse_combining_expression(unsigned int start, unsigned int end);
+    expression *parse_unary_expression(unsigned int start, unsigned int end);
+
+    expression *parse_primary(unsigned int start, unsigned int end);
+    expression *parse_parenthentical(unsigned int start, unsigned int end);
+    expression *parse_unary_primary(unsigned int start, unsigned int end);
+    expression *parse_binary_primary(unsigned int start, unsigned int end);
+    expression *parse_just_a_string(unsigned int start, unsigned int end);
+
+    static expression *parse_args(const wcstring_list_t &args, wcstring &err);
+};
+
+struct range_t
+{
+    unsigned int start;
+    unsigned int end;
+
+    range_t(unsigned s, unsigned e) : start(s), end(e) { }
+};
+
+
+/* Base class for expressions */
+class expression
+{
+protected:
+    expression(token_t what, range_t where) : token(what), range(where) { }
+
+public:
+    const token_t token;
+    range_t range;
+
+    virtual ~expression() { }
+
+    // evaluate returns true if the expression is true (i.e. BUILTIN_TEST_SUCCESS)
+    virtual bool evaluate(wcstring_list_t &errors) = 0;
+};
+
+typedef std::auto_ptr<expression> expr_ref_t;
+
+/* Single argument like -n foo or "just a string" */
+class unary_primary : public expression
+{
+public:
+    wcstring arg;
+    unary_primary(token_t tok, range_t where, const wcstring &what) : expression(tok, where), arg(what) { }
+    bool evaluate(wcstring_list_t &errors);
+};
+
+/* Two argument primary like foo != bar */
+class binary_primary : public expression
+{
+public:
+    wcstring arg_left;
+    wcstring arg_right;
+
+    binary_primary(token_t tok, range_t where, const wcstring &left, const wcstring &right) : expression(tok, where), arg_left(left), arg_right(right)
+    { }
+    bool evaluate(wcstring_list_t &errors);
+};
+
+/* Unary operator like bang */
+class unary_operator : public expression
+{
+public:
+    expr_ref_t subject;
+    unary_operator(token_t tok, range_t where, expr_ref_t &exp) : expression(tok, where), subject(exp) { }
+    bool evaluate(wcstring_list_t &errors);
+};
+
+/* Combining expression. Contains a list of AND or OR expressions. It takes more than two so that we don't have to worry about precedence in the parser. */
+class combining_expression : public expression
+{
+public:
+    const std::vector<expression *> subjects;
+    const std::vector<token_t> combiners;
+
+    combining_expression(token_t tok, range_t where, const std::vector<expression *> &exprs, const std::vector<token_t> &combs) : expression(tok, where), subjects(exprs), combiners(combs)
+    {
+        /* We should have one more subject than combiner */
+        assert(subjects.size() == combiners.size() + 1);
+    }
+
+    /* We are responsible for destroying our expressions */
+    virtual ~combining_expression()
+    {
+        for (size_t i=0; i < subjects.size(); i++)
+        {
+            delete subjects[i];
+        }
+    }
+
+    bool evaluate(wcstring_list_t &errors);
+};
+
+/* Parenthetical expression */
+class parenthetical_expression : public expression
+{
+public:
+    expr_ref_t contents;
+    parenthetical_expression(token_t tok, range_t where, expr_ref_t &expr) : expression(tok, where), contents(expr) { }
+
+    virtual bool evaluate(wcstring_list_t &errors);
+};
+
+void test_parser::add_error(const wchar_t *fmt, ...)
+{
+    assert(fmt != NULL);
+    va_list va;
+    va_start(va, fmt);
+    this->errors.push_back(vformat_string(fmt, va));
+    va_end(va);
+}
+
+expression *test_parser::error(const wchar_t *fmt, ...)
+{
+    assert(fmt != NULL);
+    va_list va;
+    va_start(va, fmt);
+    this->errors.push_back(vformat_string(fmt, va));
+    va_end(va);
+    return NULL;
+}
+
+expression *test_parser::parse_unary_expression(unsigned int start, unsigned int end)
+{
+    if (start >= end)
+    {
+        return error(L"Missing argument at index %u", start);
+    }
+    token_t tok = token_for_string(arg(start))->tok;
+    if (tok == test_bang)
+    {
+        expr_ref_t subject(parse_unary_expression(start + 1, end));
+        if (subject.get())
+        {
+            return new unary_operator(tok, range_t(start, subject->range.end), subject);
+        }
+        else
+        {
+            return NULL;
+        }
+    }
+    else
+    {
+        return parse_primary(start, end);
+    }
+}
+
+/* Parse a combining expression (AND, OR) */
+expression *test_parser::parse_combining_expression(unsigned int start, unsigned int end)
+{
+    if (start >= end)
+        return NULL;
+
+    std::vector<expression *> subjects;
+    std::vector<token_t> combiners;
+    unsigned int idx = start;
+    bool first = true;
+
+    while (idx < end)
+    {
+
+        if (! first)
+        {
+            /* This is not the first expression, so we expect a combiner. */
+            token_t combiner = token_for_string(arg(idx))->tok;
+            if (combiner != test_combine_and && combiner != test_combine_or)
+            {
+                /* Not a combiner, we're done */
+                this->errors.insert(this->errors.begin(), format_string(L"Expected a combining operator like '-a' at index %u", idx));
+                break;
+            }
+            combiners.push_back(combiner);
+            idx++;
+        }
+
+        /* Parse another expression */
+        expression *expr = parse_unary_expression(idx, end);
+        if (! expr)
+        {
+            add_error(L"Missing argument at index %u", idx);
+            if (! first)
+            {
+                /* Clean up the dangling combiner, since it never got its right hand expression */
+                combiners.pop_back();
+            }
+            break;
+        }
+
+        /* Go to the end of this expression */
+        idx = expr->range.end;
+        subjects.push_back(expr);
+        first = false;
+    }
+
+    if (! subjects.empty())
+    {
+        /* Our new expression takes ownership of all expressions we created. The token we pass is irrelevant. */
+        return new combining_expression(test_combine_and, range_t(start, idx), subjects, combiners);
+    }
+    else
+    {
+        /* No subjects */
+        return NULL;
+    }
+}
+
+expression *test_parser::parse_unary_primary(unsigned int start, unsigned int end)
+{
+    /* We need two arguments */
+    if (start >= end)
+    {
+        return error(L"Missing argument at index %u", start);
+    }
+    if (start + 1 >= end)
+    {
+        return error(L"Missing argument at index %u", start + 1);
+    }
+
+    /* All our unary primaries are prefix, so the operator is at start. */
+    const token_info_t *info = token_for_string(arg(start));
+    if (!(info->flags & UNARY_PRIMARY))
+        return NULL;
+
+    return new unary_primary(info->tok, range_t(start, start + 2), arg(start + 1));
+}
+
+expression *test_parser::parse_just_a_string(unsigned int start, unsigned int end)
+{
+    /* Handle a string as a unary primary that is not a token of any other type.
+       e.g. 'test foo -a bar' should evaluate to true
+       We handle this with a unary primary of test_string_n
+    */
+
+    /* We need one arguments */
+    if (start >= end)
+    {
+        return error(L"Missing argument at index %u", start);
+    }
+
+    const token_info_t *info = token_for_string(arg(start));
+    if (info->tok != test_unknown)
+    {
+        return error(L"Unexpected argument type at index %u", start);
+    }
+
+    /* This is hackish; a nicer way to implement this would be with a "just a string" expression type */
+    return new unary_primary(test_string_n, range_t(start, start + 1), arg(start));
+}
+
+#if 0
+expression *test_parser::parse_unary_primary(unsigned int start, unsigned int end)
+{
+    /* We need either one or two arguments */
+    if (start >= end)
+    {
+        return error(L"Missing argument at index %u", start);
+    }
+
+    /* The index of the argument to the unary primary */
+    unsigned int arg_idx;
+
+    /* All our unary primaries are prefix, so any operator is at start. But it also may just be a string, with no operator. */
+    const token_info_t *info = token_for_string(arg(start));
+    if (info->flags & UNARY_PRIMARY)
+    {
+        /* We have an operator. Skip the operator argument */
+        arg_idx = start + 1;
+
+        /* We have some freedom here...do we allow other tokens for the argument to operate on?
+           For example, should 'test -n =' work? I say yes. So no typechecking on the next token. */
+
+    }
+    else if (info->tok == test_unknown)
+    {
+        /* "Just a string. */
+        arg_idx = start;
+    }
+    else
+    {
+        /* Here we don't allow arbitrary tokens as "just a string." I.e. 'test = -a =' should have a parse error. We could relax this at some point. */
+        return error(L"Parse error at argument index %u", start);
+    }
+
+    /* Verify we have the argument we want, i.e. test -n should fail to parse */
+    if (arg_idx >= end)
+    {
+        return error(L"Missing argument at index %u", arg_idx);
+    }
+
+    return new unary_primary(info->tok, range_t(start, arg_idx + 1), arg(arg_idx));
+}
+#endif
+
+expression *test_parser::parse_binary_primary(unsigned int start, unsigned int end)
+{
+    /* We need three arguments */
+    for (unsigned int idx = start; idx < start + 3; idx++)
+    {
+        if (idx >= end)
+        {
+            return error(L"Missing argument at index %u", idx);
+        }
+    }
+
+    /* All our binary primaries are infix, so the operator is at start + 1. */
+    const token_info_t *info = token_for_string(arg(start + 1));
+    if (!(info->flags & BINARY_PRIMARY))
+        return NULL;
+
+    return new binary_primary(info->tok, range_t(start, start + 3), arg(start), arg(start + 2));
+}
+
+expression *test_parser::parse_parenthentical(unsigned int start, unsigned int end)
+{
+    /* We need at least three arguments: open paren, argument, close paren */
+    if (start + 3 >= end)
+        return NULL;
+
+    /* Must start with an open expression */
+    const token_info_t *open_paren = token_for_string(arg(start));
+    if (open_paren->tok != test_paren_open)
+        return NULL;
+
+    /* Parse a subexpression */
+    expression *subexr_ptr = parse_expression(start + 1, end);
+    if (! subexr_ptr)
+        return NULL;
+    expr_ref_t subexpr(subexr_ptr);
+
+    /* Parse a close paren */
+    unsigned close_index = subexpr->range.end;
+    assert(close_index <= end);
+    if (close_index == end)
+    {
+        return error(L"Missing close paren at index %u", close_index);
+    }
+    const token_info_t *close_paren = token_for_string(arg(close_index));
+    if (close_paren->tok != test_paren_close)
+    {
+        return error(L"Expected close paren at index %u", close_index);
+    }
+
+    /* Success */
+    return new parenthetical_expression(test_paren_open, range_t(start, close_index+1), subexpr);
+}
+
+expression *test_parser::parse_primary(unsigned int start, unsigned int end)
+{
+    if (start >= end)
+    {
+        return error(L"Missing argument at index %u", start);
+    }
+
+    expression *expr = NULL;
+    if (! expr) expr = parse_parenthentical(start, end);
+    if (! expr) expr = parse_unary_primary(start, end);
+    if (! expr) expr = parse_binary_primary(start, end);
+    if (! expr) expr = parse_just_a_string(start, end);
+    return expr;
+}
+
+// See IEEE 1003.1 breakdown of the behavior for different parameter counts
+expression *test_parser::parse_3_arg_expression(unsigned int start, unsigned int end)
+{
+    assert(end - start == 3);
+    expression *result = NULL;
+
+    const token_info_t *center_token = token_for_string(arg(start + 1));
+    if (center_token->flags & BINARY_PRIMARY)
+    {
+        result = parse_binary_primary(start, end);
+    }
+    else if (center_token->tok == test_combine_and || center_token->tok == test_combine_or)
+    {
+        expr_ref_t left(parse_unary_expression(start, start + 1));
+        expr_ref_t right(parse_unary_expression(start + 2, start + 3));
+        if (left.get() && right.get())
+        {
+            // Transfer ownership to the vector of subjects
+            std::vector<token_t> combiners(1, center_token->tok);
+            std::vector<expression *> subjects;
+            subjects.push_back(left.release());
+            subjects.push_back(right.release());
+            result = new combining_expression(center_token->tok, range_t(start, end), subjects, combiners);
+        }
+    }
+    else
+    {
+        result = parse_unary_expression(start, end);
+    }
+    return result;
+}
+
+expression *test_parser::parse_4_arg_expression(unsigned int start, unsigned int end)
+{
+    assert(end - start == 4);
+    expression *result = NULL;
+
+    token_t first_token = token_for_string(arg(start))->tok;
+    if (first_token == test_bang)
+    {
+        expr_ref_t subject(parse_3_arg_expression(start + 1, end));
+        if (subject.get())
+        {
+            result = new unary_operator(first_token, range_t(start, subject->range.end), subject);
+        }
+    }
+    else if (first_token == test_paren_open)
+    {
+        result = parse_parenthentical(start, end);
+    }
+    else
+    {
+        result = parse_combining_expression(start, end);
+    }
+    return result;
+}
+
+
+expression *test_parser::parse_expression(unsigned int start, unsigned int end)
+{
+    if (start >= end)
+    {
+        return error(L"Missing argument at index %u", start);
+    }
+
+    unsigned int argc = end - start;
+    switch (argc)
+    {
+        case 0:
+            assert(0); //should have been caught by the above test
+            return NULL;
+
+        case 1:
+        {
+            return error(L"Missing argument at index %u", start + 1);
+        }
+        case 2:
+        {
+            return parse_unary_expression(start, end);
+        }
+
+        case 3:
+        {
+            return parse_3_arg_expression(start, end);
+        }
+
+        case 4:
+        {
+            return parse_4_arg_expression(start, end);
+        }
+
+        default:
+        {
+            return parse_combining_expression(start, end);
+        }
+    }
+}
+
+expression *test_parser::parse_args(const wcstring_list_t &args, wcstring &err)
+{
+    /* Empty list and one-arg list should be handled by caller */
+    assert(args.size() > 1);
+
+    test_parser parser(args);
+    expression *result = parser.parse_expression(0, (unsigned int)args.size());
+
+    /* Handle errors */
+    for (size_t i = 0; i < parser.errors.size(); i++)
+    {
+        err.append(L"test: ");
+        err.append(parser.errors.at(i));
+        err.push_back(L'\n');
+        // For now we only show the first error
+        break;
+    }
+
+    if (result)
+    {
+        /* It's also an error if there are any unused arguments. This is not detected by parse_expression() */
+        assert(result->range.end <= args.size());
+        if (result->range.end < args.size())
+        {
+            if (err.empty())
+            {
+                append_format(err, L"test: unexpected argument at index %lu: '%ls'\n", (unsigned long)result->range.end, args.at(result->range.end).c_str());
+            }
+
+            delete result;
+            result = NULL;
+        }
+    }
+
+    return result;
+}
+
+bool unary_primary::evaluate(wcstring_list_t &errors)
+{
+    return unary_primary_evaluate(token, arg, errors);
+}
+
+bool binary_primary::evaluate(wcstring_list_t &errors)
+{
+    return binary_primary_evaluate(token, arg_left, arg_right, errors);
+}
+
+bool unary_operator::evaluate(wcstring_list_t &errors)
+{
+    switch (token)
+    {
+        case test_bang:
+            assert(subject.get());
+            return ! subject->evaluate(errors);
+        default:
+            errors.push_back(format_string(L"Unknown token type in %s", __func__));
+            return false;
+
+    }
+}
+
+bool combining_expression::evaluate(wcstring_list_t &errors)
+{
+    switch (token)
+    {
+        case test_combine_and:
+        case test_combine_or:
+        {
+            /* One-element case */
+            if (subjects.size() == 1)
+                return subjects.at(0)->evaluate(errors);
+
+            /* Evaluate our lists, remembering that AND has higher precedence than OR. We can visualize this as a sequence of OR expressions of AND expressions. */
+            assert(combiners.size() + 1 == subjects.size());
+            assert(! subjects.empty());
+
+            size_t idx = 0, max = subjects.size();
+            bool or_result = false;
+            while (idx < max)
+            {
+                if (or_result)
+                {
+                    /* Short circuit */
+                    break;
+                }
+
+                /* Evaluate a stream of AND starting at given subject index. It may only have one element.  */
+                bool and_result = true;
+                for (; idx < max; idx++)
+                {
+                    /* Evaluate it, short-circuiting */
+                    and_result = and_result && subjects.at(idx)->evaluate(errors);
+
+                    /* If the combiner at this index (which corresponding to how we combine with the next subject) is not AND, then exit the loop */
+                    if (idx + 1 < max && combiners.at(idx) != test_combine_and)
+                    {
+                        idx++;
+                        break;
+                    }
+                }
+
+                /* OR it in */
+                or_result = or_result || and_result;
+            }
+            return or_result;
+        }
+
+        default:
+            errors.push_back(format_string(L"Unknown token type in %s", __func__));
+            return BUILTIN_TEST_FAIL;
+
+    }
+}
+
+bool parenthetical_expression::evaluate(wcstring_list_t &errors)
+{
+    return contents->evaluate(errors);
+}
+
+/* IEEE 1003.1 says nothing about what it means for two strings to be "algebraically equal". For example, should we interpret 0x10 as 0, 10, or 16? Here we use only base 10 and use wcstoll, which allows for leading + and -, and leading whitespace. This matches bash. */
+static bool parse_number(const wcstring &arg, long long *out)
+{
+    const wchar_t *str = arg.c_str();
+    wchar_t *endptr = NULL;
+    *out = wcstoll(str, &endptr, 10);
+    return endptr && *endptr == L'\0';
+}
+
+static bool binary_primary_evaluate(test_expressions::token_t token, const wcstring &left, const wcstring &right, wcstring_list_t &errors)
+{
+    using namespace test_expressions;
+    long long left_num, right_num;
+    switch (token)
+    {
+        case test_string_equal:
+            return left == right;
+
+        case test_string_not_equal:
+            return left != right;
+
+        case test_number_equal:
+            return parse_number(left, &left_num) && parse_number(right, &right_num) && left_num == right_num;
+
+        case test_number_not_equal:
+            return parse_number(left, &left_num) && parse_number(right, &right_num) && left_num != right_num;
+
+        case test_number_greater:
+            return parse_number(left, &left_num) && parse_number(right, &right_num) && left_num > right_num;
+
+        case test_number_greater_equal:
+            return parse_number(left, &left_num) && parse_number(right, &right_num) && left_num >= right_num;
+
+        case test_number_lesser:
+            return parse_number(left, &left_num) && parse_number(right, &right_num) && left_num < right_num;
+
+        case test_number_lesser_equal:
+            return parse_number(left, &left_num) && parse_number(right, &right_num) && left_num <= right_num;
+
+        default:
+            errors.push_back(format_string(L"Unknown token type in %s", __func__));
+            return false;
+    }
+}
+
+
+static bool unary_primary_evaluate(test_expressions::token_t token, const wcstring &arg, wcstring_list_t &errors)
+{
+    using namespace test_expressions;
+    struct stat buf;
+    long long num;
+    switch (token)
+    {
+        case test_filetype_b:            // "-b", for block special files
+            return !wstat(arg, &buf) && S_ISBLK(buf.st_mode);
+
+        case test_filetype_c:            // "-c", for character special files
+            return !wstat(arg, &buf) && S_ISCHR(buf.st_mode);
+
+        case test_filetype_d:            // "-d", for directories
+            return !wstat(arg, &buf) && S_ISDIR(buf.st_mode);
+
+        case test_filetype_e:            // "-e", for files that exist
+            return !wstat(arg, &buf);
+
+        case test_filetype_f:            // "-f", for for regular files
+            return !wstat(arg, &buf) && S_ISREG(buf.st_mode);
+
+        case test_filetype_G:            // "-G", for check effective group id
+            return !wstat(arg, &buf) && getegid() == buf.st_gid;
+
+        case test_filetype_g:            // "-g", for set-group-id
+            return !wstat(arg, &buf) && (S_ISGID & buf.st_mode);
+
+        case test_filetype_h:            // "-h", for symbolic links
+        case test_filetype_L:            // "-L", same as -h
+            return !lwstat(arg, &buf) && S_ISLNK(buf.st_mode);
+
+        case test_filetype_O:            // "-O", for check effective user id
+            return !wstat(arg, &buf) && geteuid() == buf.st_uid;
+
+        case test_filetype_p:            // "-p", for FIFO
+            return !wstat(arg, &buf) && S_ISFIFO(buf.st_mode);
+
+        case test_filetype_S:            // "-S", socket
+            return !wstat(arg, &buf) && S_ISSOCK(buf.st_mode);
+
+        case test_filesize_s:            // "-s", size greater than zero
+            return !wstat(arg, &buf) && buf.st_size > 0;
+
+        case test_filedesc_t:            // "-t", whether the fd is associated with a terminal
+            return parse_number(arg, &num) && num == (int)num && isatty((int)num);
+
+        case test_fileperm_r:            // "-r", read permission
+            return !waccess(arg, R_OK);
+
+        case test_fileperm_u:            // "-u", whether file is setuid
+            return !wstat(arg, &buf) && (S_ISUID & buf.st_mode);
+
+        case test_fileperm_w:            // "-w", whether file write permission is allowed
+            return !waccess(arg, W_OK);
+
+        case test_fileperm_x:            // "-x", whether file execute/search is allowed
+            return !waccess(arg, X_OK);
+
+        case test_string_n:              // "-n", non-empty string
+            return ! arg.empty();
+
+        case test_string_z:              // "-z", true if length of string is 0
+            return arg.empty();
+
+        default:
+            errors.push_back(format_string(L"Unknown token type in %s", __func__));
+            return false;
+    }
+}
+
+};
+
+/*
+ * Evaluate a conditional expression given the arguments.
+ * If fromtest is set, the caller is the test or [ builtin;
+ * with the pointer giving the name of the command.
+ * for POSIX conformance this supports a more limited range
+ * of functionality.
+ *
+ * Return status is the final shell status, i.e. 0 for true,
+ * 1 for false and 2 for error.
+ */
+int builtin_test(parser_t &parser, wchar_t **argv)
+{
+    using namespace test_expressions;
+
+    /* The first argument should be the name of the command ('test') */
+    if (! argv[0])
+        return BUILTIN_TEST_FAIL;
+
+    /* Whether we are invoked with bracket '[' or not */
+    const bool is_bracket = ! wcscmp(argv[0], L"[");
+
+    size_t argc = 0;
+    while (argv[argc + 1])
+        argc++;
+
+    /* If we're bracket, the last argument ought to be ]; we ignore it. Note that argc is the number of arguments after the command name; thus argv[argc] is the last argument. */
+    if (is_bracket)
+    {
+        if (! wcscmp(argv[argc], L"]"))
+        {
+            /* Ignore the closing bracketp */
+            argc--;
+        }
+        else
+        {
+            builtin_show_error(L"[: the last argument must be ']'\n");
+            return BUILTIN_TEST_FAIL;
+        }
+
+    }
+
+    /* Collect the arguments into a list */
+    const wcstring_list_t args(argv + 1, argv + 1 + argc);
+
+    switch (argc)
+    {
+        case 0:
+        {
+            // Per 1003.1, exit false
+            return BUILTIN_TEST_FAIL;
+        }
+        case 1:
+        {
+            // Per 1003.1, exit true if the arg is non-empty
+            return args.at(0).empty() ? BUILTIN_TEST_FAIL : BUILTIN_TEST_SUCCESS;
+        }
+        default:
+        {
+            // Try parsing. If expr is not nil, we are responsible for deleting it.
+            wcstring err;
+            expression *expr = test_parser::parse_args(args, err);
+            if (! expr)
+            {
+#if 0
+                printf("Oops! test was given args:\n");
+                for (size_t i=0; i < argc; i++)
+                {
+                    printf("\t%ls\n", args.at(i).c_str());
+                }
+                printf("and returned parse error: %ls\n", err.c_str());
+#endif
+                builtin_show_error(err);
+                return BUILTIN_TEST_FAIL;
+            }
+            else
+            {
+                wcstring_list_t eval_errors;
+                bool result = expr->evaluate(eval_errors);
+                if (! eval_errors.empty())
+                {
+                    printf("test returned eval errors:\n");
+                    for (size_t i=0; i < eval_errors.size(); i++)
+                    {
+                        printf("\t%ls\n", eval_errors.at(i).c_str());
+                    }
+                }
+                delete expr;
+                return result ? BUILTIN_TEST_SUCCESS : BUILTIN_TEST_FAIL;
+            }
+        }
+    }
+    return 1;
+}
diff --git a/src/builtin_ulimit.cpp b/src/builtin_ulimit.cpp
new file mode 100644
index 00000000..64b126b7
--- /dev/null
+++ b/src/builtin_ulimit.cpp
@@ -0,0 +1,513 @@
+/** \file builtin_ulimit.c Functions defining the ulimit builtin
+
+Functions used for implementing the ulimit builtin.
+
+*/
+#include "config.h"
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <wchar.h>
+#include <wctype.h>
+#include <sys/time.h>
+#include <sys/resource.h>
+#include <unistd.h>
+#include <errno.h>
+
+#include "fallback.h"
+#include "util.h"
+
+#include "builtin.h"
+#include "common.h"
+#include "wgetopt.h"
+
+
+/**
+   Struct describing a resource limit
+*/
+struct resource_t
+{
+    /**
+       Resource id
+     */
+    int resource;
+    /**
+       Description of resource
+    */
+    const wchar_t *desc;
+    /**
+       Switch used on commandline to specify resource
+    */
+    wchar_t switch_char;
+    /**
+       The implicit multiplier used when setting getting values
+    */
+    int multiplier;
+}
+;
+
+/**
+   Array of resource_t structs, describing all known resource types.
+*/
+static const struct resource_t resource_arr[] =
+{
+    {
+        RLIMIT_CORE, L"Maximum size of core files created", L'c', 1024
+    }
+    ,
+    {
+        RLIMIT_DATA, L"Maximum size of a process’s data segment", L'd', 1024
+    }
+    ,
+    {
+        RLIMIT_FSIZE, L"Maximum size of files created by the shell", L'f', 1024
+    }
+    ,
+#ifdef RLIMIT_MEMLOCK
+    {
+        RLIMIT_MEMLOCK, L"Maximum size that may be locked into memory", L'l', 1024
+    }
+    ,
+#endif
+#ifdef RLIMIT_RSS
+    {
+        RLIMIT_RSS, L"Maximum resident set size", L'm', 1024
+    }
+    ,
+#endif
+    {
+        RLIMIT_NOFILE, L"Maximum number of open file descriptors", L'n', 1
+    }
+    ,
+    {
+        RLIMIT_STACK, L"Maximum stack size", L's', 1024
+    }
+    ,
+    {
+        RLIMIT_CPU, L"Maximum amount of cpu time in seconds", L't', 1
+    }
+    ,
+#ifdef RLIMIT_NPROC
+    {
+        RLIMIT_NPROC, L"Maximum number of processes available to a single user", L'u', 1
+    }
+    ,
+#endif
+#ifdef RLIMIT_AS
+    {
+        RLIMIT_AS, L"Maximum amount of virtual memory available to the shell", L'v', 1024
+    }
+    ,
+#endif
+    {
+        0, 0, 0, 0
+    }
+}
+;
+
+/**
+   Get the implicit multiplication factor for the specified resource limit
+*/
+static int get_multiplier(int what)
+{
+    int i;
+
+    for (i=0; resource_arr[i].desc; i++)
+    {
+        if (resource_arr[i].resource == what)
+        {
+            return resource_arr[i].multiplier;
+        }
+    }
+    return -1;
+}
+
+/**
+   Return the value for the specified resource limit. This function
+   does _not_ multiply the limit value by the multiplier constant used
+   by the commandline ulimit.
+*/
+static rlim_t get(int resource, int hard)
+{
+    struct rlimit ls;
+
+    getrlimit(resource, &ls);
+
+    return hard ? ls.rlim_max:ls.rlim_cur;
+}
+
+/**
+   Print the value of the specified resource limit
+*/
+static void print(int resource, int hard)
+{
+    rlim_t l = get(resource, hard);
+
+    if (l == RLIM_INFINITY)
+        stdout_buffer.append(L"unlimited\n");
+    else
+        append_format(stdout_buffer, L"%d\n", l / get_multiplier(resource));
+
+}
+
+/**
+   Print values of all resource limits
+*/
+static void print_all(int hard)
+{
+    int i;
+    int w=0;
+
+    for (i=0; resource_arr[i].desc; i++)
+    {
+        w=maxi(w, fish_wcswidth(resource_arr[i].desc));
+    }
+
+    for (i=0; resource_arr[i].desc; i++)
+    {
+        struct rlimit ls;
+        rlim_t l;
+        getrlimit(resource_arr[i].resource, &ls);
+        l = hard ? ls.rlim_max:ls.rlim_cur;
+
+        const wchar_t *unit = ((resource_arr[i].resource==RLIMIT_CPU)?L"(seconds, ":(get_multiplier(resource_arr[i].resource)==1?L"(":L"(kB, "));
+
+        append_format(stdout_buffer,
+                      L"%-*ls %10ls-%lc) ",
+                      w,
+                      resource_arr[i].desc,
+                      unit,
+                      resource_arr[i].switch_char);
+
+        if (l == RLIM_INFINITY)
+        {
+            stdout_buffer.append(L"unlimited\n");
+        }
+        else
+        {
+            append_format(stdout_buffer, L"%d\n", l/get_multiplier(resource_arr[i].resource));
+        }
+    }
+
+}
+
+/**
+   Returns the description for the specified resource limit
+*/
+static const wchar_t *get_desc(int what)
+{
+    int i;
+
+    for (i=0; resource_arr[i].desc; i++)
+    {
+        if (resource_arr[i].resource == what)
+        {
+            return resource_arr[i].desc;
+        }
+    }
+    return L"Not a resource";
+}
+
+/**
+   Set the new value of the specified resource limit. This function
+   does _not_ multiply the limit value by the multiplier constant used
+   by the commandline ulimit.
+*/
+static int set(int resource, int hard, int soft, rlim_t value)
+{
+    struct rlimit ls;
+    getrlimit(resource, &ls);
+
+    if (hard)
+    {
+        ls.rlim_max = value;
+    }
+
+    if (soft)
+    {
+        ls.rlim_cur = value;
+
+        /*
+          Do not attempt to set the soft limit higher than the hard limit
+        */
+        if ((value == RLIM_INFINITY && ls.rlim_max != RLIM_INFINITY) ||
+                (value != RLIM_INFINITY && ls.rlim_max != RLIM_INFINITY && value > ls.rlim_max))
+        {
+            ls.rlim_cur = ls.rlim_max;
+        }
+    }
+
+    if (setrlimit(resource, &ls))
+    {
+        if (errno == EPERM)
+            append_format(stderr_buffer, L"ulimit: Permission denied when changing resource of type '%ls'\n", get_desc(resource));
+        else
+            builtin_wperror(L"ulimit");
+        return 1;
+    }
+    return 0;
+}
+
+/**
+   The ulimit builtin, used for setting resource limits. Defined in
+   builtin_ulimit.c.
+*/
+static int builtin_ulimit(parser_t &parser, wchar_t ** argv)
+{
+    wgetopter_t w;
+    int hard=0;
+    int soft=0;
+
+    int what = RLIMIT_FSIZE;
+    int report_all = 0;
+
+    int argc = builtin_count_args(argv);
+
+    w.woptind=0;
+
+    while (1)
+    {
+        static const struct woption
+                long_options[] =
+        {
+            {
+                L"all", no_argument, 0, 'a'
+            }
+            ,
+            {
+                L"hard", no_argument, 0, 'H'
+            }
+            ,
+            {
+                L"soft", no_argument, 0, 'S'
+            }
+            ,
+            {
+                L"core-size", no_argument, 0, 'c'
+            }
+            ,
+            {
+                L"data-size", no_argument, 0, 'd'
+            }
+            ,
+            {
+                L"file-size", no_argument, 0, 'f'
+            }
+            ,
+            {
+                L"lock-size", no_argument, 0, 'l'
+            }
+            ,
+            {
+                L"resident-set-size", no_argument, 0, 'm'
+            }
+            ,
+            {
+                L"file-descriptor-count", no_argument, 0, 'n'
+            }
+            ,
+            {
+                L"stack-size", no_argument, 0, 's'
+            }
+            ,
+            {
+                L"cpu-time", no_argument, 0, 't'
+            }
+            ,
+            {
+                L"process-count", no_argument, 0, 'u'
+            }
+            ,
+            {
+                L"virtual-memory-size", no_argument, 0, 'v'
+            }
+            ,
+            {
+                L"help", no_argument, 0, 'h'
+            }
+            ,
+            {
+                0, 0, 0, 0
+            }
+        }
+        ;
+
+
+        int opt_index = 0;
+
+        int opt = w.wgetopt_long(argc,
+                                 argv,
+                                 L"aHScdflmnstuvh",
+                                 long_options,
+                                 &opt_index);
+        if (opt == -1)
+            break;
+
+        switch (opt)
+        {
+            case 0:
+                if (long_options[opt_index].flag != 0)
+                    break;
+                append_format(stderr_buffer,
+                              BUILTIN_ERR_UNKNOWN,
+                              argv[0],
+                              long_options[opt_index].name);
+                builtin_print_help(parser, argv[0], stderr_buffer);
+
+                return 1;
+
+            case L'a':
+                report_all=1;
+                break;
+
+            case L'H':
+                hard=1;
+                break;
+
+            case L'S':
+                soft=1;
+                break;
+
+            case L'c':
+                what=RLIMIT_CORE;
+                break;
+
+            case L'd':
+                what=RLIMIT_DATA;
+                break;
+
+            case L'f':
+                what=RLIMIT_FSIZE;
+                break;
+#ifdef RLIMIT_MEMLOCK
+            case L'l':
+                what=RLIMIT_MEMLOCK;
+                break;
+#endif
+
+#ifdef RLIMIT_RSS
+            case L'm':
+                what=RLIMIT_RSS;
+                break;
+#endif
+
+            case L'n':
+                what=RLIMIT_NOFILE;
+                break;
+
+            case L's':
+                what=RLIMIT_STACK;
+                break;
+
+            case L't':
+                what=RLIMIT_CPU;
+                break;
+
+#ifdef RLIMIT_NPROC
+            case L'u':
+                what=RLIMIT_NPROC;
+                break;
+#endif
+
+#ifdef RLIMIT_AS
+            case L'v':
+                what=RLIMIT_AS;
+                break;
+#endif
+
+            case L'h':
+                builtin_print_help(parser, argv[0], stdout_buffer);
+                return 0;
+
+            case L'?':
+                builtin_unknown_option(parser, argv[0], argv[w.woptind-1]);
+                return 1;
+        }
+    }
+
+    if (report_all)
+    {
+        if (argc - w.woptind == 0)
+        {
+            print_all(hard);
+        }
+        else
+        {
+            stderr_buffer.append(argv[0]);
+            stderr_buffer.append(L": Too many arguments\n");
+            builtin_print_help(parser, argv[0], stderr_buffer);
+            return 1;
+        }
+
+        return 0;
+    }
+
+    switch (argc - w.woptind)
+    {
+        case 0:
+        {
+            /*
+              Show current limit value
+            */
+            print(what, hard);
+            break;
+        }
+
+        case 1:
+        {
+            /*
+              Change current limit value
+            */
+            rlim_t new_limit;
+            wchar_t *end;
+
+            /*
+              Set both hard and soft limits if nothing else was specified
+            */
+            if (!(hard+soft))
+            {
+                hard=soft=1;
+            }
+
+            if (wcscasecmp(argv[w.woptind], L"unlimited")==0)
+            {
+                new_limit = RLIM_INFINITY;
+            }
+            else if (wcscasecmp(argv[w.woptind], L"hard")==0)
+            {
+                new_limit = get(what, 1);
+            }
+            else if (wcscasecmp(argv[w.woptind], L"soft")==0)
+            {
+                new_limit = get(what, soft);
+            }
+            else
+            {
+                errno=0;
+                new_limit = wcstol(argv[w.woptind], &end, 10);
+                if (errno || *end)
+                {
+                    append_format(stderr_buffer,
+                                  L"%ls: Invalid limit '%ls'\n",
+                                  argv[0],
+                                  argv[w.woptind]);
+                    builtin_print_help(parser, argv[0], stderr_buffer);
+                    return 1;
+                }
+                new_limit *= get_multiplier(what);
+            }
+
+            return set(what, hard, soft, new_limit);
+        }
+
+        default:
+        {
+            stderr_buffer.append(argv[0]);
+            stderr_buffer.append(L": Too many arguments\n");
+            builtin_print_help(parser, argv[0], stderr_buffer);
+            return 1;
+        }
+
+    }
+    return 0;
+}
diff --git a/src/color.cpp b/src/color.cpp
new file mode 100644
index 00000000..d076ea17
--- /dev/null
+++ b/src/color.cpp
@@ -0,0 +1,376 @@
+/** \file color.cpp Color class implementation
+*/
+
+#include "color.h"
+#include "fallback.h" // IWYU pragma: keep
+#include <assert.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <wchar.h>
+#include <cstddef>
+
+bool rgb_color_t::try_parse_special(const wcstring &special)
+{
+    memset(&data, 0, sizeof data);
+    const wchar_t *name = special.c_str();
+    if (! wcscasecmp(name, L"normal"))
+    {
+        this->type = type_normal;
+    }
+    else if (! wcscasecmp(name, L"reset"))
+    {
+        this->type = type_reset;
+    }
+    else if (! wcscasecmp(name, L"ignore"))
+    {
+        this->type = type_ignore;
+    }
+    else
+    {
+        this->type = type_none;
+    }
+    return this->type != type_none;
+}
+
+static int parse_hex_digit(wchar_t x)
+{
+    switch (x)
+    {
+        case L'0':
+            return 0x0;
+        case L'1':
+            return 0x1;
+        case L'2':
+            return 0x2;
+        case L'3':
+            return 0x3;
+        case L'4':
+            return 0x4;
+        case L'5':
+            return 0x5;
+        case L'6':
+            return 0x6;
+        case L'7':
+            return 0x7;
+        case L'8':
+            return 0x8;
+        case L'9':
+            return 0x9;
+        case L'a':
+        case L'A':
+            return 0xA;
+        case L'b':
+        case L'B':
+            return 0xB;
+        case L'c':
+        case L'C':
+            return 0xC;
+        case L'd':
+        case L'D':
+            return 0xD;
+        case L'e':
+        case L'E':
+            return 0xE;
+        case L'f':
+        case L'F':
+            return 0xF;
+        default:
+            return -1;
+    }
+}
+
+static unsigned long squared_difference(long p1, long p2)
+{
+    unsigned long diff = (unsigned long)labs(p1 - p2);
+    return diff * diff;
+}
+
+static unsigned char convert_color(const unsigned char rgb[3], const uint32_t *colors, size_t color_count)
+{
+    long r = rgb[0], g = rgb[1], b = rgb[2];
+    unsigned long best_distance = (unsigned long)(-1);
+    unsigned char best_index = (unsigned char)(-1);
+    for (unsigned char idx = 0; idx < color_count; idx++)
+    {
+        uint32_t color = colors[idx];
+        long test_r = (color >> 16) & 0xFF, test_g = (color >> 8) & 0xFF, test_b = (color >> 0) & 0xFF;
+        unsigned long distance = squared_difference(r, test_r) + squared_difference(g, test_g) + squared_difference(b, test_b);
+        if (distance <= best_distance)
+        {
+            best_index = idx;
+            best_distance = distance;
+        }
+    }
+    return best_index;
+
+}
+
+bool rgb_color_t::try_parse_rgb(const wcstring &name)
+{
+    memset(&data, 0, sizeof data);
+    /* We support the following style of rgb formats (case insensitive):
+        #FA3
+        #F3A035
+        FA3
+        F3A035
+    */
+
+    size_t digit_idx = 0, len = name.size();
+
+    /* Skip any leading # */
+    if (len > 0 && name.at(0) == L'#')
+        digit_idx++;
+
+    bool success = false;
+    size_t i;
+    if (len - digit_idx == 3)
+    {
+        // type FA3
+        for (i=0; i < 3; i++)
+        {
+            int val = parse_hex_digit(name.at(digit_idx++));
+            if (val < 0) break;
+            data.color.rgb[i] = val*16+val;
+        }
+        success = (i == 3);
+    }
+    else if (len - digit_idx == 6)
+    {
+        // type F3A035
+        for (i=0; i < 3; i++)
+        {
+            int hi = parse_hex_digit(name.at(digit_idx++));
+            int lo = parse_hex_digit(name.at(digit_idx++));
+            if (lo < 0 || hi < 0) break;
+            data.color.rgb[i] = hi*16+lo;
+        }
+        success = (i == 3);
+    }
+    if (success)
+    {
+        this->type = type_rgb;
+    }
+    return success;
+}
+
+struct named_color_t
+{
+    const wchar_t * name;
+    unsigned char idx;
+    unsigned char rgb[3];
+};
+
+static const named_color_t named_colors[11] =
+{
+    {L"black", 0, {0, 0, 0}},
+    {L"red", 1, {0xFF, 0, 0}},
+    {L"green", 2, {0, 0xFF, 0}},
+    {L"brown", 3, {0x72, 0x50, 0}},
+    {L"yellow", 3, {0xFF, 0xFF, 0}},
+    {L"blue", 4, {0, 0, 0xFF}},
+    {L"magenta", 5, {0xFF, 0, 0xFF}},
+    {L"purple", 5, {0xFF, 0, 0xFF}},
+    {L"cyan", 6, {0, 0xFF, 0xFF}},
+    {L"white", 7, {0xFF, 0xFF, 0xFF}},
+    {L"normal", 8, {0xFF, 0xFF, 0XFF}}
+};
+
+wcstring_list_t rgb_color_t::named_color_names(void)
+{
+    size_t count = sizeof named_colors / sizeof *named_colors;
+    wcstring_list_t result;
+    result.reserve(count);
+    for (size_t i=0; i < count; i++)
+    {
+        result.push_back(named_colors[i].name);
+    }
+    return result;
+}
+
+bool rgb_color_t::try_parse_named(const wcstring &str)
+{
+    memset(&data, 0, sizeof data);
+    size_t max = sizeof named_colors / sizeof *named_colors;
+    for (size_t idx=0; idx < max; idx++)
+    {
+        if (0 == wcscasecmp(str.c_str(), named_colors[idx].name))
+        {
+            data.name_idx = named_colors[idx].idx;
+            this->type = type_named;
+            return true;
+        }
+    }
+    return false;
+}
+
+static const wchar_t *name_for_color_idx(unsigned char idx)
+{
+    size_t max = sizeof named_colors / sizeof *named_colors;
+    for (size_t i=0; i < max; i++)
+    {
+        if (named_colors[i].idx == idx)
+        {
+            return named_colors[i].name;
+        }
+    }
+    return L"unknown";
+}
+
+rgb_color_t::rgb_color_t(unsigned char t, unsigned char i) : type(t), flags(), data()
+{
+    data.name_idx = i;
+}
+
+rgb_color_t rgb_color_t::normal()
+{
+    return rgb_color_t(type_normal);
+}
+rgb_color_t rgb_color_t::reset()
+{
+    return rgb_color_t(type_reset);
+}
+rgb_color_t rgb_color_t::ignore()
+{
+    return rgb_color_t(type_ignore);
+}
+rgb_color_t rgb_color_t::none()
+{
+    return rgb_color_t(type_none);
+}
+rgb_color_t rgb_color_t::white()
+{
+    return rgb_color_t(type_named, 7);
+}
+rgb_color_t rgb_color_t::black()
+{
+    return rgb_color_t(type_named, 0);
+}
+
+static unsigned char term8_color_for_rgb(const unsigned char rgb[3])
+{
+    const uint32_t kColors[] =
+    {
+        0x000000, //Black
+        0xFF0000, //Red
+        0x00FF00, //Green
+        0xFFFF00, //Yellow
+        0x0000FF, //Blue
+        0xFF00FF, //Magenta
+        0x00FFFF, //Cyan
+        0xFFFFFF, //White
+    };
+    return convert_color(rgb, kColors, sizeof kColors / sizeof *kColors);
+}
+
+static unsigned char term256_color_for_rgb(const unsigned char rgb[3])
+{
+    const uint32_t kColors[240] =
+    {
+        0x000000, 0x00005f, 0x000087, 0x0000af, 0x0000d7, 0x0000ff, 0x005f00, 0x005f5f,
+        0x005f87, 0x005faf, 0x005fd7, 0x005fff, 0x008700, 0x00875f, 0x008787, 0x0087af,
+        0x0087d7, 0x0087ff, 0x00af00, 0x00af5f, 0x00af87, 0x00afaf, 0x00afd7, 0x00afff,
+        0x00d700, 0x00d75f, 0x00d787, 0x00d7af, 0x00d7d7, 0x00d7ff, 0x00ff00, 0x00ff5f,
+        0x00ff87, 0x00ffaf, 0x00ffd7, 0x00ffff, 0x5f0000, 0x5f005f, 0x5f0087, 0x5f00af,
+        0x5f00d7, 0x5f00ff, 0x5f5f00, 0x5f5f5f, 0x5f5f87, 0x5f5faf, 0x5f5fd7, 0x5f5fff,
+        0x5f8700, 0x5f875f, 0x5f8787, 0x5f87af, 0x5f87d7, 0x5f87ff, 0x5faf00, 0x5faf5f,
+        0x5faf87, 0x5fafaf, 0x5fafd7, 0x5fafff, 0x5fd700, 0x5fd75f, 0x5fd787, 0x5fd7af,
+        0x5fd7d7, 0x5fd7ff, 0x5fff00, 0x5fff5f, 0x5fff87, 0x5fffaf, 0x5fffd7, 0x5fffff,
+        0x870000, 0x87005f, 0x870087, 0x8700af, 0x8700d7, 0x8700ff, 0x875f00, 0x875f5f,
+        0x875f87, 0x875faf, 0x875fd7, 0x875fff, 0x878700, 0x87875f, 0x878787, 0x8787af,
+        0x8787d7, 0x8787ff, 0x87af00, 0x87af5f, 0x87af87, 0x87afaf, 0x87afd7, 0x87afff,
+        0x87d700, 0x87d75f, 0x87d787, 0x87d7af, 0x87d7d7, 0x87d7ff, 0x87ff00, 0x87ff5f,
+        0x87ff87, 0x87ffaf, 0x87ffd7, 0x87ffff, 0xaf0000, 0xaf005f, 0xaf0087, 0xaf00af,
+        0xaf00d7, 0xaf00ff, 0xaf5f00, 0xaf5f5f, 0xaf5f87, 0xaf5faf, 0xaf5fd7, 0xaf5fff,
+        0xaf8700, 0xaf875f, 0xaf8787, 0xaf87af, 0xaf87d7, 0xaf87ff, 0xafaf00, 0xafaf5f,
+        0xafaf87, 0xafafaf, 0xafafd7, 0xafafff, 0xafd700, 0xafd75f, 0xafd787, 0xafd7af,
+        0xafd7d7, 0xafd7ff, 0xafff00, 0xafff5f, 0xafff87, 0xafffaf, 0xafffd7, 0xafffff,
+        0xd70000, 0xd7005f, 0xd70087, 0xd700af, 0xd700d7, 0xd700ff, 0xd75f00, 0xd75f5f,
+        0xd75f87, 0xd75faf, 0xd75fd7, 0xd75fff, 0xd78700, 0xd7875f, 0xd78787, 0xd787af,
+        0xd787d7, 0xd787ff, 0xd7af00, 0xd7af5f, 0xd7af87, 0xd7afaf, 0xd7afd7, 0xd7afff,
+        0xd7d700, 0xd7d75f, 0xd7d787, 0xd7d7af, 0xd7d7d7, 0xd7d7ff, 0xd7ff00, 0xd7ff5f,
+        0xd7ff87, 0xd7ffaf, 0xd7ffd7, 0xd7ffff, 0xff0000, 0xff005f, 0xff0087, 0xff00af,
+        0xff00d7, 0xff00ff, 0xff5f00, 0xff5f5f, 0xff5f87, 0xff5faf, 0xff5fd7, 0xff5fff,
+        0xff8700, 0xff875f, 0xff8787, 0xff87af, 0xff87d7, 0xff87ff, 0xffaf00, 0xffaf5f,
+        0xffaf87, 0xffafaf, 0xffafd7, 0xffafff, 0xffd700, 0xffd75f, 0xffd787, 0xffd7af,
+        0xffd7d7, 0xffd7ff, 0xffff00, 0xffff5f, 0xffff87, 0xffffaf, 0xffffd7, 0xffffff,
+        0x080808, 0x121212, 0x1c1c1c, 0x262626, 0x303030, 0x3a3a3a, 0x444444, 0x4e4e4e,
+        0x585858, 0x626262, 0x6c6c6c, 0x767676, 0x808080, 0x8a8a8a, 0x949494, 0x9e9e9e,
+        0xa8a8a8, 0xb2b2b2, 0xbcbcbc, 0xc6c6c6, 0xd0d0d0, 0xdadada, 0xe4e4e4, 0xeeeeee
+    };
+    return 16 + convert_color(rgb, kColors, sizeof kColors / sizeof *kColors);
+}
+
+unsigned char rgb_color_t::to_term256_index() const
+{
+    assert(type == type_rgb);
+    return term256_color_for_rgb(data.color.rgb);
+}
+
+color24_t rgb_color_t::to_color24() const
+{
+    assert(type == type_rgb);
+    return data.color;
+}
+
+unsigned char rgb_color_t::to_name_index() const
+{
+    assert(type == type_named || type == type_rgb);
+    if (type == type_named)
+    {
+        return data.name_idx;
+    }
+    else if (type == type_rgb)
+    {
+        return term8_color_for_rgb(data.color.rgb);
+    }
+    else
+    {
+        /* This is an error */
+        return (unsigned char)(-1);
+    }
+}
+
+void rgb_color_t::parse(const wcstring &str)
+{
+    bool success = false;
+    if (! success) success = try_parse_special(str);
+    if (! success) success = try_parse_named(str);
+    if (! success) success = try_parse_rgb(str);
+    if (! success)
+    {
+        memset(&this->data, 0, sizeof this->data);
+        this->type = type_none;
+    }
+}
+
+rgb_color_t::rgb_color_t(const wcstring &str) : type(), flags()
+{
+    this->parse(str);
+}
+
+rgb_color_t::rgb_color_t(const std::string &str) : type(), flags()
+{
+    this->parse(str2wcstring(str));
+}
+
+wcstring rgb_color_t::description() const
+{
+    switch (type)
+    {
+        case type_none:
+            return L"none";
+        case type_named:
+            return format_string(L"named(%d: %ls)", (int)data.name_idx, name_for_color_idx(data.name_idx));
+        case type_rgb:
+            return format_string(L"rgb(0x%02x%02x%02x)", data.color.rgb[0], data.color.rgb[1], data.color.rgb[2]);
+        case type_reset:
+            return L"reset";
+        case type_normal:
+            return L"normal";
+        case type_ignore:
+            return L"ignore";
+        default:
+            abort();
+            return L"";
+    }
+}
diff --git a/src/color.h b/src/color.h
new file mode 100644
index 00000000..988bfdb8
--- /dev/null
+++ b/src/color.h
@@ -0,0 +1,184 @@
+/** \file color.h Color class.
+  */
+#ifndef FISH_COLOR_H
+#define FISH_COLOR_H
+
+#include <string.h>
+#include <string>
+#include "common.h"
+
+/* 24 bit color */
+struct color24_t
+{
+    unsigned char rgb[3];
+};
+
+/* A type that represents a color. We work hard to keep it at a size of 4 bytes. */
+class rgb_color_t
+{
+
+    /* Types */
+    enum
+    {
+        type_none,
+        type_named,
+        type_rgb,
+        type_normal,
+        type_reset,
+        type_ignore
+    };
+    unsigned char type:4;
+
+    /* Flags */
+    enum
+    {
+        flag_bold = 1 << 0,
+        flag_underline = 1 << 1
+    };
+    unsigned char flags:4;
+
+    union
+    {
+        unsigned char name_idx; //0-10
+        color24_t color;
+    } data;
+
+    /** Try parsing a special color name like "normal" */
+    bool try_parse_special(const wcstring &str);
+
+    /** Try parsing an rgb color like "#F0A030" */
+    bool try_parse_rgb(const wcstring &str);
+
+    /** Try parsing an explicit color name like "magenta" */
+    bool try_parse_named(const wcstring &str);
+
+    /* Parsing entry point */
+    void parse(const wcstring &str);
+
+    /** Private constructor */
+    explicit rgb_color_t(unsigned char t, unsigned char i=0);
+
+public:
+
+    /** Default constructor of type none */
+    explicit rgb_color_t() : type(type_none), flags(), data() {}
+
+    /** Parse a color from a string */
+    explicit rgb_color_t(const wcstring &str);
+    explicit rgb_color_t(const std::string &str);
+
+    /** Returns white */
+    static rgb_color_t white();
+
+    /** Returns black */
+    static rgb_color_t black();
+
+    /** Returns the reset special color */
+    static rgb_color_t reset();
+
+    /** Returns the normal special color */
+    static rgb_color_t normal();
+
+    /** Returns the ignore special color */
+    static rgb_color_t ignore();
+
+    /** Returns the none special color */
+    static rgb_color_t none();
+
+    /** Returns whether the color is the ignore special color */
+    bool is_ignore(void) const
+    {
+        return type == type_ignore;
+    }
+
+    /** Returns whether the color is the normal special color */
+    bool is_normal(void) const
+    {
+        return type == type_normal;
+    }
+
+    /** Returns whether the color is the reset special color */
+    bool is_reset(void) const
+    {
+        return type == type_reset;
+    }
+
+    /** Returns whether the color is the none special color */
+    bool is_none(void) const
+    {
+        return type == type_none;
+    }
+
+    /** Returns whether the color is a named color (like "magenta") */
+    bool is_named(void) const
+    {
+        return type == type_named;
+    }
+
+    /** Returns whether the color is specified via RGB components */
+    bool is_rgb(void) const
+    {
+        return type == type_rgb;
+    }
+
+    /** Returns whether the color is special, that is, not rgb or named */
+    bool is_special(void) const
+    {
+        return type != type_named && type != type_rgb;
+    }
+
+    /** Returns a description of the color */
+    wcstring description() const;
+
+    /** Returns the name index for the given color. Requires that the color be named or RGB. */
+    unsigned char to_name_index() const;
+
+    /** Returns the term256 index for the given color. Requires that the color be RGB. */
+    unsigned char to_term256_index() const;
+
+    /** Returns the 24 bit color for the given color. Requires that the color be RGB. */
+    color24_t to_color24() const;
+
+    /** Returns whether the color is bold */
+    bool is_bold() const
+    {
+        return !!(flags & flag_bold);
+    }
+
+    /** Set whether the color is bold */
+    void set_bold(bool x)
+    {
+        if (x) flags |= flag_bold;
+        else flags &= ~flag_bold;
+    }
+
+    /** Returns whether the color is underlined */
+    bool is_underline() const
+    {
+        return !!(flags & flag_underline);
+    }
+
+    /** Set whether the color is underlined */
+    void set_underline(bool x)
+    {
+        if (x) flags |= flag_underline;
+        else flags &= ~flag_underline;
+    }
+
+    /** Compare two colors for equality */
+    bool operator==(const rgb_color_t &other) const
+    {
+        return type == other.type && ! memcmp(&data, &other.data, sizeof data);
+    }
+
+    /** Compare two colors for inequality */
+    bool operator!=(const rgb_color_t &other) const
+    {
+        return !(*this == other);
+    }
+
+    /** Returns the names of all named colors */
+    static wcstring_list_t named_color_names(void);
+};
+
+#endif
diff --git a/src/common.cpp b/src/common.cpp
new file mode 100644
index 00000000..ea1cc4e3
--- /dev/null
+++ b/src/common.cpp
@@ -0,0 +1,2405 @@
+/** \file common.c
+
+Various functions, mostly string utilities, that are used by most
+parts of fish.
+*/
+
+#include "config.h"
+
+
+#include <unistd.h>
+
+#ifdef HAVE_SIGINFO_H
+#include <siginfo.h>
+#endif
+
+#include <stdlib.h>
+#include <termios.h>
+#include <wchar.h>
+#include <string.h>
+#include <stdio.h>
+#include <sys/types.h>
+#include <assert.h>
+#include <math.h>
+#include <signal.h>
+
+
+#ifdef HAVE_SYS_IOCTL_H
+#include <sys/ioctl.h>
+#endif
+
+#include <sys/stat.h>
+#include <wctype.h>
+#include <errno.h>
+#include <limits.h>
+#include <stdarg.h>
+#include <locale.h>
+#include <sys/time.h>
+#include <algorithm>
+
+#ifdef HAVE_EXECINFO_H
+#include <execinfo.h>
+#endif
+
+#include "fallback.h"
+
+#include "wutil.h"
+#include "common.h"
+#include "expand.h"
+#include "wildcard.h"
+
+#include "util.cpp"
+#include "fallback.cpp"
+
+#define NOT_A_WCHAR (static_cast<wint_t>(WEOF))
+
+struct termios shell_modes;
+
+// Note we foolishly assume that pthread_t is just a primitive. But it might be a struct.
+static pthread_t main_thread_id = 0;
+static bool thread_assertions_configured_for_testing = false;
+
+wchar_t ellipsis_char;
+wchar_t omitted_newline_char;
+
+bool g_profiling_active = false;
+
+const wchar_t *program_name;
+
+int debug_level=1;
+
+/**
+   This struct maintains the current state of the terminal size. It is updated on demand after receiving a SIGWINCH.
+   Do not touch this struct directly, it's managed with a rwlock. Use common_get_width()/common_get_height().
+*/
+static struct winsize termsize;
+static volatile bool termsize_valid;
+static rwlock_t termsize_rwlock;
+
+static char *wcs2str_internal(const wchar_t *in, char *out);
+
+void show_stackframe()
+{
+    ASSERT_IS_NOT_FORKED_CHILD();
+
+    /* Hack to avoid showing backtraces in the tester */
+    if (program_name && ! wcscmp(program_name, L"(ignore)"))
+        return;
+
+    void *trace[32];
+    int trace_size = 0;
+
+    trace_size = backtrace(trace, 32);
+    char **messages = backtrace_symbols(trace, trace_size);
+
+    if (messages)
+    {
+        debug(0, L"Backtrace:");
+        for (int i=0; i<trace_size; i++)
+        {
+            fwprintf(stderr, L"%s\n", messages[i]);
+        }
+        free(messages);
+    }
+}
+
+int fgetws2(wcstring *s, FILE *f)
+{
+    int i=0;
+    wint_t c;
+
+    while (1)
+    {
+        errno=0;
+
+        c = getwc(f);
+
+        if (errno == EILSEQ || errno == EINTR)
+        {
+            continue;
+        }
+
+        switch (c)
+        {
+                /* End of line */
+            case WEOF:
+            case L'\n':
+            case L'\0':
+                return i;
+                /* Ignore carriage returns */
+            case L'\r':
+                break;
+
+            default:
+                i++;
+                s->push_back((wchar_t)c);
+                break;
+        }
+    }
+}
+
+/**
+   Converts the narrow character string \c in into its wide
+   equivalent, and return it
+
+   The string may contain embedded nulls.
+
+   This function encodes illegal character sequences in a reversible
+   way using the private use area.
+*/
+
+static wcstring str2wcs_internal(const char *in, const size_t in_len)
+{
+    if (in_len == 0)
+        return wcstring();
+
+    assert(in != NULL);
+
+    wcstring result;
+    result.reserve(in_len);
+    mbstate_t state = {};
+    size_t in_pos = 0;
+    while (in_pos < in_len)
+    {
+        wchar_t wc = 0;
+        size_t ret = mbrtowc(&wc, &in[in_pos], in_len-in_pos, &state);
+
+        /* Determine whether to encode this characters with our crazy scheme */
+        bool use_encode_direct = false;
+        if (wc >= ENCODE_DIRECT_BASE && wc < ENCODE_DIRECT_BASE+256)
+        {
+            use_encode_direct = true;
+        }
+        else if (wc == INTERNAL_SEPARATOR)
+        {
+            use_encode_direct = true;
+        }
+        else if (ret == (size_t)(-2))
+        {
+            /* Incomplete sequence */
+            use_encode_direct = true;
+        }
+        else if (ret == (size_t)(-1))
+        {
+            /* Invalid data */
+            use_encode_direct = true;
+        }
+        else if (ret > in_len - in_pos)
+        {
+            /* Other error codes? Terrifying, should never happen */
+            use_encode_direct = true;
+        }
+
+        if (use_encode_direct)
+        {
+            wc = ENCODE_DIRECT_BASE + (unsigned char)in[in_pos];
+            result.push_back(wc);
+            in_pos++;
+            memset(&state, 0, sizeof state);
+        }
+        else if (ret == 0)
+        {
+            /* Embedded null byte! */
+            result.push_back(L'\0');
+            in_pos++;
+            memset(&state, 0, sizeof state);
+        }
+        else
+        {
+            /* Normal case */
+            result.push_back(wc);
+            in_pos += ret;
+        }
+    }
+    return result;
+}
+
+wcstring str2wcstring(const char *in, size_t len)
+{
+    return str2wcs_internal(in, len);
+}
+
+wcstring str2wcstring(const char *in)
+{
+    return str2wcs_internal(in, strlen(in));
+}
+
+wcstring str2wcstring(const std::string &in)
+{
+    /* Handles embedded nulls! */
+    return str2wcs_internal(in.data(), in.size());
+}
+
+char *wcs2str(const wchar_t *in)
+{
+    if (! in)
+        return NULL;
+    size_t desired_size = MAX_UTF8_BYTES*wcslen(in)+1;
+    char local_buff[512];
+    if (desired_size <= sizeof local_buff / sizeof *local_buff)
+    {
+        // convert into local buff, then use strdup() so we don't waste malloc'd space
+        char *result = wcs2str_internal(in, local_buff);
+        if (result)
+        {
+            // It converted into the local buffer, so copy it
+            result = strdup(result);
+            if (! result)
+            {
+                DIE_MEM();
+            }
+        }
+        return result;
+
+    }
+    else
+    {
+        // here we fall into the bad case of allocating a buffer probably much larger than necessary
+        char *out = (char *)malloc(MAX_UTF8_BYTES*wcslen(in)+1);
+        if (!out)
+        {
+            DIE_MEM();
+        }
+        return wcs2str_internal(in, out);
+    }
+}
+
+char *wcs2str(const wcstring &in)
+{
+    return wcs2str(in.c_str());
+}
+
+/* This function is distinguished from wcs2str_internal in that it allows embedded null bytes */
+std::string wcs2string(const wcstring &input)
+{
+    std::string result;
+    result.reserve(input.size());
+
+    mbstate_t state;
+    memset(&state, 0, sizeof(state));
+
+    char converted[MB_LEN_MAX + 1];
+
+    for (size_t i=0; i < input.size(); i++)
+    {
+        wchar_t wc = input[i];
+        if (wc == INTERNAL_SEPARATOR)
+        {
+        }
+        else if ((wc >= ENCODE_DIRECT_BASE) &&
+                 (wc < ENCODE_DIRECT_BASE+256))
+        {
+            result.push_back(wc - ENCODE_DIRECT_BASE);
+        }
+        else
+        {
+            memset(converted, 0, sizeof converted);
+            size_t len = wcrtomb(converted, wc, &state);
+            if (len == (size_t)(-1))
+            {
+                debug(1, L"Wide character %d has no narrow representation", wc);
+                memset(&state, 0, sizeof(state));
+            }
+            else
+            {
+                result.append(converted, len);
+            }
+        }
+    }
+
+    return result;
+}
+
+/**
+   Converts the wide character string \c in into it's narrow
+   equivalent, stored in \c out. \c out must have enough space to fit
+   the entire string.
+
+   This function decodes illegal character sequences in a reversible
+   way using the private use area.
+*/
+static char *wcs2str_internal(const wchar_t *in, char *out)
+{
+    size_t res=0;
+    size_t in_pos=0;
+    size_t out_pos = 0;
+    mbstate_t state;
+
+    CHECK(in, 0);
+    CHECK(out, 0);
+
+    memset(&state, 0, sizeof(state));
+
+    while (in[in_pos])
+    {
+        if (in[in_pos] == INTERNAL_SEPARATOR)
+        {
+        }
+        else if ((in[in_pos] >= ENCODE_DIRECT_BASE) &&
+                 (in[in_pos] < ENCODE_DIRECT_BASE+256))
+        {
+            out[out_pos++] = in[in_pos]- ENCODE_DIRECT_BASE;
+        }
+        else
+        {
+            res = wcrtomb(&out[out_pos], in[in_pos], &state);
+
+            if (res == (size_t)(-1))
+            {
+                debug(1, L"Wide character %d has no narrow representation", in[in_pos]);
+                memset(&state, 0, sizeof(state));
+            }
+            else
+            {
+                out_pos += res;
+            }
+        }
+        in_pos++;
+    }
+    out[out_pos] = 0;
+
+    return out;
+}
+
+wcstring format_string(const wchar_t *format, ...)
+{
+    va_list va;
+    va_start(va, format);
+    wcstring result = vformat_string(format, va);
+    va_end(va);
+    return result;
+}
+
+void append_formatv(wcstring &target, const wchar_t *format, va_list va_orig)
+{
+    const int saved_err = errno;
+    /*
+      As far as I know, there is no way to check if a
+      vswprintf-call failed because of a badly formated string
+      option or because the supplied destination string was to
+      small. In GLIBC, errno seems to be set to EINVAL either way.
+
+      Because of this, on failiure we try to
+      increase the buffer size until the free space is
+      larger than max_size, at which point it will
+      conclude that the error was probably due to a badly
+      formated string option, and return an error. Make
+      sure to null terminate string before that, though.
+    */
+    const size_t max_size = (128*1024*1024);
+    wchar_t static_buff[256];
+    size_t size = 0;
+    wchar_t *buff = NULL;
+    int status = -1;
+    while (status < 0)
+    {
+        /* Reallocate if necessary */
+        if (size == 0)
+        {
+            buff = static_buff;
+            size = sizeof static_buff;
+        }
+        else
+        {
+            size *= 2;
+            if (size >= max_size)
+            {
+                buff[0] = '\0';
+                break;
+            }
+            buff = (wchar_t *)realloc((buff == static_buff ? NULL : buff), size);
+            if (buff == NULL)
+            {
+                DIE_MEM();
+            }
+        }
+
+        /* Try printing */
+        va_list va;
+        va_copy(va, va_orig);
+        status = vswprintf(buff, size / sizeof(wchar_t), format, va);
+        va_end(va);
+    }
+
+    target.append(buff);
+
+    if (buff != static_buff)
+    {
+        free(buff);
+    }
+
+    errno = saved_err;
+}
+
+wcstring vformat_string(const wchar_t *format, va_list va_orig)
+{
+    wcstring result;
+    append_formatv(result, format, va_orig);
+    return result;
+}
+
+void append_format(wcstring &str, const wchar_t *format, ...)
+{
+    va_list va;
+    va_start(va, format);
+    append_formatv(str, format, va);
+    va_end(va);
+}
+
+const wchar_t *wcsvarname(const wchar_t *str)
+{
+    while (*str)
+    {
+        if ((!iswalnum(*str)) && (*str != L'_'))
+        {
+            return str;
+        }
+        str++;
+    }
+    return NULL;
+}
+
+const wchar_t *wcsvarname(const wcstring &str)
+{
+    return wcsvarname(str.c_str());
+}
+
+const wchar_t *wcsfuncname(const wcstring &str)
+{
+    return wcschr(str.c_str(), L'/');
+}
+
+
+bool wcsvarchr(wchar_t chr)
+{
+    return iswalnum(chr) || chr == L'_';
+}
+
+int fish_wcswidth(const wchar_t *str)
+{
+    return fish_wcswidth(str, wcslen(str));
+}
+
+int fish_wcswidth(const wcstring& str)
+{
+    return fish_wcswidth(str.c_str(), str.size());
+}
+
+wchar_t *quote_end(const wchar_t *pos)
+{
+    wchar_t c = *pos;
+
+    while (1)
+    {
+        pos++;
+
+        if (!*pos)
+            return 0;
+
+        if (*pos == L'\\')
+        {
+            pos++;
+            if (!*pos)
+                return 0;
+        }
+        else
+        {
+            if (*pos == c)
+            {
+                return (wchar_t *)pos;
+            }
+        }
+    }
+    return 0;
+
+}
+
+
+wcstring wsetlocale(int category, const wchar_t *locale)
+{
+
+    char *lang = locale ? wcs2str(locale) : NULL;
+    char *res = setlocale(category, lang);
+    free(lang);
+
+    /*
+      Use ellipsis if on known unicode system, otherwise use $
+    */
+    char *ctype = setlocale(LC_CTYPE, NULL);
+    bool unicode = (strstr(ctype, ".UTF") || strstr(ctype, ".utf"));
+
+    ellipsis_char = unicode ? L'\x2026' : L'$';
+
+    // U+23CE is the "return" character
+    omitted_newline_char = unicode ? L'\x23CE' : L'~';
+
+    if (!res)
+        return wcstring();
+    else
+        return format_string(L"%s", res);
+}
+
+bool contains_internal(const wchar_t *a, int vararg_handle, ...)
+{
+    const wchar_t *arg;
+    va_list va;
+    bool res = false;
+
+    CHECK(a, 0);
+
+    va_start(va, vararg_handle);
+    while ((arg=va_arg(va, const wchar_t *))!= 0)
+    {
+        if (wcscmp(a,arg) == 0)
+        {
+            res = true;
+            break;
+        }
+
+    }
+    va_end(va);
+    return res;
+}
+
+/* wcstring variant of contains_internal. The first parameter is a wcstring, the rest are const wchar_t *. vararg_handle exists only to give us a POD-value to apss to va_start */
+__sentinel bool contains_internal(const wcstring &needle, int vararg_handle, ...)
+{
+    const wchar_t *arg;
+    va_list va;
+    int res = 0;
+
+    const wchar_t *needle_cstr = needle.c_str();
+    va_start(va, vararg_handle);
+    while ((arg=va_arg(va, const wchar_t *))!= 0)
+    {
+        /* libc++ has an unfortunate implementation of operator== that unconditonally wcslen's the wchar_t* parameter, so prefer wcscmp directly */
+        if (! wcscmp(needle_cstr, arg))
+        {
+            res=1;
+            break;
+        }
+
+    }
+    va_end(va);
+    return res;
+}
+
+long read_blocked(int fd, void *buf, size_t count)
+{
+    ssize_t res;
+    sigset_t chldset, oldset;
+
+    sigemptyset(&chldset);
+    sigaddset(&chldset, SIGCHLD);
+    VOMIT_ON_FAILURE(pthread_sigmask(SIG_BLOCK, &chldset, &oldset));
+    res = read(fd, buf, count);
+    VOMIT_ON_FAILURE(pthread_sigmask(SIG_SETMASK, &oldset, NULL));
+    return res;
+}
+
+ssize_t write_loop(int fd, const char *buff, size_t count)
+{
+    size_t out_cum=0;
+    while (out_cum < count)
+    {
+        ssize_t out = write(fd, &buff[out_cum], count - out_cum);
+        if (out < 0)
+        {
+            if (errno != EAGAIN && errno != EINTR)
+            {
+                return -1;
+            }
+        }
+        else
+        {
+            out_cum += (size_t)out;
+        }
+    }
+    return (ssize_t)out_cum;
+}
+
+ssize_t read_loop(int fd, void *buff, size_t count)
+{
+    ssize_t result;
+    do
+    {
+        result = read(fd, buff, count);
+    }
+    while (result < 0 && (errno == EAGAIN || errno == EINTR));
+    return result;
+}
+
+static bool should_debug(int level)
+{
+    if (level > debug_level)
+        return false;
+
+    /* Hack to not print error messages in the tests */
+    if (program_name && ! wcscmp(program_name, L"(ignore)"))
+        return false;
+
+    return true;
+}
+
+static void debug_shared(const wcstring &msg)
+{
+    const wcstring sb = wcstring(program_name) + L": " + msg;
+    wcstring sb2;
+    write_screen(sb, sb2);
+    fwprintf(stderr, L"%ls", sb2.c_str());
+}
+
+void debug(int level, const wchar_t *msg, ...)
+{
+    if (! should_debug(level))
+        return;
+    int errno_old = errno;
+    va_list va;
+    va_start(va, msg);
+    wcstring local_msg = vformat_string(msg, va);
+    va_end(va);
+    debug_shared(local_msg);
+    errno = errno_old;
+}
+
+void debug(int level, const char *msg, ...)
+{
+    if (! should_debug(level))
+        return;
+    int errno_old = errno;
+    char local_msg[512];
+    va_list va;
+    va_start(va, msg);
+    vsnprintf(local_msg, sizeof local_msg, msg, va);
+    va_end(va);
+    debug_shared(str2wcstring(local_msg));
+    errno = errno_old;
+}
+
+void print_stderr(const wcstring &str)
+{
+    fprintf(stderr, "%ls\n", str.c_str());
+}
+
+void read_ignore(int fd, void *buff, size_t count)
+{
+    size_t ignore __attribute__((unused));
+    ignore = read(fd, buff, count);
+}
+
+void write_ignore(int fd, const void *buff, size_t count)
+{
+    size_t ignore __attribute__((unused));
+    ignore = write(fd, buff, count);
+}
+
+
+void debug_safe(int level, const char *msg, const char *param1, const char *param2, const char *param3, const char *param4, const char *param5, const char *param6, const char *param7, const char *param8, const char *param9, const char *param10, const char *param11, const char *param12)
+{
+    const char * const params[] = {param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12};
+    if (! msg)
+        return;
+
+    /* Can't call printf, that may allocate memory Just call write() over and over. */
+    if (level > debug_level)
+        return;
+    int errno_old = errno;
+
+    size_t param_idx = 0;
+    const char *cursor = msg;
+    while (*cursor != '\0')
+    {
+        const char *end = strchr(cursor, '%');
+        if (end == NULL)
+            end = cursor + strlen(cursor);
+
+        write_ignore(STDERR_FILENO, cursor, end - cursor);
+
+        if (end[0] == '%' && end[1] == 's')
+        {
+            /* Handle a format string */
+            assert(param_idx < sizeof params / sizeof *params);
+            const char *format = params[param_idx++];
+            if (! format)
+                format = "(null)";
+            write_ignore(STDERR_FILENO, format, strlen(format));
+            cursor = end + 2;
+        }
+        else if (end[0] == '\0')
+        {
+            /* Must be at the end of the string */
+            cursor = end;
+        }
+        else
+        {
+            /* Some other format specifier, just skip it */
+            cursor = end + 1;
+        }
+    }
+
+    // We always append a newline
+    write_ignore(STDERR_FILENO, "\n", 1);
+
+    errno = errno_old;
+}
+
+void format_long_safe(char buff[64], long val)
+{
+    if (val == 0)
+    {
+        strcpy(buff, "0");
+    }
+    else
+    {
+        /* Generate the string in reverse */
+        size_t idx = 0;
+        bool negative = (val < 0);
+
+        /* Note that we can't just negate val if it's negative, because it may be the most negative value. We do rely on round-towards-zero division though. */
+
+        while (val != 0)
+        {
+            long rem = val % 10;
+            buff[idx++] = '0' + (rem < 0 ? -rem : rem);
+            val /= 10;
+        }
+        if (negative)
+            buff[idx++] = '-';
+        buff[idx] = 0;
+
+        size_t left = 0, right = idx - 1;
+        while (left < right)
+        {
+            char tmp = buff[left];
+            buff[left++] = buff[right];
+            buff[right--] = tmp;
+        }
+    }
+}
+
+void format_long_safe(wchar_t buff[64], long val)
+{
+    if (val == 0)
+    {
+        wcscpy(buff, L"0");
+    }
+    else
+    {
+        /* Generate the string in reverse */
+        size_t idx = 0;
+        bool negative = (val < 0);
+
+        while (val != 0)
+        {
+            long rem = val % 10;
+            buff[idx++] = L'0' + (wchar_t)(rem < 0 ? -rem : rem);
+            val /= 10;
+        }
+        if (negative)
+            buff[idx++] = L'-';
+        buff[idx] = 0;
+
+        size_t left = 0, right = idx - 1;
+        while (left < right)
+        {
+            wchar_t tmp = buff[left];
+            buff[left++] = buff[right];
+            buff[right--] = tmp;
+        }
+    }
+}
+
+void write_screen(const wcstring &msg, wcstring &buff)
+{
+    int line_width = 0;
+    int screen_width = common_get_width();
+
+    if (screen_width)
+    {
+        const wchar_t *start = msg.c_str();
+        const wchar_t *pos = start;
+        while (1)
+        {
+            int overflow = 0;
+
+            int tok_width=0;
+
+            /*
+              Tokenize on whitespace, and also calculate the width of the token
+            */
+            while (*pos && (!wcschr(L" \n\r\t", *pos)))
+            {
+
+                /*
+                  Check is token is wider than one line.
+                  If so we mark it as an overflow and break the token.
+                */
+                if ((tok_width + fish_wcwidth(*pos)) > (screen_width-1))
+                {
+                    overflow = 1;
+                    break;
+                }
+
+                tok_width += fish_wcwidth(*pos);
+                pos++;
+            }
+
+            /*
+              If token is zero character long, we don't do anything
+            */
+            if (pos == start)
+            {
+                start = pos = pos+1;
+            }
+            else if (overflow)
+            {
+                /*
+                  In case of overflow, we print a newline, except if we already are at position 0
+                */
+                wchar_t *token = wcsndup(start, pos-start);
+                if (line_width != 0)
+                    buff.push_back(L'\n');
+                buff.append(format_string(L"%ls-\n", token));
+                free(token);
+                line_width=0;
+            }
+            else
+            {
+                /*
+                  Print the token
+                */
+                wchar_t *token = wcsndup(start, pos-start);
+                if ((line_width + (line_width!=0?1:0) + tok_width) > screen_width)
+                {
+                    buff.push_back(L'\n');
+                    line_width=0;
+                }
+                buff.append(format_string(L"%ls%ls", line_width?L" ":L"", token));
+                free(token);
+                line_width += (line_width!=0?1:0) + tok_width;
+            }
+
+            /*
+              Break on end of string
+            */
+            if (!*pos)
+            {
+                break;
+            }
+
+            start=pos;
+        }
+    }
+    else
+    {
+        buff.append(msg);
+    }
+    buff.push_back(L'\n');
+}
+
+/* Escape a string, storing the result in out_str */
+static void escape_string_internal(const wchar_t *orig_in, size_t in_len, wcstring *out_str, escape_flags_t flags)
+{
+    assert(orig_in != NULL);
+
+    const wchar_t *in = orig_in;
+    bool escape_all = !!(flags & ESCAPE_ALL);
+    bool no_quoted  = !!(flags & ESCAPE_NO_QUOTED);
+    bool no_tilde = !!(flags & ESCAPE_NO_TILDE);
+
+    int need_escape=0;
+    int need_complex_escape=0;
+
+    /* Avoid dereferencing all over the place */
+    wcstring &out = *out_str;
+
+    if (!no_quoted && in_len == 0)
+    {
+        out.assign(L"''");
+        return;
+    }
+
+    while (*in != 0)
+    {
+
+        if ((*in >= ENCODE_DIRECT_BASE) &&
+                (*in < ENCODE_DIRECT_BASE+256))
+        {
+            int val = *in - ENCODE_DIRECT_BASE;
+            int tmp;
+
+            out += L'\\';
+            out += L'X';
+
+            tmp = val/16;
+            out += tmp > 9? L'a'+(tmp-10):L'0'+tmp;
+
+            tmp = val%16;
+            out += tmp > 9? L'a'+(tmp-10):L'0'+tmp;
+            need_escape=need_complex_escape=1;
+
+        }
+        else
+        {
+            wchar_t c = *in;
+            switch (c)
+            {
+                case L'\t':
+                    out += L'\\';
+                    out += L't';
+                    need_escape=need_complex_escape=1;
+                    break;
+
+                case L'\n':
+                    out += L'\\';
+                    out += L'n';
+                    need_escape=need_complex_escape=1;
+                    break;
+
+                case L'\b':
+                    out += L'\\';
+                    out += L'b';
+                    need_escape=need_complex_escape=1;
+                    break;
+
+                case L'\r':
+                    out += L'\\';
+                    out += L'r';
+                    need_escape=need_complex_escape=1;
+                    break;
+
+                case L'\x1b':
+                    out += L'\\';
+                    out += L'e';
+                    need_escape=need_complex_escape=1;
+                    break;
+
+
+                case L'\\':
+                case L'\'':
+                {
+                    need_escape=need_complex_escape=1;
+                    if (escape_all)
+                        out += L'\\';
+                    out += *in;
+                    break;
+                }
+                
+                
+                // Experimental fix for #1614
+                // The hope is that any time these appear in a string, they came from wildcard expansion
+                case ANY_CHAR:
+                    out += L'?';
+                    break;
+
+                case ANY_STRING:
+                    out += L'*';
+                    break;
+                    
+                case ANY_STRING_RECURSIVE:
+                    out += L"**";
+                    break;
+                    
+                case L'&':
+                case L'$':
+                case L' ':
+                case L'#':
+                case L'^':
+                case L'<':
+                case L'>':
+                case L'(':
+                case L')':
+                case L'[':
+                case L']':
+                case L'{':
+                case L'}':
+                case L'?':
+                case L'*':
+                case L'|':
+                case L';':
+                case L'"':
+                case L'%':
+                case L'~':
+                {
+                    if (! no_tilde || c != L'~')
+                    {
+                        need_escape=1;
+                        if (escape_all)
+                            out += L'\\';
+                    }
+                    out += *in;
+                    break;
+                }
+
+                default:
+                {
+                    if (*in < 32)
+                    {
+                        if (*in <27 && *in > 0)
+                        {
+                            out += L'\\';
+                            out += L'c';
+                            out += L'a' + *in -1;
+
+                            need_escape=need_complex_escape=1;
+                            break;
+
+                        }
+
+
+                        int tmp = (*in)%16;
+                        out += L'\\';
+                        out += L'x';
+                        out += ((*in>15)? L'1' : L'0');
+                        out += tmp > 9? L'a'+(tmp-10):L'0'+tmp;
+                        need_escape=need_complex_escape=1;
+                    }
+                    else
+                    {
+                        out += *in;
+                    }
+                    break;
+                }
+            }
+        }
+
+        in++;
+    }
+
+    /*
+      Use quoted escaping if possible, since most people find it
+      easier to read.
+     */
+    if (!no_quoted && need_escape && !need_complex_escape && escape_all)
+    {
+        wchar_t single_quote = L'\'';
+        out.clear();
+        out.reserve(2 + in_len);
+        out.push_back(single_quote);
+        out.append(orig_in, in_len);
+        out.push_back(single_quote);
+    }
+}
+
+wcstring escape(const wchar_t *in, escape_flags_t flags)
+{
+    if (!in)
+    {
+        debug(0, L"%s called with null input", __func__);
+        FATAL_EXIT();
+    }
+
+    wcstring result;
+    escape_string_internal(in, wcslen(in), &result, flags);
+    return result;
+}
+
+wcstring escape_string(const wcstring &in, escape_flags_t flags)
+{
+    wcstring result;
+    escape_string_internal(in.c_str(), in.size(), &result, flags);
+    return result;
+}
+
+/* Helper to return the last character in a string, or NOT_A_WCHAR */
+static wint_t string_last_char(const wcstring &str)
+{
+    size_t len = str.size();
+    return len == 0 ? NOT_A_WCHAR : str.at(len - 1);
+}
+
+/* Given a null terminated string starting with a backslash, read the escape as if it is unquoted, appending to result. Return the number of characters consumed, or 0 on error */
+static size_t read_unquoted_escape(const wchar_t *input, wcstring *result, bool allow_incomplete, bool unescape_special)
+{
+    if (input[0] != L'\\')
+    {
+        // not an escape
+        return 0;
+    }
+
+    /* Here's the character we'll ultimately append, or NOT_A_WCHAR for none. Note that L'\0' is a valid thing to append. */
+    wint_t result_char_or_none = NOT_A_WCHAR;
+
+    bool errored = false;
+    size_t in_pos = 1; //in_pos always tracks the next character to read (and therefore the number of characters read so far)
+    const wchar_t c = input[in_pos++];
+    switch (c)
+    {
+
+            /* A null character after a backslash is an error */
+        case L'\0':
+        {
+            /* Adjust in_pos to only include the backslash */
+            assert(in_pos > 0);
+            in_pos--;
+
+            /* It's an error, unless we're allowing incomplete escapes */
+            if (! allow_incomplete)
+                errored = true;
+            break;
+        }
+
+        /* Numeric escape sequences. No prefix means octal escape, otherwise hexadecimal. */
+        case L'0':
+        case L'1':
+        case L'2':
+        case L'3':
+        case L'4':
+        case L'5':
+        case L'6':
+        case L'7':
+        case L'u':
+        case L'U':
+        case L'x':
+        case L'X':
+        {
+            long long res=0;
+            size_t chars=2;
+            int base=16;
+
+            bool byte_literal = false;
+            wchar_t max_val = ASCII_MAX;
+
+            switch (c)
+            {
+                case L'u':
+                {
+                    chars=4;
+                    max_val = UCS2_MAX;
+                    break;
+                }
+
+                case L'U':
+                {
+                    chars=8;
+                    max_val = WCHAR_MAX;
+
+                    // Don't exceed the largest Unicode code point - see #1107
+                    if (0x10FFFF < max_val)
+                        max_val = (wchar_t)0x10FFFF;
+
+                    break;
+                }
+
+                case L'x':
+                {
+                    chars = 2;
+                    max_val = ASCII_MAX;
+                    break;
+                }
+
+                case L'X':
+                {
+                    byte_literal = true;
+                    max_val = BYTE_MAX;
+                    break;
+                }
+
+                default:
+                {
+                    base=8;
+                    chars=3;
+                    // note that in_pos currently is just after the first post-backslash character; we want to start our escape from there
+                    assert(in_pos > 0);
+                    in_pos--;
+                    break;
+                }
+            }
+
+            for (size_t i=0; i<chars; i++)
+            {
+                long d = convert_digit(input[in_pos],base);
+                if (d < 0)
+                {
+                    break;
+                }
+
+                res=(res*base)+d;
+                in_pos++;
+            }
+
+            if (res <= max_val)
+            {
+                result_char_or_none = (wchar_t)((byte_literal ? ENCODE_DIRECT_BASE : 0)+res);
+            }
+            else
+            {
+                errored = true;
+            }
+
+            break;
+        }
+
+        /* \a means bell (alert) */
+        case L'a':
+        {
+            result_char_or_none = L'\a';
+            break;
+        }
+
+        /* \b means backspace */
+        case L'b':
+        {
+            result_char_or_none = L'\b';
+            break;
+        }
+
+        /* \cX means control sequence X */
+        case L'c':
+        {
+            const wchar_t sequence_char = input[in_pos++];
+            if (sequence_char >= L'a' && sequence_char <= (L'a'+32))
+            {
+                result_char_or_none = sequence_char-L'a'+1;
+            }
+            else if (sequence_char >= L'A' && sequence_char <= (L'A'+32))
+            {
+                result_char_or_none = sequence_char-L'A'+1;
+            }
+            else
+            {
+                errored = true;
+            }
+            break;
+        }
+
+        /* \x1b means escape */
+        case L'e':
+        {
+            result_char_or_none = L'\x1b';
+            break;
+        }
+
+        /*
+          \f means form feed
+        */
+        case L'f':
+        {
+            result_char_or_none = L'\f';
+            break;
+        }
+
+        /*
+          \n means newline
+        */
+        case L'n':
+        {
+            result_char_or_none = L'\n';
+            break;
+        }
+
+        /*
+          \r means carriage return
+        */
+        case L'r':
+        {
+            result_char_or_none = L'\r';
+            break;
+        }
+
+        /*
+          \t means tab
+         */
+        case L't':
+        {
+            result_char_or_none = L'\t';
+            break;
+        }
+
+        /*
+          \v means vertical tab
+        */
+        case L'v':
+        {
+            result_char_or_none = L'\v';
+            break;
+        }
+
+        /* If a backslash is followed by an actual newline, swallow them both */
+        case L'\n':
+        {
+            result_char_or_none = NOT_A_WCHAR;
+            break;
+        }
+
+        default:
+        {
+            if (unescape_special)
+                result->push_back(INTERNAL_SEPARATOR);
+            result_char_or_none = c;
+            break;
+        }
+    }
+
+    if (! errored && result_char_or_none != NOT_A_WCHAR)
+    {
+        wchar_t result_char = static_cast<wchar_t>(result_char_or_none);
+        // if result_char is not NOT_A_WCHAR, it must be a valid wchar
+        assert((wint_t)result_char == result_char_or_none);
+        result->push_back(result_char);
+    }
+    return errored ? 0 : in_pos;
+}
+
+/* Returns the unescaped version of input_str into output_str (by reference). Returns true if successful. If false, the contents of output_str are undefined (!) */
+static bool unescape_string_internal(const wchar_t * const input, const size_t input_len, wcstring *output_str, unescape_flags_t flags)
+{
+    /* Set up result string, which we'll swap with the output on success */
+    wcstring result;
+    result.reserve(input_len);
+
+    const bool unescape_special = !!(flags & UNESCAPE_SPECIAL);
+    const bool allow_incomplete = !!(flags & UNESCAPE_INCOMPLETE);
+
+    int bracket_count = 0;
+
+    bool errored = false;
+    enum
+    {
+        mode_unquoted,
+        mode_single_quotes,
+        mode_double_quotes
+    } mode = mode_unquoted;
+
+    for (size_t input_position = 0; input_position < input_len && ! errored; input_position++)
+    {
+        const wchar_t c = input[input_position];
+        /* Here's the character we'll append to result, or NOT_A_WCHAR to suppress it */
+        wint_t to_append_or_none = c;
+        if (mode == mode_unquoted)
+        {
+
+            switch (c)
+            {
+                case L'\\':
+                {
+                    /* Backslashes (escapes) are complicated and may result in errors, or appending INTERNAL_SEPARATORs, so we have to handle them specially */
+                    size_t escape_chars = read_unquoted_escape(input + input_position, &result, allow_incomplete, unescape_special);
+                    if (escape_chars == 0)
+                    {
+                        /* A 0 return indicates an error */
+                        errored = true;
+                    }
+                    else
+                    {
+                        /* Skip over the characters we read, minus one because the outer loop will increment it */
+                        assert(escape_chars > 0);
+                        input_position += escape_chars - 1;
+                    }
+                    /* We've already appended, don't append anything else */
+                    to_append_or_none = NOT_A_WCHAR;
+                    break;
+                }
+
+                case L'~':
+                {
+                    if (unescape_special && (input_position == 0))
+                    {
+                        to_append_or_none = HOME_DIRECTORY;
+                    }
+                    break;
+                }
+
+                case L'%':
+                {
+                    if (unescape_special && (input_position == 0))
+                    {
+                        to_append_or_none = PROCESS_EXPAND;
+                    }
+                    break;
+                }
+
+                case L'*':
+                {
+                    if (unescape_special)
+                    {
+                        /* In general, this is ANY_STRING. But as a hack, if the last appended char is ANY_STRING, delete the last char and store ANY_STRING_RECURSIVE to reflect the fact that ** is the recursive wildcard. */
+                        if (string_last_char(result) == ANY_STRING)
+                        {
+                            assert(result.size() > 0);
+                            result.resize(result.size() - 1);
+                            to_append_or_none = ANY_STRING_RECURSIVE;
+                        }
+                        else
+                        {
+                            to_append_or_none = ANY_STRING;
+                        }
+                    }
+                    break;
+                }
+
+                case L'?':
+                {
+                    if (unescape_special)
+                    {
+                        to_append_or_none = ANY_CHAR;
+                    }
+                    break;
+                }
+
+                case L'$':
+                {
+                    if (unescape_special)
+                    {
+                        to_append_or_none = VARIABLE_EXPAND;
+                    }
+                    break;
+                }
+
+                case L'{':
+                {
+                    if (unescape_special)
+                    {
+                        bracket_count++;
+                        to_append_or_none = BRACKET_BEGIN;
+                    }
+                    break;
+                }
+
+                case L'}':
+                {
+                    if (unescape_special)
+                    {
+                        bracket_count--;
+                        to_append_or_none = BRACKET_END;
+                    }
+                    break;
+                }
+
+                case L',':
+                {
+                    /* If the last character was a separator, then treat this as a literal comma */
+                    if (unescape_special && bracket_count > 0 && string_last_char(result) != BRACKET_SEP)
+                    {
+                        to_append_or_none = BRACKET_SEP;
+                    }
+                    break;
+                }
+
+                case L'\'':
+                {
+                    mode = mode_single_quotes;
+                    to_append_or_none = unescape_special ? INTERNAL_SEPARATOR : NOT_A_WCHAR;
+                    break;
+                }
+
+                case L'\"':
+                {
+                    mode = mode_double_quotes;
+                    to_append_or_none = unescape_special ? INTERNAL_SEPARATOR : NOT_A_WCHAR;
+                    break;
+                }
+            }
+        }
+        else if (mode == mode_single_quotes)
+        {
+            if (c == L'\\')
+            {
+                /* A backslash may or may not escape something in single quotes */
+                switch (input[input_position + 1])
+                {
+                    case '\\':
+                    case L'\'':
+                    {
+                        to_append_or_none = input[input_position + 1];
+                        input_position += 1; /* Skip over the backslash */
+                        break;
+                    }
+
+                    case L'\0':
+                    {
+                        if (!allow_incomplete)
+                        {
+                            errored = true;
+                        }
+                        else
+                        {
+                            // PCA this line had the following cryptic comment:
+                            // 'We may ever escape a NULL character, but still appending a \ in case I am wrong.'
+                            // Not sure what it means or the importance of this
+                            input_position += 1; /* Skip over the backslash */
+                            to_append_or_none = L'\\';
+                        }
+                    }
+                    break;
+
+                    default:
+                    {
+                        /* Literal backslash that doesn't escape anything! Leave things alone; we'll append the backslash itself */
+                        break;
+                    }
+                }
+            }
+            else if (c == L'\'')
+            {
+                to_append_or_none = unescape_special ? INTERNAL_SEPARATOR : NOT_A_WCHAR;
+                mode = mode_unquoted;
+            }
+        }
+        else if (mode == mode_double_quotes)
+        {
+            switch (c)
+            {
+                case L'"':
+                {
+                    mode = mode_unquoted;
+                    to_append_or_none = unescape_special ? INTERNAL_SEPARATOR : NOT_A_WCHAR;
+                    break;
+                }
+
+                case '\\':
+                {
+                    switch (input[input_position + 1])
+                    {
+                        case L'\0':
+                        {
+                            if (!allow_incomplete)
+                            {
+                                errored = true;
+                            }
+                            else
+                            {
+                                to_append_or_none = L'\0';
+                            }
+                        }
+                        break;
+
+                        case '\\':
+                        case L'$':
+                        case '"':
+                        {
+                            to_append_or_none = input[input_position + 1];
+                            input_position += 1; /* Skip over the backslash */
+                            break;
+                        }
+
+                        case '\n':
+                        {
+                            /* Swallow newline */
+                            to_append_or_none = NOT_A_WCHAR;
+                            input_position += 1; /* Skip over the backslash */
+                            break;
+                        }
+
+                        default:
+                        {
+                            /* Literal backslash that doesn't escape anything! Leave things alone; we'll append the backslash itself */
+                            break;
+                        }
+                    }
+                    break;
+                }
+
+                case '$':
+                {
+                    if (unescape_special)
+                    {
+                        to_append_or_none = VARIABLE_EXPAND_SINGLE;
+                    }
+                    break;
+                }
+
+            }
+        }
+
+        /* Now maybe append the char */
+        if (to_append_or_none != NOT_A_WCHAR)
+        {
+            wchar_t to_append_char = static_cast<wchar_t>(to_append_or_none);
+            // if result_char is not NOT_A_WCHAR, it must be a valid wchar
+            assert((wint_t)to_append_char == to_append_or_none);
+            result.push_back(to_append_char);
+        }
+    }
+
+    /* Return the string by reference, and then success */
+    if (! errored)
+    {
+        output_str->swap(result);
+    }
+    return ! errored;
+}
+
+bool unescape_string_in_place(wcstring *str, unescape_flags_t escape_special)
+{
+    assert(str != NULL);
+    wcstring output;
+    bool success = unescape_string_internal(str->c_str(), str->size(), &output, escape_special);
+    if (success)
+    {
+        str->swap(output);
+    }
+    return success;
+}
+
+bool unescape_string(const wchar_t *input, wcstring *output, unescape_flags_t escape_special)
+{
+    bool success = unescape_string_internal(input, wcslen(input), output, escape_special);
+    if (! success)
+        output->clear();
+    return success;
+}
+
+bool unescape_string(const wcstring &input, wcstring *output, unescape_flags_t escape_special)
+{
+    bool success = unescape_string_internal(input.c_str(), input.size(), output, escape_special);
+    if (! success)
+        output->clear();
+    return success;
+}
+
+
+void common_handle_winch(int signal)
+{
+    /* don't run ioctl() here, it's not safe to use in signals */
+    termsize_valid = false;
+}
+
+/* updates termsize as needed, and returns a copy of the winsize. */
+static struct winsize get_current_winsize()
+{
+#ifndef HAVE_WINSIZE
+    struct winsize retval = {0};
+    retval.ws_col = 80;
+    retval.ws_row = 24;
+    return retval;
+#endif
+    scoped_rwlock guard(termsize_rwlock, true);
+    struct winsize retval = termsize;
+    if (!termsize_valid)
+    {
+        struct winsize size;
+        if (ioctl(1,TIOCGWINSZ,&size) == 0)
+        {
+            retval = size;
+            guard.upgrade();
+            termsize = retval;
+        }
+        termsize_valid = true;
+    }
+    return retval;
+}
+
+int common_get_width()
+{
+    return get_current_winsize().ws_col;
+}
+
+
+int common_get_height()
+{
+    return get_current_winsize().ws_row;
+}
+
+void tokenize_variable_array(const wcstring &val, std::vector<wcstring> &out)
+{
+    size_t pos = 0, end = val.size();
+    while (pos <= end)
+    {
+        size_t next_pos = val.find(ARRAY_SEP, pos);
+        if (next_pos == wcstring::npos)
+        {
+            next_pos = end;
+        }
+        out.resize(out.size() + 1);
+        out.back().assign(val, pos, next_pos - pos);
+        pos = next_pos + 1; //skip the separator, or skip past the end
+    }
+}
+
+bool string_prefixes_string(const wchar_t *proposed_prefix, const wcstring &value)
+{
+    size_t prefix_size = wcslen(proposed_prefix);
+    return prefix_size <= value.size() && value.compare(0, prefix_size, proposed_prefix) == 0;
+}
+
+bool string_prefixes_string(const wcstring &proposed_prefix, const wcstring &value)
+{
+    size_t prefix_size = proposed_prefix.size();
+    return prefix_size <= value.size() && value.compare(0, prefix_size, proposed_prefix) == 0;
+}
+
+bool string_prefixes_string_case_insensitive(const wcstring &proposed_prefix, const wcstring &value)
+{
+    size_t prefix_size = proposed_prefix.size();
+    return prefix_size <= value.size() && wcsncasecmp(proposed_prefix.c_str(), value.c_str(), prefix_size) == 0;
+}
+
+bool string_suffixes_string(const wcstring &proposed_suffix, const wcstring &value)
+{
+    size_t suffix_size = proposed_suffix.size();
+    return suffix_size <= value.size() && value.compare(value.size() - suffix_size, suffix_size, proposed_suffix) == 0;
+}
+
+bool string_suffixes_string(const wchar_t *proposed_suffix, const wcstring &value)
+{
+    size_t suffix_size = wcslen(proposed_suffix);
+    return suffix_size <= value.size() && value.compare(value.size() - suffix_size, suffix_size, proposed_suffix) == 0;
+}
+
+// Returns true if seq, represented as a subsequence, is contained within string
+static bool subsequence_in_string(const wcstring &seq, const wcstring &str)
+{
+    /* Impossible if seq is larger than string */
+    if (seq.size() > str.size())
+    {
+        return false;
+    }
+
+    /* Empty strings are considered to be subsequences of everything */
+    if (seq.empty())
+    {
+        return true;
+    }
+
+    size_t str_idx, seq_idx;
+    for (seq_idx = str_idx = 0; seq_idx < seq.size() && str_idx < str.size(); seq_idx++)
+    {
+        wchar_t c = seq.at(seq_idx);
+        size_t char_loc = str.find(c, str_idx);
+        if (char_loc == wcstring::npos)
+        {
+            /* Didn't find this character */
+            break;
+        }
+        else
+        {
+            /* We found it. Continue the search just after it. */
+            str_idx = char_loc + 1;
+        }
+    }
+
+    /* We succeeded if we exhausted our sequence */
+    assert(seq_idx <= seq.size());
+    return seq_idx == seq.size();
+}
+
+string_fuzzy_match_t::string_fuzzy_match_t(enum fuzzy_match_type_t t, size_t distance_first, size_t distance_second) :
+    type(t),
+    match_distance_first(distance_first),
+    match_distance_second(distance_second)
+{
+}
+
+
+string_fuzzy_match_t string_fuzzy_match_string(const wcstring &string, const wcstring &match_against, fuzzy_match_type_t limit_type)
+{
+    // Distances are generally the amount of text not matched
+    string_fuzzy_match_t result(fuzzy_match_none, 0, 0);
+    size_t location;
+    if (limit_type >= fuzzy_match_exact && string == match_against)
+    {
+        result.type = fuzzy_match_exact;
+    }
+    else if (limit_type >= fuzzy_match_prefix && string_prefixes_string(string, match_against))
+    {
+        result.type = fuzzy_match_prefix;
+        assert(match_against.size() >= string.size());
+        result.match_distance_first = match_against.size() - string.size();
+    }
+    else if (limit_type >= fuzzy_match_case_insensitive && wcscasecmp(string.c_str(), match_against.c_str()) == 0)
+    {
+        result.type = fuzzy_match_case_insensitive;
+    }
+    else if (limit_type >= fuzzy_match_prefix_case_insensitive && string_prefixes_string_case_insensitive(string, match_against))
+    {
+        result.type = fuzzy_match_prefix_case_insensitive;
+        assert(match_against.size() >= string.size());
+        result.match_distance_first = match_against.size() - string.size();
+    }
+    else if (limit_type >= fuzzy_match_substring && (location = match_against.find(string)) != wcstring::npos)
+    {
+        // string is contained within match against
+        result.type = fuzzy_match_substring;
+        assert(match_against.size() >= string.size());
+        result.match_distance_first = match_against.size() - string.size();
+        result.match_distance_second = location; //prefer earlier matches
+    }
+    else if (limit_type >= fuzzy_match_subsequence_insertions_only && subsequence_in_string(string, match_against))
+    {
+        result.type = fuzzy_match_subsequence_insertions_only;
+        assert(match_against.size() >= string.size());
+        result.match_distance_first = match_against.size() - string.size();
+        // it would be nice to prefer matches with greater matching runs here
+    }
+    return result;
+}
+
+template<typename T>
+static inline int compare_ints(T a, T b)
+{
+    if (a < b) return -1;
+    if (a == b) return 0;
+    return 1;
+}
+
+// Compare types; if the types match, compare distances
+int string_fuzzy_match_t::compare(const string_fuzzy_match_t &rhs) const
+{
+    if (this->type != rhs.type)
+    {
+        return compare_ints(this->type, rhs.type);
+    }
+    else if (this->match_distance_first != rhs.match_distance_first)
+    {
+        return compare_ints(this->match_distance_first, rhs.match_distance_first);
+    }
+    else if (this->match_distance_second != rhs.match_distance_second)
+    {
+        return compare_ints(this->match_distance_second, rhs.match_distance_second);
+    }
+    return 0; //equal
+}
+
+bool list_contains_string(const wcstring_list_t &list, const wcstring &str)
+{
+    return std::find(list.begin(), list.end(), str) != list.end();
+}
+
+int create_directory(const wcstring &d)
+{
+    int ok = 0;
+    struct stat buf;
+    int stat_res = 0;
+
+    while ((stat_res = wstat(d, &buf)) != 0)
+    {
+        if (errno != EAGAIN)
+            break;
+    }
+
+    if (stat_res == 0)
+    {
+        if (S_ISDIR(buf.st_mode))
+        {
+            ok = 1;
+        }
+    }
+    else
+    {
+        if (errno == ENOENT)
+        {
+            wcstring dir = wdirname(d);
+            if (!create_directory(dir))
+            {
+                if (!wmkdir(d, 0700))
+                {
+                    ok = 1;
+                }
+            }
+        }
+    }
+
+    return ok?0:-1;
+}
+
+__attribute__((noinline))
+void bugreport()
+{
+    debug(1,
+          _(L"This is a bug. Break on bugreport to debug."
+            L"If you can reproduce it, please send a bug report to %s."),
+          PACKAGE_BUGREPORT);
+}
+
+wcstring format_size(long long sz)
+{
+    wcstring result;
+    const wchar_t *sz_name[]=
+    {
+        L"kB", L"MB", L"GB", L"TB", L"PB", L"EB", L"ZB", L"YB", 0
+    };
+
+    if (sz < 0)
+    {
+        result.append(L"unknown");
+    }
+    else if (sz < 1)
+    {
+        result.append(_(L"empty"));
+    }
+    else if (sz < 1024)
+    {
+        result.append(format_string(L"%lldB", sz));
+    }
+    else
+    {
+        int i;
+
+        for (i=0; sz_name[i]; i++)
+        {
+            if (sz < (1024*1024) || !sz_name[i+1])
+            {
+                long isz = ((long)sz)/1024;
+                if (isz > 9)
+                    result.append(format_string(L"%d%ls", isz, sz_name[i]));
+                else
+                    result.append(format_string(L"%.1f%ls", (double)sz/1024, sz_name[i]));
+                break;
+            }
+            sz /= 1024;
+
+        }
+    }
+    return result;
+}
+
+/* Crappy function to extract the most significant digit of an unsigned long long value */
+static char extract_most_significant_digit(unsigned long long *xp)
+{
+    unsigned long long place_value = 1;
+    unsigned long long x = *xp;
+    while (x >= 10)
+    {
+        x /= 10;
+        place_value *= 10;
+    }
+    *xp -= (place_value * x);
+    return x + '0';
+}
+
+void append_ull(char *buff, unsigned long long val, size_t *inout_idx, size_t max_len)
+{
+    size_t idx = *inout_idx;
+    while (val > 0 && idx < max_len)
+        buff[idx++] = extract_most_significant_digit(&val);
+    *inout_idx = idx;
+}
+
+void append_str(char *buff, const char *str, size_t *inout_idx, size_t max_len)
+{
+    size_t idx = *inout_idx;
+    while (*str && idx < max_len)
+        buff[idx++] = *str++;
+    *inout_idx = idx;
+}
+
+void format_size_safe(char buff[128], unsigned long long sz)
+{
+    const size_t buff_size = 128;
+    const size_t max_len = buff_size - 1; //need to leave room for a null terminator
+    memset(buff, 0, buff_size);
+    size_t idx = 0;
+    const char * const sz_name[]=
+    {
+        "kB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB", NULL
+    };
+    if (sz < 1)
+    {
+        strncpy(buff, "empty", buff_size);
+    }
+    else if (sz < 1024)
+    {
+        append_ull(buff, sz, &idx, max_len);
+        append_str(buff, "B", &idx, max_len);
+    }
+    else
+    {
+        for (size_t i=0; sz_name[i]; i++)
+        {
+            if (sz < (1024*1024) || !sz_name[i+1])
+            {
+                unsigned long long isz = sz/1024;
+                if (isz > 9)
+                {
+                    append_ull(buff, isz, &idx, max_len);
+                }
+                else
+                {
+                    if (isz == 0)
+                    {
+                        append_str(buff, "0", &idx, max_len);
+                    }
+                    else
+                    {
+                        append_ull(buff, isz, &idx, max_len);
+                    }
+
+                    // Maybe append a single fraction digit
+                    unsigned long long remainder = sz % 1024;
+                    if (remainder > 0)
+                    {
+                        char tmp[3] = {'.', extract_most_significant_digit(&remainder), 0};
+                        append_str(buff, tmp, &idx, max_len);
+                    }
+                }
+                append_str(buff, sz_name[i], &idx, max_len);
+                break;
+            }
+            sz /= 1024;
+        }
+    }
+}
+
+double timef()
+{
+    int time_res;
+    struct timeval tv;
+
+    time_res = gettimeofday(&tv, 0);
+
+    if (time_res)
+    {
+        /*
+          Fixme: What on earth is the correct parameter value for NaN?
+          The man pages and the standard helpfully state that this
+          parameter is implementation defined. Gcc gives a warning if
+          a null pointer is used. But not even all mighty Google gives
+          a hint to what value should actually be returned.
+         */
+        return nan("");
+    }
+
+    return (double)tv.tv_sec + 0.000001*tv.tv_usec;
+}
+
+void exit_without_destructors(int code)
+{
+    _exit(code);
+}
+
+/* Helper function to convert from a null_terminated_array_t<wchar_t> to a null_terminated_array_t<char_t> */
+void convert_wide_array_to_narrow(const null_terminated_array_t<wchar_t> &wide_arr, null_terminated_array_t<char> *output)
+{
+    const wchar_t *const *arr = wide_arr.get();
+    if (! arr)
+    {
+        output->clear();
+        return;
+    }
+
+    std::vector<std::string> list;
+    for (size_t i=0; arr[i]; i++)
+    {
+        list.push_back(wcs2string(arr[i]));
+    }
+    output->set(list);
+}
+
+void append_path_component(wcstring &path, const wcstring &component)
+{
+    if (path.empty() || component.empty())
+    {
+        path.append(component);
+    }
+    else
+    {
+        size_t path_len = path.size();
+        bool path_slash = path.at(path_len-1) == L'/';
+        bool comp_slash = component.at(0) == L'/';
+        if (! path_slash && ! comp_slash)
+        {
+            // Need a slash
+            path.push_back(L'/');
+        }
+        else if (path_slash && comp_slash)
+        {
+            // Too many slashes
+            path.erase(path_len - 1, 1);
+        }
+        path.append(component);
+    }
+}
+
+extern "C" {
+    __attribute__((noinline)) void debug_thread_error(void)
+    {
+        while (1) sleep(9999999);
+    }
+}
+
+
+void set_main_thread()
+{
+    main_thread_id = pthread_self();
+}
+
+void configure_thread_assertions_for_testing(void)
+{
+    thread_assertions_configured_for_testing = true;
+}
+
+/* Notice when we've forked */
+static pid_t initial_pid = 0;
+
+/* Be able to restore the term's foreground process group */
+static pid_t initial_foreground_process_group = -1;
+
+bool is_forked_child(void)
+{
+    /* Just bail if nobody's called setup_fork_guards, e.g. some of our tools */
+    if (! initial_pid) return false;
+
+    bool is_child_of_fork = (getpid() != initial_pid);
+    if (is_child_of_fork)
+    {
+        printf("Uh-oh: %d\n", getpid());
+        while (1) sleep(10000);
+    }
+    return is_child_of_fork;
+}
+
+void setup_fork_guards(void)
+{
+    /* Notice when we fork by stashing our pid. This seems simpler than pthread_atfork(). */
+    initial_pid = getpid();
+}
+
+void save_term_foreground_process_group(void)
+{
+    initial_foreground_process_group = tcgetpgrp(STDIN_FILENO);
+}
+
+void restore_term_foreground_process_group(void)
+{
+    if (initial_foreground_process_group != -1)
+    {
+        /* This is called during shutdown and from a signal handler. We don't bother to complain on failure. */
+        if (0 > tcsetpgrp(STDIN_FILENO, initial_foreground_process_group))
+        {
+            /* Ignore failure */
+        }
+    }
+}
+
+bool is_main_thread()
+{
+    assert(main_thread_id != 0);
+    return main_thread_id == pthread_self();
+}
+
+void assert_is_main_thread(const char *who)
+{
+    if (! is_main_thread() && ! thread_assertions_configured_for_testing)
+    {
+        fprintf(stderr, "Warning: %s called off of main thread. Break on debug_thread_error to debug.\n", who);
+        debug_thread_error();
+    }
+}
+
+void assert_is_not_forked_child(const char *who)
+{
+    if (is_forked_child())
+    {
+        fprintf(stderr, "Warning: %s called in a forked child. Break on debug_thread_error to debug.\n", who);
+        debug_thread_error();
+    }
+}
+
+void assert_is_background_thread(const char *who)
+{
+    if (is_main_thread() && ! thread_assertions_configured_for_testing)
+    {
+        fprintf(stderr, "Warning: %s called on the main thread (may block!). Break on debug_thread_error to debug.\n", who);
+        debug_thread_error();
+    }
+}
+
+void assert_is_locked(void *vmutex, const char *who, const char *caller)
+{
+    pthread_mutex_t *mutex = static_cast<pthread_mutex_t*>(vmutex);
+    if (0 == pthread_mutex_trylock(mutex))
+    {
+        fprintf(stderr, "Warning: %s is not locked when it should be in '%s'. Break on debug_thread_error to debug.\n", who, caller);
+        debug_thread_error();
+        pthread_mutex_unlock(mutex);
+    }
+}
+
+void scoped_lock::lock(void)
+{
+    assert(! locked);
+    assert(! is_forked_child());
+    VOMIT_ON_FAILURE_NO_ERRNO(pthread_mutex_lock(lock_obj));
+    locked = true;
+}
+
+void scoped_lock::unlock(void)
+{
+    assert(locked);
+    assert(! is_forked_child());
+    VOMIT_ON_FAILURE_NO_ERRNO(pthread_mutex_unlock(lock_obj));
+    locked = false;
+}
+
+scoped_lock::scoped_lock(pthread_mutex_t &mutex) : lock_obj(&mutex), locked(false)
+{
+    this->lock();
+}
+
+scoped_lock::scoped_lock(mutex_lock_t &lock) : lock_obj(&lock.mutex), locked(false)
+{
+    this->lock();
+}
+
+scoped_lock::~scoped_lock()
+{
+    if (locked) this->unlock();
+}
+
+void scoped_rwlock::lock(void)
+{
+    assert(! (locked || locked_shared));
+    assert(! is_forked_child());
+    VOMIT_ON_FAILURE_NO_ERRNO(pthread_rwlock_rdlock(rwlock_obj));
+    locked = true;
+}
+
+void scoped_rwlock::unlock(void)
+{
+    assert(locked);
+    assert(! is_forked_child());
+    VOMIT_ON_FAILURE_NO_ERRNO(pthread_rwlock_unlock(rwlock_obj));
+    locked = false;
+}
+
+void scoped_rwlock::lock_shared(void)
+{
+    assert(! (locked || locked_shared));
+    assert(! is_forked_child());
+    VOMIT_ON_FAILURE_NO_ERRNO(pthread_rwlock_wrlock(rwlock_obj));
+    locked_shared = true;
+}
+
+void scoped_rwlock::unlock_shared(void)
+{
+    assert(locked_shared);
+    assert(! is_forked_child());
+    VOMIT_ON_FAILURE_NO_ERRNO(pthread_rwlock_unlock(rwlock_obj));
+    locked_shared = false;
+}
+
+void scoped_rwlock::upgrade(void)
+{
+    assert(locked_shared);
+    assert(! is_forked_child());
+    VOMIT_ON_FAILURE_NO_ERRNO(pthread_rwlock_unlock(rwlock_obj));
+    locked = false;
+    VOMIT_ON_FAILURE_NO_ERRNO(pthread_rwlock_wrlock(rwlock_obj));
+    locked_shared = true;
+}
+
+scoped_rwlock::scoped_rwlock(pthread_rwlock_t &rwlock, bool shared) : rwlock_obj(&rwlock), locked(false), locked_shared(false)
+{
+    if (shared)
+    {
+        this->lock_shared();
+    }
+    else
+    {
+        this->lock();
+    }
+}
+
+scoped_rwlock::scoped_rwlock(rwlock_t &rwlock, bool shared) : rwlock_obj(&rwlock.rwlock), locked(false), locked_shared(false)
+{
+    if (shared)
+    {
+        this->lock_shared();
+    }
+    else
+    {
+        this->lock();
+    }
+}
+
+scoped_rwlock::~scoped_rwlock()
+{
+    if (locked)
+    {
+        this->unlock();
+    }
+    else if (locked_shared)
+    {
+        this->unlock_shared();
+    }
+}
+
+wcstokenizer::wcstokenizer(const wcstring &s, const wcstring &separator) :
+    buffer(),
+    str(),
+    state(),
+    sep(separator)
+{
+    buffer = wcsdup(s.c_str());
+    str = buffer;
+    state = NULL;
+}
+
+bool wcstokenizer::next(wcstring &result)
+{
+    wchar_t *tmp = wcstok(str, sep.c_str(), &state);
+    str = NULL;
+    if (tmp) result = tmp;
+    return tmp != NULL;
+}
+
+wcstokenizer::~wcstokenizer()
+{
+    free(buffer);
+}
+
+
+template <typename CharType_t>
+static CharType_t **make_null_terminated_array_helper(const std::vector<std::basic_string<CharType_t> > &argv)
+{
+    size_t count = argv.size();
+
+    /* We allocate everything in one giant block. First compute how much space we need. */
+
+    /* N + 1 pointers */
+    size_t pointers_allocation_len = (count + 1) * sizeof(CharType_t *);
+
+    /* In the very unlikely event that CharType_t has stricter alignment requirements than does a pointer, round us up to the size of a CharType_t */
+    pointers_allocation_len += sizeof(CharType_t) - 1;
+    pointers_allocation_len -= pointers_allocation_len % sizeof(CharType_t);
+
+    /* N null terminated strings */
+    size_t strings_allocation_len = 0;
+    for (size_t i=0; i < count; i++)
+    {
+        /* The size of the string, plus a null terminator */
+        strings_allocation_len += (argv.at(i).size() + 1) * sizeof(CharType_t);
+    }
+
+    /* Now allocate their sum */
+    unsigned char *base = static_cast<unsigned char *>(malloc(pointers_allocation_len + strings_allocation_len));
+    if (! base) return NULL;
+
+    /* Divvy it up into the pointers and strings */
+    CharType_t **pointers = reinterpret_cast<CharType_t **>(base);
+    CharType_t *strings = reinterpret_cast<CharType_t *>(base + pointers_allocation_len);
+
+    /* Start copying */
+    for (size_t i=0; i < count; i++)
+    {
+        const std::basic_string<CharType_t> &str = argv.at(i);
+        // store the current string pointer into self
+        *pointers++ = strings;
+
+        // copy the string into strings
+        strings = std::copy(str.begin(), str.end(), strings);
+        // each string needs a null terminator
+        *strings++ = (CharType_t)(0);
+    }
+    // array of pointers needs a null terminator
+    *pointers++ = NULL;
+
+    // Make sure we know what we're doing
+    assert((unsigned char *)pointers - base == (std::ptrdiff_t)pointers_allocation_len);
+    assert((unsigned char *)strings - (unsigned char *)pointers == (std::ptrdiff_t)strings_allocation_len);
+    assert((unsigned char *)strings - base == (std::ptrdiff_t)(pointers_allocation_len + strings_allocation_len));
+
+    // Return what we did
+    return reinterpret_cast<CharType_t**>(base);
+}
+
+wchar_t **make_null_terminated_array(const wcstring_list_t &lst)
+{
+    return make_null_terminated_array_helper(lst);
+}
+
+char **make_null_terminated_array(const std::vector<std::string> &lst)
+{
+    return make_null_terminated_array_helper(lst);
+}
diff --git a/src/common.h b/src/common.h
new file mode 100644
index 00000000..e27968fd
--- /dev/null
+++ b/src/common.h
@@ -0,0 +1,926 @@
+/** \file common.h
+	Prototypes for various functions, mostly string utilities, that are used by most parts of fish.
+*/
+
+#ifndef FISH_COMMON_H
+/**
+   Header guard
+*/
+#define FISH_COMMON_H
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <wchar.h>
+#include <termios.h>
+#include <string>
+#include <sstream>
+#include <vector>
+#include <pthread.h>
+#include <string.h>
+#include <stdarg.h>
+#include <stddef.h>
+#include <sys/types.h>
+
+#include <errno.h>
+#include "config.h"
+#include "fallback.h"
+
+/**
+   Avoid writing the type name twice in a common "static_cast-initialization".
+   Caveat: This doesn't work with type names containing commas!
+*/
+#define CAST_INIT(type, dst, src) type dst = static_cast<type >(src)
+
+/* Common string type */
+typedef std::wstring wcstring;
+typedef std::vector<wcstring> wcstring_list_t;
+
+/**
+   Maximum number of bytes used by a single utf-8 character
+*/
+#define MAX_UTF8_BYTES 6
+
+/**
+   This is in the unicode private use area.
+*/
+#define ENCODE_DIRECT_BASE 0xf100
+
+/**
+  Highest legal ascii value
+*/
+#define ASCII_MAX 127u
+
+/**
+  Highest legal 16-bit unicode value
+*/
+#define UCS2_MAX 0xffffu
+
+/**
+  Highest legal byte value
+*/
+#define BYTE_MAX 0xffu
+
+/** BOM value */
+#define UTF8_BOM_WCHAR 0xFEFFu
+
+/* Flags for unescape_string functions */
+enum
+{
+    /* Default behavior */
+    UNESCAPE_DEFAULT = 0,
+
+    /* Escape special fish syntax characters like the semicolon */
+    UNESCAPE_SPECIAL = 1 << 0,
+
+    /* Allow incomplete escape sequences */
+    UNESCAPE_INCOMPLETE = 1 << 1
+};
+typedef unsigned int unescape_flags_t;
+
+/* Flags for the escape() and escape_string() functions */
+enum
+{
+    /** Escape all characters, including magic characters like the semicolon */
+    ESCAPE_ALL = 1 << 0,
+
+    /** Do not try to use 'simplified' quoted escapes, and do not use empty quotes as the empty string */
+    ESCAPE_NO_QUOTED = 1 << 1,
+
+    /** Do not escape tildes */
+    ESCAPE_NO_TILDE = 1 << 2
+};
+typedef unsigned int escape_flags_t;
+
+/* Directions */
+enum selection_direction_t
+{
+    /* visual directions */
+    direction_north,
+    direction_east,
+    direction_south,
+    direction_west,
+    direction_page_north,
+    direction_page_south,
+
+    /* logical directions */
+    direction_next,
+    direction_prev,
+
+    /* special value that means deselect */
+    direction_deselect
+};
+
+inline bool selection_direction_is_cardinal(selection_direction_t dir)
+{
+    switch (dir)
+    {
+        case direction_north:
+        case direction_page_north:
+        case direction_east:
+        case direction_page_south:
+        case direction_south:
+        case direction_west:
+            return true;
+        default:
+            return false;
+    }
+}
+
+/**
+ Helper macro for errors
+ */
+#define VOMIT_ON_FAILURE(a) do { if (0 != (a)) { VOMIT_ABORT(errno, #a); } } while (0)
+#define VOMIT_ON_FAILURE_NO_ERRNO(a) do { int err = (a); if (0 != err) { VOMIT_ABORT(err, #a); } } while (0)
+#define VOMIT_ABORT(err, str) do { int code = (err); fprintf(stderr, "%s failed on line %d in file %s: %d (%s)\n", str, __LINE__, __FILE__, code, strerror(code)); abort(); } while(0)
+
+/** Exits without invoking destructors (via _exit), useful for code after fork. */
+void exit_without_destructors(int code) __attribute__((noreturn));
+
+/**
+	Save the shell mode on startup so we can restore them on exit
+*/
+extern struct termios shell_modes;
+
+/**
+   The character to use where the text has been truncated. Is an
+   ellipsis on unicode system and a $ on other systems.
+*/
+extern wchar_t ellipsis_char;
+
+/* Character representing an omitted newline at the end of text */
+extern wchar_t omitted_newline_char;
+
+/**
+   The verbosity level of fish. If a call to debug has a severity
+   level higher than \c debug_level, it will not be printed.
+*/
+extern int debug_level;
+
+/**
+   Profiling flag. True if commands should be profiled.
+*/
+extern bool g_profiling_active;
+
+/**
+   Name of the current program. Should be set at startup. Used by the
+   debug function.
+*/
+extern const wchar_t *program_name;
+
+/* Variants of read() and write() that ignores return values, defeating a warning */
+void read_ignore(int fd, void *buff, size_t count);
+void write_ignore(int fd, const void *buff, size_t count);
+
+/**
+   This macro is used to check that an input argument is not null. It
+   is a bit lika a non-fatal form of assert. Instead of exit-ing on
+   failure, the current function is ended at once. The second
+   parameter is the return value of the current function on failure.
+*/
+#define CHECK( arg, retval )											\
+	if (!(arg)) 														\
+	{																	\
+		debug( 0,														\
+			   "function %s called with null value for argument %s. ",  \
+			   __func__,												\
+			   #arg );													\
+		bugreport();													\
+		show_stackframe();												\
+		return retval;													\
+	}
+
+/**
+   Pause for input, then exit the program. If supported, print a backtrace first.
+*/
+#define FATAL_EXIT()											\
+	{															\
+        char exit_read_buff;                                    \
+        show_stackframe();										\
+        read_ignore( 0, &exit_read_buff, 1 );                   \
+        exit_without_destructors( 1 );                          \
+	}															\
+ 
+
+/**
+   Exit program at once, leaving an error message about running out of memory.
+*/
+#define DIE_MEM()														\
+	{																	\
+		fwprintf( stderr,												\
+				  L"fish: Out of memory on line %ld of file %s, shutting down fish\n", \
+				  (long)__LINE__,												\
+				  __FILE__ );											\
+		FATAL_EXIT();														\
+	}
+
+/**
+   Check if signals are blocked. If so, print an error message and
+   return from the function performing this check.
+*/
+#define CHECK_BLOCK(retval) 											\
+	if (signal_is_blocked()) 											\
+	{																	\
+		debug( 0,														\
+			    "function %s called while blocking signals. ",  		\
+			   __func__);												\
+		bugreport();													\
+		show_stackframe();												\
+		return retval;													\
+	}
+
+/**
+   Shorthand for wgettext call
+*/
+#define _(wstr) wgettext(wstr)
+
+/**
+   Noop, used to tell xgettext that a string should be translated,
+   even though it is not directly sent to wgettext.
+*/
+#define N_(wstr) wstr
+
+/**
+   Check if the specified string element is a part of the specified string list
+ */
+#define contains( str, ... ) contains_internal( str, 0, __VA_ARGS__, NULL )
+
+/**
+  Print a stack trace to stderr
+*/
+void show_stackframe();
+
+
+/**
+   Read a line from the stream f into the string. Returns
+   the number of bytes read or -1 on failure.
+
+   If the carriage return character is encountered, it is
+   ignored. fgetws() considers the line to end if reading the file
+   results in either a newline (L'\n') character, the null (L'\\0')
+   character or the end of file (WEOF) character.
+*/
+int fgetws2(wcstring *s, FILE *f);
+
+
+/**
+ Returns a  wide character string equivalent of the
+ specified multibyte character string
+
+ This function encodes illegal character sequences in a reversible
+ way using the private use area.
+ */
+wcstring str2wcstring(const char *in);
+wcstring str2wcstring(const char *in, size_t len);
+wcstring str2wcstring(const std::string &in);
+
+/**
+   Returns a newly allocated multibyte character string equivalent of
+   the specified wide character string
+
+   This function decodes illegal character sequences in a reversible
+   way using the private use area.
+*/
+char *wcs2str(const wchar_t *in);
+char *wcs2str(const wcstring &in);
+std::string wcs2string(const wcstring &input);
+
+/** Test if a string prefixes another. Returns true if a is a prefix of b */
+bool string_prefixes_string(const wcstring &proposed_prefix, const wcstring &value);
+bool string_prefixes_string(const wchar_t *proposed_prefix, const wcstring &value);
+
+/** Test if a string is a suffix of another */
+bool string_suffixes_string(const wcstring &proposed_suffix, const wcstring &value);
+bool string_suffixes_string(const wchar_t *proposed_suffix, const wcstring &value);
+
+/** Test if a string prefixes another without regard to case. Returns true if a is a prefix of b */
+bool string_prefixes_string_case_insensitive(const wcstring &proposed_prefix, const wcstring &value);
+
+enum fuzzy_match_type_t
+{
+    /* We match the string exactly: FOOBAR matches FOOBAR */
+    fuzzy_match_exact = 0,
+
+    /* We match a prefix of the string: FO matches FOOBAR */
+    fuzzy_match_prefix,
+
+    /* We match the string exactly, but in a case insensitive way: foobar matches FOOBAR */
+    fuzzy_match_case_insensitive,
+
+    /* We match a prefix of the string, in a case insensitive way: foo matches FOOBAR */
+    fuzzy_match_prefix_case_insensitive,
+
+    /* We match a substring of the string: OOBA matches FOOBAR */
+    fuzzy_match_substring,
+
+    /* A subsequence match with insertions only: FBR matches FOOBAR */
+    fuzzy_match_subsequence_insertions_only,
+
+    /* We don't match the string */
+    fuzzy_match_none
+};
+
+/* Indicates where a match type requires replacing the entire token */
+static inline bool match_type_requires_full_replacement(fuzzy_match_type_t t)
+{
+    switch (t)
+    {
+        case fuzzy_match_exact:
+        case fuzzy_match_prefix:
+            return false;
+        default:
+            return true;
+    }
+}
+
+/* Indicates where a match shares a prefix with the string it matches */
+static inline bool match_type_shares_prefix(fuzzy_match_type_t t)
+{
+    switch (t)
+    {
+        case fuzzy_match_exact:
+        case fuzzy_match_prefix:
+        case fuzzy_match_case_insensitive:
+        case fuzzy_match_prefix_case_insensitive:
+            return true;
+        default:
+            return false;
+    }
+}
+
+/** Test if string is a fuzzy match to another */
+struct string_fuzzy_match_t
+{
+    enum fuzzy_match_type_t type;
+
+    /* Strength of the match. The value depends on the type. Lower is stronger. */
+    size_t match_distance_first;
+    size_t match_distance_second;
+
+    /* Constructor */
+    string_fuzzy_match_t(enum fuzzy_match_type_t t, size_t distance_first = 0, size_t distance_second = 0);
+
+    /* Return -1, 0, 1 if this match is (respectively) better than, equal to, or worse than rhs */
+    int compare(const string_fuzzy_match_t &rhs) const;
+};
+
+/* Compute a fuzzy match for a string. If maximum_match is not fuzzy_match_none, limit the type to matches at or below that type. */
+string_fuzzy_match_t string_fuzzy_match_string(const wcstring &string, const wcstring &match_against, fuzzy_match_type_t limit_type = fuzzy_match_none);
+
+
+/** Test if a list contains a string using a linear search. */
+bool list_contains_string(const wcstring_list_t &list, const wcstring &str);
+
+
+void assert_is_main_thread(const char *who);
+#define ASSERT_IS_MAIN_THREAD_TRAMPOLINE(x) assert_is_main_thread(x)
+#define ASSERT_IS_MAIN_THREAD() ASSERT_IS_MAIN_THREAD_TRAMPOLINE(__FUNCTION__)
+
+void assert_is_background_thread(const char *who);
+#define ASSERT_IS_BACKGROUND_THREAD_TRAMPOLINE(x) assert_is_background_thread(x)
+#define ASSERT_IS_BACKGROUND_THREAD() ASSERT_IS_BACKGROUND_THREAD_TRAMPOLINE(__FUNCTION__)
+
+/* Useful macro for asserting that a lock is locked. This doesn't check whether this thread locked it, which it would be nice if it did, but here it is anyways. */
+void assert_is_locked(void *mutex, const char *who, const char *caller);
+#define ASSERT_IS_LOCKED(x) assert_is_locked((void *)(&x), #x, __FUNCTION__)
+
+/** Format the specified size (in bytes, kilobytes, etc.) into the specified stringbuffer. */
+wcstring format_size(long long sz);
+
+/** Version of format_size that does not allocate memory. */
+void format_size_safe(char buff[128], unsigned long long sz);
+
+/** Our crappier versions of debug which is guaranteed to not allocate any memory, or do anything other than call write(). This is useful after a call to fork() with threads. */
+void debug_safe(int level, const char *msg, const char *param1 = NULL, const char *param2 = NULL, const char *param3 = NULL, const char *param4 = NULL, const char *param5 = NULL, const char *param6 = NULL, const char *param7 = NULL, const char *param8 = NULL, const char *param9 = NULL, const char *param10 = NULL, const char *param11 = NULL, const char *param12 = NULL);
+
+/** Writes out a long safely */
+void format_long_safe(char buff[64], long val);
+void format_long_safe(wchar_t buff[64], long val);
+
+
+template<typename T>
+T from_string(const wcstring &x)
+{
+    T result;
+    std::wstringstream stream(x);
+    stream >> result;
+    return result;
+}
+
+template<typename T>
+T from_string(const std::string &x)
+{
+    T result = T();
+    std::stringstream stream(x);
+    stream >> result;
+    return result;
+}
+
+template<typename T>
+wcstring to_string(const T &x)
+{
+    std::wstringstream stream;
+    stream << x;
+    return stream.str();
+}
+
+/* wstringstream is a huge memory pig. Let's provide some specializations where we can. */
+template<>
+inline wcstring to_string(const long &x)
+{
+    wchar_t buff[128];
+    format_long_safe(buff, x);
+    return wcstring(buff);
+}
+
+template<>
+inline bool from_string(const std::string &x)
+{
+    return ! x.empty() && strchr("YTyt1", x.at(0));
+}
+
+template<>
+inline bool from_string(const wcstring &x)
+{
+    return ! x.empty() && wcschr(L"YTyt1", x.at(0));
+}
+
+template<>
+inline wcstring to_string(const int &x)
+{
+    return to_string(static_cast<long>(x));
+}
+
+wchar_t **make_null_terminated_array(const wcstring_list_t &lst);
+char **make_null_terminated_array(const std::vector<std::string> &lst);
+
+/* Helper class for managing a null-terminated array of null-terminated strings (of some char type) */
+template <typename CharType_t>
+class null_terminated_array_t
+{
+    CharType_t **array;
+
+    /* No assignment or copying */
+    void operator=(null_terminated_array_t rhs);
+    null_terminated_array_t(const null_terminated_array_t &);
+
+    typedef std::vector<std::basic_string<CharType_t> > string_list_t;
+
+    size_t size() const
+    {
+        size_t len = 0;
+        if (array != NULL)
+        {
+            while (array[len] != NULL)
+            {
+                len++;
+            }
+        }
+        return len;
+    }
+
+    void free(void)
+    {
+        ::free((void *)array);
+        array = NULL;
+    }
+
+public:
+    null_terminated_array_t() : array(NULL) { }
+    null_terminated_array_t(const string_list_t &argv) : array(make_null_terminated_array(argv))
+    {
+    }
+
+    ~null_terminated_array_t()
+    {
+        this->free();
+    }
+
+    void set(const string_list_t &argv)
+    {
+        this->free();
+        this->array = make_null_terminated_array(argv);
+    }
+
+    const CharType_t * const *get() const
+    {
+        return array;
+    }
+
+    void clear()
+    {
+        this->free();
+    }
+};
+
+/* Helper function to convert from a null_terminated_array_t<wchar_t> to a null_terminated_array_t<char_t> */
+void convert_wide_array_to_narrow(const null_terminated_array_t<wchar_t> &arr, null_terminated_array_t<char> *output);
+
+/* Helper class to cache a narrow version of a wcstring in a malloc'd buffer, so that we can read it after fork() */
+class narrow_string_rep_t
+{
+private:
+    const char *str;
+
+    /* No copying */
+    narrow_string_rep_t &operator=(const narrow_string_rep_t &);
+    narrow_string_rep_t(const narrow_string_rep_t &x);
+
+public:
+    ~narrow_string_rep_t()
+    {
+        free((void *)str);
+    }
+
+    narrow_string_rep_t() : str(NULL) {}
+
+    void set(const wcstring &s)
+    {
+        free((void *)str);
+        str = wcs2str(s.c_str());
+    }
+
+    const char *get() const
+    {
+        return str;
+    }
+};
+
+bool is_forked_child();
+
+
+class mutex_lock_t
+{
+    public:
+    pthread_mutex_t mutex;
+    mutex_lock_t()
+    {
+        VOMIT_ON_FAILURE_NO_ERRNO(pthread_mutex_init(&mutex, NULL));
+    }
+    
+    ~mutex_lock_t()
+    {
+        VOMIT_ON_FAILURE_NO_ERRNO(pthread_mutex_destroy(&mutex));
+    }
+};
+
+/* Basic scoped lock class */
+class scoped_lock
+{
+    pthread_mutex_t *lock_obj;
+    bool locked;
+
+    /* No copying */
+    scoped_lock &operator=(const scoped_lock &);
+    scoped_lock(const scoped_lock &);
+
+public:
+    void lock(void);
+    void unlock(void);
+    scoped_lock(pthread_mutex_t &mutex);
+    scoped_lock(mutex_lock_t &lock);
+    ~scoped_lock();
+};
+
+class rwlock_t
+{
+public:
+    pthread_rwlock_t rwlock;
+    rwlock_t()
+    {
+        VOMIT_ON_FAILURE_NO_ERRNO(pthread_rwlock_init(&rwlock, NULL));
+    }
+
+    ~rwlock_t()
+    {
+        VOMIT_ON_FAILURE_NO_ERRNO(pthread_rwlock_destroy(&rwlock));
+    }
+};
+
+/*
+   Scoped lock class for rwlocks
+ */
+class scoped_rwlock
+{
+    pthread_rwlock_t *rwlock_obj;
+    bool locked;
+    bool locked_shared;
+
+    /* No copying */
+    scoped_rwlock &operator=(const scoped_lock &);
+    scoped_rwlock(const scoped_lock &);
+
+public:
+    void lock(void);
+    void unlock(void);
+    void lock_shared(void);
+    void unlock_shared(void);
+    /*
+       upgrade shared lock to exclusive.
+       equivalent to `lock.unlock_shared(); lock.lock();`
+     */
+    void upgrade(void);
+    scoped_rwlock(pthread_rwlock_t &rwlock, bool shared = false);
+    scoped_rwlock(rwlock_t &rwlock, bool shared = false);
+    ~scoped_rwlock();
+};
+
+/**
+   A scoped manager to save the current value of some variable, and optionally
+   set it to a new value. On destruction it restores the variable to its old
+   value.
+
+   This can be handy when there are multiple code paths to exit a block.
+*/
+template <typename T>
+class scoped_push
+{
+    T * const ref;
+    T saved_value;
+    bool restored;
+
+public:
+    scoped_push(T *r): ref(r), saved_value(*r), restored(false)
+    {
+    }
+
+    scoped_push(T *r, const T &new_value) : ref(r), saved_value(*r), restored(false)
+    {
+        *r = new_value;
+    }
+
+    ~scoped_push()
+    {
+        restore();
+    }
+
+    void restore()
+    {
+        if (!restored)
+        {
+            std::swap(*ref, saved_value);
+            restored = true;
+        }
+    }
+};
+
+
+/* Wrapper around wcstok */
+class wcstokenizer
+{
+    wchar_t *buffer, *str, *state;
+    const wcstring sep;
+
+    /* No copying */
+    wcstokenizer &operator=(const wcstokenizer &);
+    wcstokenizer(const wcstokenizer &);
+
+public:
+    wcstokenizer(const wcstring &s, const wcstring &separator);
+    bool next(wcstring &result);
+    ~wcstokenizer();
+};
+
+/**
+ Appends a path component, with a / if necessary
+ */
+void append_path_component(wcstring &path, const wcstring &component);
+
+wcstring format_string(const wchar_t *format, ...);
+wcstring vformat_string(const wchar_t *format, va_list va_orig);
+void append_format(wcstring &str, const wchar_t *format, ...);
+void append_formatv(wcstring &str, const wchar_t *format, va_list ap);
+
+/**
+   Test if the given string is a valid variable name.
+
+   \return null if this is a valid name, and a pointer to the first invalid character otherwise
+*/
+
+const wchar_t *wcsvarname(const wchar_t *str);
+const wchar_t *wcsvarname(const wcstring &str);
+
+
+/**
+   Test if the given string is a valid function name.
+
+   \return null if this is a valid name, and a pointer to the first invalid character otherwise
+*/
+
+const wchar_t *wcsfuncname(const wcstring &str);
+
+/**
+   Test if the given string is valid in a variable name
+
+   \return true if this is a valid name, false otherwise
+*/
+
+bool wcsvarchr(wchar_t chr);
+
+/**
+   Convenience variants on fish_wcwswidth().
+
+   See fallback.h for the normal definitions.
+*/
+int fish_wcswidth(const wchar_t *str);
+int fish_wcswidth(const wcstring& str);
+
+/**
+   This functions returns the end of the quoted substring beginning at
+   \c in. The type of quoting character is detemrined by examining \c
+   in. Returns 0 on error.
+
+   \param in the position of the opening quote
+*/
+wchar_t *quote_end(const wchar_t *in);
+
+/**
+   A call to this function will reset the error counter. Some
+   functions print out non-critical error messages. These should check
+   the error_count before, and skip printing the message if
+   MAX_ERROR_COUNT messages have been printed. The error_reset()
+   should be called after each interactive command executes, to allow
+   new messages to be printed.
+*/
+void error_reset();
+
+/**
+   This function behaves exactly like a wide character equivalent of
+   the C function setlocale, except that it will also try to detect if
+   the user is using a Unicode character set, and if so, use the
+   unicode ellipsis character as ellipsis, instead of '$'.
+*/
+wcstring wsetlocale(int category, const wchar_t *locale);
+
+/**
+   Checks if \c needle is included in the list of strings specified. A warning is printed if needle is zero.
+
+   \param needle the string to search for in the list
+
+   \return zero if needle is not found, of if needle is null, non-zero otherwise
+*/
+__sentinel bool contains_internal(const wchar_t *needle, int vararg_handle, ...);
+__sentinel bool contains_internal(const wcstring &needle, int vararg_handle, ...);
+
+/**
+   Call read while blocking the SIGCHLD signal. Should only be called
+   if you _know_ there is data available for reading, or the program
+   will hang until there is data.
+*/
+long read_blocked(int fd, void *buf, size_t count);
+
+/**
+   Loop a write request while failure is non-critical. Return -1 and set errno
+   in case of critical error.
+ */
+ssize_t write_loop(int fd, const char *buff, size_t count);
+
+/**
+   Loop a read request while failure is non-critical. Return -1 and set errno
+   in case of critical error.
+ */
+ssize_t read_loop(int fd, void *buff, size_t count);
+
+
+/**
+   Issue a debug message with printf-style string formating and
+   automatic line breaking. The string will begin with the string \c
+   program_name, followed by a colon and a whitespace.
+
+   Because debug is often called to tell the user about an error,
+   before using wperror to give a specific error message, debug will
+   never ever modify the value of errno.
+
+   \param level the priority of the message. Lower number means higher priority. Messages with a priority_number higher than \c debug_level will be ignored..
+   \param msg the message format string.
+
+   Example:
+
+   <code>debug( 1, L"Pi = %.3f", M_PI );</code>
+
+   will print the string 'fish: Pi = 3.141', given that debug_level is 1 or higher, and that program_name is 'fish'.
+*/
+void debug(int level, const char *msg, ...);
+void debug(int level, const wchar_t *msg, ...);
+
+/** Writes a string to stderr, followed by a newline */
+void print_stderr(const wcstring &str);
+
+/**
+   Replace special characters with backslash escape sequences. Newline is
+   replaced with \n, etc.
+
+   \param in The string to be escaped
+   \param flags Flags to control the escaping
+   \return The escaped string
+*/
+
+wcstring escape(const wchar_t *in, escape_flags_t flags);
+wcstring escape_string(const wcstring &in, escape_flags_t flags);
+
+/**
+   Expand backslashed escapes and substitute them with their unescaped
+   counterparts. Also optionally change the wildcards, the tilde
+   character and a few more into constants which are defined in a
+   private use area of Unicode. This assumes wchar_t is a unicode
+   character set.
+*/
+
+/** Unescapes a string in-place. A true result indicates the string was unescaped, a false result indicates the string was unmodified. */
+bool unescape_string_in_place(wcstring *str, unescape_flags_t escape_special);
+
+/** Unescapes a string, returning the unescaped value by reference. On failure, the output is set to an empty string. */
+bool unescape_string(const wchar_t *input, wcstring *output, unescape_flags_t escape_special);
+bool unescape_string(const wcstring &input, wcstring *output, unescape_flags_t escape_special);
+
+
+/**
+    Returns the width of the terminal window, so that not all
+    functions that use these values continually have to keep track of
+    it separately.
+
+    Only works if common_handle_winch is registered to handle winch signals.
+*/
+int common_get_width();
+/**
+   Returns the height of the terminal window, so that not all
+   functions that use these values continually have to keep track of
+   it separatly.
+
+   Only works if common_handle_winch is registered to handle winch signals.
+*/
+int common_get_height();
+
+/**
+  Handle a window change event by looking up the new window size and
+  saving it in an internal variable used by common_get_wisth and
+  common_get_height().
+*/
+void common_handle_winch(int signal);
+
+/**
+   Write paragraph of output to the specified stringbuffer, and redo
+   the linebreaks to fit the current screen.
+*/
+void write_screen(const wcstring &msg, wcstring &buff);
+
+/**
+   Tokenize the specified string into the specified wcstring_list_t.
+   \param val the input string. The contents of this string is not changed.
+   \param out the list in which to place the elements.
+*/
+void tokenize_variable_array(const wcstring &val, wcstring_list_t &out);
+
+/**
+   Make sure the specified direcotry exists. If needed, try to create
+   it and any currently not existing parent directories..
+
+   \return 0 if, at the time of function return the directory exists, -1 otherwise.
+*/
+int create_directory(const wcstring &d);
+
+/**
+   Print a short message about how to file a bug report to stderr
+*/
+void bugreport();
+
+/**
+   Return the number of seconds from the UNIX epoch, with subsecond
+   precision. This function uses the gettimeofday function, and will
+   have the same precision as that function.
+
+   If an error occurs, NAN is returned.
+ */
+double timef();
+
+/**
+	Call the following function early in main to set the main thread.
+    This is our replacement for pthread_main_np().
+ */
+void set_main_thread();
+bool is_main_thread();
+
+/** Configures thread assertions for testing */
+void configure_thread_assertions_for_testing();
+
+/** Set up a guard to complain if we try to do certain things (like take a lock) after calling fork */
+void setup_fork_guards(void);
+
+/** Save the value of tcgetpgrp so we can restore it on exit */
+void save_term_foreground_process_group(void);
+void restore_term_foreground_process_group(void);
+
+/** Return whether we are the child of a fork */
+bool is_forked_child(void);
+void assert_is_not_forked_child(const char *who);
+#define ASSERT_IS_NOT_FORKED_CHILD_TRAMPOLINE(x) assert_is_not_forked_child(x)
+#define ASSERT_IS_NOT_FORKED_CHILD() ASSERT_IS_NOT_FORKED_CHILD_TRAMPOLINE(__FUNCTION__)
+
+/** Macro to help suppress potentially unused variable warnings */
+#define USE(var) (void)(var)
+
+extern "C" {
+    __attribute__((noinline)) void debug_thread_error(void);
+}
+
+
+#endif
diff --git a/src/complete.cpp b/src/complete.cpp
new file mode 100644
index 00000000..877ac984
--- /dev/null
+++ b/src/complete.cpp
@@ -0,0 +1,2287 @@
+/** \file complete.c Functions related to tab-completion.
+
+  These functions are used for storing and retrieving tab-completion data, as well as for performing tab-completion.
+*/
+#include "config.h"
+
+#include <assert.h>
+#include <stdlib.h>
+#include <stddef.h>
+#include <wchar.h>
+#include <wctype.h>
+#include <pwd.h>
+#include <pthread.h>
+#include <algorithm>
+#include <list>
+#include <map>
+#include <set>
+#include <string>
+#include <utility>
+
+#include "fallback.h" // IWYU pragma: keep
+#include "util.h"
+
+#include "wildcard.h"
+#include "proc.h"
+#include "parser.h"
+#include "function.h"
+#include "complete.h"
+#include "builtin.h"
+#include "env.h"
+#include "exec.h"
+#include "expand.h"
+#include "common.h"
+#include "parse_util.h"
+#include "wutil.h"
+#include "path.h"
+#include "parse_tree.h"
+#include "iothread.h"
+#include "autoload.h"
+#include "parse_constants.h"
+
+/*
+  Completion description strings, mostly for different types of files, such as sockets, block devices, etc.
+
+  There are a few more completion description strings defined in
+  expand.c. Maybe all completion description strings should be defined
+  in the same file?
+*/
+
+/**
+   Description for ~USER completion
+*/
+#define COMPLETE_USER_DESC _( L"Home for %ls" )
+
+/**
+   Description for short variables. The value is concatenated to this description
+*/
+#define COMPLETE_VAR_DESC_VAL _( L"Variable: %ls" )
+
+/**
+   The maximum number of commands on which to perform description
+   lookup. The lookup process is quite time consuming, so this should
+   be set to a pretty low number.
+*/
+#define MAX_CMD_DESC_LOOKUP 10
+
+/**
+   Condition cache value returned from hashtable when this condition
+   has not yet been tested. This value is NULL, so that when the hash
+   table returns NULL, this wil be seen as an untested condition.
+*/
+#define CC_NOT_TESTED 0
+
+/**
+   Condition cache value returned from hashtable when the condition is
+   met. This can be any value, that is a valid pointer, and that is
+   different from CC_NOT_TESTED and CC_FALSE.
+*/
+#define CC_TRUE L"true"
+
+/**
+   Condition cache value returned from hashtable when the condition is
+   not met. This can be any value, that is a valid pointer, and that
+   is different from CC_NOT_TESTED and CC_TRUE.
+
+*/
+#define CC_FALSE L"false"
+
+/**
+   The special cased translation macro for completions. The empty
+   string needs to be special cased, since it can occur, and should
+   not be translated. (Gettext returns the version information as the
+   response)
+*/
+#ifdef USE_GETTEXT
+static const wchar_t *C_(const wcstring &s)
+{
+    return s.empty() ? L"" : wgettext(s.c_str());
+}
+#else
+static const wcstring &C_(const wcstring &s)
+{
+    return s;
+}
+#endif
+
+/* Testing apparatus */
+const wcstring_list_t *s_override_variable_names = NULL;
+
+void complete_set_variable_names(const wcstring_list_t *names)
+{
+    s_override_variable_names = names;
+}
+
+static inline wcstring_list_t complete_get_variable_names(void)
+{
+    if (s_override_variable_names != NULL)
+    {
+        return *s_override_variable_names;
+    }
+    else
+    {
+        return env_get_names(0);
+    }
+}
+
+/**
+   Struct describing a completion option entry.
+
+   If short_opt and long_opt are both zero, the comp field must not be
+   empty and contains a list of arguments to the command.
+
+   If either short_opt or long_opt are non-zero, they specify a switch
+   for the command. If \c comp is also not empty, it contains a list
+   of non-switch arguments that may only follow directly after the
+   specified switch.
+*/
+typedef struct complete_entry_opt
+{
+    /** Short style option */
+    wchar_t short_opt;
+    /** Long style option */
+    wcstring long_opt;
+    /** Arguments to the option */
+    wcstring comp;
+    /** Description of the completion */
+    wcstring desc;
+    /** Condition under which to use the option */
+    wcstring condition;
+    /** Must be one of the values SHARED, NO_FILES, NO_COMMON,
+      EXCLUSIVE, and determines how completions should be performed
+      on the argument after the switch. */
+    int result_mode;
+    /** True if old style long options are used */
+    int old_mode;
+    /** Completion flags */
+    complete_flags_t flags;
+
+    const wcstring localized_desc() const
+    {
+        return C_(desc);
+    }
+} complete_entry_opt_t;
+
+/* Last value used in the order field of completion_entry_t */
+static unsigned int kCompleteOrder = 0;
+
+/**
+   Struct describing a command completion
+*/
+typedef std::list<complete_entry_opt_t> option_list_t;
+class completion_entry_t
+{
+public:
+    /** List of all options */
+    option_list_t options;
+
+    /** String containing all short option characters */
+    wcstring short_opt_str;
+
+public:
+
+    /** Command string */
+    const wcstring cmd;
+
+    /** True if command is a path */
+    const bool cmd_is_path;
+
+    /** True if no other options than the ones supplied are possible */
+    bool authoritative;
+
+    /** Order for when this completion was created. This aids in outputting completions sorted by time. */
+    const unsigned int order;
+
+    /** Getters for option list. */
+    const option_list_t &get_options() const;
+
+    /** Adds or removes an option. */
+    void add_option(const complete_entry_opt_t &opt);
+    bool remove_option(wchar_t short_opt, const wchar_t *long_opt, int old_mode);
+
+    /** Getter for short_opt_str. */
+    wcstring &get_short_opt_str();
+    const wcstring &get_short_opt_str() const;
+
+    completion_entry_t(const wcstring &c, bool type, const wcstring &options, bool author) :
+        short_opt_str(options),
+        cmd(c),
+        cmd_is_path(type),
+        authoritative(author),
+        order(++kCompleteOrder)
+    {
+    }
+};
+
+/** Set of all completion entries */
+struct completion_entry_set_comparer
+{
+    /** Comparison for std::set */
+    bool operator()(completion_entry_t *p1, completion_entry_t *p2) const
+    {
+        /* Paths always come last for no particular reason */
+        if (p1->cmd_is_path != p2->cmd_is_path)
+        {
+            return p1->cmd_is_path < p2->cmd_is_path;
+        }
+        else
+        {
+            return p1->cmd < p2->cmd;
+        }
+    }
+};
+typedef std::set<completion_entry_t *, completion_entry_set_comparer> completion_entry_set_t;
+static completion_entry_set_t completion_set;
+
+// Comparison function to sort completions by their order field
+static bool compare_completions_by_order(const completion_entry_t *p1, const completion_entry_t *p2)
+{
+    return p1->order < p2->order;
+}
+
+/** The lock that guards the list of completion entries */
+static pthread_mutex_t completion_lock = PTHREAD_MUTEX_INITIALIZER;
+
+/**
+ * The lock that guards the options list of individual completion entries.
+ * If both completion_lock and completion_entry_lock are to be taken,
+ * completion_lock must be taken first.
+ */
+static pthread_mutex_t completion_entry_lock = PTHREAD_MUTEX_INITIALIZER;
+
+
+void completion_entry_t::add_option(const complete_entry_opt_t &opt)
+{
+    ASSERT_IS_LOCKED(completion_entry_lock);
+    options.push_front(opt);
+}
+
+const option_list_t &completion_entry_t::get_options() const
+{
+    ASSERT_IS_LOCKED(completion_entry_lock);
+    return options;
+}
+
+wcstring &completion_entry_t::get_short_opt_str()
+{
+    ASSERT_IS_LOCKED(completion_entry_lock);
+    return short_opt_str;
+}
+
+const wcstring &completion_entry_t::get_short_opt_str() const
+{
+    ASSERT_IS_LOCKED(completion_entry_lock);
+    return short_opt_str;
+}
+
+completion_t::~completion_t()
+{
+}
+
+/* Clear the COMPLETE_AUTO_SPACE flag, and set COMPLETE_NO_SPACE appropriately depending on the suffix of the string */
+static complete_flags_t resolve_auto_space(const wcstring &comp, complete_flags_t flags)
+{
+    if (flags & COMPLETE_AUTO_SPACE)
+    {
+        flags = flags & ~COMPLETE_AUTO_SPACE;
+        size_t len = comp.size();
+        if (len > 0  && (wcschr(L"/=@:", comp.at(len-1)) != 0))
+            flags |= COMPLETE_NO_SPACE;
+    }
+    return flags;
+}
+
+/* completion_t functions. Note that the constructor resolves flags! */
+completion_t::completion_t(const wcstring &comp, const wcstring &desc, string_fuzzy_match_t mat, complete_flags_t flags_val) :
+    completion(comp),
+    description(desc),
+    match(mat),
+    flags(resolve_auto_space(comp, flags_val))
+{
+}
+
+completion_t::completion_t(const completion_t &him) : completion(him.completion), description(him.description), match(him.match), flags(him.flags)
+{
+}
+
+completion_t &completion_t::operator=(const completion_t &him)
+{
+    if (this != &him)
+    {
+        this->completion = him.completion;
+        this->description = him.description;
+        this->match = him.match;
+        this->flags = him.flags;
+    }
+    return *this;
+}
+
+bool completion_t::is_naturally_less_than(const completion_t &a, const completion_t &b)
+{
+    return wcsfilecmp(a.completion.c_str(), b.completion.c_str()) < 0;
+}
+
+bool completion_t::is_alphabetically_equal_to(const completion_t &a, const completion_t &b)
+{
+    return a.completion == b.completion;
+}
+
+
+/** Class representing an attempt to compute completions */
+class completer_t
+{
+    const completion_request_flags_t flags;
+    const wcstring initial_cmd;
+    std::vector<completion_t> completions;
+
+    /** Table of completions conditions that have already been tested and the corresponding test results */
+    typedef std::map<wcstring, bool> condition_cache_t;
+    condition_cache_t condition_cache;
+
+    enum complete_type_t
+    {
+        COMPLETE_DEFAULT,
+        COMPLETE_AUTOSUGGEST
+    };
+
+    complete_type_t type() const
+    {
+        return (flags & COMPLETION_REQUEST_AUTOSUGGESTION) ? COMPLETE_AUTOSUGGEST : COMPLETE_DEFAULT;
+    }
+
+    bool wants_descriptions() const
+    {
+        return !!(flags & COMPLETION_REQUEST_DESCRIPTIONS);
+    }
+
+    bool fuzzy() const
+    {
+        return !!(flags & COMPLETION_REQUEST_FUZZY_MATCH);
+    }
+
+    fuzzy_match_type_t max_fuzzy_match_type() const
+    {
+        /* If we are doing fuzzy matching, request all types; if not request only prefix matching */
+        return (flags & COMPLETION_REQUEST_FUZZY_MATCH) ? fuzzy_match_none : fuzzy_match_prefix_case_insensitive;
+    }
+
+
+public:
+    completer_t(const wcstring &c, completion_request_flags_t f) :
+        flags(f),
+        initial_cmd(c)
+    {
+    }
+
+    bool empty() const
+    {
+        return completions.empty();
+    }
+    const std::vector<completion_t> &get_completions(void)
+    {
+        return completions;
+    }
+
+    bool try_complete_variable(const wcstring &str);
+    bool try_complete_user(const wcstring &str);
+
+    bool complete_param(const wcstring &cmd_orig,
+                        const wcstring &popt,
+                        const wcstring &str,
+                        bool use_switches);
+
+    void complete_param_expand(const wcstring &str, bool do_file, bool directories_only = false);
+
+    void complete_cmd(const wcstring &str,
+                      bool use_function,
+                      bool use_builtin,
+                      bool use_command,
+                      bool use_implicit_cd);
+
+    void complete_from_args(const wcstring &str,
+                            const wcstring &args,
+                            const wcstring &desc,
+                            complete_flags_t flags);
+
+    void complete_cmd_desc(const wcstring &str);
+
+    bool complete_variable(const wcstring &str, size_t start_offset);
+
+    bool condition_test(const wcstring &condition);
+
+    void complete_strings(const wcstring &wc_escaped,
+                          const wchar_t *desc,
+                          wcstring(*desc_func)(const wcstring &),
+                          std::vector<completion_t> &possible_comp,
+                          complete_flags_t flags);
+
+    expand_flags_t expand_flags() const
+    {
+        /* Never do command substitution in autosuggestions. Sadly, we also can't yet do job expansion because it's not thread safe. */
+        expand_flags_t result = 0;
+        if (this->type() == COMPLETE_AUTOSUGGEST)
+            result |= EXPAND_SKIP_CMDSUBST;
+
+        /* Allow fuzzy matching */
+        if (this->fuzzy())
+            result |= EXPAND_FUZZY_MATCH;
+
+        return result;
+    }
+};
+
+/* Autoloader for completions */
+class completion_autoload_t : public autoload_t
+{
+public:
+    completion_autoload_t();
+    virtual void command_removed(const wcstring &cmd);
+};
+
+static completion_autoload_t completion_autoloader;
+
+/** Constructor */
+completion_autoload_t::completion_autoload_t() : autoload_t(L"fish_complete_path", NULL, 0)
+{
+}
+
+/** Callback when an autoloaded completion is removed */
+void completion_autoload_t::command_removed(const wcstring &cmd)
+{
+    complete_remove(cmd.c_str(), COMMAND, 0, 0, 0);
+}
+
+
+/** Create a new completion entry. */
+void append_completion(std::vector<completion_t> &completions, const wcstring &comp, const wcstring &desc, complete_flags_t flags, string_fuzzy_match_t match)
+{
+    /* If we just constructed the completion and used push_back, we would get two string copies. Try to avoid that by making a stubby completion in the vector first, and then copying our string in. Note that completion_t's constructor will munge 'flags' so it's important that we pass those to the constructor.
+
+       Nasty hack for #1241 - since the constructor needs the completion string to resolve AUTO_SPACE, and we aren't providing it with the completion, we have to do the resolution ourselves. We should get this resolving out of the constructor.
+    */
+    const wcstring empty;
+    completions.push_back(completion_t(empty, empty, match, resolve_auto_space(comp, flags)));
+    completion_t *last = &completions.back();
+    last->completion = comp;
+    last->description = desc;
+}
+
+/**
+   Test if the specified script returns zero. The result is cached, so
+   that if multiple completions use the same condition, it needs only
+   be evaluated once. condition_cache_clear must be called after a
+   completion run to make sure that there are no stale completions.
+*/
+bool completer_t::condition_test(const wcstring &condition)
+{
+    if (condition.empty())
+    {
+//    fwprintf( stderr, L"No condition specified\n" );
+        return 1;
+    }
+
+    if (this->type() == COMPLETE_AUTOSUGGEST)
+    {
+        /* Autosuggestion can't support conditions */
+        return 0;
+    }
+
+    ASSERT_IS_MAIN_THREAD();
+
+    bool test_res;
+    condition_cache_t::iterator cached_entry = condition_cache.find(condition);
+    if (cached_entry == condition_cache.end())
+    {
+        /* Compute new value and reinsert it */
+        test_res = (0 == exec_subshell(condition, false /* don't apply exit status */));
+        condition_cache[condition] = test_res;
+    }
+    else
+    {
+        /* Use the old value */
+        test_res = cached_entry->second;
+    }
+    return test_res;
+}
+
+
+/** Search for an exactly matching completion entry. Must be called while locked. */
+static completion_entry_t *complete_find_exact_entry(const wcstring &cmd, const bool cmd_is_path)
+{
+    ASSERT_IS_LOCKED(completion_lock);
+    completion_entry_t *result = NULL;
+    completion_entry_t tmp_entry(cmd, cmd_is_path, L"", false);
+    completion_entry_set_t::iterator iter = completion_set.find(&tmp_entry);
+    if (iter != completion_set.end())
+    {
+        result = *iter;
+    }
+    return result;
+}
+
+/** Locate the specified entry. Create it if it doesn't exist. Must be called while locked. */
+static completion_entry_t *complete_get_exact_entry(const wcstring &cmd, bool cmd_is_path)
+{
+    ASSERT_IS_LOCKED(completion_lock);
+    completion_entry_t *c;
+
+    c = complete_find_exact_entry(cmd, cmd_is_path);
+
+    if (c == NULL)
+    {
+        c = new completion_entry_t(cmd, cmd_is_path, L"", false);
+        completion_set.insert(c);
+    }
+
+    return c;
+}
+
+
+void complete_set_authoritative(const wchar_t *cmd, bool cmd_is_path, bool authoritative)
+{
+    completion_entry_t *c;
+
+    CHECK(cmd,);
+    scoped_lock lock(completion_lock);
+    c = complete_get_exact_entry(cmd, cmd_is_path);
+    c->authoritative = authoritative;
+}
+
+
+void complete_add(const wchar_t *cmd,
+                  bool cmd_is_path,
+                  wchar_t short_opt,
+                  const wchar_t *long_opt,
+                  int old_mode,
+                  int result_mode,
+                  const wchar_t *condition,
+                  const wchar_t *comp,
+                  const wchar_t *desc,
+                  complete_flags_t flags)
+{
+    CHECK(cmd,);
+
+    /* Lock the lock that allows us to edit the completion entry list */
+    scoped_lock lock(completion_lock);
+
+    /* Lock the lock that allows us to edit individual completion entries */
+    scoped_lock lock2(completion_entry_lock);
+
+    completion_entry_t *c;
+    c = complete_get_exact_entry(cmd, cmd_is_path);
+
+    /* Create our new option */
+    complete_entry_opt_t opt;
+    if (short_opt != L'\0')
+    {
+        int len = 1 + ((result_mode & NO_COMMON) != 0);
+
+        c->get_short_opt_str().push_back(short_opt);
+        if (len == 2)
+        {
+            c->get_short_opt_str().push_back(L':');
+        }
+    }
+
+    opt.short_opt = short_opt;
+    opt.result_mode = result_mode;
+    opt.old_mode=old_mode;
+
+    if (comp) opt.comp = comp;
+    if (condition) opt.condition = condition;
+    if (long_opt) opt.long_opt = long_opt;
+    if (desc) opt.desc = desc;
+    opt.flags = flags;
+
+    c->add_option(opt);
+}
+
+/**
+   Remove all completion options in the specified entry that match the
+   specified short / long option strings. Returns true if it is now
+   empty and should be deleted, false if it's not empty. Must be called while locked.
+*/
+bool completion_entry_t::remove_option(wchar_t short_opt, const wchar_t *long_opt, int old_mode)
+{
+    ASSERT_IS_LOCKED(completion_lock);
+    ASSERT_IS_LOCKED(completion_entry_lock);
+    if ((short_opt == 0) && (long_opt == 0))
+    {
+        this->options.clear();
+    }
+    else
+    {
+        for (option_list_t::iterator iter = this->options.begin(); iter != this->options.end();)
+        {
+            complete_entry_opt_t &o = *iter;
+            if ((short_opt && short_opt == o.short_opt) ||
+                (long_opt && long_opt == o.long_opt && old_mode == o.old_mode))
+            {
+                /*      fwprintf( stderr,
+                      L"remove option -%lc --%ls\n",
+                      o->short_opt?o->short_opt:L' ',
+                      o->long_opt );
+                */
+                if (o.short_opt)
+                {
+                    wcstring &short_opt_str = this->get_short_opt_str();
+                    size_t idx = short_opt_str.find(o.short_opt);
+                    if (idx != wcstring::npos)
+                    {
+                        /* Consume all colons */
+                        size_t first_non_colon = idx + 1;
+                        while (first_non_colon < short_opt_str.size() && short_opt_str.at(first_non_colon) == L':')
+                            first_non_colon++;
+                        short_opt_str.erase(idx, first_non_colon - idx);
+                    }
+                }
+
+                /* Destroy this option and go to the next one */
+                iter = this->options.erase(iter);
+            }
+            else
+            {
+                /* Just go to the next one */
+                ++iter;
+            }
+        }
+    }
+    return this->options.empty();
+}
+
+
+void complete_remove(const wchar_t *cmd,
+                     bool cmd_is_path,
+                     wchar_t short_opt,
+                     const wchar_t *long_opt,
+                     int old_mode)
+{
+    CHECK(cmd,);
+    scoped_lock lock(completion_lock);
+    scoped_lock lock2(completion_entry_lock);
+
+    completion_entry_t tmp_entry(cmd, cmd_is_path, L"", false);
+    completion_entry_set_t::iterator iter = completion_set.find(&tmp_entry);
+    if (iter != completion_set.end())
+    {
+        completion_entry_t *entry = *iter;
+        bool delete_it = entry->remove_option(short_opt, long_opt, old_mode);
+        if (delete_it)
+        {
+            /* Delete this entry */
+            completion_set.erase(iter);
+            delete entry;
+        }
+    }
+}
+
+/* Formats an error string by prepending the prefix and then appending the str in single quotes */
+static wcstring format_error(const wchar_t *prefix, const wcstring &str)
+{
+    wcstring result = prefix;
+    result.push_back(L'\'');
+    result.append(str);
+    result.push_back(L'\'');
+    return result;
+}
+
+/**
+   Find the full path and commandname from a command string 'str'.
+*/
+static void parse_cmd_string(const wcstring &str, wcstring &path, wcstring &cmd)
+{
+    if (! path_get_path(str, &path))
+    {
+        /** Use the empty string as the 'path' for commands that can not be found. */
+        path = L"";
+    }
+
+    /* Make sure the path is not included in the command */
+    size_t last_slash = str.find_last_of(L'/');
+    if (last_slash != wcstring::npos)
+    {
+        cmd = str.substr(last_slash + 1);
+    }
+    else
+    {
+        cmd = str;
+    }
+}
+
+int complete_is_valid_option(const wcstring &str,
+                             const wcstring &opt,
+                             wcstring_list_t *errors,
+                             bool allow_autoload)
+{
+    wcstring cmd, path;
+    bool found_match = false;
+    bool authoritative = true;
+    int opt_found=0;
+    std::set<wcstring> gnu_match_set;
+    bool is_gnu_opt=false;
+    bool is_old_opt=false;
+    bool is_short_opt=false;
+    bool is_gnu_exact=false;
+    size_t gnu_opt_len=0;
+
+    if (opt.empty())
+        return false;
+
+    std::vector<bool> short_validated;
+    /*
+      Check some generic things like -- and - options.
+    */
+    switch (opt.size())
+    {
+
+        case 0:
+        case 1:
+        {
+            return true;
+        }
+
+        case 2:
+        {
+            if (opt == L"--")
+            {
+                return true;
+            }
+            break;
+        }
+    }
+
+    if (opt.at(0) != L'-')
+    {
+        if (errors)
+            errors->push_back(L"Option does not begin with a '-'");
+        return false;
+    }
+
+
+    short_validated.resize(opt.size(), 0);
+
+    is_gnu_opt = opt.at(1) == L'-';
+    if (is_gnu_opt)
+    {
+        size_t opt_end = opt.find(L'=');
+        if (opt_end != wcstring::npos)
+        {
+            gnu_opt_len = opt_end-2;
+        }
+        else
+        {
+            gnu_opt_len = opt.size() - 2;
+        }
+    }
+
+    parse_cmd_string(str, path, cmd);
+
+    /*
+      Make sure completions are loaded for the specified command
+    */
+    if (allow_autoload)
+    {
+        complete_load(cmd, false);
+    }
+
+    scoped_lock lock(completion_lock);
+    scoped_lock lock2(completion_entry_lock);
+    for (completion_entry_set_t::const_iterator iter = completion_set.begin(); iter != completion_set.end(); ++iter)
+    {
+        const completion_entry_t *i = *iter;
+        const wcstring &match = i->cmd_is_path ? path : cmd;
+
+        if (!wildcard_match(match, i->cmd))
+        {
+            continue;
+        }
+
+        found_match = true;
+
+        if (! i->authoritative)
+        {
+            authoritative = false;
+            break;
+        }
+
+        const option_list_t &options = i->get_options();
+        if (is_gnu_opt)
+        {
+            for (option_list_t::const_iterator iter = options.begin(); iter != options.end(); ++iter)
+            {
+                const complete_entry_opt_t &o = *iter;
+                if (o.old_mode)
+                {
+                    continue;
+                }
+
+                if (opt.compare(2, gnu_opt_len, o.long_opt) == 0)
+                {
+                    gnu_match_set.insert(o.long_opt);
+                    if (opt.compare(2, o.long_opt.size(), o.long_opt))
+                    {
+                        is_gnu_exact = true;
+                    }
+                }
+            }
+        }
+        else
+        {
+            /* Check for old style options */
+            for (option_list_t::const_iterator iter = options.begin(); iter != options.end(); ++iter)
+            {
+                const complete_entry_opt_t &o = *iter;
+
+                if (!o.old_mode)
+                    continue;
+
+
+                if (opt.compare(1, wcstring::npos, o.long_opt)==0)
+                {
+                    opt_found = true;
+                    is_old_opt = true;
+                    break;
+                }
+
+            }
+
+            if (is_old_opt)
+                break;
+
+            for (size_t opt_idx = 1; opt_idx < opt.size(); opt_idx++)
+            {
+                const wcstring &short_opt_str = i->get_short_opt_str();
+                size_t str_idx = short_opt_str.find(opt.at(opt_idx));
+                if (str_idx != wcstring::npos)
+                {
+                    if (str_idx + 1 < short_opt_str.size() && short_opt_str.at(str_idx + 1) == L':')
+                    {
+                        /*
+                          This is a short option with an embedded argument,
+                          call complete_is_valid_argument on the argument.
+                        */
+                        const wcstring nopt = L"-" + opt.substr(1, 1);
+                        short_validated.at(opt_idx) =
+                            complete_is_valid_argument(str, nopt, opt.substr(2));
+                    }
+                    else
+                    {
+                        short_validated.at(opt_idx) = true;
+                    }
+                }
+            }
+        }
+    }
+
+    if (authoritative)
+    {
+
+        if (!is_gnu_opt && !is_old_opt)
+            is_short_opt = 1;
+
+
+        if (is_short_opt)
+        {
+            opt_found=1;
+            for (size_t j=1; j<opt.size(); j++)
+            {
+                if (!short_validated.at(j))
+                {
+                    if (errors)
+                    {
+                        const wcstring str = opt.substr(j, 1);
+                        errors->push_back(format_error(_(L"Unknown option: "), str));
+                    }
+
+                    opt_found = 0;
+                    break;
+                }
+
+            }
+        }
+
+        if (is_gnu_opt)
+        {
+            opt_found = is_gnu_exact || (gnu_match_set.size() == 1);
+            if (errors && !opt_found)
+            {
+                const wchar_t *prefix;
+                if (gnu_match_set.empty())
+                {
+                    prefix = _(L"Unknown option: ");
+                }
+                else
+                {
+                    prefix = _(L"Multiple matches for option: ");
+                }
+                errors->push_back(format_error(prefix, opt));
+            }
+        }
+    }
+
+    return (authoritative && found_match)?opt_found:true;
+}
+
+bool complete_is_valid_argument(const wcstring &str, const wcstring &opt, const wcstring &arg)
+{
+    return true;
+}
+
+
+/**
+   Copy any strings in possible_comp which have the specified prefix
+   to the completer's completion array. The prefix may contain wildcards.
+   The output will consist of completion_t structs.
+
+   There are three ways to specify descriptions for each
+   completion. Firstly, if a description has already been added to the
+   completion, it is _not_ replaced. Secondly, if the desc_func
+   function is specified, use it to determine a dynamic
+   completion. Thirdly, if none of the above are available, the desc
+   string is used as a description.
+
+   \param wc_escaped the prefix, possibly containing wildcards. The wildcard should not have been unescaped, i.e. '*' should be used for any string, not the ANY_STRING character.
+   \param desc the default description, used for completions with no embedded description. The description _may_ contain a COMPLETE_SEP character, if not, one will be prefixed to it
+   \param desc_func the function that generates a description for those completions witout an embedded description
+   \param possible_comp the list of possible completions to iterate over
+*/
+
+void completer_t::complete_strings(const wcstring &wc_escaped,
+                                   const wchar_t *desc,
+                                   wcstring(*desc_func)(const wcstring &),
+                                   std::vector<completion_t> &possible_comp,
+                                   complete_flags_t flags)
+{
+    wcstring tmp = wc_escaped;
+    if (! expand_one(tmp, EXPAND_SKIP_CMDSUBST | EXPAND_SKIP_WILDCARDS | this->expand_flags(), NULL))
+        return;
+
+    const wchar_t *wc = parse_util_unescape_wildcards(tmp.c_str());
+
+    for (size_t i=0; i< possible_comp.size(); i++)
+    {
+        wcstring temp = possible_comp.at(i).completion;
+        const wchar_t *next_str = temp.empty()?NULL:temp.c_str();
+
+        if (next_str)
+        {
+            wildcard_complete(next_str, wc, desc, desc_func, this->completions, this->expand_flags(), flags);
+        }
+    }
+
+    free((void *)wc);
+}
+
+/**
+   If command to complete is short enough, substitute
+   the description with the whatis information for the executable.
+*/
+void completer_t::complete_cmd_desc(const wcstring &str)
+{
+    ASSERT_IS_MAIN_THREAD();
+
+    const wchar_t *cmd_start;
+    int skip;
+
+    const wchar_t * const cmd = str.c_str();
+    cmd_start=wcsrchr(cmd, L'/');
+
+    if (cmd_start)
+        cmd_start++;
+    else
+        cmd_start = cmd;
+
+    /*
+      Using apropos with a single-character search term produces far
+      to many results - require at least two characters if we don't
+      know the location of the whatis-database.
+    */
+    if (wcslen(cmd_start) < 2)
+        return;
+
+    if (wildcard_has(cmd_start, 0))
+    {
+        return;
+    }
+
+    skip = 1;
+
+    for (size_t i=0; i< this->completions.size(); i++)
+    {
+        const completion_t &c = this->completions.at(i);
+
+        if (c.completion.empty() || (c.completion[c.completion.size()-1] != L'/'))
+        {
+            skip = 0;
+            break;
+        }
+
+    }
+
+    if (skip)
+    {
+        return;
+    }
+
+
+    wcstring lookup_cmd(L"__fish_describe_command ");
+    lookup_cmd.append(escape_string(cmd_start, 1));
+
+    std::map<wcstring, wcstring> lookup;
+
+    /*
+      First locate a list of possible descriptions using a single
+      call to apropos or a direct search if we know the location
+      of the whatis database. This can take some time on slower
+      systems with a large set of manuals, but it should be ok
+      since apropos is only called once.
+    */
+    wcstring_list_t list;
+    if (exec_subshell(lookup_cmd, list, false /* don't apply exit status */) != -1)
+    {
+
+        /*
+          Then discard anything that is not a possible completion and put
+          the result into a hashtable with the completion as key and the
+          description as value.
+
+          Should be reasonably fast, since no memory allocations are needed.
+        */
+        for (size_t i=0; i < list.size(); i++)
+        {
+            const wcstring &elstr = list.at(i);
+
+            const wcstring fullkey(elstr, wcslen(cmd_start));
+
+            size_t tab_idx = fullkey.find(L'\t');
+            if (tab_idx == wcstring::npos)
+                continue;
+
+            const wcstring key(fullkey, 0, tab_idx);
+            wcstring val(fullkey, tab_idx + 1);
+
+            /*
+              And once again I make sure the first character is uppercased
+              because I like it that way, and I get to decide these
+              things.
+            */
+            if (! val.empty())
+                val[0]=towupper(val[0]);
+
+            lookup[key] = val;
+        }
+
+        /*
+          Then do a lookup on every completion and if a match is found,
+          change to the new description.
+
+          This needs to do a reallocation for every description added, but
+          there shouldn't be that many completions, so it should be ok.
+        */
+        for (size_t i=0; i<this->completions.size(); i++)
+        {
+            completion_t &completion = this->completions.at(i);
+            const wcstring &el = completion.completion;
+            if (el.empty())
+                continue;
+
+            std::map<wcstring, wcstring>::iterator new_desc_iter = lookup.find(el);
+            if (new_desc_iter != lookup.end())
+                completion.description = new_desc_iter->second;
+        }
+    }
+
+}
+
+/**
+   Returns a description for the specified function, or an empty string if none
+*/
+static wcstring complete_function_desc(const wcstring &fn)
+{
+    wcstring result;
+    bool has_description = function_get_desc(fn, &result);
+    if (! has_description)
+    {
+        function_get_definition(fn, &result);
+    }
+    return result;
+}
+
+
+/**
+   Complete the specified command name. Search for executables in the
+   path, executables defined using an absolute path, functions,
+   builtins and directories for implicit cd commands.
+
+   \param cmd the command string to find completions for
+
+   \param comp the list to add all completions to
+*/
+void completer_t::complete_cmd(const wcstring &str_cmd, bool use_function, bool use_builtin, bool use_command, bool use_implicit_cd)
+{
+    /* Paranoia */
+    if (str_cmd.empty())
+        return;
+
+    std::vector<completion_t> possible_comp;
+
+    env_var_t cdpath = env_get_string(L"CDPATH");
+    if (cdpath.missing_or_empty())
+        cdpath = L".";
+
+    if (use_command)
+    {
+
+        if (expand_string(str_cmd, this->completions, ACCEPT_INCOMPLETE | EXECUTABLES_ONLY | this->expand_flags(), NULL) != EXPAND_ERROR)
+        {
+            if (this->wants_descriptions())
+            {
+                this->complete_cmd_desc(str_cmd);
+            }
+        }
+    }
+    if (use_implicit_cd)
+    {
+        (void)expand_string(str_cmd, this->completions, ACCEPT_INCOMPLETE | DIRECTORIES_ONLY | this->expand_flags(), NULL);
+    }
+    if (str_cmd.find(L'/') == wcstring::npos && str_cmd.at(0) != L'~')
+    {
+        if (use_command)
+        {
+
+            const env_var_t path = env_get_string(L"PATH");
+            if (!path.missing())
+            {
+                wcstring base_path;
+                wcstokenizer tokenizer(path, ARRAY_SEP_STR);
+                while (tokenizer.next(base_path))
+                {
+                    if (base_path.empty())
+                        continue;
+
+                    /* Make sure the base path ends with a slash */
+                    if (base_path.at(base_path.size() - 1) != L'/')
+                        base_path.push_back(L'/');
+
+                    wcstring nxt_completion = base_path;
+                    nxt_completion.append(str_cmd);
+
+                    size_t prev_count =  this->completions.size();
+                    if (expand_string(nxt_completion,
+                                      this->completions,
+                                      ACCEPT_INCOMPLETE | EXECUTABLES_ONLY | this->expand_flags(), NULL) != EXPAND_ERROR)
+                    {
+                        /* For all new completions, if COMPLETE_NO_CASE is set, then use only the last path component */
+                        for (size_t i=prev_count; i< this->completions.size(); i++)
+                        {
+                            completion_t &c =  this->completions.at(i);
+                            if (c.flags & COMPLETE_REPLACES_TOKEN)
+                            {
+
+                                c.completion.erase(0, base_path.size());
+                            }
+                        }
+                    }
+                }
+                if (this->wants_descriptions())
+                    this->complete_cmd_desc(str_cmd);
+            }
+        }
+
+        if (use_function)
+        {
+            //function_get_names( &possible_comp, cmd[0] == L'_' );
+            wcstring_list_t names = function_get_names(str_cmd.at(0) == L'_');
+            for (size_t i=0; i < names.size(); i++)
+            {
+                append_completion(possible_comp, names.at(i));
+            }
+
+            this->complete_strings(str_cmd, 0, &complete_function_desc, possible_comp, 0);
+        }
+
+        possible_comp.clear();
+
+        if (use_builtin)
+        {
+            builtin_get_names(possible_comp);
+            this->complete_strings(str_cmd, 0, &builtin_get_desc, possible_comp, 0);
+        }
+
+    }
+}
+
+
+/**
+   Evaluate the argument list (as supplied by complete -a) and insert
+   any return matching completions. Matching is done using \c
+   copy_strings_with_prefix, meaning the completion may contain
+   wildcards. Logically, this is not always the right thing to do, but
+   I have yet to come up with a case where this matters.
+
+   \param str The string to complete.
+   \param args The list of option arguments to be evaluated.
+   \param desc Description of the completion
+   \param comp_out The list into which the results will be inserted
+*/
+void completer_t::complete_from_args(const wcstring &str,
+                                     const wcstring &args,
+                                     const wcstring &desc,
+                                     complete_flags_t flags)
+{
+
+    std::vector<completion_t> possible_comp;
+
+    bool is_autosuggest = (this->type() == COMPLETE_AUTOSUGGEST);
+    parser_t parser(is_autosuggest ? PARSER_TYPE_COMPLETIONS_ONLY : PARSER_TYPE_GENERAL, false /* don't show errors */);
+
+    /* If type is COMPLETE_AUTOSUGGEST, it means we're on a background thread, so don't call proc_push_interactive */
+    if (! is_autosuggest)
+        proc_push_interactive(0);
+
+    parser.expand_argument_list(args, possible_comp);
+
+    if (! is_autosuggest)
+        proc_pop_interactive();
+
+    this->complete_strings(escape_string(str, ESCAPE_ALL), desc.c_str(), 0, possible_comp, flags);
+}
+
+/**
+   Match against an old style long option
+*/
+static int param_match_old(const complete_entry_opt_t *e,
+                           const wchar_t *optstr)
+{
+    return (optstr[0] == L'-') && (e->long_opt == &optstr[1]);
+}
+
+/**
+   Match a parameter
+*/
+static int param_match(const complete_entry_opt_t *e,
+                       const wchar_t *optstr)
+{
+    if (e->short_opt != L'\0' &&
+            e->short_opt == optstr[1])
+        return 1;
+
+    if (!e->old_mode && (wcsncmp(L"--", optstr, 2) == 0))
+    {
+        if (e->long_opt == &optstr[2])
+        {
+            return 1;
+        }
+    }
+
+    return 0;
+}
+
+/**
+   Test if a string is an option with an argument, like --color=auto or -I/usr/include
+*/
+static wchar_t *param_match2(const complete_entry_opt_t *e,
+                             const wchar_t *optstr)
+{
+    if (e->short_opt != L'\0' && e->short_opt == optstr[1])
+        return (wchar_t *)&optstr[2];
+    if (!e->old_mode && (wcsncmp(L"--", optstr, 2) == 0))
+    {
+        size_t len = e->long_opt.size();
+
+        if (wcsncmp(e->long_opt.c_str(), &optstr[2],len) == 0)
+        {
+            if (optstr[len+2] == L'=')
+                return (wchar_t *)&optstr[len+3];
+        }
+    }
+    return 0;
+}
+
+/**
+   Tests whether a short option is a viable completion
+*/
+static int short_ok(const wcstring &arg_str, wchar_t nextopt, const wcstring &allopt_str)
+{
+    const wchar_t *arg = arg_str.c_str();
+    const wchar_t *allopt = allopt_str.c_str();
+    const wchar_t *ptr;
+
+    if (arg[0] != L'-')
+        return arg[0] == L'\0';
+    if (arg[1] == L'-')
+        return 0;
+
+    if (wcschr(arg, nextopt) != 0)
+        return 0;
+
+    for (ptr = arg+1; *ptr; ptr++)
+    {
+        const wchar_t *tmp = wcschr(allopt, *ptr);
+        /* Unknown option */
+        if (tmp == 0)
+        {
+            /*fwprintf( stderr, L"Unknown option %lc", *ptr );*/
+
+            return 0;
+        }
+
+        if (*(tmp+1) == L':')
+        {
+            /*      fwprintf( stderr, L"Woot %ls", allopt );*/
+            return 0;
+        }
+
+    }
+
+    return 1;
+}
+
+void complete_load(const wcstring &name, bool reload)
+{
+    completion_autoloader.load(name, reload);
+}
+
+// Performed on main thread, from background thread. Return type is ignored.
+static int complete_load_no_reload(wcstring *name)
+{
+    assert(name != NULL);
+    ASSERT_IS_MAIN_THREAD();
+    complete_load(*name, false);
+    return 0;
+}
+
+/**
+   complete_param: Given a command, find completions for the argument str of command cmd_orig with previous option popt.
+   
+   Examples in format (cmd, popt, str):
+   
+      echo hello world <tab> -> ("echo", "world", "")
+      echo hello world<tab> -> ("echo", "hello", "world")
+ 
+   Insert results into comp_out. Return true to perform file completion, false to disable it.
+*/
+struct local_options_t
+{
+    wcstring short_opt_str;
+    option_list_t options;
+};
+bool completer_t::complete_param(const wcstring &scmd_orig, const wcstring &spopt, const wcstring &sstr, bool use_switches)
+{
+    const wchar_t * const cmd_orig = scmd_orig.c_str();
+    const wchar_t * const popt = spopt.c_str();
+    const wchar_t * const str = sstr.c_str();
+
+    bool use_common=1, use_files=1;
+
+    wcstring cmd, path;
+    parse_cmd_string(cmd_orig, path, cmd);
+
+    if (this->type() == COMPLETE_DEFAULT)
+    {
+        ASSERT_IS_MAIN_THREAD();
+        complete_load(cmd, true);
+    }
+    else if (this->type() == COMPLETE_AUTOSUGGEST)
+    {
+        /* Maybe load this command (on the main thread) */
+        if (! completion_autoloader.has_tried_loading(cmd))
+        {
+            iothread_perform_on_main(complete_load_no_reload, &cmd);
+        }
+    }
+
+    /* Make a list of lists of all options that we care about */
+    std::vector<local_options_t> all_options;
+    {
+        scoped_lock lock(completion_lock);
+        scoped_lock lock2(completion_entry_lock);
+        for (completion_entry_set_t::const_iterator iter = completion_set.begin(); iter != completion_set.end(); ++iter)
+        {
+            const completion_entry_t *i = *iter;
+            const wcstring &match = i->cmd_is_path ? path : cmd;
+            if (! wildcard_match(match, i->cmd))
+            {
+                continue;
+            }
+
+            /* Copy all of their options into our list */
+            all_options.push_back(local_options_t());
+            all_options.back().short_opt_str = i->get_short_opt_str();
+            all_options.back().options = i->get_options(); //Oof, this is a lot of copying
+        }
+    }
+
+    /* Now release the lock and test each option that we captured above.
+       We have to do this outside the lock because callouts (like the condition) may add or remove completions.
+       See https://github.com/ridiculousfish/fishfish/issues/2 */
+    for (std::vector<local_options_t>::const_iterator iter = all_options.begin(); iter != all_options.end(); ++iter)
+    {
+        const option_list_t &options = iter->options;
+        use_common=1;
+        if (use_switches)
+        {
+
+            if (str[0] == L'-')
+            {
+                /* Check if we are entering a combined option and argument
+                   (like --color=auto or -I/usr/include) */
+                for (option_list_t::const_iterator oiter = options.begin(); oiter != options.end(); ++oiter)
+                {
+                    const complete_entry_opt_t *o = &*oiter;
+                    wchar_t *arg;
+                    if ((arg=param_match2(o, str))!=0 && this->condition_test(o->condition))
+                    {
+                        if (o->result_mode & NO_COMMON) use_common = false;
+                        if (o->result_mode & NO_FILES) use_files = false;
+                        complete_from_args(arg, o->comp, o->localized_desc(), o->flags);
+                    }
+
+                }
+            }
+            else if (popt[0] == L'-')
+            {
+                /* Set to true if we found a matching old-style switch */
+                int old_style_match = 0;
+
+                /*
+                  If we are using old style long options, check for them
+                  first
+                */
+                for (option_list_t::const_iterator oiter = options.begin(); oiter != options.end(); ++oiter)
+                {
+                    const complete_entry_opt_t *o = &*oiter;
+                    if (o->old_mode)
+                    {
+                        if (param_match_old(o, popt) && this->condition_test(o->condition))
+                        {
+                            old_style_match = 1;
+                            if (o->result_mode & NO_COMMON) use_common = false;
+                            if (o->result_mode & NO_FILES) use_files = false;
+                            complete_from_args(str, o->comp, o->localized_desc(), o->flags);
+                        }
+                    }
+                }
+
+                /*
+                  No old style option matched, or we are not using old
+                  style options. We check if any short (or gnu style
+                  options do.
+                */
+                if (!old_style_match)
+                {
+                    for (option_list_t::const_iterator oiter = options.begin(); oiter != options.end(); ++oiter)
+                    {
+                        const complete_entry_opt_t *o = &*oiter;
+                        /*
+                          Gnu-style options with _optional_ arguments must
+                          be specified as a single token, so that it can
+                          be differed from a regular argument.
+                        */
+                        if (!o->old_mode && ! o->long_opt.empty() && !(o->result_mode & NO_COMMON))
+                            continue;
+
+                        if (param_match(o, popt) && this->condition_test(o->condition))
+                        {
+                            if (o->result_mode & NO_COMMON) use_common = false;
+                            if (o->result_mode & NO_FILES) use_files = false;
+                            complete_from_args(str, o->comp, o->localized_desc(), o->flags);
+
+                        }
+                    }
+                }
+            }
+        }
+
+        if (use_common)
+        {
+
+            for (option_list_t::const_iterator oiter = options.begin(); oiter != options.end(); ++oiter)
+            {
+                const complete_entry_opt_t *o = &*oiter;
+                /*
+                  If this entry is for the base command,
+                  check if any of the arguments match
+                */
+
+                if (!this->condition_test(o->condition))
+                    continue;
+
+
+                if ((o->short_opt == L'\0') && (o->long_opt[0]==L'\0'))
+                {
+                    use_files = use_files && ((o->result_mode & NO_FILES)==0);
+                    complete_from_args(str, o->comp, o->localized_desc(), o->flags);
+                }
+
+                if (wcslen(str) > 0 && use_switches)
+                {
+                    /*
+                      Check if the short style option matches
+                    */
+                    if (o->short_opt != L'\0' &&
+                            short_ok(str, o->short_opt, iter->short_opt_str))
+                    {
+                        const wcstring desc = o->localized_desc();
+                        wchar_t completion[2];
+                        completion[0] = o->short_opt;
+                        completion[1] = 0;
+
+                        append_completion(this->completions, completion, desc, 0);
+
+                    }
+
+                    /*
+                      Check if the long style option matches
+                    */
+                    if (o->long_opt[0] != L'\0')
+                    {
+                        int match=0, match_no_case=0;
+
+                        wcstring whole_opt;
+                        whole_opt.append(o->old_mode?L"-":L"--");
+                        whole_opt.append(o->long_opt);
+
+                        match = string_prefixes_string(str, whole_opt);
+
+                        if (!match)
+                        {
+                            match_no_case = wcsncasecmp(str, whole_opt.c_str(), wcslen(str))==0;
+                        }
+
+                        if (match || match_no_case)
+                        {
+                            int has_arg=0; /* Does this switch have any known arguments  */
+                            int req_arg=0; /* Does this switch _require_ an argument */
+
+                            size_t offset = 0;
+                            complete_flags_t flags = 0;
+
+                            if (match)
+                            {
+                                offset = wcslen(str);
+                            }
+                            else
+                            {
+                                flags = COMPLETE_REPLACES_TOKEN;
+                            }
+
+                            has_arg = ! o->comp.empty();
+                            req_arg = (o->result_mode & NO_COMMON);
+
+                            if (!o->old_mode && (has_arg && !req_arg))
+                            {
+
+                                /*
+                                  Optional arguments to a switch can
+                                  only be handled using the '=', so we
+                                  add it as a completion. By default
+                                  we avoid using '=' and instead rely
+                                  on '--switch switch-arg', since it
+                                  is more commonly supported by
+                                  homebrew getopt-like functions.
+                                */
+                                wcstring completion = format_string(L"%ls=", whole_opt.c_str()+offset);
+                                append_completion(this->completions,
+                                                  completion,
+                                                  C_(o->desc),
+                                                  flags);
+
+                            }
+
+                            append_completion(this->completions,
+                                              whole_opt.c_str() + offset,
+                                              C_(o->desc),
+                                              flags);
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+    return use_files;
+}
+
+/**
+   Perform file completion on the specified string
+*/
+void completer_t::complete_param_expand(const wcstring &sstr, bool do_file, bool directories_only)
+{
+    const wchar_t * const str = sstr.c_str();
+    const wchar_t *comp_str;
+
+    if (string_prefixes_string(L"--", sstr) && (comp_str = wcschr(str, L'=')))
+    {
+        comp_str++;
+    }
+    else
+    {
+        comp_str = str;
+    }
+
+    expand_flags_t flags = EXPAND_SKIP_CMDSUBST | ACCEPT_INCOMPLETE | this->expand_flags();
+
+    if (! do_file)
+        flags |= EXPAND_SKIP_WILDCARDS;
+    
+    if (directories_only && do_file)
+        flags |= DIRECTORIES_ONLY;
+
+    /* Squelch file descriptions per issue 254 */
+    if (this->type() == COMPLETE_AUTOSUGGEST || do_file)
+        flags |= EXPAND_NO_DESCRIPTIONS;
+
+    /* Don't do fuzzy matching for files if the string begins with a dash (#568). We could consider relaxing this if there was a preceding double-dash argument */
+    if (string_prefixes_string(L"-", sstr))
+        flags &= ~EXPAND_FUZZY_MATCH;
+
+    if (expand_string(comp_str,
+                      this->completions,
+                      flags, NULL) == EXPAND_ERROR)
+    {
+        debug(3, L"Error while expanding string '%ls'", comp_str);
+    }
+}
+
+/**
+   Complete the specified string as an environment variable
+*/
+bool completer_t::complete_variable(const wcstring &str, size_t start_offset)
+{
+    const wchar_t * const whole_var = str.c_str();
+    const wchar_t *var = &whole_var[start_offset];
+    size_t varlen = wcslen(var);
+    bool res = false;
+
+    const wcstring_list_t names = complete_get_variable_names();
+    for (size_t i=0; i<names.size(); i++)
+    {
+        const wcstring & env_name = names.at(i);
+
+        string_fuzzy_match_t match = string_fuzzy_match_string(var, env_name, this->max_fuzzy_match_type());
+        if (match.type == fuzzy_match_none)
+        {
+            // No match
+            continue;
+        }
+
+        wcstring comp;
+        int flags = 0;
+
+        if (! match_type_requires_full_replacement(match.type))
+        {
+            // Take only the suffix
+            comp.append(env_name.c_str() + varlen);
+        }
+        else
+        {
+            comp.append(whole_var, start_offset);
+            comp.append(env_name);
+            flags = COMPLETE_REPLACES_TOKEN | COMPLETE_DONT_ESCAPE;
+        }
+
+        wcstring desc;
+        if (this->wants_descriptions())
+        {
+            env_var_t value_unescaped = env_get_string(env_name);
+            if (value_unescaped.missing())
+                continue;
+
+            wcstring value = expand_escape_variable(value_unescaped);
+            if (this->type() != COMPLETE_AUTOSUGGEST)
+                desc = format_string(COMPLETE_VAR_DESC_VAL, value.c_str());
+        }
+
+        append_completion(this->completions,  comp, desc, flags, match);
+
+        res = true;
+    }
+
+    return res;
+}
+
+bool completer_t::try_complete_variable(const wcstring &str)
+{
+    enum {e_unquoted, e_single_quoted, e_double_quoted} mode = e_unquoted;
+    const size_t len = str.size();
+
+    /* Get the position of the dollar heading a run of valid variable characters. -1 means none. */
+    size_t variable_start = -1;
+
+    for (size_t in_pos=0; in_pos<len; in_pos++)
+    {
+        wchar_t c = str.at(in_pos);
+        if (! wcsvarchr(c))
+        {
+            /* This character cannot be in a variable, reset the dollar */
+            variable_start = -1;
+        }
+
+        switch (c)
+        {
+            case L'\\':
+                in_pos++;
+                break;
+
+            case L'$':
+                if (mode == e_unquoted || mode == e_double_quoted)
+                {
+                    variable_start = in_pos;
+                }
+                break;
+
+            case L'\'':
+                if (mode == e_single_quoted)
+                {
+                    mode = e_unquoted;
+                }
+                else if (mode == e_unquoted)
+                {
+                    mode = e_single_quoted;
+                }
+                break;
+
+            case L'"':
+                if (mode == e_double_quoted)
+                {
+                    mode = e_unquoted;
+                }
+                else if (mode == e_unquoted)
+                {
+                    mode = e_double_quoted;
+                }
+                break;
+        }
+    }
+
+    /* Now complete if we have a variable start that's also not the last character */
+    bool result = false;
+    if (variable_start != static_cast<size_t>(-1) && variable_start + 1 < len)
+    {
+        result = this->complete_variable(str, variable_start + 1);
+    }
+    return result;
+}
+
+/**
+   Try to complete the specified string as a username. This is used by
+   ~USER type expansion.
+
+   \return 0 if unable to complete, 1 otherwise
+*/
+bool completer_t::try_complete_user(const wcstring &str)
+{
+    const wchar_t *cmd = str.c_str();
+    const wchar_t *first_char=cmd;
+    int res=0;
+    double start_time = timef();
+
+    if (*first_char ==L'~' && !wcschr(first_char, L'/'))
+    {
+        const wchar_t *user_name = first_char+1;
+        const wchar_t *name_end = wcschr(user_name, L'~');
+        if (name_end == 0)
+        {
+            struct passwd *pw;
+            size_t name_len = wcslen(user_name);
+
+            setpwent();
+
+            while ((pw=getpwent()) != 0)
+            {
+                double current_time = timef();
+
+                if (current_time - start_time > 0.2)
+                {
+                    return 1;
+                }
+
+                if (pw->pw_name)
+                {
+                    const wcstring pw_name_str = str2wcstring(pw->pw_name);
+                    const wchar_t *pw_name = pw_name_str.c_str();
+                    if (wcsncmp(user_name, pw_name, name_len)==0)
+                    {
+                        wcstring desc = format_string(COMPLETE_USER_DESC, pw_name);
+                        append_completion(this->completions,
+                                          &pw_name[name_len],
+                                          desc,
+                                          COMPLETE_NO_SPACE);
+
+                        res=1;
+                    }
+                    else if (wcsncasecmp(user_name, pw_name, name_len)==0)
+                    {
+                        wcstring name = format_string(L"~%ls", pw_name);
+                        wcstring desc = format_string(COMPLETE_USER_DESC, pw_name);
+
+                        append_completion(this->completions,
+                                          name,
+                                          desc,
+                                          COMPLETE_REPLACES_TOKEN | COMPLETE_DONT_ESCAPE | COMPLETE_NO_SPACE);
+                        res=1;
+                    }
+                }
+            }
+            endpwent();
+        }
+    }
+
+    return res;
+}
+
+void complete(const wcstring &cmd_with_subcmds, std::vector<completion_t> &comps, completion_request_flags_t flags)
+{
+    /* Determine the innermost subcommand */
+    const wchar_t *cmdsubst_begin, *cmdsubst_end;
+    parse_util_cmdsubst_extent(cmd_with_subcmds.c_str(), cmd_with_subcmds.size(), &cmdsubst_begin, &cmdsubst_end);
+    assert(cmdsubst_begin != NULL && cmdsubst_end != NULL && cmdsubst_end >= cmdsubst_begin);
+    const wcstring cmd = wcstring(cmdsubst_begin, cmdsubst_end - cmdsubst_begin);
+
+    /* Make our completer */
+    completer_t completer(cmd, flags);
+
+    wcstring current_command;
+    const size_t pos = cmd.size();
+    bool done=false;
+    bool use_command = 1;
+    bool use_function = 1;
+    bool use_builtin = 1;
+    bool use_implicit_cd = 1;
+
+    //debug( 1, L"Complete '%ls'", cmd.c_str() );
+
+    const wchar_t *cmd_cstr = cmd.c_str();
+    const wchar_t *tok_begin = NULL, *prev_begin = NULL, *prev_end = NULL;
+    parse_util_token_extent(cmd_cstr, cmd.size(), &tok_begin, NULL, &prev_begin, &prev_end);
+
+    /**
+     If we are completing a variable name or a tilde expansion user
+     name, we do that and return. No need for any other completions.
+     */
+    const wcstring current_token = tok_begin;
+
+    /* Unconditionally complete variables and processes. This is a little weird since we will happily complete variables even in e.g. command position, despite the fact that they are invalid there. */
+    if (!done)
+    {
+        done = completer.try_complete_variable(current_token) || completer.try_complete_user(current_token);
+    }
+
+    if (!done)
+    {
+        //const size_t prev_token_len = (prev_begin ? prev_end - prev_begin : 0);
+        //const wcstring prev_token(prev_begin, prev_token_len);
+
+        parse_node_tree_t tree;
+        parse_tree_from_string(cmd, parse_flag_continue_after_error | parse_flag_accept_incomplete_tokens | parse_flag_include_comments, &tree, NULL);
+
+        /* Find any plain statement that contains the position. We have to backtrack past spaces (#1261). So this will be at either the last space character, or after the end of the string */
+        size_t adjusted_pos = pos;
+        while (adjusted_pos > 0 && cmd.at(adjusted_pos - 1) == L' ')
+        {
+            adjusted_pos--;
+        }
+
+        const parse_node_t *plain_statement = tree.find_node_matching_source_location(symbol_plain_statement, adjusted_pos, NULL);
+        if (plain_statement == NULL)
+        {
+            /* Not part of a plain statement. This could be e.g. a for loop header, case expression, etc. Do generic file completions (#1309). If we had to backtrack, it means there was whitespace; don't do an autosuggestion in that case. Also don't do it if we are just after a pipe, semicolon, or & (#1631), or in a comment.
+            
+            Overall this logic is a total mess. A better approach would be to return the "possible next token" from the parse tree directly (this data is available as the first of the sequence of nodes without source locations at the very end of the parse tree). */
+            bool do_file = true;
+            if (flags & COMPLETION_REQUEST_AUTOSUGGESTION)
+            {
+                if (adjusted_pos < pos)
+                {
+                    do_file = false;
+                }
+                else if (pos > 0)
+                {
+                    // If the previous character is in one of these types, we don't do file suggestions
+                    parse_token_type_t bad_types[] = {parse_token_type_pipe, parse_token_type_end, parse_token_type_background, parse_special_type_comment};
+                    for (size_t i=0; i < sizeof bad_types / sizeof *bad_types; i++)
+                    {
+                        if (tree.find_node_matching_source_location(bad_types[i], pos - 1, NULL))
+                        {
+                            do_file = false;
+                            break;
+                        }
+                    }
+                }
+            }
+            completer.complete_param_expand(current_token, do_file);
+        }
+        else
+        {
+            assert(plain_statement->has_source() && plain_statement->type == symbol_plain_statement);
+
+            /* Get the command node */
+            const parse_node_t *cmd_node = tree.get_child(*plain_statement, 0, parse_token_type_string);
+
+            /* Get the actual command string */
+            if (cmd_node != NULL)
+                current_command = cmd_node->get_source(cmd);
+
+            /* Check the decoration */
+            switch (tree.decoration_for_plain_statement(*plain_statement))
+            {
+                case parse_statement_decoration_none:
+                    use_command = true;
+                    use_function = true;
+                    use_builtin = true;
+                    use_implicit_cd = true;
+                    break;
+
+                case parse_statement_decoration_command:
+                case parse_statement_decoration_exec:
+                    use_command = true;
+                    use_function = false;
+                    use_builtin = false;
+                    use_implicit_cd = false;
+                    break;
+
+                case parse_statement_decoration_builtin:
+                    use_command = false;
+                    use_function = false;
+                    use_builtin = true;
+                    use_implicit_cd = false;
+                    break;
+            }
+
+            if (cmd_node && cmd_node->location_in_or_at_end_of_source_range(pos))
+            {
+                /* Complete command filename */
+                completer.complete_cmd(current_token, use_function, use_builtin, use_command, use_implicit_cd);
+            }
+            else
+            {
+                /* Get all the arguments */
+                const parse_node_tree_t::parse_node_list_t all_arguments = tree.find_nodes(*plain_statement, symbol_argument);
+
+                /* See whether we are in an argument. We may also be in a redirection, or nothing at all. */
+                size_t matching_arg_index = -1;
+                for (size_t i=0; i < all_arguments.size(); i++)
+                {
+                    const parse_node_t *node = all_arguments.at(i);
+                    if (node->location_in_or_at_end_of_source_range(adjusted_pos))
+                    {
+                        matching_arg_index = i;
+                        break;
+                    }
+                }
+
+                bool had_ddash = false;
+                wcstring current_argument, previous_argument;
+                if (matching_arg_index != (size_t)(-1))
+                {
+                    const wcstring matching_arg = all_arguments.at(matching_arg_index)->get_source(cmd);
+                    
+                    /* If the cursor is in whitespace, then the "current" argument is empty and the previous argument is the matching one. But if the cursor was in or at the end of the argument, then the current argument is the matching one, and the previous argument is the one before it. */
+                    bool cursor_in_whitespace = (adjusted_pos < pos);
+                    if (cursor_in_whitespace)
+                    {
+                        current_argument = L"";
+                        previous_argument = matching_arg;
+                    }
+                    else
+                    {
+                        current_argument = matching_arg;
+                        if (matching_arg_index > 0)
+                        {
+                            previous_argument = all_arguments.at(matching_arg_index - 1)->get_source(cmd);
+                        }
+                    }
+
+                    /* Check to see if we have a preceding double-dash */
+                    for (size_t i=0; i < matching_arg_index; i++)
+                    {
+                        if (all_arguments.at(i)->get_source(cmd) == L"--")
+                        {
+                            had_ddash = true;
+                            break;
+                        }
+                    }
+                }
+                
+                /* If we are not in an argument, we may be in a redirection */
+                bool in_redirection = false;
+                if (matching_arg_index == (size_t)(-1))
+                {
+                    const parse_node_t *redirection = tree.find_node_matching_source_location(symbol_redirection, adjusted_pos, plain_statement);
+                    in_redirection = (redirection != NULL);
+                }
+                
+                bool do_file = false, directories_only = false;
+                if (in_redirection)
+                {
+                    do_file = true;
+                }
+                else
+                {
+                    wcstring current_command_unescape, previous_argument_unescape, current_argument_unescape;
+                    if (unescape_string(current_command, &current_command_unescape, UNESCAPE_DEFAULT) &&
+                            unescape_string(previous_argument, &previous_argument_unescape, UNESCAPE_DEFAULT) &&
+                            unescape_string(current_argument, &current_argument_unescape, UNESCAPE_INCOMPLETE))
+                    {
+                        // Have to walk over the command and its entire wrap chain
+                        // If any command disables do_file, then they all do
+                        do_file = true;
+                        const wcstring_list_t wrap_chain = complete_get_wrap_chain(current_command_unescape);
+                        for (size_t i=0; i < wrap_chain.size(); i++)
+                        {
+                            // Hackish, this. The first command in the chain is always the given command. For every command past the first, we need to create a transient commandline for builtin_commandline. But not for COMPLETION_REQUEST_AUTOSUGGESTION, which may occur on background threads.
+                            builtin_commandline_scoped_transient_t *transient_cmd = NULL;
+                            if (i == 0)
+                            {
+                                assert(wrap_chain.at(i) == current_command_unescape);
+                            }
+                            else if (! (flags & COMPLETION_REQUEST_AUTOSUGGESTION))
+                            {
+                                assert(cmd_node != NULL);
+                                wcstring faux_cmdline = cmd;
+                                faux_cmdline.replace(cmd_node->source_start, cmd_node->source_length, wrap_chain.at(i));
+                                transient_cmd = new builtin_commandline_scoped_transient_t(faux_cmdline);
+                            }
+                            if (! completer.complete_param(wrap_chain.at(i),
+                                                           previous_argument_unescape,
+                                                           current_argument_unescape,
+                                                           !had_ddash))
+                            {
+                                do_file = false;
+                            }
+                            delete transient_cmd; //may be null
+                        }
+                    }
+
+                    /* If we have found no command specific completions at all, fall back to using file completions. */
+                    if (completer.empty())
+                        do_file = true;
+                    
+                    /* Hack. If we're cd, do directories only (#1059) */
+                    directories_only = (current_command_unescape == L"cd");
+                    
+                    /* And if we're autosuggesting, and the token is empty, don't do file suggestions */
+                    if ((flags & COMPLETION_REQUEST_AUTOSUGGESTION) && current_argument_unescape.empty())
+                    {
+                        do_file = false;
+                    }
+                }
+
+                /* This function wants the unescaped string */
+                completer.complete_param_expand(current_token, do_file, directories_only);
+            }
+        }
+    }
+
+    comps = completer.get_completions();
+}
+
+
+
+/**
+   Print the GNU longopt style switch \c opt, and the argument \c
+   argument to the specified stringbuffer, but only if arguemnt is
+   non-null and longer than 0 characters.
+*/
+static void append_switch(wcstring &out,
+                          const wcstring &opt,
+                          const wcstring &argument)
+{
+    if (argument.empty())
+        return;
+
+    wcstring esc = escape_string(argument, 1);
+    append_format(out, L" --%ls %ls", opt.c_str(), esc.c_str());
+}
+
+void complete_print(wcstring &out)
+{
+    scoped_lock locker(completion_lock);
+    scoped_lock locker2(completion_entry_lock);
+
+    // Get a list of all completions in a vector, then sort it by order
+    std::vector<const completion_entry_t *> all_completions(completion_set.begin(), completion_set.end());
+    sort(all_completions.begin(), all_completions.end(), compare_completions_by_order);
+
+    for (std::vector<const completion_entry_t *>::const_iterator iter = all_completions.begin(); iter != all_completions.end(); ++iter)
+    {
+        const completion_entry_t *e = *iter;
+        const option_list_t &options = e->get_options();
+        for (option_list_t::const_iterator oiter = options.begin(); oiter != options.end(); ++oiter)
+        {
+            const complete_entry_opt_t *o = &*oiter;
+            const wchar_t *modestr[] =
+            {
+                L"",
+                L" --no-files",
+                L" --require-parameter",
+                L" --exclusive"
+            }
+            ;
+
+            append_format(out,
+                          L"complete%ls",
+                          modestr[o->result_mode]);
+
+            append_switch(out,
+                          e->cmd_is_path ? L"path" : L"command",
+                          escape_string(e->cmd, ESCAPE_ALL));
+
+
+            if (o->short_opt != 0)
+            {
+                append_format(out,
+                              L" --short-option '%lc'",
+                              o->short_opt);
+            }
+
+
+            append_switch(out,
+                          o->old_mode?L"old-option":L"long-option",
+                          o->long_opt);
+
+            append_switch(out,
+                          L"description",
+                          C_(o->desc));
+
+            append_switch(out,
+                          L"arguments",
+                          o->comp);
+
+            append_switch(out,
+                          L"condition",
+                          o->condition);
+
+            out.append(L"\n");
+        }
+    }
+    
+    /* Append wraps. This is a wonky interface where even values are the commands, and odd values are the targets that they wrap. */
+    const wcstring_list_t wrap_pairs = complete_get_wrap_pairs();
+    assert(wrap_pairs.size() % 2 == 0);
+    for (size_t i=0; i < wrap_pairs.size(); )
+    {
+        const wcstring &cmd = wrap_pairs.at(i++);
+        const wcstring &target = wrap_pairs.at(i++);
+        append_format(out, L"complete --command %ls --wraps %ls\n", cmd.c_str(), target.c_str());
+    }
+}
+
+
+/* Completion "wrapper" support. The map goes from wrapping-command to wrapped-command-list */
+static pthread_mutex_t wrapper_lock = PTHREAD_MUTEX_INITIALIZER;
+typedef std::map<wcstring, wcstring_list_t> wrapper_map_t;
+static wrapper_map_t &wrap_map()
+{
+    ASSERT_IS_LOCKED(wrapper_lock);
+    // A pointer is a little more efficient than an object as a static because we can elide the thread-safe initialization
+    static wrapper_map_t *wrapper_map = NULL;
+    if (wrapper_map == NULL)
+    {
+        wrapper_map = new wrapper_map_t();
+    }
+    return *wrapper_map;
+}
+
+/* Add a new target that is wrapped by command. Example: sgrep (command) wraps grep (target). */
+bool complete_add_wrapper(const wcstring &command, const wcstring &new_target)
+{
+    if (command.empty() || new_target.empty())
+    {
+        return false;
+    }
+    
+    scoped_lock locker(wrapper_lock);
+    wrapper_map_t &wraps = wrap_map();
+    wcstring_list_t *targets = &wraps[command];
+    // If it's already present, we do nothing
+    if (std::find(targets->begin(), targets->end(), new_target) == targets->end())
+    {
+        targets->push_back(new_target);
+    }
+    return true;
+}
+
+bool complete_remove_wrapper(const wcstring &command, const wcstring &target_to_remove)
+{
+    if (command.empty() || target_to_remove.empty())
+    {
+        return false;
+    }
+    
+    scoped_lock locker(wrapper_lock);
+    wrapper_map_t &wraps = wrap_map();
+    bool result = false;
+    wrapper_map_t::iterator current_targets_iter = wraps.find(command);
+    if (current_targets_iter != wraps.end())
+    {
+        wcstring_list_t *targets = &current_targets_iter->second;
+        wcstring_list_t::iterator where = std::find(targets->begin(), targets->end(), target_to_remove);
+        if (where != targets->end())
+        {
+            targets->erase(where);
+            result = true;
+        }
+    }
+    return result;
+}
+
+wcstring_list_t complete_get_wrap_chain(const wcstring &command)
+{
+    if (command.empty())
+    {
+        return wcstring_list_t();
+    }
+    scoped_lock locker(wrapper_lock);
+    const wrapper_map_t &wraps = wrap_map();
+    
+    wcstring_list_t result;
+    std::set<wcstring> visited; // set of visited commands
+    wcstring_list_t to_visit(1, command); // stack of remaining-to-visit commands
+    
+    wcstring target;
+    while (! to_visit.empty())
+    {
+        // Grab the next command to visit, put it in target
+        target.swap(to_visit.back());
+        to_visit.pop_back();
+        
+        // Try inserting into visited. If it was already present, we skip it; this is how we avoid loops.
+        if (! visited.insert(target).second)
+        {
+            continue;
+        }
+        
+        // Insert the target in the result. Note this is the command itself, if this is the first iteration of the loop.
+        result.push_back(target);
+        
+        // Enqueue its children
+        wrapper_map_t::const_iterator target_children_iter = wraps.find(target);
+        if (target_children_iter != wraps.end())
+        {
+            const wcstring_list_t &children = target_children_iter->second;
+            to_visit.insert(to_visit.end(), children.begin(), children.end());
+        }
+    }
+    
+    return result;
+}
+
+wcstring_list_t complete_get_wrap_pairs()
+{
+    wcstring_list_t result;
+    scoped_lock locker(wrapper_lock);
+    const wrapper_map_t &wraps = wrap_map();
+    for (wrapper_map_t::const_iterator outer = wraps.begin(); outer != wraps.end(); ++outer)
+    {
+        const wcstring &cmd = outer->first;
+        const wcstring_list_t &targets = outer->second;
+        for (size_t i=0; i < targets.size(); i++)
+        {
+            result.push_back(cmd);
+            result.push_back(targets.at(i));
+        }
+    }
+    return result;
+}
diff --git a/src/complete.h b/src/complete.h
new file mode 100644
index 00000000..c1e156e1
--- /dev/null
+++ b/src/complete.h
@@ -0,0 +1,276 @@
+/** \file complete.h
+  Prototypes for functions related to tab-completion.
+
+  These functions are used for storing and retrieving tab-completion
+  data, as well as for performing tab-completion.
+*/
+
+#ifndef FISH_COMPLETE_H
+
+/**
+   Header guard
+*/
+#define FISH_COMPLETE_H
+
+#include <vector>
+#include <stdint.h>
+
+#include "common.h"
+/**
+ * Use all completions
+ */
+#define SHARED 0
+/**
+ * Do not use file completion
+ */
+#define NO_FILES 1
+/**
+ * Require a parameter after completion
+ */
+#define NO_COMMON 2
+/**
+ * Only use the argument list specifies with completion after
+ * option. This is the same as (NO_FILES | NO_COMMON)
+ */
+#define EXCLUSIVE 3
+
+/**
+ * Command is a path
+ */
+#define PATH 1
+/**
+ * Command is not a path
+ */
+#define COMMAND 0
+
+/**
+ * Separator between completion and description
+ */
+#define COMPLETE_SEP L'\004'
+
+/**
+ * Separator between completion and description
+ */
+#define COMPLETE_SEP_STR L"\004"
+
+/**
+ * Character that separates the completion and description on
+ * programmable completions
+ */
+#define PROG_COMPLETE_SEP L'\t'
+
+enum
+{
+    /**
+       Do not insert space afterwards if this is the only completion. (The
+       default is to try insert a space)
+    */
+    COMPLETE_NO_SPACE = 1 << 0,
+
+    /** This is not the suffix of a token, but replaces it entirely */
+    COMPLETE_REPLACES_TOKEN = 1 << 2,
+
+    /** This completion may or may not want a space at the end - guess by
+       checking the last character of the completion. */
+    COMPLETE_AUTO_SPACE = 1 << 3,
+
+    /** This completion should be inserted as-is, without escaping. */
+    COMPLETE_DONT_ESCAPE = 1 << 4,
+
+    /** If you do escape, don't escape tildes */
+    COMPLETE_DONT_ESCAPE_TILDES = 1 << 5
+};
+typedef int complete_flags_t;
+
+
+class completion_t
+{
+
+private:
+    /* No public default constructor */
+    completion_t();
+public:
+
+    /* Destructor. Not inlining it saves code size. */
+    ~completion_t();
+
+    /** The completion string */
+    wcstring completion;
+
+    /** The description for this completion */
+    wcstring description;
+
+    /** The type of fuzzy match */
+    string_fuzzy_match_t match;
+
+    /**
+       Flags determining the completion behaviour.
+
+       Determines whether a space should be inserted after this
+       completion if it is the only possible completion using the
+       COMPLETE_NO_SPACE flag.
+
+       The COMPLETE_NO_CASE can be used to signal that this completion
+       is case insensitive.
+    */
+    complete_flags_t flags;
+
+    /* Construction. Note: defining these so that they are not inlined reduces the executable size. */
+    completion_t(const wcstring &comp, const wcstring &desc = wcstring(), string_fuzzy_match_t match = string_fuzzy_match_t(fuzzy_match_exact), complete_flags_t flags_val = 0);
+    completion_t(const completion_t &);
+    completion_t &operator=(const completion_t &);
+
+    /* Compare two completions. No operating overlaoding to make this always explicit (there's potentially multiple ways to compare completions). */
+    
+    /* "Naturally less than" means in a natural ordering, where digits are treated as numbers. For example, foo10 is naturally greater than foo2 (but alphabetically less than it) */
+    static bool is_naturally_less_than(const completion_t &a, const completion_t &b);
+    static bool is_alphabetically_equal_to(const completion_t &a, const completion_t &b);
+};
+
+enum
+{
+    COMPLETION_REQUEST_DEFAULT = 0,
+    COMPLETION_REQUEST_AUTOSUGGESTION = 1 << 0, // indicates the completion is for an autosuggestion
+    COMPLETION_REQUEST_DESCRIPTIONS = 1 << 1, // indicates that we want descriptions
+    COMPLETION_REQUEST_FUZZY_MATCH = 1 << 2 // indicates that we don't require a prefix match
+};
+typedef uint32_t completion_request_flags_t;
+
+/**
+
+  Add a completion.
+
+  All supplied values are copied, they should be freed by or otherwise
+  disposed by the caller.
+
+  Examples:
+
+  The command 'gcc -o' requires that a file follows it, so the
+  NO_COMMON option is suitable. This can be done using the following
+  line:
+
+  complete -c gcc -s o -r
+
+  The command 'grep -d' required that one of the strings 'read',
+  'skip' or 'recurse' is used. As such, it is suitable to specify that
+  a completion requires one of them. This can be done using the
+  following line:
+
+  complete -c grep -s d -x -a "read skip recurse"
+
+
+  \param cmd Command to complete.
+  \param cmd_type If cmd_type is PATH, cmd will be interpreted as the absolute
+      path of the program (optionally containing wildcards), otherwise it
+      will be interpreted as the command name.
+  \param short_opt The single character name of an option. (-a is a short option,
+      --all and  -funroll are long options)
+  \param long_opt The multi character name of an option. (-a is a short option,
+      --all and  -funroll are long options)
+  \param long_mode Whether to use old style, single dash long options.
+  \param result_mode Whether to search further completions when this
+      completion has been succesfully matched. If result_mode is SHARED,
+      any other completions may also be used. If result_mode is NO_FILES,
+      file completion should not be used, but other completions may be
+      used. If result_mode is NO_COMMON, on option may follow it - only a
+      parameter. If result_mode is EXCLUSIVE, no option may follow it, and
+      file completion is not performed.
+  \param comp A space separated list of completions which may contain subshells.
+  \param desc A description of the completion.
+  \param condition a command to be run to check it this completion should be used.
+      If \c condition is empty, the completion is always used.
+  \param flags A set of completion flags
+*/
+void complete_add(const wchar_t *cmd,
+                  bool cmd_is_path,
+                  wchar_t short_opt,
+                  const wchar_t *long_opt,
+                  int long_mode,
+                  int result_mode,
+                  const wchar_t *condition,
+                  const wchar_t *comp,
+                  const wchar_t *desc,
+                  int flags);
+/**
+  Sets whether the completion list for this command is complete. If
+  true, any options not matching one of the provided options will be
+  flagged as an error by syntax highlighting.
+*/
+void complete_set_authoritative(const wchar_t *cmd, bool cmd_type, bool authoritative);
+
+/**
+  Remove a previously defined completion
+*/
+void complete_remove(const wchar_t *cmd,
+                     bool cmd_is_path,
+                     wchar_t short_opt,
+                     const wchar_t *long_opt,
+                     int long_mode);
+
+
+/** Find all completions of the command cmd, insert them into out.
+ */
+void complete(const wcstring &cmd,
+              std::vector<completion_t> &comp,
+              completion_request_flags_t flags);
+
+/**
+   Print a list of all current completions into the string.
+
+   \param out The string to write completions to
+*/
+void complete_print(wcstring &out);
+
+/**
+   Tests if the specified option is defined for the specified command
+*/
+int complete_is_valid_option(const wcstring &str,
+                             const wcstring &opt,
+                             wcstring_list_t *inErrorsOrNull,
+                             bool allow_autoload);
+
+/**
+   Tests if the specified argument is valid for the specified option
+   and command
+*/
+bool complete_is_valid_argument(const wcstring &str,
+                                const wcstring &opt,
+                                const wcstring &arg);
+
+
+/**
+   Load command-specific completions for the specified command. This
+   is done automatically whenever completing any given command, so
+   there is no need to call this except in the case of completions
+   with internal dependencies.
+
+   \param cmd the command for which to load command-specific completions
+   \param reload should the commands completions be reloaded, even if they where
+      previously loaded. (This is set to true on actual completions, so that
+      changed completion are updated in running shells)
+*/
+void complete_load(const wcstring &cmd, bool reload);
+
+/**
+   Create a new completion entry
+
+   \param completions The array of completions to append to
+   \param comp The completion string
+   \param desc The description of the completion
+   \param flags completion flags
+
+*/
+void append_completion(std::vector<completion_t> &completions, const wcstring &comp, const wcstring &desc = wcstring(), int flags = 0, string_fuzzy_match_t match = string_fuzzy_match_t(fuzzy_match_exact));
+
+/* Function used for testing */
+void complete_set_variable_names(const wcstring_list_t *names);
+
+/* Support for "wrap targets." A wrap target is a command that completes liek another command. The target chain is the sequence of wraps (A wraps B wraps C...). Any loops in the chain are silently ignored. */
+bool complete_add_wrapper(const wcstring &command, const wcstring &wrap_target);
+bool complete_remove_wrapper(const wcstring &command, const wcstring &wrap_target);
+wcstring_list_t complete_get_wrap_chain(const wcstring &command);
+
+/* Wonky interface: returns all wraps. Even-values are the commands, odd values are the targets. */
+wcstring_list_t complete_get_wrap_pairs();
+
+#endif
diff --git a/src/env.cpp b/src/env.cpp
new file mode 100644
index 00000000..649e0f2a
--- /dev/null
+++ b/src/env.cpp
@@ -0,0 +1,1422 @@
+/** \file env.c
+  Functions for setting and getting environment variables.
+*/
+#include "config.h" // IWYU pragma: keep
+
+#include <stdlib.h>
+#include <wchar.h>
+#include <locale.h>
+#include <unistd.h>
+#include <assert.h>
+#include <sys/stat.h>
+#include <pthread.h>
+#include <pwd.h>
+#include <set>
+#include <map>
+#include <algorithm>
+#include <errno.h>
+#include <stddef.h>
+#include <wctype.h>
+#include <utility>
+#include <vector>
+
+#include "fallback.h"
+
+#include "wutil.h"
+#include "proc.h"
+#include "common.h"
+#include "env.h"
+#include "sanity.h"
+#include "expand.h"
+#include "history.h"
+#include "reader.h"
+#include "env_universal_common.h"
+#include "input.h"
+#include "event.h"
+#include "path.h"
+
+#include "fish_version.h"
+
+/** Value denoting a null string */
+#define ENV_NULL L"\x1d"
+
+/** Some configuration path environment variables */
+#define FISH_DATADIR_VAR L"__fish_datadir"
+#define FISH_SYSCONFDIR_VAR L"__fish_sysconfdir"
+#define FISH_HELPDIR_VAR L"__fish_help_dir"
+#define FISH_BIN_DIR L"__fish_bin_dir"
+
+/**
+   At init, we read all the environment variables from this array.
+*/
+extern char **environ;
+/**
+   This should be the same thing as \c environ, but it is possible only one of the two work...
+*/
+extern char **__environ;
+
+
+bool g_log_forks = false;
+bool g_use_posix_spawn = false; //will usually be set to true
+
+
+/**
+   Struct representing one level in the function variable stack
+*/
+struct env_node_t
+{
+    /**
+      Variable table
+    */
+    var_table_t env;
+    /**
+      Does this node imply a new variable scope? If yes, all
+      non-global variables below this one in the stack are
+      invisible. If new_scope is set for the global variable node,
+      the universe will explode.
+    */
+    bool new_scope;
+    /**
+       Does this node contain any variables which are exported to subshells
+    */
+    bool exportv;
+
+    /**
+      Pointer to next level
+    */
+    struct env_node_t *next;
+
+
+    env_node_t() : new_scope(false), exportv(false), next(NULL) { }
+
+    /* Returns a pointer to the given entry if present, or NULL. */
+    const var_entry_t *find_entry(const wcstring &key);
+
+    /* Returns the next scope to search in order, respecting the new_scope flag, or NULL if we're done. */
+    env_node_t *next_scope_to_search(void);
+};
+
+class variable_entry_t
+{
+    wcstring value; /**< Value of the variable */
+};
+
+static pthread_mutex_t env_lock = PTHREAD_MUTEX_INITIALIZER;
+
+/** Top node on the function stack */
+static env_node_t *top = NULL;
+
+/** Bottom node on the function stack */
+static env_node_t *global_env = NULL;
+
+/** Universal variables global instance. Initialized in env_init. */
+static env_universal_t *s_universal_variables = NULL;
+
+/* Getter for universal variables */
+static env_universal_t *uvars() {
+    return s_universal_variables;
+}
+
+/**
+   Table for global variables
+*/
+static var_table_t *global;
+
+/* Helper class for storing constant strings, without needing to wrap them in a wcstring */
+
+/* Comparer for const string set */
+struct const_string_set_comparer
+{
+    bool operator()(const wchar_t *a, const wchar_t *b)
+    {
+        return wcscmp(a, b) < 0;
+    }
+};
+typedef std::set<const wchar_t *, const_string_set_comparer> const_string_set_t;
+
+/** Table of variables that may not be set using the set command. */
+static const_string_set_t env_read_only;
+
+static bool is_read_only(const wcstring &key)
+{
+    return env_read_only.find(key.c_str()) != env_read_only.end();
+}
+
+/**
+   Table of variables whose value is dynamically calculated, such as umask, status, etc
+*/
+static const_string_set_t env_electric;
+
+static bool is_electric(const wcstring &key)
+{
+    return env_electric.find(key.c_str()) != env_electric.end();
+}
+
+/**
+   Exported variable array used by execv
+*/
+static null_terminated_array_t<char> export_array;
+
+/**
+   Flag for checking if we need to regenerate the exported variable
+   array
+*/
+static bool has_changed_exported = true;
+static void mark_changed_exported()
+{
+    has_changed_exported = true;
+}
+
+/**
+   List of all locale variable names
+*/
+static const wchar_t * const locale_variable[] =
+{
+    L"LANG",
+    L"LC_ALL",
+    L"LC_COLLATE",
+    L"LC_CTYPE",
+    L"LC_MESSAGES",
+    L"LC_MONETARY",
+    L"LC_NUMERIC",
+    L"LC_TIME",
+    NULL
+};
+
+
+const var_entry_t *env_node_t::find_entry(const wcstring &key)
+{
+    const var_entry_t *result = NULL;
+    var_table_t::const_iterator where = env.find(key);
+    if (where != env.end())
+    {
+        result = &where->second;
+    }
+    return result;
+}
+
+env_node_t *env_node_t::next_scope_to_search(void)
+{
+    return this->new_scope ? global_env : this->next;
+}
+
+/**
+   Return the current umask value.
+*/
+static mode_t get_umask()
+{
+    mode_t res;
+    res = umask(0);
+    umask(res);
+    return res;
+}
+
+/** Checks if the specified variable is a locale variable */
+static bool var_is_locale(const wcstring &key)
+{
+    for (size_t i=0; locale_variable[i]; i++)
+    {
+        if (key == locale_variable[i])
+        {
+            return true;
+        }
+    }
+    return false;
+}
+
+/**
+  Properly sets all locale information
+*/
+static void handle_locale()
+{
+    const env_var_t lc_all = env_get_string(L"LC_ALL");
+    const wcstring old_locale = wsetlocale(LC_MESSAGES, NULL);
+
+    /*
+      Array of locale constants corresponding to the local variable names defined in locale_variable
+    */
+    static const int cat[] =
+    {
+        0,
+        LC_ALL,
+        LC_COLLATE,
+        LC_CTYPE,
+        LC_MESSAGES,
+        LC_MONETARY,
+        LC_NUMERIC,
+        LC_TIME
+    }
+    ;
+
+    if (!lc_all.missing())
+    {
+        wsetlocale(LC_ALL, lc_all.c_str());
+    }
+    else
+    {
+        const env_var_t lang = env_get_string(L"LANG");
+        if (!lang.missing())
+        {
+            wsetlocale(LC_ALL, lang.c_str());
+        }
+
+        for (int i=2; locale_variable[i]; i++)
+        {
+            const env_var_t val = env_get_string(locale_variable[i]);
+
+            if (!val.missing())
+            {
+                wsetlocale(cat[i], val.c_str());
+            }
+        }
+    }
+
+    const wcstring new_locale = wsetlocale(LC_MESSAGES, NULL);
+    if (old_locale != new_locale)
+    {
+
+        /*
+           Try to make change known to gettext. Both changing
+           _nl_msg_cat_cntr and calling dcgettext might potentially
+           tell some gettext implementation that the translation
+           strings should be reloaded. We do both and hope for the
+           best.
+        */
+
+        extern int _nl_msg_cat_cntr;
+        _nl_msg_cat_cntr++;
+
+        fish_dcgettext("fish", "Changing language to English", LC_MESSAGES);
+
+        if (get_is_interactive())
+        {
+            debug(2, _(L"Changing language to English"));
+        }
+    }
+}
+
+
+/** React to modifying hte given variable */
+static void react_to_variable_change(const wcstring &key)
+{
+    if (var_is_locale(key))
+    {
+        handle_locale();
+    }
+    else if (key == L"fish_term256" || key == L"fish_term24bit")
+    {
+        update_fish_color_support();
+        reader_react_to_color_change();
+    }
+    else if (string_prefixes_string(L"fish_color_", key))
+    {
+        reader_react_to_color_change();
+    }
+}
+
+/**
+   Universal variable callback function. This function makes sure the
+   proper events are triggered when an event occurs.
+*/
+static void universal_callback(fish_message_type_t type, const wchar_t *name, const wchar_t *val)
+{
+    const wchar_t *str = NULL;
+
+    switch (type)
+    {
+        case SET:
+        case SET_EXPORT:
+        {
+            str=L"SET";
+            break;
+        }
+        
+        case ERASE:
+        {
+            str=L"ERASE";
+            break;
+        }
+        
+        default:
+            break;
+    }
+
+    if (str)
+    {
+        mark_changed_exported();
+
+        event_t ev = event_t::variable_event(name);
+        ev.arguments.push_back(L"VARIABLE");
+        ev.arguments.push_back(str);
+        ev.arguments.push_back(name);
+        event_fire(&ev);
+    }
+
+    if (name)
+        react_to_variable_change(name);
+}
+
+/**
+   Make sure the PATH variable contains something
+*/
+static void setup_path()
+{
+    const env_var_t path = env_get_string(L"PATH");
+    if (path.missing_or_empty())
+    {
+        const wchar_t *value = L"/usr/bin" ARRAY_SEP_STR L"/bin";
+        env_set(L"PATH", value, ENV_GLOBAL | ENV_EXPORT);
+    }
+}
+
+int env_set_pwd()
+{
+    wchar_t dir_path[4096];
+    wchar_t *res = wgetcwd(dir_path, 4096);
+    if (!res)
+    {
+        return 0;
+    }
+    env_set(L"PWD", dir_path, ENV_EXPORT | ENV_GLOBAL);
+    return 1;
+}
+
+wcstring env_get_pwd_slash(void)
+{
+    env_var_t pwd = env_get_string(L"PWD");
+    if (pwd.missing_or_empty())
+    {
+        return L"";
+    }
+    if (! string_suffixes_string(L"/", pwd))
+    {
+        pwd.push_back(L'/');
+    }
+    return pwd;
+}
+
+/* Here is the whitelist of variables that we colon-delimit, both incoming from the environment and outgoing back to it. This is deliberately very short - we don't want to add language-specific values like CLASSPATH. */
+static bool variable_is_colon_delimited_array(const wcstring &str)
+{
+    return contains(str, L"PATH", L"MANPATH", L"CDPATH");
+}
+
+void env_init(const struct config_paths_t *paths /* or NULL */)
+{
+    /*
+      env_read_only variables can not be altered directly by the user
+    */
+
+    const wchar_t * const ro_keys[] =
+    {
+        L"status",
+        L"history",
+        L"version",
+        L"_",
+        L"LINES",
+        L"COLUMNS",
+        L"PWD",
+        //L"SHLVL", // will be inserted a bit lower down
+        L"FISH_VERSION",
+    };
+    for (size_t i=0; i < sizeof ro_keys / sizeof *ro_keys; i++)
+    {
+        env_read_only.insert(ro_keys[i]);
+    }
+
+    /*
+       Names of all dynamically calculated variables
+       */
+    env_electric.insert(L"history");
+    env_electric.insert(L"status");
+    env_electric.insert(L"umask");
+    env_electric.insert(L"COLUMNS");
+    env_electric.insert(L"LINES");
+
+    top = new env_node_t;
+    global_env = top;
+    global = &top->env;
+
+    /*
+      Now the environemnt variable handling is set up, the next step
+      is to insert valid data
+    */
+
+    /*
+      Import environment variables
+    */
+    for (char **p = (environ ? environ : __environ); p && *p; p++)
+    {
+        const wcstring key_and_val = str2wcstring(*p); //like foo=bar
+        size_t eql = key_and_val.find(L'=');
+        if (eql == wcstring::npos)
+        {
+            // no equals found
+            if (is_read_only(key_and_val) || is_electric(key_and_val)) continue;
+            env_set(key_and_val, L"", ENV_EXPORT | ENV_GLOBAL);
+        }
+        else
+        {
+            wcstring key = key_and_val.substr(0, eql);
+            if (is_read_only(key) || is_electric(key)) continue;
+            wcstring val = key_and_val.substr(eql + 1);
+            if (variable_is_colon_delimited_array(key))
+            {
+                std::replace(val.begin(), val.end(), L':', ARRAY_SEP);
+            }
+
+            env_set(key, val.c_str(), ENV_EXPORT | ENV_GLOBAL);
+        }
+    }
+
+    /* Set the given paths in the environment, if we have any */
+    if (paths != NULL)
+    {
+        env_set(FISH_DATADIR_VAR, paths->data.c_str(), ENV_GLOBAL | ENV_EXPORT);
+        env_set(FISH_SYSCONFDIR_VAR, paths->sysconf.c_str(), ENV_GLOBAL | ENV_EXPORT);
+        env_set(FISH_HELPDIR_VAR, paths->doc.c_str(), ENV_GLOBAL | ENV_EXPORT);
+        env_set(FISH_BIN_DIR, paths->bin.c_str(), ENV_GLOBAL | ENV_EXPORT);
+    }
+
+    /*
+      Set up the PATH variable
+    */
+    setup_path();
+
+    /*
+      Set up the USER variable
+    */
+    if (env_get_string(L"USER").missing_or_empty())
+    {
+        const struct passwd *pw = getpwuid(getuid());
+        if (pw && pw->pw_name)
+        {
+            const wcstring uname = str2wcstring(pw->pw_name);
+            env_set(L"USER", uname.c_str(), ENV_GLOBAL | ENV_EXPORT);
+        }
+    }
+
+    /*
+      Set up the version variables
+    */
+    wcstring version = str2wcstring(get_fish_version());
+    env_set(L"version", version.c_str(), ENV_GLOBAL);
+    env_set(L"FISH_VERSION", version.c_str(), ENV_GLOBAL);
+
+    /*
+      Set up SHLVL variable
+    */
+    const env_var_t shlvl_str = env_get_string(L"SHLVL");
+    wcstring nshlvl_str = L"1";
+    if (! shlvl_str.missing())
+    {
+        wchar_t *end;
+        long shlvl_i = wcstol(shlvl_str.c_str(), &end, 10);
+        while (iswspace(*end)) ++end; /* skip trailing whitespace */
+        if (shlvl_i >= 0 && *end == '\0')
+        {
+            nshlvl_str = to_string<long>(shlvl_i + 1);
+        }
+    }
+    env_set(L"SHLVL", nshlvl_str.c_str(), ENV_GLOBAL | ENV_EXPORT);
+    env_read_only.insert(L"SHLVL");
+
+    /* Set up the HOME variable */
+    if (env_get_string(L"HOME").missing_or_empty())
+    {
+        const env_var_t unam = env_get_string(L"USER");
+        char *unam_narrow = wcs2str(unam.c_str());
+        struct passwd *pw = getpwnam(unam_narrow);
+        if (pw->pw_dir != NULL)
+        {
+            const wcstring dir = str2wcstring(pw->pw_dir);
+            env_set(L"HOME", dir.c_str(), ENV_GLOBAL | ENV_EXPORT);
+        }
+        free(unam_narrow);
+    }
+
+    /* Set PWD */
+    env_set_pwd();
+
+    /* Set up universal variables. The empty string means to use the deafult path. */
+    assert(s_universal_variables == NULL);
+    s_universal_variables = new env_universal_t(L"");
+    s_universal_variables->load();
+
+    /* Set g_log_forks */
+    env_var_t log_forks = env_get_string(L"fish_log_forks");
+    g_log_forks = ! log_forks.missing_or_empty() && from_string<bool>(log_forks);
+
+    /* Set g_use_posix_spawn. Default to true. */
+    env_var_t use_posix_spawn = env_get_string(L"fish_use_posix_spawn");
+    g_use_posix_spawn = (use_posix_spawn.missing_or_empty() ? true : from_string<bool>(use_posix_spawn));
+
+    /* Set fish_bind_mode to "default" */
+    env_set(FISH_BIND_MODE_VAR, DEFAULT_BIND_MODE, ENV_GLOBAL);
+
+    /*
+      Now that the global scope is fully initialized, add a toplevel local
+      scope. This same local scope will persist throughout the lifetime of the
+      fish process, and it will ensure that `set -l` commands run at the
+      command-line don't affect the global scope.
+    */
+    env_push(false);
+}
+
+/**
+   Search all visible scopes in order for the specified key. Return
+   the first scope in which it was found.
+*/
+static env_node_t *env_get_node(const wcstring &key)
+{
+    env_node_t *env = top;
+    while (env != NULL)
+    {
+        if (env->find_entry(key) != NULL)
+        {
+            break;
+        }
+
+        env = env->next_scope_to_search();
+    }
+    return env;
+}
+
+int env_set(const wcstring &key, const wchar_t *val, env_mode_flags_t var_mode)
+{
+    ASSERT_IS_MAIN_THREAD();
+    bool has_changed_old = has_changed_exported;
+    bool has_changed_new = false;
+    int done=0;
+
+    if (val && contains(key, L"PWD", L"HOME"))
+    {
+        /* Canoncalize our path; if it changes, recurse and try again. */
+        wcstring val_canonical = val;
+        path_make_canonical(val_canonical);
+        if (val != val_canonical)
+        {
+            return env_set(key, val_canonical.c_str(), var_mode);
+        }
+    }
+
+    if ((var_mode & (ENV_LOCAL | ENV_UNIVERSAL)) && (is_read_only(key) || is_electric(key)))
+    {
+        return ENV_SCOPE;
+    }
+    if ((var_mode & ENV_EXPORT) && is_electric(key))
+    {
+        return ENV_SCOPE;
+    }
+
+    if ((var_mode & ENV_USER) && is_read_only(key))
+    {
+        return ENV_PERM;
+    }
+
+    if (key == L"umask")
+    {
+        wchar_t *end;
+
+        /*
+         Set the new umask
+         */
+        if (val && wcslen(val))
+        {
+            errno=0;
+            long mask = wcstol(val, &end, 8);
+
+            if (!errno && (!*end) && (mask <= 0777) && (mask >= 0))
+            {
+                umask(mask);
+                /* Do not actually create a umask variable, on env_get, it will be calculated dynamically */
+                return 0;
+            }
+        }
+
+        return ENV_INVALID;
+    }
+
+    /*
+     Zero element arrays are internaly not coded as null but as this
+     placeholder string
+     */
+    if (!val)
+    {
+        val = ENV_NULL;
+    }
+
+    if (var_mode & ENV_UNIVERSAL)
+    {
+        const bool old_export = uvars() && uvars()->get_export(key);
+        bool new_export;
+        if (var_mode & ENV_EXPORT)
+        {
+            // export
+            new_export = true;
+        }
+        else if (var_mode & ENV_UNEXPORT)
+        {
+            // unexport
+            new_export = false;
+        }
+        else
+        {
+            // not changing the export
+            new_export = old_export;
+        }
+        if (uvars())
+        {
+            uvars()->set(key, val, new_export);
+            env_universal_barrier();
+            if (old_export || new_export)
+            {
+                mark_changed_exported();
+            }
+        }
+    }
+    else
+    {
+        // Determine the node
+
+        env_node_t *preexisting_node = env_get_node(key);
+        bool preexisting_entry_exportv = false;
+        if (preexisting_node != NULL)
+        {
+            var_table_t::const_iterator result = preexisting_node->env.find(key);
+            assert(result != preexisting_node->env.end());
+            const var_entry_t &entry = result->second;
+            if (entry.exportv)
+            {
+                preexisting_entry_exportv = true;
+                has_changed_new = true;
+            }
+        }
+
+        env_node_t *node = NULL;
+        if (var_mode & ENV_GLOBAL)
+        {
+            node = global_env;
+        }
+        else if (var_mode & ENV_LOCAL)
+        {
+            node = top;
+        }
+        else if (preexisting_node != NULL)
+        {
+            node = preexisting_node;
+
+            if ((var_mode & (ENV_EXPORT | ENV_UNEXPORT)) == 0)
+            {
+                // use existing entry's exportv
+                var_mode = preexisting_entry_exportv ? ENV_EXPORT : 0;
+            }
+        }
+        else
+        {
+            if (! get_proc_had_barrier())
+            {
+                set_proc_had_barrier(true);
+                env_universal_barrier();
+            }
+
+            if (uvars() && ! uvars()->get(key).missing())
+            {
+                bool exportv;
+                if (var_mode & ENV_EXPORT)
+                {
+                    exportv = true;
+                }
+                else if (var_mode & ENV_UNEXPORT)
+                {
+                    exportv = false;
+                }
+                else
+                {
+                    exportv = uvars()->get_export(key);
+                }
+
+                uvars()->set(key, val, exportv);
+                env_universal_barrier();
+
+                done = 1;
+
+            }
+            else
+            {
+                /*
+                 New variable with unspecified scope. The default
+                 scope is the innermost scope that is shadowing,
+                 which will be either the current function or the
+                 global scope.
+                 */
+                node = top;
+                while (node->next && !node->new_scope)
+                {
+                    node = node->next;
+                }
+            }
+        }
+
+        if (!done)
+        {
+            // Set the entry in the node
+            // Note that operator[] accesses the existing entry, or creates a new one
+            var_entry_t &entry = node->env[key];
+            if (entry.exportv)
+            {
+                // this variable already existed, and was exported
+                has_changed_new = true;
+            }
+            entry.val = val;
+            if (var_mode & ENV_EXPORT)
+            {
+                // the new variable is exported
+                entry.exportv = true;
+                node->exportv = true;
+                has_changed_new = true;
+            }
+            else
+            {
+                entry.exportv = false;
+            }
+
+            if (has_changed_old || has_changed_new)
+                mark_changed_exported();
+        }
+    }
+
+    event_t ev = event_t::variable_event(key);
+    ev.arguments.reserve(3);
+    ev.arguments.push_back(L"VARIABLE");
+    ev.arguments.push_back(L"SET");
+    ev.arguments.push_back(key);
+
+    //  debug( 1, L"env_set: fire events on variable %ls", key );
+    event_fire(&ev);
+    //  debug( 1, L"env_set: return from event firing" );
+
+    react_to_variable_change(key);
+
+    return 0;
+}
+
+
+/**
+   Attempt to remove/free the specified key/value pair from the
+   specified map.
+
+   \return zero if the variable was not found, non-zero otherwise
+*/
+static bool try_remove(env_node_t *n, const wchar_t *key, int var_mode)
+{
+    if (n == NULL)
+    {
+        return false;
+    }
+
+    var_table_t::iterator result = n->env.find(key);
+    if (result != n->env.end())
+    {
+        if (result->second.exportv)
+        {
+            mark_changed_exported();
+        }
+        n->env.erase(result);
+        return true;
+    }
+
+    if (var_mode & ENV_LOCAL)
+    {
+        return false;
+    }
+
+    if (n->new_scope)
+    {
+        return try_remove(global_env, key, var_mode);
+    }
+    else
+    {
+        return try_remove(n->next, key, var_mode);
+    }
+}
+
+
+int env_remove(const wcstring &key, int var_mode)
+{
+    ASSERT_IS_MAIN_THREAD();
+    env_node_t *first_node;
+    int erased = 0;
+
+    if ((var_mode & ENV_USER) && is_read_only(key))
+    {
+        return 2;
+    }
+
+    first_node = top;
+
+    if (!(var_mode & ENV_UNIVERSAL))
+    {
+
+        if (var_mode & ENV_GLOBAL)
+        {
+            first_node = global_env;
+        }
+
+        if (try_remove(first_node, key.c_str(), var_mode))
+        {
+            event_t ev = event_t::variable_event(key);
+            ev.arguments.push_back(L"VARIABLE");
+            ev.arguments.push_back(L"ERASE");
+            ev.arguments.push_back(key);
+            event_fire(&ev);
+
+            erased = 1;
+        }
+    }
+
+    if (!erased &&
+            !(var_mode & ENV_GLOBAL) &&
+            !(var_mode & ENV_LOCAL))
+    {
+        bool is_exported = uvars()->get_export(key);
+        erased = uvars() && uvars()->remove(key);
+        if (erased)
+        {
+            env_universal_barrier();
+            event_t ev = event_t::variable_event(key);
+            ev.arguments.push_back(L"VARIABLE");
+            ev.arguments.push_back(L"ERASE");
+            ev.arguments.push_back(key);
+            event_fire(&ev);
+        }
+        
+        if (is_exported)
+            mark_changed_exported();
+    }
+
+    react_to_variable_change(key);
+
+    return !erased;
+}
+
+const wchar_t *env_var_t::c_str(void) const
+{
+    assert(! is_missing);
+    return wcstring::c_str();
+}
+
+env_var_t env_get_string(const wcstring &key, env_mode_flags_t mode)
+{
+    const bool has_scope = mode & (ENV_LOCAL | ENV_GLOBAL | ENV_UNIVERSAL);
+    const bool search_local = !has_scope || (mode & ENV_LOCAL);
+    const bool search_global = !has_scope || (mode & ENV_GLOBAL);
+    const bool search_universal = !has_scope || (mode & ENV_UNIVERSAL);
+
+    const bool search_exported = (mode & ENV_EXPORT) || !(mode & ENV_UNEXPORT);
+    const bool search_unexported = (mode & ENV_UNEXPORT) || !(mode & ENV_EXPORT);
+
+    /* Make the assumption that electric keys can't be shadowed elsewhere, since we currently block that in env_set() */
+    if (is_electric(key))
+    {
+        if (!search_global) return env_var_t::missing_var();
+        /* Big hack...we only allow getting the history on the main thread. Note that history_t may ask for an environment variable, so don't take the lock here (we don't need it) */
+        if (key == L"history" && is_main_thread())
+        {
+            env_var_t result;
+
+            history_t *history = reader_get_history();
+            if (! history)
+            {
+                history = &history_t::history_with_name(L"fish");
+            }
+            if (history)
+                history->get_string_representation(&result, ARRAY_SEP_STR);
+            return result;
+        }
+        else if (key == L"COLUMNS")
+        {
+            return to_string(common_get_width());
+        }
+        else if (key == L"LINES")
+        {
+            return to_string(common_get_height());
+        }
+        else if (key == L"status")
+        {
+            return to_string(proc_get_last_status());
+        }
+        else if (key == L"umask")
+        {
+            return format_string(L"0%0.3o", get_umask());
+        }
+        // we should never get here unless the electric var list is out of sync
+    }
+
+    if (search_local || search_global) {
+        /* Lock around a local region */
+        scoped_lock lock(env_lock);
+
+        env_node_t *env = search_local ? top : global_env;
+
+        while (env != NULL)
+        {
+            const var_entry_t *entry = env->find_entry(key);
+            if (entry != NULL && (entry->exportv ? search_exported : search_unexported))
+            {
+                if (entry->val == ENV_NULL)
+                {
+                    return env_var_t::missing_var();
+                }
+                else
+                {
+                    return entry->val;
+                }
+            }
+
+            if (has_scope)
+            {
+                if (!search_global || env == global_env) break;
+                env = global_env;
+            }
+            else
+            {
+                env = env->next_scope_to_search();
+            }
+        }
+    }
+
+    if (!search_universal) return env_var_t::missing_var();
+
+    /* Another big hack - only do a universal barrier on the main thread (since it can change variable values)
+        Make sure we do this outside the env_lock because it may itself call env_get_string */
+    if (is_main_thread() && ! get_proc_had_barrier())
+    {
+        set_proc_had_barrier(true);
+        env_universal_barrier();
+    }
+
+    if (uvars())
+    {
+        env_var_t env_var = uvars()->get(key);
+        if (env_var == ENV_NULL || !(uvars()->get_export(key) ? search_exported : search_unexported))
+        {
+            env_var = env_var_t::missing_var();
+        }
+        return env_var;
+    }
+    return env_var_t::missing_var();
+}
+
+bool env_exist(const wchar_t *key, env_mode_flags_t mode)
+{
+    env_node_t *env;
+
+    CHECK(key, false);
+
+    const bool has_scope = mode & (ENV_LOCAL | ENV_GLOBAL | ENV_UNIVERSAL);
+    const bool test_local = !has_scope || (mode & ENV_LOCAL);
+    const bool test_global = !has_scope || (mode & ENV_GLOBAL);
+    const bool test_universal = !has_scope || (mode & ENV_UNIVERSAL);
+
+    const bool test_exported = (mode & ENV_EXPORT) || !(mode & ENV_UNEXPORT);
+    const bool test_unexported = (mode & ENV_UNEXPORT) || !(mode & ENV_EXPORT);
+
+    if (is_electric(key))
+    {
+        /*
+        Electric variables all exist, and they are all global. A local or
+        universal version can not exist. They are also never exported.
+        */
+        if (test_global && test_unexported)
+        {
+            return true;
+        }
+        return false;
+    }
+
+    if (test_local || test_global)
+    {
+        env = test_local ? top : global_env;
+
+        while (env)
+        {
+            var_table_t::iterator result = env->env.find(key);
+
+            if (result != env->env.end())
+            {
+                const var_entry_t &res = result->second;
+                return res.exportv ? test_exported : test_unexported;
+            }
+
+            if (has_scope)
+            {
+                if (!test_global || env == global_env) break;
+                env = global_env;
+            }
+            else
+            {
+                env = env->next_scope_to_search();
+            }
+        }
+    }
+
+    if (test_universal)
+    {
+        if (! get_proc_had_barrier())
+        {
+            set_proc_had_barrier(true);
+            env_universal_barrier();
+        }
+
+        if (uvars() && ! uvars()->get(key).missing())
+        {
+            return uvars()->get_export(key) ? test_exported : test_unexported;
+        }
+    }
+
+    return 0;
+}
+
+/**
+   Returns true if the specified scope or any non-shadowed non-global subscopes contain an exported variable.
+*/
+static int local_scope_exports(env_node_t *n)
+{
+
+    if (n==global_env)
+        return 0;
+
+    if (n->exportv)
+        return 1;
+
+    if (n->new_scope)
+        return 0;
+
+    return local_scope_exports(n->next);
+}
+
+void env_push(bool new_scope)
+{
+    env_node_t *node = new env_node_t;
+    node->next = top;
+    node->new_scope=new_scope;
+
+    if (new_scope)
+    {
+        if (local_scope_exports(top))
+            mark_changed_exported();
+    }
+    top = node;
+
+}
+
+
+void env_pop()
+{
+    if (&top->env != global)
+    {
+        int i;
+        int locale_changed = 0;
+
+        env_node_t *killme = top;
+
+        for (i=0; locale_variable[i]; i++)
+        {
+            var_table_t::iterator result =  killme->env.find(locale_variable[i]);
+            if (result != killme->env.end())
+            {
+                locale_changed = 1;
+                break;
+            }
+        }
+
+        if (killme->new_scope)
+        {
+            if (killme->exportv || local_scope_exports(killme->next))
+                mark_changed_exported();
+        }
+
+        top = top->next;
+
+        var_table_t::iterator iter;
+        for (iter = killme->env.begin(); iter != killme->env.end(); ++iter)
+        {
+            const var_entry_t &entry = iter->second;
+            if (entry.exportv)
+            {
+                mark_changed_exported();
+                break;
+            }
+        }
+
+        delete killme;
+
+        if (locale_changed)
+            handle_locale();
+
+    }
+    else
+    {
+        debug(0,
+              _(L"Tried to pop empty environment stack."));
+        sanity_lose();
+    }
+}
+
+/**
+   Function used with to insert keys of one table into a set::set<wcstring>
+*/
+static void add_key_to_string_set(const var_table_t &envs, std::set<wcstring> *str_set, bool show_exported, bool show_unexported)
+{
+    var_table_t::const_iterator iter;
+    for (iter = envs.begin(); iter != envs.end(); ++iter)
+    {
+        const var_entry_t &e = iter->second;
+
+        if ((e.exportv && show_exported) ||
+                (!e.exportv && show_unexported))
+        {
+            /* Insert this key */
+            str_set->insert(iter->first);
+        }
+
+    }
+}
+
+wcstring_list_t env_get_names(int flags)
+{
+    scoped_lock lock(env_lock);
+
+    wcstring_list_t result;
+    std::set<wcstring> names;
+    int show_local = flags & ENV_LOCAL;
+    int show_global = flags & ENV_GLOBAL;
+    int show_universal = flags & ENV_UNIVERSAL;
+
+    env_node_t *n=top;
+    const bool show_exported = (flags & ENV_EXPORT) || !(flags & ENV_UNEXPORT);
+    const bool show_unexported = (flags & ENV_UNEXPORT) || !(flags & ENV_EXPORT);
+
+    if (!show_local && !show_global && !show_universal)
+    {
+        show_local =show_universal = show_global=1;
+    }
+
+    if (show_local)
+    {
+        while (n)
+        {
+            if (n == global_env)
+                break;
+
+            add_key_to_string_set(n->env, &names, show_exported, show_unexported);
+            if (n->new_scope)
+                break;
+            else
+                n = n->next;
+
+        }
+    }
+
+    if (show_global)
+    {
+        add_key_to_string_set(global_env->env, &names, show_exported, show_unexported);
+        if (show_unexported)
+        {
+            result.insert(result.end(), env_electric.begin(), env_electric.end());
+        }
+    }
+
+    if (show_universal && uvars())
+    {
+        const wcstring_list_t uni_list = uvars()->get_names(show_exported, show_unexported);
+        names.insert(uni_list.begin(), uni_list.end());
+    }
+
+    result.insert(result.end(), names.begin(), names.end());
+    return result;
+}
+
+/**
+  Get list of all exported variables
+*/
+
+static void get_exported(const env_node_t *n, std::map<wcstring, wcstring> *h)
+{
+    if (!n)
+        return;
+
+    if (n->new_scope)
+        get_exported(global_env, h);
+    else
+        get_exported(n->next, h);
+
+    var_table_t::const_iterator iter;
+    for (iter = n->env.begin(); iter != n->env.end(); ++iter)
+    {
+        const wcstring &key = iter->first;
+        const var_entry_t &val_entry = iter->second;
+
+        if (val_entry.exportv && val_entry.val != ENV_NULL)
+        {
+            // Export the variable
+            // Don't use std::map::insert here, since we need to overwrite existing
+            // values from previous scopes
+            (*h)[key] = val_entry.val;
+        }
+        else
+        {
+            // We need to erase from the map if we are not exporting,
+            // since a lower scope may have exported. See #2132
+            h->erase(key);
+        }
+    }
+}
+
+/* Given a map from key to value, add values to out of the form key=value */
+static void export_func(const std::map<wcstring, wcstring> &envs, std::vector<std::string> &out)
+{
+    out.reserve(out.size() + envs.size());
+    std::map<wcstring, wcstring>::const_iterator iter;
+    for (iter = envs.begin(); iter != envs.end(); ++iter)
+    {
+        const wcstring &key = iter->first;
+        const std::string &ks = wcs2string(key);
+        std::string vs = wcs2string(iter->second);
+
+        /* Arrays in the value are ASCII record separator (0x1e) delimited. But some variables should have colons. Add those. */
+        if (variable_is_colon_delimited_array(key))
+        {
+            /* Replace ARRAY_SEP with colon */
+            std::replace(vs.begin(), vs.end(), (char)ARRAY_SEP, ':');
+        }
+
+        /* Put a string on the vector */
+        out.push_back(std::string());
+        std::string &str = out.back();
+        str.reserve(ks.size() + 1 + vs.size());
+
+        /* Append our environment variable data to it */
+        str.append(ks);
+        str.append("=");
+        str.append(vs);
+    }
+}
+
+static void update_export_array_if_necessary(bool recalc)
+{
+    ASSERT_IS_MAIN_THREAD();
+    if (recalc && ! get_proc_had_barrier())
+    {
+        set_proc_had_barrier(true);
+        env_universal_barrier();
+    }
+
+    if (has_changed_exported)
+    {
+        std::map<wcstring, wcstring> vals;
+        size_t i;
+
+        debug(4, L"env_export_arr() recalc");
+
+        get_exported(top, &vals);
+
+        if (uvars())
+        {
+            const wcstring_list_t uni = uvars()->get_names(true, false);
+            for (i=0; i<uni.size(); i++)
+            {
+                const wcstring &key = uni.at(i);
+                const env_var_t val = uvars()->get(key);
+
+                if (! val.missing() && val != ENV_NULL)
+                {
+                    // Note that std::map::insert does NOT overwrite a value already in the map,
+                    // which we depend on here
+                    vals.insert(std::pair<wcstring, wcstring>(key, val));
+                }
+            }
+        }
+
+        std::vector<std::string> local_export_buffer;
+        export_func(vals, local_export_buffer);
+        export_array.set(local_export_buffer);
+        has_changed_exported=false;
+    }
+
+}
+
+const char * const *env_export_arr(bool recalc)
+{
+    ASSERT_IS_MAIN_THREAD();
+    update_export_array_if_necessary(recalc);
+    return export_array.get();
+}
+
+env_vars_snapshot_t::env_vars_snapshot_t(const wchar_t * const *keys)
+{
+    ASSERT_IS_MAIN_THREAD();
+    wcstring key;
+    for (size_t i=0; keys[i]; i++)
+    {
+        key.assign(keys[i]);
+        const env_var_t val = env_get_string(key);
+        if (! val.missing())
+        {
+            vars[key] = val;
+        }
+    }
+}
+
+
+void env_universal_barrier()
+{
+    ASSERT_IS_MAIN_THREAD();
+    if (uvars())
+    {
+        callback_data_list_t changes;
+        bool changed = uvars()->sync(&changes);
+        if (changed)
+        {
+            universal_notifier_t::default_notifier().post_notification();
+        }
+        
+        /* Post callbacks */
+        for (size_t i=0; i < changes.size(); i++)
+        {
+            const callback_data_t &data = changes.at(i);
+            universal_callback(data.type, data.key.c_str(), data.val.c_str());
+        }
+    }
+}
+
+env_vars_snapshot_t::env_vars_snapshot_t() { }
+
+/* The "current" variables are not a snapshot at all, but instead trampoline to env_get_string, etc. We identify the current snapshot based on pointer values. */
+static const env_vars_snapshot_t sCurrentSnapshot;
+const env_vars_snapshot_t &env_vars_snapshot_t::current()
+{
+    return sCurrentSnapshot;
+}
+
+bool env_vars_snapshot_t::is_current() const
+{
+    return this == &sCurrentSnapshot;
+}
+
+env_var_t env_vars_snapshot_t::get(const wcstring &key) const
+{
+    /* If we represent the current state, bounce to env_get_string */
+    if (this->is_current())
+    {
+        return env_get_string(key);
+    }
+    else
+    {
+        std::map<wcstring, wcstring>::const_iterator iter = vars.find(key);
+        return (iter == vars.end() ? env_var_t::missing_var() : env_var_t(iter->second));
+    }
+}
+
+const wchar_t * const env_vars_snapshot_t::highlighting_keys[] = {L"PATH", L"CDPATH", L"fish_function_path", NULL};
diff --git a/src/env.h b/src/env.h
new file mode 100644
index 00000000..411b51ec
--- /dev/null
+++ b/src/env.h
@@ -0,0 +1,260 @@
+/** \file env.h
+  Prototypes for functions for setting and getting environment variables.
+*/
+
+#ifndef FISH_ENV_H
+#define FISH_ENV_H
+
+#include <wchar.h>
+#include <stdint.h>
+#include <string>
+#include <map>
+
+#include "common.h"
+
+/* Flags that may be passed as the 'mode' in env_set / env_get_string */
+enum
+{
+    /* Default mode */
+    ENV_DEFAULT = 0,
+    
+    /** Flag for local (to the current block) variable */
+    ENV_LOCAL = 1,
+    
+    /** Flag for exported (to commands) variable */
+    ENV_EXPORT = 2,
+    
+    /** Flag for unexported variable */
+    ENV_UNEXPORT = 16,
+    
+    /** Flag for global variable */
+    ENV_GLOBAL = 4,
+    
+    /** Flag for variable update request from the user. All variable
+       changes that are made directly by the user, such as those from the
+       'set' builtin must have this flag set. */
+    ENV_USER = 8,
+    
+    /** Flag for universal variable */
+    ENV_UNIVERSAL = 32
+};
+typedef uint32_t env_mode_flags_t;
+
+/**
+   Error code for trying to alter read-only variable
+*/
+enum
+{
+    ENV_PERM = 1,
+    ENV_SCOPE,
+    ENV_INVALID
+}
+;
+
+/* A struct of configuration directories, determined in main() that fish will optionally pass to env_init.
+ */
+struct config_paths_t
+{
+    wcstring data;      // e.g. /usr/local/share
+    wcstring sysconf;   // e.g. /usr/local/etc
+    wcstring doc;       // e.g. /usr/local/share/doc/fish
+    wcstring bin;       // e.g. /usr/local/bin
+};
+
+/**
+   Initialize environment variable data
+*/
+void env_init(const struct config_paths_t *paths = NULL);
+
+/**
+   Set the value of the environment variable whose name matches key to val.
+
+   Memory policy: All keys and values are copied, the parameters can and should be freed by the caller afterwards
+
+   \param key The key
+   \param val The value
+   \param mode The type of the variable. Can be any combination of ENV_GLOBAL, ENV_LOCAL, ENV_EXPORT and ENV_USER. If mode is zero, the current variable space is searched and the current mode is used. If no current variable with the same name is found, ENV_LOCAL is assumed.
+
+   \returns 0 on suicess or an error code on failiure.
+
+   The current error codes are:
+
+   * ENV_PERM, can only be returned when setting as a user, e.g. ENV_USER is set. This means that the user tried to change a read-only variable.
+   * ENV_SCOPE, the variable cannot be set in the given scope. This applies to readonly/electric variables set from the local or universal scopes, or set as exported.
+   * ENV_INVALID, the variable value was invalid. This applies only to special variables.
+*/
+
+int env_set(const wcstring &key, const wchar_t *val, env_mode_flags_t mode);
+
+
+/**
+  Return the value of the variable with the specified name.  Returns 0
+  if the key does not exist.  The returned string should not be
+  modified or freed. The returned string is only guaranteed to be
+  valid until the next call to env_get(), env_set(), env_push() or
+  env_pop() takes place.
+*/
+//const wchar_t *env_get( const wchar_t *key );
+
+class env_var_t : public wcstring
+{
+private:
+    bool is_missing;
+public:
+    static env_var_t missing_var(void)
+    {
+        env_var_t result(L"");
+        result.is_missing = true;
+        return result;
+
+    }
+
+    env_var_t(const env_var_t &x) : wcstring(x), is_missing(x.is_missing) { }
+    env_var_t(const wcstring & x) : wcstring(x), is_missing(false) { }
+    env_var_t(const wchar_t *x) : wcstring(x), is_missing(false) { }
+    env_var_t() : wcstring(L""), is_missing(false) { }
+
+    bool missing(void) const
+    {
+        return is_missing;
+    }
+
+    bool missing_or_empty(void) const
+    {
+        return missing() || empty();
+    }
+
+    const wchar_t *c_str(void) const;
+
+    env_var_t &operator=(const env_var_t &s)
+    {
+        is_missing = s.is_missing;
+        wcstring::operator=(s);
+        return *this;
+    }
+
+    bool operator==(const env_var_t &s) const
+    {
+        return is_missing == s.is_missing && static_cast<const wcstring &>(*this) == static_cast<const wcstring &>(s);
+    }
+
+    bool operator==(const wcstring &s) const
+    {
+        return ! is_missing && static_cast<const wcstring &>(*this) == s;
+    }
+
+    bool operator!=(const env_var_t &s) const
+    {
+        return !(*this == s);
+    }
+
+    bool operator!=(const wcstring &s) const
+    {
+        return !(*this == s);
+    }
+
+    bool operator==(const wchar_t *s) const
+    {
+        return ! is_missing && static_cast<const wcstring &>(*this) == s;
+    }
+
+    bool operator!=(const wchar_t *s) const
+    {
+        return !(*this == s);
+    }
+
+
+};
+
+/**
+   Gets the variable with the specified name, or env_var_t::missing_var if it does not exist or is an empty array.
+
+   \param key The name of the variable to get
+   \param mode An optional scope to search in. All scopes are searched if unset
+*/
+env_var_t env_get_string(const wcstring &key, env_mode_flags_t mode = ENV_DEFAULT);
+
+/**
+   Returns true if the specified key exists. This can't be reliably done
+   using env_get, since env_get returns null for 0-element arrays
+
+   \param key The name of the variable to remove
+   \param mode the scope to search in. All scopes are searched if set to default
+*/
+bool env_exist(const wchar_t *key, env_mode_flags_t mode);
+
+/**
+   Remove environemnt variable
+
+   \param key The name of the variable to remove
+   \param mode should be ENV_USER if this is a remove request from the user, 0 otherwise. If this is a user request, read-only variables can not be removed. The mode may also specify the scope of the variable that should be erased.
+
+   \return zero if the variable existed, and non-zero if the variable did not exist
+*/
+int env_remove(const wcstring &key, int mode);
+
+/**
+  Push the variable stack. Used for implementing local variables for functions and for-loops.
+*/
+void env_push(bool new_scope);
+
+/**
+  Pop the variable stack. Used for implementing local variables for functions and for-loops.
+*/
+void env_pop();
+
+/** Synchronizes all universal variable changes: writes everything out, reads stuff in */
+void env_universal_barrier();
+
+/** Returns an array containing all exported variables in a format suitable for execv. */
+const char * const * env_export_arr(bool recalc);
+
+/**
+  Returns all variable names.
+*/
+wcstring_list_t env_get_names(int flags);
+
+/** Update the PWD variable directory */
+int env_set_pwd();
+
+/* Returns the PWD with a terminating slash */
+wcstring env_get_pwd_slash();
+
+class env_vars_snapshot_t
+{
+    std::map<wcstring, wcstring> vars;
+    bool is_current() const;
+
+public:
+    env_vars_snapshot_t(const wchar_t * const * keys);
+    env_vars_snapshot_t(void);
+
+    env_var_t get(const wcstring &key) const;
+
+    // Returns the fake snapshot representing the live variables array
+    static const env_vars_snapshot_t &current();
+
+    // vars necessary for highlighting
+    static const wchar_t * const highlighting_keys[];
+};
+
+extern bool g_log_forks;
+extern int g_fork_count;
+
+extern bool g_use_posix_spawn;
+
+/**
+ A variable entry. Stores the value of a variable and whether it
+ should be exported.
+ */
+struct var_entry_t
+{
+    wcstring val; /**< The value of the variable */
+    bool exportv; /**< Whether the variable should be exported */
+    
+    var_entry_t() : exportv(false) { }
+};
+
+typedef std::map<wcstring, var_entry_t> var_table_t;
+
+#endif
diff --git a/src/env_universal_common.cpp b/src/env_universal_common.cpp
new file mode 100644
index 00000000..d29381f5
--- /dev/null
+++ b/src/env_universal_common.cpp
@@ -0,0 +1,1709 @@
+/**
+   \file env_universal_common.c
+
+   The utility library for universal variables. Used both by the
+   client library and by the daemon.
+
+*/
+#include "config.h"
+
+#include "env_universal_common.h"
+
+#include <fcntl.h>
+#include <sys/ioctl.h>
+#include <sys/mman.h>
+#include <sys/file.h>
+#include <sys/socket.h>
+#include <arpa/inet.h> // IWYU pragma: keep - needed for htonl
+#include <pwd.h>
+#include <assert.h>
+#include <errno.h>
+#include <limits.h>
+#include <stdarg.h>
+#include <stddef.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/stat.h>
+#include <sys/time.h>
+#include <sys/types.h>
+#include <unistd.h>
+#include <wchar.h>
+#include <wctype.h>
+#include <map>
+#include <utility>
+
+#ifdef HAVE_SYS_SELECT_H
+#include <sys/select.h>
+#endif
+
+#include "fallback.h" // IWYU pragma: keep
+#include "util.h"
+
+#include "common.h"
+#include "wutil.h"
+#include "utf8.h"
+
+#if __APPLE__
+#define FISH_NOTIFYD_AVAILABLE 1
+#include <notify.h>
+#endif
+
+// NAME_MAX is not defined on Solaris and suggests the use of pathconf()
+// There is no obvious sensible pathconf() for shared memory and _XPG_NAME_MAX
+// seems a reasonable choice.
+#if !defined(NAME_MAX) && defined(_XOPEN_NAME_MAX)
+#define NAME_MAX _XOPEN_NAME_MAX
+#endif
+
+/**
+   The set command
+*/
+#define SET_STR L"SET"
+
+/**
+   The set_export command
+*/
+#define SET_EXPORT_STR L"SET_EXPORT"
+
+
+/**
+   Non-wide version of the set command
+*/
+#define SET_MBS "SET"
+
+/**
+   Non-wide version of the set_export command
+*/
+#define SET_EXPORT_MBS "SET_EXPORT"
+
+/**
+   Error message
+*/
+#define PARSE_ERR L"Unable to parse universal variable message: '%ls'"
+
+/** Small note about not editing ~/.fishd manually. Inserted at the top of all .fishd files. */
+#define SAVE_MSG "# This file is automatically generated by the fish.\n# Do NOT edit it directly, your changes will be overwritten.\n"
+
+static wcstring fishd_get_config();
+static wcstring get_machine_identifier();
+static bool get_hostname_identifier(wcstring *result);
+
+static wcstring vars_filename_in_directory(const wcstring &wdir)
+{
+    if (wdir.empty())
+        return L"";
+    
+    wcstring result = wdir;
+    result.append(L"/fishd.");
+    result.append(get_machine_identifier());
+    return result;
+}
+
+static const wcstring &default_vars_path()
+{
+    static wcstring cached_result = vars_filename_in_directory(fishd_get_config());
+    return cached_result;
+}
+
+/**
+   Check, and create if necessary, a secure runtime path
+   Derived from tmux.c in tmux (http://tmux.sourceforge.net/)
+*/
+static int check_runtime_path(const char * path)
+{
+    /*
+     * Copyright (c) 2007 Nicholas Marriott <nicm@users.sourceforge.net>
+     *
+     * Permission to use, copy, modify, and distribute this software for any
+     * purpose with or without fee is hereby granted, provided that the above
+     * copyright notice and this permission notice appear in all copies.
+     *
+     * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+     * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+     * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+     * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+     * WHATSOEVER RESULTING FROM LOSS OF MIND, USE, DATA OR PROFITS, WHETHER
+     * IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING
+     * OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+     */
+
+    struct stat statpath;
+    u_int uid = geteuid();
+
+    if (mkdir(path, S_IRWXU) != 0 && errno != EEXIST)
+        return errno;
+    if (lstat(path, &statpath) != 0)
+        return errno;
+    if (!S_ISDIR(statpath.st_mode)
+            || statpath.st_uid != uid
+            || (statpath.st_mode & (S_IRWXG|S_IRWXO)) != 0)
+        return EACCES;
+    return 0;
+}
+
+/** Return the path of an appropriate runtime data directory */
+static wcstring get_runtime_path()
+{
+    wcstring result;
+    const char *dir = getenv("XDG_RUNTIME_DIR");
+    if (dir != NULL)
+    {
+        result = str2wcstring(dir);
+    }
+    else
+    {
+        const char *uname = getenv("USER");
+        if (uname == NULL)
+        {
+            const struct passwd *pw = getpwuid(getuid());
+            uname = pw->pw_name;
+        }
+
+        // /tmp/fish.user
+        std::string tmpdir = "/tmp/fish.";
+        tmpdir.append(uname);
+        if (check_runtime_path(tmpdir.c_str()) != 0)
+        {
+            debug(0, L"Runtime path not available. Try deleting the directory %s and restarting fish.", tmpdir.c_str());
+        }
+        else
+        {
+            result = str2wcstring(tmpdir);
+        }
+    }
+    return result;
+}
+
+
+/* Returns a "variables" file in the appropriate runtime directory. This is called infrequently and so does not need to be cached. */
+static wcstring default_named_pipe_path()
+{
+    // Note that vars_filename_in_directory returns empty string when passed the empty string
+    return vars_filename_in_directory(get_runtime_path());
+}
+
+/**
+   Test if the message msg contains the command cmd
+*/
+static bool match(const wchar_t *msg, const wchar_t *cmd)
+{
+    size_t len = wcslen(cmd);
+    if (wcsncasecmp(msg, cmd, len) != 0)
+        return false;
+
+    if (msg[len] && msg[len]!= L' ' && msg[len] != L'\t')
+        return false;
+
+    return true;
+}
+
+static void report_error(int err_code, const wchar_t *err_format, ...)
+{
+    va_list va;
+    va_start(va, err_format);
+    const wcstring err_text = vformat_string(err_format, va);
+    va_end(va);
+    
+    if (! err_text.empty())
+    {
+        fwprintf(stderr, L"%ls: ", err_text.c_str());
+    }
+    fwprintf(stderr, L"%s\n", strerror(err_code));
+}
+
+/* The universal variable format has some funny escaping requirements; here we try to be safe */
+static bool is_universal_safe_to_encode_directly(wchar_t c)
+{
+    if (c < 32 || c > 128)
+        return false;
+
+    return iswalnum(c) || wcschr(L"/_", c);
+}
+
+/**
+   Escape specified string
+ */
+static wcstring full_escape(const wchar_t *in)
+{
+    wcstring out;
+    for (; *in; in++)
+    {
+        wchar_t c = *in;
+        if (is_universal_safe_to_encode_directly(c))
+        {
+            out.push_back(c);
+        }
+        else if (c <= (wchar_t)ASCII_MAX)
+        {
+            // See #1225 for discussion of use of ASCII_MAX here
+            append_format(out, L"\\x%.2x", c);
+        }
+        else if (c < 65536)
+        {
+            append_format(out, L"\\u%.4x", c);
+        }
+        else
+        {
+            append_format(out, L"\\U%.8x", c);
+        }
+    }
+    return out;
+}
+
+/* Converts input to UTF-8 and appends it to receiver, using storage as temp storage */
+static bool append_utf8(const wcstring &input, std::string *receiver, std::string *storage)
+{
+    bool result = false;
+    if (wchar_to_utf8_string(input, storage))
+    {
+        receiver->append(*storage);
+        result = true;
+    }
+    return result;
+}
+
+/* Creates a file entry like "SET fish_color_cwd:FF0". Appends the result to *result (as UTF8). Returns true on success. storage may be used for temporary storage, to avoid allocations */
+static bool append_file_entry(fish_message_type_t type, const wcstring &key_in, const wcstring &val_in, std::string *result, std::string *storage)
+{
+    assert(storage != NULL);
+    assert(result != NULL);
+    
+    // Record the length on entry, in case we need to back up
+    bool success = true;
+    const size_t result_length_on_entry = result->size();
+    
+    // Append header like "SET "
+    result->append(type==SET ? SET_MBS : SET_EXPORT_MBS);
+    result->push_back(' ');
+
+    // Append variable name like "fish_color_cwd"
+    if (wcsvarname(key_in.c_str()))
+    {
+        debug(0, L"Illegal variable name: '%ls'", key_in.c_str());
+        success = false;
+    }
+    if (success && ! append_utf8(key_in, result, storage))
+    {
+        debug(0, L"Could not convert %ls to narrow character string", key_in.c_str());
+        success = false;
+    }
+    
+    // Append ":"
+    if (success)
+    {
+        result->push_back(':');
+    }
+    
+    // Append value
+    if (success && ! append_utf8(full_escape(val_in.c_str()), result, storage))
+    {
+        debug(0, L"Could not convert %ls to narrow character string", val_in.c_str());
+        success = false;
+    }
+    
+    // Append newline
+    if (success)
+    {
+        result->push_back('\n');
+    }
+    
+    // Don't modify result on failure. It's sufficient to simply resize it since all we ever did was append to it.
+    if (! success)
+    {
+        result->resize(result_length_on_entry);
+    }
+    
+    return success;
+}
+
+env_universal_t::env_universal_t(const wcstring &path) : explicit_vars_path(path), tried_renaming(false), last_read_file(kInvalidFileID)
+{
+    VOMIT_ON_FAILURE(pthread_mutex_init(&lock, NULL));
+}
+
+env_universal_t::~env_universal_t()
+{
+    pthread_mutex_destroy(&lock);
+}
+
+env_var_t env_universal_t::get(const wcstring &name) const
+{
+    env_var_t result = env_var_t::missing_var();
+    var_table_t::const_iterator where = vars.find(name);
+    if (where != vars.end())
+    {
+        result = where->second.val;
+    }
+    return result;
+}
+
+bool env_universal_t::get_export(const wcstring &name) const
+{
+    bool result = false;
+    var_table_t::const_iterator where = vars.find(name);
+    if (where != vars.end())
+    {
+        result = where->second.exportv;
+    }
+    return result;
+}
+
+
+void env_universal_t::set_internal(const wcstring &key, const wcstring &val, bool exportv, bool overwrite)
+{
+    ASSERT_IS_LOCKED(lock);
+    if (! overwrite && this->modified.find(key) != this->modified.end())
+    {
+        /* This value has been modified and we're not overwriting it. Skip it. */
+        return;
+    }
+    
+    var_entry_t *entry = &vars[key];
+    if (entry->exportv != exportv || entry->val != val)
+    {
+        entry->val = val;
+        entry->exportv = exportv;
+        
+        /* If we are overwriting, then this is now modified */
+        if (overwrite)
+        {
+            this->modified.insert(key);
+        }
+    }
+}
+
+void env_universal_t::set(const wcstring &key, const wcstring &val, bool exportv)
+{
+    scoped_lock locker(lock);
+    this->set_internal(key, val, exportv, true /* overwrite */);
+}
+
+bool env_universal_t::remove_internal(const wcstring &key)
+{
+    ASSERT_IS_LOCKED(lock);
+    size_t erased = this->vars.erase(key);
+    if (erased > 0)
+    {
+        this->modified.insert(key);
+    }
+    return erased > 0;
+}
+
+bool env_universal_t::remove(const wcstring &key)
+{
+    scoped_lock locker(lock);
+    return this->remove_internal(key);
+}
+
+wcstring_list_t env_universal_t::get_names(bool show_exported, bool show_unexported) const
+{
+    wcstring_list_t result;
+    scoped_lock locker(lock);
+    var_table_t::const_iterator iter;
+    for (iter = vars.begin(); iter != vars.end(); ++iter)
+    {
+        const wcstring &key = iter->first;
+        const var_entry_t &e = iter->second;
+        if ((e.exportv && show_exported) || (! e.exportv && show_unexported))
+        {
+            result.push_back(key);
+        }
+    }
+    return result;
+}
+
+/* Given a variable table, generate callbacks representing the difference between our vars and the new vars */
+void env_universal_t::generate_callbacks(const var_table_t &new_vars, callback_data_list_t *callbacks) const
+{
+    assert(callbacks != NULL);
+    
+    /* Construct callbacks for erased values */
+    for (var_table_t::const_iterator iter = this->vars.begin(); iter != this->vars.end(); ++iter)
+    {
+        const wcstring &key = iter->first;
+        
+        /* Skip modified values */
+        if (this->modified.find(key) != this->modified.end())
+        {
+            continue;
+        }
+        
+        /* If the value is not present in new_vars, it has been erased */
+        if (new_vars.find(key) == new_vars.end())
+        {
+            callbacks->push_back(callback_data_t(ERASE, key, L""));
+        }
+    }
+    
+    /* Construct callbacks for newly inserted or changed values */
+    for (var_table_t::const_iterator iter = new_vars.begin(); iter != new_vars.end(); ++iter)
+    {
+        const wcstring &key = iter->first;
+        
+        /* Skip modified values */
+        if (this->modified.find(key) != this->modified.end())
+        {
+            continue;
+        }
+        
+        /* See if the value has changed */
+        const var_entry_t &new_entry = iter->second;
+        var_table_t::const_iterator existing = this->vars.find(key);
+        if (existing == this->vars.end() || existing->second.exportv != new_entry.exportv || existing->second.val != new_entry.val)
+        {
+            /* Value has changed */
+            callbacks->push_back(callback_data_t(new_entry.exportv ? SET_EXPORT : SET, key, new_entry.val));
+        }
+    }
+}
+
+
+void env_universal_t::acquire_variables(var_table_t *vars_to_acquire)
+{
+    /* Copy modified values from existing vars to vars_to_acquire */
+    for (std::set<wcstring>::iterator iter = this->modified.begin(); iter != this->modified.end(); ++iter)
+    {
+        const wcstring &key = *iter;
+        var_table_t::iterator src_iter = this->vars.find(key);
+        if (src_iter == this->vars.end())
+        {
+            /* The value has been deleted. */
+            vars_to_acquire->erase(key);
+        }
+        else
+        {
+            /* The value has been modified. Copy it over. Note we can destructively modify the source entry in vars since we are about to get rid of this->vars entirely. */
+            var_entry_t &src = src_iter->second;
+            var_entry_t &dst = (*vars_to_acquire)[key];
+            dst.val.swap(src.val);
+            dst.exportv = src.exportv;
+        }
+    }
+    
+    /* We have constructed all the callbacks and updated vars_to_acquire. Acquire it! */
+    this->vars.swap(*vars_to_acquire);
+}
+
+void env_universal_t::load_from_fd(int fd, callback_data_list_t *callbacks)
+{
+    ASSERT_IS_LOCKED(lock);
+    assert(fd >= 0);
+    /* Get the dev / inode */
+    const file_id_t current_file = file_id_for_fd(fd);
+    if (current_file == last_read_file)
+    {
+        UNIVERSAL_LOG("Sync elided based on fstat()");
+    }
+    else
+    {
+        /* Read a variables table from the file. */
+        var_table_t new_vars = this->read_message_internal(fd);
+        
+        /* Announce changes */
+        if (callbacks != NULL)
+        {
+            this->generate_callbacks(new_vars, callbacks);
+        }
+        
+        /* Acquire the new variables */
+        this->acquire_variables(&new_vars);
+        
+        last_read_file = current_file;
+    }
+}
+
+bool env_universal_t::load_from_path(const wcstring &path, callback_data_list_t *callbacks)
+{
+    ASSERT_IS_LOCKED(lock);
+    
+    /* Check to see if the file is unchanged. We do this again in load_from_fd, but this avoids opening the file unnecessarily. */
+    if (last_read_file != kInvalidFileID && file_id_for_path(path) == last_read_file)
+    {
+        UNIVERSAL_LOG("Sync elided based on fast stat()");
+        return true;
+    }
+    
+    bool result = false;
+    int fd = wopen_cloexec(path, O_RDONLY);
+    if (fd >= 0)
+    {
+        UNIVERSAL_LOG("Reading from file");
+        this->load_from_fd(fd, callbacks);
+        close(fd);
+        result = true;
+    }
+    return result;
+}
+
+/* Writes our state to the fd. path is provided only for error reporting */
+bool env_universal_t::write_to_fd(int fd, const wcstring &path)
+{
+    ASSERT_IS_LOCKED(lock);
+    assert(fd >= 0);
+    bool success = true;
+
+    // Stuff we output to fd
+    std::string contents;
+    
+    // Temporary storage
+    std::string storage;
+    
+    // Write the save message. If this fails, we don't bother complaining.
+    write_loop(fd, SAVE_MSG, strlen(SAVE_MSG));
+    
+    var_table_t::const_iterator iter = vars.begin();
+    while (iter != vars.end())
+    {
+        // Append the entry. Note that append_file_entry may fail, but that only affects one variable; soldier on.
+        const wcstring &key = iter->first;
+        const var_entry_t &entry = iter->second;
+        append_file_entry(entry.exportv ? SET_EXPORT : SET, key, entry.val, &contents, &storage);
+        
+        // Go to next
+        ++iter;
+        
+        // Flush if this is the last iteration or we exceed a page
+        if (iter == vars.end() || contents.size() >= 4096)
+        {
+            if (write_loop(fd, contents.data(), contents.size()) < 0)
+            {
+                int err = errno;
+                report_error(err, L"Unable to write to universal variables file '%ls'", path.c_str());
+                success = false;
+                break;
+            }
+            contents.clear();
+        }
+    }
+    
+    /* Since we just wrote out this file, it matches our internal state; pretend we read from it */
+    this->last_read_file = file_id_for_fd(fd);
+    
+    /* We don't close the file */
+    return success;
+}
+
+bool env_universal_t::move_new_vars_file_into_place(const wcstring &src, const wcstring &dst)
+{
+    int ret = wrename(src, dst);
+    if (ret != 0)
+    {
+        int err = errno;
+        report_error(err, L"Unable to rename file from '%ls' to '%ls'", src.c_str(), dst.c_str());
+    }
+    return ret == 0;
+}
+
+static wcstring fishd_get_config()
+{
+    bool done = false;
+    wcstring result;
+
+    env_var_t xdg_dir = env_get_string(L"XDG_CONFIG_HOME", ENV_GLOBAL | ENV_EXPORT);
+    if (! xdg_dir.missing_or_empty())
+    {
+        result = xdg_dir;
+        append_path_component(result, L"/fish");
+        if (!create_directory(result))
+        {
+            done = true;
+        }
+    }
+    else
+    {
+        env_var_t home = env_get_string(L"HOME", ENV_GLOBAL | ENV_EXPORT);
+        if (! home.missing_or_empty())
+        {
+            result = home;
+            append_path_component(result, L"/.config/fish");
+            if (!create_directory(result))
+            {
+                done = 1;
+            }
+        }
+    }
+    
+    if (! done)
+    {
+        /* Bad juju */
+        debug(0, _(L"Unable to create a configuration directory for fish. Your personal settings will not be saved. Please set the $XDG_CONFIG_HOME variable to a directory where the current user has write access."));
+        result.clear();
+    }
+    
+    return result;
+}
+
+bool env_universal_t::load()
+{
+    scoped_lock locker(lock);
+    callback_data_list_t callbacks;
+    const wcstring vars_path = explicit_vars_path.empty() ? default_vars_path() : explicit_vars_path;
+    bool success = load_from_path(vars_path, &callbacks);
+    if (! success && ! tried_renaming && errno == ENOENT)
+    {
+        /* We failed to load, because the file was not found. Older fish used the hostname only. Try *moving* the filename based on the hostname into place; if that succeeds try again. Silently "upgraded." */
+        tried_renaming = true;
+        wcstring hostname_id;
+        if (get_hostname_identifier(&hostname_id))
+        {
+            const wcstring hostname_path = wdirname(vars_path) + L'/' + hostname_id;
+            if (0 == wrename(hostname_path, vars_path))
+            {
+                /* We renamed - try again */
+                success = this->load();
+            }
+        }
+    }
+    return success;
+}
+
+bool env_universal_t::open_temporary_file(const wcstring &directory, wcstring *out_path, int *out_fd)
+{
+    /* Create and open a temporary file for writing within the given directory */
+    /* Try to create a temporary file, up to 10 times. We don't use mkstemps because we want to open it CLO_EXEC. This should almost always succeed on the first try. */
+    assert(! string_suffixes_string(L"/", directory));
+    
+    bool success = false;
+    const wcstring tmp_name_template = directory + L"/fishd.tmp.XXXXXX";
+    wcstring tmp_name;
+    for (size_t attempt = 0; attempt < 10 && ! success; attempt++)
+    {
+        int result_fd = -1;
+        char *narrow_str = wcs2str(tmp_name_template.c_str());
+#if HAVE_MKOSTEMP
+        result_fd = mkostemp(narrow_str, O_CLOEXEC);
+        if (result_fd >= 0)
+        {
+            tmp_name = str2wcstring(narrow_str);
+        }
+#else
+        if (mktemp(narrow_str))
+        {
+            /* It was successfully templated; try opening it atomically */
+            tmp_name = str2wcstring(narrow_str);
+            result_fd = wopen_cloexec(tmp_name, O_WRONLY | O_CREAT | O_EXCL | O_TRUNC, 0644);
+        }
+#endif
+        
+        if (result_fd >= 0)
+        {
+            /* Success */
+            *out_fd = result_fd;
+            *out_path = str2wcstring(narrow_str);
+            success = true;
+        }
+        free(narrow_str);
+    }
+    if (! success)
+    {
+        int err = errno;
+        report_error(err, L"Unable to open file '%ls'", tmp_name.c_str());
+    }
+    return success;
+}
+
+bool env_universal_t::open_and_acquire_lock(const wcstring &path, int *out_fd)
+{
+    /* Attempt to open the file for reading at the given path, atomically acquiring a lock. On BSD, we can use O_EXLOCK. On Linux, we open the file, take a lock, and then compare fstat() to stat(); if they match, it means that the file was not replaced before we acquired the lock.
+    
+    We pass O_RDONLY with O_CREAT; this creates a potentially empty file. We do this so that we have something to lock on.
+    */
+    int result_fd = -1;
+    bool needs_lock = true;
+    int flags = O_RDWR | O_CREAT;
+#ifdef O_EXLOCK
+    flags |= O_EXLOCK;
+    needs_lock = false;
+#endif
+    for (;;)
+    {
+        int fd = wopen_cloexec(path, flags, 0644);
+        if (fd < 0)
+        {
+            int err = errno;
+            if (err == EINTR)
+            {
+                /* Signal; try again */
+                continue;
+            }
+#ifdef O_EXLOCK
+            else if (err == EOPNOTSUPP)
+            {
+                /* Filesystem probably does not support locking. Clear the flag and try again. Note that we try taking the lock via flock anyways. */
+                flags &= ~O_EXLOCK;
+                needs_lock = true;
+                continue;
+            }
+#endif
+            else
+            {
+                report_error(err, L"Unable to open universal variable file '%ls'", path.c_str());
+                break;
+            }
+        }
+        
+        /* If we get here, we must have a valid fd */
+        assert(fd >= 0);
+        
+        /* Try taking the lock, if necessary. If we failed, we may be on lockless NFS, etc.; in that case we pretend we succeeded. See the comment in save_to_path for the rationale. */
+        if (needs_lock)
+        {
+            while (flock(fd, LOCK_EX) < 0)
+            {
+                /* error */
+                if (errno != EINTR)
+                {
+                    /* Do nothing per #2149 */
+                    break;
+                }
+            }
+        }
+        
+        /* Hopefully we got the lock. However, it's possible the file changed out from under us while we were waiting for the lock. Make sure that didn't happen. */
+        if (file_id_for_fd(fd) != file_id_for_path(path))
+        {
+            /* Oops, it changed! Try again */
+            close(fd);
+            continue;
+        }
+
+        
+        /* Finally, we have an fd that's valid and hopefully locked. We're done */
+        assert(fd >= 0);
+        result_fd = fd;
+        break;
+    }
+    
+    *out_fd = result_fd;
+    
+    return result_fd >= 0;
+}
+
+/* Returns true if modified variables were written, false if not. (There may still be variable changes due to other processes on a false return). */
+bool env_universal_t::sync(callback_data_list_t *callbacks)
+{
+    UNIVERSAL_LOG("sync");
+    scoped_lock locker(lock);
+    /* Our saving strategy:
+    
+    1. Open the file, producing an fd.
+    2. Lock the file (may be combined with step 1 on systems with O_EXLOCK)
+    3. After taking the lock, check if the file at the given path is different from what we opened. If so, start over.
+    4. Read from the file. This can be elided if its dev/inode is unchanged since the last read
+    5. Open an adjacent temporary file
+    6. Write our changes to an adjacent file
+    7. Move the adjacent file into place via rename. This is assumed to be atomic.
+    8. Release the lock and close the file
+    
+    Consider what happens if Process 1 and 2 both do this simultaneously. Can there be data loss? Process 1 opens the file and then attempts to take the lock. Now, either process 1 will see the original file, or process 2's new file. If it sees the new file, we're OK: it's going to read from the new file, and so there's no data loss. If it sees the old file, then process 2 must have locked it (if process 1 locks it, switch their roles). The lock will block until process 2 reaches step 7; at that point process 1 will reach step 2, notice that the file has changed, and then start over.
+    
+    It's possible that the underlying filesystem does not support locks (lockless NFS). In this case, we risk data loss if two shells try to write their universal variables simultaneously. In practice this is unlikely, since uvars are usually written interactively.
+    
+    Prior versions of fish used a hard link scheme to support file locking on lockless NFS. The risk here is that if the process crashes or is killed while holding the lock, future instances of fish will not be able to obtain it. This seems to be a greater risk than that of data loss on lockless NFS. Users who put their home directory on lockless NFS are playing with fire anyways.
+    */
+    const wcstring &vars_path = explicit_vars_path.empty() ? default_vars_path() : explicit_vars_path;
+    
+    /* If we have no changes, just load */
+    if (modified.empty())
+    {
+        this->load_from_path(vars_path, callbacks);
+        return false;
+    }
+    
+    const wcstring directory = wdirname(vars_path);
+    bool success = true;
+    int vars_fd = -1;
+    int private_fd = -1;
+    wcstring private_file_path;
+    
+    UNIVERSAL_LOG("Performing full sync");
+    
+    /* Open the file */
+    if (success)
+    {
+        success = this->open_and_acquire_lock(vars_path, &vars_fd);
+    }
+
+    /* Read from it */
+    if (success)
+    {
+        assert(vars_fd >= 0);
+        this->load_from_fd(vars_fd, callbacks);
+    }
+
+    /* Open adjacent temporary file */
+    if (success)
+    {
+        success = this->open_temporary_file(directory, &private_file_path, &private_fd);
+    }
+    
+    /* Write to it */
+    if (success)
+    {
+        assert(private_fd >= 0);
+        success = this->write_to_fd(private_fd, private_file_path);
+    }
+    
+    if (success)
+    {
+        /* Apply new file */
+        success = this->move_new_vars_file_into_place(private_file_path, vars_path);
+    }
+    
+    if (success)
+    {
+        /* Since we moved the new file into place, clear the path so we don't try to unlink it */
+         private_file_path.clear();
+    }
+    
+    /* Clean up */
+    if (vars_fd >= 0)
+    {
+        close(vars_fd);
+    }
+    if (private_fd >= 0)
+    {
+        close(private_fd);
+    }
+    if (! private_file_path.empty())
+    {
+        wunlink(private_file_path);
+    }
+    
+    if (success)
+    {
+        /* All of our modified variables have now been written out. */
+        modified.clear();
+    }
+    
+    return success;
+}
+
+var_table_t env_universal_t::read_message_internal(int fd)
+{
+    var_table_t result;
+    
+    // Temp value used to avoid repeated allocations
+    wcstring storage;
+    
+    // The line we construct (and then parse)
+    std::string line;
+    wcstring wide_line;
+    for (;;)
+    {
+        // Read into a buffer. Note this is NOT null-terminated!
+        char buffer[1024];
+        ssize_t amt = read_loop(fd, buffer, sizeof buffer);
+        if (amt <= 0)
+        {
+            break;
+        }
+        const size_t bufflen = (size_t)amt;
+        
+        // Walk over it by lines. The contents of an unterminated line will be left in 'line' for the next iteration.
+        size_t line_start = 0;
+        while (line_start < amt)
+        {
+            // Run until we hit a newline
+            size_t cursor = line_start;
+            while (cursor < bufflen && buffer[cursor] != '\n')
+            {
+                cursor++;
+            }
+            
+            // Copy over what we read
+            line.append(buffer + line_start, cursor - line_start);
+            
+            // Process it if it's a newline (which is true if we are before the end of the buffer)
+            if (cursor < bufflen && ! line.empty())
+            {
+                if (utf8_to_wchar_string(line, &wide_line))
+                {
+                    env_universal_t::parse_message_internal(wide_line, &result, &storage);
+                }
+                line.clear();
+            }
+            
+            // Skip over the newline (or skip past the end)
+            line_start = cursor + 1;
+        }
+    }
+    
+    // We make no effort to handle an unterminated last line
+    return result;
+}
+
+/**
+ Parse message msg
+ */
+void env_universal_t::parse_message_internal(const wcstring &msgstr, var_table_t *vars, wcstring *storage)
+{
+    const wchar_t *msg = msgstr.c_str();
+    
+    //  debug( 3, L"parse_message( %ls );", msg );
+    
+    if (msg[0] == L'#')
+        return;
+    
+    bool is_set_export = match(msg, SET_EXPORT_STR);
+    bool is_set = ! is_set_export && match(msg, SET_STR);
+    if (is_set || is_set_export)
+    {
+        const wchar_t *name, *tmp;
+        const bool exportv = is_set_export;
+        
+        name = msg+(exportv?wcslen(SET_EXPORT_STR):wcslen(SET_STR));
+        while (name[0] == L'\t' || name[0] == L' ')
+            name++;
+        
+        tmp = wcschr(name, L':');
+        if (tmp)
+        {
+            /* Use 'storage' to hold our key to avoid allocations */
+            storage->assign(name, tmp - name);
+            const wcstring &key = *storage;
+            
+            wcstring val;
+            if (unescape_string(tmp + 1, &val, 0))
+            {
+                var_entry_t &entry = (*vars)[key];
+                entry.exportv = exportv;
+                entry.val.swap(val); //acquire the value
+            }
+        }
+        else
+        {
+            debug(1, PARSE_ERR, msg);
+        }
+    }
+    else
+    {
+        debug(1, PARSE_ERR, msg);
+    }
+}
+
+/**
+ Maximum length of hostname. Longer hostnames are truncated
+ */
+#define HOSTNAME_LEN 32
+
+/* Length of a MAC address */
+#define MAC_ADDRESS_MAX_LEN 6
+
+
+/* Thanks to Jan Brittenson
+ http://lists.apple.com/archives/xcode-users/2009/May/msg00062.html
+ */
+#ifdef SIOCGIFHWADDR
+
+/* Linux */
+#include <net/if.h>
+static bool get_mac_address(unsigned char macaddr[MAC_ADDRESS_MAX_LEN], const char *interface = "eth0")
+{
+    bool result = false;
+    const int dummy = socket(AF_INET, SOCK_STREAM, 0);
+    if (dummy >= 0)
+    {
+        struct ifreq r;
+        strncpy((char *)r.ifr_name, interface, sizeof r.ifr_name - 1);
+        r.ifr_name[sizeof r.ifr_name - 1] = 0;
+        if (ioctl(dummy, SIOCGIFHWADDR, &r) >= 0)
+        {
+            memcpy(macaddr, r.ifr_hwaddr.sa_data, MAC_ADDRESS_MAX_LEN);
+            result = true;
+        }
+        close(dummy);
+    }
+    return result;
+}
+
+#elif defined(HAVE_GETIFADDRS)
+
+/* OS X and BSD */
+#include <ifaddrs.h>
+#include <net/if_dl.h>
+static bool get_mac_address(unsigned char macaddr[MAC_ADDRESS_MAX_LEN], const char *interface = "en0")
+{
+    // BSD, Mac OS X
+    struct ifaddrs *ifap;
+    bool ok = false;
+    
+    if (getifaddrs(&ifap) == 0)
+    {
+        for (const ifaddrs *p = ifap; p; p = p->ifa_next)
+        {
+            if (p->ifa_addr->sa_family == AF_LINK)
+            {
+                if (p->ifa_name && p->ifa_name[0] &&
+                    ! strcmp((const char*)p->ifa_name, interface))
+                {
+                    
+                    const sockaddr_dl& sdl = *(sockaddr_dl*)p->ifa_addr;
+                    
+                    size_t alen = sdl.sdl_alen;
+                    if (alen > MAC_ADDRESS_MAX_LEN) alen = MAC_ADDRESS_MAX_LEN;
+                    memcpy(macaddr, sdl.sdl_data + sdl.sdl_nlen, alen);
+                    ok = true;
+                    break;
+                }
+            }
+        }
+        freeifaddrs(ifap);
+    }
+    return ok;
+}
+
+#else
+
+/* Unsupported */
+static bool get_mac_address(unsigned char macaddr[MAC_ADDRESS_MAX_LEN])
+{
+    return false;
+}
+
+#endif
+
+/* Function to get an identifier based on the hostname */
+static bool get_hostname_identifier(wcstring *result)
+{
+    bool success = false;
+    char hostname[HOSTNAME_LEN + 1] = {};
+    if (gethostname(hostname, HOSTNAME_LEN) == 0)
+    {
+        result->assign(str2wcstring(hostname));
+        success = true;
+    }
+    return success;
+}
+
+/* Get a sort of unique machine identifier. Prefer the MAC address; if that fails, fall back to the hostname; if that fails, pick something. */
+wcstring get_machine_identifier()
+{
+    wcstring result;
+    unsigned char mac_addr[MAC_ADDRESS_MAX_LEN] = {};
+    if (get_mac_address(mac_addr))
+    {
+        result.reserve(2 * MAC_ADDRESS_MAX_LEN);
+        for (size_t i=0; i < MAC_ADDRESS_MAX_LEN; i++)
+        {
+            append_format(result, L"%02x", mac_addr[i]);
+        }
+    }
+    else if (get_hostname_identifier(&result))
+    {
+        /* Hooray */
+    }
+    else
+    {
+        /* Fallback */
+        result.assign(L"nohost");
+    }
+    return result;
+}
+
+class universal_notifier_shmem_poller_t : public universal_notifier_t
+{
+    /* This is what our shared memory looks like. Everything here is stored in network byte order (big-endian) */
+    struct universal_notifier_shmem_t
+    {
+        uint32_t magic;
+        uint32_t version;
+        uint32_t universal_variable_seed;
+    };
+    
+#define SHMEM_MAGIC_NUMBER 0xF154
+#define SHMEM_VERSION_CURRENT 1000
+
+    private:
+    long long last_change_time;
+    uint32_t last_seed;
+    volatile universal_notifier_shmem_t *region;
+    
+    void open_shmem()
+    {
+        assert(region == NULL);
+        
+        // Use a path based on our uid to avoid collisions
+        char path[NAME_MAX];
+        snprintf(path, sizeof path, "/%ls_shmem_%d", program_name ? program_name : L"fish", getuid());
+        
+        bool errored = false;
+        int fd = shm_open(path, O_RDWR | O_CREAT, 0600);
+        if (fd < 0)
+        {
+            int err = errno;
+            report_error(err, L"Unable to open shared memory with path '%s'", path);
+            errored = true;
+        }
+
+        /* Get the size */
+        size_t size = 0;
+        if (! errored)
+        {
+            struct stat buf = {};
+            if (fstat(fd, &buf) < 0)
+            {
+                int err = errno;
+                report_error(err, L"Unable to fstat shared memory object with path '%s'", path);
+                errored = true;
+            }
+            size = buf.st_size;
+        }
+        
+        /* Set the size, if it's too small */
+        if (! errored && size < sizeof(universal_notifier_shmem_t))
+        {
+            if (ftruncate(fd, sizeof(universal_notifier_shmem_t)) < 0)
+            {
+                int err = errno;
+                report_error(err, L"Unable to truncate shared memory object with path '%s'", path);
+                errored = true;
+            }
+        }
+        
+        /* Memory map the region */
+        if (! errored)
+        {
+            void *addr = mmap(NULL, sizeof(universal_notifier_shmem_t), PROT_READ | PROT_WRITE, MAP_FILE | MAP_SHARED, fd, 0);
+            if (addr == MAP_FAILED)
+            {
+                int err = errno;
+                report_error(err, L"Unable to memory map shared memory object with path '%s'", path);
+                region = NULL;
+            }
+            else
+            {
+                region = static_cast<universal_notifier_shmem_t*>(addr);
+            }
+        }
+        
+        /* Close the fd, even if the mapping succeeded */
+        if (fd >= 0)
+        {
+            close(fd);
+        }
+        
+        /* Read the current seed */
+        this->poll();
+    }
+    
+    public:
+    
+    /* Our notification involves changing the value in our shared memory. In practice, all clients will be in separate processes, so it suffices to set the value to a pid. For testing purposes, however, it's useful to keep them in the same process, so we increment the value. This isn't "safe" in the sense that multiple simultaneous increments may result in one being lost, but it should always result in the value being changed, which is sufficient. */
+    void post_notification()
+    {
+        if (region != NULL)
+        {
+            /* Read off the seed */
+            uint32_t seed = ntohl(region->universal_variable_seed);
+            
+            /* Increment it. Don't let it wrap to zero. */
+            do
+            {
+                seed++;
+            }
+            while (seed == 0);
+            last_seed = seed;
+            
+            /* Write out our data */
+            region->magic = htonl(SHMEM_MAGIC_NUMBER);
+            region->version = htonl(SHMEM_VERSION_CURRENT);
+            region->universal_variable_seed = htonl(seed);
+        }
+    }
+    
+    universal_notifier_shmem_poller_t() : last_change_time(0), last_seed(0), region(NULL)
+    {
+        open_shmem();
+    }
+    
+    ~universal_notifier_shmem_poller_t()
+    {
+        if (region != NULL)
+        {
+            // Behold: C++ in all its glory!
+            void *address = const_cast<void *>(static_cast<volatile void *>(region));
+            if (munmap(address, sizeof(universal_notifier_shmem_t)) < 0)
+            {
+                wperror(L"munmap");
+            }
+        }
+    }
+    
+    bool poll()
+    {
+        bool result = false;
+        if (region != NULL)
+        {
+            uint32_t seed = ntohl(region->universal_variable_seed);
+            if (seed != last_seed)
+            {
+                result = true;
+                last_seed = seed;
+                last_change_time = get_time();
+            }
+        }
+        return result;
+    }
+    
+    unsigned long usec_delay_between_polls() const
+    {
+        // If it's been less than five seconds since the last change, we poll quickly
+        // Otherwise we poll more slowly
+        // Note that a poll is a very cheap shmem read. The bad part about making this high
+        // is the process scheduling/wakeups it produces
+        unsigned long usec_per_sec = 1000000;
+        if (get_time() - last_change_time < 5LL * usec_per_sec)
+        {
+            return usec_per_sec / 10; //10 times a second
+        }
+        else
+        {
+            return usec_per_sec / 3; //3 times a second
+        }
+    }
+};
+
+/* A notifyd-based notifier. Very straightforward. */
+class universal_notifier_notifyd_t : public universal_notifier_t
+{
+    int notify_fd;
+    int token;
+    std::string name;
+    
+    void setup_notifyd()
+    {
+#if FISH_NOTIFYD_AVAILABLE
+        // per notify(3), the user.uid.%d style is only accessible to processes with that uid
+        char local_name[256];
+        snprintf(local_name, sizeof local_name, "user.uid.%d.%ls.uvars",  getuid(), program_name ? program_name : L"fish");
+        name.assign(local_name);
+        
+        uint32_t status = notify_register_file_descriptor(name.c_str(), &this->notify_fd, 0, &this->token);
+        if (status != NOTIFY_STATUS_OK)
+        {
+            fprintf(stderr, "Warning: notify_register_file_descriptor() failed with status %u. Universal variable notifications may not be received.", status);
+        }
+        if (this->notify_fd >= 0)
+        {
+            // Mark us for non-blocking reads, and CLO_EXEC
+            int flags = fcntl(this->notify_fd, F_GETFL, 0);
+            if (flags >= 0 && ! (flags & O_NONBLOCK))
+            {
+                fcntl(this->notify_fd, F_SETFL, flags | O_NONBLOCK);
+            }
+            
+            set_cloexec(this->notify_fd);
+            // Serious hack: notify_fd is likely the read end of a pipe. The other end is owned by libnotify, which does not mark it as CLO_EXEC (it should!)
+            // The next fd is probably notify_fd + 1
+            // Do it ourselves. If the implementation changes and some other FD gets marked as CLO_EXEC, that's probably a good thing.
+            set_cloexec(this->notify_fd + 1);
+        }
+#endif
+    }
+    
+public:
+    universal_notifier_notifyd_t() : notify_fd(-1), token(-1 /* NOTIFY_TOKEN_INVALID */)
+    {
+        setup_notifyd();
+    }
+    
+    ~universal_notifier_notifyd_t()
+    {
+        if (token != -1 /* NOTIFY_TOKEN_INVALID */)
+        {
+#if FISH_NOTIFYD_AVAILABLE
+            notify_cancel(token);
+#endif
+        }
+    }
+    
+    int notification_fd()
+    {
+        return notify_fd;
+    }
+    
+    bool notification_fd_became_readable(int fd)
+    {
+        /* notifyd notifications come in as 32 bit values. We don't care about the value. We set ourselves as non-blocking, so just read until we can't read any more. */
+        assert(fd == notify_fd);
+        bool read_something = false;
+        unsigned char buff[64];
+        ssize_t amt_read;
+        do
+        {
+            amt_read = read(notify_fd, buff, sizeof buff);
+            read_something = (read_something || amt_read > 0);
+        } while (amt_read == sizeof buff);
+        return read_something;
+    }
+    
+    void post_notification()
+    {
+#if FISH_NOTIFYD_AVAILABLE
+        uint32_t status = notify_post(name.c_str());
+        if (status != NOTIFY_STATUS_OK)
+        {
+            fprintf(stderr, "Warning: notify_post() failed with status %u. Universal variable notifications may not be sent.", status);
+        }
+#endif
+    }
+};
+
+#define NAMED_PIPE_FLASH_DURATION_USEC (1000000 / 10)
+#define SUSTAINED_READABILITY_CLEANUP_DURATION_USEC (1000000 * 5)
+
+/* Named-pipe based notifier. All clients open the same named pipe for reading and writing. The pipe's readability status is a trigger to enter polling mode.
+
+  To post a notification, write some data to the pipe, wait a little while, and then read it back.
+  
+  To receive a notification, watch for the pipe to become readable. When it does, enter a polling mode until the pipe is no longer readable. To guard against the possibility of a shell exiting when there is data remaining in the pipe, if the pipe is kept readable too long, clients will attempt to read data out of it (to render it no longer readable).
+*/
+class universal_notifier_named_pipe_t : public universal_notifier_t
+{
+    int pipe_fd;
+    long long readback_time_usec;
+    size_t readback_amount;
+    
+    bool polling_due_to_readable_fd;
+    long long drain_if_still_readable_time_usec;
+    
+    void make_pipe(const wchar_t *test_path)
+    {
+        wcstring vars_path = test_path ? wcstring(test_path) : default_named_pipe_path();
+        vars_path.append(L".notifier");
+        const std::string narrow_path = wcs2string(vars_path);
+        
+        int fd = wopen_cloexec(vars_path, O_RDWR | O_NONBLOCK, 0600);
+        if (fd < 0 && errno == ENOENT)
+        {
+            /* File doesn't exist, try creating it */
+            if (mkfifo(narrow_path.c_str(), 0600) >= 0)
+            {
+                fd = wopen_cloexec(vars_path, O_RDWR | O_NONBLOCK, 0600);
+            }
+        }
+        if (fd < 0)
+        {
+            // Maybe open failed, maybe mkfifo failed
+            int err = errno;
+            // We explicitly do NOT report an error for ENOENT or EACCESS
+            // This works around #1955, where $XDG_RUNTIME_DIR may get a bogus value under suc
+            if (err != ENOENT && err != EPERM)
+            {
+                report_error(err, L"Unable to make or open a FIFO for universal variables with path '%ls'", vars_path.c_str());
+            }
+            pipe_fd= -1;
+        }
+        else
+        {
+            pipe_fd = fd;
+        }
+    }
+    
+    void drain_excessive_data()
+    {
+        // The pipe seems to have data on it, that won't go away
+        // Read a big chunk out of it.
+        // We don't read until it's exhausted, because if someone were to pipe say /dev/null, that would cause us to hang!
+        size_t read_amt = 64 * 1024;
+        void *buff = malloc(read_amt);
+        read_ignore(this->pipe_fd, buff, read_amt);
+        free(buff);
+    }
+    
+    public:
+    universal_notifier_named_pipe_t(const wchar_t *test_path) : pipe_fd(-1), readback_time_usec(0), readback_amount(0), polling_due_to_readable_fd(false), drain_if_still_readable_time_usec(0)
+    {
+        make_pipe(test_path);
+    }
+    
+    ~universal_notifier_named_pipe_t()
+    {
+        if (pipe_fd >= 0)
+        {
+            close(pipe_fd);
+        }
+    }
+    
+    int notification_fd()
+    {
+        if (polling_due_to_readable_fd)
+        {
+            // We are in polling mode because we think our fd is readable
+            // This means that, if we return it to be select()'d on, we'll be called back immediately
+            // So don't return it
+            return -1;
+        }
+        else
+        {
+            // We are not in polling mode
+            // Return the fd so it can be watched
+            return pipe_fd;
+        }
+    }
+    
+    bool notification_fd_became_readable(int fd)
+    {
+        // Our fd is readable. We deliberately do not read anything out of it: if we did, other sessions may miss the notification.
+        // Instead, we go into "polling mode:" we do not select() on our fd for a while, and sync periodically until the fd is no longer readable.
+        // However, if we are the one who posted the notification, we don't sync (until we clean up!)
+        bool should_sync = false;
+        if (readback_time_usec == 0)
+        {
+            polling_due_to_readable_fd = true;
+            drain_if_still_readable_time_usec = get_time() + SUSTAINED_READABILITY_CLEANUP_DURATION_USEC;
+            should_sync = true;
+        }
+        return should_sync;
+    }
+    
+    void post_notification()
+    {
+        if (pipe_fd >= 0)
+        {
+            // We need to write some data (any data) to the pipe, then wait for a while, then read it back.
+            // Nobody is expected to read it except us.
+            int pid_nbo = htonl(getpid());
+            ssize_t amt_written = write(this->pipe_fd, &pid_nbo, sizeof pid_nbo);
+            if (amt_written < 0)
+            {
+                if (errno == EWOULDBLOCK || errno == EAGAIN)
+                {
+                    // Very unsual: the pipe is full!
+                    drain_excessive_data();
+                }
+            }
+            
+            // Now schedule a read for some time in the future
+            this->readback_time_usec = get_time() + NAMED_PIPE_FLASH_DURATION_USEC;
+            this->readback_amount += sizeof pid_nbo;
+        }
+    }
+    
+    unsigned long usec_delay_between_polls() const
+    {
+        unsigned long readback_delay = ULONG_MAX;
+        if (this->readback_time_usec > 0)
+        {
+            // How long until the readback?
+            long long now = get_time();
+            if (now >= this->readback_time_usec)
+            {
+                // Oops, it already passed! Return something tiny.
+                readback_delay = 1000;
+            }
+            else
+            {
+                readback_delay = (unsigned long)(this->readback_time_usec - now);
+            }
+        }
+        
+        unsigned long polling_delay = ULONG_MAX;
+        if (polling_due_to_readable_fd)
+        {
+            // We're in polling mode
+            // Don't return a value less than our polling interval
+            polling_delay = NAMED_PIPE_FLASH_DURATION_USEC;
+        }
+        
+        // Now return the smaller of the two values
+        // If we get ULONG_MAX, it means there's no more need to poll; in that case return 0
+        unsigned long result = mini(readback_delay, polling_delay);
+        if (result == ULLONG_MAX)
+        {
+            result = 0;
+        }
+        return result;
+    }
+    
+    bool poll()
+    {
+        bool result = false;
+        
+        // Check if we are past the readback time
+        if (this->readback_time_usec > 0 && get_time() >= this->readback_time_usec)
+        {
+            // Read back what we wrote. We do nothing with the value.
+            while (this->readback_amount > 0)
+            {
+                char buff[64];
+                size_t amt_to_read = mini(this->readback_amount, sizeof buff);
+                read_ignore(this->pipe_fd, buff, amt_to_read);
+                this->readback_amount -= amt_to_read;
+            }
+            assert(this->readback_amount == 0);
+            this->readback_time_usec = 0;
+        }
+        
+        // Check to see if we are doing readability polling
+        if (polling_due_to_readable_fd && pipe_fd >= 0)
+        {
+            // We are polling, so we are definitely going to sync
+            result = true;
+            
+            // See if this is still readable
+            fd_set fds;
+            FD_ZERO(&fds);
+            FD_SET(this->pipe_fd, &fds);
+            struct timeval timeout = {};
+            select(this->pipe_fd + 1, &fds, NULL, NULL, &timeout);
+            if (! FD_ISSET(this->pipe_fd, &fds))
+            {
+                // No longer readable, no longer polling
+                polling_due_to_readable_fd = false;
+                drain_if_still_readable_time_usec = 0;
+            }
+            else
+            {
+                // Still readable. If it's been readable for a long time, there is probably lingering data on the pipe
+                if (get_time() >= drain_if_still_readable_time_usec)
+                {
+                    drain_excessive_data();
+                }
+            }
+        }
+        
+        return result;
+    }
+};
+
+class universal_notifier_null_t : public universal_notifier_t
+{
+    /* Does nothing! */
+};
+
+static universal_notifier_t::notifier_strategy_t fetch_default_strategy_from_environment()
+{
+    universal_notifier_t::notifier_strategy_t result = universal_notifier_t::strategy_default;
+
+    const struct
+    {
+        const char *name;
+        universal_notifier_t::notifier_strategy_t strat;
+    } options[] =
+    {
+        {"default", universal_notifier_t::strategy_default},
+        {"shmem", universal_notifier_t::strategy_shmem_polling},
+        {"pipe", universal_notifier_t::strategy_named_pipe},
+        {"notifyd", universal_notifier_t::strategy_notifyd}
+    };
+    const size_t opt_count = sizeof options / sizeof *options;
+
+    const char *var = getenv(UNIVERSAL_NOTIFIER_ENV_NAME);
+    if (var != NULL && var[0] != '\0')
+    {
+        size_t i;
+        for (i=0; i < opt_count; i++)
+        {
+            if (! strcmp(var, options[i].name))
+            {
+                result = options[i].strat;
+                break;
+            }
+        }
+        if (i >= opt_count)
+        {
+            fprintf(stderr, "Warning: unrecognized value for %s: '%s'\n", UNIVERSAL_NOTIFIER_ENV_NAME, var);
+            fprintf(stderr, "Warning: valid values are ");
+            for (size_t j=0; j < opt_count; j++)
+            {
+                fprintf(stderr, "%s%s", j > 0 ? ", " : "", options[j].name);
+            }
+            fputc('\n', stderr);
+        }
+    }
+    return result;
+}
+
+universal_notifier_t::notifier_strategy_t universal_notifier_t::resolve_default_strategy()
+{
+    static universal_notifier_t::notifier_strategy_t s_explicit_strategy = fetch_default_strategy_from_environment();
+    if (s_explicit_strategy != strategy_default)
+    {
+        return s_explicit_strategy;
+    }
+#if FISH_NOTIFYD_AVAILABLE
+    return strategy_notifyd;
+#else
+    return strategy_named_pipe;
+#endif
+}
+
+universal_notifier_t &universal_notifier_t::default_notifier()
+{
+    static universal_notifier_t *result = new_notifier_for_strategy(strategy_default);
+    return *result;
+}
+
+universal_notifier_t *universal_notifier_t::new_notifier_for_strategy(universal_notifier_t::notifier_strategy_t strat, const wchar_t *test_path)
+{
+    if (strat == strategy_default)
+    {
+        strat = resolve_default_strategy();
+    }
+    switch (strat)
+    {
+        case strategy_shmem_polling:
+            return new universal_notifier_shmem_poller_t();
+            
+        case strategy_notifyd:
+            return new universal_notifier_notifyd_t();
+            
+        case strategy_named_pipe:
+            return new universal_notifier_named_pipe_t(test_path);
+            
+        case strategy_null:
+            return new universal_notifier_null_t();
+        
+        default:
+            fprintf(stderr, "Unsupported strategy %d\n", strat);
+            return NULL;
+    }
+}
+
+/* Default implementations. */
+universal_notifier_t::universal_notifier_t()
+{
+}
+
+universal_notifier_t::~universal_notifier_t()
+{
+}
+
+int universal_notifier_t::notification_fd()
+{
+    return -1;
+}
+
+void universal_notifier_t::post_notification()
+{
+}
+
+bool universal_notifier_t::poll()
+{
+    return false;
+}
+
+unsigned long universal_notifier_t::usec_delay_between_polls() const
+{
+    return 0;
+}
+
+bool universal_notifier_t::notification_fd_became_readable(int fd)
+{
+    return false;
+}
+
+static bool bool_from_env_var(const char *name, bool default_value)
+{
+    const char *var = getenv(name);
+    return var ? from_string<bool>(var) : default_value;
+}
+
+bool universal_log_enabled()
+{
+    return bool_from_env_var(UNIVERSAL_LOGGING_ENV_NAME, false);
+}
+
diff --git a/src/env_universal_common.h b/src/env_universal_common.h
new file mode 100644
index 00000000..7aca60df
--- /dev/null
+++ b/src/env_universal_common.h
@@ -0,0 +1,184 @@
+#ifndef FISH_ENV_UNIVERSAL_COMMON_H
+#define FISH_ENV_UNIVERSAL_COMMON_H
+
+#include <string>
+#include <set>
+#include <pthread.h>
+#include <stdio.h>
+#include <vector>
+#include "common.h"
+#include "wutil.h"
+#include "env.h"
+
+/**
+   The different types of messages found in the fishd file
+*/
+typedef enum
+{
+    SET,
+    SET_EXPORT,
+    ERASE
+} fish_message_type_t;
+
+/**
+   The size of the buffer used for reading from the file
+*/
+#define ENV_UNIVERSAL_BUFFER_SIZE 1024
+
+/**
+   Callback data, reflecting a change in universal variables
+*/
+struct callback_data_t
+{
+    fish_message_type_t type;
+    wcstring key;
+    wcstring val;
+    
+    callback_data_t(fish_message_type_t t, const wcstring &k, const wcstring &v) : type(t), key(k), val(v)
+    {
+    }
+};
+
+typedef std::vector<struct callback_data_t> callback_data_list_t;
+
+/** Class representing universal variables */
+class env_universal_t
+{
+    /* Current values */
+    var_table_t vars;
+    
+    /* Keys that have been modified, and need to be written. A value here that is not present in vars indicates a deleted value. */
+    std::set<wcstring> modified;
+    
+    /* Path that we save to. If empty, use the default */
+    const wcstring explicit_vars_path;
+    
+    mutable pthread_mutex_t lock;
+    bool tried_renaming;
+    bool load_from_path(const wcstring &path, callback_data_list_t *callbacks);
+    void load_from_fd(int fd, callback_data_list_t *callbacks);
+    
+    void set_internal(const wcstring &key, const wcstring &val, bool exportv, bool overwrite);
+    bool remove_internal(const wcstring &name);
+    
+    /* Functions concerned with saving */
+    bool open_and_acquire_lock(const wcstring &path, int *out_fd);
+    bool open_temporary_file(const wcstring &directory, wcstring *out_path, int *out_fd);
+    bool write_to_fd(int fd, const wcstring &path);
+    bool move_new_vars_file_into_place(const wcstring &src, const wcstring &dst);
+    
+    /* File id from which we last read */
+    file_id_t last_read_file;
+    
+    /* Given a variable table, generate callbacks representing the difference between our vars and the new vars */
+    void generate_callbacks(const var_table_t &new_vars, callback_data_list_t *callbacks) const;
+    
+    /* Given a variable table, copy unmodified values into self. May destructively modified vars_to_acquire. */
+    void acquire_variables(var_table_t *vars_to_acquire);
+    
+    static void parse_message_internal(const wcstring &msg, var_table_t *vars, wcstring *storage);
+    static var_table_t read_message_internal(int fd);
+    
+public:
+    env_universal_t(const wcstring &path);
+    ~env_universal_t();
+    
+    /* Get the value of the variable with the specified name */
+    env_var_t get(const wcstring &name) const;
+    
+    /* Returns whether the variable with the given name is exported, or false if it does not exist */
+    bool get_export(const wcstring &name) const;
+    
+    /* Sets a variable */
+    void set(const wcstring &key, const wcstring &val, bool exportv);
+    
+    /* Removes a variable. Returns true if it was found, false if not. */
+    bool remove(const wcstring &name);
+    
+    /* Gets variable names */
+    wcstring_list_t get_names(bool show_exported, bool show_unexported) const;
+    
+    /** Loads variables at the correct path */
+    bool load();
+
+    /** Reads and writes variables at the correct path. Returns true if modified variables were written. */
+    bool sync(callback_data_list_t *callbacks);
+};
+
+/** The "universal notifier" is an object responsible for broadcasting and receiving universal variable change notifications. These notifications do not contain the change, but merely indicate that the uvar file has changed. It is up to the uvar subsystem to re-read the file.
+
+   We support a few notificatins strategies. Not all strategies are supported on all platforms.
+   
+   Notifiers may request polling, and/or provide a file descriptor to be watched for readability in select().
+   
+   To request polling, the notifier overrides usec_delay_between_polls() to return a positive value. That value will be used as the timeout in select(). When select returns, the loop invokes poll(). poll() should return true to indicate that the file may have changed.
+    
+   To provide a file descriptor, the notifier overrides notification_fd() to return a non-negative fd. This will be added to the "read" file descriptor list in select(). If the fd is readable, notification_fd_became_readable() will be called; that function should be overridden to return true if the file may have changed.
+
+*/
+class universal_notifier_t
+{
+public:
+    enum notifier_strategy_t
+    {
+        // Default meta-strategy to use the 'best' notifier for the system
+        strategy_default,
+        
+        // Use a value in shared memory. Simple, but requires polling and therefore semi-frequent wakeups.
+        strategy_shmem_polling,
+        
+        // Strategy that uses a named pipe. Somewhat complex, but portable and doesn't require polling most of the time.
+        strategy_named_pipe,
+                
+        // Strategy that uses notify(3). Simple and efficient, but OS X only.
+        strategy_notifyd,
+        
+        // Null notifier, does nothing
+        strategy_null
+    };
+
+    protected:
+    universal_notifier_t();
+    
+    private:
+    /* No copying */
+    universal_notifier_t &operator=(const universal_notifier_t &);
+    universal_notifier_t(const universal_notifier_t &x);
+    static notifier_strategy_t resolve_default_strategy();
+    
+    public:
+    
+    virtual ~universal_notifier_t();
+    
+    /* Factory constructor. Free with delete */
+    static universal_notifier_t *new_notifier_for_strategy(notifier_strategy_t strat, const wchar_t *test_path = NULL);
+    
+    /* Default instance. Other instances are possible for testing. */
+    static universal_notifier_t &default_notifier();
+    
+    /* Does a fast poll(). Returns true if changed. */
+    virtual bool poll();
+    
+    /* Triggers a notification */
+    virtual void post_notification();
+    
+    /* Recommended delay between polls. A value of 0 means no polling required (so no timeout) */
+    virtual unsigned long usec_delay_between_polls() const;
+    
+    /* Returns the fd from which to watch for events, or -1 if none */
+    virtual int notification_fd();
+    
+    /* The notification_fd is readable; drain it. Returns true if a notification is considered to have been posted. */
+    virtual bool notification_fd_became_readable(int fd);
+};
+
+bool universal_log_enabled();
+#define UNIVERSAL_LOG(x) if (universal_log_enabled()) fprintf(stderr, "UNIVERSAL LOG: %s\n", x)
+
+/* Environment variable for requesting a particular universal notifier. See fetch_default_strategy_from_environment for names. */
+#define UNIVERSAL_NOTIFIER_ENV_NAME "fish_universal_notifier"
+
+/* Environment variable for enabling universal variable logging (to stderr) */
+#define UNIVERSAL_LOGGING_ENV_NAME "fish_universal_log"
+
+#endif
diff --git a/src/event.cpp b/src/event.cpp
new file mode 100644
index 00000000..eeff3eb5
--- /dev/null
+++ b/src/event.cpp
@@ -0,0 +1,737 @@
+/** \file event.c
+
+  Functions for handling event triggers
+
+*/
+#include "config.h" // IWYU pragma: keep
+
+#include <signal.h>
+#include <algorithm>
+#include <assert.h>
+#include <stddef.h>
+#include <string>
+
+#include "fallback.h" // IWYU pragma: keep
+#include "wutil.h" // IWYU pragma: keep - needed for gettext
+#include "input_common.h"
+#include "proc.h"
+#include "parser.h"
+#include "common.h"
+#include "event.h"
+#include "signal.h"
+#include "io.h"
+
+
+/**
+   Number of signals that can be queued before an overflow occurs
+*/
+#define SIG_UNHANDLED_MAX 64
+
+/**
+   This struct contains a list of generated signals waiting to be
+   dispatched
+*/
+typedef struct
+{
+    /**
+       Number of delivered signals
+    */
+    volatile int count;
+    /**
+       Whether signals have been skipped
+    */
+    volatile int overflow;
+    /**
+       Array of signal events
+    */
+    volatile int signal[SIG_UNHANDLED_MAX];
+}
+signal_list_t;
+
+/**
+  The signal event list. Actually two separate lists. One which is
+  active, which is the one that new events is written to. The inactive
+  one contains the events that are currently beeing performed.
+*/
+static signal_list_t sig_list[2]= {{},{}};
+
+/**
+   The index of sig_list that is the list of signals currently written to
+*/
+static volatile int active_list=0;
+
+typedef std::vector<event_t *> event_list_t;
+
+/**
+   List of event handlers.
+*/
+static event_list_t s_event_handlers;
+/**
+   List of event handlers that should be removed
+*/
+static event_list_t killme;
+
+/**
+   List of events that have been sent but have not yet been delivered because they are blocked.
+*/
+static event_list_t blocked;
+
+/** Variables (one per signal) set when a signal is observed. This is inspected by a signal handler. */
+static volatile bool s_observed_signals[NSIG] = {};
+static void set_signal_observed(int sig, bool val)
+{
+    ASSERT_IS_MAIN_THREAD();
+    if (sig >= 0 && (size_t)sig < sizeof s_observed_signals / sizeof *s_observed_signals)
+    {
+        s_observed_signals[sig] = val;
+    }
+}
+
+/**
+   Tests if one event instance matches the definition of a event
+   class. If both the class and the instance name a function,
+   they must name the same function.
+
+*/
+static int event_match(const event_t &classv, const event_t &instance)
+{
+
+    /* If the function names are both non-empty and different, then it's not a match */
+    if (! classv.function_name.empty() &&
+            ! instance.function_name.empty() &&
+            classv.function_name != instance.function_name)
+    {
+        return 0;
+    }
+
+    if (classv.type == EVENT_ANY)
+        return 1;
+
+    if (classv.type != instance.type)
+        return 0;
+
+
+    switch (classv.type)
+    {
+
+        case EVENT_SIGNAL:
+            if (classv.param1.signal == EVENT_ANY_SIGNAL)
+                return 1;
+            return classv.param1.signal == instance.param1.signal;
+
+        case EVENT_VARIABLE:
+            return instance.str_param1 == classv.str_param1;
+
+        case EVENT_EXIT:
+            if (classv.param1.pid == EVENT_ANY_PID)
+                return 1;
+            return classv.param1.pid == instance.param1.pid;
+
+        case EVENT_JOB_ID:
+            return classv.param1.job_id == instance.param1.job_id;
+
+        case EVENT_GENERIC:
+            return instance.str_param1 == classv.str_param1;
+
+    }
+
+    /**
+       This should never be reached
+    */
+    debug(0, "Warning: Unreachable code reached in event_match in event.cpp\n");
+    return 0;
+}
+
+
+
+/**
+   Test if specified event is blocked
+*/
+static int event_is_blocked(const event_t &e)
+{
+    const block_t *block;
+    parser_t &parser = parser_t::principal_parser();
+
+    size_t idx = 0;
+    while ((block = parser.block_at_index(idx++)))
+    {
+        if (event_block_list_blocks_type(block->event_blocks, e.type))
+            return true;
+
+    }
+    return event_block_list_blocks_type(parser.global_event_blocks, e.type);
+}
+
+wcstring event_get_desc(const event_t &e)
+{
+
+    wcstring result;
+    switch (e.type)
+    {
+
+        case EVENT_SIGNAL:
+            result = format_string(_(L"signal handler for %ls (%ls)"), sig2wcs(e.param1.signal), signal_get_desc(e.param1.signal));
+            break;
+
+        case EVENT_VARIABLE:
+            result = format_string(_(L"handler for variable '%ls'"), e.str_param1.c_str());
+            break;
+
+        case EVENT_EXIT:
+            if (e.param1.pid > 0)
+            {
+                result = format_string(_(L"exit handler for process %d"), e.param1.pid);
+            }
+            else
+            {
+                job_t *j = job_get_from_pid(-e.param1.pid);
+                if (j)
+                    result = format_string(_(L"exit handler for job %d, '%ls'"), j->job_id, j->command_wcstr());
+                else
+                    result = format_string(_(L"exit handler for job with process group %d"), -e.param1.pid);
+            }
+
+            break;
+
+        case EVENT_JOB_ID:
+        {
+            job_t *j = job_get(e.param1.job_id);
+            if (j)
+                result = format_string(_(L"exit handler for job %d, '%ls'"), j->job_id, j->command_wcstr());
+            else
+                result = format_string(_(L"exit handler for job with job id %d"), e.param1.job_id);
+
+            break;
+        }
+
+        case EVENT_GENERIC:
+            result = format_string(_(L"handler for generic event '%ls'"), e.str_param1.c_str());
+            break;
+
+        default:
+            result = format_string(_(L"Unknown event type '0x%x'"), e.type);
+            break;
+
+    }
+
+    return result;
+}
+
+#if 0
+static void show_all_handlers(void)
+{
+    puts("event handlers:");
+    for (event_list_t::const_iterator iter = events.begin(); iter != events.end(); ++iter)
+    {
+        const event_t *foo = *iter;
+        wcstring tmp = event_get_desc(foo);
+        printf("    handler now %ls\n", tmp.c_str());
+    }
+}
+#endif
+
+/*
+    Give a more condensed description of \c event compared to \c event_get_desc.
+    It includes what function will fire if the \c event is an event handler.
+ */
+static wcstring event_desc_compact(const event_t &event)
+{
+    wcstring res;
+    wchar_t const *temp;
+    int sig;
+    switch (event.type)
+    {
+        case EVENT_ANY:
+            res = L"EVENT_ANY";
+            break;
+        case EVENT_VARIABLE:
+            if (event.str_param1.c_str())
+            {
+                res = format_string(L"EVENT_VARIABLE($%ls)", event.str_param1.c_str());
+            }
+            else
+            {
+                res = L"EVENT_VARIABLE([any])";
+            }
+            break;
+        case EVENT_SIGNAL:
+            sig = event.param1.signal;
+            if (sig == EVENT_ANY_SIGNAL)
+            {
+                temp = L"[all signals]";
+            }
+            else if (sig == 0)
+            {
+                temp = L"not set";
+            }
+            else
+            {
+                temp = sig2wcs(sig);
+            }
+            res = format_string(L"EVENT_SIGNAL(%ls)", temp);
+            break;
+        case EVENT_EXIT:
+            if (event.param1.pid == EVENT_ANY_PID)
+            {
+                res = wcstring(L"EVENT_EXIT([all child processes])");
+            }
+            else if (event.param1.pid > 0)
+            {
+                res = format_string(L"EVENT_EXIT(pid %d)", event.param1.pid);
+            }
+            else
+            {
+                job_t *j = job_get_from_pid(-event.param1.pid);
+                if (j)
+                    res = format_string(L"EVENT_EXIT(jobid %d: \"%ls\")", j->job_id, j->command_wcstr());
+                else
+                    res = format_string(L"EVENT_EXIT(pgid %d)", -event.param1.pid);
+            }
+            break;
+        case EVENT_JOB_ID:
+        {
+            job_t *j = job_get(event.param1.job_id);
+            if (j)
+                res = format_string(L"EVENT_JOB_ID(job %d: \"%ls\")", j->job_id, j->command_wcstr());
+            else
+                res = format_string(L"EVENT_JOB_ID(jobid %d)", event.param1.job_id);
+            break;
+        }
+        case EVENT_GENERIC:
+            res = format_string(L"EVENT_GENERIC(%ls)", event.str_param1.c_str());
+            break;
+        default:
+            res = format_string(L"unknown/illegal event(%x)", event.type);
+    }
+    if (event.function_name.size())
+    {
+        return format_string(L"%ls: \"%ls\"", res.c_str(), event.function_name.c_str());
+    }
+    else
+    {
+        return res;
+    }
+}
+
+
+void event_add_handler(const event_t &event)
+{
+    event_t *e;
+
+    if (debug_level >= 3)
+    {
+        wcstring desc = event_desc_compact(event);
+        debug(3, "register: %ls\n", desc.c_str());
+    }
+
+    e = new event_t(event);
+
+    if (e->type == EVENT_SIGNAL)
+    {
+        signal_handle(e->param1.signal, 1);
+        set_signal_observed(e->param1.signal, true);
+    }
+
+    s_event_handlers.push_back(e);
+}
+
+void event_remove(const event_t &criterion)
+{
+    event_list_t new_list;
+
+    if (debug_level >= 3)
+    {
+        wcstring desc = event_desc_compact(criterion);
+        debug(3, "unregister: %ls\n", desc.c_str());
+    }
+
+    /*
+      Because of concurrency issues (env_remove could remove an event
+      that is currently being executed), env_remove does not actually
+      free any events - instead it simply moves all events that should
+      be removed from the event list to the killme list, and the ones
+      that shouldn't be killed to new_list, and then drops the empty
+      events-list.
+    */
+
+    if (s_event_handlers.empty())
+        return;
+
+    for (size_t i=0; i<s_event_handlers.size(); i++)
+    {
+        event_t *n = s_event_handlers.at(i);
+        if (event_match(criterion, *n))
+        {
+            killme.push_back(n);
+
+            /*
+              If this event was a signal handler and no other handler handles
+              the specified signal type, do not handle that type of signal any
+              more.
+            */
+            if (n->type == EVENT_SIGNAL)
+            {
+                event_t e = event_t::signal_event(n->param1.signal);
+                if (event_get(e, 0) == 1)
+                {
+                    signal_handle(e.param1.signal, 0);
+                    set_signal_observed(e.param1.signal, 0);
+                }
+            }
+        }
+        else
+        {
+            new_list.push_back(n);
+        }
+    }
+    s_event_handlers.swap(new_list);
+}
+
+int event_get(const event_t &criterion, std::vector<event_t *> *out)
+{
+    int found = 0;
+    for (size_t i=0; i < s_event_handlers.size(); i++)
+    {
+        event_t *n = s_event_handlers.at(i);
+        if (event_match(criterion, *n))
+        {
+            found++;
+            if (out)
+                out->push_back(n);
+        }
+    }
+    return found;
+}
+
+bool event_is_signal_observed(int sig)
+{
+    /* We are in a signal handler! Don't allocate memory, etc.
+    */
+    bool result = false;
+    if (sig >= 0 && sig < sizeof s_observed_signals / sizeof *s_observed_signals)
+    {
+        result = s_observed_signals[sig];
+    }
+    return result;
+}
+
+/**
+   Free all events in the kill list
+*/
+static void event_free_kills()
+{
+    for_each(killme.begin(), killme.end(), event_free);
+    killme.resize(0);
+}
+
+/**
+   Test if the specified event is waiting to be killed
+*/
+static int event_is_killed(const event_t &e)
+{
+    return std::find(killme.begin(), killme.end(), &e) != killme.end();
+}
+
+/* Callback for firing (and then deleting) an event */
+static void fire_event_callback(void *arg)
+{
+    ASSERT_IS_MAIN_THREAD();
+    assert(arg != NULL);
+    event_t *event = static_cast<event_t *>(arg);
+    event_fire(event);
+    delete event;
+}
+
+/**
+   Perform the specified event. Since almost all event firings will
+   not be matched by even a single event handler, we make sure to
+   optimize the 'no matches' path. This means that nothing is
+   allocated/initialized unless needed.
+*/
+static void event_fire_internal(const event_t &event)
+{
+
+    event_list_t fire;
+
+    /*
+      First we free all events that have been removed, but only if this
+    		invocation of event_fire_internal is not a recursive call.
+    */
+    if (is_event <= 1)
+        event_free_kills();
+
+    if (s_event_handlers.empty())
+        return;
+
+    /*
+      Then we iterate over all events, adding events that should be
+      fired to a second list. We need to do this in a separate step
+      since an event handler might call event_remove or
+      event_add_handler, which will change the contents of the \c
+      events list.
+    */
+    for (size_t i=0; i<s_event_handlers.size(); i++)
+    {
+        event_t *criterion = s_event_handlers.at(i);
+
+        /*
+          Check if this event is a match
+        */
+        if (event_match(*criterion, event))
+        {
+            fire.push_back(criterion);
+        }
+    }
+
+    /*
+      No matches. Time to return.
+    */
+    if (fire.empty())
+        return;
+
+    if (signal_is_blocked())
+    {
+        /* Fix for https://github.com/fish-shell/fish-shell/issues/608. Don't run event handlers while signals are blocked. */
+        event_t *heap_event = new event_t(event);
+        input_common_add_callback(fire_event_callback, heap_event);
+        return;
+    }
+
+    /*
+      Iterate over our list of matching events
+    */
+
+    for (size_t i=0; i<fire.size(); i++)
+    {
+        event_t *criterion = fire.at(i);
+        int prev_status;
+
+        /*
+          Check if this event has been removed, if so, dont fire it
+        */
+        if (event_is_killed(*criterion))
+            continue;
+
+        /*
+          Fire event
+        */
+        wcstring buffer = criterion->function_name;
+
+        for (size_t j=0; j < event.arguments.size(); j++)
+        {
+            wcstring arg_esc = escape_string(event.arguments.at(j), 1);
+            buffer += L" ";
+            buffer += arg_esc;
+        }
+
+        // debug( 1, L"Event handler fires command '%ls'", buffer.c_str() );
+
+        /*
+          Event handlers are not part of the main flow of code, so
+          they are marked as non-interactive
+        */
+        proc_push_interactive(0);
+        prev_status = proc_get_last_status();
+        parser_t &parser = parser_t::principal_parser();
+
+        block_t *block = new event_block_t(event);
+        parser.push_block(block);
+        parser.eval(buffer, io_chain_t(), TOP);
+        parser.pop_block();
+        proc_pop_interactive();
+        proc_set_last_status(prev_status);
+    }
+
+    /*
+      Free killed events
+    */
+    if (is_event <= 1)
+        event_free_kills();
+
+}
+
+/**
+   Handle all pending signal events
+*/
+static void event_fire_delayed()
+{
+    /*
+      If is_event is one, we are running the event-handler non-recursively.
+
+      When the event handler has called a piece of code that triggers
+      another event, we do not want to fire delayed events because of
+      concurrency problems.
+    */
+    if (! blocked.empty() && is_event==1)
+    {
+        event_list_t new_blocked;
+
+        for (size_t i=0; i<blocked.size(); i++)
+        {
+            event_t *e = blocked.at(i);
+            if (event_is_blocked(*e))
+            {
+                new_blocked.push_back(new event_t(*e));
+            }
+            else
+            {
+                event_fire_internal(*e);
+                event_free(e);
+            }
+        }
+        blocked.swap(new_blocked);
+    }
+
+    int al = active_list;
+
+    while (sig_list[al].count > 0)
+    {
+        signal_list_t *lst;
+
+        /*
+          Switch signal lists
+        */
+        sig_list[1-al].count=0;
+        sig_list[1-al].overflow=0;
+        al = 1-al;
+        active_list=al;
+
+        /*
+          Set up
+        */
+        lst = &sig_list[1-al];
+        event_t e = event_t::signal_event(0);
+        e.arguments.resize(1);
+
+        if (lst->overflow)
+        {
+            debug(0, _(L"Signal list overflow. Signals have been ignored."));
+        }
+
+        /*
+          Send all signals in our private list
+        */
+        for (int i=0; i < lst->count; i++)
+        {
+            e.param1.signal = lst->signal[i];
+            e.arguments.at(0) = sig2wcs(e.param1.signal);
+            if (event_is_blocked(e))
+            {
+                blocked.push_back(new event_t(e));
+            }
+            else
+            {
+                event_fire_internal(e);
+            }
+        }
+
+    }
+}
+
+void event_fire_signal(int signal)
+{
+    /*
+      This means we are in a signal handler. We must be very
+      careful not do do anything that could cause a memory
+      allocation or something else that might be bad when in a
+      signal handler.
+    */
+    if (sig_list[active_list].count < SIG_UNHANDLED_MAX)
+        sig_list[active_list].signal[sig_list[active_list].count++]=signal;
+    else
+        sig_list[active_list].overflow=1;
+}
+
+
+void event_fire(const event_t *event)
+{
+
+    if (event && event->type == EVENT_SIGNAL)
+    {
+        event_fire_signal(event->param1.signal);
+    }
+    else
+    {
+        is_event++;
+
+        /*
+          Fire events triggered by signals
+        */
+        event_fire_delayed();
+
+        if (event)
+        {
+            if (event_is_blocked(*event))
+            {
+                blocked.push_back(new event_t(*event));
+            }
+            else
+            {
+                event_fire_internal(*event);
+            }
+        }
+        is_event--;
+    }
+}
+
+
+void event_init()
+{
+}
+
+void event_destroy()
+{
+
+    for_each(s_event_handlers.begin(), s_event_handlers.end(), event_free);
+    s_event_handlers.clear();
+
+    for_each(killme.begin(), killme.end(), event_free);
+    killme.clear();
+}
+
+void event_free(event_t *e)
+{
+    CHECK(e,);
+    delete e;
+}
+
+void event_fire_generic(const wchar_t *name, wcstring_list_t *args)
+{
+    CHECK(name,);
+
+    event_t ev(EVENT_GENERIC);
+    ev.str_param1 = name;
+    if (args)
+        ev.arguments = *args;
+    event_fire(&ev);
+}
+
+event_t::event_t(int t) : type(t), param1(), str_param1(), function_name(), arguments()
+{
+}
+
+event_t::~event_t()
+{
+}
+
+event_t event_t::signal_event(int sig)
+{
+    event_t event(EVENT_SIGNAL);
+    event.param1.signal = sig;
+    return event;
+}
+
+event_t event_t::variable_event(const wcstring &str)
+{
+    event_t event(EVENT_VARIABLE);
+    event.str_param1 = str;
+    return event;
+}
+
+event_t event_t::generic_event(const wcstring &str)
+{
+    event_t event(EVENT_GENERIC);
+    event.str_param1 = str;
+    return event;
+}
+
diff --git a/src/event.h b/src/event.h
new file mode 100644
index 00000000..80737596
--- /dev/null
+++ b/src/event.h
@@ -0,0 +1,177 @@
+/** \file event.h
+
+  Functions for handling event triggers
+
+  Because most of these functions can be called by signal
+  handler, it is important to make it well defined when these
+  functions produce output or perform memory allocations, since
+  such functions may not be safely called by signal handlers.
+
+
+*/
+#ifndef FISH_EVENT_H
+#define FISH_EVENT_H
+
+#include <ctime>
+#include <vector>
+
+#include "common.h"
+
+/**
+   The signal number that is used to match any signal
+*/
+#define EVENT_ANY_SIGNAL -1
+
+/**
+   The process id that is used to match any process id
+*/
+#define EVENT_ANY_PID 0
+
+/**
+   Enumeration of event types
+*/
+enum
+{
+    EVENT_ANY, /**< Matches any event type (Not always any event, as the function name may limit the choice as well */
+    EVENT_SIGNAL, /**< An event triggered by a signal */
+    EVENT_VARIABLE, /**< An event triggered by a variable update */
+    EVENT_EXIT, /**< An event triggered by a job or process exit */
+    EVENT_JOB_ID, /**< An event triggered by a job exit */
+    EVENT_GENERIC, /**< A generic event */
+}
+;
+
+/**
+   The structure which represents an event. The event_t struct has
+   several event-related use-cases:
+
+   - When used as a parameter to event_add, it represents a class of events, and function_name is the name of the function which will be called whenever an event matching the specified class occurs. This is also how events are stored internally.
+   - When used as a parameter to event_get, event_remove and event_fire, it represents a class of events, and if the function_name field is non-zero, only events which call the specified function will be returned.
+*/
+struct event_t
+{
+public:
+
+    /** Type of event */
+    int type;
+
+    /** The type-specific parameter. The int types are one of the following:
+
+         signal: Signal number for signal-type events.Use EVENT_ANY_SIGNAL to match any signal
+             pid: Process id for process-type events. Use EVENT_ANY_PID to match any pid.
+             job_id: Job id for EVENT_JOB_ID type events
+      */
+    union
+    {
+        int signal;
+        int job_id;
+        pid_t pid;
+    } param1;
+
+    /** The string types are one of the following:
+
+          variable: Variable name for variable-type events.
+          param: The parameter describing this generic event.
+    */
+    wcstring str_param1;
+
+    /**
+       The name of the event handler function
+    */
+    wcstring function_name;
+
+    /**
+       The argument list. Only used when sending a new event using
+       event_fire. In all other situations, the value of this variable
+       is ignored.
+    */
+    wcstring_list_t arguments;
+
+    event_t(int t);
+    ~event_t();
+
+    static event_t signal_event(int sig);
+    static event_t variable_event(const wcstring &str);
+    static event_t generic_event(const wcstring &str);
+};
+
+/**
+   Add an event handler
+
+   May not be called by a signal handler, since it may allocate new memory.
+*/
+void event_add_handler(const event_t &event);
+
+/**
+   Remove all events matching the specified criterion.
+
+   May not be called by a signal handler, since it may free allocated memory.
+*/
+void event_remove(const event_t &event);
+
+/**
+   Return all events which match the specified event class
+
+   This function is safe to call from a signal handler _ONLY_ if the
+   out parameter is null.
+
+   \param criterion Is the class of events to return. If the criterion has a non-null function_name, only events which trigger the specified function will return.
+   \param out the list to add events to. May be 0, in which case no events will be added, but the result count will still be valid
+
+   \return the number of found matches
+*/
+int event_get(const event_t &criterion, std::vector<event_t *> *out);
+
+/**
+    Returns whether an event listener is registered for the given signal.
+    This is safe to call from a signal handler.
+*/
+bool event_is_signal_observed(int signal);
+
+/**
+   Fire the specified event. The function_name field of the event must
+   be set to 0. If the event is of type EVENT_SIGNAL, no the event is
+   queued, and will be dispatched the next time event_fire is
+   called. If event is a null-pointer, all pending events are
+   dispatched.
+
+   This function is safe to call from a signal handler _ONLY_ if the
+   event parameter is for a signal. Signal events not be fired, by the
+   call to event_fire, instead they will be fired the next time
+   event_fire is called with anull argument. This is needed to make
+   sure that no code evaluation is ever performed by a signal handler.
+
+   \param event the specific event whose handlers should fire. If
+   null, then all delayed events will be fired.
+*/
+void event_fire(const event_t *event);
+
+/** Like event_fire, but takes a signal directly. */
+void event_fire_signal(int signal);
+
+/**
+   Initialize the event-handling library
+*/
+void event_init();
+
+/**
+   Destroy the event-handling library
+*/
+void event_destroy();
+
+/**
+   Free all memory used by the specified event
+*/
+void event_free(event_t *e);
+
+/**
+   Returns a string describing the specified event.
+*/
+wcstring event_get_desc(const event_t &e);
+
+/**
+   Fire a generic event with the specified name
+*/
+void event_fire_generic(const wchar_t *name, wcstring_list_t *args = NULL);
+
+#endif
diff --git a/src/exec.cpp b/src/exec.cpp
new file mode 100644
index 00000000..d6df94e6
--- /dev/null
+++ b/src/exec.cpp
@@ -0,0 +1,1494 @@
+/** \file exec.c
+  Functions for executing a program.
+
+  Some of the code in this file is based on code from the Glibc
+  manual, though the changes performed have been massive.
+*/
+
+#include "config.h" // IWYU pragma: keep
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <errno.h>
+#include <wchar.h>
+#include <string.h>
+#include <signal.h>
+#include <assert.h>
+#include <vector>
+#include <algorithm>
+#include <spawn.h>
+#include <wctype.h>
+#include <map>
+#include <string>
+#include <memory> // IWYU pragma: keep - suggests <tr1/memory> instead
+#include <utility>
+
+#ifdef HAVE_SIGINFO_H
+#include <siginfo.h>
+#endif
+
+#include "fallback.h" // IWYU pragma: keep
+#include "postfork.h"
+#include "common.h"
+#include "wutil.h"
+#include "proc.h"
+#include "exec.h"
+#include "parser.h"
+#include "builtin.h"
+#include "function.h"
+#include "env.h"
+#include "signal.h"
+#include "parse_util.h"
+#include "io.h"
+#include "parse_tree.h"
+
+/**
+   file descriptor redirection error message
+*/
+#define FD_ERROR   _( L"An error occurred while redirecting file descriptor %d" )
+
+/**
+   file descriptor redirection error message
+*/
+#define WRITE_ERROR   _( L"An error occurred while writing output" )
+
+/**
+   file redirection error message
+*/
+#define FILE_ERROR _( L"An error occurred while redirecting file '%s'" )
+
+/**
+   Base open mode to pass to calls to open
+*/
+#define OPEN_MASK 0666
+
+// Called in a forked child
+static void exec_write_and_exit(int fd, const char *buff, size_t count, int status)
+{
+    if (write_loop(fd, buff, count) == -1)
+    {
+        debug(0, WRITE_ERROR);
+        wperror(L"write");
+        exit_without_destructors(status);
+    }
+    exit_without_destructors(status);
+}
+
+
+void exec_close(int fd)
+{
+    ASSERT_IS_MAIN_THREAD();
+
+    /* This may be called in a child of fork(), so don't allocate memory */
+    if (fd < 0)
+    {
+        debug(0, L"Called close on invalid file descriptor ");
+        return;
+    }
+
+    while (close(fd) == -1)
+    {
+        if (errno != EINTR)
+        {
+            debug(1, FD_ERROR, fd);
+            wperror(L"close");
+            break;
+        }
+    }
+}
+
+int exec_pipe(int fd[2])
+{
+    ASSERT_IS_MAIN_THREAD();
+
+    int res;
+    while ((res=pipe(fd)))
+    {
+        if (errno != EINTR)
+        {
+            // caller will call wperror
+            return res;
+        }
+    }
+
+    debug(4, L"Created pipe using fds %d and %d", fd[0], fd[1]);
+    
+    // Pipes ought to be cloexec
+    // Pipes are dup2'd the corresponding fds; the resulting fds are not cloexec
+    set_cloexec(fd[0]);
+    set_cloexec(fd[1]);
+    return res;
+}
+
+/* Returns true if the redirection is a file redirection to a file other than /dev/null */
+static bool redirection_is_to_real_file(const io_data_t *io)
+{
+    bool result = false;
+    if (io != NULL && io->io_mode == IO_FILE)
+    {
+        /* It's a file redirection. Compare the path to /dev/null */
+        CAST_INIT(const io_file_t *, io_file, io);
+        const char *path = io_file->filename_cstr;
+        if (strcmp(path, "/dev/null") != 0)
+        {
+            /* It's not /dev/null */
+            result = true;
+        }
+
+    }
+    return result;
+}
+
+static bool chain_contains_redirection_to_real_file(const io_chain_t &io_chain)
+{
+    bool result = false;
+    for (size_t idx=0; idx < io_chain.size(); idx++)
+    {
+        const shared_ptr<const io_data_t> &io = io_chain.at(idx);
+        if (redirection_is_to_real_file(io.get()))
+        {
+            result = true;
+            break;
+        }
+    }
+    return result;
+}
+
+/**
+   Returns the interpreter for the specified script. Returns NULL if file
+   is not a script with a shebang.
+ */
+char *get_interpreter(const char *command, char *interpreter, size_t buff_size)
+{
+    // OK to not use CLO_EXEC here because this is only called after fork
+    int fd = open(command, O_RDONLY);
+    if (fd >= 0)
+    {
+        size_t idx = 0;
+        while (idx + 1 < buff_size)
+        {
+            char ch;
+            ssize_t amt = read(fd, &ch, sizeof ch);
+            if (amt <= 0)
+                break;
+            if (ch == '\n')
+                break;
+            interpreter[idx++] = ch;
+        }
+        interpreter[idx++] = '\0';
+        close(fd);
+    }
+    if (strncmp(interpreter, "#! /", 4) == 0)
+    {
+        return interpreter + 3;
+    }
+    else if (strncmp(interpreter, "#!/", 3) == 0)
+    {
+        return interpreter + 2;
+    }
+    else
+    {
+        return NULL;
+    }
+}
+
+/**
+   This function is executed by the child process created by a call to
+   fork(). It should be called after \c setup_child_process. It calls
+   execve to replace the fish process image with the command specified
+   in \c p. It never returns.
+*/
+/* Called in a forked child! Do not allocate memory, etc. */
+static void safe_launch_process(process_t *p, const char *actual_cmd, const char *const* cargv, const char *const *cenvv)
+{
+    int err;
+
+//  debug( 1, L"exec '%ls'", p->argv[0] );
+
+    // This function never returns, so we take certain liberties with constness
+    char * const * envv = const_cast<char* const *>(cenvv);
+    char * const * argv = const_cast<char* const *>(cargv);
+
+    execve(actual_cmd, argv, envv);
+
+    err = errno;
+
+    /*
+       Something went wrong with execve, check for a ":", and run
+       /bin/sh if encountered. This is a weird predecessor to the shebang
+       that is still sometimes used since it is supported on Windows.
+    */
+    /* OK to not use CLO_EXEC here because this is called after fork and the file is immediately closed */
+    int fd = open(actual_cmd, O_RDONLY);
+    if (fd >= 0)
+    {
+        char begin[1] = {0};
+        ssize_t amt_read = read(fd, begin, 1);
+        close(fd);
+
+        if ((amt_read==1) && (begin[0] == ':'))
+        {
+            // Relaunch it with /bin/sh. Don't allocate memory, so if you have more args than this, update your silly script! Maybe this should be changed to be based on ARG_MAX somehow.
+            char sh_command[] = "/bin/sh";
+            char *argv2[128];
+            argv2[0] = sh_command;
+            for (size_t i=1; i < sizeof argv2 / sizeof *argv2; i++)
+            {
+                argv2[i] = argv[i-1];
+                if (argv2[i] == NULL)
+                    break;
+            }
+
+            execve(sh_command, argv2, envv);
+        }
+    }
+
+    errno = err;
+    safe_report_exec_error(errno, actual_cmd, argv, envv);
+    exit_without_destructors(STATUS_EXEC_FAIL);
+}
+
+/**
+   This function is similar to launch_process, except it is not called after a fork (i.e. it only calls exec) and therefore it can allocate memory.
+*/
+static void launch_process_nofork(process_t *p)
+{
+    ASSERT_IS_MAIN_THREAD();
+    ASSERT_IS_NOT_FORKED_CHILD();
+
+    null_terminated_array_t<char> argv_array;
+    convert_wide_array_to_narrow(p->get_argv_array(), &argv_array);
+
+    const char *const *envv = env_export_arr(false);
+    char *actual_cmd = wcs2str(p->actual_cmd.c_str());
+
+    /* Bounce to launch_process. This never returns. */
+    safe_launch_process(p, actual_cmd, argv_array.get(), envv);
+}
+
+
+/**
+   Check if the IO redirection chains contains redirections for the
+   specified file descriptor
+*/
+static int has_fd(const io_chain_t &d, int fd)
+{
+    return io_chain_get(d, fd).get() != NULL;
+}
+
+/**
+   Close a list of fds.
+*/
+static void io_cleanup_fds(const std::vector<int> &opened_fds)
+{
+    std::for_each(opened_fds.begin(), opened_fds.end(), close);
+}
+
+/**
+   Make a copy of the specified io redirection chain, but change file
+   redirection into fd redirection. This makes the redirection chain
+   suitable for use as block-level io, since the file won't be
+   repeatedly reopened for every command in the block, which would
+   reset the cursor position.
+
+   \return true on success, false on failure. Returns the output chain and opened_fds by reference
+*/
+static bool io_transmogrify(const io_chain_t &in_chain, io_chain_t *out_chain, std::vector<int> *out_opened_fds)
+{
+    ASSERT_IS_MAIN_THREAD();
+    assert(out_chain != NULL && out_opened_fds != NULL);
+    assert(out_chain->empty());
+
+    /* Just to be clear what we do for an empty chain */
+    if (in_chain.empty())
+    {
+        return true;
+    }
+
+    bool success = true;
+
+    /* Make our chain of redirections */
+    io_chain_t result_chain;
+
+    /* In the event we can't finish transmorgrifying, we'll have to close all the files we opened. */
+    std::vector<int> opened_fds;
+
+    for (size_t idx = 0; idx < in_chain.size(); idx++)
+    {
+        const shared_ptr<io_data_t> &in = in_chain.at(idx);
+        shared_ptr<io_data_t> out; //gets allocated via new
+
+        switch (in->io_mode)
+        {
+            default:
+                /* Unknown type, should never happen */
+                fprintf(stderr, "Unknown io_mode %ld\n", (long)in->io_mode);
+                abort();
+                break;
+
+                /*
+                  These redirections don't need transmogrification. They can be passed through.
+                */
+            case IO_PIPE:
+            case IO_FD:
+            case IO_BUFFER:
+            case IO_CLOSE:
+            {
+                out = in;
+                break;
+            }
+
+            /*
+              Transmogrify file redirections
+            */
+            case IO_FILE:
+            {
+                int fd;
+                CAST_INIT(io_file_t *, in_file, in.get());
+                if ((fd=open(in_file->filename_cstr, in_file->flags, OPEN_MASK))==-1)
+                {
+                    debug(1,
+                          FILE_ERROR,
+                          in_file->filename_cstr);
+
+                    wperror(L"open");
+                    success = false;
+                    break;
+                }
+
+                opened_fds.push_back(fd);
+                out.reset(new io_fd_t(in->fd, fd, false));
+
+                break;
+            }
+        }
+
+        if (out.get() != NULL)
+            result_chain.push_back(out);
+
+        /* Don't go any further if we failed */
+        if (! success)
+        {
+            break;
+        }
+    }
+
+    /* Now either return success, or clean up */
+    if (success)
+    {
+        /* Yay */
+        out_chain->swap(result_chain);
+        out_opened_fds->swap(opened_fds);
+    }
+    else
+    {
+        /* No dice - clean up */
+        result_chain.clear();
+        io_cleanup_fds(opened_fds);
+    }
+    return success;
+}
+
+
+/**
+   Morph an io redirection chain into redirections suitable for
+   passing to eval, call eval, and clean up morphed redirections.
+
+   \param def the code to evaluate, or the empty string if none
+   \param node_offset the offset of the node to evalute, or NODE_OFFSET_INVALID
+   \param block_type the type of block to push on evaluation
+   \param io the io redirections to be performed on this block
+*/
+
+static void internal_exec_helper(parser_t &parser,
+                                 const wcstring &def,
+                                 node_offset_t node_offset,
+                                 enum block_type_t block_type,
+                                 const io_chain_t &ios)
+{
+    // If we have a valid node offset, then we must not have a string to execute
+    assert(node_offset == NODE_OFFSET_INVALID || def.empty());
+
+    io_chain_t morphed_chain;
+    std::vector<int> opened_fds;
+    bool transmorgrified = io_transmogrify(ios, &morphed_chain, &opened_fds);
+
+    /*
+      Did the transmogrification fail - if so, set error status and return
+    */
+    if (! transmorgrified)
+    {
+        proc_set_last_status(STATUS_EXEC_FAIL);
+        return;
+    }
+
+    signal_unblock();
+
+    if (node_offset == NODE_OFFSET_INVALID)
+    {
+        parser.eval(def, morphed_chain, block_type);
+    }
+    else
+    {
+        parser.eval_block_node(node_offset, morphed_chain, block_type);
+    }
+
+    signal_block();
+
+    morphed_chain.clear();
+    io_cleanup_fds(opened_fds);
+    job_reap(0);
+}
+
+/* Returns whether we can use posix spawn for a given process in a given job.
+ Per https://github.com/fish-shell/fish-shell/issues/364 , error handling for file redirections is too difficult with posix_spawn,
+ so in that case we use fork/exec.
+
+ Furthermore, to avoid the race between the caller calling tcsetpgrp() and the client checking the foreground process group, we don't use posix_spawn if we're going to foreground the process. (If we use fork(), we can call tcsetpgrp after the fork, before the exec, and avoid the race).
+*/
+static bool can_use_posix_spawn_for_job(const job_t *job, const process_t *process)
+{
+    if (job_get_flag(job, JOB_CONTROL))
+    {
+        /* We are going to use job control; therefore when we launch this job it will get its own process group ID. But will it be foregrounded? */
+        if (job_get_flag(job, JOB_TERMINAL) && job_get_flag(job, JOB_FOREGROUND))
+        {
+            /* It will be foregrounded, so we will call tcsetpgrp(), therefore do not use posix_spawn */
+            return false;
+        }
+    }
+
+    /* Now see if we have a redirection involving a file. The only one we allow is /dev/null, which we assume will not fail. */
+    bool result = true;
+    if (chain_contains_redirection_to_real_file(job->block_io_chain()) || chain_contains_redirection_to_real_file(process->io_chain()))
+    {
+        result = false;
+    }
+    return result;
+}
+
+void exec_job(parser_t &parser, job_t *j)
+{
+    pid_t pid = 0;
+    sigset_t chldset;
+
+    /*
+      Set to true if something goes wrong while exec:ing the job, in
+      which case the cleanup code will kick in.
+    */
+    bool exec_error = false;
+
+    bool needs_keepalive = false;
+    process_t keepalive;
+
+
+    CHECK(j,);
+    CHECK_BLOCK();
+
+    /* If fish was invoked with -n or --no-execute, then no_exec will be set and we do nothing. */
+    if (no_exec)
+    {
+        return;
+    }
+
+    sigemptyset(&chldset);
+    sigaddset(&chldset, SIGCHLD);
+
+    debug(4, L"Exec job '%ls' with id %d", j->command_wcstr(), j->job_id);
+
+    /* Verify that all IO_BUFFERs are output. We used to support a (single, hacked-in) magical input IO_BUFFER used by fish_pager, but now the claim is that there are no more clients and it is removed. This assertion double-checks that. */
+    const io_chain_t all_ios = j->all_io_redirections();
+    for (size_t idx = 0; idx < all_ios.size(); idx++)
+    {
+        const shared_ptr<io_data_t> &io = all_ios.at(idx);
+
+        if ((io->io_mode == IO_BUFFER))
+        {
+            CAST_INIT(io_buffer_t *, io_buffer, io.get());
+            assert(! io_buffer->is_input);
+        }
+    }
+
+    if (j->first_process->type==INTERNAL_EXEC)
+    {
+        /*
+          Do a regular launch -  but without forking first...
+        */
+        signal_block();
+
+        /*
+          setup_child_process makes sure signals are properly set
+          up. It will also call signal_unblock
+        */
+
+        /* PCA This is for handling exec. Passing all_ios here matches what fish 2.0.0 and 1.x did. It's known to be wrong - for example, it means that redirections bound for subsequent commands in the pipeline will apply to exec. However, using exec in a pipeline doesn't really make sense, so I'm not trying to fix it here. */
+        if (!setup_child_process(j, 0, all_ios))
+        {
+            /* decrement SHLVL as we're removing ourselves from the shell "stack" */
+            const env_var_t shlvl_str = env_get_string(L"SHLVL", ENV_GLOBAL | ENV_EXPORT);
+            wcstring nshlvl_str = L"0";
+            if (!shlvl_str.missing())
+            {
+                wchar_t *end;
+                long shlvl_i = wcstol(shlvl_str.c_str(), &end, 10);
+                while (iswspace(*end)) ++end; /* skip trailing whitespace */
+                if (shlvl_i > 0 && *end == '\0')
+                {
+                    nshlvl_str = to_string<long>(shlvl_i - 1);
+                }
+            }
+            env_set(L"SHLVL", nshlvl_str.c_str(), ENV_GLOBAL | ENV_EXPORT);
+
+            /*
+              launch_process _never_ returns
+            */
+            launch_process_nofork(j->first_process);
+        }
+        else
+        {
+            job_set_flag(j, JOB_CONSTRUCTED, 1);
+            j->first_process->completed=1;
+            return;
+        }
+        assert(0 && "This should be unreachable");
+    }
+    
+    /* We may have block IOs that conflict with fd redirections. For example, we may have a command with a redireciton like <&3; we may also have chosen 3 as the fd for our pipe. Ensure we have no conflicts. */
+    for (size_t i=0; i < all_ios.size(); i++)
+    {
+        io_data_t *io = all_ios.at(i).get();
+        if (io->io_mode == IO_BUFFER)
+        {
+            CAST_INIT(io_buffer_t *, io_buffer, io);
+            if (! io_buffer->avoid_conflicts_with_io_chain(all_ios))
+            {
+                /* We could not avoid conflicts, probably due to fd exhaustion. Mark an error. */
+                exec_error = true;
+                job_mark_process_as_failed(j, j->first_process);
+                break;
+            }
+        }
+    }
+    
+
+    signal_block();
+
+    /*
+      See if we need to create a group keepalive process. This is
+      a process that we create to make sure that the process group
+      doesn't die accidentally, and is often needed when a
+      builtin/block/function is inside a pipeline, since that
+      usually means we have to wait for one program to exit before
+      continuing in the pipeline, causing the group leader to
+      exit.
+    */
+
+    if (job_get_flag(j, JOB_CONTROL) && ! exec_error)
+    {
+        for (const process_t *p = j->first_process; p; p = p->next)
+        {
+            if (p->type != EXTERNAL)
+            {
+                if (p->next)
+                {
+                    needs_keepalive = true;
+                    break;
+                }
+                if (p != j->first_process)
+                {
+                    needs_keepalive = true;
+                    break;
+                }
+
+            }
+
+        }
+    }
+
+    if (needs_keepalive)
+    {
+        /* Call fork. No need to wait for threads since our use is confined and simple. */
+        if (g_log_forks)
+        {
+            printf("fork #%d: Executing keepalive fork for '%ls'\n", g_fork_count, j->command_wcstr());
+        }
+        keepalive.pid = execute_fork(false);
+        if (keepalive.pid == 0)
+        {
+            /* Child */
+            keepalive.pid = getpid();
+            set_child_group(j, &keepalive, 1);
+            pause();
+            exit_without_destructors(0);
+        }
+        else
+        {
+            /* Parent */
+            set_child_group(j, &keepalive, 0);
+        }
+    }
+
+    /*
+      This loop loops over every process_t in the job, starting it as
+      appropriate. This turns out to be rather complex, since a
+      process_t can be one of many rather different things.
+
+      The loop also has to handle pipelining between the jobs.
+    */
+
+    /* We can have up to three pipes "in flight" at a time:
+
+    1. The pipe the current process should read from (courtesy of the previous process)
+    2. The pipe that the current process should write to
+    3. The pipe that the next process should read from (courtesy of us)
+
+    We are careful to set these to -1 when closed, so if we exit the loop abruptly, we can still close them.
+
+    */
+
+    int pipe_current_read = -1, pipe_current_write = -1, pipe_next_read = -1;
+    for (process_t *p=j->first_process; p != NULL && ! exec_error; p = p->next)
+    {
+        /* The IO chain for this process. It starts with the block IO, then pipes, and then gets any from the process */
+        io_chain_t process_net_io_chain = j->block_io_chain();
+
+        /* "Consume" any pipe_next_read by making it current */
+        assert(pipe_current_read == -1);
+        pipe_current_read = pipe_next_read;
+        pipe_next_read = -1;
+
+        /* See if we need a pipe */
+        const bool pipes_to_next_command = (p->next != NULL);
+
+        /* The pipes the current process write to and read from.
+           Unfortunately these can't be just allocated on the stack, since
+           j->io wants shared_ptr.
+
+          The write pipe (destined for stdout) needs to occur before redirections. For example, with a redirection like this:
+            `foo 2>&1 | bar`, what we want to happen is this:
+
+            dup2(pipe, stdout)
+            dup2(stdout, stderr)
+
+            so that stdout and stderr both wind up referencing the pipe.
+
+            The read pipe (destined for stdin) is more ambiguous. Imagine a pipeline like this:
+
+               echo alpha | cat < beta.txt
+
+            Should cat output alpha or beta? bash and ksh output 'beta', tcsh gets it right and complains about ambiguity, and zsh outputs both (!). No shells appear to output 'alpha', so we match bash here. That would mean putting the pipe first, so that it gets trumped by the file redirection.
+
+            However, eval does this:
+
+               echo "begin; $argv "\n" ;end <&3 3<&-" | source 3<&0
+
+            which depends on the redirection being evaluated before the pipe. So the write end of the pipe comes first, the read pipe of the pipe comes last. See issue #966.
+        */
+
+        shared_ptr<io_pipe_t> pipe_write;
+        shared_ptr<io_pipe_t> pipe_read;
+
+        /* Write pipe goes first */
+        if (p->next)
+        {
+            pipe_write.reset(new io_pipe_t(p->pipe_write_fd, false));
+            process_net_io_chain.push_back(pipe_write);
+        }
+
+        /* The explicit IO redirections associated with the process */
+        process_net_io_chain.append(p->io_chain());
+
+        /* Read pipe goes last */
+        if (p != j->first_process)
+        {
+            pipe_read.reset(new io_pipe_t(p->pipe_read_fd, true));
+            /* Record the current read in pipe_read */
+            pipe_read->pipe_fd[0] = pipe_current_read;
+            process_net_io_chain.push_back(pipe_read);
+        }
+
+
+        /*
+           This call is used so the global environment variable array
+           is regenerated, if needed, before the fork. That way, we
+           avoid a lot of duplicate work where EVERY child would need
+           to generate it, since that result would not get written
+           back to the parent. This call could be safely removed, but
+           it would result in slightly lower performance - at least on
+           uniprocessor systems.
+        */
+        if (p->type == EXTERNAL)
+            env_export_arr(true);
+
+
+        /* Set up fds that will be used in the pipe. */
+        if (pipes_to_next_command)
+        {
+//      debug( 1, L"%ls|%ls" , p->argv[0], p->next->argv[0]);
+            int local_pipe[2] = {-1, -1};
+            if (exec_pipe(local_pipe) == -1)
+            {
+                debug(1, PIPE_ERROR);
+                wperror(L"pipe");
+                exec_error = true;
+                job_mark_process_as_failed(j, p);
+                break;
+            }
+
+            /* Ensure our pipe fds not conflict with any fd redirections. E.g. if the process is like 'cat <&5' then fd 5 must not be used by the pipe. */
+            if (! pipe_avoid_conflicts_with_io_chain(local_pipe, all_ios))
+            {
+                /* We failed. The pipes were closed for us. */
+                wperror(L"dup");
+                exec_error = true;
+                job_mark_process_as_failed(j, p);
+                break;
+            }
+
+            // This tells the redirection about the fds, but the redirection does not close them
+            assert(local_pipe[0] >= 0);
+            assert(local_pipe[1] >= 0);
+            memcpy(pipe_write->pipe_fd, local_pipe, sizeof(int)*2);
+
+            // Record our pipes
+            // The fds should be negative to indicate that we aren't overwriting an fd we failed to close
+            assert(pipe_current_write == -1);
+            pipe_current_write = local_pipe[1];
+
+            assert(pipe_next_read == -1);
+            pipe_next_read = local_pipe[0];
+        }
+
+        // This is the IO buffer we use for storing the output of a block or function when it is in a pipeline
+        shared_ptr<io_buffer_t> block_output_io_buffer;
+        switch (p->type)
+        {
+            case INTERNAL_FUNCTION:
+            {
+                /*
+                  Calls to function_get_definition might need to
+                  source a file as a part of autoloading, hence there
+                  must be no blocks.
+                */
+
+                signal_unblock();
+                wcstring def;
+                bool function_exists = function_get_definition(p->argv0(), &def);
+
+                wcstring_list_t named_arguments = function_get_named_arguments(p->argv0());
+                bool shadows = function_get_shadows(p->argv0());
+                std::map<wcstring,env_var_t> inherit_vars = function_get_inherit_vars(p->argv0());
+
+                signal_block();
+
+                if (! function_exists)
+                {
+                    debug(0, _(L"Unknown function '%ls'"), p->argv0());
+                    break;
+                }
+                function_block_t *newv = new function_block_t(p, p->argv0(), shadows);
+                parser.push_block(newv);
+
+                /*
+                  setting variables might trigger an event
+                  handler, hence we need to unblock
+                  signals.
+                */
+                signal_unblock();
+                parse_util_set_argv(p->get_argv()+1, named_arguments);
+                for (std::map<wcstring,env_var_t>::const_iterator it = inherit_vars.begin(), end = inherit_vars.end(); it != end; ++it)
+                {
+                    env_set(it->first, it->second.missing() ? NULL : it->second.c_str(), ENV_LOCAL | ENV_USER);
+                }
+                signal_block();
+
+                parser.forbid_function(p->argv0());
+
+                if (p->next)
+                {
+                    // Be careful to handle failure, e.g. too many open fds
+                    block_output_io_buffer.reset(io_buffer_t::create(STDOUT_FILENO, all_ios));
+                    if (block_output_io_buffer.get() == NULL)
+                    {
+                        exec_error = true;
+                        job_mark_process_as_failed(j, p);
+                    }
+                    else
+                    {
+                        /* This looks sketchy, because we're adding this io buffer locally - they aren't in the process or job redirection list. Therefore select_try won't be able to read them. However we call block_output_io_buffer->read() below, which reads until EOF. So there's no need to select on this. */
+                        process_net_io_chain.push_back(block_output_io_buffer);
+                    }
+                }
+
+                if (! exec_error)
+                {
+                    internal_exec_helper(parser, def, NODE_OFFSET_INVALID, TOP, process_net_io_chain);
+                }
+
+                parser.allow_function();
+                parser.pop_block();
+
+                break;
+            }
+
+            case INTERNAL_BLOCK_NODE:
+            {
+                if (p->next)
+                {
+                    block_output_io_buffer.reset(io_buffer_t::create(STDOUT_FILENO, all_ios));
+                    if (block_output_io_buffer.get() == NULL)
+                    {
+                        /* We failed (e.g. no more fds could be created). */
+                        exec_error = true;
+                        job_mark_process_as_failed(j, p);
+                    }
+                    else
+                    {
+                        /* See the comment above about it's OK to add an IO redirection to this local buffer, even though it won't be handled in select_try */
+                        process_net_io_chain.push_back(block_output_io_buffer);
+                    }
+                }
+
+                if (! exec_error)
+                {
+                    internal_exec_helper(parser, wcstring(), p->internal_block_node, TOP, process_net_io_chain);
+                }
+                break;
+            }
+
+            case INTERNAL_BUILTIN:
+            {
+                int local_builtin_stdin = STDIN_FILENO;
+                bool close_stdin = false;
+
+                /*
+                  If this is the first process, check the io
+                  redirections and see where we should be reading
+                  from.
+                */
+                if (p == j->first_process)
+                {
+                    const shared_ptr<const io_data_t> in = process_net_io_chain.get_io_for_fd(STDIN_FILENO);
+
+                    if (in)
+                    {
+                        switch (in->io_mode)
+                        {
+
+                            case IO_FD:
+                            {
+                                CAST_INIT(const io_fd_t *, in_fd, in.get());
+                                /* Ignore user-supplied fd redirections from an fd other than the standard ones. e.g. in source <&3 don't actually read from fd 3, which is internal to fish. We still respect this redirection in that we pass it on as a block IO to the code that source runs, and therefore this is not an error. Non-user supplied fd redirections come about through transmogrification, and we need to respect those here. */
+                                if (! in_fd->user_supplied || (in_fd->old_fd >= 0 && in_fd->old_fd < 3))
+                                {
+                                    local_builtin_stdin = in_fd->old_fd;
+                                }
+                                break;
+                            }
+                            case IO_PIPE:
+                            {
+                                CAST_INIT(const io_pipe_t *, in_pipe, in.get());
+                                local_builtin_stdin = in_pipe->pipe_fd[0];
+                                break;
+                            }
+
+                            case IO_FILE:
+                            {
+                                /* Do not set CLO_EXEC because child needs access */
+                                CAST_INIT(const io_file_t *, in_file, in.get());
+                                local_builtin_stdin=open(in_file->filename_cstr,
+                                                   in_file->flags, OPEN_MASK);
+                                if (local_builtin_stdin == -1)
+                                {
+                                    debug(1,
+                                          FILE_ERROR,
+                                          in_file->filename_cstr);
+                                    wperror(L"open");
+                                }
+                                else
+                                {
+                                    close_stdin = true;
+                                }
+
+                                break;
+                            }
+
+                            case IO_CLOSE:
+                            {
+                                /*
+                                  FIXME:
+
+                                  When requesting that stdin be closed, we
+                                  really don't do anything. How should this be
+                                  handled?
+                                 */
+                                local_builtin_stdin = -1;
+
+                                break;
+                            }
+
+                            default:
+                            {
+                                local_builtin_stdin=-1;
+                                debug(1,
+                                      _(L"Unknown input redirection type %d"),
+                                      in->io_mode);
+                                break;
+                            }
+
+                        }
+                    }
+                }
+                else
+                {
+                    local_builtin_stdin = pipe_read->pipe_fd[0];
+                }
+
+                if (local_builtin_stdin == -1)
+                {
+                    exec_error = true;
+                    break;
+                }
+                else
+                {
+                    int old_out = builtin_out_redirect;
+                    int old_err = builtin_err_redirect;
+
+                    /*
+                       Since this may be the foreground job, and since
+                       a builtin may execute another foreground job,
+                       we need to pretend to suspend this job while
+                       running the builtin, in order to avoid a
+                       situation where two jobs are running at once.
+
+                       The reason this is done here, and not by the
+                       relevant builtins, is that this way, the
+                       builtin does not need to know what job it is
+                       part of. It could probably figure that out by
+                       walking the job list, but it seems more robust
+                       to make exec handle things.
+                    */
+
+                    builtin_push_io(parser, local_builtin_stdin);
+
+                    builtin_out_redirect = has_fd(process_net_io_chain, STDOUT_FILENO);
+                    builtin_err_redirect = has_fd(process_net_io_chain, STDERR_FILENO);
+
+                    const int fg = job_get_flag(j, JOB_FOREGROUND);
+                    job_set_flag(j, JOB_FOREGROUND, 0);
+
+                    signal_unblock();
+                    
+                    p->status = builtin_run(parser, p->get_argv(), process_net_io_chain);
+
+                    builtin_out_redirect=old_out;
+                    builtin_err_redirect=old_err;
+
+                    signal_block();
+
+                    /*
+                      Restore the fg flag, which is temporarily set to
+                      false during builtin execution so as not to confuse
+                      some job-handling builtins.
+                    */
+                    job_set_flag(j, JOB_FOREGROUND, fg);
+                }
+
+                /*
+                  If stdin has been redirected, close the redirection
+                  stream.
+                */
+                if (close_stdin)
+                {
+                    exec_close(local_builtin_stdin);
+                }
+                break;
+            }
+
+            case EXTERNAL:
+                /* External commands are handled in the next switch statement below */
+                break;
+
+            case INTERNAL_EXEC:
+                /* We should have handled exec up above */
+                assert(0 && "INTERNAL_EXEC process found in pipeline, where it should never be. Aborting.");
+                break;
+        }
+
+        if (exec_error)
+        {
+            break;
+        }
+
+        switch (p->type)
+        {
+
+            case INTERNAL_BLOCK_NODE:
+            case INTERNAL_FUNCTION:
+            {
+                int status = proc_get_last_status();
+
+                /*
+                  Handle output from a block or function. This usually
+                  means do nothing, but in the case of pipes, we have
+                  to buffer such io, since otherwise the internal pipe
+                  buffer might overflow.
+                */
+                if (! block_output_io_buffer.get())
+                {
+                    /*
+                      No buffer, so we exit directly. This means we
+                      have to manually set the exit status.
+                    */
+                    if (p->next == NULL)
+                    {
+                        proc_set_last_status(job_get_flag(j, JOB_NEGATE)?(!status):status);
+                    }
+                    p->completed = 1;
+                    break;
+                }
+
+                // Here we must have a non-NULL block_output_io_buffer
+                assert(block_output_io_buffer.get() != NULL);
+                process_net_io_chain.remove(block_output_io_buffer);
+
+                block_output_io_buffer->read();
+
+                const char *buffer = block_output_io_buffer->out_buffer_ptr();
+                size_t count = block_output_io_buffer->out_buffer_size();
+
+                if (block_output_io_buffer->out_buffer_size() > 0)
+                {
+                    /* We don't have to drain threads here because our child process is simple */
+                    if (g_log_forks)
+                    {
+                        printf("Executing fork for internal block or function for '%ls'\n", p->argv0());
+                    }
+                    pid = execute_fork(false);
+                    if (pid == 0)
+                    {
+
+                        /*
+                          This is the child process. Write out the contents of the pipeline.
+                        */
+                        p->pid = getpid();
+                        setup_child_process(j, p, process_net_io_chain);
+
+                        exec_write_and_exit(block_output_io_buffer->fd, buffer, count, status);
+                    }
+                    else
+                    {
+                        /*
+                           This is the parent process. Store away
+                           information on the child, and possibly give
+                           it control over the terminal.
+                        */
+                        p->pid = pid;
+                        set_child_group(j, p, 0);
+
+                    }
+
+                }
+                else
+                {
+                    if (p->next == 0)
+                    {
+                        proc_set_last_status(job_get_flag(j, JOB_NEGATE)?(!status):status);
+                    }
+                    p->completed = 1;
+                }
+
+                block_output_io_buffer.reset();
+                break;
+
+            }
+
+            case INTERNAL_BUILTIN:
+            {
+                /*
+                  Handle output from builtin commands. In the general
+                  case, this means forking of a worker process, that
+                  will write out the contents of the stdout and stderr
+                  buffers to the correct file descriptor. Since
+                  forking is expensive, fish tries to avoid it when
+                  possible.
+                */
+
+                bool fork_was_skipped = false;
+
+                const shared_ptr<io_data_t> stdout_io = process_net_io_chain.get_io_for_fd(STDOUT_FILENO);
+                const shared_ptr<io_data_t> stderr_io = process_net_io_chain.get_io_for_fd(STDERR_FILENO);
+
+                /* If we are outputting to a file, we have to actually do it, even if we have no output, so that we can truncate the file. Does not apply to /dev/null. */
+                bool must_fork = redirection_is_to_real_file(stdout_io.get()) || redirection_is_to_real_file(stderr_io.get());
+                if (! must_fork)
+                {
+                    if (p->next == NULL)
+                    {
+                        const bool stdout_is_to_buffer = stdout_io && stdout_io->io_mode == IO_BUFFER;
+                        const bool no_stdout_output =  get_stdout_buffer().empty();
+                        const bool no_stderr_output =  get_stderr_buffer().empty();
+
+                        if (no_stdout_output && no_stderr_output)
+                        {
+                            /* The builtin produced no output and is not inside of a pipeline. No need to fork or even output anything. */
+                            if (g_log_forks)
+                            {
+                                // This one is very wordy
+                                //printf("fork #-: Skipping fork due to no output for internal builtin for '%ls'\n", p->argv0());
+                            }
+                            fork_was_skipped = true;
+                        }
+                        else if (no_stderr_output && stdout_is_to_buffer)
+                        {
+                            /* The builtin produced no stderr, and its stdout is going to an internal buffer. There is no need to fork. This helps out the performance quite a bit in complex completion code. */
+                            if (g_log_forks)
+                            {
+                                printf("fork #-: Skipping fork due to buffered output for internal builtin for '%ls'\n", p->argv0());
+                            }
+
+                            CAST_INIT(io_buffer_t *, io_buffer, stdout_io.get());
+                            const std::string res = wcs2string(get_stdout_buffer());
+                            io_buffer->out_buffer_append(res.data(), res.size());
+                            fork_was_skipped = true;
+                        }
+                        else if (stdout_io.get() == NULL && stderr_io.get() == NULL)
+                        {
+                            /* We are writing to normal stdout and stderr. Just do it - no need to fork. */
+                            if (g_log_forks)
+                            {
+                                printf("fork #-: Skipping fork due to ordinary output for internal builtin for '%ls'\n", p->argv0());
+                            }
+                            const wcstring &out = get_stdout_buffer(), &err = get_stderr_buffer();
+                            const std::string outbuff = wcs2string(out);
+                            const std::string errbuff = wcs2string(err);
+                            bool builtin_io_done = do_builtin_io(outbuff.data(), outbuff.size(), errbuff.data(), errbuff.size());
+                            if (! builtin_io_done)
+                            {
+                                show_stackframe();
+                            }
+                            fork_was_skipped = true;
+                        }
+                    }
+                }
+
+
+                if (fork_was_skipped)
+                {
+                    p->completed=1;
+                    if (p->next == 0)
+                    {
+                        debug(3, L"Set status of %ls to %d using short circuit", j->command_wcstr(), p->status);
+
+                        int status = p->status;
+                        proc_set_last_status(job_get_flag(j, JOB_NEGATE)?(!status):status);
+                    }
+                }
+                else
+                {
+
+
+                    /* Ok, unfortunately, we have to do a real fork. Bummer. We work hard to make sure we don't have to wait for all our threads to exit, by arranging things so that we don't have to allocate memory or do anything except system calls in the child. */
+
+                    /* Get the strings we'll write before we fork (since they call malloc) */
+                    const wcstring &out = get_stdout_buffer(), &err = get_stderr_buffer();
+
+                    /* These strings may contain embedded nulls, so don't treat them as C strings */
+                    const std::string outbuff_str = wcs2string(out);
+                    const char *outbuff = outbuff_str.data();
+                    size_t outbuff_len = outbuff_str.size();
+
+                    const std::string errbuff_str = wcs2string(err);
+                    const char *errbuff = errbuff_str.data();
+                    size_t errbuff_len = errbuff_str.size();
+
+                    fflush(stdout);
+                    fflush(stderr);
+                    if (g_log_forks)
+                    {
+                        printf("fork #%d: Executing fork for internal builtin for '%ls'\n", g_fork_count, p->argv0());
+                    }
+                    pid = execute_fork(false);
+                    if (pid == 0)
+                    {
+                        /*
+                          This is the child process. Setup redirections,
+                          print correct output to stdout and stderr, and
+                          then exit.
+                        */
+                        p->pid = getpid();
+                        setup_child_process(j, p, process_net_io_chain);
+                        do_builtin_io(outbuff, outbuff_len, errbuff, errbuff_len);
+                        exit_without_destructors(p->status);
+                    }
+                    else
+                    {
+                        /*
+                           This is the parent process. Store away
+                           information on the child, and possibly give
+                           it control over the terminal.
+                        */
+                        p->pid = pid;
+
+                        set_child_group(j, p, 0);
+                    }
+                }
+
+                break;
+            }
+
+            case EXTERNAL:
+            {
+                /* Get argv and envv before we fork */
+                null_terminated_array_t<char> argv_array;
+                convert_wide_array_to_narrow(p->get_argv_array(), &argv_array);
+
+                /* Ensure that stdin is blocking before we hand it off (see issue #176). It's a little strange that we only do this with stdin and not with stdout or stderr. However in practice, setting or clearing O_NONBLOCK on stdin also sets it for the other two fds, presumably because they refer to the same underlying file (/dev/tty?) */
+                make_fd_blocking(STDIN_FILENO);
+
+                const char * const *argv = argv_array.get();
+                const char * const *envv = env_export_arr(false);
+
+                std::string actual_cmd_str = wcs2string(p->actual_cmd);
+                const char *actual_cmd = actual_cmd_str.c_str();
+                
+                const wchar_t *reader_current_filename(void);
+                if (g_log_forks)
+                {
+                    const wchar_t *file = reader_current_filename();
+                    printf("fork #%d: forking for '%s' in '%ls'\n", g_fork_count, actual_cmd, file ? file : L"");
+                }
+
+#if FISH_USE_POSIX_SPAWN
+                /* Prefer to use posix_spawn, since it's faster on some systems like OS X */
+                bool use_posix_spawn = g_use_posix_spawn && can_use_posix_spawn_for_job(j, p);
+                if (use_posix_spawn)
+                {
+                    /* Create posix spawn attributes and actions */
+                    posix_spawnattr_t attr = posix_spawnattr_t();
+                    posix_spawn_file_actions_t actions = posix_spawn_file_actions_t();
+                    bool made_it = fork_actions_make_spawn_properties(&attr, &actions, j, p, process_net_io_chain);
+                    if (made_it)
+                    {
+                        /* We successfully made the attributes and actions; actually call posix_spawn */
+                        int spawn_ret = posix_spawn(&pid, actual_cmd, &actions, &attr, const_cast<char * const *>(argv), const_cast<char * const *>(envv));
+
+                        /* This usleep can be used to test for various race conditions (https://github.com/fish-shell/fish-shell/issues/360) */
+                        //usleep(10000);
+
+                        if (spawn_ret != 0)
+                        {
+                            safe_report_exec_error(spawn_ret, actual_cmd, argv, envv);
+                            /* Make sure our pid isn't set */
+                            pid = 0;
+                        }
+
+                        /* Clean up our actions */
+                        posix_spawn_file_actions_destroy(&actions);
+                        posix_spawnattr_destroy(&attr);
+                    }
+
+                    /* A 0 pid means we failed to posix_spawn. Since we have no pid, we'll never get told when it's exited, so we have to mark the process as failed. */
+                    if (pid == 0)
+                    {
+                        job_mark_process_as_failed(j, p);
+                        exec_error = true;
+                    }
+                }
+                else
+#endif
+                {
+                    pid = execute_fork(false);
+                    if (pid == 0)
+                    {
+                        /* This is the child process. */
+                        p->pid = getpid();
+                        setup_child_process(j, p, process_net_io_chain);
+                        safe_launch_process(p, actual_cmd, argv, envv);
+
+                        /*
+                          safe_launch_process _never_ returns...
+                        */
+                        assert(0 && "safe_launch_process should not have returned");
+                    }
+                    else if (pid < 0)
+                    {
+                        job_mark_process_as_failed(j, p);
+                        exec_error = true;
+                    }
+                }
+
+
+                /*
+                   This is the parent process. Store away
+                   information on the child, and possibly fice
+                   it control over the terminal.
+                */
+                p->pid = pid;
+
+                set_child_group(j, p, 0);
+
+                break;
+            }
+
+            case INTERNAL_EXEC:
+            {
+                /* We should have handled exec up above */
+                assert(0 && "INTERNAL_EXEC process found in pipeline, where it should never be. Aborting.");
+                break;
+            }
+        }
+
+        if (p->type == INTERNAL_BUILTIN)
+            builtin_pop_io(parser);
+
+        /*
+           Close the pipe the current process uses to read from the
+           previous process_t
+        */
+        if (pipe_current_read >= 0)
+        {
+            exec_close(pipe_current_read);
+            pipe_current_read = -1;
+        }
+
+        /* Close the write end too, since the curent child subprocess already has a copy of it. */
+        if (pipe_current_write >= 0)
+        {
+            exec_close(pipe_current_write);
+            pipe_current_write = -1;
+        }
+    }
+
+    /* Clean up any file descriptors we left open */
+    if (pipe_current_read >= 0)
+        exec_close(pipe_current_read);
+    if (pipe_current_write >= 0)
+        exec_close(pipe_current_write);
+    if (pipe_next_read >= 0)
+        exec_close(pipe_next_read);
+
+    /* The keepalive process is no longer needed, so we terminate it with extreme prejudice */
+    if (needs_keepalive)
+    {
+        kill(keepalive.pid, SIGKILL);
+    }
+
+    signal_unblock();
+
+    debug(3, L"Job is constructed");
+
+    job_set_flag(j, JOB_CONSTRUCTED, 1);
+
+    if (!job_get_flag(j, JOB_FOREGROUND))
+    {
+        proc_last_bg_pid = j->pgid;
+    }
+
+    if (! exec_error)
+    {
+        job_continue(j, false);
+    }
+    else
+    {
+        /* Mark the errored job as not in the foreground. I can't fully justify whether this is the right place, but it prevents sanity_lose from complaining. */
+        job_set_flag(j, JOB_FOREGROUND, 0);
+    }
+
+}
+
+
+static int exec_subshell_internal(const wcstring &cmd, wcstring_list_t *lst, bool apply_exit_status)
+{
+    ASSERT_IS_MAIN_THREAD();
+    int prev_subshell = is_subshell;
+    const int prev_status = proc_get_last_status();
+    bool split_output=false;
+
+    //fprintf(stderr, "subcmd %ls\n", cmd.c_str());
+
+    const env_var_t ifs = env_get_string(L"IFS");
+
+    if (! ifs.missing_or_empty())
+    {
+        split_output=true;
+    }
+
+    is_subshell=1;
+
+
+    int subcommand_status = -1; //assume the worst
+
+    // IO buffer creation may fail (e.g. if we have too many open files to make a pipe), so this may be null
+    const shared_ptr<io_buffer_t> io_buffer(io_buffer_t::create(STDOUT_FILENO, io_chain_t()));
+    if (io_buffer.get() != NULL)
+    {
+        parser_t &parser = parser_t::principal_parser();
+        if (parser.eval(cmd, io_chain_t(io_buffer), SUBST) == 0)
+        {
+            subcommand_status = proc_get_last_status();
+        }
+
+        io_buffer->read();
+    }
+
+    // If the caller asked us to preserve the exit status, restore the old status
+    // Otherwise set the status of the subcommand
+    proc_set_last_status(apply_exit_status ? subcommand_status : prev_status);
+
+
+    is_subshell = prev_subshell;
+
+    if (lst != NULL && io_buffer.get() != NULL)
+    {
+        const char *begin = io_buffer->out_buffer_ptr();
+        const char *end = begin + io_buffer->out_buffer_size();
+        if (split_output)
+        {
+            const char *cursor = begin;
+            while (cursor < end)
+            {
+                // Look for the next separator
+                const char *stop = (const char *)memchr(cursor, '\n', end - cursor);
+                const bool hit_separator = (stop != NULL);
+                if (! hit_separator)
+                {
+                    // If it's not found, just use the end
+                    stop = end;
+                }
+                // Stop now points at the first character we do not want to copy
+                const wcstring wc = str2wcstring(cursor, stop - cursor);
+                lst->push_back(wc);
+
+                // If we hit a separator, skip over it; otherwise we're at the end
+                cursor = stop + (hit_separator ? 1 : 0);
+            }
+        }
+        else
+        {
+            // we're not splitting output, but we still want to trim off a trailing newline
+            if (end != begin && end[-1] == '\n')
+            {
+                --end;
+            }
+            const wcstring wc = str2wcstring(begin, end - begin);
+            lst->push_back(wc);
+        }
+    }
+
+    return subcommand_status;
+}
+
+int exec_subshell(const wcstring &cmd, std::vector<wcstring> &outputs, bool apply_exit_status)
+{
+    ASSERT_IS_MAIN_THREAD();
+    return exec_subshell_internal(cmd, &outputs, apply_exit_status);
+}
+
+int exec_subshell(const wcstring &cmd, bool apply_exit_status)
+{
+    ASSERT_IS_MAIN_THREAD();
+    return exec_subshell_internal(cmd, NULL, apply_exit_status);
+}
diff --git a/src/exec.h b/src/exec.h
new file mode 100644
index 00000000..41dc3193
--- /dev/null
+++ b/src/exec.h
@@ -0,0 +1,75 @@
+/** \file exec.h
+  Prototypes for functions for executing a program
+*/
+
+#ifndef FISH_EXEC_H
+/**
+   Header guard
+*/
+#define FISH_EXEC_H
+
+#include <stddef.h>
+#include <vector>
+
+#include "common.h"
+
+/**
+   pipe redirection error message
+*/
+#define PIPE_ERROR _(L"An error occurred while setting up pipe")
+
+/**
+  Execute the processes specified by j.
+
+   I've put a fair bit of work into making builtins behave like other
+   programs as far as pipes are concerned. Unlike i.e. bash, builtins
+   can pipe to other builtins with arbitrary amounts of data, and so
+   on. To do this, after a builtin is run in the real process, it
+   forks and a dummy process is created, responsible for writing the
+   output of the builtin. This is surprisingly cheap on my computer,
+   probably because of the marvels of copy on write forking.
+
+   This rule is short circuited in the case where a builtin does not
+   output to a pipe and does in fact not output anything. The speed
+   improvement from this optimization is not noticable on a normal
+   computer/OS in regular use, but the promiscous amounts of forking
+   that resulted was responsible for a huge slowdown when using
+   Valgrind as well as when doing complex command-specific
+   completions.
+
+
+*/
+class job_t;
+class parser_t;
+void exec_job(parser_t &parser, job_t *j);
+
+/**
+  Evaluate the expression cmd in a subshell, add the outputs into the
+  list l. On return, the status flag as returned bu \c
+  proc_gfet_last_status will not be changed.
+
+  \param cmd the command to execute
+  \param outputs The list to insert output into.
+
+  \return the status of the last job to exit, or -1 if en error was encountered.
+*/
+int exec_subshell(const wcstring &cmd, std::vector<wcstring> &outputs, bool preserve_exit_status);
+int exec_subshell(const wcstring &cmd, bool preserve_exit_status);
+
+
+/**
+   Loops over close until the syscall was run without being
+   interrupted.
+*/
+void exec_close(int fd);
+
+/**
+   Call pipe(), and add resulting fds to open_fds, the list of opened
+   file descriptors for pipes. The pipes are marked CLO_EXEC.
+*/
+int exec_pipe(int fd[2]);
+
+/** Gets the interpreter for a given command */
+char *get_interpreter(const char *command, char *interpreter, size_t buff_size);
+
+#endif
diff --git a/src/expand.cpp b/src/expand.cpp
new file mode 100644
index 00000000..6d978582
--- /dev/null
+++ b/src/expand.cpp
@@ -0,0 +1,2045 @@
+/**\file expand.c
+
+String expansion functions. These functions perform several kinds of
+parameter expansion.
+
+*/
+
+#include "config.h" // IWYU pragma: keep
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <stdarg.h>
+#include <stddef.h>
+#include <wchar.h>
+#include <string.h>
+#include <wctype.h>
+#include <errno.h>
+#include <pwd.h>
+#include <dirent.h>
+#include <sys/stat.h>
+#include <unistd.h>
+#include <algorithm>
+
+#include <assert.h>
+#include <vector>
+
+#ifdef SunOS
+#include <procfs.h>
+#endif
+
+#include "fallback.h" // IWYU pragma: keep
+#include "util.h"
+
+#include "common.h"
+#include "wutil.h"
+#include "env.h"
+#include "proc.h"
+#include "parser.h"
+#include "expand.h"
+#include "wildcard.h"
+#include "exec.h"
+#include "tokenizer.h"
+#include "complete.h"
+#include "iothread.h"
+
+#include "parse_util.h"
+
+/**
+   Description for child process
+*/
+#define COMPLETE_CHILD_PROCESS_DESC _( L"Child process")
+
+/**
+   Description for non-child process
+*/
+#define COMPLETE_PROCESS_DESC _( L"Process")
+
+/**
+   Description for long job
+*/
+#define COMPLETE_JOB_DESC _( L"Job")
+
+/**
+   Description for short job. The job command is concatenated
+*/
+#define COMPLETE_JOB_DESC_VAL _( L"Job: %ls")
+
+/**
+   Description for the shells own pid
+*/
+#define COMPLETE_SELF_DESC _( L"Shell process")
+
+/**
+   Description for the shells own pid
+*/
+#define COMPLETE_LAST_DESC _( L"Last background job")
+
+/**
+   String in process expansion denoting ourself
+*/
+#define SELF_STR L"self"
+
+/**
+   String in process expansion denoting last background job
+*/
+#define LAST_STR L"last"
+
+/**
+   Characters which make a string unclean if they are the first
+   character of the string. See \c expand_is_clean().
+*/
+#define UNCLEAN_FIRST L"~%"
+/**
+   Unclean characters. See \c expand_is_clean().
+*/
+#define UNCLEAN L"$*?\\\"'({})"
+
+static void remove_internal_separator(wcstring &s, bool conv);
+
+int expand_is_clean(const wchar_t *in)
+{
+
+    const wchar_t * str = in;
+
+    CHECK(in, 1);
+
+    /*
+      Test characters that have a special meaning in the first character position
+    */
+    if (wcschr(UNCLEAN_FIRST, *str))
+        return 0;
+
+    /*
+      Test characters that have a special meaning in any character position
+    */
+    while (*str)
+    {
+        if (wcschr(UNCLEAN, *str))
+            return 0;
+        str++;
+    }
+
+    return 1;
+}
+
+
+/* Append a syntax error to the given error list */
+static void append_syntax_error(parse_error_list_t *errors, size_t source_start, const wchar_t *fmt, ...)
+{
+    if (errors != NULL)
+    {
+        parse_error_t error;
+        error.source_start = source_start;
+        error.source_length = 0;
+        error.code = parse_error_syntax;
+
+        va_list va;
+        va_start(va, fmt);
+        error.text = vformat_string(fmt, va);
+        va_end(va);
+
+        errors->push_back(error);
+    }
+}
+
+/* Append a cmdsub error to the given error list */
+static void append_cmdsub_error(parse_error_list_t *errors, size_t source_start, const wchar_t *fmt, ...)
+{
+    if (errors != NULL)
+    {
+        parse_error_t error;
+        error.source_start = source_start;
+        error.source_length = 0;
+        error.code = parse_error_cmdsubst;
+
+        va_list va;
+        va_start(va, fmt);
+        error.text = vformat_string(fmt, va);
+        va_end(va);
+
+        errors->push_back(error);
+    }
+}
+
+
+/**
+   Return the environment variable value for the string starting at \c in.
+*/
+static env_var_t expand_var(const wchar_t *in)
+{
+    if (!in)
+        return env_var_t::missing_var();
+    return env_get_string(in);
+}
+
+/**
+   Test if the specified string does not contain character which can
+   not be used inside a quoted string.
+*/
+static int is_quotable(const wchar_t *str)
+{
+    switch (*str)
+    {
+        case 0:
+            return 1;
+
+        case L'\n':
+        case L'\t':
+        case L'\r':
+        case L'\b':
+        case L'\x1b':
+            return 0;
+
+        default:
+            return is_quotable(str+1);
+    }
+    return 0;
+
+}
+
+static int is_quotable(const wcstring &str)
+{
+    return is_quotable(str.c_str());
+}
+
+wcstring expand_escape_variable(const wcstring &in)
+{
+
+    wcstring_list_t lst;
+    wcstring buff;
+
+    tokenize_variable_array(in, lst);
+
+    switch (lst.size())
+    {
+        case 0:
+            buff.append(L"''");
+            break;
+
+        case 1:
+        {
+            const wcstring &el = lst.at(0);
+
+            if (el.find(L' ') != wcstring::npos && is_quotable(el))
+            {
+                buff.append(L"'");
+                buff.append(el);
+                buff.append(L"'");
+            }
+            else
+            {
+                buff.append(escape_string(el, 1));
+            }
+            break;
+        }
+        default:
+        {
+            for (size_t j=0; j<lst.size(); j++)
+            {
+                const wcstring &el = lst.at(j);
+                if (j)
+                    buff.append(L"  ");
+
+                if (is_quotable(el))
+                {
+                    buff.append(L"'");
+                    buff.append(el);
+                    buff.append(L"'");
+                }
+                else
+                {
+                    buff.append(escape_string(el, 1));
+                }
+            }
+        }
+    }
+    return buff;
+}
+
+/**
+   Tests if all characters in the wide string are numeric
+*/
+static int iswnumeric(const wchar_t *n)
+{
+    for (; *n; n++)
+    {
+        if (*n < L'0' || *n > L'9')
+        {
+            return 0;
+        }
+    }
+    return 1;
+}
+
+/**
+   See if the process described by \c proc matches the commandline \c
+   cmd
+*/
+static bool match_pid(const wcstring &cmd,
+                      const wchar_t *proc,
+                      int flags,
+                      size_t *offset)
+{
+    /* Test for a direct match. If the proc string is empty (e.g. the user tries to complete against %), then return an offset pointing at the base command. That ensures that you don't see a bunch of dumb paths when completing against all processes. */
+    if (proc[0] != L'\0' && wcsncmp(cmd.c_str(), proc, wcslen(proc)) == 0)
+    {
+        if (offset)
+            *offset = 0;
+        return true;
+    }
+
+    /* Get the command to match against. We're only interested in the last path component. */
+    const wcstring base_cmd = wbasename(cmd);
+
+    bool result = string_prefixes_string(proc, base_cmd);
+    if (result)
+    {
+        /* It's a match. Return the offset within the full command. */
+        if (offset)
+            *offset = cmd.size() - base_cmd.size();
+    }
+    return result;
+}
+
+/** Helper class for iterating over processes. The names returned have been unescaped (e.g. may include spaces) */
+#ifdef KERN_PROCARGS2
+
+/* BSD / OS X process completions */
+
+class process_iterator_t
+{
+    std::vector<pid_t> pids;
+    size_t idx;
+
+    wcstring name_for_pid(pid_t pid);
+
+public:
+    process_iterator_t();
+    bool next_process(wcstring *str, pid_t *pid);
+};
+
+wcstring process_iterator_t::name_for_pid(pid_t pid)
+{
+    wcstring result;
+    int mib[4], maxarg = 0, numArgs = 0;
+    size_t size = 0;
+    char *args = NULL, *stringPtr = NULL;
+
+    mib[0] = CTL_KERN;
+    mib[1] = KERN_ARGMAX;
+
+    size = sizeof(maxarg);
+    if (sysctl(mib, 2, &maxarg, &size, NULL, 0) == -1)
+    {
+        return result;
+    }
+
+    args = (char *)malloc(maxarg);
+    if (args == NULL)
+    {
+        return result;
+    }
+
+    mib[0] = CTL_KERN;
+    mib[1] = KERN_PROCARGS2;
+    mib[2] = pid;
+
+    size = (size_t)maxarg;
+    if (sysctl(mib, 3, args, &size, NULL, 0) == -1)
+    {
+        free(args);
+        return result;;
+    }
+
+    memcpy(&numArgs, args, sizeof(numArgs));
+    stringPtr = args + sizeof(numArgs);
+    result = str2wcstring(stringPtr);
+    free(args);
+    return result;
+}
+
+bool process_iterator_t::next_process(wcstring *out_str, pid_t *out_pid)
+{
+    wcstring name;
+    pid_t pid = 0;
+    bool result = false;
+    while (idx < pids.size())
+    {
+        pid = pids.at(idx++);
+        name = name_for_pid(pid);
+        if (! name.empty())
+        {
+            result = true;
+            break;
+        }
+    }
+    if (result)
+    {
+        *out_str = name;
+        *out_pid = pid;
+    }
+    return result;
+}
+
+process_iterator_t::process_iterator_t() : idx(0)
+{
+    int                 err;
+    struct kinfo_proc * result;
+    bool                done;
+    static const int    name[] = { CTL_KERN, KERN_PROC, KERN_PROC_ALL, 0 };
+    // Declaring name as const requires us to cast it when passing it to
+    // sysctl because the prototype doesn't include the const modifier.
+    size_t              length;
+
+
+    // We start by calling sysctl with result == NULL and length == 0.
+    // That will succeed, and set length to the appropriate length.
+    // We then allocate a buffer of that size and call sysctl again
+    // with that buffer.  If that succeeds, we're done.  If that fails
+    // with ENOMEM, we have to throw away our buffer and loop.  Note
+    // that the loop causes use to call sysctl with NULL again; this
+    // is necessary because the ENOMEM failure case sets length to
+    // the amount of data returned, not the amount of data that
+    // could have been returned.
+
+    result = NULL;
+    done = false;
+    do
+    {
+        assert(result == NULL);
+
+        // Call sysctl with a NULL buffer.
+
+        length = 0;
+        err = sysctl((int *) name, (sizeof(name) / sizeof(*name)) - 1,
+                     NULL, &length,
+                     NULL, 0);
+        if (err == -1)
+        {
+            err = errno;
+        }
+
+        // Allocate an appropriately sized buffer based on the results
+        // from the previous call.
+
+        if (err == 0)
+        {
+            result = (struct kinfo_proc *)malloc(length);
+            if (result == NULL)
+            {
+                err = ENOMEM;
+            }
+        }
+
+        // Call sysctl again with the new buffer.  If we get an ENOMEM
+        // error, toss away our buffer and start again.
+
+        if (err == 0)
+        {
+            err = sysctl((int *) name, (sizeof(name) / sizeof(*name)) - 1,
+                         result, &length,
+                         NULL, 0);
+            if (err == -1)
+            {
+                err = errno;
+            }
+            if (err == 0)
+            {
+                done = true;
+            }
+            else if (err == ENOMEM)
+            {
+                assert(result != NULL);
+                free(result);
+                result = NULL;
+                err = 0;
+            }
+        }
+    }
+    while (err == 0 && ! done);
+
+    // Clean up and establish post conditions.
+    if (err == 0 && result != NULL)
+    {
+        for (size_t idx = 0; idx < length / sizeof(struct kinfo_proc); idx++)
+            pids.push_back(result[idx].kp_proc.p_pid);
+    }
+
+    if (result)
+        free(result);
+}
+
+#else
+
+/* /proc style process completions */
+class process_iterator_t
+{
+    DIR *dir;
+
+public:
+    process_iterator_t();
+    ~process_iterator_t();
+
+    bool next_process(wcstring *out_str, pid_t *out_pid);
+};
+
+process_iterator_t::process_iterator_t(void)
+{
+    dir = opendir("/proc");
+}
+
+process_iterator_t::~process_iterator_t(void)
+{
+    if (dir)
+        closedir(dir);
+}
+
+bool process_iterator_t::next_process(wcstring *out_str, pid_t *out_pid)
+{
+    wcstring cmd;
+    pid_t pid = 0;
+    while (cmd.empty())
+    {
+        wcstring name;
+        if (! dir || ! wreaddir(dir, name))
+            break;
+
+        if (!iswnumeric(name.c_str()))
+            continue;
+
+        wcstring path = wcstring(L"/proc/") + name;
+        struct stat buf;
+        if (wstat(path, &buf))
+            continue;
+
+        if (buf.st_uid != getuid())
+            continue;
+
+        /* remember the pid */
+        pid = fish_wcstoi(name.c_str(), NULL, 10);
+
+        /* the 'cmdline' file exists, it should contain the commandline */
+        FILE *cmdfile;
+        if ((cmdfile=wfopen(path + L"/cmdline", "r")))
+        {
+            wcstring full_command_line;
+            fgetws2(&full_command_line, cmdfile);
+
+            /* The command line needs to be escaped */
+            cmd = tok_first(full_command_line.c_str());
+        }
+#ifdef SunOS
+        else if ((cmdfile=wfopen(path + L"/psinfo", "r")))
+        {
+            psinfo_t info;
+            if (fread(&info, sizeof(info), 1, cmdfile))
+            {
+                /* The filename is unescaped */
+                cmd = str2wcstring(info.pr_fname);
+            }
+        }
+#endif
+        if (cmdfile)
+            fclose(cmdfile);
+    }
+
+    bool result = ! cmd.empty();
+    if (result)
+    {
+        *out_str = cmd;
+        *out_pid = pid;
+    }
+    return result;
+}
+
+#endif
+
+std::vector<wcstring> expand_get_all_process_names(void)
+{
+    wcstring name;
+    pid_t pid;
+    process_iterator_t iterator;
+    std::vector<wcstring> result;
+    while (iterator.next_process(&name, &pid))
+    {
+        result.push_back(name);
+    }
+    return result;
+}
+
+/* Helper function to do a job search. */
+struct find_job_data_t
+{
+    const wchar_t *proc; /* The process to search for - possibly numeric, possibly a name */
+    expand_flags_t flags;
+    std::vector<completion_t> *completions;
+};
+
+/* The following function is invoked on the main thread, because the job list is not thread safe. It should search the job list for something matching the given proc, and then return 1 to stop the search, 0 to continue it  */
+static int find_job(const struct find_job_data_t *info)
+{
+    ASSERT_IS_MAIN_THREAD();
+
+    const wchar_t * const proc = info->proc;
+    const expand_flags_t flags = info->flags;
+    std::vector<completion_t> &completions = *info->completions;
+
+    const job_t *j;
+    int found = 0;
+    // do the empty param check first, because an empty string passes our 'numeric' check
+    if (wcslen(proc)==0)
+    {
+        /*
+          This is an empty job expansion: '%'
+          It expands to the last job backgrounded.
+        */
+        job_iterator_t jobs;
+        while ((j = jobs.next()))
+        {
+            if (!j->command_is_empty())
+            {
+                append_completion(completions, to_string<long>(j->pgid));
+                break;
+            }
+        }
+        /*
+          You don't *really* want to flip a coin between killing
+          the last process backgrounded and all processes, do you?
+          Let's not try other match methods with the solo '%' syntax.
+        */
+        found = 1;
+    }
+    else if (iswnumeric(proc))
+    {
+        /*
+          This is a numeric job string, like '%2'
+        */
+
+        if (flags & ACCEPT_INCOMPLETE)
+        {
+            job_iterator_t jobs;
+            while ((j = jobs.next()))
+            {
+                wchar_t jid[16];
+                if (j->command_is_empty())
+                    continue;
+
+                swprintf(jid, 16, L"%d", j->job_id);
+
+                if (wcsncmp(proc, jid, wcslen(proc))==0)
+                {
+                    wcstring desc_buff = format_string(COMPLETE_JOB_DESC_VAL, j->command_wcstr());
+                    append_completion(completions,
+                                      jid+wcslen(proc),
+                                      desc_buff,
+                                      0);
+                }
+            }
+        }
+        else
+        {
+            int jid;
+            wchar_t *end;
+
+            errno = 0;
+            jid = fish_wcstoi(proc, &end, 10);
+            if (jid > 0 && !errno && !*end)
+            {
+                j = job_get(jid);
+                if ((j != 0) && (j->command_wcstr() != 0) && (!j->command_is_empty()))
+                {
+                    append_completion(completions, to_string<long>(j->pgid));
+                }
+            }
+        }
+        /*
+           Stop here so you can't match a random process name
+           when you're just trying to use job control.
+        */
+        found = 1;
+    }
+
+    if (! found)
+    {
+        job_iterator_t jobs;
+        while ((j = jobs.next()))
+        {
+
+            if (j->command_is_empty())
+                continue;
+
+            size_t offset;
+            if (match_pid(j->command(), proc, flags, &offset))
+            {
+                if (flags & ACCEPT_INCOMPLETE)
+                {
+                    append_completion(completions,
+                                      j->command_wcstr() + offset + wcslen(proc),
+                                      COMPLETE_JOB_DESC,
+                                      0);
+                }
+                else
+                {
+                    append_completion(completions, to_string<long>(j->pgid));
+                    found = 1;
+                }
+            }
+        }
+
+        if (! found)
+        {
+            jobs.reset();
+            while ((j = jobs.next()))
+            {
+                process_t *p;
+                if (j->command_is_empty())
+                    continue;
+                for (p=j->first_process; p; p=p->next)
+                {
+                    if (p->actual_cmd.empty())
+                        continue;
+
+                    size_t offset;
+                    if (match_pid(p->actual_cmd, proc, flags, &offset))
+                    {
+                        if (flags & ACCEPT_INCOMPLETE)
+                        {
+                            append_completion(completions,
+                                              wcstring(p->actual_cmd, offset + wcslen(proc)),
+                                              COMPLETE_CHILD_PROCESS_DESC,
+                                              0);
+                        }
+                        else
+                        {
+                            append_completion(completions,
+                                              to_string<long>(p->pid),
+                                              L"",
+                                              0);
+                            found = 1;
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+    return found;
+}
+
+
+/**
+   Searches for a job with the specified job id, or a job or process
+   which has the string \c proc as a prefix of its commandline. Appends
+   the name of the process as a completion in 'out'.
+
+   If the ACCEPT_INCOMPLETE flag is set, the remaining string for any matches
+   are inserted.
+
+   Otherwise, any job matching the specified string is matched, and
+   the job pgid is returned. If no job matches, all child processes
+   are searched. If no child processes match, and <tt>fish</tt> can
+   understand the contents of the /proc filesystem, all the users
+   processes are searched for matches.
+*/
+static void find_process(const wchar_t *proc, expand_flags_t flags, std::vector<completion_t> &out)
+{
+    if (!(flags & EXPAND_SKIP_JOBS))
+    {
+        const struct find_job_data_t data = {proc, flags, &out};
+        int found = iothread_perform_on_main(find_job, &data);
+        if (found)
+        {
+            return;
+        }
+    }
+
+    /* Iterate over all processes */
+    wcstring process_name;
+    pid_t process_pid;
+    process_iterator_t iterator;
+    while (iterator.next_process(&process_name, &process_pid))
+    {
+        size_t offset;
+        if (match_pid(process_name, proc, flags, &offset))
+        {
+            if (flags & ACCEPT_INCOMPLETE)
+            {
+                append_completion(out,
+                                  process_name.c_str() + offset + wcslen(proc),
+                                  COMPLETE_PROCESS_DESC,
+                                  0);
+            }
+            else
+            {
+                append_completion(out, to_string<long>(process_pid));
+            }
+        }
+    }
+}
+
+/**
+   Process id expansion
+*/
+static bool expand_pid(const wcstring &instr_with_sep, expand_flags_t flags, std::vector<completion_t> &out, parse_error_list_t *errors)
+{
+    /* Hack. If there's no INTERNAL_SEP and no PROCESS_EXPAND, then there's nothing to do. Check out this "null terminated string." */
+    const wchar_t some_chars[] = {INTERNAL_SEPARATOR, PROCESS_EXPAND, L'\0'};
+    if (instr_with_sep.find_first_of(some_chars) == wcstring::npos)
+    {
+        /* Nothing to do */
+        append_completion(out, instr_with_sep);
+        return true;
+    }
+
+    /* expand_string calls us with internal separators in instr...sigh */
+    wcstring instr = instr_with_sep;
+    remove_internal_separator(instr, false);
+
+    if (instr.empty() || instr.at(0) != PROCESS_EXPAND)
+    {
+        /* Not a process expansion */
+        append_completion(out, instr);
+        return true;
+    }
+
+    const wchar_t * const in = instr.c_str();
+
+    /* We know we are a process expansion now */
+    assert(in[0] == PROCESS_EXPAND);
+
+    if (flags & ACCEPT_INCOMPLETE)
+    {
+        if (wcsncmp(in+1, SELF_STR, wcslen(in+1))==0)
+        {
+            append_completion(out,
+                              &SELF_STR[wcslen(in+1)],
+                              COMPLETE_SELF_DESC,
+                              0);
+        }
+        else if (wcsncmp(in+1, LAST_STR, wcslen(in+1))==0)
+        {
+            append_completion(out,
+                              &LAST_STR[wcslen(in+1)],
+                              COMPLETE_LAST_DESC,
+                              0);
+        }
+    }
+    else
+    {
+        if (wcscmp((in+1), SELF_STR)==0)
+        {
+
+            append_completion(out, to_string<long>(getpid()));
+            return true;
+        }
+        if (wcscmp((in+1), LAST_STR)==0)
+        {
+            if (proc_last_bg_pid > 0)
+            {
+                append_completion(out, to_string<long>(proc_last_bg_pid));
+            }
+            return true;
+        }
+    }
+
+    /* This is sort of crummy - find_process doesn't return any indication of success, so instead we check to see if it inserted any completions */
+    const size_t prev_count = out.size();
+    find_process(in+1, flags, out);
+
+    if (prev_count == out.size())
+    {
+        if (!(flags & ACCEPT_INCOMPLETE))
+        {
+            /* We failed to find anything */
+            append_syntax_error(errors, 1, FAILED_EXPANSION_PROCESS_ERR_MSG, escape(in+1, ESCAPE_NO_QUOTED).c_str());
+            return false;
+        }
+    }
+
+    return true;
+}
+
+/**
+   Parse an array slicing specification
+   Returns 0 on success.
+   If a parse error occurs, returns the index of the bad token.
+   Note that 0 can never be a bad index because the string always starts with [.
+ */
+static size_t parse_slice(const wchar_t *in, wchar_t **end_ptr, std::vector<long> &idx, std::vector<size_t> &source_positions, size_t array_size)
+{
+    wchar_t *end;
+
+    const long size = (long)array_size;
+    size_t pos = 1; //skip past the opening square bracket
+
+    //  debug( 0, L"parse_slice on '%ls'", in );
+
+    while (1)
+    {
+        long tmp;
+
+        while (iswspace(in[pos]) || (in[pos]==INTERNAL_SEPARATOR))
+            pos++;
+
+        if (in[pos] == L']')
+        {
+            pos++;
+            break;
+        }
+
+        errno=0;
+        const size_t i1_src_pos = pos;
+        tmp = wcstol(&in[pos], &end, 10);
+        if ((errno) || (end == &in[pos]))
+        {
+            return pos;
+        }
+        //    debug( 0, L"Push idx %d", tmp );
+
+        long i1 = tmp>-1 ? tmp : (long)array_size+tmp+1;
+        pos = end-in;
+        while (in[pos]==INTERNAL_SEPARATOR)
+            pos++;
+        if (in[pos]==L'.' && in[pos+1]==L'.')
+        {
+            pos+=2;
+            while (in[pos]==INTERNAL_SEPARATOR)
+                pos++;
+
+            const size_t number_start = pos;
+            long tmp1 = wcstol(&in[pos], &end, 10);
+            if ((errno) || (end == &in[pos]))
+            {
+                return pos;
+            }
+            pos = end-in;
+
+            // debug( 0, L"Push range %d %d", tmp, tmp1 );
+            long i2 = tmp1>-1 ? tmp1 : size+tmp1+1;
+            // debug( 0, L"Push range idx %d %d", i1, i2 );
+            short direction = i2<i1 ? -1 : 1 ;
+            for (long jjj = i1; jjj*direction <= i2*direction; jjj+=direction)
+            {
+                // debug(0, L"Expand range [subst]: %i\n", jjj);
+                idx.push_back(jjj);
+                source_positions.push_back(number_start);
+            }
+            continue;
+        }
+
+        // debug( 0, L"Push idx %d", tmp );
+        idx.push_back(i1);
+        source_positions.push_back(i1_src_pos);
+    }
+
+    if (end_ptr)
+    {
+        //    debug( 0, L"Remainder is '%ls', slice def was %d characters long", in+pos, pos );
+
+        *end_ptr = (wchar_t *)(in+pos);
+    }
+    //  debug( 0, L"ok, done" );
+
+    return 0;
+}
+
+
+/**
+   Expand all environment variables in the string *ptr.
+
+   This function is slow, fragile and complicated. There are lots of
+   little corner cases, like $$foo should do a double expansion,
+   $foo$bar should not double expand bar, etc. Also, it's easy to
+   accidentally leak memory on array out of bounds errors an various
+   other situations. All in all, this function should be rewritten,
+   split out into multiple logical units and carefully tested. After
+   that, it can probably be optimized to do fewer memory allocations,
+   fewer string scans and overall just less work. But until that
+   happens, don't edit it unless you know exactly what you are doing,
+   and do proper testing afterwards.
+   
+   This function operates on strings backwards, starting at last_idx.
+
+    Note: last_idx is considered to be where it previously finished
+    procesisng. This means it actually starts operating on last_idx-1.
+    As such, to process a string fully, pass string.size() as last_idx
+    instead of string.size()-1.
+*/
+static int expand_variables(parser_t &parser, const wcstring &instr, std::vector<completion_t> &out, long last_idx, parse_error_list_t *errors)
+{
+    const size_t insize = instr.size();
+
+    // last_idx may be 1 past the end of the string, but no further
+    assert(last_idx >= 0 && (size_t)last_idx <= insize);
+
+    if (last_idx == 0)
+    {
+        append_completion(out, instr);
+        return true;
+    }
+
+    bool is_ok = true;
+    bool empty = false;
+
+    wcstring var_tmp;
+
+    // list of indexes
+    std::vector<long> var_idx_list;
+
+    // parallel array of source positions of each index in the variable list
+    std::vector<size_t> var_pos_list;
+
+    //  CHECK( out, 0 );
+
+    for (long i=last_idx-1; (i>=0) && is_ok && !empty; i--)
+    {
+        const wchar_t c = instr.at(i);
+        if ((c == VARIABLE_EXPAND) || (c == VARIABLE_EXPAND_SINGLE))
+        {
+            long start_pos = i+1;
+            long stop_pos;
+            long var_len;
+            int is_single = (c==VARIABLE_EXPAND_SINGLE);
+
+            stop_pos = start_pos;
+
+            while (stop_pos < insize)
+            {
+                const wchar_t nc = instr.at(stop_pos);
+                if (nc == VARIABLE_EXPAND_EMPTY)
+                {
+                    stop_pos++;
+                    break;
+                }
+                if (!wcsvarchr(nc))
+                    break;
+
+                stop_pos++;
+            }
+
+            /*      printf( "Stop for '%c'\n", in[stop_pos]);*/
+
+            var_len = stop_pos - start_pos;
+
+            if (var_len == 0)
+            {
+                if (errors)
+                {
+                    parse_util_expand_variable_error(instr, 0 /* global_token_pos */, i, errors);
+                }
+
+                is_ok = false;
+                break;
+            }
+
+            var_tmp.append(instr, start_pos, var_len);
+            env_var_t var_val;
+            if (var_len == 1 && var_tmp[0] == VARIABLE_EXPAND_EMPTY)
+            {
+                var_val = env_var_t::missing_var();
+            }
+            else
+            {
+                var_val = expand_var(var_tmp.c_str());
+            }
+
+            if (! var_val.missing())
+            {
+                int all_vars=1;
+                wcstring_list_t var_item_list;
+
+                if (is_ok)
+                {
+                    tokenize_variable_array(var_val, var_item_list);
+
+                    const size_t slice_start = stop_pos;
+                    if (slice_start < insize && instr.at(slice_start) == L'[')
+                    {
+                        wchar_t *slice_end;
+                        size_t bad_pos;
+                        all_vars=0;
+                        const wchar_t *in = instr.c_str();
+                        bad_pos = parse_slice(in + slice_start, &slice_end, var_idx_list, var_pos_list, var_item_list.size());
+                        if (bad_pos != 0)
+                        {
+                            append_syntax_error(errors,
+                                                stop_pos + bad_pos,
+                                                L"Invalid index value");
+                            is_ok = false;
+                            break;
+                        }
+                        stop_pos = (slice_end-in);
+                    }
+
+                    if (!all_vars)
+                    {
+                        wcstring_list_t string_values(var_idx_list.size());
+                        for (size_t j=0; j<var_idx_list.size(); j++)
+                        {
+                            long tmp = var_idx_list.at(j);
+                            /* Check that we are within array bounds. If not, truncate the list to exit. */
+                            if (tmp < 1 || (size_t)tmp > var_item_list.size())
+                            {
+                                size_t var_src_pos = var_pos_list.at(j);
+                                /* The slice was parsed starting at stop_pos, so we have to add that to the error position */
+                                append_syntax_error(errors,
+                                                    slice_start + var_src_pos,
+                                                    ARRAY_BOUNDS_ERR);
+                                is_ok = false;
+                                var_idx_list.resize(j);
+                                break;
+                            }
+                            else
+                            {
+                                /* Replace each index in var_idx_list inplace with the string value at the specified index */
+                                //al_set( var_idx_list, j, wcsdup((const wchar_t *)al_get( &var_item_list, tmp-1 ) ) );
+                                string_values.at(j) = var_item_list.at(tmp-1);
+                            }
+                        }
+
+                        // string_values is the new var_item_list
+                        var_item_list.swap(string_values);
+                    }
+                }
+
+                if (is_ok)
+                {
+                    if (is_single)
+                    {
+                        wcstring res(instr, 0, i);
+                        if (i > 0)
+                        {
+                            if (instr.at(i-1) != VARIABLE_EXPAND_SINGLE)
+                            {
+                                res.push_back(INTERNAL_SEPARATOR);
+                            }
+                            else if (var_item_list.empty() || var_item_list.front().empty())
+                            {
+                                // first expansion is empty, but we need to recursively expand
+                                res.push_back(VARIABLE_EXPAND_EMPTY);
+                            }
+                        }
+
+                        for (size_t j=0; j<var_item_list.size(); j++)
+                        {
+                            const wcstring &next = var_item_list.at(j);
+                            if (is_ok)
+                            {
+                                if (j != 0)
+                                    res.append(L" ");
+                                res.append(next);
+                            }
+                        }
+                        assert(stop_pos <= insize);
+                        res.append(instr, stop_pos, insize - stop_pos);
+                        is_ok &= expand_variables(parser, res, out, i, errors);
+                    }
+                    else
+                    {
+                        for (size_t j=0; j<var_item_list.size(); j++)
+                        {
+                            const wcstring &next = var_item_list.at(j);
+                            if (is_ok && (i == 0) && stop_pos == insize)
+                            {
+                                append_completion(out, next);
+                            }
+                            else
+                            {
+
+                                if (is_ok)
+                                {
+                                    wcstring new_in;
+                                    new_in.append(instr, 0, i);
+
+                                    if (i > 0)
+                                    {
+                                        if (instr.at(i-1) != VARIABLE_EXPAND)
+                                        {
+                                            new_in.push_back(INTERNAL_SEPARATOR);
+                                        }
+                                        else if (next.empty())
+                                        {
+                                            new_in.push_back(VARIABLE_EXPAND_EMPTY);
+                                        }
+                                    }
+                                    assert(stop_pos <= insize);
+                                    new_in.append(next);
+                                    new_in.append(instr, stop_pos, insize - stop_pos);
+                                    is_ok &= expand_variables(parser, new_in, out, i, errors);
+                                }
+                            }
+
+                        }
+                    }
+                }
+
+                return is_ok;
+            }
+            else
+            {
+                // even with no value, we still need to parse out slice syntax
+                // Behave as though we had 1 value, so $foo[1] always works.
+                const size_t slice_start = stop_pos;
+                if (slice_start < insize && instr.at(slice_start) == L'[')
+                {
+                    const wchar_t *in = instr.c_str();
+                    wchar_t *slice_end;
+                    size_t bad_pos;
+
+                    bad_pos = parse_slice(in + slice_start, &slice_end, var_idx_list, var_pos_list, 1);
+                    if (bad_pos != 0)
+                    {
+                        append_syntax_error(errors,
+                                            stop_pos + bad_pos,
+                                            L"Invalid index value");
+                        is_ok = 0;
+                        return is_ok;
+                    }
+                    stop_pos = (slice_end-in);
+
+                    // validate that the parsed indexes are valid
+                    for (size_t j=0; j<var_idx_list.size(); j++)
+                    {
+                        long tmp = var_idx_list.at(j);
+                        if (tmp != 1)
+                        {
+                            size_t var_src_pos = var_pos_list.at(j);
+                            append_syntax_error(errors,
+                                                slice_start + var_src_pos,
+                                                ARRAY_BOUNDS_ERR);
+                            is_ok = 0;
+                            return is_ok;
+                        }
+                    }
+                }
+                
+                /* Expand a non-existing variable */
+                if (c == VARIABLE_EXPAND)
+                {
+                    /* Regular expansion, i.e. expand this argument to nothing */
+                    empty = true;
+                }
+                else
+                {
+                    /* Expansion to single argument. */
+                    wcstring res;
+                    res.append(instr, 0, i);
+                    if (i > 0 && instr.at(i-1) == VARIABLE_EXPAND_SINGLE)
+                    {
+                        res.push_back(VARIABLE_EXPAND_EMPTY);
+                    }
+                    assert(stop_pos <= insize);
+                    res.append(instr, stop_pos, insize - stop_pos);
+
+                    is_ok &= expand_variables(parser, res, out, i, errors);
+                    return is_ok;
+                }
+            }
+        }
+    }
+
+    if (!empty)
+    {
+        append_completion(out, instr);
+    }
+
+    return is_ok;
+}
+
+/**
+   Perform bracket expansion
+*/
+static int expand_brackets(parser_t &parser, const wcstring &instr, int flags, std::vector<completion_t> &out, parse_error_list_t *errors)
+{
+    bool syntax_error = false;
+    int bracket_count=0;
+
+    const wchar_t *bracket_begin = NULL, *bracket_end = NULL;
+    const wchar_t *last_sep = NULL;
+
+    const wchar_t *item_begin;
+    size_t length_preceding_brackets, length_following_brackets, tot_len;
+
+    const wchar_t * const in = instr.c_str();
+
+    /* Locate the first non-nested bracket pair */
+    for (const wchar_t *pos = in; (*pos) && !syntax_error; pos++)
+    {
+        switch (*pos)
+        {
+            case BRACKET_BEGIN:
+            {
+                if (bracket_count == 0)
+                    bracket_begin = pos;
+                bracket_count++;
+                break;
+
+            }
+            case BRACKET_END:
+            {
+                bracket_count--;
+                if (bracket_count < 0)
+                {
+                    syntax_error = true;
+                }
+                else if (bracket_count == 0)
+                {
+                    bracket_end = pos;
+                    break;
+                }
+
+            }
+            case BRACKET_SEP:
+            {
+                if (bracket_count == 1)
+                    last_sep = pos;
+            }
+        }
+    }
+
+    if (bracket_count > 0)
+    {
+        if (!(flags & ACCEPT_INCOMPLETE))
+        {
+            syntax_error = true;
+        }
+        else
+        {
+            /* The user hasn't typed an end bracket yet; make one up and append it, then expand that. */
+            wcstring mod;
+            if (last_sep)
+            {
+                mod.append(in, bracket_begin-in+1);
+                mod.append(last_sep+1);
+                mod.push_back(BRACKET_END);
+            }
+            else
+            {
+                mod.append(in);
+                mod.push_back(BRACKET_END);
+            }
+
+            return expand_brackets(parser, mod, 1, out, errors);
+        }
+    }
+
+    if (syntax_error)
+    {
+        append_syntax_error(errors,
+                            SOURCE_LOCATION_UNKNOWN,
+                            _(L"Mismatched brackets"));
+        return 0;
+    }
+
+    if (bracket_begin == NULL)
+    {
+        append_completion(out, instr);
+        return 1;
+    }
+
+    length_preceding_brackets = (bracket_begin-in);
+    length_following_brackets = wcslen(bracket_end)-1;
+    tot_len = length_preceding_brackets+length_following_brackets;
+    item_begin = bracket_begin+1;
+    for (const wchar_t *pos =(bracket_begin+1); true; pos++)
+    {
+        if (bracket_count == 0)
+        {
+            if ((*pos == BRACKET_SEP) || (pos==bracket_end))
+            {
+                assert(pos >= item_begin);
+                size_t item_len = pos-item_begin;
+
+                wcstring whole_item;
+                whole_item.reserve(tot_len + item_len + 2);
+                whole_item.append(in, length_preceding_brackets);
+                whole_item.append(item_begin, item_len);
+                whole_item.append(bracket_end + 1);
+                expand_brackets(parser, whole_item, flags, out, errors);
+
+                item_begin = pos+1;
+                if (pos == bracket_end)
+                    break;
+            }
+        }
+
+        if (*pos == BRACKET_BEGIN)
+        {
+            bracket_count++;
+        }
+
+        if (*pos == BRACKET_END)
+        {
+            bracket_count--;
+        }
+    }
+    return 1;
+}
+
+/**
+ Perform cmdsubst expansion
+ */
+static int expand_cmdsubst(parser_t &parser, const wcstring &input, std::vector<completion_t> &out_list, parse_error_list_t *errors)
+{
+    wchar_t *paran_begin=0, *paran_end=0;
+    std::vector<wcstring> sub_res;
+    size_t i, j;
+    wchar_t *tail_begin = 0;
+
+    const wchar_t * const in = input.c_str();
+
+    int parse_ret;
+    switch (parse_ret = parse_util_locate_cmdsubst(in, &paran_begin, &paran_end, false))
+    {
+        case -1:
+            append_syntax_error(errors,
+                                SOURCE_LOCATION_UNKNOWN,
+                                L"Mismatched parenthesis");
+            return 0;
+        case 0:
+            append_completion(out_list, input);
+            return 1;
+        case 1:
+
+            break;
+    }
+
+    const wcstring subcmd(paran_begin + 1, paran_end-paran_begin - 1);
+
+    if (exec_subshell(subcmd, sub_res, true /* do apply exit status */) == -1)
+    {
+        append_cmdsub_error(errors, SOURCE_LOCATION_UNKNOWN, L"Unknown error while evaulating command substitution");
+        return 0;
+    }
+
+    tail_begin = paran_end + 1;
+    if (*tail_begin == L'[')
+    {
+        std::vector<long> slice_idx;
+        std::vector<size_t> slice_source_positions;
+        const wchar_t * const slice_begin = tail_begin;
+        wchar_t *slice_end;
+        size_t bad_pos;
+
+        bad_pos = parse_slice(slice_begin, &slice_end, slice_idx, slice_source_positions, sub_res.size());
+        if (bad_pos != 0)
+        {
+            append_syntax_error(errors, slice_begin - in + bad_pos, L"Invalid index value");
+            return 0;
+        }
+        else
+        {
+            wcstring_list_t sub_res2;
+            tail_begin = slice_end;
+            for (i=0; i < slice_idx.size(); i++)
+            {
+                long idx = slice_idx.at(i);
+                if (idx < 1 || (size_t)idx > sub_res.size())
+                {
+                    size_t pos = slice_source_positions.at(i);
+                    append_syntax_error(errors,
+                                        slice_begin - in + pos,
+                                        ARRAY_BOUNDS_ERR);
+                    return 0;
+                }
+                idx = idx-1;
+
+                sub_res2.push_back(sub_res.at(idx));
+                //        debug( 0, L"Pushing item '%ls' with index %d onto sliced result", al_get( sub_res, idx ), idx );
+                //sub_res[idx] = 0; // ??
+            }
+            sub_res = sub_res2;
+        }
+    }
+
+
+    /*
+       Recursively call ourselves to expand any remaining command
+       substitutions. The result of this recursive call using the tail
+       of the string is inserted into the tail_expand array list
+       */
+    std::vector<completion_t> tail_expand;
+    expand_cmdsubst(parser, tail_begin, tail_expand, errors /* TODO: offset error locations */);
+
+    /*
+       Combine the result of the current command substitution with the
+       result of the recursive tail expansion
+       */
+    for (i=0; i<sub_res.size(); i++)
+    {
+        const wcstring &sub_item = sub_res.at(i);
+        const wcstring sub_item2 = escape_string(sub_item, 1);
+
+        wcstring whole_item;
+
+        for (j=0; j < tail_expand.size(); j++)
+        {
+            whole_item.clear();
+            const wcstring &tail_item = tail_expand.at(j).completion;
+
+            //sb_append_substring( &whole_item, in, len1 );
+            whole_item.append(in, paran_begin-in);
+
+            //sb_append_char( &whole_item, INTERNAL_SEPARATOR );
+            whole_item.push_back(INTERNAL_SEPARATOR);
+
+            //sb_append_substring( &whole_item, sub_item2, item_len );
+            whole_item.append(sub_item2);
+
+            //sb_append_char( &whole_item, INTERNAL_SEPARATOR );
+            whole_item.push_back(INTERNAL_SEPARATOR);
+
+            //sb_append( &whole_item, tail_item );
+            whole_item.append(tail_item);
+
+            //al_push( out, whole_item.buff );
+            append_completion(out_list, whole_item);
+        }
+    }
+
+    return 1;
+}
+
+/* Given that input[0] is HOME_DIRECTORY or tilde (ugh), return the user's name. Return the empty string if it is just a tilde. Also return by reference the index of the first character of the remaining part of the string (e.g. the subsequent slash) */
+static wcstring get_home_directory_name(const wcstring &input, size_t *out_tail_idx)
+{
+    const wchar_t * const in = input.c_str();
+    assert(in[0] == HOME_DIRECTORY || in[0] == L'~');
+    size_t tail_idx;
+
+    const wchar_t *name_end = wcschr(in, L'/');
+    if (name_end)
+    {
+        tail_idx = name_end - in;
+    }
+    else
+    {
+        tail_idx = wcslen(in);
+    }
+    *out_tail_idx = tail_idx;
+    return input.substr(1, tail_idx - 1);
+}
+
+/** Attempts tilde expansion of the string specified, modifying it in place. */
+static void expand_home_directory(wcstring &input)
+{
+    if (! input.empty() && input.at(0) == HOME_DIRECTORY)
+    {
+        size_t tail_idx;
+        wcstring username = get_home_directory_name(input, &tail_idx);
+        
+        bool tilde_error = false;
+        wcstring home;
+        if (username.empty())
+        {
+            /* Current users home directory */
+            home = env_get_string(L"HOME");
+            tail_idx = 1;
+        }
+        else
+        {
+            /* Some other users home directory */
+            std::string name_cstr = wcs2string(username);
+            struct passwd *userinfo = getpwnam(name_cstr.c_str());
+            if (userinfo == NULL)
+            {
+                tilde_error = true;
+            }
+            else
+            {
+                home = str2wcstring(userinfo->pw_dir);
+            }
+        }
+        
+        wchar_t *realhome = wrealpath(home, NULL);
+        
+        if (! tilde_error && realhome)
+        {
+            input.replace(input.begin(), input.begin() + tail_idx, realhome);
+        }
+        else
+        {
+            input[0] = L'~';
+        }
+        free((void *)realhome);
+    }
+}
+
+void expand_tilde(wcstring &input)
+{
+    // Avoid needless COW behavior by ensuring we use const at
+    const wcstring &tmp = input;
+    if (! tmp.empty() && tmp.at(0) == L'~')
+    {
+        input.at(0) = HOME_DIRECTORY;
+        expand_home_directory(input);
+    }
+}
+
+static void unexpand_tildes(const wcstring &input, std::vector<completion_t> *completions)
+{
+    // If input begins with tilde, then try to replace the corresponding string in each completion with the tilde
+    // If it does not, there's nothing to do
+    if (input.empty() || input.at(0) != L'~')
+        return;
+
+    // We only operate on completions that replace their contents
+    // If we don't have any, we're done.
+    // In particular, empty vectors are common.
+    bool has_candidate_completion = false;
+    for (size_t i=0; i < completions->size(); i++)
+    {
+        if (completions->at(i).flags & COMPLETE_REPLACES_TOKEN)
+        {
+            has_candidate_completion = true;
+            break;
+        }
+    }
+    if (! has_candidate_completion)
+        return;
+
+    size_t tail_idx;
+    wcstring username_with_tilde = L"~";
+    username_with_tilde.append(get_home_directory_name(input, &tail_idx));
+
+    // Expand username_with_tilde
+    wcstring home = username_with_tilde;
+    expand_tilde(home);
+
+    // Now for each completion that starts with home, replace it with the username_with_tilde
+    for (size_t i=0; i < completions->size(); i++)
+    {
+        completion_t &comp = completions->at(i);
+        if ((comp.flags & COMPLETE_REPLACES_TOKEN) && string_prefixes_string(home, comp.completion))
+        {
+            comp.completion.replace(0, home.size(), username_with_tilde);
+
+            // And mark that our tilde is literal, so it doesn't try to escape it
+            comp.flags |= COMPLETE_DONT_ESCAPE_TILDES;
+        }
+    }
+}
+
+// If the given path contains the user's home directory, replace that with a tilde
+// We don't try to be smart about case insensitivity, etc.
+wcstring replace_home_directory_with_tilde(const wcstring &str)
+{
+    // only absolute paths get this treatment
+    wcstring result = str;
+    if (string_prefixes_string(L"/", result))
+    {
+        wcstring home_directory = L"~";
+        expand_tilde(home_directory);
+        if (! string_suffixes_string(L"/", home_directory))
+        {
+            home_directory.push_back(L'/');
+        }
+
+        // Now check if the home_directory prefixes the string
+        if (string_prefixes_string(home_directory, result))
+        {
+            // Success
+            result.replace(0, home_directory.size(), L"~/");
+        }
+    }
+    return result;
+}
+
+/**
+   Remove any internal separators. Also optionally convert wildcard characters to
+   regular equivalents. This is done to support EXPAND_SKIP_WILDCARDS.
+*/
+static void remove_internal_separator(wcstring &str, bool conv)
+{
+    /* Remove all instances of INTERNAL_SEPARATOR */
+    str.erase(std::remove(str.begin(), str.end(), (wchar_t)INTERNAL_SEPARATOR), str.end());
+
+    /* If conv is true, replace all instances of ANY_CHAR with '?', ANY_STRING with '*', ANY_STRING_RECURSIVE with '*' */
+    if (conv)
+    {
+        for (size_t idx = 0; idx < str.size(); idx++)
+        {
+            switch (str.at(idx))
+            {
+                case ANY_CHAR:
+                    str.at(idx) = L'?';
+                    break;
+                case ANY_STRING:
+                case ANY_STRING_RECURSIVE:
+                    str.at(idx) = L'*';
+                    break;
+            }
+        }
+    }
+}
+
+
+int expand_string(const wcstring &input, std::vector<completion_t> &output, expand_flags_t flags, parse_error_list_t *errors)
+{
+    parser_t parser(PARSER_TYPE_ERRORS_ONLY, true /* show errors */);
+
+    size_t i;
+    int res = EXPAND_OK;
+
+    if ((!(flags & ACCEPT_INCOMPLETE)) && expand_is_clean(input.c_str()))
+    {
+        append_completion(output, input);
+        return EXPAND_OK;
+    }
+
+    std::vector<completion_t> clist1, clist2;
+    std::vector<completion_t> *in = &clist1, *out = &clist2;
+
+    if (EXPAND_SKIP_CMDSUBST & flags)
+    {
+        wchar_t *begin, *end;
+
+        if (parse_util_locate_cmdsubst(input.c_str(), &begin, &end, true) != 0)
+        {
+            append_cmdsub_error(errors, SOURCE_LOCATION_UNKNOWN, L"Command substitutions not allowed");
+            return EXPAND_ERROR;
+        }
+        append_completion(*in, input);
+    }
+    else
+    {
+        int cmdsubst_ok = expand_cmdsubst(parser, input, *in, errors);
+        if (! cmdsubst_ok)
+            return EXPAND_ERROR;
+    }
+
+    for (i=0; i < in->size(); i++)
+    {
+        /*
+         We accept incomplete strings here, since complete uses
+         expand_string to expand incomplete strings from the
+         commandline.
+         */
+        wcstring next;
+        unescape_string(in->at(i).completion, &next, UNESCAPE_SPECIAL | UNESCAPE_INCOMPLETE);
+
+        if (EXPAND_SKIP_VARIABLES & flags)
+        {
+            for (size_t i=0; i < next.size(); i++)
+            {
+                if (next.at(i) == VARIABLE_EXPAND)
+                {
+                    next[i] = L'$';
+                }
+            }
+            append_completion(*out, next);
+        }
+        else
+        {
+            if (!expand_variables(parser, next, *out, next.size(), errors))
+            {
+                return EXPAND_ERROR;
+            }
+        }
+    }
+
+    in->clear();
+    std::swap(in, out); // note: this swaps the pointers only (last output is next input)
+
+    for (i=0; i < in->size(); i++)
+    {
+        const wcstring &next = in->at(i).completion;
+
+        if (!expand_brackets(parser, next, flags, *out, errors))
+        {
+            return EXPAND_ERROR;
+        }
+    }
+    in->clear();
+    std::swap(in, out); // note: this swaps the pointers only (last output is next input)
+
+    for (i=0; i < in->size(); i++)
+    {
+        wcstring next = in->at(i).completion;
+
+        if (!(EXPAND_SKIP_HOME_DIRECTORIES & flags))
+            expand_home_directory(next);
+
+        if (flags & ACCEPT_INCOMPLETE)
+        {
+            if (! next.empty() && next.at(0) == PROCESS_EXPAND)
+            {
+                /*
+                 If process expansion matches, we are not
+                 interested in other completions, so we
+                 short-circuit and return
+                 */
+                expand_pid(next, flags, output, NULL);
+                return EXPAND_OK;
+            }
+            else
+            {
+                append_completion(*out, next);
+            }
+        }
+        else if (! expand_pid(next, flags, *out, errors))
+        {
+            return EXPAND_ERROR;
+        }
+    }
+
+    in->clear();
+    std::swap(in, out); // note: this swaps the pointers only (last output is next input)
+
+    for (i=0; i < in->size(); i++)
+    {
+        wcstring next = in->at(i).completion;
+        int wc_res;
+
+        remove_internal_separator(next, (EXPAND_SKIP_WILDCARDS & flags) ? true : false);
+        const bool has_wildcard = wildcard_has(next, 1);
+
+        if (has_wildcard && (flags & EXECUTABLES_ONLY))
+        {
+            // Don't do wildcard expansion for executables. See #785. So do nothing here.
+        }
+        else if (((flags & ACCEPT_INCOMPLETE) && (!(flags & EXPAND_SKIP_WILDCARDS))) ||
+                 has_wildcard)
+        {
+            wcstring start, rest;
+
+            if (next[0] == '/')
+            {
+                start = L"/";
+                rest = next.substr(1);
+            }
+            else
+            {
+                start = L"";
+                rest = next;
+            }
+
+            std::vector<completion_t> expanded;
+            wc_res = wildcard_expand_string(rest, start, flags, expanded);
+            if (flags & ACCEPT_INCOMPLETE)
+            {
+                out->insert(out->end(), expanded.begin(), expanded.end());
+            }
+            else
+            {
+                switch (wc_res)
+                {
+                    case 0:
+                    {
+                        if (res == EXPAND_OK)
+                            res = EXPAND_WILDCARD_NO_MATCH;
+                        break;
+                    }
+
+                    case 1:
+                    {
+                        res = EXPAND_WILDCARD_MATCH;
+                        std::sort(expanded.begin(), expanded.end(), completion_t::is_naturally_less_than);
+                        out->insert(out->end(), expanded.begin(), expanded.end());
+                        break;
+                    }
+
+                    case -1:
+                    {
+                        return EXPAND_ERROR;
+                    }
+
+                }
+            }
+        }
+        else
+        {
+            if (!(flags & ACCEPT_INCOMPLETE))
+            {
+                append_completion(*out, next);
+            }
+        }
+    }
+
+    // Hack to un-expand tildes (see #647)
+    if (!(flags & EXPAND_SKIP_HOME_DIRECTORIES))
+    {
+        unexpand_tildes(input, out);
+    }
+
+    // Return our output
+    output.insert(output.end(), out->begin(), out->end());
+
+    return res;
+}
+
+bool expand_one(wcstring &string, expand_flags_t flags, parse_error_list_t *errors)
+{
+    std::vector<completion_t> completions;
+    bool result = false;
+
+    if ((!(flags & ACCEPT_INCOMPLETE)) &&  expand_is_clean(string.c_str()))
+    {
+        return true;
+    }
+
+    if (expand_string(string, completions, flags | EXPAND_NO_DESCRIPTIONS, errors))
+    {
+        if (completions.size() == 1)
+        {
+            string = completions.at(0).completion;
+            result = true;
+        }
+    }
+    return result;
+}
+
+
+/*
+
+https://github.com/fish-shell/fish-shell/issues/367
+
+With them the Seed of Wisdom did I sow,
+And with my own hand labour'd it to grow:
+And this was all the Harvest that I reap'd---
+"I came like Water, and like Wind I go."
+
+*/
+
+static std::string escape_single_quoted_hack_hack_hack_hack(const char *str)
+{
+    std::string result;
+    size_t len = strlen(str);
+    result.reserve(len + 2);
+    result.push_back('\'');
+    for (size_t i=0; i < len; i++)
+    {
+        char c = str[i];
+        // Escape backslashes and single quotes only
+        if (c == '\\' || c == '\'')
+            result.push_back('\\');
+        result.push_back(c);
+    }
+    result.push_back('\'');
+    return result;
+}
+
+bool fish_xdm_login_hack_hack_hack_hack(std::vector<std::string> *cmds, int argc, const char * const *argv)
+{
+    bool result = false;
+    if (cmds && cmds->size() == 1)
+    {
+        const std::string &cmd = cmds->at(0);
+        if (cmd == "exec \"${@}\"" || cmd == "exec \"$@\"")
+        {
+            /* We're going to construct a new command that starts with exec, and then has the remaining arguments escaped */
+            std::string new_cmd = "exec";
+            for (int i=1; i < argc; i++)
+            {
+                const char *arg = argv[i];
+                if (arg)
+                {
+                    new_cmd.push_back(' ');
+                    new_cmd.append(escape_single_quoted_hack_hack_hack_hack(arg));
+                }
+            }
+
+            cmds->at(0) = new_cmd;
+            result = true;
+        }
+    }
+    return result;
+}
+
+bool fish_openSUSE_dbus_hack_hack_hack_hack(std::vector<completion_t> *args)
+{
+    static signed char isSUSE = -1;
+    if (isSUSE == 0)
+        return false;
+
+    bool result = false;
+    if (args && ! args->empty())
+    {
+        const wcstring &cmd = args->at(0).completion;
+        if (cmd.find(L"DBUS_SESSION_BUS_") != wcstring::npos)
+        {
+            /* See if we are SUSE */
+            if (isSUSE < 0)
+            {
+                struct stat buf = {};
+                isSUSE = (0 == stat("/etc/SuSE-release", &buf));
+            }
+
+            if (isSUSE)
+            {
+                /* Look for an equal sign */
+                size_t where = cmd.find(L'=');
+                if (where != wcstring::npos)
+                {
+                    /* Oh my. It's presumably of the form foo=bar; find the = and split */
+                    const wcstring key = wcstring(cmd, 0, where);
+
+                    /* Trim whitespace and semicolon */
+                    wcstring val = wcstring(cmd, where+1);
+                    size_t last_good = val.find_last_not_of(L"\n ;");
+                    if (last_good != wcstring::npos)
+                        val.resize(last_good + 1);
+
+                    args->clear();
+                    append_completion(*args, L"set");
+                    if (key == L"DBUS_SESSION_BUS_ADDRESS")
+                        append_completion(*args, L"-x");
+                    append_completion(*args, key);
+                    append_completion(*args, val);
+                    result = true;
+                }
+                else if (string_prefixes_string(L"export DBUS_SESSION_BUS_ADDRESS;", cmd))
+                {
+                    /* Nothing, we already exported it */
+                    args->clear();
+                    append_completion(*args, L"echo");
+                    append_completion(*args, L"-n");
+                    result = true;
+                }
+            }
+        }
+    }
+    return result;
+}
+
+bool expand_abbreviation(const wcstring &src, wcstring *output)
+{
+    if (src.empty())
+        return false;
+
+    /* Get the abbreviations. Return false if we have none */
+    env_var_t var = env_get_string(USER_ABBREVIATIONS_VARIABLE_NAME);
+    if (var.missing_or_empty())
+        return false;
+
+    bool result = false;
+    wcstring line;
+    wcstokenizer tokenizer(var, ARRAY_SEP_STR);
+    while (tokenizer.next(line))
+    {
+        /* Line is expected to be of the form 'foo=bar' or 'foo bar'. Parse out the first = or space. Silently skip on failure (no equals, or equals at the end or beginning). Try to avoid copying any strings until we are sure this is a match. */
+        size_t equals_pos = line.find(L'=');
+        size_t space_pos = line.find(L' ');
+        size_t separator = mini(equals_pos, space_pos);
+        if (separator == wcstring::npos || separator == 0 || separator + 1 == line.size())
+            continue;
+
+        /* Find the character just past the end of the command. Walk backwards, skipping spaces. */
+        size_t cmd_end = separator;
+        while (cmd_end > 0 && iswspace(line.at(cmd_end - 1)))
+            cmd_end--;
+
+        /* See if this command matches */
+        if (line.compare(0, cmd_end, src) == 0)
+        {
+            /* Success. Set output to everythign past the end of the string. */
+            if (output != NULL)
+                output->assign(line, separator + 1, wcstring::npos);
+
+            result = true;
+            break;
+        }
+    }
+    return result;
+}
diff --git a/src/expand.h b/src/expand.h
new file mode 100644
index 00000000..8fe80b3b
--- /dev/null
+++ b/src/expand.h
@@ -0,0 +1,216 @@
+/**\file expand.h
+
+   Prototypes for string expansion functions. These functions perform
+   several kinds of parameter expansion. There are a lot of issues
+   with regards to memory allocation. Overall, these functions would
+   benefit from using a more clever memory allocation scheme, perhaps
+   an evil combination of talloc, string buffers and reference
+   counting.
+
+*/
+
+#ifndef FISH_EXPAND_H
+/**
+   Header guard
+*/
+#define FISH_EXPAND_H
+
+#include "config.h"                     // for __warn_unused
+
+#include <wchar.h>
+#include <string>                       // for string
+#include <vector>                       // for vector
+
+#include "common.h"
+#include "parse_constants.h"
+
+enum
+{
+    /** Flag specifying that cmdsubst expansion should be skipped */
+    EXPAND_SKIP_CMDSUBST = 1 << 0,
+
+    /** Flag specifying that variable expansion should be skipped */
+    EXPAND_SKIP_VARIABLES = 1 << 1,
+
+    /** Flag specifying that wildcard expansion should be skipped */
+    EXPAND_SKIP_WILDCARDS = 1 << 2,
+
+    /**
+       Incomplete matches in the last segment are ok (for tab
+       completion). An incomplete match is a wildcard that matches a
+       prefix of the filename. If accept_incomplete is true, only the
+       remainder of the string is returned.
+    */
+    ACCEPT_INCOMPLETE = 1 << 3,
+
+    /** Only match files that are executable by the current user. Only applicable together with ACCEPT_INCOMPLETE. */
+    EXECUTABLES_ONLY = 1 << 4,
+
+    /** Only match directories. Only applicable together with ACCEPT_INCOMPLETE. */
+    DIRECTORIES_ONLY = 1 << 5,
+
+    /** Don't generate descriptions */
+    EXPAND_NO_DESCRIPTIONS = 1 << 6,
+
+    /** Don't expand jobs (but you can still expand processes). This is because job expansion is not thread safe. */
+    EXPAND_SKIP_JOBS = 1 << 7,
+
+    /** Don't expand home directories */
+    EXPAND_SKIP_HOME_DIRECTORIES = 1 << 8,
+
+    /** Allow fuzzy matching */
+    EXPAND_FUZZY_MATCH = 1 << 9
+};
+typedef int expand_flags_t;
+
+/**
+  Use unencoded private-use keycodes for internal characters
+*/
+#define EXPAND_RESERVED 0xf000
+/**
+   End of range reserved for expand
+ */
+#define EXPAND_RESERVED_END 0xf000f
+
+class completion_t;
+
+enum
+{
+    /** Character represeting a home directory */
+    HOME_DIRECTORY = EXPAND_RESERVED,
+
+    /** Character represeting process expansion */
+    PROCESS_EXPAND,
+
+    /** Character representing variable expansion */
+    VARIABLE_EXPAND,
+
+    /** Character rpresenting variable expansion into a single element*/
+    VARIABLE_EXPAND_SINGLE,
+
+    /** Character representing the start of a bracket expansion */
+    BRACKET_BEGIN,
+
+    /** Character representing the end of a bracket expansion */
+    BRACKET_END,
+
+    /** Character representing separation between two bracket elements */
+    BRACKET_SEP,
+    /**
+       Separate subtokens in a token with this character.
+    */
+    INTERNAL_SEPARATOR,
+
+    /**
+       Character representing an empty variable expansion.
+       Only used transitively while expanding variables.
+    */
+    VARIABLE_EXPAND_EMPTY,
+}
+;
+
+
+/**
+   These are the possible return values for expand_string
+*/
+enum
+{
+    /** Error */
+    EXPAND_ERROR,
+    /** Ok */
+    EXPAND_OK,
+    /** Ok, a wildcard in the string matched no files */
+    EXPAND_WILDCARD_NO_MATCH,
+    /* Ok, a wildcard in the string matched a file */
+    EXPAND_WILDCARD_MATCH
+};
+
+/** Character for separating two array elements. We use 30, i.e. the ascii record separator since that seems logical. */
+#define ARRAY_SEP ((wchar_t)(0x1e))
+
+/** String containing the character for separating two array elements */
+#define ARRAY_SEP_STR L"\x1e"
+
+/**
+   Error issued on array out of bounds
+*/
+#define ARRAY_BOUNDS_ERR _(L"Array index out of bounds")
+
+/**
+   Perform various forms of expansion on in, such as tilde expansion
+   (\~USER becomes the users home directory), variable expansion
+   (\$VAR_NAME becomes the value of the environment variable VAR_NAME),
+   cmdsubst expansion and wildcard expansion. The results are inserted
+   into the list out.
+
+   If the parameter does not need expansion, it is copied into the list
+   out.
+
+   \param input The parameter to expand
+   \param output The list to which the result will be appended.
+   \param flag Specifies if any expansion pass should be skipped. Legal values are any combination of EXPAND_SKIP_CMDSUBST EXPAND_SKIP_VARIABLES and EXPAND_SKIP_WILDCARDS
+   \param errors Resulting errors, or NULL to ignore
+   \return One of EXPAND_OK, EXPAND_ERROR, EXPAND_WILDCARD_MATCH and EXPAND_WILDCARD_NO_MATCH. EXPAND_WILDCARD_NO_MATCH and EXPAND_WILDCARD_MATCH are normal exit conditions used only on strings containing wildcards to tell if the wildcard produced any matches.
+*/
+__warn_unused int expand_string(const wcstring &input, std::vector<completion_t> &output, expand_flags_t flags, parse_error_list_t *errors);
+
+
+/**
+   expand_one is identical to expand_string, except it will fail if in
+   expands to more than one string. This is used for expanding command
+   names.
+
+   \param inout_str The parameter to expand in-place
+   \param flag Specifies if any expansion pass should be skipped. Legal values are any combination of EXPAND_SKIP_CMDSUBST EXPAND_SKIP_VARIABLES and EXPAND_SKIP_WILDCARDS
+   \param errors Resulting errors, or NULL to ignore
+   \return Whether expansion succeded
+*/
+bool expand_one(wcstring &inout_str, expand_flags_t flags, parse_error_list_t *errors = NULL);
+
+/**
+   Convert the variable value to a human readable form, i.e. escape things, handle arrays, etc. Suitable for pretty-printing. The result must be free'd!
+
+   \param in the value to escape
+*/
+wcstring expand_escape_variable(const wcstring &in);
+
+/**
+   Perform tilde expansion and nothing else on the specified string, which is modified in place.
+
+   \param input the string to tilde expand
+*/
+void expand_tilde(wcstring &input);
+
+/** Perform the opposite of tilde expansion on the string, which is modified in place */
+wcstring replace_home_directory_with_tilde(const wcstring &str);
+
+/**
+   Test if the specified argument is clean, i.e. it does not contain
+   any tokens which need to be expanded or otherwise altered. Clean
+   strings can be passed through expand_string and expand_one without
+   changing them. About two thirds of all strings are clean, so
+   skipping expansion on them actually does save a small amount of
+   time, since it avoids multiple memory allocations during the
+   expansion process.
+
+   \param in the string to test
+*/
+int expand_is_clean(const wchar_t *in);
+
+/**
+   Testing function for getting all process names.
+*/
+std::vector<wcstring> expand_get_all_process_names(void);
+
+/** Abbreviation support. Expand src as an abbreviation, returning true if one was found, false if not. If result is not-null, returns the abbreviation by reference. */
+#define USER_ABBREVIATIONS_VARIABLE_NAME L"fish_user_abbreviations"
+bool expand_abbreviation(const wcstring &src, wcstring *output);
+
+/* Terrible hacks */
+bool fish_xdm_login_hack_hack_hack_hack(std::vector<std::string> *cmds, int argc, const char * const *argv);
+bool fish_openSUSE_dbus_hack_hack_hack_hack(std::vector<completion_t> *args);
+
+
+#endif
+
+
diff --git a/src/fallback.cpp b/src/fallback.cpp
new file mode 100644
index 00000000..0f53e1a8
--- /dev/null
+++ b/src/fallback.cpp
@@ -0,0 +1,1521 @@
+/**
+   This file only contains fallback implementations of functions which
+   have been found to be missing or broken by the configuration
+   scripts.
+
+   Many of these functions are more or less broken and
+   incomplete. lrand28_r internally uses the regular (bad) rand_r
+   function, the gettext function doesn't actually do anything, etc.
+*/
+
+#include "config.h"
+
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <unistd.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <wchar.h>
+#include <wctype.h>
+#include <string.h>
+#include <dirent.h>
+#include <stdarg.h>
+#include <limits.h>
+#include <assert.h>
+
+#if HAVE_GETTEXT
+#include <libintl.h>
+#endif
+
+#if HAVE_NCURSES_H
+#include <ncurses.h>
+#elif HAVE_NCURSES_CURSES_H
+#include <ncurses/curses.h>
+#else
+#include <curses.h>
+#endif
+
+#if HAVE_TERM_H
+#include <term.h>
+#elif HAVE_NCURSES_TERM_H
+#include <ncurses/term.h>
+#endif
+
+#include "fallback.h"
+#include "util.h"
+
+
+#ifndef HAVE___ENVIRON
+
+char **__environ = 0;
+
+#endif
+
+#ifdef TPUTS_KLUDGE
+
+int tputs(const char *str, int affcnt, int (*fish_putc)(tputs_arg_t))
+{
+    while (*str)
+    {
+        fish_putc(*str++);
+    }
+}
+
+#endif
+
+#ifdef TPARM_SOLARIS_KLUDGE
+
+#undef tparm
+
+/**
+   Checks for known string values and maps to correct number of parameters.
+*/
+char *tparm_solaris_kludge(char *str, ...)
+{
+    long int param[9] = { };
+
+    va_list ap;
+    va_start(ap, str);
+
+    if ((set_a_foreground && ! strcmp(str, set_a_foreground))
+            || (set_a_background && ! strcmp(str, set_a_background))
+            || (set_foreground && ! strcmp(str, set_foreground))
+            || (set_background && ! strcmp(str, set_background))
+            || (enter_underline_mode && ! strcmp(str, enter_underline_mode))
+            || (exit_underline_mode && ! strcmp(str, exit_underline_mode))
+            || (enter_standout_mode && ! strcmp(str, enter_standout_mode))
+            || (exit_standout_mode && ! strcmp(str, exit_standout_mode))
+            || (flash_screen && ! strcmp(str, flash_screen))
+            || (enter_subscript_mode && ! strcmp(str, enter_subscript_mode))
+            || (exit_subscript_mode && ! strcmp(str, exit_subscript_mode))
+            || (enter_superscript_mode && ! strcmp(str, enter_superscript_mode))
+            || (exit_superscript_mode && ! strcmp(str, exit_superscript_mode))
+            || (enter_blink_mode && ! strcmp(str, enter_blink_mode))
+            || (enter_italics_mode && ! strcmp(str, enter_italics_mode))
+            || (exit_italics_mode && ! strcmp(str, exit_italics_mode))
+            || (enter_reverse_mode && ! strcmp(str, enter_reverse_mode))
+            || (enter_shadow_mode && ! strcmp(str, enter_shadow_mode))
+            || (exit_shadow_mode && ! strcmp(str, exit_shadow_mode))
+            || (enter_secure_mode && ! strcmp(str, enter_secure_mode))
+            || (enter_bold_mode && ! strcmp(str, enter_bold_mode)))
+    {
+        param[0] = va_arg(ap, long int);
+    }
+    else if (cursor_address && ! strcmp(str, cursor_address))
+    {
+        param[0] = va_arg(ap, long int);
+        param[1] = va_arg(ap, long int);
+    }
+
+    va_end(ap);
+
+
+    return tparm(str, param[0], param[1], param[2], param[3],
+                 param[4], param[5],  param[6], param[7], param[8]);
+}
+
+// Re-defining just to make sure nothing breaks further down in this file.
+#define tparm tparm_solaris_kludge
+
+#endif
+
+
+#ifndef HAVE_FWPRINTF
+#define INTERNAL_FWPRINTF 1
+#endif
+
+#ifdef HAVE_BROKEN_FWPRINTF
+#define INTERNAL_FWPRINTF 1
+#endif
+
+#ifdef INTERNAL_FWPRINTF
+
+/**
+   Internal function for the wprintf fallbacks. USed to write the
+   specified number of spaces.
+*/
+static void pad(void (*writer)(wchar_t), int count)
+{
+
+    int i;
+    if (count < 0)
+        return;
+
+    for (i=0; i<count; i++)
+    {
+        writer(L' ');
+    }
+}
+
+/**
+   Generic formatting function. All other string formatting functions
+   are secretly a wrapper around this function. vgprintf does not
+   implement all the filters supported by printf, only those that are
+   currently used by fish. vgprintf internally uses snprintf to
+   implement the number outputs, such as %f and %x.
+
+   Currently supported functionality:
+
+   - precision specification, both through .* and .N
+   - Padding through * and N
+   - Right padding using the - prefix
+   - long versions of all filters thorugh l and ll prefix
+   - Character output using %c
+   - String output through %s
+   - Floating point number output through %f
+   - Integer output through %d, %i, %u, %o, %x and %X
+
+   For a full description on the usage of *printf, see use 'man 3 printf'.
+*/
+static int vgwprintf(void (*writer)(wchar_t),
+                     const wchar_t *filter,
+                     va_list va)
+{
+    const wchar_t *filter_org=filter;
+    int count=0;
+
+    for (; *filter; filter++)
+    {
+        if (*filter == L'%')
+        {
+            int is_long=0;
+            int width = -1;
+            filter++;
+            int loop=1;
+            int precision=-1;
+            int pad_left = 1;
+
+            if (iswdigit(*filter))
+            {
+                width=0;
+                while ((*filter >= L'0') && (*filter <= L'9'))
+                {
+                    width=10*width+(*filter++ - L'0');
+                }
+            }
+
+            while (loop)
+            {
+
+                switch (*filter)
+                {
+                    case L'l':
+                        /* Long variable */
+                        is_long++;
+                        filter++;
+                        break;
+
+                    case L'*':
+                        /* Set minimum field width */
+                        width = va_arg(va, int);
+                        filter++;
+                        break;
+
+                    case L'-':
+                        filter++;
+                        pad_left=0;
+                        break;
+
+                    case L'.':
+                        /*
+                           Set precision.
+                        */
+                        filter++;
+                        if (*filter == L'*')
+                        {
+                            precision = va_arg(va, int);
+                        }
+                        else
+                        {
+                            precision=0;
+                            while ((*filter >= L'0') && (*filter <= L'9'))
+                            {
+                                precision=10*precision+(*filter++ - L'0');
+                            }
+                        }
+                        break;
+
+                    default:
+                        loop=0;
+                        break;
+                }
+            }
+
+            switch (*filter)
+            {
+                case L'c':
+                {
+                    wchar_t c;
+
+                    if ((width >= 0) && pad_left)
+                    {
+                        pad(writer, width-1);
+                        count += maxi(width-1, 0);
+                    }
+
+                    c = is_long?va_arg(va, wint_t):btowc(va_arg(va, int));
+                    if (precision != 0)
+                        writer(c);
+
+
+                    if ((width >= 0) && !pad_left)
+                    {
+                        pad(writer, width-1);
+                        count += maxi(width-1, 0);
+                    }
+
+                    count++;
+
+                    break;
+                }
+                case L's':
+                {
+
+                    wchar_t *ss=0;
+                    wcstring wide_ss;
+                    if (is_long)
+                    {
+                        ss = va_arg(va, wchar_t *);
+                    }
+                    else
+                    {
+                        char *ns = va_arg(va, char*);
+
+                        if (ns)
+                        {
+                            wide_ss = str2wcstring(ns);
+                            ss = wide_ss.c_str();
+                        }
+                    }
+
+                    if (!ss)
+                    {
+                        return -1;
+                    }
+
+                    if ((width >= 0) && pad_left)
+                    {
+                        pad(writer, width-wcslen(ss));
+                        count += maxi(width-wcslen(ss), 0);
+                    }
+
+                    wchar_t *s=ss;
+                    int precount = count;
+
+                    while (*s)
+                    {
+                        if ((precision > 0) && (precision <= (count-precount)))
+                            break;
+
+                        writer(*(s++));
+                        count++;
+                    }
+
+                    if ((width >= 0) && !pad_left)
+                    {
+                        pad(writer, width-wcslen(ss));
+                        count += maxi(width-wcslen(ss), 0);
+                    }
+
+                    break;
+                }
+
+                case L'd':
+                case L'i':
+                case L'o':
+                case L'u':
+                case L'x':
+                case L'X':
+                {
+                    char str[33];
+                    char *pos;
+                    char format[16];
+                    int len;
+
+                    format[0]=0;
+                    strcat(format, "%");
+                    if (precision >= 0)
+                        strcat(format, ".*");
+                    switch (is_long)
+                    {
+                        case 2:
+                            strcat(format, "ll");
+                            break;
+                        case 1:
+                            strcat(format, "l");
+                            break;
+                    }
+
+                    len = strlen(format);
+                    format[len++]=(char)*filter;
+                    format[len]=0;
+
+                    switch (*filter)
+                    {
+                        case L'd':
+                        case L'i':
+                        {
+
+                            switch (is_long)
+                            {
+                                case 0:
+                                {
+                                    int d = va_arg(va, int);
+                                    if (precision >= 0)
+                                        snprintf(str, 32, format, precision, d);
+                                    else
+                                        snprintf(str, 32, format, d);
+
+                                    break;
+                                }
+
+                                case 1:
+                                {
+                                    long d = va_arg(va, long);
+                                    if (precision >= 0)
+                                        snprintf(str, 32, format, precision, d);
+                                    else
+                                        snprintf(str, 32, format, d);
+                                    break;
+                                }
+
+                                case 2:
+                                {
+                                    long long d = va_arg(va, long long);
+                                    if (precision >= 0)
+                                        snprintf(str, 32, format, precision, d);
+                                    else
+                                        snprintf(str, 32, format, d);
+                                    break;
+                                }
+
+                                default:
+                                    debug(0, L"Invalid length modifier in string %ls\n", filter_org);
+                                    return -1;
+                            }
+                            break;
+
+                        }
+
+                        case L'u':
+                        case L'o':
+                        case L'x':
+                        case L'X':
+                        {
+
+                            switch (is_long)
+                            {
+                                case 0:
+                                {
+                                    unsigned d = va_arg(va, unsigned);
+                                    if (precision >= 0)
+                                        snprintf(str, 32, format, precision, d);
+                                    else
+                                        snprintf(str, 32, format, d);
+                                    break;
+                                }
+
+                                case 1:
+                                {
+                                    unsigned long d = va_arg(va, unsigned long);
+                                    if (precision >= 0)
+                                        snprintf(str, 32, format, precision, d);
+                                    else
+                                        snprintf(str, 32, format, d);
+                                    break;
+                                }
+
+                                case 2:
+                                {
+                                    unsigned long long d = va_arg(va, unsigned long long);
+                                    if (precision >= 0)
+                                        snprintf(str, 32, format, precision, d);
+                                    else
+                                        snprintf(str, 32, format, d);
+                                    break;
+                                }
+
+                                default:
+                                    debug(0, L"Invalid length modifier in string %ls\n", filter_org);
+                                    return -1;
+                            }
+                            break;
+
+                        }
+
+                        default:
+                            debug(0, L"Invalid filter %ls in string %ls\n", *filter, filter_org);
+                            return -1;
+
+                    }
+
+                    if ((width >= 0) && pad_left)
+                    {
+                        int l = maxi(width-strlen(str), 0);
+                        pad(writer, l);
+                        count += l;
+                    }
+
+                    pos = str;
+
+                    while (*pos)
+                    {
+                        writer(*(pos++));
+                        count++;
+                    }
+
+                    if ((width >= 0) && !pad_left)
+                    {
+                        int l = maxi(width-strlen(str), 0);
+                        pad(writer, l);
+                        count += l;
+                    }
+
+                    break;
+                }
+
+                case L'f':
+                {
+                    char str[32];
+                    char *pos;
+                    double val = va_arg(va, double);
+
+                    if (precision>= 0)
+                    {
+                        if (width>= 0)
+                        {
+                            snprintf(str, 32, "%*.*f", width, precision, val);
+                        }
+                        else
+                        {
+                            snprintf(str, 32, "%.*f", precision, val);
+                        }
+                    }
+                    else
+                    {
+                        if (width>= 0)
+                        {
+                            snprintf(str, 32, "%*f", width, val);
+                        }
+                        else
+                        {
+                            snprintf(str, 32, "%f", val);
+                        }
+                    }
+
+                    pos = str;
+
+                    while (*pos)
+                    {
+                        writer(*(pos++));
+                        count++;
+                    }
+
+                    break;
+                }
+
+                case L'n':
+                {
+                    int *n = va_arg(va, int *);
+
+                    *n = count;
+                    break;
+                }
+                case L'%':
+                {
+                    writer('%');
+                    count++;
+                    break;
+                }
+                default:
+                    debug(0, L"Unknown switch %lc in string %ls\n", *filter, filter_org);
+                    return -1;
+            }
+        }
+        else
+        {
+            writer(*filter);
+            count++;
+        }
+    }
+
+    return count;
+}
+
+/**
+   Holds data for swprintf writer
+*/
+static struct
+{
+    int count;
+    int max;
+    wchar_t *pos;
+}
+sw_data;
+
+/**
+   Writers for string output
+*/
+static void sw_writer(wchar_t c)
+{
+    if (sw_data.count < sw_data.max)
+        *(sw_data.pos++)=c;
+    sw_data.count++;
+}
+
+int vswprintf(wchar_t *out, size_t n, const wchar_t *filter, va_list va)
+{
+    int written;
+
+    sw_data.pos=out;
+    sw_data.max=n;
+    sw_data.count=0;
+    written=vgwprintf(&sw_writer,
+                      filter,
+                      va);
+    if (written < n)
+    {
+        *sw_data.pos = 0;
+    }
+    else
+    {
+        written=-1;
+    }
+
+    return written;
+}
+
+int swprintf(wchar_t *out, size_t n, const wchar_t *filter, ...)
+{
+    va_list va;
+    int written;
+
+    va_start(va, filter);
+    written = vswprintf(out, n, filter, va);
+    va_end(va);
+    return written;
+}
+
+/**
+   Holds auxiliary data for fwprintf and wprintf writer
+*/
+static FILE *fw_data;
+
+static void fw_writer(wchar_t c)
+{
+    putwc(c, fw_data);
+}
+
+/*
+   Writers for file output
+*/
+int vfwprintf(FILE *f, const wchar_t *filter, va_list va)
+{
+    fw_data = f;
+    return vgwprintf(&fw_writer, filter, va);
+}
+
+int fwprintf(FILE *f, const wchar_t *filter, ...)
+{
+    va_list va;
+    int written;
+
+    va_start(va, filter);
+    written = vfwprintf(f, filter, va);
+    va_end(va);
+    return written;
+}
+
+int vwprintf(const wchar_t *filter, va_list va)
+{
+    return vfwprintf(stdout, filter, va);
+}
+
+int wprintf(const wchar_t *filter, ...)
+{
+    va_list va;
+    int written;
+
+    va_start(va, filter);
+    written=vwprintf(filter, va);
+    va_end(va);
+    return written;
+}
+
+#endif
+
+#ifndef HAVE_FGETWC
+
+wint_t fgetwc(FILE *stream)
+{
+    wchar_t res=0;
+    mbstate_t state;
+    memset(&state, '\0', sizeof(state));
+
+    while (1)
+    {
+        int b = fgetc(stream);
+        char bb;
+
+        int sz;
+
+        if (b == EOF)
+            return WEOF;
+
+        bb=b;
+
+        sz = mbrtowc(&res, &bb, 1, &state);
+
+        switch (sz)
+        {
+            case -1:
+                memset(&state, '\0', sizeof(state));
+                return WEOF;
+
+            case -2:
+                break;
+            case 0:
+                return 0;
+            default:
+                return res;
+        }
+    }
+
+}
+
+
+wint_t getwc(FILE *stream)
+{
+    return fgetwc(stream);
+}
+
+
+#endif
+
+#ifndef HAVE_FPUTWC
+
+wint_t fputwc(wchar_t wc, FILE *stream)
+{
+    int res;
+    char s[MB_CUR_MAX+1];
+    memset(s, 0, MB_CUR_MAX+1);
+    wctomb(s, wc);
+    res = fputs(s, stream);
+    return res==EOF?WEOF:wc;
+}
+
+wint_t putwc(wchar_t wc, FILE *stream)
+{
+    return fputwc(wc, stream);
+}
+
+#endif
+
+#ifndef HAVE_WCSTOK
+
+/*
+  Used by fallback wcstok. Borrowed from glibc
+*/
+static size_t fish_wcsspn(const wchar_t *wcs,
+                          const wchar_t *accept)
+{
+    register const wchar_t *p;
+    register const wchar_t *a;
+    register size_t count = 0;
+
+    for (p = wcs; *p != L'\0'; ++p)
+    {
+        for (a = accept; *a != L'\0'; ++a)
+            if (*p == *a)
+                break;
+
+        if (*a == L'\0')
+            return count;
+        else
+            ++count;
+    }
+    return count;
+}
+
+/*
+  Used by fallback wcstok. Borrowed from glibc
+*/
+static wchar_t *fish_wcspbrk(const wchar_t *wcs, const wchar_t *accept)
+{
+    while (*wcs != L'\0')
+        if (wcschr(accept, *wcs) == NULL)
+            ++wcs;
+        else
+            return (wchar_t *) wcs;
+    return NULL;
+}
+
+/*
+  Fallback wcstok implementation. Borrowed from glibc.
+*/
+wchar_t *wcstok(wchar_t *wcs, const wchar_t *delim, wchar_t **save_ptr)
+{
+    wchar_t *result;
+
+    if (wcs == NULL)
+    {
+        if (*save_ptr == NULL)
+        {
+            errno = EINVAL;
+            return NULL;
+        }
+        else
+            wcs = *save_ptr;
+    }
+
+    /* Scan leading delimiters.  */
+    wcs += fish_wcsspn(wcs, delim);
+
+    if (*wcs == L'\0')
+    {
+        *save_ptr = NULL;
+        return NULL;
+    }
+
+    /* Find the end of the token.  */
+    result = wcs;
+
+    wcs = fish_wcspbrk(result, delim);
+
+    if (wcs == NULL)
+    {
+        /* This token finishes the string.  */
+        *save_ptr = NULL;
+    }
+    else
+    {
+        /* Terminate the token and make *SAVE_PTR point past it.  */
+        *wcs = L'\0';
+        *save_ptr = wcs + 1;
+    }
+    return result;
+}
+
+#endif
+
+/* Fallback implementations of wcsdup and wcscasecmp. On systems where these are not needed (e.g. building on Linux) these should end up just being stripped, as they are static functions that are not referenced in this file.
+*/
+__attribute__((unused))
+static wchar_t *wcsdup_fallback(const wchar_t *in)
+{
+    size_t len=wcslen(in);
+    wchar_t *out = (wchar_t *)malloc(sizeof(wchar_t)*(len+1));
+    if (out == 0)
+    {
+        return 0;
+    }
+
+    memcpy(out, in, sizeof(wchar_t)*(len+1));
+    return out;
+}
+
+__attribute__((unused))
+static int wcscasecmp_fallback(const wchar_t *a, const wchar_t *b)
+{
+    if (*a == 0)
+    {
+        return (*b==0)?0:-1;
+    }
+    else if (*b == 0)
+    {
+        return 1;
+    }
+    int diff = towlower(*a)-towlower(*b);
+    if (diff != 0)
+        return diff;
+    else
+        return wcscasecmp_fallback(a+1,b+1);
+}
+
+__attribute__((unused))
+static int wcsncasecmp_fallback(const wchar_t *a, const wchar_t *b, size_t count)
+{
+    if (count == 0)
+        return 0;
+
+    if (*a == 0)
+    {
+        return (*b==0)?0:-1;
+    }
+    else if (*b == 0)
+    {
+        return 1;
+    }
+    int diff = towlower(*a)-towlower(*b);
+    if (diff != 0)
+        return diff;
+    else
+        return wcsncasecmp_fallback(a+1,b+1, count-1);
+}
+
+
+#if __APPLE__ && __DARWIN_C_LEVEL >= 200809L
+/* Note parens avoid the macro expansion */
+wchar_t *wcsdup_use_weak(const wchar_t *a)
+{
+    if (&wcsdup != NULL)
+        return (wcsdup)(a);
+    return wcsdup_fallback(a);
+}
+
+int wcscasecmp_use_weak(const wchar_t *a, const wchar_t *b)
+{
+    if (&wcscasecmp != NULL)
+        return (wcscasecmp)(a, b);
+    return wcscasecmp_fallback(a, b);
+}
+
+int wcsncasecmp_use_weak(const wchar_t *s1, const wchar_t *s2, size_t n)
+{
+    if (&wcsncasecmp != NULL)
+        return (wcsncasecmp)(s1, s2, n);
+    return wcsncasecmp_fallback(s1, s2, n);
+}
+
+#else //__APPLE__
+
+#ifndef HAVE_WCSDUP
+wchar_t *wcsdup(const wchar_t *in)
+{
+    return wcsdup_fallback(in);
+
+}
+#endif
+
+
+#ifndef HAVE_WCSCASECMP
+int wcscasecmp(const wchar_t *a, const wchar_t *b)
+{
+    return wcscasecmp_fallback(a, b);
+}
+#endif
+#endif //__APPLE__
+
+#ifndef HAVE_WCSLEN
+size_t wcslen(const wchar_t *in)
+{
+    const wchar_t *end=in;
+    while (*end)
+        end++;
+    return end-in;
+}
+#endif
+
+#ifndef HAVE_WCSNCASECMP
+int wcsncasecmp(const wchar_t *a, const wchar_t *b, size_t count)
+{
+    return wcsncasecmp_fallback(a, b, count);
+}
+#endif
+
+#ifndef HAVE_WCWIDTH
+int wcwidth(wchar_t c)
+{
+    if (c < 32)
+        return 0;
+    if (c == 127)
+        return 0;
+    return 1;
+}
+#endif
+
+#ifndef HAVE_WCSNDUP
+wchar_t *wcsndup(const wchar_t *in, size_t c)
+{
+    wchar_t *res = (wchar_t *)malloc(sizeof(wchar_t)*(c+1));
+    if (res == 0)
+    {
+        return 0;
+    }
+    wcslcpy(res, in, c+1);
+    return res;
+}
+#endif
+
+long convert_digit(wchar_t d, int base)
+{
+    long res=-1;
+    if ((d <= L'9') && (d >= L'0'))
+    {
+        res = d - L'0';
+    }
+    else if ((d <= L'z') && (d >= L'a'))
+    {
+        res = d + 10 - L'a';
+    }
+    else if ((d <= L'Z') && (d >= L'A'))
+    {
+        res = d + 10 - L'A';
+    }
+    if (res >= base)
+    {
+        res = -1;
+    }
+
+    return res;
+}
+
+#ifndef HAVE_WCSTOL
+long wcstol(const wchar_t *nptr,
+            wchar_t **endptr,
+            int base)
+{
+    long long res=0;
+    int is_set=0;
+    if (base > 36)
+    {
+        errno = EINVAL;
+        return 0;
+    }
+
+    while (1)
+    {
+        long nxt = convert_digit(*nptr, base);
+        if (endptr != 0)
+            *endptr = (wchar_t *)nptr;
+        if (nxt < 0)
+        {
+            if (!is_set)
+            {
+                errno = EINVAL;
+            }
+            return res;
+        }
+        res = (res*base)+nxt;
+        is_set = 1;
+        if (res > LONG_MAX)
+        {
+            errno = ERANGE;
+            return LONG_MAX;
+        }
+        if (res < LONG_MIN)
+        {
+            errno = ERANGE;
+            return LONG_MIN;
+        }
+        nptr++;
+    }
+}
+
+#endif
+#ifndef HAVE_WCSLCAT
+
+/*$OpenBSD: strlcat.c,v 1.11 2003/06/17 21:56:24 millert Exp $*/
+
+/*
+ * Copyright (c) 1998 Todd C. Miller <Todd.Miller@courtesan.com>
+ *
+ * Permission to use, copy, modify, and distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+size_t
+wcslcat(wchar_t *dst, const wchar_t *src, size_t siz)
+{
+
+    register wchar_t *d = dst;
+    register const wchar_t *s = src;
+    register size_t n = siz;
+    size_t dlen;
+
+    /* Find the end of dst and adjust bytes left but don't go past end */
+    while (n-- != 0 && *d != '\0')
+        d++;
+
+    dlen = d - dst;
+    n = siz - dlen;
+
+    if (n == 0)
+        return(dlen + wcslen(s));
+
+    while (*s != '\0')
+    {
+        if (n != 1)
+        {
+            *d++ = *s;
+            n--;
+        }
+        s++;
+    }
+    *d = '\0';
+
+    return(dlen + (s - src));
+    /* count does not include NUL */
+}
+
+#endif
+#ifndef HAVE_WCSLCPY
+
+/*$OpenBSD: strlcpy.c,v 1.8 2003/06/17 21:56:24 millert Exp $*/
+
+/*
+ * Copyright (c) 1998 Todd C. Miller <Todd.Miller@courtesan.com>
+ *
+ * Permission to use, copy, modify, and distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+size_t
+wcslcpy(wchar_t *dst, const wchar_t *src, size_t siz)
+{
+    register wchar_t *d = dst;
+    register const wchar_t *s = src;
+    register size_t n = siz;
+
+    /* Copy as many bytes as will fit */
+    if (n != 0 && --n != 0)
+    {
+        do
+        {
+            if ((*d++ = *s++) == 0)
+                break;
+        }
+        while (--n != 0);
+    }
+
+    /* Not enough room in dst, add NUL and traverse rest of src */
+    if (n == 0)
+    {
+        if (siz != 0)
+            *d = '\0';
+        /* NUL-terminate dst */
+        while (*s++)
+            ;
+    }
+    return(s - src - 1);
+    /* count does not include NUL */
+}
+
+#endif
+
+#ifndef HAVE_LRAND48_R
+
+int lrand48_r(struct drand48_data *buffer, long int *result)
+{
+    *result = rand_r(&buffer->seed);
+    return 0;
+}
+
+int srand48_r(long int seedval, struct drand48_data *buffer)
+{
+    buffer->seed = (unsigned int)seedval;
+    return 0;
+}
+
+#endif
+
+#ifndef HAVE_FUTIMES
+
+int futimes(int fd, const struct timeval *times)
+{
+    errno = ENOSYS;
+    return -1;
+}
+
+#endif
+
+#if HAVE_GETTEXT
+
+char * fish_gettext(const char * msgid)
+{
+    return gettext(msgid);;
+}
+
+char * fish_bindtextdomain(const char * domainname, const char * dirname)
+{
+    return bindtextdomain(domainname, dirname);
+}
+
+char * fish_textdomain(const char * domainname)
+{
+    return textdomain(domainname);
+}
+
+#else
+
+char *fish_gettext(const char * msgid)
+{
+    return (char *)msgid;
+}
+
+char *fish_bindtextdomain(const char * domainname, const char * dirname)
+{
+    return NULL;
+}
+
+char *fish_textdomain(const char * domainname)
+{
+    return NULL;
+}
+
+#endif
+
+#if HAVE_DCGETTEXT
+
+char *fish_dcgettext(const char * domainname, const char * msgid, int category)
+{
+    return dcgettext(domainname, msgid, category);
+}
+
+#else
+
+char *fish_dcgettext(const char * domainname, const char * msgid, int category)
+{
+    return (char *)msgid;
+}
+
+
+#endif
+
+#ifndef HAVE__NL_MSG_CAT_CNTR
+
+int _nl_msg_cat_cntr=0;
+
+#endif
+
+#ifndef HAVE_KILLPG
+int killpg(int pgr, int sig)
+{
+    assert(pgr > 1);
+    return kill(-pgr, sig);
+}
+#endif
+
+#ifndef HAVE_WORKING_GETOPT_LONG
+
+int getopt_long(int argc,
+                char * const argv[],
+                const char *optstring,
+                const struct option *longopts,
+                int *longindex)
+{
+    return getopt(argc, argv, optstring);
+}
+
+
+#endif
+
+#ifndef HAVE_BACKTRACE
+int backtrace(void **buffer, int size)
+{
+    return 0;
+}
+#endif
+
+#ifndef HAVE_BACKTRACE_SYMBOLS
+char ** backtrace_symbols(void *const *buffer, int size)
+{
+    return 0;
+}
+#endif
+
+#ifndef HAVE_SYSCONF
+
+long sysconf(int name)
+{
+    if (name == _SC_ARG_MAX)
+    {
+#ifdef ARG_MAX
+        return ARG_MAX;
+#endif
+    }
+
+    return -1;
+
+}
+#endif
+
+#ifndef HAVE_NAN
+double nan(char *tagp)
+{
+    return 0.0/0.0;
+}
+#endif
+
+/* Big hack to use our versions of wcswidth where we know them to be broken, like on OS X */
+#ifndef HAVE_BROKEN_WCWIDTH
+#define HAVE_BROKEN_WCWIDTH 1
+#endif
+
+#if ! HAVE_BROKEN_WCWIDTH
+
+int fish_wcwidth(wchar_t wc)
+{
+    return wcwidth(wc);
+}
+
+int fish_wcswidth(const wchar_t *str, size_t n)
+{
+    return wcswidth(str, n);
+}
+
+#else
+
+static int mk_wcwidth(wchar_t wc);
+static int mk_wcswidth(const wchar_t *pwcs, size_t n);
+
+int fish_wcwidth(wchar_t wc)
+{
+    return mk_wcwidth(wc);
+}
+
+int fish_wcswidth(const wchar_t *str, size_t n)
+{
+    return mk_wcswidth(str, n);
+}
+
+/*
+ * This is an implementation of wcwidth() and wcswidth() (defined in
+ * IEEE Std 1002.1-2001) for Unicode.
+ *
+ * http://www.opengroup.org/onlinepubs/007904975/functions/wcwidth.html
+ * http://www.opengroup.org/onlinepubs/007904975/functions/wcswidth.html
+ *
+ * In fixed-width output devices, Latin characters all occupy a single
+ * "cell" position of equal width, whereas ideographic CJK characters
+ * occupy two such cells. Interoperability between terminal-line
+ * applications and (teletype-style) character terminals using the
+ * UTF-8 encoding requires agreement on which character should advance
+ * the cursor by how many cell positions. No established formal
+ * standards exist at present on which Unicode character shall occupy
+ * how many cell positions on character terminals. These routines are
+ * a first attempt of defining such behavior based on simple rules
+ * applied to data provided by the Unicode Consortium.
+ *
+ * For some graphical characters, the Unicode standard explicitly
+ * defines a character-cell width via the definition of the East Asian
+ * FullWidth (F), Wide (W), Half-width (H), and Narrow (Na) classes.
+ * In all these cases, there is no ambiguity about which width a
+ * terminal shall use. For characters in the East Asian Ambiguous (A)
+ * class, the width choice depends purely on a preference of backward
+ * compatibility with either historic CJK or Western practice.
+ * Choosing single-width for these characters is easy to justify as
+ * the appropriate long-term solution, as the CJK practice of
+ * displaying these characters as double-width comes from historic
+ * implementation simplicity (8-bit encoded characters were displayed
+ * single-width and 16-bit ones double-width, even for Greek,
+ * Cyrillic, etc.) and not any typographic considerations.
+ *
+ * Much less clear is the choice of width for the Not East Asian
+ * (Neutral) class. Existing practice does not dictate a width for any
+ * of these characters. It would nevertheless make sense
+ * typographically to allocate two character cells to characters such
+ * as for instance EM SPACE or VOLUME INTEGRAL, which cannot be
+ * represented adequately with a single-width glyph. The following
+ * routines at present merely assign a single-cell width to all
+ * neutral characters, in the interest of simplicity. This is not
+ * entirely satisfactory and should be reconsidered before
+ * establishing a formal standard in this area. At the moment, the
+ * decision which Not East Asian (Neutral) characters should be
+ * represented by double-width glyphs cannot yet be answered by
+ * applying a simple rule from the Unicode database content. Setting
+ * up a proper standard for the behavior of UTF-8 character terminals
+ * will require a careful analysis not only of each Unicode character,
+ * but also of each presentation form, something the author of these
+ * routines has avoided to do so far.
+ *
+ * http://www.unicode.org/unicode/reports/tr11/
+ *
+ * Markus Kuhn -- 2007-05-26 (Unicode 5.0)
+ *
+ * Permission to use, copy, modify, and distribute this software
+ * for any purpose and without fee is hereby granted. The author
+ * disclaims all warranties with regard to this software.
+ *
+ * Latest version: http://www.cl.cam.ac.uk/~mgk25/ucs/wcwidth.c
+ */
+
+#include <wchar.h>
+
+struct interval
+{
+    int first;
+    int last;
+};
+
+/* auxiliary function for binary search in interval table */
+static int bisearch(wchar_t ucs, const struct interval *table, int max)
+{
+    int min = 0;
+    int mid;
+
+    if (ucs < table[0].first || ucs > table[max].last)
+        return 0;
+    while (max >= min)
+    {
+        mid = (min + max) / 2;
+        if (ucs > table[mid].last)
+            min = mid + 1;
+        else if (ucs < table[mid].first)
+            max = mid - 1;
+        else
+            return 1;
+    }
+
+    return 0;
+}
+
+
+/* The following two functions define the column width of an ISO 10646
+ * character as follows:
+ *
+ *    - The null character (U+0000) has a column width of 0.
+ *
+ *    - Other C0/C1 control characters and DEL will lead to a return
+ *      value of -1.
+ *
+ *    - Non-spacing and enclosing combining characters (general
+ *      category code Mn or Me in the Unicode database) have a
+ *      column width of 0.
+ *
+ *    - SOFT HYPHEN (U+00AD) has a column width of 1.
+ *
+ *    - Other format characters (general category code Cf in the Unicode
+ *      database) and ZERO WIDTH SPACE (U+200B) have a column width of 0.
+ *
+ *    - Hangul Jamo medial vowels and final consonants (U+1160-U+11FF)
+ *      have a column width of 0.
+ *
+ *    - Spacing characters in the East Asian Wide (W) or East Asian
+ *      Full-width (F) category as defined in Unicode Technical
+ *      Report #11 have a column width of 2.
+ *
+ *    - All remaining characters (including all printable
+ *      ISO 8859-1 and WGL4 characters, Unicode control characters,
+ *      etc.) have a column width of 1.
+ *
+ * This implementation assumes that wchar_t characters are encoded
+ * in ISO 10646.
+ */
+
+static int mk_wcwidth(wchar_t ucs)
+{
+    /* sorted list of non-overlapping intervals of non-spacing characters */
+    /* generated by "uniset +cat=Me +cat=Mn +cat=Cf -00AD +1160-11FF +200B c" */
+    static const struct interval combining[] =
+    {
+        { 0x0300, 0x036F }, { 0x0483, 0x0486 }, { 0x0488, 0x0489 },
+        { 0x0591, 0x05BD }, { 0x05BF, 0x05BF }, { 0x05C1, 0x05C2 },
+        { 0x05C4, 0x05C5 }, { 0x05C7, 0x05C7 }, { 0x0600, 0x0603 },
+        { 0x0610, 0x0615 }, { 0x064B, 0x065E }, { 0x0670, 0x0670 },
+        { 0x06D6, 0x06E4 }, { 0x06E7, 0x06E8 }, { 0x06EA, 0x06ED },
+        { 0x070F, 0x070F }, { 0x0711, 0x0711 }, { 0x0730, 0x074A },
+        { 0x07A6, 0x07B0 }, { 0x07EB, 0x07F3 }, { 0x0901, 0x0902 },
+        { 0x093C, 0x093C }, { 0x0941, 0x0948 }, { 0x094D, 0x094D },
+        { 0x0951, 0x0954 }, { 0x0962, 0x0963 }, { 0x0981, 0x0981 },
+        { 0x09BC, 0x09BC }, { 0x09C1, 0x09C4 }, { 0x09CD, 0x09CD },
+        { 0x09E2, 0x09E3 }, { 0x0A01, 0x0A02 }, { 0x0A3C, 0x0A3C },
+        { 0x0A41, 0x0A42 }, { 0x0A47, 0x0A48 }, { 0x0A4B, 0x0A4D },
+        { 0x0A70, 0x0A71 }, { 0x0A81, 0x0A82 }, { 0x0ABC, 0x0ABC },
+        { 0x0AC1, 0x0AC5 }, { 0x0AC7, 0x0AC8 }, { 0x0ACD, 0x0ACD },
+        { 0x0AE2, 0x0AE3 }, { 0x0B01, 0x0B01 }, { 0x0B3C, 0x0B3C },
+        { 0x0B3F, 0x0B3F }, { 0x0B41, 0x0B43 }, { 0x0B4D, 0x0B4D },
+        { 0x0B56, 0x0B56 }, { 0x0B82, 0x0B82 }, { 0x0BC0, 0x0BC0 },
+        { 0x0BCD, 0x0BCD }, { 0x0C3E, 0x0C40 }, { 0x0C46, 0x0C48 },
+        { 0x0C4A, 0x0C4D }, { 0x0C55, 0x0C56 }, { 0x0CBC, 0x0CBC },
+        { 0x0CBF, 0x0CBF }, { 0x0CC6, 0x0CC6 }, { 0x0CCC, 0x0CCD },
+        { 0x0CE2, 0x0CE3 }, { 0x0D41, 0x0D43 }, { 0x0D4D, 0x0D4D },
+        { 0x0DCA, 0x0DCA }, { 0x0DD2, 0x0DD4 }, { 0x0DD6, 0x0DD6 },
+        { 0x0E31, 0x0E31 }, { 0x0E34, 0x0E3A }, { 0x0E47, 0x0E4E },
+        { 0x0EB1, 0x0EB1 }, { 0x0EB4, 0x0EB9 }, { 0x0EBB, 0x0EBC },
+        { 0x0EC8, 0x0ECD }, { 0x0F18, 0x0F19 }, { 0x0F35, 0x0F35 },
+        { 0x0F37, 0x0F37 }, { 0x0F39, 0x0F39 }, { 0x0F71, 0x0F7E },
+        { 0x0F80, 0x0F84 }, { 0x0F86, 0x0F87 }, { 0x0F90, 0x0F97 },
+        { 0x0F99, 0x0FBC }, { 0x0FC6, 0x0FC6 }, { 0x102D, 0x1030 },
+        { 0x1032, 0x1032 }, { 0x1036, 0x1037 }, { 0x1039, 0x1039 },
+        { 0x1058, 0x1059 }, { 0x1160, 0x11FF }, { 0x135F, 0x135F },
+        { 0x1712, 0x1714 }, { 0x1732, 0x1734 }, { 0x1752, 0x1753 },
+        { 0x1772, 0x1773 }, { 0x17B4, 0x17B5 }, { 0x17B7, 0x17BD },
+        { 0x17C6, 0x17C6 }, { 0x17C9, 0x17D3 }, { 0x17DD, 0x17DD },
+        { 0x180B, 0x180D }, { 0x18A9, 0x18A9 }, { 0x1920, 0x1922 },
+        { 0x1927, 0x1928 }, { 0x1932, 0x1932 }, { 0x1939, 0x193B },
+        { 0x1A17, 0x1A18 }, { 0x1B00, 0x1B03 }, { 0x1B34, 0x1B34 },
+        { 0x1B36, 0x1B3A }, { 0x1B3C, 0x1B3C }, { 0x1B42, 0x1B42 },
+        { 0x1B6B, 0x1B73 }, { 0x1DC0, 0x1DCA }, { 0x1DFE, 0x1DFF },
+        { 0x200B, 0x200F }, { 0x202A, 0x202E }, { 0x2060, 0x2063 },
+        { 0x206A, 0x206F }, { 0x20D0, 0x20EF }, { 0x302A, 0x302F },
+        { 0x3099, 0x309A }, { 0xA806, 0xA806 }, { 0xA80B, 0xA80B },
+        { 0xA825, 0xA826 }, { 0xFB1E, 0xFB1E }, { 0xFE00, 0xFE0F },
+        { 0xFE20, 0xFE23 }, { 0xFEFF, 0xFEFF }, { 0xFFF9, 0xFFFB },
+        { 0x10A01, 0x10A03 }, { 0x10A05, 0x10A06 }, { 0x10A0C, 0x10A0F },
+        { 0x10A38, 0x10A3A }, { 0x10A3F, 0x10A3F }, { 0x1D167, 0x1D169 },
+        { 0x1D173, 0x1D182 }, { 0x1D185, 0x1D18B }, { 0x1D1AA, 0x1D1AD },
+        { 0x1D242, 0x1D244 }, { 0xE0001, 0xE0001 }, { 0xE0020, 0xE007F },
+        { 0xE0100, 0xE01EF }
+    };
+
+    /* test for 8-bit control characters */
+    if (ucs == 0)
+        return 0;
+    if (ucs < 32 || (ucs >= 0x7f && ucs < 0xa0))
+        return -1;
+
+    /* binary search in table of non-spacing characters */
+    if (bisearch(ucs, combining,
+                 sizeof(combining) / sizeof(struct interval) - 1))
+        return 0;
+
+    /* if we arrive here, ucs is not a combining or C0/C1 control character */
+
+    return 1 +
+           (ucs >= 0x1100 &&
+            (ucs <= 0x115f ||                    /* Hangul Jamo init. consonants */
+             ucs == 0x2329 || ucs == 0x232a ||
+             (ucs >= 0x2e80 && ucs <= 0xa4cf &&
+              ucs != 0x303f) ||                  /* CJK ... Yi */
+             (ucs >= 0xac00 && ucs <= 0xd7a3) || /* Hangul Syllables */
+             (ucs >= 0xf900 && ucs <= 0xfaff) || /* CJK Compatibility Ideographs */
+             (ucs >= 0xfe10 && ucs <= 0xfe19) || /* Vertical forms */
+             (ucs >= 0xfe30 && ucs <= 0xfe6f) || /* CJK Compatibility Forms */
+             (ucs >= 0xff00 && ucs <= 0xff60) || /* Fullwidth Forms */
+             (ucs >= 0xffe0 && ucs <= 0xffe6) ||
+             (ucs >= 0x20000 && ucs <= 0x2fffd) ||
+             (ucs >= 0x30000 && ucs <= 0x3fffd)));
+}
+
+static int mk_wcswidth(const wchar_t *pwcs, size_t n)
+{
+    int width = 0;
+    for (size_t i=0; i < n; i++)
+    {
+        if (pwcs[i] == L'\0')
+            break;
+
+        int w = mk_wcwidth(pwcs[i]);
+        if (w < 0)
+        {
+            width = -1;
+            break;
+        }
+        width += w;
+    }
+    return width;
+}
+
+#endif // HAVE_BROKEN_WCWIDTH
diff --git a/src/fallback.h b/src/fallback.h
new file mode 100644
index 00000000..7d3afa13
--- /dev/null
+++ b/src/fallback.h
@@ -0,0 +1,475 @@
+
+#ifndef FISH_FALLBACK_H
+#define FISH_FALLBACK_H
+
+#include <stdio.h>
+#include <stdint.h>
+#include <stdarg.h>
+#include <wctype.h>
+#include <wchar.h>
+#include <limits.h>
+#include <sys/time.h>
+#include <sys/types.h>
+#include <signal.h>
+
+/** fish's internal versions of wcwidth and wcswidth, which can use an internal implementation if the system one is busted. */
+int fish_wcwidth(wchar_t wc);
+int fish_wcswidth(const wchar_t *str, size_t n);
+
+#ifndef WCHAR_MAX
+/**
+   This _should_ be defined by wchar.h, but e.g. OpenBSD doesn't.
+*/
+#define WCHAR_MAX INT_MAX
+#endif
+
+/**
+   Make sure __func__ is defined to some string. In C99, this should
+   be the currently compiled function. If we aren't using C99 or
+   later, older versions of GCC had __FUNCTION__.
+*/
+#if __STDC_VERSION__ < 199901L
+# if __GNUC__ >= 2
+#  define __func__ __FUNCTION__
+# else
+#  define __func__ "<unknown>"
+# endif
+#endif
+
+/**
+   Under curses, tputs expects an int (*func)(char) as its last
+   parameter, but in ncurses, tputs expects a int (*func)(int) as its
+   last parameter. tputs_arg_t is defined to always be what tputs
+   expects. Hopefully.
+*/
+
+#ifdef NCURSES_VERSION
+typedef int tputs_arg_t;
+#else
+typedef char tputs_arg_t;
+#endif
+
+#ifndef SIGIO
+#define SIGIO SIGUSR1
+#endif
+
+#ifndef SIGWINCH
+#define SIGWINCH SIGUSR2
+#endif
+
+#ifndef HAVE_WINSIZE
+/**
+   Structure used to get the size of a terminal window
+ */
+struct winsize
+{
+    /**
+       Number of rows
+     */
+    unsigned short ws_row;
+    /**
+       Number of columns
+     */
+    unsigned short ws_col;
+}
+;
+
+#endif
+
+#ifdef TPUTS_KLUDGE
+
+/**
+   Linux on PPC seems to have a tputs implementation that sometimes
+   behaves strangely. This fallback seems to fix things.
+*/
+int tputs(const char *str, int affcnt, int (*fish_putc)(tputs_arg_t));
+
+#endif
+
+#ifdef TPARM_SOLARIS_KLUDGE
+
+/**
+   Solaris tparm has a set fixed of paramters in it's curses implementation,
+   work around this here.
+*/
+
+#define tparm tparm_solaris_kludge
+char *tparm_solaris_kludge(char *str, ...);
+
+#endif
+
+#ifndef HAVE_FWPRINTF
+
+/**
+   Print formated string. Some operating systems (Like NetBSD) do not
+   have wide string formating functions.  Therefore we implement our
+   own. Not at all complete. Supports wide and narrow characters,
+   strings and decimal numbers, position (%n), field width and
+   precision.
+*/
+int fwprintf(FILE *f, const wchar_t *format, ...);
+
+
+/**
+   Print formated string. Some operating systems (Like NetBSD) do not
+   have wide string formating functions.  Therefore we define our
+   own. Not at all complete. Supports wide and narrow characters,
+   strings and decimal numbers, position (%n), field width and
+   precision.
+*/
+int swprintf(wchar_t *str, size_t l, const wchar_t *format, ...);
+
+/**
+   Print formated string. Some operating systems (Like NetBSD) do not
+   have wide string formating functions.  Therefore we define our
+   own. Not at all complete. Supports wide and narrow characters,
+   strings and decimal numbers, position (%n), field width and
+   precision.
+*/
+int wprintf(const wchar_t *format, ...);
+
+/**
+   Print formated string. Some operating systems (Like NetBSD) do not
+   have wide string formating functions.  Therefore we define our
+   own. Not at all complete. Supports wide and narrow characters,
+   strings and decimal numbers, position (%n), field width and
+   precision.
+*/
+int vwprintf(const wchar_t *filter, va_list va);
+
+/**
+   Print formated string. Some operating systems (Like NetBSD) do not
+   have wide string formating functions.  Therefore we define our
+   own. Not at all complete. Supports wide and narrow characters,
+   strings and decimal numbers, position (%n), field width and
+   precision.
+*/
+int vfwprintf(FILE *f, const wchar_t *filter, va_list va);
+
+/**
+   Print formated string. Some operating systems (Like NetBSD) do not
+   have wide string formating functions.  Therefore we define our
+   own. Not at all complete. Supports wide and narrow characters,
+   strings and decimal numbers, position (%n), field width and
+   precision.
+*/
+int vswprintf(wchar_t *out, size_t n, const wchar_t *filter, va_list va);
+
+#endif
+
+#ifndef HAVE_FGETWC
+/**
+   Fallback implementation of fgetwc
+*/
+wint_t fgetwc(FILE *stream);
+
+/**
+   Fallback implementation of getwc
+*/
+wint_t getwc(FILE *stream);
+
+#endif
+
+#ifndef HAVE_FPUTWC
+
+/**
+   Fallback implementation of fputwc
+*/
+wint_t fputwc(wchar_t wc, FILE *stream);
+/**
+   Fallback implementation of putwc
+*/
+wint_t putwc(wchar_t wc, FILE *stream);
+
+#endif
+
+#ifndef HAVE_WCSTOK
+
+/**
+   Fallback implementation of wcstok. Uses code borrowed from glibc.
+*/
+wchar_t *wcstok(wchar_t *wcs, const wchar_t *delim, wchar_t **ptr);
+
+#endif
+
+#ifndef HAVE_WCWIDTH
+
+/**
+   Return the number of columns used by a character. This is a libc
+   function, but the prototype for this function is missing in some libc
+   implementations.
+
+   Fish has a fallback implementation in case the implementation is
+   missing altogether.  In locales without a native wcwidth, Unicode
+   is probably so broken that it isn't worth trying to implement a
+   real wcwidth. Therefore, the fallback wcwidth assumes any printing
+   character takes up one column and anything else uses 0 columns.
+*/
+int wcwidth(wchar_t c);
+
+#endif
+
+
+/** On OS X, use weak linking for wcsdup and wcscasecmp. Weak linking allows you to call the function only if it exists at runtime. You can detect it by testing the function pointer against NULL. To avoid making the callers do that, redefine wcsdup to wcsdup_use_weak, and likewise with wcscasecmp. This lets us use the same binary on SnowLeopard (10.6) and Lion+ (10.7), even though these functions only exist on 10.7+.
+
+    On other platforms, use what's detected at build time.
+*/
+#if __APPLE__ && __DARWIN_C_LEVEL >= 200809L
+wchar_t *wcsdup_use_weak(const wchar_t *);
+int wcscasecmp_use_weak(const wchar_t *, const wchar_t *);
+int wcsncasecmp_use_weak(const wchar_t *s1, const wchar_t *s2, size_t n);
+#define wcsdup(a) wcsdup_use_weak((a))
+#define wcscasecmp(a, b) wcscasecmp_use_weak((a), (b))
+#define wcsncasecmp(a, b, c) wcsncasecmp_use_weak((a), (b), (c))
+
+#else
+
+#ifndef HAVE_WCSDUP
+
+/**
+   Create a duplicate string. Wide string version of strdup. Will
+   automatically exit if out of memory.
+*/
+wchar_t *wcsdup(const wchar_t *in);
+
+#endif
+
+#ifndef HAVE_WCSCASECMP
+/**
+   Case insensitive string compare function. Wide string version of
+   strcasecmp.
+
+   This implementation of wcscasecmp does not take into account
+   esoteric locales where uppercase and lowercase do not cleanly
+   transform between each other. Hopefully this should be fine since
+   fish only uses this function with one of the strings supplied by
+   fish and guaranteed to be a sane, english word. Using wcscasecmp on
+   a user-supplied string should be considered a bug.
+*/
+int wcscasecmp(const wchar_t *a, const wchar_t *b);
+
+#endif
+#endif //__APPLE__
+
+
+#ifndef HAVE_WCSLEN
+
+/**
+   Fallback for wclsen. Returns the length of the specified string.
+*/
+size_t wcslen(const wchar_t *in);
+
+#endif
+
+
+#ifndef HAVE_WCSNCASECMP
+
+/**
+   Case insensitive string compare function. Wide string version of
+   strncasecmp.
+
+   This implementation of wcsncasecmp does not take into account
+   esoteric locales where uppercase and lowercase do not cleanly
+   transform between each other. Hopefully this should be fine since
+   fish only uses this function with one of the strings supplied by
+   fish and guaranteed to be a sane, english word. Using wcsncasecmp on
+   a user-supplied string should be considered a bug.
+*/
+int wcsncasecmp(const wchar_t *a, const wchar_t *b, size_t count);
+
+/**
+   Returns a newly allocated wide character string wich is a copy of
+   the string in, but of length c or shorter. The returned string is
+   always null terminated, and the null is not included in the string
+   length.
+*/
+
+#endif
+
+#ifndef HAVE_WCSNDUP
+
+/**
+   Fallback for wcsndup function. Returns a copy of \c in, truncated
+   to a maximum length of \c c.
+*/
+wchar_t *wcsndup(const wchar_t *in, size_t c);
+
+#endif
+
+/**
+   Converts from wide char to digit in the specified base. If d is not
+   a valid digit in the specified base, return -1. This is a helper
+   function for wcstol, but it is useful itself, so it is exported.
+*/
+long convert_digit(wchar_t d, int base);
+
+#ifndef HAVE_WCSTOL
+
+/**
+   Fallback implementation. Convert a wide character string to a
+   number in the specified base. This functions is the wide character
+   string equivalent of strtol. For bases of 10 or lower, 0..9 are
+   used to represent numbers. For bases below 36, a-z and A-Z are used
+   to represent numbers higher than 9. Higher bases than 36 are not
+   supported.
+*/
+long wcstol(const wchar_t *nptr,
+            wchar_t **endptr,
+            int base);
+
+#endif
+#ifndef HAVE_WCSLCAT
+
+/**
+   Appends src to string dst of size siz (unlike wcsncat, siz is the
+   full size of dst, not space left).  At most siz-1 characters will be
+   copied.  Always NUL terminates (unless siz <= wcslen(dst)).  Returns
+   wcslen(src) + MIN(siz, wcslen(initial dst)).  If retval >= siz,
+   truncation occurred.
+
+   This is the OpenBSD strlcat function, modified for wide characters,
+   and renamed to reflect this change.
+
+*/
+size_t wcslcat(wchar_t *dst, const wchar_t *src, size_t siz);
+
+#endif
+#ifndef HAVE_WCSLCPY
+
+/**
+   Copy src to string dst of size siz.  At most siz-1 characters will
+   be copied.  Always NUL terminates (unless siz == 0).  Returns
+   wcslen(src); if retval >= siz, truncation occurred.
+
+   This is the OpenBSD strlcpy function, modified for wide characters,
+   and renamed to reflect this change.
+*/
+size_t wcslcpy(wchar_t *dst, const wchar_t *src, size_t siz);
+
+#endif
+
+#ifndef HAVE_LRAND48_R
+
+/**
+   Datastructure for the lrand48_r fallback implementation.
+*/
+struct drand48_data
+{
+    /**
+       Seed value
+    */
+    unsigned int seed;
+}
+;
+
+/**
+   Fallback implementation of lrand48_r. Internally uses rand_r, so it is pretty weak.
+*/
+int lrand48_r(struct drand48_data *buffer, long int *result);
+
+/**
+   Fallback implementation of srand48_r, the seed function for lrand48_r.
+*/
+int srand48_r(long int seedval, struct drand48_data *buffer);
+
+#endif
+
+#ifndef HAVE_FUTIMES
+
+int futimes(int fd, const struct timeval *times);
+
+#endif
+
+/* autoconf may fail to detect gettext (645), so don't define a function call gettext or we'll get build errors */
+
+/** Cover for gettext() */
+char * fish_gettext(const char * msgid);
+
+/** Cover for bindtextdomain() */
+char * fish_bindtextdomain(const char * domainname, const char * dirname);
+
+/** Cover for textdomain() */
+char * fish_textdomain(const char * domainname);
+
+/* Cover for dcgettext */
+char * fish_dcgettext(const char * domainname, const char * msgid, int category);
+
+#ifndef HAVE__NL_MSG_CAT_CNTR
+
+/**
+   Some gettext implementation use have this variable, and by
+   increasing it, one can tell the system that the translations need
+   to be reloaded.
+*/
+extern int _nl_msg_cat_cntr;
+
+#endif
+
+
+#ifndef HAVE_KILLPG
+/**
+   Send specified signal to specified process group.
+ */
+int killpg(int pgr, int sig);
+#endif
+
+
+#ifndef HAVE_WORKING_GETOPT_LONG
+
+/**
+   Struct describing a long getopt option
+ */
+struct option
+{
+    /**
+       Name of option
+     */
+    const char *name;
+    /**
+       Flag
+     */
+    int has_arg;
+    /**
+       Flag
+     */
+    int *flag;
+    /**
+       Return value
+     */
+    int val;
+}
+;
+
+#ifndef no_argument
+#define  no_argument 0
+#endif
+
+#ifndef required_argument
+#define  required_argument 1
+#endif
+
+#ifndef optional_argument
+#define  optional_argument 2
+#endif
+
+int getopt_long(int argc,
+                char * const argv[],
+                const char *optstring,
+                const struct option *longopts,
+                int *longindex);
+
+#endif
+
+#ifndef HAVE_SYSCONF
+
+#define _SC_ARG_MAX 1
+long sysconf(int name);
+
+#endif
+
+#ifndef HAVE_NAN
+double nan(char *tagp);
+#endif
+
+
+#endif
diff --git a/src/fish.cpp b/src/fish.cpp
new file mode 100644
index 00000000..a50e0886
--- /dev/null
+++ b/src/fish.cpp
@@ -0,0 +1,647 @@
+/*
+Copyright (C) 2005-2008 Axel Liljencrantz
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License version 2 as
+published by the Free Software Foundation.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA
+*/
+
+
+/** \file fish.c
+	The main loop of <tt>fish</tt>.
+*/
+
+#include "config.h"
+
+#include <limits.h>
+#include <stddef.h>
+#include <stdint.h>
+#include <sys/stat.h>
+#include <string>
+#include <vector>
+#include <stdlib.h>
+#include <stdio.h>
+#include <wchar.h>
+#include <string.h>
+#include <unistd.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <sys/socket.h> // IWYU pragma: keep - suggests internal header
+#include <sys/un.h>
+#include <pwd.h>
+
+#ifdef HAVE_GETOPT_H
+#include <getopt.h>
+#endif
+
+#include <locale.h>
+
+#include "fallback.h" // IWYU pragma: keep
+
+#include "common.h"
+#include "reader.h"
+#include "builtin.h"
+#include "function.h"
+#include "wutil.h"
+#include "env.h"
+#include "proc.h"
+#include "parser.h"
+#include "expand.h"
+#include "intern.h"
+#include "event.h"
+#include "history.h"
+#include "path.h"
+#include "input.h"
+#include "io.h"
+#include "fish_version.h"
+
+/* PATH_MAX may not exist */
+#ifndef PATH_MAX
+#define PATH_MAX 1024
+#endif
+
+/**
+   The string describing the single-character options accepted by the main fish binary
+*/
+#define GETOPT_STRING "+hilnvc:p:d:"
+
+/* If we are doing profiling, the filename to output to */
+static const char *s_profiling_output_filename = NULL;
+
+static bool has_suffix(const std::string &path, const char *suffix, bool ignore_case)
+{
+    size_t pathlen = path.size(), suffixlen = strlen(suffix);
+    return pathlen >= suffixlen && !(ignore_case ? strcasecmp : strcmp)(path.c_str() + pathlen - suffixlen, suffix);
+}
+
+/* Modifies the given path by calling realpath. Returns true if realpath succeeded, false otherwise */
+static bool get_realpath(std::string &path)
+{
+    char buff[PATH_MAX], *ptr;
+    if ((ptr = realpath(path.c_str(), buff)))
+    {
+        path = ptr;
+    }
+    return ptr != NULL;
+}
+
+/* OS X function for getting the executable path */
+extern "C" {
+    int _NSGetExecutablePath(char* buf, uint32_t* bufsize);
+}
+
+/* Return the path to the current executable. This needs to be realpath'd. */
+static std::string get_executable_path(const char *argv0)
+{
+    char buff[PATH_MAX];
+#if __APPLE__
+    {
+        /* Returns 0 on success, -1 if the buffer is too small */
+        uint32_t buffSize = sizeof buff;
+        if (0 == _NSGetExecutablePath(buff, &buffSize))
+            return std::string(buff);
+
+        /* Loop until we're big enough */
+        char *mbuff = (char *)malloc(buffSize);
+        while (0 > _NSGetExecutablePath(mbuff, &buffSize))
+            mbuff = (char *)realloc(mbuff, buffSize);
+
+        /* Return the string */
+        std::string result = mbuff;
+        free(mbuff);
+        return result;
+    }
+#endif
+    {
+        /* On other Unixes, try /proc directory. This might be worth breaking out into macros. */
+        if (0 < readlink("/proc/self/exe", buff, sizeof buff) || // Linux
+                0 < readlink("/proc/curproc/file", buff, sizeof buff) || // BSD
+                0 < readlink("/proc/self/path/a.out", buff, sizeof buff)) // Solaris
+        {
+            return std::string(buff);
+        }
+    }
+
+    /* Just return argv0, which probably won't work (i.e. it's not an absolute path or a path relative to the working directory, but instead something the caller found via $PATH). We'll eventually fall back to the compile time paths. */
+    return std::string(argv0 ? argv0 : "");
+}
+
+
+static struct config_paths_t determine_config_directory_paths(const char *argv0)
+{
+    struct config_paths_t paths;
+    bool done = false;
+    std::string exec_path = get_executable_path(argv0);
+    if (get_realpath(exec_path))
+    {
+#if __APPLE__
+
+        /* On OS X, maybe we're an app bundle, and should use the bundle's files. Since we don't link CF, use this lame approach to test it: see if the resolved path ends with /Contents/MacOS/fish, case insensitive since HFS+ usually is.
+         */
+        if (! done)
+        {
+            const char *suffix = "/Contents/MacOS/fish";
+            const size_t suffixlen = strlen(suffix);
+            if (has_suffix(exec_path, suffix, true))
+            {
+                /* Looks like we're a bundle. Cut the string at the / prefixing /Contents... and then the rest */
+                wcstring wide_resolved_path = str2wcstring(exec_path);
+                wide_resolved_path.resize(exec_path.size() - suffixlen);
+                wide_resolved_path.append(L"/Contents/Resources/");
+
+                /* Append share, etc, doc */
+                paths.data = wide_resolved_path + L"share/fish";
+                paths.sysconf = wide_resolved_path + L"etc/fish";
+                paths.doc = wide_resolved_path + L"doc/fish";
+
+                /* But the bin_dir is the resolved_path, minus fish (aka the MacOS directory) */
+                paths.bin = str2wcstring(exec_path);
+                paths.bin.resize(paths.bin.size() - strlen("/fish"));
+
+                done = true;
+            }
+        }
+#endif
+
+        if (! done)
+        {
+            /* The next check is that we are in a reloctable directory tree like this:
+                 bin/fish
+                 etc/fish
+                 share/fish
+
+                 Check it!
+            */
+            const char *suffix = "/bin/fish";
+            if (has_suffix(exec_path, suffix, false))
+            {
+                wcstring base_path = str2wcstring(exec_path);
+                base_path.resize(base_path.size() - strlen(suffix));
+
+                paths.data = base_path + L"/share/fish";
+                paths.sysconf = base_path + L"/etc/fish";
+                paths.doc = base_path + L"/share/doc/fish";
+                paths.bin = base_path + L"/bin";
+
+                /* Check only that the data and sysconf directories exist. Handle the doc directories separately */
+                struct stat buf;
+                if (0 == wstat(paths.data, &buf) && 0 == wstat(paths.sysconf, &buf))
+                {
+                    /* The docs dir may not exist; in that case fall back to the compiled in path */
+                    if (0 != wstat(paths.doc, &buf))
+                    {
+                        paths.doc = L"" DOCDIR;
+                    }
+                    done = true;
+                }
+            }
+        }
+    }
+
+    if (! done)
+    {
+        /* Fall back to what got compiled in. */
+        paths.data = L"" DATADIR "/fish";
+        paths.sysconf = L"" SYSCONFDIR "/fish";
+        paths.doc = L"" DOCDIR;
+        paths.bin = L"" BINDIR;
+
+        done = true;
+    }
+
+    return paths;
+}
+
+/* Source the file config.fish in the given directory */
+static void source_config_in_directory(const wcstring &dir)
+{
+    /* We want to execute a command like 'builtin source dir/config.fish 2>/dev/null' */
+    const wcstring escaped_dir = escape_string(dir, ESCAPE_ALL);
+    const wcstring cmd = L"builtin source " + escaped_dir + L"/config.fish 2>/dev/null";
+    parser_t &parser = parser_t::principal_parser();
+    parser.set_is_within_fish_initialization(true);
+    parser.eval(cmd, io_chain_t(), TOP);
+    parser.set_is_within_fish_initialization(false);
+}
+
+static int try_connect_socket(std::string &name)
+{
+    int s, r, ret = -1;
+
+    /** Connect to a DGRAM socket rather than the expected STREAM.
+      This avoids any notification to a remote socket that we have connected,
+      preventing any surprising behaviour.
+      If the connection fails with EPROTOTYPE, the connection is probably a
+      STREAM; if it succeeds or fails any other way, there is no cause for
+      alarm.
+      With thanks to Andrew Lutomirski <github.com/amluto>
+    */
+
+    if ((s = socket(AF_UNIX, SOCK_DGRAM, 0)) == -1)
+    {
+        wperror(L"socket");
+        return -1;
+    }
+
+    debug(3, L"Connect to socket %s at fd %d", name.c_str(), s);
+
+    struct sockaddr_un local = {};
+    local.sun_family = AF_UNIX;
+    strncpy(local.sun_path, name.c_str(), (sizeof local.sun_path) - 1);
+
+    r = connect(s, (struct sockaddr *)&local, sizeof local);
+
+    if (r == -1 && errno == EPROTOTYPE)
+    {
+        ret = 0;
+    }
+
+    close(s);
+    return ret;
+}
+
+/**
+   Check for a running fishd from old versions and warn about not being able
+   to share variables.
+   https://github.com/fish-shell/fish-shell/issues/1730
+*/
+static void check_running_fishd()
+{
+    /* There are two paths to check:
+       $FISHD_SOCKET_DIR/fishd.socket.$USER or /tmp/fishd.socket.$USER
+         - referred to as the "old socket"
+       $XDG_RUNTIME_DIR/fishd.socket or /tmp/fish.$USER/fishd.socket
+         - referred to as the "new socket"
+       All existing versions of fish attempt to create the old socket, but
+       failure in newer versions is not treated as critical, so both need
+       to be checked. */
+    const char *uname = getenv("USER");
+    if (uname == NULL)
+    {
+        const struct passwd *pw = getpwuid(getuid());
+        uname = pw->pw_name;
+    }
+
+    const char *dir_old_socket = getenv("FISHD_SOCKET_DIR");
+    std::string path_old_socket;
+
+    if (dir_old_socket == NULL)
+    {
+        path_old_socket = "/tmp/";
+    }
+    else
+    {
+        path_old_socket.append(dir_old_socket);
+    }
+
+    path_old_socket.append("fishd.socket.");
+    path_old_socket.append(uname);
+
+    const char *dir_new_socket = getenv("XDG_RUNTIME_DIR");
+    std::string path_new_socket;
+    if (dir_new_socket == NULL)
+    {
+        path_new_socket = "/tmp/fish.";
+        path_new_socket.append(uname);
+        path_new_socket.push_back('/');
+    }
+    else
+    {
+        path_new_socket.append(dir_new_socket);
+    }
+
+    path_new_socket.append("fishd.socket");
+
+    if (try_connect_socket(path_old_socket) == 0 || try_connect_socket(path_new_socket) == 0)
+    {
+        debug(1, _(L"Old versions of fish appear to be running. You will not be able to share variable values between old and new fish sessions. For best results, restart all running instances of fish."));
+    }
+
+}
+
+/**
+   Parse init files. exec_path is the path of fish executable as determined by argv[0].
+*/
+static int read_init(const struct config_paths_t &paths)
+{
+    source_config_in_directory(paths.data);
+    source_config_in_directory(paths.sysconf);
+
+    /* We need to get the configuration directory before we can source the user configuration file. If path_get_config returns false then we have no configuration directory and no custom config to load. */
+    wcstring config_dir;
+    if (path_get_config(config_dir))
+    {
+        source_config_in_directory(config_dir);
+    }
+
+    return 1;
+}
+
+/**
+  Parse the argument list, return the index of the first non-switch
+  arguments.
+ */
+static int fish_parse_opt(int argc, char **argv, std::vector<std::string> *out_cmds)
+{
+    int my_optind;
+    int force_interactive=0;
+    bool has_cmd = false;
+
+    while (1)
+    {
+        static struct option
+                long_options[] =
+        {
+            { "command", required_argument, 0, 'c' },
+            { "debug-level", required_argument, 0, 'd' },
+            { "interactive", no_argument, 0, 'i' } ,
+            { "login", no_argument, 0, 'l' },
+            { "no-execute", no_argument, 0, 'n' },
+            { "profile", required_argument, 0, 'p' },
+            { "help", no_argument, 0, 'h' },
+            { "version", no_argument, 0, 'v' },
+            { 0, 0, 0, 0 }
+        }
+        ;
+
+        int opt_index = 0;
+
+        int opt = getopt_long(argc,
+                              argv,
+                              GETOPT_STRING,
+                              long_options,
+                              &opt_index);
+
+        if (opt == -1)
+            break;
+
+        switch (opt)
+        {
+            case 0:
+            {
+                break;
+            }
+
+            case 'c':
+            {
+                out_cmds->push_back(optarg ? optarg : "");
+                has_cmd = true;
+                is_interactive_session = 0;
+                break;
+            }
+
+            case 'd':
+            {
+                char *end;
+                long tmp;
+
+                errno = 0;
+                tmp = strtol(optarg, &end, 10);
+
+                if (tmp >= 0 && tmp <=10 && !*end && !errno)
+                {
+                    debug_level = (int)tmp;
+                }
+                else
+                {
+                    debug(0, _(L"Invalid value '%s' for debug level switch"), optarg);
+                    exit_without_destructors(1);
+                }
+                break;
+            }
+
+            case 'h':
+            {
+                out_cmds->push_back("__fish_print_help fish");
+                has_cmd = true;
+                break;
+            }
+
+            case 'i':
+            {
+                force_interactive = 1;
+                break;
+            }
+
+            case 'l':
+            {
+                is_login=1;
+                break;
+            }
+
+            case 'n':
+            {
+                no_exec=1;
+                break;
+            }
+
+            case 'p':
+            {
+                s_profiling_output_filename = optarg;
+                g_profiling_active = true;
+                break;
+            }
+
+            case 'v':
+            {
+                fwprintf(stderr,
+                         _(L"%s, version %s\n"),
+                         PACKAGE_NAME,
+                         get_fish_version());
+                exit_without_destructors(0);
+            }
+
+            case '?':
+            {
+                exit_without_destructors(1);
+            }
+
+        }
+    }
+
+    my_optind = optind;
+
+    is_login |= (strcmp(argv[0], "-fish") == 0);
+
+    /* We are an interactive session if we are either forced, or have not been given an explicit command to execute and stdin is a tty. */
+    if (force_interactive)
+    {
+        is_interactive_session = true;
+    }
+    else if (is_interactive_session)
+    {
+        is_interactive_session = ! has_cmd && (my_optind == argc) && isatty(STDIN_FILENO);
+    }
+
+    return my_optind;
+}
+
+extern int g_fork_count;
+int main(int argc, char **argv)
+{
+    int res=1;
+    int my_optind=0;
+
+    set_main_thread();
+    setup_fork_guards();
+
+    wsetlocale(LC_ALL, L"");
+    is_interactive_session=1;
+    program_name=L"fish";
+
+    //struct stat tmp;
+    //stat("----------FISH_HIT_MAIN----------", &tmp);
+
+    std::vector<std::string> cmds;
+    my_optind = fish_parse_opt(argc, argv, &cmds);
+
+    /*
+      No-exec is prohibited when in interactive mode
+    */
+    if (is_interactive_session && no_exec)
+    {
+        debug(1, _(L"Can not use the no-execute mode when running an interactive session"));
+        no_exec = 0;
+    }
+
+    /* Only save (and therefore restore) the fg process group if we are interactive. See #197, #1002 */
+    if (is_interactive_session)
+    {
+        save_term_foreground_process_group();
+    }
+
+    const struct config_paths_t paths = determine_config_directory_paths(argv[0]);
+
+    proc_init();
+    event_init();
+    wutil_init();
+    builtin_init();
+    function_init();
+    env_init(&paths);
+    reader_init();
+    history_init();
+    /* For setcolor to support term256 in config.fish (#1022) */
+    update_fish_color_support();
+
+    parser_t &parser = parser_t::principal_parser();
+
+    if (g_log_forks)
+        printf("%d: g_fork_count: %d\n", __LINE__, g_fork_count);
+
+    const io_chain_t empty_ios;
+    if (read_init(paths))
+    {
+        /* Stop the exit status of any initialization commands (#635) */
+        proc_set_last_status(STATUS_BUILTIN_OK);
+
+        /* Run the commands specified as arguments, if any */
+        if (! cmds.empty())
+        {
+            /* Do something nasty to support OpenSUSE assuming we're bash. This may modify cmds. */
+            if (is_login)
+            {
+                fish_xdm_login_hack_hack_hack_hack(&cmds, argc - my_optind, argv + my_optind);
+            }
+            for (size_t i=0; i < cmds.size(); i++)
+            {
+                const wcstring cmd_wcs = str2wcstring(cmds.at(i));
+                res = parser.eval(cmd_wcs, empty_ios, TOP);
+            }
+            reader_exit(0, 0);
+        }
+        else
+        {
+            if (my_optind == argc)
+            {
+                // Interactive mode
+                check_running_fishd();
+                res = reader_read(STDIN_FILENO, empty_ios);
+            }
+            else
+            {
+                char **ptr;
+                char *file = *(argv+(my_optind++));
+                int i;
+                int fd;
+
+
+                if ((fd = open(file, O_RDONLY)) == -1)
+                {
+                    wperror(L"open");
+                    return 1;
+                }
+
+                // OK to not do this atomically since we cannot have gone multithreaded yet
+                set_cloexec(fd);
+
+                if (*(argv+my_optind))
+                {
+                    wcstring sb;
+                    for (i=1,ptr = argv+my_optind; *ptr; i++, ptr++)
+                    {
+                        if (i != 1)
+                            sb.append(ARRAY_SEP_STR);
+                        sb.append(str2wcstring(*ptr));
+                    }
+
+                    env_set(L"argv", sb.c_str(), 0);
+                }
+
+                const wcstring rel_filename = str2wcstring(file);
+                const wchar_t *abs_filename = wrealpath(rel_filename, NULL);
+
+                if (!abs_filename)
+                {
+                    abs_filename = wcsdup(rel_filename.c_str());
+                }
+
+                reader_push_current_filename(intern(abs_filename));
+                free((void *)abs_filename);
+
+                res = reader_read(fd, empty_ios);
+
+                if (res)
+                {
+                    debug(1,
+                          _(L"Error while reading file %ls\n"),
+                          reader_current_filename()?reader_current_filename(): _(L"Standard input"));
+                }
+                reader_pop_current_filename();
+            }
+        }
+    }
+
+    int exit_status = res ? STATUS_UNKNOWN_COMMAND : proc_get_last_status();
+
+    proc_fire_event(L"PROCESS_EXIT", EVENT_EXIT, getpid(), exit_status);
+
+    restore_term_mode();
+    restore_term_foreground_process_group();
+
+    if (g_profiling_active)
+    {
+        parser.emit_profiling(s_profiling_output_filename);
+    }
+
+    history_destroy();
+    proc_destroy();
+    builtin_destroy();
+    reader_destroy();
+    wutil_destroy();
+    event_destroy();
+
+    if (g_log_forks)
+        printf("%d: g_fork_count: %d\n", __LINE__, g_fork_count);
+
+    exit_without_destructors(exit_status);
+    return EXIT_FAILURE; //above line should always exit
+}
diff --git a/src/fish_indent.cpp b/src/fish_indent.cpp
new file mode 100644
index 00000000..59efbd1e
--- /dev/null
+++ b/src/fish_indent.cpp
@@ -0,0 +1,396 @@
+/*
+Copyright (C) 2014 ridiculous_fish
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License version 2 as
+published by the Free Software Foundation.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA
+*/
+
+/** \file fish_indent.cpp
+  The fish_indent proegram.
+*/
+
+#include "config.h"
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <wchar.h>
+#include <vector>
+#ifdef HAVE_GETOPT_H
+#include <getopt.h>
+#endif
+#include <assert.h>
+#include <locale.h>
+#include <stddef.h>
+#include <string>
+
+#include "color.h"
+#include "highlight.h"
+#include "parse_constants.h"
+#include "wutil.h"
+#include "common.h"
+#include "output.h"
+#include "env.h"
+#include "input.h"
+#include "parse_tree.h"
+#include "print_help.h"
+#include "fish_version.h"
+
+#define SPACES_PER_INDENT 4
+
+/* An indent_t represents an abstract indent depth. 2 means we are in a doubly-nested block, etc. */
+typedef unsigned int indent_t;
+
+/* Read the entire contents of a file into the specified string */
+static wcstring read_file(FILE *f)
+{
+    wcstring result;
+    while (1)
+    {
+        wint_t c = fgetwc(f);
+        if (c == WEOF)
+        {
+            if (ferror(f))
+            {
+                wperror(L"fgetwc");
+                exit(1);
+            }
+            break;
+        }
+        result.push_back((wchar_t)c);
+    }
+    return result;
+}
+
+/* Append whitespace as necessary. If we have a newline, append the appropriate indent. Otherwise, append a space. */
+static void append_whitespace(indent_t node_indent, bool do_indent, bool has_new_line, wcstring *out_result)
+{
+    if (! has_new_line)
+    {
+        out_result->push_back(L' ');
+    }
+    else if (do_indent)
+    {
+        out_result->append(node_indent * SPACES_PER_INDENT, L' ');
+    }
+}
+
+static void prettify_node_recursive(const wcstring &source, const parse_node_tree_t &tree, node_offset_t node_idx, indent_t node_indent, parse_token_type_t parent_type, bool *has_new_line, wcstring *out_result, bool do_indent)
+{
+    const parse_node_t &node = tree.at(node_idx);
+    const parse_token_type_t node_type = node.type;
+
+    /* Increment the indent if we are either a root job_list, or root case_item_list, or in an if or while header (#1665) */
+    const bool is_root_job_list = (node_type == symbol_job_list && parent_type != symbol_job_list);
+    const bool is_root_case_item_list = (node_type == symbol_case_item_list && parent_type != symbol_case_item_list);
+    const bool is_if_while_header = (node_type == symbol_job && (parent_type == symbol_if_clause || parent_type == symbol_while_header));
+    if (is_root_job_list || is_root_case_item_list || is_if_while_header)
+    {
+        node_indent += 1;
+    }
+
+    /* Handle comments, which come before the text */
+    if (node.has_comments())
+    {
+        const parse_node_tree_t::parse_node_list_t comment_nodes = tree.comment_nodes_for_node(node);
+        for (size_t i=0; i < comment_nodes.size(); i++)
+        {
+            const parse_node_t &comment_node = *comment_nodes.at(i);
+            append_whitespace(node_indent, do_indent, *has_new_line, out_result);
+            out_result->append(source, comment_node.source_start, comment_node.source_length);
+        }
+    }
+
+    if (node_type == parse_token_type_end)
+    {
+        /* Newline */
+        out_result->push_back(L'\n');
+        *has_new_line = true;
+    }
+    else if ((node_type >= FIRST_PARSE_TOKEN_TYPE && node_type <= LAST_PARSE_TOKEN_TYPE) || node_type == parse_special_type_parse_error)
+    {
+        if (node.has_source())
+        {
+            /* Some type representing a particular token */
+            append_whitespace(node_indent, do_indent, *has_new_line, out_result);
+            out_result->append(source, node.source_start, node.source_length);
+            *has_new_line = false;
+        }
+    }
+
+    /* Recurse to all our children */
+    for (node_offset_t idx = 0; idx < node.child_count; idx++)
+    {
+        /* Note we pass our type to our child, which becomes its parent node type */
+        prettify_node_recursive(source, tree, node.child_start + idx, node_indent, node_type, has_new_line, out_result, do_indent);
+    }
+}
+
+/* Entry point for prettification. */
+static wcstring prettify(const wcstring &src, bool do_indent)
+{
+    parse_node_tree_t tree;
+    if (! parse_tree_from_string(src, parse_flag_continue_after_error | parse_flag_include_comments | parse_flag_leave_unterminated | parse_flag_show_blank_lines, &tree, NULL /* errors */))
+    {
+        /* We return the initial string on failure */
+        return src;
+    }
+
+    /* We may have a forest of disconnected trees on a parse failure. We have to handle all nodes that have no parent, and all parse errors. */
+    bool has_new_line = true;
+    wcstring result;
+    for (node_offset_t i=0; i < tree.size(); i++)
+    {
+        const parse_node_t &node = tree.at(i);
+        if (node.parent == NODE_OFFSET_INVALID || node.type == parse_special_type_parse_error)
+        {
+            /* A root node */
+            prettify_node_recursive(src, tree, i, 0, symbol_job_list, &has_new_line, &result, do_indent);
+        }
+    }
+    return result;
+}
+
+
+// Helper for output_set_writer
+static std::string output_receiver;
+static int write_to_output_receiver(char c)
+{
+    output_receiver.push_back(c);
+    return 0;
+}
+
+/* Given a string and list of colors of the same size, return the string with ANSI escape sequences representing the colors. */
+static std::string ansi_colorize(const wcstring &text, const std::vector<highlight_spec_t> &colors)
+{
+    assert(colors.size() == text.size());
+    assert(output_receiver.empty());
+
+    int (*saved)(char) = output_get_writer();
+    output_set_writer(write_to_output_receiver);
+
+    highlight_spec_t last_color = highlight_spec_normal;
+    for (size_t i=0; i < text.size(); i++)
+    {
+        highlight_spec_t color = colors.at(i);
+        if (color != last_color)
+        {
+            set_color(highlight_get_color(color, false), rgb_color_t::normal());
+            last_color = color;
+        }
+        writech(text.at(i));
+    }
+
+    output_set_writer(saved);
+    std::string result;
+    result.swap(output_receiver);
+    return result;
+}
+
+/* Given a string and list of colors of the same size, return the string with HTML span elements for the various colors. */
+static const wchar_t *html_class_name_for_color(highlight_spec_t spec)
+{
+    #define P(x) L"fish_color_"  #x
+    switch (spec & HIGHLIGHT_SPEC_PRIMARY_MASK)
+    {
+        case highlight_spec_normal: return P(normal);
+        case highlight_spec_error: return P(error);
+        case highlight_spec_command: return P(command);
+        case highlight_spec_statement_terminator: return P(statement_terminator);
+        case highlight_spec_param: return P(param);
+        case highlight_spec_comment: return P(comment);
+        case highlight_spec_match: return P(match);
+        case highlight_spec_search_match: return P(search_match);
+        case highlight_spec_operator: return P(operator);
+        case highlight_spec_escape: return P(escape);
+        case highlight_spec_quote: return P(quote);
+        case highlight_spec_redirection: return P(redirection);
+        case highlight_spec_autosuggestion: return P(autosuggestion);
+        case highlight_spec_selection: return P(selection);
+
+        default: return P(other);
+    }
+}
+
+static std::string html_colorize(const wcstring &text, const std::vector<highlight_spec_t> &colors)
+{
+    if (text.empty())
+    {
+        return "";
+    }
+
+    assert(colors.size() == text.size());
+    wcstring html = L"<pre><code>";
+    highlight_spec_t last_color = highlight_spec_normal;
+    for (size_t i=0; i < text.size(); i++)
+    {
+        /* Handle colors */
+        highlight_spec_t color = colors.at(i);
+        if (i > 0 && color != last_color)
+        {
+            html.append(L"</span>");
+        }
+        if (i == 0 || color != last_color)
+        {
+            append_format(html, L"<span class=\"%ls\">", html_class_name_for_color(color));
+        }
+        last_color = color;
+
+        /* Handle text */
+        wchar_t wc = text.at(i);
+        switch (wc)
+        {
+            case L'&':
+                html.append(L"&amp;");
+                break;
+            case L'\'':
+                html.append(L"&apos;");
+                break;
+            case L'"':
+                html.append(L"&quot;");
+                break;
+            case L'<':
+                html.append(L"&lt;");
+                break;
+            case L'>':
+                html.append(L"&gt;");
+                break;
+            default:
+                html.push_back(wc);
+                break;
+        }
+    }
+    html.append(L"</span></code></pre>");
+    return wcs2string(html);
+}
+
+static std::string no_colorize(const wcstring &text)
+{
+    return wcs2string(text);
+}
+
+int main(int argc, char *argv[])
+{
+    set_main_thread();
+    setup_fork_guards();
+
+    wsetlocale(LC_ALL, L"");
+    program_name=L"fish_indent";
+
+    env_init();
+    input_init();
+
+    /* Types of output we support */
+    enum
+    {
+        output_type_plain_text,
+        output_type_ansi,
+        output_type_html
+    } output_type = output_type_plain_text;
+
+    /* Whether to indent (true) or just reformat to one job per line (false) */
+    bool do_indent = true;
+
+    while (1)
+    {
+        const struct option long_options[] =
+        {
+            { "no-indent", no_argument, 0, 'i' },
+            { "help", no_argument, 0, 'h' },
+            { "version", no_argument, 0, 'v' },
+            { "html", no_argument, 0, 1 },
+            { "ansi", no_argument, 0, 2 },
+            { 0, 0, 0, 0 }
+        };
+
+        int opt_index = 0;
+        int opt = getopt_long(argc, argv, "hvi", long_options, &opt_index);
+        if (opt == -1)
+            break;
+
+        switch (opt)
+        {
+            case 0:
+            {
+                break;
+            }
+
+            case 'h':
+            {
+                print_help("fish_indent", 1);
+                exit(0);
+                assert(0 && "Unreachable code reached");
+                break;
+            }
+
+            case 'v':
+            {
+                fwprintf(stderr, _(L"%ls, version %s\n"), program_name, get_fish_version());
+                exit(0);
+                assert(0 && "Unreachable code reached");
+                break;
+            }
+
+            case 'i':
+            {
+                do_indent = false;
+                break;
+            }
+
+            case 1:
+            {
+                output_type = output_type_html;
+                break;
+            }
+
+            case 2:
+            {
+                output_type = output_type_ansi;
+                break;
+            }
+
+            case '?':
+            {
+                exit(1);
+            }
+        }
+    }
+
+    const wcstring src = read_file(stdin);
+    const wcstring output_wtext = prettify(src, do_indent);
+
+    /* Maybe colorize */
+    std::vector<highlight_spec_t> colors;
+    if (output_type != output_type_plain_text)
+    {
+        highlight_shell_no_io(output_wtext, colors, output_wtext.size(), NULL, env_vars_snapshot_t::current());
+    }
+
+    std::string colored_output;
+    switch (output_type)
+    {
+        case output_type_plain_text:
+            colored_output = no_colorize(output_wtext);
+            break;
+
+        case output_type_ansi:
+            colored_output = ansi_colorize(output_wtext, colors);
+            break;
+
+        case output_type_html:
+            colored_output = html_colorize(output_wtext, colors);
+            break;
+    }
+
+    fputs(colored_output.c_str(), stdout);
+    return 0;
+}
diff --git a/src/fish_tests.cpp b/src/fish_tests.cpp
new file mode 100644
index 00000000..c0b4a9f4
--- /dev/null
+++ b/src/fish_tests.cpp
@@ -0,0 +1,4009 @@
+/** \file fish_tests.c
+  Various bug and feature tests. Compiled and run by make test.
+*/
+
+#include "config.h"
+
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <wchar.h>
+#include <string.h>
+#include <unistd.h>
+#include <errno.h>
+#include <unistd.h>
+#include <termios.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <sys/wait.h>
+#include <fcntl.h>
+#include <stdarg.h>
+#include <libgen.h>
+#include <iostream>
+#include <string>
+#include <sstream>
+#include <algorithm>
+#include <iterator>
+
+#ifdef HAVE_GETOPT_H
+#include <getopt.h>
+#endif
+
+#include <signal.h>
+
+#include <locale.h>
+#include <dirent.h>
+#include <time.h>
+
+#include "fallback.h"
+#include "util.h"
+
+#include "common.h"
+#include "proc.h"
+#include "reader.h"
+#include "builtin.h"
+#include "function.h"
+#include "autoload.h"
+#include "complete.h"
+#include "wutil.h"
+#include "env.h"
+#include "expand.h"
+#include "parser.h"
+#include "tokenizer.h"
+#include "output.h"
+#include "exec.h"
+#include "event.h"
+#include "path.h"
+#include "history.h"
+#include "highlight.h"
+#include "iothread.h"
+#include "postfork.h"
+#include "signal.h"
+#include "parse_tree.h"
+#include "parse_util.h"
+#include "pager.h"
+#include "input.h"
+#include "utf8.h"
+#include "env_universal_common.h"
+#include "wcstringutil.h"
+
+static const char * const * s_arguments;
+static int s_test_run_count = 0;
+
+/* Indicate if we should test the given function. Either we test everything (all arguments) or we run only tests that have a prefix in s_arguments */
+static bool should_test_function(const char *func_name)
+{
+    /* No args, test everything */
+    bool result = false;
+    if (! s_arguments || ! s_arguments[0])
+    {
+        result = true;
+    }
+    else
+    {
+        for (size_t i=0; s_arguments[i] != NULL; i++)
+        {
+            if (! strncmp(func_name, s_arguments[i], strlen(s_arguments[i])))
+            {
+                /* Prefix match */
+                result = true;
+                break;
+            }
+        }
+    }
+    if (result)
+        s_test_run_count++;
+    return result;
+}
+
+/**
+   The number of tests to run
+ */
+#define ESCAPE_TEST_COUNT 100000
+/**
+   The average length of strings to unescape
+ */
+#define ESCAPE_TEST_LENGTH 100
+/**
+   The higest character number of character to try and escape
+ */
+#define ESCAPE_TEST_CHAR 4000
+
+/**
+   Number of laps to run performance testing loop
+*/
+#define LAPS 50
+
+/**
+   The result of one of the test passes
+*/
+#define NUM_ANS L"-7 99999999 1234567 deadbeef DEADBEEFDEADBEEF"
+
+/**
+   Number of encountered errors
+*/
+static int err_count=0;
+
+/**
+   Print formatted output
+*/
+static void say(const wchar_t *blah, ...)
+{
+    va_list va;
+    va_start(va, blah);
+    vwprintf(blah, va);
+    va_end(va);
+    wprintf(L"\n");
+}
+
+/**
+   Print formatted error string
+*/
+static void err(const wchar_t *blah, ...)
+{
+    va_list va;
+    va_start(va, blah);
+    err_count++;
+    
+    // Xcode's term doesn't support color (even though TERM claims it does)
+    bool colorize = ! getenv("RUNNING_IN_XCODE");
+
+    // show errors in red
+    if (colorize)
+    {
+        fputs("\x1b[31m", stdout);
+    }
+
+    wprintf(L"Error: ");
+    vwprintf(blah, va);
+    va_end(va);
+
+    // return to normal color
+    if (colorize)
+    {
+        fputs("\x1b[0m", stdout);
+    }
+
+    wprintf(L"\n");
+}
+
+// Joins a wcstring_list_t via commas
+static wcstring comma_join(const wcstring_list_t &lst)
+{
+    wcstring result;
+    for (size_t i=0; i < lst.size(); i++)
+    {
+        if (i > 0)
+        {
+            result.push_back(L',');
+        }
+        result.append(lst.at(i));
+    }
+    return result;
+}
+
+#define do_test(e) do { if (! (e)) err(L"Test failed on line %lu: %s", __LINE__, #e); } while (0)
+
+#define do_test1(e, msg) do { if (! (e)) err(L"Test failed on line %lu: %ls", __LINE__, (msg)); } while (0)
+
+/* Test sane escapes */
+static void test_unescape_sane()
+{
+    const struct test_t
+    {
+        const wchar_t * input;
+        const wchar_t * expected;
+    } tests[] =
+    {
+        {L"abcd", L"abcd"},
+        {L"'abcd'", L"abcd"},
+        {L"'abcd\\n'", L"abcd\\n"},
+        {L"\"abcd\\n\"", L"abcd\\n"},
+        {L"\"abcd\\n\"", L"abcd\\n"},
+        {L"\\143", L"c"},
+        {L"'\\143'", L"\\143"},
+        {L"\\n", L"\n"} // \n normally becomes newline
+    };
+    wcstring output;
+    for (size_t i=0; i < sizeof tests / sizeof *tests; i++)
+    {
+        bool ret = unescape_string(tests[i].input, &output, UNESCAPE_DEFAULT);
+        if (! ret)
+        {
+            err(L"Failed to unescape '%ls'\n", tests[i].input);
+        }
+        else if (output != tests[i].expected)
+        {
+            err(L"In unescaping '%ls', expected '%ls' but got '%ls'\n", tests[i].input, tests[i].expected, output.c_str());
+        }
+    }
+
+    // test for overflow
+    if (unescape_string(L"echo \\UFFFFFF", &output, UNESCAPE_DEFAULT))
+    {
+        err(L"Should not have been able to unescape \\UFFFFFF\n");
+    }
+    if (unescape_string(L"echo \\U110000", &output, UNESCAPE_DEFAULT))
+    {
+        err(L"Should not have been able to unescape \\U110000\n");
+    }
+    if (! unescape_string(L"echo \\U10FFFF", &output, UNESCAPE_DEFAULT))
+    {
+        err(L"Should have been able to unescape \\U10FFFF\n");
+    }
+
+
+}
+
+/**
+   Test the escaping/unescaping code by escaping/unescaping random
+   strings and verifying that the original string comes back.
+*/
+
+static void test_escape_crazy()
+{
+    say(L"Testing escaping and unescaping");
+    wcstring random_string;
+    wcstring escaped_string;
+    wcstring unescaped_string;
+    for (size_t i=0; i<ESCAPE_TEST_COUNT; i++)
+    {
+        random_string.clear();
+        while (rand() % ESCAPE_TEST_LENGTH)
+        {
+            random_string.push_back((rand() % ESCAPE_TEST_CHAR) +1);
+        }
+
+        escaped_string = escape_string(random_string, ESCAPE_ALL);
+        bool unescaped_success = unescape_string(escaped_string, &unescaped_string, UNESCAPE_DEFAULT);
+
+        if (! unescaped_success)
+        {
+            err(L"Failed to unescape string <%ls>", escaped_string.c_str());
+        }
+        else if (unescaped_string != random_string)
+        {
+            err(L"Escaped and then unescaped string '%ls', but got back a different string '%ls'", random_string.c_str(), unescaped_string.c_str());
+        }
+    }
+}
+
+static void test_format(void)
+{
+    say(L"Testing formatting functions");
+    struct
+    {
+        unsigned long long val;
+        const char *expected;
+    } tests[] =
+    {
+        { 0, "empty" },
+        { 1, "1B" },
+        { 2, "2B" },
+        { 1024, "1kB" },
+        { 1870, "1.8kB" },
+        { 4322911, "4.1MB" }
+    };
+    size_t i;
+    for (i=0; i < sizeof tests / sizeof *tests; i++)
+    {
+        char buff[128];
+        format_size_safe(buff, tests[i].val);
+        do_test(! strcmp(buff, tests[i].expected));
+    }
+
+    for (int j=-129; j <= 129; j++)
+    {
+        char buff1[128], buff2[128];
+        format_long_safe(buff1, j);
+        sprintf(buff2, "%d", j);
+        do_test(! strcmp(buff1, buff2));
+        
+        wchar_t wbuf1[128], wbuf2[128];
+        format_long_safe(wbuf1, j);
+        swprintf(wbuf2, 128, L"%d", j);
+        do_test(! wcscmp(wbuf1, wbuf2));
+
+    }
+
+    long q = LONG_MIN;
+    char buff1[128], buff2[128];
+    format_long_safe(buff1, q);
+    sprintf(buff2, "%ld", q);
+    do_test(! strcmp(buff1, buff2));
+}
+
+/**
+   Test wide/narrow conversion by creating random strings and
+   verifying that the original string comes back thorugh double
+   conversion.
+*/
+static void test_convert()
+{
+    /*  char o[] =
+        {
+          -17, -128, -121, -68, 0
+        }
+      ;
+
+      wchar_t *w = str2wcs(o);
+      char *n = wcs2str(w);
+
+      int i;
+
+      for( i=0; o[i]; i++ )
+      {
+        bitprint(o[i]);;
+        //wprintf(L"%d ", o[i]);
+      }
+      wprintf(L"\n");
+
+      for( i=0; w[i]; i++ )
+      {
+        wbitprint(w[i]);;
+        //wprintf(L"%d ", w[i]);
+      }
+      wprintf(L"\n");
+
+      for( i=0; n[i]; i++ )
+      {
+        bitprint(n[i]);;
+        //wprintf(L"%d ", n[i]);
+      }
+      wprintf(L"\n");
+
+      return;
+    */
+
+
+    int i;
+    std::vector<char> sb;
+
+    say(L"Testing wide/narrow string conversion");
+
+    for (i=0; i<ESCAPE_TEST_COUNT; i++)
+    {
+        const char *o, *n;
+
+        char c;
+
+        sb.clear();
+
+        while (rand() % ESCAPE_TEST_LENGTH)
+        {
+            c = rand();
+            sb.push_back(c);
+        }
+        c = 0;
+        sb.push_back(c);
+
+        o = &sb.at(0);
+        const wcstring w = str2wcstring(o);
+        n = wcs2str(w.c_str());
+
+        if (!o || !n)
+        {
+            err(L"Line %d - Conversion cycle of string %s produced null pointer on %s", __LINE__, o, L"wcs2str");
+        }
+
+        if (strcmp(o, n))
+        {
+            err(L"Line %d - %d: Conversion cycle of string %s produced different string %s", __LINE__, i, o, n);
+        }
+        free((void *)n);
+
+    }
+}
+
+/* Verify correct behavior with embedded nulls */
+static void test_convert_nulls(void)
+{
+    say(L"Testing embedded nulls in string conversion");
+    const wchar_t in[] = L"AAA\0BBB";
+    const size_t in_len = (sizeof in / sizeof *in) - 1;
+    const wcstring in_str = wcstring(in, in_len);
+    std::string out_str = wcs2string(in_str);
+    if (out_str.size() != in_len)
+    {
+        err(L"Embedded nulls mishandled in wcs2string");
+    }
+    for (size_t i=0; i < in_len; i++)
+    {
+        if (in[i] != out_str.at(i))
+        {
+            err(L"Embedded nulls mishandled in wcs2string at index %lu", (unsigned long)i);
+        }
+    }
+
+    wcstring out_wstr = str2wcstring(out_str);
+    if (out_wstr.size() != in_len)
+    {
+        err(L"Embedded nulls mishandled in str2wcstring");
+    }
+    for (size_t i=0; i < in_len; i++)
+    {
+        if (in[i] != out_wstr.at(i))
+        {
+            err(L"Embedded nulls mishandled in str2wcstring at index %lu", (unsigned long)i);
+        }
+    }
+
+}
+
+/**
+   Test the tokenizer
+*/
+static void test_tok()
+{
+
+    say(L"Testing tokenizer");
+
+
+    say(L"Testing invalid input");
+    tokenizer_t t(NULL, 0);
+
+    if (tok_last_type(&t) != TOK_ERROR)
+    {
+        err(L"Invalid input to tokenizer was undetected");
+    }
+
+    say(L"Testing use of broken tokenizer");
+    if (!tok_has_next(&t))
+    {
+        err(L"tok_has_next() should return 1 once on broken tokenizer");
+    }
+
+    tok_next(&t);
+    if (tok_last_type(&t) != TOK_ERROR)
+    {
+        err(L"Invalid input to tokenizer was undetected");
+    }
+
+    /*
+      This should crash if there is a bug. No reliable way to detect otherwise.
+    */
+    say(L"Test destruction of broken tokenizer");
+    {
+
+        const wchar_t *str = L"string <redirection  2>&1 'nested \"quoted\" '(string containing subshells ){and,brackets}$as[$well (as variable arrays)] not_a_redirect^ ^ ^^is_a_redirect Compress_Newlines\n  \n\t\n   \nInto_Just_One";
+        const int types[] =
+        {
+            TOK_STRING, TOK_REDIRECT_IN, TOK_STRING, TOK_REDIRECT_FD, TOK_STRING, TOK_STRING, TOK_STRING, TOK_REDIRECT_OUT, TOK_REDIRECT_APPEND, TOK_STRING, TOK_STRING, TOK_END, TOK_STRING, TOK_END
+        };
+
+        say(L"Test correct tokenization");
+
+        tokenizer_t t(str, 0);
+        for (size_t i=0; i < sizeof types / sizeof *types; i++, tok_next(&t))
+        {
+            if (types[i] != tok_last_type(&t))
+            {
+                err(L"Tokenization error:");
+                wprintf(L"Token number %d of string \n'%ls'\n, expected token type %ls, got token '%ls' of type %ls\n",
+                        i+1,
+                        str,
+                        tok_get_desc(types[i]),
+                        tok_last(&t),
+                        tok_get_desc(tok_last_type(&t)));
+            }
+        }
+    }
+
+    /* Test redirection_type_for_string */
+    if (redirection_type_for_string(L"<") != TOK_REDIRECT_IN) err(L"redirection_type_for_string failed on line %ld", (long)__LINE__);
+    if (redirection_type_for_string(L"^") != TOK_REDIRECT_OUT) err(L"redirection_type_for_string failed on line %ld", (long)__LINE__);
+    if (redirection_type_for_string(L">") != TOK_REDIRECT_OUT) err(L"redirection_type_for_string failed on line %ld", (long)__LINE__);
+    if (redirection_type_for_string(L"2>") != TOK_REDIRECT_OUT) err(L"redirection_type_for_string failed on line %ld", (long)__LINE__);
+    if (redirection_type_for_string(L">>") != TOK_REDIRECT_APPEND) err(L"redirection_type_for_string failed on line %ld", (long)__LINE__);
+    if (redirection_type_for_string(L"2>>") != TOK_REDIRECT_APPEND) err(L"redirection_type_for_string failed on line %ld", (long)__LINE__);
+    if (redirection_type_for_string(L"2>?") != TOK_REDIRECT_NOCLOB) err(L"redirection_type_for_string failed on line %ld", (long)__LINE__);
+    if (redirection_type_for_string(L"9999999999999999>?") != TOK_NONE) err(L"redirection_type_for_string failed on line %ld", (long)__LINE__);
+    if (redirection_type_for_string(L"2>&3") != TOK_REDIRECT_FD) err(L"redirection_type_for_string failed on line %ld", (long)__LINE__);
+    if (redirection_type_for_string(L"2>|") != TOK_NONE) err(L"redirection_type_for_string failed on line %ld", (long)__LINE__);
+}
+
+// Little function that runs in the main thread
+static int test_iothread_main_call(int *addr)
+{
+    *addr += 1;
+    return *addr;
+}
+
+// Little function that runs in a background thread, bouncing to the main
+static int test_iothread_thread_call(int *addr)
+{
+    int before = *addr;
+    iothread_perform_on_main(test_iothread_main_call, addr);
+    int after = *addr;
+
+    // Must have incremented it at least once
+    if (before >= after)
+    {
+        err(L"Failed to increment from background thread");
+    }
+    return after;
+}
+
+static void test_iothread(void)
+{
+    say(L"Testing iothreads");
+    int *int_ptr = new int(0);
+    int iterations = 50000;
+    int max_achieved_thread_count = 0;
+    double start = timef();
+    for (int i=0; i < iterations; i++)
+    {
+        int thread_count = iothread_perform(test_iothread_thread_call, int_ptr);
+        max_achieved_thread_count = std::max(max_achieved_thread_count, thread_count);
+    }
+
+    // Now wait until we're done
+    iothread_drain_all();
+    double end = timef();
+
+    // Should have incremented it once per thread
+    if (*int_ptr != iterations)
+    {
+        say(L"Expected int to be %d, but instead it was %d", iterations, *int_ptr);
+    }
+    
+    say(L"    (%.02f msec, with max of %d threads)", (end - start) * 1000.0, max_achieved_thread_count);
+
+    delete int_ptr;
+}
+
+static parser_test_error_bits_t detect_argument_errors(const wcstring &src)
+{
+    parse_node_tree_t tree;
+    if (! parse_tree_from_string(src, parse_flag_none, &tree, NULL, symbol_argument_list))
+    {
+        return PARSER_TEST_ERROR;
+    }
+
+    assert(! tree.empty());
+    const parse_node_t *first_arg = tree.next_node_in_node_list(tree.at(0), symbol_argument, NULL);
+    assert(first_arg != NULL);
+    return parse_util_detect_errors_in_argument(*first_arg, first_arg->get_source(src));
+}
+
+/**
+   Test the parser
+*/
+static void test_parser()
+{
+    say(L"Testing parser");
+
+    parser_t parser(PARSER_TYPE_GENERAL, true);
+
+    say(L"Testing block nesting");
+    if (!parse_util_detect_errors(L"if; end"))
+    {
+        err(L"Incomplete if statement undetected");
+    }
+    if (!parse_util_detect_errors(L"if test; echo"))
+    {
+        err(L"Missing end undetected");
+    }
+    if (!parse_util_detect_errors(L"if test; end; end"))
+    {
+        err(L"Unbalanced end undetected");
+    }
+
+    say(L"Testing detection of invalid use of builtin commands");
+    if (!parse_util_detect_errors(L"case foo"))
+    {
+        err(L"'case' command outside of block context undetected");
+    }
+    if (!parse_util_detect_errors(L"switch ggg; if true; case foo;end;end"))
+    {
+        err(L"'case' command outside of switch block context undetected");
+    }
+    if (!parse_util_detect_errors(L"else"))
+    {
+        err(L"'else' command outside of conditional block context undetected");
+    }
+    if (!parse_util_detect_errors(L"else if"))
+    {
+        err(L"'else if' command outside of conditional block context undetected");
+    }
+    if (!parse_util_detect_errors(L"if false; else if; end"))
+    {
+        err(L"'else if' missing command undetected");
+    }
+
+    if (!parse_util_detect_errors(L"break"))
+    {
+        err(L"'break' command outside of loop block context undetected");
+    }
+
+    if (parse_util_detect_errors(L"break --help"))
+    {
+        err(L"'break --help' incorrectly marked as error");
+    }
+
+    if (! parse_util_detect_errors(L"while false ; function foo ; break ; end ; end "))
+    {
+        err(L"'break' command inside function allowed to break from loop outside it");
+    }
+
+
+    if (!parse_util_detect_errors(L"exec ls|less") || !parse_util_detect_errors(L"echo|return"))
+    {
+        err(L"Invalid pipe command undetected");
+    }
+
+    if (parse_util_detect_errors(L"for i in foo ; switch $i ; case blah ; break; end; end "))
+    {
+        err(L"'break' command inside switch falsely reported as error");
+    }
+
+    if (parse_util_detect_errors(L"or cat | cat") || parse_util_detect_errors(L"and cat | cat"))
+    {
+        err(L"boolean command at beginning of pipeline falsely reported as error");
+    }
+
+    if (! parse_util_detect_errors(L"cat | and cat"))
+    {
+        err(L"'and' command in pipeline not reported as error");
+    }
+
+    if (! parse_util_detect_errors(L"cat | or cat"))
+    {
+        err(L"'or' command in pipeline not reported as error");
+    }
+
+    if (! parse_util_detect_errors(L"cat | exec") || ! parse_util_detect_errors(L"exec | cat"))
+    {
+        err(L"'exec' command in pipeline not reported as error");
+    }
+
+    if (detect_argument_errors(L"foo"))
+    {
+        err(L"simple argument reported as error");
+    }
+
+    if (detect_argument_errors(L"''"))
+    {
+        err(L"Empty string reported as error");
+    }
+
+
+    if (!(detect_argument_errors(L"foo$$") & PARSER_TEST_ERROR))
+    {
+        err(L"Bad variable expansion not reported as error");
+    }
+
+    if (!(detect_argument_errors(L"foo$@") & PARSER_TEST_ERROR))
+    {
+        err(L"Bad variable expansion not reported as error");
+    }
+
+    /* Within command substitutions, we should be able to detect everything that parse_util_detect_errors can detect */
+    if (!(detect_argument_errors(L"foo(cat | or cat)") & PARSER_TEST_ERROR))
+    {
+        err(L"Bad command substitution not reported as error");
+    }
+
+    if (!(detect_argument_errors(L"foo\\xFF9") & PARSER_TEST_ERROR))
+    {
+        err(L"Bad escape not reported as error");
+    }
+
+    if (!(detect_argument_errors(L"foo(echo \\xFF9)") & PARSER_TEST_ERROR))
+    {
+        err(L"Bad escape in command substitution not reported as error");
+    }
+
+    if (!(detect_argument_errors(L"foo(echo (echo (echo \\xFF9)))") & PARSER_TEST_ERROR))
+    {
+        err(L"Bad escape in nested command substitution not reported as error");
+    }
+    
+    if (! parse_util_detect_errors(L"false & ; and cat"))
+    {
+        err(L"'and' command after background not reported as error");
+    }
+
+    if (! parse_util_detect_errors(L"true & ; or cat"))
+    {
+        err(L"'or' command after background not reported as error");
+    }
+    
+    if (parse_util_detect_errors(L"true & ; not cat"))
+    {
+        err(L"'not' command after background falsely reported as error");
+    }
+
+    
+    if (! parse_util_detect_errors(L"if true & ; end"))
+    {
+        err(L"backgrounded 'if' conditional not reported as error");
+    }
+
+    if (! parse_util_detect_errors(L"if false; else if true & ; end"))
+    {
+        err(L"backgrounded 'else if' conditional not reported as error");
+    }
+
+    if (! parse_util_detect_errors(L"while true & ; end"))
+    {
+        err(L"backgrounded 'while' conditional not reported as error");
+    }
+
+    say(L"Testing basic evaluation");
+#if 0
+    /* This fails now since the parser takes a wcstring&, and NULL converts to wchar_t * converts to wcstring which crashes (thanks C++) */
+    if (!parser.eval(0, 0, TOP))
+    {
+        err(L"Null input when evaluating undetected");
+    }
+#endif
+    if (!parser.eval(L"ls", io_chain_t(), WHILE))
+    {
+        err(L"Invalid block mode when evaluating undetected");
+    }
+
+    /* Ensure that we don't crash on infinite self recursion and mutual recursion. These must use the principal parser because we cannot yet execute jobs on other parsers (!) */
+    say(L"Testing recursion detection");
+    parser_t::principal_parser().eval(L"function recursive ; recursive ; end ; recursive; ", io_chain_t(), TOP);
+#if 0
+    /* This is disabled since it produces a long backtrace. We should find a way to either visually compress the backtrace, or disable error spewing */
+    parser_t::principal_parser().eval(L"function recursive1 ; recursive2 ; end ; function recursive2 ; recursive1 ; end ; recursive1; ", io_chain_t(), TOP);
+#endif
+
+    say(L"Testing empty function name");
+    parser_t::principal_parser().eval(L"function '' ; echo fail; exit 42 ; end ; ''", io_chain_t(), TOP);
+
+    say(L"Testing eval_args");
+    completion_list_t comps;
+    parser_t::principal_parser().expand_argument_list(L"alpha 'beta gamma' delta", comps);
+    do_test(comps.size() == 3);
+    do_test(comps.at(0).completion == L"alpha");
+    do_test(comps.at(1).completion == L"beta gamma");
+    do_test(comps.at(2).completion == L"delta");
+
+}
+
+/* Wait a while and then SIGINT the main thread */
+struct test_cancellation_info_t
+{
+    pthread_t thread;
+    double delay;
+};
+
+static int signal_main(test_cancellation_info_t *info)
+{
+    usleep(info->delay * 1E6);
+    pthread_kill(info->thread, SIGINT);
+    return 0;
+}
+
+static void test_1_cancellation(const wchar_t *src)
+{
+    shared_ptr<io_buffer_t> out_buff(io_buffer_t::create(STDOUT_FILENO, io_chain_t()));
+    const io_chain_t io_chain(out_buff);
+    test_cancellation_info_t ctx = {pthread_self(), 0.25 /* seconds */ };
+    iothread_perform(signal_main, &ctx);
+    parser_t::principal_parser().eval(src, io_chain, TOP);
+    out_buff->read();
+    if (out_buff->out_buffer_size() != 0)
+    {
+        err(L"Expected 0 bytes in out_buff, but instead found %lu bytes\n", out_buff->out_buffer_size());
+    }
+    iothread_drain_all();
+}
+
+static void test_cancellation()
+{
+    if (getenv("RUNNING_IN_XCODE")) {
+        say(L"Skipping Ctrl-C cancellation test because we are running in Xcode debugger");
+        return;
+    }
+    say(L"Testing Ctrl-C cancellation. If this hangs, that's a bug!");
+
+    /* Enable fish's signal handling here. We need to make this interactive for fish to install its signal handlers */
+    proc_push_interactive(1);
+    signal_set_handlers();
+
+    /* This tests that we can correctly ctrl-C out of certain loop constructs, and that nothing gets printed if we do */
+
+    /* Here the command substitution is an infinite loop. echo never even gets its argument, so when we cancel we expect no output */
+    test_1_cancellation(L"echo (while true ; echo blah ; end)");
+
+    fprintf(stderr, ".");
+
+    /* Nasty infinite loop that doesn't actually execute anything */
+    test_1_cancellation(L"echo (while true ; end) (while true ; end) (while true ; end)");
+    fprintf(stderr, ".");
+
+    test_1_cancellation(L"while true ; end");
+    fprintf(stderr, ".");
+
+    test_1_cancellation(L"for i in (while true ; end) ; end");
+    fprintf(stderr, ".");
+
+    fprintf(stderr, "\n");
+
+    /* Restore signal handling */
+    proc_pop_interactive();
+    signal_reset_handlers();
+
+    /* Ensure that we don't think we should cancel */
+    reader_reset_interrupted();
+}
+
+static void test_indents()
+{
+    say(L"Testing indents");
+
+    // Here are the components of our source and the indents we expect those to be
+    struct indent_component_t
+    {
+        const wchar_t *txt;
+        int indent;
+    };
+
+    const indent_component_t components1[] =
+    {
+        {L"if foo", 0},
+        {L"end", 0},
+        {NULL, -1}
+    };
+
+    const indent_component_t components2[] =
+    {
+        {L"if foo", 0},
+        {L"", 1}, //trailing newline!
+        {NULL, -1}
+    };
+
+    const indent_component_t components3[] =
+    {
+        {L"if foo", 0},
+        {L"foo", 1},
+        {L"end", 0}, //trailing newline!
+        {NULL, -1}
+    };
+
+    const indent_component_t components4[] =
+    {
+        {L"if foo", 0},
+        {L"if bar", 1},
+        {L"end", 1},
+        {L"end", 0},
+        {L"", 0},
+        {NULL, -1}
+    };
+
+    const indent_component_t components5[] =
+    {
+        {L"if foo", 0},
+        {L"if bar", 1},
+        {L"", 2},
+        {NULL, -1}
+    };
+
+    const indent_component_t components6[] =
+    {
+        {L"begin", 0},
+        {L"foo", 1},
+        {L"", 1},
+        {NULL, -1}
+    };
+
+    const indent_component_t components7[] =
+    {
+        {L"begin", 0},
+        {L";", 1},
+        {L"end", 0},
+        {L"foo", 0},
+        {L"", 0},
+        {NULL, -1}
+    };
+
+    const indent_component_t components8[] =
+    {
+        {L"if foo", 0},
+        {L"if bar", 1},
+        {L"baz", 2},
+        {L"end", 1},
+        {L"", 1},
+        {NULL, -1}
+    };
+
+    const indent_component_t components9[] =
+    {
+        {L"switch foo", 0},
+        {L"", 1},
+        {NULL, -1}
+    };
+
+    const indent_component_t components10[] =
+    {
+        {L"switch foo", 0},
+        {L"case bar", 1},
+        {L"case baz", 1},
+        {L"quux", 2},
+        {L"", 2},
+        {NULL, -1}
+    };
+
+    const indent_component_t components11[] =
+    {
+        {L"switch foo", 0},
+        {L"cas", 1}, //parse error indentation handling
+        {NULL, -1}
+    };
+
+    const indent_component_t components12[] =
+    {
+        {L"while false", 0},
+        {L"# comment", 1}, //comment indentation handling
+        {L"command", 1}, //comment indentation handling
+        {L"# comment2", 1}, //comment indentation handling
+        {NULL, -1}
+    };
+
+
+    const indent_component_t *tests[] = {components1, components2, components3, components4, components5, components6, components7, components8, components9, components10, components11, components12};
+    for (size_t which = 0; which < sizeof tests / sizeof *tests; which++)
+    {
+        const indent_component_t *components = tests[which];
+        // Count how many we have
+        size_t component_count = 0;
+        while (components[component_count].txt != NULL)
+        {
+            component_count++;
+        }
+
+        // Generate the expected indents
+        wcstring text;
+        std::vector<int> expected_indents;
+        for (size_t i=0; i < component_count; i++)
+        {
+            if (i > 0)
+            {
+                text.push_back(L'\n');
+                expected_indents.push_back(components[i].indent);
+            }
+            text.append(components[i].txt);
+            expected_indents.resize(text.size(), components[i].indent);
+        }
+        do_test(expected_indents.size() == text.size());
+
+        // Compute the indents
+        std::vector<int> indents = parse_util_compute_indents(text);
+
+        if (expected_indents.size() != indents.size())
+        {
+            err(L"Indent vector has wrong size! Expected %lu, actual %lu", expected_indents.size(), indents.size());
+        }
+        do_test(expected_indents.size() == indents.size());
+        for (size_t i=0; i < text.size(); i++)
+        {
+            if (expected_indents.at(i) != indents.at(i))
+            {
+                err(L"Wrong indent at index %lu in test #%lu (expected %d, actual %d):\n%ls\n", i, which + 1, expected_indents.at(i), indents.at(i), text.c_str());
+                break; //don't keep showing errors for the rest of the line
+            }
+        }
+
+    }
+}
+
+static void test_utils()
+{
+    say(L"Testing utils");
+    const wchar_t *a = L"echo (echo (echo hi";
+
+    const wchar_t *begin = NULL, *end = NULL;
+    parse_util_cmdsubst_extent(a, 0, &begin, &end);
+    if (begin != a || end != begin + wcslen(begin)) err(L"parse_util_cmdsubst_extent failed on line %ld", (long)__LINE__);
+    parse_util_cmdsubst_extent(a, 1, &begin, &end);
+    if (begin != a || end != begin + wcslen(begin)) err(L"parse_util_cmdsubst_extent failed on line %ld", (long)__LINE__);
+    parse_util_cmdsubst_extent(a, 2, &begin, &end);
+    if (begin != a || end != begin + wcslen(begin)) err(L"parse_util_cmdsubst_extent failed on line %ld", (long)__LINE__);
+    parse_util_cmdsubst_extent(a, 3, &begin, &end);
+    if (begin != a || end != begin + wcslen(begin)) err(L"parse_util_cmdsubst_extent failed on line %ld", (long)__LINE__);
+
+    parse_util_cmdsubst_extent(a, 8, &begin, &end);
+    if (begin != a + wcslen(L"echo (")) err(L"parse_util_cmdsubst_extent failed on line %ld", (long)__LINE__);
+
+    parse_util_cmdsubst_extent(a, 17, &begin, &end);
+    if (begin != a + wcslen(L"echo (echo (")) err(L"parse_util_cmdsubst_extent failed on line %ld", (long)__LINE__);
+}
+
+/* UTF8 tests taken from Alexey Vatchenko's utf8 library. See http://www.bsdua.org/libbsdua.html */
+
+static void test_utf82wchar(const char *src, size_t slen, const wchar_t *dst, size_t dlen,
+                            int flags, size_t res, const char *descr)
+{
+    size_t size;
+    wchar_t *mem = NULL;
+
+    /* Hack: if wchar is only UCS-2, and the UTF-8 input string contains astral characters, then tweak the expected size to 0 */
+    if (src != NULL && is_wchar_ucs2())
+    {
+        /* A UTF-8 code unit may represent an astral code point if it has 4 or more leading 1s */
+        const unsigned char astral_mask = 0xF0;
+        for (size_t i=0; i < slen; i++)
+        {
+            if ((src[i] & astral_mask) == astral_mask)
+            {
+                /* Astral char. We expect this conversion to just fail. */
+                res = 0;
+                break;
+            }
+        }
+    }
+
+    if (dst != NULL)
+    {
+        mem = (wchar_t *)malloc(dlen * sizeof(*mem));
+        if (mem == NULL)
+        {
+            err(L"u2w: %s: MALLOC FAILED\n", descr);
+            return;
+        }
+    }
+
+    do
+    {
+        size = utf8_to_wchar(src, slen, mem, dlen, flags);
+        if (res != size)
+        {
+            err(L"u2w: %s: FAILED (rv: %lu, must be %lu)", descr, size, res);
+            break;
+        }
+
+        if (mem == NULL)
+            break;		/* OK */
+
+        if (memcmp(mem, dst, size * sizeof(*mem)) != 0)
+        {
+            err(L"u2w: %s: BROKEN", descr);
+            break;
+        }
+
+    }
+    while (0);
+
+    free(mem);
+}
+
+// Annoying variant to handle uchar to avoid narrowing conversion warnings
+static void test_utf82wchar(const unsigned char *usrc, size_t slen, const wchar_t *dst, size_t dlen,
+                            int flags, size_t res, const char *descr) {
+    const char *src = reinterpret_cast<const char *>(usrc);
+    return test_utf82wchar(src, slen, dst, dlen, flags, res, descr);
+}
+
+static void test_wchar2utf8(const wchar_t *src, size_t slen, const char *dst, size_t dlen,
+                            int flags, size_t res, const char *descr)
+{
+    size_t size;
+    char *mem = NULL;
+
+    /* Hack: if wchar is simulating UCS-2, and the wchar_t input string contains astral characters, then tweak the expected size to 0 */
+    if (src != NULL && is_wchar_ucs2())
+    {
+        const uint32_t astral_mask = 0xFFFF0000U;
+        for (size_t i=0; i < slen; i++)
+        {
+            if ((src[i] & astral_mask) != 0)
+            {
+                /* astral char */
+                res = 0;
+                break;
+            }
+        }
+    }
+
+    if (dst != NULL)
+    {
+        mem = (char *)malloc(dlen);
+        if (mem == NULL)
+        {
+            err(L"w2u: %s: MALLOC FAILED", descr);
+            return;
+        }
+    }
+
+    size = wchar_to_utf8(src, slen, mem, dlen, flags);
+    if (res != size)
+    {
+        err(L"w2u: %s: FAILED (rv: %lu, must be %lu)", descr, size, res);
+        goto finish;
+    }
+
+    if (mem == NULL)
+        goto finish;		/* OK */
+
+    if (memcmp(mem, dst, size) != 0)
+    {
+        err(L"w2u: %s: BROKEN", descr);
+        goto finish;
+    }
+
+    finish:
+    free(mem);
+}
+
+// Annoying variant to handle uchar to avoid narrowing conversion warnings
+static void test_wchar2utf8(const wchar_t *src, size_t slen, const unsigned char *udst, size_t dlen,
+                            int flags, size_t res, const char *descr)
+{
+    const char *dst = reinterpret_cast<const char *>(udst);
+    return test_wchar2utf8(src, slen, dst, dlen, flags, res, descr);
+}
+
+static void test_utf8()
+{
+    wchar_t w1[] = {0x54, 0x65, 0x73, 0x74};
+    wchar_t w2[] = {0x0422, 0x0435, 0x0441, 0x0442};
+    wchar_t w3[] = {0x800, 0x1e80, 0x98c4, 0x9910, 0xff00};
+    wchar_t w4[] = {0x15555, 0xf7777, 0xa};
+    wchar_t w5[] = {0x255555, 0x1fa04ff, 0xddfd04, 0xa};
+    wchar_t w6[] = {0xf255555, 0x1dfa04ff, 0x7fddfd04, 0xa};
+    wchar_t wb[] = {-2, 0xa, (wchar_t)0xffffffff, 0x0441};
+    wchar_t wm[] = {0x41, 0x0441, 0x3042, 0xff67, 0x9b0d, 0x2e05da67};
+    wchar_t wb1[] = {0xa, 0x0422};
+    wchar_t wb2[] = {0xd800, 0xda00, 0x41, 0xdfff, 0xa};
+    wchar_t wbom[] = {0xfeff, 0x41, 0xa};
+    wchar_t wbom2[] = {0x41, 0xa};
+    wchar_t wbom22[] = {0xfeff, 0x41, 0xa};
+    unsigned char u1[] = {0x54, 0x65, 0x73, 0x74};
+    unsigned char u2[] = {0xd0, 0xa2, 0xd0, 0xb5, 0xd1, 0x81, 0xd1, 0x82};
+    unsigned char u3[] = {0xe0, 0xa0, 0x80, 0xe1, 0xba, 0x80, 0xe9, 0xa3, 0x84,
+                 0xe9, 0xa4, 0x90, 0xef, 0xbc, 0x80
+                };
+    unsigned char u4[] = {0xf0, 0x95, 0x95, 0x95, 0xf3, 0xb7, 0x9d, 0xb7, 0xa};
+    unsigned char u5[] = {0xf8, 0x89, 0x95, 0x95, 0x95, 0xf9, 0xbe, 0xa0, 0x93,
+                 0xbf, 0xf8, 0xb7, 0x9f, 0xb4, 0x84, 0x0a
+                };
+    unsigned char u6[] = {0xfc, 0x8f, 0x89, 0x95, 0x95, 0x95, 0xfc, 0x9d, 0xbe,
+                 0xa0, 0x93, 0xbf, 0xfd, 0xbf, 0xb7, 0x9f, 0xb4, 0x84, 0x0a
+                };
+    unsigned char ub[] = {0xa, 0xd1, 0x81};
+    unsigned char um[] = {0x41, 0xd1, 0x81, 0xe3, 0x81, 0x82, 0xef, 0xbd, 0xa7,
+                 0xe9, 0xac, 0x8d, 0xfc, 0xae, 0x81, 0x9d, 0xa9, 0xa7
+                };
+    unsigned char ub1[] = {0xa, 0xff, 0xd0, 0xa2, 0xfe, 0x8f, 0xe0, 0x80};
+    unsigned char uc080[] = {0xc0, 0x80};
+    unsigned char ub2[] = {0xed, 0xa1, 0x8c, 0xed, 0xbe, 0xb4, 0xa};
+    unsigned char ubom[] = {0x41, 0xa};
+    unsigned char ubom2[] = {0xef, 0xbb, 0xbf, 0x41, 0xa};
+
+    /*
+     * UTF-8 -> UCS-4 string.
+     */
+    test_utf82wchar(ubom2, sizeof(ubom2), wbom2,
+                    sizeof(wbom2) / sizeof(*wbom2), UTF8_SKIP_BOM,
+                    sizeof(wbom2) / sizeof(*wbom2), "skip BOM");
+    test_utf82wchar(ubom2, sizeof(ubom2), wbom22,
+                    sizeof(wbom22) / sizeof(*wbom22), 0,
+                    sizeof(wbom22) / sizeof(*wbom22), "BOM");
+    test_utf82wchar(uc080, sizeof(uc080), NULL, 0, 0, 0,
+                    "c0 80 - forbitten by rfc3629");
+    test_utf82wchar(ub2, sizeof(ub2), NULL, 0, 0, is_wchar_ucs2() ? 0 : 3,
+                    "resulted in forbitten wchars (len)");
+    test_utf82wchar(ub2, sizeof(ub2), wb2, sizeof(wb2) / sizeof(*wb2), 0, 0,
+                    "resulted in forbitten wchars");
+    test_utf82wchar(ub2, sizeof(ub2), L"\x0a", 1, UTF8_IGNORE_ERROR,
+                    1, "resulted in ignored forbitten wchars");
+    test_utf82wchar(u1, sizeof(u1), w1, sizeof(w1) / sizeof(*w1), 0,
+                    sizeof(w1) / sizeof(*w1), "1 octet chars");
+    test_utf82wchar(u2, sizeof(u2), w2, sizeof(w2) / sizeof(*w2), 0,
+                    sizeof(w2) / sizeof(*w2), "2 octets chars");
+    test_utf82wchar(u3, sizeof(u3), w3, sizeof(w3) / sizeof(*w3), 0,
+                    sizeof(w3) / sizeof(*w3), "3 octets chars");
+    test_utf82wchar(u4, sizeof(u4), w4, sizeof(w4) / sizeof(*w4), 0,
+                    sizeof(w4) / sizeof(*w4), "4 octets chars");
+    test_utf82wchar(u5, sizeof(u5), w5, sizeof(w5) / sizeof(*w5), 0,
+                    sizeof(w5) / sizeof(*w5), "5 octets chars");
+    test_utf82wchar(u6, sizeof(u6), w6, sizeof(w6) / sizeof(*w6), 0,
+                    sizeof(w6) / sizeof(*w6), "6 octets chars");
+    test_utf82wchar("\xff", 1, NULL, 0, 0, 0, "broken utf-8 0xff symbol");
+    test_utf82wchar("\xfe", 1, NULL, 0, 0, 0, "broken utf-8 0xfe symbol");
+    test_utf82wchar("\x8f", 1, NULL, 0, 0, 0,
+                    "broken utf-8, start from 10 higher bits");
+    if (! is_wchar_ucs2()) test_utf82wchar(ub1, sizeof(ub1), wb1, sizeof(wb1) / sizeof(*wb1),
+                                               UTF8_IGNORE_ERROR, sizeof(wb1) / sizeof(*wb1), "ignore bad chars");
+    test_utf82wchar(um, sizeof(um), wm, sizeof(wm) / sizeof(*wm), 0,
+                    sizeof(wm) / sizeof(*wm), "mixed languages");
+    test_utf82wchar(um, sizeof(um), wm, sizeof(wm) / sizeof(*wm) - 1, 0,
+                    0, "boundaries -1");
+    test_utf82wchar(um, sizeof(um), wm, sizeof(wm) / sizeof(*wm) + 1, 0,
+                    sizeof(wm) / sizeof(*wm), "boundaries +1");
+    test_utf82wchar(um, sizeof(um), NULL, 0, 0,
+                    sizeof(wm) / sizeof(*wm), "calculate length");
+    test_utf82wchar(ub1, sizeof(ub1), NULL, 0, 0,
+                    0, "calculate length of bad chars");
+    test_utf82wchar(ub1, sizeof(ub1), NULL, 0,
+                    UTF8_IGNORE_ERROR, sizeof(wb1) / sizeof(*wb1),
+                    "calculate length, ignore bad chars");
+    test_utf82wchar((const char *)NULL, 0, NULL, 0, 0, 0, "invalid params, all 0");
+    test_utf82wchar(u1, 0, NULL, 0, 0, 0,
+                    "invalid params, src buf not NULL");
+    test_utf82wchar((const char *)NULL, 10, NULL, 0, 0, 0,
+                    "invalid params, src length is not 0");
+    test_utf82wchar(u1, sizeof(u1), w1, 0, 0, 0,
+                    "invalid params, dst is not NULL");
+
+    /*
+     * UCS-4 -> UTF-8 string.
+     */
+    const char * const nullc = NULL;
+    test_wchar2utf8(wbom, sizeof(wbom) / sizeof(*wbom), ubom, sizeof(ubom),
+                    UTF8_SKIP_BOM, sizeof(ubom), "BOM");
+    test_wchar2utf8(wb2, sizeof(wb2) / sizeof(*wb2), nullc, 0, 0,
+                    0, "prohibited wchars");
+    test_wchar2utf8(wb2, sizeof(wb2) / sizeof(*wb2), nullc, 0,
+                    UTF8_IGNORE_ERROR, 2, "ignore prohibited wchars");
+    test_wchar2utf8(w1, sizeof(w1) / sizeof(*w1), u1, sizeof(u1), 0,
+                    sizeof(u1), "1 octet chars");
+    test_wchar2utf8(w2, sizeof(w2) / sizeof(*w2), u2, sizeof(u2), 0,
+                    sizeof(u2), "2 octets chars");
+    test_wchar2utf8(w3, sizeof(w3) / sizeof(*w3), u3, sizeof(u3), 0,
+                    sizeof(u3), "3 octets chars");
+    test_wchar2utf8(w4, sizeof(w4) / sizeof(*w4), u4, sizeof(u4), 0,
+                    sizeof(u4), "4 octets chars");
+    test_wchar2utf8(w5, sizeof(w5) / sizeof(*w5), u5, sizeof(u5), 0,
+                    sizeof(u5), "5 octets chars");
+    test_wchar2utf8(w6, sizeof(w6) / sizeof(*w6), u6, sizeof(u6), 0,
+                    sizeof(u6), "6 octets chars");
+    test_wchar2utf8(wb, sizeof(wb) / sizeof(*wb), ub, sizeof(ub), 0,
+                    0, "bad chars");
+    test_wchar2utf8(wb, sizeof(wb) / sizeof(*wb), ub, sizeof(ub),
+                    UTF8_IGNORE_ERROR, sizeof(ub), "ignore bad chars");
+    test_wchar2utf8(wm, sizeof(wm) / sizeof(*wm), um, sizeof(um), 0,
+                    sizeof(um), "mixed languages");
+    test_wchar2utf8(wm, sizeof(wm) / sizeof(*wm), um, sizeof(um) - 1, 0,
+                    0, "boundaries -1");
+    test_wchar2utf8(wm, sizeof(wm) / sizeof(*wm), um, sizeof(um) + 1, 0,
+                    sizeof(um), "boundaries +1");
+    test_wchar2utf8(wm, sizeof(wm) / sizeof(*wm), nullc, 0, 0,
+                    sizeof(um), "calculate length");
+    test_wchar2utf8(wb, sizeof(wb) / sizeof(*wb), nullc, 0, 0,
+                    0, "calculate length of bad chars");
+    test_wchar2utf8(wb, sizeof(wb) / sizeof(*wb), nullc, 0,
+                    UTF8_IGNORE_ERROR, sizeof(ub),
+                    "calculate length, ignore bad chars");
+    test_wchar2utf8(NULL, 0, nullc, 0, 0, 0, "invalid params, all 0");
+    test_wchar2utf8(w1, 0, nullc, 0, 0, 0,
+                    "invalid params, src buf not NULL");
+    test_wchar2utf8(NULL, 10, nullc, 0, 0, 0,
+                    "invalid params, src length is not 0");
+    test_wchar2utf8(w1, sizeof(w1) / sizeof(*w1), u1, 0, 0, 0,
+                    "invalid params, dst is not NULL");
+}
+
+static void test_escape_sequences(void)
+{
+    say(L"Testing escape codes");
+    if (escape_code_length(L"") != 0) err(L"test_escape_sequences failed on line %d\n", __LINE__);
+    if (escape_code_length(L"abcd") != 0) err(L"test_escape_sequences failed on line %d\n", __LINE__);
+    if (escape_code_length(L"\x1b[2J") != 4) err(L"test_escape_sequences failed on line %d\n", __LINE__);
+    if (escape_code_length(L"\x1b[38;5;123mABC") != strlen("\x1b[38;5;123m")) err(L"test_escape_sequences failed on line %d\n", __LINE__);
+    if (escape_code_length(L"\x1b@") != 2) err(L"test_escape_sequences failed on line %d\n", __LINE__);
+    
+    // iTerm2 escape sequences
+    if (escape_code_length(L"\x1b]50;CurrentDir=/tmp/foo\x07NOT_PART_OF_SEQUENCE") != 25) err(L"test_escape_sequences failed on line %d\n", __LINE__);
+    if (escape_code_length(L"\x1b]50;SetMark\x07NOT_PART_OF_SEQUENCE") != 13) err(L"test_escape_sequences failed on line %d\n", __LINE__);
+    if (escape_code_length(L"\x1b" L"]6;1;bg;red;brightness;255\x07NOT_PART_OF_SEQUENCE") != 28) err(L"test_escape_sequences failed on line %d\n", __LINE__);
+    if (escape_code_length(L"\x1b]Pg4040ff\x1b\\NOT_PART_OF_SEQUENCE") != 12) err(L"test_escape_sequences failed on line %d\n", __LINE__);
+    if (escape_code_length(L"\x1b]blahblahblah\x1b\\") != 16) err(L"test_escape_sequences failed on line %d\n", __LINE__);
+    if (escape_code_length(L"\x1b]blahblahblah\x07") != 15) err(L"test_escape_sequences failed on line %d\n", __LINE__);
+}
+
+class lru_node_test_t : public lru_node_t
+{
+public:
+    lru_node_test_t(const wcstring &tmp) : lru_node_t(tmp) { }
+};
+
+class test_lru_t : public lru_cache_t<lru_node_test_t>
+{
+public:
+    test_lru_t() : lru_cache_t<lru_node_test_t>(16) { }
+
+    std::vector<lru_node_test_t *> evicted_nodes;
+
+    virtual void node_was_evicted(lru_node_test_t *node)
+    {
+        do_test(find(evicted_nodes.begin(), evicted_nodes.end(), node) == evicted_nodes.end());
+        evicted_nodes.push_back(node);
+    }
+};
+
+static void test_lru(void)
+{
+    say(L"Testing LRU cache");
+
+    test_lru_t cache;
+    std::vector<lru_node_test_t *> expected_evicted;
+    size_t total_nodes = 20;
+    for (size_t i=0; i < total_nodes; i++)
+    {
+        do_test(cache.size() == std::min(i, (size_t)16));
+        lru_node_test_t *node = new lru_node_test_t(to_string(i));
+        if (i < 4) expected_evicted.push_back(node);
+        // Adding the node the first time should work, and subsequent times should fail
+        do_test(cache.add_node(node));
+        do_test(! cache.add_node(node));
+    }
+    do_test(cache.evicted_nodes == expected_evicted);
+    cache.evict_all_nodes();
+    do_test(cache.evicted_nodes.size() == total_nodes);
+    while (! cache.evicted_nodes.empty())
+    {
+        lru_node_t *node = cache.evicted_nodes.back();
+        cache.evicted_nodes.pop_back();
+        delete node;
+    }
+}
+
+/**
+   Perform parameter expansion and test if the output equals the zero-terminated parameter list supplied.
+
+   \param in the string to expand
+   \param flags the flags to send to expand_string
+   \param ... A zero-terminated parameter list of values to test.
+              After the zero terminator comes one more arg, a string, which is the error
+              message to print if the test fails.
+*/
+
+static bool expand_test(const wchar_t *in, expand_flags_t flags, ...)
+{
+    std::vector<completion_t> output;
+    va_list va;
+    bool res=true;
+    wchar_t *arg;
+    parse_error_list_t errors;
+
+    if (expand_string(in, output, flags, &errors) == EXPAND_ERROR)
+    {
+        if (errors.empty())
+        {
+            err(L"Bug: Parse error reported but no error text found.");
+        }
+        else
+        {
+            err(L"%ls", errors.at(0).describe(wcstring(in)).c_str());
+        }
+        return false;
+    }
+
+    wcstring_list_t expected;
+
+    va_start(va, flags);
+    while ((arg=va_arg(va, wchar_t *))!= 0)
+    {
+        expected.push_back(wcstring(arg));
+    }
+    va_end(va);
+
+    wcstring_list_t::const_iterator exp_it = expected.begin(), exp_end = expected.end();
+    std::vector<completion_t>::const_iterator out_it = output.begin(), out_end = output.end();
+    for (; exp_it != exp_end || out_it != out_end; ++exp_it, ++out_it)
+    {
+        if (exp_it == exp_end || out_it == out_end)
+        {
+            // sizes don't match
+            res = false;
+            break;
+        }
+
+        if (out_it->completion != *exp_it)
+        {
+            res = false;
+            break;
+        }
+    }
+
+    if (!res)
+    {
+        if ((arg = va_arg(va, wchar_t *)) != 0)
+        {
+            wcstring msg = L"Expected [";
+            bool first = true;
+            for (wcstring_list_t::const_iterator it = expected.begin(), end = expected.end(); it != end; ++it)
+            {
+                if (!first) msg += L", ";
+                first = false;
+                msg += '"';
+                msg += *it;
+                msg += '"';
+            }
+            msg += L"], found [";
+            first = true;
+            for (std::vector<completion_t>::const_iterator it = output.begin(), end = output.end(); it != end; ++it)
+            {
+                if (!first) msg += L", ";
+                first = false;
+                msg += '"';
+                msg += it->completion;
+                msg += '"';
+            }
+            msg += L"]";
+            err(L"%ls\n%ls", arg, msg.c_str());
+        }
+    }
+
+    va_end(va);
+
+    return res;
+
+}
+
+/**
+   Test globbing and other parameter expansion
+*/
+static void test_expand()
+{
+    say(L"Testing parameter expansion");
+
+    expand_test(L"foo", 0, L"foo", 0,
+                L"Strings do not expand to themselves");
+
+    expand_test(L"a{b,c,d}e", 0, L"abe", L"ace", L"ade", 0,
+                L"Bracket expansion is broken");
+
+    expand_test(L"a*", EXPAND_SKIP_WILDCARDS, L"a*", 0,
+                L"Cannot skip wildcard expansion");
+
+    expand_test(L"/bin/l\\0", ACCEPT_INCOMPLETE, 0,
+                L"Failed to handle null escape in expansion");
+
+    expand_test(L"foo\\$bar", EXPAND_SKIP_VARIABLES, L"foo$bar", 0,
+                L"Failed to handle dollar sign in variable-skipping expansion");
+
+    if (system("mkdir -p /tmp/fish_expand_test/")) err(L"mkdir failed");
+    if (system("touch /tmp/fish_expand_test/.foo")) err(L"touch failed");
+    if (system("touch /tmp/fish_expand_test/bar")) err(L"touch failed");
+
+    // This is checking that .* does NOT match . and .. (https://github.com/fish-shell/fish-shell/issues/270). But it does have to match literal components (e.g. "./*" has to match the same as "*"
+    expand_test(L"/tmp/fish_expand_test/.*", 0, L"/tmp/fish_expand_test/.foo", 0,
+                L"Expansion not correctly handling dotfiles");
+    expand_test(L"/tmp/fish_expand_test/./.*", 0, L"/tmp/fish_expand_test/./.foo", 0,
+                L"Expansion not correctly handling literal path components in dotfiles");
+
+    if (! expand_test(L"/tmp/fish_expand_test/.*", 0, L"/tmp/fish_expand_test/.foo", 0))
+    {
+        err(L"Expansion not correctly handling dotfiles");
+    }
+    if (! expand_test(L"/tmp/fish_expand_test/./.*", 0, L"/tmp/fish_expand_test/./.foo", 0))
+    {
+        err(L"Expansion not correctly handling literal path components in dotfiles");
+    }
+
+    if (system("rm -Rf /tmp/fish_expand_test")) err(L"rm failed");
+}
+
+static void test_fuzzy_match(void)
+{
+    say(L"Testing fuzzy string matching");
+
+    if (string_fuzzy_match_string(L"", L"").type != fuzzy_match_exact) err(L"test_fuzzy_match failed on line %ld", __LINE__);
+    if (string_fuzzy_match_string(L"alpha", L"alpha").type != fuzzy_match_exact) err(L"test_fuzzy_match failed on line %ld", __LINE__);
+    if (string_fuzzy_match_string(L"alp", L"alpha").type != fuzzy_match_prefix) err(L"test_fuzzy_match failed on line %ld", __LINE__);
+    if (string_fuzzy_match_string(L"ALPHA!", L"alPhA!").type != fuzzy_match_case_insensitive) err(L"test_fuzzy_match failed on line %ld", __LINE__);
+    if (string_fuzzy_match_string(L"alPh", L"ALPHA!").type != fuzzy_match_prefix_case_insensitive) err(L"test_fuzzy_match failed on line %ld", __LINE__);
+    if (string_fuzzy_match_string(L"LPH", L"ALPHA!").type != fuzzy_match_substring) err(L"test_fuzzy_match failed on line %ld", __LINE__);
+    if (string_fuzzy_match_string(L"AA", L"ALPHA!").type != fuzzy_match_subsequence_insertions_only) err(L"test_fuzzy_match failed on line %ld", __LINE__);
+    if (string_fuzzy_match_string(L"BB", L"ALPHA!").type != fuzzy_match_none) err(L"test_fuzzy_match failed on line %ld", __LINE__);
+}
+
+static void test_abbreviations(void)
+{
+    say(L"Testing abbreviations");
+
+    const wchar_t *abbreviations =
+        L"gc=git checkout" ARRAY_SEP_STR
+        L"foo=" ARRAY_SEP_STR
+        L"gc=something else" ARRAY_SEP_STR
+        L"=" ARRAY_SEP_STR
+        L"=foo" ARRAY_SEP_STR
+        L"foo" ARRAY_SEP_STR
+        L"foo=bar" ARRAY_SEP_STR
+        L"gx git checkout";
+
+    env_push(true);
+
+    int ret = env_set(USER_ABBREVIATIONS_VARIABLE_NAME, abbreviations, ENV_LOCAL);
+    if (ret != 0) err(L"Unable to set abbreviation variable");
+
+    wcstring result;
+    if (expand_abbreviation(L"", &result)) err(L"Unexpected success with empty abbreviation");
+    if (expand_abbreviation(L"nothing", &result)) err(L"Unexpected success with missing abbreviation");
+
+    if (! expand_abbreviation(L"gc", &result)) err(L"Unexpected failure with gc abbreviation");
+    if (result != L"git checkout") err(L"Wrong abbreviation result for gc");
+    result.clear();
+
+    if (! expand_abbreviation(L"foo", &result)) err(L"Unexpected failure with foo abbreviation");
+    if (result != L"bar") err(L"Wrong abbreviation result for foo");
+
+    bool expanded;
+    expanded = reader_expand_abbreviation_in_command(L"just a command", 3, &result);
+    if (expanded) err(L"Command wrongly expanded on line %ld", (long)__LINE__);
+    expanded = reader_expand_abbreviation_in_command(L"gc somebranch", 0, &result);
+    if (! expanded) err(L"Command not expanded on line %ld", (long)__LINE__);
+
+    expanded = reader_expand_abbreviation_in_command(L"gc somebranch", wcslen(L"gc"), &result);
+    if (! expanded) err(L"gc not expanded");
+    if (result != L"git checkout somebranch") err(L"gc incorrectly expanded on line %ld to '%ls'", (long)__LINE__, result.c_str());
+    
+    /* space separation */
+    expanded = reader_expand_abbreviation_in_command(L"gx somebranch", wcslen(L"gc"), &result);
+    if (! expanded) err(L"gx not expanded");
+    if (result != L"git checkout somebranch") err(L"gc incorrectly expanded on line %ld to '%ls'", (long)__LINE__, result.c_str());
+
+    expanded = reader_expand_abbreviation_in_command(L"echo hi ; gc somebranch", wcslen(L"echo hi ; g"), &result);
+    if (! expanded) err(L"gc not expanded on line %ld", (long)__LINE__);
+    if (result != L"echo hi ; git checkout somebranch") err(L"gc incorrectly expanded on line %ld", (long)__LINE__);
+
+    expanded = reader_expand_abbreviation_in_command(L"echo (echo (echo (echo (gc ", wcslen(L"echo (echo (echo (echo (gc"), &result);
+    if (! expanded) err(L"gc not expanded on line %ld", (long)__LINE__);
+    if (result != L"echo (echo (echo (echo (git checkout ") err(L"gc incorrectly expanded on line %ld to '%ls'", (long)__LINE__, result.c_str());
+
+    /* if commands should be expanded */
+    expanded = reader_expand_abbreviation_in_command(L"if gc", wcslen(L"if gc"), &result);
+    if (! expanded) err(L"gc not expanded on line %ld", (long)__LINE__);
+    if (result != L"if git checkout") err(L"gc incorrectly expanded on line %ld to '%ls'", (long)__LINE__, result.c_str());
+
+    /* others should not be */
+    expanded = reader_expand_abbreviation_in_command(L"of gc", wcslen(L"of gc"), &result);
+    if (expanded) err(L"gc incorrectly expanded on line %ld", (long)__LINE__);
+
+    /* others should not be */
+    expanded = reader_expand_abbreviation_in_command(L"command gc", wcslen(L"command gc"), &result);
+    if (expanded) err(L"gc incorrectly expanded on line %ld", (long)__LINE__);
+
+
+    env_pop();
+}
+
+/** Test path functions */
+static void test_path()
+{
+    say(L"Testing path functions");
+
+    wcstring path = L"//foo//////bar/";
+    path_make_canonical(path);
+    if (path != L"/foo/bar")
+    {
+        err(L"Bug in canonical PATH code");
+    }
+
+    path = L"/";
+    path_make_canonical(path);
+    if (path != L"/")
+    {
+        err(L"Bug in canonical PATH code");
+    }
+
+    if (paths_are_equivalent(L"/foo/bar/baz", L"foo/bar/baz")) err(L"Bug in canonical PATH code on line %ld", (long)__LINE__);
+    if (! paths_are_equivalent(L"///foo///bar/baz", L"/foo/bar////baz//")) err(L"Bug in canonical PATH code on line %ld", (long)__LINE__);
+    if (! paths_are_equivalent(L"/foo/bar/baz", L"/foo/bar/baz")) err(L"Bug in canonical PATH code on line %ld", (long)__LINE__);
+    if (! paths_are_equivalent(L"/", L"/")) err(L"Bug in canonical PATH code on line %ld", (long)__LINE__);
+}
+
+static void test_pager_navigation()
+{
+    say(L"Testing pager navigation");
+
+    /* Generate 19 strings of width 10. There's 2 spaces between completions, and our term size is 80; these can therefore fit into 6 columns (6 * 12 - 2 = 70) or 5 columns (58) but not 7 columns (7 * 12 - 2 = 82).
+
+       You can simulate this test by creating 19 files named "file00.txt" through "file_18.txt".
+    */
+    completion_list_t completions;
+    for (size_t i=0; i < 19; i++)
+    {
+        append_completion(completions, L"abcdefghij");
+    }
+
+    pager_t pager;
+    pager.set_completions(completions);
+    pager.set_term_size(80, 24);
+    page_rendering_t render = pager.render();
+
+    if (render.term_width != 80)
+        err(L"Wrong term width");
+    if (render.term_height != 24)
+        err(L"Wrong term height");
+
+    size_t rows = 4, cols = 5;
+
+    /* We have 19 completions. We can fit into 6 columns with 4 rows or 5 columns with 4 rows; the second one is better and so is what we ought to have picked. */
+    if (render.rows != rows)
+        err(L"Wrong row count");
+    if (render.cols != cols)
+        err(L"Wrong column count");
+
+    /* Initially expect to have no completion index */
+    if (render.selected_completion_idx != (size_t)(-1))
+    {
+        err(L"Wrong initial selection");
+    }
+
+    /* Here are navigation directions and where we expect the selection to be */
+    const struct
+    {
+        selection_direction_t dir;
+        size_t sel;
+    }
+    cmds[] =
+    {
+        /* Tab completion to get into the list */
+        {direction_next, 0},
+
+        /* Westward motion in upper left wraps along the top row */
+        {direction_west, 16},
+        {direction_east, 1},
+
+        /* "Next" motion goes down the column */
+        {direction_next, 2},
+        {direction_next, 3},
+
+        {direction_west, 18},
+        {direction_east, 3},
+        {direction_east, 7},
+        {direction_east, 11},
+        {direction_east, 15},
+        {direction_east, 3},
+
+        {direction_west, 18},
+        {direction_east, 3},
+
+        /* Eastward motion wraps along the bottom, westward goes to the prior column */
+        {direction_east, 7},
+        {direction_east, 11},
+        {direction_east, 15},
+        {direction_east, 3},
+
+        /* Column memory */
+        {direction_west, 18},
+        {direction_south, 15},
+        {direction_north, 18},
+        {direction_west, 14},
+        {direction_south, 15},
+        {direction_north, 14},
+
+        /* pages */
+        {direction_page_north, 12},
+        {direction_page_south, 15},
+        {direction_page_north, 12},
+        {direction_east, 16},
+        {direction_page_south, 18},
+        {direction_east, 3},
+        {direction_north, 2},
+        {direction_page_north, 0},
+        {direction_page_south, 3},
+
+    };
+    for (size_t i=0; i < sizeof cmds / sizeof *cmds; i++)
+    {
+        pager.select_next_completion_in_direction(cmds[i].dir, render);
+        pager.update_rendering(&render);
+        if (cmds[i].sel != render.selected_completion_idx)
+        {
+            err(L"For command %lu, expected selection %lu, but found instead %lu\n", i, cmds[i].sel, render.selected_completion_idx);
+        }
+    }
+
+}
+
+enum word_motion_t
+{
+    word_motion_left,
+    word_motion_right
+};
+static void test_1_word_motion(word_motion_t motion, move_word_style_t style, const wcstring &test)
+{
+    wcstring command;
+    std::set<size_t> stops;
+
+    // Carets represent stops and should be cut out of the command
+    for (size_t i=0; i < test.size(); i++)
+    {
+        wchar_t wc = test.at(i);
+        if (wc == L'^')
+        {
+            stops.insert(command.size());
+        }
+        else
+        {
+            command.push_back(wc);
+        }
+    }
+
+    size_t idx, end;
+    if (motion == word_motion_left)
+    {
+        idx = command.size();
+        end = 0;
+    }
+    else
+    {
+        idx = 0;
+        end = command.size();
+    }
+
+    move_word_state_machine_t sm(style);
+    while (idx != end)
+    {
+        size_t char_idx = (motion == word_motion_left ? idx - 1 : idx);
+        wchar_t wc = command.at(char_idx);
+        bool will_stop = ! sm.consume_char(wc);
+        //printf("idx %lu, looking at %lu (%c): %d\n", idx, char_idx, (char)wc, will_stop);
+        bool expected_stop = (stops.count(idx) > 0);
+        if (will_stop != expected_stop)
+        {
+            wcstring tmp = command;
+            tmp.insert(idx, L"^");
+            const char *dir = (motion == word_motion_left ? "left" : "right");
+            if (will_stop)
+            {
+                err(L"Word motion: moving %s, unexpected stop at idx %lu: '%ls'", dir, idx, tmp.c_str());
+            }
+            else if (! will_stop && expected_stop)
+            {
+                err(L"Word motion: moving %s, should have stopped at idx %lu: '%ls'", dir, idx, tmp.c_str());
+            }
+        }
+        // We don't expect to stop here next time
+        if (expected_stop)
+        {
+            stops.erase(idx);
+        }
+        if (will_stop)
+        {
+            sm.reset();
+        }
+        else
+        {
+            idx += (motion == word_motion_left ? -1 : 1);
+        }
+    }
+}
+
+/** Test word motion (forward-word, etc.). Carets represent cursor stops. */
+static void test_word_motion()
+{
+    say(L"Testing word motion");
+    test_1_word_motion(word_motion_left, move_word_style_punctuation, L"^echo ^hello_^world.^txt");
+    test_1_word_motion(word_motion_right, move_word_style_punctuation, L"echo^ hello^_world^.txt^");
+
+    test_1_word_motion(word_motion_left, move_word_style_punctuation, L"echo ^foo_^foo_^foo/^/^/^/^/^    ");
+    test_1_word_motion(word_motion_right, move_word_style_punctuation, L"echo^ foo^_foo^_foo^/^/^/^/^/    ^");
+
+    test_1_word_motion(word_motion_left, move_word_style_path_components, L"^/^foo/^bar/^baz/");
+    test_1_word_motion(word_motion_left, move_word_style_path_components, L"^echo ^--foo ^--bar");
+    test_1_word_motion(word_motion_left, move_word_style_path_components, L"^echo ^hi ^> /^dev/^null");
+
+    test_1_word_motion(word_motion_left, move_word_style_path_components, L"^echo /^foo/^bar{^aaa,^bbb,^ccc}^bak/");
+}
+
+/** Test is_potential_path */
+static void test_is_potential_path()
+{
+    say(L"Testing is_potential_path");
+    if (system("rm -Rf /tmp/is_potential_path_test/"))
+    {
+        err(L"Failed to remove /tmp/is_potential_path_test/");
+    }
+
+    /* Directories */
+    if (system("mkdir -p /tmp/is_potential_path_test/alpha/")) err(L"mkdir failed");
+    if (system("mkdir -p /tmp/is_potential_path_test/beta/")) err(L"mkdir failed");
+
+    /* Files */
+    if (system("touch /tmp/is_potential_path_test/aardvark")) err(L"touch failed");
+    if (system("touch /tmp/is_potential_path_test/gamma")) err(L"touch failed");
+
+    const wcstring wd = L"/tmp/is_potential_path_test/";
+    const wcstring_list_t wds(1, wd);
+
+    wcstring tmp;
+    do_test(is_potential_path(L"al", wds, PATH_REQUIRE_DIR, &tmp) && tmp == L"alpha/");
+    do_test(is_potential_path(L"alpha/", wds, PATH_REQUIRE_DIR, &tmp) && tmp == L"alpha/");
+    do_test(is_potential_path(L"aard", wds, 0, &tmp) && tmp == L"aardvark");
+
+    do_test(! is_potential_path(L"balpha/", wds, PATH_REQUIRE_DIR, &tmp));
+    do_test(! is_potential_path(L"aard", wds, PATH_REQUIRE_DIR, &tmp));
+    do_test(! is_potential_path(L"aarde", wds, PATH_REQUIRE_DIR, &tmp));
+    do_test(! is_potential_path(L"aarde", wds, 0, &tmp));
+
+    do_test(is_potential_path(L"/tmp/is_potential_path_test/aardvark", wds, 0, &tmp) && tmp == L"/tmp/is_potential_path_test/aardvark");
+    do_test(is_potential_path(L"/tmp/is_potential_path_test/al", wds, PATH_REQUIRE_DIR, &tmp) && tmp == L"/tmp/is_potential_path_test/alpha/");
+    do_test(is_potential_path(L"/tmp/is_potential_path_test/aardv", wds, 0, &tmp) && tmp == L"/tmp/is_potential_path_test/aardvark");
+
+    do_test(! is_potential_path(L"/tmp/is_potential_path_test/aardvark", wds, PATH_REQUIRE_DIR, &tmp));
+    do_test(! is_potential_path(L"/tmp/is_potential_path_test/al/", wds, 0, &tmp));
+    do_test(! is_potential_path(L"/tmp/is_potential_path_test/ar", wds, 0, &tmp));
+
+    do_test(is_potential_path(L"/usr", wds, PATH_REQUIRE_DIR, &tmp) && tmp == L"/usr/");
+
+}
+
+/** Test the 'test' builtin */
+int builtin_test(parser_t &parser, wchar_t **argv);
+static bool run_one_test_test(int expected, wcstring_list_t &lst, bool bracket)
+{
+    parser_t parser(PARSER_TYPE_GENERAL, true);
+    size_t i, count = lst.size();
+    wchar_t **argv = new wchar_t *[count+3];
+    argv[0] = (wchar_t *)(bracket ? L"[" : L"test");
+    for (i=0; i < count; i++)
+    {
+        argv[i+1] = (wchar_t *)lst.at(i).c_str();
+    }
+    if (bracket)
+    {
+        argv[i+1] = (wchar_t *)L"]";
+        i++;
+    }
+    argv[i+1] = NULL;
+    int result = builtin_test(parser, argv);
+    delete[] argv;
+    return expected == result;
+}
+
+static bool run_test_test(int expected, const wcstring &str)
+{
+    using namespace std;
+    wcstring_list_t lst;
+
+    wistringstream iss(str);
+    copy(istream_iterator<wcstring, wchar_t, std::char_traits<wchar_t> >(iss),
+         istream_iterator<wstring, wchar_t, std::char_traits<wchar_t> >(),
+         back_inserter<vector<wcstring> >(lst));
+
+    bool bracket = run_one_test_test(expected, lst, true);
+    bool nonbracket = run_one_test_test(expected, lst, false);
+    do_test(bracket == nonbracket);
+    return nonbracket;
+}
+
+static void test_test_brackets()
+{
+    // Ensure [ knows it needs a ]
+    parser_t parser(PARSER_TYPE_GENERAL, true);
+
+    const wchar_t *argv1[] = {L"[", L"foo", NULL};
+    do_test(builtin_test(parser, (wchar_t **)argv1) != 0);
+
+    const wchar_t *argv2[] = {L"[", L"foo", L"]", NULL};
+    do_test(builtin_test(parser, (wchar_t **)argv2) == 0);
+
+    const wchar_t *argv3[] = {L"[", L"foo", L"]", L"bar", NULL};
+    do_test(builtin_test(parser, (wchar_t **)argv3) != 0);
+
+}
+
+static void test_test()
+{
+    say(L"Testing test builtin");
+    test_test_brackets();
+
+    do_test(run_test_test(0, L"5 -ne 6"));
+    do_test(run_test_test(0, L"5 -eq 5"));
+    do_test(run_test_test(0, L"0 -eq 0"));
+    do_test(run_test_test(0, L"-1 -eq -1"));
+    do_test(run_test_test(0, L"1 -ne -1"));
+    do_test(run_test_test(1, L"-1 -ne -1"));
+    do_test(run_test_test(0, L"abc != def"));
+    do_test(run_test_test(1, L"abc = def"));
+    do_test(run_test_test(0, L"5 -le 10"));
+    do_test(run_test_test(0, L"10 -le 10"));
+    do_test(run_test_test(1, L"20 -le 10"));
+    do_test(run_test_test(0, L"-1 -le 0"));
+    do_test(run_test_test(1, L"0 -le -1"));
+    do_test(run_test_test(0, L"15 -ge 10"));
+    do_test(run_test_test(0, L"15 -ge 10"));
+    do_test(run_test_test(1, L"! 15 -ge 10"));
+    do_test(run_test_test(0, L"! ! 15 -ge 10"));
+
+    do_test(run_test_test(0, L"0 -ne 1 -a 0 -eq 0"));
+    do_test(run_test_test(0, L"0 -ne 1 -a -n 5"));
+    do_test(run_test_test(0, L"-n 5 -a 10 -gt 5"));
+    do_test(run_test_test(0, L"-n 3 -a -n 5"));
+
+    /* test precedence:
+            '0 == 0 || 0 == 1 && 0 == 2'
+        should be evaluated as:
+            '0 == 0 || (0 == 1 && 0 == 2)'
+        and therefore true. If it were
+            '(0 == 0 || 0 == 1) && 0 == 2'
+        it would be false. */
+    do_test(run_test_test(0, L"0 = 0 -o 0 = 1 -a 0 = 2"));
+    do_test(run_test_test(0, L"-n 5 -o 0 = 1 -a 0 = 2"));
+    do_test(run_test_test(1, L"( 0 = 0 -o  0 = 1 ) -a 0 = 2"));
+    do_test(run_test_test(0, L"0 = 0 -o ( 0 = 1 -a 0 = 2 )"));
+
+    /* A few lame tests for permissions; these need to be a lot more complete. */
+    do_test(run_test_test(0, L"-e /bin/ls"));
+    do_test(run_test_test(1, L"-e /bin/ls_not_a_path"));
+    do_test(run_test_test(0, L"-x /bin/ls"));
+    do_test(run_test_test(1, L"-x /bin/ls_not_a_path"));
+    do_test(run_test_test(0, L"-d /bin/"));
+    do_test(run_test_test(1, L"-d /bin/ls"));
+
+    /* This failed at one point */
+    do_test(run_test_test(1, L"-d /bin -a 5 -eq 3"));
+    do_test(run_test_test(0, L"-d /bin -o 5 -eq 3"));
+    do_test(run_test_test(0, L"-d /bin -a ! 5 -eq 3"));
+
+    /* We didn't properly handle multiple "just strings" either */
+    do_test(run_test_test(0, L"foo"));
+    do_test(run_test_test(0, L"foo -a bar"));
+
+    /* These should be errors */
+    do_test(run_test_test(1, L"foo bar"));
+    do_test(run_test_test(1, L"foo bar baz"));
+
+    /* This crashed */
+    do_test(run_test_test(1, L"1 = 1 -a = 1"));
+
+    /* Make sure we can treat -S as a parameter instead of an operator. https://github.com/fish-shell/fish-shell/issues/601 */
+    do_test(run_test_test(0, L"-S = -S"));
+    do_test(run_test_test(1, L"! ! ! A"));
+}
+
+/** Testing colors */
+static void test_colors()
+{
+    say(L"Testing colors");
+    do_test(rgb_color_t(L"#FF00A0").is_rgb());
+    do_test(rgb_color_t(L"FF00A0").is_rgb());
+    do_test(rgb_color_t(L"#F30").is_rgb());
+    do_test(rgb_color_t(L"F30").is_rgb());
+    do_test(rgb_color_t(L"f30").is_rgb());
+    do_test(rgb_color_t(L"#FF30a5").is_rgb());
+    do_test(rgb_color_t(L"3f30").is_none());
+    do_test(rgb_color_t(L"##f30").is_none());
+    do_test(rgb_color_t(L"magenta").is_named());
+    do_test(rgb_color_t(L"MaGeNTa").is_named());
+    do_test(rgb_color_t(L"mooganta").is_none());
+}
+
+static void test_complete(void)
+{
+    say(L"Testing complete");
+
+    const wchar_t *name_strs[] = {L"Foo1", L"Foo2", L"Foo3", L"Bar1", L"Bar2", L"Bar3"};
+    size_t count = sizeof name_strs / sizeof *name_strs;
+    const wcstring_list_t names(name_strs, name_strs + count);
+
+    complete_set_variable_names(&names);
+
+    std::vector<completion_t> completions;
+    complete(L"$F", completions, COMPLETION_REQUEST_DEFAULT);
+    do_test(completions.size() == 3);
+    do_test(completions.at(0).completion == L"oo1");
+    do_test(completions.at(1).completion == L"oo2");
+    do_test(completions.at(2).completion == L"oo3");
+
+    completions.clear();
+    complete(L"$1", completions, COMPLETION_REQUEST_DEFAULT);
+    do_test(completions.empty());
+
+    completions.clear();
+    complete(L"$1", completions, COMPLETION_REQUEST_DEFAULT | COMPLETION_REQUEST_FUZZY_MATCH);
+    do_test(completions.size() == 2);
+    do_test(completions.at(0).completion == L"$Foo1");
+    do_test(completions.at(1).completion == L"$Bar1");
+
+    completions.clear();
+    complete(L"echo (/bin/mkdi", completions, COMPLETION_REQUEST_DEFAULT);
+    do_test(completions.size() == 1);
+    do_test(completions.at(0).completion == L"r");
+
+    completions.clear();
+    complete(L"echo (ls /bin/mkdi", completions, COMPLETION_REQUEST_DEFAULT);
+    do_test(completions.size() == 1);
+    do_test(completions.at(0).completion == L"r");
+
+    completions.clear();
+    complete(L"echo (command ls /bin/mkdi", completions, COMPLETION_REQUEST_DEFAULT);
+    do_test(completions.size() == 1);
+    do_test(completions.at(0).completion == L"r");
+
+    /* Add a function and test completing it in various ways */
+    struct function_data_t func_data = {};
+    func_data.name = L"scuttlebutt";
+    func_data.definition = L"echo gongoozle";
+    function_add(func_data, parser_t::principal_parser());
+
+    /* Complete a function name */
+    completions.clear();
+    complete(L"echo (scuttlebut", completions, COMPLETION_REQUEST_DEFAULT);
+    do_test(completions.size() == 1);
+    do_test(completions.at(0).completion == L"t");
+
+    /* But not with the command prefix */
+    completions.clear();
+    complete(L"echo (command scuttlebut", completions, COMPLETION_REQUEST_DEFAULT);
+    do_test(completions.size() == 0);
+
+    /* Not with the builtin prefix */
+    completions.clear();
+    complete(L"echo (builtin scuttlebut", completions, COMPLETION_REQUEST_DEFAULT);
+    do_test(completions.size() == 0);
+
+    /* Not after a redirection */
+    completions.clear();
+    complete(L"echo hi > scuttlebut", completions, COMPLETION_REQUEST_DEFAULT);
+    do_test(completions.size() == 0);
+
+    /* Trailing spaces (#1261) */
+    complete_add(L"foobarbaz", false, 0, NULL, 0, NO_FILES, NULL, L"qux", NULL, COMPLETE_AUTO_SPACE);
+    completions.clear();
+    complete(L"foobarbaz ", completions, COMPLETION_REQUEST_DEFAULT);
+    do_test(completions.size() == 1);
+    do_test(completions.at(0).completion == L"qux");
+
+    /* Don't complete variable names in single quotes (#1023) */
+    completions.clear();
+    complete(L"echo '$Foo", completions, COMPLETION_REQUEST_DEFAULT);
+    do_test(completions.empty());
+    completions.clear();
+    complete(L"echo \\$Foo", completions, COMPLETION_REQUEST_DEFAULT);
+    do_test(completions.empty());
+    
+    /* File completions */
+    char saved_wd[PATH_MAX + 1] = {};
+    getcwd(saved_wd, sizeof saved_wd);
+    if (system("mkdir -p '/tmp/complete_test/'")) err(L"mkdir failed");
+    if (system("touch '/tmp/complete_test/testfile'")) err(L"touch failed");
+    if (chdir("/tmp/complete_test/")) err(L"chdir failed");
+    complete(L"cat te", completions, COMPLETION_REQUEST_DEFAULT);
+    do_test(completions.size() == 1);
+    do_test(completions.at(0).completion == L"stfile");
+    completions.clear();
+    complete(L"cat /tmp/complete_test/te", completions, COMPLETION_REQUEST_DEFAULT);
+    do_test(completions.size() == 1);
+    do_test(completions.at(0).completion == L"stfile");
+    completions.clear();
+    complete(L"echo sup > /tmp/complete_test/te", completions, COMPLETION_REQUEST_DEFAULT);
+    do_test(completions.size() == 1);
+    do_test(completions.at(0).completion == L"stfile");
+    completions.clear();
+    complete(L"echo sup > /tmp/complete_test/te", completions, COMPLETION_REQUEST_DEFAULT);
+    do_test(completions.size() == 1);
+    do_test(completions.at(0).completion == L"stfile");
+    completions.clear();
+    
+    // Zero escapes can cause problems. See #1631
+    complete(L"cat foo\\0", completions, COMPLETION_REQUEST_DEFAULT);
+    do_test(completions.empty());
+    completions.clear();
+    complete(L"cat foo\\0bar", completions, COMPLETION_REQUEST_DEFAULT);
+    do_test(completions.empty());
+    completions.clear();
+    complete(L"cat \\0", completions, COMPLETION_REQUEST_DEFAULT);
+    do_test(completions.empty());
+    completions.clear();
+    complete(L"cat te\\0", completions, COMPLETION_REQUEST_DEFAULT);
+    do_test(completions.empty());
+    completions.clear();
+
+    if (chdir(saved_wd)) err(L"chdir failed");
+    if (system("rm -Rf '/tmp/complete_test/'")) err(L"rm failed");
+    
+    complete_set_variable_names(NULL);
+    
+    /* Test wraps */
+    do_test(comma_join(complete_get_wrap_chain(L"wrapper1")) == L"wrapper1");
+    complete_add_wrapper(L"wrapper1", L"wrapper2");
+    do_test(comma_join(complete_get_wrap_chain(L"wrapper1")) == L"wrapper1,wrapper2");
+    complete_add_wrapper(L"wrapper2", L"wrapper3");
+    do_test(comma_join(complete_get_wrap_chain(L"wrapper1")) == L"wrapper1,wrapper2,wrapper3");
+    complete_add_wrapper(L"wrapper3", L"wrapper1"); //loop!
+    do_test(comma_join(complete_get_wrap_chain(L"wrapper1")) == L"wrapper1,wrapper2,wrapper3");
+    complete_remove_wrapper(L"wrapper1", L"wrapper2");
+    do_test(comma_join(complete_get_wrap_chain(L"wrapper1")) == L"wrapper1");
+    do_test(comma_join(complete_get_wrap_chain(L"wrapper2")) == L"wrapper2,wrapper3,wrapper1");
+}
+
+static void test_1_completion(wcstring line, const wcstring &completion, complete_flags_t flags, bool append_only, wcstring expected, long source_line)
+{
+    // str is given with a caret, which we use to represent the cursor position
+    // find it
+    const size_t in_cursor_pos = line.find(L'^');
+    do_test(in_cursor_pos != wcstring::npos);
+    line.erase(in_cursor_pos, 1);
+
+    const size_t out_cursor_pos = expected.find(L'^');
+    do_test(out_cursor_pos != wcstring::npos);
+    expected.erase(out_cursor_pos, 1);
+
+    size_t cursor_pos = in_cursor_pos;
+    wcstring result = completion_apply_to_command_line(completion, flags, line, &cursor_pos, append_only);
+    if (result != expected)
+    {
+        fprintf(stderr, "line %ld: %ls + %ls -> [%ls], expected [%ls]\n", source_line, line.c_str(), completion.c_str(), result.c_str(), expected.c_str());
+    }
+    do_test(result == expected);
+    do_test(cursor_pos == out_cursor_pos);
+}
+
+static void test_completion_insertions()
+{
+#define TEST_1_COMPLETION(a, b, c, d, e) test_1_completion(a, b, c, d, e, __LINE__)
+    say(L"Testing completion insertions");
+    TEST_1_COMPLETION(L"foo^", L"bar", 0, false, L"foobar ^");
+    TEST_1_COMPLETION(L"foo^ baz", L"bar", 0, false, L"foobar ^ baz"); //we really do want to insert two spaces here - otherwise it's hidden by the cursor
+    TEST_1_COMPLETION(L"'foo^", L"bar", 0, false, L"'foobar' ^");
+    TEST_1_COMPLETION(L"'foo'^", L"bar", 0, false, L"'foobar' ^");
+    TEST_1_COMPLETION(L"'foo\\'^", L"bar", 0, false, L"'foo\\'bar' ^");
+    TEST_1_COMPLETION(L"foo\\'^", L"bar", 0, false, L"foo\\'bar ^");
+
+    // Test append only
+    TEST_1_COMPLETION(L"foo^", L"bar", 0, true, L"foobar ^");
+    TEST_1_COMPLETION(L"foo^ baz", L"bar", 0, true, L"foobar ^ baz");
+    TEST_1_COMPLETION(L"'foo^", L"bar", 0, true, L"'foobar' ^");
+    TEST_1_COMPLETION(L"'foo'^", L"bar", 0, true, L"'foo'bar ^");
+    TEST_1_COMPLETION(L"'foo\\'^", L"bar", 0, true, L"'foo\\'bar' ^");
+    TEST_1_COMPLETION(L"foo\\'^", L"bar", 0, true, L"foo\\'bar ^");
+
+    TEST_1_COMPLETION(L"foo^", L"bar", COMPLETE_NO_SPACE, false, L"foobar^");
+    TEST_1_COMPLETION(L"'foo^", L"bar", COMPLETE_NO_SPACE, false, L"'foobar^");
+    TEST_1_COMPLETION(L"'foo'^", L"bar", COMPLETE_NO_SPACE, false, L"'foobar'^");
+    TEST_1_COMPLETION(L"'foo\\'^", L"bar", COMPLETE_NO_SPACE, false, L"'foo\\'bar^");
+    TEST_1_COMPLETION(L"foo\\'^", L"bar", COMPLETE_NO_SPACE, false, L"foo\\'bar^");
+
+    TEST_1_COMPLETION(L"foo^", L"bar", COMPLETE_REPLACES_TOKEN, false, L"bar ^");
+    TEST_1_COMPLETION(L"'foo^", L"bar", COMPLETE_REPLACES_TOKEN, false, L"bar ^");
+}
+
+static void perform_one_autosuggestion_special_test(const wcstring &command, const wcstring &wd, const wcstring &expected, long line)
+{
+    wcstring suggestion;
+    bool success = autosuggest_suggest_special(command, wd, suggestion);
+    if (! success)
+    {
+        printf("line %ld: autosuggest_suggest_special() failed for command %ls\n", line, command.c_str());
+        do_test(success);
+    }
+    if (suggestion != expected)
+    {
+        printf("line %ld: autosuggest_suggest_special() returned the wrong expected string for command %ls\n", line, command.c_str());
+        printf("  actual: %ls\n", suggestion.c_str());
+        printf("expected: %ls\n", expected.c_str());
+        do_test(suggestion == expected);
+    }
+}
+
+/* Testing test_autosuggest_suggest_special, in particular for properly handling quotes and backslashes */
+static void test_autosuggest_suggest_special()
+{
+    if (system("mkdir -p '/tmp/autosuggest_test/0foobar'")) err(L"mkdir failed");
+    if (system("mkdir -p '/tmp/autosuggest_test/1foo bar'")) err(L"mkdir failed");
+    if (system("mkdir -p '/tmp/autosuggest_test/2foo  bar'")) err(L"mkdir failed");
+    if (system("mkdir -p '/tmp/autosuggest_test/3foo\\bar'")) err(L"mkdir failed");
+    if (system("mkdir -p /tmp/autosuggest_test/4foo\\'bar")) err(L"mkdir failed"); //a path with a single quote
+    if (system("mkdir -p /tmp/autosuggest_test/5foo\\\"bar")) err(L"mkdir failed"); //a path with a double quote
+    if (system("mkdir -p ~/test_autosuggest_suggest_special/")) err(L"mkdir failed"); //make sure tilde is handled
+
+    const wcstring wd = L"/tmp/autosuggest_test/";
+    perform_one_autosuggestion_special_test(L"cd /tmp/autosuggest_test/0", wd, L"cd /tmp/autosuggest_test/0foobar/", __LINE__);
+    perform_one_autosuggestion_special_test(L"cd \"/tmp/autosuggest_test/0", wd, L"cd \"/tmp/autosuggest_test/0foobar/\"", __LINE__);
+    perform_one_autosuggestion_special_test(L"cd '/tmp/autosuggest_test/0", wd, L"cd '/tmp/autosuggest_test/0foobar/'", __LINE__);
+    perform_one_autosuggestion_special_test(L"cd 0", wd, L"cd 0foobar/", __LINE__);
+    perform_one_autosuggestion_special_test(L"cd \"0", wd, L"cd \"0foobar/\"", __LINE__);
+    perform_one_autosuggestion_special_test(L"cd '0", wd, L"cd '0foobar/'", __LINE__);
+
+    perform_one_autosuggestion_special_test(L"cd /tmp/autosuggest_test/1", wd, L"cd /tmp/autosuggest_test/1foo\\ bar/", __LINE__);
+    perform_one_autosuggestion_special_test(L"cd \"/tmp/autosuggest_test/1", wd, L"cd \"/tmp/autosuggest_test/1foo bar/\"", __LINE__);
+    perform_one_autosuggestion_special_test(L"cd '/tmp/autosuggest_test/1", wd, L"cd '/tmp/autosuggest_test/1foo bar/'", __LINE__);
+    perform_one_autosuggestion_special_test(L"cd 1", wd, L"cd 1foo\\ bar/", __LINE__);
+    perform_one_autosuggestion_special_test(L"cd \"1", wd, L"cd \"1foo bar/\"", __LINE__);
+    perform_one_autosuggestion_special_test(L"cd '1", wd, L"cd '1foo bar/'", __LINE__);
+
+    perform_one_autosuggestion_special_test(L"cd /tmp/autosuggest_test/2", wd, L"cd /tmp/autosuggest_test/2foo\\ \\ bar/", __LINE__);
+    perform_one_autosuggestion_special_test(L"cd \"/tmp/autosuggest_test/2", wd, L"cd \"/tmp/autosuggest_test/2foo  bar/\"", __LINE__);
+    perform_one_autosuggestion_special_test(L"cd '/tmp/autosuggest_test/2", wd, L"cd '/tmp/autosuggest_test/2foo  bar/'", __LINE__);
+    perform_one_autosuggestion_special_test(L"cd 2", wd, L"cd 2foo\\ \\ bar/", __LINE__);
+    perform_one_autosuggestion_special_test(L"cd \"2", wd, L"cd \"2foo  bar/\"", __LINE__);
+    perform_one_autosuggestion_special_test(L"cd '2", wd, L"cd '2foo  bar/'", __LINE__);
+
+    perform_one_autosuggestion_special_test(L"cd /tmp/autosuggest_test/3", wd, L"cd /tmp/autosuggest_test/3foo\\\\bar/", __LINE__);
+    perform_one_autosuggestion_special_test(L"cd \"/tmp/autosuggest_test/3", wd, L"cd \"/tmp/autosuggest_test/3foo\\bar/\"", __LINE__);
+    perform_one_autosuggestion_special_test(L"cd '/tmp/autosuggest_test/3", wd, L"cd '/tmp/autosuggest_test/3foo\\bar/'", __LINE__);
+    perform_one_autosuggestion_special_test(L"cd 3", wd, L"cd 3foo\\\\bar/", __LINE__);
+    perform_one_autosuggestion_special_test(L"cd \"3", wd, L"cd \"3foo\\bar/\"", __LINE__);
+    perform_one_autosuggestion_special_test(L"cd '3", wd, L"cd '3foo\\bar/'", __LINE__);
+
+    perform_one_autosuggestion_special_test(L"cd /tmp/autosuggest_test/4", wd, L"cd /tmp/autosuggest_test/4foo\\'bar/", __LINE__);
+    perform_one_autosuggestion_special_test(L"cd \"/tmp/autosuggest_test/4", wd, L"cd \"/tmp/autosuggest_test/4foo'bar/\"", __LINE__);
+    perform_one_autosuggestion_special_test(L"cd '/tmp/autosuggest_test/4", wd, L"cd '/tmp/autosuggest_test/4foo\\'bar/'", __LINE__);
+    perform_one_autosuggestion_special_test(L"cd 4", wd, L"cd 4foo\\'bar/", __LINE__);
+    perform_one_autosuggestion_special_test(L"cd \"4", wd, L"cd \"4foo'bar/\"", __LINE__);
+    perform_one_autosuggestion_special_test(L"cd '4", wd, L"cd '4foo\\'bar/'", __LINE__);
+
+    perform_one_autosuggestion_special_test(L"cd /tmp/autosuggest_test/5", wd, L"cd /tmp/autosuggest_test/5foo\\\"bar/", __LINE__);
+    perform_one_autosuggestion_special_test(L"cd \"/tmp/autosuggest_test/5", wd, L"cd \"/tmp/autosuggest_test/5foo\\\"bar/\"", __LINE__);
+    perform_one_autosuggestion_special_test(L"cd '/tmp/autosuggest_test/5", wd, L"cd '/tmp/autosuggest_test/5foo\"bar/'", __LINE__);
+    perform_one_autosuggestion_special_test(L"cd 5", wd, L"cd 5foo\\\"bar/", __LINE__);
+    perform_one_autosuggestion_special_test(L"cd \"5", wd, L"cd \"5foo\\\"bar/\"", __LINE__);
+    perform_one_autosuggestion_special_test(L"cd '5", wd, L"cd '5foo\"bar/'", __LINE__);
+
+    perform_one_autosuggestion_special_test(L"cd ~/test_autosuggest_suggest_specia", wd, L"cd ~/test_autosuggest_suggest_special/", __LINE__);
+    
+    // A single quote should defeat tilde expansion
+    perform_one_autosuggestion_special_test(L"cd '~/test_autosuggest_suggest_specia'", wd, L"", __LINE__);
+
+    if (system("rm -Rf '/tmp/autosuggest_test/'")) err(L"rm failed");
+    if (system("rm -Rf ~/test_autosuggest_suggest_special/")) err(L"rm failed");
+}
+
+static void perform_one_autosuggestion_should_ignore_test(const wcstring &command, const wcstring &wd, long line)
+{
+    completion_list_t comps;
+    complete(command, comps, COMPLETION_REQUEST_AUTOSUGGESTION);
+    do_test(comps.empty());
+    if (! comps.empty())
+    {
+        const wcstring &suggestion = comps.front().completion;
+        printf("line %ld: complete() expected to return nothing for %ls\n", line, command.c_str());
+        printf("  instead got: %ls\n", suggestion.c_str());
+    }
+}
+
+static void test_autosuggestion_ignores()
+{
+    say(L"Testing scenarios that should produce no autosuggestions");
+    const wcstring wd = L"/tmp/autosuggest_test/";
+    // Do not do file autosuggestions immediately after certain statement terminators - see #1631
+    perform_one_autosuggestion_should_ignore_test(L"echo PIPE_TEST|", wd, __LINE__);
+    perform_one_autosuggestion_should_ignore_test(L"echo PIPE_TEST&", wd, __LINE__);
+    perform_one_autosuggestion_should_ignore_test(L"echo PIPE_TEST#comment", wd, __LINE__);
+    perform_one_autosuggestion_should_ignore_test(L"echo PIPE_TEST;", wd, __LINE__);
+}
+
+static void test_autosuggestion_combining()
+{
+    say(L"Testing autosuggestion combining");
+    do_test(combine_command_and_autosuggestion(L"alpha", L"alphabeta") == L"alphabeta");
+
+    // when the last token contains no capital letters, we use the case of the autosuggestion
+    do_test(combine_command_and_autosuggestion(L"alpha", L"ALPHABETA") == L"ALPHABETA");
+
+    // when the last token contains capital letters, we use its case
+    do_test(combine_command_and_autosuggestion(L"alPha", L"alphabeTa") == L"alPhabeTa");
+
+    // if autosuggestion is not longer than input, use the input's case
+    do_test(combine_command_and_autosuggestion(L"alpha", L"ALPHAA") == L"ALPHAA");
+    do_test(combine_command_and_autosuggestion(L"alpha", L"ALPHA") == L"alpha");
+}
+
+
+/**
+   Test speed of completion calculations
+*/
+void perf_complete()
+{
+    wchar_t c;
+    std::vector<completion_t> out;
+    long long t1, t2;
+    int matches=0;
+    double t;
+    wchar_t str[3]=
+    {
+        0, 0, 0
+    }
+    ;
+    int i;
+
+
+    say(L"Testing completion performance");
+
+    reader_push(L"");
+    say(L"Here we go");
+
+    t1 = get_time();
+
+
+    for (c=L'a'; c<=L'z'; c++)
+    {
+        str[0]=c;
+        reader_set_buffer(str, 0);
+
+        complete(str, out, COMPLETION_REQUEST_DEFAULT);
+
+        matches += out.size();
+        out.clear();
+    }
+    t2=get_time();
+
+    t = (double)(t2-t1)/(1000000*26);
+
+    say(L"One letter command completion took %f seconds per completion, %f microseconds/match", t, (double)(t2-t1)/matches);
+
+    matches=0;
+    t1 = get_time();
+    for (i=0; i<LAPS; i++)
+    {
+        str[0]='a'+(rand()%26);
+        str[1]='a'+(rand()%26);
+
+        reader_set_buffer(str, 0);
+
+        complete(str, out, COMPLETION_REQUEST_DEFAULT);
+
+        matches += out.size();
+        out.clear();
+    }
+    t2=get_time();
+
+    t = (double)(t2-t1)/(1000000*LAPS);
+
+    say(L"Two letter command completion took %f seconds per completion, %f microseconds/match", t, (double)(t2-t1)/matches);
+
+    reader_pop();
+
+}
+
+static void test_history_matches(history_search_t &search, size_t matches)
+{
+    size_t i;
+    for (i=0; i < matches; i++)
+    {
+        do_test(search.go_backwards());
+        wcstring item = search.current_string();
+    }
+    do_test(! search.go_backwards());
+
+    for (i=1; i < matches; i++)
+    {
+        do_test(search.go_forwards());
+    }
+    do_test(! search.go_forwards());
+}
+
+static bool history_contains(history_t *history, const wcstring &txt)
+{
+    bool result = false;
+    size_t i;
+    for (i=1; ; i++)
+    {
+        history_item_t item = history->item_at_index(i);
+        if (item.empty())
+            break;
+
+        if (item.str() == txt)
+        {
+            result = true;
+            break;
+        }
+    }
+    return result;
+}
+
+static void test_input()
+{
+    say(L"Testing input");
+    /* Ensure sequences are order independent. Here we add two bindings where the first is a prefix of the second, and then emit the second key list. The second binding should be invoked, not the first! */
+    wcstring prefix_binding = L"qqqqqqqa";
+    wcstring desired_binding = prefix_binding + L'a';
+    input_mapping_add(prefix_binding.c_str(), L"up-line");
+    input_mapping_add(desired_binding.c_str(), L"down-line");
+
+    /* Push the desired binding to the queue */
+    for (size_t idx = 0; idx < desired_binding.size(); idx++) {
+        input_queue_ch(desired_binding.at(idx));
+    }
+
+    /* Now test */
+    wint_t c = input_readch();
+    if (c != R_DOWN_LINE)
+    {
+        err(L"Expected to read char R_DOWN_LINE, but instead got %ls\n", describe_char(c).c_str());
+    }
+}
+
+#define UVARS_PER_THREAD 8
+#define UVARS_TEST_PATH L"/tmp/fish_uvars_test/varsfile.txt"
+
+static int test_universal_helper(int *x)
+{
+    env_universal_t uvars(UVARS_TEST_PATH);
+    for (int j=0; j < UVARS_PER_THREAD; j++)
+    {
+        const wcstring key = format_string(L"key_%d_%d", *x, j);
+        const wcstring val = format_string(L"val_%d_%d", *x, j);
+        uvars.set(key, val, false);
+        bool synced = uvars.sync(NULL);
+        if (! synced)
+        {
+            err(L"Failed to sync universal variables");
+        }
+        fputc('.', stderr);
+    }
+    
+    /* Last step is to delete the first key */
+    uvars.remove(format_string(L"key_%d_%d", *x, 0));
+    bool synced = uvars.sync(NULL);
+    if (! synced)
+    {
+        err(L"Failed to sync universal variables");
+    }
+    fputc('.', stderr);
+    
+    return 0;
+}
+
+static void test_universal()
+{
+    say(L"Testing universal variables");
+    if (system("mkdir -p /tmp/fish_uvars_test/")) err(L"mkdir failed");
+    
+    const int threads = 16;
+    for (int i=0; i < threads; i++)
+    {
+        iothread_perform(test_universal_helper, new int(i));
+    }
+    iothread_drain_all();
+    
+    env_universal_t uvars(UVARS_TEST_PATH);
+    bool loaded = uvars.load();
+    if (! loaded)
+    {
+        err(L"Failed to load universal variables");
+    }
+    for (int i=0; i < threads; i++)
+    {
+        for (int j=0; j < UVARS_PER_THREAD; j++)
+        {
+            const wcstring key = format_string(L"key_%d_%d", i, j);
+            env_var_t expected_val;
+            if (j == 0)
+            {
+                expected_val = env_var_t::missing_var();
+            }
+            else
+            {
+                expected_val = format_string(L"val_%d_%d", i, j);
+            }
+            const env_var_t var = uvars.get(key);
+            if (j == 0)
+            {
+                assert(expected_val.missing());
+            }
+            if (var != expected_val)
+            {
+                const wchar_t *missing_desc = L"<missing>";
+                err(L"Wrong value for key %ls: expected %ls, got %ls\n", key.c_str(), (expected_val.missing() ? missing_desc : expected_val.c_str()), (var.missing() ? missing_desc : var.c_str()));
+            }
+        }
+    }
+    
+    if (system("rm -Rf /tmp/fish_uvars_test")) err(L"rm failed");
+    putc('\n', stderr);
+}
+
+static bool callback_data_less_than(const callback_data_t &a, const callback_data_t &b) {
+    return a.key < b.key;
+}
+
+static void test_universal_callbacks()
+{
+    say(L"Testing universal callbacks");
+    if (system("mkdir -p /tmp/fish_uvars_test/")) err(L"mkdir failed");
+    env_universal_t uvars1(UVARS_TEST_PATH);
+    env_universal_t uvars2(UVARS_TEST_PATH);
+    
+    /* Put some variables into both */
+    uvars1.set(L"alpha", L"1", false);
+    uvars1.set(L"beta", L"1", false);
+    uvars1.set(L"delta", L"1", false);
+    uvars1.set(L"epsilon", L"1", false);
+    uvars1.set(L"lambda", L"1", false);
+    uvars1.set(L"kappa", L"1", false);
+    uvars1.set(L"omicron", L"1", false);
+    
+    uvars1.sync(NULL);
+    uvars2.sync(NULL);
+    
+    /* Change uvars1 */
+    uvars1.set(L"alpha", L"2", false); //changes value
+    uvars1.set(L"beta", L"1", true); //changes export
+    uvars1.remove(L"delta"); //erases value
+    uvars1.set(L"epsilon", L"1", false); //changes nothing
+    uvars1.sync(NULL);
+    
+    /* Change uvars2. It should treat its value as correct and ignore changes from uvars1. */
+    uvars2.set(L"lambda", L"1", false); //same value
+    uvars2.set(L"kappa", L"2", false); //different value
+
+    /* Now see what uvars2 sees */
+    callback_data_list_t callbacks;
+    uvars2.sync(&callbacks);
+    
+    /* Sort them to get them in a predictable order */
+    std::sort(callbacks.begin(), callbacks.end(), callback_data_less_than);
+    
+    /* Should see exactly two changes */
+    do_test(callbacks.size() == 3);
+    do_test(callbacks.at(0).type == SET);
+    do_test(callbacks.at(0).key == L"alpha");
+    do_test(callbacks.at(0).val == L"2");
+    do_test(callbacks.at(1).type == SET_EXPORT);
+    do_test(callbacks.at(1).key == L"beta");
+    do_test(callbacks.at(1).val == L"1");
+    do_test(callbacks.at(2).type == ERASE);
+    do_test(callbacks.at(2).key == L"delta");
+    do_test(callbacks.at(2).val == L"");
+
+    
+    if (system("rm -Rf /tmp/fish_uvars_test")) err(L"rm failed");
+}
+
+bool poll_notifier(universal_notifier_t *note)
+{
+    bool result = false;
+    if (note->usec_delay_between_polls() > 0)
+    {
+        result = note->poll();
+    }
+    
+    int fd = note->notification_fd();
+    if (! result && fd >= 0)
+    {
+        fd_set fds;
+        FD_ZERO(&fds);
+        FD_SET(fd, &fds);
+        struct timeval tv = {0, 0};
+        if (select(fd + 1, &fds, NULL, NULL, &tv) > 0 && FD_ISSET(fd, &fds))
+        {
+            result = note->notification_fd_became_readable(fd);
+        }
+    }
+    return result;
+}
+
+static void trigger_or_wait_for_notification(universal_notifier_t *notifier, universal_notifier_t::notifier_strategy_t strategy)
+{
+    switch (strategy)
+    {
+        case universal_notifier_t::strategy_default:
+            assert(0 && "strategy_default should be passed");
+            break;
+            
+        case universal_notifier_t::strategy_shmem_polling:
+            // nothing required
+            break;
+            
+        case universal_notifier_t::strategy_notifyd:
+            // notifyd requires a round trip to the notifyd server, which means we have to wait a little bit to receive it
+            // In practice, this seems to be enough
+            usleep(1000000 / 25);
+            break;
+            
+        case universal_notifier_t::strategy_named_pipe:
+        case universal_notifier_t::strategy_null:
+            break;
+    }
+}
+
+static void test_notifiers_with_strategy(universal_notifier_t::notifier_strategy_t strategy)
+{
+    assert(strategy != universal_notifier_t::strategy_default);
+    say(L"Testing universal notifiers with strategy %d", (int)strategy);
+    universal_notifier_t *notifiers[16];
+    size_t notifier_count = sizeof notifiers / sizeof *notifiers;
+    
+    // Populate array of notifiers
+    for (size_t i=0; i < notifier_count; i++)
+    {
+        notifiers[i] = universal_notifier_t::new_notifier_for_strategy(strategy, UVARS_TEST_PATH);
+    }
+    
+    // Nobody should poll yet
+    for (size_t i=0; i < notifier_count; i++)
+    {
+        if (poll_notifier(notifiers[i]))
+        {
+            err(L"Universal variable notifier polled true before any changes, with strategy %d", (int)strategy);
+        }
+    }
+
+    // Tweak each notifier. Verify that others see it.
+    for (size_t post_idx=0; post_idx < notifier_count; post_idx++)
+    {
+        notifiers[post_idx]->post_notification();
+        
+        // Do special stuff to "trigger" a notification for testing
+        trigger_or_wait_for_notification(notifiers[post_idx], strategy);
+        
+        for (size_t i=0; i < notifier_count; i++)
+        {
+            // We aren't concerned with the one who posted
+            // Poll from it (to drain it), and then skip it
+            if (i == post_idx)
+            {
+                poll_notifier(notifiers[i]);
+                continue;
+            }
+            
+            if (! poll_notifier(notifiers[i]))
+            {
+                err(L"Universal variable notifier (%lu) %p polled failed to notice changes, with strategy %d", i, notifiers[i], (int)strategy);
+            }
+        }
+        
+        // Named pipes have special cleanup requirements
+        if (strategy == universal_notifier_t::strategy_named_pipe)
+        {
+            usleep(1000000 / 10); //corresponds to NAMED_PIPE_FLASH_DURATION_USEC
+            // Have to clean up the posted one first, so that the others see the pipe become no longer readable
+            poll_notifier(notifiers[post_idx]);
+            for (size_t i=0; i < notifier_count; i++)
+            {
+                poll_notifier(notifiers[i]);
+            }
+        }
+    }
+    
+    // Nobody should poll now
+    for (size_t i=0; i < notifier_count; i++)
+    {
+        if (poll_notifier(notifiers[i]))
+        {
+            err(L"Universal variable notifier polled true after all changes, with strategy %d", (int)strategy);
+        }
+    }
+    
+    // Clean up
+    for (size_t i=0; i < notifier_count; i++)
+    {
+        delete notifiers[i];
+    }
+}
+
+static void test_universal_notifiers()
+{
+    if (system("mkdir -p /tmp/fish_uvars_test/ && touch /tmp/fish_uvars_test/varsfile.txt")) err(L"mkdir failed");
+    test_notifiers_with_strategy(universal_notifier_t::strategy_shmem_polling);
+    test_notifiers_with_strategy(universal_notifier_t::strategy_named_pipe);
+#if __APPLE__
+    test_notifiers_with_strategy(universal_notifier_t::strategy_notifyd);
+#endif
+    
+    if (system("rm -Rf /tmp/fish_uvars_test/")) err(L"rm failed");
+}
+
+class history_tests_t
+{
+public:
+    static void test_history(void);
+    static void test_history_merge(void);
+    static void test_history_formats(void);
+    static void test_history_speed(void);
+
+    static void test_history_races(void);
+    static void test_history_races_pound_on_history();
+};
+
+static wcstring random_string(void)
+{
+    wcstring result;
+    size_t max = 1 + rand() % 32;
+    while (max--)
+    {
+        wchar_t c = 1 + rand()%ESCAPE_TEST_CHAR;
+        result.push_back(c);
+    }
+    return result;
+}
+
+void history_tests_t::test_history(void)
+{
+    say(L"Testing history");
+
+    history_t &history = history_t::history_with_name(L"test_history");
+    history.clear();
+    history.add(L"Gamma");
+    history.add(L"Beta");
+    history.add(L"Alpha");
+
+    /* All three items match "a" */
+    history_search_t search1(history, L"a");
+    test_history_matches(search1, 3);
+    do_test(search1.current_string() == L"Alpha");
+
+    /* One item matches "et" */
+    history_search_t search2(history, L"et");
+    test_history_matches(search2, 1);
+    do_test(search2.current_string() == L"Beta");
+
+    /* Test item removal */
+    history.remove(L"Alpha");
+    history_search_t search3(history, L"Alpha");
+    test_history_matches(search3, 0);
+
+    /* Test history escaping and unescaping, yaml, etc. */
+    history_item_list_t before, after;
+    history.clear();
+    size_t i, max = 100;
+    for (i=1; i <= max; i++)
+    {
+
+        /* Generate a value */
+        wcstring value = wcstring(L"test item ") + to_string(i);
+
+        /* Maybe add some backslashes */
+        if (i % 3 == 0)
+            value.append(L"(slashies \\\\\\ slashies)");
+
+        /* Generate some paths */
+        path_list_t paths;
+        size_t count = rand() % 6;
+        while (count--)
+        {
+            paths.push_back(random_string());
+        }
+
+        /* Record this item */
+        history_item_t item(value, time(NULL));
+        item.required_paths = paths;
+        before.push_back(item);
+        history.add(item);
+    }
+    history.save();
+
+    /* Read items back in reverse order and ensure they're the same */
+    for (i=100; i >= 1; i--)
+    {
+        history_item_t item = history.item_at_index(i);
+        do_test(! item.empty());
+        after.push_back(item);
+    }
+    do_test(before.size() == after.size());
+    for (size_t i=0; i < before.size(); i++)
+    {
+        const history_item_t &bef = before.at(i), &aft = after.at(i);
+        do_test(bef.contents == aft.contents);
+        do_test(bef.creation_timestamp == aft.creation_timestamp);
+        do_test(bef.required_paths == aft.required_paths);
+    }
+
+    /* Clean up after our tests */
+    history.clear();
+}
+
+// wait until the next second
+static void time_barrier(void)
+{
+    time_t start = time(NULL);
+    do
+    {
+        usleep(1000);
+    }
+    while (time(NULL) == start);
+}
+
+static wcstring_list_t generate_history_lines(int pid)
+{
+    wcstring_list_t result;
+    long max = 256;
+    result.reserve(max);
+    for (long i=0; i < max; i++)
+    {
+        result.push_back(format_string(L"%ld %ld", (long)pid, i));
+    }
+    return result;
+}
+
+void history_tests_t::test_history_races_pound_on_history()
+{
+    /* Called in child process to modify history */
+    history_t *hist = new history_t(L"race_test");
+    hist->chaos_mode = true;
+    const wcstring_list_t lines = generate_history_lines(getpid());
+    for (size_t idx = 0; idx < lines.size(); idx++)
+    {
+        const wcstring &line = lines.at(idx);
+        hist->add(line);
+        hist->save();
+    }
+    delete hist;
+}
+
+void history_tests_t::test_history_races(void)
+{
+    say(L"Testing history race conditions");
+
+    // Ensure history is clear
+    history_t *hist = new history_t(L"race_test");
+    hist->clear();
+    delete hist;
+
+    // Test concurrent history writing
+#define RACE_COUNT 10
+    pid_t children[RACE_COUNT];
+
+    for (size_t i=0; i < RACE_COUNT; i++)
+    {
+        pid_t pid = fork();
+        if (! pid)
+        {
+            // Child process
+            setup_fork_guards();
+            test_history_races_pound_on_history();
+            exit_without_destructors(0);
+        }
+        else
+        {
+            // Parent process
+            children[i] = pid;
+        }
+    }
+
+    // Wait for all children
+    for (size_t i=0; i < RACE_COUNT; i++)
+    {
+        int stat;
+        waitpid(children[i], &stat, WUNTRACED);
+    }
+
+    // Compute the expected lines
+    wcstring_list_t lines[RACE_COUNT];
+    for (size_t i=0; i < RACE_COUNT; i++)
+    {
+        lines[i] = generate_history_lines(children[i]);
+    }
+
+    // Count total lines
+    size_t line_count = 0;
+    for (size_t i=0; i < RACE_COUNT; i++)
+    {
+        line_count += lines[i].size();
+    }
+
+    // Ensure we consider the lines that have been outputted as part of our history
+    time_barrier();
+
+    /* Ensure that we got sane, sorted results */
+    hist = new history_t(L"race_test");
+    hist->chaos_mode = true;
+    size_t hist_idx;
+    for (hist_idx = 1; ; hist_idx ++)
+    {
+        history_item_t item = hist->item_at_index(hist_idx);
+        if (item.empty())
+            break;
+
+        // The item must be present in one of our 'lines' arrays
+        // If it is present, then every item after it is assumed to be missed
+        size_t i;
+        for (i=0; i < RACE_COUNT; i++)
+        {
+            wcstring_list_t::iterator where = std::find(lines[i].begin(), lines[i].end(), item.str());
+            if (where != lines[i].end())
+            {
+                // Delete everything from the found location onwards
+                lines[i].resize(where - lines[i].begin());
+
+                // Break because we found it
+                break;
+            }
+        }
+        if (i >= RACE_COUNT)
+        {
+            err(L"Line '%ls' found in history not found in some array", item.str().c_str());
+        }
+    }
+    // every write should add at least one item
+    do_test(hist_idx >= RACE_COUNT);
+
+    //hist->clear();
+    delete hist;
+}
+
+void history_tests_t::test_history_merge(void)
+{
+    // In a single fish process, only one history is allowed to exist with the given name
+    // But it's common to have multiple history instances with the same name active in different processes,
+    // e.g. when you have multiple shells open.
+    // We try to get that right and merge all their history together. Test that case.
+    say(L"Testing history merge");
+    const size_t count = 3;
+    const wcstring name = L"merge_test";
+    history_t *hists[count] = {new history_t(name), new history_t(name), new history_t(name)};
+    const wcstring texts[count] = {L"History 1", L"History 2", L"History 3"};
+    const wcstring alt_texts[count] = {L"History Alt 1", L"History Alt 2", L"History Alt 3"};
+
+    /* Make sure history is clear */
+    for (size_t i=0; i < count; i++)
+    {
+        hists[i]->clear();
+    }
+
+    /* Make sure we don't add an item in the same second as we created the history */
+    time_barrier();
+
+    /* Add a different item to each */
+    for (size_t i=0; i < count; i++)
+    {
+        hists[i]->add(texts[i]);
+    }
+
+    /* Save them */
+    for (size_t i=0; i < count; i++)
+    {
+        hists[i]->save();
+    }
+
+    /* Make sure each history contains what it ought to, but they have not leaked into each other */
+    for (size_t i = 0; i < count; i++)
+    {
+        for (size_t j=0; j < count; j++)
+        {
+            bool does_contain = history_contains(hists[i], texts[j]);
+            bool should_contain = (i == j);
+            do_test(should_contain == does_contain);
+        }
+    }
+
+    /* Make a new history. It should contain everything. The time_barrier() is so that the timestamp is newer, since we only pick up items whose timestamp is before the birth stamp. */
+    time_barrier();
+    history_t *everything = new history_t(name);
+    for (size_t i=0; i < count; i++)
+    {
+        do_test(history_contains(everything, texts[i]));
+    }
+
+    /* Tell all histories to merge. Now everybody should have everything. */
+    for (size_t i=0; i < count; i++)
+    {
+        hists[i]->incorporate_external_changes();
+    }
+    /* Add some more per-history items */
+    for (size_t i=0; i < count; i++)
+    {
+        hists[i]->add(alt_texts[i]);
+    }
+    /* Everybody should have old items, but only one history should have each new item */
+    for (size_t i = 0; i < count; i++)
+    {
+        for (size_t j=0; j < count; j++)
+        {
+            /* Old item */
+            do_test(history_contains(hists[i], texts[j]));
+
+            /* New item */
+            bool does_contain = history_contains(hists[i], alt_texts[j]);
+            bool should_contain = (i == j);
+            do_test(should_contain == does_contain);
+        }
+    }
+
+
+    /* Clean up */
+    for (size_t i=0; i < 3; i++)
+    {
+        delete hists[i];
+    }
+    everything->clear();
+    delete everything; //not as scary as it looks
+}
+
+static bool install_sample_history(const wchar_t *name)
+{
+    char command[512];
+    snprintf(command, sizeof command, "cp tests/%ls ~/.config/fish/%ls_history", name, name);
+    if (system(command))
+    {
+        err(L"Failed to copy sample history");
+        return false;
+    }
+    return true;
+}
+
+/* Indicates whether the history is equal to the given null-terminated array of strings. */
+static bool history_equals(history_t &hist, const wchar_t * const *strings)
+{
+    /* Count our expected items */
+    size_t expected_count = 0;
+    while (strings[expected_count])
+    {
+        expected_count++;
+    }
+
+    /* Ensure the contents are the same */
+    size_t history_idx = 1;
+    size_t array_idx = 0;
+    for (;;)
+    {
+        const wchar_t *expected = strings[array_idx];
+        history_item_t item = hist.item_at_index(history_idx);
+        if (expected == NULL)
+        {
+            if (! item.empty())
+            {
+                err(L"Expected empty item at history index %lu", history_idx);
+            }
+            break;
+        }
+        else
+        {
+            if (item.str() != expected)
+            {
+                err(L"Expected '%ls', found '%ls' at index %lu", expected, item.str().c_str(), history_idx);
+            }
+        }
+        history_idx++;
+        array_idx++;
+    }
+
+    return true;
+}
+
+void history_tests_t::test_history_formats(void)
+{
+    const wchar_t *name;
+    
+    // Test inferring and reading legacy and bash history formats
+    name = L"history_sample_fish_1_x";
+    say(L"Testing %ls", name);
+    if (! install_sample_history(name))
+    {
+        err(L"Couldn't open file tests/%ls", name);
+    }
+    else
+    {
+        /* Note: This is backwards from what appears in the file */
+        const wchar_t * const expected[] =
+        {
+            L"#def",
+
+            L"echo #abc",
+
+            L"function yay\n"
+            "echo hi\n"
+            "end",
+
+            L"cd foobar",
+
+            L"ls /",
+
+            NULL
+        };
+
+        history_t &test_history = history_t::history_with_name(name);
+        if (! history_equals(test_history, expected))
+        {
+            err(L"test_history_formats failed for %ls\n", name);
+        }
+        test_history.clear();
+    }
+
+    name = L"history_sample_fish_2_0";
+    say(L"Testing %ls", name);
+    if (! install_sample_history(name))
+    {
+        err(L"Couldn't open file tests/%ls", name);
+    }
+    else
+    {
+        const wchar_t * const expected[] =
+        {
+            L"echo this has\\\nbackslashes",
+
+            L"function foo\n"
+            "echo bar\n"
+            "end",
+
+            L"echo alpha",
+
+            NULL
+        };
+
+        history_t &test_history = history_t::history_with_name(name);
+        if (! history_equals(test_history, expected))
+        {
+            err(L"test_history_formats failed for %ls\n", name);
+        }
+        test_history.clear();
+    }
+
+    say(L"Testing bash import");
+    FILE *f = fopen("tests/history_sample_bash", "r");
+    if (! f)
+    {
+        err(L"Couldn't open file tests/history_sample_bash");
+    }
+    else
+    {
+        // It should skip over the export command since that's a bash-ism
+        const wchar_t *expected[] =
+        {
+            L"echo supsup",
+
+            L"history --help",
+
+            L"echo foo",
+
+            NULL
+        };
+        history_t &test_history = history_t::history_with_name(L"bash_import");
+        test_history.populate_from_bash(f);
+        if (! history_equals(test_history, expected))
+        {
+            err(L"test_history_formats failed for bash import\n");
+        }
+        test_history.clear();
+        fclose(f);
+    }
+    
+    name = L"history_sample_corrupt1";
+    say(L"Testing %ls", name);
+    if (! install_sample_history(name))
+    {
+        err(L"Couldn't open file tests/%ls", name);
+    }
+    else
+    {
+        /* We simply invoke get_string_representation. If we don't die, the test is a success. */
+        history_t &test_history = history_t::history_with_name(name);
+        const wchar_t *expected[] =
+        {
+            L"no_newline_at_end_of_file",
+            
+            L"corrupt_prefix",
+            
+            L"this_command_is_ok",
+            
+            NULL
+        };
+        if (! history_equals(test_history, expected))
+        {
+            err(L"test_history_formats failed for %ls\n", name);
+        }
+        test_history.clear();
+    }
+}
+
+void history_tests_t::test_history_speed(void)
+{
+    say(L"Testing history speed (pid is %d)", getpid());
+    history_t *hist = new history_t(L"speed_test");
+    wcstring item = L"History Speed Test - X";
+
+    /* Test for 10 seconds */
+    double start = timef();
+    double end = start + 10;
+    double stop = 0;
+    size_t count = 0;
+    for (;;)
+    {
+        item[item.size() - 1] = L'0' + (count % 10);
+        hist->add(item);
+        count++;
+
+        stop = timef();
+        if (stop >= end)
+            break;
+    }
+    printf("%lu items - %.2f msec per item\n", (unsigned long)count, (stop - start) * 1E6 / count);
+    hist->clear();
+    delete hist;
+}
+
+static void test_new_parser_correctness(void)
+{
+    say(L"Testing new parser!");
+    const struct parser_test_t
+    {
+        const wchar_t *src;
+        bool ok;
+    }
+    parser_tests[] =
+    {
+        {L"; ; ; ", true},
+        {L"if ; end", false},
+        {L"if true ; end", true},
+        {L"if true; end ; end", false},
+        {L"if end; end ; end", false},
+        {L"if end", false},
+        {L"end", false},
+        {L"for i i", false},
+        {L"for i in a b c ; end", true},
+        {L"begin end", true},
+        {L"begin; end", true},
+        {L"begin if true; end; end;", true},
+        {L"begin if true ; echo hi ; end; end", true},
+    };
+
+    for (size_t i=0; i < sizeof parser_tests / sizeof *parser_tests; i++)
+    {
+        const parser_test_t *test = &parser_tests[i];
+
+        parse_node_tree_t parse_tree;
+        bool success = parse_tree_from_string(test->src, parse_flag_none, &parse_tree, NULL);
+        say(L"%lu / %lu: Parse \"%ls\": %s", i+1, sizeof parser_tests / sizeof *parser_tests, test->src, success ? "yes" : "no");
+        if (success && ! test->ok)
+        {
+            err(L"\"%ls\" should NOT have parsed, but did", test->src);
+        }
+        else if (! success && test->ok)
+        {
+            err(L"\"%ls\" should have parsed, but failed", test->src);
+        }
+    }
+    say(L"Parse tests complete");
+}
+
+/* Given that we have an array of 'fuzz_count' strings, we wish to enumerate all permutations of 'len' values. We do this by incrementing an integer, interpreting it as "base fuzz_count". */
+static inline bool string_for_permutation(const wcstring *fuzzes, size_t fuzz_count, size_t len, size_t permutation, wcstring *out_str)
+{
+    out_str->clear();
+
+    size_t remaining_permutation = permutation;
+    for (size_t i=0; i < len; i++)
+    {
+        size_t idx = remaining_permutation % fuzz_count;
+        remaining_permutation /= fuzz_count;
+
+        out_str->append(fuzzes[idx]);
+        out_str->push_back(L' ');
+    }
+    // Return false if we wrapped
+    return remaining_permutation == 0;
+}
+
+static void test_new_parser_fuzzing(void)
+{
+    say(L"Fuzzing parser (node size: %lu)", sizeof(parse_node_t));
+    const wcstring fuzzes[] =
+    {
+        L"if",
+        L"else",
+        L"for",
+        L"in",
+        L"while",
+        L"begin",
+        L"function",
+        L"switch",
+        L"case",
+        L"end",
+        L"and",
+        L"or",
+        L"not",
+        L"command",
+        L"builtin",
+        L"foo",
+        L"|",
+        L"^",
+        L"&",
+        L";",
+    };
+
+    /* Generate a list of strings of all keyword / token combinations. */
+    wcstring src;
+    src.reserve(128);
+
+    parse_node_tree_t node_tree;
+    parse_error_list_t errors;
+
+    double start = timef();
+    bool log_it = true;
+    unsigned long max_len = 5;
+    for (unsigned long len = 0; len < max_len; len++)
+    {
+        if (log_it)
+            fprintf(stderr, "%lu / %lu...", len, max_len);
+
+        /* We wish to look at all permutations of 4 elements of 'fuzzes' (with replacement). Construct an int and keep incrementing it. */
+        unsigned long permutation = 0;
+        while (string_for_permutation(fuzzes, sizeof fuzzes / sizeof *fuzzes, len, permutation++, &src))
+        {
+            parse_tree_from_string(src, parse_flag_continue_after_error, &node_tree, &errors);
+        }
+        if (log_it)
+            fprintf(stderr, "done (%lu)\n", permutation);
+
+    }
+    double end = timef();
+    if (log_it)
+        say(L"All fuzzed in %f seconds!", end - start);
+}
+
+// Parse a statement, returning the command, args (joined by spaces), and the decoration. Returns true if successful.
+static bool test_1_parse_ll2(const wcstring &src, wcstring *out_cmd, wcstring *out_joined_args, enum parse_statement_decoration_t *out_deco)
+{
+    out_cmd->clear();
+    out_joined_args->clear();
+    *out_deco = parse_statement_decoration_none;
+
+    bool result = false;
+    parse_node_tree_t tree;
+    if (parse_tree_from_string(src, parse_flag_none, &tree, NULL))
+    {
+        /* Get the statement. Should only have one */
+        const parse_node_tree_t::parse_node_list_t stmt_nodes = tree.find_nodes(tree.at(0), symbol_plain_statement);
+        if (stmt_nodes.size() != 1)
+        {
+            say(L"Unexpected number of statements (%lu) found in '%ls'", stmt_nodes.size(), src.c_str());
+            return false;
+        }
+        const parse_node_t &stmt = *stmt_nodes.at(0);
+
+        /* Return its decoration */
+        *out_deco = tree.decoration_for_plain_statement(stmt);
+
+        /* Return its command */
+        tree.command_for_plain_statement(stmt, src, out_cmd);
+
+        /* Return arguments separated by spaces */
+        const parse_node_tree_t::parse_node_list_t arg_nodes = tree.find_nodes(stmt, symbol_argument);
+        for (size_t i=0; i < arg_nodes.size(); i++)
+        {
+            if (i > 0) out_joined_args->push_back(L' ');
+            out_joined_args->append(arg_nodes.at(i)->get_source(src));
+        }
+        result = true;
+    }
+    return result;
+}
+
+/* Test the LL2 (two token lookahead) nature of the parser by exercising the special builtin and command handling. In particular, 'command foo' should be a decorated statement 'foo' but 'command --help' should be an undecorated statement 'command' with argument '--help', and NOT attempt to run a command called '--help' */
+static void test_new_parser_ll2(void)
+{
+    say(L"Testing parser two-token lookahead");
+
+    const struct
+    {
+        wcstring src;
+        wcstring cmd;
+        wcstring args;
+        enum parse_statement_decoration_t deco;
+    } tests[] =
+    {
+        {L"echo hello", L"echo", L"hello", parse_statement_decoration_none},
+        {L"command echo hello", L"echo", L"hello", parse_statement_decoration_command},
+        {L"exec echo hello", L"echo", L"hello", parse_statement_decoration_exec},
+        {L"command command hello", L"command", L"hello", parse_statement_decoration_command},
+        {L"builtin command hello", L"command", L"hello", parse_statement_decoration_builtin},
+        {L"command --help", L"command", L"--help", parse_statement_decoration_none},
+        {L"command -h", L"command", L"-h", parse_statement_decoration_none},
+        {L"command", L"command", L"", parse_statement_decoration_none},
+        {L"command -", L"command", L"-", parse_statement_decoration_none},
+        {L"command --", L"command", L"--", parse_statement_decoration_none},
+        {L"builtin --names", L"builtin", L"--names", parse_statement_decoration_none},
+        {L"function", L"function", L"", parse_statement_decoration_none},
+        {L"function --help", L"function", L"--help", parse_statement_decoration_none}
+    };
+
+    for (size_t i=0; i < sizeof tests / sizeof *tests; i++)
+    {
+        wcstring cmd, args;
+        enum parse_statement_decoration_t deco = parse_statement_decoration_none;
+        bool success = test_1_parse_ll2(tests[i].src, &cmd, &args, &deco);
+        if (! success)
+            err(L"Parse of '%ls' failed on line %ld", tests[i].cmd.c_str(), (long)__LINE__);
+        if (cmd != tests[i].cmd)
+            err(L"When parsing '%ls', expected command '%ls' but got '%ls' on line %ld", tests[i].src.c_str(), tests[i].cmd.c_str(), cmd.c_str(), (long)__LINE__);
+        if (args != tests[i].args)
+            err(L"When parsing '%ls', expected args '%ls' but got '%ls' on line %ld", tests[i].src.c_str(), tests[i].args.c_str(), args.c_str(), (long)__LINE__);
+        if (deco != tests[i].deco)
+            err(L"When parsing '%ls', expected decoration %d but got %d on line %ld", tests[i].src.c_str(), (int)tests[i].deco, (int)deco, (long)__LINE__);
+    }
+
+    /* Verify that 'function -h' and 'function --help' are plain statements but 'function --foo' is not (#1240) */
+    const struct
+    {
+        wcstring src;
+        parse_token_type_t type;
+    }
+    tests2[] =
+    {
+        {L"function -h", symbol_plain_statement},
+        {L"function --help", symbol_plain_statement},
+        {L"function --foo ; end", symbol_function_header},
+        {L"function foo ; end", symbol_function_header},
+    };
+    for (size_t i=0; i < sizeof tests2 / sizeof *tests2; i++)
+    {
+        parse_node_tree_t tree;
+        if (! parse_tree_from_string(tests2[i].src, parse_flag_none, &tree, NULL))
+        {
+            err(L"Failed to parse '%ls'", tests2[i].src.c_str());
+        }
+
+        const parse_node_tree_t::parse_node_list_t node_list = tree.find_nodes(tree.at(0), tests2[i].type);
+        if (node_list.size() == 0)
+        {
+            err(L"Failed to find node of type '%ls'", token_type_description(tests2[i].type).c_str());
+        }
+        else if (node_list.size() > 1)
+        {
+            err(L"Found too many nodes of type '%ls'", token_type_description(tests2[i].type).c_str());
+        }
+    }
+}
+
+static void test_new_parser_ad_hoc()
+{
+    /* Very ad-hoc tests for issues encountered */
+    say(L"Testing new parser ad hoc tests");
+
+    /* Ensure that 'case' terminates a job list */
+    const wcstring src = L"switch foo ; case bar; case baz; end";
+    parse_node_tree_t parse_tree;
+    bool success = parse_tree_from_string(src, parse_flag_none, &parse_tree, NULL);
+    if (! success)
+    {
+        err(L"Parsing failed");
+    }
+
+    /* Expect three case_item_lists: one for each case, and a terminal one. The bug was that we'd try to run a command 'case' */
+    const parse_node_t &root = parse_tree.at(0);
+    const parse_node_tree_t::parse_node_list_t node_list = parse_tree.find_nodes(root, symbol_case_item_list);
+    if (node_list.size() != 3)
+    {
+        err(L"Expected 3 case item nodes, found %lu", node_list.size());
+    }
+}
+
+static void test_new_parser_errors(void)
+{
+    say(L"Testing new parser error reporting");
+    const struct
+    {
+        const wchar_t *src;
+        parse_error_code_t code;
+    }
+    tests[] =
+    {
+        {L"echo 'abc", parse_error_tokenizer_unterminated_quote},
+        {L"'", parse_error_tokenizer_unterminated_quote},
+        {L"echo (abc", parse_error_tokenizer_unterminated_subshell},
+
+        {L"end", parse_error_unbalancing_end},
+        {L"echo hi ; end", parse_error_unbalancing_end},
+
+        {L"else", parse_error_unbalancing_else},
+        {L"if true ; end ; else", parse_error_unbalancing_else},
+
+        {L"case", parse_error_unbalancing_case},
+        {L"if true ; case ; end", parse_error_unbalancing_case},
+
+        {L"foo || bar", parse_error_double_pipe},
+        {L"foo && bar", parse_error_double_background},
+    };
+
+    for (size_t i = 0; i < sizeof tests / sizeof *tests; i++)
+    {
+        const wcstring src = tests[i].src;
+        parse_error_code_t expected_code = tests[i].code;
+
+        parse_error_list_t errors;
+        parse_node_tree_t parse_tree;
+        bool success = parse_tree_from_string(src, parse_flag_none, &parse_tree, &errors);
+        if (success)
+        {
+            err(L"Source '%ls' was expected to fail to parse, but succeeded", src.c_str());
+        }
+
+        if (errors.size() != 1)
+        {
+            err(L"Source '%ls' was expected to produce 1 error, but instead produced %lu errors", src.c_str(), errors.size());
+        }
+        else if (errors.at(0).code != expected_code)
+        {
+            err(L"Source '%ls' was expected to produce error code %lu, but instead produced error code %lu", src.c_str(), expected_code, (unsigned long)errors.at(0).code);
+            for (size_t i=0; i < errors.size(); i++)
+            {
+                err(L"\t\t%ls", errors.at(i).describe(src).c_str());
+            }
+        }
+
+    }
+}
+
+/* Given a format string, returns a list of non-empty strings separated by format specifiers. The format specifiers themselves are omitted. */
+static wcstring_list_t separate_by_format_specifiers(const wchar_t *format)
+{
+    wcstring_list_t result;
+    const wchar_t *cursor = format;
+    const wchar_t *end = format + wcslen(format);
+    while (cursor < end)
+    {
+        const wchar_t *next_specifier = wcschr(cursor, '%');
+        if (next_specifier == NULL)
+        {
+            next_specifier = end;
+        }
+        assert(next_specifier != NULL);
+        
+        /* Don't return empty strings */
+        if (next_specifier > cursor)
+        {
+            result.push_back(wcstring(cursor, next_specifier - cursor));
+        }
+        
+        /* Walk over the format specifier (if any) */
+        cursor = next_specifier;
+        if (*cursor == '%')
+        {
+            cursor++;
+            // Flag
+            if (wcschr(L"#0- +'", *cursor))
+                cursor++;
+            // Minimum field width
+            while (iswdigit(*cursor))
+                cursor++;
+            // Precision
+            if (*cursor == L'.')
+            {
+                cursor++;
+                while (iswdigit(*cursor))
+                    cursor++;
+            }
+            // Length modifier
+            if (! wcsncmp(cursor, L"ll", 2) || ! wcsncmp(cursor, L"hh", 2))
+            {
+                cursor += 2;
+            }
+            else if (wcschr(L"hljtzqL", *cursor))
+            {
+                cursor++;
+            }
+            // The format specifier itself. We allow any character except NUL
+            if (*cursor != L'\0')
+            {
+                cursor += 1;
+            }
+            assert(cursor <= end);
+        }
+    }
+    return result;
+}
+
+/* Given a format string 'format', return true if the string may have been produced by that format string. We do this by splitting the format string around the format specifiers, and then ensuring that each of the remaining chunks is found (in order) in the string. */
+static bool string_matches_format(const wcstring &string, const wchar_t *format)
+{
+    bool result = true;
+    wcstring_list_t components = separate_by_format_specifiers(format);
+    size_t idx = 0;
+    for (size_t i=0; i < components.size(); i++)
+    {
+        const wcstring &component = components.at(i);
+        size_t where = string.find(component, idx);
+        if (where == wcstring::npos)
+        {
+            result = false;
+            break;
+        }
+        idx = where + component.size();
+        assert(idx <= string.size());
+    }
+    return result;
+}
+
+static void test_error_messages()
+{
+    say(L"Testing error messages");
+    const struct error_test_t
+    {
+        const wchar_t *src;
+        const wchar_t *error_text_format;
+    }
+    error_tests[] =
+    {
+        {L"echo $^", ERROR_BAD_VAR_CHAR1},
+        {L"echo foo${a}bar", ERROR_BRACKETED_VARIABLE1},
+        {L"echo foo\"${a}\"bar", ERROR_BRACKETED_VARIABLE_QUOTED1},
+        {L"echo foo\"${\"bar", ERROR_BAD_VAR_CHAR1},
+        {L"echo $?", ERROR_NOT_STATUS},
+        {L"echo $$", ERROR_NOT_PID},
+        {L"echo $#", ERROR_NOT_ARGV_COUNT},
+        {L"echo $@", ERROR_NOT_ARGV_AT},
+        {L"echo $*", ERROR_NOT_ARGV_STAR},
+        {L"echo $", ERROR_NO_VAR_NAME},
+        {L"echo foo\"$\"bar", ERROR_NO_VAR_NAME},
+        {L"echo \"foo\"$\"bar\"", ERROR_NO_VAR_NAME},
+        {L"echo foo $ bar", ERROR_NO_VAR_NAME},
+        {L"echo foo$(foo)bar", ERROR_BAD_VAR_SUBCOMMAND1},
+        {L"echo \"foo$(foo)bar\"", ERROR_BAD_VAR_SUBCOMMAND1},
+        {L"echo foo || echo bar", ERROR_BAD_OR},
+        {L"echo foo && echo bar", ERROR_BAD_AND}
+    };
+    
+    parse_error_list_t errors;
+    for (size_t i=0; i < sizeof error_tests / sizeof *error_tests; i++)
+    {
+        const struct error_test_t *test = &error_tests[i];
+        errors.clear();
+        parse_util_detect_errors(test->src, &errors, false /* allow_incomplete */);
+        do_test(! errors.empty());
+        if (! errors.empty())
+        {
+            do_test1(string_matches_format(errors.at(0).text, test->error_text_format), test->src);
+        }
+    }
+}
+
+static void test_highlighting(void)
+{
+    say(L"Testing syntax highlighting");
+    if (system("mkdir -p /tmp/fish_highlight_test/")) err(L"mkdir failed");
+    if (system("touch /tmp/fish_highlight_test/foo")) err(L"touch failed");
+    if (system("touch /tmp/fish_highlight_test/bar")) err(L"touch failed");
+
+    // Here are the components of our source and the colors we expect those to be
+    struct highlight_component_t
+    {
+        const wchar_t *txt;
+        int color;
+    };
+
+    const highlight_component_t components1[] =
+    {
+        {L"echo", highlight_spec_command},
+        {L"/tmp/fish_highlight_test/foo", highlight_spec_param | highlight_modifier_valid_path},
+        {L"&", highlight_spec_statement_terminator},
+        {NULL, -1}
+    };
+
+    const highlight_component_t components2[] =
+    {
+        {L"command", highlight_spec_command},
+        {L"echo", highlight_spec_command},
+        {L"abc", highlight_spec_param},
+        {L"/tmp/fish_highlight_test/foo", highlight_spec_param | highlight_modifier_valid_path},
+        {L"&", highlight_spec_statement_terminator},
+        {NULL, -1}
+    };
+
+    const highlight_component_t components3[] =
+    {
+        {L"if command ls", highlight_spec_command},
+        {L"; ", highlight_spec_statement_terminator},
+        {L"echo", highlight_spec_command},
+        {L"abc", highlight_spec_param},
+        {L"; ", highlight_spec_statement_terminator},
+        {L"/bin/definitely_not_a_command", highlight_spec_error},
+        {L"; ", highlight_spec_statement_terminator},
+        {L"end", highlight_spec_command},
+        {NULL, -1}
+    };
+
+    /* Verify that cd shows errors for non-directories */
+    const highlight_component_t components4[] =
+    {
+        {L"cd", highlight_spec_command},
+        {L"/tmp/fish_highlight_test", highlight_spec_param | highlight_modifier_valid_path},
+        {NULL, -1}
+    };
+
+    const highlight_component_t components5[] =
+    {
+        {L"cd", highlight_spec_command},
+        {L"/tmp/fish_highlight_test/foo", highlight_spec_error},
+        {NULL, -1}
+    };
+
+    const highlight_component_t components6[] =
+    {
+        {L"cd", highlight_spec_command},
+        {L"--help", highlight_spec_param},
+        {L"-h", highlight_spec_param},
+        {L"definitely_not_a_directory", highlight_spec_error},
+        {NULL, -1}
+    };
+
+    // Command substitutions
+    const highlight_component_t components7[] =
+    {
+        {L"echo", highlight_spec_command},
+        {L"param1", highlight_spec_param},
+        {L"(", highlight_spec_operator},
+        {L"ls", highlight_spec_command},
+        {L"param2", highlight_spec_param},
+        {L")", highlight_spec_operator},
+        {L"|", highlight_spec_statement_terminator},
+        {L"cat", highlight_spec_command},
+        {NULL, -1}
+    };
+
+    // Redirections substitutions
+    const highlight_component_t components8[] =
+    {
+        {L"echo", highlight_spec_command},
+        {L"param1", highlight_spec_param},
+
+        /* Input redirection */
+        {L"<", highlight_spec_redirection},
+        {L"/bin/echo", highlight_spec_redirection},
+
+        /* Output redirection to a valid fd */
+        {L"1>&2", highlight_spec_redirection},
+
+        /* Output redirection to an invalid fd */
+        {L"2>&", highlight_spec_redirection},
+        {L"LOL", highlight_spec_error},
+
+        /* Just a param, not a redirection */
+        {L"/tmp/blah", highlight_spec_param},
+
+        /* Input redirection from directory */
+        {L"<", highlight_spec_redirection},
+        {L"/tmp/", highlight_spec_error},
+
+        /* Output redirection to an invalid path */
+        {L"3>", highlight_spec_redirection},
+        {L"/not/a/valid/path/nope", highlight_spec_error},
+
+        /* Output redirection to directory */
+        {L"3>", highlight_spec_redirection},
+        {L"/tmp/nope/", highlight_spec_error},
+
+
+        /* Redirections to overflow fd */
+        {L"99999999999999999999>&2", highlight_spec_error},
+        {L"2>&", highlight_spec_redirection},
+        {L"99999999999999999999", highlight_spec_error},
+
+        /* Output redirection containing a command substitution */
+        {L"4>", highlight_spec_redirection},
+        {L"(", highlight_spec_operator},
+        {L"echo", highlight_spec_command},
+        {L"/tmp/somewhere", highlight_spec_param},
+        {L")", highlight_spec_operator},
+
+        /* Just another param */
+        {L"param2", highlight_spec_param},
+        {NULL, -1}
+    };
+
+    const highlight_component_t components9[] =
+    {
+        {L"end", highlight_spec_error},
+        {L";", highlight_spec_statement_terminator},
+        {L"if", highlight_spec_command},
+        {L"end", highlight_spec_error},
+        {NULL, -1}
+    };
+
+    const highlight_component_t components10[] =
+    {
+        {L"echo", highlight_spec_command},
+        {L"'single_quote", highlight_spec_error},
+        {NULL, -1}
+    };
+
+    const highlight_component_t components11[] =
+    {
+        {L"echo", highlight_spec_command},
+        {L"$foo", highlight_spec_operator},
+        {L"\"", highlight_spec_quote},
+        {L"$bar", highlight_spec_operator},
+        {L"\"", highlight_spec_quote},
+        {L"$baz[", highlight_spec_operator},
+        {L"1 2..3", highlight_spec_param},
+        {L"]", highlight_spec_operator},
+        {NULL, -1}
+    };
+
+    const highlight_component_t components12[] =
+    {
+        {L"for", highlight_spec_command},
+        {L"i", highlight_spec_param},
+        {L"in", highlight_spec_command},
+        {L"1 2 3", highlight_spec_param},
+        {L";", highlight_spec_statement_terminator},
+        {L"end", highlight_spec_command},
+        {NULL, -1}
+    };
+
+    const highlight_component_t components13[] =
+    {
+        {L"echo", highlight_spec_command},
+        {L"$$foo[", highlight_spec_operator},
+        {L"1", highlight_spec_param},
+        {L"][", highlight_spec_operator},
+        {L"2", highlight_spec_param},
+        {L"]", highlight_spec_operator},
+        {L"[3]", highlight_spec_param}, // two dollar signs, so last one is not an expansion
+        {NULL, -1}
+    };
+
+    const highlight_component_t *tests[] = {components1, components2, components3, components4, components5, components6, components7, components8, components9, components10, components11, components12, components13};
+    for (size_t which = 0; which < sizeof tests / sizeof *tests; which++)
+    {
+        const highlight_component_t *components = tests[which];
+        // Count how many we have
+        size_t component_count = 0;
+        while (components[component_count].txt != NULL)
+        {
+            component_count++;
+        }
+
+        // Generate the text
+        wcstring text;
+        std::vector<highlight_spec_t> expected_colors;
+        for (size_t i=0; i < component_count; i++)
+        {
+            if (i > 0)
+            {
+                text.push_back(L' ');
+                expected_colors.push_back(0);
+            }
+            text.append(components[i].txt);
+            expected_colors.resize(text.size(), components[i].color);
+        }
+        do_test(expected_colors.size() == text.size());
+
+        std::vector<highlight_spec_t> colors(text.size());
+        highlight_shell(text, colors, 20, NULL, env_vars_snapshot_t());
+
+        if (expected_colors.size() != colors.size())
+        {
+            err(L"Color vector has wrong size! Expected %lu, actual %lu", expected_colors.size(), colors.size());
+        }
+        do_test(expected_colors.size() == colors.size());
+        for (size_t i=0; i < text.size(); i++)
+        {
+            // Hackish space handling. We don't care about the colors in spaces.
+            if (text.at(i) == L' ')
+                continue;
+
+            if (expected_colors.at(i) != colors.at(i))
+            {
+                const wcstring spaces(i, L' ');
+                err(L"Wrong color at index %lu in text (expected %#x, actual %#x):\n%ls\n%ls^", i, expected_colors.at(i), colors.at(i), text.c_str(), spaces.c_str());
+            }
+        }
+    }
+
+    if (system("rm -Rf /tmp/fish_highlight_test"))
+    {
+        err(L"rm failed");
+    }
+}
+
+static void test_wcstring_tok(void)
+{
+    say(L"Testing wcstring_tok");
+    wcstring buff = L"hello world";
+    wcstring needle = L" \t\n";
+    wcstring_range loc = wcstring_tok(buff, needle);
+    if (loc.first == wcstring::npos || buff.substr(loc.first, loc.second) != L"hello")
+    {
+        err(L"Wrong results from first wcstring_tok(): {%zu, %zu}", loc.first, loc.second);
+    }
+    loc = wcstring_tok(buff, needle, loc);
+    if (loc.first == wcstring::npos || buff.substr(loc.first, loc.second) != L"world")
+    {
+        err(L"Wrong results from second wcstring_tok(): {%zu, %zu}", loc.first, loc.second);
+    }
+    loc = wcstring_tok(buff, needle, loc);
+    if (loc.first != wcstring::npos)
+    {
+        err(L"Wrong results from third wcstring_tok(): {%zu, %zu}", loc.first, loc.second);
+    }
+
+    buff = L"hello world";
+    loc = wcstring_tok(buff, needle);
+    // loc is "hello" again
+    loc = wcstring_tok(buff, L"", loc);
+    if (loc.first == wcstring::npos || buff.substr(loc.first, loc.second) != L"world")
+    {
+        err(L"Wrong results from wcstring_tok with empty needle: {%zu, %zu}", loc.first, loc.second);
+    }
+}
+
+/**
+   Main test
+*/
+int main(int argc, char **argv)
+{
+    // Look for the file tests/test.fish. We expect to run in a directory containing that file.
+    // If we don't find it, walk up the directory hierarchy until we do, or error
+    while (access("./tests/test.fish", F_OK) != 0)
+    {
+        char wd[PATH_MAX + 1] = {};
+        getcwd(wd, sizeof wd);
+        if (! strcmp(wd, "/"))
+        {
+            fprintf(stderr, "Unable to find 'tests' directory, which should contain file test.fish\n");
+            exit(EXIT_FAILURE);
+        }
+        if (chdir(dirname(wd)) < 0)
+        {
+            perror("chdir");
+        }
+    }
+    
+    setlocale(LC_ALL, "");
+    //srand(time(0));
+    configure_thread_assertions_for_testing();
+
+    program_name=L"(ignore)";
+    s_arguments = argv + 1;
+
+    say(L"Testing low-level functionality");
+    set_main_thread();
+    setup_fork_guards();
+    proc_init();
+    event_init();
+    function_init();
+    builtin_init();
+    reader_init();
+    env_init();
+
+    /* Set default signal handlers, so we can ctrl-C out of this */
+    signal_reset_handlers();
+
+    if (should_test_function("highlighting")) test_highlighting();
+    if (should_test_function("new_parser_ll2")) test_new_parser_ll2();
+    if (should_test_function("new_parser_fuzzing")) test_new_parser_fuzzing(); //fuzzing is expensive
+    if (should_test_function("new_parser_correctness")) test_new_parser_correctness();
+    if (should_test_function("new_parser_ad_hoc")) test_new_parser_ad_hoc();
+    if (should_test_function("new_parser_errors")) test_new_parser_errors();
+    if (should_test_function("error_messages")) test_error_messages();
+    if (should_test_function("escape")) test_unescape_sane();
+    if (should_test_function("escape")) test_escape_crazy();
+    if (should_test_function("format")) test_format();
+    if (should_test_function("convert")) test_convert();
+    if (should_test_function("convert_nulls")) test_convert_nulls();
+    if (should_test_function("tok")) test_tok();
+    if (should_test_function("iothread")) test_iothread();
+    if (should_test_function("parser")) test_parser();
+    if (should_test_function("cancellation")) test_cancellation();
+    if (should_test_function("indents")) test_indents();
+    if (should_test_function("utils")) test_utils();
+    if (should_test_function("utf8")) test_utf8();
+    if (should_test_function("escape_sequences")) test_escape_sequences();
+    if (should_test_function("lru")) test_lru();
+    if (should_test_function("expand")) test_expand();
+    if (should_test_function("fuzzy_match")) test_fuzzy_match();
+    if (should_test_function("abbreviations")) test_abbreviations();
+    if (should_test_function("test")) test_test();
+    if (should_test_function("path")) test_path();
+    if (should_test_function("pager_navigation")) test_pager_navigation();
+    if (should_test_function("word_motion")) test_word_motion();
+    if (should_test_function("is_potential_path")) test_is_potential_path();
+    if (should_test_function("colors")) test_colors();
+    if (should_test_function("complete")) test_complete();
+    if (should_test_function("input")) test_input();
+    if (should_test_function("universal")) test_universal();
+    if (should_test_function("universal")) test_universal_callbacks();
+    if (should_test_function("notifiers")) test_universal_notifiers();
+    if (should_test_function("completion_insertions")) test_completion_insertions();
+    if (should_test_function("autosuggestion_ignores")) test_autosuggestion_ignores();
+    if (should_test_function("autosuggestion_combining")) test_autosuggestion_combining();
+    if (should_test_function("autosuggest_suggest_special")) test_autosuggest_suggest_special();
+    if (should_test_function("wcstring_tok")) test_wcstring_tok();
+    if (should_test_function("history")) history_tests_t::test_history();
+    if (should_test_function("history_merge")) history_tests_t::test_history_merge();
+    if (should_test_function("history_races")) history_tests_t::test_history_races();
+    if (should_test_function("history_formats")) history_tests_t::test_history_formats();
+    //history_tests_t::test_history_speed();
+
+    say(L"Encountered %d errors in low-level tests", err_count);
+    if (s_test_run_count == 0)
+        say(L"*** No Tests Were Actually Run! ***");
+
+    /*
+      Skip performance tests for now, since they seem to hang when running from inside make (?)
+    */
+//  say( L"Testing performance" );
+//  perf_complete();
+
+    reader_destroy();
+    builtin_destroy();
+    wutil_destroy();
+    event_destroy();
+    proc_destroy();
+
+    if (err_count != 0)
+    {
+        return(1);
+    }
+}
diff --git a/src/fish_version.cpp b/src/fish_version.cpp
new file mode 100644
index 00000000..e434c3c8
--- /dev/null
+++ b/src/fish_version.cpp
@@ -0,0 +1,14 @@
+/** \file fish_version.c Fish version receiver.
+
+  This file has a specific purpose of shortening compilation times when
+  the only change is different `git describe` version.
+*/
+
+#include "fish_version.h"
+
+/**
+ * Return fish shell version.
+ */
+const char *get_fish_version() {
+    return FISH_BUILD_VERSION;
+}
diff --git a/src/fish_version.h b/src/fish_version.h
new file mode 100644
index 00000000..61938c16
--- /dev/null
+++ b/src/fish_version.h
@@ -0,0 +1,5 @@
+/** \file fish_version.h
+  Prototype for version receiver.
+*/
+
+const char *get_fish_version();
diff --git a/src/function.cpp b/src/function.cpp
new file mode 100644
index 00000000..2a8c20be
--- /dev/null
+++ b/src/function.cpp
@@ -0,0 +1,389 @@
+/** \file function.c
+
+    Prototypes for functions for storing and retrieving function
+  information. These functions also take care of autoloading
+  functions in the $fish_function_path. Actual function evaluation
+  is taken care of by the parser and to some degree the builtin
+  handling library.
+*/
+
+#include "config.h" // IWYU pragma: keep
+
+#include <wchar.h>
+#include <pthread.h>
+#include <map>
+#include <set>
+#include <dirent.h>
+#include <stddef.h>
+#include <string>
+#include <utility>
+
+#include "wutil.h"
+#include "fallback.h" // IWYU pragma: keep
+
+#include "autoload.h"
+#include "function.h"
+#include "common.h"
+#include "intern.h"
+#include "event.h"
+#include "reader.h"
+#include "parser_keywords.h"
+#include "env.h"
+
+/**
+   Table containing all functions
+*/
+typedef std::map<wcstring, function_info_t> function_map_t;
+static function_map_t loaded_functions;
+
+/**
+   Functions that shouldn't be autoloaded (anymore).
+*/
+static std::set<wcstring> function_tombstones;
+
+/* Lock for functions */
+static pthread_mutex_t functions_lock;
+
+/* Autoloader for functions */
+class function_autoload_t : public autoload_t
+{
+public:
+    function_autoload_t();
+    virtual void command_removed(const wcstring &cmd);
+};
+
+static function_autoload_t function_autoloader;
+
+/** Constructor */
+function_autoload_t::function_autoload_t() : autoload_t(L"fish_function_path", NULL, 0)
+{
+}
+
+static bool function_remove_ignore_autoload(const wcstring &name, bool tombstone = true);
+
+/** Callback when an autoloaded function is removed */
+void function_autoload_t::command_removed(const wcstring &cmd)
+{
+    function_remove_ignore_autoload(cmd, false);
+}
+
+/**
+   Kludgy flag set by the load function in order to tell function_add
+   that the function being defined is autoloaded. There should be a
+   better way to do this...
+*/
+static bool is_autoload = false;
+
+/**
+   Make sure that if the specified function is a dynamically loaded
+   function, it has been fully loaded.
+*/
+static int load(const wcstring &name)
+{
+    ASSERT_IS_MAIN_THREAD();
+    scoped_lock lock(functions_lock);
+    bool was_autoload = is_autoload;
+    int res;
+
+    bool no_more_autoload = function_tombstones.count(name) > 0;
+    if (no_more_autoload)
+        return 0;
+
+    function_map_t::iterator iter = loaded_functions.find(name);
+    if (iter !=  loaded_functions.end() && !iter->second.is_autoload)
+    {
+        /* We have a non-autoload version already */
+        return 0;
+    }
+
+    is_autoload = true;
+    res = function_autoloader.load(name, true);
+    is_autoload = was_autoload;
+    return res;
+}
+
+/**
+   Insert a list of all dynamically loaded functions into the
+   specified list.
+*/
+static void autoload_names(std::set<wcstring> &names, int get_hidden)
+{
+    size_t i;
+
+    const env_var_t path_var_wstr =  env_get_string(L"fish_function_path");
+    if (path_var_wstr.missing())
+        return;
+    const wchar_t *path_var = path_var_wstr.c_str();
+
+    wcstring_list_t path_list;
+
+    tokenize_variable_array(path_var, path_list);
+    for (i=0; i<path_list.size(); i++)
+    {
+        const wcstring &ndir_str = path_list.at(i);
+        const wchar_t *ndir = (wchar_t *)ndir_str.c_str();
+        DIR *dir = wopendir(ndir);
+        if (!dir)
+            continue;
+
+        wcstring name;
+        while (wreaddir(dir, name))
+        {
+            const wchar_t *fn = name.c_str();
+            const wchar_t *suffix;
+            if (!get_hidden && fn[0] == L'_')
+                continue;
+
+            suffix = wcsrchr(fn, L'.');
+            if (suffix && (wcscmp(suffix, L".fish") == 0))
+            {
+                wcstring name(fn, suffix - fn);
+                names.insert(name);
+            }
+        }
+        closedir(dir);
+    }
+}
+
+void function_init()
+{
+    /* PCA: This recursive lock was introduced early in my work. I would like to make this a non-recursive lock but I haven't fully investigated all the call paths (for autoloading functions, etc.) */
+    pthread_mutexattr_t a;
+    VOMIT_ON_FAILURE(pthread_mutexattr_init(&a));
+    VOMIT_ON_FAILURE(pthread_mutexattr_settype(&a,PTHREAD_MUTEX_RECURSIVE));
+    VOMIT_ON_FAILURE(pthread_mutex_init(&functions_lock, &a));
+    VOMIT_ON_FAILURE(pthread_mutexattr_destroy(&a));
+}
+
+static std::map<wcstring,env_var_t> snapshot_vars(const wcstring_list_t &vars)
+{
+    std::map<wcstring,env_var_t> result;
+    for (wcstring_list_t::const_iterator it = vars.begin(), end = vars.end(); it != end; ++it)
+    {
+        result.insert(std::make_pair(*it, env_get_string(*it)));
+    }
+    return result;
+}
+
+function_info_t::function_info_t(const function_data_t &data, const wchar_t *filename, int def_offset, bool autoload) :
+    definition(data.definition),
+    description(data.description),
+    definition_file(intern(filename)),
+    definition_offset(def_offset),
+    named_arguments(data.named_arguments),
+    inherit_vars(snapshot_vars(data.inherit_vars)),
+    is_autoload(autoload),
+    shadows(data.shadows)
+{
+}
+
+function_info_t::function_info_t(const function_info_t &data, const wchar_t *filename, int def_offset, bool autoload) :
+    definition(data.definition),
+    description(data.description),
+    definition_file(intern(filename)),
+    definition_offset(def_offset),
+    named_arguments(data.named_arguments),
+    inherit_vars(data.inherit_vars),
+    is_autoload(autoload),
+    shadows(data.shadows)
+{
+}
+
+void function_add(const function_data_t &data, const parser_t &parser, int definition_line_offset)
+{
+    ASSERT_IS_MAIN_THREAD();
+
+    CHECK(! data.name.empty(),);
+    CHECK(data.definition,);
+    scoped_lock lock(functions_lock);
+
+    /* Remove the old function */
+    function_remove(data.name);
+
+    /* Create and store a new function */
+    const wchar_t *filename = reader_current_filename();
+
+    const function_map_t::value_type new_pair(data.name, function_info_t(data, filename, definition_line_offset, is_autoload));
+    loaded_functions.insert(new_pair);
+
+    /* Add event handlers */
+    for (std::vector<event_t>::const_iterator iter = data.events.begin(); iter != data.events.end(); ++iter)
+    {
+        event_add_handler(*iter);
+    }
+}
+
+int function_exists(const wcstring &cmd)
+{
+    if (parser_keywords_is_reserved(cmd))
+        return 0;
+    scoped_lock lock(functions_lock);
+    load(cmd);
+    return loaded_functions.find(cmd) != loaded_functions.end();
+}
+
+int function_exists_no_autoload(const wcstring &cmd, const env_vars_snapshot_t &vars)
+{
+    if (parser_keywords_is_reserved(cmd))
+        return 0;
+    scoped_lock lock(functions_lock);
+    return loaded_functions.find(cmd) != loaded_functions.end() || function_autoloader.can_load(cmd, vars);
+}
+
+static bool function_remove_ignore_autoload(const wcstring &name, bool tombstone)
+{
+    // Note: the lock may be held at this point, but is recursive
+    scoped_lock lock(functions_lock);
+
+    function_map_t::iterator iter = loaded_functions.find(name);
+
+    // not found.  not erasing.
+    if (iter == loaded_functions.end())
+        return false;
+
+    // removing an auto-loaded function.  prevent it from being
+    // auto-reloaded.
+    if (iter->second.is_autoload && tombstone)
+        function_tombstones.insert(name);
+
+    loaded_functions.erase(iter);
+
+    event_t ev(EVENT_ANY);
+    ev.function_name=name;
+    event_remove(ev);
+
+    return true;
+}
+
+void function_remove(const wcstring &name)
+{
+    if (function_remove_ignore_autoload(name))
+        function_autoloader.unload(name);
+}
+
+static const function_info_t *function_get(const wcstring &name)
+{
+    // The caller must lock the functions_lock before calling this; however our mutex is currently recursive, so trylock will never fail
+    // We need a way to correctly check if a lock is locked (or better yet, make our lock non-recursive)
+    //ASSERT_IS_LOCKED(functions_lock);
+    function_map_t::iterator iter = loaded_functions.find(name);
+    if (iter == loaded_functions.end())
+    {
+        return NULL;
+    }
+    else
+    {
+        return &iter->second;
+    }
+}
+
+bool function_get_definition(const wcstring &name, wcstring *out_definition)
+{
+    scoped_lock lock(functions_lock);
+    const function_info_t *func = function_get(name);
+    if (func && out_definition)
+    {
+        out_definition->assign(func->definition);
+    }
+    return func != NULL;
+}
+
+wcstring_list_t function_get_named_arguments(const wcstring &name)
+{
+    scoped_lock lock(functions_lock);
+    const function_info_t *func = function_get(name);
+    return func ? func->named_arguments : wcstring_list_t();
+}
+
+std::map<wcstring,env_var_t> function_get_inherit_vars(const wcstring &name)
+{
+    scoped_lock lock(functions_lock);
+    const function_info_t *func = function_get(name);
+    return func ? func->inherit_vars : std::map<wcstring,env_var_t>();
+}
+
+int function_get_shadows(const wcstring &name)
+{
+    scoped_lock lock(functions_lock);
+    const function_info_t *func = function_get(name);
+    return func ? func->shadows : false;
+}
+
+
+bool function_get_desc(const wcstring &name, wcstring *out_desc)
+{
+    /* Empty length string goes to NULL */
+    scoped_lock lock(functions_lock);
+    const function_info_t *func = function_get(name);
+    if (out_desc && func && ! func->description.empty())
+    {
+        out_desc->assign(_(func->description.c_str()));
+        return true;
+    }
+    else
+    {
+        return false;
+    }
+}
+
+void function_set_desc(const wcstring &name, const wcstring &desc)
+{
+    load(name);
+    scoped_lock lock(functions_lock);
+    function_map_t::iterator iter = loaded_functions.find(name);
+    if (iter != loaded_functions.end())
+    {
+        iter->second.description = desc;
+    }
+}
+
+bool function_copy(const wcstring &name, const wcstring &new_name)
+{
+    bool result = false;
+    scoped_lock lock(functions_lock);
+    function_map_t::const_iterator iter = loaded_functions.find(name);
+    if (iter != loaded_functions.end())
+    {
+        // This new instance of the function shouldn't be tied to the definition file of the original, so pass NULL filename, etc.
+        const function_map_t::value_type new_pair(new_name, function_info_t(iter->second, NULL, 0, false));
+        loaded_functions.insert(new_pair);
+        result = true;
+    }
+    return result;
+}
+
+wcstring_list_t function_get_names(int get_hidden)
+{
+    std::set<wcstring> names;
+    scoped_lock lock(functions_lock);
+    autoload_names(names, get_hidden);
+
+    function_map_t::const_iterator iter;
+    for (iter = loaded_functions.begin(); iter != loaded_functions.end(); ++iter)
+    {
+        const wcstring &name = iter->first;
+
+        /* Maybe skip hidden */
+        if (! get_hidden)
+        {
+            if (name.empty() || name.at(0) == L'_') continue;
+        }
+        names.insert(name);
+    }
+    return wcstring_list_t(names.begin(), names.end());
+}
+
+const wchar_t *function_get_definition_file(const wcstring &name)
+{
+    scoped_lock lock(functions_lock);
+    const function_info_t *func = function_get(name);
+    return func ? func->definition_file : NULL;
+}
+
+
+int function_get_definition_offset(const wcstring &name)
+{
+    scoped_lock lock(functions_lock);
+    const function_info_t *func = function_get(name);
+    return func ? func->definition_offset : -1;
+}
diff --git a/src/function.h b/src/function.h
new file mode 100644
index 00000000..cd3f6f5a
--- /dev/null
+++ b/src/function.h
@@ -0,0 +1,188 @@
+/** \file function.h
+
+    Prototypes for functions for storing and retrieving function
+  information. These functions also take care of autoloading
+  functions in the $fish_function_path. Actual function evaluation
+  is taken care of by the parser and to some degree the builtin
+  handling library.
+*/
+
+#ifndef FISH_FUNCTION_H
+#define FISH_FUNCTION_H
+
+#include <vector>
+#include <map>
+
+#include "common.h"
+#include "event.h"
+#include "env.h"
+
+class parser_t;
+
+/**
+   Structure describing a function. This is used by the parser to
+   store data on a function while parsing it. It is not used
+   internally to store functions, the function_internal_data_t
+   structure is used for that purpose. Parhaps these two should be
+   merged.
+  */
+struct function_data_t
+{
+    /**
+       Name of function
+     */
+    wcstring name;
+    /**
+       Description of function
+     */
+    wcstring description;
+    /**
+       Function definition
+     */
+    const wchar_t *definition;
+    /**
+       List of all event handlers for this function
+     */
+    std::vector<event_t> events;
+    /**
+       List of all named arguments for this function
+     */
+    wcstring_list_t named_arguments;
+    /**
+       List of all variables that are inherited from the function definition scope.
+       The variable values are snapshotted when function_add() is called.
+      */
+    wcstring_list_t inherit_vars;
+    /**
+       Set to non-zero if invoking this function shadows the variables
+       of the underlying function.
+     */
+    int shadows;
+};
+
+class function_info_t
+{
+public:
+    /** Constructs relevant information from the function_data */
+    function_info_t(const function_data_t &data, const wchar_t *filename, int def_offset, bool autoload);
+
+    /** Used by function_copy */
+    function_info_t(const function_info_t &data, const wchar_t *filename, int def_offset, bool autoload);
+
+    /** Function definition */
+    const wcstring definition;
+
+    /** Function description. Only the description may be changed after the function is created. */
+    wcstring description;
+
+    /** File where this function was defined (intern'd string) */
+    const wchar_t * const definition_file;
+
+    /** Line where definition started */
+    const int definition_offset;
+
+    /** List of all named arguments for this function */
+    const wcstring_list_t named_arguments;
+
+    /** Mapping of all variables that were inherited from the function definition scope to their values */
+    const std::map<wcstring,env_var_t> inherit_vars;
+
+    /** Flag for specifying that this function was automatically loaded */
+    const bool is_autoload;
+
+    /** Set to true if invoking this function shadows the variables of the underlying function. */
+    const bool shadows;
+};
+
+
+/**
+   Initialize function data
+*/
+void function_init();
+
+/** Add a function. definition_line_offset is the line number of the function's definition within its source file */
+void function_add(const function_data_t &data, const parser_t &parser, int definition_line_offset = 0);
+
+/**
+   Remove the function with the specified name.
+*/
+void function_remove(const wcstring &name);
+
+/**
+   Returns by reference the definition of the function with the name \c name.
+   Returns true if successful, false if no function with the given name exists.
+*/
+bool function_get_definition(const wcstring &name, wcstring *out_definition);
+
+/**
+   Returns by reference the description of the function with the name \c name.
+   Returns true if the function exists and has a nonempty description, false if it does not.
+*/
+bool function_get_desc(const wcstring &name, wcstring *out_desc);
+
+/**
+   Sets the description of the function with the name \c name.
+*/
+void function_set_desc(const wcstring &name, const wcstring &desc);
+
+/**
+   Returns true if the function with the name name exists.
+*/
+int function_exists(const wcstring &name);
+
+/**
+   Returns true if the function with the name name exists, without triggering autoload.
+*/
+int function_exists_no_autoload(const wcstring &name, const env_vars_snapshot_t &vars);
+
+/**
+   Returns all function names.
+
+   \param get_hidden whether to include hidden functions, i.e. ones starting with an underscore
+*/
+wcstring_list_t function_get_names(int get_hidden);
+
+/**
+   Returns tha absolute path of the file where the specified function
+   was defined. Returns 0 if the file was defined on the commandline.
+
+   This function does not autoload functions, it will only work on
+   functions that have already been defined.
+
+   This returns an intern'd string.
+*/
+const wchar_t *function_get_definition_file(const wcstring &name);
+
+/**
+   Returns the linenumber where the definition of the specified
+   function started.
+
+   This function does not autoload functions, it will only work on
+   functions that have already been defined.
+*/
+int function_get_definition_offset(const wcstring &name);
+
+/**
+   Returns a list of all named arguments of the specified function.
+*/
+wcstring_list_t function_get_named_arguments(const wcstring &name);
+
+/**
+   Returns a mapping of all variables of the specified function that were inherited
+   from the scope of the function definition to their values.
+ */
+std::map<wcstring,env_var_t> function_get_inherit_vars(const wcstring &name);
+
+/**
+   Creates a new function using the same definition as the specified function.
+   Returns true if copy is successful.
+*/
+bool function_copy(const wcstring &name, const wcstring &new_name);
+
+
+/**
+   Returns whether this function shadows variables of the underlying function
+*/
+int function_get_shadows(const wcstring &name);
+
+#endif
diff --git a/src/highlight.cpp b/src/highlight.cpp
new file mode 100644
index 00000000..e07c7040
--- /dev/null
+++ b/src/highlight.cpp
@@ -0,0 +1,1621 @@
+/** \file highlight.c
+  Functions for syntax highlighting
+*/
+
+#include "config.h" // IWYU pragma: keep
+
+#include <sys/stat.h>
+#include <unistd.h>
+#include <errno.h>
+#include <wchar.h>
+#include <algorithm>
+#include <dirent.h>
+#include <map>
+#include <set>
+#include <string>
+
+#include "fallback.h"
+
+#include "wutil.h"
+#include "highlight.h"
+#include "tokenizer.h"
+#include "parse_util.h"
+#include "parse_constants.h"
+#include "builtin.h"
+#include "function.h"
+#include "env.h"
+#include "expand.h"
+#include "common.h"
+#include "output.h"
+#include "wildcard.h"
+#include "path.h"
+#include "history.h"
+#include "parse_tree.h"
+
+#define CURSOR_POSITION_INVALID ((size_t)(-1))
+
+/**
+   Number of elements in the highlight_var array
+*/
+#define VAR_COUNT ( sizeof(highlight_var)/sizeof(wchar_t *) )
+
+/** The environment variables used to specify the color of different tokens. This matches the order in highlight_spec_t */
+static const wchar_t * const highlight_var[] =
+{
+    L"fish_color_normal",
+    L"fish_color_error",
+    L"fish_color_command",
+    L"fish_color_end",
+    L"fish_color_param",
+    L"fish_color_comment",
+    L"fish_color_match",
+    L"fish_color_search_match",
+    L"fish_color_operator",
+    L"fish_color_escape",
+    L"fish_color_quote",
+    L"fish_color_redirection",
+    L"fish_color_autosuggestion",
+    L"fish_color_selection",
+
+    L"fish_pager_color_prefix",
+    L"fish_pager_color_completion",
+    L"fish_pager_color_description",
+    L"fish_pager_color_progress",
+    L"fish_pager_color_secondary"
+
+};
+
+/* If the given path looks like it's relative to the working directory, then prepend that working directory. */
+static wcstring apply_working_directory(const wcstring &path, const wcstring &working_directory)
+{
+    if (path.empty() || working_directory.empty())
+        return path;
+
+    /* We're going to make sure that if we want to prepend the wd, that the string has no leading / */
+    bool prepend_wd;
+    switch (path.at(0))
+    {
+        case L'/':
+        case L'~':
+            prepend_wd = false;
+            break;
+        default:
+            prepend_wd = true;
+            break;
+    }
+
+    if (! prepend_wd)
+    {
+        /* No need to prepend the wd, so just return the path we were given */
+        return path;
+    }
+    else
+    {
+        /* Remove up to one ./ */
+        wcstring path_component = path;
+        if (string_prefixes_string(L"./", path_component))
+        {
+            path_component.erase(0, 2);
+        }
+
+        /* Removing leading /s */
+        while (string_prefixes_string(L"/", path_component))
+        {
+            path_component.erase(0, 1);
+        }
+
+        /* Construct and return a new path */
+        wcstring new_path = working_directory;
+        append_path_component(new_path, path_component);
+        return new_path;
+    }
+}
+
+/* Determine if the filesystem containing the given fd is case insensitive. */
+typedef std::map<wcstring, bool> case_sensitivity_cache_t;
+bool fs_is_case_insensitive(const wcstring &path, int fd, case_sensitivity_cache_t &case_sensitivity_cache)
+{
+    /* If _PC_CASE_SENSITIVE is not defined, assume case sensitive */
+    bool result = false;
+#ifdef _PC_CASE_SENSITIVE
+    /* Try the cache first */
+    case_sensitivity_cache_t::iterator cache = case_sensitivity_cache.find(path);
+    if (cache != case_sensitivity_cache.end())
+    {
+        /* Use the cached value */
+        result = cache->second;
+    }
+    else
+    {
+        /* Ask the system. A -1 value means error (so assume case sensitive), a 1 value means case sensitive, and a 0 value means case insensitive */
+        long ret = fpathconf(fd, _PC_CASE_SENSITIVE);
+        result = (ret == 0);
+        case_sensitivity_cache[path] = result;
+    }
+#endif
+    return result;
+}
+
+/* Tests whether the specified string cpath is the prefix of anything we could cd to. directories is a list of possible parent directories (typically either the working directory, or the cdpath). This does I/O!
+
+   We expect the path to already be unescaped.
+*/
+bool is_potential_path(const wcstring &const_path, const wcstring_list_t &directories, path_flags_t flags, wcstring *out_path)
+{
+    ASSERT_IS_BACKGROUND_THREAD();
+
+    const bool require_dir = !!(flags & PATH_REQUIRE_DIR);
+    wcstring clean_path;
+    int has_magic = 0;
+    bool result = false;
+
+    wcstring path(const_path);
+    if (flags & PATH_EXPAND_TILDE)
+        expand_tilde(path);
+
+    //  debug( 1, L"%ls -> %ls ->%ls", path, tilde, unescaped );
+
+    for (size_t i=0; i < path.size(); i++)
+    {
+        wchar_t c = path.at(i);
+        switch (c)
+        {
+            case PROCESS_EXPAND:
+            case VARIABLE_EXPAND:
+            case VARIABLE_EXPAND_SINGLE:
+            case BRACKET_BEGIN:
+            case BRACKET_END:
+            case BRACKET_SEP:
+            case ANY_CHAR:
+            case ANY_STRING:
+            case ANY_STRING_RECURSIVE:
+            {
+                has_magic = 1;
+                break;
+            }
+
+            case INTERNAL_SEPARATOR:
+            {
+                break;
+            }
+
+            default:
+            {
+                clean_path.push_back(c);
+                break;
+            }
+
+        }
+
+    }
+
+    if (! has_magic && ! clean_path.empty())
+    {
+        /* Don't test the same path multiple times, which can happen if the path is absolute and the CDPATH contains multiple entries */
+        std::set<wcstring> checked_paths;
+
+        /* Keep a cache of which paths / filesystems are case sensitive */
+        case_sensitivity_cache_t case_sensitivity_cache;
+
+        for (size_t wd_idx = 0; wd_idx < directories.size() && ! result; wd_idx++)
+        {
+            const wcstring &wd = directories.at(wd_idx);
+
+            const wcstring abs_path = apply_working_directory(clean_path, wd);
+
+            /* Skip this if it's empty or we've already checked it */
+            if (abs_path.empty() || checked_paths.count(abs_path))
+                continue;
+            checked_paths.insert(abs_path);
+
+            /* If we end with a slash, then it must be a directory */
+            bool must_be_full_dir = abs_path.at(abs_path.size()-1) == L'/';
+            if (must_be_full_dir)
+            {
+                struct stat buf;
+                if (0 == wstat(abs_path, &buf) && S_ISDIR(buf.st_mode))
+                {
+                    result = true;
+                    /* Return the path suffix, not the whole absolute path */
+                    if (out_path)
+                        *out_path = clean_path;
+                }
+            }
+            else
+            {
+                DIR *dir = NULL;
+
+                /* We do not end with a slash; it does not have to be a directory */
+                const wcstring dir_name = wdirname(abs_path);
+                const wcstring base_name = wbasename(abs_path);
+                if (dir_name == L"/" && base_name == L"/")
+                {
+                    result = true;
+                    if (out_path)
+                        *out_path = clean_path;
+                }
+                else if ((dir = wopendir(dir_name)))
+                {
+                    // We opened the dir_name; look for a string where the base name prefixes it
+                    wcstring ent;
+
+                    // Check if we're case insensitive
+                    bool case_insensitive = fs_is_case_insensitive(dir_name, dirfd(dir), case_sensitivity_cache);
+
+                    // Don't ask for the is_dir value unless we care, because it can cause extra filesystem acces */
+                    bool is_dir = false;
+                    while (wreaddir_resolving(dir, dir_name, ent, require_dir ? &is_dir : NULL))
+                    {
+
+                        /* Determine which function to call to check for prefixes */
+                        bool (*prefix_func)(const wcstring &, const wcstring &);
+                        if (case_insensitive)
+                        {
+                            prefix_func = string_prefixes_string_case_insensitive;
+                        }
+                        else
+                        {
+                            prefix_func = string_prefixes_string;
+                        }
+
+                        if (prefix_func(base_name, ent) && (! require_dir || is_dir))
+                        {
+                            result = true;
+                            if (out_path)
+                            {
+                                /* We want to return the path in the same "form" as it was given. Take the given path, get its basename. Append that to the output if the basename actually prefixes the path (which it won't if the given path contains no slashes), and isn't a slash (so we don't duplicate slashes). Then append the directory entry. */
+
+                                out_path->clear();
+                                const wcstring path_base = wdirname(const_path);
+
+
+                                if (prefix_func(path_base, const_path))
+                                {
+                                    out_path->append(path_base);
+                                    if (! string_suffixes_string(L"/", *out_path))
+                                        out_path->push_back(L'/');
+                                }
+                                out_path->append(ent);
+                                /* We actually do want a trailing / for directories, since it makes autosuggestion a bit nicer */
+                                if (is_dir)
+                                    out_path->push_back(L'/');
+                            }
+                            break;
+                        }
+                    }
+                    closedir(dir);
+                }
+            }
+        }
+    }
+    return result;
+}
+
+
+/* Given a string, return whether it prefixes a path that we could cd into. Return that path in out_path. Expects path to be unescaped. */
+static bool is_potential_cd_path(const wcstring &path, const wcstring &working_directory, path_flags_t flags, wcstring *out_path)
+{
+    wcstring_list_t directories;
+
+    if (string_prefixes_string(L"./", path))
+    {
+        /* Ignore the CDPATH in this case; just use the working directory */
+        directories.push_back(working_directory);
+    }
+    else
+    {
+        /* Get the CDPATH */
+        env_var_t cdpath = env_get_string(L"CDPATH");
+        if (cdpath.missing_or_empty())
+            cdpath = L".";
+
+        /* Tokenize it into directories */
+        wcstokenizer tokenizer(cdpath, ARRAY_SEP_STR);
+        wcstring next_path;
+        while (tokenizer.next(next_path))
+        {
+            /* Ensure that we use the working directory for relative cdpaths like "." */
+            directories.push_back(apply_working_directory(next_path, working_directory));
+        }
+    }
+
+    /* Call is_potential_path with all of these directories */
+    bool result = is_potential_path(path, directories, flags | PATH_REQUIRE_DIR, out_path);
+#if 0
+    if (out_path)
+    {
+        printf("%ls -> %ls\n", path.c_str(), out_path->c_str());
+    }
+#endif
+    return result;
+}
+
+/* Given a plain statement node in a parse tree, get the command and return it, expanded appropriately for commands. If we succeed, return true. */
+bool plain_statement_get_expanded_command(const wcstring &src, const parse_node_tree_t &tree, const parse_node_t &plain_statement, wcstring *out_cmd)
+{
+    assert(plain_statement.type == symbol_plain_statement);
+    bool result = false;
+
+    /* Get the command */
+    wcstring cmd;
+    if (tree.command_for_plain_statement(plain_statement, src, &cmd))
+    {
+        /* Try expanding it. If we cannot, it's an error. */
+        if (expand_one(cmd, EXPAND_SKIP_CMDSUBST | EXPAND_SKIP_VARIABLES | EXPAND_SKIP_JOBS))
+        {
+            /* Success, return the expanded string by reference */
+            out_cmd->swap(cmd);
+            result = true;
+        }
+    }
+    return result;
+}
+
+
+rgb_color_t highlight_get_color(highlight_spec_t highlight, bool is_background)
+{
+    rgb_color_t result = rgb_color_t::normal();
+
+    /* If sloppy_background is set, then we look at the foreground color even if is_background is set */
+    bool treat_as_background = is_background && !(highlight & highlight_modifier_sloppy_background);
+
+    /* Get the primary variable */
+    size_t idx = highlight_get_primary(highlight);
+    if (idx >= VAR_COUNT)
+    {
+        return rgb_color_t::normal();
+    }
+
+    env_var_t val_wstr = env_get_string(highlight_var[idx]);
+
+//  debug( 1, L"%d -> %d -> %ls", highlight, idx, val );
+
+    if (val_wstr.missing())
+        val_wstr = env_get_string(highlight_var[0]);
+
+    if (! val_wstr.missing())
+        result = parse_color(val_wstr, treat_as_background);
+
+    /* Handle modifiers. */
+    if (highlight & highlight_modifier_valid_path)
+    {
+        env_var_t val2_wstr =  env_get_string(L"fish_color_valid_path");
+        const wcstring val2 = val2_wstr.missing() ? L"" : val2_wstr.c_str();
+
+        rgb_color_t result2 = parse_color(val2, is_background);
+        if (result.is_normal())
+            result = result2;
+        else
+        {
+            if (result2.is_bold())
+                result.set_bold(true);
+            if (result2.is_underline())
+                result.set_underline(true);
+        }
+    }
+
+    if (highlight & highlight_modifier_force_underline)
+    {
+        result.set_underline(true);
+    }
+
+    return result;
+}
+
+
+static bool has_expand_reserved(const wcstring &str)
+{
+    bool result = false;
+    for (size_t i=0; i < str.size(); i++)
+    {
+        wchar_t wc = str.at(i);
+        if (wc >= EXPAND_RESERVED && wc <= EXPAND_RESERVED_END)
+        {
+            result = true;
+            break;
+        }
+    }
+    return result;
+}
+
+/* Parse a command line. Return by reference the last command, and the last argument to that command (as a copied node), if any. This is used by autosuggestions */
+static bool autosuggest_parse_command(const wcstring &buff, wcstring *out_expanded_command, parse_node_t *out_last_arg)
+{
+    bool result = false;
+
+    /* Parse the buffer */
+    parse_node_tree_t parse_tree;
+    parse_tree_from_string(buff, parse_flag_continue_after_error | parse_flag_accept_incomplete_tokens, &parse_tree, NULL);
+
+    /* Find the last statement */
+    const parse_node_t *last_statement = parse_tree.find_last_node_of_type(symbol_plain_statement, NULL);
+    if (last_statement != NULL)
+    {
+        if (plain_statement_get_expanded_command(buff, parse_tree, *last_statement, out_expanded_command))
+        {
+            /* We got it */
+            result = true;
+
+            /* Find the last argument. If we don't get one, return an invalid node. */
+            const parse_node_t *last_arg = parse_tree.find_last_node_of_type(symbol_argument, last_statement);
+            if (last_arg != NULL)
+            {
+                *out_last_arg = *last_arg;
+            }
+        }
+    }
+    return result;
+}
+
+/* We have to return an escaped string here */
+bool autosuggest_suggest_special(const wcstring &str, const wcstring &working_directory, wcstring &out_suggestion)
+{
+    if (str.empty())
+        return false;
+
+    ASSERT_IS_BACKGROUND_THREAD();
+
+    /* Parse the string */
+    wcstring parsed_command;
+    parse_node_t last_arg_node(token_type_invalid);
+    if (! autosuggest_parse_command(str, &parsed_command, &last_arg_node))
+        return false;
+
+    bool result = false;
+    if (parsed_command == L"cd" && last_arg_node.type == symbol_argument && last_arg_node.has_source())
+    {
+        /* We can possibly handle this specially */
+        const wcstring escaped_dir = last_arg_node.get_source(str);
+        wcstring suggested_path;
+
+        /* We always return true because we recognized the command. This prevents us from falling back to dumber algorithms; for example we won't suggest a non-directory for the cd command. */
+        result = true;
+        out_suggestion.clear();
+
+        /* Unescape the parameter */
+        wcstring unescaped_dir;
+        bool unescaped = unescape_string(escaped_dir, &unescaped_dir, UNESCAPE_INCOMPLETE);
+
+        /* Determine the quote type we got from the input directory. */
+        wchar_t quote = L'\0';
+        parse_util_get_parameter_info(escaped_dir, 0, &quote, NULL, NULL);
+
+        /* Big hack to avoid expanding a tilde inside quotes */
+        path_flags_t path_flags = (quote == L'\0') ? PATH_EXPAND_TILDE : 0;
+        if (unescaped && is_potential_cd_path(unescaped_dir, working_directory, path_flags, &suggested_path))
+        {
+            /* Note: this looks really wrong for strings that have an "unescapable" character in them, e.g. a \t, because parse_util_escape_string_with_quote will insert that character */
+            wcstring escaped_suggested_path = parse_util_escape_string_with_quote(suggested_path, quote);
+
+            /* Return it */
+            out_suggestion = str;
+            out_suggestion.erase(last_arg_node.source_start);
+            if (quote != L'\0') out_suggestion.push_back(quote);
+            out_suggestion.append(escaped_suggested_path);
+            if (quote != L'\0') out_suggestion.push_back(quote);
+        }
+    }
+    else
+    {
+        /* Either an error or some other command, so we don't handle it specially */
+    }
+    return result;
+}
+
+bool autosuggest_validate_from_history(const history_item_t &item, file_detection_context_t &detector, const wcstring &working_directory, const env_vars_snapshot_t &vars)
+{
+    ASSERT_IS_BACKGROUND_THREAD();
+
+    bool handled = false, suggestionOK = false;
+
+    /* Parse the string */
+    wcstring parsed_command;
+    parse_node_t last_arg_node(token_type_invalid);
+    if (! autosuggest_parse_command(item.str(), &parsed_command, &last_arg_node))
+        return false;
+
+    if (parsed_command == L"cd" && last_arg_node.type == symbol_argument && last_arg_node.has_source())
+    {
+        /* We can possibly handle this specially */
+        wcstring dir = last_arg_node.get_source(item.str());
+        if (expand_one(dir, EXPAND_SKIP_CMDSUBST))
+        {
+            handled = true;
+            bool is_help = string_prefixes_string(dir, L"--help") || string_prefixes_string(dir, L"-h");
+            if (is_help)
+            {
+                suggestionOK = false;
+            }
+            else
+            {
+                wcstring path;
+                bool can_cd = path_get_cdpath(dir, &path, working_directory.c_str(), vars);
+                if (! can_cd)
+                {
+                    suggestionOK = false;
+                }
+                else if (paths_are_same_file(working_directory, path))
+                {
+                    /* Don't suggest the working directory as the path! */
+                    suggestionOK = false;
+                }
+                else
+                {
+                    suggestionOK = true;
+                }
+            }
+        }
+    }
+
+    /* If not handled specially, handle it here */
+    if (! handled)
+    {
+        bool cmd_ok = false;
+
+        if (path_get_path(parsed_command, NULL))
+        {
+            cmd_ok = true;
+        }
+        else if (builtin_exists(parsed_command) || function_exists_no_autoload(parsed_command, vars))
+        {
+            cmd_ok = true;
+        }
+
+        if (cmd_ok)
+        {
+            const path_list_t &paths = item.get_required_paths();
+            if (paths.empty())
+            {
+                suggestionOK= true;
+            }
+            else
+            {
+                suggestionOK = detector.paths_are_valid(paths);
+            }
+        }
+    }
+
+    return suggestionOK;
+}
+
+/* Highlights the variable starting with 'in', setting colors within the 'colors' array. Returns the number of characters consumed. */
+static size_t color_variable(const wchar_t *in, size_t in_len, std::vector<highlight_spec_t>::iterator colors)
+{
+    assert(in_len > 0);
+    assert(in[0] == L'$');
+
+    // Handle an initial run of $s.
+    size_t idx = 0;
+    size_t dollar_count = 0;
+    while (in[idx] == '$')
+    {
+        // Our color depends on the next char
+        wchar_t next = in[idx + 1];
+        if (next == L'$' || wcsvarchr(next))
+        {
+            colors[idx] = highlight_spec_operator;
+        }
+        else
+        {
+            colors[idx] = highlight_spec_error;
+        }
+        idx++;
+        dollar_count++;
+    }
+
+    // Handle a sequence of variable characters
+    while (wcsvarchr(in[idx]))
+    {
+        colors[idx++] = highlight_spec_operator;
+    }
+
+    // Handle a slice, up to dollar_count of them. Note that we currently don't do any validation of the slice's contents, e.g. $foo[blah] will not show an error even though it's invalid.
+    for (size_t slice_count=0; slice_count < dollar_count && in[idx] == L'['; slice_count++)
+    {
+        wchar_t *slice_begin = NULL, *slice_end = NULL;
+        int located = parse_util_locate_slice(in + idx, &slice_begin, &slice_end, false);
+        if (located == 1)
+        {
+            size_t slice_begin_idx = slice_begin - in, slice_end_idx = slice_end - in;
+            assert(slice_end_idx > slice_begin_idx);
+            colors[slice_begin_idx] = highlight_spec_operator;
+            colors[slice_end_idx] = highlight_spec_operator;
+            idx = slice_end_idx + 1;
+        }
+        else if (located == 0)
+        {
+            // not a slice
+            break;
+        }
+        else
+        {
+            assert(located < 0);
+            // syntax error
+            // Normally the entire token is colored red for us, but inside a double-quoted string
+            // that doesn't happen. As such, color the variable + the slice start red. Coloring any
+            // more than that looks bad, unless we're willing to try and detect where the double-quoted
+            // string ends, and I'd rather not do that.
+            std::fill(colors, colors + idx + 1, (highlight_spec_t)highlight_spec_error);
+            break;
+        }
+    }
+    return idx;
+}
+
+/* This function is a disaster badly in need of refactoring. It colors an argument, without regard to command substitutions. */
+static void color_argument_internal(const wcstring &buffstr, std::vector<highlight_spec_t>::iterator colors)
+{
+    const size_t buff_len = buffstr.size();
+    std::fill(colors, colors + buff_len, (highlight_spec_t)highlight_spec_param);
+
+    enum {e_unquoted, e_single_quoted, e_double_quoted} mode = e_unquoted;
+    int bracket_count=0;
+    for (size_t in_pos=0; in_pos < buff_len; in_pos++)
+    {
+        const wchar_t c = buffstr.at(in_pos);
+        switch (mode)
+        {
+            case e_unquoted:
+            {
+                if (c == L'\\')
+                {
+                    int fill_color = highlight_spec_escape; //may be set to highlight_error
+                    const size_t backslash_pos = in_pos;
+                    size_t fill_end = backslash_pos;
+
+                    // Move to the escaped character
+                    in_pos++;
+                    const wchar_t escaped_char = (in_pos < buff_len ? buffstr.at(in_pos) : L'\0');
+
+                    if (escaped_char == L'\0')
+                    {
+                        fill_end = in_pos;
+                        fill_color = highlight_spec_error;
+                    }
+                    else if (wcschr(L"~%", escaped_char))
+                    {
+                        if (in_pos == 1)
+                        {
+                            fill_end = in_pos + 1;
+                        }
+                    }
+                    else if (escaped_char == L',')
+                    {
+                        if (bracket_count)
+                        {
+                            fill_end = in_pos + 1;
+                        }
+                    }
+                    else if (wcschr(L"abefnrtv*?$(){}[]'\"<>^ \\#;|&", escaped_char))
+                    {
+                        fill_end = in_pos + 1;
+                    }
+                    else if (wcschr(L"c", escaped_char))
+                    {
+                        // Like \ci. So highlight three characters
+                        fill_end = in_pos + 1;
+                    }
+                    else if (wcschr(L"uUxX01234567", escaped_char))
+                    {
+                        long long res=0;
+                        int chars=2;
+                        int base=16;
+
+                        wchar_t max_val = ASCII_MAX;
+
+                        switch (escaped_char)
+                        {
+                            case L'u':
+                            {
+                                chars=4;
+                                max_val = UCS2_MAX;
+                                in_pos++;
+                                break;
+                            }
+
+                            case L'U':
+                            {
+                                chars=8;
+                                max_val = WCHAR_MAX;
+                                in_pos++;
+                                break;
+                            }
+
+                            case L'x':
+                            {
+                                in_pos++;
+                                break;
+                            }
+
+                            case L'X':
+                            {
+                                max_val = BYTE_MAX;
+                                in_pos++;
+                                break;
+                            }
+
+                            default:
+                            {
+                                // a digit like \12
+                                base=8;
+                                chars=3;
+                                break;
+                            }
+                        }
+
+                        // Consume
+                        for (int i=0; i < chars && in_pos < buff_len; i++)
+                        {
+                            long d = convert_digit(buffstr.at(in_pos), base);
+                            if (d < 0)
+                                break;
+                            res = (res * base) + d;
+                            in_pos++;
+                        }
+                        //in_pos is now at the first character that could not be converted (or buff_len)
+                        assert(in_pos >= backslash_pos && in_pos <= buff_len);
+                        fill_end = in_pos;
+
+                        // It's an error if we exceeded the max value
+                        if (res > max_val)
+                            fill_color = highlight_spec_error;
+
+                        // Subtract one from in_pos, so that the increment in the loop will move to the next character
+                        in_pos--;
+                    }
+                    assert(fill_end >= backslash_pos);
+                    std::fill(colors + backslash_pos, colors + fill_end, fill_color);
+                }
+                else
+                {
+                    // Not a backslash
+                    switch (c)
+                    {
+                        case L'~':
+                        case L'%':
+                        {
+                            if (in_pos == 0)
+                            {
+                                colors[in_pos] = highlight_spec_operator;
+                            }
+                            break;
+                        }
+
+                        case L'$':
+                        {
+                            assert(in_pos < buff_len);
+                            in_pos += color_variable(buffstr.c_str() + in_pos, buff_len - in_pos, colors + in_pos);
+                            /* subtract one to account for the upcoming loop increment */
+                            in_pos -= 1;
+                            break;
+                        }
+
+
+                        case L'*':
+                        case L'?':
+                        case L'(':
+                        case L')':
+                        {
+                            colors[in_pos] = highlight_spec_operator;
+                            break;
+                        }
+
+                        case L'{':
+                        {
+                            colors[in_pos] = highlight_spec_operator;
+                            bracket_count++;
+                            break;
+                        }
+
+                        case L'}':
+                        {
+                            colors[in_pos] = highlight_spec_operator;
+                            bracket_count--;
+                            break;
+                        }
+
+                        case L',':
+                        {
+                            if (bracket_count > 0)
+                            {
+                                colors[in_pos] = highlight_spec_operator;
+                            }
+
+                            break;
+                        }
+
+                        case L'\'':
+                        {
+                            colors[in_pos] = highlight_spec_quote;
+                            mode = e_single_quoted;
+                            break;
+                        }
+
+                        case L'\"':
+                        {
+                            colors[in_pos] = highlight_spec_quote;
+                            mode = e_double_quoted;
+                            break;
+                        }
+
+                    }
+                }
+                break;
+            }
+
+            /*
+             Mode 1 means single quoted string, i.e 'foo'
+             */
+            case e_single_quoted:
+            {
+                colors[in_pos] = highlight_spec_quote;
+                if (c == L'\\')
+                {
+                    // backslash
+                    if (in_pos + 1 < buff_len)
+                    {
+                        const wchar_t escaped_char = buffstr.at(in_pos + 1);
+                        if (escaped_char == L'\\' || escaped_char == L'\'')
+                        {
+                            colors[in_pos] = highlight_spec_escape; //backslash
+                            colors[in_pos + 1] = highlight_spec_escape; //escaped char
+                            in_pos += 1; //skip over backslash
+                        }
+                    }
+                }
+                else if (c == L'\'')
+                {
+                    mode = e_unquoted;
+                }
+                break;
+            }
+
+            /*
+             Mode 2 means double quoted string, i.e. "foo"
+             */
+            case e_double_quoted:
+            {
+                // slices are colored in advance, past `in_pos`, and we don't want to overwrite that
+                if (colors[in_pos] == highlight_spec_param)
+                {
+                    colors[in_pos] = highlight_spec_quote;
+                }
+                switch (c)
+                {
+                    case L'"':
+                    {
+                        mode = e_unquoted;
+                        break;
+                    }
+
+                    case L'\\':
+                    {
+                        // backslash
+                        if (in_pos + 1 < buff_len)
+                        {
+                            const wchar_t escaped_char = buffstr.at(in_pos + 1);
+                            if (wcschr(L"\\\"\n$", escaped_char))
+                            {
+                                colors[in_pos] = highlight_spec_escape; //backslash
+                                colors[in_pos + 1] = highlight_spec_escape; //escaped char
+                                in_pos += 1; //skip over backslash
+                            }
+                        }
+                        break;
+                    }
+
+                    case L'$':
+                    {
+                        in_pos += color_variable(buffstr.c_str() + in_pos, buff_len - in_pos, colors + in_pos);
+                        /* subtract one to account for the upcoming increment in the loop */
+                        in_pos -= 1;
+                        break;
+                    }
+
+                }
+                break;
+            }
+        }
+    }
+}
+
+/* Syntax highlighter helper */
+class highlighter_t
+{
+    /* The string we're highlighting. Note this is a reference memmber variable (to avoid copying)! We must not outlive this! */
+    const wcstring &buff;
+
+    /* Cursor position */
+    const size_t cursor_pos;
+
+    /* Environment variables. Again, a reference member variable! */
+    const env_vars_snapshot_t &vars;
+
+    /* Whether it's OK to do I/O */
+    const bool io_ok;
+
+    /* Working directory */
+    const wcstring working_directory;
+
+    /* The resulting colors */
+    typedef std::vector<highlight_spec_t> color_array_t;
+    color_array_t color_array;
+
+    /* The parse tree of the buff */
+    parse_node_tree_t parse_tree;
+
+    /* Color an argument */
+    void color_argument(const parse_node_t &node);
+
+    /* Color a redirection */
+    void color_redirection(const parse_node_t &node);
+
+    /* Color the arguments of the given node */
+    void color_arguments(const parse_node_t &list_node);
+
+    /* Color the redirections of the given node */
+    void color_redirections(const parse_node_t &list_node);
+
+    /* Color all the children of the command with the given type */
+    void color_children(const parse_node_t &parent, parse_token_type_t type, highlight_spec_t color);
+
+    /* Colors the source range of a node with a given color */
+    void color_node(const parse_node_t &node, highlight_spec_t color);
+
+public:
+
+    /* Constructor */
+    highlighter_t(const wcstring &str, size_t pos, const env_vars_snapshot_t &ev, const wcstring &wd, bool can_do_io) : buff(str), cursor_pos(pos), vars(ev), io_ok(can_do_io), working_directory(wd), color_array(str.size())
+    {
+        /* Parse the tree */
+        parse_tree_from_string(buff, parse_flag_continue_after_error | parse_flag_include_comments, &this->parse_tree, NULL);
+    }
+
+    /* Perform highlighting, returning an array of colors */
+    const color_array_t &highlight();
+};
+
+void highlighter_t::color_node(const parse_node_t &node, highlight_spec_t color)
+{
+    // Can only color nodes with valid source ranges
+    if (! node.has_source() || node.source_length == 0)
+        return;
+
+    // Fill the color array with our color in the corresponding range
+    size_t source_end = node.source_start + node.source_length;
+    assert(source_end >= node.source_start);
+    assert(source_end <= color_array.size());
+
+    std::fill(this->color_array.begin() + node.source_start, this->color_array.begin() + source_end, color);
+}
+
+/* node does not necessarily have type symbol_argument here */
+void highlighter_t::color_argument(const parse_node_t &node)
+{
+    if (! node.has_source())
+        return;
+
+    const wcstring arg_str = node.get_source(this->buff);
+
+    /* Get an iterator to the colors associated with the argument */
+    const size_t arg_start = node.source_start;
+    const color_array_t::iterator arg_colors = color_array.begin() + arg_start;
+
+    /* Color this argument without concern for command substitutions */
+    color_argument_internal(arg_str, arg_colors);
+
+    /* Now do command substitutions */
+    size_t cmdsub_cursor = 0, cmdsub_start = 0, cmdsub_end = 0;
+    wcstring cmdsub_contents;
+    while (parse_util_locate_cmdsubst_range(arg_str, &cmdsub_cursor, &cmdsub_contents, &cmdsub_start, &cmdsub_end, true /* accept incomplete */) > 0)
+    {
+        /* The cmdsub_start is the open paren. cmdsub_end is either the close paren or the end of the string. cmdsub_contents extends from one past cmdsub_start to cmdsub_end */
+        assert(cmdsub_end > cmdsub_start);
+        assert(cmdsub_end - cmdsub_start - 1 == cmdsub_contents.size());
+
+        /* Found a command substitution. Compute the position of the start and end of the cmdsub contents, within our overall src. */
+        const size_t arg_subcmd_start = arg_start + cmdsub_start, arg_subcmd_end = arg_start + cmdsub_end;
+
+        /* Highlight the parens. The open paren must exist; the closed paren may not if it was incomplete. */
+        assert(cmdsub_start < arg_str.size());
+        this->color_array.at(arg_subcmd_start) = highlight_spec_operator;
+        if (arg_subcmd_end < this->buff.size())
+            this->color_array.at(arg_subcmd_end) = highlight_spec_operator;
+
+        /* Compute the cursor's position within the cmdsub. We must be past the open paren (hence >) but can be at the end of the string or closed paren (hence <=) */
+        size_t cursor_subpos = CURSOR_POSITION_INVALID;
+        if (cursor_pos != CURSOR_POSITION_INVALID && cursor_pos > arg_subcmd_start && cursor_pos <= arg_subcmd_end)
+        {
+            /* The -1 because the cmdsub_contents does not include the open paren */
+            cursor_subpos = cursor_pos - arg_subcmd_start - 1;
+        }
+
+        /* Highlight it recursively. */
+        highlighter_t cmdsub_highlighter(cmdsub_contents, cursor_subpos, this->vars, this->working_directory, this->io_ok);
+        const color_array_t &subcolors = cmdsub_highlighter.highlight();
+
+        /* Copy out the subcolors back into our array */
+        assert(subcolors.size() == cmdsub_contents.size());
+        std::copy(subcolors.begin(), subcolors.end(), this->color_array.begin() + arg_subcmd_start + 1);
+    }
+}
+
+// Indicates whether the source range of the given node forms a valid path in the given working_directory
+static bool node_is_potential_path(const wcstring &src, const parse_node_t &node, const wcstring &working_directory)
+{
+    if (! node.has_source())
+        return false;
+
+
+    /* Get the node source, unescape it, and then pass it to is_potential_path along with the working directory (as a one element list) */
+    bool result = false;
+    wcstring token(src, node.source_start, node.source_length);
+    if (unescape_string_in_place(&token, UNESCAPE_SPECIAL))
+    {
+        /* Big hack: is_potential_path expects a tilde, but unescape_string gives us HOME_DIRECTORY. Put it back. */
+        if (! token.empty() && token.at(0) == HOME_DIRECTORY)
+            token.at(0) = L'~';
+
+        const wcstring_list_t working_directory_list(1, working_directory);
+        result = is_potential_path(token, working_directory_list, PATH_EXPAND_TILDE);
+    }
+    return result;
+}
+
+// Color all of the arguments of the given command
+void highlighter_t::color_arguments(const parse_node_t &list_node)
+{
+    /* Hack: determine whether the parent is the cd command, so we can show errors for non-directories */
+    bool cmd_is_cd = false;
+    if (this->io_ok)
+    {
+        const parse_node_t *parent = this->parse_tree.get_parent(list_node, symbol_plain_statement);
+        if (parent != NULL)
+        {
+            wcstring cmd_str;
+            if (plain_statement_get_expanded_command(this->buff, this->parse_tree, *parent, &cmd_str))
+            {
+                cmd_is_cd = (cmd_str == L"cd");
+            }
+        }
+    }
+
+    /* Find all the arguments of this list */
+    const parse_node_tree_t::parse_node_list_t nodes = this->parse_tree.find_nodes(list_node, symbol_argument);
+
+    for (size_t i=0; i < nodes.size(); i++)
+    {
+        const parse_node_t *child = nodes.at(i);
+        assert(child != NULL && child->type == symbol_argument);
+        this->color_argument(*child);
+
+        if (cmd_is_cd)
+        {
+            /* Mark this as an error if it's not 'help' and not a valid cd path */
+            wcstring param = child->get_source(this->buff);
+            if (expand_one(param, EXPAND_SKIP_CMDSUBST))
+            {
+                bool is_help = string_prefixes_string(param, L"--help") || string_prefixes_string(param, L"-h");
+                if (! is_help && this->io_ok && ! is_potential_cd_path(param, working_directory, PATH_EXPAND_TILDE, NULL))
+                {
+                    this->color_node(*child, highlight_spec_error);
+                }
+            }
+        }
+    }
+}
+
+void highlighter_t::color_redirection(const parse_node_t &redirection_node)
+{
+    assert(redirection_node.type == symbol_redirection);
+    if (! redirection_node.has_source())
+        return;
+
+    const parse_node_t *redirection_primitive = this->parse_tree.get_child(redirection_node, 0, parse_token_type_redirection); //like 2>
+    const parse_node_t *redirection_target = this->parse_tree.get_child(redirection_node, 1, parse_token_type_string); //like &1 or file path
+
+    if (redirection_primitive != NULL)
+    {
+        wcstring target;
+        const enum token_type redirect_type = this->parse_tree.type_for_redirection(redirection_node, this->buff, NULL, &target);
+
+        /* We may get a TOK_NONE redirection type, e.g. if the redirection is invalid */
+        this->color_node(*redirection_primitive, redirect_type == TOK_NONE ? highlight_spec_error : highlight_spec_redirection);
+
+        /* Check if the argument contains a command substitution. If so, highlight it as a param even though it's a command redirection, and don't try to do any other validation. */
+        if (parse_util_locate_cmdsubst(target.c_str(), NULL, NULL, true) != 0)
+        {
+            if (redirection_target != NULL)
+            {
+                this->color_argument(*redirection_target);
+            }
+        }
+        else
+        {
+            /* No command substitution, so we can highlight the target file or fd. For example, disallow redirections into a non-existent directory */
+            bool target_is_valid = true;
+
+            if (! this->io_ok)
+            {
+                /* I/O is disallowed, so we don't have much hope of catching anything but gross errors. Assume it's valid. */
+                target_is_valid = true;
+            }
+            else if (! expand_one(target, EXPAND_SKIP_CMDSUBST))
+            {
+                /* Could not be expanded */
+                target_is_valid = false;
+            }
+            else
+            {
+                /* Ok, we successfully expanded our target. Now verify that it works with this redirection. We will probably need it as a path (but not in the case of fd redirections */
+                const wcstring target_path = apply_working_directory(target, this->working_directory);
+                switch (redirect_type)
+                {
+                    case TOK_REDIRECT_FD:
+                    {
+                        /* target should be an fd. It must be all digits, and must not overflow. fish_wcstoi returns INT_MAX on overflow; we could instead check errno to disambiguiate this from a real INT_MAX fd, but instead we just disallow that. */
+                        const wchar_t *target_cstr = target.c_str();
+                        wchar_t *end = NULL;
+                        int fd = fish_wcstoi(target_cstr, &end, 10);
+
+                        /* The iswdigit check ensures there's no leading whitespace, the *end check ensures the entire string was consumed, and the numeric checks ensure the fd is at least zero and there was no overflow */
+                        target_is_valid = (iswdigit(target_cstr[0]) && *end == L'\0' && fd >= 0 && fd < INT_MAX);
+                    }
+                    break;
+
+                    case TOK_REDIRECT_IN:
+                    {
+                        /* Input redirections must have a readable non-directory */
+                        struct stat buf = {};
+                        target_is_valid = ! waccess(target_path, R_OK) && ! wstat(target_path, &buf) && ! S_ISDIR(buf.st_mode);
+                    }
+                    break;
+
+                    case TOK_REDIRECT_OUT:
+                    case TOK_REDIRECT_APPEND:
+                    case TOK_REDIRECT_NOCLOB:
+                    {
+                        /* Test whether the file exists, and whether it's writable (possibly after creating it). access() returns failure if the file does not exist. */
+                        bool file_exists = false, file_is_writable = false;
+                        int err = 0;
+
+                        struct stat buf = {};
+                        if (wstat(target_path, &buf) < 0)
+                        {
+                            err = errno;
+                        }
+
+                        if (string_suffixes_string(L"/", target))
+                        {
+                            /* Redirections to things that are directories is definitely not allowed */
+                            file_exists = false;
+                            file_is_writable = false;
+                        }
+                        else if (err == 0)
+                        {
+                            /* No err. We can write to it if it's not a directory and we have permission */
+                            file_exists = true;
+                            file_is_writable = ! S_ISDIR(buf.st_mode) && ! waccess(target_path, W_OK);
+                        }
+                        else if (err == ENOENT)
+                        {
+                            /* File does not exist. Check if its parent directory is writable. */
+                            wcstring parent = wdirname(target_path);
+
+                            /* Ensure that the parent ends with the path separator. This will ensure that we get an error if the parent directory is not really a directory. */
+                            if (! string_suffixes_string(L"/", parent))
+                                parent.push_back(L'/');
+
+                            /* Now the file is considered writable if the parent directory is writable */
+                            file_exists = false;
+                            file_is_writable = (0 == waccess(parent, W_OK));
+                        }
+                        else
+                        {
+                            /* Other errors we treat as not writable. This includes things like ENOTDIR. */
+                            file_exists = false;
+                            file_is_writable = false;
+                        }
+
+                        /* NOCLOB means that we must not overwrite files that exist */
+                        target_is_valid = file_is_writable && !(file_exists && redirect_type == TOK_REDIRECT_NOCLOB);
+                    }
+                    break;
+
+                    default:
+                        /* We should not get here, since the node was marked as a redirection, but treat it as an error for paranoia */
+                        target_is_valid = false;
+                        break;
+                }
+            }
+
+            if (redirection_target != NULL)
+            {
+                this->color_node(*redirection_target, target_is_valid ? highlight_spec_redirection : highlight_spec_error);
+            }
+        }
+    }
+}
+
+// Color all of the redirections of the given command
+void highlighter_t::color_redirections(const parse_node_t &list_node)
+{
+    const parse_node_tree_t::parse_node_list_t nodes = this->parse_tree.find_nodes(list_node, symbol_redirection);
+    for (size_t i=0; i < nodes.size(); i++)
+    {
+        this->color_redirection(*nodes.at(i));
+    }
+}
+
+/* Color all the children of the command with the given type */
+void highlighter_t::color_children(const parse_node_t &parent, parse_token_type_t type, highlight_spec_t color)
+{
+    for (node_offset_t idx=0; idx < parent.child_count; idx++)
+    {
+        const parse_node_t *child = this->parse_tree.get_child(parent, idx);
+        if (child != NULL && child->type == type)
+        {
+            this->color_node(*child, color);
+        }
+    }
+}
+
+/* Determine if a command is valid */
+static bool command_is_valid(const wcstring &cmd, enum parse_statement_decoration_t decoration, const wcstring &working_directory, const env_vars_snapshot_t &vars)
+{
+    /* Determine which types we check, based on the decoration */
+    bool builtin_ok = true, function_ok = true, abbreviation_ok = true, command_ok = true, implicit_cd_ok = true;
+    if (decoration == parse_statement_decoration_command || decoration == parse_statement_decoration_exec)
+    {
+        builtin_ok = false;
+        function_ok = false;
+        abbreviation_ok = false;
+        command_ok = true;
+        implicit_cd_ok = false;
+    }
+    else if (decoration == parse_statement_decoration_builtin)
+    {
+        builtin_ok = true;
+        function_ok = false;
+        abbreviation_ok = false;
+        command_ok = false;
+        implicit_cd_ok = false;
+    }
+
+    /* Check them */
+    bool is_valid = false;
+
+    /* Builtins */
+    if (! is_valid && builtin_ok)
+        is_valid = builtin_exists(cmd);
+
+    /* Functions */
+    if (! is_valid && function_ok)
+        is_valid = function_exists_no_autoload(cmd, vars);
+
+    /* Abbreviations */
+    if (! is_valid && abbreviation_ok)
+        is_valid = expand_abbreviation(cmd, NULL);
+
+    /* Regular commands */
+    if (! is_valid && command_ok)
+        is_valid = path_get_path(cmd, NULL, vars);
+
+    /* Implicit cd */
+    if (! is_valid && implicit_cd_ok)
+        is_valid = path_can_be_implicit_cd(cmd, NULL, working_directory.c_str(), vars);
+
+    /* Return what we got */
+    return is_valid;
+}
+
+const highlighter_t::color_array_t & highlighter_t::highlight()
+{
+    // If we are doing I/O, we must be in a background thread
+    if (io_ok)
+    {
+        ASSERT_IS_BACKGROUND_THREAD();
+    }
+
+    const size_t length = buff.size();
+    assert(this->buff.size() == this->color_array.size());
+
+    if (length == 0)
+        return color_array;
+
+    /* Start out at zero */
+    std::fill(this->color_array.begin(), this->color_array.end(), 0);
+
+#if 0
+    const wcstring dump = parse_dump_tree(parse_tree, buff);
+    fprintf(stderr, "%ls\n", dump.c_str());
+#endif
+
+    /* Walk the node tree */
+    for (parse_node_tree_t::const_iterator iter = parse_tree.begin(); iter != parse_tree.end(); ++iter)
+    {
+        const parse_node_t &node = *iter;
+
+        switch (node.type)
+        {
+                // Color direct string descendants, e.g. 'for' and 'in'.
+            case symbol_while_header:
+            case symbol_begin_header:
+            case symbol_function_header:
+            case symbol_if_clause:
+            case symbol_else_clause:
+            case symbol_case_item:
+            case symbol_boolean_statement:
+            case symbol_decorated_statement:
+            case symbol_if_statement:
+            {
+                this->color_children(node, parse_token_type_string, highlight_spec_command);
+            }
+            break;
+
+            case symbol_switch_statement:
+            {
+                const parse_node_t *literal_switch = this->parse_tree.get_child(node, 0, parse_token_type_string);
+                const parse_node_t *switch_arg = this->parse_tree.get_child(node, 1, symbol_argument);
+                this->color_node(*literal_switch, highlight_spec_command);
+                this->color_node(*switch_arg, highlight_spec_param);
+            }
+            break;
+
+            case symbol_for_header:
+            {
+                // Color the 'for' and 'in' as commands
+                const parse_node_t *literal_for_node = this->parse_tree.get_child(node, 0, parse_token_type_string);
+                const parse_node_t *literal_in_node = this->parse_tree.get_child(node, 2, parse_token_type_string);
+                this->color_node(*literal_for_node, highlight_spec_command);
+                this->color_node(*literal_in_node, highlight_spec_command);
+
+                // Color the variable name as a parameter
+                const parse_node_t *var_name_node = this->parse_tree.get_child(node, 1, parse_token_type_string);
+                this->color_argument(*var_name_node);
+            }
+            break;
+
+            case parse_token_type_pipe:
+            case parse_token_type_background:
+            case parse_token_type_end:
+            case symbol_optional_background:
+            {
+                this->color_node(node, highlight_spec_statement_terminator);
+            }
+            break;
+
+            case symbol_plain_statement:
+            {
+                /* Get the decoration from the parent */
+                enum parse_statement_decoration_t decoration = parse_tree.decoration_for_plain_statement(node);
+
+                /* Color the command */
+                const parse_node_t *cmd_node = parse_tree.get_child(node, 0, parse_token_type_string);
+                if (cmd_node != NULL && cmd_node->has_source())
+                {
+                    bool is_valid_cmd = false;
+                    if (! this->io_ok)
+                    {
+                        /* We cannot check if the command is invalid, so just assume it's valid */
+                        is_valid_cmd = true;
+                    }
+                    else
+                    {
+                        /* Check to see if the command is valid */
+                        wcstring cmd(buff, cmd_node->source_start, cmd_node->source_length);
+
+                        /* Try expanding it. If we cannot, it's an error. */
+                        bool expanded = expand_one(cmd, EXPAND_SKIP_CMDSUBST | EXPAND_SKIP_VARIABLES | EXPAND_SKIP_JOBS);
+                        if (expanded && ! has_expand_reserved(cmd))
+                        {
+                            is_valid_cmd = command_is_valid(cmd, decoration, working_directory, vars);
+                        }
+                    }
+                    this->color_node(*cmd_node, is_valid_cmd ? highlight_spec_command : highlight_spec_error);
+                }
+            }
+            break;
+
+
+            case symbol_arguments_or_redirections_list:
+            case symbol_argument_list:
+            {
+                /* Only work on root lists, so that we don't re-color child lists */
+                if (parse_tree.argument_list_is_root(node))
+                {
+                    this->color_arguments(node);
+                    this->color_redirections(node);
+                }
+            }
+            break;
+
+            case symbol_end_command:
+                this->color_node(node, highlight_spec_command);
+                break;
+
+            case parse_special_type_parse_error:
+            case parse_special_type_tokenizer_error:
+                this->color_node(node, highlight_spec_error);
+                break;
+
+            case parse_special_type_comment:
+                this->color_node(node, highlight_spec_comment);
+                break;
+
+            default:
+                break;
+        }
+    }
+
+    if (this->io_ok && this->cursor_pos <= this->buff.size())
+    {
+        /* If the cursor is over an argument, and that argument is a valid path, underline it */
+        for (parse_node_tree_t::const_iterator iter = parse_tree.begin(); iter != parse_tree.end(); ++iter)
+        {
+            const parse_node_t &node = *iter;
+
+            /* Must be an argument with source */
+            if (node.type != symbol_argument || ! node.has_source())
+                continue;
+
+            /* See if this node contains the cursor. We check <= source_length so that, when backspacing (and the cursor is just beyond the last token), we may still underline it */
+            if (this->cursor_pos >= node.source_start && this->cursor_pos - node.source_start <= node.source_length)
+            {
+                /* See if this is a valid path */
+                if (node_is_potential_path(buff, node, working_directory))
+                {
+                    /* It is, underline it. */
+                    for (size_t i=node.source_start; i < node.source_start + node.source_length; i++)
+                    {
+                        /* Don't color highlight_spec_error because it looks dorky. For example, trying to cd into a non-directory would show an underline and also red. */
+                        if (highlight_get_primary(this->color_array.at(i)) != highlight_spec_error)
+                        {
+                            this->color_array.at(i) |= highlight_modifier_valid_path;
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+    return color_array;
+}
+
+void highlight_shell(const wcstring &buff, std::vector<highlight_spec_t> &color, size_t pos, wcstring_list_t *error, const env_vars_snapshot_t &vars)
+{
+    /* Do something sucky and get the current working directory on this background thread. This should really be passed in. */
+    const wcstring working_directory = env_get_pwd_slash();
+
+    /* Highlight it! */
+    highlighter_t highlighter(buff, pos, vars, working_directory, true /* can do IO */);
+    color = highlighter.highlight();
+}
+
+void highlight_shell_no_io(const wcstring &buff, std::vector<highlight_spec_t> &color, size_t pos, wcstring_list_t *error, const env_vars_snapshot_t &vars)
+{
+    /* Do something sucky and get the current working directory on this background thread. This should really be passed in. */
+    const wcstring working_directory = env_get_pwd_slash();
+
+    /* Highlight it! */
+    highlighter_t highlighter(buff, pos, vars, working_directory, false /* no IO allowed */);
+    color = highlighter.highlight();
+}
+
+/**
+   Perform quote and parenthesis highlighting on the specified string.
+*/
+static void highlight_universal_internal(const wcstring &buffstr, std::vector<highlight_spec_t> &color, size_t pos)
+{
+    assert(buffstr.size() == color.size());
+    if (pos < buffstr.size())
+    {
+
+        /*
+          Highlight matching quotes
+        */
+        if ((buffstr.at(pos) == L'\'') || (buffstr.at(pos) == L'\"'))
+        {
+            std::vector<size_t> lst;
+
+            int level=0;
+            wchar_t prev_q=0;
+
+            const wchar_t * const buff = buffstr.c_str();
+            const wchar_t *str = buff;
+
+            int match_found=0;
+
+            while (*str)
+            {
+                switch (*str)
+                {
+                    case L'\\':
+                        str++;
+                        break;
+                    case L'\"':
+                    case L'\'':
+                        if (level == 0)
+                        {
+                            level++;
+                            lst.push_back(str-buff);
+                            prev_q = *str;
+                        }
+                        else
+                        {
+                            if (prev_q == *str)
+                            {
+                                size_t pos1, pos2;
+
+                                level--;
+                                pos1 = lst.back();
+                                pos2 = str-buff;
+                                if (pos1==pos || pos2==pos)
+                                {
+                                    color.at(pos1)|=highlight_make_background(highlight_spec_match);
+                                    color.at(pos2)|=highlight_make_background(highlight_spec_match);
+                                    match_found = 1;
+
+                                }
+                                prev_q = *str==L'\"'?L'\'':L'\"';
+                            }
+                            else
+                            {
+                                level++;
+                                lst.push_back(str-buff);
+                                prev_q = *str;
+                            }
+                        }
+
+                        break;
+                }
+                if ((*str == L'\0'))
+                    break;
+
+                str++;
+            }
+
+            if (!match_found)
+                color.at(pos) = highlight_make_background(highlight_spec_error);
+        }
+
+        /*
+          Highlight matching parenthesis
+        */
+        const wchar_t c = buffstr.at(pos);
+        if (wcschr(L"()[]{}", c))
+        {
+            int step = wcschr(L"({[", c)?1:-1;
+            wchar_t dec_char = *(wcschr(L"()[]{}", c) + step);
+            wchar_t inc_char = c;
+            int level = 0;
+            int match_found=0;
+            for (long i=pos; i >= 0 && (size_t)i < buffstr.size(); i+=step)
+            {
+                const wchar_t test_char = buffstr.at(i);
+                if (test_char == inc_char)
+                    level++;
+                if (test_char == dec_char)
+                    level--;
+                if (level == 0)
+                {
+                    long pos2 = i;
+                    color.at(pos)|=highlight_spec_match<<16;
+                    color.at(pos2)|=highlight_spec_match<<16;
+                    match_found=1;
+                    break;
+                }
+            }
+
+            if (!match_found)
+                color[pos] = highlight_make_background(highlight_spec_error);
+        }
+    }
+}
+
+void highlight_universal(const wcstring &buff, std::vector<highlight_spec_t> &color, size_t pos, wcstring_list_t *error, const env_vars_snapshot_t &vars)
+{
+    assert(buff.size() == color.size());
+    std::fill(color.begin(), color.end(), 0);
+    highlight_universal_internal(buff, color, pos);
+}
diff --git a/src/highlight.h b/src/highlight.h
new file mode 100644
index 00000000..9b89c0f2
--- /dev/null
+++ b/src/highlight.h
@@ -0,0 +1,136 @@
+/** \file highlight.h
+  Prototypes for functions for syntax highlighting
+*/
+
+#ifndef FISH_HIGHLIGHT_H
+#define FISH_HIGHLIGHT_H
+
+#include <assert.h>                     // for assert
+#include <stddef.h>                     // for size_t
+#include <stdint.h>                     // for uint32_t
+#include <vector>                       // for vector
+
+#include "common.h"                     // for wcstring, wcstring_list_t
+#include "env.h"
+#include "color.h"
+
+/* Internally, we specify highlight colors using a set of bits. Each highlight_spec is a 32 bit uint. We divide this into low 16 (foreground) and high 16 (background). Each half we further subdivide into low 8 (primary) and high 8 (modifiers). The primary is not a bitmask; specify exactly one. The modifiers are a bitmask; specify any number */
+enum
+{
+    /* The following values are mutually exclusive; specify at most one */
+    highlight_spec_normal = 0, // normal text
+    highlight_spec_error, // error
+    highlight_spec_command, //command
+    highlight_spec_statement_terminator, //process separator
+    highlight_spec_param, //command parameter (argument)
+    highlight_spec_comment, //comment
+    highlight_spec_match, //matching parenthesis, etc.
+    highlight_spec_search_match, //search match
+    highlight_spec_operator, //operator
+    highlight_spec_escape, //escape sequences
+    highlight_spec_quote, //quoted string
+    highlight_spec_redirection, //redirection
+    highlight_spec_autosuggestion, //autosuggestion
+    highlight_spec_selection,
+
+    // Pager support
+    highlight_spec_pager_prefix,
+    highlight_spec_pager_completion,
+    highlight_spec_pager_description,
+    highlight_spec_pager_progress,
+    highlight_spec_pager_secondary,
+
+    HIGHLIGHT_SPEC_PRIMARY_MASK = 0xFF,
+
+    /* The following values are modifiers */
+    highlight_modifier_valid_path = 0x100,
+    highlight_modifier_force_underline = 0x200,
+    highlight_modifier_sloppy_background = 0x300, //hackish, indicates that we should treat a foreground color as background, per certain historical behavior
+
+    /* Very special value */
+    highlight_spec_invalid = 0xFFFF
+
+};
+typedef uint32_t highlight_spec_t;
+
+inline highlight_spec_t highlight_get_primary(highlight_spec_t val)
+{
+    return val & HIGHLIGHT_SPEC_PRIMARY_MASK;
+}
+
+inline highlight_spec_t highlight_make_background(highlight_spec_t val)
+{
+    assert(val >> 16 == 0); //should have nothing in upper bits, otherwise this is already a background
+    return val << 16;
+}
+
+class history_item_t;
+struct file_detection_context_t;
+
+/**
+   Perform syntax highlighting for the shell commands in buff. The result is
+   stored in the color array as a color_code from the HIGHLIGHT_ enum
+   for each character in buff.
+
+   \param buff The buffer on which to perform syntax highlighting
+   \param color The array in wchich to store the color codes. The first 8 bits are used for fg color, the next 8 bits for bg color.
+   \param pos the cursor position. Used for quote matching, etc.
+   \param error a list in which a description of each error will be inserted. May be 0, in whcich case no error descriptions will be generated.
+*/
+void highlight_shell(const wcstring &buffstr, std::vector<highlight_spec_t> &color, size_t pos, wcstring_list_t *error, const env_vars_snapshot_t &vars);
+
+/**
+   Perform a non-blocking shell highlighting. The function will not do any I/O that may block. As a result, invalid commands may not be detected, etc.
+*/
+void highlight_shell_no_io(const wcstring &buffstr, std::vector<highlight_spec_t> &color, size_t pos, wcstring_list_t *error, const env_vars_snapshot_t &vars);
+
+/**
+   Perform syntax highlighting for the text in buff. Matching quotes and paranthesis are highlighted. The result is
+   stored in the color array as a color_code from the HIGHLIGHT_ enum
+   for each character in buff.
+
+   \param buff The buffer on which to perform syntax highlighting
+   \param color The array in wchich to store the color codes. The first 8 bits are used for fg color, the next 8 bits for bg color.
+   \param pos the cursor position. Used for quote matching, etc.
+   \param error a list in which a description of each error will be inserted. May be 0, in whcich case no error descriptions will be generated.
+*/
+void highlight_universal(const wcstring &buffstr, std::vector<highlight_spec_t> &color, size_t pos, wcstring_list_t *error, const env_vars_snapshot_t &vars);
+
+/**
+   Translate from HIGHLIGHT_* to FISH_COLOR_* according to environment
+   variables. Defaults to FISH_COLOR_NORMAL.
+
+   Example:
+
+   If the environment variable FISH_FISH_COLOR_ERROR is set to 'red', a
+   call to highlight_get_color( highlight_error) will return
+   FISH_COLOR_RED.
+*/
+rgb_color_t highlight_get_color(highlight_spec_t highlight, bool is_background);
+
+/** Given a command 'str' from the history, try to determine whether we ought to suggest it by specially recognizing the command.
+    Returns true if we validated the command. If so, returns by reference whether the suggestion is valid or not.
+*/
+bool autosuggest_validate_from_history(const history_item_t &item, file_detection_context_t &detector, const wcstring &working_directory, const env_vars_snapshot_t &vars);
+
+/** Given the command line contents 'str', return via reference a suggestion by specially recognizing the command. The suggestion is escaped. Returns true if we recognized the command (even if we couldn't think of a suggestion for it).
+*/
+bool autosuggest_suggest_special(const wcstring &str, const wcstring &working_directory, wcstring &outString);
+
+/* Tests whether the specified string cpath is the prefix of anything we could cd to. directories is a list of possible parent directories (typically either the working directory, or the cdpath). This does I/O!
+
+    This is used only internally to this file, and is exposed only for testing.
+*/
+enum
+{
+    /* The path must be to a directory */
+    PATH_REQUIRE_DIR = 1 << 0,
+
+    /* Expand any leading tilde in the path */
+    PATH_EXPAND_TILDE = 1 << 1
+};
+typedef unsigned int path_flags_t;
+bool is_potential_path(const wcstring &const_path, const wcstring_list_t &directories, path_flags_t flags, wcstring *out_path = NULL);
+
+#endif
+
diff --git a/src/history.cpp b/src/history.cpp
new file mode 100644
index 00000000..c526eb39
--- /dev/null
+++ b/src/history.cpp
@@ -0,0 +1,1905 @@
+/** \file history.c
+  History functions, part of the user interface.
+*/
+#include "config.h"
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <wchar.h>
+#include <errno.h>
+#include <sys/stat.h>
+#include <unistd.h>
+#include <sys/mman.h>
+#include <fcntl.h>
+#include <string.h>
+#include <time.h>
+#include <assert.h>
+#include <ctype.h>
+#include <wctype.h>
+#include <iterator>
+
+#include "fallback.h" // IWYU pragma: keep
+#include "sanity.h"
+#include "reader.h"
+#include "parse_tree.h"
+#include "wutil.h"
+#include "history.h"
+#include "common.h"
+#include "path.h"
+#include "signal.h"
+#include "iothread.h"
+#include "env.h"
+#include "lru.h"
+#include "parse_constants.h"
+#include <map>
+#include <algorithm>
+
+/*
+
+Our history format is intended to be valid YAML. Here it is:
+
+  - cmd: ssh blah blah blah
+    when: 2348237
+    paths:
+      - /path/to/something
+      - /path/to/something_else
+
+  Newlines are replaced by \n. Backslashes are replaced by \\.
+*/
+
+/** When we rewrite the history, the number of items we keep */
+#define HISTORY_SAVE_MAX (1024 * 256)
+
+/** Whether we print timing information */
+#define LOG_TIMES 0
+
+/** Default buffer size for flushing to the history file */
+#define HISTORY_OUTPUT_BUFFER_SIZE (4096 * 4)
+
+/* Helper class for certain output. This is basically a string that allows us to ensure we only flush at record boundaries, and avoids the copying of ostringstream. Have you ever tried to implement your own streambuf? Total insanity. */
+class history_output_buffer_t
+{
+    /* A null-terminated C string */
+    std::vector<char> buffer;
+
+    /* Offset is the offset of the null terminator */
+    size_t offset;
+
+    static size_t safe_strlen(const char *s)
+    {
+        return s ? strlen(s) : 0;
+    }
+
+public:
+
+    /* Add a bit more to HISTORY_OUTPUT_BUFFER_SIZE because we flush once we've exceeded that size */
+    history_output_buffer_t() : buffer(HISTORY_OUTPUT_BUFFER_SIZE + 128, '\0'), offset(0)
+    {
+    }
+
+    /* Append one or more strings */
+    void append(const char *s1, const char *s2 = NULL, const char *s3 = NULL)
+    {
+        const char *ptrs[4] = {s1, s2, s3, NULL};
+        const size_t lengths[4] = {safe_strlen(s1), safe_strlen(s2), safe_strlen(s3), 0};
+
+        /* Determine the additional size we'll need */
+        size_t additional_length = 0;
+        for (size_t i=0; i < sizeof lengths / sizeof *lengths; i++)
+        {
+            additional_length += lengths[i];
+        }
+
+        /* Allocate that much, plus a null terminator */
+        size_t required_size = offset + additional_length + 1;
+        if (required_size > buffer.size())
+        {
+            buffer.resize(required_size, '\0');
+        }
+
+        /* Copy */
+        for (size_t i=0; ptrs[i] != NULL; i++)
+        {
+            memmove(&buffer.at(offset), ptrs[i], lengths[i]);
+            offset += lengths[i];
+        }
+
+        /* Null terminator was appended by virtue of the resize() above (or in a previous invocation). */
+        assert(buffer.at(buffer.size() - 1) == '\0');
+    }
+
+    /* Output to a given fd, resetting our buffer. Returns true on success, false on error */
+    bool flush_to_fd(int fd)
+    {
+        bool result = write_loop(fd, &buffer.at(0), offset) >= 0;
+        offset = 0;
+        return result;
+    }
+
+    /* Return how much data we've accumulated */
+    size_t output_size() const
+    {
+        return offset;
+    }
+};
+
+class time_profiler_t
+{
+    const char *what;
+    double start;
+public:
+
+    time_profiler_t(const char *w)
+    {
+        if (LOG_TIMES)
+        {
+            what = w;
+            start = timef();
+        }
+    }
+
+    ~time_profiler_t()
+    {
+        if (LOG_TIMES)
+        {
+            double end = timef();
+            fprintf(stderr, "(LOG_TIMES %s: %02f msec)\n", what, (end - start) * 1000);
+        }
+    }
+};
+
+/* Lock a file via fcntl; returns true on success, false on failure. */
+static bool history_file_lock(int fd, short type)
+{
+    assert(type == F_RDLCK || type == F_WRLCK);
+    struct flock flk = {};
+    flk.l_type = type;
+    flk.l_whence = SEEK_SET;
+    int ret = fcntl(fd, F_SETLKW, (void *)&flk);
+    return ret != -1;
+}
+
+/* Our LRU cache is used for restricting the amount of history we have, and limiting how long we order it. */
+class history_lru_node_t : public lru_node_t
+{
+public:
+    time_t timestamp;
+    path_list_t required_paths;
+    history_lru_node_t(const history_item_t &item) :
+        lru_node_t(item.str()),
+        timestamp(item.timestamp()),
+        required_paths(item.required_paths)
+    {}
+};
+
+class history_lru_cache_t : public lru_cache_t<history_lru_node_t>
+{
+protected:
+
+    /* Override to delete evicted nodes */
+    virtual void node_was_evicted(history_lru_node_t *node)
+    {
+        delete node;
+    }
+
+public:
+    history_lru_cache_t(size_t max) : lru_cache_t<history_lru_node_t>(max) { }
+
+    /* Function to add a history item */
+    void add_item(const history_item_t &item)
+    {
+        /* Skip empty items */
+        if (item.empty())
+            return;
+
+        /* See if it's in the cache. If it is, update the timestamp. If not, we create a new node and add it. Note that calling get_node promotes the node to the front. */
+        history_lru_node_t *node = this->get_node(item.str());
+        if (node != NULL)
+        {
+            node->timestamp = std::max(node->timestamp, item.timestamp());
+            /* What to do about paths here? Let's just ignore them */
+        }
+        else
+        {
+            node = new history_lru_node_t(item);
+            this->add_node(node);
+        }
+    }
+};
+
+static pthread_mutex_t hist_lock = PTHREAD_MUTEX_INITIALIZER;
+
+static std::map<wcstring, history_t *> histories;
+
+static wcstring history_filename(const wcstring &name, const wcstring &suffix);
+
+/** Replaces newlines with a literal backslash followed by an n, and replaces backslashes with two backslashes. */
+static void escape_yaml(std::string *str);
+
+/** Undoes escape_yaml */
+static void unescape_yaml(std::string *str);
+
+/* We can merge two items if they are the same command. We use the more recent timestamp, more recent identifier, and the longer list of required paths. */
+bool history_item_t::merge(const history_item_t &item)
+{
+    bool result = false;
+    if (this->contents == item.contents)
+    {
+        this->creation_timestamp = std::max(this->creation_timestamp, item.creation_timestamp);
+        if (this->required_paths.size() < item.required_paths.size())
+        {
+            this->required_paths = item.required_paths;
+        }
+        if (this->identifier < item.identifier)
+        {
+            this->identifier = item.identifier;
+        }
+        result = true;
+    }
+    return result;
+}
+
+history_item_t::history_item_t(const wcstring &str) : contents(str), creation_timestamp(time(NULL)), identifier(0)
+{
+}
+
+history_item_t::history_item_t(const wcstring &str, time_t when, history_identifier_t ident) : contents(str), creation_timestamp(when), identifier(ident)
+{
+}
+
+bool history_item_t::matches_search(const wcstring &term, enum history_search_type_t type) const
+{
+    switch (type)
+    {
+
+        case HISTORY_SEARCH_TYPE_CONTAINS:
+            /* We consider equal strings to NOT match a contains search (so that you don't have to see history equal to what you typed). The length check ensures that. */
+            return contents.size() > term.size() && contents.find(term) != wcstring::npos;
+
+        case HISTORY_SEARCH_TYPE_PREFIX:
+            /* We consider equal strings to match a prefix search, so that autosuggest will allow suggesting what you've typed */
+            return string_prefixes_string(term, contents);
+
+        default:
+            sanity_lose();
+            return false;
+    }
+}
+
+/* Append our YAML history format to the provided vector at the given offset, updating the offset */
+static void append_yaml_to_buffer(const wcstring &wcmd, time_t timestamp, const path_list_t &required_paths, history_output_buffer_t *buffer)
+{
+    std::string cmd = wcs2string(wcmd);
+    escape_yaml(&cmd);
+    buffer->append("- cmd: ", cmd.c_str(), "\n");
+
+    char timestamp_str[96];
+    snprintf(timestamp_str, sizeof timestamp_str, "%ld", (long) timestamp);
+    buffer->append("  when: ", timestamp_str, "\n");
+
+    if (! required_paths.empty())
+    {
+        buffer->append("  paths:\n");
+
+        for (path_list_t::const_iterator iter = required_paths.begin(); iter != required_paths.end(); ++iter)
+        {
+            std::string path = wcs2string(*iter);
+            escape_yaml(&path);
+            buffer->append("    - ", path.c_str(), "\n");
+        }
+    }
+}
+
+// Parse a timestamp line that looks like this: spaces, "when:", spaces, timestamp, newline
+// The string is NOT null terminated; however we do know it contains a newline, so stop when we reach it
+static bool parse_timestamp(const char *str, time_t *out_when)
+{
+    const char *cursor = str;
+    /* Advance past spaces */
+    while (*cursor == ' ')
+        cursor++;
+
+    /* Look for "when:" */
+    size_t when_len = 5;
+    if (strncmp(cursor, "when:", when_len) != 0)
+        return false;
+    cursor += when_len;
+
+    /* Advance past spaces */
+    while (*cursor == ' ')
+        cursor++;
+
+    /* Try to parse a timestamp. */
+    long timestamp = 0;
+    if (isdigit(*cursor) && (timestamp = strtol(cursor, NULL, 0)) > 0)
+    {
+        *out_when = (time_t)timestamp;
+        return true;
+    }
+    return false;
+}
+
+// Returns a pointer to the start of the next line, or NULL
+// The next line must itself end with a newline
+// Note that the string is not null terminated
+static const char *next_line(const char *start, size_t length)
+{
+    /* Handle the hopeless case */
+    if (length < 1)
+        return NULL;
+
+    /* Get a pointer to the end, that we must not pass */
+    const char * const end = start + length;
+
+    /* Skip past the next newline */
+    const char *nextline = (const char *)memchr(start, '\n', length);
+    if (! nextline || nextline >= end)
+    {
+        return NULL;
+    }
+    /* Skip past the newline character itself */
+    if (++nextline >= end)
+    {
+        return NULL;
+    }
+
+    /* Make sure this new line is itself "newline terminated". If it's not, return NULL; */
+    const char *next_newline = (const char *)memchr(nextline, '\n', end - nextline);
+    if (! next_newline)
+    {
+        return NULL;
+    }
+
+    /* Done */
+    return nextline;
+}
+
+// Support for iteratively locating the offsets of history items
+// Pass the address and length of a mapped region.
+// Pass a pointer to a cursor size_t, initially 0
+// If custoff_timestamp is nonzero, skip items created at or after that timestamp
+// Returns (size_t)(-1) when done
+static size_t offset_of_next_item_fish_2_0(const char *begin, size_t mmap_length, size_t *inout_cursor, time_t cutoff_timestamp)
+{
+    size_t cursor = *inout_cursor;
+    size_t result = (size_t)(-1);
+    while (cursor < mmap_length)
+    {
+        const char *line_start = begin + cursor;
+
+        /* Advance the cursor to the next line */
+        const char *newline = (const char *)memchr(line_start, '\n', mmap_length - cursor);
+        if (newline == NULL)
+            break;
+
+        /* Advance the cursor past this line. +1 is for the newline */
+        cursor = newline - begin + 1;
+
+        /* Skip lines with a leading space, since these are in the interior of one of our items */
+        if (line_start[0] == ' ')
+            continue;
+
+        /* Skip very short lines to make one of the checks below easier */
+        if (newline - line_start < 3)
+            continue;
+
+        /* Try to be a little YAML compatible. Skip lines with leading %, ---, or ... */
+        if (! memcmp(line_start, "%", 1) ||
+                ! memcmp(line_start, "---", 3) ||
+                ! memcmp(line_start, "...", 3))
+            continue;
+        
+        
+        /* Hackish: fish 1.x rewriting a fish 2.0 history file can produce lines with lots of leading "- cmd: - cmd: - cmd:". Trim all but one leading "- cmd:". */
+        const char *double_cmd = "- cmd: - cmd: ";
+        const size_t double_cmd_len = strlen(double_cmd);
+        while (newline - line_start > double_cmd_len && ! memcmp(line_start, double_cmd, double_cmd_len))
+        {
+            /* Skip over just one of the - cmd. In the end there will be just one left. */
+            line_start += strlen("- cmd: ");
+        }
+        
+        /* Hackish: fish 1.x rewriting a fish 2.0 history file can produce commands like "when: 123456". Ignore those. */
+        const char *cmd_when = "- cmd:    when:";
+        const size_t cmd_when_len = strlen(cmd_when);
+        if (newline - line_start >= cmd_when_len && ! memcmp(line_start, cmd_when, cmd_when_len))
+            continue;
+        
+
+        /* At this point, we know line_start is at the beginning of an item. But maybe we want to skip this item because of timestamps. A 0 cutoff means we don't care; if we do care, then try parsing out a timestamp. */
+        if (cutoff_timestamp != 0)
+        {
+            /* Hackish fast way to skip items created after our timestamp. This is the mechanism by which we avoid "seeing" commands from other sessions that started after we started. We try hard to ensure that our items are sorted by their timestamps, so in theory we could just break, but I don't think that works well if (for example) the clock changes. So we'll read all subsequent items.
+             */
+            const char * const end = begin + mmap_length;
+
+            /* Walk over lines that we think are interior. These lines are not null terminated, but are guaranteed to contain a newline. */
+            bool has_timestamp = false;
+            time_t timestamp = 0;
+            const char *interior_line;
+
+            for (interior_line = next_line(line_start, end - line_start);
+                    interior_line != NULL && ! has_timestamp;
+                    interior_line = next_line(interior_line, end - interior_line))
+            {
+
+                /* If the first character is not a space, it's not an interior line, so we're done */
+                if (interior_line[0] != ' ')
+                    break;
+
+                /* Hackish optimization: since we just stepped over some interior line, update the cursor so we don't have to look at these lines next time */
+                cursor = interior_line - begin;
+
+                /* Try parsing a timestamp from this line. If we succeed, the loop will break. */
+                has_timestamp = parse_timestamp(interior_line, &timestamp);
+            }
+
+            /* Skip this item if the timestamp is past our cutoff. */
+            if (has_timestamp && timestamp > cutoff_timestamp)
+            {
+                continue;
+            }
+        }
+
+        /* We made it through the gauntlet. */
+        result = line_start - begin;
+        break;
+    }
+    *inout_cursor = cursor;
+    return result;
+}
+
+
+// Same as offset_of_next_item_fish_2_0, but for fish 1.x (pre fishfish)
+// Adapted from history_populate_from_mmap in history.c
+static size_t offset_of_next_item_fish_1_x(const char *begin, size_t mmap_length, size_t *inout_cursor, time_t cutoff_timestamp)
+{
+    if (mmap_length == 0 || *inout_cursor >= mmap_length)
+        return (size_t)(-1);
+
+    const char *end = begin + mmap_length;
+    const char *pos;
+
+    bool ignore_newline = false;
+    bool do_push = true;
+    bool all_done = false;
+
+    size_t result = *inout_cursor;
+    for (pos = begin + *inout_cursor; pos < end && ! all_done; pos++)
+    {
+
+        if (do_push)
+        {
+            ignore_newline = (*pos == '#');
+            do_push = false;
+        }
+
+        switch (*pos)
+        {
+            case '\\':
+            {
+                pos++;
+                break;
+            }
+
+            case '\n':
+            {
+                if (ignore_newline)
+                {
+                    ignore_newline = false;
+                }
+                else
+                {
+                    /* Note: pos will be left pointing just after this newline, because of the ++ in the loop */
+                    all_done = true;
+                }
+                break;
+            }
+        }
+    }
+    *inout_cursor = (pos - begin);
+    return result;
+}
+
+// Returns the offset of the next item based on the given history type, or -1
+static size_t offset_of_next_item(const char *begin, size_t mmap_length, history_file_type_t mmap_type, size_t *inout_cursor, time_t cutoff_timestamp)
+{
+    size_t result;
+    switch (mmap_type)
+    {
+        case history_type_fish_2_0:
+            result = offset_of_next_item_fish_2_0(begin, mmap_length, inout_cursor, cutoff_timestamp);
+            break;
+
+        case history_type_fish_1_x:
+            result = offset_of_next_item_fish_1_x(begin, mmap_length, inout_cursor, cutoff_timestamp);
+            break;
+
+        default:
+        case history_type_unknown:
+            // Oh well
+            result = (size_t)(-1);
+            break;
+    }
+    return result;
+}
+
+history_t & history_t::history_with_name(const wcstring &name)
+{
+    /* Note that histories are currently never deleted, so we can return a reference to them without using something like shared_ptr */
+    scoped_lock locker(hist_lock);
+    history_t *& current = histories[name];
+    if (current == NULL)
+        current = new history_t(name);
+    return *current;
+}
+
+history_t::history_t(const wcstring &pname) :
+    name(pname),
+    first_unwritten_new_item_index(0),
+    has_pending_item(false),
+    disable_automatic_save_counter(0),
+    mmap_start(NULL),
+    mmap_length(0),
+    mmap_type(history_file_type_t(-1)),
+    mmap_file_id(kInvalidFileID),
+    boundary_timestamp(time(NULL)),
+    countdown_to_vacuum(-1),
+    loaded_old(false),
+    chaos_mode(false)
+{
+    pthread_mutex_init(&lock, NULL);
+}
+
+history_t::~history_t()
+{
+    pthread_mutex_destroy(&lock);
+}
+
+void history_t::add(const history_item_t &item, bool pending)
+{
+    scoped_lock locker(lock);
+
+    /* Try merging with the last item */
+    if (! new_items.empty() && new_items.back().merge(item))
+    {
+        /* We merged, so we don't have to add anything. Maybe this item was pending, but it just got merged with an item that is not pending, so pending just becomes false. */
+        this->has_pending_item = false;
+    }
+    else
+    {
+        /* We have to add a new item */
+        new_items.push_back(item);
+        this->has_pending_item = pending;
+        save_internal_unless_disabled();
+    }
+}
+
+void history_t::save_internal_unless_disabled()
+{
+    /* This must be called while locked */
+    ASSERT_IS_LOCKED(lock);
+
+    /* Respect disable_automatic_save_counter */
+    if (disable_automatic_save_counter > 0)
+    {
+        return;
+    }
+
+    /* We may or may not vacuum. We try to vacuum every kVacuumFrequency items, but start the countdown at a random number so that even if the user never runs more than 25 commands, we'll eventually vacuum.  If countdown_to_vacuum is -1, it means we haven't yet picked a value for the counter. */
+    const int kVacuumFrequency = 25;
+    if (countdown_to_vacuum < 0)
+    {
+        static unsigned int seed = (unsigned int)time(NULL);
+        /* Generate a number in the range [0, kVacuumFrequency) */
+        countdown_to_vacuum = rand_r(&seed) / (RAND_MAX / kVacuumFrequency + 1);
+    }
+
+    /* Determine if we're going to vacuum */
+    bool vacuum = false;
+    if (countdown_to_vacuum == 0)
+    {
+        countdown_to_vacuum = kVacuumFrequency;
+        vacuum = true;
+    }
+
+    /* This might be a good candidate for moving to a background thread */
+    time_profiler_t profiler(vacuum ? "save_internal vacuum" : "save_internal no vacuum");
+    this->save_internal(vacuum);
+
+    /* Update our countdown */
+    assert(countdown_to_vacuum > 0);
+    countdown_to_vacuum--;
+}
+
+void history_t::add(const wcstring &str, history_identifier_t ident, bool pending)
+{
+    time_t when = time(NULL);
+    /* Big hack: do not allow timestamps equal to our boundary date. This is because we include items whose timestamps are equal to our boundary when reading old history, so we can catch "just closed" items. But this means that we may interpret our own items, that we just wrote, as old items, if we wrote them in the same second as our birthdate.
+    */
+    if (when == this->boundary_timestamp)
+    {
+        when++;
+    }
+
+    this->add(history_item_t(str, when, ident), pending);
+}
+
+void history_t::remove(const wcstring &str)
+{
+    /* Add to our list of deleted items */
+    deleted_items.insert(str);
+
+    /* Remove from our list of new items */
+    size_t idx = new_items.size();
+    while (idx--)
+    {
+        if (new_items.at(idx).str() == str)
+        {
+            new_items.erase(new_items.begin() + idx);
+
+            /* If this index is before our first_unwritten_new_item_index, then subtract one from that index so it stays pointing at the same item. If it is equal to or larger, then we have not yet writen this item, so we don't have to adjust the index. */
+            if (idx < first_unwritten_new_item_index)
+            {
+                first_unwritten_new_item_index--;
+            }
+        }
+    }
+    assert(first_unwritten_new_item_index <= new_items.size());
+}
+
+void history_t::set_valid_file_paths(const wcstring_list_t &valid_file_paths, history_identifier_t ident)
+{
+    /* 0 identifier is used to mean "not necessary" */
+    if (ident == 0)
+    {
+        return;
+    }
+
+    scoped_lock locker(lock);
+
+    /* Look for an item with the given identifier. It is likely to be at the end of new_items */
+    for (history_item_list_t::reverse_iterator iter = new_items.rbegin(); iter != new_items.rend(); ++iter)
+    {
+        if (iter->identifier == ident)
+        {
+            /* Found it */
+            iter->required_paths = valid_file_paths;
+            break;
+        }
+    }
+}
+
+void history_t::get_string_representation(wcstring *result, const wcstring &separator)
+{
+    scoped_lock locker(lock);
+
+    bool first = true;
+
+    std::set<wcstring> seen;
+
+    /* If we have a pending item, we skip the first encountered (i.e. last) new item */
+    bool next_is_pending = this->has_pending_item;
+
+    /* Append new items. Note that in principle we could use const_reverse_iterator, but we do not because reverse_iterator is not convertible to const_reverse_iterator ( http://github.com/fish-shell/fish-shell/issues/431 ) */
+    for (history_item_list_t::reverse_iterator iter=new_items.rbegin(); iter < new_items.rend(); ++iter)
+    {
+        /* Skip a pending item if we have one */
+        if (next_is_pending)
+        {
+            next_is_pending = false;
+            continue;
+        }
+
+        /* Skip duplicates */
+        if (! seen.insert(iter->str()).second)
+            continue;
+
+        if (! first)
+            result->append(separator);
+        result->append(iter->str());
+        first = false;
+    }
+
+    /* Append old items */
+    load_old_if_needed();
+    for (std::deque<size_t>::reverse_iterator iter = old_item_offsets.rbegin(); iter != old_item_offsets.rend(); ++iter)
+    {
+        size_t offset = *iter;
+        const history_item_t item = history_t::decode_item(mmap_start + offset, mmap_length - offset, mmap_type);
+
+        /* Skip duplicates */
+        if (! seen.insert(item.str()).second)
+            continue;
+
+        if (! first)
+            result->append(separator);
+        result->append(item.str());
+        first = false;
+    }
+}
+
+history_item_t history_t::item_at_index(size_t idx)
+{
+    scoped_lock locker(lock);
+
+    /* 0 is considered an invalid index */
+    assert(idx > 0);
+    idx--;
+
+    /* Determine how many "resolved" (non-pending) items we have. We can have at most one pending item, and it's always the last one. */
+    size_t resolved_new_item_count = new_items.size();
+    if (this->has_pending_item && resolved_new_item_count > 0)
+    {
+        resolved_new_item_count -= 1;
+    }
+    
+    /* idx=0 corresponds to the last resolved item */
+    if (idx < resolved_new_item_count)
+    {
+        return new_items.at(resolved_new_item_count - idx - 1);
+    }
+
+    /* Now look in our old items */
+    idx -= resolved_new_item_count;
+    load_old_if_needed();
+    size_t old_item_count = old_item_offsets.size();
+    if (idx < old_item_count)
+    {
+        /* idx=0 corresponds to last item in old_item_offsets */
+        size_t offset = old_item_offsets.at(old_item_count - idx - 1);
+        return history_t::decode_item(mmap_start + offset, mmap_length - offset, mmap_type);
+    }
+
+    /* Index past the valid range, so return an empty history item */
+    return history_item_t(wcstring(), 0);
+}
+
+/* Read one line, stripping off any newline, and updating cursor. Note that our input string is NOT null terminated; it's just a memory mapped file. */
+static size_t read_line(const char *base, size_t cursor, size_t len, std::string &result)
+{
+    /* Locate the newline */
+    assert(cursor <= len);
+    const char *start = base + cursor;
+    const char *newline = (char *)memchr(start, '\n', len - cursor);
+    if (newline != NULL)
+    {
+        /* We found a newline. */
+        result.assign(start, newline - start);
+
+        /* Return the amount to advance the cursor; skip over the newline */
+        return newline - start + 1;
+    }
+    else
+    {
+        /* We ran off the end */
+        result.clear();
+        return len - cursor;
+    }
+}
+
+/* Trims leading spaces in the given string, returning how many there were */
+static size_t trim_leading_spaces(std::string &str)
+{
+    size_t i = 0, max = str.size();
+    while (i < max && str[i] == ' ')
+        i++;
+    str.erase(0, i);
+    return i;
+}
+
+static bool extract_prefix_and_unescape_yaml(std::string *key, std::string *value, const std::string &line)
+{
+    size_t where = line.find(":");
+    if (where != std::string::npos)
+    {
+        key->assign(line, 0, where);
+
+        // skip a space after the : if necessary
+        size_t val_start = where + 1;
+        if (val_start < line.size() && line.at(val_start) == ' ')
+            val_start++;
+        value->assign(line, val_start, line.size() - val_start);
+
+        unescape_yaml(key);
+        unescape_yaml(value);
+    }
+    return where != std::string::npos;
+}
+
+/* Decode an item via the fish 2.0 format */
+history_item_t history_t::decode_item_fish_2_0(const char *base, size_t len)
+{
+    wcstring cmd;
+    time_t when = 0;
+    path_list_t paths;
+
+    size_t indent = 0, cursor = 0;
+    std::string key, value, line;
+    
+    /* Read the "- cmd:" line */
+    size_t advance = read_line(base, cursor, len, line);
+    trim_leading_spaces(line);
+    if (! extract_prefix_and_unescape_yaml(&key, &value, line) || key != "- cmd")
+    {
+        goto done;
+    }
+    
+    cursor += advance;
+    cmd = str2wcstring(value);
+
+    /* Read the remaining lines */
+    for (;;)
+    {
+        /* Read a line */
+        size_t advance = read_line(base, cursor, len, line);
+
+        /* Count and trim leading spaces */
+        size_t this_indent = trim_leading_spaces(line);
+        if (indent == 0)
+            indent = this_indent;
+
+        if (this_indent == 0 || indent != this_indent)
+            break;
+
+        if (! extract_prefix_and_unescape_yaml(&key, &value, line))
+            break;
+
+        /* We are definitely going to consume this line */
+        cursor += advance;
+
+        if (key == "when")
+        {
+            /* Parse an int from the timestamp. Should this fail, strtol returns 0; that's acceptable. */
+            char *end = NULL;
+            long tmp = strtol(value.c_str(), &end, 0);
+            when = tmp;
+        }
+        else if (key == "paths")
+        {
+            /* Read lines starting with " - " until we can't read any more */
+            for (;;)
+            {
+                size_t advance = read_line(base, cursor, len, line);
+                if (trim_leading_spaces(line) <= indent)
+                    break;
+
+                if (strncmp(line.c_str(), "- ", 2))
+                    break;
+
+                /* We're going to consume this line */
+                cursor += advance;
+
+                /* Skip the leading dash-space and then store this path it */
+                line.erase(0, 2);
+                unescape_yaml(&line);
+                paths.push_back(str2wcstring(line));
+            }
+        }
+    }
+done:
+    history_item_t result(cmd, when);
+    result.required_paths.swap(paths);
+    return result;
+}
+
+history_item_t history_t::decode_item(const char *base, size_t len, history_file_type_t type)
+{
+    switch (type)
+    {
+        case history_type_fish_1_x:
+            return history_t::decode_item_fish_1_x(base, len);
+        case history_type_fish_2_0:
+            return history_t::decode_item_fish_2_0(base, len);
+        default:
+            return history_item_t(L"");
+    }
+}
+
+/**
+   Remove backslashes from all newlines. This makes a string from the
+   history file better formated for on screen display.
+*/
+static wcstring history_unescape_newlines_fish_1_x(const wcstring &in_str)
+{
+    wcstring out;
+    for (const wchar_t *in = in_str.c_str(); *in; in++)
+    {
+        if (*in == L'\\')
+        {
+            if (*(in+1)!= L'\n')
+            {
+                out.push_back(*in);
+            }
+        }
+        else
+        {
+            out.push_back(*in);
+        }
+    }
+    return out;
+}
+
+
+/* Decode an item via the fish 1.x format. Adapted from fish 1.x's item_get(). */
+history_item_t history_t::decode_item_fish_1_x(const char *begin, size_t length)
+{
+
+    const char *end = begin + length;
+    const char *pos=begin;
+
+    bool was_backslash = 0;
+    wcstring out;
+    bool first_char = true;
+    bool timestamp_mode = false;
+    time_t timestamp = 0;
+
+    while (1)
+    {
+        wchar_t c;
+        mbstate_t state;
+        size_t res;
+
+        memset(&state, 0, sizeof(state));
+
+        res = mbrtowc(&c, pos, end-pos, &state);
+
+        if (res == (size_t)-1)
+        {
+            pos++;
+            continue;
+        }
+        else if (res == (size_t)-2)
+        {
+            break;
+        }
+        else if (res == (size_t)0)
+        {
+            pos++;
+            continue;
+        }
+        pos += res;
+
+        if (c == L'\n')
+        {
+            if (timestamp_mode)
+            {
+                const wchar_t *time_string = out.c_str();
+                while (*time_string && !iswdigit(*time_string))
+                    time_string++;
+                errno=0;
+
+                if (*time_string)
+                {
+                    time_t tm;
+                    wchar_t *end;
+
+                    errno = 0;
+                    tm = (time_t)wcstol(time_string, &end, 10);
+
+                    if (tm && !errno && !*end)
+                    {
+                        timestamp = tm;
+                    }
+
+                }
+
+                out.clear();
+                timestamp_mode = false;
+                continue;
+            }
+            if (!was_backslash)
+                break;
+        }
+
+        if (first_char)
+        {
+            if (c == L'#')
+                timestamp_mode = true;
+        }
+
+        first_char = false;
+
+        out.push_back(c);
+
+        was_backslash = ((c == L'\\') && !was_backslash);
+
+    }
+
+    out = history_unescape_newlines_fish_1_x(out);
+    return history_item_t(out, timestamp);
+}
+
+
+/* Try to infer the history file type based on inspecting the data */
+static history_file_type_t infer_file_type(const char *data, size_t len)
+{
+    history_file_type_t result = history_type_unknown;
+    if (len > 0)
+    {
+        /* Old fish started with a # */
+        if (data[0] == '#')
+        {
+            result = history_type_fish_1_x;
+        }
+        else
+        {
+            /* Assume new fish */
+            result = history_type_fish_2_0;
+        }
+    }
+    return result;
+}
+
+void history_t::populate_from_mmap(void)
+{
+    mmap_type = infer_file_type(mmap_start, mmap_length);
+    size_t cursor = 0;
+    for (;;)
+    {
+        size_t offset = offset_of_next_item(mmap_start, mmap_length, mmap_type, &cursor, boundary_timestamp);
+        // If we get back -1, we're done
+        if (offset == (size_t)(-1))
+            break;
+
+        // Remember this item
+        old_item_offsets.push_back(offset);
+    }
+}
+
+/* Do a private, read-only map of the entirety of a history file with the given name. Returns true if successful. Returns the mapped memory region by reference. */
+bool history_t::map_file(const wcstring &name, const char **out_map_start, size_t *out_map_len, file_id_t *file_id)
+{
+    bool result = false;
+    wcstring filename = history_filename(name, L"");
+    if (! filename.empty())
+    {
+        int fd = wopen_cloexec(filename, O_RDONLY);
+        if (fd >= 0)
+        {
+
+            /* Get the file ID if requested */
+            if (file_id != NULL)
+                *file_id = file_id_for_fd(fd);
+
+            /* Take a read lock to guard against someone else appending. This is released when the file is closed (below). We will read the file after releasing the lock, but that's not a problem, because we never modify already written data. In short, the purpose of this lock is to ensure we don't see the file size change mid-update.
+
+                We may fail to lock (e.g. on lockless NFS - see https://github.com/fish-shell/fish-shell/issues/685 ). In that case, we proceed as if it did not fail. The risk is that we may get an incomplete history item; this is unlikely because we only treat an item as valid if it has a terminating newline.
+
+                Simulate a failing lock in chaos_mode
+            */
+            if (! chaos_mode) history_file_lock(fd, F_RDLCK);
+            off_t len = lseek(fd, 0, SEEK_END);
+            if (len != (off_t)-1)
+            {
+                size_t mmap_length = (size_t)len;
+                if (lseek(fd, 0, SEEK_SET) == 0)
+                {
+                    char *mmap_start;
+                    if ((mmap_start = (char *)mmap(0, mmap_length, PROT_READ, MAP_PRIVATE, fd, 0)) != MAP_FAILED)
+                    {
+                        result = true;
+                        *out_map_start = mmap_start;
+                        *out_map_len = mmap_length;
+                    }
+                }
+            }
+            close(fd);
+        }
+    }
+    return result;
+}
+
+bool history_t::load_old_if_needed(void)
+{
+    if (loaded_old) return true;
+    loaded_old = true;
+
+
+    // PCA not sure why signals were blocked here
+    //signal_block();
+
+    bool ok = false;
+    if (map_file(name, &mmap_start, &mmap_length, &mmap_file_id))
+    {
+        // Here we've mapped the file
+        ok = true;
+        time_profiler_t profiler("populate_from_mmap");
+        this->populate_from_mmap();
+    }
+
+    //signal_unblock();
+    return ok;
+}
+
+void history_search_t::skip_matches(const wcstring_list_t &skips)
+{
+    external_skips = skips;
+    std::sort(external_skips.begin(), external_skips.end());
+}
+
+bool history_search_t::should_skip_match(const wcstring &str) const
+{
+    return std::binary_search(external_skips.begin(), external_skips.end(), str);
+}
+
+bool history_search_t::go_forwards()
+{
+    /* Pop the top index (if more than one) and return if we have any left */
+    if (prev_matches.size() > 1)
+    {
+        prev_matches.pop_back();
+        return true;
+    }
+    return false;
+}
+
+bool history_search_t::go_backwards()
+{
+    /* Backwards means increasing our index */
+    const size_t max_idx = (size_t)(-1);
+
+    size_t idx = 0;
+    if (! prev_matches.empty())
+        idx = prev_matches.back().first;
+
+    if (idx == max_idx)
+        return false;
+
+    const bool main_thread = is_main_thread();
+
+    while (++idx < max_idx)
+    {
+        if (main_thread ? reader_interrupted() : reader_thread_job_is_stale())
+        {
+            return false;
+        }
+
+        const history_item_t item = history->item_at_index(idx);
+        /* We're done if it's empty or we cancelled */
+        if (item.empty())
+        {
+            return false;
+        }
+
+        /* Look for a term that matches and that we haven't seen before */
+        const wcstring &str = item.str();
+        if (item.matches_search(term, search_type) && ! match_already_made(str) && ! should_skip_match(str))
+        {
+            prev_matches.push_back(prev_match_t(idx, item));
+            return true;
+        }
+    }
+    return false;
+}
+
+/** Goes to the end (forwards) */
+void history_search_t::go_to_end(void)
+{
+    prev_matches.clear();
+}
+
+/** Returns if we are at the end, which is where we start. */
+bool history_search_t::is_at_end(void) const
+{
+    return prev_matches.empty();
+}
+
+
+/** Goes to the beginning (backwards) */
+void history_search_t::go_to_beginning(void)
+{
+    /* Just go backwards as far as we can */
+    while (go_backwards())
+        ;
+}
+
+
+history_item_t history_search_t::current_item() const
+{
+    assert(! prev_matches.empty());
+    return prev_matches.back().second;
+}
+
+wcstring history_search_t::current_string() const
+{
+    history_item_t item = this->current_item();
+    return item.str();
+}
+
+bool history_search_t::match_already_made(const wcstring &match) const
+{
+    for (std::vector<prev_match_t>::const_iterator iter = prev_matches.begin(); iter != prev_matches.end(); ++iter)
+    {
+        if (iter->second.str() == match)
+            return true;
+    }
+    return false;
+}
+
+static void replace_all(std::string *str, const char *needle, const char *replacement)
+{
+    size_t needle_len = strlen(needle), replacement_len = strlen(replacement);
+    size_t offset = 0;
+    while ((offset = str->find(needle, offset)) != std::string::npos)
+    {
+        str->replace(offset, needle_len, replacement);
+        offset += replacement_len;
+    }
+}
+
+static void escape_yaml(std::string *str)
+{
+    replace_all(str, "\\", "\\\\"); //replace one backslash with two
+    replace_all(str, "\n", "\\n"); //replace newline with backslash + literal n
+}
+
+/* This function is called frequently, so it ought to be fast. */
+static void unescape_yaml(std::string *str)
+{
+    size_t cursor = 0, size = str->size();
+    while (cursor < size)
+    {
+        // Operate on a const version of str, to avoid needless COWs that at() does.
+        const std::string &const_str = *str;
+
+        // Look for a backslash
+        size_t backslash = const_str.find('\\', cursor);
+        if (backslash == std::string::npos || backslash + 1 >= size)
+        {
+            // Either not found, or found as the last character
+            break;
+        }
+        else
+        {
+            // Backslash found. Maybe we'll do something about it. Be sure to invoke the const version of at().
+            char escaped_char = const_str.at(backslash + 1);
+            if (escaped_char == '\\')
+            {
+                // Two backslashes in a row. Delete the second one.
+                str->erase(backslash + 1, 1);
+                size--;
+            }
+            else if (escaped_char == 'n')
+            {
+                // Backslash + n. Replace with a newline.
+                str->replace(backslash, 2, "\n");
+                size--;
+            }
+            // The character at index backslash has now been made whole; start at the next character
+            cursor = backslash + 1;
+        }
+    }
+}
+
+static wcstring history_filename(const wcstring &name, const wcstring &suffix)
+{
+    wcstring path;
+    if (! path_get_config(path))
+        return L"";
+
+    wcstring result = path;
+    result.append(L"/");
+    result.append(name);
+    result.append(L"_history");
+    result.append(suffix);
+    return result;
+}
+
+void history_t::clear_file_state()
+{
+    ASSERT_IS_LOCKED(lock);
+    /* Erase everything we know about our file */
+    if (mmap_start != NULL && mmap_start != MAP_FAILED)
+    {
+        munmap((void *)mmap_start, mmap_length);
+    }
+    mmap_start = NULL;
+    mmap_length = 0;
+    loaded_old = false;
+    old_item_offsets.clear();
+}
+
+void history_t::compact_new_items()
+{
+    /* Keep only the most recent items with the given contents. This algorithm could be made more efficient, but likely would consume more memory too. */
+    std::set<wcstring> seen;
+    size_t idx = new_items.size();
+    while (idx--)
+    {
+        const history_item_t &item = new_items[idx];
+        if (! seen.insert(item.contents).second)
+        {
+            // This item was not inserted because it was already in the set, so delete the item at this index
+            new_items.erase(new_items.begin() + idx);
+
+            if (idx < first_unwritten_new_item_index)
+            {
+                /* Decrement first_unwritten_new_item_index if we are deleting a previously written item */
+                first_unwritten_new_item_index--;
+            }
+        }
+    }
+}
+
+bool history_t::save_internal_via_rewrite()
+{
+    /* This must be called while locked */
+    ASSERT_IS_LOCKED(lock);
+
+    bool ok = false;
+
+    wcstring tmp_name_template = history_filename(name, L".XXXXXX");
+    if (! tmp_name_template.empty())
+    {
+        /* Make an LRU cache to save only the last N elements */
+        history_lru_cache_t lru(HISTORY_SAVE_MAX);
+
+        /* Insert old items in, from old to new. Merge them with our new items, inserting items with earlier timestamps first. */
+        history_item_list_t::const_iterator new_item_iter = new_items.begin();
+
+        /* Map in existing items (which may have changed out from underneath us, so don't trust our old mmap'd data) */
+        const char *local_mmap_start = NULL;
+        size_t local_mmap_size = 0;
+        if (map_file(name, &local_mmap_start, &local_mmap_size, NULL))
+        {
+            const history_file_type_t local_mmap_type = infer_file_type(local_mmap_start, local_mmap_size);
+            size_t cursor = 0;
+            for (;;)
+            {
+                size_t offset = offset_of_next_item(local_mmap_start, local_mmap_size, local_mmap_type, &cursor, 0);
+                /* If we get back -1, we're done */
+                if (offset == (size_t)(-1))
+                    break;
+
+                /* Try decoding an old item */
+                const history_item_t old_item = history_t::decode_item(local_mmap_start + offset, local_mmap_size - offset, local_mmap_type);
+                if (old_item.empty() || deleted_items.count(old_item.str()) > 0)
+                {
+//                    debug(0, L"Item is deleted : %s\n", old_item.str().c_str());
+                    continue;
+                }
+                /* The old item may actually be more recent than our new item, if it came from another session. Insert all new items at the given index with an earlier timestamp. */
+                for (; new_item_iter != new_items.end(); ++new_item_iter)
+                {
+                    if (new_item_iter->timestamp() < old_item.timestamp())
+                    {
+                        /* This "new item" is in fact older. */
+                        lru.add_item(*new_item_iter);
+                    }
+                    else
+                    {
+                        /* The new item is not older. */
+                        break;
+                    }
+                }
+
+                /* Now add this old item */
+                lru.add_item(old_item);
+            }
+            munmap((void *)local_mmap_start, local_mmap_size);
+        }
+
+        /* Insert any remaining new items */
+        for (; new_item_iter != new_items.end(); ++new_item_iter)
+        {
+            lru.add_item(*new_item_iter);
+        }
+
+        signal_block();
+
+        /* Try to create a temporary file, up to 10 times. We don't use mkstemps because we want to open it CLO_EXEC. This should almost always succeed on the first try. */
+        int out_fd = -1;
+        wcstring tmp_name;
+        for (size_t attempt = 0; attempt < 10 && out_fd == -1; attempt++)
+        {
+            char *narrow_str = wcs2str(tmp_name_template.c_str());
+#if HAVE_MKOSTEMP
+            out_fd = mkostemp(narrow_str, O_CLOEXEC);
+            if (out_fd >= 0)
+            {
+                tmp_name = str2wcstring(narrow_str);
+            }
+#else
+            if (narrow_str && mktemp(narrow_str))
+            {
+                /* It was successfully templated; try opening it atomically */
+                tmp_name = str2wcstring(narrow_str);
+                out_fd = wopen_cloexec(tmp_name, O_WRONLY | O_CREAT | O_EXCL | O_TRUNC, 0600);
+            }
+#endif
+            free(narrow_str);
+        }
+
+        if (out_fd >= 0)
+        {
+            /* Write them out */
+            bool errored = false;
+            history_output_buffer_t buffer;
+            for (history_lru_cache_t::iterator iter = lru.begin(); iter != lru.end(); ++iter)
+            {
+                const history_lru_node_t *node = *iter;
+                append_yaml_to_buffer(node->key, node->timestamp, node->required_paths, &buffer);
+                if (buffer.output_size() >= HISTORY_OUTPUT_BUFFER_SIZE && ! buffer.flush_to_fd(out_fd))
+                {
+                    errored = true;
+                    break;
+                }
+            }
+
+            if (! errored && buffer.flush_to_fd(out_fd))
+            {
+                ok = true;
+            }
+
+            if (! ok)
+            {
+                /*
+                  This message does not have high enough priority to
+                  be shown by default.
+                */
+                debug(2, L"Error when writing history file");
+            }
+            else
+            {
+                wcstring new_name = history_filename(name, wcstring());
+                if (0 > wrename(tmp_name, new_name))
+                {
+                    debug(2, L"Error when renaming history file");
+                }
+            }
+            close(out_fd);
+        }
+
+        signal_unblock();
+
+        /* Make sure we clear all nodes, since this doesn't happen automatically */
+        lru.evict_all_nodes();
+    }
+
+    if (ok)
+    {
+        /* We've saved everything, so we have no more unsaved items */
+        this->first_unwritten_new_item_index = new_items.size();
+
+        /* We deleted our deleted items */
+        this->deleted_items.clear();
+
+        /* Our history has been written to the file, so clear our state so we can re-reference the file. */
+        this->clear_file_state();
+    }
+
+
+    return ok;
+}
+
+bool history_t::save_internal_via_appending()
+{
+    /* This must be called while locked */
+    ASSERT_IS_LOCKED(lock);
+
+    /* No deleting allowed */
+    assert(deleted_items.empty());
+
+    bool ok = false;
+
+    /* If the file is different (someone vacuumed it) then we need to update our mmap */
+    bool file_changed = false;
+
+    /* Get the path to the real history file */
+    wcstring history_path = history_filename(name, wcstring());
+
+    signal_block();
+
+    /* Open the file */
+    int out_fd = wopen_cloexec(history_path, O_WRONLY | O_APPEND);
+    if (out_fd >= 0)
+    {
+        /* Check to see if the file changed */
+        if (file_id_for_fd(out_fd) != mmap_file_id)
+            file_changed = true;
+
+        /* Exclusive lock on the entire file. This is released when we close the file (below). This may fail on (e.g.) lockless NFS. If so, proceed as if it did not fail; the risk is that we may get interleaved history items, which is considered better than no history, or forcing everything through the slow copy-move mode. We try to minimize this possibility by writing with O_APPEND.
+
+           Simulate a failing lock in chaos_mode
+        */
+        if (! chaos_mode) history_file_lock(out_fd, F_WRLCK);
+
+        /* We (hopefully successfully) took the exclusive lock. Append to the file.
+           Note that this is sketchy for a few reasons:
+             - Another shell may have appended its own items with a later timestamp, so our file may no longer be sorted by timestamp.
+             - Another shell may have appended the same items, so our file may now contain duplicates.
+
+           We cannot modify any previous parts of our file, because other instances may be reading those portions. We can only append.
+
+           Originally we always rewrote the file on saving, which avoided both of these problems. However, appending allows us to save history after every command, which is nice!
+
+           Periodically we "clean up" the file by rewriting it, so that most of the time it doesn't have duplicates, although we don't yet sort by timestamp (the timestamp isn't really used for much anyways).
+        */
+
+        /* So far so good. Write all items at or after first_unwritten_new_item_index. Note that we write even a pending item - pending items are ignored by history within the command itself, but should still be written to the file. */
+
+        bool errored = false;
+        history_output_buffer_t buffer;
+        while (first_unwritten_new_item_index < new_items.size())
+        {
+            const history_item_t &item = new_items.at(first_unwritten_new_item_index);
+            append_yaml_to_buffer(item.str(), item.timestamp(), item.get_required_paths(), &buffer);
+            if (buffer.output_size() >= HISTORY_OUTPUT_BUFFER_SIZE)
+            {
+                errored = ! buffer.flush_to_fd(out_fd);
+                if (errored) break;
+            }
+
+            /* We wrote this item, hooray */
+            first_unwritten_new_item_index++;
+        }
+
+        if (! errored && buffer.flush_to_fd(out_fd))
+        {
+            ok = true;
+        }
+
+        close(out_fd);
+    }
+
+    signal_unblock();
+
+    /* If someone has replaced the file, forget our file state */
+    if (file_changed)
+    {
+        this->clear_file_state();
+    }
+
+    return ok;
+}
+
+/** Save the specified mode to file; optionally also vacuums */
+void history_t::save_internal(bool vacuum)
+{
+    /* This must be called while locked */
+    ASSERT_IS_LOCKED(lock);
+
+    /* Nothing to do if there's no new items */
+    if (first_unwritten_new_item_index >= new_items.size() && deleted_items.empty())
+        return;
+
+    /* Compact our new items so we don't have duplicates */
+    this->compact_new_items();
+
+    /* Try saving. If we have items to delete, we have to rewrite the file. If we do not, we can append to it. */
+    bool ok = false;
+    if (! vacuum && deleted_items.empty())
+    {
+        /* Try doing a fast append */
+        ok = save_internal_via_appending();
+    }
+    if (! ok)
+    {
+        /* We did not or could not append; rewrite the file ("vacuum" it) */
+        ok = this->save_internal_via_rewrite();
+    }
+}
+
+void history_t::save(void)
+{
+    scoped_lock locker(lock);
+    this->save_internal(false);
+}
+
+void history_t::disable_automatic_saving()
+{
+    scoped_lock locker(lock);
+    disable_automatic_save_counter++;
+    assert(disable_automatic_save_counter != 0); // overflow!
+}
+
+void history_t::enable_automatic_saving()
+{
+    scoped_lock locker(lock);
+    assert(disable_automatic_save_counter > 0); //underflow
+    disable_automatic_save_counter--;
+    save_internal_unless_disabled();
+}
+
+
+void history_t::clear(void)
+{
+    scoped_lock locker(lock);
+    new_items.clear();
+    deleted_items.clear();
+    first_unwritten_new_item_index = 0;
+    old_item_offsets.clear();
+    wcstring filename = history_filename(name, L"");
+    if (! filename.empty())
+        wunlink(filename);
+    this->clear_file_state();
+
+}
+
+bool history_t::is_empty(void)
+{
+    scoped_lock locker(lock);
+
+    /* If we have new items, we're not empty */
+    if (! new_items.empty())
+        return false;
+
+    bool empty = false;
+    if (loaded_old)
+    {
+        /* If we've loaded old items, see if we have any offsets */
+        empty = old_item_offsets.empty();
+    }
+    else
+    {
+        /* If we have not loaded old items, don't actually load them (which may be expensive); just stat the file and see if it exists and is nonempty */
+        const wcstring where = history_filename(name, L"");
+        struct stat buf = {};
+        if (wstat(where, &buf) != 0)
+        {
+            /* Access failed, assume missing */
+            empty = true;
+        }
+        else
+        {
+            /* We're empty if the file is empty */
+            empty = (buf.st_size == 0);
+        }
+    }
+    return empty;
+}
+
+/* Indicate whether we ought to import the bash history file into fish */
+static bool should_import_bash_history_line(const std::string &line)
+{
+    if (line.empty())
+        return false;
+
+    /* Very naive tests! Skip export; probably should skip others. */
+    const char * const ignore_prefixes[] =
+    {
+        "export ",
+        "#"
+    };
+
+    for (size_t i=0; i < sizeof ignore_prefixes / sizeof *ignore_prefixes; i++)
+    {
+        const char *prefix = ignore_prefixes[i];
+        if (! line.compare(0, strlen(prefix), prefix))
+        {
+            return false;
+        }
+    }
+
+    /* Skip lines with backticks */
+    if (line.find('`') != std::string::npos)
+        return false;
+
+    return true;
+}
+
+void history_t::populate_from_bash(FILE *stream)
+{
+    /* Bash's format is very simple: just lines with #s for comments.
+       Ignore a few commands that are bash-specific. This list ought to be expanded.
+    */
+    std::string line;
+    for (;;)
+    {
+        line.clear();
+        bool success = false, has_newline = false;
+
+        /* Loop until we've read a line */
+        do
+        {
+            char buff[128];
+            success = !! fgets(buff, sizeof buff, stream);
+            if (success)
+            {
+                /* Skip the newline */
+                char *newline = strchr(buff, '\n');
+                if (newline) *newline = '\0';
+                has_newline = (newline != NULL);
+
+                /* Append what we've got */
+                line.append(buff);
+            }
+        }
+        while (success && ! has_newline);
+
+        /* Maybe add this line */
+        if (should_import_bash_history_line(line))
+        {
+            this->add(str2wcstring(line));
+        }
+
+        if (line.empty())
+            break;
+    }
+}
+
+void history_t::incorporate_external_changes()
+{
+    /* To incorporate new items, we simply update our timestamp to now, so that items from previous instances get added. We then clear the file state so that we remap the file. Note that this is somehwhat expensive because we will be going back over old items. An optimization would be to preserve old_item_offsets so that they don't have to be recomputed. (However, then items *deleted* in other instances would not show up here). */
+    time_t new_timestamp = time(NULL);
+    scoped_lock locker(lock);
+
+    /* If for some reason the clock went backwards, we don't want to start dropping items; therefore we only do work if time has progressed. This also makes multiple calls cheap. */
+    if (new_timestamp > this->boundary_timestamp)
+    {
+        this->boundary_timestamp = new_timestamp;
+        this->clear_file_state();
+    }
+}
+
+void history_init()
+{
+}
+
+
+void history_destroy()
+{
+    /* Save all histories */
+    for (std::map<wcstring, history_t *>::iterator iter = histories.begin(); iter != histories.end(); ++iter)
+    {
+        iter->second->save();
+    }
+}
+
+
+void history_sanity_check()
+{
+    /*
+      No sanity checking implemented yet...
+    */
+}
+
+int file_detection_context_t::perform_file_detection(bool test_all)
+{
+    ASSERT_IS_BACKGROUND_THREAD();
+    valid_paths.clear();
+    int result = 1;
+    for (path_list_t::const_iterator iter = potential_paths.begin(); iter != potential_paths.end(); ++iter)
+    {
+        if (path_is_valid(*iter, working_directory))
+        {
+            /* Push the original (possibly relative) path */
+            valid_paths.push_back(*iter);
+        }
+        else
+        {
+            /* Not a valid path */
+            result = 0;
+            if (! test_all)
+                break;
+        }
+    }
+    return result;
+}
+
+bool file_detection_context_t::paths_are_valid(const path_list_t &paths)
+{
+    this->potential_paths = paths;
+    return perform_file_detection(false) > 0;
+}
+
+file_detection_context_t::file_detection_context_t(history_t *hist, history_identifier_t ident) :
+    history(hist),
+    working_directory(env_get_pwd_slash()),
+    history_item_identifier(ident)
+{
+}
+
+static int threaded_perform_file_detection(file_detection_context_t *ctx)
+{
+    ASSERT_IS_BACKGROUND_THREAD();
+    assert(ctx != NULL);
+    return ctx->perform_file_detection(true /* test all */);
+}
+
+static void perform_file_detection_done(file_detection_context_t *ctx, int success)
+{
+    ASSERT_IS_MAIN_THREAD();
+
+    /* Now that file detection is done, update the history item with the valid file paths */
+    ctx->history->set_valid_file_paths(ctx->valid_paths, ctx->history_item_identifier);
+
+    /* Allow saving again */
+    ctx->history->enable_automatic_saving();
+
+    /* Done with the context. */
+    delete ctx;
+}
+
+static bool string_could_be_path(const wcstring &potential_path)
+{
+    // Assume that things with leading dashes aren't paths
+    if (potential_path.empty() || potential_path.at(0) == L'-')
+    {
+        return false;
+    }
+    return true;
+}
+
+void history_t::add_pending_with_file_detection(const wcstring &str)
+{
+    ASSERT_IS_MAIN_THREAD();
+    path_list_t potential_paths;
+
+    /* Find all arguments that look like they could be file paths */
+    bool impending_exit = false;
+    parse_node_tree_t tree;
+    parse_tree_from_string(str, parse_flag_none, &tree, NULL);
+    size_t count = tree.size();
+
+    for (size_t i=0; i < count; i++)
+    {
+        const parse_node_t &node = tree.at(i);
+        if (! node.has_source())
+        {
+            continue;
+        }
+
+        if (node.type == symbol_argument)
+        {
+            wcstring potential_path = node.get_source(str);
+            bool unescaped = unescape_string_in_place(&potential_path, UNESCAPE_DEFAULT);
+            if (unescaped && string_could_be_path(potential_path))
+            {
+                potential_paths.push_back(potential_path);
+            }
+        }
+        else if (node.type == symbol_plain_statement)
+        {
+            /* Hack hack hack - if the command is likely to trigger an exit, then don't do background file detection, because we won't be able to write it to our history file before we exit. */
+            if (tree.decoration_for_plain_statement(node) == parse_statement_decoration_exec)
+            {
+                impending_exit = true;
+            }
+
+            wcstring command;
+            tree.command_for_plain_statement(node, str, &command);
+            unescape_string_in_place(&command, UNESCAPE_DEFAULT);
+            if (contains(command, L"exit", L"reboot"))
+            {
+                impending_exit = true;
+            }
+        }
+    }
+
+    /* If we got a path, we'll perform file detection for autosuggestion hinting */
+    history_identifier_t identifier = 0;
+    if (! potential_paths.empty() && ! impending_exit)
+    {
+        /* Grab the next identifier */
+        static history_identifier_t sLastIdentifier = 0;
+        identifier = ++sLastIdentifier;
+
+        /* Create a new detection context */
+        file_detection_context_t *context = new file_detection_context_t(this, identifier);
+        context->potential_paths.swap(potential_paths);
+
+        /* Prevent saving until we're done, so we have time to get the paths */
+        this->disable_automatic_saving();
+
+        /* Kick it off. Even though we haven't added the item yet, it updates the item on the main thread, so we can't race */
+        iothread_perform(threaded_perform_file_detection, perform_file_detection_done, context);
+    }
+
+    /* Actually add the item to the history. */
+    this->add(str, identifier, true /* pending */);
+
+    /* If we think we're about to exit, save immediately, regardless of any disabling. This may cause us to lose file hinting for some commands, but it beats losing history items */
+    if (impending_exit)
+    {
+        this->save();
+    }
+}
+
+/* Very simple, just mark that we have no more pending items */
+void history_t::resolve_pending()
+{
+    scoped_lock locker(lock);
+    this->has_pending_item = false;
+}
diff --git a/src/history.h b/src/history.h
new file mode 100644
index 00000000..7f04890e
--- /dev/null
+++ b/src/history.h
@@ -0,0 +1,375 @@
+/** \file history.h
+  Prototypes for history functions, part of the user interface.
+*/
+
+#ifndef FISH_HISTORY_H
+#define FISH_HISTORY_H
+
+#include "common.h"
+#include "wutil.h"
+#include <deque>
+#include <vector>
+#include <utility>
+#include <set>
+#include <pthread.h>
+#include <stddef.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <time.h>
+#include <string>
+
+/* fish supports multiple shells writing to history at once. Here is its strategy:
+
+1. All history files are append-only. Data, once written, is never modified.
+2. A history file may be re-written ("vacuumed"). This involves reading in the file and writing a new one, while performing maintenance tasks: discarding items in an LRU fashion until we reach the desired maximum count, removing duplicates, and sorting them by timestamp (eventually, not implemented yet). The new file is atomically moved into place via rename().
+3. History files are mapped in via mmap(). Before the file is mapped, the file takes a fcntl read lock. The purpose of this lock is to avoid seeing a transient state where partial data has been written to the file.
+4. History is appended to under a fcntl write lock.
+5. The chaos_mode boolean can be set to true to do things like lower buffer sizes which can trigger race conditions. This is useful for testing.
+*/
+
+typedef std::vector<wcstring> path_list_t;
+
+enum history_search_type_t
+{
+    /** The history searches for strings containing the given string */
+    HISTORY_SEARCH_TYPE_CONTAINS,
+
+    /** The history searches for strings starting with the given string */
+    HISTORY_SEARCH_TYPE_PREFIX
+};
+
+typedef uint32_t history_identifier_t;
+
+class history_item_t
+{
+    friend class history_t;
+    friend class history_lru_node_t;
+    friend class history_tests_t;
+
+private:
+    explicit history_item_t(const wcstring &str);
+    explicit history_item_t(const wcstring &, time_t, history_identifier_t ident = 0);
+
+    /** Attempts to merge two compatible history items together */
+    bool merge(const history_item_t &item);
+
+    /** The actual contents of the entry */
+    wcstring contents;
+
+    /** Original creation time for the entry */
+    time_t creation_timestamp;
+
+    /** Sometimes unique identifier used for hinting */
+    history_identifier_t identifier;
+
+    /** Paths that we require to be valid for this item to be autosuggested */
+    path_list_t required_paths;
+    
+public:
+    const wcstring &str() const
+    {
+        return contents;
+    }
+
+    bool empty() const
+    {
+        return contents.empty();
+    }
+
+    /* Whether our contents matches a search term. */
+    bool matches_search(const wcstring &term, enum history_search_type_t type) const;
+
+    time_t timestamp() const
+    {
+        return creation_timestamp;
+    }
+
+    const path_list_t &get_required_paths() const
+    {
+        return required_paths;
+    }
+
+    bool operator==(const history_item_t &other) const
+    {
+        return contents == other.contents &&
+               creation_timestamp == other.creation_timestamp &&
+               required_paths == other.required_paths;
+    }
+};
+
+typedef std::deque<history_item_t> history_item_list_t;
+
+/* The type of file that we mmap'd */
+enum history_file_type_t
+{
+    history_type_unknown,
+    history_type_fish_2_0,
+    history_type_fish_1_x
+};
+
+class history_t
+{
+    friend class history_tests_t;
+private:
+    /** No copying */
+    history_t(const history_t&);
+    history_t &operator=(const history_t&);
+
+    /** Private creator */
+    history_t(const wcstring &pname);
+
+    /** Privately add an item. If pending, the item will not be returned by history searches until a call to resolve_pending. */
+    void add(const history_item_t &item, bool pending = false);
+
+    /** Destructor */
+    ~history_t();
+
+    /** Lock for thread safety */
+    pthread_mutex_t lock;
+
+    /** Internal function */
+    void clear_file_state();
+
+    /** The name of this list. Used for picking a suitable filename and for switching modes. */
+    const wcstring name;
+
+    /** New items. Note that these are NOT discarded on save. We need to keep these around so we can distinguish between items in our history and items in the history of other shells that were started after we were started. */
+    history_item_list_t new_items;
+
+    /** The index of the first new item that we have not yet written. */
+    size_t first_unwritten_new_item_index;
+
+    /** Whether we have a pending item. If so, the most recently added item is ignored by item_at_index. */
+    bool has_pending_item;
+    
+    /** Whether we should disable saving to the file for a time */
+    uint32_t disable_automatic_save_counter;
+
+    /** Deleted item contents. */
+    std::set<wcstring> deleted_items;
+
+    /** The mmaped region for the history file */
+    const char *mmap_start;
+
+    /** The size of the mmap'd region */
+    size_t mmap_length;
+
+    /** The type of file we mmap'd */
+    history_file_type_t mmap_type;
+
+    /** The file ID of the file we mmap'd */
+    file_id_t mmap_file_id;
+
+    /** The boundary timestamp distinguishes old items from new items. Items whose timestamps are <= the boundary are considered "old". Items whose timestemps are > the boundary are new, and are ignored by this instance (unless they came from this instance). The timestamp may be adjusted by incorporate_external_changes() */
+    time_t boundary_timestamp;
+
+    /** How many items we add until the next vacuum. Initially a random value. */
+    int countdown_to_vacuum;
+
+    /** Figure out the offsets of our mmap data */
+    void populate_from_mmap(void);
+
+    /** List of old items, as offsets into out mmap data */
+    std::deque<size_t> old_item_offsets;
+
+    /** Whether we've loaded old items */
+    bool loaded_old;
+
+    /** Loads old if necessary */
+    bool load_old_if_needed(void);
+
+    /** Memory maps the history file if necessary */
+    bool mmap_if_needed(void);
+
+    /** Deletes duplicates in new_items. */
+    void compact_new_items();
+
+    /** Saves history by rewriting the file */
+    bool save_internal_via_rewrite();
+
+    /** Saves history by appending to the file */
+    bool save_internal_via_appending();
+
+    /** Saves history */
+    void save_internal(bool vacuum);
+
+    /** Saves history, maybe */
+    void save_internal_unless_disabled();
+
+    /* Do a private, read-only map of the entirety of a history file with the given name. Returns true if successful. Returns the mapped memory region by reference. */
+    bool map_file(const wcstring &name, const char **out_map_start, size_t *out_map_len, file_id_t *file_id);
+
+    /** Whether we're in maximum chaos mode, useful for testing */
+    bool chaos_mode;
+
+    /* Versioned decoding */
+    static history_item_t decode_item_fish_2_0(const char *base, size_t len);
+    static history_item_t decode_item_fish_1_x(const char *base, size_t len);
+    static history_item_t decode_item(const char *base, size_t len, history_file_type_t type);
+
+public:
+    /** Returns history with the given name, creating it if necessary */
+    static history_t & history_with_name(const wcstring &name);
+
+    /** Determines whether the history is empty. Unfortunately this cannot be const, since it may require populating the history. */
+    bool is_empty(void);
+
+    /** Add a new history item to the end. If pending is set, the item will not be returned by item_at_index until a call to resolve_pending(). Pending items are tracked with an offset into the array of new items, so adding a non-pending item has the effect of resolving all pending items. */
+    void add(const wcstring &str, history_identifier_t ident = 0, bool pending = false);
+
+    /** Remove a history item */
+    void remove(const wcstring &str);
+
+    /** Add a new pending history item to the end, and then begin file detection on the items to determine which arguments are paths */
+    void add_pending_with_file_detection(const wcstring &str);
+    
+    /** Resolves any pending history items, so that they may be returned in history searches. */
+    void resolve_pending();
+
+    /** Saves history */
+    void save();
+
+    /** Enable / disable automatic saving. Main thread only! */
+    void disable_automatic_saving();
+    void enable_automatic_saving();
+
+    /** Irreversibly clears history */
+    void clear();
+
+    /** Populates from a bash history file */
+    void populate_from_bash(FILE *f);
+
+    /** Incorporates the history of other shells into this history */
+    void incorporate_external_changes();
+
+    /* Gets all the history into a string with ARRAY_SEP_STR. This is intended for the $history environment variable. This may be long! */
+    void get_string_representation(wcstring *result, const wcstring &separator);
+
+    /** Sets the valid file paths for the history item with the given identifier */
+    void set_valid_file_paths(const wcstring_list_t &valid_file_paths, history_identifier_t ident);
+
+    /** Return the specified history at the specified index. 0 is the index of the current commandline. (So the most recent item is at index 1.) */
+    history_item_t item_at_index(size_t idx);
+};
+
+class history_search_t
+{
+
+    /** The history in which we are searching */
+    history_t * history;
+
+    /** Our type */
+    enum history_search_type_t search_type;
+
+    /** Our list of previous matches as index, value. The end is the current match. */
+    typedef std::pair<size_t, history_item_t> prev_match_t;
+    std::vector<prev_match_t> prev_matches;
+
+    /** Returns yes if a given term is in prev_matches. */
+    bool match_already_made(const wcstring &match) const;
+
+    /** The search term */
+    wcstring term;
+
+    /** Additional strings to skip (sorted) */
+    wcstring_list_t external_skips;
+
+    bool should_skip_match(const wcstring &str) const;
+
+public:
+
+    /** Gets the search term */
+    const wcstring &get_term() const
+    {
+        return term;
+    }
+
+    /** Sets additional string matches to skip */
+    void skip_matches(const wcstring_list_t &skips);
+
+    /** Finds the next search term (forwards in time). Returns true if one was found. */
+    bool go_forwards(void);
+
+    /** Finds the previous search result (backwards in time). Returns true if one was found. */
+    bool go_backwards(void);
+
+    /** Goes to the end (forwards) */
+    void go_to_end(void);
+
+    /** Returns if we are at the end. We start out at the end. */
+    bool is_at_end(void) const;
+
+    /** Goes to the beginning (backwards) */
+    void go_to_beginning(void);
+
+    /** Returns the current search result item. asserts if there is no current item. */
+    history_item_t current_item(void) const;
+
+    /** Returns the current search result item contents. asserts if there is no current item. */
+    wcstring current_string(void) const;
+
+
+    /** Constructor */
+    history_search_t(history_t &hist, const wcstring &str, enum history_search_type_t type = HISTORY_SEARCH_TYPE_CONTAINS) :
+        history(&hist),
+        search_type(type),
+        term(str)
+    {}
+
+    /* Default constructor */
+    history_search_t() :
+        history(),
+        search_type(HISTORY_SEARCH_TYPE_CONTAINS),
+        term()
+    {}
+
+};
+
+/**
+   Init history library. The history file won't actually be loaded
+   until the first time a history search is performed.
+*/
+void history_init();
+
+/**
+   Saves the new history to disc.
+*/
+void history_destroy();
+
+/**
+   Perform sanity checks
+*/
+void history_sanity_check();
+
+/* A helper class for threaded detection of paths */
+struct file_detection_context_t
+{
+    /* Constructor */
+    file_detection_context_t(history_t *hist, history_identifier_t ident = 0);
+
+    /* Determine which of potential_paths are valid, and put them in valid_paths */
+    int perform_file_detection();
+
+    /* The history associated with this context */
+    history_t * const history;
+
+    /* The working directory at the time the command was issued */
+    wcstring working_directory;
+
+    /* Paths to test */
+    path_list_t potential_paths;
+
+    /* Paths that were found to be valid */
+    path_list_t valid_paths;
+
+    /* Identifier of the history item to which we are associated */
+    const history_identifier_t history_item_identifier;
+
+    /* Performs file detection. Returns 1 if every path in potential_paths is valid, 0 otherwise. If test_all is true, tests every path; otherwise stops as soon as it reaches an invalid path. */
+    int perform_file_detection(bool test_all);
+
+    /* Determine whether the given paths are all valid */
+    bool paths_are_valid(const path_list_t &paths);
+};
+
+#endif
diff --git a/src/input.cpp b/src/input.cpp
new file mode 100644
index 00000000..2629506f
--- /dev/null
+++ b/src/input.cpp
@@ -0,0 +1,1116 @@
+/** \file input.c
+
+    Functions for reading a character of input from stdin.
+
+*/
+
+#include "config.h"
+
+#include <assert.h>
+#include <errno.h>
+#include <unistd.h>
+#include <wchar.h>
+
+#if HAVE_NCURSES_H
+#include <ncurses.h>
+#elif HAVE_NCURSES_CURSES_H
+#include <ncurses/curses.h>
+#else
+#include <curses.h>
+#endif
+
+#if HAVE_TERM_H
+#include <term.h>
+#elif HAVE_NCURSES_TERM_H
+#include <ncurses/term.h>
+#endif
+
+#include <wctype.h>
+
+#include "fallback.h" // IWYU pragma: keep
+#include "wutil.h" // IWYU pragma: keep - needed for wgettext
+#include "reader.h"
+#include "proc.h"
+#include "common.h"
+#include "input_common.h"
+#include "input.h"
+#include "parser.h"
+#include "env.h"
+#include "event.h"
+#include "signal.h" // IWYU pragma: keep - needed for CHECK_BLOCK
+#include "io.h"
+#include "output.h"
+#include <vector>
+#include <algorithm>
+
+#define DEFAULT_TERM L"ansi"
+#define MAX_INPUT_FUNCTION_ARGS 20
+
+/**
+   Struct representing a keybinding. Returned by input_get_mappings.
+ */
+
+struct input_mapping_t
+{
+    wcstring seq; /**< Character sequence which generates this event */
+    wcstring_list_t commands; /**< commands that should be evaluated by this mapping */
+
+    /* We wish to preserve the user-specified order. This is just an incrementing value. */
+    unsigned int specification_order;
+
+    wcstring mode; /**< mode in which this command should be evaluated */
+    wcstring sets_mode; /** new mode that should be switched to after command evaluation */
+
+    input_mapping_t(const wcstring &s, const std::vector<wcstring> &c,
+                    const wcstring &m = DEFAULT_BIND_MODE,
+                    const wcstring &sm = DEFAULT_BIND_MODE) : seq(s), commands(c), mode(m), sets_mode(sm)
+    {
+        static unsigned int s_last_input_mapping_specification_order = 0;
+        specification_order = ++s_last_input_mapping_specification_order;
+
+    }
+};
+
+/**
+   A struct representing the mapping from a terminfo key name to a terminfo character sequence
+ */
+struct terminfo_mapping_t
+{
+    const wchar_t *name; /**< Name of key */
+    const char *seq; /**< Character sequence generated on keypress. Constant string. */
+};
+
+
+/**
+   Names of all the input functions supported
+*/
+static const wchar_t * const name_arr[] =
+{
+    L"beginning-of-line",
+    L"end-of-line",
+    L"forward-char",
+    L"backward-char",
+    L"forward-word",
+    L"backward-word",
+    L"forward-bigword",
+    L"backward-bigword",
+    L"history-search-backward",
+    L"history-search-forward",
+    L"delete-char",
+    L"backward-delete-char",
+    L"kill-line",
+    L"yank",
+    L"yank-pop",
+    L"complete",
+    L"complete-and-search",
+    L"beginning-of-history",
+    L"end-of-history",
+    L"backward-kill-line",
+    L"kill-whole-line",
+    L"kill-word",
+    L"kill-bigword",
+    L"backward-kill-word",
+    L"backward-kill-path-component",
+    L"backward-kill-bigword",
+    L"history-token-search-backward",
+    L"history-token-search-forward",
+    L"self-insert",
+    L"transpose-chars",
+    L"transpose-words",
+    L"upcase-word",
+    L"downcase-word",
+    L"capitalize-word",
+    L"vi-arg-digit",
+    L"vi-delete-to",
+    L"execute",
+    L"beginning-of-buffer",
+    L"end-of-buffer",
+    L"repaint",
+    L"force-repaint",
+    L"up-line",
+    L"down-line",
+    L"suppress-autosuggestion",
+    L"accept-autosuggestion",
+    L"begin-selection",
+    L"end-selection",
+    L"kill-selection",
+    L"forward-jump",
+    L"backward-jump",
+    L"and",
+    L"cancel"
+};
+
+wcstring describe_char(wint_t c)
+{
+    wint_t initial_cmd_char = R_BEGINNING_OF_LINE;
+    size_t name_count = sizeof name_arr / sizeof *name_arr;
+    if (c >= initial_cmd_char && c < initial_cmd_char + name_count)
+    {
+        return format_string(L"%02x (%ls)", c, name_arr[c - initial_cmd_char]);
+    }
+    return format_string(L"%02x", c);
+}
+
+/**
+   Description of each supported input function
+*/
+/*
+static const wchar_t *desc_arr[] =
+{
+  L"Move to beginning of line",
+  L"Move to end of line",
+  L"Move forward one character",
+  L"Move backward one character",
+  L"Move forward one word",
+  L"Move backward one word",
+  L"Search backward through list of previous commands",
+  L"Search forward through list of previous commands",
+  L"Delete one character forward",
+  L"Delete one character backward",
+  L"Move contents from cursor to end of line to killring",
+  L"Paste contents of killring",
+  L"Rotate to previous killring entry",
+  L"Guess the rest of the next input token",
+  L"Move to first item of history",
+  L"Move to last item of history",
+  L"Clear current line",
+  L"Move contents from beginning of line to cursor to killring",
+  L"Move entire line to killring",
+  L"Move next word to killring",
+  L"Move previous word to killring",
+  L"Write out key bindings",
+  L"Clear entire screen",
+  L"Quit the running program",
+  L"Search backward through list of previous commands for matching token",
+  L"Search forward through list of previous commands for matching token",
+  L"Insert the pressed key",
+  L"Do nothing",
+  L"End of file",
+  L"Repeat command"
+}
+  ;
+*/
+
+/**
+   Internal code for each supported input function
+*/
+static const wchar_t code_arr[] =
+{
+    R_BEGINNING_OF_LINE,
+    R_END_OF_LINE,
+    R_FORWARD_CHAR,
+    R_BACKWARD_CHAR,
+    R_FORWARD_WORD,
+    R_BACKWARD_WORD,
+    R_FORWARD_BIGWORD,
+    R_BACKWARD_BIGWORD,
+    R_HISTORY_SEARCH_BACKWARD,
+    R_HISTORY_SEARCH_FORWARD,
+    R_DELETE_CHAR,
+    R_BACKWARD_DELETE_CHAR,
+    R_KILL_LINE,
+    R_YANK,
+    R_YANK_POP,
+    R_COMPLETE,
+    R_COMPLETE_AND_SEARCH,
+    R_BEGINNING_OF_HISTORY,
+    R_END_OF_HISTORY,
+    R_BACKWARD_KILL_LINE,
+    R_KILL_WHOLE_LINE,
+    R_KILL_WORD,
+    R_KILL_BIGWORD,
+    R_BACKWARD_KILL_WORD,
+    R_BACKWARD_KILL_PATH_COMPONENT,
+    R_BACKWARD_KILL_BIGWORD,
+    R_HISTORY_TOKEN_SEARCH_BACKWARD,
+    R_HISTORY_TOKEN_SEARCH_FORWARD,
+    R_SELF_INSERT,
+    R_TRANSPOSE_CHARS,
+    R_TRANSPOSE_WORDS,
+    R_UPCASE_WORD,
+    R_DOWNCASE_WORD,
+    R_CAPITALIZE_WORD,
+    R_VI_ARG_DIGIT,
+    R_VI_DELETE_TO,
+    R_EXECUTE,
+    R_BEGINNING_OF_BUFFER,
+    R_END_OF_BUFFER,
+    R_REPAINT,
+    R_FORCE_REPAINT,
+    R_UP_LINE,
+    R_DOWN_LINE,
+    R_SUPPRESS_AUTOSUGGESTION,
+    R_ACCEPT_AUTOSUGGESTION,
+    R_BEGIN_SELECTION,
+    R_END_SELECTION,
+    R_KILL_SELECTION,
+    R_FORWARD_JUMP,
+    R_BACKWARD_JUMP,
+    R_AND,
+    R_CANCEL
+};
+
+/** Mappings for the current input mode */
+static std::vector<input_mapping_t> mapping_list;
+
+/* Terminfo map list */
+static std::vector<terminfo_mapping_t> terminfo_mappings;
+
+#define TERMINFO_ADD(key) { (L ## #key) + 4, key }
+
+
+/**
+   List of all terminfo mappings
+ */
+static std::vector<terminfo_mapping_t> mappings;
+
+
+/**
+   Set to one when the input subsytem has been initialized.
+*/
+static bool is_init = false;
+
+/**
+   Initialize terminfo.
+ */
+static void input_terminfo_init();
+
+static wchar_t input_function_args[MAX_INPUT_FUNCTION_ARGS];
+static bool input_function_status;
+static int input_function_args_index = 0;
+
+/**
+    Return the current bind mode
+*/
+wcstring input_get_bind_mode()
+{
+    env_var_t mode = env_get_string(FISH_BIND_MODE_VAR);
+    return mode.missing() ? DEFAULT_BIND_MODE : mode;
+}
+
+/**
+    Set the current bind mode
+*/
+void input_set_bind_mode(const wcstring &bm)
+{
+    env_set(FISH_BIND_MODE_VAR, bm.c_str(), ENV_GLOBAL);
+}
+
+
+/**
+    Returns the arity of a given input function
+*/
+int input_function_arity(int function)
+{
+    switch (function)
+    {
+        case R_FORWARD_JUMP:
+        case R_BACKWARD_JUMP:
+            return 1;
+        default:
+            return 0;
+    }
+}
+
+/**
+    Sets the return status of the most recently executed input function
+*/
+void input_function_set_status(bool status)
+{
+    input_function_status = status;
+}
+
+/* Helper function to compare the lengths of sequences */
+static bool length_is_greater_than(const input_mapping_t &m1, const input_mapping_t &m2)
+{
+    return m1.seq.size() > m2.seq.size();
+}
+
+static bool specification_order_is_less_than(const input_mapping_t &m1, const input_mapping_t &m2)
+{
+    return m1.specification_order < m2.specification_order;
+}
+
+/* Inserts an input mapping at the correct position. We sort them in descending order by length, so that we test longer sequences first. */
+static void input_mapping_insert_sorted(const input_mapping_t &new_mapping)
+{
+    std::vector<input_mapping_t>::iterator loc = std::lower_bound(mapping_list.begin(), mapping_list.end(), new_mapping, length_is_greater_than);
+    mapping_list.insert(loc, new_mapping);
+}
+
+/* Adds an input mapping */
+void input_mapping_add(const wchar_t *sequence, const wchar_t **commands, size_t commands_len,
+                       const wchar_t *mode, const wchar_t *sets_mode)
+{
+    CHECK(sequence,);
+    CHECK(commands,);
+    CHECK(mode,);
+    CHECK(sets_mode,);
+
+    // debug( 0, L"Add mapping from %ls to %ls in mode %ls", escape(sequence, ESCAPE_ALL).c_str(), escape(command, ESCAPE_ALL).c_str(), mode);
+
+    // remove existing mappings with this sequence
+    const wcstring_list_t commands_vector(commands, commands + commands_len);
+
+    for (size_t i=0; i<mapping_list.size(); i++)
+    {
+        input_mapping_t &m = mapping_list.at(i);
+        if (m.seq == sequence && m.mode == mode)
+        {
+            m.commands = commands_vector;
+            m.sets_mode = sets_mode;
+            return;
+        }
+    }
+
+    // add a new mapping, using the next order
+    const input_mapping_t new_mapping = input_mapping_t(sequence, commands_vector, mode, sets_mode);
+    input_mapping_insert_sorted(new_mapping);
+}
+
+void input_mapping_add(const wchar_t *sequence, const wchar_t *command,
+                       const wchar_t *mode, const wchar_t *sets_mode)
+{
+    input_mapping_add(sequence, &command, 1, mode, sets_mode);
+}
+
+/**
+   Handle interruptions to key reading by reaping finshed jobs and
+   propagating the interrupt to the reader.
+*/
+static int interrupt_handler()
+{
+    /*
+      Fire any pending events
+    */
+    event_fire(NULL);
+
+    /*
+      Reap stray processes, including printing exit status messages
+    */
+    if (job_reap(1))
+        reader_repaint_needed();
+
+    /*
+      Tell the reader an event occured
+    */
+    if (reader_reading_interrupted())
+    {
+        /*
+          Return 3, i.e. the character read by a Control-C.
+        */
+        return 3;
+    }
+
+    return R_NULL;
+}
+
+void update_fish_color_support(void)
+{
+    /* Infer term256 support. If fish_term256 is set, we respect it; otherwise try to detect it from the TERM variable */
+    env_var_t fish_term256 = env_get_string(L"fish_term256");
+    bool support_term256;
+    if (! fish_term256.missing_or_empty())
+    {
+        support_term256 = from_string<bool>(fish_term256);
+    }
+    else
+    {
+        env_var_t term = env_get_string(L"TERM");
+        if (term.missing())
+        {
+            support_term256 = false;
+        }
+        else if (term.find(L"256color") != wcstring::npos)
+        {
+            /* Explicitly supported */
+            support_term256 = true;
+        }
+        else if (term.find(L"xterm") != wcstring::npos)
+        {
+            // assume that all xterms are 256, except for OS X SnowLeopard
+            env_var_t prog = env_get_string(L"TERM_PROGRAM");
+            support_term256 = (prog != L"Apple_Terminal");
+        }
+        else
+        {
+            // Don't know, default to false
+            support_term256 = false;
+        }
+    }
+
+    env_var_t fish_term24bit = env_get_string(L"fish_term24bit");
+    bool support_term24bit;
+    if (! fish_term24bit.missing_or_empty())
+    {
+        support_term24bit = from_string<bool>(fish_term24bit);
+    }
+    else
+    {
+        /* We don't attempt to infer term24 bit support yet. */
+        support_term24bit = false;
+    }
+
+    color_support_t support = (support_term256 ? color_support_term256 : 0) | (support_term24bit ? color_support_term24bit : 0);
+    output_set_color_support(support);
+}
+
+int input_init()
+{
+    if (is_init)
+        return 1;
+
+    is_init = true;
+
+    input_common_init(&interrupt_handler);
+
+    const env_var_t term = env_get_string(L"TERM");
+    int errret;
+    if (setupterm(0, STDOUT_FILENO, &errret) == ERR)
+    {
+        debug(0, _(L"Could not set up terminal"));
+        if (errret == 0)
+        {
+            debug(0, _(L"Check that your terminal type, '%ls', is supported on this system"),
+                  term.c_str());
+            debug(0, _(L"Attempting to use '%ls' instead"), DEFAULT_TERM);
+            env_set(L"TERM", DEFAULT_TERM, ENV_GLOBAL | ENV_EXPORT);
+            const std::string default_term = wcs2string(DEFAULT_TERM);
+            if (setupterm(const_cast<char *>(default_term.c_str()), STDOUT_FILENO, &errret) == ERR)
+            {
+                debug(0, _(L"Could not set up terminal"));
+                exit_without_destructors(1);
+            }
+        }
+        else
+        {
+            exit_without_destructors(1);
+        }
+    }
+    assert(! term.missing());
+    output_set_term(term);
+
+    input_terminfo_init();
+
+    update_fish_color_support();
+
+    /* If we have no keybindings, add a few simple defaults */
+    if (mapping_list.empty())
+    {
+        input_mapping_add(L"", L"self-insert");
+        input_mapping_add(L"\n", L"execute");
+        input_mapping_add(L"\t", L"complete");
+        input_mapping_add(L"\x3", L"commandline \"\"");
+        input_mapping_add(L"\x4", L"exit");
+        input_mapping_add(L"\x5", L"bind");
+    }
+
+    return 1;
+}
+
+void input_destroy()
+{
+    if (!is_init)
+        return;
+
+
+    is_init = false;
+
+    input_common_destroy();
+
+    if (del_curterm(cur_term) == ERR)
+    {
+        debug(0, _(L"Error while closing terminfo"));
+    }
+}
+
+void input_function_push_arg(wchar_t arg)
+{
+    input_function_args[input_function_args_index++] = arg;
+}
+
+wchar_t input_function_pop_arg()
+{
+    return input_function_args[--input_function_args_index];
+}
+
+void input_function_push_args(int code)
+{
+    int arity = input_function_arity(code);
+    for (int i = 0; i < arity; i++)
+    {
+        input_function_push_arg(input_common_readch(0));
+    }
+}
+
+/**
+   Perform the action of the specified binding
+   allow_commands controls whether fish commands should be executed, or should
+   be deferred until later.
+*/
+static void input_mapping_execute(const input_mapping_t &m, bool allow_commands)
+{
+    /* has_functions: there are functions that need to be put on the input
+       queue
+       has_commands: there are shell commands that need to be evaluated */
+    bool has_commands = false, has_functions = false;
+
+    for (wcstring_list_t::const_iterator it = m.commands.begin(), end = m.commands.end(); it != end; it++)
+    {
+        if (input_function_get_code(*it) != -1)
+            has_functions = true;
+        else
+            has_commands = true;
+    }
+
+    /* !has_functions && !has_commands: only set bind mode */
+    if (!has_commands && !has_functions)
+    {
+        input_set_bind_mode(m.sets_mode);
+        return;
+    }
+
+    if (!allow_commands)
+    {
+        /* We don't want to run commands yet. Put the characters back and return
+           R_NULL */
+        for (wcstring::const_reverse_iterator it = m.seq.rbegin(), end = m.seq.rend(); it != end; ++it)
+        {
+            input_common_next_ch(*it);
+        }
+        input_common_next_ch(R_NULL);
+        return; /* skip the input_set_bind_mode */
+    }
+    else if (has_functions && !has_commands)
+    {
+        /* functions are added at the head of the input queue */
+        for (wcstring_list_t::const_reverse_iterator it = m.commands.rbegin(), end = m.commands.rend (); it != end; it++)
+        {
+            wchar_t code = input_function_get_code(*it);
+            input_function_push_args(code);
+            input_common_next_ch(code);
+        }
+    }
+    else if (has_commands && !has_functions)
+    {
+        /* Execute all commands.
+
+           FIXME(snnw): if commands add stuff to input queue (e.g. commandline
+           -f execute), we won't see that until all other commands have also
+           been run. */
+        int last_status = proc_get_last_status();
+        for (wcstring_list_t::const_iterator it = m.commands.begin(), end = m.commands.end(); it != end; it++)
+        {
+            parser_t::principal_parser().eval(it->c_str(), io_chain_t(), TOP);
+        }
+        proc_set_last_status(last_status);
+        input_common_next_ch(R_NULL);
+    }
+    else
+    {
+        /* invalid binding, mixed commands and functions.  We would need to
+           execute these one by one. */
+        input_common_next_ch(R_NULL);
+    }
+
+    input_set_bind_mode(m.sets_mode);
+}
+
+
+
+/**
+   Try reading the specified function mapping
+*/
+static bool input_mapping_is_match(const input_mapping_t &m)
+{
+    wint_t c = 0;
+    int j;
+
+    //debug(0, L"trying mapping %ls\n", escape(m.seq.c_str(), ESCAPE_ALL).c_str());
+    const wchar_t *str = m.seq.c_str();
+    for (j=0; str[j] != L'\0'; j++)
+    {
+        bool timed = (j > 0 && iswcntrl(str[0]));
+
+        c = input_common_readch(timed);
+        if (str[j] != c)
+        {
+            break;
+        }
+    }
+
+    if (str[j] == L'\0')
+    {
+        //debug(0, L"matched mapping %ls (%ls)\n", escape(m.seq.c_str(), ESCAPE_ALL).c_str(), m.command.c_str());
+        /* We matched the entire sequence */
+        return true;
+    }
+    else
+    {
+        int k;
+        /*
+          Return the read characters
+        */
+        input_common_next_ch(c);
+        for (k=j-1; k>=0; k--)
+        {
+            input_common_next_ch(m.seq[k]);
+        }
+    }
+
+    return false;
+
+}
+
+void input_queue_ch(wint_t ch)
+{
+    input_common_queue_ch(ch);
+}
+
+static void input_mapping_execute_matching_or_generic(bool allow_commands)
+{
+    const input_mapping_t *generic = NULL;
+
+    const wcstring bind_mode = input_get_bind_mode();
+
+    for (int i = 0; i < mapping_list.size(); i++)
+    {
+        const input_mapping_t &m = mapping_list.at(i);
+
+        //debug(0, L"trying mapping (%ls,%ls,%ls)\n", escape(m.seq.c_str(), ESCAPE_ALL).c_str(),
+        //           m.mode.c_str(), m.sets_mode.c_str());
+
+        if (m.mode != bind_mode)
+        {
+            //debug(0, L"skipping mapping because mode %ls != %ls\n", m.mode.c_str(), input_get_bind_mode().c_str());
+            continue;
+        }
+
+        if (m.seq.length() == 0)
+        {
+            generic = &m;
+        }
+        else if (input_mapping_is_match(m))
+        {
+            input_mapping_execute(m, allow_commands);
+            return;
+        }
+    }
+
+    if (generic)
+    {
+        input_mapping_execute(*generic, allow_commands);
+    }
+    else
+    {
+        //debug(0, L"no generic found, ignoring...");
+        wchar_t c = input_common_readch(0);
+        if (c == R_EOF)
+            input_common_next_ch(c);
+    }
+}
+
+/* Helper function. Picks through the queue of incoming characters until we get to one that's not a readline function. */
+static wchar_t input_read_characters_only()
+{
+    std::vector<wchar_t> functions_to_put_back;
+    wchar_t char_to_return;
+    for (;;)
+    {
+        char_to_return = input_common_readch(0);
+        bool is_readline_function = (char_to_return >= R_MIN && char_to_return <= R_MAX);
+        // R_NULL and R_EOF are more control characters than readline functions, so check specially for those
+        if (!is_readline_function || char_to_return == R_NULL || char_to_return == R_EOF)
+        {
+            break;
+        }
+        // This is a readline function; save it off for later re-enqueing and try again
+        functions_to_put_back.push_back(char_to_return);
+    }
+    // Restore any readline functions, in reverse to preserve their original order
+    size_t idx = functions_to_put_back.size();
+    while (idx--)
+    {
+        input_common_next_ch(functions_to_put_back.at(idx));
+    }
+    return char_to_return;
+}
+
+wint_t input_readch(bool allow_commands)
+{
+    CHECK_BLOCK(R_NULL);
+
+    /*
+       Clear the interrupted flag
+    */
+    reader_reset_interrupted();
+
+    /*
+       Search for sequence in mapping tables
+    */
+
+    while (1)
+    {
+        wchar_t c = input_common_readch(0);
+
+        if (c >= R_MIN && c <= R_MAX)
+        {
+            switch (c)
+            {
+                case R_EOF: /* If it's closed, then just return */
+                {
+                    return R_EOF;
+                }
+                case R_SELF_INSERT:
+                {
+                    /* #1595: ensure we only insert characters, not readline functions. The common case is that this will be empty. */
+                    return input_read_characters_only();
+                }
+                case R_AND:
+                {
+                    if (input_function_status)
+                    {
+                        return input_readch();
+                    }
+                    else
+                    {
+                        while ((c = input_common_readch(0)) && c >= R_MIN && c <= R_MAX);
+                        input_common_next_ch(c);
+                        return input_readch();
+                    }
+                }
+                default:
+                {
+                    return c;
+                }
+            }
+        }
+        else
+        {
+            input_common_next_ch(c);
+            input_mapping_execute_matching_or_generic(allow_commands);
+            // regarding allow_commands, we're in a loop, but if a fish command
+            // is executed, R_NULL is unread, so the next pass through the loop
+            // we'll break out and return it.
+        }
+    }
+}
+
+std::vector<input_mapping_name_t> input_mapping_get_names()
+{
+    // Sort the mappings by the user specification order, so we can return them in the same order that the user specified them in
+    std::vector<input_mapping_t> local_list = mapping_list;
+    std::sort(local_list.begin(), local_list.end(), specification_order_is_less_than);
+    std::vector<input_mapping_name_t> result;
+    result.reserve(local_list.size());
+
+    for (size_t i=0; i<local_list.size(); i++)
+    {
+        const input_mapping_t &m = local_list.at(i);
+        result.push_back((input_mapping_name_t){m.seq, m.mode});
+    }
+    return result;
+}
+
+
+bool input_mapping_erase(const wcstring &sequence, const wcstring &mode)
+{
+    ASSERT_IS_MAIN_THREAD();
+    bool result = false;
+
+    for (std::vector<input_mapping_t>::iterator it = mapping_list.begin(), end = mapping_list.end();
+         it != end;
+         ++it)
+    {
+        if (sequence == it->seq && mode == it->mode)
+        {
+            mapping_list.erase(it);
+            result = true;
+            break;
+        }
+    }
+    return result;
+}
+
+bool input_mapping_get(const wcstring &sequence, const wcstring &mode, wcstring_list_t *out_cmds, wcstring *out_sets_mode)
+{
+    bool result = false;
+    for (std::vector<input_mapping_t>::const_iterator it = mapping_list.begin(), end = mapping_list.end();
+         it != end;
+         ++it)
+    {
+        if (sequence == it->seq && mode == it->mode)
+        {
+            *out_cmds = it->commands;
+            *out_sets_mode = it->sets_mode;
+            result = true;
+            break;
+        }
+    }
+    return result;
+}
+
+/**
+   Add all terminfo mappings
+ */
+static void input_terminfo_init()
+{
+    const terminfo_mapping_t tinfos[] =
+    {
+        TERMINFO_ADD(key_a1),
+        TERMINFO_ADD(key_a3),
+        TERMINFO_ADD(key_b2),
+        TERMINFO_ADD(key_backspace),
+        TERMINFO_ADD(key_beg),
+        TERMINFO_ADD(key_btab),
+        TERMINFO_ADD(key_c1),
+        TERMINFO_ADD(key_c3),
+        TERMINFO_ADD(key_cancel),
+        TERMINFO_ADD(key_catab),
+        TERMINFO_ADD(key_clear),
+        TERMINFO_ADD(key_close),
+        TERMINFO_ADD(key_command),
+        TERMINFO_ADD(key_copy),
+        TERMINFO_ADD(key_create),
+        TERMINFO_ADD(key_ctab),
+        TERMINFO_ADD(key_dc),
+        TERMINFO_ADD(key_dl),
+        TERMINFO_ADD(key_down),
+        TERMINFO_ADD(key_eic),
+        TERMINFO_ADD(key_end),
+        TERMINFO_ADD(key_enter),
+        TERMINFO_ADD(key_eol),
+        TERMINFO_ADD(key_eos),
+        TERMINFO_ADD(key_exit),
+        TERMINFO_ADD(key_f0),
+        TERMINFO_ADD(key_f1),
+        TERMINFO_ADD(key_f2),
+        TERMINFO_ADD(key_f3),
+        TERMINFO_ADD(key_f4),
+        TERMINFO_ADD(key_f5),
+        TERMINFO_ADD(key_f6),
+        TERMINFO_ADD(key_f7),
+        TERMINFO_ADD(key_f8),
+        TERMINFO_ADD(key_f9),
+        TERMINFO_ADD(key_f10),
+        TERMINFO_ADD(key_f11),
+        TERMINFO_ADD(key_f12),
+        TERMINFO_ADD(key_f13),
+        TERMINFO_ADD(key_f14),
+        TERMINFO_ADD(key_f15),
+        TERMINFO_ADD(key_f16),
+        TERMINFO_ADD(key_f17),
+        TERMINFO_ADD(key_f18),
+        TERMINFO_ADD(key_f19),
+        TERMINFO_ADD(key_f20),
+        /*
+        I know of no keyboard with more than 20 function keys, so
+        adding the rest here makes very little sense, since it will
+        take up a lot of room in any listings (like tab completions),
+        but with no benefit.
+        */
+        /*
+        TERMINFO_ADD(key_f21),
+        TERMINFO_ADD(key_f22),
+        TERMINFO_ADD(key_f23),
+        TERMINFO_ADD(key_f24),
+        TERMINFO_ADD(key_f25),
+        TERMINFO_ADD(key_f26),
+        TERMINFO_ADD(key_f27),
+        TERMINFO_ADD(key_f28),
+        TERMINFO_ADD(key_f29),
+        TERMINFO_ADD(key_f30),
+        TERMINFO_ADD(key_f31),
+        TERMINFO_ADD(key_f32),
+        TERMINFO_ADD(key_f33),
+        TERMINFO_ADD(key_f34),
+        TERMINFO_ADD(key_f35),
+        TERMINFO_ADD(key_f36),
+        TERMINFO_ADD(key_f37),
+        TERMINFO_ADD(key_f38),
+        TERMINFO_ADD(key_f39),
+        TERMINFO_ADD(key_f40),
+        TERMINFO_ADD(key_f41),
+        TERMINFO_ADD(key_f42),
+        TERMINFO_ADD(key_f43),
+        TERMINFO_ADD(key_f44),
+        TERMINFO_ADD(key_f45),
+        TERMINFO_ADD(key_f46),
+        TERMINFO_ADD(key_f47),
+        TERMINFO_ADD(key_f48),
+        TERMINFO_ADD(key_f49),
+        TERMINFO_ADD(key_f50),
+        TERMINFO_ADD(key_f51),
+        TERMINFO_ADD(key_f52),
+        TERMINFO_ADD(key_f53),
+        TERMINFO_ADD(key_f54),
+        TERMINFO_ADD(key_f55),
+        TERMINFO_ADD(key_f56),
+        TERMINFO_ADD(key_f57),
+        TERMINFO_ADD(key_f58),
+        TERMINFO_ADD(key_f59),
+        TERMINFO_ADD(key_f60),
+        TERMINFO_ADD(key_f61),
+        TERMINFO_ADD(key_f62),
+        TERMINFO_ADD(key_f63),*/
+        TERMINFO_ADD(key_find),
+        TERMINFO_ADD(key_help),
+        TERMINFO_ADD(key_home),
+        TERMINFO_ADD(key_ic),
+        TERMINFO_ADD(key_il),
+        TERMINFO_ADD(key_left),
+        TERMINFO_ADD(key_ll),
+        TERMINFO_ADD(key_mark),
+        TERMINFO_ADD(key_message),
+        TERMINFO_ADD(key_move),
+        TERMINFO_ADD(key_next),
+        TERMINFO_ADD(key_npage),
+        TERMINFO_ADD(key_open),
+        TERMINFO_ADD(key_options),
+        TERMINFO_ADD(key_ppage),
+        TERMINFO_ADD(key_previous),
+        TERMINFO_ADD(key_print),
+        TERMINFO_ADD(key_redo),
+        TERMINFO_ADD(key_reference),
+        TERMINFO_ADD(key_refresh),
+        TERMINFO_ADD(key_replace),
+        TERMINFO_ADD(key_restart),
+        TERMINFO_ADD(key_resume),
+        TERMINFO_ADD(key_right),
+        TERMINFO_ADD(key_save),
+        TERMINFO_ADD(key_sbeg),
+        TERMINFO_ADD(key_scancel),
+        TERMINFO_ADD(key_scommand),
+        TERMINFO_ADD(key_scopy),
+        TERMINFO_ADD(key_screate),
+        TERMINFO_ADD(key_sdc),
+        TERMINFO_ADD(key_sdl),
+        TERMINFO_ADD(key_select),
+        TERMINFO_ADD(key_send),
+        TERMINFO_ADD(key_seol),
+        TERMINFO_ADD(key_sexit),
+        TERMINFO_ADD(key_sf),
+        TERMINFO_ADD(key_sfind),
+        TERMINFO_ADD(key_shelp),
+        TERMINFO_ADD(key_shome),
+        TERMINFO_ADD(key_sic),
+        TERMINFO_ADD(key_sleft),
+        TERMINFO_ADD(key_smessage),
+        TERMINFO_ADD(key_smove),
+        TERMINFO_ADD(key_snext),
+        TERMINFO_ADD(key_soptions),
+        TERMINFO_ADD(key_sprevious),
+        TERMINFO_ADD(key_sprint),
+        TERMINFO_ADD(key_sr),
+        TERMINFO_ADD(key_sredo),
+        TERMINFO_ADD(key_sreplace),
+        TERMINFO_ADD(key_sright),
+        TERMINFO_ADD(key_srsume),
+        TERMINFO_ADD(key_ssave),
+        TERMINFO_ADD(key_ssuspend),
+        TERMINFO_ADD(key_stab),
+        TERMINFO_ADD(key_sundo),
+        TERMINFO_ADD(key_suspend),
+        TERMINFO_ADD(key_undo),
+        TERMINFO_ADD(key_up)
+    };
+    const size_t count = sizeof tinfos / sizeof *tinfos;
+    terminfo_mappings.reserve(terminfo_mappings.size() + count);
+    terminfo_mappings.insert(terminfo_mappings.end(), tinfos, tinfos + count);
+}
+
+bool input_terminfo_get_sequence(const wchar_t *name, wcstring *out_seq)
+{
+    ASSERT_IS_MAIN_THREAD();
+
+    const char *res = 0;
+    int err = ENOENT;
+
+    CHECK(name, 0);
+    input_init();
+
+    for (size_t i=0; i<terminfo_mappings.size(); i++)
+    {
+        const terminfo_mapping_t &m = terminfo_mappings.at(i);
+        if (!wcscmp(name, m.name))
+        {
+            res = m.seq;
+            err = EILSEQ;
+            break;
+        }
+    }
+
+    if (!res)
+    {
+        errno = err;
+        return false;
+    }
+
+    *out_seq = format_string(L"%s", res);
+    return true;
+
+}
+
+bool input_terminfo_get_name(const wcstring &seq, wcstring *out_name)
+{
+    input_init();
+
+    for (size_t i=0; i<terminfo_mappings.size(); i++)
+    {
+        terminfo_mapping_t &m = terminfo_mappings.at(i);
+
+        if (!m.seq)
+        {
+            continue;
+        }
+
+        const wcstring map_buf = format_string(L"%s",  m.seq);
+        if (map_buf == seq)
+        {
+            out_name->assign(m.name);
+            return true;
+        }
+    }
+
+    return false;
+}
+
+wcstring_list_t input_terminfo_get_names(bool skip_null)
+{
+    wcstring_list_t result;
+    result.reserve(terminfo_mappings.size());
+
+    input_init();
+
+    for (size_t i=0; i<terminfo_mappings.size(); i++)
+    {
+        terminfo_mapping_t &m = terminfo_mappings.at(i);
+
+        if (skip_null && !m.seq)
+        {
+            continue;
+        }
+        result.push_back(wcstring(m.name));
+    }
+    return result;
+}
+
+wcstring_list_t input_function_get_names(void)
+{
+    size_t count = sizeof name_arr / sizeof *name_arr;
+    return wcstring_list_t(name_arr, name_arr + count);
+}
+
+wchar_t input_function_get_code(const wcstring &name)
+{
+
+    size_t i;
+    for (i = 0; i<(sizeof(code_arr)/sizeof(wchar_t)) ; i++)
+    {
+        if (name == name_arr[i])
+        {
+            return code_arr[i];
+        }
+    }
+    return -1;
+}
diff --git a/src/input.h b/src/input.h
new file mode 100644
index 00000000..38facddb
--- /dev/null
+++ b/src/input.h
@@ -0,0 +1,206 @@
+/** \file input.h
+
+Functions for reading a character of input from stdin, using the
+inputrc information for key bindings.
+
+*/
+
+#ifndef FISH_INPUT_H
+#define FISH_INPUT_H
+
+#include <stddef.h>
+#include <string>
+#include <vector>
+
+#include "common.h"
+#include "env.h"
+#include "input_common.h"
+
+
+#define DEFAULT_BIND_MODE L"default"
+#define FISH_BIND_MODE_VAR L"fish_bind_mode"
+
+/**
+   Key codes for inputrc-style keyboard functions that are passed on
+   to the caller of input_read()
+
+   NOTE: IF YOU MODIFY THIS YOU MUST UPDATE THE name_arr AND code_arr VARIABLES TO MATCH!
+*/
+enum
+{
+    R_BEGINNING_OF_LINE = R_NULL+10,  /* This give input_common ten slots for lowlevel keycodes */
+    R_END_OF_LINE,
+    R_FORWARD_CHAR,
+    R_BACKWARD_CHAR,
+    R_FORWARD_WORD,
+    R_BACKWARD_WORD,
+    R_FORWARD_BIGWORD,
+    R_BACKWARD_BIGWORD,
+    R_HISTORY_SEARCH_BACKWARD,
+    R_HISTORY_SEARCH_FORWARD,
+    R_DELETE_CHAR,
+    R_BACKWARD_DELETE_CHAR,
+    R_KILL_LINE,
+    R_YANK,
+    R_YANK_POP,
+    R_COMPLETE,
+    R_COMPLETE_AND_SEARCH,
+    R_BEGINNING_OF_HISTORY,
+    R_END_OF_HISTORY,
+    R_BACKWARD_KILL_LINE,
+    R_KILL_WHOLE_LINE,
+    R_KILL_WORD,
+    R_KILL_BIGWORD,
+    R_BACKWARD_KILL_WORD,
+    R_BACKWARD_KILL_PATH_COMPONENT,
+    R_BACKWARD_KILL_BIGWORD,
+    R_HISTORY_TOKEN_SEARCH_BACKWARD,
+    R_HISTORY_TOKEN_SEARCH_FORWARD,
+    R_SELF_INSERT,
+    R_TRANSPOSE_CHARS,
+    R_TRANSPOSE_WORDS,
+    R_UPCASE_WORD,
+    R_DOWNCASE_WORD,
+    R_CAPITALIZE_WORD,
+    R_VI_ARG_DIGIT,
+    R_VI_DELETE_TO,
+    R_EXECUTE,
+    R_BEGINNING_OF_BUFFER,
+    R_END_OF_BUFFER,
+    R_REPAINT,
+    R_FORCE_REPAINT,
+    R_UP_LINE,
+    R_DOWN_LINE,
+    R_SUPPRESS_AUTOSUGGESTION,
+    R_ACCEPT_AUTOSUGGESTION,
+    R_BEGIN_SELECTION,
+    R_END_SELECTION,
+    R_KILL_SELECTION,
+    R_FORWARD_JUMP,
+    R_BACKWARD_JUMP,
+    R_AND,
+    R_CANCEL
+};
+
+wcstring describe_char(wint_t c);
+
+#define R_MIN R_NULL
+#define R_MAX R_CANCEL
+
+/**
+   Initialize the terminal by calling setupterm, and set up arrays
+   used by readch to detect escape sequences for special keys.
+
+   Before calling input_init, terminfo is not initialized and MUST not be used
+*/
+int input_init();
+
+/**
+   free up memory used by terminal functions.
+*/
+void input_destroy();
+
+/**
+   Read a character from fd 0. Try to convert some escape sequences
+   into character constants, but do not permanently block the escape
+   character.
+
+   This is performed in the same way vim does it, i.e. if an escape
+   character is read, wait for more input for a short time (a few
+   milliseconds). If more input is avaialable, it is assumed to be an
+   escape sequence for a special character (such as an arrow key), and
+   readch attempts to parse it. If no more input follows after the
+   escape key, it is assumed to be an actual escape key press, and is
+   returned as such.
+
+   The argument determines whether fish commands are allowed to be run
+   as bindings. If false, when a character is encountered that would
+   invoke a fish command, it is unread and R_NULL is returned.
+*/
+wint_t input_readch(bool allow_commands = true);
+
+/**
+   Enqueue a character or a readline function to the queue of unread
+   characters that input_readch will return before actually reading from fd
+   0.
+ */
+void input_queue_ch(wint_t ch);
+
+
+/**
+   Add a key mapping from the specified sequence to the specified command
+
+   \param sequence the sequence to bind
+   \param command an input function that will be run whenever the key sequence occurs
+*/
+void input_mapping_add(const wchar_t *sequence, const wchar_t *command,
+                       const wchar_t *mode = DEFAULT_BIND_MODE,
+                       const wchar_t *new_mode = DEFAULT_BIND_MODE);
+
+void input_mapping_add(const wchar_t *sequence, const wchar_t **commands, size_t commands_len,
+                       const wchar_t *mode = DEFAULT_BIND_MODE, const wchar_t *new_mode = DEFAULT_BIND_MODE);
+
+struct input_mapping_name_t {
+    wcstring seq;
+    wcstring mode;
+};
+
+/**
+   Returns all mapping names and modes
+ */
+std::vector<input_mapping_name_t> input_mapping_get_names();
+
+/**
+   Erase binding for specified key sequence
+ */
+bool input_mapping_erase(const wcstring &sequence, const wcstring &mode = DEFAULT_BIND_MODE);
+
+/**
+   Gets the command bound to the specified key sequence in the specified mode. Returns true if it exists, false if not.
+ */
+bool input_mapping_get(const wcstring &sequence, const wcstring &mode, wcstring_list_t *out_cmds, wcstring *out_new_mode);
+
+/**
+    Return the current bind mode
+*/
+wcstring input_get_bind_mode();
+
+/**
+    Set the current bind mode
+*/
+void input_set_bind_mode(const wcstring &bind_mode);
+
+
+wchar_t input_function_pop_arg();
+
+
+/**
+    Sets the return status of the most recently executed input function
+*/
+void input_function_set_status(bool status);
+
+/**
+   Return the sequence for the terminfo variable of the specified name.
+
+   If no terminfo variable of the specified name could be found, return false and set errno to ENOENT.
+   If the terminfo variable does not have a value, return false and set errno to EILSEQ.
+ */
+bool input_terminfo_get_sequence(const wchar_t *name, wcstring *out_seq);
+
+/** Return the name of the terminfo variable with the specified sequence, in out_name. Returns true if found, false if not found. */
+bool input_terminfo_get_name(const wcstring &seq, wcstring *out_name);
+
+/** Return a list of all known terminfo names */
+wcstring_list_t input_terminfo_get_names(bool skip_null);
+
+/** Returns the input function code for the given input function name. */
+wchar_t input_function_get_code(const wcstring &name);
+
+/** Returns a list of all existing input function names */
+wcstring_list_t input_function_get_names(void);
+
+/** Updates our idea of whether we support term256 and term24bit */
+void update_fish_color_support();
+
+
+#endif
diff --git a/src/input_common.cpp b/src/input_common.cpp
new file mode 100644
index 00000000..24fde6e2
--- /dev/null
+++ b/src/input_common.cpp
@@ -0,0 +1,326 @@
+/** \file input_common.c
+
+Implementation file for the low level input library
+
+*/
+#include "config.h"
+
+
+#include <string.h>
+#include <errno.h>
+#include <sys/time.h>
+#include <unistd.h>
+#include <list>
+#include <queue>
+#include <cwchar>                       // for wint_t
+#include <deque>                        // for deque
+#include <utility>                      // for swap, pair
+#ifdef HAVE_SYS_SELECT_H
+#include <sys/select.h>
+#endif
+
+#include "fallback.h" // IWYU pragma: keep
+#include "util.h"
+
+#include "common.h"
+#include "input_common.h"
+#include "env_universal_common.h"
+#include "env.h"
+#include "iothread.h"
+
+/**
+   Time in milliseconds to wait for another byte to be available for
+   reading after \\x1b is read before assuming that escape key was
+   pressed, and not an escape sequence.
+*/
+#define WAIT_ON_ESCAPE 10
+
+/** Characters that have been read and returned by the sequence matching code */
+static std::deque<wint_t> lookahead_list;
+
+/* Queue of pairs of (function pointer, argument) to be invoked. Expected to be mostly empty. */
+typedef std::pair<void (*)(void *), void *> callback_info_t;
+typedef std::queue<callback_info_t, std::list<callback_info_t> > callback_queue_t;
+static callback_queue_t callback_queue;
+static void input_flush_callbacks(void);
+
+static bool has_lookahead(void)
+{
+    return ! lookahead_list.empty();
+}
+
+static wint_t lookahead_pop(void)
+{
+    wint_t result = lookahead_list.front();
+    lookahead_list.pop_front();
+    return result;
+}
+
+static void lookahead_push_back(wint_t c)
+{
+    lookahead_list.push_back(c);
+}
+
+static void lookahead_push_front(wint_t c)
+{
+    lookahead_list.push_front(c);
+}
+
+static wint_t lookahead_front(void)
+{
+    return lookahead_list.front();
+}
+
+/** Callback function for handling interrupts on reading */
+static int (*interrupt_handler)();
+
+void input_common_init(int (*ih)())
+{
+    interrupt_handler = ih;
+}
+
+void input_common_destroy()
+{
+
+}
+
+/**
+   Internal function used by input_common_readch to read one byte from fd 0. This function should only be called by
+   input_common_readch().
+*/
+static wint_t readb()
+{
+    /* do_loop must be set on every path through the loop; leaving it uninitialized allows the static analyzer to assist in catching mistakes. */
+    unsigned char arr[1];
+    bool do_loop;
+
+    do
+    {
+        /* Flush callbacks */
+        input_flush_callbacks();
+
+        fd_set fdset;
+        int fd_max = 0;
+        int ioport = iothread_port();
+        int res;
+
+        FD_ZERO(&fdset);
+        FD_SET(0, &fdset);
+        if (ioport > 0)
+        {
+            FD_SET(ioport, &fdset);
+            fd_max = maxi(fd_max, ioport);
+        }
+        
+        /* Get our uvar notifier */
+        universal_notifier_t &notifier = universal_notifier_t::default_notifier();
+        
+        /* Get the notification fd (possibly none) */
+        int notifier_fd = notifier.notification_fd();
+        if (notifier_fd > 0)
+        {
+            FD_SET(notifier_fd, &fdset);
+            fd_max = maxi(fd_max, notifier_fd);
+        }
+        
+        /* Get its suggested delay (possibly none) */
+        struct timeval tv = {};
+        const unsigned long usecs_delay = notifier.usec_delay_between_polls();
+        if (usecs_delay > 0)
+        {
+            unsigned long usecs_per_sec = 1000000;
+            tv.tv_sec = (int)(usecs_delay / usecs_per_sec);
+            tv.tv_usec = (int)(usecs_delay % usecs_per_sec);
+        }
+        
+        res = select(fd_max + 1, &fdset, 0, 0, usecs_delay > 0 ? &tv : NULL);
+        if (res==-1)
+        {
+            switch (errno)
+            {
+                case EINTR:
+                case EAGAIN:
+                {
+                    if (interrupt_handler)
+                    {
+                        int res = interrupt_handler();
+                        if (res)
+                        {
+                            return res;
+                        }
+                        if (has_lookahead())
+                        {
+                            return lookahead_pop();
+                        }
+
+                    }
+
+
+                    do_loop = true;
+                    break;
+                }
+                default:
+                {
+                    /*
+                      The terminal has been closed. Save and exit.
+                    */
+                    return R_EOF;
+                }
+            }
+        }
+        else
+        {
+            /* Assume we loop unless we see a character in stdin */
+            do_loop = true;
+
+            /* Check to see if we want a universal variable barrier */
+            bool barrier_from_poll = notifier.poll();
+            bool barrier_from_readability = false;
+            if (notifier_fd > 0 && FD_ISSET(notifier_fd, &fdset))
+            {
+                barrier_from_readability = notifier.notification_fd_became_readable(notifier_fd);
+            }
+            if (barrier_from_poll || barrier_from_readability)
+            {
+                env_universal_barrier();
+            }
+
+            if (ioport > 0 && FD_ISSET(ioport, &fdset))
+            {
+                iothread_service_completion();
+                if (has_lookahead())
+                {
+                    return lookahead_pop();
+                }
+            }
+
+            if (FD_ISSET(STDIN_FILENO, &fdset))
+            {
+                if (read_blocked(0, arr, 1) != 1)
+                {
+                    /* The teminal has been closed. Save and exit. */
+                    return R_EOF;
+                }
+
+                /* We read from stdin, so don't loop */
+                do_loop = false;
+            }
+        }
+    }
+    while (do_loop);
+
+    return arr[0];
+}
+
+wchar_t input_common_readch(int timed)
+{
+    if (! has_lookahead())
+    {
+        if (timed)
+        {
+            int count;
+            fd_set fds;
+            struct timeval tm=
+            {
+                0,
+                1000 * WAIT_ON_ESCAPE
+            }
+            ;
+
+            FD_ZERO(&fds);
+            FD_SET(0, &fds);
+            count = select(1, &fds, 0, 0, &tm);
+
+            switch (count)
+            {
+                case 0:
+                    return WEOF;
+
+                case -1:
+                    return WEOF;
+                    break;
+                default:
+                    break;
+
+            }
+        }
+
+        wchar_t res;
+        mbstate_t state = {};
+
+        while (1)
+        {
+            wint_t b = readb();
+            char bb;
+
+            size_t sz;
+
+            if ((b >= R_NULL) && (b < R_NULL + 1000))
+                return b;
+
+            bb=b;
+
+            sz = mbrtowc(&res, &bb, 1, &state);
+
+            switch (sz)
+            {
+                case (size_t)(-1):
+                    memset(&state, '\0', sizeof(state));
+                    debug(2, L"Illegal input");
+                    return R_NULL;
+                case (size_t)(-2):
+                    break;
+                case 0:
+                    return 0;
+                default:
+                    return res;
+            }
+        }
+    }
+    else
+    {
+        if (!timed)
+        {
+            while (has_lookahead() && lookahead_front() == WEOF)
+                lookahead_pop();
+            if (! has_lookahead())
+                return input_common_readch(0);
+        }
+
+        return lookahead_pop();
+    }
+}
+
+
+void input_common_queue_ch(wint_t ch)
+{
+    lookahead_push_back(ch);
+}
+
+void input_common_next_ch(wint_t ch)
+{
+    lookahead_push_front(ch);
+}
+
+void input_common_add_callback(void (*callback)(void *), void *arg)
+{
+    ASSERT_IS_MAIN_THREAD();
+    callback_queue.push(callback_info_t(callback, arg));
+}
+
+static void input_flush_callbacks(void)
+{
+    /* Nothing to do if nothing to do */
+    if (callback_queue.empty())
+        return;
+
+    /* We move the queue into a local variable, so that events queued up during a callback don't get fired until next round. */
+    callback_queue_t local_queue;
+    std::swap(local_queue, callback_queue);
+    while (! local_queue.empty())
+    {
+        const callback_info_t &callback = local_queue.front();
+        callback.first(callback.second); //cute
+        local_queue.pop();
+    }
+}
diff --git a/src/input_common.h b/src/input_common.h
new file mode 100644
index 00000000..5ec34b39
--- /dev/null
+++ b/src/input_common.h
@@ -0,0 +1,66 @@
+/** \file input_common.h
+
+Header file for the low level input library
+
+*/
+#ifndef INPUT_COMMON_H
+#define INPUT_COMMON_H
+
+#include <stddef.h>
+
+/**
+  Use unencoded private-use keycodes for internal characters
+*/
+#define INPUT_COMMON_RESERVED 0xe000
+
+enum
+{
+    /**
+       R_NULL is sometimes returned by the input when a character was
+       requested but none could be delivered, or when an exception
+       happened.
+    */
+    R_NULL = INPUT_COMMON_RESERVED,
+    R_EOF
+}
+;
+
+/**
+   Init the library
+*/
+void input_common_init(int (*ih)());
+
+/**
+   Free memory used by the library
+*/
+void input_common_destroy();
+
+/**
+   Function used by input_readch to read bytes from stdin until enough
+   bytes have been read to convert them to a wchar_t. Conversion is
+   done using mbrtowc. If a character has previously been read and
+   then 'unread' using \c input_common_unreadch, that character is
+   returned. If timed is true, readch2 will wait at most
+   WAIT_ON_ESCAPE milliseconds for a character to be available for
+   reading before returning with the value WEOF.
+*/
+wchar_t input_common_readch(int timed);
+
+/**
+   Enqueue a character or a readline function to the queue of unread
+   characters that input_readch will return before actually reading from fd
+   0.
+*/
+void input_common_queue_ch(wint_t ch);
+
+/**
+   Add a character or a readline function to the front of the queue of unread
+   characters.  This will be the first character returned by input_readch
+   (unless this function is called more than once).
+*/
+void input_common_next_ch(wint_t ch);
+
+/** Adds a callback to be invoked at the next turn of the "event loop." The callback function will be invoked and passed arg. */
+void input_common_add_callback(void (*callback)(void *), void *arg);
+
+#endif
diff --git a/src/intern.cpp b/src/intern.cpp
new file mode 100644
index 00000000..2ec0af8a
--- /dev/null
+++ b/src/intern.cpp
@@ -0,0 +1,86 @@
+/** \file intern.c
+
+    Library for pooling common strings
+
+*/
+#include "config.h" // IWYU pragma: keep
+
+#include <wchar.h>
+#include <pthread.h>
+#include <vector>
+#include <algorithm>
+
+#include "fallback.h" // IWYU pragma: keep
+#include "common.h"
+#include "intern.h"
+
+/** Comparison function for intern'd strings */
+class string_table_compare_t
+{
+public:
+    bool operator()(const wchar_t *a, const wchar_t *b) const
+    {
+        return wcscmp(a, b) < 0;
+    }
+};
+
+/* A sorted vector ends up being a little more memory efficient than a std::set for the intern'd string table */
+#define USE_SET 0
+#if USE_SET
+/** The table of intern'd strings */
+typedef std::set<const wchar_t *, string_table_compare_t> string_table_t;
+#else
+/** The table of intern'd strings */
+typedef std::vector<const wchar_t *> string_table_t;
+#endif
+
+static string_table_t string_table;
+
+/** The lock to provide thread safety for intern'd strings */
+static pthread_mutex_t intern_lock = PTHREAD_MUTEX_INITIALIZER;
+
+static const wchar_t *intern_with_dup(const wchar_t *in, bool dup)
+{
+    if (!in)
+        return NULL;
+
+//  debug( 0, L"intern %ls", in );
+    scoped_lock lock(intern_lock);
+    const wchar_t *result;
+
+#if USE_SET
+    string_table_t::const_iterator iter = string_table.find(in);
+    if (iter != string_table.end())
+    {
+        result = *iter;
+    }
+    else
+    {
+        result = dup ? wcsdup(in) : in;
+        string_table.insert(result);
+    }
+#else
+    string_table_t::iterator iter = std::lower_bound(string_table.begin(), string_table.end(), in, string_table_compare_t());
+    if (iter != string_table.end() && wcscmp(*iter, in) == 0)
+    {
+        result = *iter;
+    }
+    else
+    {
+        result = dup ? wcsdup(in) : in;
+        string_table.insert(iter, result);
+    }
+#endif
+    return result;
+}
+
+const wchar_t *intern(const wchar_t *in)
+{
+    return intern_with_dup(in, true);
+}
+
+
+const wchar_t *intern_static(const wchar_t *in)
+{
+    return intern_with_dup(in, false);
+}
diff --git a/src/intern.h b/src/intern.h
new file mode 100644
index 00000000..b9d07526
--- /dev/null
+++ b/src/intern.h
@@ -0,0 +1,26 @@
+/** \file intern.h
+
+    Library for pooling common strings
+
+*/
+
+#ifndef FISH_INTERN_H
+#define FISH_INTERN_H
+
+/**
+   Return an identical copy of the specified string from a pool of unique strings. If the string was not in the pool, add a copy.
+
+   \param in the string to return an interned copy of
+*/
+const wchar_t *intern(const wchar_t *in);
+
+/**
+   Insert the specified string literal into the pool of unique
+   strings. The string will not first be copied, and it will not be
+   free'd on exit.
+
+   \param in the string to add to the interned pool
+*/
+const wchar_t *intern_static(const wchar_t *in);
+
+#endif
diff --git a/src/io.cpp b/src/io.cpp
new file mode 100644
index 00000000..a013b9ef
--- /dev/null
+++ b/src/io.cpp
@@ -0,0 +1,324 @@
+/** \file io.c
+
+Utilities for io redirection.
+
+*/
+#include "config.h" // IWYU pragma: keep
+
+
+#include <stdio.h>
+#include <errno.h>
+#include <assert.h>
+#include <unistd.h>
+
+#include "fallback.h" // IWYU pragma: keep
+#include "wutil.h"
+#include "exec.h"
+#include "common.h"
+#include "io.h"
+
+
+io_data_t::~io_data_t()
+{
+}
+
+void io_close_t::print() const
+{
+    fprintf(stderr, "close %d\n", fd);
+}
+
+void io_fd_t::print() const
+{
+    fprintf(stderr, "FD map %d -> %d\n", old_fd, fd);
+}
+
+void io_file_t::print() const
+{
+    fprintf(stderr, "file (%s)\n", filename_cstr);
+}
+
+void io_pipe_t::print() const
+{
+    fprintf(stderr, "pipe {%d, %d} (input: %s)\n", pipe_fd[0], pipe_fd[1],
+            is_input ? "yes" : "no");
+}
+
+void io_buffer_t::print() const
+{
+    fprintf(stderr, "buffer %p (input: %s, size %lu)\n", out_buffer_ptr(),
+            is_input ? "yes" : "no", (unsigned long) out_buffer_size());
+}
+
+void io_buffer_t::read()
+{
+    exec_close(pipe_fd[1]);
+
+    if (io_mode == IO_BUFFER)
+    {
+        /*    if( fcntl( pipe_fd[0], F_SETFL, 0 ) )
+            {
+              wperror( L"fcntl" );
+              return;
+              }  */
+        debug(4, L"io_buffer_t::read: blocking read on fd %d", pipe_fd[0]);
+        while (1)
+        {
+            char b[4096];
+            long l;
+            l=read_blocked(pipe_fd[0], b, 4096);
+            if (l==0)
+            {
+                break;
+            }
+            else if (l<0)
+            {
+                /*
+                  exec_read_io_buffer is only called on jobs that have
+                  exited, and will therefore never block. But a broken
+                  pipe seems to cause some flags to reset, causing the
+                  EOF flag to not be set. Therefore, EAGAIN is ignored
+                  and we exit anyway.
+                */
+                if (errno != EAGAIN)
+                {
+                    debug(1,
+                          _(L"An error occured while reading output from code block on file descriptor %d"),
+                          pipe_fd[0]);
+                    wperror(L"io_buffer_t::read");
+                }
+
+                break;
+            }
+            else
+            {
+                out_buffer_append(b, l);
+            }
+        }
+    }
+}
+
+bool io_buffer_t::avoid_conflicts_with_io_chain(const io_chain_t &ios)
+{
+    bool result = pipe_avoid_conflicts_with_io_chain(this->pipe_fd, ios);
+    if (! result)
+    {
+        wperror(L"dup");
+    }
+    return result;
+}
+
+io_buffer_t *io_buffer_t::create(int fd, const io_chain_t &conflicts)
+{
+    bool success = true;
+    assert(fd >= 0);
+    io_buffer_t *buffer_redirect = new io_buffer_t(fd);
+
+    if (exec_pipe(buffer_redirect->pipe_fd) == -1)
+    {
+        debug(1, PIPE_ERROR);
+        wperror(L"pipe");
+        success = false;
+    }
+    else if (! buffer_redirect->avoid_conflicts_with_io_chain(conflicts))
+    {
+        // The above call closes the fds on error
+        success = false;
+    }
+    else if (make_fd_nonblocking(buffer_redirect->pipe_fd[0]) != 0)
+    {
+        debug(1, PIPE_ERROR);
+        wperror(L"fcntl");
+        success = false;
+    }
+
+    if (! success)
+    {
+        delete buffer_redirect;
+        buffer_redirect = NULL;
+    }
+    return buffer_redirect;
+}
+
+io_buffer_t::~io_buffer_t()
+{
+    if (pipe_fd[0] >= 0)
+    {
+        exec_close(pipe_fd[0]);
+    }
+
+    /*
+      Dont free fd for writing. This should already be free'd before
+      calling exec_read_io_buffer on the buffer
+    */
+}
+
+void io_chain_t::remove(const shared_ptr<const io_data_t> &element)
+{
+    // See if you can guess why std::find doesn't work here
+    for (io_chain_t::iterator iter = this->begin(); iter != this->end(); ++iter)
+    {
+        if (*iter == element)
+        {
+            this->erase(iter);
+            break;
+        }
+    }
+}
+
+void io_chain_t::push_back(const shared_ptr<io_data_t> &element)
+{
+    // Ensure we never push back NULL
+    assert(element.get() != NULL);
+    std::vector<shared_ptr<io_data_t> >::push_back(element);
+}
+
+void io_chain_t::push_front(const shared_ptr<io_data_t> &element)
+{
+    assert(element.get() != NULL);
+    this->insert(this->begin(), element);
+}
+
+void io_chain_t::append(const io_chain_t &chain)
+{
+    this->insert(this->end(), chain.begin(), chain.end());
+}
+
+void io_print(const io_chain_t &chain)
+{
+    if (chain.empty())
+    {
+        fprintf(stderr, "Empty chain %p\n", &chain);
+        return;
+    }
+
+    fprintf(stderr, "Chain %p (%ld items):\n", &chain, (long)chain.size());
+    for (size_t i=0; i < chain.size(); i++)
+    {
+        const shared_ptr<const io_data_t> &io = chain.at(i);
+        if (io.get() == NULL)
+        {
+            fprintf(stderr, "\t(null)\n");
+        }
+        else
+        {
+            fprintf(stderr, "\t%lu: fd:%d, ", (unsigned long)i, io->fd);
+            io->print();
+        }
+    }
+}
+
+/* If the given fd is used by the io chain, duplicates it repeatedly until an fd not used in the io chain is found, or we run out. If we return a new fd or an error, closes the old one. Any fd created is marked close-on-exec. Returns -1 on failure (in which case the given fd is still closed). */
+static int move_fd_to_unused(int fd, const io_chain_t &io_chain)
+{
+    int new_fd = fd;
+    if (fd >= 0 && io_chain.get_io_for_fd(fd).get() != NULL)
+    {
+        /* We have fd >= 0, and it's a conflict. dup it and recurse. Note that we recurse before anything is closed; this forces the kernel to give us a new one (or report fd exhaustion). */
+        int tmp_fd;
+        do
+        {
+            tmp_fd = dup(fd);
+        } while (tmp_fd < 0 && errno == EINTR);
+        
+        assert(tmp_fd != fd);
+        if (tmp_fd < 0)
+        {
+            /* Likely fd exhaustion. */
+            new_fd = -1;
+        }
+        else
+        {
+            /* Ok, we have a new candidate fd. Recurse. If we get a valid fd, either it's the same as what we gave it, or it's a new fd and what we gave it has been closed. If we get a negative value, the fd also has been closed. */
+            set_cloexec(tmp_fd);
+            new_fd = move_fd_to_unused(tmp_fd, io_chain);
+        }
+        
+        /* We're either returning a new fd or an error. In both cases, we promise to close the old one. */
+        assert(new_fd != fd);
+        int saved_errno = errno;
+        exec_close(fd);
+        errno = saved_errno;
+    }
+    return new_fd;
+}
+
+bool pipe_avoid_conflicts_with_io_chain(int fds[2], const io_chain_t &ios)
+{
+    bool success = true;
+    for (int i=0; i < 2; i++)
+    {
+        fds[i] = move_fd_to_unused(fds[i], ios);
+        if (fds[i] < 0)
+        {
+            success = false;
+            break;
+        }
+    }
+    
+    /* If any fd failed, close all valid fds */
+    if (! success)
+    {
+        int saved_errno = errno;
+        for (int i=0; i < 2; i++)
+        {
+            if (fds[i] >= 0)
+            {
+                exec_close(fds[i]);
+                fds[i] = -1;
+            }
+        }
+        errno = saved_errno;
+    }
+    return success;
+}
+
+/* Return the last IO for the given fd */
+shared_ptr<const io_data_t> io_chain_t::get_io_for_fd(int fd) const
+{
+    size_t idx = this->size();
+    while (idx--)
+    {
+        const shared_ptr<const io_data_t> &data = this->at(idx);
+        if (data->fd == fd)
+        {
+            return data;
+        }
+    }
+    return shared_ptr<const io_data_t>();
+}
+
+shared_ptr<io_data_t> io_chain_t::get_io_for_fd(int fd)
+{
+    size_t idx = this->size();
+    while (idx--)
+    {
+        const shared_ptr<io_data_t> &data = this->at(idx);
+        if (data->fd == fd)
+        {
+            return data;
+        }
+    }
+    return shared_ptr<io_data_t>();
+}
+
+/* The old function returned the last match, so we mimic that. */
+shared_ptr<const io_data_t> io_chain_get(const io_chain_t &src, int fd)
+{
+    return src.get_io_for_fd(fd);
+}
+
+shared_ptr<io_data_t> io_chain_get(io_chain_t &src, int fd)
+{
+    return src.get_io_for_fd(fd);
+}
+
+io_chain_t::io_chain_t(const shared_ptr<io_data_t> &data) :
+    std::vector<shared_ptr<io_data_t> >(1, data)
+{
+
+}
+
+io_chain_t::io_chain_t() : std::vector<shared_ptr<io_data_t> >()
+{
+}
+
diff --git a/src/io.h b/src/io.h
new file mode 100644
index 00000000..86fc7962
--- /dev/null
+++ b/src/io.h
@@ -0,0 +1,222 @@
+#ifndef FISH_IO_H
+#define FISH_IO_H
+
+#include <vector>
+#include <stddef.h>
+#include <stdlib.h>
+
+// Note that we have to include something to get any _LIBCPP_VERSION defined so we can detect libc++
+// So it's key that vector go above. If we didn't need vector for other reasons, we might include ciso646, which does nothing
+
+#if defined(_LIBCPP_VERSION) || __cplusplus > 199711L
+// C++11 or libc++ (which is a C++11-only library, but the memory header works OK in C++03)
+#include <memory>
+using std::shared_ptr;
+#else
+// C++03 or libstdc++
+#include <tr1/memory>
+using std::tr1::shared_ptr;
+#endif
+
+#include "common.h"
+
+/**
+   Describes what type of IO operation an io_data_t represents
+*/
+enum io_mode_t
+{
+    IO_FILE, IO_PIPE, IO_FD, IO_BUFFER, IO_CLOSE
+};
+
+/** Represents an FD redirection */
+class io_data_t
+{
+private:
+    /* No assignment or copying allowed */
+    io_data_t(const io_data_t &rhs);
+    void operator=(const io_data_t &rhs);
+
+protected:
+    io_data_t(io_mode_t m, int f) :
+        io_mode(m),
+        fd(f)
+    {
+    }
+
+public:
+    /** Type of redirect */
+    const io_mode_t io_mode;
+    /** FD to redirect */
+    const int fd;
+
+    virtual void print() const = 0;
+    virtual ~io_data_t() = 0;
+};
+
+class io_close_t : public io_data_t
+{
+public:
+    io_close_t(int f) :
+        io_data_t(IO_CLOSE, f)
+    {
+    }
+
+    virtual void print() const;
+};
+
+class io_fd_t : public io_data_t
+{
+public:
+    /** fd to redirect specified fd to. For example, in 2>&1, old_fd is 1, and io_data_t::fd is 2 */
+    const int old_fd;
+    
+    /** Whether this redirection was supplied by a script. For example, 'cmd <&3' would have user_supplied set to true. But a redirection that comes about through transmogrification would not. */
+    const bool user_supplied;
+    
+    virtual void print() const;
+
+    io_fd_t(int f, int old, bool us) :
+        io_data_t(IO_FD, f),
+        old_fd(old),
+        user_supplied(us)
+    {
+    }
+};
+
+class io_file_t : public io_data_t
+{
+public:
+    /** Filename, malloc'd. This needs to be used after fork, so don't use wcstring here. */
+    const char * const filename_cstr;
+    /** file creation flags to send to open */
+    const int flags;
+
+    virtual void print() const;
+
+    io_file_t(int f, const wcstring &fname, int fl = 0) :
+        io_data_t(IO_FILE, f),
+        filename_cstr(wcs2str(fname)),
+        flags(fl)
+    {
+    }
+
+    virtual ~io_file_t()
+    {
+        free((void *)filename_cstr);
+    }
+};
+
+class io_pipe_t : public io_data_t
+{
+protected:
+    io_pipe_t(io_mode_t m, int f, bool i):
+        io_data_t(m, f),
+        is_input(i)
+    {
+        pipe_fd[0] = pipe_fd[1] = -1;
+    }
+
+public:
+    int pipe_fd[2];
+    const bool is_input;
+
+    virtual void print() const;
+
+    io_pipe_t(int f, bool i):
+        io_data_t(IO_PIPE, f),
+        is_input(i)
+    {
+        pipe_fd[0] = pipe_fd[1] = -1;
+    }
+};
+
+class io_chain_t;
+class io_buffer_t : public io_pipe_t
+{
+private:
+    /** buffer to save output in */
+    std::vector<char> out_buffer;
+
+    io_buffer_t(int f):
+        io_pipe_t(IO_BUFFER, f, false /* not input */),
+        out_buffer()
+    {
+    }
+
+public:
+    virtual void print() const;
+
+    virtual ~io_buffer_t();
+
+    /** Function to append to the buffer */
+    void out_buffer_append(const char *ptr, size_t count)
+    {
+        out_buffer.insert(out_buffer.end(), ptr, ptr + count);
+    }
+
+    /** Function to get a pointer to the buffer */
+    char *out_buffer_ptr(void)
+    {
+        return out_buffer.empty() ? NULL : &out_buffer.at(0);
+    }
+
+    const char *out_buffer_ptr(void) const
+    {
+        return out_buffer.empty() ? NULL : &out_buffer.at(0);
+    }
+
+    /** Function to get the size of the buffer */
+    size_t out_buffer_size(void) const
+    {
+        return out_buffer.size();
+    }
+    
+    /* Ensures that the pipes do not conflict with any fd redirections in the chain */
+    bool avoid_conflicts_with_io_chain(const io_chain_t &ios);
+
+    /**
+       Close output pipe, and read from input pipe until eof.
+    */
+    void read();
+
+    /**
+       Create a IO_BUFFER type io redirection, complete with a pipe and a
+       vector<char> for output. The default file descriptor used is STDOUT_FILENO
+       for buffering. 
+
+       \param fd the fd that will be mapped in the child process, typically STDOUT_FILENO
+       \param conflicts A set of IO redirections. The function ensures that any pipe it makes
+              does not conflict with an fd redirection in this list.
+    */
+    static io_buffer_t *create(int fd, const io_chain_t &conflicts);
+};
+
+class io_chain_t : public std::vector<shared_ptr<io_data_t> >
+{
+public:
+    io_chain_t();
+    io_chain_t(const shared_ptr<io_data_t> &);
+
+    void remove(const shared_ptr<const io_data_t> &element);
+    void push_back(const shared_ptr<io_data_t> &element);
+    void push_front(const shared_ptr<io_data_t> &element);
+    void append(const io_chain_t &chain);
+
+    shared_ptr<const io_data_t> get_io_for_fd(int fd) const;
+    shared_ptr<io_data_t> get_io_for_fd(int fd);
+};
+
+
+/**
+   Return the last io redirection in the chain for the specified file descriptor.
+*/
+shared_ptr<const io_data_t> io_chain_get(const io_chain_t &src, int fd);
+shared_ptr<io_data_t> io_chain_get(io_chain_t &src, int fd);
+
+/* Given a pair of fds, if an fd is used by the given io chain, duplicate that fd repeatedly until we find one that does not conflict, or we run out of fds. Returns the new fds by reference, closing the old ones. If we get an error, returns false (in which case both fds are closed and set to -1). */
+bool pipe_avoid_conflicts_with_io_chain(int fds[2], const io_chain_t &ios);
+
+/** Print debug information about the specified IO redirection chain to stderr. */
+void io_print(const io_chain_t &chain);
+
+#endif
diff --git a/src/iothread.cpp b/src/iothread.cpp
new file mode 100644
index 00000000..7a37b63b
--- /dev/null
+++ b/src/iothread.cpp
@@ -0,0 +1,390 @@
+#include "config.h" // IWYU pragma: keep
+#include "iothread.h"
+#include "common.h"
+#include <pthread.h>
+#include <assert.h>
+#include <stdio.h>
+#include <limits.h> // IWYU pragma: keep - for _POSIX_THREADS_MAX, suggests internal header
+#include <sys/select.h>
+#include <sys/time.h>
+#include <unistd.h>
+#include <signal.h>
+#include <fcntl.h>
+#include <queue>
+#include <algorithm>
+
+#ifdef _POSIX_THREAD_THREADS_MAX
+#if _POSIX_THREAD_THREADS_MAX < 64
+#define IO_MAX_THREADS _POSIX_THREAD_THREADS_MAX
+#endif
+#endif
+
+#ifndef IO_MAX_THREADS
+#define IO_MAX_THREADS 64
+#endif
+
+/* Values for the wakeup bytes sent to the ioport */
+#define IO_SERVICE_MAIN_THREAD_REQUEST_QUEUE 99
+#define IO_SERVICE_RESULT_QUEUE 100
+
+#define IOTHREAD_LOG if (0)
+
+static void iothread_service_main_thread_requests(void);
+static void iothread_service_result_queue();
+
+struct SpawnRequest_t
+{
+    int (*handler)(void *);
+    void (*completionCallback)(void *, int);
+    void *context;
+    int handlerResult;
+};
+
+struct MainThreadRequest_t
+{
+    int (*handler)(void *);
+    void *context;
+    volatile int handlerResult;
+    volatile bool done;
+};
+
+/* Spawn support. Requests are allocated and come in on request_queue. They go out on result_queue, at which point they can be deallocated. s_active_thread_count is also protected by the lock. */
+static pthread_mutex_t s_spawn_queue_lock;
+static std::queue<SpawnRequest_t *> s_request_queue;
+static int s_active_thread_count;
+
+static pthread_mutex_t s_result_queue_lock;
+static std::queue<SpawnRequest_t *> s_result_queue;
+
+/* "Do on main thread" support */
+static pthread_mutex_t s_main_thread_performer_lock; // protects the main thread requests
+static pthread_cond_t s_main_thread_performer_condition; //protects the main thread requests
+static pthread_mutex_t s_main_thread_request_queue_lock; // protects the queue
+static std::queue<MainThreadRequest_t *> s_main_thread_request_queue;
+
+/* Notifying pipes */
+static int s_read_pipe, s_write_pipe;
+
+static void iothread_init(void)
+{
+    static bool inited = false;
+    if (! inited)
+    {
+        inited = true;
+
+        /* Initialize some locks */
+        VOMIT_ON_FAILURE(pthread_mutex_init(&s_spawn_queue_lock, NULL));
+        VOMIT_ON_FAILURE(pthread_mutex_init(&s_result_queue_lock, NULL));
+        VOMIT_ON_FAILURE(pthread_mutex_init(&s_main_thread_request_queue_lock, NULL));
+        VOMIT_ON_FAILURE(pthread_mutex_init(&s_main_thread_performer_lock, NULL));
+        VOMIT_ON_FAILURE(pthread_cond_init(&s_main_thread_performer_condition, NULL));
+
+        /* Initialize the completion pipes */
+        int pipes[2] = {0, 0};
+        VOMIT_ON_FAILURE(pipe(pipes));
+        s_read_pipe = pipes[0];
+        s_write_pipe = pipes[1];
+
+        // 0 means success to VOMIT_ON_FAILURE. Arrange to pass 0 if fcntl returns anything other than -1.
+        VOMIT_ON_FAILURE(-1 == fcntl(s_read_pipe, F_SETFD, FD_CLOEXEC));
+        VOMIT_ON_FAILURE(-1 == fcntl(s_write_pipe, F_SETFD, FD_CLOEXEC));
+    }
+}
+
+static void add_to_queue(struct SpawnRequest_t *req)
+{
+    ASSERT_IS_LOCKED(s_spawn_queue_lock);
+    s_request_queue.push(req);
+}
+
+static SpawnRequest_t *dequeue_spawn_request(void)
+{
+    ASSERT_IS_LOCKED(s_spawn_queue_lock);
+    SpawnRequest_t *result = NULL;
+    if (! s_request_queue.empty())
+    {
+        result = s_request_queue.front();
+        s_request_queue.pop();
+    }
+    return result;
+}
+
+static void enqueue_thread_result(SpawnRequest_t *req)
+{
+    scoped_lock lock(s_result_queue_lock);
+    s_result_queue.push(req);
+}
+
+static void *this_thread()
+{
+    return (void *)(intptr_t)pthread_self();
+}
+
+/* The function that does thread work. */
+static void *iothread_worker(void *unused)
+{
+    scoped_lock locker(s_spawn_queue_lock);
+    struct SpawnRequest_t *req;
+    while ((req = dequeue_spawn_request()) != NULL)
+    {
+        IOTHREAD_LOG fprintf(stderr, "pthread %p dequeued %p\n", this_thread(), req);
+        /* Unlock the queue while we execute the request */
+        locker.unlock();
+        
+        /* Perfor the work */
+        req->handlerResult = req->handler(req->context);
+        
+        /* If there's a completion handler, we have to enqueue it on the result queue. Otherwise, we can just delete the request! */
+        if (req->completionCallback == NULL)
+        {
+            delete req;
+        }
+        else
+        {
+            /* Enqueue the result, and tell the main thread about it */
+            enqueue_thread_result(req);
+            const char wakeup_byte = IO_SERVICE_RESULT_QUEUE;
+            VOMIT_ON_FAILURE(! write_loop(s_write_pipe, &wakeup_byte, sizeof wakeup_byte));
+        }
+        
+        /* Lock us up again */
+        locker.lock();
+    }
+    
+    /* We believe we have exhausted the thread request queue. We want to decrement s_active_thread_count and exit. But it's possible that a request just came in. Furthermore, it's possible that the main thread saw that s_active_thread_count is full, and decided to not spawn a new thread, trusting in one of the existing threads to handle it. But we've already committed to not handling anything else. Therefore, we have to decrement s_active_thread_count under the lock, which we still hold. Likewise, the main thread must check the value under the lock. */
+    ASSERT_IS_LOCKED(s_spawn_queue_lock);
+    assert(s_active_thread_count > 0);
+    s_active_thread_count -= 1;
+    
+    IOTHREAD_LOG fprintf(stderr, "pthread %p exiting\n", this_thread());
+
+    /* We're done */
+    return NULL;
+}
+
+/* Spawn another thread. No lock is held when this is called. */
+static void iothread_spawn()
+{
+    /* The spawned thread inherits our signal mask. We don't want the thread to ever receive signals on the spawned thread, so temporarily block all signals, spawn the thread, and then restore it. */
+    sigset_t new_set, saved_set;
+    sigfillset(&new_set);
+    VOMIT_ON_FAILURE(pthread_sigmask(SIG_BLOCK, &new_set, &saved_set));
+
+    /* Spawn a thread. If this fails, it means there's already a bunch of threads; it is very unlikely that they are all on the verge of exiting, so one is likely to be ready to handle extant requests. So we can ignore failure with some confidence. */
+    pthread_t thread = 0;
+    pthread_create(&thread, NULL, iothread_worker, NULL);
+    
+    /* We will never join this thread */
+    VOMIT_ON_FAILURE(pthread_detach(thread));
+    
+    IOTHREAD_LOG fprintf(stderr, "pthread %p spawned\n", (void *)(intptr_t)thread);
+    
+    /* Restore our sigmask */
+    VOMIT_ON_FAILURE(pthread_sigmask(SIG_SETMASK, &saved_set, NULL));
+}
+
+int iothread_perform_base(int (*handler)(void *), void (*completionCallback)(void *, int), void *context)
+{
+    ASSERT_IS_MAIN_THREAD();
+    ASSERT_IS_NOT_FORKED_CHILD();
+    iothread_init();
+
+    /* Create and initialize a request. */
+    struct SpawnRequest_t *req = new SpawnRequest_t();
+    req->handler = handler;
+    req->completionCallback = completionCallback;
+    req->context = context;
+
+    int local_thread_count = -1;
+    bool spawn_new_thread = false;
+    {
+        /* Lock around a local region. Note that we can only access s_active_thread_count under the lock. */
+        scoped_lock lock(s_spawn_queue_lock);
+        add_to_queue(req);
+        if (s_active_thread_count < IO_MAX_THREADS)
+        {
+            s_active_thread_count++;
+            spawn_new_thread = true;
+        }
+        local_thread_count = s_active_thread_count;
+    }
+    
+    /* Kick off the thread if we decided to do so */
+    if (spawn_new_thread)
+    {
+        iothread_spawn();
+    }
+    
+    /* We return the active thread count for informational purposes only */
+    return local_thread_count;
+}
+
+int iothread_port(void)
+{
+    iothread_init();
+    return s_read_pipe;
+}
+
+void iothread_service_completion(void)
+{
+    ASSERT_IS_MAIN_THREAD();
+    char wakeup_byte = 0;
+    VOMIT_ON_FAILURE(1 != read_loop(iothread_port(), &wakeup_byte, sizeof wakeup_byte));
+    switch (wakeup_byte)
+    {
+        case IO_SERVICE_MAIN_THREAD_REQUEST_QUEUE:
+            iothread_service_main_thread_requests();
+            break;
+        case IO_SERVICE_RESULT_QUEUE:
+            iothread_service_result_queue();
+            break;
+        default:
+            fprintf(stderr, "Unknown wakeup byte %02x in %s\n", wakeup_byte, __FUNCTION__);
+            break;
+    }
+}
+
+static bool iothread_wait_for_pending_completions(long timeout_usec)
+{
+    const long usec_per_sec = 1000000;
+    struct timeval tv;
+    tv.tv_sec = timeout_usec / usec_per_sec;
+    tv.tv_usec = timeout_usec % usec_per_sec;
+    
+    const int fd = iothread_port();
+    fd_set fds;
+    FD_ZERO(&fds);
+    FD_SET(fd, &fds);
+    int ret = select(fd + 1, &fds, NULL, NULL, &tv);
+    return ret > 0;
+}
+
+/* Note that this function is quite sketchy. In particular, it drains threads, not requests, meaning that it may leave requests on the queue. This is the desired behavior (it may be called before fork, and we don't want to bother servicing requests before we fork), but in the test suite we depend on it draining all requests. In practice, this works, because a thread in practice won't exit while there is outstanding requests.
+ 
+  At the moment, this function is only used in the test suite and in a drain-all-threads-before-fork compatibility mode that no architecture requires, so it's OK that it's terrible.
+*/
+void iothread_drain_all(void)
+{
+    ASSERT_IS_MAIN_THREAD();
+    ASSERT_IS_NOT_FORKED_CHILD();
+    
+    scoped_lock locker(s_spawn_queue_lock);
+    
+#define TIME_DRAIN 0
+#if TIME_DRAIN
+    int thread_count = s_active_thread_count;
+    double now = timef();
+#endif
+    
+    /* Nasty polling via select(). */
+    while (s_active_thread_count > 0)
+    {
+        locker.unlock();
+        if (iothread_wait_for_pending_completions(1000))
+        {
+            iothread_service_completion();
+        }
+        locker.lock();
+    }
+#if TIME_DRAIN
+    double after = timef();
+    printf("(Waited %.02f msec for %d thread(s) to drain)\n", 1000 * (after - now), thread_count);
+#endif
+}
+
+/* "Do on main thread" support */
+static void iothread_service_main_thread_requests(void)
+{
+    ASSERT_IS_MAIN_THREAD();
+
+    // Move the queue to a local variable
+    std::queue<MainThreadRequest_t *> request_queue;
+    {
+        scoped_lock queue_lock(s_main_thread_request_queue_lock);
+        std::swap(request_queue, s_main_thread_request_queue);
+    }
+
+    if (! request_queue.empty())
+    {
+        // Perform each of the functions
+        // Note we are NOT responsible for deleting these. They are stack allocated in their respective threads!
+        while (! request_queue.empty())
+        {
+            MainThreadRequest_t *req = request_queue.front();
+            request_queue.pop();
+            req->handlerResult = req->handler(req->context);
+            req->done = true;
+        }
+
+        /* Ok, we've handled everybody. Announce the good news, and allow ourselves to be unlocked. Note we must do this while holding the lock. Otherwise we race with the waiting threads:
+         1. waiting thread checks for done, sees false
+         2. main thread performs request, sets done to true, posts to condition
+         3. waiting thread unlocks lock, waits on condition (forever)
+         Because the waiting thread performs step 1 under the lock, if we take the lock, we avoid posting before the waiting thread is waiting.
+        */
+        scoped_lock broadcast_lock(s_main_thread_performer_lock);
+        VOMIT_ON_FAILURE(pthread_cond_broadcast(&s_main_thread_performer_condition));
+    }
+}
+
+/* Service the queue of results */
+static void iothread_service_result_queue()
+{
+    // Move the queue to a local variable
+    std::queue<SpawnRequest_t *> result_queue;
+    {
+        scoped_lock queue_lock(s_result_queue_lock);
+        std::swap(result_queue, s_result_queue);
+    }
+    
+    // Perform each completion in order
+    // We are responsibile for cleaning them up
+    while (! result_queue.empty())
+    {
+        SpawnRequest_t *req = result_queue.front();
+        result_queue.pop();
+        if (req->completionCallback)
+        {
+            req->completionCallback(req->context, req->handlerResult);
+        }
+        delete req;
+    }
+}
+
+int iothread_perform_on_main_base(int (*handler)(void *), void *context)
+{
+    // If this is the main thread, just do it
+    if (is_main_thread())
+    {
+        return handler(context);
+    }
+
+    // Make a new request. Note we are synchronous, so this can be stack allocated!
+    MainThreadRequest_t req;
+    req.handler = handler;
+    req.context = context;
+    req.handlerResult = 0;
+    req.done = false;
+
+    // Append it
+    {
+        scoped_lock queue_lock(s_main_thread_request_queue_lock);
+        s_main_thread_request_queue.push(&req);
+    }
+
+    // Tell the pipe
+    const char wakeup_byte = IO_SERVICE_MAIN_THREAD_REQUEST_QUEUE;
+    VOMIT_ON_FAILURE(! write_loop(s_write_pipe, &wakeup_byte, sizeof wakeup_byte));
+
+    // Wait on the condition, until we're done
+    scoped_lock perform_lock(s_main_thread_performer_lock);
+    while (! req.done)
+    {
+        // It would be nice to support checking for cancellation here, but the clients need a deterministic way to clean up to avoid leaks
+        VOMIT_ON_FAILURE(pthread_cond_wait(&s_main_thread_performer_condition, &s_main_thread_performer_lock));
+    }
+
+    // Ok, the request must now be done
+    assert(req.done);
+    return req.handlerResult;
+}
diff --git a/src/iothread.h b/src/iothread.h
new file mode 100644
index 00000000..bdec2195
--- /dev/null
+++ b/src/iothread.h
@@ -0,0 +1,55 @@
+/** \file iothread.h
+    Handles IO that may hang.
+*/
+
+#ifndef FISH_IOTHREAD_H
+#define FISH_IOTHREAD_H
+
+/**
+ Runs a command on a thread.
+
+ \param handler The function to execute on a background thread. Accepts an arbitrary context pointer, and returns an int, which is passed to the completionCallback.
+ \param completionCallback The function to execute on the main thread once the background thread is complete. Accepts an int (the return value of handler) and the context.
+ \param context A arbitary context pointer to pass to the handler and completion callback.
+ \return A sequence number, currently not very useful.
+*/
+int iothread_perform_base(int (*handler)(void *), void (*completionCallback)(void *, int), void *context);
+
+/**
+  Gets the fd on which to listen for completion callbacks.
+
+  \return A file descriptor on which to listen for completion callbacks.
+*/
+int iothread_port(void);
+
+/** Services one iothread competion callback. */
+void iothread_service_completion(void);
+
+/** Waits for all iothreads to terminate. */
+void iothread_drain_all(void);
+
+/** Performs a function on the main thread, blocking until it completes */
+int iothread_perform_on_main_base(int (*handler)(void *), void *context);
+
+/** Helper templates */
+template<typename T>
+int iothread_perform(int (*handler)(T *), void (*completionCallback)(T *, int), T *context)
+{
+    return iothread_perform_base((int (*)(void *))handler, (void (*)(void *, int))completionCallback, static_cast<void *>(context));
+}
+
+/* Variant that takes no completion callback */
+template<typename T>
+int iothread_perform(int (*handler)(T *), T *context)
+{
+    return iothread_perform_base((int (*)(void *))handler, (void (*)(void *, int))0, static_cast<void *>(context));
+}
+
+template<typename T>
+int iothread_perform_on_main(int (*handler)(T *), T *context)
+{
+    return iothread_perform_on_main_base((int (*)(void *))handler, (void *)(context));
+}
+
+
+#endif
diff --git a/src/key_reader.cpp b/src/key_reader.cpp
new file mode 100644
index 00000000..382f4b22
--- /dev/null
+++ b/src/key_reader.cpp
@@ -0,0 +1,97 @@
+/*
+    A small utility to print the resulting key codes from pressing a
+  key. Servers the same function as hitting ^V in bash, but I prefer
+  the way key_reader works.
+
+  Type ^C to exit the program.
+*/
+#include "config.h"
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <termios.h>
+#include <unistd.h>
+#include <locale.h>
+#include <termcap.h>
+
+#include "common.h"
+#include "fallback.h"
+
+#include "input_common.h"
+
+int writestr(char *str)
+{
+    write_ignore(1, str, strlen(str));
+    return 0;
+}
+
+int main(int argc, char **argv)
+{
+    set_main_thread();
+    setup_fork_guards();
+    setlocale(LC_ALL, "");
+
+
+    if (argc == 2)
+    {
+        static char term_buffer[2048];
+        char *termtype = getenv("TERM");
+        char *tbuff = new char[9999];
+        char *res;
+
+        tgetent(term_buffer, termtype);
+        res = tgetstr(argv[1], &tbuff);
+        if (res != 0)
+        {
+            while (*res != 0)
+            {
+                printf("%d ", *res);
+
+
+                res++;
+            }
+            printf("\n");
+        }
+        else
+        {
+            printf("Undefined sequence\n");
+        }
+    }
+    else
+    {
+        char scratch[1024];
+        unsigned int c;
+
+        struct termios modes,      /* so we can change the modes */
+                savemodes;  /* so we can reset the modes when we're done */
+
+        input_common_init(0);
+
+
+        tcgetattr(0,&modes);        /* get the current terminal modes */
+        savemodes = modes;          /* save a copy so we can reset them */
+
+        modes.c_lflag &= ~ICANON;   /* turn off canonical mode */
+        modes.c_lflag &= ~ECHO;   /* turn off echo mode */
+        modes.c_cc[VMIN]=1;
+        modes.c_cc[VTIME]=0;
+        tcsetattr(0,TCSANOW,&modes);      /* set the new modes */
+        while (1)
+    {
+            if ((c=input_common_readch(0)) == EOF)
+                break;
+            if ((c > 31) && (c != 127))
+                sprintf(scratch, "dec: %d hex: %x char: %c\n", c, c, c);
+            else
+                sprintf(scratch, "dec: %d hex: %x\n", c, c);
+            writestr(scratch);
+        }
+        /* reset the terminal to the saved mode */
+        tcsetattr(0,TCSANOW,&savemodes);
+
+        input_common_destroy();
+    }
+
+    return 0;
+}
diff --git a/src/kill.cpp b/src/kill.cpp
new file mode 100644
index 00000000..22f21475
--- /dev/null
+++ b/src/kill.cpp
@@ -0,0 +1,219 @@
+/** \file kill.c
+  The killring.
+
+  Works like the killring in emacs and readline. The killring is cut
+  and paste with a memory of previous cuts. It supports integration
+  with the X clipboard.
+*/
+
+#include "config.h" // IWYU pragma: keep
+
+#include <stddef.h>
+#include <algorithm>
+#include <list>
+#include <string>
+
+#include "fallback.h" // IWYU pragma: keep
+#include "kill.h"
+#include "common.h"
+#include "env.h"
+#include "exec.h"
+#include "path.h"
+
+/**
+   Maximum entries in killring
+*/
+#define KILL_MAX 8192
+
+/** Last kill string */
+//static ll_node_t *kill_last=0;
+
+/** Current kill string */
+//static ll_node_t *kill_current=0;
+
+/** Kill ring */
+typedef std::list<wcstring> kill_list_t;
+static kill_list_t kill_list;
+
+/**
+   Contents of the X clipboard, at last time we checked it
+*/
+static wcstring cut_buffer;
+
+/**
+   Test if the xsel command is installed.  Since this is called often,
+   cache the result.
+*/
+static int has_xsel()
+{
+    static signed char res=-1;
+    if (res < 0)
+    {
+        res = !! path_get_path(L"xsel", NULL);
+    }
+
+    return res;
+}
+
+void kill_add(const wcstring &str)
+{
+    ASSERT_IS_MAIN_THREAD();
+    if (str.empty())
+        return;
+
+    wcstring cmd;
+    wcstring escaped_str;
+    kill_list.push_front(str);
+
+    /*
+       Check to see if user has set the FISH_CLIPBOARD_CMD variable,
+       and, if so, use it instead of checking the display, etc.
+
+       I couldn't think of a safe way to allow overide of the echo
+       command too, so, the command used must accept the input via stdin.
+    */
+
+    const env_var_t clipboard_wstr = env_get_string(L"FISH_CLIPBOARD_CMD");
+    if (!clipboard_wstr.missing())
+    {
+        escaped_str = escape(str.c_str(), ESCAPE_ALL);
+        cmd.assign(L"echo -n ");
+        cmd.append(escaped_str);
+        cmd.append(clipboard_wstr);
+    }
+    else
+    {
+        /* This is for sending the kill to the X copy-and-paste buffer */
+        if (!has_xsel())
+        {
+            return;
+        }
+
+        const env_var_t disp_wstr = env_get_string(L"DISPLAY");
+        if (!disp_wstr.missing())
+        {
+            escaped_str = escape(str.c_str(), ESCAPE_ALL);
+            cmd.assign(L"echo -n ");
+            cmd.append(escaped_str);
+            cmd.append(L" | xsel -i -b");
+        }
+    }
+
+    if (! cmd.empty())
+    {
+        if (exec_subshell(cmd, false /* do not apply exit status */) == -1)
+        {
+            /*
+               Do nothing on failiure
+            */
+        }
+
+        cut_buffer = escaped_str;
+    }
+}
+
+/**
+   Remove first match for specified string from circular list
+*/
+static void kill_remove(const wcstring &s)
+{
+    ASSERT_IS_MAIN_THREAD();
+    kill_list_t::iterator iter = std::find(kill_list.begin(), kill_list.end(), s);
+    if (iter != kill_list.end())
+        kill_list.erase(iter);
+}
+
+
+
+void kill_replace(const wcstring &old, const wcstring &newv)
+{
+    kill_remove(old);
+    kill_add(newv);
+}
+
+const wchar_t *kill_yank_rotate()
+{
+    ASSERT_IS_MAIN_THREAD();
+    // Move the first element to the end
+    if (kill_list.empty())
+    {
+        return NULL;
+    }
+    else
+    {
+        kill_list.splice(kill_list.end(), kill_list, kill_list.begin());
+        return kill_list.front().c_str();
+    }
+}
+
+/**
+   Check the X clipboard. If it has been changed, add the new
+   clipboard contents to the fish killring.
+*/
+static void kill_check_x_buffer()
+{
+    if (!has_xsel())
+        return;
+
+    const env_var_t disp = env_get_string(L"DISPLAY");
+    if (! disp.missing())
+    {
+        size_t i;
+        wcstring cmd = L"xsel -t 500 -b";
+        wcstring new_cut_buffer=L"";
+        wcstring_list_t list;
+        if (exec_subshell(cmd, list, false /* do not apply exit status */) != -1)
+        {
+
+            for (i=0; i<list.size(); i++)
+            {
+                wcstring next_line = escape_string(list.at(i), 0);
+                if (i > 0) new_cut_buffer += L"\\n";
+                new_cut_buffer += next_line;
+            }
+
+            if (new_cut_buffer.size() > 0)
+            {
+                /*
+                  The buffer is inserted with backslash escapes,
+                  since we don't really like tabs, newlines,
+                  etc. anyway.
+                */
+
+                if (cut_buffer != new_cut_buffer)
+                {
+                    cut_buffer = new_cut_buffer;
+                    kill_list.push_front(new_cut_buffer);
+                }
+            }
+        }
+    }
+}
+
+
+const wchar_t *kill_yank()
+{
+    kill_check_x_buffer();
+    if (kill_list.empty())
+    {
+        return L"";
+    }
+    else
+    {
+        return kill_list.front().c_str();
+    }
+}
+
+void kill_sanity_check()
+{
+}
+
+void kill_init()
+{
+}
+
+void kill_destroy()
+{
+    cut_buffer.clear();
+}
+
diff --git a/src/kill.h b/src/kill.h
new file mode 100644
index 00000000..686122a2
--- /dev/null
+++ b/src/kill.h
@@ -0,0 +1,36 @@
+/** \file kill.h
+  Prototypes for the killring.
+
+  Works like the killring in emacs and readline. The killring is cut and paste whith a memory of previous cuts.
+*/
+
+#ifndef FISH_KILL_H
+#define FISH_KILL_H
+
+#include "common.h"
+
+/**
+   Replace the specified string in the killring
+*/
+void kill_replace(const wcstring &old, const wcstring &newv);
+
+
+/** Add a string to the top of the killring */
+void kill_add(const wcstring &str);
+
+/** Rotate the killring */
+const wchar_t *kill_yank_rotate();
+
+/** Paste from the killring */
+const wchar_t *kill_yank();
+
+/** Sanity check */
+void kill_sanity_check();
+
+/** Initialize the killring */
+void kill_init();
+
+/** Destroy the killring */
+void kill_destroy();
+
+#endif
diff --git a/src/lru.h b/src/lru.h
new file mode 100644
index 00000000..779c2edc
--- /dev/null
+++ b/src/lru.h
@@ -0,0 +1,257 @@
+/** \file lru.h
+
+    Least-recently-used cache implementation
+*/
+
+#ifndef FISH_LRU_H
+#define FISH_LRU_H
+
+#include <assert.h>
+#include <wchar.h>
+#include <map>
+#include <set>
+#include <list>
+#include "common.h"
+
+/** A predicate to compare dereferenced pointers */
+struct dereference_less_t
+{
+    template <typename ptr_t>
+    bool operator()(ptr_t p1, ptr_t p2) const
+    {
+        return *p1 < *p2;
+    }
+};
+
+class lru_node_t
+{
+    template<class T> friend class lru_cache_t;
+
+    /** Our linked list pointer */
+    lru_node_t *prev, *next;
+
+public:
+    /** The key used to look up in the cache */
+    const wcstring key;
+
+    /** Constructor */
+    lru_node_t(const wcstring &pkey) : prev(NULL), next(NULL), key(pkey) { }
+
+    /** Virtual destructor that does nothing for classes that inherit lru_node_t */
+    virtual ~lru_node_t() {}
+
+    /** operator< for std::set */
+    bool operator<(const lru_node_t &other) const
+    {
+        return key < other.key;
+    }
+};
+
+template<class node_type_t>
+class lru_cache_t
+{
+private:
+
+    /** Max node count. This may be (transiently) exceeded by add_node_without_eviction, which is used from background threads. */
+    const size_t max_node_count;
+
+    /** Count of nodes */
+    size_t node_count;
+
+    /** The set of nodes */
+    typedef std::set<lru_node_t *, dereference_less_t> node_set_t;
+    node_set_t node_set;
+
+    void promote_node(node_type_t *node)
+    {
+        /* We should never promote the mouth */
+        assert(node != &mouth);
+
+        /* First unhook us */
+        node->prev->next = node->next;
+        node->next->prev = node->prev;
+
+        /* Put us after the mouth */
+        node->next = mouth.next;
+        node->next->prev = node;
+        node->prev = &mouth;
+        mouth.next = node;
+    }
+
+    void evict_node(node_type_t *condemned_node)
+    {
+        /* We should never evict the mouth */
+        assert(condemned_node != NULL && condemned_node != &mouth);
+
+        /* Remove it from the linked list */
+        condemned_node->prev->next = condemned_node->next;
+        condemned_node->next->prev = condemned_node->prev;
+
+        /* Remove us from the set */
+        node_set.erase(condemned_node);
+        node_count--;
+
+        /* Tell ourselves */
+        this->node_was_evicted(condemned_node);
+    }
+
+    void evict_last_node(void)
+    {
+        /* Simple */
+        evict_node((node_type_t *)mouth.prev);
+    }
+
+    static lru_node_t *get_previous(lru_node_t *node)
+    {
+        return node->prev;
+    }
+
+protected:
+
+    /** Head of the linked list */
+    lru_node_t mouth;
+
+    /** Overridable callback for when a node is evicted */
+    virtual void node_was_evicted(node_type_t *node) { }
+
+public:
+
+    /** Constructor */
+    lru_cache_t(size_t max_size = 1024) : max_node_count(max_size), node_count(0), mouth(wcstring())
+    {
+        /* Hook up the mouth to itself: a one node circularly linked list! */
+        mouth.prev = mouth.next = &mouth;
+    }
+
+    /** Note that we do not evict nodes in our destructor (even though they typically need to be deleted by their creator). */
+    virtual ~lru_cache_t() { }
+
+
+    /** Returns the node for a given key, or NULL */
+    node_type_t *get_node(const wcstring &key)
+    {
+        node_type_t *result = NULL;
+
+        /* Construct a fake node as our key */
+        lru_node_t node_key(key);
+
+        /* Look for it in the set */
+        node_set_t::iterator iter = node_set.find(&node_key);
+
+        /* If we found a node, promote and return it */
+        if (iter != node_set.end())
+        {
+            result = static_cast<node_type_t*>(*iter);
+            promote_node(result);
+        }
+        return result;
+    }
+
+    /** Evicts the node for a given key, returning true if a node was evicted. */
+    bool evict_node(const wcstring &key)
+    {
+        /* Construct a fake node as our key */
+        lru_node_t node_key(key);
+
+        /* Look for it in the set */
+        node_set_t::iterator iter = node_set.find(&node_key);
+        if (iter == node_set.end())
+            return false;
+
+        /* Evict the given node */
+        evict_node(static_cast<node_type_t*>(*iter));
+        return true;
+    }
+
+    /** Adds a node under the given key. Returns true if the node was added, false if the node was not because a node with that key is already in the set. */
+    bool add_node(node_type_t *node)
+    {
+        /* Add our node without eviction */
+        if (! this->add_node_without_eviction(node))
+            return false;
+
+        /* Evict */
+        while (node_count > max_node_count)
+            evict_last_node();
+
+        /* Success */
+        return true;
+    }
+
+    /** Adds a node under the given key without triggering eviction. Returns true if the node was added, false if the node was not because a node with that key is already in the set. */
+    bool add_node_without_eviction(node_type_t *node)
+    {
+        assert(node != NULL && node != &mouth);
+
+        /* Try inserting; return false if it was already in the set */
+        if (! node_set.insert(node).second)
+            return false;
+
+        /* Add the node after the mouth */
+        node->next = mouth.next;
+        node->next->prev = node;
+        node->prev = &mouth;
+        mouth.next = node;
+
+        /* Update the count. This may push us over the maximum node count. */
+        node_count++;
+        
+        /* Success */
+        return true;
+    }
+
+    /** Counts nodes */
+    size_t size(void)
+    {
+        return node_count;
+    }
+
+    /** Evicts all nodes */
+    void evict_all_nodes(void)
+    {
+        while (node_count > 0)
+        {
+            evict_last_node();
+        }
+    }
+
+    /** Iterator for walking nodes, from least recently used to most */
+    class iterator
+    {
+        lru_node_t *node;
+    public:
+        iterator(lru_node_t *val) : node(val) { }
+        void operator++()
+        {
+            node = lru_cache_t::get_previous(node);
+        }
+        void operator++(int x)
+        {
+            node = lru_cache_t::get_previous(node);
+        }
+        bool operator==(const iterator &other)
+        {
+            return node == other.node;
+        }
+        bool operator!=(const iterator &other)
+        {
+            return !(*this == other);
+        }
+        node_type_t *operator*()
+        {
+            return static_cast<node_type_t *>(node);
+        }
+    };
+
+    iterator begin()
+    {
+        return iterator(mouth.prev);
+    }
+    iterator end()
+    {
+        return iterator(&mouth);
+    }
+};
+
+
+#endif
diff --git a/src/output.cpp b/src/output.cpp
new file mode 100644
index 00000000..8f97ce61
--- /dev/null
+++ b/src/output.cpp
@@ -0,0 +1,595 @@
+/** \file output.c
+ Generic output functions
+ */
+
+#include "config.h"
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+
+#if HAVE_NCURSES_H
+#include <ncurses.h>
+#elif HAVE_NCURSES_CURSES_H
+#include <ncurses/curses.h>
+#else
+#include <curses.h>
+#endif
+
+#if HAVE_TERM_H
+#include <term.h>
+#elif HAVE_NCURSES_TERM_H
+#include <ncurses/term.h>
+#endif
+
+#include <wchar.h>
+#include <limits.h>
+#include <string>
+
+#include "fallback.h"
+#include "wutil.h" // IWYU pragma: keep - needed for wgettext
+#include "common.h"
+#include "output.h"
+
+/**
+ Number of color names in the col array
+ */
+#define FISH_COLORS (sizeof(col)/sizeof(wchar_t *))
+
+static int writeb_internal(char c);
+
+
+/**
+ The function used for output
+ */
+
+static int (*out)(char c) = &writeb_internal;
+
+/**
+ Name of terminal
+ */
+static wcstring current_term;
+
+/* Whether term256 and term24bit are supported */
+static color_support_t color_support = 0;
+
+
+void output_set_writer(int (*writer)(char))
+{
+    CHECK(writer,);
+    out = writer;
+}
+
+int (*output_get_writer())(char)
+{
+    return out;
+}
+
+static bool term256_support_is_native(void)
+{
+    /* Return YES if we think the term256 support is "native" as opposed to forced. */
+    return max_colors >= 256;
+}
+
+color_support_t output_get_color_support(void)
+{
+    return color_support;
+}
+
+void output_set_color_support(color_support_t val)
+{
+    color_support = val;
+}
+
+unsigned char index_for_color(rgb_color_t c)
+{
+    if (c.is_named() || ! (output_get_color_support() & color_support_term256))
+    {
+        return c.to_name_index();
+    }
+    else
+    {
+        return c.to_term256_index();
+    }
+}
+
+
+static bool write_color_escape(char *todo, unsigned char idx, bool is_fg)
+{
+    bool result = false;
+    if (idx < 16 || term256_support_is_native())
+    {
+        /* Use tparm */
+        writembs(tparm(todo, idx));
+        result = true;
+    }
+    else
+    {
+        /* We are attempting to bypass the term here. Generate the ANSI escape sequence ourself. */
+        char stridx[128];
+        format_long_safe(stridx, idx);
+        char buff[128] = "\x1b[";
+        strcat(buff, is_fg ? "38;5;" : "48;5;");
+        strcat(buff, stridx);
+        strcat(buff, "m");
+
+        int (*writer)(char) = output_get_writer();
+        if (writer)
+        {
+            for (size_t i=0; buff[i]; i++)
+            {
+                writer(buff[i]);
+            }
+        }
+
+        result = true;
+    }
+    return result;
+}
+
+static bool write_foreground_color(unsigned char idx)
+{
+    if (set_a_foreground && set_a_foreground[0])
+    {
+        return write_color_escape(set_a_foreground, idx, true);
+    }
+    else if (set_foreground && set_foreground[0])
+    {
+        return write_color_escape(set_foreground, idx, true);
+    }
+    else
+    {
+        return false;
+    }
+}
+
+static bool write_background_color(unsigned char idx)
+{
+    if (set_a_background && set_a_background[0])
+    {
+        return write_color_escape(set_a_background, idx, false);
+    }
+    else if (set_background && set_background[0])
+    {
+        return write_color_escape(set_background, idx, false);
+    }
+    else
+    {
+        return false;
+    }
+}
+
+void write_color(rgb_color_t color, bool is_fg)
+{
+    bool supports_term24bit = !! (output_get_color_support() & color_support_term24bit);
+    if (! supports_term24bit || ! color.is_rgb())
+    {
+        /* Indexed or non-24 bit color */
+        unsigned char idx = index_for_color(color);
+        (is_fg ? write_foreground_color : write_background_color)(idx);
+    }
+    else
+    {
+        /* 24 bit! No tparm here, just ANSI escape sequences.
+           Foreground: ^[38;2;<r>;<g>;<b>m
+           Background: ^[48;2;<r>;<g>;<b>m
+        */
+        color24_t rgb = color.to_color24();
+        char buff[128];
+        snprintf(buff, sizeof buff, "\x1b[%u;2;%u;%u;%um", is_fg ? 38 : 48, rgb.rgb[0], rgb.rgb[1], rgb.rgb[2]);
+        int (*writer)(char) = output_get_writer();
+        if (writer)
+        {
+            for (size_t i=0; buff[i]; i++)
+            {
+                writer(buff[i]);
+            }
+        }
+    }
+}
+
+void set_color(rgb_color_t c, rgb_color_t c2)
+{
+
+#if 0
+    wcstring tmp = c.description();
+    wcstring tmp2 = c2.description();
+    printf("set_color %ls : %ls\n", tmp.c_str(), tmp2.c_str());
+#endif
+    ASSERT_IS_MAIN_THREAD();
+
+    const rgb_color_t normal = rgb_color_t::normal();
+    static rgb_color_t last_color = rgb_color_t::normal();
+    static rgb_color_t last_color2 = rgb_color_t::normal();
+    static int was_bold=0;
+    static int was_underline=0;
+    int bg_set=0, last_bg_set=0;
+
+    int is_bold = 0;
+    int is_underline = 0;
+
+    /*
+       Test if we have at least basic support for setting fonts, colors
+       and related bits - otherwise just give up...
+       */
+    if (!exit_attribute_mode)
+    {
+        return;
+    }
+
+
+    is_bold |= c.is_bold();
+    is_bold |= c2.is_bold();
+
+    is_underline |= c.is_underline();
+    is_underline |= c2.is_underline();
+
+    if (c.is_reset() || c2.is_reset())
+    {
+        c = c2 = normal;
+        was_bold=0;
+        was_underline=0;
+        /*
+         If we exit attibute mode, we must first set a color, or
+         previously coloured text might lose it's
+         color. Terminals are weird...
+         */
+        write_foreground_color(0);
+        writembs(exit_attribute_mode);
+        return;
+    }
+
+    if (was_bold && !is_bold)
+    {
+        /*
+             Only way to exit bold mode is a reset of all attributes.
+             */
+        writembs(exit_attribute_mode);
+        last_color = normal;
+        last_color2 = normal;
+        was_bold=0;
+        was_underline=0;
+    }
+
+    if (! last_color2.is_normal() &&
+            ! last_color2.is_reset() &&
+            ! last_color2.is_ignore())
+    {
+        /*
+             Background was set
+             */
+        last_bg_set=1;
+    }
+
+    if (! c2.is_normal() &&
+            ! c2.is_ignore())
+    {
+        /*
+             Background is set
+             */
+        bg_set=1;
+        if (c==c2)
+            c = (c2==rgb_color_t::white())?rgb_color_t::black():rgb_color_t::white();
+    }
+
+    if ((enter_bold_mode != 0) && (strlen(enter_bold_mode) > 0))
+    {
+        if (bg_set && !last_bg_set)
+        {
+            /*
+                   Background color changed and is set, so we enter bold
+                   mode to make reading easier. This means bold mode is
+                   _always_ on when the background color is set.
+                   */
+            writembs(enter_bold_mode);
+        }
+        if (!bg_set && last_bg_set)
+        {
+            /*
+                   Background color changed and is no longer set, so we
+                   exit bold mode
+                   */
+            writembs(exit_attribute_mode);
+            was_bold=0;
+            was_underline=0;
+            /*
+                   We don't know if exit_attribute_mode resets colors, so
+                   we set it to something known.
+                   */
+            if (write_foreground_color(0))
+            {
+                last_color=rgb_color_t::black();
+            }
+        }
+    }
+
+    if (last_color != c)
+    {
+        if (c.is_normal())
+        {
+            write_foreground_color(0);
+            writembs(exit_attribute_mode);
+
+            last_color2 = rgb_color_t::normal();
+            was_bold=0;
+            was_underline=0;
+        }
+        else if (! c.is_special())
+        {
+            write_color(c, true /* foreground */);
+        }
+    }
+
+    last_color = c;
+
+    if (last_color2 != c2)
+    {
+        if (c2.is_normal())
+        {
+            write_background_color(0);
+
+            writembs(exit_attribute_mode);
+            if (! last_color.is_normal())
+            {
+                write_color(last_color, true /* foreground */);
+            }
+
+
+            was_bold=0;
+            was_underline=0;
+            last_color2 = c2;
+        }
+        else if (! c2.is_special())
+        {
+            write_color(c2, false /* not foreground */);
+            last_color2 = c2;
+        }
+    }
+
+    /*
+       Lastly, we set bold mode and underline mode correctly
+       */
+    if ((enter_bold_mode != 0) && (strlen(enter_bold_mode) > 0) && !bg_set)
+    {
+        if (is_bold && !was_bold)
+        {
+            if (enter_bold_mode)
+            {
+                writembs(tparm(enter_bold_mode));
+            }
+        }
+        was_bold = is_bold;
+    }
+
+    if (was_underline && !is_underline)
+    {
+        writembs(exit_underline_mode);
+    }
+
+    if (!was_underline && is_underline)
+    {
+        writembs(enter_underline_mode);
+    }
+    was_underline = is_underline;
+
+}
+
+/**
+ Default output method, simply calls write() on stdout
+ */
+static int writeb_internal(char c)
+{
+    write_loop(1, &c, 1);
+    return 0;
+}
+
+int writeb(tputs_arg_t b)
+{
+    out(b);
+    return 0;
+}
+
+int writech(wint_t ch)
+{
+    mbstate_t state;
+    size_t i;
+    char buff[MB_LEN_MAX+1];
+    size_t bytes;
+
+    if ((ch >= ENCODE_DIRECT_BASE) &&
+            (ch < ENCODE_DIRECT_BASE+256))
+    {
+        buff[0] = ch - ENCODE_DIRECT_BASE;
+        bytes=1;
+    }
+    else
+    {
+        memset(&state, 0, sizeof(state));
+        bytes= wcrtomb(buff, ch, &state);
+
+        switch (bytes)
+        {
+            case (size_t)(-1):
+            {
+                return 1;
+            }
+        }
+    }
+
+    for (i=0; i<bytes; i++)
+    {
+        out(buff[i]);
+    }
+    return 0;
+}
+
+void writestr(const wchar_t *str)
+{
+    char *pos;
+
+    CHECK(str,);
+
+    //  while( *str )
+    //    writech( *str++ );
+
+    /*
+       Check amount of needed space
+       */
+    size_t len = wcstombs(0, str, 0);
+
+    if (len == (size_t)-1)
+    {
+        debug(1, L"Tried to print invalid wide character string");
+        return;
+    }
+
+    len++;
+
+    /*
+       Convert
+       */
+    char *buffer, static_buffer[256];
+    if (len <= sizeof static_buffer)
+        buffer = static_buffer;
+    else
+        buffer = new char[len];
+
+    wcstombs(buffer,
+             str,
+             len);
+
+    /*
+       Write
+       */
+    for (pos = buffer; *pos; pos++)
+    {
+        out(*pos);
+    }
+
+    if (buffer != static_buffer)
+        delete[] buffer;
+}
+
+rgb_color_t best_color(const std::vector<rgb_color_t> &candidates, color_support_t support)
+{
+    if (candidates.empty())
+    {
+        return rgb_color_t::none();
+    }
+    
+    rgb_color_t first_rgb = rgb_color_t::none(), first_named = rgb_color_t::none();
+    for (size_t i=0; i < candidates.size(); i++)
+    {
+        const rgb_color_t &color = candidates.at(i);
+        if (first_rgb.is_none() && color.is_rgb())
+        {
+            first_rgb = color;
+        }
+        if (first_named.is_none() && color.is_named())
+        {
+            first_named = color;
+        }
+    }
+    // If we have both RGB and named colors, then prefer rgb if term256 is supported
+    rgb_color_t result = rgb_color_t::none();
+    bool has_term256 = !! (support & color_support_term256);
+    if ((!first_rgb.is_none() && has_term256) || first_named.is_none())
+    {
+        result = first_rgb;
+    }
+    else
+    {
+        result = first_named;
+    }
+    if (result.is_none())
+    {
+        result = candidates.at(0);
+    }
+    return result;
+}
+
+/* This code should be refactored to enable sharing with builtin_set_color */
+rgb_color_t parse_color(const wcstring &val, bool is_background)
+{
+    int is_bold=0;
+    int is_underline=0;
+
+    std::vector<rgb_color_t> candidates;
+
+    wcstring_list_t el;
+    tokenize_variable_array(val, el);
+
+    for (size_t j=0; j < el.size(); j++)
+    {
+        const wcstring &next = el.at(j);
+        wcstring color_name;
+        if (is_background)
+        {
+            // look for something like "--background=red"
+            const wcstring prefix = L"--background=";
+            if (string_prefixes_string(prefix, next))
+            {
+                color_name = wcstring(next, prefix.size());
+            }
+        }
+        else
+        {
+            if (next == L"--bold" || next == L"-o")
+                is_bold = true;
+            else if (next == L"--underline" || next == L"-u")
+                is_underline = true;
+            else
+                color_name = next;
+        }
+
+        if (! color_name.empty())
+        {
+            rgb_color_t color = rgb_color_t(color_name);
+            if (! color.is_none())
+            {
+                candidates.push_back(color);
+            }
+        }
+    }
+    rgb_color_t result = best_color(candidates, output_get_color_support());
+    
+    if (result.is_none())
+        result = rgb_color_t::normal();
+
+    result.set_bold(is_bold);
+    result.set_underline(is_underline);
+
+#if 0
+    wcstring desc = result.description();
+    printf("Parsed %ls from %ls (%s)\n", desc.c_str(), val.c_str(), is_background ? "background" : "foreground");
+#endif
+
+    return result;
+}
+
+void output_set_term(const wcstring &term)
+{
+    current_term.assign(term);
+}
+
+const wchar_t *output_get_term()
+{
+    return current_term.empty() ? L"<unknown>" : current_term.c_str();
+}
+
+void writembs_check(char *mbs, const char *mbs_name, const char *file, long line)
+{
+    if (mbs != NULL)
+    {
+        tputs(mbs, 1, &writeb);
+    }
+    else
+    {
+        debug( 0, _(L"Tried to use terminfo string %s on line %ld of %s, which is undefined in terminal of type \"%ls\". Please report this error to %s"),
+              mbs_name,
+              line,
+              file,
+              output_get_term(),
+              PACKAGE_BUGREPORT);
+    }
+}
diff --git a/src/output.h b/src/output.h
new file mode 100644
index 00000000..a952d8a4
--- /dev/null
+++ b/src/output.h
@@ -0,0 +1,144 @@
+/** \file output.h
+  Generic output functions
+*/
+/**
+   Constants for various character classifications. Each character of a command string can be classified as one of the following types.
+*/
+
+#ifndef FISH_OUTPUT_H
+#define FISH_OUTPUT_H
+
+#include <stddef.h>
+#include <vector>
+#include "common.h"
+#include "fallback.h"
+#include "color.h"
+
+/**
+   Constants for various colors as used by the set_color function.
+*/
+enum
+{
+    FISH_COLOR_BLACK,
+    FISH_COLOR_RED,
+    FISH_COLOR_GREEN,
+    FISH_COLOR_YELLOW,
+    FISH_COLOR_BLUE,
+    FISH_COLOR_MAGENTA,
+    FISH_COLOR_CYAN,
+    FISH_COLOR_WHITE,
+    /** The default fg color of the terminal */
+    FISH_COLOR_NORMAL,
+    FISH_COLOR_IGNORE,
+    FISH_COLOR_RESET
+}
+;
+
+/**
+   The value to send to set_color to tell it to use a bold font
+*/
+#define FISH_COLOR_BOLD 0x80
+/**
+   The value to send to set_color to tell it to underline the text
+*/
+#define FISH_COLOR_UNDERLINE 0x100
+
+/**
+   Sets the fg and bg color. May be called as often as you like, since
+   if the new color is the same as the previous, nothing will be
+   written. Negative values for set_color will also be ignored. Since
+   the terminfo string this function emits can potentially cause the
+   screen to flicker, the function takes care to write as little as
+   possible.
+
+   Possible values for color are any form the FISH_COLOR_* enum,
+   FISH_COLOR_IGNORE and FISH_COLOR_RESET. FISH_COLOR_IGNORE will
+   leave the color unchanged, and FISH_COLOR_RESET will perform an
+   exit_attribute_mode, even if set_color thinks it is already in
+   FISH_COLOR_NORMAL mode.
+
+   In order to set the color to normal, three terminfo strings may
+   have to be written.
+
+   - First a string to set the color, such as set_a_foreground. This
+   is needed because otherwise the previous strings colors might be
+   removed as well.
+
+   - After that we write the exit_attribute_mode string to reset all
+   color attributes.
+
+   - Lastly we may need to write set_a_background or set_a_foreground
+   to set the other half of the color pair to what it should be.
+
+   \param c Foreground color.
+   \param c2 Background color.
+*/
+
+
+void set_color(rgb_color_t c, rgb_color_t c2);
+
+/**
+   Write specified multibyte string
+ */
+void writembs_check(char *mbs, const char *mbs_name, const char *file, long line);
+#define writembs(mbs) writembs_check((mbs), #mbs, __FILE__, __LINE__)
+
+/**
+   Write a wide character using the output method specified using output_set_writer().
+*/
+int writech(wint_t ch);
+
+/**
+   Write a wide character string to FD 1.
+*/
+void writestr(const wchar_t *str);
+
+/**
+   Return the internal color code representing the specified color
+*/
+rgb_color_t parse_color(const wcstring &val, bool is_background);
+
+/**
+   This is for writing process notification messages. Has to write to
+   stdout, so clr_eol and such functions will work correctly. Not an
+   issue since this function is only used in interactive mode anyway.
+*/
+int writeb(tputs_arg_t b);
+
+/**
+   Set the function used for writing in move_cursor, writespace and
+   set_color and all other output functions in this library. By
+   default, the write call is used to give completely unbuffered
+   output to stdout.
+*/
+void output_set_writer(int (*writer)(char));
+
+/**
+   Return the current output writer
+ */
+int (*output_get_writer())(char) ;
+
+/** Set the terminal name */
+void output_set_term(const wcstring &term);
+
+/** Return the terminal name */
+const wchar_t *output_get_term();
+
+/** Sets what colors are supported */
+enum
+{
+    color_support_term256 = 1 << 0,
+    color_support_term24bit = 1 << 1
+};
+typedef unsigned int color_support_t;
+color_support_t output_get_color_support();
+void output_set_color_support(color_support_t support);
+
+/** Given a list of rgb_color_t, pick the "best" one, as determined by the color support. Returns rgb_color_t::none() if empty */
+rgb_color_t best_color(const std::vector<rgb_color_t> &colors, color_support_t support);
+
+/* Exported for builtin_set_color's usage only */
+void write_color(rgb_color_t color, bool is_fg);
+unsigned char index_for_color(rgb_color_t c);
+
+#endif
diff --git a/src/pager.cpp b/src/pager.cpp
new file mode 100644
index 00000000..a5cf0ecc
--- /dev/null
+++ b/src/pager.cpp
@@ -0,0 +1,998 @@
+#include "config.h" // IWYU pragma: keep
+
+#include <assert.h>
+#include <wchar.h>
+#include <wctype.h>
+#include <vector>
+#include <map>
+#include "util.h"
+#include "wutil.h" // IWYU pragma: keep - needed for wgettext
+#include "pager.h"
+#include "highlight.h"
+
+typedef pager_t::comp_t comp_t;
+typedef std::vector<completion_t> completion_list_t;
+typedef std::vector<comp_t> comp_info_list_t;
+
+/** The minimum width (in characters) the terminal must to show completions at all */
+#define PAGER_MIN_WIDTH 16
+
+/** The maximum number of columns of completion to attempt to fit onto the screen */
+#define PAGER_MAX_COLS 6
+
+/** Width of the search field */
+#define PAGER_SEARCH_FIELD_WIDTH 12
+
+/** Text we use for the search field */
+#define SEARCH_FIELD_PROMPT _(L"search: ")
+
+/* Returns numer / denom, rounding up. As a "courtesy" 0/0 is 0. */
+static size_t divide_round_up(size_t numer, size_t denom)
+{
+    if (numer == 0)
+        return 0;
+
+    assert(denom > 0);
+    bool has_rem = (numer % denom) > 0;
+    return numer / denom + (has_rem ? 1 : 0);
+}
+
+/**
+   This function calculates the minimum width for each completion
+   entry in the specified array_list. This width depends on the
+   terminal size, so this function should be called when the terminal
+   changes size.
+*/
+void pager_t::recalc_min_widths(comp_info_list_t * lst) const
+{
+    for (size_t i=0; i<lst->size(); i++)
+    {
+        comp_t *c = &lst->at(i);
+
+        c->min_width = mini(c->desc_width, maxi(0, available_term_width/3 - 2)) +
+                       mini(c->desc_width, maxi(0, available_term_width/5 - 4)) +4;
+    }
+
+}
+
+/**
+   Print the specified string, but use at most the specified amount of
+   space. If the whole string can't be fitted, ellipsize it.
+
+   \param str the string to print
+   \param color the color to apply to every printed character
+   \param max the maximum space that may be used for printing
+   \param has_more if this flag is true, this is not the entire string, and the string should be ellisiszed even if the string fits but takes up the whole space.
+*/
+
+static int print_max(const wcstring &str, highlight_spec_t color, int max, bool has_more, line_t *line)
+{
+    int written = 0;
+    for (size_t i=0; i < str.size(); i++)
+    {
+        wchar_t c = str.at(i);
+
+        if (written + wcwidth(c) > max)
+            break;
+        if ((written + wcwidth(c) == max) && (has_more || i + 1 < str.size()))
+        {
+            line->append(ellipsis_char, color);
+            written += wcwidth(ellipsis_char);
+            break;
+        }
+
+        line->append(c, color);
+        written += wcwidth(c);
+    }
+    return written;
+}
+
+
+/**
+   Print the specified item using at the specified amount of space
+*/
+line_t pager_t::completion_print_item(const wcstring &prefix, const comp_t *c, size_t row, size_t column, int width, bool secondary, bool selected, page_rendering_t *rendering) const
+{
+    int comp_width=0, desc_width=0;
+    int written=0;
+
+    line_t line_data;
+
+    if (c->pref_width <= width)
+    {
+        /*
+          The entry fits, we give it as much space as it wants
+        */
+        comp_width = c->comp_width;
+        desc_width = c->desc_width;
+    }
+    else
+    {
+        /*
+          The completion and description won't fit on the
+          allocated space. Give a maximum of 2/3 of the
+          space to the completion, and whatever is left to
+          the description.
+        */
+        int desc_all = c->desc_width?c->desc_width+4:0;
+
+        comp_width = maxi(mini(c->comp_width, 2*(width-4)/3), width - desc_all);
+        if (c->desc_width)
+            desc_width = width-comp_width-4;
+
+    }
+
+    int bg_color = secondary ? highlight_spec_pager_secondary : highlight_spec_normal;
+    if (selected)
+    {
+        bg_color = highlight_spec_search_match;
+    }
+
+    for (size_t i=0; i<c->comp.size(); i++)
+    {
+        const wcstring &comp = c->comp.at(i);
+
+        if (i != 0)
+            written += print_max(PAGER_SPACER_STRING, highlight_spec_normal, comp_width - written, true /* has_more */, &line_data);
+
+        int packed_color = highlight_spec_pager_prefix | highlight_make_background(bg_color);
+        written += print_max(prefix, packed_color, comp_width - written, ! comp.empty(), &line_data);
+
+        packed_color = highlight_spec_pager_completion | highlight_make_background(bg_color);
+        written += print_max(comp, packed_color, comp_width - written, i + 1 < c->comp.size(), &line_data);
+    }
+
+    if (desc_width)
+    {
+        int packed_color = highlight_spec_pager_description | highlight_make_background(bg_color);
+        while (written < (width-desc_width-2)) //the 2 here refers to the parenthesis below
+        {
+            written += print_max(L" ", packed_color, 1, false, &line_data);
+        }
+        written += print_max(L"(", packed_color, 1, false, &line_data);
+        written += print_max(c->desc, packed_color, desc_width, false, &line_data);
+        written += print_max(L")", packed_color, 1, false, &line_data);
+    }
+    else
+    {
+        while (written < width)
+        {
+            written += print_max(L" ", 0, 1, false, &line_data);
+        }
+    }
+
+    return line_data;
+}
+
+/**
+   Print the specified part of the completion list, using the
+   specified column offsets and quoting style.
+
+   \param l The list of completions to print
+   \param cols number of columns to print in
+   \param width An array specifying the width of each column
+   \param row_start The first row to print
+   \param row_stop the row after the last row to print
+   \param prefix The string to print before each completion
+*/
+
+void pager_t::completion_print(size_t cols, int *width_per_column, size_t row_start, size_t row_stop, const wcstring &prefix, const comp_info_list_t &lst, page_rendering_t *rendering) const
+{
+    /* Teach the rendering about the rows it printed */
+    assert(row_start >= 0);
+    assert(row_stop >= row_start);
+    rendering->row_start = row_start;
+    rendering->row_end = row_stop;
+
+    size_t rows = (lst.size()-1)/cols+1;
+
+    size_t effective_selected_idx = this->visual_selected_completion_index(rows, cols);
+
+    for (size_t row = row_start; row < row_stop; row++)
+    {
+        for (size_t col = 0; col < cols; col++)
+        {
+            int is_last = (col==(cols-1));
+
+            if (lst.size() <= col * rows + row)
+                continue;
+
+            size_t idx = col * rows + row;
+            const comp_t *el = &lst.at(idx);
+            bool is_selected = (idx == effective_selected_idx);
+
+            /* Print this completion on its own "line" */
+            line_t line = completion_print_item(prefix, el, row, col, width_per_column[col] - (is_last ? 0 : PAGER_SPACER_STRING_WIDTH), row%2, is_selected, rendering);
+
+            /* If there's more to come, append two spaces */
+            if (col + 1 < cols)
+            {
+                line.append(PAGER_SPACER_STRING, 0);
+            }
+
+            /* Append this to the real line */
+            rendering->screen_data.create_line(row - row_start).append_line(line);
+        }
+    }
+}
+
+
+/* Trim leading and trailing whitespace, and compress other whitespace runs into a single space. */
+static void mangle_1_completion_description(wcstring *str)
+{
+    size_t leading = 0, trailing = 0, len = str->size();
+
+    // Skip leading spaces
+    for (; leading < len; leading++)
+    {
+        if (! iswspace(str->at(leading)))
+            break;
+    }
+
+    // Compress runs of spaces to a single space
+    bool was_space = false;
+    for (; leading < len; leading++)
+    {
+        wchar_t wc = str->at(leading);
+        bool is_space = iswspace(wc);
+        if (! is_space)
+        {
+            // normal character
+            str->at(trailing++) = wc;
+        }
+        else if (! was_space)
+        {
+            // initial space in a run
+            str->at(trailing++) = L' ';
+        }
+        else
+        {
+            // non-initial space in a run, do nothing
+        }
+        was_space = is_space;
+    }
+
+    // leading is now at len, trailing is the new length of the string
+    // Delete trailing spaces
+    while (trailing > 0 && iswspace(str->at(trailing - 1)))
+    {
+        trailing--;
+    }
+
+    str->resize(trailing);
+}
+
+static void join_completions(comp_info_list_t *comps)
+{
+    // A map from description to index in the completion list of the element with that description
+    // The indexes are stored +1
+    std::map<wcstring, size_t> desc_table;
+
+    // note that we mutate the completion list as we go, so the size changes
+    for (size_t i=0; i < comps->size(); i++)
+    {
+        const comp_t &new_comp = comps->at(i);
+        const wcstring &desc = new_comp.desc;
+        if (desc.empty())
+            continue;
+
+        // See if it's in the table
+        size_t prev_idx_plus_one = desc_table[desc];
+        if (prev_idx_plus_one == 0)
+        {
+            // We're the first with this description
+            desc_table[desc] = i+1;
+        }
+        else
+        {
+            // There's a prior completion with this description. Append the new ones to it.
+            comp_t *prior_comp = &comps->at(prev_idx_plus_one - 1);
+            prior_comp->comp.insert(prior_comp->comp.end(), new_comp.comp.begin(), new_comp.comp.end());
+
+            // Erase the element at this index, and decrement the index to reflect that fact
+            comps->erase(comps->begin() + i);
+            i -= 1;
+        }
+    }
+}
+
+/** Generate a list of comp_t structures from a list of completions */
+static comp_info_list_t process_completions_into_infos(const completion_list_t &lst, const wcstring &prefix)
+{
+    const size_t lst_size = lst.size();
+
+    // Make the list of the correct size up-front
+    comp_info_list_t result(lst_size);
+    for (size_t i=0; i<lst_size; i++)
+    {
+        const completion_t &comp = lst.at(i);
+        comp_t *comp_info = &result.at(i);
+
+        // Append the single completion string. We may later merge these into multiple.
+        comp_info->comp.push_back(escape_string(comp.completion, ESCAPE_ALL | ESCAPE_NO_QUOTED));
+
+        // Append the mangled description
+        comp_info->desc = comp.description;
+        mangle_1_completion_description(&comp_info->desc);
+
+        // Set the representative completion
+        comp_info->representative = comp;
+    }
+    return result;
+}
+
+void pager_t::measure_completion_infos(comp_info_list_t *infos, const wcstring &prefix) const
+{
+    size_t prefix_len = fish_wcswidth(prefix.c_str());
+    for (size_t i=0; i < infos->size(); i++)
+    {
+        comp_t *comp = &infos->at(i);
+
+        // Compute comp_width
+        const wcstring_list_t &comp_strings = comp->comp;
+        for (size_t j=0; j < comp_strings.size(); j++)
+        {
+            // If there's more than one, append the length of ', '
+            if (j >= 1)
+                comp->comp_width += 2;
+
+            comp->comp_width += prefix_len + fish_wcswidth(comp_strings.at(j).c_str());
+        }
+
+        // Compute desc_width
+        comp->desc_width = fish_wcswidth(comp->desc.c_str());
+
+        // Compute preferred width
+        comp->pref_width = comp->comp_width + comp->desc_width + (comp->desc_width?4:0);
+    }
+
+    recalc_min_widths(infos);
+}
+
+/* Indicates if the given completion info passes any filtering we have */
+bool pager_t::completion_info_passes_filter(const comp_t &info) const
+{
+    /* If we have no filter, everything passes */
+    if (! search_field_shown || this->search_field_line.empty())
+        return true;
+
+    const wcstring &needle = this->search_field_line.text;
+
+    /* We do substring matching */
+    const fuzzy_match_type_t limit = fuzzy_match_substring;
+
+    /* Match against the description */
+    if (string_fuzzy_match_string(needle, info.desc, limit).type != fuzzy_match_none)
+    {
+        return true;
+    }
+
+    /* Match against the completion strings */
+    for (size_t i=0; i < info.comp.size(); i++)
+    {
+        if (string_fuzzy_match_string(needle, prefix + info.comp.at(i), limit).type != fuzzy_match_none)
+        {
+            return true;
+        }
+    }
+
+    /* No match */
+    return false;
+}
+
+/* Update completion_infos from unfiltered_completion_infos, to reflect the filter */
+void pager_t::refilter_completions()
+{
+    this->completion_infos.clear();
+    for (size_t i=0; i < this->unfiltered_completion_infos.size(); i++)
+    {
+        const comp_t &info = this->unfiltered_completion_infos.at(i);
+        if (this->completion_info_passes_filter(info))
+        {
+            this->completion_infos.push_back(info);
+        }
+    }
+}
+
+void pager_t::set_completions(const completion_list_t &raw_completions)
+{
+    // Get completion infos out of it
+    unfiltered_completion_infos = process_completions_into_infos(raw_completions, prefix);
+
+    // Maybe join them
+    if (prefix == L"-")
+        join_completions(&unfiltered_completion_infos);
+
+    // Compute their various widths
+    measure_completion_infos(&unfiltered_completion_infos, prefix);
+
+    // Refilter them
+    this->refilter_completions();
+}
+
+void pager_t::set_prefix(const wcstring &pref)
+{
+    prefix = pref;
+}
+
+void pager_t::set_term_size(int w, int h)
+{
+    assert(w > 0);
+    assert(h > 0);
+    available_term_width = w;
+    available_term_height = h;
+    recalc_min_widths(&completion_infos);
+}
+
+/**
+   Try to print the list of completions l with the prefix prefix using
+   cols as the number of columns. Return true if the completion list was
+   printed, false if the terminal is to narrow for the specified number of
+   columns. Always succeeds if cols is 1.
+*/
+
+bool pager_t::completion_try_print(size_t cols, const wcstring &prefix, const comp_info_list_t &lst, page_rendering_t *rendering, size_t suggested_start_row) const
+{
+    /*
+      The calculated preferred width of each column
+    */
+    int pref_width[PAGER_MAX_COLS] = {0};
+    /*
+      The calculated minimum width of each column
+    */
+    int min_width[PAGER_MAX_COLS] = {0};
+    /*
+      If the list can be printed with this width, width will contain the width of each column
+    */
+    int *width=pref_width;
+
+    /* Set to one if the list should be printed at this width */
+    bool print = false;
+
+    /* Compute the effective term width and term height, accounting for disclosure */
+    int term_width = this->available_term_width;
+    int term_height = this->available_term_height - 1 - (search_field_shown ? 1 : 0); // we always subtract 1 to make room for a comment row
+    if (! this->fully_disclosed)
+    {
+        term_height = mini(term_height, PAGER_UNDISCLOSED_MAX_ROWS);
+    }
+
+    size_t row_count = divide_round_up(lst.size(), cols);
+
+    /* We have more to disclose if we are not fully disclosed and there's more rows than we have in our term height */
+    if (! this->fully_disclosed && row_count > term_height)
+    {
+        rendering->remaining_to_disclose = row_count - term_height;
+    }
+    else
+    {
+        rendering->remaining_to_disclose = 0;
+    }
+    
+    /* If we have only one row remaining to disclose, then squelch the comment row. This prevents us from consuming a line to show "...and 1 more row" */
+    if (! this->fully_disclosed && rendering->remaining_to_disclose == 1)
+    {
+        term_height += 1;
+        rendering->remaining_to_disclose = 0;
+    }
+
+    int pref_tot_width=0;
+    int min_tot_width = 0;
+
+    /* Skip completions on tiny terminals */
+    if (term_width < PAGER_MIN_WIDTH)
+        return true;
+
+    /* Calculate how wide the list would be */
+    for (long col = 0; col < cols; col++)
+    {
+        for (long row = 0; row<row_count; row++)
+        {
+            int pref,min;
+            const comp_t *c;
+            if (lst.size() <= col*row_count + row)
+                continue;
+
+            c = &lst.at(col*row_count + row);
+            pref = c->pref_width;
+            min = c->min_width;
+
+            if (col != cols-1)
+            {
+                pref += 2;
+                min += 2;
+            }
+            min_width[col] = maxi(min_width[col],
+                                  min);
+            pref_width[col] = maxi(pref_width[col],
+                                   pref);
+        }
+        min_tot_width += min_width[col];
+        pref_tot_width += pref_width[col];
+    }
+    /*
+      Force fit if one column
+    */
+    if (cols == 1)
+    {
+        if (pref_tot_width > term_width)
+        {
+            pref_width[0] = term_width;
+        }
+        width = pref_width;
+        print = true;
+    }
+    else if (pref_tot_width <= term_width)
+    {
+        /* Terminal is wide enough. Print the list! */
+        width = pref_width;
+        print = true;
+    }
+
+    if (print)
+    {
+        /* Determine the starting and stop row */
+        size_t start_row = 0, stop_row = 0;
+        if (row_count <= term_height)
+        {
+            /* Easy, we can show everything */
+            start_row = 0;
+            stop_row = row_count;
+        }
+        else
+        {
+            /* We can only show part of the full list. Determine which part based on the suggested_start_row */
+            assert(row_count > term_height);
+            size_t last_starting_row = row_count - term_height;
+            start_row = mini(suggested_start_row, last_starting_row);
+            stop_row = start_row + term_height;
+            assert(start_row >= 0 && start_row <= last_starting_row);
+        }
+
+        assert(stop_row >= start_row);
+        assert(stop_row <= row_count);
+        assert(stop_row - start_row <= term_height);
+        completion_print(cols, width, start_row, stop_row, prefix, lst, rendering);
+
+        /* Ellipsis helper string. Either empty or containing the ellipsis char */
+        const wchar_t ellipsis_string[] = {ellipsis_char == L'\x2026' ? L'\x2026' : L'\0', L'\0'};
+
+        /* Add the progress line. It's a "more to disclose" line if necessary, or a row listing if it's scrollable; otherwise ignore it */
+        wcstring progress_text;
+        if (rendering->remaining_to_disclose == 1)
+        {
+            /* I don't expect this case to ever happen */
+            progress_text = format_string(_(L"%lsand 1 more row"), ellipsis_string);
+        }
+        else if (rendering->remaining_to_disclose > 1)
+        {
+            progress_text = format_string(_(L"%lsand %lu more rows"), ellipsis_string, (unsigned long)rendering->remaining_to_disclose);
+        }
+        else if (start_row > 0 || stop_row < row_count)
+        {
+            /* We have a scrollable interface. The +1 here is because we are zero indexed, but want to present things as 1-indexed. We do not add 1 to stop_row or row_count because these are the "past the last value" */
+            progress_text = format_string(_(L"rows %lu to %lu of %lu"), start_row + 1, stop_row, row_count);
+        }
+        else if (completion_infos.empty() && ! unfiltered_completion_infos.empty())
+        {
+            /* Everything is filtered */
+            progress_text = _(L"(no matches)");
+        }
+
+        if (! progress_text.empty())
+        {
+            line_t &line = rendering->screen_data.add_line();
+            print_max(progress_text, highlight_spec_pager_progress | highlight_make_background(highlight_spec_pager_progress), term_width, true /* has_more */, &line);
+        }
+
+        if (search_field_shown)
+        {
+            /* Add the search field */
+            wcstring search_field_text = search_field_line.text;
+            /* Append spaces to make it at least the required width */
+            if (search_field_text.size() < PAGER_SEARCH_FIELD_WIDTH)
+            {
+                search_field_text.append(PAGER_SEARCH_FIELD_WIDTH - search_field_text.size(), L' ');
+            }
+            line_t *search_field = &rendering->screen_data.insert_line_at_index(0);
+
+            /* We limit the width to term_width - 1 */
+            int search_field_written = print_max(SEARCH_FIELD_PROMPT, highlight_spec_normal, term_width - 1, false, search_field);
+            search_field_written += print_max(search_field_text, highlight_modifier_force_underline, term_width - search_field_written - 1, false, search_field);
+        }
+
+    }
+    return print;
+}
+
+
+page_rendering_t pager_t::render() const
+{
+
+    /**
+       Try to print the completions. Start by trying to print the
+       list in PAGER_MAX_COLS columns, if the completions won't
+       fit, reduce the number of columns by one. Printing a single
+       column never fails.
+    */
+    page_rendering_t rendering;
+    rendering.term_width = this->available_term_width;
+    rendering.term_height = this->available_term_height;
+    rendering.search_field_shown = this->search_field_shown;
+    rendering.search_field_line = this->search_field_line;
+
+    for (int cols = PAGER_MAX_COLS; cols > 0; cols--)
+    {
+        /* Initially empty rendering */
+        rendering.screen_data.resize(0);
+
+        /* Determine how many rows we would need if we had 'cols' columns. Then determine how many columns we want from that. For example, say we had 19 completions. We can fit them into 6 columns, 4 rows, with the last row containing only 1 entry. Or we can fit them into 5 columns, 4 rows, the last row containing 4 entries. Since fewer columns with the same number of rows is better, skip cases where we know we can do better. */
+        size_t min_rows_required_for_cols = divide_round_up(completion_infos.size(), cols);
+        size_t min_cols_required_for_rows = divide_round_up(completion_infos.size(), min_rows_required_for_cols);
+
+        assert(min_cols_required_for_rows <= cols);
+        if (cols > 1 && min_cols_required_for_rows < cols)
+        {
+            /* Next iteration will be better, so skip this one */
+            continue;
+        }
+
+        rendering.cols = (size_t)cols;
+        rendering.rows = min_rows_required_for_cols;
+        rendering.selected_completion_idx = this->visual_selected_completion_index(rendering.rows, rendering.cols);
+
+        if (completion_try_print(cols, prefix, completion_infos, &rendering, suggested_row_start))
+        {
+            break;
+        }
+    }
+    return rendering;
+}
+
+void pager_t::update_rendering(page_rendering_t *rendering) const
+{
+    if (rendering->term_width != this->available_term_width ||
+            rendering->term_height != this->available_term_height ||
+            rendering->selected_completion_idx != this->visual_selected_completion_index(rendering->rows, rendering->cols) ||
+            rendering->search_field_shown != this->search_field_shown ||
+            rendering->search_field_line.text != this->search_field_line.text ||
+            rendering->search_field_line.position != this->search_field_line.position ||
+            (rendering->remaining_to_disclose > 0 && this->fully_disclosed))
+    {
+        *rendering = this->render();
+    }
+}
+
+pager_t::pager_t() : available_term_width(0), available_term_height(0), selected_completion_idx(PAGER_SELECTION_NONE), suggested_row_start(0), fully_disclosed(false), search_field_shown(false)
+{
+}
+
+bool pager_t::empty() const
+{
+    return unfiltered_completion_infos.empty();
+}
+
+bool pager_t::select_next_completion_in_direction(selection_direction_t direction, const page_rendering_t &rendering)
+{
+    /* Must have something to select */
+    if (this->completion_infos.empty())
+    {
+        return false;
+    }
+
+    /* Handle the case of nothing selected yet */
+    if (selected_completion_idx == PAGER_SELECTION_NONE)
+    {
+        switch (direction)
+        {
+                /* These directions do something sane */
+            case direction_south:
+            case direction_page_south:
+            case direction_next:
+            case direction_prev:
+                if (direction == direction_prev)
+                {
+                    selected_completion_idx = completion_infos.size() - 1;
+                }
+                else
+                {
+                    selected_completion_idx = 0;
+                }
+                return true;
+
+                /* These do nothing */
+            case direction_north:
+            case direction_page_north:
+            case direction_east:
+            case direction_west:
+            case direction_deselect:
+            default:
+                return false;
+        }
+    }
+
+    /* Ok, we had something selected already. Select something different. */
+    size_t new_selected_completion_idx = selected_completion_idx;
+    if (! selection_direction_is_cardinal(direction))
+    {
+        /* Next, previous, or deselect, all easy */
+        if (direction == direction_deselect)
+        {
+            new_selected_completion_idx = PAGER_SELECTION_NONE;
+        }
+        else if (direction == direction_next)
+        {
+            new_selected_completion_idx = selected_completion_idx + 1;
+            if (new_selected_completion_idx >= completion_infos.size())
+            {
+                new_selected_completion_idx = 0;
+            }
+        }
+        else if (direction == direction_prev)
+        {
+            if (selected_completion_idx == 0)
+            {
+                new_selected_completion_idx = completion_infos.size() - 1;
+            }
+            else
+            {
+                new_selected_completion_idx = selected_completion_idx - 1;
+            }
+        }
+        else
+        {
+            assert(0 && "Unknown non-cardinal direction");
+        }
+    }
+    else
+    {
+        /* Cardinal directions. We have a completion index; we wish to compute its row and column. */
+        size_t current_row = this->get_selected_row(rendering);
+        size_t current_col = this->get_selected_column(rendering);
+        size_t page_height = maxi(rendering.term_height - 1, 1);
+
+        switch (direction)
+        {
+            case direction_page_north:
+            {
+                if (current_row > page_height)
+                   current_row = current_row - page_height;
+                else
+                   current_row = 0;
+                break;
+            }
+            case direction_north:
+            {
+                /* Go up a whole row. If we cycle, go to the previous column. */
+                if (current_row > 0)
+                {
+                    current_row--;
+                }
+                else
+                {
+                    current_row = rendering.rows - 1;
+                    if (current_col > 0)
+                        current_col--;
+                }
+                break;
+            }
+
+            case direction_page_south:
+            {
+                if (current_row + page_height < rendering.rows)
+                {
+                    current_row += page_height;
+                }
+                else
+                {
+                    current_row = rendering.rows - 1;
+                    if (current_col * rendering.rows + current_row >= completion_infos.size()) {
+                        current_row = (completion_infos.size() - 1) % rendering.rows;
+                    }
+                }
+                break;
+            }
+            case direction_south:
+            {
+                /* Go down, unless we are in the last row. Note that this means that we may set selected_completion_idx to an out-of-bounds value if the last row is incomplete; this is a feature (it allows "last column memory"). */
+                if (current_row + 1 < rendering.rows)
+                {
+                    current_row++;
+                }
+                else
+                {
+                    current_row = 0;
+                    if (current_col + 1 < rendering.cols)
+                        current_col++;
+
+                }
+                break;
+            }
+
+            case direction_east:
+            {
+                /* Go east, wrapping to the next row. There is no "row memory," so if we run off the end, wrap. */
+                if (current_col + 1 < rendering.cols && (current_col + 1) * rendering.rows + current_row < completion_infos.size())
+                {
+                    current_col++;
+                }
+                else
+                {
+                    current_col = 0;
+                    if (current_row + 1 < rendering.rows)
+                        current_row++;
+                }
+                break;
+            }
+
+            case direction_west:
+            {
+                /* Go west, wrapping to the previous row */
+                if (current_col > 0)
+                {
+                    current_col--;
+                }
+                else
+                {
+                    current_col = rendering.cols - 1;
+                    if (current_row > 0)
+                        current_row--;
+                }
+                break;
+            }
+
+            default:
+                assert(0 && "Unknown cardinal direction");
+                break;
+        }
+
+        /* Compute the new index based on the changed row */
+        new_selected_completion_idx = current_col * rendering.rows + current_row;
+    }
+
+    if (new_selected_completion_idx != selected_completion_idx)
+    {
+        selected_completion_idx = new_selected_completion_idx;
+
+        /* Update suggested_row_start to ensure the selection is visible. suggested_row_start * rendering.cols is the first suggested visible completion; add the visible completion count to that to get the last one */
+        size_t visible_row_count = rendering.row_end - rendering.row_start;
+
+        if (visible_row_count > 0 && selected_completion_idx != PAGER_SELECTION_NONE) //paranoia
+        {
+            size_t row_containing_selection = this->get_selected_row(rendering);
+
+            /* Ensure our suggested row start is not past the selected row */
+            if (suggested_row_start > row_containing_selection)
+            {
+                suggested_row_start = row_containing_selection;
+            }
+
+            /* Ensure our suggested row start is not too early before it */
+            if (suggested_row_start + visible_row_count <= row_containing_selection)
+            {
+                /* The user moved south past the bottom completion */
+                if (! fully_disclosed && rendering.remaining_to_disclose > 0)
+                {
+                    /* Perform disclosure */
+                    fully_disclosed = true;
+                }
+                else
+                {
+                    /* Scroll */
+                    suggested_row_start = row_containing_selection - visible_row_count + 1;
+
+                    /* Ensure fully_disclosed is set. I think we can hit this case if the user resizes the window - we don't want to drop back to the disclosed style */
+                    fully_disclosed = true;
+                }
+            }
+        }
+
+        return true;
+    }
+    else
+    {
+        return false;
+    }
+}
+
+size_t pager_t::visual_selected_completion_index(size_t rows, size_t cols) const
+{
+    /* No completions -> no selection */
+    if (completion_infos.empty() || rows == 0 || cols == 0)
+    {
+        return PAGER_SELECTION_NONE;
+    }
+
+    size_t result = selected_completion_idx;
+    if (result != PAGER_SELECTION_NONE)
+    {
+        /* If the selected completion is beyond the last selection, go left by columns until it's within it. This is how we implement "column memory." */
+        while (result >= completion_infos.size() && result >= rows)
+        {
+            result -= rows;
+        }
+
+        /* If we are still beyond the last selection, clamp it */
+        if (result >= completion_infos.size())
+            result = completion_infos.size() - 1;
+    }
+    assert(result == PAGER_SELECTION_NONE || result < completion_infos.size());
+    return result;
+}
+
+/* It's possible we have no visual selection but are still navigating the contents, e.g. every completion is filtered */
+bool pager_t::is_navigating_contents() const
+{
+    return selected_completion_idx != PAGER_SELECTION_NONE;
+}
+
+void pager_t::set_fully_disclosed(bool flag)
+{
+    fully_disclosed = flag;
+}
+
+const completion_t *pager_t::selected_completion(const page_rendering_t &rendering) const
+{
+    const completion_t * result = NULL;
+    size_t idx = visual_selected_completion_index(rendering.rows, rendering.cols);
+    if (idx != PAGER_SELECTION_NONE)
+    {
+        result = &completion_infos.at(idx).representative;
+    }
+    return result;
+}
+
+/* Get the selected row and column. Completions are rendered column first, i.e. we go south before we go west. So if we have N rows, and our selected index is N + 2, then our row is 2 (mod by N) and our column is 1 (divide by N) */
+size_t pager_t::get_selected_row(const page_rendering_t &rendering) const
+{
+    if (rendering.rows == 0)
+        return PAGER_SELECTION_NONE;
+
+    return selected_completion_idx == PAGER_SELECTION_NONE ? PAGER_SELECTION_NONE : selected_completion_idx % rendering.rows;
+}
+
+size_t pager_t::get_selected_column(const page_rendering_t &rendering) const
+{
+    if (rendering.rows == 0)
+        return PAGER_SELECTION_NONE;
+
+    return selected_completion_idx == PAGER_SELECTION_NONE ? PAGER_SELECTION_NONE : selected_completion_idx / rendering.rows;
+}
+
+void pager_t::clear()
+{
+    unfiltered_completion_infos.clear();
+    completion_infos.clear();
+    prefix.clear();
+    selected_completion_idx = PAGER_SELECTION_NONE;
+    fully_disclosed = false;
+    search_field_shown = false;
+    search_field_line.clear();
+}
+
+void pager_t::set_search_field_shown(bool flag)
+{
+    this->search_field_shown = flag;
+}
+
+bool pager_t::is_search_field_shown() const
+{
+    return this->search_field_shown;
+}
+
+size_t pager_t::cursor_position() const
+{
+    size_t result = wcslen(SEARCH_FIELD_PROMPT) + this->search_field_line.position;
+    /* Clamp it to the right edge */
+    if (available_term_width > 0 && result + 1 > available_term_width)
+    {
+        result = available_term_width - 1;
+    }
+    return result;
+}
+
+
+/* Constructor */
+page_rendering_t::page_rendering_t() : term_width(-1), term_height(-1), rows(0), cols(0), row_start(0), row_end(0), selected_completion_idx(-1), remaining_to_disclose(0), search_field_shown(false)
+{
+}
diff --git a/src/pager.h b/src/pager.h
new file mode 100644
index 00000000..61c37ce6
--- /dev/null
+++ b/src/pager.h
@@ -0,0 +1,172 @@
+/** \file pager.h
+  Pager support
+*/
+
+#include <stddef.h>
+#include <string>
+#include <vector>
+#include "common.h"
+#include "complete.h"
+#include "screen.h"
+#include "reader.h"
+
+#define PAGER_SELECTION_NONE ((size_t)(-1))
+
+/* Represents rendering from the pager */
+class page_rendering_t
+{
+public:
+    int term_width;
+    int term_height;
+    size_t rows;
+    size_t cols;
+    size_t row_start;
+    size_t row_end;
+    size_t selected_completion_idx;
+    screen_data_t screen_data;
+
+    size_t remaining_to_disclose;
+
+    bool search_field_shown;
+    editable_line_t search_field_line;
+
+    /* Returns a rendering with invalid data, useful to indicate "no rendering" */
+    page_rendering_t();
+};
+
+/* The space between adjacent completions */
+#define PAGER_SPACER_STRING L"  "
+#define PAGER_SPACER_STRING_WIDTH 2
+
+/* How many rows we will show in the "initial" pager */
+#define PAGER_UNDISCLOSED_MAX_ROWS 4
+
+typedef std::vector<completion_t> completion_list_t;
+page_rendering_t render_completions(const completion_list_t &raw_completions, const wcstring &prefix);
+
+class pager_t
+{
+    int available_term_width;
+    int available_term_height;
+
+    size_t selected_completion_idx;
+    size_t suggested_row_start;
+
+    /* Fully disclosed means that we show all completions */
+    bool fully_disclosed;
+
+    /* Whether we show the search field */
+    bool search_field_shown;
+
+    /* Returns the index of the completion that should draw selected, using the given number of columns */
+    size_t visual_selected_completion_index(size_t rows, size_t cols) const;
+
+    /** Data structure describing one or a group of related completions */
+public:
+    struct comp_t
+    {
+        /** The list of all completin strings this entry applies to */
+        wcstring_list_t comp;
+
+        /** The description */
+        wcstring desc;
+
+        /** The representative completion */
+        completion_t representative;
+
+        /** On-screen width of the completion string */
+        int comp_width;
+
+        /** On-screen width of the description information */
+        int desc_width;
+
+        /** Preferred total width */
+        int pref_width;
+
+        /** Minimum acceptable width */
+        int min_width;
+
+        comp_t() : comp(), desc(), representative(L""), comp_width(0), desc_width(0), pref_width(0), min_width(0)
+        {
+        }
+    };
+
+private:
+    typedef std::vector<comp_t> comp_info_list_t;
+
+    /* The filtered list of completion infos */
+    comp_info_list_t completion_infos;
+
+    /* The unfiltered list. Note there's a lot of duplication here. */
+    comp_info_list_t unfiltered_completion_infos;
+
+    wcstring prefix;
+
+    bool completion_try_print(size_t cols, const wcstring &prefix, const comp_info_list_t &lst, page_rendering_t *rendering, size_t suggested_start_row) const;
+
+    void recalc_min_widths(comp_info_list_t * lst) const;
+    void measure_completion_infos(std::vector<comp_t> *infos, const wcstring &prefix) const;
+
+    bool completion_info_passes_filter(const comp_t &info) const;
+
+    void completion_print(size_t cols, int *width_per_column, size_t row_start, size_t row_stop, const wcstring &prefix, const comp_info_list_t &lst, page_rendering_t *rendering) const;
+    line_t completion_print_item(const wcstring &prefix, const comp_t *c, size_t row, size_t column, int width, bool secondary, bool selected, page_rendering_t *rendering) const;
+
+
+public:
+
+    /* The text of the search field */
+    editable_line_t search_field_line;
+
+    /* Sets the set of completions */
+    void set_completions(const completion_list_t &comp);
+
+    /* Sets the prefix */
+    void set_prefix(const wcstring &pref);
+
+    /* Sets the terminal width and height */
+    void set_term_size(int w, int h);
+
+    /* Changes the selected completion in the given direction according to the layout of the given rendering. Returns true if the selection changed. */
+    bool select_next_completion_in_direction(selection_direction_t direction, const page_rendering_t &rendering);
+
+    /* Returns the currently selected completion for the given rendering */
+    const completion_t *selected_completion(const page_rendering_t &rendering) const;
+
+    /* Indicates the row and column for the given rendering. Returns -1 if no selection. */
+    size_t get_selected_row(const page_rendering_t &rendering) const;
+    size_t get_selected_column(const page_rendering_t &rendering) const;
+
+    /* Produces a rendering of the completions, at the given term size */
+    page_rendering_t render() const;
+
+    /* Updates the rendering if it's stale */
+    void update_rendering(page_rendering_t *rendering) const;
+
+    /* Indicates if there are no completions, and therefore nothing to render */
+    bool empty() const;
+
+    /* Clears all completions and the prefix */
+    void clear();
+
+    /* Updates the completions list per the filter */
+    void refilter_completions();
+
+    /* Sets whether the search field is shown */
+    void set_search_field_shown(bool flag);
+
+    /* Gets whether the search field shown */
+    bool is_search_field_shown() const;
+
+    /* Indicates if we are navigating our contents */
+    bool is_navigating_contents() const;
+
+    /* Become fully disclosed */
+    void set_fully_disclosed(bool flag);
+
+    /* Position of the cursor */
+    size_t cursor_position() const;
+
+    /* Constructor */
+    pager_t();
+};
diff --git a/src/parse_constants.h b/src/parse_constants.h
new file mode 100644
index 00000000..552085f8
--- /dev/null
+++ b/src/parse_constants.h
@@ -0,0 +1,352 @@
+/**\file parse_constants.h
+
+    Constants used in the programmatic representation of fish code.
+*/
+
+#ifndef fish_parse_constants_h
+#define fish_parse_constants_h
+
+#include "config.h"
+
+#define PARSE_ASSERT(a) assert(a)
+#define PARSER_DIE() do { fprintf(stderr, "Parser dying!\n"); exit_without_destructors(-1); } while (0)
+
+enum parse_token_type_t
+{
+    token_type_invalid,
+
+    // Non-terminal tokens
+    symbol_job_list,
+    symbol_job,
+    symbol_job_continuation,
+    symbol_statement,
+    symbol_block_statement,
+    symbol_block_header,
+    symbol_for_header,
+    symbol_while_header,
+    symbol_begin_header,
+    symbol_function_header,
+
+    symbol_if_statement,
+    symbol_if_clause,
+    symbol_else_clause,
+    symbol_else_continuation,
+
+    symbol_switch_statement,
+    symbol_case_item_list,
+    symbol_case_item,
+
+    symbol_boolean_statement,
+    symbol_decorated_statement,
+    symbol_plain_statement,
+    symbol_arguments_or_redirections_list,
+    symbol_argument_or_redirection,
+
+    symbol_argument_list,
+
+    // "freestanding" argument lists are parsed from the argument list supplied to 'complete -a'
+    // They are not generated by parse trees rooted in symbol_job_list
+    symbol_freestanding_argument_list,
+
+    symbol_argument,
+    symbol_redirection,
+
+    symbol_optional_background,
+
+    symbol_end_command,
+
+    // Terminal types
+    parse_token_type_string,
+    parse_token_type_pipe,
+    parse_token_type_redirection,
+    parse_token_type_background,
+    parse_token_type_end,
+
+    // Special terminal type that means no more tokens forthcoming
+    parse_token_type_terminate,
+
+    // Very special terminal types that don't appear in the production list
+    parse_special_type_parse_error,
+    parse_special_type_tokenizer_error,
+    parse_special_type_comment,
+
+    FIRST_TERMINAL_TYPE = parse_token_type_string,
+    LAST_TERMINAL_TYPE = parse_token_type_terminate,
+
+    LAST_TOKEN_OR_SYMBOL = parse_token_type_terminate,
+
+    FIRST_PARSE_TOKEN_TYPE = parse_token_type_string,
+    LAST_PARSE_TOKEN_TYPE = parse_token_type_end
+} __packed;
+
+/* These must be maintained in sorted order (except for none, which isn't a keyword). This enables us to do binary search. */
+enum parse_keyword_t
+{
+    parse_keyword_none,
+    parse_keyword_and,
+    parse_keyword_begin,
+    parse_keyword_builtin,
+    parse_keyword_case,
+    parse_keyword_command,
+    parse_keyword_else,
+    parse_keyword_end,
+    parse_keyword_exec,
+    parse_keyword_for,
+    parse_keyword_function,
+    parse_keyword_if,
+    parse_keyword_in,
+    parse_keyword_not,
+    parse_keyword_or,
+    parse_keyword_switch,
+    parse_keyword_while,
+    LAST_KEYWORD = parse_keyword_while
+} __packed;
+
+/* Statement decorations. This matches the order of productions in decorated_statement */
+enum parse_statement_decoration_t
+{
+    parse_statement_decoration_none,
+    parse_statement_decoration_command,
+    parse_statement_decoration_builtin,
+    parse_statement_decoration_exec
+};
+
+/* Boolean statement types */
+enum parse_bool_statement_type_t
+{
+    parse_bool_and,
+    parse_bool_or,
+    parse_bool_not
+};
+
+/* Parse error code list */
+enum parse_error_code_t
+{
+    parse_error_none,
+
+    /* Matching values from enum parser_error */
+    parse_error_syntax,
+    parse_error_eval,
+    parse_error_cmdsubst,
+
+    parse_error_generic, // unclassified error types
+
+    //tokenizer errors
+    parse_error_tokenizer_unterminated_quote,
+    parse_error_tokenizer_unterminated_subshell,
+    parse_error_tokenizer_unterminated_escape,
+    parse_error_tokenizer_other,
+
+    parse_error_unbalancing_end, //end outside of block
+    parse_error_unbalancing_else, //else outside of if
+    parse_error_unbalancing_case, //case outside of switch
+
+    parse_error_double_pipe, // foo || bar, has special error message
+    parse_error_double_background // foo && bar, has special error message
+};
+
+enum
+{
+    PARSER_TEST_ERROR = 1,
+    PARSER_TEST_INCOMPLETE = 2
+};
+typedef unsigned int parser_test_error_bits_t;
+
+struct parse_error_t
+{
+    /** Text of the error */
+    wcstring text;
+
+    /** Code for the error */
+    enum parse_error_code_t code;
+
+    /** Offset and length of the token in the source code that triggered this error */
+    size_t source_start;
+    size_t source_length;
+
+    /** Return a string describing the error, suitable for presentation to the user. If skip_caret is false, the offending line with a caret is printed as well */
+    wcstring describe(const wcstring &src) const;
+
+    /** Return a string describing the error, suitable for presentation to the user, with the given prefix. If skip_caret is false, the offending line with a caret is printed as well */
+    wcstring describe_with_prefix(const wcstring &src, const wcstring &prefix, bool is_interactive, bool skip_caret) const;
+};
+typedef std::vector<parse_error_t> parse_error_list_t;
+
+/* Special source_start value that means unknown */
+#define SOURCE_LOCATION_UNKNOWN (static_cast<size_t>(-1))
+
+/* Helper function to offset error positions by the given amount. This is used when determining errors in a substring of a larger source buffer. */
+void parse_error_offset_source_start(parse_error_list_t *errors, size_t amt);
+
+/** Maximum number of function calls. */
+#define FISH_MAX_STACK_DEPTH 128
+
+/** Error message on a function that calls itself immediately */
+#define INFINITE_FUNC_RECURSION_ERR_MSG _( L"The function '%ls' calls itself immediately, which would result in an infinite loop.")
+
+/** Error message on reaching maximum call stack depth */
+#define CALL_STACK_LIMIT_EXCEEDED_ERR_MSG _( L"The function call stack limit has been exceeded. Do you have an accidental infinite loop?")
+
+/** Error message when encountering an illegal command name */
+#define ILLEGAL_CMD_ERR_MSG _( L"Illegal command name '%ls'")
+
+/** Error message when encountering an unknown builtin name */
+#define UNKNOWN_BUILTIN_ERR_MSG _( L"Unknown builtin '%ls'")
+
+/** Error message when encountering a failed expansion, e.g. for the variable name in for loops */
+#define FAILED_EXPANSION_VARIABLE_NAME_ERR_MSG _( L"Unable to expand variable name '%ls'")
+
+/** Error message when encountering a failed process expansion, e.g. %notaprocess */
+#define FAILED_EXPANSION_PROCESS_ERR_MSG _( L"Unable to find a process '%ls'")
+
+/** Error message when encountering an illegal file descriptor */
+#define ILLEGAL_FD_ERR_MSG _( L"Illegal file descriptor in redirection '%ls'")
+
+/** Error message for wildcards with no matches */
+#define WILDCARD_ERR_MSG _( L"No matches for wildcard '%ls'.")
+
+/** Error when using break outside of loop */
+#define INVALID_BREAK_ERR_MSG _( L"'break' while not inside of loop" )
+
+/** Error when using continue outside of loop */
+#define INVALID_CONTINUE_ERR_MSG _( L"'continue' while not inside of loop" )
+
+/** Error when using return builtin outside of function definition */
+#define INVALID_RETURN_ERR_MSG _( L"'return' outside of function definition" )
+
+
+/*** Error messages. The number is a reminder of how many format specifiers are contained. */
+
+/** Error for (e.g.) $^ */
+#define ERROR_BAD_VAR_CHAR1 _( L"$%lc is not a valid variable in fish." )
+
+/** Error for ${a} */
+#define ERROR_BRACKETED_VARIABLE1 _( L"Variables cannot be bracketed. In fish, please use {$%ls}." )
+
+/** Error for "${a}" */
+#define ERROR_BRACKETED_VARIABLE_QUOTED1 _( L"Variables cannot be bracketed. In fish, please use \"$%ls\"." )
+
+/** Error issued on $? */
+#define ERROR_NOT_STATUS _( L"$? is not the exit status. In fish, please use $status.")
+
+/** Error issued on $$ */
+#define ERROR_NOT_PID _( L"$$ is not the pid. In fish, please use %%self.")
+
+/** Error issued on $# */
+#define ERROR_NOT_ARGV_COUNT _( L"$# is not supported. In fish, please use 'count $argv'.")
+
+/** Error issued on $@ */
+#define ERROR_NOT_ARGV_AT _( L"$@ is not supported. In fish, please use $argv.")
+
+/** Error issued on $(...) */
+#define ERROR_BAD_VAR_SUBCOMMAND1 _( L"$(...) is not supported. In fish, please use '(%ls)'." )
+
+/** Error issued on $* */
+#define ERROR_NOT_ARGV_STAR _( L"$* is not supported. In fish, please use $argv." )
+
+/** Error issued on $ */
+#define ERROR_NO_VAR_NAME _( L"Expected a variable name after this $.")
+
+/** Error on || */
+#define ERROR_BAD_OR _( L"Unsupported use of '||'. In fish, please use 'COMMAND; or COMMAND'.")
+
+/** Error on && */
+#define ERROR_BAD_AND _( L"Unsupported use of '&&'. In fish, please use 'COMMAND; and COMMAND'.")
+
+/** Error on foo=bar */
+#define ERROR_BAD_EQUALS_IN_COMMAND5 _( L"Unsupported use of '='. To run '%ls' with a modified environment, please use 'env %ls=%ls %ls%ls'")
+
+/** Error message for Posix-style assignment: foo=bar */
+#define ERROR_BAD_COMMAND_ASSIGN_ERR_MSG _( L"Unsupported use of '='. In fish, please use 'set %ls %ls'.")
+
+
+
+/**
+   While block description
+*/
+#define WHILE_BLOCK N_( L"'while' block" )
+
+/**
+   For block description
+*/
+#define FOR_BLOCK N_( L"'for' block" )
+
+/**
+   Breakpoint block
+*/
+#define BREAKPOINT_BLOCK N_( L"Block created by breakpoint" )
+
+
+
+/**
+   If block description
+*/
+#define IF_BLOCK N_( L"'if' conditional block" )
+
+
+/**
+   Function definition block description
+*/
+#define FUNCTION_DEF_BLOCK N_( L"function definition block" )
+
+
+/**
+   Function invocation block description
+*/
+#define FUNCTION_CALL_BLOCK N_( L"function invocation block" )
+
+/**
+   Function invocation block description
+*/
+#define FUNCTION_CALL_NO_SHADOW_BLOCK N_( L"function invocation block with no variable shadowing" )
+
+
+/**
+   Switch block description
+*/
+#define SWITCH_BLOCK N_( L"'switch' block" )
+
+
+/**
+   Fake block description
+*/
+#define FAKE_BLOCK N_( L"unexecutable block" )
+
+
+/**
+   Top block description
+*/
+#define TOP_BLOCK N_( L"global root block" )
+
+
+/**
+   Command substitution block description
+*/
+#define SUBST_BLOCK N_( L"command substitution block" )
+
+
+/**
+   Begin block description
+*/
+#define BEGIN_BLOCK N_( L"'begin' unconditional block" )
+
+
+/**
+   Source block description
+*/
+#define SOURCE_BLOCK N_( L"Block created by the . builtin" )
+
+/**
+   Source block description
+*/
+#define EVENT_BLOCK N_( L"event handler block" )
+
+
+/**
+   Unknown block description
+*/
+#define UNKNOWN_BLOCK N_( L"unknown/invalid block" )
+
+
+
+#endif
diff --git a/src/parse_execution.cpp b/src/parse_execution.cpp
new file mode 100644
index 00000000..8381958f
--- /dev/null
+++ b/src/parse_execution.cpp
@@ -0,0 +1,1628 @@
+/**\file parse_execution.cpp
+
+   Provides the "linkage" between a parse_node_tree_t and actual execution structures (job_t, etc.)
+
+   A note on error handling: fish has two kind of errors, fatal parse errors non-fatal runtime errors. A fatal error prevents execution of the entire file, while a non-fatal error skips that job.
+
+   Non-fatal errors are printed as soon as they are encountered; otherwise you would have to wait for the execution to finish to see them.
+*/
+
+#include "parse_execution.h"
+#include <assert.h>
+#include <errno.h>
+#include <stdarg.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <termios.h>
+#include <unistd.h>
+#include <wchar.h>
+#include <wctype.h>
+#include <string>
+#include <memory> // IWYU pragma: keep - suggests <tr1/memory> instead
+#include <vector>
+#include "env.h"
+#include "event.h"
+#include "tokenizer.h"
+#include "util.h"
+#include "parse_util.h"
+#include "complete.h"
+#include "wildcard.h"
+#include "parser.h"
+#include "expand.h"
+#include "reader.h"
+#include "wutil.h"
+#include "path.h"
+#include "function.h"
+#include "builtin.h"
+#include "exec.h"
+
+/* These are the specific statement types that support redirections */
+static bool specific_statement_type_is_redirectable_block(const parse_node_t &node)
+{
+    return node.type == symbol_block_statement || node.type == symbol_if_statement || node.type == symbol_switch_statement;
+
+}
+
+/* Get the name of a redirectable block, for profiling purposes */
+static wcstring profiling_cmd_name_for_redirectable_block(const parse_node_t &node, const parse_node_tree_t &tree, const wcstring &src)
+{
+    assert(specific_statement_type_is_redirectable_block(node));
+    assert(node.has_source());
+
+    /* Get the source for the block, and cut it at the next statement terminator. */
+    const size_t src_start = node.source_start;
+    size_t src_len = node.source_length;
+
+    const parse_node_tree_t::parse_node_list_t statement_terminator_nodes = tree.find_nodes(node, parse_token_type_end, 1);
+    if (! statement_terminator_nodes.empty())
+    {
+        const parse_node_t *term = statement_terminator_nodes.at(0);
+        assert(term->source_start >= src_start);
+        src_len = term->source_start - src_start;
+    }
+
+    wcstring result = wcstring(src, src_start, src_len);
+    result.append(L"...");
+    return result;
+}
+
+parse_execution_context_t::parse_execution_context_t(const parse_node_tree_t &t, const wcstring &s, parser_t *p, int initial_eval_level) : tree(t), src(s), parser(p), eval_level(initial_eval_level), executing_node_idx(NODE_OFFSET_INVALID), cached_lineno_offset(0), cached_lineno_count(0)
+{
+}
+
+/* Utilities */
+
+wcstring parse_execution_context_t::get_source(const parse_node_t &node) const
+{
+    return node.get_source(this->src);
+}
+
+const parse_node_t *parse_execution_context_t::get_child(const parse_node_t &parent, node_offset_t which, parse_token_type_t expected_type) const
+{
+    return this->tree.get_child(parent, which, expected_type);
+}
+
+node_offset_t parse_execution_context_t::get_offset(const parse_node_t &node) const
+{
+    /* Get the offset of a node via pointer arithmetic, very hackish */
+    const parse_node_t *addr = &node;
+    const parse_node_t *base = &this->tree.at(0);
+    assert(addr >= base);
+    assert(addr - base < SOURCE_OFFSET_INVALID);
+    node_offset_t offset = static_cast<node_offset_t>(addr - base);
+    assert(offset < this->tree.size());
+    assert(&tree.at(offset) == &node);
+    return offset;
+}
+
+const parse_node_t *parse_execution_context_t::infinite_recursive_statement_in_job_list(const parse_node_t &job_list, wcstring *out_func_name) const
+{
+    assert(job_list.type == symbol_job_list);
+    /*
+      This is a bit fragile. It is a test to see if we are
+      inside of function call, but not inside a block in that
+      function call. If, in the future, the rules for what
+      block scopes are pushed on function invocation changes,
+      then this check will break.
+    */
+    const block_t *current = parser->block_at_index(0), *parent = parser->block_at_index(1);
+    bool is_within_function_call = (current && parent && current->type() == TOP && parent->type() == FUNCTION_CALL);
+    if (! is_within_function_call)
+    {
+        return NULL;
+    }
+
+    /* Check to see which function call is forbidden */
+    if (parser->forbidden_function.empty())
+    {
+        return NULL;
+    }
+    const wcstring &forbidden_function_name = parser->forbidden_function.back();
+
+    /* Get the first job in the job list. */
+    const parse_node_t *first_job = tree.next_node_in_node_list(job_list, symbol_job, NULL);
+    if (first_job == NULL)
+    {
+        return NULL;
+    }
+
+    /* Here's the statement node we find that's infinite recursive */
+    const parse_node_t *infinite_recursive_statement = NULL;
+
+    /* Get the list of statements */
+    const parse_node_tree_t::parse_node_list_t statements = tree.specific_statements_for_job(*first_job);
+
+    /* Find all the decorated statements. We are interested in statements with no decoration (i.e. not command, not builtin) whose command expands to the forbidden function */
+    for (size_t i=0; i < statements.size(); i++)
+    {
+        /* We only care about decorated statements, not while statements, etc. */
+        const parse_node_t &statement = *statements.at(i);
+        if (statement.type != symbol_decorated_statement)
+        {
+            continue;
+        }
+
+        const parse_node_t &plain_statement = tree.find_child(statement, symbol_plain_statement);
+        if (tree.decoration_for_plain_statement(plain_statement) != parse_statement_decoration_none)
+        {
+            /* This statement has a decoration like 'builtin' or 'command', and therefore is not infinite recursion. In particular this is what enables 'wrapper functions' */
+            continue;
+        }
+
+        /* Ok, this is an undecorated plain statement. Get and expand its command */
+        wcstring cmd;
+        tree.command_for_plain_statement(plain_statement, src, &cmd);
+
+        if (expand_one(cmd, EXPAND_SKIP_CMDSUBST | EXPAND_SKIP_VARIABLES, NULL) &&
+            cmd == forbidden_function_name)
+        {
+            /* This is it */
+            infinite_recursive_statement = &statement;
+            if (out_func_name != NULL)
+            {
+                *out_func_name = forbidden_function_name;
+            }
+            break;
+        }
+    }
+
+    assert(infinite_recursive_statement == NULL || infinite_recursive_statement->type == symbol_decorated_statement);
+    return infinite_recursive_statement;
+}
+
+enum process_type_t parse_execution_context_t::process_type_for_command(const parse_node_t &plain_statement, const wcstring &cmd) const
+{
+    assert(plain_statement.type == symbol_plain_statement);
+    enum process_type_t process_type = EXTERNAL;
+
+    /* Determine the process type, which depends on the statement decoration (command, builtin, etc) */
+    enum parse_statement_decoration_t decoration = tree.decoration_for_plain_statement(plain_statement);
+
+    if (decoration == parse_statement_decoration_exec)
+    {
+        /* Always exec */
+        process_type = INTERNAL_EXEC;
+    }
+    else if (decoration == parse_statement_decoration_command)
+    {
+        /* Always a command */
+        process_type = EXTERNAL;
+    }
+    else if (decoration == parse_statement_decoration_builtin)
+    {
+        /* What happens if this builtin is not valid? */
+        process_type = INTERNAL_BUILTIN;
+    }
+    else if (function_exists(cmd))
+    {
+        process_type = INTERNAL_FUNCTION;
+    }
+    else if (builtin_exists(cmd))
+    {
+        process_type = INTERNAL_BUILTIN;
+    }
+    else
+    {
+        process_type = EXTERNAL;
+    }
+    return process_type;
+}
+
+bool parse_execution_context_t::should_cancel_execution(const block_t *block) const
+{
+    return cancellation_reason(block) != execution_cancellation_none;
+}
+
+parse_execution_context_t::execution_cancellation_reason_t parse_execution_context_t::cancellation_reason(const block_t *block) const
+{
+    if (shell_is_exiting())
+    {
+        return execution_cancellation_exit;
+    }
+    else if (parser && parser->cancellation_requested)
+    {
+        return execution_cancellation_skip;
+    }
+    else if (block && block->loop_status != LOOP_NORMAL)
+    {
+        /* Nasty hack - break and continue set the 'skip' flag as well as the loop status flag. */
+        return execution_cancellation_loop_control;
+    }
+    else if (block && block->skip)
+    {
+        return execution_cancellation_skip;
+    }
+    else
+    {
+        return execution_cancellation_none;
+    }
+}
+
+/* Return whether the job contains a single statement, of block type, with no redirections */
+bool parse_execution_context_t::job_is_simple_block(const parse_node_t &job_node) const
+{
+    assert(job_node.type == symbol_job);
+
+    /* Must have one statement */
+    const parse_node_t &statement = *get_child(job_node, 0, symbol_statement);
+    const parse_node_t &specific_statement = *get_child(statement, 0);
+    if (! specific_statement_type_is_redirectable_block(specific_statement))
+    {
+        /* Not an appropriate block type */
+        return false;
+    }
+
+
+    /* Must be no pipes */
+    const parse_node_t &continuation = *get_child(job_node, 1, symbol_job_continuation);
+    if (continuation.child_count > 0)
+    {
+        /* Multiple statements in this job, so there's pipes involved */
+        return false;
+    }
+
+    /* Check for arguments and redirections. All of the above types have an arguments / redirections list. It must be empty. */
+    const parse_node_t &args_and_redirections = tree.find_child(specific_statement, symbol_arguments_or_redirections_list);
+    if (args_and_redirections.child_count > 0)
+    {
+        /* Non-empty, we have an argument or redirection */
+        return false;
+    }
+
+    /* Ok, we are a simple block! */
+    return true;
+}
+
+parse_execution_result_t parse_execution_context_t::run_if_statement(const parse_node_t &statement)
+{
+    assert(statement.type == symbol_if_statement);
+
+    /* Push an if block */
+    if_block_t *ib = new if_block_t();
+    ib->node_offset = this->get_offset(statement);
+    parser->push_block(ib);
+
+    parse_execution_result_t result = parse_execution_success;
+
+    /* We have a sequence of if clauses, with a final else, resulting in a single job list that we execute */
+    const parse_node_t *job_list_to_execute = NULL;
+    const parse_node_t *if_clause = get_child(statement, 0, symbol_if_clause);
+    const parse_node_t *else_clause = get_child(statement, 1, symbol_else_clause);
+    for (;;)
+    {
+        if (should_cancel_execution(ib))
+        {
+            result = parse_execution_cancelled;
+            break;
+        }
+
+        assert(if_clause != NULL && else_clause != NULL);
+        const parse_node_t &condition = *get_child(*if_clause, 1, symbol_job);
+
+        /* Check the condition. We treat parse_execution_errored here as failure, in accordance with historic behavior */
+        parse_execution_result_t cond_ret = run_1_job(condition, ib);
+        bool take_branch = (cond_ret == parse_execution_success) && proc_get_last_status() == EXIT_SUCCESS;
+
+        if (take_branch)
+        {
+            /* condition succeeded */
+            job_list_to_execute = get_child(*if_clause, 3, symbol_job_list);
+            break;
+        }
+        else if (else_clause->child_count == 0)
+        {
+            /* 'if' condition failed, no else clause, return 0, we're done. */
+            job_list_to_execute = NULL;
+            proc_set_last_status(STATUS_BUILTIN_OK);
+            break;
+        }
+        else
+        {
+            /* We have an 'else continuation' (either else-if or else) */
+            const parse_node_t &else_cont = *get_child(*else_clause, 1, symbol_else_continuation);
+            assert(else_cont.production_idx < 2);
+            if (else_cont.production_idx == 0)
+            {
+                /* it's an 'else if', go to the next one */
+                if_clause = get_child(else_cont, 0, symbol_if_clause);
+                else_clause = get_child(else_cont, 1, symbol_else_clause);
+            }
+            else
+            {
+                /* it's the final 'else', we're done */
+                assert(else_cont.production_idx == 1);
+                job_list_to_execute = get_child(else_cont, 1, symbol_job_list);
+                break;
+            }
+        }
+    }
+
+    /* Execute any job list we got */
+    if (job_list_to_execute != NULL)
+    {
+        run_job_list(*job_list_to_execute, ib);
+    }
+    else
+    {   /* No job list means no sucessful conditions, so return 0 (#1443). */
+        proc_set_last_status(STATUS_BUILTIN_OK);
+    }
+
+    /* It's possible there's a last-minute cancellation (#1297). */
+    if (should_cancel_execution(ib))
+    {
+        result = parse_execution_cancelled;
+    }
+
+    /* Done */
+    parser->pop_block(ib);
+
+    /* Otherwise, take the exit status of the job list. Reversal of #1061. */
+    return result;
+}
+
+parse_execution_result_t parse_execution_context_t::run_begin_statement(const parse_node_t &header, const parse_node_t &contents)
+{
+    assert(header.type == symbol_begin_header);
+    assert(contents.type == symbol_job_list);
+
+    /* Basic begin/end block. Push a scope block. */
+    scope_block_t *sb = new scope_block_t(BEGIN);
+    parser->push_block(sb);
+
+    /* Run the job list */
+    parse_execution_result_t ret = run_job_list(contents, sb);
+
+    /* Pop the block */
+    parser->pop_block(sb);
+
+    return ret;
+}
+
+/* Define a function */
+parse_execution_result_t parse_execution_context_t::run_function_statement(const parse_node_t &header, const parse_node_t &block_end_command)
+{
+    assert(header.type == symbol_function_header);
+    assert(block_end_command.type == symbol_end_command);
+    
+    /* Get arguments */
+    wcstring_list_t argument_list;
+    parse_execution_result_t result = this->determine_arguments(header, &argument_list);
+
+    if (result == parse_execution_success)
+    {
+        /* The function definition extends from the end of the header to the function end. It's not just the range of the contents because that loses comments - see issue #1710 */
+        assert(block_end_command.has_source());
+        size_t contents_start = header.source_start + header.source_length;
+        size_t contents_end = block_end_command.source_start; // 1 past the last character in the function definition
+        assert(contents_end >= contents_start);
+        
+        // Swallow whitespace at both ends
+        while (contents_start < contents_end && iswspace(this->src.at(contents_start)))
+        {
+            contents_start++;
+        }
+        while (contents_start < contents_end && iswspace(this->src.at(contents_end - 1)))
+        {
+            contents_end--;
+        }
+        
+        assert(contents_end >= contents_start);
+        const wcstring contents_str = wcstring(this->src, contents_start, contents_end - contents_start);
+        int definition_line_offset = this->line_offset_of_character_at_offset(contents_start);
+        wcstring error_str;
+        int err = define_function(*parser, argument_list, contents_str, definition_line_offset, &error_str);
+        proc_set_last_status(err);
+
+        if (! error_str.empty())
+        {
+            this->report_error(header, L"%ls", error_str.c_str());
+            result = parse_execution_errored;
+        }
+    }
+    return result;
+
+}
+
+parse_execution_result_t parse_execution_context_t::run_block_statement(const parse_node_t &statement)
+{
+    assert(statement.type == symbol_block_statement);
+
+    const parse_node_t &block_header = *get_child(statement, 0, symbol_block_header); //block header
+    const parse_node_t &header = *get_child(block_header, 0); //specific header type (e.g. for loop)
+    const parse_node_t &contents = *get_child(statement, 1, symbol_job_list); //block contents
+
+    parse_execution_result_t ret = parse_execution_success;
+    switch (header.type)
+    {
+        case symbol_for_header:
+            ret = run_for_statement(header, contents);
+            break;
+
+        case symbol_while_header:
+            ret = run_while_statement(header, contents);
+            break;
+
+        case symbol_function_header:
+        {
+            const parse_node_t &function_end = *get_child(statement, 2, symbol_end_command); //the 'end' associated with the block
+            ret = run_function_statement(header, function_end);
+            break;
+        }
+
+        case symbol_begin_header:
+            ret = run_begin_statement(header, contents);
+            break;
+
+        default:
+            fprintf(stderr, "Unexpected block header: %ls\n", header.describe().c_str());
+            PARSER_DIE();
+            break;
+    }
+
+    return ret;
+}
+
+parse_execution_result_t parse_execution_context_t::run_for_statement(const parse_node_t &header, const parse_node_t &block_contents)
+{
+    assert(header.type == symbol_for_header);
+    assert(block_contents.type == symbol_job_list);
+
+    /* Get the variable name: `for var_name in ...`. We expand the variable name. It better result in just one. */
+    const parse_node_t &var_name_node = *get_child(header, 1, parse_token_type_string);
+    wcstring for_var_name = get_source(var_name_node);
+    if (! expand_one(for_var_name, 0, NULL))
+    {
+        report_error(var_name_node, FAILED_EXPANSION_VARIABLE_NAME_ERR_MSG, for_var_name.c_str());
+        return parse_execution_errored;
+    }
+
+    /* Get the contents to iterate over. */
+    wcstring_list_t argument_sequence;
+    parse_execution_result_t ret = this->determine_arguments(header, &argument_sequence);
+    if (ret != parse_execution_success)
+    {
+        return ret;
+    }
+
+    for_block_t *fb = new for_block_t();
+    parser->push_block(fb);
+
+    /* Now drive the for loop. */
+    const size_t arg_count = argument_sequence.size();
+    for (size_t i=0; i < arg_count; i++)
+    {
+        if (should_cancel_execution(fb))
+        {
+            ret = parse_execution_cancelled;
+            break;
+        }
+
+        const wcstring &val = argument_sequence.at(i);
+        env_set(for_var_name, val.c_str(),  ENV_LOCAL);
+        fb->loop_status = LOOP_NORMAL;
+        fb->skip = 0;
+
+        this->run_job_list(block_contents, fb);
+
+        if (this->cancellation_reason(fb) == execution_cancellation_loop_control)
+        {
+            /* Handle break or continue */
+            if (fb->loop_status == LOOP_CONTINUE)
+            {
+                /* Reset the loop state */
+                fb->loop_status = LOOP_NORMAL;
+                fb->skip = false;
+                continue;
+            }
+            else if (fb->loop_status == LOOP_BREAK)
+            {
+                break;
+            }
+        }
+    }
+
+    parser->pop_block(fb);
+
+    return ret;
+}
+
+
+parse_execution_result_t parse_execution_context_t::run_switch_statement(const parse_node_t &statement)
+{
+    assert(statement.type == symbol_switch_statement);
+    const parse_node_t *matching_case_item = NULL;
+
+    parse_execution_result_t result = parse_execution_success;
+
+    /* Get the switch variable */
+    const parse_node_t &switch_value_node = *get_child(statement, 1, symbol_argument);
+    const wcstring switch_value = get_source(switch_value_node);
+
+    /* Expand it. We need to offset any errors by the position of the string */
+    std::vector<completion_t> switch_values_expanded;
+    parse_error_list_t errors;
+    int expand_ret = expand_string(switch_value, switch_values_expanded, EXPAND_NO_DESCRIPTIONS, &errors);
+    parse_error_offset_source_start(&errors, switch_value_node.source_start);
+
+    switch (expand_ret)
+    {
+        case EXPAND_ERROR:
+        {
+            result = report_errors(errors);
+            break;
+        }
+
+        case EXPAND_WILDCARD_NO_MATCH:
+        {
+            result = report_unmatched_wildcard_error(switch_value_node);
+            break;
+        }
+
+        case EXPAND_WILDCARD_MATCH:
+        case EXPAND_OK:
+        {
+            break;
+        }
+    }
+
+    if (result == parse_execution_success && switch_values_expanded.size() != 1)
+    {
+        result = report_error(switch_value_node,
+                              _(L"switch: Expected exactly one argument, got %lu\n"),
+                              switch_values_expanded.size());
+    }
+
+    if (result == parse_execution_success)
+    {
+        const wcstring &switch_value_expanded = switch_values_expanded.at(0).completion;
+
+        switch_block_t *sb = new switch_block_t();
+        parser->push_block(sb);
+
+
+        /* Expand case statements */
+        const parse_node_t *case_item_list = get_child(statement, 3, symbol_case_item_list);
+
+        /* Loop while we don't have a match but do have more of the list */
+        while (matching_case_item == NULL && case_item_list != NULL)
+        {
+            if (should_cancel_execution(sb))
+            {
+                result = parse_execution_cancelled;
+                break;
+            }
+
+            /* Get the next item and the remainder of the list */
+            const parse_node_t *case_item = tree.next_node_in_node_list(*case_item_list, symbol_case_item, &case_item_list);
+            if (case_item == NULL)
+            {
+                /* No more items */
+                break;
+            }
+
+            /* Pull out the argument list */
+            const parse_node_t &arg_list = *get_child(*case_item, 1, symbol_argument_list);
+
+            /* Expand arguments. A case item list may have a wildcard that fails to expand to anything. We also report case errors, but don't stop execution; i.e. a case item that contains an unexpandable process will report and then fail to match. */
+            wcstring_list_t case_args;
+            parse_execution_result_t case_result = this->determine_arguments(arg_list, &case_args);
+            if (case_result == parse_execution_success)
+            {
+                for (size_t i=0; i < case_args.size(); i++)
+                {
+                    const wcstring &arg = case_args.at(i);
+
+                    /* Unescape wildcards so they can be expanded again */
+                    wchar_t *unescaped_arg = parse_util_unescape_wildcards(arg.c_str());
+                    bool match = wildcard_match(switch_value_expanded, unescaped_arg);
+                    free(unescaped_arg);
+
+                    /* If this matched, we're done */
+                    if (match)
+                    {
+                        matching_case_item = case_item;
+                        break;
+                    }
+                }
+            }
+        }
+
+        if (result == parse_execution_success && matching_case_item != NULL)
+        {
+            /* Success, evaluate the job list */
+            const parse_node_t *job_list = get_child(*matching_case_item, 3, symbol_job_list);
+            result = this->run_job_list(*job_list, sb);
+        }
+
+        parser->pop_block(sb);
+    }
+
+    return result;
+}
+
+parse_execution_result_t parse_execution_context_t::run_while_statement(const parse_node_t &header, const parse_node_t &block_contents)
+{
+    assert(header.type == symbol_while_header);
+    assert(block_contents.type == symbol_job_list);
+
+    /* Push a while block */
+    while_block_t *wb = new while_block_t();
+    wb->node_offset = this->get_offset(header);
+    parser->push_block(wb);
+
+    parse_execution_result_t ret = parse_execution_success;
+
+    /* The condition and contents of the while loop, as a job and job list respectively */
+    const parse_node_t &while_condition = *get_child(header, 1, symbol_job);
+
+    /* Run while the condition is true */
+    for (;;)
+    {
+        /* Check the condition */
+        parse_execution_result_t cond_result = this->run_1_job(while_condition, wb);
+
+        /* We only continue on successful execution and EXIT_SUCCESS */
+        if (cond_result != parse_execution_success || proc_get_last_status() != EXIT_SUCCESS)
+        {
+            break;
+        }
+
+        /* Check cancellation */
+        if (this->should_cancel_execution(wb))
+        {
+            ret = parse_execution_cancelled;
+            break;
+        }
+
+
+        /* The block ought to go inside the loop (see #1212) */
+        this->run_job_list(block_contents, wb);
+
+        if (this->cancellation_reason(wb) == execution_cancellation_loop_control)
+        {
+            /* Handle break or continue */
+            if (wb->loop_status == LOOP_CONTINUE)
+            {
+                /* Reset the loop state */
+                wb->loop_status = LOOP_NORMAL;
+                wb->skip = false;
+                continue;
+            }
+            else if (wb->loop_status == LOOP_BREAK)
+            {
+                break;
+            }
+        }
+        
+        /* no_exec means that fish was invoked with -n or --no-execute. If set, we allow the loop to not-execute once so its contents can be checked, and then break */
+        if (no_exec)
+        {
+            break;
+        }
+    }
+
+    /* Done */
+    parser->pop_block(wb);
+
+    return ret;
+}
+
+/* Reports an error. Always returns parse_execution_errored, so you can assign the result to an 'errored' variable */
+parse_execution_result_t parse_execution_context_t::report_error(const parse_node_t &node, const wchar_t *fmt, ...) const
+{
+    if (parser->show_errors)
+    {
+        /* Create an error */
+        parse_error_list_t error_list = parse_error_list_t(1);
+        parse_error_t *error = &error_list.at(0);
+        error->source_start = node.source_start;
+        error->source_length = node.source_length;
+        error->code = parse_error_syntax; //hackish
+
+        va_list va;
+        va_start(va, fmt);
+        error->text = vformat_string(fmt, va);
+        va_end(va);
+
+        this->report_errors(error_list);
+    }
+    return parse_execution_errored;
+}
+
+parse_execution_result_t parse_execution_context_t::report_errors(const parse_error_list_t &error_list) const
+{
+    if (parser->show_errors && ! parser->cancellation_requested)
+    {
+        if (error_list.empty())
+        {
+            fprintf(stderr, "Bug: Error reported but no error text found.");
+        }
+
+        /* Get a backtrace */
+        wcstring backtrace_and_desc;
+        parser->get_backtrace(src, error_list, &backtrace_and_desc);
+
+        /* Print it */
+        fprintf(stderr, "%ls", backtrace_and_desc.c_str());
+    }
+    return parse_execution_errored;
+}
+
+/* Reoports an unmatched wildcard error and returns parse_execution_errored */
+parse_execution_result_t parse_execution_context_t::report_unmatched_wildcard_error(const parse_node_t &unmatched_wildcard)
+{
+    proc_set_last_status(STATUS_UNMATCHED_WILDCARD);
+    // unmatched wildcards are only reported in interactive use because scripts have legitimate reasons
+    // to want to use wildcards without knowing whether they expand to anything.
+    if (get_is_interactive())
+    {
+        // Check if we're running code that was typed at the commandline.
+        // We can't just use `is_block` or the eval level, because `begin; echo *.unmatched; end` would not report
+        // the error even though it's run interactively.
+        // But any non-interactive use must have at least one function / event handler / source on the stack.
+        bool interactive = true;
+        for (size_t i = 0, count = parser->block_count(); i < count; ++i)
+        {
+            switch (parser->block_at_index(i)->type())
+            {
+                case FUNCTION_CALL:
+                case FUNCTION_CALL_NO_SHADOW:
+                case EVENT:
+                case SOURCE:
+                    interactive = false;
+                    break;
+                default:
+                    break;
+            }
+        }
+        if (interactive)
+        {
+            report_error(unmatched_wildcard, WILDCARD_ERR_MSG, get_source(unmatched_wildcard).c_str());
+        }
+    }
+    return parse_execution_errored;
+}
+
+/* Handle the case of command not found */
+parse_execution_result_t parse_execution_context_t::handle_command_not_found(const wcstring &cmd_str, const parse_node_t &statement_node, int err_code)
+{
+    assert(statement_node.type == symbol_plain_statement);
+
+    /* We couldn't find the specified command. This is a non-fatal error. We want to set the exit status to 127, which is the standard number used by other shells like bash and zsh. */
+
+    const wchar_t * const cmd = cmd_str.c_str();
+    const wchar_t * const equals_ptr = wcschr(cmd, L'=');
+    if (equals_ptr != NULL)
+    {
+        /* Try to figure out if this is a pure variable assignment (foo=bar), or if this appears to be running a command (foo=bar ruby...) */
+
+        const wcstring name_str = wcstring(cmd, equals_ptr - cmd); //variable name, up to the =
+        const wcstring val_str = wcstring(equals_ptr + 1); //variable value, past the =
+
+
+        const parse_node_tree_t::parse_node_list_t args = tree.find_nodes(statement_node, symbol_argument, 1);
+
+        if (! args.empty())
+        {
+            const wcstring argument = get_source(*args.at(0));
+
+            wcstring ellipsis_str = wcstring(1, ellipsis_char);
+            if (ellipsis_str == L"$")
+                ellipsis_str = L"...";
+
+            /* Looks like a command */
+            this->report_error(statement_node,
+                               ERROR_BAD_EQUALS_IN_COMMAND5,
+                               argument.c_str(),
+                               name_str.c_str(),
+                               val_str.c_str(),
+                               argument.c_str(),
+                               ellipsis_str.c_str());
+        }
+        else
+        {
+            this->report_error(statement_node,
+                               ERROR_BAD_COMMAND_ASSIGN_ERR_MSG,
+                               name_str.c_str(),
+                               val_str.c_str());
+        }
+    }
+    else if ((cmd[0] == L'$' || cmd[0] == VARIABLE_EXPAND || cmd[0] == VARIABLE_EXPAND_SINGLE) && cmd[1] != L'\0')
+    {
+        this->report_error(statement_node,
+                           _(L"Variables may not be used as commands. In fish, please define a function or use 'eval %ls'."),
+                           cmd+1);
+    }
+    else if (wcschr(cmd, L'$'))
+    {
+        this->report_error(statement_node,
+                           _(L"Commands may not contain variables. In fish, please use 'eval %ls'."),
+                           cmd);
+    }
+    else if (err_code!=ENOENT)
+    {
+        this->report_error(statement_node,
+                           _(L"The file '%ls' is not executable by this user"),
+                           cmd?cmd:L"UNKNOWN");
+    }
+    else
+    {
+        /*
+         Handle unrecognized commands with standard
+         command not found handler that can make better
+         error messages
+         */
+
+        wcstring_list_t event_args;
+        {
+            parse_execution_result_t arg_result = this->determine_arguments(statement_node, &event_args);
+
+            if (arg_result != parse_execution_success)
+            {
+                return arg_result;
+            }
+
+            event_args.insert(event_args.begin(), cmd_str);
+        }
+
+        event_fire_generic(L"fish_command_not_found", &event_args);
+
+        /* Here we want to report an error (so it shows a backtrace), but with no text */
+        this->report_error(statement_node, L"");
+    }
+
+    /* Set the last proc status appropriately */
+    proc_set_last_status(err_code==ENOENT?STATUS_UNKNOWN_COMMAND:STATUS_NOT_EXECUTABLE);
+
+    return parse_execution_errored;
+}
+
+/* Creates a 'normal' (non-block) process */
+parse_execution_result_t parse_execution_context_t::populate_plain_process(job_t *job, process_t *proc, const parse_node_t &statement)
+{
+    assert(job != NULL);
+    assert(proc != NULL);
+    assert(statement.type == symbol_plain_statement);
+
+    /* We may decide that a command should be an implicit cd */
+    bool use_implicit_cd = false;
+
+    /* Get the command. We expect to always get it here. */
+    wcstring cmd;
+    bool got_cmd = tree.command_for_plain_statement(statement, src, &cmd);
+    assert(got_cmd);
+
+    /* Expand it as a command. Return an error on failure. */
+    bool expanded = expand_one(cmd, EXPAND_SKIP_CMDSUBST | EXPAND_SKIP_VARIABLES, NULL);
+    if (! expanded)
+    {
+        report_error(statement, ILLEGAL_CMD_ERR_MSG, cmd.c_str());
+        return parse_execution_errored;
+    }
+
+    /* Determine the process type */
+    enum process_type_t process_type = process_type_for_command(statement, cmd);
+
+    /* Check for stack overflow */
+    if (process_type == INTERNAL_FUNCTION && parser->forbidden_function.size() > FISH_MAX_STACK_DEPTH)
+    {
+        this->report_error(statement, CALL_STACK_LIMIT_EXCEEDED_ERR_MSG);
+        return parse_execution_errored;
+    }
+
+    wcstring path_to_external_command;
+    if (process_type == EXTERNAL || process_type == INTERNAL_EXEC)
+    {
+        /* Determine the actual command. This may be an implicit cd. */
+        bool has_command = path_get_path(cmd, &path_to_external_command);
+
+        /* If there was no command, then we care about the value of errno after checking for it, to distinguish between e.g. no file vs permissions problem */
+        const int no_cmd_err_code = errno;
+
+        /* If the specified command does not exist, and is undecorated, try using an implicit cd. */
+        if (! has_command && tree.decoration_for_plain_statement(statement) == parse_statement_decoration_none)
+        {
+            /* Implicit cd requires an empty argument and redirection list */
+            const parse_node_t *args = get_child(statement, 1, symbol_arguments_or_redirections_list);
+            if (args->child_count == 0)
+            {
+                /* Ok, no arguments or redirections; check to see if the first argument is a directory */
+                wcstring implicit_cd_path;
+                use_implicit_cd = path_can_be_implicit_cd(cmd, &implicit_cd_path);
+            }
+        }
+
+        if (! has_command && ! use_implicit_cd)
+        {
+            /* No command */
+            return this->handle_command_not_found(cmd, statement, no_cmd_err_code);
+        }
+    }
+
+    /* The argument list and set of IO redirections that we will construct for the process */
+    io_chain_t process_io_chain;
+    wcstring_list_t argument_list;
+    if (use_implicit_cd)
+    {
+        /* Implicit cd is simple */
+        argument_list.push_back(L"cd");
+        argument_list.push_back(cmd);
+        path_to_external_command.clear();
+
+        /* If we have defined a wrapper around cd, use it, otherwise use the cd builtin */
+        process_type = function_exists(L"cd") ? INTERNAL_FUNCTION : INTERNAL_BUILTIN;
+    }
+    else
+    {
+        /* Form the list of arguments. The command is the first argument. TODO: count hack, where we treat 'count --help' as different from 'count $foo' that expands to 'count --help'. fish 1.x never successfully did this, but it tried to! */
+        parse_execution_result_t arg_result = this->determine_arguments(statement, &argument_list);
+        if (arg_result != parse_execution_success)
+        {
+            return arg_result;
+        }
+        argument_list.insert(argument_list.begin(), cmd);
+
+        /* The set of IO redirections that we construct for the process */
+        if (! this->determine_io_chain(statement, &process_io_chain))
+        {
+            return parse_execution_errored;
+        }
+
+        /* Determine the process type */
+        process_type = process_type_for_command(statement, cmd);
+    }
+
+
+    /* Populate the process */
+    proc->type = process_type;
+    proc->set_argv(argument_list);
+    proc->set_io_chain(process_io_chain);
+    proc->actual_cmd = path_to_external_command;
+    return parse_execution_success;
+}
+
+/* Determine the list of arguments, expanding stuff. Reports any errors caused by expansion. If we have a wildcard that could not be expanded, report the error and continue. */
+parse_execution_result_t parse_execution_context_t::determine_arguments(const parse_node_t &parent, wcstring_list_t *out_arguments)
+{
+    /* The ultimate result */
+    enum parse_execution_result_t result = parse_execution_success;
+
+    /* Get all argument nodes underneath the statement. We guess we'll have that many arguments (but may have more or fewer, if there are wildcards involved) */
+    const parse_node_tree_t::parse_node_list_t argument_nodes = tree.find_nodes(parent, symbol_argument);
+    out_arguments->reserve(out_arguments->size() + argument_nodes.size());
+    for (size_t i=0; i < argument_nodes.size(); i++)
+    {
+        const parse_node_t &arg_node = *argument_nodes.at(i);
+
+        /* Expect all arguments to have source */
+        assert(arg_node.has_source());
+        const wcstring arg_str = arg_node.get_source(src);
+
+        /* Expand this string */
+        std::vector<completion_t> arg_expanded;
+        parse_error_list_t errors;
+        int expand_ret = expand_string(arg_str, arg_expanded, EXPAND_NO_DESCRIPTIONS, &errors);
+        parse_error_offset_source_start(&errors, arg_node.source_start);
+        switch (expand_ret)
+        {
+            case EXPAND_ERROR:
+            {
+                this->report_errors(errors);
+                result = parse_execution_errored;
+                break;
+            }
+
+            case EXPAND_WILDCARD_NO_MATCH:
+            {
+                // report the unmatched wildcard error but don't stop processing.
+                // this will only print an error in interactive mode, though it does set the
+                // process status (similar to a command substitution failing)
+                report_unmatched_wildcard_error(arg_node);
+                break;
+            }
+
+            case EXPAND_WILDCARD_MATCH:
+            case EXPAND_OK:
+            {
+                break;
+            }
+        }
+
+        /* Now copy over any expanded arguments */
+        for (size_t i=0; i < arg_expanded.size(); i++)
+        {
+            out_arguments->push_back(arg_expanded.at(i).completion);
+        }
+    }
+
+    return result;
+}
+
+bool parse_execution_context_t::determine_io_chain(const parse_node_t &statement_node, io_chain_t *out_chain)
+{
+    io_chain_t result;
+    bool errored = false;
+
+    /* We are called with a statement of varying types. We require that the statement have an arguments_or_redirections_list child. */
+    const parse_node_t &args_and_redirections_list = tree.find_child(statement_node, symbol_arguments_or_redirections_list);
+
+    /* Get all redirection nodes underneath the statement */
+    const parse_node_tree_t::parse_node_list_t redirect_nodes = tree.find_nodes(args_and_redirections_list, symbol_redirection);
+    for (size_t i=0; i < redirect_nodes.size(); i++)
+    {
+        const parse_node_t &redirect_node = *redirect_nodes.at(i);
+
+        int source_fd = -1; /* source fd */
+        wcstring target; /* file path or target fd */
+        enum token_type redirect_type = tree.type_for_redirection(redirect_node, src, &source_fd, &target);
+
+        /* PCA: I can't justify this EXPAND_SKIP_VARIABLES flag. It was like this when I got here. */
+        bool target_expanded = expand_one(target, no_exec ? EXPAND_SKIP_VARIABLES : 0, NULL);
+        if (! target_expanded || target.empty())
+        {
+            /* Should improve this error message */
+            errored = report_error(redirect_node,
+                                   _(L"Invalid redirection target: %ls"),
+                                   target.c_str());
+        }
+
+
+        /* Generate the actual IO redirection */
+        shared_ptr<io_data_t> new_io;
+        assert(redirect_type != TOK_NONE);
+        switch (redirect_type)
+        {
+            case TOK_REDIRECT_FD:
+            {
+                if (target == L"-")
+                {
+                    new_io.reset(new io_close_t(source_fd));
+                }
+                else
+                {
+                    wchar_t *end = NULL;
+                    errno = 0;
+                    int old_fd = fish_wcstoi(target.c_str(), &end, 10);
+                    if (old_fd < 0 || errno || *end)
+                    {
+                        errored = report_error(redirect_node,
+                                               _(L"Requested redirection to '%ls', which is not a valid file descriptor"),
+                                               target.c_str());
+                    }
+                    else
+                    {
+                        new_io.reset(new io_fd_t(source_fd, old_fd, true));
+                    }
+                }
+                break;
+            }
+
+            case TOK_REDIRECT_OUT:
+            case TOK_REDIRECT_APPEND:
+            case TOK_REDIRECT_IN:
+            case TOK_REDIRECT_NOCLOB:
+            {
+                int oflags = oflags_for_redirection_type(redirect_type);
+                io_file_t *new_io_file = new io_file_t(source_fd, target, oflags);
+                new_io.reset(new_io_file);
+                break;
+            }
+
+            default:
+            {
+                // Should be unreachable
+                fprintf(stderr, "Unexpected redirection type %ld. aborting.\n", (long)redirect_type);
+                PARSER_DIE();
+                break;
+            }
+        }
+
+        /* Append the new_io if we got one */
+        if (new_io.get() != NULL)
+        {
+            result.push_back(new_io);
+        }
+    }
+
+    if (out_chain && ! errored)
+    {
+        out_chain->swap(result);
+    }
+    return ! errored;
+}
+
+parse_execution_result_t parse_execution_context_t::populate_boolean_process(job_t *job, process_t *proc, const parse_node_t &bool_statement)
+{
+    // Handle a boolean statement
+    bool skip_job = false;
+    assert(bool_statement.type == symbol_boolean_statement);
+    switch (parse_node_tree_t::statement_boolean_type(bool_statement))
+    {
+        case parse_bool_and:
+            // AND. Skip if the last job failed.
+            skip_job = (proc_get_last_status() != 0);
+            break;
+
+        case parse_bool_or:
+            // OR. Skip if the last job succeeded.
+            skip_job = (proc_get_last_status() == 0);
+            break;
+
+        case parse_bool_not:
+            // NOT. Negate it.
+            job_set_flag(job, JOB_NEGATE, !job_get_flag(job, JOB_NEGATE));
+            break;
+    }
+
+    if (skip_job)
+    {
+        return parse_execution_skipped;
+    }
+    else
+    {
+        const parse_node_t &subject = *tree.get_child(bool_statement, 1, symbol_statement);
+        return this->populate_job_process(job, proc, subject);
+    }
+}
+
+parse_execution_result_t parse_execution_context_t::populate_block_process(job_t *job, process_t *proc, const parse_node_t &statement_node)
+{
+    /* We handle block statements by creating INTERNAL_BLOCK_NODE, that will bounce back to us when it's time to execute them */
+    assert(statement_node.type == symbol_block_statement || statement_node.type == symbol_if_statement || statement_node.type == symbol_switch_statement);
+
+    /* The set of IO redirections that we construct for the process */
+    io_chain_t process_io_chain;
+    bool errored = ! this->determine_io_chain(statement_node, &process_io_chain);
+    if (errored)
+        return parse_execution_errored;
+
+    proc->type = INTERNAL_BLOCK_NODE;
+    proc->internal_block_node = this->get_offset(statement_node);
+    proc->set_io_chain(process_io_chain);
+    return parse_execution_success;
+}
+
+
+/* Returns a process_t allocated with new. It's the caller's responsibility to delete it (!) */
+parse_execution_result_t parse_execution_context_t::populate_job_process(job_t *job, process_t *proc, const parse_node_t &statement_node)
+{
+    assert(statement_node.type == symbol_statement);
+    assert(statement_node.child_count == 1);
+
+    // Get the "specific statement" which is boolean / block / if / switch / decorated
+    const parse_node_t &specific_statement = *get_child(statement_node, 0);
+
+    parse_execution_result_t result = parse_execution_success;
+
+    switch (specific_statement.type)
+    {
+        case symbol_boolean_statement:
+        {
+            result = this->populate_boolean_process(job, proc, specific_statement);
+            break;
+        }
+
+        case symbol_block_statement:
+        case symbol_if_statement:
+        case symbol_switch_statement:
+        {
+            result = this->populate_block_process(job, proc, specific_statement);
+            break;
+        }
+
+        case symbol_decorated_statement:
+        {
+            /* Get the plain statement. It will pull out the decoration itself */
+            const parse_node_t &plain_statement = tree.find_child(specific_statement, symbol_plain_statement);
+            result = this->populate_plain_process(job, proc, plain_statement);
+            break;
+        }
+
+        default:
+            fprintf(stderr, "'%ls' not handled by new parser yet\n", specific_statement.describe().c_str());
+            PARSER_DIE();
+            break;
+    }
+
+    return result;
+}
+
+
+parse_execution_result_t parse_execution_context_t::populate_job_from_job_node(job_t *j, const parse_node_t &job_node, const block_t *associated_block)
+{
+    assert(job_node.type == symbol_job);
+
+    /* Tell the job what its command is */
+    j->set_command(get_source(job_node));
+
+    /* We are going to construct process_t structures for every statement in the job. Get the first statement. */
+    const parse_node_t *statement_node = get_child(job_node, 0, symbol_statement);
+    assert(statement_node != NULL);
+
+    parse_execution_result_t result = parse_execution_success;
+
+    /* Create processes. Each one may fail. */
+    std::vector<process_t *> processes;
+    processes.push_back(new process_t());
+    result = this->populate_job_process(j, processes.back(), *statement_node);
+
+    /* Construct process_ts for job continuations (pipelines), by walking the list until we hit the terminal (empty) job continuation */
+    const parse_node_t *job_cont = get_child(job_node, 1, symbol_job_continuation);
+    assert(job_cont != NULL);
+    while (result == parse_execution_success && job_cont->child_count > 0)
+    {
+        assert(job_cont->type == symbol_job_continuation);
+
+        /* Handle the pipe, whose fd may not be the obvious stdout */
+        const parse_node_t &pipe_node = *get_child(*job_cont, 0, parse_token_type_pipe);
+        int pipe_write_fd = fd_redirected_by_pipe(get_source(pipe_node));
+        if (pipe_write_fd == -1)
+        {
+            result = report_error(pipe_node, ILLEGAL_FD_ERR_MSG, get_source(pipe_node).c_str());
+            break;
+        }
+        processes.back()->pipe_write_fd = pipe_write_fd;
+
+        /* Get the statement node and make a process from it */
+        const parse_node_t *statement_node = get_child(*job_cont, 1, symbol_statement);
+        assert(statement_node != NULL);
+
+        /* Store the new process (and maybe with an error) */
+        processes.push_back(new process_t());
+        result = this->populate_job_process(j, processes.back(), *statement_node);
+
+        /* Get the next continuation */
+        job_cont = get_child(*job_cont, 2, symbol_job_continuation);
+        assert(job_cont != NULL);
+    }
+
+    /* Return what happened */
+    if (result == parse_execution_success)
+    {
+        /* Link up the processes */
+        assert(! processes.empty());
+        j->first_process = processes.at(0);
+        for (size_t i=1 ; i < processes.size(); i++)
+        {
+            processes.at(i-1)->next = processes.at(i);
+        }
+    }
+    else
+    {
+        /* Clean up processes */
+        for (size_t i=0; i < processes.size(); i++)
+        {
+            const process_t *proc = processes.at(i);
+            processes.at(i) = NULL;
+            delete proc;
+        }
+    }
+    return result;
+}
+
+parse_execution_result_t parse_execution_context_t::run_1_job(const parse_node_t &job_node, const block_t *associated_block)
+{
+    if (should_cancel_execution(associated_block))
+    {
+        return parse_execution_cancelled;
+    }
+
+    // Get terminal modes
+    struct termios tmodes = {};
+    if (get_is_interactive())
+    {
+        if (tcgetattr(STDIN_FILENO, &tmodes))
+        {
+            // need real error handling here
+            wperror(L"tcgetattr");
+            return parse_execution_errored;
+        }
+    }
+
+    /* Increment the eval_level for the duration of this command */
+    scoped_push<int> saved_eval_level(&eval_level, eval_level + 1);
+
+    /* Save the node index */
+    scoped_push<node_offset_t> saved_node_offset(&executing_node_idx, this->get_offset(job_node));
+
+    /* Profiling support */
+    long long start_time = 0, parse_time = 0, exec_time = 0;
+    profile_item_t *profile_item = this->parser->create_profile_item();
+    if (profile_item != NULL)
+    {
+        start_time = get_time();
+    }
+
+    /* When we encounter a block construct (e.g. while loop) in the general case, we create a "block process" that has a pointer to its source. This allows us to handle block-level redirections. However, if there are no redirections, then we can just jump into the block directly, which is significantly faster. */
+    if (job_is_simple_block(job_node))
+    {
+        parse_execution_result_t result = parse_execution_success;
+        
+        const parse_node_t &statement = *get_child(job_node, 0, symbol_statement);
+        const parse_node_t &specific_statement = *get_child(statement, 0);
+        assert(specific_statement_type_is_redirectable_block(specific_statement));
+        switch (specific_statement.type)
+        {
+            case symbol_block_statement:
+                result = this->run_block_statement(specific_statement);
+                break;
+
+            case symbol_if_statement:
+                result = this->run_if_statement(specific_statement);
+                break;
+
+            case symbol_switch_statement:
+                result = this->run_switch_statement(specific_statement);
+                break;
+
+            default:
+                /* Other types should be impossible due to the specific_statement_type_is_redirectable_block check */
+                PARSER_DIE();
+                break;
+        }
+
+        if (profile_item != NULL)
+        {
+            /* Block-types profile a little weird. They have no 'parse' time, and their command is just the block type */
+            exec_time = get_time();
+            profile_item->level=eval_level;
+            profile_item->parse = 0;
+            profile_item->exec=(int)(exec_time-start_time);
+            profile_item->cmd = profiling_cmd_name_for_redirectable_block(specific_statement, this->tree, this->src);
+            profile_item->skipped = false;
+        }
+
+        return result;
+    }
+
+    job_t *j = new job_t(acquire_job_id(), block_io);
+    j->tmodes = tmodes;
+    job_set_flag(j, JOB_CONTROL,
+                 (job_control_mode==JOB_CONTROL_ALL) ||
+                 ((job_control_mode == JOB_CONTROL_INTERACTIVE) && (get_is_interactive())));
+
+    job_set_flag(j, JOB_FOREGROUND, ! tree.job_should_be_backgrounded(job_node));
+
+    job_set_flag(j, JOB_TERMINAL, job_get_flag(j, JOB_CONTROL) \
+                 && (!is_subshell && !is_event));
+
+    job_set_flag(j, JOB_SKIP_NOTIFICATION, is_subshell \
+                 || is_block \
+                 || is_event \
+                 || (!get_is_interactive()));
+
+    /* Tell the current block what its job is. This has to happen before we populate it (#1394) */
+    parser->current_block()->job = j;
+
+    /* Populate the job. This may fail for reasons like command_not_found. If this fails, an error will have been printed */
+    parse_execution_result_t pop_result = this->populate_job_from_job_node(j, job_node, associated_block);
+
+    /* Clean up the job on failure or cancellation */
+    bool populated_job = (pop_result == parse_execution_success);
+    if (! populated_job || this->should_cancel_execution(associated_block))
+    {
+        assert(parser->current_block()->job == j);
+        parser->current_block()->job = NULL;
+        delete j;
+        j = NULL;
+        populated_job = false;
+    }
+
+
+    /* Store time it took to 'parse' the command */
+    if (profile_item != NULL)
+    {
+        parse_time = get_time();
+    }
+
+    if (populated_job)
+    {
+        /* Success. Give the job to the parser - it will clean it up. */
+        parser->job_add(j);
+
+        /* Check to see if this contained any external commands */
+        bool job_contained_external_command = false;
+        for (const process_t *proc = j->first_process; proc != NULL; proc = proc->next)
+        {
+            if (proc->type == EXTERNAL)
+            {
+                job_contained_external_command = true;
+                break;
+            }
+        }
+
+        /* Actually execute the job */
+        exec_job(*this->parser, j);
+
+        /* Only external commands require a new fishd barrier */
+        if (job_contained_external_command)
+        {
+            set_proc_had_barrier(false);
+        }
+    }
+
+    if (profile_item != NULL)
+    {
+        exec_time = get_time();
+        profile_item->level=eval_level;
+        profile_item->parse = (int)(parse_time-start_time);
+        profile_item->exec=(int)(exec_time-parse_time);
+        profile_item->cmd = j ? j->command() : wcstring();
+        profile_item->skipped = ! populated_job;
+    }
+
+
+    /* Clean up jobs. */
+    job_reap(0);
+
+    /* All done */
+    return parse_execution_success;
+}
+
+parse_execution_result_t parse_execution_context_t::run_job_list(const parse_node_t &job_list_node, const block_t *associated_block)
+{
+    assert(job_list_node.type == symbol_job_list);
+
+    parse_execution_result_t result = parse_execution_success;
+    const parse_node_t *job_list = &job_list_node;
+    while (job_list != NULL && ! should_cancel_execution(associated_block))
+    {
+        assert(job_list->type == symbol_job_list);
+
+        // Try pulling out a job
+        const parse_node_t *job = tree.next_node_in_node_list(*job_list, symbol_job, &job_list);
+
+        if (job != NULL)
+        {
+            result = this->run_1_job(*job, associated_block);
+        }
+    }
+
+    /* Returns the last job executed */
+    return result;
+}
+
+parse_execution_result_t parse_execution_context_t::eval_node_at_offset(node_offset_t offset, const block_t *associated_block, const io_chain_t &io)
+{
+    /* Don't ever expect to have an empty tree if this is called */
+    assert(! tree.empty());
+    assert(offset < tree.size());
+
+    /* Apply this block IO for the duration of this function */
+    scoped_push<io_chain_t> block_io_push(&block_io, io);
+
+    const parse_node_t &node = tree.at(offset);
+
+    /* Currently, we only expect to execute the top level job list, or a block node. Assert that. */
+    assert(node.type == symbol_job_list || specific_statement_type_is_redirectable_block(node));
+
+    enum parse_execution_result_t status = parse_execution_success;
+    switch (node.type)
+    {
+        case symbol_job_list:
+        {
+            /* We should only get a job list if it's the very first node. This is because this is the entry point for both top-level execution (the first node) and INTERNAL_BLOCK_NODE execution (which does block statements, but never job lists) */
+            assert(offset == 0);
+            wcstring func_name;
+            const parse_node_t *infinite_recursive_node = this->infinite_recursive_statement_in_job_list(node, &func_name);
+            if (infinite_recursive_node != NULL)
+            {
+                /* We have an infinite recursion */
+                this->report_error(*infinite_recursive_node, INFINITE_FUNC_RECURSION_ERR_MSG, func_name.c_str());
+                status = parse_execution_errored;
+            }
+            else
+            {
+                /* No infinite recursion */
+                status = this->run_job_list(node, associated_block);
+            }
+            break;
+        }
+
+        case symbol_block_statement:
+            status = this->run_block_statement(node);
+            break;
+
+        case symbol_if_statement:
+            status = this->run_if_statement(node);
+            break;
+
+        case symbol_switch_statement:
+            status = this->run_switch_statement(node);
+            break;
+
+        default:
+            /* In principle, we could support other node types. However we never expect to be passed them - see above. */
+            fprintf(stderr, "Unexpected node %ls found in %s\n", node.describe().c_str(), __FUNCTION__);
+            PARSER_DIE();
+            break;
+    }
+
+    return status;
+}
+
+int parse_execution_context_t::line_offset_of_node_at_offset(node_offset_t requested_index)
+{
+    /* If we're not executing anything, return -1 */
+    if (requested_index == NODE_OFFSET_INVALID)
+    {
+        return -1;
+    }
+
+    /* If for some reason we're executing a node without source, return -1 */
+    const parse_node_t &node = tree.at(requested_index);
+    if (! node.has_source())
+    {
+        return -1;
+    }
+    
+    size_t char_offset = tree.at(requested_index).source_start;
+    return this->line_offset_of_character_at_offset(char_offset);
+}
+
+int parse_execution_context_t::line_offset_of_character_at_offset(size_t offset)
+{
+    /* Count the number of newlines, leveraging our cache */
+    assert(offset <= src.size());
+
+    /* Easy hack to handle 0 */
+    if (offset == 0)
+    {
+        return 0;
+    }
+
+    /* We want to return (one plus) the number of newlines at offsets less than the given offset. cached_lineno_count is the number of newlines at indexes less than cached_lineno_offset. */
+    const wchar_t *str = src.c_str();
+    if (offset > cached_lineno_offset)
+    {
+        size_t i;
+        for (i = cached_lineno_offset; str[i] != L'\0' && i < offset; i++)
+        {
+            /* Add one for every newline we find in the range [cached_lineno_offset, offset) */
+            if (str[i] == L'\n')
+            {
+                cached_lineno_count++;
+            }
+        }
+        cached_lineno_offset = i; //note: i, not offset, in case offset is beyond the length of the string
+    }
+    else if (offset < cached_lineno_offset)
+    {
+        /* Subtract one for every newline we find in the range [offset, cached_lineno_offset) */
+        for (size_t i = offset; i < cached_lineno_offset; i++)
+        {
+            if (str[i] == L'\n')
+            {
+                cached_lineno_count--;
+            }
+        }
+        cached_lineno_offset = offset;
+    }
+    return cached_lineno_count;
+}
+
+int parse_execution_context_t::get_current_line_number()
+{
+    int line_number = -1;
+    int line_offset = this->line_offset_of_node_at_offset(this->executing_node_idx);
+    if (line_offset >= 0)
+    {
+        /* The offset is 0 based; the number is 1 based */
+        line_number = line_offset + 1;
+    }
+    return line_number;
+}
+
+int parse_execution_context_t::get_current_source_offset() const
+{
+    int result = -1;
+    if (executing_node_idx != NODE_OFFSET_INVALID)
+    {
+        const parse_node_t &node = tree.at(executing_node_idx);
+        if (node.has_source())
+        {
+            result = static_cast<int>(node.source_start);
+        }
+    }
+    return result;
+}
diff --git a/src/parse_execution.h b/src/parse_execution.h
new file mode 100644
index 00000000..7e1c1f46
--- /dev/null
+++ b/src/parse_execution.h
@@ -0,0 +1,145 @@
+/**\file parse_execution.h
+
+   Provides the "linkage" between a parse_node_tree_t and actual execution structures (job_t, etc.).
+*/
+
+#ifndef FISH_PARSE_EXECUTION_H
+#define FISH_PARSE_EXECUTION_H
+
+#include <stddef.h>
+#include "common.h"
+#include "io.h"
+#include "parse_constants.h"
+#include "parse_tree.h"
+#include "proc.h"
+
+class parser_t;
+struct block_t;
+
+enum parse_execution_result_t
+{
+    /* The job was successfully executed (though it have failed on its own). */
+    parse_execution_success,
+
+    /* The job did not execute due to some error (e.g. failed to wildcard expand). An error will have been printed and proc_last_status will have been set. */
+    parse_execution_errored,
+
+    /* The job was cancelled (e.g. Ctrl-C) */
+    parse_execution_cancelled,
+
+    /* The job was skipped (e.g. due to a not-taken 'and' command). This is a special return allowed only from the populate functions, not the run functions. */
+    parse_execution_skipped
+};
+
+class parse_execution_context_t
+{
+private:
+    const parse_node_tree_t tree;
+    const wcstring src;
+    io_chain_t block_io;
+    parser_t * const parser;
+    //parse_error_list_t errors;
+
+    int eval_level;
+
+    /* The currently executing node index, used to indicate the line number */
+    node_offset_t executing_node_idx;
+
+    /* Cached line number information */
+    size_t cached_lineno_offset;
+    int cached_lineno_count;
+
+    /* No copying allowed */
+    parse_execution_context_t(const parse_execution_context_t&);
+    parse_execution_context_t& operator=(const parse_execution_context_t&);
+
+    /* Should I cancel? */
+    bool should_cancel_execution(const block_t *block) const;
+
+    /* Ways that we can stop executing a block. These are in a sort of ascending order of importance, e.g. `exit` should trump `break` */
+    enum execution_cancellation_reason_t
+    {
+        execution_cancellation_none,
+        execution_cancellation_loop_control,
+        execution_cancellation_skip,
+        execution_cancellation_exit
+    };
+    execution_cancellation_reason_t cancellation_reason(const block_t *block) const;
+
+    /* Report an error. Always returns true. */
+    parse_execution_result_t report_error(const parse_node_t &node, const wchar_t *fmt, ...) const;
+    parse_execution_result_t report_errors(const parse_error_list_t &errors) const;
+
+    /* Wildcard error helper */
+    parse_execution_result_t report_unmatched_wildcard_error(const parse_node_t &unmatched_wildcard);
+
+    /* Command not found support */
+    parse_execution_result_t handle_command_not_found(const wcstring &cmd, const parse_node_t &statement_node, int err_code);
+
+    /* Utilities */
+    wcstring get_source(const parse_node_t &node) const;
+    const parse_node_t *get_child(const parse_node_t &parent, node_offset_t which, parse_token_type_t expected_type = token_type_invalid) const;
+    node_offset_t get_offset(const parse_node_t &node) const;
+    const parse_node_t *infinite_recursive_statement_in_job_list(const parse_node_t &job_list, wcstring *out_func_name) const;
+
+    /* Indicates whether a job is a simple block (one block, no redirections) */
+    bool job_is_simple_block(const parse_node_t &node) const;
+
+    enum process_type_t process_type_for_command(const parse_node_t &plain_statement, const wcstring &cmd) const;
+
+    /* These create process_t structures from statements */
+    parse_execution_result_t populate_job_process(job_t *job, process_t *proc, const parse_node_t &statement_node);
+    parse_execution_result_t populate_boolean_process(job_t *job, process_t *proc, const parse_node_t &bool_statement);
+    parse_execution_result_t populate_plain_process(job_t *job, process_t *proc, const parse_node_t &statement);
+    parse_execution_result_t populate_block_process(job_t *job, process_t *proc, const parse_node_t &statement_node);
+
+    /* These encapsulate the actual logic of various (block) statements. */
+    parse_execution_result_t run_block_statement(const parse_node_t &statement);
+    parse_execution_result_t run_for_statement(const parse_node_t &header, const parse_node_t &contents);
+    parse_execution_result_t run_if_statement(const parse_node_t &statement);
+    parse_execution_result_t run_switch_statement(const parse_node_t &statement);
+    parse_execution_result_t run_while_statement(const parse_node_t &header, const parse_node_t &contents);
+    parse_execution_result_t run_function_statement(const parse_node_t &header, const parse_node_t &block_end_command);
+    parse_execution_result_t run_begin_statement(const parse_node_t &header, const parse_node_t &contents);
+
+    parse_execution_result_t determine_arguments(const parse_node_t &parent, wcstring_list_t *out_arguments);
+
+    /* Determines the IO chain. Returns true on success, false on error */
+    bool determine_io_chain(const parse_node_t &statement, io_chain_t *out_chain);
+
+    parse_execution_result_t run_1_job(const parse_node_t &job_node, const block_t *associated_block);
+    parse_execution_result_t run_job_list(const parse_node_t &job_list_node, const block_t *associated_block);
+    parse_execution_result_t populate_job_from_job_node(job_t *j, const parse_node_t &job_node, const block_t *associated_block);
+
+    /* Returns the line number of the node at the given index, indexed from 0. Not const since it touches cached_lineno_offset */
+    int line_offset_of_node_at_offset(node_offset_t idx);
+    int line_offset_of_character_at_offset(size_t char_idx);
+
+public:
+    parse_execution_context_t(const parse_node_tree_t &t, const wcstring &s, parser_t *p, int initial_eval_level);
+
+    /* Returns the current eval level */
+    int current_eval_level() const
+    {
+        return eval_level;
+    }
+
+    /* Returns the current line number, indexed from 1. Not const since it touches cached_lineno_offset */
+    int get_current_line_number();
+
+    /* Returns the source offset, or -1 */
+    int get_current_source_offset() const;
+
+    /* Returns the source string */
+    const wcstring &get_source() const
+    {
+        return src;
+    }
+
+    /* Start executing at the given node offset. Returns 0 if there was no error, 1 if there was an error */
+    parse_execution_result_t eval_node_at_offset(node_offset_t offset, const block_t *associated_block, const io_chain_t &io);
+
+};
+
+
+#endif
diff --git a/src/parse_productions.cpp b/src/parse_productions.cpp
new file mode 100644
index 00000000..0d28c6a1
--- /dev/null
+++ b/src/parse_productions.cpp
@@ -0,0 +1,579 @@
+#include "parse_productions.h"
+#include <assert.h>
+#include <stdio.h>
+#include "parse_tree.h"
+
+using namespace parse_productions;
+#define NO_PRODUCTION ((production_option_idx_t)(-1))
+
+static bool production_is_empty(const production_t production)
+{
+    return production[0] == token_type_invalid;
+}
+
+/* Empty productions are allowed but must be first. Validate that the given production is in the valid range, i.e. it is either not empty or there is a non-empty production after it */
+static bool production_is_valid(const production_options_t production_list, production_option_idx_t which)
+{
+    assert(which >= 0);
+    if (which >= MAX_PRODUCTIONS)
+        return false;
+
+    bool nonempty_found = false;
+    for (int i=which; i < MAX_PRODUCTIONS; i++)
+    {
+        if (! production_is_empty(production_list[i]))
+        {
+            nonempty_found = true;
+            break;
+        }
+    }
+    return nonempty_found;
+}
+
+#define PRODUCTIONS(sym) static const production_options_t productions_##sym
+#define RESOLVE(sym) static production_option_idx_t resolve_##sym (const parse_token_t &token1, const parse_token_t &token2)
+#define RESOLVE_ONLY(sym) static production_option_idx_t resolve_##sym (const parse_token_t &input1, const parse_token_t &input2) { return 0; }
+
+#define KEYWORD(x) ((x) + LAST_TOKEN_OR_SYMBOL + 1)
+
+
+/* A job_list is a list of jobs, separated by semicolons or newlines */
+PRODUCTIONS(job_list) =
+{
+    {},
+    {symbol_job, symbol_job_list},
+    {parse_token_type_end, symbol_job_list}
+};
+
+RESOLVE(job_list)
+{
+    switch (token1.type)
+    {
+        case parse_token_type_string:
+            // some keywords are special
+            switch (token1.keyword)
+            {
+                case parse_keyword_end:
+                case parse_keyword_else:
+                case parse_keyword_case:
+                    // End this job list
+                    return 0;
+
+                default:
+                    // Normal string
+                    return 1;
+            }
+
+        case parse_token_type_pipe:
+        case parse_token_type_redirection:
+        case parse_token_type_background:
+            return 1;
+
+        case parse_token_type_end:
+            // Empty line
+            return 2;
+
+        case parse_token_type_terminate:
+            // no more commands, just transition to empty
+            return 0;
+
+        default:
+            return NO_PRODUCTION;
+    }
+}
+
+/* A job is a non-empty list of statements, separated by pipes. (Non-empty is useful for cases like if statements, where we require a command). To represent "non-empty", we require a statement, followed by a possibly empty job_continuation */
+
+PRODUCTIONS(job) =
+{
+    {symbol_statement, symbol_job_continuation, symbol_optional_background}
+};
+RESOLVE_ONLY(job)
+
+PRODUCTIONS(job_continuation) =
+{
+    {},
+    {parse_token_type_pipe, symbol_statement, symbol_job_continuation}
+};
+RESOLVE(job_continuation)
+{
+    switch (token1.type)
+    {
+        case parse_token_type_pipe:
+            // Pipe, continuation
+            return 1;
+
+        default:
+            // Not a pipe, no job continuation
+            return 0;
+    }
+}
+
+/* A statement is a normal command, or an if / while / and etc */
+PRODUCTIONS(statement) =
+{
+    {symbol_boolean_statement},
+    {symbol_block_statement},
+    {symbol_if_statement},
+    {symbol_switch_statement},
+    {symbol_decorated_statement}
+};
+RESOLVE(statement)
+{
+    /* The only block-like builtin that takes any parameters is 'function' So go to decorated statements if the subsequent token looks like '--'.
+        The logic here is subtle:
+            If we are 'begin', then we expect to be invoked with no arguments.
+            If we are 'function', then we are a non-block if we are invoked with -h or --help
+            If we are anything else, we require an argument, so do the same thing if the subsequent token is a statement terminator.
+    */
+
+    if (token1.type == parse_token_type_string)
+    {
+        // If we are a function, then look for help arguments
+        // Otherwise, if the next token looks like an option (starts with a dash), then parse it as a decorated statement
+        if (token1.keyword == parse_keyword_function && token2.is_help_argument)
+        {
+            return 4;
+        }
+        else if (token1.keyword != parse_keyword_function && token2.has_dash_prefix)
+        {
+            return 4;
+        }
+
+        // Likewise if the next token doesn't look like an argument at all. This corresponds to e.g. a "naked if".
+        bool naked_invocation_invokes_help = (token1.keyword != parse_keyword_begin && token1.keyword != parse_keyword_end);
+        if (naked_invocation_invokes_help && (token2.type == parse_token_type_end || token2.type == parse_token_type_terminate))
+        {
+            return 4;
+        }
+
+    }
+
+    switch (token1.type)
+    {
+        case parse_token_type_string:
+            switch (token1.keyword)
+            {
+                case parse_keyword_and:
+                case parse_keyword_or:
+                case parse_keyword_not:
+                    return 0;
+
+                case parse_keyword_for:
+                case parse_keyword_while:
+                case parse_keyword_function:
+                case parse_keyword_begin:
+                    return 1;
+
+                case parse_keyword_if:
+                    return 2;
+
+                case parse_keyword_else:
+                    return NO_PRODUCTION;
+
+                case parse_keyword_switch:
+                    return 3;
+
+                case parse_keyword_end:
+                    return NO_PRODUCTION;
+
+                    // All other keywords fall through to decorated statement
+                default:
+                    return 4;
+            }
+            break;
+
+        case parse_token_type_pipe:
+        case parse_token_type_redirection:
+        case parse_token_type_background:
+        case parse_token_type_terminate:
+            return NO_PRODUCTION;
+            //parse_error(L"statement", token);
+
+        default:
+            return NO_PRODUCTION;
+    }
+}
+
+PRODUCTIONS(if_statement) =
+{
+    {symbol_if_clause, symbol_else_clause, symbol_end_command, symbol_arguments_or_redirections_list}
+};
+RESOLVE_ONLY(if_statement)
+
+PRODUCTIONS(if_clause) =
+{
+    { KEYWORD(parse_keyword_if), symbol_job, parse_token_type_end, symbol_job_list }
+};
+RESOLVE_ONLY(if_clause)
+
+PRODUCTIONS(else_clause) =
+{
+    { },
+    { KEYWORD(parse_keyword_else), symbol_else_continuation }
+};
+RESOLVE(else_clause)
+{
+    switch (token1.keyword)
+    {
+        case parse_keyword_else:
+            return 1;
+        default:
+            return 0;
+    }
+}
+
+PRODUCTIONS(else_continuation) =
+{
+    {symbol_if_clause, symbol_else_clause},
+    {parse_token_type_end, symbol_job_list}
+};
+RESOLVE(else_continuation)
+{
+    switch (token1.keyword)
+    {
+        case parse_keyword_if:
+            return 0;
+        default:
+            return 1;
+    }
+}
+
+PRODUCTIONS(switch_statement) =
+{
+    { KEYWORD(parse_keyword_switch), symbol_argument, parse_token_type_end, symbol_case_item_list, symbol_end_command, symbol_arguments_or_redirections_list}
+};
+RESOLVE_ONLY(switch_statement)
+
+PRODUCTIONS(case_item_list) =
+{
+    {},
+    {symbol_case_item, symbol_case_item_list},
+    {parse_token_type_end, symbol_case_item_list}
+};
+RESOLVE(case_item_list)
+{
+    if (token1.keyword == parse_keyword_case) return 1;
+    else if (token1.type == parse_token_type_end) return 2; //empty line
+    else return 0;
+}
+
+PRODUCTIONS(case_item) =
+{
+    {KEYWORD(parse_keyword_case), symbol_argument_list, parse_token_type_end, symbol_job_list}
+};
+RESOLVE_ONLY(case_item)
+
+PRODUCTIONS(argument_list) =
+{
+    {},
+    {symbol_argument, symbol_argument_list}
+};
+RESOLVE(argument_list)
+{
+    switch (token1.type)
+    {
+        case parse_token_type_string:
+            return 1;
+        default:
+            return 0;
+    }
+}
+
+PRODUCTIONS(freestanding_argument_list) =
+{
+    {},
+    {symbol_argument, symbol_freestanding_argument_list},
+    {parse_token_type_end, symbol_freestanding_argument_list},
+};
+RESOLVE(freestanding_argument_list)
+{
+    switch (token1.type)
+    {
+        case parse_token_type_string:
+            return 1;
+        case parse_token_type_end:
+            return 2;
+        default:
+            return 0;
+    }
+}
+
+
+PRODUCTIONS(block_statement) =
+{
+    {symbol_block_header, symbol_job_list, symbol_end_command, symbol_arguments_or_redirections_list}
+};
+RESOLVE_ONLY(block_statement)
+
+PRODUCTIONS(block_header) =
+{
+    {symbol_for_header},
+    {symbol_while_header},
+    {symbol_function_header},
+    {symbol_begin_header}
+};
+RESOLVE(block_header)
+{
+    switch (token1.keyword)
+    {
+        case parse_keyword_for:
+            return 0;
+        case parse_keyword_while:
+            return 1;
+        case parse_keyword_function:
+            return 2;
+        case parse_keyword_begin:
+            return 3;
+        default:
+            return NO_PRODUCTION;
+    }
+}
+
+PRODUCTIONS(for_header) =
+{
+    {KEYWORD(parse_keyword_for), parse_token_type_string, KEYWORD
+        (parse_keyword_in), symbol_argument_list, parse_token_type_end}
+};
+RESOLVE_ONLY(for_header)
+
+PRODUCTIONS(while_header) =
+{
+    {KEYWORD(parse_keyword_while), symbol_job, parse_token_type_end}
+};
+RESOLVE_ONLY(while_header)
+
+PRODUCTIONS(begin_header) =
+{
+    {KEYWORD(parse_keyword_begin)}
+};
+RESOLVE_ONLY(begin_header)
+
+PRODUCTIONS(function_header) =
+{
+    {KEYWORD(parse_keyword_function), symbol_argument, symbol_argument_list, parse_token_type_end}
+};
+RESOLVE_ONLY(function_header)
+
+/* A boolean statement is AND or OR or NOT */
+PRODUCTIONS(boolean_statement) =
+{
+    {KEYWORD(parse_keyword_and), symbol_statement},
+    {KEYWORD(parse_keyword_or), symbol_statement},
+    {KEYWORD(parse_keyword_not), symbol_statement}
+};
+RESOLVE(boolean_statement)
+{
+    switch (token1.keyword)
+    {
+        case parse_keyword_and:
+            return 0;
+        case parse_keyword_or:
+            return 1;
+        case parse_keyword_not:
+            return 2;
+        default:
+            return NO_PRODUCTION;
+    }
+}
+
+PRODUCTIONS(decorated_statement) =
+{
+    {symbol_plain_statement},
+    {KEYWORD(parse_keyword_command), symbol_plain_statement},
+    {KEYWORD(parse_keyword_builtin), symbol_plain_statement},
+    {KEYWORD(parse_keyword_exec), symbol_plain_statement}
+};
+RESOLVE(decorated_statement)
+{
+    /* If this is e.g. 'command --help' then the command is 'command' and not a decoration. If the second token is not a string, then this is a naked 'command' and we should execute it as undecorated. */
+    if (token2.type != parse_token_type_string || token2.has_dash_prefix)
+    {
+        return 0;
+    }
+
+    switch (token1.keyword)
+    {
+        default:
+            return 0;
+        case parse_keyword_command:
+            return 1;
+        case parse_keyword_builtin:
+            return 2;
+        case parse_keyword_exec:
+            return 3;
+    }
+}
+
+PRODUCTIONS(plain_statement) =
+{
+    {parse_token_type_string, symbol_arguments_or_redirections_list}
+};
+RESOLVE_ONLY(plain_statement)
+
+PRODUCTIONS(arguments_or_redirections_list) =
+{
+    {},
+    {symbol_argument_or_redirection, symbol_arguments_or_redirections_list}
+};
+RESOLVE(arguments_or_redirections_list)
+{
+    switch (token1.type)
+    {
+        case parse_token_type_string:
+        case parse_token_type_redirection:
+            return 1;
+        default:
+            return 0;
+    }
+}
+
+PRODUCTIONS(argument_or_redirection) =
+{
+    {symbol_argument},
+    {symbol_redirection}
+};
+RESOLVE(argument_or_redirection)
+{
+    switch (token1.type)
+    {
+        case parse_token_type_string:
+            return 0;
+        case parse_token_type_redirection:
+            return 1;
+        default:
+            return NO_PRODUCTION;
+    }
+}
+
+PRODUCTIONS(argument) =
+{
+    {parse_token_type_string}
+};
+RESOLVE_ONLY(argument)
+
+PRODUCTIONS(redirection) =
+{
+    {parse_token_type_redirection, parse_token_type_string}
+};
+RESOLVE_ONLY(redirection)
+
+PRODUCTIONS(optional_background) =
+{
+    {},
+    { parse_token_type_background }
+};
+
+RESOLVE(optional_background)
+{
+    switch (token1.type)
+    {
+        case parse_token_type_background:
+            return 1;
+        default:
+            return 0;
+    }
+}
+
+PRODUCTIONS(end_command) =
+{
+    {KEYWORD(parse_keyword_end)}
+};
+RESOLVE_ONLY(end_command)
+
+#define TEST(sym) case (symbol_##sym): production_list = & productions_ ## sym ; resolver = resolve_ ## sym ; break;
+const production_t *parse_productions::production_for_token(parse_token_type_t node_type, const parse_token_t &input1, const parse_token_t &input2, production_option_idx_t *out_which_production, wcstring *out_error_text)
+{
+    const bool log_it = false;
+    if (log_it)
+    {
+        fprintf(stderr, "Resolving production for %ls with input token <%ls>\n", token_type_description(node_type).c_str(), input1.describe().c_str());
+    }
+
+    /* Fetch the list of productions and the function to resolve them */
+    const production_options_t *production_list = NULL;
+    production_option_idx_t (*resolver)(const parse_token_t &input1, const parse_token_t &input2) = NULL;
+    switch (node_type)
+    {
+            TEST(job_list)
+            TEST(job)
+            TEST(statement)
+            TEST(job_continuation)
+            TEST(boolean_statement)
+            TEST(block_statement)
+            TEST(if_statement)
+            TEST(if_clause)
+            TEST(else_clause)
+            TEST(else_continuation)
+            TEST(switch_statement)
+            TEST(decorated_statement)
+            TEST(case_item_list)
+            TEST(case_item)
+            TEST(argument_list)
+            TEST(freestanding_argument_list)
+            TEST(block_header)
+            TEST(for_header)
+            TEST(while_header)
+            TEST(begin_header)
+            TEST(function_header)
+            TEST(plain_statement)
+            TEST(arguments_or_redirections_list)
+            TEST(argument_or_redirection)
+            TEST(argument)
+            TEST(redirection)
+            TEST(optional_background)
+            TEST(end_command)
+
+        case parse_token_type_string:
+        case parse_token_type_pipe:
+        case parse_token_type_redirection:
+        case parse_token_type_background:
+        case parse_token_type_end:
+        case parse_token_type_terminate:
+            fprintf(stderr, "Terminal token type %ls passed to %s\n", token_type_description(node_type).c_str(), __FUNCTION__);
+            PARSER_DIE();
+            break;
+
+        case parse_special_type_parse_error:
+        case parse_special_type_tokenizer_error:
+        case parse_special_type_comment:
+            fprintf(stderr, "Special type %ls passed to %s\n", token_type_description(node_type).c_str(), __FUNCTION__);
+            PARSER_DIE();
+            break;
+
+
+        case token_type_invalid:
+            fprintf(stderr, "token_type_invalid passed to %s\n", __FUNCTION__);
+            PARSER_DIE();
+            break;
+
+    }
+    PARSE_ASSERT(production_list != NULL);
+    PARSE_ASSERT(resolver != NULL);
+
+    const production_t *result = NULL;
+    production_option_idx_t which = resolver(input1, input2);
+
+    if (log_it)
+    {
+        fprintf(stderr, "\tresolved to %u\n", (unsigned)which);
+    }
+
+
+    if (which == NO_PRODUCTION)
+    {
+        if (log_it)
+        {
+            fprintf(stderr, "Node type '%ls' has no production for input '%ls' (in %s)\n", token_type_description(node_type).c_str(), input1.describe().c_str(), __FUNCTION__);
+        }
+        result = NULL;
+    }
+    else
+    {
+        PARSE_ASSERT(production_is_valid(*production_list, which));
+        result = &((*production_list)[which]);
+    }
+    *out_which_production = which;
+    return result;
+}
+
diff --git a/src/parse_productions.h b/src/parse_productions.h
new file mode 100644
index 00000000..1d64ccfc
--- /dev/null
+++ b/src/parse_productions.h
@@ -0,0 +1,74 @@
+/**\file parse_tree.h
+
+    Programmatic representation of fish code.
+*/
+
+#ifndef FISH_PARSE_TREE_CONSTRUCTION_H
+#define FISH_PARSE_TREE_CONSTRUCTION_H
+
+#include <stdint.h>                     // for uint8_t, uint32_t
+#include "common.h"                     // for wcstring
+#include "parse_constants.h"            // for parse_token_type_t, etc
+
+struct parse_token_t;
+
+namespace parse_productions
+{
+
+#define MAX_PRODUCTIONS 5
+#define MAX_SYMBOLS_PER_PRODUCTION 6
+
+typedef uint32_t production_tag_t;
+
+/* A production is an array of unsigned char. Symbols are encoded directly as their symbol value. Keywords are encoded with an offset of LAST_TOKEN_OR_SYMBOL + 1. So essentially we glom together keywords and symbols. */
+typedef uint8_t production_element_t;
+
+/* An index into a production option list */
+typedef uint8_t production_option_idx_t;
+
+/* A production is an array of production elements */
+typedef production_element_t const production_t[MAX_SYMBOLS_PER_PRODUCTION];
+
+/* A production options is an array of (possible) productions */
+typedef production_t production_options_t[MAX_PRODUCTIONS];
+
+/* Resolve the type from a production element */
+inline parse_token_type_t production_element_type(production_element_t elem)
+{
+    if (elem > LAST_TOKEN_OR_SYMBOL)
+    {
+        return parse_token_type_string;
+    }
+    else
+    {
+        return static_cast<parse_token_type_t>(elem);
+    }
+}
+
+/* Resolve the keyword from a production element */
+inline parse_keyword_t production_element_keyword(production_element_t elem)
+{
+    if (elem > LAST_TOKEN_OR_SYMBOL)
+    {
+        // First keyword is LAST_TOKEN_OR_SYMBOL + 1
+        return static_cast<parse_keyword_t>(elem - LAST_TOKEN_OR_SYMBOL - 1);
+    }
+    else
+    {
+        return parse_keyword_none;
+    }
+}
+
+/* Check if an element is valid */
+inline bool production_element_is_valid(production_element_t elem)
+{
+    return elem != token_type_invalid;
+}
+
+/* Fetch a production. We are passed two input tokens. The first input token is guaranteed to not be invalid; the second token may be invalid if there's no more tokens. */
+const production_t *production_for_token(parse_token_type_t node_type, const parse_token_t &input1, const parse_token_t &input2, production_option_idx_t *out_which_production, wcstring *out_error_text);
+
+}
+
+
+#endif
diff --git a/src/parse_tree.cpp b/src/parse_tree.cpp
new file mode 100644
index 00000000..f357d5ff
--- /dev/null
+++ b/src/parse_tree.cpp
@@ -0,0 +1,1778 @@
+#include <assert.h>
+#include <stdarg.h>
+#include <stddef.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <wchar.h>
+#include <string>
+#include "common.h"
+#include "parse_constants.h"
+#include "parse_productions.h"
+#include "parse_tree.h"
+#include "tokenizer.h"
+#include "fallback.h"
+#include "wutil.h" // IWYU pragma: keep - needed for wgettext
+#include "proc.h"
+#include <vector>
+#include <algorithm>
+
+using namespace parse_productions;
+
+static bool production_is_empty(const production_t *production)
+{
+    return (*production)[0] == token_type_invalid;
+}
+
+/** Returns a string description of this parse error */
+wcstring parse_error_t::describe_with_prefix(const wcstring &src, const wcstring &prefix, bool is_interactive, bool skip_caret) const
+{
+    wcstring result = text;
+    if (! skip_caret && source_start < src.size() && source_start + source_length <= src.size())
+    {
+        // Locate the beginning of this line of source
+        size_t line_start = 0;
+
+        // Look for a newline prior to source_start. If we don't find one, start at the beginning of the string; otherwise start one past the newline. Note that source_start may itself point at a newline; we want to find the newline before it.
+        if (source_start > 0)
+        {
+            size_t newline = src.find_last_of(L'\n', source_start - 1);
+            if (newline != wcstring::npos)
+            {
+                line_start = newline + 1;
+            }
+        }
+
+        // Look for the newline after the source range. If the source range itself includes a newline, that's the one we want, so start just before the end of the range
+        size_t last_char_in_range = (source_length == 0 ? source_start : source_start + source_length - 1);
+        size_t line_end = src.find(L'\n', last_char_in_range);
+        if (line_end == wcstring::npos)
+        {
+            line_end = src.size();
+        }
+
+        assert(line_end >= line_start);
+        assert(source_start >= line_start);
+
+        // Don't include the caret and line if we're interactive this is the first line, because then it's obvious
+        bool skip_caret = (is_interactive && source_start == 0);
+
+        if (! skip_caret)
+        {
+            // Append the line of text.
+            if (! result.empty())
+            {
+                result.push_back(L'\n');
+            }
+            result.append(prefix);
+            result.append(src, line_start, line_end - line_start);
+
+            // Append the caret line. The input source may include tabs; for that reason we construct a "caret line" that has tabs in corresponding positions
+            const wcstring line_to_measure = prefix + wcstring(src, line_start, source_start - line_start);
+            wcstring caret_space_line;
+            caret_space_line.reserve(source_start - line_start);
+            for (size_t i=0; i < line_to_measure.size(); i++)
+            {
+                wchar_t wc = line_to_measure.at(i);
+                if (wc == L'\t')
+                {
+                    caret_space_line.push_back(L'\t');
+                }
+                else if (wc == L'\n')
+                {
+                    /* It's possible that the source_start points at a newline itself. In that case, pretend it's a space. We only expect this to be at the end of the string. */
+                    caret_space_line.push_back(L' ');
+                }
+                else
+                {
+                    int width = fish_wcwidth(wc);
+                    if (width > 0)
+                    {
+                        caret_space_line.append(static_cast<size_t>(width), L' ');
+                    }
+                }
+            }
+            result.push_back(L'\n');
+            result.append(caret_space_line);
+            result.push_back(L'^');
+        }
+    }
+    return result;
+}
+
+wcstring parse_error_t::describe(const wcstring &src) const
+{
+    return this->describe_with_prefix(src, wcstring(), get_is_interactive(), false);
+}
+
+wcstring parse_errors_description(const parse_error_list_t &errors, const wcstring &src, const wchar_t *prefix)
+{
+    wcstring target;
+    for (size_t i=0; i < errors.size(); i++)
+    {
+        if (i > 0)
+        {
+            target.push_back(L'\n');
+        }
+        if (prefix != NULL)
+        {
+            target.append(prefix);
+            target.append(L": ");
+        }
+        target.append(errors.at(i).describe(src));
+    }
+    return target;
+}
+
+void parse_error_offset_source_start(parse_error_list_t *errors, size_t amt)
+{
+    assert(errors != NULL);
+    if (amt > 0)
+    {
+        size_t i, max = errors->size();
+        for (i=0; i < max; i++)
+        {
+            parse_error_t *error = &errors->at(i);
+            /* preserve the special meaning of -1 as 'unknown' */
+            if (error->source_start != SOURCE_LOCATION_UNKNOWN)
+            {
+                error->source_start += amt;
+            }
+        }
+    }
+}
+
+/** Returns a string description of the given token type */
+wcstring token_type_description(parse_token_type_t type)
+{
+    switch (type)
+    {
+        case token_type_invalid:
+            return L"invalid";
+
+        case symbol_job_list:
+            return L"job_list";
+        case symbol_job:
+            return L"job";
+        case symbol_job_continuation:
+            return L"job_continuation";
+
+        case symbol_statement:
+            return L"statement";
+        case symbol_block_statement:
+            return L"block_statement";
+        case symbol_block_header:
+            return L"block_header";
+        case symbol_for_header:
+            return L"for_header";
+        case symbol_while_header:
+            return L"while_header";
+        case symbol_begin_header:
+            return L"begin_header";
+        case symbol_function_header:
+            return L"function_header";
+
+        case symbol_if_statement:
+            return L"if_statement";
+        case symbol_if_clause:
+            return L"if_clause";
+        case symbol_else_clause:
+            return L"else_clause";
+        case symbol_else_continuation:
+            return L"else_continuation";
+
+        case symbol_switch_statement:
+            return L"switch_statement";
+        case symbol_case_item_list:
+            return L"case_item_list";
+        case symbol_case_item:
+            return L"case_item";
+
+        case symbol_argument_list:
+            return L"argument_list";
+        case symbol_freestanding_argument_list:
+            return L"freestanding_argument_list";
+
+        case symbol_boolean_statement:
+            return L"boolean_statement";
+        case symbol_decorated_statement:
+            return L"decorated_statement";
+        case symbol_plain_statement:
+            return L"plain_statement";
+        case symbol_arguments_or_redirections_list:
+            return L"arguments_or_redirections_list";
+        case symbol_argument_or_redirection:
+            return L"argument_or_redirection";
+        case symbol_argument:
+            return L"symbol_argument";
+        case symbol_redirection:
+            return L"symbol_redirection";
+        case symbol_optional_background:
+            return L"optional_background";
+        case symbol_end_command:
+            return L"symbol_end_command";
+
+
+        case parse_token_type_string:
+            return L"token_string";
+        case parse_token_type_pipe:
+            return L"token_pipe";
+        case parse_token_type_redirection:
+            return L"token_redirection";
+        case parse_token_type_background:
+            return L"token_background";
+        case parse_token_type_end:
+            return L"token_end";
+        case parse_token_type_terminate:
+            return L"token_terminate";
+
+        case parse_special_type_parse_error:
+            return L"parse_error";
+        case parse_special_type_tokenizer_error:
+            return L"tokenizer_error";
+        case parse_special_type_comment:
+            return L"comment";
+
+    }
+    return format_string(L"Unknown token type %ld", static_cast<long>(type));
+}
+
+#define LONGIFY(x) L ## x
+#define KEYWORD_MAP(x) { parse_keyword_ ## x , LONGIFY(#x) }
+static const struct
+{
+    const parse_keyword_t keyword;
+    const wchar_t * const name;
+}
+keyword_map[] =
+{
+    /* Note that these must be sorted (except for the first), so that we can do binary search */
+    KEYWORD_MAP(none),
+    KEYWORD_MAP(and),
+    KEYWORD_MAP(begin),
+    KEYWORD_MAP(builtin),
+    KEYWORD_MAP(case),
+    KEYWORD_MAP(command),
+    KEYWORD_MAP(else),
+    KEYWORD_MAP(end),
+    KEYWORD_MAP(exec),
+    KEYWORD_MAP(for),
+    KEYWORD_MAP(function),
+    KEYWORD_MAP(if),
+    KEYWORD_MAP(in),
+    KEYWORD_MAP(not),
+    KEYWORD_MAP(or),
+    KEYWORD_MAP(switch),
+    KEYWORD_MAP(while)
+};
+
+wcstring keyword_description(parse_keyword_t k)
+{
+    if (k >= 0 && k <= LAST_KEYWORD)
+    {
+        return keyword_map[k].name;
+    }
+    else
+    {
+        return format_string(L"Unknown keyword type %ld", static_cast<long>(k));
+    }
+}
+
+static wcstring token_type_user_presentable_description(parse_token_type_t type, parse_keyword_t keyword = parse_keyword_none)
+{
+    if (keyword != parse_keyword_none)
+    {
+        return format_string(L"keyword '%ls'", keyword_description(keyword).c_str());
+    }
+
+    switch (type)
+    {
+        /* Hackish. We only support the following types. */
+        case symbol_statement:
+            return L"a command";
+
+        case symbol_argument:
+            return L"an argument";
+
+        case parse_token_type_string:
+            return L"a string";
+
+        case parse_token_type_pipe:
+            return L"a pipe";
+
+        case parse_token_type_redirection:
+            return L"a redirection";
+
+        case parse_token_type_background:
+            return L"a '&'";
+
+        case parse_token_type_end:
+            return L"end of the statement";
+
+        case parse_token_type_terminate:
+            return L"end of the input";
+
+        default:
+            return format_string(L"a %ls", token_type_description(type).c_str());
+    }
+}
+
+static wcstring block_type_user_presentable_description(parse_token_type_t type)
+{
+    switch (type)
+    {
+        case symbol_for_header:
+            return L"for loop";
+
+        case symbol_while_header:
+            return L"while loop";
+
+        case symbol_function_header:
+            return L"function definition";
+
+        case symbol_begin_header:
+            return L"begin";
+
+        case symbol_if_statement:
+            return L"if statement";
+
+        case symbol_switch_statement:
+            return L"switch statement";
+
+        default:
+            return token_type_description(type);
+    }
+}
+
+/** Returns a string description of the given parse node */
+wcstring parse_node_t::describe(void) const
+{
+    wcstring result = token_type_description(type);
+    if (type < FIRST_TERMINAL_TYPE)
+    {
+        append_format(result, L" (prod %d)", this->production_idx);
+    }
+    return result;
+}
+
+
+/** Returns a string description of the given parse token */
+wcstring parse_token_t::describe() const
+{
+    wcstring result = token_type_description(type);
+    if (keyword != parse_keyword_none)
+    {
+        append_format(result, L" <%ls>", keyword_description(keyword).c_str());
+    }
+    return result;
+}
+
+/** A string description appropriate for presentation to the user */
+wcstring parse_token_t::user_presentable_description() const
+{
+    return token_type_user_presentable_description(type, keyword);
+}
+
+/* Convert from tokenizer_t's token type to a parse_token_t type */
+static inline parse_token_type_t parse_token_type_from_tokenizer_token(enum token_type tokenizer_token_type)
+{
+    parse_token_type_t result = token_type_invalid;
+    switch (tokenizer_token_type)
+    {
+        case TOK_STRING:
+            result = parse_token_type_string;
+            break;
+
+        case TOK_PIPE:
+            result = parse_token_type_pipe;
+            break;
+
+        case TOK_END:
+            result = parse_token_type_end;
+            break;
+
+        case TOK_BACKGROUND:
+            result = parse_token_type_background;
+            break;
+
+        case TOK_REDIRECT_OUT:
+        case TOK_REDIRECT_APPEND:
+        case TOK_REDIRECT_IN:
+        case TOK_REDIRECT_FD:
+        case TOK_REDIRECT_NOCLOB:
+            result = parse_token_type_redirection;
+            break;
+
+        case TOK_ERROR:
+            result = parse_special_type_tokenizer_error;
+            break;
+
+        case TOK_COMMENT:
+            result = parse_special_type_comment;
+            break;
+
+
+        default:
+            fprintf(stderr, "Bad token type %d passed to %s\n", (int)tokenizer_token_type, __FUNCTION__);
+            assert(0);
+            break;
+    }
+    return result;
+}
+
+/* Helper function for dump_tree */
+static void dump_tree_recursive(const parse_node_tree_t &nodes, const wcstring &src, node_offset_t node_idx, size_t indent, wcstring *result, size_t *line, node_offset_t *inout_first_node_not_dumped)
+{
+    assert(node_idx < nodes.size());
+
+    // Update first_node_not_dumped
+    // This takes a bit of explanation. While it's true that a parse tree may be a "forest",  its individual trees are "compact," meaning they are not interleaved. Thus we keep track of the largest node index as we descend a tree. One past the largest is the start of the next tree.
+    if (*inout_first_node_not_dumped <= node_idx)
+    {
+        *inout_first_node_not_dumped = node_idx + 1;
+    }
+
+    const parse_node_t &node = nodes.at(node_idx);
+
+    const size_t spacesPerIndent = 2;
+
+    // unindent statement lists by 1 to flatten them
+    if (node.type == symbol_job_list || node.type == symbol_arguments_or_redirections_list)
+    {
+        if (indent > 0) indent -= 1;
+    }
+
+    append_format(*result, L"%2lu - %l2u  ", *line, node_idx);
+    result->append(indent * spacesPerIndent, L' ');;
+    result->append(node.describe());
+    if (node.child_count > 0)
+    {
+        append_format(*result, L" <%lu children>", node.child_count);
+    }
+    if (node.has_comments())
+    {
+        append_format(*result, L" <has_comments>", node.child_count);
+    }
+
+    if (node.has_source() && node.type == parse_token_type_string)
+    {
+        result->append(L": \"");
+        result->append(src, node.source_start, node.source_length);
+        result->append(L"\"");
+    }
+
+    if (node.type != parse_token_type_string)
+    {
+        if (node.has_source())
+        {
+            append_format(*result, L"  [%ld, %ld]", (long)node.source_start, (long)node.source_length);
+        }
+        else
+        {
+            append_format(*result, L"  [no src]", (long)node.source_start, (long)node.source_length);
+        }
+    }
+
+    result->push_back(L'\n');
+    ++*line;
+    for (node_offset_t child_idx = node.child_start; child_idx < node.child_start + node.child_count; child_idx++)
+    {
+        dump_tree_recursive(nodes, src, child_idx, indent + 1, result, line, inout_first_node_not_dumped);
+    }
+}
+
+/* Gives a debugging textual description of a parse tree. Note that this supports "parse forests" too. That is, our tree may not really be a tree, but instead a collection of trees. */
+wcstring parse_dump_tree(const parse_node_tree_t &nodes, const wcstring &src)
+{
+    if (nodes.empty())
+        return L"(empty!)";
+
+    node_offset_t first_node_not_dumped = 0;
+    size_t line = 0;
+    wcstring result;
+    while (first_node_not_dumped < nodes.size())
+    {
+        if (first_node_not_dumped > 0)
+        {
+            result.append(L"---New Tree---\n");
+        }
+        dump_tree_recursive(nodes, src, first_node_not_dumped, 0, &result, &line, &first_node_not_dumped);
+    }
+    return result;
+}
+
+/* Struct representing elements of the symbol stack, used in the internal state of the LL parser */
+struct parse_stack_element_t
+{
+    enum parse_token_type_t type;
+    enum parse_keyword_t keyword;
+    node_offset_t node_idx;
+
+    explicit parse_stack_element_t(parse_token_type_t t, node_offset_t idx) : type(t), keyword(parse_keyword_none), node_idx(idx)
+    {
+    }
+
+    explicit parse_stack_element_t(production_element_t e, node_offset_t idx) : type(production_element_type(e)), keyword(production_element_keyword(e)), node_idx(idx)
+    {
+    }
+
+    wcstring describe(void) const
+    {
+        wcstring result = token_type_description(type);
+        if (keyword != parse_keyword_none)
+        {
+            append_format(result, L" <%ls>", keyword_description(keyword).c_str());
+        }
+        return result;
+    }
+
+    /* Returns a name that we can show to the user, e.g. "a command" */
+    wcstring user_presentable_description(void) const
+    {
+        return token_type_user_presentable_description(type, keyword);
+    }
+};
+
+/* The parser itself, private implementation of class parse_t. This is a hand-coded table-driven LL parser. Most hand-coded LL parsers are recursive descent, but recursive descent parsers are difficult to "pause", unlike table-driven parsers. */
+class parse_ll_t
+{
+    /* Traditional symbol stack of the LL parser */
+    std::vector<parse_stack_element_t> symbol_stack;
+
+    /* Parser output. This is a parse tree, but stored in an array. */
+    parse_node_tree_t nodes;
+
+    /* Whether we ran into a fatal error, including parse errors or tokenizer errors */
+    bool fatal_errored;
+
+    /* Whether we should collect error messages or not */
+    bool should_generate_error_messages;
+
+    /* List of errors we have encountered */
+    parse_error_list_t errors;
+
+    /* The symbol stack can contain terminal types or symbols. Symbols go on to do productions, but terminal types are just matched against input tokens. */
+    bool top_node_handle_terminal_types(parse_token_t token);
+
+    void parse_error_unexpected_token(const wchar_t *expected, parse_token_t token);
+    void parse_error(parse_token_t token, parse_error_code_t code, const wchar_t *format, ...);
+    void parse_error_at_location(size_t location, parse_error_code_t code, const wchar_t *format, ...);
+    void parse_error_failed_production(struct parse_stack_element_t &elem, parse_token_t token);
+    void parse_error_unbalancing_token(parse_token_t token);
+
+    /* Reports an error for an unclosed block, e.g. 'begin;'. Returns true on success, false on failure (e.g. it is not an unclosed block) */
+    bool report_error_for_unclosed_block();
+
+    void dump_stack(void) const;
+
+    // Get the node corresponding to the top element of the stack
+    parse_node_t &node_for_top_symbol()
+    {
+        PARSE_ASSERT(! symbol_stack.empty());
+        const parse_stack_element_t &top_symbol = symbol_stack.back();
+        PARSE_ASSERT(top_symbol.node_idx != NODE_OFFSET_INVALID);
+        PARSE_ASSERT(top_symbol.node_idx < nodes.size());
+        return nodes.at(top_symbol.node_idx);
+    }
+
+    // Pop from the top of the symbol stack, then push the given production, updating node counts. Note that production_t has type "pointer to array" so some care is required.
+    inline void symbol_stack_pop_push_production(const production_t *production)
+    {
+        bool logit = false;
+        if (logit)
+        {
+            size_t count = 0;
+            fprintf(stderr, "Applying production:\n");
+            for (size_t i=0; i < MAX_SYMBOLS_PER_PRODUCTION; i++)
+            {
+                production_element_t elem = (*production)[i];
+                if (production_element_is_valid(elem))
+                {
+                    parse_token_type_t type = production_element_type(elem);
+                    parse_keyword_t keyword = production_element_keyword(elem);
+                    fprintf(stderr, "\t%ls <%ls>\n", token_type_description(type).c_str(), keyword_description(keyword).c_str());
+                    count++;
+                }
+            }
+            if (! count) fprintf(stderr, "\t<empty>\n");
+        }
+
+        // Get the parent index. But we can't get the parent parse node yet, since it may be made invalid by adding children
+        const node_offset_t parent_node_idx = symbol_stack.back().node_idx;
+
+        // Add the children. Confusingly, we want our nodes to be in forwards order (last token last, so dumps look nice), but the symbols should be reverse order (last token first, so it's lowest on the stack)
+        const size_t child_start_big = nodes.size();
+        assert(child_start_big < NODE_OFFSET_INVALID);
+        node_offset_t child_start = static_cast<node_offset_t>(child_start_big);
+
+        // To avoid constructing multiple nodes, we make a single one that we modify
+        parse_node_t representative_child(token_type_invalid);
+        representative_child.parent = parent_node_idx;
+
+        node_offset_t child_count = 0;
+        for (size_t i=0; i < MAX_SYMBOLS_PER_PRODUCTION; i++)
+        {
+            production_element_t elem = (*production)[i];
+            if (! production_element_is_valid(elem))
+            {
+                // All done, bail out
+                break;
+            }
+
+            // Append the parse node.
+            representative_child.type = production_element_type(elem);
+            nodes.push_back(representative_child);
+            child_count++;
+        }
+
+        // Update the parent
+        parse_node_t &parent_node = nodes.at(parent_node_idx);
+
+        // Should have no children yet
+        PARSE_ASSERT(parent_node.child_count == 0);
+
+        // Tell the node about its children
+        parent_node.child_start = child_start;
+        parent_node.child_count = child_count;
+
+        // Replace the top of the stack with new stack elements corresponding to our new nodes. Note that these go in reverse order.
+        symbol_stack.pop_back();
+        symbol_stack.reserve(symbol_stack.size() + child_count);
+        node_offset_t idx = child_count;
+        while (idx--)
+        {
+            production_element_t elem = (*production)[idx];
+            PARSE_ASSERT(production_element_is_valid(elem));
+            symbol_stack.push_back(parse_stack_element_t(elem, child_start + idx));
+        }
+    }
+
+public:
+
+    /* Constructor */
+    parse_ll_t(enum parse_token_type_t goal) : fatal_errored(false), should_generate_error_messages(true)
+    {
+        this->symbol_stack.reserve(16);
+        this->nodes.reserve(64);
+        this->reset_symbols_and_nodes(goal);
+    }
+
+    /* Input */
+    void accept_tokens(parse_token_t token1, parse_token_t token2);
+
+    /* Report tokenizer errors */
+    void report_tokenizer_error(parse_token_t token, int tok_err, const wchar_t *tok_error);
+
+    /* Indicate if we hit a fatal error */
+    bool has_fatal_error(void) const
+    {
+        return this->fatal_errored;
+    }
+
+    /* Indicate whether we want to generate error messages */
+    void set_should_generate_error_messages(bool flag)
+    {
+        this->should_generate_error_messages = flag;
+    }
+
+    /* Clear the parse symbol stack (but not the node tree). Add a node of the given type as the goal node. This is called from the constructor */
+    void reset_symbols(enum parse_token_type_t goal);
+
+    /* Clear the parse symbol stack and the node tree. Add a node of the given type as the goal node. This is called from the constructor. */
+    void reset_symbols_and_nodes(enum parse_token_type_t goal);
+
+    /* Once parsing is complete, determine the ranges of intermediate nodes */
+    void determine_node_ranges();
+
+    /* Acquire output after parsing. This transfers directly from within self */
+    void acquire_output(parse_node_tree_t *output, parse_error_list_t *errors);
+};
+
+void parse_ll_t::dump_stack(void) const
+{
+    // Walk backwards from the top, looking for parents
+    wcstring_list_t lines;
+    if (symbol_stack.empty())
+    {
+        lines.push_back(L"(empty)");
+    }
+    else
+    {
+        node_offset_t child = symbol_stack.back().node_idx;
+        node_offset_t cursor = child;
+        lines.push_back(nodes.at(cursor).describe());
+        while (cursor--)
+        {
+            const parse_node_t &node = nodes.at(cursor);
+            if (node.child_start <= child && node.child_start + node.child_count > child)
+            {
+                lines.push_back(node.describe());
+                child = cursor;
+            }
+        }
+    }
+
+    fprintf(stderr, "Stack dump (%lu elements):\n", symbol_stack.size());
+    for (size_t idx = 0; idx < lines.size(); idx++)
+    {
+        fprintf(stderr, "    %ls\n", lines.at(idx).c_str());
+    }
+}
+
+// Give each node a source range equal to the union of the ranges of its children
+// Terminal nodes already have source ranges (and no children)
+// Since children always appear after their parents, we can implement this very simply by walking backwards
+// We then do a second pass to give empty nodes an empty source range (but with a valid offset)
+// We do this by walking forward. If a child of a node has an invalid source range, we set it equal to the end of the source range of its previous child
+void parse_ll_t::determine_node_ranges(void)
+{
+    size_t idx = nodes.size();
+    while (idx--)
+    {
+        parse_node_t *parent = &nodes[idx];
+
+        // Skip nodes that already have a source range. These are terminal nodes.
+        if (parent->source_start != SOURCE_OFFSET_INVALID)
+            continue;
+
+        // Ok, this node needs a source range. Get all of its children, and then set its range.
+        source_offset_t min_start = SOURCE_OFFSET_INVALID, max_end = 0; //note SOURCE_OFFSET_INVALID is huge
+        for (node_offset_t i=0; i < parent->child_count; i++)
+        {
+            const parse_node_t &child = nodes.at(parent->child_offset(i));
+            if (child.has_source())
+            {
+                min_start = std::min(min_start, child.source_start);
+                max_end = std::max(max_end, child.source_start + child.source_length);
+            }
+        }
+
+        if (min_start != SOURCE_OFFSET_INVALID)
+        {
+            assert(max_end >= min_start);
+            parent->source_start = min_start;
+            parent->source_length = max_end - min_start;
+        }
+    }
+
+    /* Forwards pass */
+    size_t size = nodes.size();
+    for (idx = 0; idx < size; idx++)
+    {
+        /* Since we populate the source range based on the sibling node, it's simpler to walk over the children of each node.
+        We keep a running "child_source_cursor" which is meant to be the end of the child's source range. It's initially set to the beginning of the parent' source range. */
+        parse_node_t *parent = &nodes[idx];
+        // If the parent doesn't have a valid source range, then none of its children will either; skip it entirely
+        if (parent->source_start == SOURCE_OFFSET_INVALID)
+        {
+            continue;
+        }
+        source_offset_t child_source_cursor = parent->source_start;
+        for (size_t child_idx = 0; child_idx < parent->child_count; child_idx++)
+        {
+            parse_node_t *child = &nodes[parent->child_start + child_idx];
+            if (child->source_start == SOURCE_OFFSET_INVALID)
+            {
+                child->source_start = child_source_cursor;
+            }
+            child_source_cursor = child->source_start + child->source_length;
+        }
+    }
+}
+
+void parse_ll_t::acquire_output(parse_node_tree_t *output, parse_error_list_t *errors)
+{
+    if (output != NULL)
+    {
+        output->swap(this->nodes);
+    }
+    this->nodes.clear();
+
+    if (errors != NULL)
+    {
+        errors->swap(this->errors);
+    }
+    this->errors.clear();
+    this->symbol_stack.clear();
+}
+
+void parse_ll_t::parse_error(parse_token_t token, parse_error_code_t code, const wchar_t *fmt, ...)
+{
+    this->fatal_errored = true;
+    if (this->should_generate_error_messages)
+    {
+        //this->dump_stack();
+        parse_error_t err;
+
+        va_list va;
+        va_start(va, fmt);
+        err.text = vformat_string(fmt, va);
+        err.code = code;
+        va_end(va);
+
+        err.source_start = token.source_start;
+        err.source_length = token.source_length;
+        this->errors.push_back(err);
+    }
+}
+
+void parse_ll_t::parse_error_at_location(size_t source_location, parse_error_code_t code, const wchar_t *fmt, ...)
+{
+    this->fatal_errored = true;
+    if (this->should_generate_error_messages)
+    {
+        //this->dump_stack();
+        parse_error_t err;
+
+        va_list va;
+        va_start(va, fmt);
+        err.text = vformat_string(fmt, va);
+        err.code = code;
+        va_end(va);
+
+        err.source_start = source_location;
+        err.source_length = 0;
+        this->errors.push_back(err);
+    }
+}
+
+// Unbalancing token. This includes 'else' or 'case' or 'end' outside of the appropriate block
+// This essentially duplicates some logic from resolving the production for symbol_statement_list - yuck
+void parse_ll_t::parse_error_unbalancing_token(parse_token_t token)
+{
+    this->fatal_errored = true;
+    if (this->should_generate_error_messages)
+    {
+        switch (token.keyword)
+        {
+            case parse_keyword_end:
+                this->parse_error(token, parse_error_unbalancing_end, L"'end' outside of a block");
+                break;
+
+            case parse_keyword_else:
+                this->parse_error(token, parse_error_unbalancing_else, L"'else' builtin not inside of if block");
+                break;
+
+            case parse_keyword_case:
+                this->parse_error(token, parse_error_unbalancing_case, L"'case' builtin not inside of switch block");
+                break;
+
+            default:
+                // At the moment, this case should only be hit if you parse a freestanding_argument_list
+                // For example, 'complete -c foo -a 'one & three'
+                // Hackish error message for that case
+                if (! symbol_stack.empty() && symbol_stack.back().type == symbol_freestanding_argument_list)
+                {
+                    this->parse_error(token, parse_error_generic, L"Expected %ls, but found %ls", token_type_user_presentable_description(symbol_argument).c_str(), token.user_presentable_description().c_str());
+                }
+                else
+                {
+                    this->parse_error(token, parse_error_generic, L"Did not expect %ls", token.user_presentable_description().c_str());
+                }
+                break;
+        }
+    }
+}
+
+// This is a 'generic' parse error when we can't match the top of the stack element
+void parse_ll_t::parse_error_failed_production(struct parse_stack_element_t &stack_elem, parse_token_t token)
+{
+    fatal_errored = true;
+    if (this->should_generate_error_messages)
+    {
+        bool done = false;
+
+        /* Check for || */
+        if (token.type == parse_token_type_pipe && token.source_start > 0)
+        {
+            /* Here we wanted a statement and instead got a pipe. See if this is a double pipe: foo || bar. If so, we have a special error message. */
+            const parse_node_t *prev_pipe = nodes.find_node_matching_source_location(parse_token_type_pipe, token.source_start - 1, NULL);
+            if (prev_pipe != NULL)
+            {
+                /* The pipe of the previous job abuts our current token. So we have ||. */
+                this->parse_error(token, parse_error_double_pipe, ERROR_BAD_OR);
+                done = true;
+            }
+        }
+
+        /* Check for && */
+        if (! done && token.type == parse_token_type_background && token.source_start > 0)
+        {
+            /* Check to see if there was a previous token_background */
+            const parse_node_t *prev_background = nodes.find_node_matching_source_location(parse_token_type_background, token.source_start - 1, NULL);
+            if (prev_background != NULL)
+            {
+                /* We have &&. */
+                this->parse_error(token, parse_error_double_background, ERROR_BAD_AND);
+                done = true;
+            }
+        }
+
+        if (! done)
+        {
+            const wcstring expected = stack_elem.user_presentable_description();
+            this->parse_error_unexpected_token(expected.c_str(), token);
+        }
+    }
+}
+
+void parse_ll_t::report_tokenizer_error(parse_token_t token, int tok_err_code, const wchar_t *tok_error)
+{
+    assert(tok_error != NULL);
+    parse_error_code_t parse_error_code;
+    switch (tok_err_code)
+    {
+        case TOK_UNTERMINATED_QUOTE:
+            parse_error_code = parse_error_tokenizer_unterminated_quote;
+            break;
+
+        case TOK_UNTERMINATED_SUBSHELL:
+            parse_error_code = parse_error_tokenizer_unterminated_subshell;
+            break;
+
+        case TOK_UNTERMINATED_ESCAPE:
+            parse_error_code = parse_error_tokenizer_unterminated_escape;
+            break;
+
+        case TOK_OTHER:
+        default:
+            parse_error_code = parse_error_tokenizer_other;
+            break;
+
+    }
+    this->parse_error(token, parse_error_code, L"%ls", tok_error);
+}
+
+void parse_ll_t::parse_error_unexpected_token(const wchar_t *expected, parse_token_t token)
+{
+    fatal_errored = true;
+    if (this->should_generate_error_messages)
+    {
+        this->parse_error(token, parse_error_generic, L"Expected %ls, but instead found %ls", expected, token.user_presentable_description().c_str());
+    }
+}
+
+void parse_ll_t::reset_symbols(enum parse_token_type_t goal)
+{
+    /* Add a new goal node, and then reset our symbol list to point at it */
+    node_offset_t where = static_cast<node_offset_t>(nodes.size());
+    nodes.push_back(parse_node_t(goal));
+
+    symbol_stack.clear();
+    symbol_stack.push_back(parse_stack_element_t(goal, where)); // goal token
+    this->fatal_errored = false;
+}
+
+/* Reset both symbols and nodes */
+void parse_ll_t::reset_symbols_and_nodes(enum parse_token_type_t goal)
+{
+    nodes.clear();
+    this->reset_symbols(goal);
+}
+
+static bool type_is_terminal_type(parse_token_type_t type)
+{
+    switch (type)
+    {
+        case parse_token_type_string:
+        case parse_token_type_pipe:
+        case parse_token_type_redirection:
+        case parse_token_type_background:
+        case parse_token_type_end:
+        case parse_token_type_terminate:
+            return true;
+
+        default:
+            return false;
+    }
+}
+
+bool parse_ll_t::report_error_for_unclosed_block()
+{
+    bool reported_error = false;
+    /* Unclosed block, for example, 'while true ; '. We want to show the block node that opened it. */
+    const parse_node_t &top_node = this->node_for_top_symbol();
+
+    /* Hacktastic. We want to point at the source location of the block, but our block doesn't have a source range yet - only the terminal tokens do. So get the block statement corresponding to this end command. In general this block may be of a variety of types: if_statement, switch_statement, etc., each with different node structures. But keep descending the first child and eventually you hit a keyword: begin, if, etc. That's the keyword we care about. */
+    const parse_node_t *end_command = this->nodes.get_parent(top_node, symbol_end_command);
+    const parse_node_t *block_node = end_command ? this->nodes.get_parent(*end_command) : NULL;
+
+    if (block_node && block_node->type == symbol_block_statement)
+    {
+        // Get the header
+        block_node = this->nodes.get_child(*block_node, 0, symbol_block_header);
+        block_node = this->nodes.get_child(*block_node, 0); // specific statement
+    }
+    if (block_node != NULL)
+    {
+        // block_node is now an if_statement, switch_statement, for_header, while_header, function_header, or begin_header
+        // Hackish: descend down the first node until we reach the bottom. This will be a keyword node like SWITCH, which will have the source range. Ordinarily the source range would be known by the parent node too, but we haven't completed parsing yet, so we haven't yet propagated source ranges
+        const parse_node_t *cursor = block_node;
+        while (cursor->child_count > 0)
+        {
+            cursor = this->nodes.get_child(*cursor, 0);
+            assert(cursor != NULL);
+        }
+        if (cursor->source_start != NODE_OFFSET_INVALID)
+        {
+            const wcstring node_desc = block_type_user_presentable_description(block_node->type);
+            this->parse_error_at_location(cursor->source_start, parse_error_generic, L"Missing end to balance this %ls", node_desc.c_str());
+            reported_error = true;
+        }
+    }
+    return reported_error;
+}
+
+bool parse_ll_t::top_node_handle_terminal_types(parse_token_t token)
+{
+    PARSE_ASSERT(! symbol_stack.empty());
+    PARSE_ASSERT(token.type >= FIRST_PARSE_TOKEN_TYPE);
+    bool handled = false;
+    parse_stack_element_t &stack_top = symbol_stack.back();
+    if (type_is_terminal_type(stack_top.type))
+    {
+        // The top of the stack is terminal. We are going to handle this (because we can't produce from a terminal type)
+        handled = true;
+
+        // Now see if we actually matched
+        bool matched = false;
+        if (stack_top.type == token.type)
+        {
+            switch (stack_top.type)
+            {
+                case parse_token_type_string:
+                    // We matched if the keywords match, or no keyword was required
+                    matched = (stack_top.keyword == parse_keyword_none || stack_top.keyword == token.keyword);
+                    break;
+
+                default:
+                    // For other types, we only require that the types match
+                    matched = true;
+                    break;
+            }
+        }
+
+        if (matched)
+        {
+            // Success. Tell the node that it matched this token, and what its source range is
+            // In the parse phase, we only set source ranges for terminal types. We propagate ranges to parent nodes afterwards.
+            parse_node_t &node = node_for_top_symbol();
+            node.source_start = token.source_start;
+            node.source_length = token.source_length;
+        }
+        else
+        {
+            // Failure
+            if (stack_top.type == parse_token_type_string && token.type == parse_token_type_string)
+            {
+                // Keyword failure. We should unify this with the 'matched' computation above.
+                assert(stack_top.keyword != parse_keyword_none && stack_top.keyword != token.keyword);
+
+                // Check to see which keyword we got which was considered wrong
+                switch (token.keyword)
+                {
+                        // Some keywords are only valid in certain contexts. If this cascaded all the way down through the outermost job_list, it was not in a valid context.
+                    case parse_keyword_case:
+                    case parse_keyword_end:
+                    case parse_keyword_else:
+                        this->parse_error_unbalancing_token(token);
+                        break;
+
+                    case parse_keyword_none:
+                    {
+                        // This is a random other string (not a keyword)
+                        const wcstring expected = keyword_description(stack_top.keyword);
+                        this->parse_error(token, parse_error_generic, L"Expected keyword '%ls'", expected.c_str());
+                        break;
+                    }
+
+
+                    default:
+                    {
+                        // Got a real keyword we can report
+                        const wcstring actual = (token.keyword == parse_keyword_none ? token.describe() : keyword_description(token.keyword));
+                        const wcstring expected = keyword_description(stack_top.keyword);
+                        this->parse_error(token, parse_error_generic, L"Expected keyword '%ls', instead got keyword '%ls'", expected.c_str(), actual.c_str());
+                        break;
+                    }
+                }
+            }
+            else if (stack_top.keyword == parse_keyword_end && token.type == parse_token_type_terminate && this->report_error_for_unclosed_block())
+            {
+                // Handled by report_error_for_unclosed_block
+            }
+            else
+            {
+                const wcstring expected = stack_top.user_presentable_description();
+                this->parse_error_unexpected_token(expected.c_str(), token);
+            }
+        }
+
+        // We handled the token, so pop the symbol stack
+        symbol_stack.pop_back();
+    }
+    return handled;
+}
+
+void parse_ll_t::accept_tokens(parse_token_t token1, parse_token_t token2)
+{
+    bool logit = false;
+    if (logit)
+    {
+        fprintf(stderr, "Accept token %ls\n", token1.describe().c_str());
+    }
+    PARSE_ASSERT(token1.type >= FIRST_PARSE_TOKEN_TYPE);
+
+    bool consumed = false;
+
+    // Handle special types specially. Note that these are the only types that can be pushed if the symbol stack is empty.
+    if (token1.type == parse_special_type_parse_error || token1.type == parse_special_type_tokenizer_error || token1.type == parse_special_type_comment)
+    {
+        /* We set the special node's parent to the top of the stack. This means that we have an asymmetric relationship: the special node has a parent (which is the node we were trying to generate when we encountered the special node), but the parent node does not have the special node as a child. This means for example that parents don't have to worry about tracking any comment nodes, but we can still recover the parent from the comment. */
+        parse_node_t special_node(token1.type);
+        special_node.parent = symbol_stack.back().node_idx;
+        special_node.source_start = token1.source_start;
+        special_node.source_length = token1.source_length;
+        nodes.push_back(special_node);
+        consumed = true;
+
+        /* Mark special flags */
+        if (token1.type == parse_special_type_comment)
+        {
+            this->node_for_top_symbol().flags |= parse_node_flag_has_comments;
+        }
+
+        /* tokenizer errors are fatal */
+        if (token1.type == parse_special_type_tokenizer_error)
+            this->fatal_errored = true;
+    }
+
+    while (! consumed && ! this->fatal_errored)
+    {
+        PARSE_ASSERT(! symbol_stack.empty());
+
+        if (top_node_handle_terminal_types(token1))
+        {
+            if (logit)
+            {
+                fprintf(stderr, "Consumed token %ls\n", token1.describe().c_str());
+            }
+            consumed = true;
+            break;
+        }
+
+        // top_node_match_token may indicate an error if our stack is empty
+        if (this->fatal_errored)
+            break;
+
+        // Get the production for the top of the stack
+        parse_stack_element_t &stack_elem = symbol_stack.back();
+        parse_node_t &node = nodes.at(stack_elem.node_idx);
+        const production_t *production = production_for_token(stack_elem.type, token1, token2, &node.production_idx, NULL /* error text */);
+        if (production == NULL)
+        {
+            parse_error_failed_production(stack_elem, token1);
+            // the above sets fatal_errored, which ends the loop
+        }
+        else
+        {
+            bool is_terminate = (token1.type == parse_token_type_terminate);
+
+            // When a job_list encounters something like 'else', it returns an empty production to return control to the outer block. But if it's unbalanced, then we'll end up with an empty stack! So make sure that doesn't happen. This is the primary mechanism by which we detect e.g. unbalanced end. However, if we get a true terminate token, then we allow (expect) this to empty the stack
+            if (symbol_stack.size() == 1 && production_is_empty(production) && ! is_terminate)
+            {
+                this->parse_error_unbalancing_token(token1);
+                break;
+            }
+
+            // Manipulate the symbol stack.
+            // Note that stack_elem is invalidated by popping the stack.
+            symbol_stack_pop_push_production(production);
+
+            // Expect to not have an empty stack, unless this was the terminate type
+            // Note we may not have an empty stack with the terminate type (i.e. incomplete input)
+            assert(is_terminate || ! symbol_stack.empty());
+
+            if (symbol_stack.empty())
+            {
+                break;
+            }
+        }
+    }
+}
+
+/* Given an expanded string, returns any keyword it matches */
+static parse_keyword_t keyword_with_name(const wchar_t *name)
+{
+    /* Binary search on keyword_map. Start at 1 since 0 is keyword_none */
+    parse_keyword_t result = parse_keyword_none;
+    size_t left = 1, right = sizeof keyword_map / sizeof *keyword_map;
+    while (left < right)
+    {
+        size_t mid = left + (right - left)/2;
+        int cmp = wcscmp(name, keyword_map[mid].name);
+        if (cmp < 0)
+        {
+            right = mid; // name was smaller than mid
+        }
+        else if (cmp > 0)
+        {
+            left = mid + 1; // name was larger than mid
+        }
+        else
+        {
+            result = keyword_map[mid].keyword; // found it
+            break;
+        }
+    }
+    return result;
+}
+
+/* Given a token, returns the keyword it matches, or parse_keyword_none. */
+static parse_keyword_t keyword_for_token(token_type tok, const wchar_t *tok_txt)
+{
+    /* Only strings can be keywords */
+    if (tok != TOK_STRING)
+    {
+        return parse_keyword_none;
+    }
+    
+    /* If tok_txt is clean (which most are), we can compare it directly. Otherwise we have to expand it. We only expand quotes, and we don't want to do expensive expansions like tilde expansions. So we do our own "cleanliness" check; if we find a character not in our allowed set we know it's not a keyword, and if we never find a quote we don't have to expand! Note that this lowercase set could be shrunk to be just the characters that are in keywords. */
+    parse_keyword_t result = parse_keyword_none;
+    bool needs_expand = false, all_chars_valid = true;
+    const wchar_t *chars_allowed_in_keywords = L"abcdefghijklmnopqrstuvwxyz'\"";
+    for (size_t i=0; tok_txt[i] != L'\0'; i++)
+    {
+        wchar_t c = tok_txt[i];
+        if (! wcschr(chars_allowed_in_keywords, c))
+        {
+            all_chars_valid = false;
+            break;
+        }
+        // If we encounter a quote, we need expansion
+        needs_expand = needs_expand || c == L'"' || c == L'\'';
+    }
+    
+    if (all_chars_valid)
+    {
+        /* Expand if necessary */
+        if (! needs_expand)
+        {
+            result = keyword_with_name(tok_txt);
+        }
+        else
+        {
+            wcstring storage;
+            if (unescape_string(tok_txt, &storage, 0))
+            {
+                result = keyword_with_name(storage.c_str());
+            }
+        }
+    }
+    return result;
+}
+
+/* Placeholder invalid token */
+static const parse_token_t kInvalidToken = {token_type_invalid, parse_keyword_none, false, false, SOURCE_OFFSET_INVALID, 0};
+
+/* Terminal token */
+static const parse_token_t kTerminalToken = {parse_token_type_terminate, parse_keyword_none, false, false, SOURCE_OFFSET_INVALID, 0};
+
+static inline bool is_help_argument(const wchar_t *txt)
+{
+    return ! wcscmp(txt, L"-h") || ! wcscmp(txt, L"--help");
+}
+
+/* Return a new parse token, advancing the tokenizer */
+static inline parse_token_t next_parse_token(tokenizer_t *tok)
+{
+    if (! tok_has_next(tok))
+    {
+        return kTerminalToken;
+    }
+
+    token_type tok_type = static_cast<token_type>(tok_last_type(tok));
+    int tok_start = tok_get_pos(tok);
+    size_t tok_extent = tok_get_extent(tok);
+    assert(tok_extent < 10000000); //paranoia
+    const wchar_t *tok_txt = tok_last(tok);
+
+    parse_token_t result;
+
+    /* Set the type, keyword, and whether there's a dash prefix. Note that this is quite sketchy, because it ignores quotes. This is the historical behavior. For example, `builtin --names` lists builtins, but `builtin "--names"` attempts to run --names as a command. Amazingly as of this writing (10/12/13) nobody seems to have noticed this. Squint at it really hard and it even starts to look like a feature. */
+    result.type = parse_token_type_from_tokenizer_token(tok_type);
+    result.keyword = keyword_for_token(tok_type, tok_txt);
+    result.has_dash_prefix = (tok_txt[0] == L'-');
+    result.is_help_argument = result.has_dash_prefix && is_help_argument(tok_txt);
+    result.source_start = (source_offset_t)tok_start;
+    result.source_length = (source_offset_t)tok_extent;
+
+    tok_next(tok);
+    return result;
+}
+
+bool parse_tree_from_string(const wcstring &str, parse_tree_flags_t parse_flags, parse_node_tree_t *output, parse_error_list_t *errors, parse_token_type_t goal)
+{
+    parse_ll_t parser(goal);
+    parser.set_should_generate_error_messages(errors != NULL);
+
+    /* Construct the tokenizer */
+    tok_flags_t tok_options = 0;
+    if (parse_flags & parse_flag_include_comments)
+        tok_options |= TOK_SHOW_COMMENTS;
+
+    if (parse_flags & parse_flag_accept_incomplete_tokens)
+        tok_options |= TOK_ACCEPT_UNFINISHED;
+
+    if (parse_flags & parse_flag_show_blank_lines)
+        tok_options |= TOK_SHOW_BLANK_LINES;
+
+    if (errors == NULL)
+        tok_options |= TOK_SQUASH_ERRORS;
+
+    tokenizer_t tok = tokenizer_t(str.c_str(), tok_options);
+
+    /* We are an LL(2) parser. We pass two tokens at a time. New tokens come in at index 1. Seed our queue with an initial token at index 1. */
+    parse_token_t queue[2] = {kInvalidToken, kInvalidToken};
+
+    /* Loop until we have a terminal token. */
+    for (size_t token_count = 0; queue[0].type != parse_token_type_terminate; token_count++)
+    {
+        /* Push a new token onto the queue */
+        queue[0] = queue[1];
+        queue[1] = next_parse_token(&tok);
+
+        /* If we are leaving things unterminated, then don't pass parse_token_type_terminate */
+        if (queue[0].type == parse_token_type_terminate && (parse_flags & parse_flag_leave_unterminated))
+        {
+            break;
+        }
+
+        /* Pass these two tokens, unless we're still loading the queue. We know that queue[0] is valid; queue[1] may be invalid. */
+        if (token_count > 0)
+        {
+            parser.accept_tokens(queue[0], queue[1]);
+        }
+
+        /* Handle tokenizer errors. This is a hack because really the parser should report this for itself; but it has no way of getting the tokenizer message */
+        if (queue[1].type == parse_special_type_tokenizer_error)
+        {
+            parser.report_tokenizer_error(queue[1], tok_get_error(&tok), tok_last(&tok));
+        }
+
+        /* Handle errors */
+        if (parser.has_fatal_error())
+        {
+            if (parse_flags & parse_flag_continue_after_error)
+            {
+                /* Hack hack hack. Typically the parse error is due to the first token. However, if it's a tokenizer error, then has_fatal_error was set due to the check above; in that case the second token is what matters. */
+                size_t error_token_idx = 0;
+                if (queue[1].type == parse_special_type_tokenizer_error)
+                {
+                    error_token_idx = (queue[1].type == parse_special_type_tokenizer_error ? 1 : 0);
+                    token_count = -1; // so that it will be 0 after incrementing, and our tokenizer error will be ignored
+                }
+
+                /* Mark a special error token, and then keep going */
+                const parse_token_t token = {parse_special_type_parse_error, parse_keyword_none, false, false, queue[error_token_idx].source_start, queue[error_token_idx].source_length};
+                parser.accept_tokens(token, kInvalidToken);
+                parser.reset_symbols(goal);
+            }
+            else
+            {
+                /* Bail out */
+                break;
+            }
+        }
+    }
+
+    // Teach each node where its source range is
+    parser.determine_node_ranges();
+
+    // Acquire the output from the parser
+    parser.acquire_output(output, errors);
+
+#if 0
+    //wcstring result = dump_tree(this->parser->nodes, str);
+    //fprintf(stderr, "Tree (%ld nodes):\n%ls", this->parser->nodes.size(), result.c_str());
+    fprintf(stderr, "%lu nodes, node size %lu, %lu bytes\n", output->size(), sizeof(parse_node_t), output->size() * sizeof(parse_node_t));
+#endif
+
+    // Indicate if we had a fatal error
+    return ! parser.has_fatal_error();
+}
+
+const parse_node_t *parse_node_tree_t::get_child(const parse_node_t &parent, node_offset_t which, parse_token_type_t expected_type) const
+{
+    const parse_node_t *result = NULL;
+
+    /* We may get nodes with no children if we had an incomplete parse. Don't consider than an error */
+    if (parent.child_count > 0)
+    {
+        PARSE_ASSERT(which < parent.child_count);
+        node_offset_t child_offset = parent.child_offset(which);
+        if (child_offset < this->size())
+        {
+            result = &this->at(child_offset);
+
+            /* If we are given an expected type, then the node must be null or that type */
+            assert(expected_type == token_type_invalid || expected_type == result->type);
+        }
+    }
+
+    return result;
+}
+
+const parse_node_t &parse_node_tree_t::find_child(const parse_node_t &parent, parse_token_type_t type) const
+{
+    for (node_offset_t i=0; i < parent.child_count; i++)
+    {
+        const parse_node_t *child = this->get_child(parent, i);
+        if (child->type == type)
+        {
+            return *child;
+        }
+    }
+    PARSE_ASSERT(0);
+    return *(parse_node_t *)(NULL); //unreachable
+}
+
+const parse_node_t *parse_node_tree_t::get_parent(const parse_node_t &node, parse_token_type_t expected_type) const
+{
+    const parse_node_t *result = NULL;
+    if (node.parent != NODE_OFFSET_INVALID)
+    {
+        PARSE_ASSERT(node.parent < this->size());
+        const parse_node_t &parent = this->at(node.parent);
+        if (expected_type == token_type_invalid || expected_type == parent.type)
+        {
+            // The type matches (or no type was requested)
+            result = &parent;
+        }
+    }
+    return result;
+}
+
+static void find_nodes_recursive(const parse_node_tree_t &tree, const parse_node_t &parent, parse_token_type_t type, parse_node_tree_t::parse_node_list_t *result, size_t max_count)
+{
+    if (result->size() < max_count)
+    {
+        if (parent.type == type) result->push_back(&parent);
+        for (node_offset_t i=0; i < parent.child_count; i++)
+        {
+            const parse_node_t *child = tree.get_child(parent, i);
+            assert(child != NULL);
+            find_nodes_recursive(tree, *child, type, result, max_count);
+        }
+    }
+}
+
+parse_node_tree_t::parse_node_list_t parse_node_tree_t::find_nodes(const parse_node_t &parent, parse_token_type_t type, size_t max_count) const
+{
+    parse_node_list_t result;
+    find_nodes_recursive(*this, parent, type, &result, max_count);
+    return result;
+}
+
+/* Return true if the given node has the proposed ancestor as an ancestor (or is itself that ancestor) */
+static bool node_has_ancestor(const parse_node_tree_t &tree, const parse_node_t &node, const parse_node_t &proposed_ancestor)
+{
+    if (&node == &proposed_ancestor)
+    {
+        /* Found it */
+        return true;
+    }
+    else if (node.parent == NODE_OFFSET_INVALID)
+    {
+        /* No more parents */
+        return false;
+    }
+    else
+    {
+        /* Recurse to the parent */
+        return node_has_ancestor(tree, tree.at(node.parent), proposed_ancestor);
+    }
+}
+
+const parse_node_t *parse_node_tree_t::find_last_node_of_type(parse_token_type_t type, const parse_node_t *parent) const
+{
+    const parse_node_t *result = NULL;
+    // Find nodes of the given type in the tree, working backwards
+    size_t idx = this->size();
+    while (idx--)
+    {
+        const parse_node_t &node = this->at(idx);
+        if (node.type == type)
+        {
+            // Types match. Check if it has the right parent
+            if (parent == NULL || node_has_ancestor(*this, node, *parent))
+            {
+                // Success
+                result = &node;
+                break;
+            }
+        }
+    }
+    return result;
+}
+
+const parse_node_t *parse_node_tree_t::find_node_matching_source_location(parse_token_type_t type, size_t source_loc, const parse_node_t *parent) const
+{
+    const parse_node_t *result = NULL;
+    // Find nodes of the given type in the tree, working backwards
+    const size_t len = this->size();
+    for (size_t idx=0; idx < len; idx++)
+    {
+        const parse_node_t &node = this->at(idx);
+
+        /* Types must match */
+        if (node.type != type)
+            continue;
+
+        /* Must contain source location */
+        if (! node.location_in_or_at_end_of_source_range(source_loc))
+            continue;
+
+        /* If a parent is given, it must be an ancestor */
+        if (parent != NULL && ! node_has_ancestor(*this, node, *parent))
+            continue;
+
+        /* Found it */
+        result = &node;
+        break;
+    }
+    return result;
+}
+
+
+bool parse_node_tree_t::argument_list_is_root(const parse_node_t &node) const
+{
+    bool result = true;
+    assert(node.type == symbol_argument_list || node.type == symbol_arguments_or_redirections_list);
+    const parse_node_t *parent = this->get_parent(node);
+    if (parent != NULL)
+    {
+        /* We have a parent - check to make sure it's not another list! */
+        result = parent->type != symbol_arguments_or_redirections_list && parent->type != symbol_argument_list;
+    }
+    return result;
+}
+
+enum parse_statement_decoration_t parse_node_tree_t::decoration_for_plain_statement(const parse_node_t &node) const
+{
+    assert(node.type == symbol_plain_statement);
+    enum parse_statement_decoration_t decoration = parse_statement_decoration_none;
+    const parse_node_t *decorated_statement = this->get_parent(node, symbol_decorated_statement);
+    if (decorated_statement != NULL)
+    {
+        decoration = static_cast<enum parse_statement_decoration_t>(decorated_statement->production_idx);
+    }
+    return decoration;
+}
+
+bool parse_node_tree_t::command_for_plain_statement(const parse_node_t &node, const wcstring &src, wcstring *out_cmd) const
+{
+    bool result = false;
+    assert(node.type == symbol_plain_statement);
+    const parse_node_t *cmd_node = this->get_child(node, 0, parse_token_type_string);
+    if (cmd_node != NULL && cmd_node->has_source())
+    {
+        out_cmd->assign(src, cmd_node->source_start, cmd_node->source_length);
+        result = true;
+    }
+    else
+    {
+        out_cmd->clear();
+    }
+    return result;
+}
+
+bool parse_node_tree_t::statement_is_in_pipeline(const parse_node_t &node, bool include_first) const
+{
+    // Moderately nasty hack! Walk up our ancestor chain and see if we are in a job_continuation. This checks if we are in the second or greater element in a pipeline; if we are the first element we treat this as false
+    // This accepts a few statement types
+    bool result = false;
+    const parse_node_t *ancestor = &node;
+
+    // If we're given a plain statement, try to get its decorated statement parent
+    if (ancestor && ancestor->type == symbol_plain_statement)
+        ancestor = this->get_parent(*ancestor, symbol_decorated_statement);
+    if (ancestor)
+        ancestor = this->get_parent(*ancestor, symbol_statement);
+    if (ancestor)
+        ancestor = this->get_parent(*ancestor);
+
+    if (ancestor)
+    {
+        if (ancestor->type == symbol_job_continuation)
+        {
+            // Second or more in a pipeline
+            result = true;
+        }
+        else if (ancestor->type == symbol_job && include_first)
+        {
+            // Check to see if we have a job continuation that's not empty
+            const parse_node_t *continuation = this->get_child(*ancestor, 1, symbol_job_continuation);
+            result = (continuation != NULL && continuation->child_count > 0);
+        }
+    }
+
+    return result;
+}
+
+enum token_type parse_node_tree_t::type_for_redirection(const parse_node_t &redirection_node, const wcstring &src, int *out_fd, wcstring *out_target) const
+{
+    assert(redirection_node.type == symbol_redirection);
+    enum token_type result = TOK_NONE;
+    const parse_node_t *redirection_primitive = this->get_child(redirection_node, 0, parse_token_type_redirection); //like 2>
+    const parse_node_t *redirection_target = this->get_child(redirection_node, 1, parse_token_type_string); //like &1 or file path
+
+    if (redirection_primitive != NULL && redirection_primitive->has_source())
+    {
+        result = redirection_type_for_string(redirection_primitive->get_source(src), out_fd);
+    }
+    if (out_target != NULL)
+    {
+        *out_target = redirection_target ? redirection_target->get_source(src) : L"";
+    }
+    return result;
+}
+
+const parse_node_t *parse_node_tree_t::header_node_for_block_statement(const parse_node_t &node) const
+{
+    const parse_node_t *result = NULL;
+    if (node.type == symbol_block_statement)
+    {
+        const parse_node_t *block_header = this->get_child(node, 0, symbol_block_header);
+        if (block_header != NULL)
+        {
+            result = this->get_child(*block_header, 0);
+        }
+    }
+    return result;
+}
+
+parse_node_tree_t::parse_node_list_t parse_node_tree_t::specific_statements_for_job(const parse_node_t &job) const
+{
+    assert(job.type == symbol_job);
+    parse_node_list_t result;
+
+    /* Initial statement (non-specific) */
+    result.push_back(get_child(job, 0, symbol_statement));
+
+    /* Our cursor variable. Walk over the list of continuations. */
+    const parse_node_t *continuation = get_child(job, 1, symbol_job_continuation);
+    while (continuation != NULL && continuation->child_count > 0)
+    {
+        result.push_back(get_child(*continuation, 1, symbol_statement));
+        continuation = get_child(*continuation, 2, symbol_job_continuation);
+    }
+
+    /* Result now contains a list of statements. But we want a list of specific statements e.g. symbol_switch_statement. So replace them in-place in the vector. */
+    for (size_t i=0; i < result.size(); i++)
+    {
+        const parse_node_t *statement = result.at(i);
+        assert(statement->type == symbol_statement);
+        result.at(i) = this->get_child(*statement, 0);
+    }
+
+    return result;
+}
+
+parse_node_tree_t::parse_node_list_t parse_node_tree_t::comment_nodes_for_node(const parse_node_t &parent) const
+{
+    parse_node_list_t result;
+    if (parent.has_comments())
+    {
+        /* Walk all our nodes, looking for comment nodes that have the given node as a parent */
+        for (size_t i=0; i < this->size(); i++)
+        {
+            const parse_node_t &potential_comment = this->at(i);
+            if (potential_comment.type == parse_special_type_comment && this->get_parent(potential_comment) == &parent)
+            {
+                result.push_back(&potential_comment);
+            }
+        }
+    }
+    return result;
+}
+
+enum parse_bool_statement_type_t parse_node_tree_t::statement_boolean_type(const parse_node_t &node)
+{
+    assert(node.type == symbol_boolean_statement);
+    switch (node.production_idx)
+    {
+        // These magic numbers correspond to productions for boolean_statement
+        case 0:
+            return parse_bool_and;
+
+        case 1:
+            return parse_bool_or;
+
+        case 2:
+            return parse_bool_not;
+
+        default:
+        {
+            fprintf(stderr, "Unexpected production in boolean statement\n");
+            PARSER_DIE();
+            return (enum parse_bool_statement_type_t)(-1);
+        }
+    }
+}
+
+bool parse_node_tree_t::job_should_be_backgrounded(const parse_node_t &job) const
+{
+    assert(job.type == symbol_job);
+    assert(job.production_idx == 0);
+    bool result = false;
+    const parse_node_t *opt_background = get_child(job, 2, symbol_optional_background);
+    if (opt_background != NULL)
+    {
+        // We may get the value -1 if the node is not yet materialized (i.e. an incomplete parse tree)
+        assert(opt_background->production_idx == uint8_t(-1) || opt_background->production_idx <= 1);
+        result = (opt_background->production_idx == 1);
+    }
+    return result;
+}
+
+const parse_node_t *parse_node_tree_t::next_node_in_node_list(const parse_node_t &node_list, parse_token_type_t entry_type, const parse_node_t **out_list_tail) const
+{
+    parse_token_type_t list_type = node_list.type;
+
+    /* Paranoia - it doesn't make sense for a list type to contain itself */
+    assert(list_type != entry_type);
+
+    const parse_node_t *list_cursor = &node_list;
+    const parse_node_t *list_entry = NULL;
+
+    /* Loop while we don't have an item but do have a list. Note that some nodes may contain nothing - e.g. job_list contains blank lines as a production */
+    while (list_entry == NULL && list_cursor != NULL)
+    {
+        const parse_node_t *next_cursor = NULL;
+
+        /* Walk through the children */
+        for (node_offset_t i=0; i < list_cursor->child_count; i++)
+        {
+            const parse_node_t *child = this->get_child(*list_cursor, i);
+            if (child->type == entry_type)
+            {
+                /* This is the list entry */
+                list_entry = child;
+            }
+            else if (child->type == list_type)
+            {
+                /* This is the next in the list */
+                next_cursor = child;
+            }
+        }
+        /* Go to the next entry, even if it's NULL */
+        list_cursor = next_cursor;
+    }
+
+    /* Return what we got */
+    assert(list_cursor == NULL || list_cursor->type == list_type);
+    assert(list_entry == NULL || list_entry->type == entry_type);
+    if (out_list_tail != NULL)
+        *out_list_tail = list_cursor;
+    return list_entry;
+}
diff --git a/src/parse_tree.h b/src/parse_tree.h
new file mode 100644
index 00000000..e4d4eb36
--- /dev/null
+++ b/src/parse_tree.h
@@ -0,0 +1,288 @@
+/**\file parse_tree.h
+
+    Programmatic representation of fish code.
+*/
+
+#ifndef FISH_PARSE_PRODUCTIONS_H
+#define FISH_PARSE_PRODUCTIONS_H
+
+#include <assert.h>
+#include <stddef.h>
+#include <stdint.h>
+
+#include "common.h"
+#include "tokenizer.h"
+#include "parse_constants.h"
+#include <vector>
+
+class parse_node_tree_t;
+
+typedef uint32_t node_offset_t;
+
+#define NODE_OFFSET_INVALID (static_cast<node_offset_t>(-1))
+
+typedef uint32_t source_offset_t;
+
+#define SOURCE_OFFSET_INVALID (static_cast<source_offset_t>(-1))
+
+/* Returns a description of a list of parse errors */
+wcstring parse_errors_description(const parse_error_list_t &errors, const wcstring &src, const wchar_t *prefix = NULL);
+
+/** A struct representing the token type that we use internally */
+struct parse_token_t
+{
+    enum parse_token_type_t type; // The type of the token as represented by the parser
+    enum parse_keyword_t keyword; // Any keyword represented by this token
+    bool has_dash_prefix; // Hackish: whether the source contains a dash prefix
+    bool is_help_argument; // Hackish: whether the source looks like '-h' or '--help'
+    source_offset_t source_start;
+    source_offset_t source_length;
+
+    wcstring describe() const;
+    wcstring user_presentable_description() const;
+};
+
+
+enum
+{
+    parse_flag_none = 0,
+
+    /* Attempt to build a "parse tree" no matter what. This may result in a 'forest' of disconnected trees. This is intended to be used by syntax highlighting. */
+    parse_flag_continue_after_error = 1 << 0,
+
+    /* Include comment tokens */
+    parse_flag_include_comments = 1 << 1,
+
+    /* Indicate that the tokenizer should accept incomplete tokens */
+    parse_flag_accept_incomplete_tokens = 1 << 2,
+
+    /* Indicate that the parser should not generate the terminate token, allowing an 'unfinished' tree where some nodes may have no productions. */
+    parse_flag_leave_unterminated = 1 << 3,
+
+    /* Indicate that the parser should generate job_list entries for blank lines. */
+    parse_flag_show_blank_lines = 1 << 4
+};
+typedef unsigned int parse_tree_flags_t;
+
+wcstring parse_dump_tree(const parse_node_tree_t &tree, const wcstring &src);
+
+wcstring token_type_description(parse_token_type_t type);
+wcstring keyword_description(parse_keyword_t type);
+
+enum
+{
+    /* Flag indicating that the node has associated comment nodes */
+    parse_node_flag_has_comments = 1 << 0
+};
+typedef uint8_t parse_node_flags_t;
+
+/** Class for nodes of a parse tree. Since there's a lot of these, the size and order of the fields is important. */
+class parse_node_t
+{
+public:
+    /* Start in the source code */
+    source_offset_t source_start;
+
+    /* Length of our range in the source code */
+    source_offset_t source_length;
+
+    /* Parent */
+    node_offset_t parent;
+
+    /* Children */
+    node_offset_t child_start;
+
+    /* Number of children */
+    uint8_t child_count;
+
+    /* Which production was used */
+    uint8_t production_idx;
+
+    /* Type of the node */
+    enum parse_token_type_t type;
+
+    /* Node flags */
+    parse_node_flags_t flags;
+
+    /* Description */
+    wcstring describe(void) const;
+
+    /* Constructor */
+    explicit parse_node_t(parse_token_type_t ty) : source_start(SOURCE_OFFSET_INVALID), source_length(0), parent(NODE_OFFSET_INVALID), child_start(0), child_count(0), production_idx(-1), type(ty), flags(0)
+    {
+    }
+
+    node_offset_t child_offset(node_offset_t which) const
+    {
+        PARSE_ASSERT(which < child_count);
+        return child_start + which;
+    }
+
+    /* Indicate if this node has a range of source code associated with it */
+    bool has_source() const
+    {
+        /* Should never have a nonempty range with an invalid offset */
+        assert(this->source_start != SOURCE_OFFSET_INVALID || this->source_length == 0);
+        return this->source_length > 0;
+    }
+
+    /* Indicate if the node has comment nodes */
+    bool has_comments() const
+    {
+        return !! (this->flags & parse_node_flag_has_comments);
+    }
+
+    /* Gets source for the node, or the empty string if it has no source */
+    wcstring get_source(const wcstring &str) const
+    {
+        if (! has_source())
+            return wcstring();
+        else
+            return wcstring(str, this->source_start, this->source_length);
+    }
+
+    /* Returns whether the given location is within the source range or at its end */
+    bool location_in_or_at_end_of_source_range(size_t loc) const
+    {
+        return has_source() && source_start <= loc && loc - source_start <= source_length;
+    }
+};
+
+/* The parse tree itself */
+class parse_node_tree_t : public std::vector<parse_node_t>
+{
+public:
+
+    /* Get the node corresponding to a child of the given node, or NULL if there is no such child. If expected_type is provided, assert that the node has that type.
+     */
+    const parse_node_t *get_child(const parse_node_t &parent, node_offset_t which, parse_token_type_t expected_type = token_type_invalid) const;
+
+    /* Find the first direct child of the given node of the given type. asserts on failure
+     */
+    const parse_node_t &find_child(const parse_node_t &parent, parse_token_type_t type) const;
+
+    /* Get the node corresponding to the parent of the given node, or NULL if there is no such child. If expected_type is provided, only returns the parent if it is of that type. Note the asymmetry: get_child asserts since the children are known, but get_parent does not, since the parent may not be known. */
+    const parse_node_t *get_parent(const parse_node_t &node, parse_token_type_t expected_type = token_type_invalid) const;
+
+    /* Find all the nodes of a given type underneath a given node, up to max_count of them */
+    typedef std::vector<const parse_node_t *> parse_node_list_t;
+    parse_node_list_t find_nodes(const parse_node_t &parent, parse_token_type_t type, size_t max_count = (size_t)(-1)) const;
+
+    /* Finds the last node of a given type underneath a given node, or NULL if it could not be found. If parent is NULL, this finds the last node in the tree of that type. */
+    const parse_node_t *find_last_node_of_type(parse_token_type_t type, const parse_node_t *parent = NULL) const;
+
+    /* Finds a node containing the given source location. If 'parent' is not NULL, it must be an ancestor. */
+    const parse_node_t *find_node_matching_source_location(parse_token_type_t type, size_t source_loc, const parse_node_t *parent) const;
+
+    /* Indicate if the given argument_list or arguments_or_redirections_list is a root list, or has a parent */
+    bool argument_list_is_root(const parse_node_t &node) const;
+
+    /* Utilities */
+
+    /* Given a plain statement, get the decoration (from the parent node), or none if there is no decoration */
+    enum parse_statement_decoration_t decoration_for_plain_statement(const parse_node_t &node) const;
+
+    /* Given a plain statement, get the command by reference (from the child node). Returns true if successful. Clears the command on failure. */
+    bool command_for_plain_statement(const parse_node_t &node, const wcstring &src, wcstring *out_cmd) const;
+
+    /* Given a plain statement, return true if the statement is part of a pipeline. If include_first is set, the first command in a pipeline is considered part of it; otherwise only the second or additional commands are */
+    bool statement_is_in_pipeline(const parse_node_t &node, bool include_first) const;
+
+    /* Given a redirection, get the redirection type (or TOK_NONE) and target (file path, or fd) */
+    enum token_type type_for_redirection(const parse_node_t &node, const wcstring &src, int *out_fd, wcstring *out_target) const;
+
+    /* If the given node is a block statement, returns the header node (for_header, while_header, begin_header, or function_header). Otherwise returns NULL */
+    const parse_node_t *header_node_for_block_statement(const parse_node_t &node) const;
+
+    /* Given a node list (e.g. of type symbol_job_list) and a node type (e.g. symbol_job), return the next element of the given type in that list, and the tail (by reference). Returns NULL if we've exhausted the list. */
+    const parse_node_t *next_node_in_node_list(const parse_node_t &node_list, parse_token_type_t item_type, const parse_node_t **list_tail) const;
+
+    /* Given a job, return all of its statements. These are 'specific statements' (e.g. symbol_decorated_statement, not symbol_statement) */
+    parse_node_list_t specific_statements_for_job(const parse_node_t &job) const;
+
+    /* Given a node, return all of its comment nodes. */
+    parse_node_list_t comment_nodes_for_node(const parse_node_t &node) const;
+
+    /* Returns the boolean type for a boolean node */
+    static enum parse_bool_statement_type_t statement_boolean_type(const parse_node_t &node);
+
+    /* Given a job, return whether it should be backgrounded, because it has a & specifier */
+    bool job_should_be_backgrounded(const parse_node_t &job) const;
+};
+
+/* The big entry point. Parse a string, attempting to produce a tree for the given goal type */
+bool parse_tree_from_string(const wcstring &str, parse_tree_flags_t flags, parse_node_tree_t *output, parse_error_list_t *errors, parse_token_type_t goal = symbol_job_list);
+
+/* Fish grammar:
+
+# A job_list is a list of jobs, separated by semicolons or newlines
+
+    job_list = <empty> |
+                job job_list
+                <TOK_END> job_list
+
+# A job is a non-empty list of statements, separated by pipes. (Non-empty is useful for cases like if statements, where we require a command). To represent "non-empty", we require a statement, followed by a possibly empty job_continuation, and then optionally a background specifier '&'
+
+    job = statement job_continuation optional_background
+    job_continuation = <empty> |
+                       <TOK_PIPE> statement job_continuation
+
+# A statement is a normal command, or an if / while / and etc
+
+    statement = boolean_statement | block_statement | if_statement | switch_statement | decorated_statement
+
+# A block is a conditional, loop, or begin/end
+
+    if_statement = if_clause else_clause end_command arguments_or_redirections_list
+    if_clause = <IF> job <TOK_END> job_list
+    else_clause = <empty> |
+                 <ELSE> else_continuation
+    else_continuation = if_clause else_clause |
+                        <TOK_END> job_list
+
+    switch_statement = SWITCH argument <TOK_END> case_item_list end_command arguments_or_redirections_list
+    case_item_list = <empty> |
+                    case_item case_item_list |
+                    <TOK_END> case_item_list
+
+    case_item = CASE argument_list <TOK_END> job_list
+
+    block_statement = block_header  job_list end_command arguments_or_redirections_list
+    block_header = for_header | while_header  | function_header | begin_header
+    for_header = FOR var_name IN argument_list <TOK_END>
+    while_header = WHILE job <TOK_END>
+    begin_header = BEGIN
+
+# Functions take arguments, and require at least one (the name). No redirections allowed.
+    function_header = FUNCTION argument argument_list <TOK_END>
+
+# A boolean statement is AND or OR or NOT
+
+    boolean_statement = AND statement | OR statement | NOT statement
+
+# A decorated_statement is a command with a list of arguments_or_redirections, possibly with "builtin" or "command" or "exec"
+
+    decorated_statement = plain_statement | COMMAND plain_statement | BUILTIN plain_statement | EXEC plain_statement
+    plain_statement = <TOK_STRING> arguments_or_redirections_list
+
+    argument_list = <empty> | argument argument_list
+
+    arguments_or_redirections_list = <empty> |
+                                     argument_or_redirection arguments_or_redirections_list
+    argument_or_redirection = argument | redirection
+    argument = <TOK_STRING>
+
+    redirection = <TOK_REDIRECTION> <TOK_STRING>
+
+    optional_background = <empty> | <TOK_BACKGROUND>
+
+    end_command = END
+
+ # A freestanding_argument_list is equivalent to a normal argument list, except it may contain TOK_END (newlines, and even semicolons, for historical reasons
+
+     freestanding_argument_list = <empty> |
+                                  argument freestanding_argument_list |
+                                  <TOK_END> freestanding_argument_list
+*/
+
+#endif
diff --git a/src/parse_util.cpp b/src/parse_util.cpp
new file mode 100644
index 00000000..10388cb7
--- /dev/null
+++ b/src/parse_util.cpp
@@ -0,0 +1,1623 @@
+/** \file parse_util.c
+
+    Various mostly unrelated utility functions related to parsing,
+    loading and evaluating fish code.
+
+  This library can be seen as a 'toolbox' for functions that are
+  used in many places in fish and that are somehow related to
+  parsing the code.
+*/
+
+#include "config.h" // IWYU pragma: keep
+
+#include <stdlib.h>
+#include <stdarg.h>
+#include <wchar.h>
+#include <string>
+#include <assert.h>
+
+#include "fallback.h"
+#include "util.h"
+#include "wutil.h" // IWYU pragma: keep
+#include "common.h"
+#include "tokenizer.h"
+#include "parse_util.h"
+#include "expand.h"
+#include "env.h"
+#include "wildcard.h"
+#include "parse_tree.h"
+#include "builtin.h"
+
+/** Error message for improper use of the exec builtin */
+#define EXEC_ERR_MSG _(L"The '%ls' command can not be used in a pipeline")
+
+/** Error message for use of backgrounded commands before and/or */
+#define BOOL_AFTER_BACKGROUND_ERROR_MSG _(L"The '%ls' command can not be used immediately after a backgrounded job")
+
+/** Error message for backgrounded commands as conditionals */
+#define BACKGROUND_IN_CONDITIONAL_ERROR_MSG _(L"Backgrounded commands can not be used as conditionals")
+
+
+int parse_util_lineno(const wchar_t *str, size_t offset)
+{
+    if (! str)
+        return 0;
+
+    int res = 1;
+    for (size_t i=0; i < offset && str[i] != L'\0'; i++)
+    {
+        if (str[i] == L'\n')
+        {
+            res++;
+        }
+    }
+    return res;
+}
+
+
+int parse_util_get_line_from_offset(const wcstring &str, size_t pos)
+{
+    const wchar_t *buff = str.c_str();
+    int count = 0;
+    for (size_t i=0; i<pos; i++)
+    {
+        if (!buff[i])
+        {
+            return -1;
+        }
+
+        if (buff[i] == L'\n')
+        {
+            count++;
+        }
+    }
+    return count;
+}
+
+
+size_t parse_util_get_offset_from_line(const wcstring &str, int line)
+{
+    const wchar_t *buff = str.c_str();
+    size_t i;
+    int count = 0;
+
+    if (line < 0)
+    {
+        return (size_t)(-1);
+    }
+
+    if (line == 0)
+        return 0;
+
+    for (i=0;; i++)
+    {
+        if (!buff[i])
+        {
+            return -1;
+        }
+
+        if (buff[i] == L'\n')
+        {
+            count++;
+            if (count == line)
+            {
+                return (i+1)<str.size()?i+1:i;
+            }
+
+        }
+    }
+}
+
+size_t parse_util_get_offset(const wcstring &str, int line, long line_offset)
+{
+    const wchar_t *buff = str.c_str();
+    size_t off = parse_util_get_offset_from_line(buff, line);
+    size_t off2 = parse_util_get_offset_from_line(buff, line+1);
+    long line_offset2 = line_offset;
+
+    if (off == (size_t)(-1))
+    {
+        return -1;
+    }
+
+    if (off2 == (size_t)(-1))
+    {
+        off2 = wcslen(buff)+1;
+    }
+
+    if (line_offset2 < 0)
+    {
+        line_offset2 = 0;
+    }
+
+    if (line_offset2 >= off2-off-1)
+    {
+        line_offset2 = off2-off-1;
+    }
+
+    return off + line_offset2;
+}
+
+static int parse_util_locate_brackets_of_type(const wchar_t *in, wchar_t **begin, wchar_t **end, bool allow_incomplete, wchar_t open_type, wchar_t close_type)
+{
+    /* open_type is typically ( or [, and close type is the corresponding value */
+    wchar_t *pos;
+    wchar_t prev=0;
+    int syntax_error=0;
+    int paran_count=0;
+
+    wchar_t *paran_begin=0, *paran_end=0;
+
+    CHECK(in, 0);
+
+    for (pos = const_cast<wchar_t *>(in); *pos; pos++)
+    {
+        if (prev != '\\')
+        {
+            if (wcschr(L"\'\"", *pos))
+            {
+                wchar_t *q_end = quote_end(pos);
+                if (q_end && *q_end)
+                {
+                    pos=q_end;
+                }
+                else
+                {
+                    break;
+                }
+            }
+            else
+            {
+                if (*pos == open_type)
+                {
+                    if ((paran_count == 0)&&(paran_begin==0))
+                    {
+                        paran_begin = pos;
+                    }
+
+                    paran_count++;
+                }
+                else if (*pos == close_type)
+                {
+
+                    paran_count--;
+
+                    if ((paran_count == 0) && (paran_end == 0))
+                    {
+                        paran_end = pos;
+                        break;
+                    }
+
+                    if (paran_count < 0)
+                    {
+                        syntax_error = 1;
+                        break;
+                    }
+                }
+            }
+
+        }
+        prev = *pos;
+    }
+
+    syntax_error |= (paran_count < 0);
+    syntax_error |= ((paran_count>0)&&(!allow_incomplete));
+
+    if (syntax_error)
+    {
+        return -1;
+    }
+
+    if (paran_begin == 0)
+    {
+        return 0;
+    }
+
+    if (begin)
+    {
+        *begin = paran_begin;
+    }
+
+    if (end)
+    {
+        *end = paran_count?(wchar_t *)in+wcslen(in):paran_end;
+    }
+
+    return 1;
+}
+
+
+int parse_util_locate_cmdsubst(const wchar_t *in, wchar_t **begin, wchar_t **end, bool accept_incomplete)
+{
+    return parse_util_locate_brackets_of_type(in, begin, end, accept_incomplete, L'(', L')');
+}
+
+int parse_util_locate_slice(const wchar_t *in, wchar_t **begin, wchar_t **end, bool accept_incomplete)
+{
+    return parse_util_locate_brackets_of_type(in, begin, end, accept_incomplete, L'[', L']');
+}
+
+
+static int parse_util_locate_brackets_range(const wcstring &str, size_t *inout_cursor_offset, wcstring *out_contents, size_t *out_start, size_t *out_end, bool accept_incomplete, wchar_t open_type, wchar_t close_type)
+{
+    /* Clear the return values */
+    out_contents->clear();
+    *out_start = 0;
+    *out_end = str.size();
+
+    /* Nothing to do if the offset is at or past the end of the string. */
+    if (*inout_cursor_offset >= str.size())
+        return 0;
+
+    /* Defer to the wonky version */
+    const wchar_t * const buff = str.c_str();
+    const wchar_t * const valid_range_start = buff + *inout_cursor_offset, *valid_range_end = buff + str.size();
+    wchar_t *bracket_range_begin = NULL, *bracket_range_end = NULL;
+    int ret = parse_util_locate_brackets_of_type(valid_range_start, &bracket_range_begin, &bracket_range_end, accept_incomplete, open_type, close_type);
+    if (ret > 0)
+    {
+        /* The command substitutions must not be NULL and must be in the valid pointer range, and the end must be bigger than the beginning */
+        assert(bracket_range_begin != NULL && bracket_range_begin >= valid_range_start && bracket_range_begin <= valid_range_end);
+        assert(bracket_range_end != NULL && bracket_range_end > bracket_range_begin && bracket_range_end >= valid_range_start && bracket_range_end <= valid_range_end);
+
+        /* Assign the substring to the out_contents */
+        const wchar_t *interior_begin = bracket_range_begin + 1;
+        out_contents->assign(interior_begin, bracket_range_end - interior_begin);
+
+        /* Return the start and end */
+        *out_start = bracket_range_begin - buff;
+        *out_end = bracket_range_end - buff;
+
+        /* Update the inout_cursor_offset. Note this may cause it to exceed str.size(), though overflow is not likely */
+        *inout_cursor_offset = 1 + *out_end;
+    }
+    return ret;
+}
+
+int parse_util_locate_cmdsubst_range(const wcstring &str, size_t *inout_cursor_offset, wcstring *out_contents, size_t *out_start, size_t *out_end, bool accept_incomplete)
+{
+    return parse_util_locate_brackets_range(str, inout_cursor_offset, out_contents, out_start, out_end, accept_incomplete, L'(', L')');
+}
+
+void parse_util_cmdsubst_extent(const wchar_t *buff, size_t cursor_pos, const wchar_t **a, const wchar_t **b)
+{
+    const wchar_t * const cursor = buff + cursor_pos;
+
+    CHECK(buff,);
+
+    const size_t bufflen = wcslen(buff);
+    assert(cursor_pos <= bufflen);
+
+    /* ap and bp are the beginning and end of the tightest command substitition found so far */
+    const wchar_t *ap = buff, *bp = buff + bufflen;
+    const wchar_t *pos = buff;
+    for (;;)
+    {
+        wchar_t *begin = NULL, *end = NULL;
+        if (parse_util_locate_cmdsubst(pos, &begin, &end, true) <= 0)
+        {
+            /* No subshell found, all done */
+            break;
+        }
+        /* Interpret NULL to mean the end */
+        if (end == NULL)
+        {
+            end = const_cast<wchar_t *>(buff) + bufflen;
+        }
+
+        if (begin < cursor && end >= cursor)
+        {
+            /* This command substitution surrounds the cursor, so it's a tighter fit */
+            begin++;
+            ap = begin;
+            bp = end;
+            /* pos is where to begin looking for the next one. But if we reached the end there's no next one. */
+            if (begin >= end)
+                break;
+            pos = begin + 1;
+        }
+        else if (begin >= cursor)
+        {
+            /* This command substitution starts at or after the cursor. Since it was the first command substitution in the string, we're done. */
+            break;
+        }
+        else
+        {
+            /* This command substitution ends before the cursor. Skip it. */
+            assert(end < cursor);
+            pos = end + 1;
+            assert(pos <= buff + bufflen);
+        }
+    }
+
+    if (a != NULL) *a = ap;
+    if (b != NULL) *b = bp;
+}
+
+/**
+   Get the beginning and end of the job or process definition under the cursor
+*/
+static void job_or_process_extent(const wchar_t *buff,
+                                  size_t cursor_pos,
+                                  const wchar_t **a,
+                                  const wchar_t **b,
+                                  int process)
+{
+    const wchar_t *begin, *end;
+    long pos;
+    wchar_t *buffcpy;
+    int finished=0;
+
+    CHECK(buff,);
+
+    if (a)
+    {
+        *a=0;
+    }
+
+    if (b)
+    {
+        *b = 0;
+    }
+
+    parse_util_cmdsubst_extent(buff, cursor_pos, &begin, &end);
+    if (!end || !begin)
+    {
+        return;
+    }
+
+    pos = cursor_pos - (begin - buff);
+
+    if (a)
+    {
+        *a = begin;
+    }
+
+    if (b)
+    {
+        *b = end;
+    }
+
+    buffcpy = wcsndup(begin, end-begin);
+
+    if (!buffcpy)
+    {
+        DIE_MEM();
+    }
+
+    tokenizer_t tok(buffcpy, TOK_ACCEPT_UNFINISHED);
+    for (; tok_has_next(&tok) && !finished; tok_next(&tok))
+    {
+        int tok_begin = tok_get_pos(&tok);
+
+        switch (tok_last_type(&tok))
+        {
+            case TOK_PIPE:
+            {
+                if (!process)
+                {
+                    break;
+                }
+            }
+
+            case TOK_END:
+            case TOK_BACKGROUND:
+            {
+
+                if (tok_begin >= pos)
+                {
+                    finished=1;
+                    if (b)
+                    {
+                        *b = (wchar_t *)begin + tok_begin;
+                    }
+                }
+                else
+                {
+                    if (a)
+                    {
+                        *a = (wchar_t *)begin + tok_begin+1;
+                    }
+                }
+
+                break;
+            }
+
+            default:
+            {
+                break;
+            }
+        }
+    }
+
+    free(buffcpy);
+}
+
+void parse_util_process_extent(const wchar_t *buff,
+                               size_t pos,
+                               const wchar_t **a,
+                               const wchar_t **b)
+{
+    job_or_process_extent(buff, pos, a, b, 1);
+}
+
+void parse_util_job_extent(const wchar_t *buff,
+                           size_t pos,
+                           const wchar_t **a,
+                           const wchar_t **b)
+{
+    job_or_process_extent(buff,pos,a, b, 0);
+}
+
+
+void parse_util_token_extent(const wchar_t *buff,
+                             size_t cursor_pos,
+                             const wchar_t **tok_begin,
+                             const wchar_t **tok_end,
+                             const wchar_t **prev_begin,
+                             const wchar_t **prev_end)
+{
+    const wchar_t *a = NULL, *b = NULL, *pa = NULL, *pb = NULL;
+
+    CHECK(buff,);
+
+    assert(cursor_pos >= 0);
+
+    const wchar_t *cmdsubst_begin, *cmdsubst_end;
+    parse_util_cmdsubst_extent(buff, cursor_pos, &cmdsubst_begin, &cmdsubst_end);
+
+    if (!cmdsubst_end || !cmdsubst_begin)
+    {
+        return;
+    }
+
+    /* pos is equivalent to cursor_pos within the range of the command substitution {begin, end} */
+    long offset_within_cmdsubst = cursor_pos - (cmdsubst_begin - buff);
+
+    a = cmdsubst_begin + offset_within_cmdsubst;
+    b = a;
+    pa = cmdsubst_begin + offset_within_cmdsubst;
+    pb = pa;
+
+    assert(cmdsubst_begin >= buff);
+    assert(cmdsubst_begin <= (buff+wcslen(buff)));
+    assert(cmdsubst_end >= cmdsubst_begin);
+    assert(cmdsubst_end <= (buff+wcslen(buff)));
+
+    const wcstring buffcpy = wcstring(cmdsubst_begin, cmdsubst_end-cmdsubst_begin);
+
+    tokenizer_t tok(buffcpy.c_str(), TOK_ACCEPT_UNFINISHED | TOK_SQUASH_ERRORS);
+    for (; tok_has_next(&tok); tok_next(&tok))
+    {
+        size_t tok_begin = tok_get_pos(&tok);
+        size_t tok_end = tok_begin;
+
+        /*
+          Calculate end of token
+        */
+        if (tok_last_type(&tok) == TOK_STRING)
+        {
+            tok_end += wcslen(tok_last(&tok));
+        }
+
+        /*
+          Cursor was before beginning of this token, means that the
+          cursor is between two tokens, so we set it to a zero element
+          string and break
+        */
+        if (tok_begin > offset_within_cmdsubst)
+        {
+            a = b = cmdsubst_begin + offset_within_cmdsubst;
+            break;
+        }
+
+        /*
+          If cursor is inside the token, this is the token we are
+          looking for. If so, set a and b and break
+        */
+        if ((tok_last_type(&tok) == TOK_STRING) && (tok_end >= offset_within_cmdsubst))
+        {
+            a = cmdsubst_begin + tok_get_pos(&tok);
+            b = a + wcslen(tok_last(&tok));
+            break;
+        }
+
+        /*
+          Remember previous string token
+        */
+        if (tok_last_type(&tok) == TOK_STRING)
+        {
+            pa = cmdsubst_begin + tok_get_pos(&tok);
+            pb = pa + wcslen(tok_last(&tok));
+        }
+    }
+
+    if (tok_begin)
+    {
+        *tok_begin = a;
+    }
+
+    if (tok_end)
+    {
+        *tok_end = b;
+    }
+
+    if (prev_begin)
+    {
+        *prev_begin = pa;
+    }
+
+    if (prev_end)
+    {
+        *prev_end = pb;
+    }
+
+    assert(pa >= buff);
+    assert(pa <= (buff+wcslen(buff)));
+    assert(pb >= pa);
+    assert(pb <= (buff+wcslen(buff)));
+
+}
+
+void parse_util_set_argv(const wchar_t * const *argv, const wcstring_list_t &named_arguments)
+{
+    if (*argv)
+    {
+        const wchar_t * const *arg;
+        wcstring sb;
+
+        for (arg=argv; *arg; arg++)
+        {
+            if (arg != argv)
+            {
+                sb.append(ARRAY_SEP_STR);
+            }
+            sb.append(*arg);
+        }
+
+        env_set(L"argv", sb.c_str(), ENV_LOCAL);
+    }
+    else
+    {
+        env_set(L"argv", 0, ENV_LOCAL);
+    }
+
+    if (! named_arguments.empty())
+    {
+        const wchar_t * const *arg;
+        size_t i;
+        for (i=0, arg=argv; i < named_arguments.size(); i++)
+        {
+            env_set(named_arguments.at(i).c_str(), *arg, ENV_LOCAL | ENV_USER);
+
+            if (*arg)
+                arg++;
+        }
+    }
+}
+
+wchar_t *parse_util_unescape_wildcards(const wchar_t *str)
+{
+    wchar_t *in, *out;
+    wchar_t *unescaped;
+
+    CHECK(str, 0);
+
+    unescaped = wcsdup(str);
+
+    if (!unescaped)
+    {
+        DIE_MEM();
+    }
+
+    for (in=out=unescaped; *in; in++)
+    {
+        switch (*in)
+        {
+            case L'\\':
+            {
+                switch (*(in + 1))
+                {
+                    case L'*':
+                    case L'?':
+                    {
+                        in++;
+                        *(out++)=*in;
+                        break;
+                    }
+                    case L'\\':
+                    {
+                        in++;
+                        *(out++)=L'\\';
+                        *(out++)=L'\\';
+                        break;
+                    }
+                    default:
+                    {
+                        *(out++)=*in;
+                        break;
+                    }
+                }
+                break;
+            }
+
+            case L'*':
+            {
+                *(out++)=ANY_STRING;
+                break;
+            }
+
+            case L'?':
+            {
+                *(out++)=ANY_CHAR;
+                break;
+            }
+
+            default:
+            {
+                *(out++)=*in;
+                break;
+            }
+        }
+    }
+    *out = *in;
+    return unescaped;
+}
+
+
+/**
+   Find the outermost quoting style of current token. Returns 0 if
+   token is not quoted.
+
+*/
+static wchar_t get_quote(const wchar_t *cmd, size_t len)
+{
+    size_t i=0;
+    wchar_t res=0;
+
+    while (1)
+    {
+        if (!cmd[i])
+            break;
+
+        if (cmd[i] == L'\\')
+        {
+            i++;
+            if (!cmd[i])
+                break;
+            i++;
+        }
+        else
+        {
+            if (cmd[i] == L'\'' || cmd[i] == L'\"')
+            {
+                const wchar_t *end = quote_end(&cmd[i]);
+                //fwprintf( stderr, L"Jump %d\n",  end-cmd );
+                if ((end == 0) || (!*end) || (end > cmd + len))
+                {
+                    res = cmd[i];
+                    break;
+                }
+                i = end-cmd+1;
+            }
+            else
+                i++;
+        }
+    }
+
+    return res;
+}
+
+void parse_util_get_parameter_info(const wcstring &cmd, const size_t pos, wchar_t *quote, size_t *offset, int *type)
+{
+    size_t prev_pos=0;
+    wchar_t last_quote = '\0';
+    int unfinished;
+
+    tokenizer_t tok(cmd.c_str(), TOK_ACCEPT_UNFINISHED | TOK_SQUASH_ERRORS);
+    for (; tok_has_next(&tok); tok_next(&tok))
+    {
+        if (tok_get_pos(&tok) > pos)
+            break;
+
+        if (tok_last_type(&tok) == TOK_STRING)
+            last_quote = get_quote(tok_last(&tok),
+                                   pos - tok_get_pos(&tok));
+
+        if (type != NULL)
+            *type = tok_last_type(&tok);
+
+        prev_pos = tok_get_pos(&tok);
+    }
+
+    wchar_t *cmd_tmp = wcsdup(cmd.c_str());
+    cmd_tmp[pos]=0;
+    size_t cmdlen = wcslen(cmd_tmp);
+    unfinished = (cmdlen==0);
+    if (!unfinished)
+    {
+        unfinished = (quote != 0);
+
+        if (!unfinished)
+        {
+            if (wcschr(L" \t\n\r", cmd_tmp[cmdlen-1]) != 0)
+            {
+                if ((cmdlen == 1) || (cmd_tmp[cmdlen-2] != L'\\'))
+                {
+                    unfinished=1;
+                }
+            }
+        }
+    }
+
+    if (quote)
+        *quote = last_quote;
+
+    if (offset != 0)
+    {
+        if (!unfinished)
+        {
+            while ((cmd_tmp[prev_pos] != 0) && (wcschr(L";|",cmd_tmp[prev_pos])!= 0))
+                prev_pos++;
+
+            *offset = prev_pos;
+        }
+        else
+        {
+            *offset = pos;
+        }
+    }
+    free(cmd_tmp);
+}
+
+wcstring parse_util_escape_string_with_quote(const wcstring &cmd, wchar_t quote)
+{
+    wcstring result;
+    if (quote == L'\0')
+    {
+        result = escape_string(cmd, ESCAPE_ALL | ESCAPE_NO_QUOTED | ESCAPE_NO_TILDE);
+    }
+    else
+    {
+        bool unescapable = false;
+        for (size_t i = 0; i < cmd.size(); i++)
+        {
+            wchar_t c = cmd.at(i);
+            switch (c)
+            {
+                case L'\n':
+                case L'\t':
+                case L'\b':
+                case L'\r':
+                    unescapable = true;
+                    break;
+                default:
+                    if (c == quote)
+                        result.push_back(L'\\');
+                    result.push_back(c);
+                    break;
+            }
+        }
+
+        if (unescapable)
+        {
+            result = escape_string(cmd, ESCAPE_ALL | ESCAPE_NO_QUOTED);
+            result.insert(0, &quote, 1);
+        }
+    }
+    return result;
+}
+
+/* We are given a parse tree, the index of a node within the tree, its indent, and a vector of indents the same size as the original source string. Set the indent correspdonding to the node's source range, if appropriate.
+
+   trailing_indent is the indent for nodes with unrealized source, i.e. if I type 'if false <ret>' then we have an if node with an empty job list (without source) but we want the last line to be indented anyways.
+
+   switch statements also indent.
+
+   max_visited_node_idx is the largest index we visited.
+*/
+static void compute_indents_recursive(const parse_node_tree_t &tree, node_offset_t node_idx, int node_indent, parse_token_type_t parent_type, std::vector<int> *indents, int *trailing_indent, node_offset_t *max_visited_node_idx)
+{
+    /* Guard against incomplete trees */
+    if (node_idx > tree.size())
+        return;
+
+    /* Update max_visited_node_idx */
+    if (node_idx > *max_visited_node_idx)
+        *max_visited_node_idx = node_idx;
+
+    /* We could implement this by utilizing the fish grammar. But there's an easy trick instead: almost everything that wraps a job list should be indented by 1. So just find all of the job lists. One exception is switch, which wraps a case_item_list instead of a job_list. The other exception is job_list itself: a job_list is a job and a job_list, and we want that child list to be indented the same as the parent. So just find all job_lists whose parent is not a job_list, and increment their indent by 1. */
+
+    const parse_node_t &node = tree.at(node_idx);
+    const parse_token_type_t node_type = node.type;
+
+    /* Increment the indent if we are either a root job_list, or root case_item_list */
+    const bool is_root_job_list = (node_type == symbol_job_list && parent_type != symbol_job_list);
+    const bool is_root_case_item_list = (node_type == symbol_case_item_list && parent_type != symbol_case_item_list);
+    if (is_root_job_list || is_root_case_item_list)
+    {
+        node_indent += 1;
+    }
+
+    /* If we have source, store the trailing indent unconditionally. If we do not have source, store the trailing indent only if ours is bigger; this prevents the trailing "run" of terminal job lists from affecting the trailing indent. For example, code like this:
+
+            if foo
+
+      will be parsed as this:
+
+      job_list
+        job
+           if_statement
+               job [if]
+               job_list [empty]
+         job_list [empty]
+
+      There's two "terminal" job lists, and we want the innermost one.
+
+      Note we are relying on the fact that nodes are in the same order as the source, i.e. an in-order traversal of the node tree also traverses the source from beginning to end.
+    */
+    if (node.has_source() || node_indent > *trailing_indent)
+    {
+        *trailing_indent = node_indent;
+    }
+
+
+    /* Store the indent into the indent array */
+    if (node.source_start != SOURCE_OFFSET_INVALID && node.source_start < indents->size())
+    {
+        if (node.has_source())
+        {
+            /* A normal non-empty node. Store the indent unconditionally. */
+            indents->at(node.source_start) = node_indent;
+        }
+        else
+        {
+            /* An empty node. We have a source offset but no source length. This can come about when a node legitimately empty:
+
+                  while true; end
+
+               The job_list inside the while loop is empty. It still has a source offset (at the end of the while statement) but no source extent.
+               We still need to capture that indent, because there may be comments inside:
+                    while true
+                       # loop forever
+                    end
+
+               The 'loop forever' comment must be indented, by virtue of storing the indent.
+
+               Now consider what happens if we remove the end:
+
+                   while true
+                     # loop forever
+
+                Now both the job_list and end_command are unmaterialized. However, we want the indent to be of the job_list and not the end_command.  Therefore, we only store the indent if it's bigger.
+            */
+            if (node_indent > indents->at(node.source_start))
+            {
+                indents->at(node.source_start) = node_indent;
+            }
+        }
+    }
+
+
+    /* Recursive to all our children */
+    for (node_offset_t idx = 0; idx < node.child_count; idx++)
+    {
+        /* Note we pass our type to our child, which becomes its parent node type */
+        compute_indents_recursive(tree, node.child_start + idx, node_indent, node_type, indents, trailing_indent, max_visited_node_idx);
+    }
+}
+
+std::vector<int> parse_util_compute_indents(const wcstring &src)
+{
+    /* Make a vector the same size as the input string, which contains the indents. Initialize them to -1. */
+    const size_t src_size = src.size();
+    std::vector<int> indents(src_size, -1);
+
+    /* Parse the string. We pass continue_after_error to produce a forest; the trailing indent of the last node we visited becomes the input indent of the next. I.e. in the case of 'switch foo ; cas', we get an invalid parse tree (since 'cas' is not valid) but we indent it as if it were a case item list */
+    parse_node_tree_t tree;
+    parse_tree_from_string(src, parse_flag_continue_after_error | parse_flag_include_comments | parse_flag_accept_incomplete_tokens, &tree, NULL /* errors */);
+
+    /* Start indenting at the first node. If we have a parse error, we'll have to start indenting from the top again */
+    node_offset_t start_node_idx = 0;
+    int last_trailing_indent = 0;
+
+    while (start_node_idx < tree.size())
+    {
+        /* The indent that we'll get for the last line */
+        int trailing_indent = 0;
+
+        /* Biggest offset we visited */
+        node_offset_t max_visited_node_idx = 0;
+
+        /* Invoke the recursive version. As a hack, pass job_list for the 'parent' token type, which will prevent the really-root job list from indenting */
+        compute_indents_recursive(tree, start_node_idx, last_trailing_indent, symbol_job_list, &indents, &trailing_indent, &max_visited_node_idx);
+
+        /* We may have more to indent. The trailing indent becomes our current indent. Start at the node after the last we visited. */
+        last_trailing_indent = trailing_indent;
+        start_node_idx = max_visited_node_idx + 1;
+    }
+
+    /* Handle comments. Each comment node has a parent (which is whatever the top of the symbol stack was when the comment was encountered). So the source range of the comment has the same indent as its parent. */
+    const size_t tree_size = tree.size();
+    for (node_offset_t i=0; i < tree_size; i++)
+    {
+        const parse_node_t &node = tree.at(i);
+        if (node.type == parse_special_type_comment && node.has_source() && node.parent < tree_size)
+        {
+            const parse_node_t &parent = tree.at(node.parent);
+            if (parent.source_start != SOURCE_OFFSET_INVALID)
+            {
+                indents.at(node.source_start) = indents.at(parent.source_start);
+            }
+        }
+    }
+
+    /* Now apply the indents. The indents array has -1 for places where the indent does not change, so start at each value and extend it along the run of -1s */
+    int last_indent = 0;
+    for (size_t i=0; i<src_size; i++)
+    {
+        int this_indent = indents.at(i);
+        if (this_indent < 0)
+        {
+            indents.at(i) = last_indent;
+        }
+        else
+        {
+            /* New indent level */
+            last_indent = this_indent;
+            /* Make all whitespace before a token have the new level. This avoid using the wrong indentation level if a new line starts with whitespace. */
+            size_t prev_char_idx = i;
+            while (prev_char_idx--)
+            {
+                if (!wcschr(L" \n\t\r", src.at(prev_char_idx)))
+                    break;
+                indents.at(prev_char_idx) = last_indent;
+            }
+        }
+    }
+
+    /* Ensure trailing whitespace has the trailing indent. This makes sure a new line is correctly indented even if it is empty. */
+    size_t suffix_idx = src_size;
+    while (suffix_idx--)
+    {
+        if (!wcschr(L" \n\t\r", src.at(suffix_idx)))
+            break;
+        indents.at(suffix_idx) = last_trailing_indent;
+    }
+
+    return indents;
+}
+
+/* Append a syntax error to the given error list */
+
+static bool append_syntax_error(parse_error_list_t *errors, size_t source_location, const wchar_t *fmt, ...)
+{
+    parse_error_t error;
+    error.source_start = source_location;
+    error.source_length = 0;
+    error.code = parse_error_syntax;
+    
+    va_list va;
+    va_start(va, fmt);
+    error.text = vformat_string(fmt, va);
+    va_end(va);
+    
+    errors->push_back(error);
+    return true;
+}
+
+
+/**
+   Returns 1 if the specified command is a builtin that may not be used in a pipeline
+*/
+static int parser_is_pipe_forbidden(const wcstring &word)
+{
+    return contains(word,
+                    L"exec",
+                    L"case",
+                    L"break",
+                    L"return",
+                    L"continue");
+}
+
+bool parse_util_argument_is_help(const wchar_t *s, int min_match)
+{
+    CHECK(s, 0);
+
+    size_t len = wcslen(s);
+
+    min_match = maxi(min_match, 3);
+
+    return (wcscmp(L"-h", s) == 0) ||
+           (len >= (size_t)min_match && (wcsncmp(L"--help", s, len) == 0));
+}
+
+// Check if the first argument under the given node is --help
+static bool first_argument_is_help(const parse_node_tree_t &node_tree, const parse_node_t &node, const wcstring &src)
+{
+    bool is_help = false;
+    const parse_node_tree_t::parse_node_list_t arg_nodes = node_tree.find_nodes(node, symbol_argument, 1);
+    if (! arg_nodes.empty())
+    {
+        // Check the first argument only
+        const parse_node_t &arg = *arg_nodes.at(0);
+        const wcstring first_arg_src = arg.get_source(src);
+        is_help = parse_util_argument_is_help(first_arg_src.c_str(), 3);
+    }
+    return is_help;
+}
+
+/* If a var name or command is too long for error reporting, make it shorter */
+static wcstring truncate_string(const wcstring &str)
+{
+    const size_t max_len = 16;
+    wcstring result(str, 0, max_len);
+    if (str.size() > max_len)
+    {
+        // Truncate!
+        if (ellipsis_char == L'\x2026')
+        {
+            result.at(max_len - 1) = ellipsis_char;
+        }
+        else
+        {
+            result.replace(max_len - 3, 3, L"...");
+        }
+    }
+    return result;
+}
+
+/* Given a wide character immediately after a dollar sign, return the appropriate error message. For example, if wc is @, then the variable name was $@ and we suggest $argv. */
+static const wchar_t *error_format_for_character(wchar_t wc)
+{
+    switch (wc)
+    {
+        case L'?': return ERROR_NOT_STATUS;
+        case L'#': return ERROR_NOT_ARGV_COUNT;
+        case L'@': return ERROR_NOT_ARGV_AT;
+        case L'*': return ERROR_NOT_ARGV_STAR;
+        case L'$':
+        case VARIABLE_EXPAND:
+        case VARIABLE_EXPAND_SINGLE:
+        case VARIABLE_EXPAND_EMPTY:
+            return ERROR_NOT_PID;
+        default: return ERROR_BAD_VAR_CHAR1;
+    }
+}
+
+void parse_util_expand_variable_error(const wcstring &token, size_t global_token_pos, size_t dollar_pos, parse_error_list_t *errors)
+{
+    // Note that dollar_pos is probably VARIABLE_EXPAND or VARIABLE_EXPAND_SINGLE, not a literal dollar sign
+    assert(errors != NULL);
+    assert(dollar_pos < token.size());
+    const bool double_quotes = (token.at(dollar_pos) == VARIABLE_EXPAND_SINGLE);
+    const size_t start_error_count = errors->size();
+    const size_t global_dollar_pos = global_token_pos + dollar_pos;
+    const size_t global_after_dollar_pos = global_dollar_pos + 1;
+    wchar_t char_after_dollar = (dollar_pos + 1 >= token.size() ? L'\0' : token.at(dollar_pos + 1));
+    switch (char_after_dollar)
+    {
+        case BRACKET_BEGIN:
+        case L'{':
+            
+        {
+            // The BRACKET_BEGIN is for unquoted, the { is for quoted. Anyways we have (possible quoted) ${. See if we have a }, and the stuff in between is variable material. If so, report a bracket error. Otherwise just complain about the ${.
+            bool looks_like_variable = false;
+            size_t closing_bracket = token.find(char_after_dollar == L'{' ? L'}' : BRACKET_END, dollar_pos + 2);
+            wcstring var_name;
+            if (closing_bracket != wcstring::npos)
+            {
+                size_t var_start = dollar_pos + 2, var_end = closing_bracket;
+                var_name = wcstring(token, var_start, var_end - var_start);
+                looks_like_variable = ! var_name.empty() && wcsvarname(var_name.c_str()) == NULL;
+            }
+            if (looks_like_variable)
+            {
+                append_syntax_error(errors,
+                                    global_after_dollar_pos,
+                                    double_quotes ? ERROR_BRACKETED_VARIABLE_QUOTED1 : ERROR_BRACKETED_VARIABLE1,
+                                    truncate_string(var_name).c_str());
+            }
+            else
+            {
+                append_syntax_error(errors,
+                                    global_after_dollar_pos,
+                                    ERROR_BAD_VAR_CHAR1,
+                                    L'{');
+            }
+            break;
+        }
+            
+        case INTERNAL_SEPARATOR:
+        {
+            //e.g.: echo foo"$"baz
+            // These are only ever quotes, not command substitutions. Command substitutions are handled earlier.
+            append_syntax_error(errors,
+                                global_dollar_pos,
+                                ERROR_NO_VAR_NAME);
+            break;
+        }
+        
+        case '(':
+        {
+            // e.g.: 'echo "foo$(bar)baz"
+            // Try to determine what's in the parens.
+            wcstring token_after_parens;
+            wcstring paren_text;
+            size_t open_parens = dollar_pos + 1, cmdsub_start = 0, cmdsub_end = 0;
+            if (parse_util_locate_cmdsubst_range(token,
+                                                 &open_parens,
+                                                 &paren_text,
+                                                 &cmdsub_start,
+                                                 &cmdsub_end,
+                                                 true) > 0)
+            {
+                token_after_parens = tok_first(paren_text.c_str());
+            }
+            
+            /* Make sure we always show something */
+            if (token_after_parens.empty())
+            {
+                token_after_parens = L"...";
+            }
+            
+            append_syntax_error(errors,
+                                global_dollar_pos,
+                                ERROR_BAD_VAR_SUBCOMMAND1,
+                                truncate_string(token_after_parens).c_str());
+            break;
+        }
+        
+        case L'\0':
+        {
+            append_syntax_error(errors,
+                                global_dollar_pos,
+                                ERROR_NO_VAR_NAME);
+            break;
+        }
+        
+        default:
+        {
+            wchar_t token_stop_char = char_after_dollar;
+            // Unescape (see #50)
+            if (token_stop_char == ANY_CHAR)
+                token_stop_char = L'?';
+            else if (token_stop_char == ANY_STRING || token_stop_char == ANY_STRING_RECURSIVE)
+                token_stop_char = L'*';
+            
+            /* Determine which error message to use. The format string may not consume all the arguments we pass but that's harmless. */
+            const wchar_t *error_fmt = error_format_for_character(token_stop_char);
+            
+            append_syntax_error(errors,
+                                global_after_dollar_pos,
+                                error_fmt,
+                                token_stop_char);
+            break;
+        }
+            
+    }
+    
+    // We should have appended exactly one error
+    assert(errors->size() == start_error_count + 1);
+}
+
+/* Detect cases like $(abc). Given an arg like foo(bar), let arg_src be foo and cmdsubst_src be bar. If arg ends with VARIABLE_EXPAND, then report an error. */
+static parser_test_error_bits_t detect_dollar_cmdsub_errors(size_t arg_src_offset, const wcstring &arg_src, const wcstring &cmdsubst_src, parse_error_list_t *out_errors)
+{
+    parser_test_error_bits_t result_bits = 0;
+    wcstring unescaped_arg_src;
+    if (unescape_string(arg_src, &unescaped_arg_src, UNESCAPE_SPECIAL))
+    {
+        if (! unescaped_arg_src.empty())
+        {
+            wchar_t last = unescaped_arg_src.at(unescaped_arg_src.size() - 1);
+            if (last == VARIABLE_EXPAND)
+            {
+                result_bits |= PARSER_TEST_ERROR;
+                if (out_errors != NULL)
+                {
+                    wcstring subcommand_first_token = tok_first(cmdsubst_src.c_str());
+                    if (subcommand_first_token.empty())
+                    {
+                        // e.g. $(). Report somthing.
+                        subcommand_first_token = L"...";
+                    }
+                    append_syntax_error(out_errors,
+                                        arg_src_offset + arg_src.size() - 1, // global position of the dollar
+                                        ERROR_BAD_VAR_SUBCOMMAND1,
+                                        truncate_string(subcommand_first_token).c_str());
+                }
+            }
+        }
+    }
+    return result_bits;
+}
+
+/**
+   Test if this argument contains any errors. Detected errors include
+   syntax errors in command substitutions, improperly escaped
+   characters and improper use of the variable expansion operator.
+*/
+parser_test_error_bits_t parse_util_detect_errors_in_argument(const parse_node_t &node, const wcstring &arg_src, parse_error_list_t *out_errors)
+{
+    assert(node.type == symbol_argument);
+
+    int err=0;
+
+    wchar_t *paran_begin, *paran_end;
+    int do_loop = 1;
+    
+    wcstring working_copy = arg_src;
+
+    while (do_loop)
+    {
+        const wchar_t *working_copy_cstr = working_copy.c_str();
+        switch (parse_util_locate_cmdsubst(working_copy_cstr,
+                                           &paran_begin,
+                                           &paran_end,
+                                           false))
+        {
+            case -1:
+            {
+                err=1;
+                if (out_errors)
+                {
+                    append_syntax_error(out_errors, node.source_start, L"Mismatched parenthesis");
+                }
+                return err;
+            }
+
+            case 0:
+            {
+                do_loop = 0;
+                break;
+            }
+
+            case 1:
+            {
+
+                const wcstring subst(paran_begin + 1, paran_end);
+
+                // Replace the command substitution with just INTERNAL_SEPARATOR
+                size_t cmd_sub_start = paran_begin - working_copy_cstr;
+                size_t cmd_sub_len = paran_end + 1 - paran_begin;
+                working_copy.replace(cmd_sub_start, cmd_sub_len, wcstring(1, INTERNAL_SEPARATOR));
+
+                parse_error_list_t subst_errors;
+                err |= parse_util_detect_errors(subst, &subst_errors, false /* do not accept incomplete */);
+
+                /* Our command substitution produced error offsets relative to its source. Tweak the offsets of the errors in the command substitution to account for both its offset within the string, and the offset of the node */
+                size_t error_offset = cmd_sub_start + 1 + node.source_start;
+                parse_error_offset_source_start(&subst_errors, error_offset);
+                
+                if (out_errors != NULL)
+                {
+                    out_errors->insert(out_errors->end(), subst_errors.begin(), subst_errors.end());
+                    
+                    /* Hackish. Take this opportunity to report $(...) errors. We do this because after we've replaced with internal separators, we can't distinguish between "" and (), and also we no longer have the source of the command substitution. As an optimization, this is only necessary if the last character is a $. */
+                    if (cmd_sub_start > 0 && working_copy.at(cmd_sub_start - 1) == L'$')
+                    {
+                        err |= detect_dollar_cmdsub_errors(node.source_start,
+                                                           working_copy.substr(0, cmd_sub_start),
+                                                           subst,
+                                                           out_errors);
+                    }
+                }
+                break;
+            }
+        }
+    }
+
+    wcstring unesc;
+    if (! unescape_string(working_copy, &unesc, UNESCAPE_SPECIAL))
+    {
+        if (out_errors)
+        {
+            append_syntax_error(out_errors, node.source_start, L"Invalid token '%ls'", working_copy.c_str());
+        }
+        return 1;
+    }
+    else
+    {
+        /* Check for invalid variable expansions */
+        const size_t unesc_size = unesc.size();
+        for (size_t idx = 0; idx < unesc_size; idx++)
+        {
+            switch (unesc.at(idx))
+            {
+                case VARIABLE_EXPAND:
+                case VARIABLE_EXPAND_SINGLE:
+                {
+                    wchar_t next_char = (idx + 1 < unesc_size ? unesc.at(idx + 1) : L'\0');
+
+                    if (next_char != VARIABLE_EXPAND && next_char != VARIABLE_EXPAND_SINGLE && ! wcsvarchr(next_char))
+                    {
+                        err=1;
+                        if (out_errors)
+                        {
+                            /* We have something like $$$^....  Back up until we reach the first $ */
+                            size_t first_dollar = idx;
+                            while (first_dollar > 0 && (unesc.at(first_dollar-1) == VARIABLE_EXPAND || unesc.at(first_dollar-1) == VARIABLE_EXPAND_SINGLE))
+                            {
+                                first_dollar--;
+                            }
+                            parse_util_expand_variable_error(unesc, node.source_start, first_dollar, out_errors);
+                        }
+                    }
+
+                    break;
+                }
+            }
+        }
+    }
+
+    return err;
+}
+
+parser_test_error_bits_t parse_util_detect_errors(const wcstring &buff_src, parse_error_list_t *out_errors, bool allow_incomplete)
+{
+    parse_node_tree_t node_tree;
+    parse_error_list_t parse_errors;
+
+    parser_test_error_bits_t res = 0;
+
+    // Whether we encountered a parse error
+    bool errored = false;
+
+    // Whether we encountered an unclosed block
+    // We detect this via an 'end_command' block without source
+    bool has_unclosed_block = false;
+
+    // Whether there's an unclosed quote, and therefore unfinished
+    // This is only set if allow_incomplete is set
+    bool has_unclosed_quote = false;
+
+    // Parse the input string into a parse tree
+    // Some errors are detected here
+    bool parsed = parse_tree_from_string(buff_src, allow_incomplete ? parse_flag_leave_unterminated : parse_flag_none, &node_tree, &parse_errors);
+
+    if (allow_incomplete)
+    {
+        for (size_t i=0; i < parse_errors.size(); i++)
+        {
+            if (parse_errors.at(i).code == parse_error_tokenizer_unterminated_quote)
+            {
+                // Remove this error, since we don't consider it a real error
+                has_unclosed_quote = true;
+                parse_errors.erase(parse_errors.begin() + i);
+                i--;
+            }
+        }
+    }
+    
+    // #1238: If the only error was unterminated quote, then consider this to have parsed successfully. A better fix would be to have parse_tree_from_string return this information directly (but it would be a shame to munge up its nice bool return).
+    if (parse_errors.empty() && has_unclosed_quote)
+    {
+        parsed = true;
+    }
+
+    if (! parsed)
+    {
+        errored = true;
+    }
+
+    // has_unclosed_quote may only be set if allow_incomplete is true
+    assert(! has_unclosed_quote || allow_incomplete);
+    
+    // Expand all commands
+    // Verify 'or' and 'and' not used inside pipelines
+    // Verify pipes via parser_is_pipe_forbidden
+    // Verify return only within a function
+    // Verify no variable expansions
+
+    if (! errored)
+    {
+        const size_t node_tree_size = node_tree.size();
+        for (size_t i=0; i < node_tree_size; i++)
+        {
+            const parse_node_t &node = node_tree.at(i);
+            if (node.type == symbol_end_command && ! node.has_source())
+            {
+                // an 'end' without source is an unclosed block
+                has_unclosed_block = true;
+            }
+            else if (node.type == symbol_boolean_statement)
+            {
+                // 'or' and 'and' can be in a pipeline, as long as they're first
+                parse_bool_statement_type_t type = parse_node_tree_t::statement_boolean_type(node);
+                if ((type ==  parse_bool_and || type == parse_bool_or) && node_tree.statement_is_in_pipeline(node, false /* don't count first */))
+                {
+                    errored = append_syntax_error(&parse_errors, node.source_start, EXEC_ERR_MSG, (type ==  parse_bool_and) ? L"and" : L"or");
+                }
+            }
+            else if (node.type == symbol_argument)
+            {
+                const wcstring arg_src = node.get_source(buff_src);
+                res |= parse_util_detect_errors_in_argument(node, arg_src, &parse_errors);
+            }
+            else if (node.type == symbol_job)
+            {
+                if (node_tree.job_should_be_backgrounded(node))
+                {
+                    /* Disallow background in the following cases:
+                    
+                       foo & ; and bar
+                       foo & ; or bar
+                       if foo & ; end
+                       while foo & ; end
+                    */
+                    const parse_node_t *job_parent = node_tree.get_parent(node);
+                    assert(job_parent != NULL);
+                    switch (job_parent->type)
+                    {
+                        case symbol_if_clause:
+                        case symbol_while_header:
+                        {
+                            assert(node_tree.get_child(*job_parent, 1) == &node);
+                            errored = append_syntax_error(&parse_errors, node.source_start, BACKGROUND_IN_CONDITIONAL_ERROR_MSG);
+                            break;
+                        }
+                        
+                        case symbol_job_list:
+                        {
+                            // This isn't very complete, e.g. we don't catch 'foo & ; not and bar'
+                            assert(node_tree.get_child(*job_parent, 0) == &node);
+                            const parse_node_t *next_job_list = node_tree.get_child(*job_parent, 1, symbol_job_list);
+                            assert(next_job_list != NULL);
+                            const parse_node_t *next_job = node_tree.next_node_in_node_list(*next_job_list, symbol_job, NULL);
+                            if (next_job != NULL)
+                            {
+                                const parse_node_t *next_statement = node_tree.get_child(*next_job, 0, symbol_statement);
+                                if (next_statement != NULL)
+                                {
+                                    const parse_node_t *spec_statement = node_tree.get_child(*next_statement, 0);
+                                    if (spec_statement && spec_statement->type == symbol_boolean_statement)
+                                    {
+                                        switch (parse_node_tree_t::statement_boolean_type(*spec_statement))
+                                        {
+                                            // These are not allowed
+                                            case parse_bool_and:
+                                                errored = append_syntax_error(&parse_errors, spec_statement->source_start, BOOL_AFTER_BACKGROUND_ERROR_MSG, L"and");
+                                                break;
+                                            case parse_bool_or:
+                                                errored = append_syntax_error(&parse_errors, spec_statement->source_start, BOOL_AFTER_BACKGROUND_ERROR_MSG, L"or");
+                                                break;
+                                            case parse_bool_not:
+                                                // This one is OK
+                                                break;
+                                        }
+                                    }
+                                }
+                            }
+                            break;
+                        }
+                        
+                        default:
+                        break;
+                    }
+                }
+            }
+            else if (node.type == symbol_plain_statement)
+            {
+                // In a few places below, we want to know if we are in a pipeline
+                const bool is_in_pipeline = node_tree.statement_is_in_pipeline(node, true /* count first */);
+
+                // We need to know the decoration
+                const enum parse_statement_decoration_t decoration = node_tree.decoration_for_plain_statement(node);
+
+                // Check that we don't try to pipe through exec
+                if (is_in_pipeline && decoration == parse_statement_decoration_exec)
+                {
+                    errored = append_syntax_error(&parse_errors, node.source_start, EXEC_ERR_MSG, L"exec");
+                }
+
+                wcstring command;
+                if (node_tree.command_for_plain_statement(node, buff_src, &command))
+                {
+                    // Check that we can expand the command
+                    if (! expand_one(command, EXPAND_SKIP_CMDSUBST | EXPAND_SKIP_VARIABLES | EXPAND_SKIP_JOBS, NULL))
+                    {
+                        // TODO: leverage the resulting errors
+                        errored = append_syntax_error(&parse_errors, node.source_start, ILLEGAL_CMD_ERR_MSG, command.c_str());
+                    }
+
+                    // Check that pipes are sound
+                    if (! errored && parser_is_pipe_forbidden(command) && is_in_pipeline)
+                    {
+                        errored = append_syntax_error(&parse_errors, node.source_start, EXEC_ERR_MSG, command.c_str());
+                    }
+
+                    // Check that we don't return from outside a function
+                    // But we allow it if it's 'return --help'
+                    if (! errored && command == L"return")
+                    {
+                        const parse_node_t *ancestor = &node;
+                        bool found_function = false;
+                        while (ancestor != NULL)
+                        {
+                            const parse_node_t *possible_function_header = node_tree.header_node_for_block_statement(*ancestor);
+                            if (possible_function_header != NULL && possible_function_header->type == symbol_function_header)
+                            {
+                                found_function = true;
+                                break;
+                            }
+                            ancestor = node_tree.get_parent(*ancestor);
+
+                        }
+                        if (! found_function && ! first_argument_is_help(node_tree, node, buff_src))
+                        {
+                            errored = append_syntax_error(&parse_errors, node.source_start, INVALID_RETURN_ERR_MSG);
+                        }
+                    }
+
+                    // Check that we don't break or continue from outside a loop
+                    if (! errored && (command == L"break" || command == L"continue"))
+                    {
+                        // Walk up until we hit a 'for' or 'while' loop. If we hit a function first, stop the search; we can't break an outer loop from inside a function.
+                        // This is a little funny because we can't tell if it's a 'for' or 'while' loop from the ancestor alone; we need the header. That is, we hit a block_statement, and have to check its header.
+                        bool found_loop = false, end_search = false;
+                        const parse_node_t *ancestor = &node;
+                        while (ancestor != NULL && ! end_search)
+                        {
+                            const parse_node_t *loop_or_function_header = node_tree.header_node_for_block_statement(*ancestor);
+                            if (loop_or_function_header != NULL)
+                            {
+                                switch (loop_or_function_header->type)
+                                {
+                                    case symbol_while_header:
+                                    case symbol_for_header:
+                                        // this is a loop header, so we can break or continue
+                                        found_loop = true;
+                                        end_search = true;
+                                        break;
+
+                                    case symbol_function_header:
+                                        // this is a function header, so we cannot break or continue. We stop our search here.
+                                        found_loop = false;
+                                        end_search = true;
+                                        break;
+
+                                    default:
+                                        // most likely begin / end style block, which makes no difference
+                                        break;
+                                }
+                            }
+                            ancestor = node_tree.get_parent(*ancestor);
+                        }
+
+                        if (! found_loop && ! first_argument_is_help(node_tree, node, buff_src))
+                        {
+                            errored = append_syntax_error(&parse_errors,
+                                                          node.source_start,
+                                                          (command == L"break" ? INVALID_BREAK_ERR_MSG : INVALID_CONTINUE_ERR_MSG));
+                        }
+                    }
+
+                    // Check that we don't do an invalid builtin (#1252)
+                    if (! errored && decoration == parse_statement_decoration_builtin && ! builtin_exists(command))
+                    {
+                        errored = append_syntax_error(&parse_errors,
+                                                      node.source_start,
+                                                      UNKNOWN_BUILTIN_ERR_MSG,
+                                                      command.c_str());
+                    }
+
+                }
+            }
+        }
+    }
+
+    if (errored)
+        res |= PARSER_TEST_ERROR;
+
+    if (has_unclosed_block || has_unclosed_quote)
+        res |= PARSER_TEST_INCOMPLETE;
+
+    if (out_errors)
+    {
+        out_errors->swap(parse_errors);
+    }
+
+    return res;
+
+}
diff --git a/src/parse_util.h b/src/parse_util.h
new file mode 100644
index 00000000..760d4477
--- /dev/null
+++ b/src/parse_util.h
@@ -0,0 +1,194 @@
+/** \file parse_util.h
+
+    Various mostly unrelated utility functions related to parsing,
+    loading and evaluating fish code.
+*/
+
+#ifndef FISH_PARSE_UTIL_H
+#define FISH_PARSE_UTIL_H
+
+#include <stddef.h>
+#include <vector>
+#include "common.h"
+#include "parse_constants.h"
+
+/**
+   Find the beginning and end of the first subshell in the specified string.
+
+   \param in the string to search for subshells
+   \param begin the starting paranthesis of the subshell
+   \param end the ending paranthesis of the subshell
+   \param accept_incomplete whether to permit missing closing parenthesis
+   \return -1 on syntax error, 0 if no subshells exist and 1 on success
+*/
+
+int parse_util_locate_cmdsubst(const wchar_t *in,
+                               wchar_t **begin,
+                               wchar_t **end,
+                               bool accept_incomplete);
+
+/** Same as parse_util_locate_cmdsubst, but handles square brackets [ ] */
+int parse_util_locate_slice(const wchar_t *in,
+                            wchar_t **begin,
+                            wchar_t **end,
+                            bool accept_incomplete);
+
+/**
+   Alternative API. Iterate over command substitutions.
+
+   \param str the string to search for subshells
+   \param inout_cursor_offset On input, the location to begin the search. On output, either the end of the string, or just after the closed-paren.
+   \param out_contents On output, the contents of the command substitution
+   \param out_start On output, the offset of the start of the command substitution (open paren)
+   \param out_end On output, the offset of the end of the command substitution (close paren), or the end of the string if it was incomplete
+   \param accept_incomplete whether to permit missing closing parenthesis
+   \return -1 on syntax error, 0 if no subshells exist and 1 on success
+*/
+
+int parse_util_locate_cmdsubst_range(const wcstring &str,
+                                     size_t *inout_cursor_offset,
+                                     wcstring *out_contents,
+                                     size_t *out_start,
+                                     size_t *out_end,
+                                     bool accept_incomplete);
+
+/**
+   Find the beginning and end of the command substitution under the
+   cursor. If no subshell is found, the entire string is returned. If
+   the current command substitution is not ended, i.e. the closing
+   parenthesis is missing, then the string from the beginning of the
+   substitution to the end of the string is returned.
+
+   \param buff the string to search for subshells
+   \param cursor_pos the position of the cursor
+   \param a the start of the searched string
+   \param b the end of the searched string
+*/
+void parse_util_cmdsubst_extent(const wchar_t *buff,
+                                size_t cursor_pos,
+                                const wchar_t **a,
+                                const wchar_t **b);
+
+/**
+   Find the beginning and end of the process definition under the cursor
+
+   \param buff the string to search for subshells
+   \param cursor_pos the position of the cursor
+   \param a the start of the searched string
+   \param b the end of the searched string
+*/
+void parse_util_process_extent(const wchar_t *buff,
+                               size_t cursor_pos,
+                               const wchar_t **a,
+                               const wchar_t **b);
+
+
+/**
+   Find the beginning and end of the job definition under the cursor
+
+   \param buff the string to search for subshells
+   \param cursor_pos the position of the cursor
+   \param a the start of the searched string
+   \param b the end of the searched string
+*/
+void parse_util_job_extent(const wchar_t *buff,
+                           size_t cursor_pos,
+                           const wchar_t **a,
+                           const wchar_t **b);
+
+/**
+   Find the beginning and end of the token under the cursor and the
+   token before the current token. Any combination of tok_begin,
+   tok_end, prev_begin and prev_end may be null.
+
+   \param buff the string to search for subshells
+   \param cursor_pos the position of the cursor
+   \param tok_begin the start of the current token
+   \param tok_end the end of the current token
+   \param prev_begin the start o the token before the current token
+   \param prev_end the end of the token before the current token
+*/
+void parse_util_token_extent(const wchar_t *buff,
+                             size_t cursor_pos,
+                             const wchar_t **tok_begin,
+                             const wchar_t **tok_end,
+                             const wchar_t **prev_begin,
+                             const wchar_t **prev_end);
+
+
+/**
+   Get the linenumber at the specified character offset
+*/
+int parse_util_lineno(const wchar_t *str, size_t len);
+
+/**
+   Calculate the line number of the specified cursor position
+ */
+int parse_util_get_line_from_offset(const wcstring &str, size_t pos);
+
+/**
+   Get the offset of the first character on the specified line
+ */
+size_t parse_util_get_offset_from_line(const wcstring &str, int line);
+
+
+/**
+   Return the total offset of the buffer for the cursor position nearest to the specified poition
+ */
+size_t parse_util_get_offset(const wcstring &str, int line, long line_offset);
+
+/**
+   Set the argv environment variable to the specified null-terminated
+   array of strings.
+*/
+void parse_util_set_argv(const wchar_t * const *argv, const wcstring_list_t &named_arguments);
+
+/**
+   Make a duplicate of the specified string, unescape wildcard
+   characters but not performing any other character transformation.
+*/
+wchar_t *parse_util_unescape_wildcards(const wchar_t *in);
+
+/**
+   Checks if the specified string is a help option.
+
+   \param s the string to test
+   \param min_match is the minimum number of characters that must match in a long style option, i.e. the longest common prefix between --help and any other option. If less than 3, 3 will be assumed.
+*/
+bool parse_util_argument_is_help(const wchar_t *s, int min_match);
+
+
+/**
+   Calculates information on the parameter at the specified index.
+
+   \param cmd The command to be analyzed
+   \param pos An index in the string which is inside the parameter
+   \param quote If not NULL, store the type of quote this parameter has, can be either ', " or \\0, meaning the string is not quoted.
+   \param offset If not NULL, get_param will store the offset to the beginning of the parameter.
+   \param type If not NULL, get_param will store the token type as returned by tok_last.
+*/
+void parse_util_get_parameter_info(const wcstring &cmd, const size_t pos, wchar_t *quote, size_t *offset, int *type);
+
+/**
+   Attempts to escape the string 'cmd' using the given quote type, as determined by the quote character. The quote can be a single quote or double quote, or L'\0' to indicate no quoting (and thus escaping should be with backslashes).
+*/
+wcstring parse_util_escape_string_with_quote(const wcstring &cmd, wchar_t quote);
+
+/** Given a string, parse it as fish code and then return the indents. The return value has the same size as the string */
+std::vector<int> parse_util_compute_indents(const wcstring &src);
+
+/** Given a string, detect parse errors in it. If allow_incomplete is set, then if the string is incomplete (e.g. an unclosed quote), an error is not returned and the PARSER_TEST_INCOMPLETE bit is set in the return value. If allow_incomplete is not set, then incomplete strings result in an error. */
+parser_test_error_bits_t parse_util_detect_errors(const wcstring &buff_src, parse_error_list_t *out_errors = NULL, bool allow_incomplete = true);
+
+/**
+   Test if this argument contains any errors. Detected errors include syntax errors in command substitutions, improperly escaped characters and improper use of the variable expansion operator.
+
+   This does NOT currently detect unterminated quotes.
+*/
+class parse_node_t;
+parser_test_error_bits_t parse_util_detect_errors_in_argument(const parse_node_t &node, const wcstring &arg_src, parse_error_list_t *out_errors = NULL);
+
+/* Given a string containing a variable expansion error, append an appropriate error to the errors list. The global_token_pos is the offset of the token in the larger source, and the dollar_pos is the offset of the offending dollar sign within the token. */
+void parse_util_expand_variable_error(const wcstring &token, size_t global_token_pos, size_t dollar_pos, parse_error_list_t *out_errors);
+
+#endif
diff --git a/src/parser.cpp b/src/parser.cpp
new file mode 100644
index 00000000..2e5ebe75
--- /dev/null
+++ b/src/parser.cpp
@@ -0,0 +1,1196 @@
+/** \file parser.c
+
+The fish parser. Contains functions for parsing and evaluating code.
+
+*/
+
+#include "config.h" // IWYU pragma: keep
+
+#include <stdio.h>
+#include <wchar.h>
+#include <assert.h>
+#include <string>
+#include <algorithm>
+
+#include "fallback.h"
+#include "common.h"
+#include "wutil.h"
+#include "proc.h"
+#include "parser.h"
+#include "function.h"
+#include "env.h"
+#include "expand.h"
+#include "reader.h"
+#include "sanity.h"
+#include "event.h"
+#include "intern.h"
+#include "signal.h" // IWYU pragma: keep - needed for CHECK_BLOCK
+#include "parse_util.h"
+#include "parse_tree.h"
+#include "parse_execution.h"
+
+/**
+   Error message for tokenizer error. The tokenizer message is
+   appended to this message.
+*/
+#define TOK_ERR_MSG _( L"Tokenizer error: '%ls'")
+
+/**
+   Error for evaluating in illegal scope
+*/
+#define INVALID_SCOPE_ERR_MSG _( L"Tried to evaluate commands using invalid block type '%ls'" )
+
+/**
+   Error for wrong token type
+*/
+#define UNEXPECTED_TOKEN_ERR_MSG _( L"Unexpected token of type '%ls'")
+
+/**
+   While block description
+*/
+#define WHILE_BLOCK N_( L"'while' block" )
+
+/**
+   For block description
+*/
+#define FOR_BLOCK N_( L"'for' block" )
+
+/**
+   Breakpoint block
+*/
+#define BREAKPOINT_BLOCK N_( L"Block created by breakpoint" )
+
+/**
+   If block description
+*/
+#define IF_BLOCK N_( L"'if' conditional block" )
+
+/**
+   Function definition block description
+*/
+#define FUNCTION_DEF_BLOCK N_( L"function definition block" )
+
+/**
+   Function invocation block description
+*/
+#define FUNCTION_CALL_BLOCK N_( L"function invocation block" )
+
+/**
+   Function invocation block description
+*/
+#define FUNCTION_CALL_NO_SHADOW_BLOCK N_( L"function invocation block with no variable shadowing" )
+
+/**
+   Switch block description
+*/
+#define SWITCH_BLOCK N_( L"'switch' block" )
+
+/**
+   Fake block description
+*/
+#define FAKE_BLOCK N_( L"unexecutable block" )
+
+/**
+   Top block description
+*/
+#define TOP_BLOCK N_( L"global root block" )
+
+/**
+   Command substitution block description
+*/
+#define SUBST_BLOCK N_( L"command substitution block" )
+
+/**
+   Begin block description
+*/
+#define BEGIN_BLOCK N_( L"'begin' unconditional block" )
+
+/**
+   Source block description
+*/
+#define SOURCE_BLOCK N_( L"Block created by the . builtin" )
+
+/**
+   Source block description
+*/
+#define EVENT_BLOCK N_( L"event handler block" )
+
+/**
+   Unknown block description
+*/
+#define UNKNOWN_BLOCK N_( L"unknown/invalid block" )
+
+
+/**
+   Datastructure to describe a block type, like while blocks, command substitution blocks, etc.
+*/
+struct block_lookup_entry
+{
+
+    /**
+       The block type id. The legal values are defined in parser.h.
+    */
+    block_type_t type;
+
+    /**
+       The name of the builtin that creates this type of block, if any.
+    */
+    const wchar_t *name;
+
+    /**
+       A description of this block type
+    */
+    const wchar_t *desc;
+}
+;
+
+/**
+   List of all legal block types
+*/
+static const struct block_lookup_entry block_lookup[]=
+{
+    { WHILE, L"while", WHILE_BLOCK },
+    { FOR, L"for", FOR_BLOCK },
+    { IF, L"if", IF_BLOCK },
+    { FUNCTION_DEF, L"function", FUNCTION_DEF_BLOCK },
+    { FUNCTION_CALL, 0, FUNCTION_CALL_BLOCK },
+    { FUNCTION_CALL_NO_SHADOW, 0, FUNCTION_CALL_NO_SHADOW_BLOCK },
+    { SWITCH, L"switch", SWITCH_BLOCK },
+    { FAKE, 0, FAKE_BLOCK },
+    { TOP, 0, TOP_BLOCK },
+    { SUBST, 0, SUBST_BLOCK },
+    { BEGIN, L"begin", BEGIN_BLOCK },
+    { SOURCE, L".", SOURCE_BLOCK },
+    { EVENT, 0, EVENT_BLOCK },
+    { BREAKPOINT, L"breakpoint", BREAKPOINT_BLOCK },
+    { (block_type_t)0, 0, 0 }
+};
+
+// Given a file path, return something nicer. Currently we just "unexpand" tildes.
+static wcstring user_presentable_path(const wcstring &path)
+{
+    return replace_home_directory_with_tilde(path);
+}
+
+
+parser_t::parser_t(enum parser_type_t type, bool errors) :
+    parser_type(type),
+    show_errors(errors),
+    cancellation_requested(false),
+    is_within_fish_initialization(false)
+{
+}
+
+/* A pointer to the principal parser (which is a static local) */
+static parser_t *s_principal_parser = NULL;
+
+parser_t &parser_t::principal_parser(void)
+{
+    ASSERT_IS_NOT_FORKED_CHILD();
+    ASSERT_IS_MAIN_THREAD();
+    static parser_t parser(PARSER_TYPE_GENERAL, true);
+    if (! s_principal_parser)
+    {
+        s_principal_parser = &parser;
+    }
+    return parser;
+}
+
+void parser_t::set_is_within_fish_initialization(bool flag)
+{
+    is_within_fish_initialization = flag;
+}
+
+void parser_t::skip_all_blocks(void)
+{
+    /* Tell all blocks to skip */
+    if (s_principal_parser)
+    {
+        s_principal_parser->cancellation_requested = true;
+
+        //write(2, "Cancelling blocks\n", strlen("Cancelling blocks\n"));
+        for (size_t i=0; i < s_principal_parser->block_count(); i++)
+        {
+            s_principal_parser->block_at_index(i)->skip = true;
+        }
+    }
+}
+
+void parser_t::push_block(block_t *new_current)
+{
+    const enum block_type_t type = new_current->type();
+    new_current->src_lineno = parser_t::get_lineno();
+
+    const wchar_t *filename = parser_t::current_filename();
+    if (filename != NULL)
+    {
+        new_current->src_filename = intern(filename);
+    }
+
+    const block_t *old_current = this->current_block();
+    if (old_current && old_current->skip)
+    {
+        new_current->skip = true;
+    }
+
+    /*
+      New blocks should be skipped if the outer block is skipped,
+      except TOP ans SUBST block, which open up new environments. Fake
+      blocks should always be skipped. Rather complicated... :-(
+    */
+    new_current->skip = old_current ? old_current->skip : 0;
+
+    /*
+      Type TOP and SUBST are never skipped
+    */
+    if (type == TOP || type == SUBST)
+    {
+        new_current->skip = 0;
+    }
+
+    /*
+      Fake blocks and function definition blocks are never executed
+    */
+    if (type == FAKE || type == FUNCTION_DEF)
+    {
+        new_current->skip = 1;
+    }
+
+    new_current->job = 0;
+    new_current->loop_status=LOOP_NORMAL;
+
+    this->block_stack.push_back(new_current);
+
+    // Types TOP and SUBST are not considered blocks for the purposes of `status -b`
+    if (type != TOP && type != SUBST)
+    {
+        is_block = 1;
+    }
+
+    if ((new_current->type() != FUNCTION_DEF) &&
+            (new_current->type() != FAKE) &&
+            (new_current->type() != TOP))
+    {
+        env_push(type == FUNCTION_CALL);
+        new_current->wants_pop_env = true;
+    }
+}
+
+void parser_t::pop_block()
+{
+    if (block_stack.empty())
+    {
+        debug(1,
+              L"function %s called on empty block stack.",
+              __func__);
+        bugreport();
+        return;
+    }
+
+    block_t *old = block_stack.back();
+    block_stack.pop_back();
+
+    if (old->wants_pop_env)
+        env_pop();
+
+    delete old;
+
+    // Figure out if `status -b` should consider us to be in a block now
+    int new_is_block=0;
+    for (std::vector<block_t*>::const_iterator it = block_stack.begin(), end = block_stack.end(); it != end; ++it)
+    {
+        const enum block_type_t type = (*it)->type();
+        if (type != TOP && type != SUBST)
+        {
+            new_is_block = 1;
+            break;
+        }
+    }
+    is_block = new_is_block;
+}
+
+void parser_t::pop_block(const block_t *expected)
+{
+    assert(expected == this->current_block());
+    this->pop_block();
+}
+
+const wchar_t *parser_t::get_block_desc(int block) const
+{
+    for (size_t i=0; block_lookup[i].desc; i++)
+    {
+        if (block_lookup[i].type == block)
+        {
+            return _(block_lookup[i].desc);
+        }
+    }
+    return _(UNKNOWN_BLOCK);
+}
+
+wcstring parser_t::block_stack_description() const
+{
+    wcstring result;
+    size_t idx = this->block_count();
+    size_t spaces = 0;
+    while (idx--)
+    {
+        if (spaces > 0)
+        {
+            result.push_back(L'\n');
+        }
+        for (size_t j=0; j < spaces; j++)
+        {
+            result.push_back(L' ');
+        }
+        result.append(this->block_at_index(idx)->description());
+        spaces++;
+    }
+    return result;
+}
+
+const block_t *parser_t::block_at_index(size_t idx) const
+{
+    /* 0 corresponds to the last element in our vector */
+    size_t count = block_stack.size();
+    return idx < count ? block_stack.at(count - idx - 1) : NULL;
+}
+
+block_t *parser_t::block_at_index(size_t idx)
+{
+    size_t count = block_stack.size();
+    return idx < count ? block_stack.at(count - idx - 1) : NULL;
+}
+
+const block_t *parser_t::current_block() const
+{
+    return block_stack.empty() ? NULL : block_stack.back();
+}
+
+block_t *parser_t::current_block()
+{
+    return block_stack.empty() ? NULL : block_stack.back();
+}
+
+void parser_t::forbid_function(const wcstring &function)
+{
+    forbidden_function.push_back(function);
+}
+
+void parser_t::allow_function()
+{
+    forbidden_function.pop_back();
+}
+
+/**
+   Print profiling information to the specified stream
+*/
+static void print_profile(const std::vector<profile_item_t*> &items,
+                          FILE *out)
+{
+    for (size_t pos = 0; pos < items.size(); pos++)
+    {
+        const profile_item_t *me, *prev;
+        size_t i;
+        int my_time;
+
+        me = items.at(pos);
+        if (!me->skipped)
+        {
+            my_time=me->parse+me->exec;
+
+            for (i=pos+1; i<items.size(); i++)
+            {
+                prev = items.at(i);
+                if (prev->skipped)
+                {
+                    continue;
+                }
+
+                if (prev->level <= me->level)
+                {
+                    break;
+                }
+
+                if (prev->level > me->level+1)
+                {
+                    continue;
+                }
+
+                my_time -= prev->parse;
+                my_time -= prev->exec;
+            }
+
+            if (me->cmd.size() > 0)
+            {
+                if (fwprintf(out, L"%d\t%d\t", my_time, me->parse+me->exec) < 0)
+                {
+                    wperror(L"fwprintf");
+                    return;
+                }
+
+                for (i=0; i<me->level; i++)
+                {
+                    if (fwprintf(out, L"-") < 0)
+                    {
+                        wperror(L"fwprintf");
+                        return;
+                    }
+
+                }
+                if (fwprintf(out, L"> %ls\n", me->cmd.c_str()) < 0)
+                {
+                    wperror(L"fwprintf");
+                    return;
+                }
+
+            }
+        }
+    }
+}
+
+void parser_t::emit_profiling(const char *path) const
+{
+    /* Save profiling information. OK to not use CLO_EXEC here because this is called while fish is dying (and hence will not fork) */
+    FILE *f = fopen(path, "w");
+    if (!f)
+    {
+        debug(1,
+              _(L"Could not write profiling information to file '%s'"),
+              path);
+    }
+    else
+    {
+        if (fwprintf(f,
+                     _(L"Time\tSum\tCommand\n"),
+                     profile_items.size()) < 0)
+        {
+            wperror(L"fwprintf");
+        }
+        else
+        {
+            print_profile(profile_items, f);
+        }
+
+        if (fclose(f))
+        {
+            wperror(L"fclose");
+        }
+    }
+}
+
+void parser_t::expand_argument_list(const wcstring &arg_list_src, std::vector<completion_t> &output_arg_list)
+{
+    expand_flags_t eflags = 0;
+    if (! show_errors)
+        eflags |= EXPAND_NO_DESCRIPTIONS;
+    if (this->parser_type != PARSER_TYPE_GENERAL)
+        eflags |= EXPAND_SKIP_CMDSUBST;
+
+    /* Suppress calling proc_push_interactive off of the main thread. */
+    if (this->parser_type == PARSER_TYPE_GENERAL)
+    {
+        proc_push_interactive(0);
+    }
+
+    /* Parse the string as an argument list */
+    parse_node_tree_t tree;
+    if (! parse_tree_from_string(arg_list_src, parse_flag_none, &tree, NULL /* errors */, symbol_freestanding_argument_list))
+    {
+        /* Failed to parse. Here we expect to have reported any errors in test_args */
+        return;
+    }
+
+    /* Get the root argument list */
+    assert(! tree.empty());
+    const parse_node_t *arg_list = &tree.at(0);
+    assert(arg_list->type == symbol_freestanding_argument_list);
+
+    /* Extract arguments from it */
+    while (arg_list != NULL)
+    {
+        const parse_node_t *arg_node = tree.next_node_in_node_list(*arg_list, symbol_argument, &arg_list);
+        if (arg_node != NULL)
+        {
+            const wcstring arg_src = arg_node->get_source(arg_list_src);
+            if (expand_string(arg_src, output_arg_list, eflags, NULL) == EXPAND_ERROR)
+            {
+                /* Failed to expand a string */
+                break;
+            }
+        }
+    }
+
+    if (this->parser_type == PARSER_TYPE_GENERAL)
+    {
+        proc_pop_interactive();
+    }
+}
+
+void parser_t::stack_trace(size_t block_idx, wcstring &buff) const
+{
+    /*
+      Check if we should end the recursion
+    */
+    if (block_idx >= this->block_count())
+        return;
+
+    const block_t *b = this->block_at_index(block_idx);
+
+    if (b->type()==EVENT)
+    {
+        /*
+          This is an event handler
+        */
+        const event_block_t *eb = static_cast<const event_block_t *>(b);
+        wcstring description = event_get_desc(eb->event);
+        append_format(buff, _(L"in event handler: %ls\n"), description.c_str());
+        buff.append(L"\n");
+
+        /*
+          Stop recursing at event handler. No reason to believe that
+          any other code is relevant.
+
+          It might make sense in the future to continue printing the
+          stack trace of the code that invoked the event, if this is a
+          programmatic event, but we can't currently detect that.
+        */
+        return;
+    }
+
+    if (b->type() == FUNCTION_CALL || b->type() == FUNCTION_CALL_NO_SHADOW || b->type()==SOURCE || b->type()==SUBST)
+    {
+        /*
+          These types of blocks should be printed
+        */
+
+        int i;
+
+        switch (b->type())
+        {
+            case SOURCE:
+            {
+                const source_block_t *sb = static_cast<const source_block_t*>(b);
+                const wchar_t *source_dest = sb->source_file;
+                append_format(buff, _(L"from sourcing file %ls\n"), user_presentable_path(source_dest).c_str());
+                break;
+            }
+            case FUNCTION_CALL:
+            case FUNCTION_CALL_NO_SHADOW:
+            {
+                const function_block_t *fb = static_cast<const function_block_t*>(b);
+                append_format(buff, _(L"in function '%ls'\n"), fb->name.c_str());
+                break;
+            }
+            case SUBST:
+            {
+                append_format(buff, _(L"in command substitution\n"));
+                break;
+            }
+
+            default: /* Can't get here */
+                break;
+        }
+
+        const wchar_t *file = b->src_filename;
+
+        if (file)
+        {
+            append_format(buff,
+                          _(L"\tcalled on line %d of file %ls\n"),
+                          b->src_lineno,
+                          user_presentable_path(file).c_str());
+        }
+        else if (is_within_fish_initialization)
+        {
+            append_format(buff, _(L"\tcalled during startup\n"));
+        }
+        else
+        {
+            append_format(buff, _(L"\tcalled on standard input\n"));
+        }
+
+        if (b->type() == FUNCTION_CALL)
+        {
+            const function_block_t *fb = static_cast<const function_block_t *>(b);
+            const process_t * const process = fb->process;
+            if (process->argv(1))
+            {
+                wcstring tmp;
+
+                for (i=1; process->argv(i); i++)
+                {
+                    if (i > 1)
+                        tmp.push_back(L' ');
+                    tmp.append(process->argv(i));
+                }
+                append_format(buff, _(L"\twith parameter list '%ls'\n"), tmp.c_str());
+            }
+        }
+
+        append_format(buff, L"\n");
+    }
+
+    /*
+      Recursively print the next block
+    */
+    parser_t::stack_trace(block_idx + 1, buff);
+}
+
+/**
+   Returns the name of the currently evaluated function if we are
+   currently evaluating a function, null otherwise. This is tested by
+   moving down the block-scope-stack, checking every block if it is of
+   type FUNCTION_CALL.
+*/
+const wchar_t *parser_t::is_function() const
+{
+    // PCA: Have to make this a string somehow
+    ASSERT_IS_MAIN_THREAD();
+
+    const wchar_t *result = NULL;
+    for (size_t block_idx = 0; block_idx < this->block_count(); block_idx++)
+    {
+        const block_t *b = this->block_at_index(block_idx);
+        if (b->type() == FUNCTION_CALL || b->type() == FUNCTION_CALL_NO_SHADOW)
+        {
+            const function_block_t *fb = static_cast<const function_block_t *>(b);
+            result = fb->name.c_str();
+            break;
+        }
+        else if (b->type() == SOURCE)
+        {
+            /* If a function sources a file, obviously that function's offset doesn't contribute */
+            break;
+        }
+    }
+    return result;
+}
+
+
+int parser_t::get_lineno() const
+{
+    int lineno = -1;
+    if (! execution_contexts.empty())
+    {
+        lineno = execution_contexts.back()->get_current_line_number();
+
+        /* If we are executing a function, we have to add in its offset */
+        const wchar_t *function_name = is_function();
+        if (function_name != NULL)
+        {
+            lineno += function_get_definition_offset(function_name);
+        }
+
+    }
+    return lineno;
+}
+
+const wchar_t *parser_t::current_filename() const
+{
+    ASSERT_IS_MAIN_THREAD();
+
+    for (size_t i=0; i < this->block_count(); i++)
+    {
+        const block_t *b = this->block_at_index(i);
+        if (b->type() == FUNCTION_CALL || b->type() == FUNCTION_CALL_NO_SHADOW)
+        {
+            const function_block_t *fb = static_cast<const function_block_t *>(b);
+            return function_get_definition_file(fb->name);
+        }
+        else if (b->type() == SOURCE)
+        {
+            const source_block_t *sb = static_cast<const source_block_t *>(b);
+            return sb->source_file;
+        }
+    }
+
+    /* We query a global array for the current file name, but only do that if we are the principal parser */
+    if (this == &principal_parser())
+    {
+        return reader_current_filename();
+    }
+    return NULL;
+}
+
+wcstring parser_t::current_line()
+{
+    if (execution_contexts.empty())
+    {
+        return wcstring();
+    }
+    const parse_execution_context_t *context = execution_contexts.back();
+    assert(context != NULL);
+
+    int source_offset = context->get_current_source_offset();
+    if (source_offset < 0)
+    {
+        return wcstring();
+    }
+
+    const int lineno = this->get_lineno();
+    const wchar_t *file = this->current_filename();
+
+    wcstring prefix;
+
+    /* If we are not going to print a stack trace, at least print the line number and filename */
+    if (!get_is_interactive() || is_function())
+    {
+        if (file)
+        {
+            append_format(prefix, _(L"%ls (line %d): "), user_presentable_path(file).c_str(), lineno);
+        }
+        else if (is_within_fish_initialization)
+        {
+            append_format(prefix, L"%ls: ", _(L"Startup"), lineno);
+        }
+        else
+        {
+            append_format(prefix, L"%ls: ", _(L"Standard input"), lineno);
+        }
+    }
+
+    bool is_interactive = get_is_interactive();
+    bool skip_caret = is_interactive && ! is_function();
+
+    /* Use an error with empty text */
+    assert(source_offset >= 0);
+    parse_error_t empty_error = {};
+    empty_error.source_start = source_offset;
+
+    wcstring line_info = empty_error.describe_with_prefix(context->get_source(), prefix, is_interactive, skip_caret);
+    if (! line_info.empty())
+    {
+        line_info.push_back(L'\n');
+    }
+
+    parser_t::stack_trace(0, line_info);
+    return line_info;
+}
+
+void parser_t::job_add(job_t *job)
+{
+    assert(job != NULL);
+    assert(job->first_process != NULL);
+    this->my_job_list.push_front(job);
+}
+
+bool parser_t::job_remove(job_t *j)
+{
+    job_list_t::iterator iter = std::find(my_job_list.begin(), my_job_list.end(), j);
+    if (iter != my_job_list.end())
+    {
+        my_job_list.erase(iter);
+        return true;
+    }
+    else
+    {
+        debug(1, _(L"Job inconsistency"));
+        sanity_lose();
+        return false;
+    }
+}
+
+void parser_t::job_promote(job_t *job)
+{
+    job_list_t::iterator loc = std::find(my_job_list.begin(), my_job_list.end(), job);
+    assert(loc != my_job_list.end());
+
+    /* Move the job to the beginning */
+    my_job_list.splice(my_job_list.begin(), my_job_list, loc);
+}
+
+job_t *parser_t::job_get(job_id_t id)
+{
+    job_iterator_t jobs(my_job_list);
+    job_t *job;
+    while ((job = jobs.next()))
+    {
+        if (id <= 0 || job->job_id == id)
+            return job;
+    }
+    return NULL;
+}
+
+job_t *parser_t::job_get_from_pid(int pid)
+{
+    job_iterator_t jobs;
+    job_t *job;
+    while ((job = jobs.next()))
+    {
+        if (job->pgid == pid)
+            return job;
+    }
+    return 0;
+}
+
+profile_item_t *parser_t::create_profile_item()
+{
+    profile_item_t *result = NULL;
+    if (g_profiling_active)
+    {
+        result = new profile_item_t();
+        profile_items.push_back(result);
+    }
+    return result;
+}
+
+
+int parser_t::eval(const wcstring &cmd, const io_chain_t &io, enum block_type_t block_type)
+{
+    CHECK_BLOCK(1);
+
+    if (block_type != TOP && block_type != SUBST)
+    {
+        debug(1, INVALID_SCOPE_ERR_MSG, parser_t::get_block_desc(block_type));
+        bugreport();
+        return 1;
+    }
+
+    /* Parse the source into a tree, if we can */
+    parse_node_tree_t tree;
+    parse_error_list_t error_list;
+    if (! parse_tree_from_string(cmd, parse_flag_none, &tree, this->show_errors ? &error_list : NULL))
+    {
+        if (this->show_errors)
+        {
+            /* Get a backtrace */
+            wcstring backtrace_and_desc;
+            this->get_backtrace(cmd, error_list, &backtrace_and_desc);
+
+            /* Print it */
+            fprintf(stderr, "%ls", backtrace_and_desc.c_str());
+        }
+
+        return 1;
+    }
+
+    //print_stderr(block_stack_description());
+
+
+    /* Determine the initial eval level. If this is the first context, it's -1; otherwise it's the eval level of the top context. This is sort of wonky because we're stitching together a global notion of eval level from these separate objects. A better approach would be some profile object that all contexts share, and that tracks the eval levels on its own. */
+    int exec_eval_level = (execution_contexts.empty() ? -1 : execution_contexts.back()->current_eval_level());
+
+    /* Append to the execution context stack */
+    parse_execution_context_t *ctx = new parse_execution_context_t(tree, cmd, this, exec_eval_level);
+    execution_contexts.push_back(ctx);
+
+    /* Execute the first node */
+    if (! tree.empty())
+    {
+        this->eval_block_node(0, io, block_type);
+    }
+
+    /* Clean up the execution context stack */
+    assert(! execution_contexts.empty() && execution_contexts.back() == ctx);
+    execution_contexts.pop_back();
+    delete ctx;
+
+    return 0;
+}
+
+int parser_t::eval_block_node(node_offset_t node_idx, const io_chain_t &io, enum block_type_t block_type)
+{
+    /* Paranoia. It's a little frightening that we're given only a node_idx and we interpret this in the topmost execution context's tree. What happens if two trees were to be interleaved? Fortunately that cannot happen (yet); in the future we probably want some sort of reference counted trees.
+    */
+    parse_execution_context_t *ctx = execution_contexts.back();
+    assert(ctx != NULL);
+
+    CHECK_BLOCK(1);
+
+    /* Handle cancellation requests. If our block stack is currently empty, then we already did successfully cancel (or there was nothing to cancel); clear the flag. If our block stack is not empty, we are still in the process of cancelling; refuse to evaluate anything */
+    if (this->cancellation_requested)
+    {
+        if (! block_stack.empty())
+        {
+            return 1;
+        }
+        else
+        {
+            this->cancellation_requested = false;
+        }
+    }
+
+    /* Only certain blocks are allowed */
+    if ((block_type != TOP) &&
+            (block_type != SUBST))
+    {
+        debug(1,
+              INVALID_SCOPE_ERR_MSG,
+              parser_t::get_block_desc(block_type));
+        bugreport();
+        return 1;
+    }
+
+    /* Not sure why we reap jobs here */
+    job_reap(0);
+
+    /* Start it up */
+    const block_t * const start_current_block = current_block();
+    block_t *scope_block = new scope_block_t(block_type);
+    this->push_block(scope_block);
+    int result = ctx->eval_node_at_offset(node_idx, scope_block, io);
+
+    /* Clean up the block stack */
+    this->pop_block();
+    while (start_current_block != current_block())
+    {
+        if (current_block() == NULL)
+        {
+            debug(0,
+                  _(L"End of block mismatch. Program terminating."));
+            bugreport();
+            FATAL_EXIT();
+            break;
+        }
+        this->pop_block();
+    }
+
+    /* Reap again */
+    job_reap(0);
+
+    return result;
+}
+
+bool parser_t::detect_errors_in_argument_list(const wcstring &arg_list_src, wcstring *out, const wchar_t *prefix)
+{
+    bool errored = false;
+    parse_error_list_t errors;
+
+    /* Use empty string for the prefix if it's NULL */
+    if (prefix == NULL)
+    {
+        prefix = L"";
+    }
+
+    /* Parse the string as an argument list */
+    parse_node_tree_t tree;
+    if (! parse_tree_from_string(arg_list_src, parse_flag_none, &tree, &errors, symbol_freestanding_argument_list))
+    {
+        /* Failed to parse. */
+        errored = true;
+    }
+
+    if (! errored)
+    {
+        /* Get the root argument list */
+        assert(! tree.empty());
+        const parse_node_t *arg_list = &tree.at(0);
+        assert(arg_list->type == symbol_freestanding_argument_list);
+
+        /* Extract arguments from it */
+        while (arg_list != NULL && ! errored)
+        {
+            const parse_node_t *arg_node = tree.next_node_in_node_list(*arg_list, symbol_argument, &arg_list);
+            if (arg_node != NULL)
+            {
+                const wcstring arg_src = arg_node->get_source(arg_list_src);
+                if (parse_util_detect_errors_in_argument(*arg_node, arg_src, &errors))
+                {
+                    errored = true;
+                }
+            }
+        }
+    }
+
+    if (! errors.empty() && out != NULL)
+    {
+        out->assign(errors.at(0).describe_with_prefix(arg_list_src, prefix, false /* not interactive */, false /* don't skip caret */));
+    }
+    return errored;
+}
+
+void parser_t::get_backtrace(const wcstring &src, const parse_error_list_t &errors, wcstring *output) const
+{
+    assert(output != NULL);
+    if (! errors.empty())
+    {
+        const parse_error_t &err = errors.at(0);
+
+        const bool is_interactive = get_is_interactive();
+
+        // Determine if we want to try to print a caret to point at the source error
+        // The err.source_start <= src.size() check is due to the nasty way that slices work,
+        // which is by rewriting the source (!)
+        size_t which_line = 0;
+        bool skip_caret = true;
+        if (err.source_start != SOURCE_LOCATION_UNKNOWN && err.source_start <= src.size())
+        {
+            // Determine which line we're on
+            which_line = 1 + std::count(src.begin(), src.begin() + err.source_start, L'\n');
+
+            // Don't include the caret if we're interactive, this is the first line of text, and our source is at its beginning, because then it's obvious
+            skip_caret = (is_interactive && which_line == 1 && err.source_start == 0);
+        }
+
+        wcstring prefix;
+        const wchar_t *filename = this->current_filename();
+        if (filename)
+        {
+            if (which_line > 0)
+            {
+                prefix = format_string(_(L"%ls (line %lu): "), user_presentable_path(filename).c_str(), which_line);
+            }
+            else
+            {
+                prefix = format_string(_(L"%ls: "), user_presentable_path(filename).c_str());
+            }
+        }
+        else
+        {
+            prefix = L"fish: ";
+        }
+
+        const wcstring description = err.describe_with_prefix(src, prefix, is_interactive, skip_caret);
+        if (! description.empty())
+        {
+            output->append(description);
+            output->push_back(L'\n');
+        }
+        this->stack_trace(0, *output);
+    }
+}
+
+block_t::block_t(block_type_t t) :
+    block_type(t),
+    skip(),
+    tok_pos(),
+    node_offset(NODE_OFFSET_INVALID),
+    loop_status(LOOP_NORMAL),
+    job(),
+    src_filename(),
+    src_lineno(),
+    wants_pop_env(false),
+    event_blocks()
+{
+}
+
+block_t::~block_t()
+{
+}
+
+wcstring block_t::description() const
+{
+    wcstring result;
+    switch (this->type())
+    {
+        case WHILE:
+            result.append(L"while");
+            break;
+
+        case FOR:
+            result.append(L"for");
+            break;
+
+        case IF:
+            result.append(L"if");
+            break;
+
+        case FUNCTION_DEF:
+            result.append(L"function_def");
+            break;
+
+        case FUNCTION_CALL:
+            result.append(L"function_call");
+            break;
+
+        case FUNCTION_CALL_NO_SHADOW:
+            result.append(L"function_call_no_shadow");
+            break;
+
+        case SWITCH:
+            result.append(L"switch");
+            break;
+
+        case FAKE:
+            result.append(L"fake");
+            break;
+
+        case SUBST:
+            result.append(L"substitution");
+            break;
+
+        case TOP:
+            result.append(L"top");
+            break;
+
+        case BEGIN:
+            result.append(L"begin");
+            break;
+
+        case SOURCE:
+            result.append(L"source");
+            break;
+
+        case EVENT:
+            result.append(L"event");
+            break;
+
+        case BREAKPOINT:
+            result.append(L"breakpoint");
+            break;
+
+        default:
+            append_format(result, L"unknown type %ld", (long)this->type());
+            break;
+    }
+
+    if (this->src_lineno >= 0)
+    {
+        append_format(result, L" (line %d)", this->src_lineno);
+    }
+    if (this->src_filename != NULL)
+    {
+        append_format(result, L" (file %ls)", this->src_filename);
+    }
+    return result;
+}
+
+/* Various block constructors */
+
+if_block_t::if_block_t() : block_t(IF)
+{
+}
+
+event_block_t::event_block_t(const event_t &evt) :
+    block_t(EVENT),
+    event(evt)
+{
+}
+
+function_block_t::function_block_t(const process_t *p, const wcstring &n, bool shadows) :
+    block_t(shadows ? FUNCTION_CALL : FUNCTION_CALL_NO_SHADOW),
+    process(p),
+    name(n)
+{
+}
+
+source_block_t::source_block_t(const wchar_t *src) :
+    block_t(SOURCE),
+    source_file(src)
+{
+}
+
+for_block_t::for_block_t() : block_t(FOR)
+{
+}
+
+while_block_t::while_block_t() : block_t(WHILE)
+{
+}
+
+switch_block_t::switch_block_t() : block_t(SWITCH)
+{
+}
+
+fake_block_t::fake_block_t() : block_t(FAKE)
+{
+}
+
+scope_block_t::scope_block_t(block_type_t type) : block_t(type)
+{
+    assert(type == BEGIN || type == TOP || type == SUBST);
+}
+
+breakpoint_block_t::breakpoint_block_t() : block_t(BREAKPOINT)
+{
+}
diff --git a/src/parser.h b/src/parser.h
new file mode 100644
index 00000000..ba0edd6a
--- /dev/null
+++ b/src/parser.h
@@ -0,0 +1,433 @@
+/** \file parser.h
+  The fish parser.
+*/
+
+#ifndef FISH_PARSER_H
+#define FISH_PARSER_H
+
+#include <stddef.h>                     // for size_t
+#include <list>                         // for _List_const_iterator, list, etc
+
+#include "common.h"
+#include "proc.h"
+#include "event.h"
+#include "parse_tree.h"
+#include "io.h"
+#include "parse_constants.h"
+
+#include <vector>
+
+/**
+   event_blockage_t represents a block on events of the specified type
+*/
+struct event_blockage_t
+{
+    /**
+       The types of events to block. This is interpreted as a bitset
+       whete the value is 1 for every bit corresponding to a blocked
+       event type. For example, if EVENT_VARIABLE type events should
+       be blocked, (type & 1<<EVENT_BLOCKED) should be set.
+
+       Note that EVENT_ANY can be used to specify any event.
+    */
+    unsigned int typemask;
+};
+
+typedef std::list<event_blockage_t> event_blockage_list_t;
+
+inline bool event_block_list_blocks_type(const event_blockage_list_t &ebls, int type)
+{
+    for (event_blockage_list_t::const_iterator iter = ebls.begin(); iter != ebls.end(); ++iter)
+    {
+        if (iter->typemask & (1<<EVENT_ANY))
+            return true;
+        if (iter->typemask & (1<<type))
+            return true;
+    }
+    return false;
+}
+
+
+/**
+  Types of blocks
+*/
+enum block_type_t
+{
+    WHILE, /**< While loop block */
+    FOR,  /**< For loop block */
+    IF, /**< If block */
+    FUNCTION_DEF, /**< Function definition block */
+    FUNCTION_CALL, /**< Function invocation block */
+    FUNCTION_CALL_NO_SHADOW, /**< Function invocation block with no variable shadowing */
+    SWITCH, /**< Switch block */
+    FAKE, /**< Fake block */
+    SUBST, /**< Command substitution scope */
+    TOP, /**< Outermost block */
+    BEGIN, /**< Unconditional block */
+    SOURCE, /**< Block created by the . (source) builtin */
+    EVENT, /**< Block created on event notifier invocation */
+    BREAKPOINT, /**< Breakpoint block */
+};
+
+/** Possible states for a loop */
+enum loop_status_t
+{
+    LOOP_NORMAL, /**< Current loop block executed as normal */
+    LOOP_BREAK, /**< Current loop block should be removed */
+    LOOP_CONTINUE, /**< Current loop block should be skipped */
+};
+
+/**
+   block_t represents a block of commands.
+*/
+struct block_t
+{
+protected:
+    /** Protected constructor. Use one of the subclasses below. */
+    block_t(block_type_t t);
+
+private:
+    const block_type_t block_type; /**< Type of block. */
+
+public:
+    block_type_t type() const
+    {
+        return this->block_type;
+    }
+
+    /** Description of the block, for debugging */
+    wcstring description() const;
+
+    bool skip; /**< Whether execution of the commands in this block should be skipped */
+    int tok_pos; /**< The start index of the block */
+
+    node_offset_t node_offset; /* Offset of the node */
+
+    /** Status for the current loop block. Can be any of the values from the loop_status enum. */
+    enum loop_status_t loop_status;
+
+    /** The job that is currently evaluated in the specified block. */
+    job_t *job;
+
+    /** Name of file that created this block. This string is intern'd. */
+    const wchar_t *src_filename;
+
+    /** Line number where this block was created */
+    int src_lineno;
+
+    /** Whether we should pop the environment variable stack when we're popped off of the block stack */
+    bool wants_pop_env;
+
+    /** List of event blocks. */
+    event_blockage_list_t event_blocks;
+
+    /** Destructor */
+    virtual ~block_t();
+};
+
+struct if_block_t : public block_t
+{
+    if_block_t();
+};
+
+struct event_block_t : public block_t
+{
+    event_t const event;
+    event_block_t(const event_t &evt);
+};
+
+struct function_block_t : public block_t
+{
+    const process_t *process;
+    wcstring name;
+    function_block_t(const process_t *p, const wcstring &n, bool shadows);
+};
+
+struct source_block_t : public block_t
+{
+    const wchar_t * const source_file;
+    source_block_t(const wchar_t *src);
+};
+
+struct for_block_t : public block_t
+{
+    for_block_t();
+};
+
+struct while_block_t : public block_t
+{
+    while_block_t();
+};
+
+struct switch_block_t : public block_t
+{
+    switch_block_t();
+};
+
+struct fake_block_t : public block_t
+{
+    fake_block_t();
+};
+
+struct scope_block_t : public block_t
+{
+    scope_block_t(block_type_t type); //must be BEGIN, TOP or SUBST
+};
+
+struct breakpoint_block_t : public block_t
+{
+    breakpoint_block_t();
+};
+
+/**
+   Errors that can be generated by the parser
+*/
+enum parser_error
+{
+    /**
+       No error
+    */
+    NO_ERR=0,
+    /**
+       An error in the syntax
+    */
+    SYNTAX_ERROR,
+    /**
+       Error occured while evaluating commands
+    */
+    EVAL_ERROR,
+    /**
+       Error while evaluating cmdsubst
+    */
+    CMDSUBST_ERROR,
+};
+
+enum parser_type_t
+{
+    PARSER_TYPE_NONE,
+    PARSER_TYPE_GENERAL,
+    PARSER_TYPE_FUNCTIONS_ONLY,
+    PARSER_TYPE_COMPLETIONS_ONLY,
+    PARSER_TYPE_ERRORS_ONLY
+};
+
+struct profile_item_t
+{
+    /** Time spent executing the specified command, including parse time for nested blocks. */
+    int exec;
+
+    /** Time spent parsing the specified command, including execution time for command substitutions. */
+    int parse;
+
+    /** The block level of the specified command. nested blocks and command substitutions both increase the block level. */
+    size_t level;
+
+    /** If the execution of this command was skipped. */
+    bool skipped;
+
+    /** The command string. */
+    wcstring cmd;
+};
+
+class parse_execution_context_t;
+class completion_t;
+
+class parser_t
+{
+    friend class parse_execution_context_t;
+private:
+    enum parser_type_t parser_type;
+
+    /** Whether or not we output errors */
+    const bool show_errors;
+
+    /** Indication that we should skip all blocks */
+    bool cancellation_requested;
+
+    /** Indicates that we are within the process of initializing fish */
+    bool is_within_fish_initialization;
+
+    /** Stack of execution contexts. We own these pointers and must delete them */
+    std::vector<parse_execution_context_t *> execution_contexts;
+
+    /** List of called functions, used to help prevent infinite recursion */
+    wcstring_list_t forbidden_function;
+
+    /** The jobs associated with this parser */
+    job_list_t my_job_list;
+
+    /** The list of blocks, allocated with new. It's our responsibility to delete these */
+    std::vector<block_t *> block_stack;
+
+    /** Gets a description of the block stack, for debugging */
+    wcstring block_stack_description() const;
+
+    /** List of profile items, allocated with new */
+    std::vector<profile_item_t *> profile_items;
+
+    /* No copying allowed */
+    parser_t(const parser_t&);
+    parser_t& operator=(const parser_t&);
+
+    /** Adds a job to the beginning of the job list. */
+    void job_add(job_t *job);
+
+    /**
+       Returns the name of the currently evaluated function if we are
+       currently evaluating a function, null otherwise. This is tested by
+       moving down the block-scope-stack, checking every block if it is of
+       type FUNCTION_CALL.
+    */
+    const wchar_t *is_function() const;
+
+public:
+
+    /** Get the "principal" parser, whatever that is */
+    static parser_t &principal_parser();
+
+    /** Indicates that execution of all blocks in the principal parser should stop.
+        This is called from signal handlers!
+    */
+    static void skip_all_blocks();
+
+    /** Create a parser of the given type */
+    parser_t(enum parser_type_t type, bool show_errors);
+
+    /** Global event blocks */
+    event_blockage_list_t global_event_blocks;
+
+    /**
+      Evaluate the expressions contained in cmd.
+
+      \param cmd the string to evaluate
+      \param io io redirections to perform on all started jobs
+      \param block_type The type of block to push on the block stack
+
+      \return 0 on success, 1 otherwise
+    */
+    int eval(const wcstring &cmd, const io_chain_t &io, enum block_type_t block_type);
+
+    /** Evaluates a block node at the given node offset in the topmost execution context */
+    int eval_block_node(node_offset_t node_idx, const io_chain_t &io, enum block_type_t block_type);
+
+    /**
+      Evaluate line as a list of parameters, i.e. tokenize it and perform parameter expansion and cmdsubst execution on the tokens.
+      The output is inserted into output.
+      Errors are ignored.
+
+      \param arg_src String to evaluate as an argument list
+      \param output List to insert output into
+    */
+    void expand_argument_list(const wcstring &arg_src, std::vector<completion_t> &output);
+
+    /**
+       Returns a string describing the current parser pisition in the format 'FILENAME (line LINE_NUMBER): LINE'.
+       Example:
+
+       init.fish (line 127): ls|grep pancake
+    */
+    wcstring current_line();
+
+    /** Returns the current line number */
+    int get_lineno() const;
+
+    /** Returns the block at the given index. 0 corresponds to the innermost block. Returns NULL when idx is at or equal to the number of blocks. */
+    const block_t *block_at_index(size_t idx) const;
+    block_t *block_at_index(size_t idx);
+
+    /** Returns the current (innermost) block */
+    const block_t *current_block() const;
+    block_t *current_block();
+
+    /** Count of blocks */
+    size_t block_count() const
+    {
+        return block_stack.size();
+    }
+
+    /** Get the list of jobs */
+    job_list_t &job_list()
+    {
+        return my_job_list;
+    }
+
+    /* Hackish. In order to correctly report the origin of code with no associated file, we need to know whether it's run during initialization or not. */
+    void set_is_within_fish_initialization(bool flag);
+
+    /** Pushes the block. pop_block will call delete on it. */
+    void push_block(block_t *newv);
+
+    /** Remove the outermost block namespace */
+    void pop_block();
+
+    /** Remove the outermost block, asserting it's the given one */
+    void pop_block(const block_t *b);
+
+    /** Return a description of the given blocktype */
+    const wchar_t *get_block_desc(int block) const;
+
+    /** Removes a job */
+    bool job_remove(job_t *job);
+
+    /** Promotes a job to the front of the list */
+    void job_promote(job_t *job);
+
+    /** Return the job with the specified job id. If id is 0 or less, return the last job used. */
+    job_t *job_get(int job_id);
+
+    /** Returns the job with the given pid */
+    job_t *job_get_from_pid(int pid);
+
+    /* Returns a new profile item if profiling is active. The caller should fill it in. The parser_t will clean it up. */
+    profile_item_t *create_profile_item();
+
+    /**
+       Test if the specified string can be parsed, or if more bytes need
+       to be read first. The result will have the PARSER_TEST_ERROR bit
+       set if there is a syntax error in the code, and the
+       PARSER_TEST_INCOMPLETE bit set if the code contains unclosed
+       blocks.
+
+       \param buff the text buffer to test
+       \param block_level if non-null, the block nesting level will be filled out into this array
+       \param out if non-null, any errors in the command will be filled out into this buffer
+       \param prefix the prefix string to prepend to each error message written to the \c out buffer
+    */
+    void get_backtrace(const wcstring &src, const parse_error_list_t &errors, wcstring *output) const;
+
+    /**
+       Detect errors in the specified string when parsed as an argument list. Returns true if an error occurred.
+    */
+    bool detect_errors_in_argument_list(const wcstring &arg_list_src, wcstring *out_err, const wchar_t *prefix);
+
+    /**
+       Tell the parser that the specified function may not be run if not
+       inside of a conditional block. This is to remove some possibilities
+       of infinite recursion.
+    */
+    void forbid_function(const wcstring &function);
+
+    /**
+       Undo last call to parser_forbid_function().
+    */
+    void allow_function();
+
+    /**
+       Output profiling data to the given filename
+    */
+    void emit_profiling(const char *path) const;
+
+    /**
+       Returns the file currently evaluated by the parser. This can be
+       different than reader_current_filename, e.g. if we are evaulating a
+       function defined in a different file than the one curently read.
+    */
+    const wchar_t *current_filename() const;
+
+    /**
+       Write a stack trace starting at the specified block to the specified wcstring
+    */
+    void stack_trace(size_t block_idx, wcstring &buff) const;
+};
+
+#endif
diff --git a/src/parser_keywords.cpp b/src/parser_keywords.cpp
new file mode 100644
index 00000000..adf9c40c
--- /dev/null
+++ b/src/parser_keywords.cpp
@@ -0,0 +1,59 @@
+/** \file parser_keywords.c
+
+Functions having to do with parser keywords, like testing if a function is a block command.
+*/
+
+#include "config.h" // IWYU pragma: keep
+
+#include "fallback.h" // IWYU pragma: keep
+#include "common.h"
+#include "parser_keywords.h"
+
+bool parser_keywords_skip_arguments(const wcstring &cmd)
+{
+    return contains(cmd,
+                    L"else",
+                    L"begin");
+}
+
+
+bool parser_keywords_is_subcommand(const wcstring &cmd)
+{
+
+    return parser_keywords_skip_arguments(cmd) ||
+           contains(cmd,
+                    L"command",
+                    L"builtin",
+                    L"while",
+                    L"exec",
+                    L"if",
+                    L"and",
+                    L"or",
+                    L"not");
+
+}
+
+bool parser_keywords_is_block(const wcstring &word)
+{
+    return contains(word,
+                    L"for",
+                    L"while",
+                    L"if",
+                    L"function",
+                    L"switch",
+                    L"begin");
+}
+
+bool parser_keywords_is_reserved(const wcstring &word)
+{
+    return parser_keywords_is_block(word) ||
+           parser_keywords_is_subcommand(word) ||
+           contains(word,
+                    L"end",
+                    L"case",
+                    L"else",
+                    L"return",
+                    L"continue",
+                    L"break");
+}
+
diff --git a/src/parser_keywords.h b/src/parser_keywords.h
new file mode 100644
index 00000000..94cd0eb7
--- /dev/null
+++ b/src/parser_keywords.h
@@ -0,0 +1,45 @@
+/** \file parser_keywords.h
+
+Functions having to do with parser keywords, like testing if a function is a block command.
+*/
+
+#ifndef FISH_PARSER_KEYWORD_H
+#define FISH_PARSER_KEYWORD_H
+
+#include "common.h"
+
+/**
+   Tests if the specified commands parameters should be interpreted as another command, which will be true if the command is either 'command', 'exec', 'if', 'while', or 'builtin'.  This does not handle "else if" which is more complicated.
+
+   \param cmd The command name to test
+   \return 1 of the command parameter is a command, 0 otherwise
+*/
+
+bool parser_keywords_is_subcommand(const wcstring &cmd);
+
+/**
+   Tests if the specified command is a reserved word, i.e. if it is
+   the name of one of the builtin functions that change the block or
+   command scope, like 'for', 'end' or 'command' or 'exec'. These
+   functions may not be overloaded, so their names are reserved.
+
+   \param word The command name to test
+   \return 1 of the command parameter is a command, 0 otherwise
+*/
+bool parser_keywords_is_reserved(const wcstring &word);
+
+/**
+   Test if the specified string is command that opens a new block
+*/
+
+bool parser_keywords_is_block(const wcstring &word);
+
+/**
+   Check if the specified command is one of the builtins that cannot
+   have arguments, any followin argument is interpreted as a new
+   command
+*/
+bool parser_keywords_skip_arguments(const wcstring &cmd);
+
+
+#endif
diff --git a/src/path.cpp b/src/path.cpp
new file mode 100644
index 00000000..62a31a63
--- /dev/null
+++ b/src/path.cpp
@@ -0,0 +1,404 @@
+#include "config.h" // IWYU pragma: keep
+
+#include <wchar.h>
+#include <sys/stat.h>
+#include <unistd.h>
+#include <errno.h>
+#include <assert.h>
+#include <string>
+#include <vector>
+
+#include "fallback.h" // IWYU pragma: keep
+#include "common.h"
+#include "env.h"
+#include "wutil.h"
+#include "path.h"
+#include "expand.h"
+
+/**
+   Unexpected error in path_get_path()
+*/
+#define MISSING_COMMAND_ERR_MSG _( L"Error while searching for command '%ls'" )
+
+static bool path_get_path_core(const wcstring &cmd, wcstring *out_path, const env_var_t &bin_path_var)
+{
+    int err = ENOENT;
+
+    debug(3, L"path_get_path( '%ls' )", cmd.c_str());
+
+    /* If the command has a slash, it must be a full path */
+    if (cmd.find(L'/') != wcstring::npos)
+    {
+        if (waccess(cmd, X_OK)==0)
+        {
+            struct stat buff;
+            if (wstat(cmd, &buff))
+            {
+                return false;
+            }
+
+            if (S_ISREG(buff.st_mode))
+            {
+                if (out_path)
+                    out_path->assign(cmd);
+                return true;
+            }
+            else
+            {
+                errno = EACCES;
+                return false;
+            }
+        }
+        else
+        {
+            return false;
+        }
+
+    }
+    else
+    {
+        wcstring bin_path;
+        if (! bin_path_var.missing())
+        {
+            bin_path = bin_path_var;
+        }
+        else
+        {
+            if (contains(PREFIX L"/bin", L"/bin", L"/usr/bin"))
+            {
+                bin_path = L"/bin" ARRAY_SEP_STR L"/usr/bin";
+            }
+            else
+            {
+                bin_path = L"/bin" ARRAY_SEP_STR L"/usr/bin" ARRAY_SEP_STR PREFIX L"/bin";
+            }
+        }
+
+        wcstring nxt_path;
+        wcstokenizer tokenizer(bin_path, ARRAY_SEP_STR);
+        while (tokenizer.next(nxt_path))
+        {
+            if (nxt_path.empty())
+                continue;
+            append_path_component(nxt_path, cmd);
+            if (waccess(nxt_path, X_OK)==0)
+            {
+                struct stat buff;
+                if (wstat(nxt_path, &buff)==-1)
+                {
+                    if (errno != EACCES)
+                    {
+                        wperror(L"stat");
+                    }
+                    continue;
+                }
+                if (S_ISREG(buff.st_mode))
+                {
+                    if (out_path)
+                        out_path->swap(nxt_path);
+                    return true;
+                }
+                err = EACCES;
+
+            }
+            else
+            {
+                switch (errno)
+                {
+                    case ENOENT:
+                    case ENAMETOOLONG:
+                    case EACCES:
+                    case ENOTDIR:
+                        break;
+                    default:
+                    {
+                        debug(1,
+                              MISSING_COMMAND_ERR_MSG,
+                              nxt_path.c_str());
+                        wperror(L"access");
+                    }
+                }
+            }
+        }
+    }
+
+    errno = err;
+    return false;
+}
+
+bool path_get_path(const wcstring &cmd, wcstring *out_path, const env_vars_snapshot_t &vars)
+{
+    return path_get_path_core(cmd, out_path, vars.get(L"PATH"));
+}
+
+bool path_get_path(const wcstring &cmd, wcstring *out_path)
+{
+    return path_get_path_core(cmd, out_path, env_get_string(L"PATH"));
+}
+
+bool path_get_cdpath(const wcstring &dir, wcstring *out, const wchar_t *wd, const env_vars_snapshot_t &env_vars)
+{
+    int err = ENOENT;
+    if (dir.empty())
+        return false;
+
+    if (wd)
+    {
+        size_t len = wcslen(wd);
+        assert(wd[len - 1] == L'/');
+    }
+
+    wcstring_list_t paths;
+    if (dir.at(0) == L'/')
+    {
+        /* Absolute path */
+        paths.push_back(dir);
+    }
+    else if (string_prefixes_string(L"./", dir) ||
+             string_prefixes_string(L"../", dir) ||
+             dir == L"." || dir == L"..")
+    {
+        /* Path is relative to the working directory */
+        wcstring path;
+        if (wd)
+            path.append(wd);
+        path.append(dir);
+        paths.push_back(path);
+    }
+    else
+    {
+        // Respect CDPATH
+        env_var_t path = env_vars.get(L"CDPATH");
+        if (path.missing_or_empty())
+            path = L"."; //We'll change this to the wd if we have one
+
+        wcstring nxt_path;
+        wcstokenizer tokenizer(path, ARRAY_SEP_STR);
+        while (tokenizer.next(nxt_path))
+        {
+
+            if (nxt_path == L"." && wd != NULL)
+            {
+                // nxt_path is just '.', and we have a working directory, so use the wd instead
+                // TODO: if nxt_path starts with ./ we need to replace the . with the wd
+                nxt_path = wd;
+            }
+            expand_tilde(nxt_path);
+
+//      debug( 2, L"woot %ls\n", expanded_path.c_str() );
+
+            if (nxt_path.empty())
+                continue;
+
+            wcstring whole_path = nxt_path;
+            append_path_component(whole_path, dir);
+            paths.push_back(whole_path);
+        }
+    }
+
+    bool success = false;
+    for (wcstring_list_t::const_iterator iter = paths.begin(); iter != paths.end(); ++iter)
+    {
+        struct stat buf;
+        const wcstring &dir = *iter;
+        if (wstat(dir, &buf) == 0)
+        {
+            if (S_ISDIR(buf.st_mode))
+            {
+                success = true;
+                if (out)
+                    out->assign(dir);
+                break;
+            }
+            else
+            {
+                err = ENOTDIR;
+            }
+        }
+    }
+
+    if (! success)
+        errno = err;
+    return success;
+}
+
+bool path_can_be_implicit_cd(const wcstring &path, wcstring *out_path, const wchar_t *wd, const env_vars_snapshot_t &vars)
+{
+    wcstring exp_path = path;
+    expand_tilde(exp_path);
+
+    bool result = false;
+    if (string_prefixes_string(L"/", exp_path) ||
+            string_prefixes_string(L"./", exp_path) ||
+            string_prefixes_string(L"../", exp_path) ||
+            string_suffixes_string(L"/", exp_path) ||
+            exp_path == L"..")
+    {
+        /* These paths can be implicit cd, so see if you cd to the path. Note that a single period cannot (that's used for sourcing files anyways) */
+        result = path_get_cdpath(exp_path, out_path, wd, vars);
+    }
+    return result;
+}
+
+static wcstring path_create_config()
+{
+    bool done = false;
+    wcstring res;
+
+    const env_var_t xdg_dir  = env_get_string(L"XDG_CONFIG_HOME");
+    if (! xdg_dir.missing())
+    {
+        res = xdg_dir + L"/fish";
+        if (!create_directory(res))
+        {
+            done = true;
+        }
+    }
+    else
+    {
+        const env_var_t home = env_get_string(L"HOME");
+        if (! home.missing())
+        {
+            res = home + L"/.config/fish";
+            if (!create_directory(res))
+            {
+                done = true;
+            }
+        }
+    }
+
+    if (! done)
+    {
+        res.clear();
+
+        debug(0, _(L"Unable to create a configuration directory for fish. Your personal settings will not be saved. Please set the $XDG_CONFIG_HOME variable to a directory where the current user has write access."));
+    }
+    return res;
+}
+
+/* Cache the config path */
+bool path_get_config(wcstring &path)
+{
+    static const wcstring result = path_create_config();
+    path = result;
+    return ! result.empty();
+}
+
+__attribute__((unused))
+static void replace_all(wcstring &str, const wchar_t *needle, const wchar_t *replacement)
+{
+    size_t needle_len = wcslen(needle);
+    size_t offset = 0;
+    while ((offset = str.find(needle, offset)) != wcstring::npos)
+    {
+        str.replace(offset, needle_len, replacement);
+        offset += needle_len;
+    }
+}
+
+void path_make_canonical(wcstring &path)
+{
+    // Ignore trailing slashes, unless it's the first character
+    size_t len = path.size();
+    while (len > 1 && path.at(len - 1) == L'/')
+        len--;
+
+    // Turn runs of slashes into a single slash
+    size_t trailing = 0;
+    bool prev_was_slash = false;
+    for (size_t leading = 0; leading < len; leading++)
+    {
+        wchar_t c = path.at(leading);
+        bool is_slash = (c == '/');
+        if (! prev_was_slash || ! is_slash)
+        {
+            // This is either the first slash in a run, or not a slash at all
+            path.at(trailing++) = c;
+        }
+        prev_was_slash = is_slash;
+    }
+    assert(trailing <= len);
+    if (trailing < len)
+        path.resize(trailing);
+}
+
+bool paths_are_equivalent(const wcstring &p1, const wcstring &p2)
+{
+    if (p1 == p2)
+        return true;
+
+    size_t len1 = p1.size(), len2 = p2.size();
+
+    // Ignore trailing slashes after the first character
+    while (len1 > 1 && p1.at(len1 - 1) == L'/') len1--;
+    while (len2 > 1 && p2.at(len2 - 1) == L'/') len2--;
+
+    // Start walking
+    size_t idx1 = 0, idx2 = 0;
+    while (idx1 < len1 && idx2 < len2)
+    {
+        wchar_t c1 = p1.at(idx1), c2 = p2.at(idx2);
+
+        // If the characters are different, the strings are not equivalent
+        if (c1 != c2)
+            break;
+
+        idx1++;
+        idx2++;
+
+        // If the character was a slash, walk forwards until we hit the end of the string, or a non-slash
+        // Note the first condition is invariant within the loop
+        while (c1 == L'/' && idx1 < len1 && p1.at(idx1) == L'/') idx1++;
+        while (c2 == L'/' && idx2 < len2 && p2.at(idx2) == L'/') idx2++;
+    }
+
+    // We matched if we consumed all of the characters in both strings
+    return idx1 == len1 && idx2 == len2;
+}
+
+bool path_is_valid(const wcstring &path, const wcstring &working_directory)
+{
+    bool path_is_valid;
+    /* Some special paths are always valid */
+    if (path.empty())
+    {
+        path_is_valid = false;
+    }
+    else if (path == L"." || path == L"./")
+    {
+        path_is_valid = true;
+    }
+    else if (path == L".." || path == L"../")
+    {
+        path_is_valid = (! working_directory.empty() && working_directory != L"/");
+    }
+    else if (path.at(0) != '/')
+    {
+        /* Prepend the working directory. Note that we know path is not empty here. */
+        wcstring tmp = working_directory;
+        tmp.append(path);
+        path_is_valid = (0 == waccess(tmp, F_OK));
+    }
+    else
+    {
+        /* Simple check */
+        path_is_valid = (0 == waccess(path, F_OK));
+    }
+    return path_is_valid;
+}
+
+bool paths_are_same_file(const wcstring &path1, const wcstring &path2)
+{
+    if (paths_are_equivalent(path1, path2))
+        return true;
+
+    struct stat s1, s2;
+    if (wstat(path1, &s1) == 0 && wstat(path2, &s2) == 0)
+    {
+        return s1.st_ino == s2.st_ino && s1.st_dev == s2.st_dev;
+    }
+    else
+    {
+        return false;
+    }
+}
diff --git a/src/path.h b/src/path.h
new file mode 100644
index 00000000..eb79ee8a
--- /dev/null
+++ b/src/path.h
@@ -0,0 +1,86 @@
+/** \file path.h
+
+  Directory utilities. This library contains functions for locating
+  configuration directories, for testing if a command with a given
+  name can be found in the PATH, and various other path-related
+  issues.
+*/
+
+#ifndef FISH_PATH_H
+#define FISH_PATH_H
+
+#include <stddef.h>
+#include "common.h"
+#include "env.h"
+
+/**
+   Return value for path_cdpath_get when locatied a rotten symlink
+ */
+#define EROTTEN 1
+
+/**
+   Returns the user configuration directory for fish. If the directory
+   or one of it's parents doesn't exist, they are first created.
+
+   \param path The directory as an out param
+   \return whether the directory was returned successfully
+*/
+bool path_get_config(wcstring &path);
+
+/**
+   Finds the full path of an executable. Returns YES if successful.
+
+   \param cmd The name of the executable.
+   \param output_or_NULL If non-NULL, store the full path.
+   \param vars The environment variables snapshot to use
+   \return 0 if the command can not be found, the path of the command otherwise. The result should be freed with free().
+*/
+bool path_get_path(const wcstring &cmd,
+                   wcstring *output_or_NULL,
+                   const env_vars_snapshot_t &vars = env_vars_snapshot_t::current());
+
+/**
+   Returns the full path of the specified directory, using the CDPATH
+   variable as a list of base directories for relative paths. The
+   returned string is allocated using halloc and the specified
+   context.
+
+   If no valid path is found, null is returned and errno is set to
+   ENOTDIR if at least one such path was found, but it did not point
+   to a directory, EROTTEN if a arotten symbolic link was found, or
+   ENOENT if no file of the specified name was found. If both a rotten
+   symlink and a file are found, it is undefined which error status
+   will be returned.
+
+   \param dir The name of the directory.
+   \param out_or_NULL If non-NULL, return the path to the resolved directory
+   \param wd The working directory, or NULL to use the default. The working directory should have a slash appended at the end.
+   \param vars The environment variable snapshot to use (for the CDPATH variable)
+   \return 0 if the command can not be found, the path of the command otherwise. The path should be free'd with free().
+*/
+bool path_get_cdpath(const wcstring &dir,
+                     wcstring *out_or_NULL,
+                     const wchar_t *wd = NULL,
+                     const env_vars_snapshot_t &vars = env_vars_snapshot_t::current());
+
+/** Returns whether the path can be used for an implicit cd command; if so, also returns the path by reference (if desired). This requires it to start with one of the allowed prefixes (., .., ~) and resolve to a directory. */
+bool path_can_be_implicit_cd(const wcstring &path,
+                             wcstring *out_path = NULL,
+                             const wchar_t *wd = NULL,
+                             const env_vars_snapshot_t &vars = env_vars_snapshot_t::current());
+
+/**
+   Remove double slashes and trailing slashes from a path,
+   e.g. transform foo//bar/ into foo/bar. The string is modified in-place.
+ */
+void path_make_canonical(wcstring &path);
+
+/** Check if two paths are equivalent, which means to ignore runs of multiple slashes (or trailing slashes) */
+bool paths_are_equivalent(const wcstring &p1, const wcstring &p2);
+
+bool path_is_valid(const wcstring &path, const wcstring &working_directory);
+
+/** Returns whether the two paths refer to the same file */
+bool paths_are_same_file(const wcstring &path1, const wcstring &path2);
+
+#endif
diff --git a/src/postfork.cpp b/src/postfork.cpp
new file mode 100644
index 00000000..9416b35d
--- /dev/null
+++ b/src/postfork.cpp
@@ -0,0 +1,589 @@
+/** \file postfork.cpp
+
+  Functions that we may safely call after fork().
+*/
+
+#include <fcntl.h>
+#include <errno.h>
+#include <signal.h>
+#include <stdio.h>
+#include <string.h>
+#include <time.h>
+#include <memory> // IWYU pragma: keep - suggests <tr1/memory> instead
+#include "common.h"
+#include "proc.h"
+#include "wutil.h"
+#include "signal.h"
+#include "postfork.h"
+#include "iothread.h"
+#include "exec.h"
+
+#ifndef JOIN_THREADS_BEFORE_FORK
+#define JOIN_THREADS_BEFORE_FORK 0
+#endif
+
+/** The number of times to try to call fork() before giving up */
+#define FORK_LAPS 5
+
+/** The number of nanoseconds to sleep between attempts to call fork() */
+#define FORK_SLEEP_TIME 1000000
+
+/** Base open mode to pass to calls to open */
+#define OPEN_MASK 0666
+
+/** fork error message */
+#define FORK_ERROR "Could not create child process - exiting"
+
+/** file redirection clobbering error message */
+#define NOCLOB_ERROR "The file '%s' already exists"
+
+/** file redirection error message */
+#define FILE_ERROR "An error occurred while redirecting file '%s'"
+
+/** file descriptor redirection error message */
+#define FD_ERROR   "An error occurred while redirecting file descriptor %s"
+
+/** pipe error */
+#define LOCAL_PIPE_ERROR "An error occurred while setting up pipe"
+
+static bool log_redirections = false;
+
+/* Cover for debug_safe that can take an int. The format string should expect a %s */
+static void debug_safe_int(int level, const char *format, int val)
+{
+    char buff[128];
+    format_long_safe(buff, val);
+    debug_safe(level, format, buff);
+}
+
+// PCA These calls to debug are rather sketchy because they may allocate memory. Fortunately they only occur if an error occurs.
+int set_child_group(job_t *j, process_t *p, int print_errors)
+{
+    int res = 0;
+
+    if (job_get_flag(j, JOB_CONTROL))
+    {
+        if (!j->pgid)
+        {
+            j->pgid = p->pid;
+        }
+
+        if (setpgid(p->pid, j->pgid))
+        {
+            if (getpgid(p->pid) != j->pgid && print_errors)
+            {
+                char pid_buff[128];
+                char job_id_buff[128];
+                char getpgid_buff[128];
+                char job_pgid_buff[128];
+
+                format_long_safe(pid_buff, p->pid);
+                format_long_safe(job_id_buff, j->job_id);
+                format_long_safe(getpgid_buff, getpgid(p->pid));
+                format_long_safe(job_pgid_buff, j->pgid);
+
+                debug_safe(1,
+                           "Could not send process %s, '%s' in job %s, '%s' from group %s to group %s",
+                           pid_buff,
+                           p->argv0_cstr(),
+                           job_id_buff,
+                           j->command_cstr(),
+                           getpgid_buff,
+                           job_pgid_buff);
+
+                safe_perror("setpgid");
+                res = -1;
+            }
+        }
+    }
+    else
+    {
+        j->pgid = getpid();
+    }
+
+    if (job_get_flag(j, JOB_TERMINAL) && job_get_flag(j, JOB_FOREGROUND))
+    {
+        if (tcsetpgrp(0, j->pgid) && print_errors)
+        {
+            char job_id_buff[128];
+            format_long_safe(job_id_buff, j->job_id);
+            debug_safe(1, "Could not send job %s ('%s') to foreground", job_id_buff, j->command_cstr());
+            safe_perror("tcsetpgrp");
+            res = -1;
+        }
+    }
+
+    return res;
+}
+
+/**
+   Set up a childs io redirections. Should only be called by
+   setup_child_process(). Does the following: First it closes any open
+   file descriptors not related to the child by calling
+   close_unused_internal_pipes() and closing the universal variable
+   server file descriptor. It then goes on to perform all the
+   redirections described by \c io.
+
+   \param io the list of IO redirections for the child
+
+   \return 0 on sucess, -1 on failiure
+*/
+static int handle_child_io(const io_chain_t &io_chain)
+{
+    for (size_t idx = 0; idx < io_chain.size(); idx++)
+    {
+        const io_data_t *io = io_chain.at(idx).get();
+        int tmp;
+
+        if (io->io_mode == IO_FD && io->fd == static_cast<const io_fd_t*>(io)->old_fd)
+        {
+            continue;
+        }
+
+        switch (io->io_mode)
+        {
+            case IO_CLOSE:
+            {
+                if (log_redirections) fprintf(stderr, "%d: close %d\n", getpid(), io->fd);
+                if (close(io->fd))
+                {
+                    debug_safe_int(0, "Failed to close file descriptor %s", io->fd);
+                    safe_perror("close");
+                }
+                break;
+            }
+
+            case IO_FILE:
+            {
+                // Here we definitely do not want to set CLO_EXEC because our child needs access
+                CAST_INIT(const io_file_t *, io_file, io);
+                if ((tmp=open(io_file->filename_cstr,
+                              io_file->flags, OPEN_MASK))==-1)
+                {
+                    if ((io_file->flags & O_EXCL) &&
+                            (errno ==EEXIST))
+                    {
+                        debug_safe(1, NOCLOB_ERROR, io_file->filename_cstr);
+                    }
+                    else
+                    {
+                        debug_safe(1, FILE_ERROR, io_file->filename_cstr);
+                        safe_perror("open");
+                    }
+
+                    return -1;
+                }
+                else if (tmp != io->fd)
+                {
+                    /*
+                      This call will sometimes fail, but that is ok,
+                      this is just a precausion.
+                    */
+                    close(io->fd);
+
+                    if (dup2(tmp, io->fd) == -1)
+                    {
+                        debug_safe_int(1,  FD_ERROR, io->fd);
+                        safe_perror("dup2");
+                        exec_close(tmp);
+                        return -1;
+                    }
+                    exec_close(tmp);
+                }
+                break;
+            }
+
+            case IO_FD:
+            {
+                int old_fd = static_cast<const io_fd_t *>(io)->old_fd;
+                if (log_redirections) fprintf(stderr, "%d: fd dup %d to %d\n", getpid(), old_fd, io->fd);
+
+                /*
+                  This call will sometimes fail, but that is ok,
+                  this is just a precausion.
+                */
+                close(io->fd);
+
+
+                if (dup2(old_fd, io->fd) == -1)
+                {
+                    debug_safe_int(1, FD_ERROR, io->fd);
+                    safe_perror("dup2");
+                    return -1;
+                }
+                break;
+            }
+
+            case IO_BUFFER:
+            case IO_PIPE:
+            {
+                CAST_INIT(const io_pipe_t *, io_pipe, io);
+                /* If write_pipe_idx is 0, it means we're connecting to the read end (first pipe fd). If it's 1, we're connecting to the write end (second pipe fd). */
+                unsigned int write_pipe_idx = (io_pipe->is_input ? 0 : 1);
+                /*
+                        debug( 0,
+                             L"%ls %ls on fd %d (%d %d)",
+                             write_pipe?L"write":L"read",
+                             (io->io_mode == IO_BUFFER)?L"buffer":L"pipe",
+                             io->fd,
+                             io->pipe_fd[0],
+                             io->pipe_fd[1]);
+                */
+                if (log_redirections) fprintf(stderr, "%d: %s dup %d to %d\n", getpid(), io->io_mode == IO_BUFFER ? "buffer" : "pipe", io_pipe->pipe_fd[write_pipe_idx], io->fd);
+                if (dup2(io_pipe->pipe_fd[write_pipe_idx], io->fd) != io->fd)
+                {
+                    debug_safe(1, LOCAL_PIPE_ERROR);
+                    safe_perror("dup2");
+                    return -1;
+                }
+
+                if (io_pipe->pipe_fd[0] >= 0)
+                    exec_close(io_pipe->pipe_fd[0]);
+                if (io_pipe->pipe_fd[1] >= 0)
+                    exec_close(io_pipe->pipe_fd[1]);
+                break;
+            }
+
+        }
+    }
+
+    return 0;
+
+}
+
+
+int setup_child_process(job_t *j, process_t *p, const io_chain_t &io_chain)
+{
+    bool ok=true;
+
+    if (p)
+    {
+        ok = (0 == set_child_group(j, p, 1));
+    }
+
+    if (ok)
+    {
+        ok = (0 == handle_child_io(io_chain));
+        if (p != 0 && ! ok)
+        {
+            exit_without_destructors(1);
+        }
+    }
+
+    /* Set the handling for job control signals back to the default.  */
+    if (ok)
+    {
+        signal_reset_handlers();
+    }
+
+    /* Remove all signal blocks */
+    signal_unblock();
+
+    return ok ? 0 : -1;
+}
+
+int g_fork_count = 0;
+
+/**
+   This function is a wrapper around fork. If the fork calls fails
+   with EAGAIN, it is retried FORK_LAPS times, with a very slight
+   delay between each lap. If fork fails even then, the process will
+   exit with an error message.
+*/
+pid_t execute_fork(bool wait_for_threads_to_die)
+{
+    ASSERT_IS_MAIN_THREAD();
+
+    if (wait_for_threads_to_die || JOIN_THREADS_BEFORE_FORK)
+    {
+        /* Make sure we have no outstanding threads before we fork. This is a pretty sketchy thing to do here, both because exec.cpp shouldn't have to know about iothreads, and because the completion handlers may do unexpected things. */
+        iothread_drain_all();
+    }
+
+    pid_t pid;
+    struct timespec pollint;
+    int i;
+
+    g_fork_count++;
+
+    for (i=0; i<FORK_LAPS; i++)
+    {
+        pid = fork();
+        if (pid >= 0)
+        {
+            return pid;
+        }
+
+        if (errno != EAGAIN)
+        {
+            break;
+        }
+
+        pollint.tv_sec = 0;
+        pollint.tv_nsec = FORK_SLEEP_TIME;
+
+        /*
+          Don't sleep on the final lap - sleeping might change the
+          value of errno, which will break the error reporting below.
+        */
+        if (i != FORK_LAPS-1)
+        {
+            nanosleep(&pollint, NULL);
+        }
+    }
+
+    debug_safe(0, FORK_ERROR);
+    safe_perror("fork");
+    FATAL_EXIT();
+    return 0;
+}
+
+#if FISH_USE_POSIX_SPAWN
+bool fork_actions_make_spawn_properties(posix_spawnattr_t *attr, posix_spawn_file_actions_t *actions, job_t *j, process_t *p, const io_chain_t &io_chain)
+{
+    /* Initialize the output */
+    if (posix_spawnattr_init(attr) != 0)
+    {
+        return false;
+    }
+
+    if (posix_spawn_file_actions_init(actions) != 0)
+    {
+        posix_spawnattr_destroy(attr);
+        return false;
+    }
+
+    bool should_set_parent_group_id = false;
+    int desired_parent_group_id = 0;
+    if (job_get_flag(j, JOB_CONTROL))
+    {
+        should_set_parent_group_id = true;
+
+        // PCA: I'm quite fuzzy on process groups,
+        // but I believe that the default value of 0
+        // means that the process becomes its own
+        // group leader, which is what set_child_group did
+        // in this case. So we want this to be 0 if j->pgid is 0.
+        desired_parent_group_id = j->pgid;
+    }
+
+    /* Set the handling for job control signals back to the default.  */
+    bool reset_signal_handlers = true;
+
+    /* Remove all signal blocks */
+    bool reset_sigmask = true;
+
+    /* Set our flags */
+    short flags = 0;
+    if (reset_signal_handlers)
+        flags |= POSIX_SPAWN_SETSIGDEF;
+    if (reset_sigmask)
+        flags |= POSIX_SPAWN_SETSIGMASK;
+    if (should_set_parent_group_id)
+        flags |= POSIX_SPAWN_SETPGROUP;
+
+    int err = 0;
+    if (! err)
+        err = posix_spawnattr_setflags(attr, flags);
+
+    if (! err && should_set_parent_group_id)
+        err = posix_spawnattr_setpgroup(attr, desired_parent_group_id);
+
+    /* Everybody gets default handlers */
+    if (! err && reset_signal_handlers)
+    {
+        sigset_t sigdefault;
+        get_signals_with_handlers(&sigdefault);
+        err = posix_spawnattr_setsigdefault(attr, &sigdefault);
+    }
+
+    /* No signals blocked */
+    sigset_t sigmask;
+    sigemptyset(&sigmask);
+    if (! err && reset_sigmask)
+        err = posix_spawnattr_setsigmask(attr, &sigmask);
+    
+    for (size_t idx = 0; idx < io_chain.size(); idx++)
+    {
+        const shared_ptr<const io_data_t> io = io_chain.at(idx);
+
+        if (io->io_mode == IO_FD)
+        {
+            CAST_INIT(const io_fd_t *, io_fd, io.get());
+            if (io->fd == io_fd->old_fd)
+                continue;
+        }
+
+        switch (io->io_mode)
+        {
+            case IO_CLOSE:
+            {
+                if (! err)
+                    err = posix_spawn_file_actions_addclose(actions, io->fd);
+                break;
+            }
+
+            case IO_FILE:
+            {
+                CAST_INIT(const io_file_t *, io_file, io.get());
+                if (! err)
+                    err = posix_spawn_file_actions_addopen(actions, io->fd, io_file->filename_cstr, io_file->flags /* mode */, OPEN_MASK);
+                break;
+            }
+
+            case IO_FD:
+            {
+                CAST_INIT(const io_fd_t *, io_fd, io.get());
+                if (! err)
+                    err = posix_spawn_file_actions_adddup2(actions, io_fd->old_fd /* from */, io->fd /* to */);
+                break;
+            }
+
+            case IO_BUFFER:
+            case IO_PIPE:
+            {
+                CAST_INIT(const io_pipe_t *, io_pipe, io.get());
+                unsigned int write_pipe_idx = (io_pipe->is_input ? 0 : 1);
+                int from_fd = io_pipe->pipe_fd[write_pipe_idx];
+                int to_fd = io->fd;
+                if (! err)
+                    err = posix_spawn_file_actions_adddup2(actions, from_fd, to_fd);
+
+
+                if (write_pipe_idx > 0)
+                {
+                    if (! err)
+                        err = posix_spawn_file_actions_addclose(actions, io_pipe->pipe_fd[0]);
+                    if (! err)
+                        err = posix_spawn_file_actions_addclose(actions, io_pipe->pipe_fd[1]);
+                }
+                else
+                {
+                    if (! err)
+                        err = posix_spawn_file_actions_addclose(actions, io_pipe->pipe_fd[0]);
+
+                }
+                break;
+            }
+        }
+    }
+
+    /* Clean up on error */
+    if (err)
+    {
+        posix_spawnattr_destroy(attr);
+        posix_spawn_file_actions_destroy(actions);
+    }
+
+    return ! err;
+}
+#endif //FISH_USE_POSIX_SPAWN
+
+void safe_report_exec_error(int err, const char *actual_cmd, const char * const *argv, const char *const *envv)
+{
+    debug_safe(0, "Failed to execute process '%s'. Reason:", actual_cmd);
+
+    switch (err)
+    {
+
+        case E2BIG:
+        {
+            char sz1[128], sz2[128];
+
+            long arg_max = -1;
+
+            size_t sz = 0;
+            const char * const *p;
+            for (p=argv; *p; p++)
+            {
+                sz += strlen(*p)+1;
+            }
+
+            for (p=envv; *p; p++)
+            {
+                sz += strlen(*p)+1;
+            }
+
+            format_size_safe(sz1, sz);
+            arg_max = sysconf(_SC_ARG_MAX);
+
+            if (arg_max > 0)
+            {
+                format_size_safe(sz2, sz);
+                debug_safe(0, "The total size of the argument and environment lists %s exceeds the operating system limit of %s.", sz1, sz2);
+            }
+            else
+            {
+                debug_safe(0, "The total size of the argument and environment lists (%s) exceeds the operating system limit.", sz1);
+            }
+
+            debug_safe(0, "Try running the command again with fewer arguments.");
+            break;
+        }
+
+        case ENOEXEC:
+        {
+            const char *err = safe_strerror(errno);
+            debug_safe(0, "exec: %s", err);
+
+            debug_safe(0, "The file '%s' is marked as an executable but could not be run by the operating system.", actual_cmd);
+            break;
+        }
+
+        case ENOENT:
+        {
+            /* ENOENT is returned by exec() when the path fails, but also returned by posix_spawn if an open file action fails. These cases appear to be impossible to distinguish. We address this by not using posix_spawn for file redirections, so all the ENOENTs we find must be errors from exec(). */
+            char interpreter_buff[128] = {}, *interpreter;
+            interpreter = get_interpreter(actual_cmd, interpreter_buff, sizeof interpreter_buff);
+            if (interpreter && 0 != access(interpreter, X_OK))
+            {
+                debug_safe(0, "The file '%s' specified the interpreter '%s', which is not an executable command.", actual_cmd, interpreter);
+            }
+            else
+            {
+                debug_safe(0, "The file '%s' does not exist or could not be executed.", actual_cmd);
+            }
+            break;
+        }
+
+        case ENOMEM:
+        {
+            debug_safe(0, "Out of memory");
+            break;
+        }
+
+        default:
+        {
+            const char *err = safe_strerror(errno);
+            debug_safe(0, "exec: %s", err);
+
+            //    debug(0, L"The file '%ls' is marked as an executable but could not be run by the operating system.", p->actual_cmd);
+            break;
+        }
+    }
+}
+
+/** Perform output from builtins. May be called from a forked child, so don't do anything that may allocate memory, etc.. */
+bool do_builtin_io(const char *out, size_t outlen, const char *err, size_t errlen)
+{
+    bool success = true;
+    if (out && outlen)
+    {
+
+        if (write_loop(STDOUT_FILENO, out, outlen) < 0)
+        {
+            debug_safe(0, "Error while writing to stdout");
+            safe_perror("write_loop");
+            success = false;
+        }
+    }
+
+    if (err && errlen)
+    {
+        if (write_loop(STDERR_FILENO, err, errlen) < 0)
+        {
+            success = false;
+        }
+    }
+    return success;
+}
diff --git a/src/postfork.h b/src/postfork.h
new file mode 100644
index 00000000..c277da52
--- /dev/null
+++ b/src/postfork.h
@@ -0,0 +1,74 @@
+/** \file postfork.h
+
+  Functions that we may safely call after fork(), of which there are very few. In particular we cannot allocate memory, since we're insane enough to call fork from a multithreaded process.
+*/
+
+#ifndef FISH_POSTFORK_H
+#define FISH_POSTFORK_H
+
+#include <stddef.h>
+#include <unistd.h>
+
+#include "config.h"
+#include "io.h"
+
+#if HAVE_SPAWN_H
+#include <spawn.h>
+#endif
+
+#ifndef FISH_USE_POSIX_SPAWN
+#define FISH_USE_POSIX_SPAWN HAVE_SPAWN_H
+#endif
+
+
+/**
+   This function should be called by both the parent process and the
+   child right after fork() has been called. If job control is
+   enabled, the child is put in the jobs group, and if the child is
+   also in the foreground, it is also given control of the
+   terminal. When called in the parent process, this function may
+   fail, since the child might have already finished and called
+   exit. The parent process may safely ignore the exit status of this
+   call.
+
+   Returns 0 on sucess, -1 on failiure.
+*/
+class job_t;
+class process_t;
+int set_child_group(job_t *j, process_t *p, int print_errors);
+
+/**
+   Initialize a new child process. This should be called right away
+   after forking in the child process. If job control is enabled for
+   this job, the process is put in the process group of the job, all
+   signal handlers are reset, signals are unblocked (this function may
+   only be called inside the exec function, which blocks all signals),
+   and all IO redirections and other file descriptor actions are
+   performed.
+
+   \param j the job to set up the IO for
+   \param p the child process to set up
+   \param io_chain the IO chain to use
+
+   \return 0 on sucess, -1 on failiure. When this function returns,
+   signals are always unblocked. On failiure, signal handlers, io
+   redirections and process group of the process is undefined.
+*/
+int setup_child_process(job_t *j, process_t *p, const io_chain_t &io_chain);
+
+/* Call fork(), optionally waiting until we are no longer multithreaded. If the forked child doesn't do anything that could allocate memory, take a lock, etc. (like call exec), then it's not necessary to wait for threads to die. If the forked child may do those things, it should wait for threads to die.
+*/
+pid_t execute_fork(bool wait_for_threads_to_die);
+
+/* Perform output from builtins. Returns true on success. */
+bool do_builtin_io(const char *out, size_t outlen, const char *err, size_t errlen);
+
+/** Report an error from failing to exec or posix_spawn a command */
+void safe_report_exec_error(int err, const char *actual_cmd, const char * const *argv, const char * const *envv);
+
+#if FISH_USE_POSIX_SPAWN
+/* Initializes and fills in a posix_spawnattr_t; on success, the caller should destroy it via posix_spawnattr_destroy */
+bool fork_actions_make_spawn_properties(posix_spawnattr_t *attr, posix_spawn_file_actions_t *actions, job_t *j, process_t *p, const io_chain_t &io_chain);
+#endif
+
+#endif
diff --git a/src/print_help.cpp b/src/print_help.cpp
new file mode 100644
index 00000000..da401134
--- /dev/null
+++ b/src/print_help.cpp
@@ -0,0 +1,36 @@
+
+/** \file print_help.c
+  Print help message for the specified command
+*/
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <stddef.h>
+#include <sys/types.h>
+
+#include "print_help.h"
+
+#define CMD_LEN 1024
+
+#define HELP_ERR "Could not show help message\n"
+
+/* defined in common.h */
+ssize_t write_loop(int fd, const char *buff, size_t count);
+
+
+void print_help(const char *c, int fd)
+{
+    char cmd[ CMD_LEN];
+    int printed = snprintf(cmd, CMD_LEN, "fish -c '__fish_print_help %s >&%d'", c, fd);
+
+    if (printed < CMD_LEN)
+    {
+        if ((system(cmd) == -1))
+        {
+            write_loop(2, HELP_ERR, strlen(HELP_ERR));
+        }
+
+    }
+
+}
diff --git a/src/print_help.h b/src/print_help.h
new file mode 100644
index 00000000..005800b1
--- /dev/null
+++ b/src/print_help.h
@@ -0,0 +1,15 @@
+
+/** \file print_help.h
+  Print help message for the specified command
+*/
+
+#ifndef FISH_PRINT_HELP_H
+#define FISH_PRINT_HELP_H
+
+/**
+  Print help message for the specified command
+*/
+
+void print_help(const char *cmd, int fd);
+
+#endif
diff --git a/src/proc.cpp b/src/proc.cpp
new file mode 100644
index 00000000..0a0da0a8
--- /dev/null
+++ b/src/proc.cpp
@@ -0,0 +1,1414 @@
+/** \file proc.c
+
+Utilities for keeping track of jobs, processes and subshells, as
+well as signal handling functions for tracking children. These
+functions do not themselves launch new processes, the exec library
+will call proc to create representations of the running jobs as
+needed.
+
+Some of the code in this file is based on code from the Glibc manual.
+
+*/
+#include "config.h"
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <sys/wait.h>
+#include <wchar.h>
+#include <string.h>
+#include <errno.h>
+#include <termios.h>
+#include <pthread.h>
+#include <wctype.h>
+#include <algorithm>
+#include <memory> // IWYU pragma: keep - suggests <tr1/memory> instead
+#include <vector>
+
+#include <unistd.h>
+#include <signal.h>
+#include <sys/time.h>
+
+#if HAVE_TERM_H
+#include <term.h>
+#elif HAVE_NCURSES_TERM_H
+#include <ncurses/term.h>
+#endif
+
+#ifdef HAVE_SIGINFO_H
+#include <siginfo.h>
+#endif
+
+#ifdef HAVE_SYS_SELECT_H
+#include <sys/select.h>
+#endif
+
+#include "fallback.h" // IWYU pragma: keep
+#include "util.h"
+
+#include "wutil.h"
+#include "proc.h"
+#include "common.h"
+#include "reader.h"
+#include "sanity.h"
+#include "parser.h"
+#include "signal.h"
+#include "event.h"
+
+#include "output.h"
+
+/**
+   Size of message buffer
+*/
+#define MESS_SIZE 256
+
+/**
+   Size of buffer for reading buffered output
+*/
+#define BUFFER_SIZE 4096
+
+/**
+	Status of last process to exit
+*/
+static int last_status=0;
+
+bool job_list_is_empty(void)
+{
+    ASSERT_IS_MAIN_THREAD();
+    return parser_t::principal_parser().job_list().empty();
+}
+
+void job_iterator_t::reset()
+{
+    this->current = job_list->begin();
+    this->end = job_list->end();
+}
+
+job_iterator_t::job_iterator_t(job_list_t &jobs) : job_list(&jobs)
+{
+    this->reset();
+}
+
+job_iterator_t::job_iterator_t() : job_list(&parser_t::principal_parser().job_list())
+{
+    ASSERT_IS_MAIN_THREAD();
+    this->reset();
+}
+
+size_t job_iterator_t::count() const
+{
+    return this->job_list->size();
+}
+
+void print_jobs(void)
+{
+    job_iterator_t jobs;
+    job_t *j;
+    while ((j = jobs.next()))
+    {
+        printf("%p -> %ls -> (foreground %d, complete %d, stopped %d, constructed %d)\n", j, j->command_wcstr(), job_get_flag(j, JOB_FOREGROUND), job_is_completed(j), job_is_stopped(j), job_get_flag(j, JOB_CONSTRUCTED));
+    }
+}
+
+int is_interactive_session=0;
+int is_subshell=0;
+int is_block=0;
+int is_login=0;
+int is_event=0;
+pid_t proc_last_bg_pid = 0;
+int job_control_mode = JOB_CONTROL_INTERACTIVE;
+int no_exec=0;
+
+static int is_interactive = -1;
+
+static bool proc_had_barrier = false;
+
+int get_is_interactive(void)
+{
+    ASSERT_IS_MAIN_THREAD();
+    /* is_interactive is initialized to -1; ensure someone has popped/pushed it before then */
+    assert(is_interactive >= 0);
+    return is_interactive > 0;
+}
+
+bool get_proc_had_barrier()
+{
+    ASSERT_IS_MAIN_THREAD();
+    return proc_had_barrier;
+}
+
+void set_proc_had_barrier(bool flag)
+{
+    ASSERT_IS_MAIN_THREAD();
+    proc_had_barrier = flag;
+}
+
+/**
+   The event variable used to send all process event
+*/
+static event_t event(0);
+
+/**
+   A stack containing the values of is_interactive. Used by proc_push_interactive and proc_pop_interactive.
+*/
+static std::vector<int> interactive_stack;
+
+void proc_init()
+{
+    proc_push_interactive(0);
+}
+
+
+/**
+   Remove job from list of jobs
+*/
+static int job_remove(job_t *j)
+{
+    ASSERT_IS_MAIN_THREAD();
+    return parser_t::principal_parser().job_remove(j);
+}
+
+void job_promote(job_t *job)
+{
+    ASSERT_IS_MAIN_THREAD();
+    parser_t::principal_parser().job_promote(job);
+}
+
+
+/*
+  Remove job from the job list and free all memory associated with
+  it.
+*/
+void job_free(job_t * j)
+{
+    job_remove(j);
+    delete j;
+}
+
+void proc_destroy()
+{
+    job_list_t &jobs = parser_t::principal_parser().job_list();
+    while (! jobs.empty())
+    {
+        job_t *job = jobs.front();
+        debug(2, L"freeing leaked job %ls", job->command_wcstr());
+        job_free(job);
+    }
+}
+
+void proc_set_last_status(int s)
+{
+    ASSERT_IS_MAIN_THREAD();
+    last_status = s;
+}
+
+int proc_get_last_status()
+{
+    return last_status;
+}
+
+/* Basic thread safe job IDs. The vector consumed_job_ids has a true value wherever the job ID corresponding to that slot is in use. The job ID corresponding to slot 0 is 1. */
+static pthread_mutex_t job_id_lock = PTHREAD_MUTEX_INITIALIZER;
+static std::vector<bool> consumed_job_ids;
+
+job_id_t acquire_job_id(void)
+{
+    scoped_lock lock(job_id_lock);
+
+    /* Find the index of the first 0 slot */
+    std::vector<bool>::iterator slot = std::find(consumed_job_ids.begin(), consumed_job_ids.end(), false);
+    if (slot != consumed_job_ids.end())
+    {
+        /* We found a slot. Note that slot 0 corresponds to job ID 1. */
+        *slot = true;
+        return (job_id_t)(slot - consumed_job_ids.begin() + 1);
+    }
+    else
+    {
+        /* We did not find a slot; create a new slot. The size of the vector is now the job ID (since it is one larger than the slot). */
+        consumed_job_ids.push_back(true);
+        return (job_id_t)consumed_job_ids.size();
+    }
+}
+
+void release_job_id(job_id_t jid)
+{
+    assert(jid > 0);
+    scoped_lock lock(job_id_lock);
+    size_t slot = (size_t)(jid - 1), count = consumed_job_ids.size();
+
+    /* Make sure this slot is within our vector and is currently set to consumed */
+    assert(slot < count);
+    assert(consumed_job_ids.at(slot) == true);
+
+    /* Clear it and then resize the vector to eliminate unused trailing job IDs */
+    consumed_job_ids.at(slot) = false;
+    while (count--)
+    {
+        if (consumed_job_ids.at(count))
+            break;
+    }
+    consumed_job_ids.resize(count + 1);
+}
+
+job_t *job_get(job_id_t id)
+{
+    ASSERT_IS_MAIN_THREAD();
+    return parser_t::principal_parser().job_get(id);
+}
+
+job_t *job_get_from_pid(int pid)
+{
+    ASSERT_IS_MAIN_THREAD();
+    return parser_t::principal_parser().job_get_from_pid(pid);
+}
+
+
+/*
+   Return true if all processes in the job have stopped or completed.
+
+   \param j the job to test
+*/
+int job_is_stopped(const job_t *j)
+{
+    process_t *p;
+
+    for (p = j->first_process; p; p = p->next)
+    {
+        if (!p->completed && !p->stopped)
+        {
+            return 0;
+        }
+    }
+    return 1;
+}
+
+
+/*
+   Return true if the last processes in the job has completed.
+
+   \param j the job to test
+*/
+bool job_is_completed(const job_t *j)
+{
+    assert(j->first_process != NULL);
+    bool result = true;
+    for (process_t *p = j->first_process; p != NULL; p = p->next)
+    {
+        if (! p->completed)
+        {
+            result = false;
+            break;
+        }
+    }
+    return result;
+}
+
+void job_set_flag(job_t *j, unsigned int flag, int set)
+{
+    if (set)
+    {
+        j->flags |= flag;
+    }
+    else
+    {
+        j->flags &= ~flag;
+    }
+}
+
+int job_get_flag(const job_t *j, unsigned int flag)
+{
+    return !!(j->flags & flag);
+}
+
+int job_signal(job_t *j, int signal)
+{
+    pid_t my_pid = getpid();
+    int res = 0;
+
+    if (j->pgid != my_pid)
+    {
+        res = killpg(j->pgid, SIGHUP);
+    }
+    else
+    {
+        for (process_t *p = j->first_process; p; p=p->next)
+        {
+            if (! p->completed)
+            {
+                if (p->pid)
+                {
+                    if (kill(p->pid, SIGHUP))
+                    {
+                        res = -1;
+                        break;
+                    }
+                }
+            }
+        }
+
+    }
+
+    return res;
+}
+
+
+/**
+   Store the status of the process pid that was returned by waitpid.
+   Return 0 if all went well, nonzero otherwise.
+   This is called from a signal handler.
+*/
+static void mark_process_status(const job_t *j, process_t *p, int status)
+{
+//	debug( 0, L"Process %ls %ls", p->argv[0], WIFSTOPPED (status)?L"stopped":(WIFEXITED( status )?L"exited":(WIFSIGNALED( status )?L"signaled to exit":L"BLARGH")) );
+    p->status = status;
+
+    if (WIFSTOPPED(status))
+    {
+        p->stopped = 1;
+    }
+    else if (WIFSIGNALED(status) || WIFEXITED(status))
+    {
+        p->completed = 1;
+    }
+    else
+    {
+        /* This should never be reached */
+        p->completed = 1;
+
+        char mess[MESS_SIZE];
+        snprintf(mess,
+                 MESS_SIZE,
+                 "Process %ld exited abnormally\n",
+                 (long) p->pid);
+        /*
+          If write fails, do nothing. We're in a signal handlers error
+          handler. If things aren't working properly, it's safer to
+          give up.
+         */
+        write_ignore(2, mess, strlen(mess));
+    }
+}
+
+void job_mark_process_as_failed(const job_t *job, process_t *p)
+{
+    /* The given process failed to even lift off (e.g. posix_spawn failed) and so doesn't have a valid pid. Mark it as dead. */
+    for (process_t *cursor = p; cursor != NULL; cursor = cursor->next)
+    {
+        cursor->completed = 1;
+    }
+}
+
+/**
+   Handle status update for child \c pid. This function is called by
+   the signal handler, so it mustn't use malloc or any such hitech
+   nonsense.
+
+   \param pid the pid of the process whose status changes
+   \param status the status as returned by wait
+*/
+static void handle_child_status(pid_t pid, int status)
+{
+    bool found_proc = false;
+    const job_t *j = NULL;
+    process_t *p = NULL;
+//	char mess[MESS_SIZE];
+    /*
+      snprintf( mess,
+      MESS_SIZE,
+      "Process %d\n",
+      (int) pid );
+      write( 2, mess, strlen(mess ));
+    */
+
+    job_iterator_t jobs;
+    while (! found_proc && (j = jobs.next()))
+    {
+        process_t *prev=0;
+        for (p=j->first_process; p; p=p->next)
+        {
+            if (pid == p->pid)
+            {
+                /*				snprintf( mess,
+                  MESS_SIZE,
+                  "Process %d is %ls from job %ls\n",
+                  (int) pid, p->actual_cmd, j->command );
+                  write( 2, mess, strlen(mess ));
+                */
+
+                mark_process_status(j, p, status);
+                if (p->completed && prev != 0)
+                {
+                    if (!prev->completed && prev->pid)
+                    {
+                        /*					snprintf( mess,
+                          MESS_SIZE,
+                          "Kill previously uncompleted process %ls (%d)\n",
+                          prev->actual_cmd,
+                          prev->pid );
+                          write( 2, mess, strlen(mess ));
+                        */
+                        kill(prev->pid,SIGPIPE);
+                    }
+                }
+                found_proc = true;
+                break;
+            }
+            prev = p;
+        }
+    }
+
+
+    if (WIFSIGNALED(status) &&
+            (WTERMSIG(status)==SIGINT  ||
+             WTERMSIG(status)==SIGQUIT))
+    {
+        if (!is_interactive_session)
+        {
+            struct sigaction act;
+            sigemptyset(& act.sa_mask);
+            act.sa_flags=0;
+            act.sa_handler=SIG_DFL;
+            sigaction(SIGINT, &act, 0);
+            sigaction(SIGQUIT, &act, 0);
+            kill(getpid(), WTERMSIG(status));
+        }
+        else
+        {
+            /* In an interactive session, tell the principal parser to skip all blocks we're executing so control-C returns control to the user. */
+            if (p && found_proc)
+            {
+                parser_t::skip_all_blocks();
+            }
+        }
+    }
+
+    if (!found_proc)
+    {
+        /*
+           A child we lost track of?
+
+           There have been bugs in both subshell handling and in
+           builtin handling that have caused this previously...
+        */
+        /*		snprintf( mess,
+          MESS_SIZE,
+          "Process %d not found by %d\n",
+          (int) pid, (int)getpid() );
+
+          write( 2, mess, strlen(mess ));
+        */
+    }
+    return;
+}
+
+process_t::process_t() :
+    argv_array(),
+    argv0_narrow(),
+    type(),
+    internal_block_node(NODE_OFFSET_INVALID),
+    actual_cmd(),
+    pid(0),
+    pipe_write_fd(0),
+    pipe_read_fd(0),
+    completed(0),
+    stopped(0),
+    status(0),
+    count_help_magic(0),
+    next(NULL)
+#ifdef HAVE__PROC_SELF_STAT
+    ,last_time(),
+    last_jiffies(0)
+#endif
+{
+}
+
+process_t::~process_t()
+{
+    if (this->next != NULL)
+        delete this->next;
+}
+
+job_t::job_t(job_id_t jobid, const io_chain_t &bio) :
+    command_str(),
+    command_narrow(),
+    block_io(bio),
+    first_process(NULL),
+    pgid(0),
+    tmodes(),
+    job_id(jobid),
+    flags(0)
+{
+}
+
+job_t::~job_t()
+{
+    if (first_process != NULL)
+        delete first_process;
+    release_job_id(job_id);
+}
+
+/* Return all the IO redirections. Start with the block IO, then walk over the processes */
+io_chain_t job_t::all_io_redirections() const
+{
+    io_chain_t result = this->block_io;
+    for (process_t *p = this->first_process; p != NULL; p = p->next)
+    {
+        result.append(p->io_chain());
+    }
+    return result;
+}
+
+typedef unsigned int process_generation_count_t;
+
+/* A static value tracking how many SIGCHLDs we have seen. This is only ever modified from within the SIGCHLD signal handler, and therefore does not need atomics or locks */
+static volatile process_generation_count_t s_sigchld_generation_count = 0;
+
+/* If we have received a SIGCHLD signal, process any children. If await is false, this returns immediately if no SIGCHLD has been received. If await is true, this waits for one. Returns true if something was processed. This returns the number of children processed, or -1 on error. */
+static int process_mark_finished_children(bool wants_await)
+{
+    ASSERT_IS_MAIN_THREAD();
+    
+    /* A static value tracking the SIGCHLD gen count at the time we last processed it. When this is different from s_sigchld_generation_count, it indicates there may be unreaped processes. There may not be if we reaped them via the other waitpid path. This is only ever modified from the main thread, and not from a signal handler. */
+    static process_generation_count_t s_last_processed_sigchld_generation_count = 0;
+    
+    int processed_count = 0;
+    bool got_error = false;
+
+    /* The critical read. This fetches a value which is only written in the signal handler. This needs to be an atomic read (we'd use sig_atomic_t, if we knew that were unsigned - fortunately aligned unsigned int is atomic on pretty much any modern chip.) It also needs to occur before we start reaping, since the signal handler can be invoked at any point. */
+    const process_generation_count_t local_count = s_sigchld_generation_count;
+    
+    /* Determine whether we have children to process. Note that we can't reliably use the difference because a single SIGCHLD may be delivered for multiple children - see #1768. Also if we are awaiting, we always process. */
+    bool wants_waitpid = wants_await || local_count != s_last_processed_sigchld_generation_count;
+
+    if (wants_waitpid)
+    {
+        for (;;)
+        {
+            /* Call waitpid until we get 0/ECHILD. If we wait, it's only on the first iteration. So we want to set NOHANG (don't wait) unless wants_await is true and this is the first iteration. */
+            int options = WUNTRACED;
+            if (! (wants_await && processed_count == 0))
+            {
+                options |= WNOHANG;
+            }
+            
+            int status = -1;
+            pid_t pid = waitpid(-1, &status, options);
+            if (pid > 0)
+            {
+                /* We got a valid pid */
+                handle_child_status(pid, status);
+                processed_count += 1;
+            }
+            else if (pid == 0)
+            {
+                /* No ready-and-waiting children, we're done */
+                break;
+            }
+            else
+            {
+                /* This indicates an error. One likely failure is ECHILD (no children), which we break on, and is not considered an error. The other likely failure is EINTR, which means we got a signal, which is considered an error. */
+                got_error = (errno != ECHILD);
+                break;
+            }
+        }
+    }
+
+    if (got_error)
+    {
+        return -1;
+    }
+    else
+    {
+        s_last_processed_sigchld_generation_count = local_count;
+        return processed_count;
+    }
+}
+
+
+/* This is called from a signal handler. The signal is always SIGCHLD. */
+void job_handle_signal(int signal, siginfo_t *info, void *con)
+{
+    /* This is the only place that this generation count is modified. It's OK if it overflows. */
+    s_sigchld_generation_count += 1;
+}
+
+/* Given a command like "cat file", truncate it to a reasonable length */
+static wcstring truncate_command(const wcstring &cmd)
+{
+    const size_t max_len = 32;
+    if (cmd.size() <= max_len)
+    {
+        // No truncation necessary
+        return cmd;
+    }
+    
+    // Truncation required
+    const bool ellipsis_is_unicode = (ellipsis_char == L'\x2026');
+    const size_t ellipsis_length = ellipsis_is_unicode ? 1 : 3;
+    size_t trunc_length = max_len - ellipsis_length;
+    // Eat trailing whitespace
+    while (trunc_length > 0 && iswspace(cmd.at(trunc_length - 1)))
+    {
+        trunc_length -= 1;
+    }
+    wcstring result = wcstring(cmd, 0, trunc_length);
+    // Append ellipsis
+    if (ellipsis_is_unicode)
+    {
+        result.push_back(ellipsis_char);
+    }
+    else
+    {
+        result.append(L"...");
+    }
+    return result;
+}
+
+/**
+	Format information about job status for the user to look at.
+
+	\param j the job to test
+	\param status a string description of the job exit type
+*/
+static void format_job_info(const job_t *j, const wchar_t *status, size_t job_count)
+{
+    fwprintf(stdout, L"\r");
+    if (job_count == 1)
+    {
+        fwprintf(stdout, _(L"\'%ls\' has %ls"), truncate_command(j->command()).c_str(), status);
+    }
+    else
+    {
+        fwprintf(stdout, _(L"Job %d, \'%ls\' has %ls"), j->job_id, truncate_command(j->command()).c_str(), status);
+    }
+    fflush(stdout);
+    tputs(clr_eol,1,&writeb);
+    fwprintf(stdout, L"\n");
+}
+
+void proc_fire_event(const wchar_t *msg, int type, pid_t pid, int status)
+{
+
+    event.type=type;
+    event.param1.pid = pid;
+
+    event.arguments.push_back(msg);
+    event.arguments.push_back(to_string<int>(pid));
+    event.arguments.push_back(to_string<int>(status));
+    event_fire(&event);
+    event.arguments.resize(0);
+}
+
+int job_reap(bool interactive)
+{
+    ASSERT_IS_MAIN_THREAD();
+    job_t *jnext;
+    int found=0;
+
+    /* job_reap may fire an event handler, we do not want to call ourselves recursively (to avoid infinite recursion). */
+    static bool locked = false;
+    if (locked)
+    {
+        return 0;
+    }
+    locked = true;
+    
+    process_mark_finished_children(false);
+
+    /* Preserve the exit status */
+    const int saved_status = proc_get_last_status();
+
+    job_iterator_t jobs;
+    const size_t job_count = jobs.count();
+    jnext = jobs.next();
+    while (jnext)
+    {
+        job_t *j = jnext;
+        jnext = jobs.next();
+
+        /*
+          If we are reaping only jobs who do not need status messages
+          sent to the console, do not consider reaping jobs that need
+          status messages
+        */
+        if ((!job_get_flag(j, JOB_SKIP_NOTIFICATION)) && (!interactive) && (!job_get_flag(j, JOB_FOREGROUND)))
+        {
+            continue;
+        }
+
+        for (process_t *p = j->first_process; p; p=p->next)
+        {
+            int s;
+            if (!p->completed)
+                continue;
+
+            if (!p->pid)
+                continue;
+
+            s = p->status;
+
+            proc_fire_event(L"PROCESS_EXIT", EVENT_EXIT, p->pid, (WIFSIGNALED(s)?-1:WEXITSTATUS(s)));
+
+            if (WIFSIGNALED(s))
+            {
+                /*
+                   Ignore signal SIGPIPE.We issue it ourselves to the pipe
+                   writer when the pipe reader dies.
+                */
+                if (WTERMSIG(s) != SIGPIPE)
+                {
+                    int proc_is_job = ((p==j->first_process) && (p->next == 0));
+                    if (proc_is_job)
+                        job_set_flag(j, JOB_NOTIFIED, 1);
+                    if (!job_get_flag(j, JOB_SKIP_NOTIFICATION))
+                    {
+                        /* Print nothing if we get SIGINT in the foreground process group, to avoid spamming obvious stuff on the console (#1119). If we get SIGINT for the foreground process, assume the user typed ^C and can see it working. It's possible they didn't, and the signal was delivered via pkill, etc., but the SIGINT/SIGTERM distinction is precisely to allow INT to be from a UI and TERM to be programmatic, so this assumption is keeping with the design of signals.
+                        If echoctl is on, then the terminal will have written ^C to the console. If off, it won't have. We don't echo ^C either way, so as to respect the user's preference. */
+                        if (WTERMSIG(p->status) != SIGINT || ! job_get_flag(j, JOB_FOREGROUND))
+                            {
+                            if (proc_is_job)
+                            {
+                                // We want to report the job number, unless it's the only job, in which case we don't need to
+                                const wcstring job_number_desc = (job_count == 1) ? wcstring() : format_string(L"Job %d, ", j->job_id);
+                                fwprintf(stdout,
+                                         _(L"%ls: %ls\'%ls\' terminated by signal %ls (%ls)"),
+                                         program_name,
+                                         job_number_desc.c_str(),
+                                         truncate_command(j->command()).c_str(),
+                                         sig2wcs(WTERMSIG(p->status)),
+                                         signal_get_desc(WTERMSIG(p->status)));
+                            }
+                            else
+                            {
+                                const wcstring job_number_desc = (job_count == 1) ? wcstring() : format_string(L"from job %d, ", j->job_id);
+                                fwprintf(stdout,
+                                         _(L"%ls: Process %d, \'%ls\' %ls\'%ls\' terminated by signal %ls (%ls)"),
+                                         program_name,
+                                         p->pid,
+                                         p->argv0(),
+                                         job_number_desc.c_str(),
+                                         truncate_command(j->command()).c_str(),
+                                         sig2wcs(WTERMSIG(p->status)),
+                                         signal_get_desc(WTERMSIG(p->status)));
+                            }
+                            tputs(clr_eol,1,&writeb);
+                            fwprintf(stdout, L"\n");
+                        }
+                        found=1;
+                    }
+
+                    /*
+                       Clear status so it is not reported more than once
+                    */
+                    p->status = 0;
+                }
+            }
+        }
+
+        /*
+           If all processes have completed, tell the user the job has
+           completed and delete it from the active job list.
+        */
+        if (job_is_completed(j))
+        {
+            if (!job_get_flag(j, JOB_FOREGROUND) && !job_get_flag(j, JOB_NOTIFIED) && !job_get_flag(j, JOB_SKIP_NOTIFICATION))
+            {
+                format_job_info(j, _(L"ended"), job_count);
+                found=1;
+            }
+            proc_fire_event(L"JOB_EXIT", EVENT_EXIT, -j->pgid, 0);
+            proc_fire_event(L"JOB_EXIT", EVENT_JOB_ID, j->job_id, 0);
+
+            job_free(j);
+        }
+        else if (job_is_stopped(j) && !job_get_flag(j, JOB_NOTIFIED))
+        {
+            /*
+               Notify the user about newly stopped jobs.
+            */
+            if (!job_get_flag(j, JOB_SKIP_NOTIFICATION))
+            {
+                format_job_info(j, _(L"stopped"), job_count);
+                found=1;
+            }
+            job_set_flag(j, JOB_NOTIFIED, 1);
+        }
+    }
+
+    if (found)
+        fflush(stdout);
+
+    /* Restore the exit status. */
+    proc_set_last_status(saved_status);
+
+    locked = false;
+
+    return found;
+}
+
+
+#ifdef HAVE__PROC_SELF_STAT
+
+/**
+   Maximum length of a /proc/[PID]/stat filename
+*/
+#define FN_SIZE 256
+
+/**
+   Get the CPU time for the specified process
+*/
+unsigned long proc_get_jiffies(process_t *p)
+{
+    wchar_t fn[FN_SIZE];
+
+    char state;
+    int pid, ppid, pgrp,
+        session, tty_nr, tpgid,
+        exit_signal, processor;
+
+    long int cutime, cstime, priority,
+         nice, placeholder, itrealvalue,
+         rss;
+    unsigned long int flags, minflt, cminflt,
+             majflt, cmajflt, utime,
+             stime, starttime, vsize,
+             rlim, startcode, endcode,
+             startstack, kstkesp, kstkeip,
+             signal, blocked, sigignore,
+             sigcatch, wchan, nswap, cnswap;
+    char comm[1024];
+
+    if (p->pid <= 0)
+        return 0;
+
+    swprintf(fn, FN_SIZE, L"/proc/%d/stat", p->pid);
+
+    FILE *f = wfopen(fn, "r");
+    if (!f)
+        return 0;
+
+    int count = fscanf(f,
+                       "%d %s %c "
+                       "%d %d %d "
+                       "%d %d %lu "
+
+                       "%lu %lu %lu "
+                       "%lu %lu %lu "
+                       "%ld %ld %ld "
+
+                       "%ld %ld %ld "
+                       "%lu %lu %ld "
+                       "%lu %lu %lu "
+
+                       "%lu %lu %lu "
+                       "%lu %lu %lu "
+                       "%lu %lu %lu "
+
+                       "%lu %d %d ",
+
+                       &pid, comm, &state,
+                       &ppid, &pgrp, &session,
+                       &tty_nr, &tpgid, &flags,
+
+                       &minflt, &cminflt, &majflt,
+                       &cmajflt, &utime, &stime,
+                       &cutime, &cstime, &priority,
+
+                       &nice, &placeholder, &itrealvalue,
+                       &starttime, &vsize, &rss,
+                       &rlim, &startcode, &endcode,
+
+                       &startstack, &kstkesp, &kstkeip,
+                       &signal, &blocked, &sigignore,
+                       &sigcatch, &wchan, &nswap,
+
+                       &cnswap, &exit_signal, &processor
+                      );
+
+    /*
+      Don't need to check exit status of fclose on read-only streams
+    */
+    fclose(f);
+    
+    if (count < 17)
+    {
+        return 0;
+    }
+
+    return utime+stime+cutime+cstime;
+
+}
+
+/**
+   Update the CPU time for all jobs
+*/
+void proc_update_jiffies()
+{
+    job_t* job;
+    process_t *p;
+    job_iterator_t j;
+
+    for (job = j.next(); job; job = j.next())
+    {
+        for (p=job->first_process; p; p=p->next)
+        {
+            gettimeofday(&p->last_time, 0);
+            p->last_jiffies = proc_get_jiffies(p);
+        }
+    }
+}
+
+
+#endif
+
+/**
+   Check if there are buffers associated with the job, and select on
+   them for a while if available.
+
+   \param j the job to test
+
+   \return 1 if buffers were available, zero otherwise
+*/
+static int select_try(job_t *j)
+{
+    fd_set fds;
+    int maxfd=-1;
+
+    FD_ZERO(&fds);
+
+    const io_chain_t chain = j->all_io_redirections();
+    for (size_t idx = 0; idx < chain.size(); idx++)
+    {
+        const io_data_t *io = chain.at(idx).get();
+        if (io->io_mode == IO_BUFFER)
+        {
+            CAST_INIT(const io_pipe_t *, io_pipe, io);
+            int fd = io_pipe->pipe_fd[0];
+//			fwprintf( stderr, L"fd %d on job %ls\n", fd, j->command );
+            FD_SET(fd, &fds);
+            maxfd = maxi(maxfd, fd);
+            debug(3, L"select_try on %d\n", fd);
+        }
+    }
+
+    if (maxfd >= 0)
+    {
+        int retval;
+        struct timeval tv;
+
+        tv.tv_sec=0;
+        tv.tv_usec=10000;
+
+        retval =select(maxfd+1, &fds, 0, 0, &tv);
+        if (retval == 0) {
+            debug(3, L"select_try hit timeout\n");
+        }
+        return retval > 0;
+    }
+
+    return -1;
+}
+
+/**
+   Read from descriptors until they are empty.
+
+   \param j the job to test
+*/
+static void read_try(job_t *j)
+{
+    io_buffer_t *buff = NULL;
+
+    /*
+      Find the last buffer, which is the one we want to read from
+    */
+    const io_chain_t chain = j->all_io_redirections();
+    for (size_t idx = 0; idx < chain.size(); idx++)
+    {
+        io_data_t *d = chain.at(idx).get();
+        if (d->io_mode == IO_BUFFER)
+        {
+            buff = static_cast<io_buffer_t *>(d);
+        }
+    }
+
+    if (buff)
+    {
+        debug(3, L"proc::read_try('%ls')\n", j->command_wcstr());
+        while (1)
+        {
+            char b[BUFFER_SIZE];
+            long l;
+
+            l=read_blocked(buff->pipe_fd[0],
+                           b, BUFFER_SIZE);
+            if (l==0)
+            {
+                break;
+            }
+            else if (l<0)
+            {
+                if (errno != EAGAIN)
+                {
+                    debug(1,
+                          _(L"An error occured while reading output from code block"));
+                    wperror(L"read_try");
+                }
+                break;
+            }
+            else
+            {
+                buff->out_buffer_append(b, l);
+            }
+        }
+    }
+}
+
+
+/**
+   Give ownership of the terminal to the specified job.
+
+   \param j The job to give the terminal to.
+
+   \param cont If this variable is set, we are giving back control to
+   a job that has previously been stopped. In that case, we need to
+   set the terminal attributes to those saved in the job.
+ */
+static bool terminal_give_to_job(job_t *j, int cont)
+{
+
+    if (tcsetpgrp(0, j->pgid))
+    {
+        debug(1,
+              _(L"Could not send job %d ('%ls') to foreground"),
+              j->job_id,
+              j->command_wcstr());
+        wperror(L"tcsetpgrp");
+        return false;
+    }
+
+    if (cont)
+    {
+        if (tcsetattr(0, TCSADRAIN, &j->tmodes))
+        {
+            debug(1,
+                  _(L"Could not send job %d ('%ls') to foreground"),
+                  j->job_id,
+                  j->command_wcstr());
+            wperror(L"tcsetattr");
+            return false;
+        }
+    }
+    return true;
+}
+
+/**
+   Returns control of the terminal to the shell, and saves the terminal
+   attribute state to the job, so that we can restore the terminal
+   ownership to the job at a later time .
+*/
+static int terminal_return_from_job(job_t *j)
+{
+
+    if (tcsetpgrp(0, getpgrp()))
+    {
+        debug(1, _(L"Could not return shell to foreground"));
+        wperror(L"tcsetpgrp");
+        return 0;
+    }
+
+    /*
+       Save jobs terminal modes.
+    */
+    if (tcgetattr(0, &j->tmodes))
+    {
+        debug(1, _(L"Could not return shell to foreground"));
+        wperror(L"tcgetattr");
+        return 0;
+    }
+
+    /* Disabling this per https://github.com/adityagodbole/fish-shell/commit/9d229cd18c3e5c25a8bd37e9ddd3b67ddc2d1b72
+       On Linux, 'cd . ; ftp' prevents you from typing into the ftp prompt
+       See https://github.com/fish-shell/fish-shell/issues/121
+    */
+#if 0
+    /*
+       Restore the shell's terminal modes.
+    */
+    if (tcsetattr(0, TCSADRAIN, &shell_modes))
+    {
+        debug(1, _(L"Could not return shell to foreground"));
+        wperror(L"tcsetattr");
+        return 0;
+    }
+#endif
+
+    return 1;
+}
+
+void job_continue(job_t *j, bool cont)
+{
+    /*
+      Put job first in the job list
+    */
+    job_promote(j);
+    job_set_flag(j, JOB_NOTIFIED, 0);
+
+    CHECK_BLOCK();
+
+    debug(4,
+          L"Continue job %d, gid %d (%ls), %ls, %ls",
+          j->job_id,
+          j->pgid,
+          j->command_wcstr(),
+          job_is_completed(j)?L"COMPLETED":L"UNCOMPLETED",
+          is_interactive?L"INTERACTIVE":L"NON-INTERACTIVE");
+
+    if (!job_is_completed(j))
+    {
+        if (job_get_flag(j, JOB_TERMINAL) && job_get_flag(j, JOB_FOREGROUND))
+        {
+            /* Put the job into the foreground. Hack: ensure that stdin is marked as blocking first (#176). */
+            make_fd_blocking(STDIN_FILENO);
+
+            signal_block();
+
+            bool ok = terminal_give_to_job(j, cont);
+
+            signal_unblock();
+
+            if (!ok)
+                return;
+        }
+
+        /*
+           Send the job a continue signal, if necessary.
+        */
+        if (cont)
+        {
+            process_t *p;
+
+            for (p=j->first_process; p; p=p->next)
+                p->stopped=0;
+
+            if (job_get_flag(j, JOB_CONTROL))
+            {
+                if (killpg(j->pgid, SIGCONT))
+                {
+                    wperror(L"killpg (SIGCONT)");
+                    return;
+                }
+            }
+            else
+            {
+                for (p=j->first_process; p; p=p->next)
+                {
+                    if (kill(p->pid, SIGCONT) < 0)
+                    {
+                        wperror(L"kill (SIGCONT)");
+                        return;
+                    }
+                }
+            }
+        }
+
+        if (job_get_flag(j, JOB_FOREGROUND))
+        {
+            /* Look for finished processes first, to avoid select() if it's already done. */
+            process_mark_finished_children(false);
+
+            /*
+               Wait for job to report.
+            */
+            while (! reader_exit_forced() && ! job_is_stopped(j) && ! job_is_completed(j))
+            {
+//					debug( 1, L"select_try()" );
+                switch (select_try(j))
+                {
+                    case 1:
+                    {
+                        read_try(j);
+                        process_mark_finished_children(false);
+                        break;
+                    }
+                    
+                    case 0:
+                    {
+                        /* No FDs are ready. Look for finished processes. */
+                        process_mark_finished_children(false);
+                        break;
+                    }
+
+                    case -1:
+                    {
+                        /*
+                          If there is no funky IO magic, we can use
+                          waitpid instead of handling child deaths
+                          through signals. This gives a rather large
+                          speed boost (A factor 3 startup time
+                          improvement on my 300 MHz machine) on
+                          short-lived jobs.
+                         
+                          This will return early if we get a signal,
+                          like SIGHUP.
+                        */
+                        process_mark_finished_children(true);
+                        break;
+                    }
+                }
+            }
+        }
+    }
+
+    if (job_get_flag(j, JOB_FOREGROUND))
+    {
+
+        if (job_is_completed(j))
+        {
+
+            // It's possible that the job will produce output and exit before we've even read from it.
+            // We'll eventually read the output, but it may be after we've executed subsequent calls
+            // This is why my prompt colors kept getting screwed up - the builtin echo calls
+            // were sometimes having their output combined with the set_color calls in the wrong order!
+            read_try(j);
+
+            process_t *p = j->first_process;
+            while (p->next)
+                p = p->next;
+
+            if (WIFEXITED(p->status) || WIFSIGNALED(p->status))
+            {
+                /*
+                   Mark process status only if we are in the foreground
+                   and the last process in a pipe, and it is not a short circuited builtin
+                */
+                if (p->pid)
+                {
+                    int status = proc_format_status(p->status);
+                    //wprintf(L"setting status %d for %ls\n", job_get_flag( j, JOB_NEGATE )?!status:status, j->command);
+                    proc_set_last_status(job_get_flag(j, JOB_NEGATE)?!status:status);
+                }
+            }
+        }
+
+        /* Put the shell back in the foreground. */
+        if (job_get_flag(j, JOB_TERMINAL) && job_get_flag(j, JOB_FOREGROUND))
+        {
+            int ok;
+
+            signal_block();
+
+            ok = terminal_return_from_job(j);
+
+            signal_unblock();
+
+            if (!ok)
+                return;
+
+        }
+    }
+
+}
+
+int proc_format_status(int status)
+{
+    if (WIFSIGNALED(status))
+    {
+        return 128+WTERMSIG(status);
+    }
+    else if (WIFEXITED(status))
+    {
+        return WEXITSTATUS(status);
+    }
+    return status;
+
+}
+
+
+void proc_sanity_check()
+{
+    job_t *j;
+    job_t *fg_job=0;
+
+    job_iterator_t jobs;
+    while ((j = jobs.next()))
+    {
+        process_t *p;
+
+        if (!job_get_flag(j, JOB_CONSTRUCTED))
+            continue;
+
+
+        validate_pointer(j->first_process,
+                         _(L"Process list pointer"),
+                         0);
+
+        /*
+          More than one foreground job?
+        */
+        if (job_get_flag(j, JOB_FOREGROUND) && !(job_is_stopped(j) || job_is_completed(j)))
+        {
+            if (fg_job != 0)
+            {
+                debug(0,
+                      _(L"More than one job in foreground: job 1: '%ls' job 2: '%ls'"),
+                      fg_job->command_wcstr(),
+                      j->command_wcstr());
+                sanity_lose();
+            }
+            fg_job = j;
+        }
+
+        p = j->first_process;
+        while (p)
+        {
+            /* Internal block nodes do not have argv - see #1545 */
+            bool null_ok = (p->type == INTERNAL_BLOCK_NODE);
+            validate_pointer(p->get_argv(), _(L"Process argument list"), null_ok);
+            validate_pointer(p->argv0(), _(L"Process name"), null_ok);
+            validate_pointer(p->next, _(L"Process list pointer"), true);
+
+            if ((p->stopped & (~0x00000001)) != 0)
+            {
+                debug(0,
+                      _(L"Job '%ls', process '%ls' has inconsistent state \'stopped\'=%d"),
+                      j->command_wcstr(),
+                      p->argv0(),
+                      p->stopped);
+                sanity_lose();
+            }
+
+            if ((p->completed & (~0x00000001)) != 0)
+            {
+                debug(0,
+                      _(L"Job '%ls', process '%ls' has inconsistent state \'completed\'=%d"),
+                      j->command_wcstr(),
+                      p->argv0(),
+                      p->completed);
+                sanity_lose();
+            }
+
+            p=p->next;
+        }
+
+    }
+}
+
+void proc_push_interactive(int value)
+{
+    ASSERT_IS_MAIN_THREAD();
+    int old = is_interactive;
+    interactive_stack.push_back(is_interactive);
+    is_interactive = value;
+    if (old != value)
+        signal_set_handlers();
+}
+
+void proc_pop_interactive()
+{
+    ASSERT_IS_MAIN_THREAD();
+    int old = is_interactive;
+    is_interactive= interactive_stack.back();
+    interactive_stack.pop_back();
+    if (is_interactive != old)
+        signal_set_handlers();
+}
diff --git a/src/proc.h b/src/proc.h
new file mode 100644
index 00000000..8dd66c7c
--- /dev/null
+++ b/src/proc.h
@@ -0,0 +1,603 @@
+/** \file proc.h
+
+    Prototypes for utilities for keeping track of jobs, processes and subshells, as
+  well as signal handling functions for tracking children. These
+  functions do not themselves launch new processes, the exec library
+  will call proc to create representations of the running jobs as
+  needed.
+
+*/
+
+#ifndef FISH_PROC_H
+#define FISH_PROC_H
+
+#include <signal.h>
+#include <sys/time.h>
+#include <list>
+#include <assert.h>                     // for assert
+#include <stddef.h>                     // for size_t
+#include <termios.h>                    // for pid_t, termios
+
+#include "config.h"                     // for HAVE__PROC_SELF_STAT
+#include "io.h"
+#include "common.h"
+#include "parse_tree.h"
+
+/**
+   The status code use when a command was not found
+*/
+#define STATUS_UNKNOWN_COMMAND 127
+
+/**
+   The status code use when an unknown error occured during execution of a command
+*/
+#define STATUS_NOT_EXECUTABLE 126
+
+/**
+   The status code use when an unknown error occured during execution of a command
+*/
+#define STATUS_EXEC_FAIL 125
+
+/**
+   The status code use when a wildcard had no matches
+*/
+#define STATUS_UNMATCHED_WILDCARD 124
+
+/**
+   The status code used for normal exit in a  builtin
+*/
+#define STATUS_BUILTIN_OK 0
+
+/**
+   The status code used for erroneous argument combinations in a builtin
+*/
+#define STATUS_BUILTIN_ERROR 1
+
+/**
+   Types of processes
+*/
+enum process_type_t
+{
+    /**
+       A regular external command
+    */
+    EXTERNAL,
+    /**
+       A builtin command
+    */
+    INTERNAL_BUILTIN,
+    /**
+       A shellscript function
+    */
+    INTERNAL_FUNCTION,
+
+    /** A block of commands, represented as a node */
+    INTERNAL_BLOCK_NODE,
+
+    /**
+       The exec builtin
+    */
+    INTERNAL_EXEC
+};
+
+enum
+{
+    JOB_CONTROL_ALL,
+    JOB_CONTROL_INTERACTIVE,
+    JOB_CONTROL_NONE,
+}
+;
+
+/**
+  A structure representing a single fish process. Contains variables
+  for tracking process state and the process argument
+  list. Actually, a fish process can be either a regular external
+  process, an internal builtin which may or may not spawn a fake IO
+  process during execution, a shellscript function or a block of
+  commands to be evaluated by calling eval. Lastly, this process can
+  be the result of an exec command. The role of this process_t is
+  determined by the type field, which can be one of EXTERNAL,
+  INTERNAL_BUILTIN, INTERNAL_FUNCTION, INTERNAL_EXEC.
+
+  The process_t contains information on how the process should be
+  started, such as command name and arguments, as well as runtime
+  information on the status of the actual physical process which
+  represents it. Shellscript functions, builtins and blocks of code
+  may all need to spawn an external process that handles the piping
+  and redirecting of IO for them.
+
+  If the process is of type EXTERNAL or INTERNAL_EXEC, argv is the
+  argument array and actual_cmd is the absolute path of the command
+  to execute.
+
+  If the process is of type INTERNAL_BUILTIN, argv is the argument
+  vector, and argv[0] is the name of the builtin command.
+
+  If the process is of type INTERNAL_FUNCTION, argv is the argument
+  vector, and argv[0] is the name of the shellscript function.
+
+*/
+class process_t
+{
+private:
+
+    null_terminated_array_t<wchar_t> argv_array;
+
+    /* narrow copy of argv0 so we don't have to convert after fork */
+    narrow_string_rep_t argv0_narrow;
+
+    io_chain_t process_io_chain;
+
+    /* No copying */
+    process_t(const process_t &rhs);
+    void operator=(const process_t &rhs);
+
+public:
+
+    process_t();
+    ~process_t();
+
+
+    /**
+      Type of process. Can be one of \c EXTERNAL, \c
+      INTERNAL_BUILTIN, \c INTERNAL_FUNCTION, \c INTERNAL_EXEC
+    */
+    enum process_type_t type;
+
+    /* For internal block processes only, the node offset of the block */
+    node_offset_t internal_block_node;
+
+    /** Sets argv */
+    void set_argv(const wcstring_list_t &argv)
+    {
+        argv_array.set(argv);
+        argv0_narrow.set(argv.empty() ? L"" : argv[0]);
+    }
+
+    /** Returns argv */
+    const wchar_t * const *get_argv(void) const
+    {
+        return argv_array.get();
+    }
+    const null_terminated_array_t<wchar_t> &get_argv_array(void) const
+    {
+        return argv_array;
+    }
+
+    /** Returns argv[idx] */
+    const wchar_t *argv(size_t idx) const
+    {
+        const wchar_t * const *argv = argv_array.get();
+        assert(argv != NULL);
+        return argv[idx];
+    }
+
+    /** Returns argv[0], or NULL */
+    const wchar_t *argv0(void) const
+    {
+        const wchar_t * const *argv = argv_array.get();
+        return argv ? argv[0] : NULL;
+    }
+
+    /** Returns argv[0] as a char * */
+    const char *argv0_cstr(void) const
+    {
+        return argv0_narrow.get();
+    }
+
+    /* IO chain getter and setter */
+    const io_chain_t &io_chain() const
+    {
+        return process_io_chain;
+    }
+
+    void set_io_chain(const io_chain_t &chain)
+    {
+        this->process_io_chain = chain;
+    }
+
+    /** actual command to pass to exec in case of EXTERNAL or INTERNAL_EXEC. */
+    wcstring actual_cmd;
+
+    /** process ID */
+    pid_t pid;
+
+    /** File descriptor that pipe output should bind to */
+    int pipe_write_fd;
+
+    /** File descriptor that the _next_ process pipe input should bind to */
+    int pipe_read_fd;
+
+    /** true if process has completed */
+    volatile int completed;
+
+    /** true if process has stopped */
+    volatile int stopped;
+
+    /** reported status value */
+    volatile int status;
+
+    /** Special flag to tell the evaluation function for count to print the help information */
+    int count_help_magic;
+
+    /** Next process in pipeline. We own this and we are responsible for deleting it. */
+    process_t *next;
+#ifdef HAVE__PROC_SELF_STAT
+    /** Last time of cpu time check */
+    struct timeval last_time;
+    /** Number of jiffies spent in process at last cpu time check */
+    unsigned long last_jiffies;
+#endif
+};
+
+/**
+  Constants for the flag variable in the job struct
+*/
+enum
+{
+    /** Whether the user has been told about stopped job */
+    JOB_NOTIFIED = 1 << 0,
+
+    /** Whether this job is in the foreground */
+    JOB_FOREGROUND = 1 << 1,
+
+    /**
+    Whether the specified job is completely constructed,
+    i.e. completely parsed, and every process in the job has been
+    forked, etc.
+    */
+    JOB_CONSTRUCTED = 1 << 2,
+
+    /** Whether the specified job is a part of a subshell, event handler or some other form of special job that should not be reported */
+    JOB_SKIP_NOTIFICATION = 1 << 3,
+
+    /** Whether the exit status should be negated. This flag can only be set by the not builtin. */
+    JOB_NEGATE = 1 << 4,
+
+    /** Whether the job is under job control  */
+    JOB_CONTROL = 1 << 5,
+
+    /** Whether the job wants to own the terminal when in the foreground  */
+    JOB_TERMINAL = 1 << 6
+};
+
+typedef int job_id_t;
+job_id_t acquire_job_id(void);
+void release_job_id(job_id_t jobid);
+
+/**
+    A struct represeting a job. A job is basically a pipeline of one
+    or more processes and a couple of flags.
+ */
+class job_t
+{
+    /**
+        The original command which led to the creation of this
+        job. It is used for displaying messages about job status
+        on the terminal.
+    */
+    wcstring command_str;
+
+    /* narrow copy so we don't have to convert after fork */
+    narrow_string_rep_t command_narrow;
+
+    /* The IO chain associated with the block */
+    const io_chain_t block_io;
+
+    /* No copying */
+    job_t(const job_t &rhs);
+    void operator=(const job_t &);
+
+public:
+
+    job_t(job_id_t jobid, const io_chain_t &bio);
+    ~job_t();
+
+    /** Returns whether the command is empty. */
+    bool command_is_empty() const
+    {
+        return command_str.empty();
+    }
+
+    /** Returns the command as a wchar_t *. */
+    const wchar_t *command_wcstr() const
+    {
+        return command_str.c_str();
+    }
+
+    /** Returns the command */
+    const wcstring &command() const
+    {
+        return command_str;
+    }
+
+    /** Returns the command as a char *. */
+    const char *command_cstr() const
+    {
+        return command_narrow.get();
+    }
+
+    /** Sets the command */
+    void set_command(const wcstring &cmd)
+    {
+        command_str = cmd;
+        command_narrow.set(cmd);
+    }
+
+    /**
+        A linked list of all the processes in this job. We are responsible for deleting this when we are deallocated.
+    */
+    process_t *first_process;
+
+    /**
+        process group ID for the process group that this job is
+        running in.
+    */
+    pid_t pgid;
+
+    /**
+        The saved terminal modes of this job. This needs to be
+        saved so that we can restore the terminal to the same
+        state after temporarily taking control over the terminal
+        when a job stops.
+    */
+    struct termios tmodes;
+
+    /**
+       The job id of the job. This is a small integer that is a
+       unique identifier of the job within this shell, and is
+       used e.g. in process expansion.
+    */
+    const job_id_t job_id;
+
+    /**
+       Bitset containing information about the job. A combination of the JOB_* constants.
+    */
+    unsigned int flags;
+
+    /* Returns the block IO redirections associated with the job. These are things like the IO redirections associated with the begin...end statement. */
+    const io_chain_t &block_io_chain() const
+    {
+        return this->block_io;
+    }
+
+    /* Fetch all the IO redirections associated with the job */
+    io_chain_t all_io_redirections() const;
+};
+
+/**
+  Whether we are running a subshell command
+*/
+extern int is_subshell;
+
+/**
+  Whether we are running a block of commands
+*/
+extern int is_block;
+
+/**
+  Whether we are reading from the keyboard right now
+*/
+int get_is_interactive(void);
+
+/**
+  Whether this shell is attached to the keyboard at all
+*/
+extern int is_interactive_session;
+
+/**
+  Whether we are a login shell
+*/
+extern int is_login;
+
+/**
+  Whether we are running an event handler
+*/
+extern int is_event;
+
+
+typedef std::list<job_t *> job_list_t;
+
+bool job_list_is_empty(void);
+
+/** A class to aid iteration over jobs list.
+    Note this is used from a signal handler, so it must be careful to not allocate memory.
+*/
+class job_iterator_t
+{
+    job_list_t * const job_list;
+    job_list_t::iterator current, end;
+public:
+
+    void reset(void);
+
+    job_t *next()
+    {
+        job_t *job = NULL;
+        if (current != end)
+        {
+            job = *current;
+            ++current;
+        }
+        return job;
+    }
+
+    job_iterator_t(job_list_t &jobs);
+    job_iterator_t();
+    size_t count() const;
+};
+
+/**
+   Whether a universal variable barrier roundtrip has already been
+   made for the currently executing command. Such a roundtrip only
+   needs to be done once on a given command, unless a universal
+   variable value is changed. Once this has been done, this variable
+   is set to 1, so that no more roundtrips need to be done.
+
+   Both setting it to one when it should be zero and the opposite may
+   cause concurrency bugs.
+*/
+bool get_proc_had_barrier();
+void set_proc_had_barrier(bool flag);
+
+/**
+   Pid of last process started in the background
+*/
+extern pid_t proc_last_bg_pid;
+
+/**
+   The current job control mode.
+
+   Must be one of JOB_CONTROL_ALL, JOB_CONTROL_INTERACTIVE and JOB_CONTROL_NONE
+*/
+extern int job_control_mode;
+
+/**
+   If this flag is set, fish will never fork or run execve. It is used
+   to put fish into a syntax verifier mode where fish tries to validate
+   the syntax of a file but doesn't actually do anything.
+  */
+extern int no_exec;
+
+/**
+   Add the specified flag to the bitset of flags for the specified job
+ */
+void job_set_flag(job_t *j, unsigned int flag, int set);
+
+/**
+   Returns one if the specified flag is set in the specified job, 0 otherwise.
+ */
+int job_get_flag(const job_t *j, unsigned int flag);
+
+/**
+   Sets the status of the last process to exit
+*/
+void proc_set_last_status(int s);
+
+/**
+   Returns the status of the last process to exit
+*/
+int proc_get_last_status();
+
+/**
+   Remove the specified job
+*/
+void job_free(job_t* j);
+
+/**
+   Promotes a job to the front of the job list.
+*/
+void job_promote(job_t *job);
+
+/**
+  Return the job with the specified job id.
+  If id is 0 or less, return the last job used.
+*/
+job_t *job_get(job_id_t id);
+
+/**
+  Return the job with the specified pid.
+*/
+job_t *job_get_from_pid(int pid);
+
+/**
+   Tests if the job is stopped
+*/
+int job_is_stopped(const job_t *j);
+
+/**
+   Tests if the job has completed, i.e. if the last process of the pipeline has ended.
+*/
+bool job_is_completed(const job_t *j);
+
+/**
+  Reassume a (possibly) stopped job. Put job j in the foreground.  If
+  cont is true, restore the saved terminal modes and send the
+  process group a SIGCONT signal to wake it up before we block.
+
+  \param j The job
+  \param cont Whether the function should wait for the job to complete before returning
+*/
+void job_continue(job_t *j, bool cont);
+
+/**
+   Notify the user about stopped or terminated jobs. Delete terminated
+   jobs from the job list.
+
+   \param interactive whether interactive jobs should be reaped as well
+*/
+int job_reap(bool interactive);
+
+/**
+   Signal handler for SIGCHLD. Mark any processes with relevant
+   information.
+*/
+void job_handle_signal(int signal, siginfo_t *info, void *con);
+
+/**
+   Send the specified signal to all processes in the specified job.
+*/
+int job_signal(job_t *j, int signal);
+
+/**
+   Mark a process as failed to execute (and therefore completed)
+*/
+void job_mark_process_as_failed(const job_t *job, process_t *p);
+
+#ifdef HAVE__PROC_SELF_STAT
+/**
+   Use the procfs filesystem to look up how many jiffies of cpu time
+   was used by this process. This function is only available on
+   systems with the procfs file entry 'stat', i.e. Linux.
+*/
+unsigned long proc_get_jiffies(process_t *p);
+
+/**
+   Update process time usage for all processes by calling the
+   proc_get_jiffies function for every process of every job.
+*/
+void proc_update_jiffies();
+
+#endif
+
+/**
+   Perform a set of simple sanity checks on the job list. This
+   includes making sure that only one job is in the foreground, that
+   every process is in a valid state, etc.
+*/
+void proc_sanity_check();
+
+/**
+   Send a process/job exit event notification. This function is a
+   convenience wrapper around event_fire().
+*/
+void proc_fire_event(const wchar_t *msg, int type, pid_t pid, int status);
+
+/**
+   Initializations
+*/
+void proc_init();
+
+/**
+   Clean up before exiting
+*/
+void proc_destroy();
+
+/**
+   Set new value for is_interactive flag, saving previous value. If
+   needed, update signal handlers.
+*/
+void proc_push_interactive(int value);
+
+/**
+   Set is_interactive flag to the previous value. If needed, update
+   signal handlers.
+*/
+void proc_pop_interactive();
+
+/**
+   Format an exit status code as returned by e.g. wait into a fish exit code number as accepted by proc_set_last_status.
+ */
+int proc_format_status(int status);
+
+#endif
diff --git a/src/reader.cpp b/src/reader.cpp
new file mode 100644
index 00000000..eaf5829f
--- /dev/null
+++ b/src/reader.cpp
@@ -0,0 +1,4358 @@
+/** \file reader.c
+
+Functions for reading data from stdin and passing to the
+parser. If stdin is a keyboard, it supplies a killring, history,
+syntax highlighting, tab-completion and various other interactive features.
+
+Internally the interactive mode functions rely in the functions of the
+input library to read individual characters of input.
+
+Token search is handled incrementally. Actual searches are only done
+on when searching backwards, since the previous results are saved. The
+last search position is remembered and a new search continues from the
+last search position. All search results are saved in the list
+'search_prev'. When the user searches forward, i.e. presses Alt-down,
+the list is consulted for previous search result, and subsequent
+backwards searches are also handled by consulting the list up until
+the end of the list is reached, at which point regular searching will
+commence.
+
+*/
+
+#include "config.h"
+#include <algorithm>
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <stdint.h>
+#include <string.h>
+#include <errno.h>
+#include <termios.h>
+#include <time.h>
+#include <sys/time.h>
+#include <unistd.h>
+#include <wctype.h>
+#include <stack>
+#include <pthread.h>
+
+#ifdef HAVE_SIGINFO_H
+#include <siginfo.h>
+#endif
+
+#ifdef HAVE_SYS_SELECT_H
+#include <sys/select.h>
+#endif
+
+#include <signal.h>
+#include <fcntl.h>
+#include <wchar.h>
+#include <assert.h>
+
+
+#include "fallback.h"
+#include "util.h"
+
+#include "wutil.h"
+#include "highlight.h"
+#include "reader.h"
+#include "proc.h"
+#include "parser.h"
+#include "complete.h"
+#include "history.h"
+#include "common.h"
+#include "sanity.h"
+#include "env.h"
+#include "exec.h"
+#include "expand.h"
+#include "tokenizer.h"
+#include "kill.h"
+#include "input_common.h"
+#include "input.h"
+#include "function.h"
+#include "output.h"
+#include "signal.h"
+#include "screen.h"
+#include "iothread.h"
+#include "intern.h"
+#include "parse_util.h"
+#include "parse_tree.h"
+#include "pager.h"
+#include "color.h"
+#include "event.h"
+
+/**
+   Maximum length of prefix string when printing completion
+   list. Longer prefixes will be ellipsized.
+*/
+#define PREFIX_MAX_LEN 9
+
+/**
+   A simple prompt for reading shell commands that does not rely on
+   fish specific commands, meaning it will work even if fish is not
+   installed. This is used by read_i.
+*/
+#define DEFAULT_PROMPT L"echo -n \"$USER@\"(hostname|cut -d . -f 1)' '(__fish_pwd)'> '"
+
+/**
+   The name of the function that prints the fish prompt
+ */
+#define LEFT_PROMPT_FUNCTION_NAME L"fish_prompt"
+
+/**
+   The name of the function that prints the fish right prompt (RPROMPT)
+ */
+#define RIGHT_PROMPT_FUNCTION_NAME L"fish_right_prompt"
+
+
+/* The name of the function for getting the input mode indicator */
+#define MODE_PROMPT_FUNCTION_NAME L"fish_mode_prompt"
+
+
+/**
+   The default title for the reader. This is used by reader_readline.
+*/
+#define DEFAULT_TITLE L"echo $_ \" \"; __fish_pwd"
+
+/**
+   The maximum number of characters to read from the keyboard without
+   repainting. Note that this readahead will only occur if new
+   characters are available for reading, fish will never block for
+   more input without repainting.
+*/
+#define READAHEAD_MAX 256
+
+/**
+   A mode for calling the reader_kill function. In this mode, the new
+   string is appended to the current contents of the kill buffer.
+ */
+#define KILL_APPEND 0
+/**
+   A mode for calling the reader_kill function. In this mode, the new
+   string is prepended to the current contents of the kill buffer.
+ */
+#define KILL_PREPEND 1
+
+/**
+   History search mode. This value means that no search is currently
+   performed.
+ */
+#define NO_SEARCH 0
+/**
+   History search mode. This value means that we are performing a line
+   history search.
+ */
+#define LINE_SEARCH 1
+/**
+   History search mode. This value means that we are performing a token
+   history search.
+ */
+#define TOKEN_SEARCH 2
+
+/**
+   History search mode. This value means we are searching backwards.
+ */
+#define SEARCH_BACKWARD 0
+/**
+   History search mode. This value means we are searching forwards.
+ */
+#define SEARCH_FORWARD 1
+
+/* Any time the contents of a buffer changes, we update the generation count. This allows for our background highlighting thread to notice it and skip doing work that it would otherwise have to do. This variable should really be of some kind of interlocked or atomic type that guarantees we're not reading stale cache values. With C++11 we should use atomics, but until then volatile should work as well, at least on x86.*/
+static volatile unsigned int s_generation_count;
+
+/* This pthreads generation count is set when an autosuggestion background thread starts up, so it can easily check if the work it is doing is no longer useful. */
+static pthread_key_t generation_count_key;
+
+static void set_command_line_and_position(editable_line_t *el, const wcstring &new_str, size_t pos);
+
+void editable_line_t::insert_string(const wcstring &str, size_t start, size_t len)
+{
+    // Clamp the range to something valid
+    size_t string_length = str.size();
+    start = mini(start, string_length);
+    len = mini(len, string_length - start);
+    this->text.insert(this->position, str, start, len);
+    this->position += len;
+}
+
+/**
+   A struct describing the state of the interactive reader. These
+   states can be stacked, in case reader_readline() calls are
+   nested. This happens when the 'read' builtin is used.
+*/
+class reader_data_t
+{
+public:
+
+    /** String containing the whole current commandline */
+    editable_line_t command_line;
+
+    /** String containing the autosuggestion */
+    wcstring autosuggestion;
+
+    /** Current pager */
+    pager_t pager;
+
+    /** Current page rendering */
+    page_rendering_t current_page_rendering;
+
+    /** Whether autosuggesting is allowed at all */
+    bool allow_autosuggestion;
+
+    /** When backspacing, we temporarily suppress autosuggestions */
+    bool suppress_autosuggestion;
+
+    /** Whether abbreviations are expanded */
+    bool expand_abbreviations;
+
+    /** The representation of the current screen contents */
+    screen_t screen;
+
+    /** The history */
+    history_t *history;
+
+    /**
+       String containing the current search item
+    */
+    wcstring search_buff;
+
+    /* History search */
+    history_search_t history_search;
+
+    /**
+       Saved position used by token history search
+    */
+    int token_history_pos;
+
+    /**
+       Saved search string for token history search. Not handled by command_line_changed.
+    */
+    wcstring token_history_buff;
+
+    /**
+       List for storing previous search results. Used to avoid duplicates.
+    */
+    wcstring_list_t search_prev;
+
+    /** The current position in search_prev */
+    size_t search_pos;
+
+    bool is_navigating_pager_contents() const
+    {
+        return this->pager.is_navigating_contents();
+    }
+
+    /* The line that is currently being edited. Typically the command line, but may be the search field */
+    editable_line_t *active_edit_line()
+    {
+        if (this->is_navigating_pager_contents() && this->pager.is_search_field_shown())
+        {
+            return &this->pager.search_field_line;
+        }
+        else
+        {
+            return &this->command_line;
+        }
+    }
+
+    /** Do what we need to do whenever our command line changes */
+    void command_line_changed(const editable_line_t *el);
+
+    /** Do what we need to do whenever our pager selection */
+    void pager_selection_changed();
+
+    /** Expand abbreviations at the current cursor position, minus backtrack_amt. */
+    bool expand_abbreviation_as_necessary(size_t cursor_backtrack);
+
+    /** Indicates whether a selection is currently active */
+    bool sel_active;
+
+    /** The position of the cursor, when selection was initiated. */
+    size_t sel_begin_pos;
+
+    /** The start position of the current selection, if one. */
+    size_t sel_start_pos;
+
+    /** The stop position of the current selection, if one. */
+    size_t sel_stop_pos;
+
+    /** Name of the current application */
+    wcstring app_name;
+
+    /** The prompt commands */
+    wcstring left_prompt;
+    wcstring right_prompt;
+
+    /** The output of the last evaluation of the prompt command */
+    wcstring left_prompt_buff;
+
+    /** The output of the last evaluation of the right prompt command */
+    wcstring right_prompt_buff;
+
+    /* Completion support */
+    wcstring cycle_command_line;
+    size_t cycle_cursor_pos;
+
+    /**
+       Color is the syntax highlighting for buff.  The format is that
+       color[i] is the classification (according to the enum in
+       highlight.h) of buff[i].
+    */
+    std::vector<highlight_spec_t> colors;
+
+    /** An array defining the block level at each character. */
+    std::vector<int> indents;
+
+    /**
+       Function for tab completion
+    */
+    complete_function_t complete_func;
+
+    /**
+       Function for syntax highlighting
+    */
+    highlight_function_t highlight_function;
+
+    /**
+       Function for testing if the string can be returned
+    */
+    parser_test_error_bits_t (*test_func)(const wchar_t *);
+
+    /**
+       When this is true, the reader will exit
+    */
+    bool end_loop;
+
+    /**
+       If this is true, exit reader even if there are running
+       jobs. This happens if we press e.g. ^D twice.
+    */
+    bool prev_end_loop;
+
+    /** The current contents of the top item in the kill ring.  */
+    wcstring kill_item;
+
+    /**
+       Pointer to previous reader_data
+    */
+    reader_data_t *next;
+
+    /**
+       This variable keeps state on if we are in search mode, and
+       if yes, what mode
+     */
+    int search_mode;
+
+    /**
+       Keep track of whether any internal code has done something
+       which is known to require a repaint.
+     */
+    bool repaint_needed;
+
+    /** Whether a screen reset is needed after a repaint. */
+    bool screen_reset_needed;
+
+    /** Whether the reader should exit on ^C. */
+    bool exit_on_interrupt;
+
+    /** Constructor */
+    reader_data_t() :
+        allow_autosuggestion(0),
+        suppress_autosuggestion(0),
+        expand_abbreviations(0),
+        history(0),
+        token_history_pos(0),
+        search_pos(0),
+        sel_active(0),
+        sel_begin_pos(0),
+        sel_start_pos(0),
+        sel_stop_pos(0),
+        cycle_cursor_pos(0),
+        complete_func(0),
+        highlight_function(0),
+        test_func(0),
+        end_loop(0),
+        prev_end_loop(0),
+        next(0),
+        search_mode(0),
+        repaint_needed(0),
+        screen_reset_needed(0),
+        exit_on_interrupt(0)
+    {
+    }
+};
+
+/* Sets the command line contents, without clearing the pager */
+static void reader_set_buffer_maintaining_pager(const wcstring &b, size_t pos);
+
+/* Clears the pager */
+static void clear_pager();
+
+/**
+   The current interactive reading context
+*/
+static reader_data_t *data=0;
+
+/**
+   This flag is set to true when fish is interactively reading from
+   stdin. It changes how a ^C is handled by the fish interrupt
+   handler.
+*/
+static int is_interactive_read;
+
+/**
+   Flag for ending non-interactive shell
+*/
+static int end_loop = 0;
+
+/** The stack containing names of files that are being parsed */
+static std::stack<const wchar_t *, std::vector<const wchar_t *> > current_filename;
+
+
+/**
+   Store the pid of the parent process, so the exit function knows whether it should reset the terminal or not.
+*/
+static pid_t original_pid;
+
+/**
+   This variable is set to true by the signal handler when ^C is pressed
+*/
+static volatile int interrupted=0;
+
+
+/*
+  Prototypes for a bunch of functions defined later on.
+*/
+
+static bool is_backslashed(const wcstring &str, size_t pos);
+static wchar_t unescaped_quote(const wcstring &str, size_t pos);
+
+/** Mode on startup, which we restore on exit */
+static struct termios terminal_mode_on_startup;
+
+/** Mode we use to execute programs */
+static struct termios terminal_mode_for_executing_programs;
+
+
+static void reader_super_highlight_me_plenty(int highlight_pos_adjust = 0, bool no_io = false);
+
+/**
+   Variable to keep track of forced exits - see \c reader_exit_forced();
+*/
+static int exit_forced;
+
+
+/**
+   Give up control of terminal
+*/
+static void term_donate()
+{
+    set_color(rgb_color_t::normal(), rgb_color_t::normal());
+
+    while (1)
+    {
+        if (tcsetattr(0, TCSANOW, &terminal_mode_for_executing_programs))
+        {
+            if (errno != EINTR)
+            {
+                debug(1, _(L"Could not set terminal mode for new job"));
+                wperror(L"tcsetattr");
+                break;
+            }
+        }
+        else
+            break;
+    }
+
+
+}
+
+
+/**
+   Update the cursor position
+*/
+static void update_buff_pos(editable_line_t *el, size_t buff_pos)
+{
+    el->position = buff_pos;
+    if (el == &data->command_line && data->sel_active)
+    {
+        if (data->sel_begin_pos <= buff_pos)
+        {
+            data->sel_start_pos = data->sel_begin_pos;
+            data->sel_stop_pos = buff_pos;
+        }
+        else
+        {
+            data->sel_start_pos = buff_pos;
+            data->sel_stop_pos = data->sel_begin_pos;
+        }
+    }
+}
+
+
+/**
+   Grab control of terminal
+*/
+static void term_steal()
+{
+
+    while (1)
+    {
+        if (tcsetattr(0,TCSANOW,&shell_modes))
+        {
+            if (errno != EINTR)
+            {
+                debug(1, _(L"Could not set terminal mode for shell"));
+                wperror(L"tcsetattr");
+                break;
+            }
+        }
+        else
+            break;
+    }
+
+    common_handle_winch(0);
+
+}
+
+int reader_exit_forced()
+{
+    return exit_forced;
+}
+
+/* Given a command line and an autosuggestion, return the string that gets shown to the user */
+wcstring combine_command_and_autosuggestion(const wcstring &cmdline, const wcstring &autosuggestion)
+{
+    // We want to compute the full line, containing the command line and the autosuggestion
+    // They may disagree on whether characters are uppercase or lowercase
+    // Here we do something funny: if the last token of the command line contains any uppercase characters, we use its case
+    // Otherwise we use the case of the autosuggestion
+    // This is an idea from https://github.com/fish-shell/fish-shell/issues/335
+    wcstring full_line;
+    if (autosuggestion.size() <= cmdline.size() || cmdline.empty())
+    {
+        // No or useless autosuggestion, or no command line
+        full_line = cmdline;
+    }
+    else if (string_prefixes_string(cmdline, autosuggestion))
+    {
+        // No case disagreements, or no extra characters in the autosuggestion
+        full_line = autosuggestion;
+    }
+    else
+    {
+        // We have an autosuggestion which is not a prefix of the command line, i.e. a case disagreement
+        // Decide whose case we want to use
+        const wchar_t *begin = NULL, *cmd = cmdline.c_str();
+        parse_util_token_extent(cmd, cmdline.size() - 1, &begin, NULL, NULL, NULL);
+        bool last_token_contains_uppercase = false;
+        if (begin)
+        {
+            const wchar_t *end = begin + wcslen(begin);
+            last_token_contains_uppercase = (std::find_if(begin, end, iswupper) != end);
+        }
+        if (! last_token_contains_uppercase)
+        {
+            // Use the autosuggestion's case
+            full_line = autosuggestion;
+        }
+        else
+        {
+            // Use the command line case for its characters, then append the remaining characters in the autosuggestion
+            // Note that we know that autosuggestion.size() > cmdline.size() due to the first test above
+            full_line = cmdline;
+            full_line.append(autosuggestion, cmdline.size(), autosuggestion.size() - cmdline.size());
+        }
+    }
+    return full_line;
+}
+
+/**
+   Repaint the entire commandline. This means reset and clear the
+   commandline, write the prompt, perform syntax highlighting, write
+   the commandline and move the cursor.
+*/
+static void reader_repaint()
+{
+    editable_line_t *cmd_line = &data->command_line;
+    // Update the indentation
+    data->indents = parse_util_compute_indents(cmd_line->text);
+
+    // Combine the command and autosuggestion into one string
+    wcstring full_line = combine_command_and_autosuggestion(cmd_line->text, data->autosuggestion);
+
+    size_t len = full_line.size();
+    if (len < 1)
+        len = 1;
+
+    std::vector<highlight_spec_t> colors = data->colors;
+    colors.resize(len, highlight_spec_autosuggestion);
+
+    if (data->sel_active)
+    {
+        highlight_spec_t selection_color = highlight_make_background(highlight_spec_selection);
+        for (size_t i = data->sel_start_pos; i <= std::min(len - 1, data->sel_stop_pos); i++)
+        {
+            colors[i] = selection_color;
+        }
+    }
+
+    std::vector<int> indents = data->indents;
+    indents.resize(len);
+
+    // Re-render our completions page if necessary
+    // We set the term size to 1 less than the true term height. This means we will always show the (bottom) line of the prompt.
+    data->pager.set_term_size(maxi(1, common_get_width()), maxi(1, common_get_height() - 1));
+    data->pager.update_rendering(&data->current_page_rendering);
+
+    bool focused_on_pager = data->active_edit_line() == &data->pager.search_field_line;
+    size_t cursor_position = focused_on_pager ? data->pager.cursor_position() : cmd_line->position;
+
+    s_write(&data->screen,
+            data->left_prompt_buff,
+            data->right_prompt_buff,
+            full_line,
+            cmd_line->size(),
+            &colors[0],
+            &indents[0],
+            cursor_position,
+            data->sel_start_pos,
+            data->sel_stop_pos,
+            data->current_page_rendering,
+            focused_on_pager);
+
+    data->repaint_needed = false;
+}
+
+/** Internal helper function for handling killing parts of text. */
+static void reader_kill(editable_line_t *el, size_t begin_idx, size_t length, int mode, int newv)
+{
+    const wchar_t *begin = el->text.c_str() + begin_idx;
+    if (newv)
+    {
+        data->kill_item = wcstring(begin, length);
+        kill_add(data->kill_item);
+    }
+    else
+    {
+        wcstring old = data->kill_item;
+        if (mode == KILL_APPEND)
+        {
+            data->kill_item.append(begin, length);
+        }
+        else
+        {
+            data->kill_item = wcstring(begin, length);
+            data->kill_item.append(old);
+        }
+
+
+        kill_replace(old, data->kill_item);
+    }
+
+    if (el->position > begin_idx)
+    {
+        /* Move the buff position back by the number of characters we deleted, but don't go past buff_pos */
+        size_t backtrack = mini(el->position - begin_idx, length);
+        update_buff_pos(el, el->position - backtrack);
+    }
+
+    el->text.erase(begin_idx, length);
+    data->command_line_changed(el);
+
+    reader_super_highlight_me_plenty();
+    reader_repaint();
+}
+
+
+/* This is called from a signal handler! */
+void reader_handle_int(int sig)
+{
+    if (!is_interactive_read)
+    {
+        parser_t::skip_all_blocks();
+    }
+
+    interrupted = 1;
+
+}
+
+const wchar_t *reader_current_filename()
+{
+    ASSERT_IS_MAIN_THREAD();
+    return current_filename.empty() ? NULL : current_filename.top();
+}
+
+
+void reader_push_current_filename(const wchar_t *fn)
+{
+    ASSERT_IS_MAIN_THREAD();
+    current_filename.push(intern(fn));
+}
+
+
+void reader_pop_current_filename()
+{
+    ASSERT_IS_MAIN_THREAD();
+    current_filename.pop();
+}
+
+
+/** Make sure buffers are large enough to hold the current string length */
+void reader_data_t::command_line_changed(const editable_line_t *el)
+{
+    ASSERT_IS_MAIN_THREAD();
+    if (el == &this->command_line)
+    {
+        size_t len = this->command_line.size();
+
+        /* When we grow colors, propagate the last color (if any), under the assumption that usually it will be correct. If it is, it avoids a repaint. */
+        highlight_spec_t last_color = colors.empty() ? highlight_spec_t() : colors.back();
+        colors.resize(len, last_color);
+
+        indents.resize(len);
+
+        /* Update the gen count */
+        s_generation_count++;
+    }
+    else if (el == &this->pager.search_field_line)
+    {
+        this->pager.refilter_completions();
+        this->pager_selection_changed();
+    }
+}
+
+void reader_data_t::pager_selection_changed()
+{
+    ASSERT_IS_MAIN_THREAD();
+
+    const completion_t *completion = this->pager.selected_completion(this->current_page_rendering);
+
+    /* Update the cursor and command line */
+    size_t cursor_pos = this->cycle_cursor_pos;
+    wcstring new_cmd_line;
+
+    if (completion == NULL)
+    {
+        new_cmd_line = this->cycle_command_line;
+    }
+    else
+    {
+        new_cmd_line = completion_apply_to_command_line(completion->completion, completion->flags, this->cycle_command_line, &cursor_pos, false);
+    }
+    reader_set_buffer_maintaining_pager(new_cmd_line, cursor_pos);
+
+    /* Since we just inserted a completion, don't immediately do a new autosuggestion */
+    this->suppress_autosuggestion = true;
+
+    /* Trigger repaint (see #765) */
+    reader_repaint_needed();
+}
+
+/* Expand abbreviations at the given cursor position. Does NOT inspect 'data'. */
+bool reader_expand_abbreviation_in_command(const wcstring &cmdline, size_t cursor_pos, wcstring *output)
+{
+    /* See if we are at "command position". Get the surrounding command substitution, and get the extent of the first token. */
+    const wchar_t * const buff = cmdline.c_str();
+    const wchar_t *cmdsub_begin = NULL, *cmdsub_end = NULL;
+    parse_util_cmdsubst_extent(buff, cursor_pos, &cmdsub_begin, &cmdsub_end);
+    assert(cmdsub_begin != NULL && cmdsub_begin >= buff);
+    assert(cmdsub_end != NULL && cmdsub_end >= cmdsub_begin);
+
+    /* Determine the offset of this command substitution */
+    const size_t subcmd_offset = cmdsub_begin - buff;
+
+    const wcstring subcmd = wcstring(cmdsub_begin, cmdsub_end - cmdsub_begin);
+    const size_t subcmd_cursor_pos = cursor_pos - subcmd_offset;
+
+    /* Parse this subcmd */
+    parse_node_tree_t parse_tree;
+    parse_tree_from_string(subcmd, parse_flag_continue_after_error | parse_flag_accept_incomplete_tokens, &parse_tree, NULL);
+
+    /* Look for plain statements where the cursor is at the end of the command */
+    const parse_node_t *matching_cmd_node = NULL;
+    const size_t len = parse_tree.size();
+    for (size_t i=0; i < len; i++)
+    {
+        const parse_node_t &node = parse_tree.at(i);
+
+        /* Only interested in plain statements with source */
+        if (node.type != symbol_plain_statement || ! node.has_source())
+            continue;
+
+        /* Skip decorated statements */
+        if (parse_tree.decoration_for_plain_statement(node) != parse_statement_decoration_none)
+            continue;
+
+        /* Get the command node. Skip it if we can't or it has no source */
+        const parse_node_t *cmd_node = parse_tree.get_child(node, 0, parse_token_type_string);
+        if (cmd_node == NULL || ! cmd_node->has_source())
+            continue;
+
+        /* Now see if its source range contains our cursor, including at the end */
+        if (subcmd_cursor_pos >= cmd_node->source_start && subcmd_cursor_pos <= cmd_node->source_start + cmd_node->source_length)
+        {
+            /* Success! */
+            matching_cmd_node = cmd_node;
+            break;
+        }
+    }
+
+    /* Now if we found a command node, expand it */
+    bool result = false;
+    if (matching_cmd_node != NULL)
+    {
+        assert(matching_cmd_node->type == parse_token_type_string);
+        const wcstring token = matching_cmd_node->get_source(subcmd);
+        wcstring abbreviation;
+        if (expand_abbreviation(token, &abbreviation))
+        {
+            /* There was an abbreviation! Replace the token in the full command. Maintain the relative position of the cursor. */
+            if (output != NULL)
+            {
+                output->assign(cmdline);
+                output->replace(subcmd_offset + matching_cmd_node->source_start, matching_cmd_node->source_length, abbreviation);
+            }
+            result = true;
+        }
+    }
+    return result;
+}
+
+/* Expand abbreviations at the current cursor position, minus the given  cursor backtrack. This may change the command line but does NOT repaint it. This is to allow the caller to coalesce repaints. */
+bool reader_data_t::expand_abbreviation_as_necessary(size_t cursor_backtrack)
+{
+    bool result = false;
+    editable_line_t *el = data->active_edit_line();
+    if (this->expand_abbreviations && el == &data->command_line)
+    {
+        /* Try expanding abbreviations */
+        wcstring new_cmdline;
+        size_t cursor_pos = el->position - mini(el->position, cursor_backtrack);
+        if (reader_expand_abbreviation_in_command(el->text, cursor_pos, &new_cmdline))
+        {
+            /* We expanded an abbreviation! The cursor moves by the difference in the command line lengths. */
+            size_t new_buff_pos = el->position + new_cmdline.size() - el->text.size();
+
+
+            el->text.swap(new_cmdline);
+            update_buff_pos(el, new_buff_pos);
+            data->command_line_changed(el);
+            result = true;
+        }
+    }
+    return result;
+}
+
+/** Sorts and remove any duplicate completions in the list. */
+static void sort_and_make_unique(std::vector<completion_t> &l)
+{
+    sort(l.begin(), l.end(), completion_t::is_naturally_less_than);
+    l.erase(std::unique(l.begin(), l.end(), completion_t::is_alphabetically_equal_to), l.end());
+}
+
+
+void reader_reset_interrupted()
+{
+    interrupted = 0;
+}
+
+int reader_interrupted()
+{
+    int res = interrupted;
+    if (res)
+    {
+        interrupted=0;
+    }
+    return res;
+}
+
+int reader_reading_interrupted()
+{
+    int res = reader_interrupted();
+    if (res && data && data->exit_on_interrupt)
+    {
+        reader_exit(1, 0);
+        parser_t::skip_all_blocks();
+        // We handled the interrupt ourselves, our caller doesn't need to
+        // handle it.
+        return 0;
+    }
+    return res;
+}
+
+bool reader_thread_job_is_stale()
+{
+    ASSERT_IS_BACKGROUND_THREAD();
+    return (void*)(uintptr_t) s_generation_count != pthread_getspecific(generation_count_key);
+}
+
+void reader_write_title(const wcstring &cmd)
+{
+    const env_var_t term_str = env_get_string(L"TERM");
+
+    /*
+      This is a pretty lame heuristic for detecting terminals that do
+      not support setting the title. If we recognise the terminal name
+      as that of a virtual terminal, we assume it supports setting the
+      title. If we recognise it as that of a console, we assume it
+      does not support setting the title. Otherwise we check the
+      ttyname and see if we believe it is a virtual terminal.
+
+      One situation in which this breaks down is with screen, since
+      screen supports setting the terminal title if the underlying
+      terminal does so, but will print garbage on terminals that
+      don't. Since we can't see the underlying terminal below screen
+      there is no way to fix this.
+    */
+    if (term_str.missing())
+        return;
+
+    const wchar_t *term = term_str.c_str();
+    bool recognized = false;
+    recognized = recognized || contains(term, L"xterm", L"screen", L"nxterm", L"rxvt");
+    recognized = recognized || ! wcsncmp(term, L"xterm-", wcslen(L"xterm-"));
+    recognized = recognized || ! wcsncmp(term, L"screen-", wcslen(L"screen-"));
+
+    if (! recognized)
+    {
+        char *n = ttyname(STDIN_FILENO);
+
+        if (contains(term, L"linux"))
+        {
+            return;
+        }
+
+        if (contains(term, L"dumb"))
+            return;
+
+        if (strstr(n, "tty") || strstr(n, "/vc/"))
+            return;
+    }
+
+    wcstring fish_title_command = DEFAULT_TITLE;
+    if (function_exists(L"fish_title"))
+    {
+        fish_title_command = L"fish_title";
+        if (! cmd.empty())
+        {
+            fish_title_command.append(L" ");
+            fish_title_command.append(parse_util_escape_string_with_quote(cmd, L'\0'));
+        }
+    }
+
+    wcstring_list_t lst;
+
+    proc_push_interactive(0);
+    if (exec_subshell(fish_title_command, lst, false /* do not apply exit status */) != -1)
+    {
+        if (! lst.empty())
+        {
+            writestr(L"\x1b]0;");
+            for (size_t i=0; i<lst.size(); i++)
+            {
+                writestr(lst.at(i).c_str());
+            }
+            writestr(L"\7");
+        }
+    }
+    proc_pop_interactive();
+    set_color(rgb_color_t::reset(), rgb_color_t::reset());
+}
+
+/**
+   Reexecute the prompt command. The output is inserted into data->prompt_buff.
+*/
+static void exec_prompt()
+{
+    /* Clear existing prompts */
+    data->left_prompt_buff.clear();
+    data->right_prompt_buff.clear();
+
+    /* Do not allow the exit status of the prompts to leak through */
+    const bool apply_exit_status = false;
+
+    /* If we have any prompts, they must be run non-interactively */
+    if (data->left_prompt.size() || data->right_prompt.size())
+    {
+        proc_push_interactive(0);
+
+        // Prepend any mode indicator to the left prompt (#1988)
+        if (function_exists(MODE_PROMPT_FUNCTION_NAME))
+        {
+            wcstring_list_t mode_indicator_list;
+            exec_subshell(MODE_PROMPT_FUNCTION_NAME, mode_indicator_list, apply_exit_status);
+            // We do not support multiple lines in the mode indicator, so just concatenate all of them
+            for (size_t i = 0; i < mode_indicator_list.size(); i++)
+            {
+                data->left_prompt_buff += mode_indicator_list.at(i);
+            }
+        }
+
+        if (! data->left_prompt.empty())
+        {
+            wcstring_list_t prompt_list;
+            // ignore return status
+            exec_subshell(data->left_prompt, prompt_list, apply_exit_status);
+            for (size_t i = 0; i < prompt_list.size(); i++)
+            {
+                if (i > 0) data->left_prompt_buff += L'\n';
+                data->left_prompt_buff += prompt_list.at(i);
+            }
+        }
+
+        if (! data->right_prompt.empty())
+        {
+            wcstring_list_t prompt_list;
+            // status is ignored
+            exec_subshell(data->right_prompt, prompt_list, apply_exit_status);
+            for (size_t i = 0; i < prompt_list.size(); i++)
+            {
+                // Right prompt does not support multiple lines, so just concatenate all of them
+                data->right_prompt_buff += prompt_list.at(i);
+            }
+        }
+
+        proc_pop_interactive();
+    }
+
+    /* Write the screen title */
+    reader_write_title(L"");
+}
+
+void reader_init()
+{
+    VOMIT_ON_FAILURE(pthread_key_create(&generation_count_key, NULL));
+
+    /* Save the initial terminal mode */
+    tcgetattr(STDIN_FILENO, &terminal_mode_on_startup);
+
+    /* Set the mode used for program execution, initialized to the current mode */
+    memcpy(&terminal_mode_for_executing_programs, &terminal_mode_on_startup, sizeof terminal_mode_for_executing_programs);
+    terminal_mode_for_executing_programs.c_iflag &= ~IXON;     /* disable flow control */
+    terminal_mode_for_executing_programs.c_iflag &= ~IXOFF;    /* disable flow control */
+
+    /* Set the mode used for the terminal, initialized to the current mode */
+    memcpy(&shell_modes, &terminal_mode_on_startup, sizeof shell_modes);
+    shell_modes.c_lflag &= ~ICANON;   /* turn off canonical mode */
+    shell_modes.c_lflag &= ~ECHO;     /* turn off echo mode */
+    shell_modes.c_iflag &= ~IXON;     /* disable flow control */
+    shell_modes.c_iflag &= ~IXOFF;    /* disable flow control */
+    shell_modes.c_cc[VMIN]=1;
+    shell_modes.c_cc[VTIME]=0;
+
+#if defined(_POSIX_VDISABLE)
+    // PCA disable VDSUSP (typically control-Y), which is a funny job control
+    // function available only on OS X and BSD systems
+    // This lets us use control-Y for yank instead
+#ifdef VDSUSP
+    shell_modes.c_cc[VDSUSP] = _POSIX_VDISABLE;
+#endif
+#endif
+}
+
+
+void reader_destroy()
+{
+    pthread_key_delete(generation_count_key);
+}
+
+void restore_term_mode()
+{
+    // Restore the term mode if we own the terminal
+    // It's important we do this before restore_foreground_process_group, otherwise we won't think we own the terminal
+    if (getpid() == tcgetpgrp(STDIN_FILENO))
+    {
+        tcsetattr(STDIN_FILENO, TCSANOW, &terminal_mode_on_startup);
+    }
+}
+
+void reader_exit(int do_exit, int forced)
+{
+    if (data)
+        data->end_loop=do_exit;
+    end_loop=do_exit;
+    if (forced)
+        exit_forced = 1;
+
+}
+
+void reader_repaint_needed()
+{
+    if (data)
+    {
+        data->repaint_needed = true;
+    }
+}
+
+void reader_repaint_if_needed()
+{
+    if (data == NULL)
+        return;
+
+    bool needs_reset = data->screen_reset_needed;
+    bool needs_repaint = needs_reset || data->repaint_needed;
+
+    if (needs_reset)
+    {
+        exec_prompt();
+        s_reset(&data->screen, screen_reset_current_line_and_prompt);
+        data->screen_reset_needed = false;
+    }
+
+    if (needs_repaint)
+    {
+        reader_repaint();
+        /* reader_repaint clears repaint_needed */
+    }
+}
+
+static void reader_repaint_if_needed_one_arg(void * unused)
+{
+    reader_repaint_if_needed();
+}
+
+void reader_react_to_color_change()
+{
+    if (! data)
+        return;
+
+    if (! data->repaint_needed || ! data->screen_reset_needed)
+    {
+        data->repaint_needed = true;
+        data->screen_reset_needed = true;
+        input_common_add_callback(reader_repaint_if_needed_one_arg, NULL);
+    }
+}
+
+
+/* Indicates if the given command char ends paging */
+static bool command_ends_paging(wchar_t c, bool focused_on_search_field)
+{
+    switch (c)
+    {
+            /* These commands always end paging */
+        case R_HISTORY_SEARCH_BACKWARD:
+        case R_HISTORY_SEARCH_FORWARD:
+        case R_HISTORY_TOKEN_SEARCH_BACKWARD:
+        case R_HISTORY_TOKEN_SEARCH_FORWARD:
+        case R_ACCEPT_AUTOSUGGESTION:
+        case R_CANCEL:
+            return true;
+
+            /* These commands never do */
+        case R_COMPLETE:
+        case R_COMPLETE_AND_SEARCH:
+        case R_BACKWARD_CHAR:
+        case R_FORWARD_CHAR:
+        case R_UP_LINE:
+        case R_DOWN_LINE:
+        case R_NULL:
+        case R_REPAINT:
+        case R_SUPPRESS_AUTOSUGGESTION:
+        case R_BEGINNING_OF_HISTORY:
+        case R_END_OF_HISTORY:
+        default:
+            return false;
+
+            /* R_EXECUTE does end paging, but only executes if it was not paging. So it's handled specially */
+        case R_EXECUTE:
+            return false;
+
+            /* These commands operate on the search field if that's where the focus is */
+        case R_BEGINNING_OF_LINE:
+        case R_END_OF_LINE:
+        case R_FORWARD_WORD:
+        case R_BACKWARD_WORD:
+        case R_FORWARD_BIGWORD:
+        case R_BACKWARD_BIGWORD:
+        case R_DELETE_CHAR:
+        case R_BACKWARD_DELETE_CHAR:
+        case R_KILL_LINE:
+        case R_YANK:
+        case R_YANK_POP:
+        case R_BACKWARD_KILL_LINE:
+        case R_KILL_WHOLE_LINE:
+        case R_KILL_WORD:
+        case R_KILL_BIGWORD:
+        case R_BACKWARD_KILL_WORD:
+        case R_BACKWARD_KILL_PATH_COMPONENT:
+        case R_BACKWARD_KILL_BIGWORD:
+        case R_SELF_INSERT:
+        case R_TRANSPOSE_CHARS:
+        case R_TRANSPOSE_WORDS:
+        case R_UPCASE_WORD:
+        case R_DOWNCASE_WORD:
+        case R_CAPITALIZE_WORD:
+        case R_VI_ARG_DIGIT:
+        case R_VI_DELETE_TO:
+        case R_BEGINNING_OF_BUFFER:
+        case R_END_OF_BUFFER:
+            return ! focused_on_search_field;
+    }
+}
+
+/**
+   Remove the previous character in the character buffer and on the
+   screen using syntax highlighting, etc.
+*/
+static void remove_backward()
+{
+    editable_line_t *el = data->active_edit_line();
+
+    if (el->position <= 0)
+        return;
+
+    /* Fake composed character sequences by continuing to delete until we delete a character of width at least 1. */
+    int width;
+    do
+    {
+        update_buff_pos(el, el->position - 1);
+        width = fish_wcwidth(el->text.at(el->position));
+        el->text.erase(el->position, 1);
+    }
+    while (width == 0 && el->position > 0);
+    data->command_line_changed(el);
+    data->suppress_autosuggestion = true;
+
+    reader_super_highlight_me_plenty();
+
+    reader_repaint_needed();
+}
+
+
+/**
+   Insert the characters of the string into the command line buffer
+   and print them to the screen using syntax highlighting, etc.
+   Optionally also expand abbreviations, after space characters.
+   Returns true if the string changed.
+*/
+static bool insert_string(editable_line_t *el, const wcstring &str, bool allow_expand_abbreviations = false)
+{
+    size_t len = str.size();
+    if (len == 0)
+        return false;
+    
+    /* Start inserting. If we are expanding abbreviations, we have to do this after every space (see #1434), so look for spaces. We try to do this efficiently (rather than the simpler character at a time) to avoid expensive work in command_line_changed() */
+    size_t cursor = 0;
+    while (cursor < len)
+    {
+        /* Determine the position of the next expansion-triggering char (possibly none), and the end of the range we wish to insert */
+        const wchar_t *expansion_triggering_chars = L" ;|&^><";
+        size_t char_triggering_expansion_pos = allow_expand_abbreviations ? str.find_first_of(expansion_triggering_chars, cursor) : wcstring::npos;
+        bool has_expansion_triggering_char = (char_triggering_expansion_pos != wcstring::npos);
+        size_t range_end = (has_expansion_triggering_char ? char_triggering_expansion_pos + 1 : len);
+        
+        /* Insert from the cursor up to but not including the range end */
+        assert(range_end > cursor);
+        el->insert_string(str, cursor, range_end - cursor);
+        
+        update_buff_pos(el, el->position);
+        data->command_line_changed(el);
+        
+        /* If we got an expansion trigger, then the last character we inserted was it (i.e. was a space). Expand abbreviations. */
+        if (has_expansion_triggering_char && allow_expand_abbreviations)
+        {
+            assert(range_end > 0);
+            assert(wcschr(expansion_triggering_chars, str.at(range_end - 1)));
+            data->expand_abbreviation_as_necessary(1);
+        }
+        cursor = range_end;
+    }
+    
+    if (el == &data->command_line)
+    {
+        data->suppress_autosuggestion = false;
+        
+        /* Syntax highlight. Note we must have that buff_pos > 0 because we just added something nonzero to its length  */
+        assert(el->position > 0);
+        reader_super_highlight_me_plenty(-1);
+    }
+    
+    reader_repaint();
+
+    return true;
+}
+
+/**
+   Insert the character into the command line buffer and print it to
+   the screen using syntax highlighting, etc.
+*/
+static bool insert_char(editable_line_t *el, wchar_t c, bool allow_expand_abbreviations = false)
+{
+    return insert_string(el, wcstring(1, c), allow_expand_abbreviations);
+}
+
+
+/**
+   Insert the string in the given command line at the given cursor
+   position. The function checks if the string is quoted or not and
+   correctly escapes the string.
+   \param val the string to insert
+   \param flags A union of all flags describing the completion to insert. See the completion_t struct for more information on possible values.
+   \param command_line The command line into which we will insert
+   \param inout_cursor_pos On input, the location of the cursor within the command line. On output, the new desired position.
+   \param append_only Whether we can only append to the command line, or also modify previous characters. This is used to determine whether we go inside a trailing quote.
+   \return The completed string
+*/
+wcstring completion_apply_to_command_line(const wcstring &val_str, complete_flags_t flags, const wcstring &command_line, size_t *inout_cursor_pos, bool append_only)
+{
+    const wchar_t *val = val_str.c_str();
+    bool add_space = !(flags & COMPLETE_NO_SPACE);
+    bool do_replace = !!(flags & COMPLETE_REPLACES_TOKEN);
+    bool do_escape = !(flags & COMPLETE_DONT_ESCAPE);
+
+    const size_t cursor_pos = *inout_cursor_pos;
+    bool back_into_trailing_quote = false;
+
+    if (do_replace)
+    {
+        size_t move_cursor;
+        const wchar_t *begin, *end;
+
+        const wchar_t *buff = command_line.c_str();
+        parse_util_token_extent(buff, cursor_pos, &begin, 0, 0, 0);
+        end = buff + cursor_pos;
+
+        wcstring sb(buff, begin - buff);
+
+        if (do_escape)
+        {
+            /* Respect COMPLETE_DONT_ESCAPE_TILDES */
+            bool no_tilde = !!(flags & COMPLETE_DONT_ESCAPE_TILDES);
+            wcstring escaped = escape(val, ESCAPE_ALL | ESCAPE_NO_QUOTED | (no_tilde ? ESCAPE_NO_TILDE : 0));
+            sb.append(escaped);
+            move_cursor = escaped.size();
+        }
+        else
+        {
+            sb.append(val);
+            move_cursor = wcslen(val);
+        }
+
+
+        if (add_space)
+        {
+            sb.append(L" ");
+            move_cursor += 1;
+        }
+        sb.append(end);
+
+        size_t new_cursor_pos = (begin - buff) + move_cursor;
+        *inout_cursor_pos = new_cursor_pos;
+        return sb;
+    }
+    else
+    {
+        wchar_t quote = L'\0';
+        wcstring replaced;
+        if (do_escape)
+        {
+            /* Note that we ignore COMPLETE_DONT_ESCAPE_TILDES here. We get away with this because unexpand_tildes only operates on completions that have COMPLETE_REPLACES_TOKEN set, but we ought to respect them */
+            parse_util_get_parameter_info(command_line, cursor_pos, &quote, NULL, NULL);
+
+            /* If the token is reported as unquoted, but ends with a (unescaped) quote, and we can modify the command line, then delete the trailing quote so that we can insert within the quotes instead of after them. See https://github.com/fish-shell/fish-shell/issues/552 */
+            if (quote == L'\0' && ! append_only && cursor_pos > 0)
+            {
+                /* The entire token is reported as unquoted...see if the last character is an unescaped quote */
+                wchar_t trailing_quote = unescaped_quote(command_line, cursor_pos - 1);
+                if (trailing_quote != L'\0')
+                {
+                    quote = trailing_quote;
+                    back_into_trailing_quote = true;
+                }
+            }
+
+            replaced = parse_util_escape_string_with_quote(val_str, quote);
+        }
+        else
+        {
+            replaced = val;
+        }
+
+        size_t insertion_point = cursor_pos;
+        if (back_into_trailing_quote)
+        {
+            /* Move the character back one so we enter the terminal quote */
+            assert(insertion_point > 0);
+            insertion_point--;
+        }
+
+        /* Perform the insertion and compute the new location */
+        wcstring result = command_line;
+        result.insert(insertion_point, replaced);
+        size_t new_cursor_pos = insertion_point + replaced.size() + (back_into_trailing_quote ? 1 : 0);
+        if (add_space)
+        {
+            if (quote != L'\0' && unescaped_quote(command_line, insertion_point) != quote)
+            {
+                /* This is a quoted parameter, first print a quote */
+                result.insert(new_cursor_pos++, wcstring(&quote, 1));
+            }
+            result.insert(new_cursor_pos++, L" ");
+        }
+        *inout_cursor_pos = new_cursor_pos;
+        return result;
+    }
+}
+
+/**
+   Insert the string at the current cursor position. The function
+   checks if the string is quoted or not and correctly escapes the
+   string.
+
+   \param val the string to insert
+   \param flags A union of all flags describing the completion to insert. See the completion_t struct for more information on possible values.
+
+*/
+static void completion_insert(const wchar_t *val, complete_flags_t flags)
+{
+    editable_line_t *el = data->active_edit_line();
+    size_t cursor = el->position;
+    wcstring new_command_line = completion_apply_to_command_line(val, flags, el->text, &cursor, false /* not append only */);
+    reader_set_buffer_maintaining_pager(new_command_line, cursor);
+
+    /* Since we just inserted a completion, don't immediately do a new autosuggestion */
+    data->suppress_autosuggestion = true;
+}
+
+struct autosuggestion_context_t
+{
+    wcstring search_string;
+    wcstring autosuggestion;
+    size_t cursor_pos;
+    history_search_t searcher;
+    file_detection_context_t detector;
+    const wcstring working_directory;
+    const env_vars_snapshot_t vars;
+    const unsigned int generation_count;
+
+    autosuggestion_context_t(history_t *history, const wcstring &term, size_t pos) :
+        search_string(term),
+        cursor_pos(pos),
+        searcher(*history, term, HISTORY_SEARCH_TYPE_PREFIX),
+        detector(history),
+        working_directory(env_get_pwd_slash()),
+        vars(env_vars_snapshot_t::highlighting_keys),
+        generation_count(s_generation_count)
+    {
+    }
+
+    /* The function run in the background thread to determine an autosuggestion */
+    int threaded_autosuggest(void)
+    {
+        ASSERT_IS_BACKGROUND_THREAD();
+
+        /* If the main thread has moved on, skip all the work */
+        if (generation_count != s_generation_count)
+        {
+            return 0;
+        }
+
+        VOMIT_ON_FAILURE(pthread_setspecific(generation_count_key, (void*)(uintptr_t) generation_count));
+
+        /* Let's make sure we aren't using the empty string */
+        if (search_string.empty())
+        {
+            return 0;
+        }
+
+        while (! reader_thread_job_is_stale() && searcher.go_backwards())
+        {
+            history_item_t item = searcher.current_item();
+
+            /* Skip items with newlines because they make terrible autosuggestions */
+            if (item.str().find('\n') != wcstring::npos)
+                continue;
+
+            if (autosuggest_validate_from_history(item, detector, working_directory, vars))
+            {
+                /* The command autosuggestion was handled specially, so we're done */
+                this->autosuggestion = searcher.current_string();
+                return 1;
+            }
+        }
+
+        /* Maybe cancel here */
+        if (reader_thread_job_is_stale())
+            return 0;
+
+        /* Try handling a special command like cd */
+        wcstring special_suggestion;
+        if (autosuggest_suggest_special(search_string, working_directory, special_suggestion))
+        {
+            this->autosuggestion = special_suggestion;
+            return 1;
+        }
+
+        /* Maybe cancel here */
+        if (reader_thread_job_is_stale())
+            return 0;
+
+        // Here we do something a little funny
+        // If the line ends with a space, and the cursor is not at the end,
+        // don't use completion autosuggestions. It ends up being pretty weird seeing stuff get spammed on the right
+        // while you go back to edit a line
+        const wchar_t last_char = search_string.at(search_string.size() - 1);
+        const bool cursor_at_end = (this->cursor_pos == search_string.size());
+        if (! cursor_at_end && iswspace(last_char))
+            return 0;
+
+        /* On the other hand, if the line ends with a quote, don't go dumping stuff after the quote */
+        if (wcschr(L"'\"", last_char) && cursor_at_end)
+            return 0;
+
+        /* Try normal completions */
+        std::vector<completion_t> completions;
+        complete(search_string, completions, COMPLETION_REQUEST_AUTOSUGGESTION);
+        if (! completions.empty())
+        {
+            const completion_t &comp = completions.at(0);
+            size_t cursor = this->cursor_pos;
+            this->autosuggestion = completion_apply_to_command_line(comp.completion, comp.flags, this->search_string, &cursor, true /* append only */);
+            return 1;
+        }
+
+        return 0;
+    }
+};
+
+static int threaded_autosuggest(autosuggestion_context_t *ctx)
+{
+    return ctx->threaded_autosuggest();
+}
+
+static bool can_autosuggest(void)
+{
+    /* We autosuggest if suppress_autosuggestion is not set, if we're not doing a history search, and our command line contains a non-whitespace character. */
+    const editable_line_t *el = data->active_edit_line();
+    const wchar_t *whitespace = L" \t\r\n\v";
+    return ! data->suppress_autosuggestion &&
+           data->history_search.is_at_end() &&
+           el == &data->command_line &&
+           el->text.find_first_not_of(whitespace) != wcstring::npos;
+}
+
+static void autosuggest_completed(autosuggestion_context_t *ctx, int result)
+{
+    if (result &&
+            can_autosuggest() &&
+            ctx->search_string == data->command_line.text &&
+            string_prefixes_string_case_insensitive(ctx->search_string, ctx->autosuggestion))
+    {
+        /* Autosuggestion is active and the search term has not changed, so we're good to go */
+        data->autosuggestion = ctx->autosuggestion;
+        sanity_check();
+        reader_repaint();
+    }
+    delete ctx;
+}
+
+
+static void update_autosuggestion(void)
+{
+    /* Updates autosuggestion. We look for an autosuggestion if the command line is non-empty and if we're not doing a history search.  */
+    data->autosuggestion.clear();
+    if (data->allow_autosuggestion && ! data->suppress_autosuggestion && ! data->command_line.empty() && data->history_search.is_at_end())
+    {
+        const editable_line_t *el = data->active_edit_line();
+        autosuggestion_context_t *ctx = new autosuggestion_context_t(data->history, el->text, el->position);
+        iothread_perform(threaded_autosuggest, autosuggest_completed, ctx);
+    }
+}
+
+/* Accept any autosuggestion by replacing the command line with it. If full is true, take the whole thing; if it's false, then take only the first "word" */
+static void accept_autosuggestion(bool full)
+{
+    if (! data->autosuggestion.empty())
+    {
+        /* Accepting an autosuggestion clears the pager */
+        clear_pager();
+        
+        /* Accept the autosuggestion */
+        if (full)
+        {
+            /* Just take the whole thing */
+            data->command_line.text = data->autosuggestion;
+        }
+        else
+        {
+            /* Accept characters up to a word separator */
+            move_word_state_machine_t state(move_word_style_punctuation);
+            for (size_t idx = data->command_line.size(); idx < data->autosuggestion.size(); idx++)
+            {
+                wchar_t wc = data->autosuggestion.at(idx);
+                if (! state.consume_char(wc))
+                    break;
+                data->command_line.text.push_back(wc);
+            }
+        }
+        update_buff_pos(&data->command_line, data->command_line.size());
+        data->command_line_changed(&data->command_line);
+        reader_super_highlight_me_plenty();
+        reader_repaint();
+    }
+}
+
+/* Ensure we have no pager contents */
+static void clear_pager()
+{
+    if (data)
+    {
+        data->pager.clear();
+        data->current_page_rendering = page_rendering_t();
+        reader_repaint_needed();
+    }
+}
+
+static void select_completion_in_direction(enum selection_direction_t dir)
+{
+    assert(data != NULL);
+    bool selection_changed = data->pager.select_next_completion_in_direction(dir, data->current_page_rendering);
+    if (selection_changed)
+    {
+        data->pager_selection_changed();
+    }
+}
+
+/**
+  Flash the screen. This function only changed the color of the
+  current line, since the flash_screen sequnce is rather painful to
+  look at in most terminal emulators.
+*/
+static void reader_flash()
+{
+    struct timespec pollint;
+
+    editable_line_t *el = &data->command_line;
+    for (size_t i=0; i<el->position; i++)
+    {
+        data->colors.at(i) = highlight_spec_search_match<<16;
+    }
+
+    reader_repaint();
+
+    pollint.tv_sec = 0;
+    pollint.tv_nsec = 100 * 1000000;
+    nanosleep(&pollint, NULL);
+
+    reader_super_highlight_me_plenty();
+
+    reader_repaint();
+}
+
+/**
+   Characters that may not be part of a token that is to be replaced
+   by a case insensitive completion.
+ */
+#define REPLACE_UNCLEAN L"$*?({})"
+
+/**
+   Check if the specified string can be replaced by a case insensitive
+   completion with the specified flags.
+
+   Advanced tokens like those containing {}-style expansion can not at
+   the moment be replaced, other than if the new token is already an
+   exact replacement, e.g. if the COMPLETE_DONT_ESCAPE flag is set.
+ */
+
+static bool reader_can_replace(const wcstring &in, int flags)
+{
+
+    const wchar_t * str = in.c_str();
+
+    if (flags & COMPLETE_DONT_ESCAPE)
+    {
+        return true;
+    }
+    /*
+      Test characters that have a special meaning in any character position
+    */
+    while (*str)
+    {
+        if (wcschr(REPLACE_UNCLEAN, *str))
+            return false;
+        str++;
+    }
+
+    return true;
+}
+
+/* Compare two completions, ordering completions with better match types first */
+bool compare_completions_by_match_type(const completion_t &a, const completion_t &b)
+{
+    /* Compare match types, unless both completions are prefix (#923) in which case we always want to compare them alphabetically */
+    if (a.match.type != fuzzy_match_prefix || b.match.type != fuzzy_match_prefix)
+    {
+        int match_compare = a.match.compare(b.match);
+        if (match_compare != 0)
+        {
+            return match_compare < 0;
+        }
+    }
+
+    /* Compare using file comparison */
+    return wcsfilecmp(a.completion.c_str(), b.completion.c_str()) < 0;
+}
+
+/* Determine the best match type for a set of completions */
+static fuzzy_match_type_t get_best_match_type(const std::vector<completion_t> &comp)
+{
+    fuzzy_match_type_t best_type = fuzzy_match_none;
+    for (size_t i=0; i < comp.size(); i++)
+    {
+        const completion_t &el = comp.at(i);
+        if (el.match.type < best_type)
+        {
+            best_type = el.match.type;
+        }
+    }
+    /* If the best type is an exact match, reduce it to prefix match. Otherwise a tab completion will only show one match if it matches a file exactly. (see issue #959) */
+    if (best_type == fuzzy_match_exact)
+    {
+        best_type = fuzzy_match_prefix;
+    }
+    return best_type;
+}
+
+/* Order completions such that case insensitive completions come first. */
+static void prioritize_completions(std::vector<completion_t> &comp)
+{
+    fuzzy_match_type_t best_type = get_best_match_type(comp);
+
+    /* Throw out completions whose match types are less suitable than the best. */
+    size_t i = comp.size();
+    while (i--)
+    {
+        if (comp.at(i).match.type > best_type)
+        {
+            comp.erase(comp.begin() + i);
+        }
+    }
+
+    /* Sort the remainder */
+    sort(comp.begin(), comp.end(), compare_completions_by_match_type);
+}
+
+/**
+   Handle the list of completions. This means the following:
+
+   - If the list is empty, flash the terminal.
+   - If the list contains one element, write the whole element, and if
+   the element does not end on a '/', '@', ':', or a '=', also write a trailing
+   space.
+   - If the list contains multiple elements with a common prefix, write
+   the prefix.
+   - If the list contains multiple elements without a common prefix, call
+   run_pager to display a list of completions. Depending on terminal size and
+   the length of the list, run_pager may either show less than a screenfull and
+   exit or use an interactive pager to allow the user to scroll through the
+   completions.
+
+   \param comp the list of completion strings
+   \param continue_after_prefix_insertion If we have a shared prefix, whether to print the list of completions after inserting it.
+
+   Return true if we inserted text into the command line, false if we did not.
+*/
+
+static bool handle_completions(const std::vector<completion_t> &comp, bool continue_after_prefix_insertion)
+{
+    bool done = false;
+    bool success = false;
+    const editable_line_t *el = &data->command_line;
+    const wchar_t *begin, *end, *buff = el->text.c_str();
+
+    parse_util_token_extent(buff, el->position, &begin, 0, 0, 0);
+    end = buff+el->position;
+
+    const wcstring tok(begin, end - begin);
+
+    /*
+      Check trivial cases
+     */
+    switch (comp.size())
+    {
+            /* No suitable completions found, flash screen and return */
+        case 0:
+        {
+            reader_flash();
+            done = true;
+            success = false;
+            break;
+        }
+
+        /* Exactly one suitable completion found - insert it */
+        case 1:
+        {
+
+            const completion_t &c = comp.at(0);
+
+            /*
+              If this is a replacement completion, check
+              that we know how to replace it, e.g. that
+              the token doesn't contain evil operators
+              like {}
+             */
+            if (!(c.flags & COMPLETE_REPLACES_TOKEN) || reader_can_replace(tok, c.flags))
+            {
+                completion_insert(c.completion.c_str(), c.flags);
+            }
+            done = true;
+            success = true;
+            break;
+        }
+    }
+
+
+    if (!done)
+    {
+        fuzzy_match_type_t best_match_type = get_best_match_type(comp);
+
+        /* Determine whether we are going to replace the token or not. If any commands of the best type do not require replacement, then ignore all those that want to use replacement */
+        bool will_replace_token = true;
+        for (size_t i=0; i< comp.size(); i++)
+        {
+            const completion_t &el = comp.at(i);
+            if (el.match.type <= best_match_type && !(el.flags & COMPLETE_REPLACES_TOKEN))
+            {
+                will_replace_token = false;
+                break;
+            }
+        }
+
+        /* Decide which completions survived. There may be a lot of them; it would be nice if we could figure out how to avoid copying them here */
+        std::vector<completion_t> surviving_completions;
+        for (size_t i=0; i < comp.size(); i++)
+        {
+            const completion_t &el = comp.at(i);
+            /* Ignore completions with a less suitable match type than the best. */
+            if (el.match.type > best_match_type)
+                continue;
+
+            /* Only use completions that match replace_token */
+            bool completion_replace_token = !!(el.flags & COMPLETE_REPLACES_TOKEN);
+            if (completion_replace_token != will_replace_token)
+                continue;
+
+            /* Don't use completions that want to replace, if we cannot replace them */
+            if (completion_replace_token && ! reader_can_replace(tok, el.flags))
+                continue;
+
+            /* This completion survived */
+            surviving_completions.push_back(el);
+        }
+
+
+        /* Try to find a common prefix to insert among the surviving completions */
+        wcstring common_prefix;
+        complete_flags_t flags = 0;
+        bool prefix_is_partial_completion = false;
+        for (size_t i=0; i < surviving_completions.size(); i++)
+        {
+            const completion_t &el = surviving_completions.at(i);
+            if (i == 0)
+            {
+                /* First entry, use the whole string */
+                common_prefix = el.completion;
+                flags = el.flags;
+            }
+            else
+            {
+                /* Determine the shared prefix length. */
+                size_t idx, max = mini(common_prefix.size(), el.completion.size());
+                for (idx=0; idx < max; idx++)
+                {
+                    wchar_t ac = common_prefix.at(idx), bc = el.completion.at(idx);
+                    bool matches = (ac == bc);
+                    /* If we are replacing the token, allow case to vary */
+                    if (will_replace_token && ! matches)
+                    {
+                        /* Hackish way to compare two strings in a case insensitive way, hopefully better than towlower(). */
+                        matches = (wcsncasecmp(&ac, &bc, 1) == 0);
+                    }
+                    if (! matches)
+                        break;
+                }
+
+                /* idx is now the length of the new common prefix */
+                common_prefix.resize(idx);
+                prefix_is_partial_completion = true;
+
+                /* Early out if we decide there's no common prefix */
+                if (idx == 0)
+                    break;
+            }
+        }
+        
+        /* Determine if we use the prefix. We use it if it's non-empty and it will actually make the command line longer. It may make the command line longer by virtue of not using REPLACE_TOKEN (so it always appends to the command line), or by virtue of replacing the token but being longer than it. */
+        bool use_prefix = common_prefix.size() > (will_replace_token ? tok.size() : 0);
+        assert(! use_prefix || ! common_prefix.empty());
+
+        if (use_prefix)
+        {
+            /* We got something. If more than one completion contributed, then it means we have a prefix; don't insert a space after it */
+            if (prefix_is_partial_completion)
+                flags |= COMPLETE_NO_SPACE;
+            completion_insert(common_prefix.c_str(), flags);
+            success = true;
+        }
+
+        if (continue_after_prefix_insertion || ! use_prefix)
+        {
+            /* We didn't get a common prefix, or we want to print the list anyways. */
+            size_t len, prefix_start = 0;
+            wcstring prefix;
+            parse_util_get_parameter_info(el->text, el->position, NULL, &prefix_start, NULL);
+
+            assert(el->position >= prefix_start);
+            len = el->position - prefix_start;
+
+            if (match_type_requires_full_replacement(best_match_type))
+            {
+                // No prefix
+                prefix.clear();
+            }
+            else if (len <= PREFIX_MAX_LEN)
+            {
+                prefix.append(el->text, prefix_start, len);
+            }
+            else
+            {
+                // append just the end of the string
+                prefix = wcstring(&ellipsis_char, 1);
+                prefix.append(el->text, prefix_start + len - PREFIX_MAX_LEN, PREFIX_MAX_LEN);
+            }
+
+            wchar_t quote;
+            parse_util_get_parameter_info(el->text, el->position, &quote, NULL, NULL);
+
+            /* Update the pager data */
+            data->pager.set_prefix(prefix);
+            data->pager.set_completions(surviving_completions);
+
+            /* Invalidate our rendering */
+            data->current_page_rendering = page_rendering_t();
+
+            /* Modify the command line to reflect the new pager */
+            data->pager_selection_changed();
+
+            reader_repaint_needed();
+
+            success = false;
+        }
+    }
+    return success;
+}
+
+/* Return true if we believe ourselves to be orphaned. loop_count is how many times we've tried to stop ourselves via SIGGTIN */
+static bool check_for_orphaned_process(unsigned long loop_count, pid_t shell_pgid)
+{
+    bool we_think_we_are_orphaned = false;
+    /* Try kill-0'ing the process whose pid corresponds to our process group ID. It's possible this will fail because we don't have permission to signal it. But more likely it will fail because it no longer exists, and we are orphaned. */
+    if (loop_count % 64 == 0)
+    {
+        if (kill(shell_pgid, 0) < 0 && errno == ESRCH)
+        {
+            we_think_we_are_orphaned = true;
+        }
+    }
+
+    if (! we_think_we_are_orphaned && loop_count % 128 == 0)
+    {
+        /* Try reading from the tty; if we get EIO we are orphaned. This is sort of bad because it may block. */
+
+        char *tty = ctermid(NULL);
+        if (! tty)
+        {
+            wperror(L"ctermid");
+            exit_without_destructors(1);
+        }
+
+        /* Open the tty. Presumably this is stdin, but maybe not? */
+        int tty_fd = open(tty, O_RDONLY | O_NONBLOCK);
+        if (tty_fd < 0)
+        {
+            wperror(L"open");
+            exit_without_destructors(1);
+        }
+
+        char tmp;
+        if (read(tty_fd, &tmp, 1) < 0 && errno == EIO)
+        {
+            we_think_we_are_orphaned = true;
+        }
+
+        close(tty_fd);
+    }
+
+    /* Just give up if we've done it a lot times */
+    if (loop_count > 4096)
+    {
+        we_think_we_are_orphaned = true;
+    }
+
+    return we_think_we_are_orphaned;
+}
+
+/**
+   Initialize data for interactive use
+*/
+static void reader_interactive_init()
+{
+    /* See if we are running interactively.  */
+    pid_t shell_pgid;
+
+    input_init();
+    kill_init();
+    shell_pgid = getpgrp();
+
+    /*
+      This should enable job control on fish, even if our parent process did
+      not enable it for us.
+    */
+
+    /*
+       Check if we are in control of the terminal, so that we don't do
+       semi-expensive things like reset signal handlers unless we
+       really have to, which we often don't.
+     */
+    if (tcgetpgrp(STDIN_FILENO) != shell_pgid)
+    {
+        int block_count = 0;
+        int i;
+
+        /*
+          Bummer, we are not in control of the terminal. Stop until
+          parent has given us control of it. Stopping in fish is a bit
+          of a challange, what with all the signal fidgeting, we need
+          to reset a bunch of signal state, making this coda a but
+          unobvious.
+
+          In theory, reseting signal handlers could cause us to miss
+          signal deliveries. In practice, this code should only be run
+          suring startup, when we're not waiting for any signals.
+        */
+        while (signal_is_blocked())
+        {
+            signal_unblock();
+            block_count++;
+        }
+        signal_reset_handlers();
+
+        /* Ok, signal handlers are taken out of the picture. Stop ourself in a loop until we are in control of the terminal. However, the call to signal(SIGTTIN) may silently not do anything if we are orphaned.
+
+            As far as I can tell there's no really good way to detect that we are orphaned. One way is to just detect if the group leader exited, via kill(shell_pgid, 0). Another possibility is that read() from the tty fails with EIO - this is more reliable but it's harder, because it may succeed or block. So we loop for a while, trying those strategies. Eventually we just give up and assume we're orphaend.
+         */
+        for (unsigned long loop_count = 0;; loop_count++)
+        {
+            pid_t owner = tcgetpgrp(STDIN_FILENO);
+            shell_pgid = getpgrp();
+            if (owner < 0 && errno == ENOTTY)
+            {
+                // No TTY, cannot be interactive?
+                debug(1,
+                      _(L"No TTY for interactive shell (tcgetpgrp failed)"));
+                wperror(L"setpgid");
+                exit_without_destructors(1);
+            }
+            if (owner == shell_pgid)
+            {
+                /* Success */
+                break;
+            }
+            else
+            {
+                if (check_for_orphaned_process(loop_count, shell_pgid))
+                {
+                    /* We're orphaned, so we just die. Another sad statistic. */
+                    debug(1,
+                          _(L"I appear to be an orphaned process, so I am quitting politely. My pid is %d."), (int)getpid());
+                    exit_without_destructors(1);
+                }
+
+                /* Try stopping us */
+                int ret = killpg(shell_pgid, SIGTTIN);
+                if (ret < 0)
+                {
+                    wperror(L"killpg");
+                    exit_without_destructors(1);
+                }
+            }
+        }
+
+        signal_set_handlers();
+
+        for (i=0; i<block_count; i++)
+        {
+            signal_block();
+        }
+
+    }
+
+
+    /* Put ourselves in our own process group.  */
+    shell_pgid = getpid();
+    if (getpgrp() != shell_pgid)
+    {
+        if (setpgid(shell_pgid, shell_pgid) < 0)
+        {
+            debug(1,
+                  _(L"Couldn't put the shell in its own process group"));
+            wperror(L"setpgid");
+            exit_without_destructors(1);
+        }
+    }
+
+    /* Grab control of the terminal.  */
+    if (tcsetpgrp(STDIN_FILENO, shell_pgid))
+    {
+        debug(1,
+              _(L"Couldn't grab control of terminal"));
+        wperror(L"tcsetpgrp");
+        exit_without_destructors(1);
+    }
+
+    common_handle_winch(0);
+
+
+    if (tcsetattr(0,TCSANOW,&shell_modes))      /* set the new modes */
+    {
+        wperror(L"tcsetattr");
+    }
+
+    /*
+       We need to know our own pid so we'll later know if we are a
+       fork
+    */
+    original_pid = getpid();
+
+    env_set(L"_", L"fish", ENV_GLOBAL);
+}
+
+/**
+   Destroy data for interactive use
+*/
+static void reader_interactive_destroy()
+{
+    kill_destroy();
+    writestr(L"\n");
+    set_color(rgb_color_t::reset(), rgb_color_t::reset());
+    input_destroy();
+}
+
+
+void reader_sanity_check()
+{
+    /* Note: 'data' is non-null if we are interactive, except in the testing environment */
+    if (get_is_interactive() && data != NULL)
+    {
+        if (data->command_line.position > data->command_line.size())
+            sanity_lose();
+
+        if (data->colors.size() != data->command_line.size())
+            sanity_lose();
+
+        if (data->indents.size() != data->command_line.size())
+            sanity_lose();
+
+    }
+}
+
+/**
+   Set the specified string as the current buffer.
+*/
+static void set_command_line_and_position(editable_line_t *el, const wcstring &new_str, size_t pos)
+{
+    el->text = new_str;
+    update_buff_pos(el, pos);
+    data->command_line_changed(el);
+    reader_super_highlight_me_plenty();
+    reader_repaint_needed();
+}
+
+static void reader_replace_current_token(const wcstring &new_token)
+{
+
+    const wchar_t *begin, *end;
+    size_t new_pos;
+
+    /* Find current token */
+    editable_line_t *el = data->active_edit_line();
+    const wchar_t *buff = el->text.c_str();
+    parse_util_token_extent(buff, el->position, &begin, &end, 0, 0);
+
+    if (!begin || !end)
+        return;
+
+    /* Make new string */
+    wcstring new_buff(buff, begin - buff);
+    new_buff.append(new_token);
+    new_buff.append(end);
+    new_pos = (begin-buff) + new_token.size();
+
+    set_command_line_and_position(el, new_buff, new_pos);
+}
+
+
+/**
+   Reset the data structures associated with the token search
+*/
+static void reset_token_history()
+{
+    const editable_line_t *el = data->active_edit_line();
+    const wchar_t *begin, *end;
+    const wchar_t *buff = el->text.c_str();
+    parse_util_token_extent((wchar_t *)buff, el->position, &begin, &end, 0, 0);
+
+    data->search_buff.clear();
+    if (begin)
+    {
+        data->search_buff.append(begin, end - begin);
+    }
+
+    data->token_history_pos = -1;
+    data->search_pos=0;
+    data->search_prev.clear();
+    data->search_prev.push_back(data->search_buff);
+
+    data->history_search = history_search_t(*data->history, data->search_buff, HISTORY_SEARCH_TYPE_CONTAINS);
+}
+
+
+/**
+   Handles a token search command.
+
+   \param forward if the search should be forward or reverse
+   \param reset whether the current token should be made the new search token
+*/
+static void handle_token_history(int forward, int reset)
+{
+    /* Paranoia */
+    if (! data)
+        return;
+
+    wcstring str;
+    long current_pos;
+
+    if (reset)
+    {
+        /*
+          Start a new token search using the current token
+        */
+        reset_token_history();
+
+    }
+
+
+    current_pos  = data->token_history_pos;
+
+    if (forward || data->search_pos + 1 < data->search_prev.size())
+    {
+        if (forward)
+        {
+            if (data->search_pos > 0)
+            {
+                data->search_pos--;
+            }
+            str = data->search_prev.at(data->search_pos);
+        }
+        else
+        {
+            data->search_pos++;
+            str = data->search_prev.at(data->search_pos);
+        }
+
+        reader_replace_current_token(str);
+        reader_super_highlight_me_plenty();
+        reader_repaint();
+    }
+    else
+    {
+        if (current_pos == -1)
+        {
+            data->token_history_buff.clear();
+
+            /*
+              Search for previous item that contains this substring
+            */
+            if (data->history_search.go_backwards())
+            {
+                data->token_history_buff = data->history_search.current_string();
+            }
+            current_pos = data->token_history_buff.size();
+
+        }
+
+        if (data->token_history_buff.empty())
+        {
+            /*
+              We have reached the end of the history - check if the
+              history already contains the search string itself, if so
+              return, otherwise add it.
+            */
+
+            const wcstring &last = data->search_prev.back();
+            if (data->search_buff != last)
+            {
+                str = data->search_buff;
+            }
+            else
+            {
+                return;
+            }
+        }
+        else
+        {
+
+            //debug( 3, L"new '%ls'", data->token_history_buff.c_str() );
+            tokenizer_t tok(data->token_history_buff.c_str(), TOK_ACCEPT_UNFINISHED);
+            for (; tok_has_next(&tok); tok_next(&tok))
+            {
+                switch (tok_last_type(&tok))
+                {
+                    case TOK_STRING:
+                    {
+                        if (wcsstr(tok_last(&tok), data->search_buff.c_str()))
+                        {
+                            //debug( 3, L"Found token at pos %d\n", tok_get_pos( &tok ) );
+                            if (tok_get_pos(&tok) >= current_pos)
+                            {
+                                break;
+                            }
+                            //debug( 3, L"ok pos" );
+
+                            const wcstring last_tok = tok_last(&tok);
+                            if (find(data->search_prev.begin(), data->search_prev.end(), last_tok) == data->search_prev.end())
+                            {
+                                data->token_history_pos = tok_get_pos(&tok);
+                                str = tok_last(&tok);
+                            }
+
+                        }
+                    }
+                    break;
+
+                    default:
+                    {
+                        break;
+                    }
+
+                }
+            }
+        }
+
+        if (!str.empty())
+        {
+            reader_replace_current_token(str);
+            reader_super_highlight_me_plenty();
+            reader_repaint();
+            data->search_pos = data->search_prev.size();
+            data->search_prev.push_back(str);
+        }
+        else if (! reader_interrupted())
+        {
+            data->token_history_pos=-1;
+            handle_token_history(0, 0);
+        }
+    }
+}
+
+/**
+   Move buffer position one word or erase one word. This function
+   updates both the internal buffer and the screen. It is used by
+   M-left, M-right and ^W to do block movement or block erase.
+
+   \param dir Direction to move/erase. 0 means move left, 1 means move right.
+   \param erase Whether to erase the characters along the way or only move past them.
+   \param new if the new kill item should be appended to the previous kill item or not.
+*/
+enum move_word_dir_t
+{
+    MOVE_DIR_LEFT,
+    MOVE_DIR_RIGHT
+};
+
+static void move_word(editable_line_t *el, bool move_right, bool erase, enum move_word_style_t style, bool newv)
+{
+    /* Return if we are already at the edge */
+    const size_t boundary = move_right ? el->size() : 0;
+    if (el->position == boundary)
+        return;
+
+    /* When moving left, a value of 1 means the character at index 0. */
+    move_word_state_machine_t state(style);
+    const wchar_t * const command_line = el->text.c_str();
+    const size_t start_buff_pos = el->position;
+
+    size_t buff_pos = el->position;
+    while (buff_pos != boundary)
+    {
+        size_t idx = (move_right ? buff_pos : buff_pos - 1);
+        wchar_t c = command_line[idx];
+        if (! state.consume_char(c))
+            break;
+        buff_pos = (move_right ? buff_pos + 1 : buff_pos - 1);
+    }
+
+    /* Always consume at least one character */
+    if (buff_pos == start_buff_pos)
+        buff_pos = (move_right ? buff_pos + 1 : buff_pos - 1);
+
+    /* If we are moving left, buff_pos-1 is the index of the first character we do not delete (possibly -1). If we are moving right, then buff_pos is that index - possibly el->size(). */
+    if (erase)
+    {
+        /* Don't autosuggest after a kill */
+        if (el == &data->command_line)
+        {
+            data->suppress_autosuggestion = true;
+        }
+
+        if (move_right)
+        {
+            reader_kill(el, start_buff_pos, buff_pos - start_buff_pos, KILL_APPEND, newv);
+        }
+        else
+        {
+            reader_kill(el, buff_pos, start_buff_pos - buff_pos, KILL_PREPEND, newv);
+        }
+    }
+    else
+    {
+        update_buff_pos(el, buff_pos);
+        reader_repaint();
+    }
+
+}
+
+const wchar_t *reader_get_buffer(void)
+{
+    ASSERT_IS_MAIN_THREAD();
+    return data ? data->command_line.text.c_str() : NULL;
+}
+
+history_t *reader_get_history(void)
+{
+    ASSERT_IS_MAIN_THREAD();
+    return data ? data->history : NULL;
+}
+
+/* Sets the command line contents, without clearing the pager */
+static void reader_set_buffer_maintaining_pager(const wcstring &b, size_t pos)
+{
+    /* Callers like to pass us pointers into ourselves, so be careful! I don't know if we can use operator= with a pointer to our interior, so use an intermediate. */
+    size_t command_line_len = b.size();
+    data->command_line.text = b;
+    data->command_line_changed(&data->command_line);
+
+    /* Don't set a position past the command line length */
+    if (pos > command_line_len)
+        pos = command_line_len;
+
+    update_buff_pos(&data->command_line, pos);
+
+    /* Clear history search and pager contents */
+    data->search_mode = NO_SEARCH;
+    data->search_buff.clear();
+    data->history_search.go_to_end();
+
+    reader_super_highlight_me_plenty();
+    reader_repaint_needed();
+}
+
+/* Sets the command line contents, clearing the pager */
+void reader_set_buffer(const wcstring &b, size_t pos)
+{
+    if (!data)
+        return;
+
+    clear_pager();
+    reader_set_buffer_maintaining_pager(b, pos);
+}
+
+
+size_t reader_get_cursor_pos()
+{
+    if (!data)
+        return (size_t)(-1);
+
+    return data->command_line.position;
+}
+
+bool reader_get_selection(size_t *start, size_t *len)
+{
+    bool result = false;
+    if (data != NULL && data->sel_active)
+    {
+        *start = data->sel_start_pos;
+        *len = std::min(data->sel_stop_pos - data->sel_start_pos + 1, data->command_line.size());
+        result = true;
+    }
+    return result;
+}
+
+
+#define ENV_CMD_DURATION L"CMD_DURATION"
+
+void set_env_cmd_duration(struct timeval *after, struct timeval *before)
+{
+    time_t secs = after->tv_sec - before->tv_sec;
+    suseconds_t usecs = after->tv_usec - before->tv_usec;
+    wchar_t buf[16];
+
+    if (after->tv_usec < before->tv_usec)
+    {
+        usecs += 1000000;
+        secs -= 1;
+    }
+
+    swprintf(buf, 16, L"%d", (secs * 1000) + (usecs / 1000));
+    env_set(ENV_CMD_DURATION, buf, ENV_UNEXPORT);
+}
+
+void reader_run_command(parser_t &parser, const wcstring &cmd)
+{
+
+    struct timeval time_before, time_after;
+
+    wcstring ft = tok_first(cmd.c_str());
+
+    if (! ft.empty())
+        env_set(L"_", ft.c_str(), ENV_GLOBAL);
+
+    reader_write_title(cmd);
+
+    term_donate();
+
+    gettimeofday(&time_before, NULL);
+
+    parser.eval(cmd, io_chain_t(), TOP);
+    job_reap(1);
+
+    gettimeofday(&time_after, NULL);
+    set_env_cmd_duration(&time_after, &time_before);
+
+    term_steal();
+
+    env_set(L"_", program_name, ENV_GLOBAL);
+
+#ifdef HAVE__PROC_SELF_STAT
+    proc_update_jiffies();
+#endif
+
+
+}
+
+
+parser_test_error_bits_t reader_shell_test(const wchar_t *b)
+{
+    assert(b != NULL);
+    wcstring bstr = b;
+
+    /* Append a newline, to act as a statement terminator */
+    bstr.push_back(L'\n');
+
+    parse_error_list_t errors;
+    parser_test_error_bits_t res = parse_util_detect_errors(bstr, &errors, true /* do accept incomplete */);
+
+    if (res & PARSER_TEST_ERROR)
+    {
+        wcstring error_desc;
+        parser_t::principal_parser().get_backtrace(bstr, errors, &error_desc);
+
+        // ensure we end with a newline. Also add an initial newline, because it's likely the user just hit enter and so there's junk on the current line
+        if (! string_suffixes_string(L"\n", error_desc))
+        {
+            error_desc.push_back(L'\n');
+        }
+        fwprintf(stderr, L"\n%ls", error_desc.c_str());
+    }
+    return res;
+}
+
+/**
+   Test if the given string contains error. Since this is the error
+   detection for general purpose, there are no invalid strings, so
+   this function always returns false.
+*/
+static parser_test_error_bits_t default_test(const wchar_t *b)
+{
+    return 0;
+}
+
+void reader_push(const wchar_t *name)
+{
+    reader_data_t *n = new reader_data_t();
+
+    n->history = & history_t::history_with_name(name);
+    n->app_name = name;
+    n->next = data;
+
+    data=n;
+
+    data->command_line_changed(&data->command_line);
+
+    if (data->next == 0)
+    {
+        reader_interactive_init();
+    }
+
+    exec_prompt();
+    reader_set_highlight_function(&highlight_universal);
+    reader_set_test_function(&default_test);
+    reader_set_left_prompt(L"");
+}
+
+void reader_pop()
+{
+    reader_data_t *n = data;
+
+    if (data == 0)
+    {
+        debug(0, _(L"Pop null reader block"));
+        sanity_lose();
+        return;
+    }
+
+    data=data->next;
+
+    /* Invoke the destructor to balance our new */
+    delete n;
+
+    if (data == 0)
+    {
+        reader_interactive_destroy();
+    }
+    else
+    {
+        end_loop = 0;
+        //history_set_mode( data->app_name.c_str() );
+        s_reset(&data->screen, screen_reset_abandon_line);
+    }
+}
+
+void reader_set_left_prompt(const wcstring &new_prompt)
+{
+    data->left_prompt = new_prompt;
+}
+
+void reader_set_right_prompt(const wcstring &new_prompt)
+{
+    data->right_prompt = new_prompt;
+}
+
+void reader_set_allow_autosuggesting(bool flag)
+{
+    data->allow_autosuggestion = flag;
+}
+
+void reader_set_expand_abbreviations(bool flag)
+{
+    data->expand_abbreviations = flag;
+}
+
+void reader_set_complete_function(complete_function_t f)
+{
+    data->complete_func = f;
+}
+
+void reader_set_highlight_function(highlight_function_t func)
+{
+    data->highlight_function = func;
+}
+
+void reader_set_test_function(parser_test_error_bits_t (*f)(const wchar_t *))
+{
+    data->test_func = f;
+}
+
+void reader_set_exit_on_interrupt(bool i)
+{
+    data->exit_on_interrupt = i;
+}
+
+void reader_import_history_if_necessary(void)
+{
+    /* Import history from bash, etc. if our current history is empty */
+    if (data->history && data->history->is_empty())
+    {
+        /* Try opening a bash file. We make an effort to respect $HISTFILE; this isn't very complete (AFAIK it doesn't have to be exported), and to really get this right we ought to ask bash itself. But this is better than nothing.
+        */
+        const env_var_t var = env_get_string(L"HISTFILE");
+        wcstring path = (var.missing() ? L"~/.bash_history" : var);
+        expand_tilde(path);
+        FILE *f = wfopen(path, "r");
+        if (f)
+        {
+            data->history->populate_from_bash(f);
+            fclose(f);
+        }
+    }
+}
+
+/** A class as the context pointer for a background (threaded) highlight operation. */
+class background_highlight_context_t
+{
+public:
+    /** The string to highlight */
+    const wcstring string_to_highlight;
+
+    /** Color buffer */
+    std::vector<highlight_spec_t> colors;
+
+    /** The position to use for bracket matching */
+    const size_t match_highlight_pos;
+
+    /** Function for syntax highlighting */
+    const highlight_function_t highlight_function;
+
+    /** Environment variables */
+    const env_vars_snapshot_t vars;
+
+    /** When the request was made */
+    const double when;
+
+    /** Gen count at the time the request was made */
+    const unsigned int generation_count;
+
+    background_highlight_context_t(const wcstring &pbuff, size_t phighlight_pos, highlight_function_t phighlight_func) :
+        string_to_highlight(pbuff),
+        colors(pbuff.size(), 0),
+        match_highlight_pos(phighlight_pos),
+        highlight_function(phighlight_func),
+        vars(env_vars_snapshot_t::highlighting_keys),
+        when(timef()),
+        generation_count(s_generation_count)
+    {
+    }
+
+    int perform_highlight()
+    {
+        if (generation_count != s_generation_count)
+        {
+            // The gen count has changed, so don't do anything
+            return 0;
+        }
+        if (! string_to_highlight.empty())
+        {
+            highlight_function(string_to_highlight, colors, match_highlight_pos, NULL /* error */, vars);
+        }
+        return 0;
+    }
+};
+
+/* Called to set the highlight flag for search results */
+static void highlight_search(void)
+{
+    if (! data->search_buff.empty() && ! data->history_search.is_at_end())
+    {
+        const editable_line_t *el = &data->command_line;
+        const wcstring &needle = data->search_buff;
+        size_t match_pos = el->text.find(needle);
+        if (match_pos != wcstring::npos)
+        {
+            size_t end = match_pos + needle.size();
+            for (size_t i=match_pos; i < end; i++)
+            {
+                data->colors.at(i) |= (highlight_spec_search_match<<16);
+            }
+        }
+    }
+}
+
+static void highlight_complete(background_highlight_context_t *ctx, int result)
+{
+    ASSERT_IS_MAIN_THREAD();
+    if (ctx->string_to_highlight == data->command_line.text)
+    {
+        /* The data hasn't changed, so swap in our colors. The colors may not have changed, so do nothing if they have not. */
+        assert(ctx->colors.size() == data->command_line.size());
+        if (data->colors != ctx->colors)
+        {
+            data->colors.swap(ctx->colors);
+            sanity_check();
+            highlight_search();
+            reader_repaint();
+        }
+    }
+
+    /* Free our context */
+    delete ctx;
+}
+
+static int threaded_highlight(background_highlight_context_t *ctx)
+{
+    return ctx->perform_highlight();
+}
+
+
+/**
+   Call specified external highlighting function and then do search
+   highlighting. Lastly, clear the background color under the cursor
+   to avoid repaint issues on terminals where e.g. syntax highlighting
+   maykes characters under the sursor unreadable.
+
+   \param match_highlight_pos_adjust the adjustment to the position to use for bracket matching. This is added to the current cursor position and may be negative.
+   \param error if non-null, any possible errors in the buffer are further descibed by the strings inserted into the specified arraylist
+    \param no_io if true, do a highlight that does not perform I/O, synchronously. If false, perform an asynchronous highlight in the background, which may perform disk I/O.
+*/
+static void reader_super_highlight_me_plenty(int match_highlight_pos_adjust, bool no_io)
+{
+    const editable_line_t *el = &data->command_line;
+    long match_highlight_pos = (long)el->position + match_highlight_pos_adjust;
+    assert(match_highlight_pos >= 0);
+
+    reader_sanity_check();
+
+    highlight_function_t highlight_func = no_io ? highlight_shell_no_io : data->highlight_function;
+    background_highlight_context_t *ctx = new background_highlight_context_t(el->text, match_highlight_pos, highlight_func);
+    if (no_io)
+    {
+        // Highlighting without IO, we just do it
+        // Note that highlight_complete deletes ctx.
+        int result = ctx->perform_highlight();
+        highlight_complete(ctx, result);
+    }
+    else
+    {
+        // Highlighting including I/O proceeds in the background
+        iothread_perform(threaded_highlight, highlight_complete, ctx);
+    }
+    highlight_search();
+
+    /* Here's a hack. Check to see if our autosuggestion still applies; if so, don't recompute it. Since the autosuggestion computation is asynchronous, this avoids "flashing" as you type into the autosuggestion. */
+    const wcstring &cmd = el->text, &suggest = data->autosuggestion;
+    if (can_autosuggest() && ! suggest.empty() && string_prefixes_string_case_insensitive(cmd, suggest))
+    {
+        /* The autosuggestion is still reasonable, so do nothing */
+    }
+    else
+    {
+        update_autosuggestion();
+    }
+}
+
+
+bool shell_is_exiting()
+{
+    if (get_is_interactive())
+        return job_list_is_empty() && data != NULL && data->end_loop;
+    else
+        return end_loop;
+}
+
+/**
+   This function is called when the main loop notices that end_loop
+   has been set while in interactive mode. It checks if it is ok to
+   exit.
+ */
+
+static void handle_end_loop()
+{
+    job_t *j;
+    int stopped_jobs_count=0;
+    int is_breakpoint=0;
+    const parser_t &parser = parser_t::principal_parser();
+
+    for (size_t i = 0; i < parser.block_count(); i++)
+    {
+        if (parser.block_at_index(i)->type() == BREAKPOINT)
+        {
+            is_breakpoint = 1;
+            break;
+        }
+    }
+
+    job_iterator_t jobs;
+    while ((j = jobs.next()))
+    {
+        if (!job_is_completed(j) && (job_is_stopped(j)))
+        {
+            stopped_jobs_count++;
+            break;
+        }
+    }
+
+    if (!reader_exit_forced() && !data->prev_end_loop && stopped_jobs_count && !is_breakpoint)
+    {
+        writestr(_(L"There are stopped jobs. A second attempt to exit will enforce their termination.\n"));
+
+        reader_exit(0, 0);
+        data->prev_end_loop=1;
+    }
+    else
+    {
+        /* PCA: we used to only hangup jobs if stdin was closed. This prevented child processes from exiting. It's unclear to my why it matters if stdin is closed, but it seems to me if we're forcing an exit, we definitely want to hang up our processes.
+
+            See https://github.com/fish-shell/fish-shell/issues/138
+        */
+        if (reader_exit_forced() || !isatty(0))
+        {
+            /*
+              We already know that stdin is a tty since we're
+              in interactive mode. If isatty returns false, it
+              means stdin must have been closed.
+            */
+            job_iterator_t jobs;
+            while ((j = jobs.next()))
+            {
+                /* Send SIGHUP only to foreground processes.
+
+                   See https://github.com/fish-shell/fish-shell/issues/1771
+                 */
+                if (! job_is_completed(j) && job_get_flag(j, JOB_FOREGROUND))
+                {
+                    job_signal(j, SIGHUP);
+                }
+            }
+        }
+    }
+}
+
+static bool selection_is_at_top()
+{
+    const pager_t *pager = &data->pager;
+    size_t row = pager->get_selected_row(data->current_page_rendering);
+    if (row != 0 && row != PAGER_SELECTION_NONE)
+        return false;
+
+    size_t col = pager->get_selected_column(data->current_page_rendering);
+    if (col != 0 && col != PAGER_SELECTION_NONE)
+        return false;
+
+    return true;
+}
+
+
+/**
+   Read interactively. Read input from stdin while providing editing
+   facilities.
+*/
+static int read_i(void)
+{
+    reader_push(L"fish");
+    reader_set_complete_function(&complete);
+    reader_set_highlight_function(&highlight_shell);
+    reader_set_test_function(&reader_shell_test);
+    reader_set_allow_autosuggesting(true);
+    reader_set_expand_abbreviations(true);
+    reader_import_history_if_necessary();
+
+    parser_t &parser = parser_t::principal_parser();
+
+    data->prev_end_loop=0;
+
+    while ((!data->end_loop) && (!sanity_check()))
+    {
+        event_fire_generic(L"fish_prompt");
+        if (function_exists(LEFT_PROMPT_FUNCTION_NAME))
+            reader_set_left_prompt(LEFT_PROMPT_FUNCTION_NAME);
+        else
+            reader_set_left_prompt(DEFAULT_PROMPT);
+
+        if (function_exists(RIGHT_PROMPT_FUNCTION_NAME))
+            reader_set_right_prompt(RIGHT_PROMPT_FUNCTION_NAME);
+        else
+            reader_set_right_prompt(L"");
+
+
+        /*
+          Put buff in temporary string and clear buff, so
+          that we can handle a call to reader_set_buffer
+          during evaluation.
+        */
+
+        const wchar_t *tmp = reader_readline(0);
+
+        if (data->end_loop)
+        {
+            handle_end_loop();
+        }
+        else if (tmp)
+        {
+            const wcstring command = tmp;
+            update_buff_pos(&data->command_line, 0);
+            data->command_line.text.clear();
+            data->command_line_changed(&data->command_line);
+            wcstring_list_t argv(1, command);
+            event_fire_generic(L"fish_preexec", &argv);
+            reader_run_command(parser, command);
+            event_fire_generic(L"fish_postexec", &argv);
+            /* Allow any pending history items to be returned in the history array. */
+            if (data->history)
+            {
+                data->history->resolve_pending();
+            }
+            if (data->end_loop)
+            {
+                handle_end_loop();
+            }
+            else
+            {
+                data->prev_end_loop=0;
+            }
+        }
+
+
+    }
+    reader_pop();
+    return 0;
+}
+
+/**
+   Test if there are bytes available for reading on the specified file
+   descriptor
+*/
+static int can_read(int fd)
+{
+    struct timeval can_read_timeout = { 0, 0 };
+    fd_set fds;
+
+    FD_ZERO(&fds);
+    FD_SET(fd, &fds);
+    return select(fd + 1, &fds, 0, 0, &can_read_timeout) == 1;
+}
+
+/**
+   Test if the specified character is in the private use area that
+   fish uses to store internal characters
+
+    Note: Allow U+F8FF because that's the Apple symbol, which is in the
+    OS X US keyboard layout.
+*/
+static int wchar_private(wchar_t c)
+{
+    return ((c >= 0xe000) && (c < 0xf8ff));
+}
+
+/**
+   Test if the specified character in the specified string is
+   backslashed. pos may be at the end of the string, which indicates
+   if there is a trailing backslash.
+*/
+static bool is_backslashed(const wcstring &str, size_t pos)
+{
+    /* note pos == str.size() is OK */
+    if (pos > str.size())
+        return false;
+
+    size_t count = 0, idx = pos;
+    while (idx--)
+    {
+        if (str.at(idx) != L'\\')
+            break;
+        count++;
+    }
+
+    return (count % 2) == 1;
+}
+
+static wchar_t unescaped_quote(const wcstring &str, size_t pos)
+{
+    wchar_t result = L'\0';
+    if (pos < str.size())
+    {
+        wchar_t c = str.at(pos);
+        if ((c == L'\'' || c == L'"') && ! is_backslashed(str, pos))
+        {
+            result = c;
+        }
+    }
+    return result;
+}
+
+/* Returns true if the last token is a comment. */
+static bool text_ends_in_comment(const wcstring &text)
+{
+    token_type last_type = TOK_NONE;
+    tokenizer_t tok(text.c_str(), TOK_ACCEPT_UNFINISHED | TOK_SHOW_COMMENTS | TOK_SQUASH_ERRORS);
+    while (tok_has_next(&tok))
+    {
+        last_type = tok_last_type(&tok);
+        tok_next(&tok);
+    }
+    return last_type == TOK_COMMENT;
+}
+
+const wchar_t *reader_readline(int nchars)
+{
+    wint_t c;
+    int last_char=0;
+    size_t yank_len=0;
+    const wchar_t *yank_str;
+    bool comp_empty = true;
+    std::vector<completion_t> comp;
+    int finished=0;
+    struct termios old_modes;
+
+    /* Coalesce redundant repaints. When we get a repaint, we set this to true, and skip repaints until we get something else. */
+    bool coalescing_repaints = false;
+
+    /* The command line before completion */
+    data->cycle_command_line.clear();
+    data->cycle_cursor_pos = 0;
+
+    data->search_buff.clear();
+    data->search_mode = NO_SEARCH;
+
+    exec_prompt();
+
+    reader_super_highlight_me_plenty();
+    s_reset(&data->screen, screen_reset_abandon_line);
+    reader_repaint();
+
+    /*
+     get the current terminal modes. These will be restored when the
+     function returns.
+     */
+    tcgetattr(0,&old_modes);
+    /* set the new modes */
+    if (tcsetattr(0,TCSANOW,&shell_modes))
+    {
+        wperror(L"tcsetattr");
+    }
+
+    while (!finished && !data->end_loop)
+    {
+        if (0 < nchars && (size_t)nchars <= data->command_line.size())
+        {
+            // we've already hit the specified character limit
+            finished = 1;
+            break;
+        }
+
+        /*
+         Sometimes strange input sequences seem to generate a zero
+         byte. I believe these simply mean a character was pressed
+         but it should be ignored. (Example: Trying to add a tilde
+         (~) to digit)
+         */
+        while (1)
+        {
+            int was_interactive_read = is_interactive_read;
+            is_interactive_read = 1;
+            c=input_readch();
+            is_interactive_read = was_interactive_read;
+            //fprintf(stderr, "C: %lx\n", (long)c);
+
+            if (((!wchar_private(c))) && (c>31) && (c != 127))
+            {
+                if (can_read(0))
+                {
+
+                    wchar_t arr[READAHEAD_MAX+1];
+                    size_t i;
+                    size_t limit = 0 < nchars ? std::min((size_t)nchars - data->command_line.size(), (size_t)READAHEAD_MAX)
+                                              : READAHEAD_MAX;
+
+                    memset(arr, 0, sizeof(arr));
+                    arr[0] = c;
+
+                    for (i = 1; i < limit; ++i)
+                    {
+
+                        if (!can_read(0))
+                        {
+                            c = 0;
+                            break;
+                        }
+                        // only allow commands on the first key; otherwise, we might
+                        // have data we need to insert on the commandline that the
+                        // commmand might need to be able to see.
+                        c = input_readch(false);
+                        if ((!wchar_private(c)) && (c>31) && (c != 127))
+                        {
+                            arr[i]=c;
+                            c=0;
+                        }
+                        else
+                            break;
+                    }
+
+                    editable_line_t *el = data->active_edit_line();
+                    insert_string(el, arr, true);
+                    
+                    /* End paging upon inserting into the normal command line */
+                    if (el == &data->command_line)
+                    {
+                        clear_pager();
+                    }
+                    last_char = c;
+                }
+            }
+
+            if (c != 0)
+                break;
+
+            if (0 < nchars && (size_t)nchars <= data->command_line.size())
+            {
+                c = R_NULL;
+                break;
+            }
+        }
+
+        /* If we get something other than a repaint, then stop coalescing them */
+        if (c != R_REPAINT)
+            coalescing_repaints = false;
+
+        if (last_char != R_YANK && last_char != R_YANK_POP)
+            yank_len=0;
+
+        /* Restore the text */
+        if (c == R_CANCEL && data->is_navigating_pager_contents())
+        {
+            set_command_line_and_position(&data->command_line, data->cycle_command_line, data->cycle_cursor_pos);
+        }
+
+
+        /* Clear the pager if necessary */
+        bool focused_on_search_field = (data->active_edit_line() == &data->pager.search_field_line);
+        if (command_ends_paging(c, focused_on_search_field))
+        {
+            clear_pager();
+        }
+
+        //fprintf(stderr, "\n\nchar: %ls\n\n", describe_char(c).c_str());
+
+        switch (c)
+        {
+                /* go to beginning of line*/
+            case R_BEGINNING_OF_LINE:
+            {
+                editable_line_t *el = data->active_edit_line();
+                while (el->position > 0 && el->text.at(el->position - 1) != L'\n')
+                {
+                    update_buff_pos(el, el->position - 1);
+                }
+
+                reader_repaint_needed();
+                break;
+            }
+
+            case R_END_OF_LINE:
+            {
+                editable_line_t *el = data->active_edit_line();
+                if (el->position < el->size())
+                {
+                    const wchar_t *buff = el->text.c_str();
+                    while (buff[el->position] &&
+                            buff[el->position] != L'\n')
+                    {
+                        update_buff_pos(el, el->position + 1);
+                    }
+                }
+                else
+                {
+                    accept_autosuggestion(true);
+                }
+
+                reader_repaint_needed();
+                break;
+            }
+
+
+            case R_BEGINNING_OF_BUFFER:
+            {
+                update_buff_pos(&data->command_line, 0);
+                reader_repaint_needed();
+                break;
+            }
+
+            /* go to EOL*/
+            case R_END_OF_BUFFER:
+            {
+                update_buff_pos(&data->command_line, data->command_line.size());
+
+                reader_repaint_needed();
+                break;
+            }
+
+            case R_NULL:
+            {
+                break;
+            }
+
+            case R_CANCEL:
+            {
+                // The only thing we can cancel right now is paging, which we handled up above
+                break;
+            }
+
+            case R_FORCE_REPAINT:
+            case R_REPAINT:
+            {
+                if (! coalescing_repaints)
+                {
+                    coalescing_repaints = true;
+                    exec_prompt();
+                    s_reset(&data->screen, screen_reset_current_line_and_prompt);
+                    data->screen_reset_needed = false;
+                    reader_repaint();
+                }
+                break;
+            }
+
+            case R_EOF:
+            {
+                exit_forced = 1;
+                data->end_loop=1;
+                break;
+            }
+
+            /* complete */
+            case R_COMPLETE:
+            case R_COMPLETE_AND_SEARCH:
+            {
+
+                if (!data->complete_func)
+                    break;
+
+                /* Use the command line only; it doesn't make sense to complete in any other line */
+                editable_line_t *el = &data->command_line;
+                if (data->is_navigating_pager_contents() || (! comp_empty && last_char == R_COMPLETE))
+                {
+                    /* The user typed R_COMPLETE more than once in a row. If we are not yet fully disclosed, then become so; otherwise cycle through our available completions. */
+                    if (data->current_page_rendering.remaining_to_disclose > 0)
+                    {
+                        data->pager.set_fully_disclosed(true);
+                        reader_repaint_needed();
+                    }
+                    else
+                    {
+                        select_completion_in_direction(c == R_COMPLETE ? direction_next : direction_prev);
+                    }
+                }
+                else
+                {
+                    /* Either the user hit tab only once, or we had no visible completion list. */
+
+                    /* Remove a trailing backslash. This may trigger an extra repaint, but this is rare. */
+                    if (is_backslashed(el->text, el->position))
+                    {
+                        remove_backward();
+                    }
+
+                    /* Get the string; we have to do this after removing any trailing backslash */
+                    const wchar_t * const buff = el->text.c_str();
+
+                    /* Clear the completion list */
+                    comp.clear();
+
+                    /* Figure out the extent of the command substitution surrounding the cursor. This is because we only look at the current command substitution to form completions - stuff happening outside of it is not interesting. */
+                    const wchar_t *cmdsub_begin, *cmdsub_end;
+                    parse_util_cmdsubst_extent(buff, el->position, &cmdsub_begin, &cmdsub_end);
+
+                    /* Figure out the extent of the token within the command substitution. Note we pass cmdsub_begin here, not buff */
+                    const wchar_t *token_begin, *token_end;
+                    parse_util_token_extent(cmdsub_begin, el->position - (cmdsub_begin-buff), &token_begin, &token_end, 0, 0);
+
+                    /* Hack: the token may extend past the end of the command substitution, e.g. in (echo foo) the last token is 'foo)'. Don't let that happen. */
+                    if (token_end > cmdsub_end) token_end = cmdsub_end;
+
+                    /* Figure out how many steps to get from the current position to the end of the current token. */
+                    size_t end_of_token_offset = token_end - buff;
+
+                    /* Move the cursor to the end */
+                    if (el->position != end_of_token_offset)
+                    {
+                        update_buff_pos(el, end_of_token_offset);
+                        reader_repaint();
+                    }
+
+                    /* Construct a copy of the string from the beginning of the command substitution up to the end of the token we're completing */
+                    const wcstring buffcpy = wcstring(cmdsub_begin, token_end);
+
+                    //fprintf(stderr, "Complete (%ls)\n", buffcpy.c_str());
+                    data->complete_func(buffcpy, comp, COMPLETION_REQUEST_DEFAULT | COMPLETION_REQUEST_DESCRIPTIONS | COMPLETION_REQUEST_FUZZY_MATCH);
+
+                    /* Munge our completions */
+                    sort_and_make_unique(comp);
+                    prioritize_completions(comp);
+
+                    /* Record our cycle_command_line */
+                    data->cycle_command_line = el->text;
+                    data->cycle_cursor_pos = el->position;
+
+                    bool continue_after_prefix_insertion = (c == R_COMPLETE_AND_SEARCH);
+                    comp_empty = handle_completions(comp, continue_after_prefix_insertion);
+
+                    /* Show the search field if requested and if we printed a list of completions */
+                    if (c == R_COMPLETE_AND_SEARCH && ! comp_empty && ! data->pager.empty())
+                    {
+                        data->pager.set_search_field_shown(true);
+                        select_completion_in_direction(direction_next);
+                        reader_repaint_needed();
+                    }
+
+                }
+
+                break;
+            }
+
+            /* kill */
+            case R_KILL_LINE:
+            {
+                editable_line_t *el = data->active_edit_line();
+                const wchar_t *buff = el->text.c_str();
+                const wchar_t *begin = &buff[el->position];
+                const wchar_t *end = begin;
+
+                while (*end && *end != L'\n')
+                    end++;
+
+                if (end==begin && *end)
+                    end++;
+
+                size_t len = end-begin;
+                if (len)
+                {
+                    reader_kill(el, begin - buff, len, KILL_APPEND, last_char!=R_KILL_LINE);
+                }
+
+                break;
+            }
+
+            case R_BACKWARD_KILL_LINE:
+            {
+                editable_line_t *el = data->active_edit_line();
+                if (el->position > 0)
+                {
+                    const wchar_t *buff = el->text.c_str();
+                    const wchar_t *end = &buff[el->position];
+                    const wchar_t *begin = end;
+
+                    /* Make sure we delete at least one character (see #580) */
+                    begin--;
+
+                    /* Delete until we hit a newline, or the beginning of the string */
+                    while (begin > buff  && *begin != L'\n')
+                        begin--;
+
+                    /* If we landed on a newline, don't delete it */
+                    if (*begin == L'\n')
+                        begin++;
+
+                    assert(end >= begin);
+                    size_t len = maxi<size_t>(end-begin, 1);
+                    begin = end - len;
+
+                    reader_kill(el, begin - buff, len, KILL_PREPEND, last_char!=R_BACKWARD_KILL_LINE);
+                }
+                break;
+
+            }
+
+            case R_KILL_WHOLE_LINE:
+            {
+                /* We match the emacs behavior here: "kills the entire line including the following newline" */
+                editable_line_t *el = data->active_edit_line();
+                const wchar_t *buff = el->text.c_str();
+
+                /* Back up to the character just past the previous newline, or go to the beginning of the command line. Note that if the position is on a newline, visually this looks like the cursor is at the end of a line. Therefore that newline is NOT the beginning of a line; this justifies the -1 check. */
+                size_t begin = el->position;
+                while (begin > 0  && buff[begin-1] != L'\n')
+                {
+                    begin--;
+                }
+                
+                /* Push end forwards to just past the next newline, or just past the last char. */
+                size_t end = el->position;
+                while (buff[end] != L'\0')
+                {
+                    end++;
+                    if (buff[end-1] == L'\n')
+                    {
+                        break;
+                    }
+                }
+                assert(end >= begin);
+
+                if (end > begin)
+                {
+                    reader_kill(el, begin, end - begin, KILL_APPEND, last_char!=R_KILL_WHOLE_LINE);
+                }
+                break;
+            }
+
+            /* yank*/
+            case R_YANK:
+            {
+                yank_str = kill_yank();
+                insert_string(data->active_edit_line(), yank_str);
+                yank_len = wcslen(yank_str);
+                break;
+            }
+
+            /* rotate killring*/
+            case R_YANK_POP:
+            {
+                if (yank_len)
+                {
+                    for (size_t i=0; i<yank_len; i++)
+                        remove_backward();
+
+                    yank_str = kill_yank_rotate();
+                    insert_string(data->active_edit_line(), yank_str);
+                    yank_len = wcslen(yank_str);
+                }
+                break;
+            }
+
+            /* Escape was pressed */
+            case L'\x1b':
+            {
+                if (data->search_mode)
+                {
+                    data->search_mode= NO_SEARCH;
+
+                    if (data->token_history_pos==-1)
+                    {
+                        //history_reset();
+                        data->history_search.go_to_end();
+                        reader_set_buffer(data->search_buff, data->search_buff.size());
+                    }
+                    else
+                    {
+                        reader_replace_current_token(data->search_buff);
+                    }
+                    data->search_buff.clear();
+                    reader_super_highlight_me_plenty();
+                    reader_repaint_needed();
+                }
+
+                break;
+            }
+
+            /* delete backward*/
+            case R_BACKWARD_DELETE_CHAR:
+            {
+                remove_backward();
+                break;
+            }
+
+            /* delete forward*/
+            case R_DELETE_CHAR:
+            {
+                /**
+                 Remove the current character in the character buffer and on the
+                 screen using syntax highlighting, etc.
+                 */
+                editable_line_t *el = data->active_edit_line();
+                if (el->position < el->size())
+                {
+                    update_buff_pos(el, el->position + 1);
+                    remove_backward();
+                }
+                break;
+            }
+
+            /*
+             Evaluate. If the current command is unfinished, or if
+             the charater is escaped using a backslash, insert a
+             newline
+             */
+            case R_EXECUTE:
+            {
+                /* Delete any autosuggestion */
+                data->autosuggestion.clear();
+
+                /* If the user hits return while navigating the pager, it only clears the pager */
+                if (data->is_navigating_pager_contents())
+                {
+                    clear_pager();
+                    break;
+                }
+
+                /* The user may have hit return with pager contents, but while not navigating them. Clear the pager in that event. */
+                clear_pager();
+
+                /* We only execute the command line */
+                editable_line_t *el = &data->command_line;
+
+                /* Allow backslash-escaped newlines, but only if the following character is whitespace, or we're at the end of the text (see issue #613) and not in a comment (#1255). */
+                if (is_backslashed(el->text, el->position))
+                {
+                    bool continue_on_next_line = false;
+                    if (el->position >= el->size())
+                    {
+                        continue_on_next_line = ! text_ends_in_comment(el->text);
+                    }
+                    else
+                    {
+                        continue_on_next_line = iswspace(el->text.at(el->position));
+                    }
+                    if (continue_on_next_line)
+                    {
+                        insert_char(el, '\n');
+                        break;
+                    }
+                }
+
+                /* See if this command is valid */
+                int command_test_result = data->test_func(el->text.c_str());
+                if (command_test_result == 0 || command_test_result == PARSER_TEST_INCOMPLETE)
+                {
+                    /* This command is valid, but an abbreviation may make it invalid. If so, we will have to test again. */
+                    bool abbreviation_expanded = data->expand_abbreviation_as_necessary(1);
+                    if (abbreviation_expanded)
+                    {
+                        /* It's our reponsibility to rehighlight and repaint. But everything we do below triggers a repaint. */
+                        command_test_result = data->test_func(el->text.c_str());
+
+                        /* If the command is OK, then we're going to execute it. We still want to do syntax highlighting, but a synchronous variant that performs no I/O, so as not to block the user */
+                        bool skip_io = (command_test_result == 0);
+                        reader_super_highlight_me_plenty(0, skip_io);
+                    }
+                }
+
+                switch (command_test_result)
+                {
+
+                    case 0:
+                    {
+                        /* Finished command, execute it. Don't add items that start with a leading space. */
+                        const editable_line_t *el = &data->command_line;
+                        if (data->history != NULL && ! el->empty() && el->text.at(0) != L' ')
+                        {
+                            data->history->add_pending_with_file_detection(el->text);
+                        }
+                        finished=1;
+                        update_buff_pos(&data->command_line, data->command_line.size());
+                        reader_repaint();
+                        break;
+                    }
+
+                    /*
+                     We are incomplete, continue editing
+                     */
+                    case PARSER_TEST_INCOMPLETE:
+                    {
+                        insert_char(el, '\n');
+                        break;
+                    }
+
+                    /*
+                     Result must be some combination including an
+                     error. The error message will already be
+                     printed, all we need to do is repaint
+                     */
+                    default:
+                    {
+                        s_reset(&data->screen, screen_reset_abandon_line);
+                        reader_repaint_needed();
+                        break;
+                    }
+
+                }
+
+                break;
+            }
+
+            /* History functions */
+            case R_HISTORY_SEARCH_BACKWARD:
+            case R_HISTORY_TOKEN_SEARCH_BACKWARD:
+            case R_HISTORY_SEARCH_FORWARD:
+            case R_HISTORY_TOKEN_SEARCH_FORWARD:
+            {
+                int reset = 0;
+                if (data->search_mode == NO_SEARCH)
+                {
+                    reset = 1;
+                    if ((c == R_HISTORY_SEARCH_BACKWARD) ||
+                            (c == R_HISTORY_SEARCH_FORWARD))
+                    {
+                        data->search_mode = LINE_SEARCH;
+                    }
+                    else
+                    {
+                        data->search_mode = TOKEN_SEARCH;
+                    }
+
+                    const editable_line_t *el = &data->command_line;
+                    data->search_buff.append(el->text);
+                    data->history_search = history_search_t(*data->history, data->search_buff, HISTORY_SEARCH_TYPE_CONTAINS);
+
+                    /* Skip the autosuggestion as history unless it was truncated */
+                    const wcstring &suggest = data->autosuggestion;
+                    if (! suggest.empty() && ! data->screen.autosuggestion_is_truncated)
+                    {
+                        data->history_search.skip_matches(wcstring_list_t(&suggest, 1 + &suggest));
+                    }
+                }
+
+                switch (data->search_mode)
+                {
+                    case LINE_SEARCH:
+                    {
+                        if ((c == R_HISTORY_SEARCH_BACKWARD) ||
+                                (c == R_HISTORY_TOKEN_SEARCH_BACKWARD))
+                        {
+                            data->history_search.go_backwards();
+                        }
+                        else
+                        {
+                            if (! data->history_search.go_forwards())
+                            {
+                                /* If you try to go forwards past the end, we just go to the end */
+                                data->history_search.go_to_end();
+                            }
+                        }
+
+                        wcstring new_text;
+                        if (data->history_search.is_at_end())
+                        {
+                            new_text = data->search_buff;
+                        }
+                        else
+                        {
+                            new_text = data->history_search.current_string();
+                        }
+                        set_command_line_and_position(&data->command_line, new_text, new_text.size());
+
+                        break;
+                    }
+
+                    case TOKEN_SEARCH:
+                    {
+                        if ((c == R_HISTORY_SEARCH_BACKWARD) ||
+                                (c == R_HISTORY_TOKEN_SEARCH_BACKWARD))
+                        {
+                            handle_token_history(SEARCH_BACKWARD, reset);
+                        }
+                        else
+                        {
+                            handle_token_history(SEARCH_FORWARD, reset);
+                        }
+
+                        break;
+                    }
+
+                }
+                break;
+            }
+
+
+            /* Move left*/
+            case R_BACKWARD_CHAR:
+            {
+                editable_line_t *el = data->active_edit_line();
+                if (data->is_navigating_pager_contents())
+                {
+                    select_completion_in_direction(direction_west);
+                }
+                else if (el->position > 0)
+                {
+                    update_buff_pos(el, el->position-1);
+                    reader_repaint_needed();
+                }
+                break;
+            }
+
+            /* Move right*/
+            case R_FORWARD_CHAR:
+            {
+                editable_line_t *el = data->active_edit_line();
+                if (data->is_navigating_pager_contents())
+                {
+                    select_completion_in_direction(direction_east);
+                }
+                else if (el->position < el->size())
+                {
+                    update_buff_pos(el, el->position + 1);
+                    reader_repaint_needed();
+                }
+                else
+                {
+                    accept_autosuggestion(true);
+                }
+                break;
+            }
+
+            /* kill one word left */
+            case R_BACKWARD_KILL_WORD:
+            case R_BACKWARD_KILL_PATH_COMPONENT:
+            case R_BACKWARD_KILL_BIGWORD:
+            {
+                move_word_style_t style =
+                    (c == R_BACKWARD_KILL_BIGWORD ? move_word_style_whitespace :
+                     c == R_BACKWARD_KILL_PATH_COMPONENT ? move_word_style_path_components : move_word_style_punctuation);
+                bool newv = (last_char != R_BACKWARD_KILL_WORD && last_char != R_BACKWARD_KILL_PATH_COMPONENT && last_char != R_BACKWARD_KILL_BIGWORD);
+                move_word(data->active_edit_line(), MOVE_DIR_LEFT, true /* erase */, style, newv);
+                break;
+            }
+
+            /* kill one word right */
+            case R_KILL_WORD:
+            {
+                move_word(data->active_edit_line(), MOVE_DIR_RIGHT, true /* erase */, move_word_style_punctuation, last_char!=R_KILL_WORD);
+                break;
+            }
+
+            /* kill one bigword right */
+            case R_KILL_BIGWORD:
+            {
+                move_word(data->active_edit_line(), MOVE_DIR_RIGHT, true /* erase */, move_word_style_whitespace, last_char!=R_KILL_BIGWORD);
+                break;
+            }
+
+            /* move one word left*/
+            case R_BACKWARD_WORD:
+            {
+                move_word(data->active_edit_line(), MOVE_DIR_LEFT, false /* do not erase */, move_word_style_punctuation, false);
+                break;
+            }
+
+            /* move one bigword left */
+            case R_BACKWARD_BIGWORD:
+            {
+                move_word(data->active_edit_line(), MOVE_DIR_LEFT, false /* do not erase */, move_word_style_whitespace, false);
+                break;
+            }
+
+            /* move one word right*/
+            case R_FORWARD_WORD:
+            {
+                editable_line_t *el = data->active_edit_line();
+                if (el->position < el->size())
+                {
+                    move_word(el, MOVE_DIR_RIGHT, false /* do not erase */, move_word_style_punctuation, false);
+                }
+                else
+                {
+                    accept_autosuggestion(false /* accept only one word */);
+                }
+                break;
+            }
+
+            /* move one bigword right */
+            case R_FORWARD_BIGWORD:
+            {
+                editable_line_t *el = data->active_edit_line();
+                if (el->position < el->size())
+                {
+                    move_word(el, MOVE_DIR_RIGHT, false /* do not erase */, move_word_style_whitespace, false);
+                }
+                else
+                {
+                    accept_autosuggestion(false /* accept only one word */);
+                }
+                break;
+            }
+
+            case R_BEGINNING_OF_HISTORY:
+            {
+                if (data->is_navigating_pager_contents())
+                {
+                        select_completion_in_direction(direction_page_north);
+                } else {
+                    const editable_line_t *el = &data->command_line;
+                    data->history_search = history_search_t(*data->history, el->text, HISTORY_SEARCH_TYPE_PREFIX);
+                    data->history_search.go_to_beginning();
+                    if (! data->history_search.is_at_end())
+                    {
+                        wcstring new_text = data->history_search.current_string();
+                        set_command_line_and_position(&data->command_line, new_text, new_text.size());
+                    }
+                }
+                break;
+            }
+
+            case R_END_OF_HISTORY:
+            {
+                if (data->is_navigating_pager_contents())
+                {
+                    select_completion_in_direction(direction_page_south);
+                } else {
+                    data->history_search.go_to_end();
+                }
+                break;
+            }
+
+            case R_UP_LINE:
+            case R_DOWN_LINE:
+            {
+                if (data->is_navigating_pager_contents())
+                {
+                    /* We are already navigating pager contents. */
+                    selection_direction_t direction;
+                    if (c == R_DOWN_LINE)
+                    {
+                        /* Down arrow is always south */
+                        direction = direction_south;
+                    }
+                    else if (selection_is_at_top())
+                    {
+                        /* Up arrow, but we are in the first column and first row. End navigation */
+                        direction = direction_deselect;
+                    }
+                    else
+                    {
+                        /* Up arrow, go north */
+                        direction = direction_north;
+                    }
+
+                    /* Now do the selection */
+                    select_completion_in_direction(direction);
+                }
+                else if (c == R_DOWN_LINE && ! data->pager.empty())
+                {
+                    /* We pressed down with a non-empty pager contents, begin navigation */
+                    select_completion_in_direction(direction_south);
+                }
+                else
+                {
+                    /* Not navigating the pager contents */
+                    editable_line_t *el = data->active_edit_line();
+                    int line_old = parse_util_get_line_from_offset(el->text, el->position);
+                    int line_new;
+
+                    if (c == R_UP_LINE)
+                        line_new = line_old-1;
+                    else
+                        line_new = line_old+1;
+
+                    int line_count = parse_util_lineno(el->text.c_str(), el->size())-1;
+
+                    if (line_new >= 0 && line_new <= line_count)
+                    {
+                        size_t base_pos_new;
+                        size_t base_pos_old;
+
+                        int indent_old;
+                        int indent_new;
+                        size_t line_offset_old;
+                        size_t total_offset_new;
+
+                        base_pos_new = parse_util_get_offset_from_line(el->text, line_new);
+
+                        base_pos_old = parse_util_get_offset_from_line(el->text,  line_old);
+
+                        assert(base_pos_new != (size_t)(-1) && base_pos_old != (size_t)(-1));
+                        indent_old = data->indents.at(base_pos_old);
+                        indent_new = data->indents.at(base_pos_new);
+
+                        line_offset_old = el->position - parse_util_get_offset_from_line(el->text, line_old);
+                        total_offset_new = parse_util_get_offset(el->text, line_new, line_offset_old - 4*(indent_new-indent_old));
+                        update_buff_pos(el, total_offset_new);
+                        reader_repaint_needed();
+                    }
+                }
+
+                break;
+            }
+
+            case R_SUPPRESS_AUTOSUGGESTION:
+            {
+                data->suppress_autosuggestion = true;
+                data->autosuggestion.clear();
+                reader_repaint_needed();
+                break;
+            }
+
+            case R_ACCEPT_AUTOSUGGESTION:
+            {
+                accept_autosuggestion(true);
+                break;
+            }
+
+            case R_TRANSPOSE_CHARS:
+            {
+                editable_line_t *el = data->active_edit_line();
+                if (el->size() < 2)
+                {
+                    break;
+                }
+
+                /* If the cursor is at the end, transpose the last two characters of the line */
+                if (el->position == el->size())
+                {
+                    update_buff_pos(el, el->position - 1);
+                }
+
+                /*
+                 Drag the character before the cursor forward over the character at the cursor, moving
+                 the cursor forward as well.
+                 */
+                if (el->position > 0)
+                {
+                    wcstring local_cmd = el->text;
+                    std::swap(local_cmd.at(el->position), local_cmd.at(el->position-1));
+                    set_command_line_and_position(el, local_cmd, el->position + 1);
+                }
+                break;
+            }
+
+            case R_TRANSPOSE_WORDS:
+            {
+                editable_line_t *el = data->active_edit_line();
+                size_t len = el->size();
+                const wchar_t *buff = el->text.c_str();
+                const wchar_t *tok_begin, *tok_end, *prev_begin, *prev_end;
+
+                /* If we are not in a token, look for one ahead */
+                size_t buff_pos = el->position;
+                while (buff_pos != len && !iswalnum(buff[buff_pos]))
+                    buff_pos++;
+
+                update_buff_pos(el, buff_pos);
+
+                parse_util_token_extent(buff, el->position, &tok_begin, &tok_end, &prev_begin, &prev_end);
+
+                /* In case we didn't find a token at or after the cursor... */
+                if (tok_begin == &buff[len])
+                {
+                    /* ...retry beginning from the previous token */
+                    size_t pos = prev_end - &buff[0];
+                    parse_util_token_extent(buff, pos, &tok_begin, &tok_end, &prev_begin, &prev_end);
+                }
+
+                /* Make sure we have two tokens */
+                if (prev_begin < prev_end && tok_begin < tok_end && tok_begin > prev_begin)
+                {
+                    const wcstring prev(prev_begin, prev_end - prev_begin);
+                    const wcstring sep(prev_end, tok_begin - prev_end);
+                    const wcstring tok(tok_begin, tok_end - tok_begin);
+                    const wcstring trail(tok_end, &buff[len] - tok_end);
+
+                    /* Compose new command line with swapped tokens */
+                    wcstring new_buff(buff, prev_begin - buff);
+                    new_buff.append(tok);
+                    new_buff.append(sep);
+                    new_buff.append(prev);
+                    new_buff.append(trail);
+                    /* Put cursor right after the second token */
+                    set_command_line_and_position(el, new_buff, tok_end - buff);
+                }
+                break;
+            }
+
+            case R_UPCASE_WORD:
+            case R_DOWNCASE_WORD:
+            case R_CAPITALIZE_WORD:
+            {
+                editable_line_t *el = data->active_edit_line();
+                // For capitalize_word, whether we've capitalized a character so far
+                bool capitalized_first = false;
+
+                // We apply the operation from the current location to the end of the word
+                size_t pos = el->position;
+                move_word(el, MOVE_DIR_RIGHT, false, move_word_style_punctuation, false);
+                for (; pos < el->position; pos++)
+                {
+                    wchar_t chr = el->text.at(pos);
+
+                    // We always change the case; this decides whether we go uppercase (true) or lowercase (false)
+                    bool make_uppercase;
+                    if (c == R_CAPITALIZE_WORD)
+                        make_uppercase = ! capitalized_first && iswalnum(chr);
+                    else
+                        make_uppercase = (c == R_UPCASE_WORD);
+
+                    // Apply the operation and then record what we did
+                    if (make_uppercase)
+                        chr = towupper(chr);
+                    else
+                        chr = towlower(chr);
+
+                    data->command_line.text.at(pos) = chr;
+                    capitalized_first = capitalized_first || make_uppercase;
+                }
+                data->command_line_changed(el);
+                reader_super_highlight_me_plenty();
+                reader_repaint_needed();
+                break;
+            }
+
+            case R_BEGIN_SELECTION:
+            {
+                data->sel_active = true;
+                data->sel_begin_pos = data->command_line.position;
+                data->sel_start_pos = data->command_line.position;
+                data->sel_stop_pos = data->command_line.position;
+                break;
+            }
+
+            case R_END_SELECTION:
+            {
+                data->sel_active = false;
+                data->sel_start_pos = data->command_line.position;
+                data->sel_stop_pos = data->command_line.position;
+                break;
+            }
+
+            case R_KILL_SELECTION:
+            {
+                bool newv = (last_char != R_KILL_SELECTION);
+                size_t start, len;
+                if (reader_get_selection(&start, &len))
+                {
+                    reader_kill(&data->command_line, start, len, KILL_APPEND, newv);
+                }
+                break;
+            }
+
+            case R_FORWARD_JUMP:
+            {
+                editable_line_t *el = data->active_edit_line();
+                wchar_t target = input_function_pop_arg();
+                bool status = false;
+
+                for (size_t i = el->position + 1; i < el->size(); i++)
+                {
+                    if (el->at(i) == target)
+                    {
+                        update_buff_pos(el, i);
+                        status = true;
+                        break;
+                    }
+                }
+                input_function_set_status(status);
+                reader_repaint_needed();
+                break;
+            }
+
+            case R_BACKWARD_JUMP:
+            {
+                editable_line_t *el = data->active_edit_line();
+                wchar_t target = input_function_pop_arg();
+                bool status = false;
+
+                size_t tmp_pos = el->position;
+                while (tmp_pos--)
+                {
+                    if (el->at(tmp_pos) == target)
+                    {
+                        update_buff_pos(el, tmp_pos);
+                        status = true;
+                        break;
+                    }
+                }
+                input_function_set_status(status);
+                reader_repaint_needed();
+                break;
+            }
+
+            /* Other, if a normal character, we add it to the command */
+            default:
+            {
+                if ((!wchar_private(c)) && (((c>31) || (c==L'\n'))&& (c != 127)))
+                {
+                    bool allow_expand_abbreviations = false;
+                    if (data->is_navigating_pager_contents())
+                    {
+                        data->pager.set_search_field_shown(true);
+                    }
+                    else
+                    {
+                        allow_expand_abbreviations = true;
+                    }
+
+                    /* Regular character */
+                    editable_line_t *el = data->active_edit_line();
+                    insert_char(data->active_edit_line(), c, allow_expand_abbreviations);
+
+                    /* End paging upon inserting into the normal command line */
+                    if (el == &data->command_line)
+                    {
+                        clear_pager();
+                    }
+
+
+                }
+                else
+                {
+                    /*
+                     Low priority debug message. These can happen if
+                     the user presses an unefined control
+                     sequnece. No reason to report.
+                     */
+                    debug(2, _(L"Unknown keybinding %d"), c);
+                }
+                break;
+            }
+
+        }
+
+        if ((c != R_HISTORY_SEARCH_BACKWARD) &&
+                (c != R_HISTORY_SEARCH_FORWARD) &&
+                (c != R_HISTORY_TOKEN_SEARCH_BACKWARD) &&
+                (c != R_HISTORY_TOKEN_SEARCH_FORWARD) &&
+                (c != R_NULL) &&
+                (c != R_REPAINT) &&
+                (c != R_FORCE_REPAINT))
+        {
+            data->search_mode = NO_SEARCH;
+            data->search_buff.clear();
+            data->history_search.go_to_end();
+            data->token_history_pos=-1;
+        }
+
+        last_char = c;
+
+        reader_repaint_if_needed();
+    }
+
+    writestr(L"\n");
+
+    /* Ensure we have no pager contents when we exit */
+    if (! data->pager.empty())
+    {
+        /* Clear to end of screen to erase the pager contents. TODO: this may fail if eos doesn't exist, in which case we should emit newlines */
+        screen_force_clear_to_end();
+        data->pager.clear();
+    }
+
+    if (!reader_exit_forced())
+    {
+        if (tcsetattr(0,TCSANOW,&old_modes))      /* return to previous mode */
+        {
+            wperror(L"tcsetattr");
+        }
+
+        set_color(rgb_color_t::reset(), rgb_color_t::reset());
+    }
+
+    return finished ? data->command_line.text.c_str() : NULL;
+}
+
+int reader_search_mode()
+{
+    if (!data)
+    {
+        return -1;
+    }
+
+    return !!data->search_mode;
+}
+
+int reader_has_pager_contents()
+{
+    if (!data)
+    {
+        return -1;
+    }
+
+    return ! data->current_page_rendering.screen_data.empty();
+}
+
+/**
+   Read non-interactively.  Read input from stdin without displaying
+   the prompt, using syntax highlighting. This is used for reading
+   scripts and init files.
+*/
+static int read_ni(int fd, const io_chain_t &io)
+{
+    parser_t &parser = parser_t::principal_parser();
+    FILE *in_stream;
+    wchar_t *buff=0;
+    std::vector<char> acc;
+
+    int des = (fd == STDIN_FILENO ? dup(STDIN_FILENO) : fd);
+    int res=0;
+
+    if (des == -1)
+    {
+        wperror(L"dup");
+        return 1;
+    }
+
+    in_stream = fdopen(des, "r");
+    if (in_stream != 0)
+    {
+        while (!feof(in_stream))
+        {
+            char buff[4096];
+            size_t c = fread(buff, 1, 4096, in_stream);
+
+            if (ferror(in_stream))
+            {
+                if (errno == EINTR)
+                {
+                    /* We got a signal, just keep going. Be sure that we call insert() below because we may get data as well as EINTR. */
+                    clearerr(in_stream);
+                }
+                else if ((errno == EAGAIN || errno == EWOULDBLOCK) && make_fd_blocking(des) == 0)
+                {
+                    /* We succeeded in making the fd blocking, keep going */
+                    clearerr(in_stream);
+                }
+                else
+                {
+                    /* Fatal error */
+                    debug(1, _(L"Error while reading from file descriptor"));
+
+                    /* Reset buffer on error. We won't evaluate incomplete files. */
+                    acc.clear();
+                    break;
+                }
+            }
+
+            acc.insert(acc.end(), buff, buff + c);
+        }
+
+        wcstring str = acc.empty() ? wcstring() : str2wcstring(&acc.at(0), acc.size());
+        acc.clear();
+
+        if (fclose(in_stream))
+        {
+            debug(1,
+                  _(L"Error while closing input stream"));
+            wperror(L"fclose");
+            res = 1;
+        }
+
+        /* Swallow a BOM (#1518) */
+        if (! str.empty() && str.at(0) == UTF8_BOM_WCHAR)
+        {
+            str.erase(0, 1);
+        }
+
+        parse_error_list_t errors;
+        if (! parse_util_detect_errors(str, &errors, false /* do not accept incomplete */))
+        {
+            parser.eval(str, io, TOP);
+        }
+        else
+        {
+            wcstring sb;
+            parser.get_backtrace(str, errors, &sb);
+            fwprintf(stderr, L"%ls", sb.c_str());
+            res = 1;
+        }
+    }
+    else
+    {
+        debug(1,
+              _(L"Error while opening input stream"));
+        wperror(L"fdopen");
+        free(buff);
+        res=1;
+    }
+    return res;
+}
+
+int reader_read(int fd, const io_chain_t &io)
+{
+    int res;
+
+    /*
+      If reader_read is called recursively through the '.' builtin, we
+      need to preserve is_interactive. This, and signal handler setup
+      is handled by proc_push_interactive/proc_pop_interactive.
+    */
+
+    int inter = ((fd == STDIN_FILENO) && isatty(STDIN_FILENO));
+    proc_push_interactive(inter);
+
+    res= get_is_interactive() ? read_i():read_ni(fd, io);
+
+    /*
+      If the exit command was called in a script, only exit the
+      script, not the program.
+    */
+    if (data)
+        data->end_loop = 0;
+    end_loop = 0;
+
+    proc_pop_interactive();
+    return res;
+}
diff --git a/src/reader.h b/src/reader.h
new file mode 100644
index 00000000..b4a159be
--- /dev/null
+++ b/src/reader.h
@@ -0,0 +1,319 @@
+/** \file reader.h
+
+    Prototypes for functions for reading data from stdin and passing
+	to the parser. If stdin is a keyboard, it supplies a killring,
+	history, syntax highlighting, tab-completion and various other
+	features.
+*/
+
+#ifndef FISH_READER_H
+#define FISH_READER_H
+
+#include <vector>
+#include <string>
+#include <stddef.h>
+
+#include "io.h"
+#include "common.h"
+#include "complete.h"
+#include "highlight.h"
+#include "parse_constants.h"
+
+class history_t;
+
+/* Helper class for storing a command line */
+class editable_line_t
+{
+public:
+
+    /** The command line */
+    wcstring text;
+
+    /** The current position of the cursor in the command line */
+    size_t position;
+
+    const wcstring &get_text() const
+    {
+        return text;
+    }
+
+    /* Gets the length of the text */
+    size_t size() const
+    {
+        return text.size();
+    }
+
+    bool empty() const
+    {
+        return text.empty();
+    }
+
+    void clear()
+    {
+        text.clear();
+        position = 0;
+    }
+
+    wchar_t at(size_t idx)
+    {
+        return text.at(idx);
+    }
+
+    editable_line_t() : text(), position(0)
+    {
+    }
+
+    /* Inserts a substring of str given by start, len at the cursor position */
+    void insert_string(const wcstring &str, size_t start = 0, size_t len = wcstring::npos);
+};
+
+/**
+  Read commands from \c fd until encountering EOF
+*/
+int reader_read(int fd, const io_chain_t &io);
+
+/**
+  Tell the shell that it should exit after the currently running command finishes.
+*/
+void reader_exit(int do_exit, int force);
+
+/**
+   Check that the reader is in a sane state
+*/
+void reader_sanity_check();
+
+/**
+   Initialize the reader
+*/
+void reader_init();
+
+/**
+   Destroy and free resources used by the reader
+*/
+void reader_destroy();
+
+/** Restore the term mode at startup */
+void restore_term_mode();
+
+/**
+   Returns the filename of the file currently read
+*/
+const wchar_t *reader_current_filename();
+
+/**
+   Push a new filename on the stack of read files
+
+   \param fn The fileanme to push
+*/
+void reader_push_current_filename(const wchar_t *fn);
+/**
+   Pop the current filename from the stack of read files
+ */
+void reader_pop_current_filename();
+
+/**
+   Write the title to the titlebar. This function is called just
+   before a new application starts executing and just after it
+   finishes.
+
+   \param cmd Command line string passed to \c fish_title if is defined.
+*/
+void reader_write_title(const wcstring &cmd);
+
+/**
+   Call this function to tell the reader that a repaint is needed, and
+   should be performed when possible.
+ */
+void reader_repaint_needed();
+
+/** Call this function to tell the reader that some color has changed. */
+void reader_react_to_color_change();
+
+/* Repaint immediately if needed. */
+void reader_repaint_if_needed();
+
+/**
+   Run the specified command with the correct terminal modes, and
+   while taking care to perform job notification, set the title, etc.
+*/
+void reader_run_command(const wcstring &buff);
+
+/**
+   Get the string of character currently entered into the command
+   buffer, or 0 if interactive mode is uninitialized.
+*/
+const wchar_t *reader_get_buffer();
+
+/** Returns the current reader's history */
+history_t *reader_get_history(void);
+
+/**
+   Set the string of characters in the command buffer, as well as the cursor position.
+
+   \param b the new buffer value
+   \param p the cursor position. If \c p is larger than the length of the command line,
+            the cursor is placed on the last character.
+*/
+void reader_set_buffer(const wcstring &b, size_t p);
+
+/**
+   Get the current cursor position in the command line. If interactive
+   mode is uninitialized, return (size_t)(-1).
+*/
+size_t reader_get_cursor_pos();
+
+
+/**
+   Get the current selection range in the command line.
+   Returns false if there is no active selection, true otherwise.
+*/
+bool reader_get_selection(size_t *start, size_t *len);
+
+/**
+   Return the value of the interrupted flag, which is set by the sigint
+   handler, and clear it if it was set.
+*/
+int reader_interrupted();
+
+/**
+   Clear the interrupted flag unconditionally without handling anything. The
+   flag could have been set e.g. when an interrupt arrived just as we were
+   ending an earlier \c reader_readline invocation but before the
+   \c is_interactive_read flag was cleared.
+*/
+void reader_reset_interrupted();
+
+/**
+   Return the value of the interrupted flag, which is set by the sigint
+   handler, and clear it if it was set. If the current reader is interruptible,
+   call \c reader_exit().
+*/
+int reader_reading_interrupted();
+
+/**
+   Returns true if the current reader generation count does not equal the
+   generation count the current thread was started with.
+   Note 1: currently only valid for autocompletion threads! Other threads don't
+   set the threadlocal generation count when they start up.
+*/
+bool reader_thread_job_is_stale();
+
+/**
+   Read one line of input. Before calling this function, reader_push() must have
+   been called in order to set up a valid reader environment. If nchars > 0,
+   return after reading that many characters even if a full line has not yet
+   been read. Note: the returned value may be longer than nchars if a single
+   keypress resulted in multiple characters being inserted into the commandline.
+*/
+const wchar_t *reader_readline(int nchars);
+
+/**
+   Push a new reader environment.
+*/
+void reader_push(const wchar_t *name);
+
+/**
+   Return to previous reader environment
+*/
+void reader_pop();
+
+/**
+   Specify function to use for finding possible tab completions. The function must take these arguments:
+
+   - The command to be completed as a null terminated array of wchar_t
+   - An array_list_t in which completions will be inserted.
+*/
+typedef void (*complete_function_t)(const wcstring &, std::vector<completion_t> &, completion_request_flags_t);
+void reader_set_complete_function(complete_function_t);
+
+/**
+ The type of a highlight function.
+ */
+class env_vars_snapshot_t;
+typedef void (*highlight_function_t)(const wcstring &, std::vector<highlight_spec_t> &, size_t, wcstring_list_t *, const env_vars_snapshot_t &vars);
+
+/**
+ Specify function for syntax highlighting. The function must take these arguments:
+
+ - The command to be highlighted as a null terminated array of wchar_t
+ - The color code of each character as an array of ints
+ - The cursor position
+ - An array_list_t used for storing error messages
+ */
+void reader_set_highlight_function(highlight_function_t);
+
+/**
+   Specify function for testing if the command buffer contains syntax
+   errors that must be corrected before returning.
+*/
+void reader_set_test_function(parser_test_error_bits_t (*f)(const wchar_t *));
+
+/**
+   Specify string of shell commands to be run in order to generate the
+   prompt.
+*/
+void reader_set_left_prompt(const wcstring &prompt);
+
+/**
+   Specify string of shell commands to be run in order to generate the
+   right prompt.
+*/
+void reader_set_right_prompt(const wcstring &prompt);
+
+
+/** Sets whether autosuggesting is allowed. */
+void reader_set_allow_autosuggesting(bool flag);
+
+/** Sets whether abbreviation expansion is performed. */
+void reader_set_expand_abbreviations(bool flag);
+
+
+/** Sets whether the reader should exit on ^C. */
+void reader_set_exit_on_interrupt(bool flag);
+
+/**
+   Returns true if the shell is exiting, 0 otherwise.
+*/
+bool shell_is_exiting();
+
+/**
+   The readers interrupt signal handler. Cancels all currently running blocks.
+*/
+void reader_handle_int(int signal);
+
+/**
+   This function returns true if fish is exiting by force, i.e. because stdin died
+*/
+int reader_exit_forced();
+
+/**
+   Test if the given shell command contains errors. Uses parser_test
+   for testing. Suitable for reader_set_test_function().
+*/
+parser_test_error_bits_t reader_shell_test(const wchar_t *b);
+
+/**
+   Test whether the interactive reader is in search mode.
+
+   \return 0 if not in search mode, 1 if in search mode and -1 if not in interactive mode
+ */
+int reader_search_mode();
+
+/**
+   Test whether the interactive reader has visible pager contents.
+
+   \return 0 if it has pager contents, 1 if it does not have pager contents, and -1 if not in interactive mode
+ */
+int reader_has_pager_contents();
+
+
+/* Given a command line and an autosuggestion, return the string that gets shown to the user. Exposed for testing purposes only. */
+wcstring combine_command_and_autosuggestion(const wcstring &cmdline, const wcstring &autosuggestion);
+
+/* Expand abbreviations at the given cursor position. Exposed for testing purposes only. */
+bool reader_expand_abbreviation_in_command(const wcstring &cmdline, size_t cursor_pos, wcstring *output);
+
+/* Apply a completion string. Exposed for testing only. */
+wcstring completion_apply_to_command_line(const wcstring &val_str, complete_flags_t flags, const wcstring &command_line, size_t *inout_cursor_pos, bool append_only);
+
+#endif
diff --git a/src/sanity.cpp b/src/sanity.cpp
new file mode 100644
index 00000000..e918c805
--- /dev/null
+++ b/src/sanity.cpp
@@ -0,0 +1,63 @@
+/** \file sanity.c
+  Functions for performing sanity checks on the program state
+*/
+#include "config.h" // IWYU pragma: keep
+
+#include <unistd.h>
+
+#include "fallback.h" // IWYU pragma: keep
+#include "common.h"
+#include "sanity.h"
+#include "proc.h"
+#include "history.h"
+#include "reader.h"
+#include "kill.h"
+
+
+/**
+   Status from earlier sanity checks
+*/
+static int insane;
+
+void sanity_lose()
+{
+    debug(0, _(L"Errors detected, shutting down. Break on sanity_lose() to debug."));
+    insane = 1;
+}
+
+int sanity_check()
+{
+    if (!insane)
+        if (get_is_interactive())
+            history_sanity_check();
+    if (!insane)
+        reader_sanity_check();
+    if (!insane)
+        kill_sanity_check();
+    if (!insane)
+        proc_sanity_check();
+
+    return insane;
+}
+
+void validate_pointer(const void *ptr, const wchar_t *err, int null_ok)
+{
+
+    /*
+       Test if the pointer data crosses a segment boundary.
+    */
+
+    if ((0x00000003l & (intptr_t)ptr) != 0)
+    {
+        debug(0, _(L"The pointer '%ls' is invalid"), err);
+        sanity_lose();
+    }
+
+    if ((!null_ok) && (ptr==0))
+    {
+        debug(0, _(L"The pointer '%ls' is null"), err);
+        sanity_lose();
+    }
+}
+
+
diff --git a/src/sanity.h b/src/sanity.h
new file mode 100644
index 00000000..7480c403
--- /dev/null
+++ b/src/sanity.h
@@ -0,0 +1,27 @@
+/** \file sanity.h
+  Prototypes for functions for performing sanity checks on the program state
+*/
+
+#ifndef FISH_SANITY_H
+#define FISH_SANITY_H
+
+/**
+  Call this function to tell the program it is not in a sane state.
+*/
+void sanity_lose();
+
+/**
+  Perform sanity checks, return 1 if program is in a sane state 0 otherwise.
+*/
+int sanity_check();
+
+/**
+  Try and determine if ptr is a valid pointer. If not, loose sanity.
+
+  \param ptr The pointer to validate
+  \param err A description of what the pointer refers to, for use in error messages
+  \param null_ok Wheter the pointer is allowed to point to 0
+*/
+void validate_pointer(const void *ptr, const wchar_t *err, int null_ok);
+
+#endif
diff --git a/src/screen.cpp b/src/screen.cpp
new file mode 100644
index 00000000..1971bb57
--- /dev/null
+++ b/src/screen.cpp
@@ -0,0 +1,1497 @@
+/** \file screen.c High level library for handling the terminal screen
+
+The screen library allows the interactive reader to write its
+output to screen efficiently by keeping an internal representation
+of the current screen contents and trying to find the most
+efficient way for transforming that to the desired screen content.
+*/
+
+#include "config.h"
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+
+#include <unistd.h>
+
+#if HAVE_NCURSES_H
+#include <ncurses.h>
+#elif HAVE_NCURSES_CURSES_H
+#include <ncurses/curses.h>
+#else
+#include <curses.h>
+#endif
+
+#if HAVE_TERM_H
+#include <term.h>
+#elif HAVE_NCURSES_TERM_H
+#include <ncurses/term.h>
+#endif
+
+#include <wchar.h>
+#include <time.h>
+
+#include <assert.h>
+#include <algorithm>
+#include <string>
+#include <vector>
+
+
+#include "fallback.h"
+#include "common.h"
+#include "util.h"
+#include "output.h"
+#include "highlight.h"
+#include "screen.h"
+#include "env.h"
+#include "pager.h"
+
+/** The number of characters to indent new blocks */
+#define INDENT_STEP 4u
+
+/** The initial screen width */
+#define SCREEN_WIDTH_UNINITIALIZED -1
+
+/** A helper value for an invalid location */
+#define INVALID_LOCATION (screen_data_t::cursor_t(-1, -1))
+
+static void invalidate_soft_wrap(screen_t *scr);
+
+/**
+   Ugly kludge. The internal buffer used to store output of
+   tputs. Since tputs external function can only take an integer and
+   not a pointer as parameter we need a static storage buffer.
+*/
+typedef std::vector<char> data_buffer_t;
+static data_buffer_t *s_writeb_buffer=0;
+
+static int s_writeb(char c);
+
+/* Class to temporarily set s_writeb_buffer and the writer function in a scoped way */
+class scoped_buffer_t
+{
+    data_buffer_t * const old_buff;
+    int (* const old_writer)(char);
+
+public:
+    scoped_buffer_t(data_buffer_t *buff) : old_buff(s_writeb_buffer), old_writer(output_get_writer())
+    {
+        s_writeb_buffer = buff;
+        output_set_writer(s_writeb);
+    }
+
+    ~scoped_buffer_t()
+    {
+        s_writeb_buffer = old_buff;
+        output_set_writer(old_writer);
+    }
+};
+
+/**
+   Tests if the specified narrow character sequence is present at the
+   specified position of the specified wide character string. All of
+   \c seq must match, but str may be longer than seq.
+*/
+static size_t try_sequence(const char *seq, const wchar_t *str)
+{
+    for (size_t i=0; ; i++)
+    {
+        if (!seq[i])
+            return i;
+
+        if (seq[i] != str[i])
+            return 0;
+    }
+
+    return 0;
+}
+
+/**
+   Returns the number of columns left until the next tab stop, given
+   the current cursor postion.
+ */
+static size_t next_tab_stop(size_t in)
+{
+    /*
+      Assume tab stops every 8 characters if undefined
+    */
+    size_t tab_width = (init_tabs > 0 ? (size_t)init_tabs : 8);
+    return ((in/tab_width)+1)*tab_width;
+}
+
+/* Like fish_wcwidth, but returns 0 for control characters instead of -1 */
+static int fish_wcwidth_min_0(wchar_t wc)
+{
+    return maxi(0, fish_wcwidth(wc));
+}
+
+/* Whether we permit soft wrapping. If so, in some cases we don't explicitly move to the second physical line on a wrapped logical line; instead we just output it. */
+static bool allow_soft_wrap(void)
+{
+    // Should we be looking at eat_newline_glitch as well?
+    return !! auto_right_margin;
+}
+
+
+/* Returns the number of characters in the escape code starting at 'code' (which should initially contain \x1b) */
+size_t escape_code_length(const wchar_t *code)
+{
+    assert(code != NULL);
+
+    /* The only escape codes we recognize start with \x1b */
+    if (code[0] != L'\x1b')
+        return 0;
+
+    size_t resulting_length = 0;
+    bool found = false;
+
+    if (cur_term != NULL)
+    {
+        /*
+         Detect these terminfo color escapes with parameter
+         value 0..7, all of which don't move the cursor
+         */
+        char * const esc[] =
+        {
+            set_a_foreground,
+            set_a_background,
+            set_foreground,
+            set_background,
+        };
+
+        for (size_t p=0; p < sizeof esc / sizeof *esc && !found; p++)
+        {
+            if (!esc[p])
+                continue;
+
+            for (size_t k=0; k<8; k++)
+            {
+                size_t len = try_sequence(tparm(esc[p],k), code);
+                if (len)
+                {
+                    resulting_length = len;
+                    found = true;
+                    break;
+                }
+            }
+        }
+    }
+
+    if (cur_term != NULL)
+    {
+        /*
+         Detect these semi-common terminfo escapes without any
+         parameter values, all of which don't move the cursor
+         */
+        char * const esc2[] =
+        {
+            enter_bold_mode,
+            exit_attribute_mode,
+            enter_underline_mode,
+            exit_underline_mode,
+            enter_standout_mode,
+            exit_standout_mode,
+            flash_screen,
+            enter_subscript_mode,
+            exit_subscript_mode,
+            enter_superscript_mode,
+            exit_superscript_mode,
+            enter_blink_mode,
+            enter_italics_mode,
+            exit_italics_mode,
+            enter_reverse_mode,
+            enter_shadow_mode,
+            exit_shadow_mode,
+            enter_standout_mode,
+            exit_standout_mode,
+            enter_secure_mode
+        };
+
+
+
+        for (size_t p=0; p < sizeof esc2 / sizeof *esc2 && !found; p++)
+        {
+            if (!esc2[p])
+                continue;
+            /*
+             Test both padded and unpadded version, just to
+             be safe. Most versions of tparm don't actually
+             seem to do anything these days.
+             */
+            size_t len = maxi(try_sequence(tparm(esc2[p]), code), try_sequence(esc2[p], code));
+            if (len)
+            {
+                resulting_length = len;
+                found = true;
+            }
+        }
+    }
+
+    if (!found)
+    {
+        if (code[1] == L'k')
+        {
+            /* This looks like the escape sequence for setting a screen name */
+            const env_var_t term_name = env_get_string(L"TERM");
+            if (!term_name.missing() && string_prefixes_string(L"screen", term_name))
+            {
+                const wchar_t * const screen_name_end_sentinel = L"\x1b\\";
+                const wchar_t *screen_name_end = wcsstr(&code[2], screen_name_end_sentinel);
+                if (screen_name_end != NULL)
+                {
+                    const wchar_t *escape_sequence_end = screen_name_end + wcslen(screen_name_end_sentinel);
+                    resulting_length = escape_sequence_end - code;
+                }
+                else
+                {
+                    /* Consider just <esc>k to be the code */
+                    resulting_length = 2;
+                }
+                found = true;
+            }
+        }
+    }
+    
+    if (! found)
+    {
+        /* iTerm2 escape codes: CSI followed by ], terminated by either BEL or escape + backslash. See https://code.google.com/p/iterm2/wiki/ProprietaryEscapeCodes */
+        if (code[1] == ']')
+        {
+            // Start at 2 to skip over <esc>]
+            size_t cursor = 2;
+            for (; code[cursor] != L'\0'; cursor++)
+            {
+                /* Consume a sequence of characters up to <esc>\ or <bel> */
+                if (code[cursor] == '\x07' || (code[cursor] == '\\' && code[cursor - 1] == '\x1b'))
+                {
+                    found = true;
+                    break;
+                }
+            }
+            if (found)
+            {
+                resulting_length = cursor + 1;
+            }
+        }
+    }
+
+    if (! found)
+    {
+        /* Generic VT100 one byte sequence: CSI followed by something in the range @ through _ */
+        if (code[1] == L'[' && (code[2] >= L'@' && code[2] <= L'_'))
+        {
+            resulting_length = 3;
+            found = true;
+        }
+    }
+
+    if (! found)
+    {
+        /* Generic VT100 CSI-style sequence. <esc>, followed by zero or more ASCII characters NOT in the range [@,_], followed by one character in that range */
+        if (code[1] == L'[')
+        {
+            // Start at 2 to skip over <esc>[
+            size_t cursor = 2;
+            for (; code[cursor] != L'\0'; cursor++)
+            {
+                /* Consume a sequence of ASCII characters not in the range [@, ~] */
+                wchar_t c = code[cursor];
+
+                /* If we're not in ASCII, just stop */
+                if (c > 127)
+                    break;
+
+                /* If we're the end character, then consume it and then stop */
+                if (c >= L'@' && c <= L'~')
+                {
+                    cursor++;
+                    break;
+                }
+            }
+            /* curs now indexes just beyond the end of the sequence (or at the terminating zero) */
+            found = true;
+            resulting_length = cursor;
+        }
+    }
+    if (! found)
+    {
+        /* Generic VT100 two byte sequence: <esc> followed by something in the range @ through _ */
+        if (code[1] >= L'@' && code[1] <= L'_')
+        {
+            resulting_length = 2;
+            found = true;
+        }
+    }
+
+    return resulting_length;
+}
+
+/* Information about a prompt layout */
+struct prompt_layout_t
+{
+    /* How many lines the prompt consumes */
+    size_t line_count;
+
+    /* Width of the longest line */
+    size_t max_line_width;
+
+    /* Width of the last line */
+    size_t last_line_width;
+};
+
+/**
+   Calculate layout information for the given prompt. Does some clever magic
+   to detect common escape sequences that may be embeded in a prompt,
+   such as color codes.
+*/
+static prompt_layout_t calc_prompt_layout(const wchar_t *prompt)
+{
+    size_t current_line_width = 0;
+    size_t j;
+
+    prompt_layout_t prompt_layout = {};
+    prompt_layout.line_count = 1;
+
+    for (j=0; prompt[j]; j++)
+    {
+        if (prompt[j] == L'\x1b')
+        {
+            /* This is the start of an escape code. Skip over it if it's at least one character long. */
+            size_t escape_len = escape_code_length(&prompt[j]);
+            if (escape_len > 0)
+            {
+                j += escape_len - 1;
+            }
+        }
+        else if (prompt[j] == L'\t')
+        {
+            current_line_width = next_tab_stop(current_line_width);
+        }
+        else if (prompt[j] == L'\n' || prompt[j] == L'\f')
+        {
+            /* PCA: At least one prompt uses \f\r as a newline. It's unclear to me what this is meant to do, but terminals seem to treat it as a newline so we do the same. */
+            current_line_width = 0;
+            prompt_layout.line_count += 1;
+        }
+        else if (prompt[j] == L'\r')
+        {
+            current_line_width = 0;
+        }
+        else
+        {
+            /* Ordinary decent character. Just add width. This returns -1 for a control character - don't add that. */
+            current_line_width += fish_wcwidth_min_0(prompt[j]);
+            prompt_layout.max_line_width = maxi(prompt_layout.max_line_width, current_line_width);
+        }
+    }
+    prompt_layout.last_line_width = current_line_width;
+    return prompt_layout;
+}
+
+static size_t calc_prompt_lines(const wcstring &prompt)
+{
+    // Hack for the common case where there's no newline at all
+    // I don't know if a newline can appear in an escape sequence,
+    // so if we detect a newline we have to defer to calc_prompt_width_and_lines
+    size_t result = 1;
+    if (prompt.find(L'\n') != wcstring::npos || prompt.find(L'\f') != wcstring::npos)
+    {
+        result = calc_prompt_layout(prompt.c_str()).line_count;
+    }
+    return result;
+}
+
+/**
+   Stat stdout and stderr and save result.
+
+   This should be done before calling a function that may cause output.
+*/
+
+static void s_save_status(screen_t *s)
+{
+
+    // PCA Let's not do this futimes stuff, because sudo dumbly uses the
+    // tty's ctime as part of its tty_tickets feature
+    // Disabling this should fix https://github.com/fish-shell/fish-shell/issues/122
+#if 0
+    /*
+      This futimes call tries to trick the system into using st_mtime
+      as a tampering flag. This of course only works on systems where
+      futimes is defined, but it should make the status saving stuff
+      failsafe.
+    */
+    struct timeval t[]=
+    {
+        {
+            time(0)-1,
+            0
+        }
+        ,
+        {
+            time(0)-1,
+            0
+        }
+    }
+    ;
+
+    /*
+      Don't check return value on these. We don't care if they fail,
+      really.  This is all just to make the prompt look ok, which is
+      impossible to do 100% reliably. We try, at least.
+    */
+    futimes(1, t);
+    futimes(2, t);
+#endif
+
+    fstat(1, &s->prev_buff_1);
+    fstat(2, &s->prev_buff_2);
+}
+
+/**
+   Stat stdout and stderr and compare result to previous result in
+   reader_save_status. Repaint if modification time has changed.
+
+   Unfortunately, for some reason this call seems to give a lot of
+   false positives, at least under Linux.
+*/
+
+static void s_check_status(screen_t *s)
+{
+    fflush(stdout);
+    fflush(stderr);
+
+    fstat(1, &s->post_buff_1);
+    fstat(2, &s->post_buff_2);
+
+    int changed = (s->prev_buff_1.st_mtime != s->post_buff_1.st_mtime) ||
+                  (s->prev_buff_2.st_mtime != s->post_buff_2.st_mtime);
+
+    #if defined HAVE_STRUCT_STAT_ST_MTIMESPEC_TV_NSEC
+        changed = changed || s->prev_buff_1.st_mtimespec.tv_nsec != s->post_buff_1.st_mtimespec.tv_nsec ||
+                    s->prev_buff_2.st_mtimespec.tv_nsec != s->post_buff_2.st_mtimespec.tv_nsec;
+    #elif defined HAVE_STRUCT_STAT_ST_MTIM_TV_NSEC
+        changed = changed || s->prev_buff_1.st_mtim.tv_nsec != s->post_buff_1.st_mtim.tv_nsec ||
+                    s->prev_buff_2.st_mtim.tv_nsec != s->post_buff_2.st_mtim.tv_nsec;
+    #endif
+
+    if (changed)
+    {
+        /*
+          Ok, someone has been messing with our screen. We will want
+          to repaint. However, we do not know where the cursor is. It
+          is our best bet that we are still on the same line, so we
+          move to the beginning of the line, reset the modelled screen
+          contents, and then set the modeled cursor y-pos to its
+          earlier value.
+        */
+
+        int prev_line = s->actual.cursor.y;
+        write_loop(STDOUT_FILENO, "\r", 1);
+        s_reset(s, screen_reset_current_line_and_prompt);
+        s->actual.cursor.y = prev_line;
+    }
+}
+
+/**
+   Appends a character to the end of the line that the output cursor is
+   on. This function automatically handles linebreaks and lines longer
+   than the screen width.
+*/
+static void s_desired_append_char(screen_t *s,
+                                  wchar_t b,
+                                  int c,
+                                  int indent,
+                                  size_t prompt_width)
+{
+    int line_no = s->desired.cursor.y;
+
+    switch (b)
+    {
+        case L'\n':
+        {
+            int i;
+            /* Current line is definitely hard wrapped */
+            s->desired.create_line(s->desired.line_count());
+            s->desired.line(s->desired.cursor.y).is_soft_wrapped = false;
+            s->desired.cursor.y++;
+            s->desired.cursor.x=0;
+            for (i=0; i < prompt_width+indent*INDENT_STEP; i++)
+            {
+                s_desired_append_char(s, L' ', 0, indent, prompt_width);
+            }
+            break;
+        }
+
+        case L'\r':
+        {
+            line_t &current = s->desired.line(line_no);
+            current.clear();
+            s->desired.cursor.x = 0;
+            break;
+        }
+
+        default:
+        {
+            int screen_width = common_get_width();
+            int cw = fish_wcwidth_min_0(b);
+
+            s->desired.create_line(line_no);
+
+            /*
+                   Check if we are at the end of the line. If so, continue on the next line.
+                   */
+            if ((s->desired.cursor.x + cw) > screen_width)
+            {
+                /* Current line is soft wrapped (assuming we support it) */
+                s->desired.line(s->desired.cursor.y).is_soft_wrapped = true;
+                //fprintf(stderr, "\n\n1 Soft wrapping %d\n\n", s->desired.cursor.y);
+
+                line_no = (int)s->desired.line_count();
+                s->desired.add_line();
+                s->desired.cursor.y++;
+                s->desired.cursor.x=0;
+            }
+
+            line_t &line = s->desired.line(line_no);
+            line.append(b, c);
+            s->desired.cursor.x+= cw;
+
+            /* Maybe wrap the cursor to the next line, even if the line itself did not wrap. This avoids wonkiness in the last column. */
+            if (s->desired.cursor.x >= screen_width)
+            {
+                line.is_soft_wrapped = true;
+                s->desired.cursor.x = 0;
+                s->desired.cursor.y++;
+            }
+            break;
+        }
+    }
+
+}
+
+/**
+   The writeb function offered to tputs.
+*/
+static int s_writeb(char c)
+{
+    s_writeb_buffer->push_back(c);
+    return 0;
+}
+
+/**
+   Write the bytes needed to move screen cursor to the specified
+   position to the specified buffer. The actual_cursor field of the
+   specified screen_t will be updated.
+
+   \param s the screen to operate on
+   \param b the buffer to send the output escape codes to
+   \param new_x the new x position
+   \param new_y the new y position
+*/
+static void s_move(screen_t *s, data_buffer_t *b, int new_x, int new_y)
+{
+    if (s->actual.cursor.x == new_x && s->actual.cursor.y == new_y)
+        return;
+
+    // If we are at the end of our window, then either the cursor stuck to the edge or it didn't. We don't know! We can fix it up though.
+    if (s->actual.cursor.x == common_get_width())
+    {
+        // Either issue a cr to go back to the beginning of this line, or a nl to go to the beginning of the next one, depending on what we think is more efficient
+        if (new_y <= s->actual.cursor.y)
+        {
+            b->push_back('\r');
+        }
+        else
+        {
+            b->push_back('\n');
+            s->actual.cursor.y++;
+        }
+        // Either way we're not in the first column
+        s->actual.cursor.x = 0;
+    }
+
+    int i;
+    int x_steps, y_steps;
+
+    char *str;
+    /*
+      debug( 0, L"move from %d %d to %d %d",
+      s->screen_cursor[0], s->screen_cursor[1],
+      new_x, new_y );
+    */
+    scoped_buffer_t scoped_buffer(b);
+
+    y_steps = new_y - s->actual.cursor.y;
+
+    if (y_steps > 0 && (strcmp(cursor_down, "\n")==0))
+    {
+        /*
+          This is very strange - it seems some (all?) consoles use a
+          simple newline as the cursor down escape. This will of
+          course move the cursor to the beginning of the line as well
+          as moving it down one step. The cursor_up does not have this
+          behaviour...
+        */
+        s->actual.cursor.x=0;
+    }
+
+    if (y_steps < 0)
+    {
+        str = cursor_up;
+    }
+    else
+    {
+        str = cursor_down;
+
+    }
+
+    for (i=0; i<abs(y_steps); i++)
+    {
+        writembs(str);
+    }
+
+
+    x_steps = new_x - s->actual.cursor.x;
+
+    if (x_steps && new_x == 0)
+    {
+        b->push_back('\r');
+        x_steps = 0;
+    }
+
+    char *multi_str = NULL;
+    if (x_steps < 0)
+    {
+        str = cursor_left;
+        multi_str = parm_left_cursor;
+    }
+    else
+    {
+        str = cursor_right;
+        multi_str = parm_right_cursor;
+    }
+    
+    // Use the bulk ('multi') output for cursor movement if it is supported and it would be shorter
+    // Note that this is required to avoid some visual glitches in iTerm (#1448)
+    bool use_multi = (multi_str != NULL && multi_str[0] != '\0' && abs(x_steps) * strlen(str) > strlen(multi_str));
+    if (use_multi)
+    {
+        char *multi_param = tparm(multi_str, abs(x_steps));
+        writembs(multi_param);
+    }
+    else
+    {
+        for (i=0; i<abs(x_steps); i++)
+        {
+            writembs(str);
+        }
+    }
+
+
+    s->actual.cursor.x = new_x;
+    s->actual.cursor.y = new_y;
+}
+
+/**
+   Set the pen color for the terminal
+*/
+static void s_set_color(screen_t *s, data_buffer_t *b, highlight_spec_t c)
+{
+    scoped_buffer_t scoped_buffer(b);
+
+    unsigned int uc = (unsigned int)c;
+    set_color(highlight_get_color(uc & 0xffff, false),
+              highlight_get_color((uc>>16)&0xffff, true));
+}
+
+/**
+   Convert a wide character to a multibyte string and append it to the
+   buffer.
+*/
+static void s_write_char(screen_t *s, data_buffer_t *b, wchar_t c)
+{
+    scoped_buffer_t scoped_buffer(b);
+    s->actual.cursor.x += fish_wcwidth_min_0(c);
+    writech(c);
+    if (s->actual.cursor.x == s->actual_width && allow_soft_wrap())
+    {
+        s->soft_wrap_location.x = 0;
+        s->soft_wrap_location.y = s->actual.cursor.y + 1;
+
+        /* Note that our cursor position may be a lie: Apple Terminal makes the right cursor stick to the margin, while Ubuntu makes it "go off the end" (but still doesn't wrap). We rely on s_move to fix this up. */
+    }
+    else
+    {
+        invalidate_soft_wrap(s);
+    }
+}
+
+/**
+   Send the specified string through tputs and append the output to
+   the specified buffer.
+*/
+static void s_write_mbs(data_buffer_t *b, char *s)
+{
+    scoped_buffer_t scoped_buffer(b);
+    writembs(s);
+}
+
+/**
+   Convert a wide string to a multibyte string and append it to the
+   buffer.
+*/
+static void s_write_str(data_buffer_t *b, const wchar_t *s)
+{
+    scoped_buffer_t scoped_buffer(b);
+    writestr(s);
+}
+
+/** Returns the length of the "shared prefix" of the two lines, which is the run of matching text and colors.
+    If the prefix ends on a combining character, do not include the previous character in the prefix.
+*/
+static size_t line_shared_prefix(const line_t &a, const line_t &b)
+{
+    size_t idx, max = std::min(a.size(), b.size());
+    for (idx=0; idx < max; idx++)
+    {
+        wchar_t ac = a.char_at(idx), bc = b.char_at(idx);
+        if (fish_wcwidth(ac) < 1 || fish_wcwidth(bc) < 1)
+        {
+            /* Possible combining mark, return one index prior */
+            if (idx > 0) idx--;
+            break;
+        }
+
+        /* We're done if the text or colors are different */
+        if (ac != bc || a.color_at(idx) != b.color_at(idx))
+            break;
+    }
+    return idx;
+}
+
+/* We are about to output one or more characters onto the screen at the given x, y. If we are at the end of previous line, and the previous line is marked as soft wrapping, then tweak the screen so we believe we are already in the target position. This lets the terminal take care of wrapping, which means that if you copy and paste the text, it won't have an embedded newline.  */
+static bool perform_any_impending_soft_wrap(screen_t *scr, int x, int y)
+{
+    if (x == scr->soft_wrap_location.x && y == scr->soft_wrap_location.y)
+    {
+        /* We can soft wrap; but do we want to? */
+        if (scr->desired.line(y - 1).is_soft_wrapped && allow_soft_wrap())
+        {
+            /* Yes. Just update the actual cursor; that will cause us to elide emitting the commands to move here, so we will just output on "one big line" (which the terminal soft wraps */
+            scr->actual.cursor = scr->soft_wrap_location;
+        }
+    }
+    return false;
+}
+
+/* Make sure we don't soft wrap */
+static void invalidate_soft_wrap(screen_t *scr)
+{
+    scr->soft_wrap_location = INVALID_LOCATION;
+}
+
+/* Various code for testing term behavior */
+#if 0
+static bool test_stuff(screen_t *scr)
+{
+    data_buffer_t output;
+    scoped_buffer_t scoped_buffer(&output);
+
+    s_move(scr, &output, 0, 0);
+    int screen_width = common_get_width();
+
+    const wchar_t *left = L"left";
+    const wchar_t *right = L"right";
+
+    for (size_t idx = 0; idx < 80; idx++)
+    {
+        output.push_back('A');
+    }
+
+    if (! output.empty())
+    {
+        write_loop(STDOUT_FILENO, &output.at(0), output.size());
+        output.clear();
+    }
+
+    sleep(5);
+
+    for (size_t i=0; i < 1; i++)
+    {
+        writembs(cursor_left);
+    }
+
+    if (! output.empty())
+    {
+        write_loop(1, &output.at(0), output.size());
+        output.clear();
+    }
+
+
+
+    while (1)
+    {
+        int c = getchar();
+        if (c != EOF) break;
+    }
+
+
+    while (1)
+    {
+        int c = getchar();
+        if (c != EOF) break;
+    }
+    puts("Bye");
+    exit(0);
+    while (1) sleep(10000);
+    return true;
+}
+#endif
+
+/**
+   Update the screen to match the desired output.
+*/
+static void s_update(screen_t *scr, const wchar_t *left_prompt, const wchar_t *right_prompt)
+{
+    //if (test_stuff(scr)) return;
+    const size_t left_prompt_width = calc_prompt_layout(left_prompt).last_line_width;
+    const size_t right_prompt_width = calc_prompt_layout(right_prompt).last_line_width;
+
+    int screen_width = common_get_width();
+
+    /* Figure out how many following lines we need to clear (probably 0) */
+    size_t actual_lines_before_reset = scr->actual_lines_before_reset;
+    scr->actual_lines_before_reset = 0;
+
+    data_buffer_t output;
+
+    bool need_clear_lines = scr->need_clear_lines;
+    bool need_clear_screen = scr->need_clear_screen;
+    bool has_cleared_screen = false;
+
+    if (scr->actual_width != screen_width)
+    {
+        /* Ensure we don't issue a clear screen for the very first output, to avoid https://github.com/fish-shell/fish-shell/issues/402 */
+        if (scr->actual_width != SCREEN_WIDTH_UNINITIALIZED)
+        {
+            need_clear_screen = true;
+            s_move(scr, &output, 0, 0);
+            s_reset(scr, screen_reset_current_line_contents);
+
+            need_clear_lines = need_clear_lines || scr->need_clear_lines;
+            need_clear_screen = need_clear_screen || scr->need_clear_screen;
+        }
+        scr->actual_width = screen_width;
+    }
+
+    scr->need_clear_lines = false;
+    scr->need_clear_screen = false;
+
+    /* Determine how many lines have stuff on them; we need to clear lines with stuff that we don't want */
+    const size_t lines_with_stuff = maxi(actual_lines_before_reset, scr->actual.line_count());
+    if (lines_with_stuff > scr->desired.line_count())
+    {
+        /* There are lines that we output to previously that will need to be cleared */
+        //need_clear_lines = true;
+    }
+
+    if (wcscmp(left_prompt, scr->actual_left_prompt.c_str()))
+    {
+        s_move(scr, &output, 0, 0);
+        s_write_str(&output, left_prompt);
+        scr->actual_left_prompt = left_prompt;
+        scr->actual.cursor.x = (int)left_prompt_width;
+    }
+
+    for (size_t i=0; i < scr->desired.line_count(); i++)
+    {
+        const line_t &o_line = scr->desired.line(i);
+        line_t &s_line = scr->actual.create_line(i);
+        size_t start_pos = (i==0 ? left_prompt_width : 0);
+        int current_width = 0;
+
+        /* If this is the last line, maybe we should clear the screen */
+        const bool should_clear_screen_this_line = (need_clear_screen && i + 1 == scr->desired.line_count() && clr_eos != NULL);
+
+        /* Note that skip_remaining is a width, not a character count */
+        size_t skip_remaining = start_pos;
+
+        if (! should_clear_screen_this_line)
+        {
+            /* Compute how much we should skip. At a minimum we skip over the prompt. But also skip over the shared prefix of what we want to output now, and what we output before, to avoid repeatedly outputting it. */
+            const size_t shared_prefix = line_shared_prefix(o_line, s_line);
+            if (shared_prefix > 0)
+            {
+                int prefix_width = fish_wcswidth(&o_line.text.at(0), shared_prefix);
+                if (prefix_width > skip_remaining)
+                    skip_remaining = prefix_width;
+            }
+
+            /* If we're soft wrapped, and if we're going to change the first character of the next line, don't skip over the last two characters so that we maintain soft-wrapping */
+            if (o_line.is_soft_wrapped && i + 1 < scr->desired.line_count())
+            {
+                bool first_character_of_next_line_will_change = true;
+                if (i + 1 < scr->actual.line_count())
+                {
+                    if (line_shared_prefix(scr->desired.line(i+1), scr->actual.line(i+1)) > 0)
+                    {
+                        first_character_of_next_line_will_change = false;
+                    }
+                }
+                if (first_character_of_next_line_will_change)
+                {
+                    skip_remaining = mini(skip_remaining, (size_t)(scr->actual_width - 2));
+                }
+            }
+        }
+
+        /* Skip over skip_remaining width worth of characters */
+        size_t j = 0;
+        for (; j < o_line.size(); j++)
+        {
+            int width = fish_wcwidth_min_0(o_line.char_at(j));
+            if (skip_remaining < width)
+                break;
+            skip_remaining -= width;
+            current_width += width;
+        }
+
+        /* Skip over zero-width characters (e.g. combining marks at the end of the prompt) */
+        for (; j < o_line.size(); j++)
+        {
+            int width = fish_wcwidth_min_0(o_line.char_at(j));
+            if (width > 0)
+                break;
+        }
+
+        /* Now actually output stuff */
+        for (; j < o_line.size(); j++)
+        {
+            /* If we are about to output into the last column, clear the screen first. If we clear the screen after we output into the last column, it can erase the last character due to the sticky right cursor. If we clear the screen too early, we can defeat soft wrapping. */
+            if (j + 1 == screen_width && should_clear_screen_this_line && ! has_cleared_screen)
+            {
+                s_move(scr, &output, current_width, (int)i);
+                s_write_mbs(&output, clr_eos);
+                has_cleared_screen = true;
+            }
+
+            perform_any_impending_soft_wrap(scr, current_width, (int)i);
+            s_move(scr, &output, current_width, (int)i);
+            s_set_color(scr, &output, o_line.color_at(j));
+            s_write_char(scr, &output, o_line.char_at(j));
+            current_width += fish_wcwidth_min_0(o_line.char_at(j));
+        }
+
+        /* Clear the screen if we have not done so yet. */
+        if (should_clear_screen_this_line && ! has_cleared_screen)
+        {
+            s_move(scr, &output, current_width, (int)i);
+            s_write_mbs(&output, clr_eos);
+            has_cleared_screen = true;
+        }
+
+        bool clear_remainder = false;
+        /* Clear the remainder of the line if we need to clear and if we didn't write to the end of the line. If we did write to the end of the line, the "sticky right edge" (as part of auto_right_margin) means that we'll be clearing the last character we wrote! */
+        if (has_cleared_screen)
+        {
+            /* Already cleared everything */
+            clear_remainder = false;
+        }
+        else if (need_clear_lines && current_width < screen_width)
+        {
+            clear_remainder = true;
+        }
+        else if (right_prompt_width < scr->last_right_prompt_width)
+        {
+            clear_remainder = true;
+        }
+        else
+        {
+            int prev_width = (s_line.text.empty() ? 0 : fish_wcswidth(&s_line.text.at(0), s_line.text.size()));
+            clear_remainder = prev_width > current_width;
+
+        }
+        if (clear_remainder)
+        {
+            s_set_color(scr, &output, 0xffffffff);
+            s_move(scr, &output, current_width, (int)i);
+            s_write_mbs(&output, clr_eol);
+        }
+
+        /* Output any rprompt if this is the first line. */
+        if (i == 0 && right_prompt_width > 0)
+        {
+            s_move(scr, &output, (int)(screen_width - right_prompt_width), (int)i);
+            s_set_color(scr, &output, 0xffffffff);
+            s_write_str(&output, right_prompt);
+            scr->actual.cursor.x += right_prompt_width;
+
+            /* We output in the last column. Some terms (Linux) push the cursor further right, past the window. Others make it "stick." Since we don't really know which is which, issue a cr so it goes back to the left.
+
+               However, if the user is resizing the window smaller, then it's possible the cursor wrapped. If so, then a cr will go to the beginning of the following line! So instead issue a bunch of "move left" commands to get back onto the line, and then jump to the front of it (!)
+            */
+
+            s_move(scr, &output, scr->actual.cursor.x - (int)right_prompt_width, scr->actual.cursor.y);
+            s_write_str(&output, L"\r");
+            scr->actual.cursor.x = 0;
+        }
+    }
+
+
+    /* Clear remaining lines (if any) if we haven't cleared the screen. */
+    if (! has_cleared_screen && scr->desired.line_count() < lines_with_stuff)
+    {
+        s_set_color(scr, &output, 0xffffffff);
+        for (size_t i=scr->desired.line_count(); i < lines_with_stuff; i++)
+        {
+            s_move(scr, &output, 0, (int)i);
+            s_write_mbs(&output, clr_eol);
+        }
+    }
+
+    s_move(scr, &output, scr->desired.cursor.x, scr->desired.cursor.y);
+    s_set_color(scr, &output, 0xffffffff);
+
+    if (! output.empty())
+    {
+        write_loop(STDOUT_FILENO, &output.at(0), output.size());
+    }
+
+    /* We have now synced our actual screen against our desired screen. Note that this is a big assignment! */
+    scr->actual = scr->desired;
+    scr->last_right_prompt_width = right_prompt_width;
+}
+
+/** Returns true if we are using a dumb terminal. */
+static bool is_dumb(void)
+{
+    return (!cursor_up || !cursor_down || !cursor_left || !cursor_right);
+}
+
+struct screen_layout_t
+{
+    /* The left prompt that we're going to use */
+    wcstring left_prompt;
+
+    /* How much space to leave for it */
+    size_t left_prompt_space;
+
+    /* The right prompt */
+    wcstring right_prompt;
+
+    /* The autosuggestion */
+    wcstring autosuggestion;
+
+    /* Whether the prompts get their own line or not */
+    bool prompts_get_own_line;
+};
+
+/* Given a vector whose indexes are offsets and whose values are the widths of the string if truncated at that offset, return the offset that fits in the given width. Returns width_by_offset.size() - 1 if they all fit. The first value in width_by_offset is assumed to be 0. */
+static size_t truncation_offset_for_width(const std::vector<size_t> &width_by_offset, size_t max_width)
+{
+    assert(! width_by_offset.empty() && width_by_offset.at(0) == 0);
+    size_t i;
+    for (i=1; i < width_by_offset.size(); i++)
+    {
+        if (width_by_offset.at(i) > max_width)
+            break;
+    }
+    /* i is the first index that did not fit; i-1 is therefore the last that did */
+    return i - 1;
+}
+
+static screen_layout_t compute_layout(screen_t *s,
+                                      size_t screen_width,
+                                      const wcstring &left_prompt_str,
+                                      const wcstring &right_prompt_str,
+                                      const wcstring &commandline,
+                                      const wcstring &autosuggestion_str,
+                                      const int *indent)
+{
+    screen_layout_t result = {};
+
+    /* Start by ensuring that the prompts themselves can fit */
+    const wchar_t *left_prompt = left_prompt_str.c_str();
+    const wchar_t *right_prompt = right_prompt_str.c_str();
+    const wchar_t *autosuggestion = autosuggestion_str.c_str();
+
+    prompt_layout_t left_prompt_layout = calc_prompt_layout(left_prompt);
+    prompt_layout_t right_prompt_layout = calc_prompt_layout(right_prompt);
+
+    size_t left_prompt_width = left_prompt_layout.last_line_width;
+    size_t right_prompt_width = right_prompt_layout.last_line_width;
+
+    if (left_prompt_layout.max_line_width > screen_width)
+    {
+        /* If we have a multi-line prompt, see if the longest line fits; if not neuter the whole left prompt */
+        left_prompt = L"> ";
+        left_prompt_width = 2;
+    }
+
+    if (left_prompt_width + right_prompt_width >= screen_width)
+    {
+        /* Nix right_prompt */
+        right_prompt = L"";
+        right_prompt_width = 0;
+    }
+
+    if (left_prompt_width + right_prompt_width >= screen_width)
+    {
+        /* Still doesn't fit, neuter left_prompt */
+        left_prompt = L"> ";
+        left_prompt_width = 2;
+    }
+
+    /* Now we should definitely fit */
+    assert(left_prompt_width + right_prompt_width < screen_width);
+
+
+    /* Convert commandline to a list of lines and their widths */
+    wcstring_list_t command_lines(1);
+    std::vector<size_t> line_widths(1);
+    for (size_t i=0; i < commandline.size(); i++)
+    {
+        wchar_t c = commandline.at(i);
+        if (c == L'\n')
+        {
+            /* Make a new line */
+            command_lines.push_back(wcstring());
+            line_widths.push_back(indent[i]*INDENT_STEP);
+        }
+        else
+        {
+            command_lines.back() += c;
+            line_widths.back() += fish_wcwidth_min_0(c);
+        }
+    }
+    const size_t first_command_line_width = line_widths.at(0);
+
+    /* If we have more than one line, ensure we have no autosuggestion */
+    if (command_lines.size() > 1)
+    {
+        autosuggestion = L"";
+    }
+
+    /* Compute the width of the autosuggestion at all possible truncation offsets */
+    std::vector<size_t> autosuggestion_truncated_widths;
+    autosuggestion_truncated_widths.reserve(1 + wcslen(autosuggestion));
+    size_t autosuggestion_total_width = 0;
+    for (size_t i=0; autosuggestion[i] != L'\0'; i++)
+    {
+        autosuggestion_truncated_widths.push_back(autosuggestion_total_width);
+        autosuggestion_total_width += fish_wcwidth_min_0(autosuggestion[i]);
+    }
+
+    /* Here are the layouts we try in turn:
+
+    1. Left prompt visible, right prompt visible, command line visible, autosuggestion visible
+    2. Left prompt visible, right prompt visible, command line visible, autosuggestion truncated (possibly to zero)
+    3. Left prompt visible, right prompt hidden, command line visible, autosuggestion hidden
+    4. Newline separator (left prompt visible, right prompt hidden, command line visible, autosuggestion visible)
+
+    A remark about layout #4: if we've pushed the command line to a new line, why can't we draw the right prompt? The issue is resizing: if you resize the window smaller, then the right prompt will wrap to the next line. This means that we can't go back to the line that we were on, and things turn to chaos very quickly.
+
+    */
+
+    bool done = false;
+
+    /* Case 1 */
+    if (! done && left_prompt_width + right_prompt_width + first_command_line_width + autosuggestion_total_width < screen_width)
+    {
+        result.left_prompt = left_prompt;
+        result.left_prompt_space = left_prompt_width;
+        result.right_prompt = right_prompt;
+        result.autosuggestion = autosuggestion;
+        done = true;
+    }
+
+    /* Case 2. Note that we require strict inequality so that there's always at least one space between the left edge and the rprompt */
+    if (! done && left_prompt_width + right_prompt_width + first_command_line_width < screen_width)
+    {
+        result.left_prompt = left_prompt;
+        result.left_prompt_space = left_prompt_width;
+        result.right_prompt = right_prompt;
+
+        /* Need at least two characters to show an autosuggestion */
+        size_t available_autosuggestion_space = screen_width - (left_prompt_width + right_prompt_width + first_command_line_width);
+        if (autosuggestion_total_width > 0 && available_autosuggestion_space > 2)
+        {
+            size_t truncation_offset = truncation_offset_for_width(autosuggestion_truncated_widths, available_autosuggestion_space - 2);
+            result.autosuggestion = wcstring(autosuggestion, truncation_offset);
+            result.autosuggestion.push_back(ellipsis_char);
+        }
+        done = true;
+    }
+
+    /* Case 3 */
+    if (! done && left_prompt_width + first_command_line_width < screen_width)
+    {
+        result.left_prompt = left_prompt;
+        result.left_prompt_space = left_prompt_width;
+        done = true;
+    }
+
+    /* Case 4 */
+    if (! done)
+    {
+        result.left_prompt = left_prompt;
+        result.left_prompt_space = left_prompt_width;
+        // See remark about for why we can't use the right prompt here
+        //result.right_prompt = right_prompt;
+
+        // If the command wraps, and the prompt is not short, place the command on its own line.
+        // A short prompt is 33% or less of the terminal's width.
+        const size_t prompt_percent_width = (100 * left_prompt_width) / screen_width;
+        if (left_prompt_width + first_command_line_width + 1 > screen_width && prompt_percent_width > 33)
+        {
+            result.prompts_get_own_line = true;
+        }
+
+        done = true;
+    }
+
+    assert(done);
+    return result;
+}
+
+
+void s_write(screen_t *s,
+             const wcstring &left_prompt,
+             const wcstring &right_prompt,
+             const wcstring &commandline,
+             size_t explicit_len,
+             const highlight_spec_t *colors,
+             const int *indent,
+             size_t cursor_pos,
+             size_t sel_start_pos,
+             size_t sel_stop_pos,
+             const page_rendering_t &pager,
+             bool cursor_position_is_within_pager)
+{
+    screen_data_t::cursor_t cursor_arr;
+
+    CHECK(s,);
+    CHECK(indent,);
+
+    /* Turn the command line into the explicit portion and the autosuggestion */
+    const wcstring explicit_command_line = commandline.substr(0, explicit_len);
+    const wcstring autosuggestion = commandline.substr(explicit_len);
+
+    /*
+      If we are using a dumb terminal, don't try any fancy stuff,
+      just print out the text. right_prompt not supported.
+     */
+    if (is_dumb())
+    {
+        const std::string prompt_narrow = wcs2string(left_prompt);
+        const std::string command_line_narrow = wcs2string(explicit_command_line);
+
+        write_loop(STDOUT_FILENO, "\r", 1);
+        write_loop(STDOUT_FILENO, prompt_narrow.c_str(), prompt_narrow.size());
+        write_loop(STDOUT_FILENO, command_line_narrow.c_str(), command_line_narrow.size());
+
+        return;
+    }
+
+    s_check_status(s);
+    const size_t screen_width = common_get_width();
+
+    /* Completely ignore impossibly small screens */
+    if (screen_width < 4)
+    {
+        return;
+    }
+
+    /* Compute a layout */
+    const screen_layout_t layout = compute_layout(s, screen_width, left_prompt, right_prompt, explicit_command_line, autosuggestion, indent);
+
+    /* Determine whether, if we have an autosuggestion, it was truncated */
+    s->autosuggestion_is_truncated = ! autosuggestion.empty() && autosuggestion != layout.autosuggestion;
+
+    /* Clear the desired screen */
+    s->desired.resize(0);
+    s->desired.cursor.x = s->desired.cursor.y = 0;
+
+    /* Append spaces for the left prompt */
+    for (size_t i=0; i < layout.left_prompt_space; i++)
+    {
+        s_desired_append_char(s, L' ', 0, 0, layout.left_prompt_space);
+    }
+
+    /* If overflowing, give the prompt its own line to improve the situation. */
+    size_t first_line_prompt_space = layout.left_prompt_space;
+    if (layout.prompts_get_own_line)
+    {
+        s_desired_append_char(s, L'\n', 0, 0, 0);
+        first_line_prompt_space = 0;
+    }
+
+    /* Reconstruct the command line */
+    wcstring effective_commandline = explicit_command_line + layout.autosuggestion;
+
+    /* Output the command line */
+    size_t i;
+    for (i=0; i < effective_commandline.size(); i++)
+    {
+        /* Grab the current cursor's x,y position if this character matches the cursor's offset */
+        if (! cursor_position_is_within_pager && i == cursor_pos)
+        {
+            cursor_arr = s->desired.cursor;
+        }
+        s_desired_append_char(s, effective_commandline.at(i), colors[i], indent[i], first_line_prompt_space);
+    }
+    
+    /* Cursor may have been at the end too */
+    if (! cursor_position_is_within_pager && i == cursor_pos)
+    {
+        cursor_arr = s->desired.cursor;
+    }
+    
+    /* Now that we've output everything, set the cursor to the position that we saved in the loop above */
+    s->desired.cursor = cursor_arr;
+
+    if (cursor_position_is_within_pager)
+    {
+        s->desired.cursor.x = (int)cursor_pos;
+        s->desired.cursor.y = (int)s->desired.line_count();
+    }
+
+    /* Append pager_data (none if empty) */
+    s->desired.append_lines(pager.screen_data);
+
+    s_update(s, layout.left_prompt.c_str(), layout.right_prompt.c_str());
+    s_save_status(s);
+}
+
+void s_reset(screen_t *s, screen_reset_mode_t mode)
+{
+    CHECK(s,);
+
+    bool abandon_line = false, repaint_prompt = false, clear_to_eos = false;
+    switch (mode)
+    {
+        case screen_reset_current_line_contents:
+            break;
+
+        case screen_reset_current_line_and_prompt:
+            repaint_prompt = true;
+            break;
+
+        case screen_reset_abandon_line:
+            abandon_line = true;
+            repaint_prompt = true;
+            break;
+
+        case screen_reset_abandon_line_and_clear_to_end_of_screen:
+            abandon_line = true;
+            repaint_prompt = true;
+            clear_to_eos = true;
+            break;
+    }
+
+    /* If we're abandoning the line, we must also be repainting the prompt */
+    assert(! abandon_line || repaint_prompt);
+
+    /* If we are not abandoning the line, we need to remember how many lines we had output to, so we can clear the remaining lines in the next call to s_update. This prevents leaving junk underneath the cursor when resizing a window wider such that it reduces our desired line count. */
+    if (! abandon_line)
+    {
+        s->actual_lines_before_reset =  maxi(s->actual_lines_before_reset, s->actual.line_count());
+    }
+
+    if (repaint_prompt && ! abandon_line)
+    {
+
+        /* If the prompt is multi-line, we need to move up to the prompt's initial line. We do this by lying to ourselves and claiming that we're really below what we consider "line 0" (which is the last line of the prompt). This will cause us to move up to try to get back to line 0, but really we're getting back to the initial line of the prompt. */
+        const size_t prompt_line_count = calc_prompt_lines(s->actual_left_prompt);
+        assert(prompt_line_count >= 1);
+        s->actual.cursor.y += (prompt_line_count - 1);
+    }
+    else if (abandon_line)
+    {
+        s->actual.cursor.y = 0;
+    }
+
+    if (repaint_prompt)
+        s->actual_left_prompt.clear();
+    s->actual.resize(0);
+    s->need_clear_lines = true;
+    s->need_clear_screen = s->need_clear_screen || clear_to_eos;
+
+    if (abandon_line)
+    {
+        /* Do the PROMPT_SP hack */
+        int screen_width = common_get_width();
+        wcstring abandon_line_string;
+        abandon_line_string.reserve(screen_width + 32); //should be enough
+
+        int non_space_width = wcwidth(omitted_newline_char);
+        if (screen_width >= non_space_width)
+        {
+            if (output_get_color_support() & color_support_term256)
+            {
+                // draw the string in term256 gray
+                abandon_line_string.append(L"\x1b[38;5;245m");
+            }
+            else
+            {
+                // draw in "bright black" (gray)
+                abandon_line_string.append(L"\x1b[0m" //bright
+                                           L"\x1b[30;1m"); //black
+            }
+            abandon_line_string.push_back(omitted_newline_char);
+            abandon_line_string.append(L"\x1b[0m"); //normal text ANSI escape sequence
+            abandon_line_string.append(screen_width - non_space_width, L' ');
+
+        }
+        abandon_line_string.push_back(L'\r');
+        // now we are certainly on a new line. But we may have dropped the omitted newline char on it. So append enough spaces to overwrite the omitted newline char, and then
+        abandon_line_string.append(non_space_width, L' ');
+        abandon_line_string.push_back(L'\r');
+
+        const std::string narrow_abandon_line_string = wcs2string(abandon_line_string);
+        write_loop(STDOUT_FILENO, narrow_abandon_line_string.c_str(), narrow_abandon_line_string.size());
+        s->actual.cursor.x = 0;
+    }
+
+    if (! abandon_line)
+    {
+        /* This should prevent resetting the cursor position during the next repaint. */
+        write_loop(STDOUT_FILENO, "\r", 1);
+        s->actual.cursor.x = 0;
+    }
+
+    fstat(1, &s->prev_buff_1);
+    fstat(2, &s->prev_buff_2);
+}
+
+bool screen_force_clear_to_end()
+{
+    bool result = false;
+    if (clr_eos)
+    {
+        data_buffer_t output;
+        s_write_mbs(&output, clr_eos);
+        if (! output.empty())
+        {
+            write_loop(STDOUT_FILENO, &output.at(0), output.size());
+            result = true;
+        }
+    }
+    return result;
+}
+
+screen_t::screen_t() :
+    desired(),
+    actual(),
+    actual_left_prompt(),
+    last_right_prompt_width(),
+    actual_width(SCREEN_WIDTH_UNINITIALIZED),
+    soft_wrap_location(INVALID_LOCATION),
+    autosuggestion_is_truncated(false),
+    need_clear_lines(false),
+    need_clear_screen(false),
+    actual_lines_before_reset(0),
+    prev_buff_1(), prev_buff_2(), post_buff_1(), post_buff_2()
+{
+}
+
diff --git a/src/screen.h b/src/screen.h
new file mode 100644
index 00000000..e84d8497
--- /dev/null
+++ b/src/screen.h
@@ -0,0 +1,284 @@
+/** \file screen.h High level library for handling the terminal screen
+
+  The screen library allows the interactive reader to write its
+  output to screen efficiently by keeping an internal representation
+  of the current screen contents and trying to find a reasonably
+  efficient way for transforming that to the desired screen content.
+
+  The current implementation is less smart than ncurses allows
+  and can not for example move blocks of text around to handle text
+  insertion.
+  */
+#ifndef FISH_SCREEN_H
+#define FISH_SCREEN_H
+
+#include <assert.h>
+#include <stddef.h>
+#include <vector>
+#include <sys/stat.h>
+#include "common.h"
+#include "highlight.h"
+
+class page_rendering_t;
+
+/**
+   A class representing a single line of a screen.
+*/
+struct line_t
+{
+    std::vector<wchar_t> text;
+    std::vector<highlight_spec_t> colors;
+    bool is_soft_wrapped;
+
+    line_t() : text(), colors(), is_soft_wrapped(false)
+    {
+    }
+
+    void clear(void)
+    {
+        text.clear();
+        colors.clear();
+    }
+
+    void append(wchar_t txt, highlight_spec_t color)
+    {
+        text.push_back(txt);
+        colors.push_back(color);
+    }
+
+    void append(const wchar_t *txt, highlight_spec_t color)
+    {
+        for (size_t i=0; txt[i]; i++)
+        {
+            text.push_back(txt[i]);
+            colors.push_back(color);
+        }
+    }
+
+
+
+    size_t size(void) const
+    {
+        return text.size();
+    }
+
+    wchar_t char_at(size_t idx) const
+    {
+        return text.at(idx);
+    }
+
+    highlight_spec_t color_at(size_t idx) const
+    {
+        return colors.at(idx);
+    }
+
+    void append_line(const line_t &line)
+    {
+        text.insert(text.end(), line.text.begin(), line.text.end());
+        colors.insert(colors.end(), line.colors.begin(), line.colors.end());
+    }
+
+};
+
+/**
+ A class representing screen contents.
+*/
+class screen_data_t
+{
+    std::vector<line_t> line_datas;
+
+public:
+
+    struct cursor_t
+    {
+        int x;
+        int y;
+        cursor_t() : x(0), y(0) { }
+        cursor_t(int a, int b) : x(a), y(b) { }
+    } cursor;
+
+    line_t &add_line(void)
+    {
+        line_datas.resize(line_datas.size() + 1);
+        return line_datas.back();
+    }
+
+    void resize(size_t size)
+    {
+        line_datas.resize(size);
+    }
+
+    line_t &create_line(size_t idx)
+    {
+        if (idx >= line_datas.size())
+        {
+            line_datas.resize(idx + 1);
+        }
+        return line_datas.at(idx);
+    }
+
+    line_t &insert_line_at_index(size_t idx)
+    {
+        assert(idx <= line_datas.size());
+        return *line_datas.insert(line_datas.begin() + idx, line_t());
+    }
+
+    line_t &line(size_t idx)
+    {
+        return line_datas.at(idx);
+    }
+
+    size_t line_count(void)
+    {
+        return line_datas.size();
+    }
+
+    void append_lines(const screen_data_t &d)
+    {
+        this->line_datas.insert(this->line_datas.end(), d.line_datas.begin(), d.line_datas.end());
+    }
+
+    bool empty() const
+    {
+        return line_datas.empty();
+    }
+};
+
+/**
+   The class representing the current and desired screen contents.
+*/
+class screen_t
+{
+public:
+
+    /** Constructor */
+    screen_t();
+
+    /**
+      The internal representation of the desired screen contents.
+    */
+    screen_data_t desired;
+    /**
+      The internal representation of the actual screen contents.
+    */
+    screen_data_t actual;
+
+    /**
+       A string containing the prompt which was last printed to
+       the screen.
+    */
+    wcstring actual_left_prompt;
+
+    /** Last right prompt width */
+    size_t last_right_prompt_width;
+
+    /**
+      The actual width of the screen at the time of the last screen
+      write.
+    */
+    int actual_width;
+
+    /** If we support soft wrapping, we can output to this location without any cursor motion. */
+    screen_data_t::cursor_t soft_wrap_location;
+
+    /** Whether the last-drawn autosuggestion (if any) is truncated, or hidden entirely */
+    bool autosuggestion_is_truncated;
+
+    /**
+     This flag is set to true when there is reason to suspect that
+     the parts of the screen lines where the actual content is not
+     filled in may be non-empty. This means that a clr_eol command
+     has to be sent to the terminal at the end of each line, including
+     actual_lines_before_reset.
+    */
+    bool need_clear_lines;
+
+    /** Whether there may be yet more content after the lines, and we issue a clr_eos if possible. */
+    bool need_clear_screen;
+
+    /** If we need to clear, this is how many lines the actual screen had, before we reset it. This is used when resizing the window larger: if the cursor jumps to the line above, we need to remember to clear the subsequent lines. */
+    size_t actual_lines_before_reset;
+
+    /**
+       These status buffers are used to check if any output has occurred
+       other than from fish's main loop, in which case we need to redraw.
+    */
+    struct stat prev_buff_1, prev_buff_2, post_buff_1, post_buff_2;
+};
+
+/**
+   This is the main function for the screen putput library. It is used
+   to define the desired contents of the screen. The screen command
+   will use it's knowlege of the current contents of the screen in
+   order to render the desired output using as few terminal commands
+   as possible.
+
+    \param s the screen on which to write
+    \param left_prompt the prompt to prepend to the command line
+    \param right_prompt the right prompt, or NULL if none
+    \param commandline the command line
+    \param explicit_len the number of characters of the "explicit" (non-autosuggestion) portion of the command line
+    \param colors the colors to use for the comand line
+    \param indent the indent to use for the command line
+    \param cursor_pos where the cursor is
+    \param sel_start_pos where the selections starts (inclusive)
+    \param sel_stop_pos where the selections ends (inclusive)
+    \param pager_data any pager data, to append to the screen
+    \param position_is_within_pager whether the position is within the pager line (first line)
+*/
+void s_write(screen_t *s,
+             const wcstring &left_prompt,
+             const wcstring &right_prompt,
+             const wcstring &commandline,
+             size_t explicit_len,
+             const highlight_spec_t *colors,
+             const int *indent,
+             size_t cursor_pos,
+             size_t sel_start_pos,
+             size_t sel_stop_pos,
+             const page_rendering_t &pager_data,
+             bool position_is_within_pager);
+
+/**
+    This function resets the screen buffers internal knowledge about
+    the contents of the screen. Use this function when some other
+    function than s_write has written to the screen.
+
+    \param s the screen to reset
+    \param reset_cursor whether the line on which the cursor has changed should be assumed to have changed. If \c reset_cursor is false, the library will attempt to make sure that the screen area does not seem to move up or down on repaint.
+    \param reset_prompt whether to reset the prompt as well.
+
+    If reset_cursor is incorrectly set to false, this may result in
+    screen contents being erased. If it is incorrectly set to true, it
+    may result in one or more lines of garbage on screen on the next
+    repaint. If this happens during a loop, such as an interactive
+    resizing, there will be one line of garbage for every repaint,
+    which will quickly fill the screen.
+*/
+void s_reset(screen_t *s, bool reset_cursor, bool reset_prompt = true);
+
+
+enum screen_reset_mode_t
+{
+    /* Do not make a new line, do not repaint the prompt. */
+    screen_reset_current_line_contents,
+
+    /* Do not make a new line, do repaint the prompt. */
+    screen_reset_current_line_and_prompt,
+
+    /* Abandon the current line, go to the next one, repaint the prompt */
+    screen_reset_abandon_line,
+
+    /* Abandon the current line, go to the next one, clear the rest of the screen */
+    screen_reset_abandon_line_and_clear_to_end_of_screen
+};
+
+void s_reset(screen_t *s, screen_reset_mode_t mode);
+
+/* Issues an immediate clr_eos, returning if it existed */
+bool screen_force_clear_to_end();
+
+/* Returns the length of an escape code. Exposed for testing purposes only. */
+size_t escape_code_length(const wchar_t *code);
+
+#endif
diff --git a/src/signal.cpp b/src/signal.cpp
new file mode 100644
index 00000000..b13f0c01
--- /dev/null
+++ b/src/signal.cpp
@@ -0,0 +1,695 @@
+/** \file signal.c
+
+The library for various signal related issues
+
+*/
+
+#include "config.h" // IWYU pragma: keep
+
+#include <wchar.h>
+#include <stdio.h>
+#include <signal.h>
+#include <errno.h>
+
+#ifdef HAVE_SIGINFO_H
+#include <siginfo.h>
+#endif
+
+#include "common.h"
+#include "fallback.h" // IWYU pragma: keep
+#include "wutil.h"
+#include "signal.h"
+#include "event.h"
+#include "reader.h"
+#include "proc.h"
+
+
+/**
+   Struct describing an entry for the lookup table used to convert
+   between signal names and signal ids, etc.
+*/
+struct lookup_entry
+{
+    /**
+       Signal id
+    */
+    int signal;
+    /**
+       Signal name
+    */
+    const wchar_t *name;
+    /**
+       Signal description
+    */
+    const wchar_t *desc;
+};
+
+/**
+   The number of signal blocks in place. Increased by signal_block, decreased by signal_unblock.
+*/
+static int block_count=0;
+
+
+/**
+   Lookup table used to convert between signal names and signal ids,
+   etc.
+*/
+static const struct lookup_entry lookup[] =
+{
+#ifdef SIGHUP
+    {
+        SIGHUP,
+        L"SIGHUP",
+        N_(L"Terminal hung up")
+    }
+    ,
+#endif
+#ifdef SIGINT
+    {
+        SIGINT,
+        L"SIGINT",
+        N_(L"Quit request from job control (^C)")
+    }
+    ,
+#endif
+#ifdef SIGQUIT
+    {
+        SIGQUIT,
+        L"SIGQUIT",
+        N_(L"Quit request from job control with core dump (^\\)")
+    }
+    ,
+#endif
+#ifdef SIGILL
+    {
+        SIGILL,
+        L"SIGILL",
+        N_(L"Illegal instruction")
+    }
+    ,
+#endif
+#ifdef SIGTRAP
+    {
+        SIGTRAP,
+        L"SIGTRAP",
+        N_(L"Trace or breakpoint trap")
+    }
+    ,
+#endif
+#ifdef SIGABRT
+    {
+        SIGABRT,
+        L"SIGABRT",
+        N_(L"Abort")
+    }
+    ,
+#endif
+#ifdef SIGBUS
+    {
+        SIGBUS,
+        L"SIGBUS",
+        N_(L"Misaligned address error")
+    }
+    ,
+#endif
+#ifdef SIGFPE
+    {
+        SIGFPE,
+        L"SIGFPE",
+        N_(L"Floating point exception")
+    }
+    ,
+#endif
+#ifdef SIGKILL
+    {
+        SIGKILL,
+        L"SIGKILL",
+        N_(L"Forced quit")
+    }
+    ,
+#endif
+#ifdef SIGUSR1
+    {
+        SIGUSR1,
+        L"SIGUSR1",
+        N_(L"User defined signal 1")
+    }
+    ,
+#endif
+#ifdef SIGUSR2
+    {
+        SIGUSR2, L"SIGUSR2",
+        N_(L"User defined signal 2")
+    }
+    ,
+#endif
+#ifdef SIGSEGV
+    {
+        SIGSEGV,
+        L"SIGSEGV",
+        N_(L"Address boundary error")
+    }
+    ,
+#endif
+#ifdef SIGPIPE
+    {
+        SIGPIPE,
+        L"SIGPIPE",
+        N_(L"Broken pipe")
+    }
+    ,
+#endif
+#ifdef SIGALRM
+    {
+        SIGALRM,
+        L"SIGALRM",
+        N_(L"Timer expired")
+    }
+    ,
+#endif
+#ifdef SIGTERM
+    {
+        SIGTERM,
+        L"SIGTERM",
+        N_(L"Polite quit request")
+    }
+    ,
+#endif
+#ifdef SIGCHLD
+    {
+        SIGCHLD,
+        L"SIGCHLD",
+        N_(L"Child process status changed")
+    }
+    ,
+#endif
+#ifdef SIGCONT
+    {
+        SIGCONT,
+        L"SIGCONT",
+        N_(L"Continue previously stopped process")
+    }
+    ,
+#endif
+#ifdef SIGSTOP
+    {
+        SIGSTOP,
+        L"SIGSTOP",
+        N_(L"Forced stop")
+    }
+    ,
+#endif
+#ifdef SIGTSTP
+    {
+        SIGTSTP,
+        L"SIGTSTP",
+        N_(L"Stop request from job control (^Z)")
+    }
+    ,
+#endif
+#ifdef SIGTTIN
+    {
+        SIGTTIN,
+        L"SIGTTIN",
+        N_(L"Stop from terminal input")
+    }
+    ,
+#endif
+#ifdef SIGTTOU
+    {
+        SIGTTOU,
+        L"SIGTTOU",
+        N_(L"Stop from terminal output")
+    }
+    ,
+#endif
+#ifdef SIGURG
+    {
+        SIGURG,
+        L"SIGURG",
+        N_(L"Urgent socket condition")
+    }
+    ,
+#endif
+#ifdef SIGXCPU
+    {
+        SIGXCPU,
+        L"SIGXCPU",
+        N_(L"CPU time limit exceeded")
+    }
+    ,
+#endif
+#ifdef SIGXFSZ
+    {
+        SIGXFSZ,
+        L"SIGXFSZ",
+        N_(L"File size limit exceeded")
+    }
+    ,
+#endif
+#ifdef SIGVTALRM
+    {
+        SIGVTALRM,
+        L"SIGVTALRM",
+        N_(L"Virtual timer expired")
+    }
+    ,
+#endif
+#ifdef SIGPROF
+    {
+        SIGPROF,
+        L"SIGPROF",
+        N_(L"Profiling timer expired")
+    }
+    ,
+#endif
+#ifdef SIGWINCH
+    {
+        SIGWINCH,
+        L"SIGWINCH",
+        N_(L"Window size change")
+    }
+    ,
+#endif
+#ifdef SIGWIND
+    {
+        SIGWIND,
+        L"SIGWIND",
+        N_(L"Window size change")
+    }
+    ,
+#endif
+#ifdef SIGIO
+    {
+        SIGIO,
+        L"SIGIO",
+        N_(L"I/O on asynchronous file descriptor is possible")
+    }
+    ,
+#endif
+#ifdef SIGPWR
+    {
+        SIGPWR,
+        L"SIGPWR",
+        N_(L"Power failure")
+    }
+    ,
+#endif
+#ifdef SIGSYS
+    {
+        SIGSYS,
+        L"SIGSYS",
+        N_(L"Bad system call")
+    }
+    ,
+#endif
+#ifdef SIGINFO
+    {
+        SIGINFO,
+        L"SIGINFO",
+        N_(L"Information request")
+    }
+    ,
+#endif
+#ifdef SIGSTKFLT
+    {
+        SIGSTKFLT,
+        L"SISTKFLT",
+        N_(L"Stack fault")
+    }
+    ,
+#endif
+#ifdef SIGEMT
+    {
+        SIGEMT,
+        L"SIGEMT",
+        N_(L"Emulator trap")
+    }
+    ,
+#endif
+#ifdef SIGIOT
+    {
+        SIGIOT,
+        L"SIGIOT",
+        N_(L"Abort (Alias for SIGABRT)")
+    }
+    ,
+#endif
+#ifdef SIGUNUSED
+    {
+        SIGUNUSED,
+        L"SIGUNUSED",
+        N_(L"Unused signal")
+    }
+    ,
+#endif
+    {
+        0,
+        0,
+        0
+    }
+}
+;
+
+
+/**
+   Test if \c name is a string describing the signal named \c canonical.
+*/
+static int match_signal_name(const wchar_t *canonical,
+                             const wchar_t *name)
+{
+    if (wcsncasecmp(name, L"sig", 3)==0)
+        name +=3;
+
+    return wcscasecmp(canonical+3,name) == 0;
+}
+
+
+int wcs2sig(const wchar_t *str)
+{
+    int i;
+    wchar_t *end=0;
+
+    for (i=0; lookup[i].desc ; i++)
+    {
+        if (match_signal_name(lookup[i].name, str))
+        {
+            return lookup[i].signal;
+        }
+    }
+    errno=0;
+    int res = fish_wcstoi(str, &end, 10);
+    if (!errno && res>=0 && !*end)
+        return res;
+
+    return -1;
+}
+
+
+const wchar_t *sig2wcs(int sig)
+{
+    int i;
+
+    for (i=0; lookup[i].desc ; i++)
+    {
+        if (lookup[i].signal == sig)
+        {
+            return lookup[i].name;
+        }
+    }
+
+    return _(L"Unknown");
+}
+
+const wchar_t *signal_get_desc(int sig)
+{
+    int i;
+
+    for (i=0; lookup[i].desc ; i++)
+    {
+        if (lookup[i].signal == sig)
+        {
+            return _(lookup[i].desc);
+        }
+    }
+
+    return _(L"Unknown");
+}
+
+/**
+   Standard signal handler
+*/
+static void default_handler(int signal, siginfo_t *info, void *context)
+{
+    if (event_is_signal_observed(signal))
+    {
+        event_fire_signal(signal);
+    }
+}
+
+/**
+   Respond to a winch signal by checking the terminal size
+*/
+static void handle_winch(int sig, siginfo_t *info, void *context)
+{
+    common_handle_winch(sig);
+    default_handler(sig, 0, 0);
+}
+
+/**
+   Respond to a hup signal by exiting, unless it is caught by a
+   shellscript function, in which case we do nothing.
+*/
+static void handle_hup(int sig, siginfo_t *info, void *context)
+{
+    if (event_is_signal_observed(SIGHUP))
+    {
+        default_handler(sig, 0, 0);
+    }
+    else
+    {
+        reader_exit(1, 1);
+    }
+}
+
+/** Handle sigterm. The only thing we do is restore the front process ID, then die. */
+static void handle_term(int sig, siginfo_t *info, void *context)
+{
+    restore_term_foreground_process_group();
+    signal(SIGTERM, SIG_DFL);
+    raise(SIGTERM);
+}
+
+/**
+   Interactive mode ^C handler. Respond to int signal by setting
+   interrupted-flag and stopping all loops and conditionals.
+*/
+static void handle_int(int sig, siginfo_t *info, void *context)
+{
+    reader_handle_int(sig);
+    default_handler(sig, info, context);
+}
+
+/**
+   sigchld handler. Does notification and calls the handler in proc.c
+*/
+static void handle_chld(int sig, siginfo_t *info, void *context)
+{
+    job_handle_signal(sig, info, context);
+    default_handler(sig, info, context);
+}
+
+void signal_reset_handlers()
+{
+    int i;
+
+    struct sigaction act;
+    sigemptyset(& act.sa_mask);
+    act.sa_flags=0;
+    act.sa_handler=SIG_DFL;
+
+    for (i=0; lookup[i].desc ; i++)
+    {
+        sigaction(lookup[i].signal, &act, 0);
+    }
+}
+
+
+/**
+   Sets appropriate signal handlers.
+*/
+void signal_set_handlers()
+{
+    struct sigaction act;
+
+    if (get_is_interactive() == -1)
+        return;
+
+    sigemptyset(& act.sa_mask);
+    act.sa_flags=SA_SIGINFO;
+    act.sa_sigaction = &default_handler;
+
+    /*
+      First reset everything to a use default_handler, a function
+      whose sole action is to fire of an event
+    */
+    sigaction(SIGINT, &act, 0);
+    sigaction(SIGQUIT, &act, 0);
+    sigaction(SIGTSTP, &act, 0);
+    sigaction(SIGTTIN, &act, 0);
+    sigaction(SIGTTOU, &act, 0);
+    sigaction(SIGCHLD, &act, 0);
+
+    /*
+      Ignore sigpipe, it is generated if fishd dies, but we can
+      recover.
+    */
+    sigaction(SIGPIPE, &act, 0);
+
+    if (get_is_interactive())
+    {
+        /*
+           Interactive mode. Ignore interactive signals.  We are a
+           shell, we know whats best for the user. ;-)
+        */
+
+        act.sa_handler=SIG_IGN;
+
+        sigaction(SIGINT, &act, 0);
+        sigaction(SIGQUIT, &act, 0);
+        sigaction(SIGTSTP, &act, 0);
+        sigaction(SIGTTIN, &act, 0);
+        sigaction(SIGTTOU, &act, 0);
+
+        act.sa_sigaction = &handle_int;
+        act.sa_flags = SA_SIGINFO;
+        if (sigaction(SIGINT, &act, 0))
+        {
+            wperror(L"sigaction");
+            FATAL_EXIT();
+        }
+
+        act.sa_sigaction = &handle_chld;
+        act.sa_flags = SA_SIGINFO;
+        if (sigaction(SIGCHLD, &act, 0))
+        {
+            wperror(L"sigaction");
+            FATAL_EXIT();
+        }
+
+#ifdef SIGWINCH
+        act.sa_flags = SA_SIGINFO;
+        act.sa_sigaction= &handle_winch;
+        if (sigaction(SIGWINCH, &act, 0))
+        {
+            wperror(L"sigaction");
+            FATAL_EXIT();
+        }
+#endif
+
+        act.sa_flags = SA_SIGINFO;
+        act.sa_sigaction= &handle_hup;
+        if (sigaction(SIGHUP, &act, 0))
+        {
+            wperror(L"sigaction");
+            FATAL_EXIT();
+        }
+
+        // SIGTERM restores the terminal controlling process before dying
+        act.sa_flags = SA_SIGINFO;
+        act.sa_sigaction= &handle_term;
+        if (sigaction(SIGTERM, &act, 0))
+        {
+            wperror(L"sigaction");
+            FATAL_EXIT();
+        }
+
+    }
+    else
+    {
+        /*
+          Non-interactive. Ignore interrupt, check exit status of
+          processes to determine result instead.
+        */
+        act.sa_handler=SIG_IGN;
+
+        sigaction(SIGINT, &act, 0);
+        sigaction(SIGQUIT, &act, 0);
+
+        act.sa_handler=SIG_DFL;
+
+        act.sa_sigaction = &handle_chld;
+        act.sa_flags = SA_SIGINFO;
+        if (sigaction(SIGCHLD, &act, 0))
+        {
+            wperror(L"sigaction");
+            exit_without_destructors(1);
+        }
+    }
+
+}
+
+void signal_handle(int sig, int do_handle)
+{
+    struct sigaction act;
+
+    /*
+      These should always be handled
+    */
+    if ((sig == SIGINT) ||
+            (sig == SIGQUIT) ||
+            (sig == SIGTSTP) ||
+            (sig == SIGTTIN) ||
+            (sig == SIGTTOU) ||
+            (sig == SIGCHLD))
+        return;
+
+    sigemptyset(&act.sa_mask);
+    if (do_handle)
+    {
+        act.sa_flags = SA_SIGINFO;
+        act.sa_sigaction = &default_handler;
+    }
+    else
+    {
+        act.sa_flags = 0;
+        act.sa_handler = SIG_DFL;
+    }
+
+    sigaction(sig, &act, 0);
+}
+
+void get_signals_with_handlers(sigset_t *set)
+{
+    sigemptyset(set);
+    for (int i=0; lookup[i].desc ; i++)
+    {
+        struct sigaction act = {};
+        sigaction(lookup[i].signal, NULL, &act);
+        if (act.sa_handler != SIG_DFL)
+            sigaddset(set, lookup[i].signal);
+    }
+}
+
+void signal_block()
+{
+    ASSERT_IS_MAIN_THREAD();
+    sigset_t chldset;
+
+    if (!block_count)
+    {
+        sigfillset(&chldset);
+        VOMIT_ON_FAILURE(pthread_sigmask(SIG_BLOCK, &chldset, NULL));
+    }
+
+    block_count++;
+//	debug( 0, L"signal block level increased to %d", block_count );
+}
+
+void signal_unblock()
+{
+    ASSERT_IS_MAIN_THREAD();
+    sigset_t chldset;
+
+    block_count--;
+
+    if (block_count < 0)
+    {
+        debug(0, _(L"Signal block mismatch"));
+        bugreport();
+        FATAL_EXIT();
+    }
+
+    if (!block_count)
+    {
+        sigfillset(&chldset);
+        VOMIT_ON_FAILURE(pthread_sigmask(SIG_UNBLOCK, &chldset, 0));
+    }
+//	debug( 0, L"signal block level decreased to %d", block_count );
+}
+
+bool signal_is_blocked()
+{
+    return !!block_count;
+}
+
diff --git a/src/signal.h b/src/signal.h
new file mode 100644
index 00000000..fc3e7e73
--- /dev/null
+++ b/src/signal.h
@@ -0,0 +1,65 @@
+/** \file signal.h
+
+The library for various signal related issues
+
+*/
+#ifndef FISH_SIGNALH
+#define FISH_SIGNALH
+
+#include <signal.h>
+
+/**
+   Get the integer signal value representing the specified signal, or
+   -1 of no signal was found
+*/
+int wcs2sig(const wchar_t *str);
+
+/**
+   Get string representation of a signal
+*/
+const wchar_t *sig2wcs(int sig);
+
+/**
+   Returns a description of the specified signal.
+*/
+const wchar_t *signal_get_desc(int sig);
+
+/**
+   Set all signal handlers to SIG_DFL
+*/
+void signal_reset_handlers();
+
+/**
+   Set signal handlers to fish default handlers
+*/
+void signal_set_handlers();
+
+/**
+   Tell fish what to do on the specified signal.
+
+   \param sig The signal to specify the action of
+   \param do_handle If true fish will catch the specified signal and fire an event, otherwise the default action (SIG_DFL) will be set
+*/
+void signal_handle(int sig, int do_handle);
+
+/**
+  Block all signals
+*/
+void signal_block();
+
+/**
+   Unblock all signals
+*/
+void signal_unblock();
+
+/**
+   Returns true if signals are being blocked
+*/
+bool signal_is_blocked();
+
+/**
+  Returns signals with non-default handlers
+*/
+void get_signals_with_handlers(sigset_t *set);
+
+#endif
diff --git a/src/tokenizer.cpp b/src/tokenizer.cpp
new file mode 100644
index 00000000..319eb2d9
--- /dev/null
+++ b/src/tokenizer.cpp
@@ -0,0 +1,988 @@
+/** \file tokenizer.c
+
+A specialized tokenizer for tokenizing the fish language. In the
+future, the tokenizer should be extended to support marks,
+tokenizing multiple strings and disposing of unused string
+segments.
+*/
+
+#include "config.h" // IWYU pragma: keep
+
+#include <wchar.h>
+#include <wctype.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <assert.h>
+#include <string>
+
+#include "fallback.h" // IWYU pragma: keep
+#include "common.h"
+#include "wutil.h" // IWYU pragma: keep - needed for wgettext
+#include "tokenizer.h"
+
+/* Wow what a hack */
+#define TOK_CALL_ERROR(t, e, x) do { tok_call_error((t), (e), (t)->squash_errors ? L"" : (x)); } while (0)
+
+/**
+   Error string for unexpected end of string
+*/
+#define QUOTE_ERROR _( L"Unexpected end of string, quotes are not balanced" )
+
+/**
+   Error string for mismatched parenthesis
+*/
+#define PARAN_ERROR _( L"Unexpected end of string, parenthesis do not match" )
+
+/**
+   Error string for mismatched square brackets
+*/
+#define SQUARE_BRACKET_ERROR _( L"Unexpected end of string, square brackets do not match" )
+
+
+/**
+   Error string for invalid redirections
+*/
+#define REDIRECT_ERROR _( L"Invalid input/output redirection" )
+
+/**
+   Error string for when trying to pipe from fd 0
+*/
+#define PIPE_ERROR _( L"Cannot use stdin (fd 0) as pipe output" )
+
+/**
+   Characters that separate tokens. They are ordered by frequency of occurrence to increase parsing speed.
+*/
+#define SEP L" \n|\t;#\r<>^&"
+
+/**
+   Descriptions of all tokenizer errors
+*/
+static const wchar_t *tok_desc[] =
+{
+    N_(L"Tokenizer not yet initialized"),
+    N_(L"Tokenizer error"),
+    N_(L"String"),
+    N_(L"Pipe"),
+    N_(L"End of command"),
+    N_(L"Redirect output to file"),
+    N_(L"Append output to file"),
+    N_(L"Redirect input to file"),
+    N_(L"Redirect to file descriptor"),
+    N_(L"Redirect output to file if file does not exist"),
+    N_(L"Run job in background"),
+    N_(L"Comment")
+};
+
+
+
+/**
+   Set the latest tokens string to be the specified error message
+*/
+static void tok_call_error(tokenizer_t *tok, int error_type, const wchar_t *error_message)
+{
+    tok->last_type = TOK_ERROR;
+    tok->error = error_type;
+    tok->last_token = error_message;
+}
+
+int tok_get_error(tokenizer_t *tok)
+{
+    return tok->error;
+}
+
+
+tokenizer_t::tokenizer_t(const wchar_t *b, tok_flags_t flags) : buff(NULL), orig_buff(NULL), last_type(TOK_NONE), last_pos(0), has_next(false), accept_unfinished(false), show_comments(false), show_blank_lines(false), last_quote(0), error(0), squash_errors(false), cached_lineno_offset(0), cached_lineno_count(0), continue_line_after_comment(false)
+{
+    CHECK(b,);
+
+    this->accept_unfinished = !!(flags & TOK_ACCEPT_UNFINISHED);
+    this->show_comments = !!(flags & TOK_SHOW_COMMENTS);
+    this->squash_errors = !!(flags & TOK_SQUASH_ERRORS);
+    this->show_blank_lines = !!(flags & TOK_SHOW_BLANK_LINES);
+
+    this->has_next = (*b != L'\0');
+    this->orig_buff = this->buff = b;
+    this->cached_lineno_offset = 0;
+    this->cached_lineno_count = 0;
+    tok_next(this);
+}
+
+enum token_type tok_last_type(tokenizer_t *tok)
+{
+    CHECK(tok, TOK_ERROR);
+    CHECK(tok->buff, TOK_ERROR);
+
+    return tok->last_type;
+}
+
+const wchar_t *tok_last(tokenizer_t *tok)
+{
+    CHECK(tok, 0);
+
+    return tok->last_token.c_str();
+}
+
+int tok_has_next(tokenizer_t *tok)
+{
+    /*
+      Return 1 on broken tokenizer
+    */
+    CHECK(tok, 1);
+    CHECK(tok->buff, 1);
+
+    /*  fwprintf( stderr, L"has_next is %ls \n", tok->has_next?L"true":L"false" );*/
+    return   tok->has_next;
+}
+
+/**
+   Tests if this character can be a part of a string. The redirect ^ is allowed unless it's the first character.
+   Hash (#) starts a comment if it's the first character in a token; otherwise it is considered a string character.
+   See #953.
+*/
+bool tok_is_string_character(wchar_t c, bool is_first)
+{
+    switch (c)
+    {
+            /* Unconditional separators */
+        case L'\0':
+        case L' ':
+        case L'\n':
+        case L'|':
+        case L'\t':
+        case L';':
+        case L'\r':
+        case L'<':
+        case L'>':
+        case L'&':
+            return false;
+
+            /* Conditional separator */
+        case L'^':
+            return ! is_first;
+
+        default:
+            return true;
+    }
+}
+
+/**
+   Quick test to catch the most common 'non-magical' characters, makes
+   read_string slightly faster by adding a fast path for the most
+   common characters. This is obviously not a suitable replacement for
+   iswalpha.
+*/
+static int myal(wchar_t c)
+{
+    return (c>=L'a' && c<=L'z') || (c>=L'A'&&c<=L'Z');
+}
+
+/**
+   Read the next token as a string
+*/
+static void read_string(tokenizer_t *tok)
+{
+    const wchar_t *start;
+    long len;
+    int do_loop=1;
+    int paran_count=0;
+
+    start = tok->buff;
+    bool is_first = true;
+
+    enum tok_mode_t
+    {
+        mode_regular_text = 0, // regular text
+        mode_subshell = 1, // inside of subshell
+        mode_array_brackets = 2, // inside of array brackets
+        mode_array_brackets_and_subshell = 3 // inside of array brackets and subshell, like in '$foo[(ech'
+    } mode = mode_regular_text;
+
+    while (1)
+    {
+        if (!myal(*tok->buff))
+        {
+            if (*tok->buff == L'\\')
+            {
+                tok->buff++;
+                if (*tok->buff == L'\0')
+                {
+                    if ((!tok->accept_unfinished))
+                    {
+                        TOK_CALL_ERROR(tok, TOK_UNTERMINATED_ESCAPE, QUOTE_ERROR);
+                        return;
+                    }
+                    else
+                    {
+                        /* Since we are about to increment tok->buff, decrement it first so the increment doesn't go past the end of the buffer. https://github.com/fish-shell/fish-shell/issues/389 */
+                        tok->buff--;
+                        do_loop = 0;
+                    }
+                }
+
+                tok->buff++;
+                continue;
+            }
+
+            switch (mode)
+            {
+                case mode_regular_text:
+                {
+                    switch (*tok->buff)
+                    {
+                        case L'(':
+                        {
+                            paran_count=1;
+                            mode = mode_subshell;
+                            break;
+                        }
+
+                        case L'[':
+                        {
+                            if (tok->buff != start)
+                                mode = mode_array_brackets;
+                            break;
+                        }
+
+                        case L'\'':
+                        case L'"':
+                        {
+
+                            const wchar_t *end = quote_end(tok->buff);
+                            tok->last_quote = *tok->buff;
+                            if (end)
+                            {
+                                tok->buff=(wchar_t *)end;
+                            }
+                            else
+                            {
+                                tok->buff += wcslen(tok->buff);
+
+                                if (! tok->accept_unfinished)
+                                {
+                                    TOK_CALL_ERROR(tok, TOK_UNTERMINATED_QUOTE, QUOTE_ERROR);
+                                    return;
+                                }
+                                do_loop = 0;
+
+                            }
+                            break;
+                        }
+
+                        default:
+                        {
+                            if (! tok_is_string_character(*(tok->buff), is_first))
+                            {
+                                do_loop=0;
+                            }
+                        }
+                    }
+                    break;
+                }
+
+                case mode_array_brackets_and_subshell:
+                case mode_subshell:
+                    switch (*tok->buff)
+                    {
+                        case L'\'':
+                        case L'\"':
+                        {
+                            const wchar_t *end = quote_end(tok->buff);
+                            if (end)
+                            {
+                                tok->buff=(wchar_t *)end;
+                            }
+                            else
+                            {
+                                tok->buff += wcslen(tok->buff);
+                                if ((!tok->accept_unfinished))
+                                {
+                                    TOK_CALL_ERROR(tok, TOK_UNTERMINATED_QUOTE, QUOTE_ERROR);
+                                    return;
+                                }
+                                do_loop = 0;
+                            }
+
+                            break;
+                        }
+
+                        case L'(':
+                            paran_count++;
+                            break;
+                        case L')':
+                            paran_count--;
+                            if (paran_count == 0)
+                            {
+                                mode = (mode == mode_array_brackets_and_subshell ? mode_array_brackets : mode_regular_text);
+                            }
+                            break;
+                        case L'\0':
+                            do_loop = 0;
+                            break;
+                    }
+                    break;
+
+                case mode_array_brackets:
+                    switch (*tok->buff)
+                    {
+                        case L'(':
+                            paran_count=1;
+                            mode = mode_array_brackets_and_subshell;
+                            break;
+
+                        case L']':
+                            mode = mode_regular_text;
+                            break;
+
+                        case L'\0':
+                            do_loop = 0;
+                            break;
+                    }
+                    break;
+            }
+        }
+
+
+        if (!do_loop)
+            break;
+
+        tok->buff++;
+        is_first = false;
+    }
+
+    if ((!tok->accept_unfinished) && (mode != mode_regular_text))
+    {
+        switch (mode)
+        {
+            case mode_subshell:
+                TOK_CALL_ERROR(tok, TOK_UNTERMINATED_SUBSHELL, PARAN_ERROR);
+                break;
+            case mode_array_brackets:
+            case mode_array_brackets_and_subshell:
+                TOK_CALL_ERROR(tok, TOK_UNTERMINATED_SUBSHELL, SQUARE_BRACKET_ERROR); // TOK_UNTERMINATED_SUBSHELL is a lie but nobody actually looks at it
+                break;
+            default:
+                assert(0 && "Unexpected mode in read_string");
+                break;
+        }
+        return;
+    }
+
+
+    len = tok->buff - start;
+
+    tok->last_token.assign(start, len);
+    tok->last_type = TOK_STRING;
+}
+
+/**
+   Read the next token as a comment.
+*/
+static void read_comment(tokenizer_t *tok)
+{
+    const wchar_t *start;
+
+    start = tok->buff;
+    while (*(tok->buff)!= L'\n' && *(tok->buff)!= L'\0')
+        tok->buff++;
+
+
+    size_t len = tok->buff - start;
+    tok->last_token.assign(start, len);
+    tok->last_type = TOK_COMMENT;
+}
+
+
+
+/* Reads a redirection or an "fd pipe" (like 2>|) from a string. Returns how many characters were consumed. If zero, then this string was not a redirection.
+
+   Also returns by reference the redirection mode, and the fd to redirection. If there is overflow, *out_fd is set to -1.
+*/
+static size_t read_redirection_or_fd_pipe(const wchar_t *buff, enum token_type *out_redirection_mode, int *out_fd)
+{
+    bool errored = false;
+    int fd = 0;
+    enum token_type redirection_mode = TOK_NONE;
+
+    size_t idx = 0;
+
+    /* Determine the fd. This may be specified as a prefix like '2>...' or it may be implicit like '>' or '^'. Try parsing out a number; if we did not get any digits then infer it from the first character. Watch out for overflow. */
+    long long big_fd = 0;
+    for (; iswdigit(buff[idx]); idx++)
+    {
+        /* Note that it's important we consume all the digits here, even if it overflows. */
+        if (big_fd <= INT_MAX)
+            big_fd = big_fd * 10 + (buff[idx] - L'0');
+    }
+
+    fd = (big_fd > INT_MAX ? -1 : static_cast<int>(big_fd));
+
+    if (idx == 0)
+    {
+        /* We did not find a leading digit, so there's no explicit fd. Infer it from the type */
+        switch (buff[idx])
+        {
+            case L'>':
+                fd = STDOUT_FILENO;
+                break;
+            case L'<':
+                fd = STDIN_FILENO;
+                break;
+            case L'^':
+                fd = STDERR_FILENO;
+                break;
+            default:
+                errored = true;
+                break;
+        }
+    }
+
+    /* Either way we should have ended on the redirection character itself like '>' */
+    wchar_t redirect_char = buff[idx++]; //note increment of idx
+    if (redirect_char == L'>' || redirect_char == L'^')
+    {
+        redirection_mode = TOK_REDIRECT_OUT;
+        if (buff[idx] == redirect_char)
+        {
+            /* Doubled up like ^^ or >>. That means append */
+            redirection_mode = TOK_REDIRECT_APPEND;
+            idx++;
+        }
+    }
+    else if (redirect_char == L'<')
+    {
+        redirection_mode = TOK_REDIRECT_IN;
+    }
+    else
+    {
+        /* Something else */
+        errored = true;
+    }
+
+    /* Optional characters like & or ?, or the pipe char | */
+    wchar_t opt_char = buff[idx];
+    if (opt_char == L'&')
+    {
+        redirection_mode = TOK_REDIRECT_FD;
+        idx++;
+    }
+    else if (opt_char == L'?')
+    {
+        redirection_mode = TOK_REDIRECT_NOCLOB;
+        idx++;
+    }
+    else if (opt_char == L'|')
+    {
+        /* So the string looked like '2>|'. This is not a redirection - it's a pipe! That gets handled elsewhere. */
+        redirection_mode = TOK_PIPE;
+        idx++;
+    }
+
+    /* Don't return valid-looking stuff on error */
+    if (errored)
+    {
+        idx = 0;
+        redirection_mode = TOK_NONE;
+    }
+
+    /* Return stuff */
+    if (out_redirection_mode != NULL)
+        *out_redirection_mode = redirection_mode;
+    if (out_fd != NULL)
+        *out_fd = fd;
+
+    return idx;
+}
+
+enum token_type redirection_type_for_string(const wcstring &str, int *out_fd)
+{
+    enum token_type mode = TOK_NONE;
+    int fd = 0;
+    read_redirection_or_fd_pipe(str.c_str(), &mode, &fd);
+    /* Redirections only, no pipes */
+    if (mode == TOK_PIPE || fd < 0)
+        mode = TOK_NONE;
+    if (out_fd != NULL)
+        *out_fd = fd;
+    return mode;
+}
+
+int fd_redirected_by_pipe(const wcstring &str)
+{
+    /* Hack for the common case */
+    if (str == L"|")
+    {
+        return STDOUT_FILENO;
+    }
+
+    enum token_type mode = TOK_NONE;
+    int fd = 0;
+    read_redirection_or_fd_pipe(str.c_str(), &mode, &fd);
+    /* Pipes only */
+    if (mode != TOK_PIPE || fd < 0)
+        fd = -1;
+    return fd;
+}
+
+int oflags_for_redirection_type(enum token_type type)
+{
+    switch (type)
+    {
+        case TOK_REDIRECT_APPEND:
+            return O_CREAT | O_APPEND | O_WRONLY;
+        case TOK_REDIRECT_OUT:
+            return O_CREAT | O_WRONLY | O_TRUNC;
+        case TOK_REDIRECT_NOCLOB:
+            return O_CREAT | O_EXCL | O_WRONLY;
+        case TOK_REDIRECT_IN:
+            return O_RDONLY;
+
+        default:
+            return -1;
+    }
+}
+
+/**
+   Test if a character is whitespace. Differs from iswspace in that it
+   does not consider a newline to be whitespace.
+*/
+static bool my_iswspace(wchar_t c)
+{
+    return c != L'\n' && iswspace(c);
+}
+
+
+const wchar_t *tok_get_desc(int type)
+{
+    if (type < 0 || (size_t)type >= (sizeof tok_desc / sizeof *tok_desc))
+    {
+        return _(L"Invalid token type");
+    }
+    return _(tok_desc[type]);
+}
+
+void tok_next(tokenizer_t *tok)
+{
+
+    CHECK(tok,);
+    CHECK(tok->buff,);
+
+    if (tok_last_type(tok) == TOK_ERROR)
+    {
+        tok->has_next=false;
+        return;
+    }
+
+    if (!tok->has_next)
+    {
+        /*    wprintf( L"EOL\n" );*/
+        tok->last_type = TOK_END;
+        return;
+    }
+
+    while (1)
+    {
+        if (tok->buff[0] == L'\\' && tok->buff[1] == L'\n')
+        {
+            tok->buff += 2;
+            tok->continue_line_after_comment = true;
+        }
+        else if (my_iswspace(tok->buff[0]))
+        {
+            tok->buff++;
+        }
+        else
+        {
+            break;
+        }
+    }
+
+
+    while (*tok->buff == L'#')
+    {
+        if (tok->show_comments)
+        {
+            tok->last_pos = tok->buff - tok->orig_buff;
+            read_comment(tok);
+
+            if (tok->buff[0] == L'\n' && tok->continue_line_after_comment)
+                tok->buff++;
+
+            return;
+        }
+        else
+        {
+            while (*(tok->buff)!= L'\n' && *(tok->buff)!= L'\0')
+                tok->buff++;
+
+            if (tok->buff[0] == L'\n' && tok->continue_line_after_comment)
+                tok->buff++;
+        }
+
+        while (my_iswspace(*(tok->buff))) {
+            tok->buff++;
+        }
+    }
+
+    tok->continue_line_after_comment = false;
+
+    tok->last_pos = tok->buff - tok->orig_buff;
+
+    switch (*tok->buff)
+    {
+        case L'\0':
+            tok->last_type = TOK_END;
+            /*fwprintf( stderr, L"End of string\n" );*/
+            tok->has_next = false;
+            break;
+        case 13: // carriage return
+        case L'\n':
+        case L';':
+            tok->last_type = TOK_END;
+            tok->buff++;
+            // Hack: when we get a newline, swallow as many as we can
+            // This compresses multiple subsequent newlines into a single one
+            if (! tok->show_blank_lines)
+            {
+                while (*tok->buff == L'\n' || *tok->buff == 13 /* CR */ || *tok->buff == ' ' || *tok->buff == '\t')
+                {
+                    tok->buff++;
+                }
+            }
+            tok->last_token.clear();
+            break;
+        case L'&':
+            tok->last_type = TOK_BACKGROUND;
+            tok->buff++;
+            break;
+
+        case L'|':
+            tok->last_token = L"1";
+            tok->last_type = TOK_PIPE;
+            tok->buff++;
+            break;
+
+        case L'>':
+        case L'<':
+        case L'^':
+        {
+            /* There's some duplication with the code in the default case below. The key difference here is that we must never parse these as a string; a failed redirection is an error! */
+            enum token_type mode = TOK_NONE;
+            int fd = -1;
+            size_t consumed = read_redirection_or_fd_pipe(tok->buff, &mode, &fd);
+            if (consumed == 0 || fd < 0)
+            {
+                TOK_CALL_ERROR(tok, TOK_OTHER, REDIRECT_ERROR);
+            }
+            else
+            {
+                tok->buff += consumed;
+                tok->last_type = mode;
+                tok->last_token = to_string(fd);
+            }
+        }
+        break;
+
+        default:
+        {
+            /* Maybe a redirection like '2>&1', maybe a pipe like 2>|, maybe just a string */
+            size_t consumed = 0;
+            enum token_type mode = TOK_NONE;
+            int fd = -1;
+            if (iswdigit(*tok->buff))
+                consumed = read_redirection_or_fd_pipe(tok->buff, &mode, &fd);
+
+            if (consumed > 0)
+            {
+                /* It looks like a redirection or a pipe. But we don't support piping fd 0. Note that fd 0 may be -1, indicating overflow; but we don't treat that as a tokenizer error. */
+                if (mode == TOK_PIPE && fd == 0)
+                {
+                    TOK_CALL_ERROR(tok, TOK_OTHER, PIPE_ERROR);
+                }
+                else
+                {
+                    tok->buff += consumed;
+                    tok->last_type = mode;
+                    tok->last_token = to_string(fd);
+                }
+            }
+            else
+            {
+                /* Not a redirection or pipe, so just a string */
+                read_string(tok);
+            }
+        }
+        break;
+
+    }
+
+}
+
+wcstring tok_first(const wchar_t *str)
+{
+    wcstring result;
+    if (str)
+    {
+        tokenizer_t t(str, TOK_SQUASH_ERRORS);
+        switch (tok_last_type(&t))
+        {
+            case TOK_STRING:
+            {
+                const wchar_t *tmp = tok_last(&t);
+                if (tmp != NULL)
+                    result = tmp;
+                break;
+            }
+            default:
+                break;
+        }
+    }
+    return result;
+}
+
+int tok_get_pos(const tokenizer_t *tok)
+{
+    CHECK(tok, 0);
+    return (int)tok->last_pos;
+}
+
+size_t tok_get_extent(const tokenizer_t *tok)
+{
+    CHECK(tok, 0);
+    size_t current_pos = tok->buff - tok->orig_buff;
+    return current_pos > tok->last_pos ? current_pos - tok->last_pos : 0;
+}
+
+
+void tok_set_pos(tokenizer_t *tok, int pos)
+{
+    CHECK(tok,);
+
+    tok->buff = tok->orig_buff + pos;
+    tok->has_next = true;
+    tok_next(tok);
+}
+
+bool move_word_state_machine_t::consume_char_punctuation(wchar_t c)
+{
+    enum
+    {
+        s_always_one = 0,
+        s_whitespace,
+        s_alphanumeric,
+        s_end
+    };
+
+    bool consumed = false;
+    while (state != s_end && ! consumed)
+    {
+        switch (state)
+        {
+            case s_always_one:
+                /* Always consume the first character */
+                consumed = true;
+                state = s_whitespace;
+                break;
+
+            case s_whitespace:
+                if (iswspace(c))
+                {
+                    /* Consumed whitespace */
+                    consumed = true;
+                }
+                else
+                {
+                    state = s_alphanumeric;
+                }
+                break;
+
+            case s_alphanumeric:
+                if (iswalnum(c))
+                {
+                    /* Consumed alphanumeric */
+                    consumed = true;
+                }
+                else
+                {
+                    state = s_end;
+                }
+                break;
+
+            case s_end:
+            default:
+                break;
+        }
+    }
+    return consumed;
+}
+
+bool move_word_state_machine_t::is_path_component_character(wchar_t c)
+{
+    /* Always treat separators as first. All this does is ensure that we treat ^ as a string character instead of as stderr redirection, which I hypothesize is usually what is desired. */
+    return tok_is_string_character(c, true) && ! wcschr(L"/={,}'\"", c);
+}
+
+bool move_word_state_machine_t::consume_char_path_components(wchar_t c)
+{
+    enum
+    {
+        s_initial_punctuation,
+        s_whitespace,
+        s_separator,
+        s_slash,
+        s_path_component_characters,
+        s_end
+    };
+
+    //printf("state %d, consume '%lc'\n", state, c);
+    bool consumed = false;
+    while (state != s_end && ! consumed)
+    {
+        switch (state)
+        {
+            case s_initial_punctuation:
+                if (! is_path_component_character(c))
+                {
+                    consumed = true;
+                }
+                state = s_whitespace;
+                break;
+
+            case s_whitespace:
+                if (iswspace(c))
+                {
+                    /* Consumed whitespace */
+                    consumed = true;
+                }
+                else if (c == L'/' || is_path_component_character(c))
+                {
+                    /* Path component */
+                    state = s_slash;
+                }
+                else
+                {
+                    /* Path separator */
+                    state = s_separator;
+                }
+                break;
+
+            case s_separator:
+                if (! iswspace(c) && ! is_path_component_character(c))
+                {
+                    /* Consumed separator */
+                    consumed = true;
+                }
+                else
+                {
+                    state = s_end;
+                }
+                break;
+
+            case s_slash:
+                if (c == L'/')
+                {
+                    /* Consumed slash */
+                    consumed = true;
+                }
+                else
+                {
+                    state = s_path_component_characters;
+                }
+                break;
+
+            case s_path_component_characters:
+                if (is_path_component_character(c))
+                {
+                    /* Consumed string character except slash */
+                    consumed = true;
+                }
+                else
+                {
+                    state = s_end;
+                }
+                break;
+
+                /* We won't get here, but keep the compiler happy */
+            case s_end:
+            default:
+                break;
+        }
+    }
+    return consumed;
+}
+
+bool move_word_state_machine_t::consume_char_whitespace(wchar_t c)
+{
+    enum
+    {
+        s_always_one = 0,
+        s_blank,
+        s_graph,
+        s_end
+    };
+
+    bool consumed = false;
+    while (state != s_end && ! consumed)
+    {
+        switch (state)
+        {
+            case s_always_one:
+                /* Always consume the first character */
+                consumed = true;
+                state = s_blank;
+                break;
+
+            case s_blank:
+                if (iswblank(c))
+                {
+                    /* Consumed whitespace */
+                    consumed = true;
+                }
+                else
+                {
+                    state = s_graph;
+                }
+                break;
+
+            case s_graph:
+                if (iswgraph(c))
+                {
+                    /* Consumed printable non-space */
+                    consumed = true;
+                }
+                else
+                {
+                    state = s_end;
+                }
+                break;
+
+            case s_end:
+            default:
+                break;
+        }
+    }
+    return consumed;
+}
+
+bool move_word_state_machine_t::consume_char(wchar_t c)
+{
+    switch (style)
+    {
+        case move_word_style_punctuation:
+            return consume_char_punctuation(c);
+        case move_word_style_path_components:
+            return consume_char_path_components(c);
+        case move_word_style_whitespace:
+            return consume_char_whitespace(c);
+        default:
+            return false;
+    }
+}
+
+move_word_state_machine_t::move_word_state_machine_t(move_word_style_t syl) : state(0), style(syl)
+{
+}
+
+void move_word_state_machine_t::reset()
+{
+    state = 0;
+}
diff --git a/src/tokenizer.h b/src/tokenizer.h
new file mode 100644
index 00000000..09a49509
--- /dev/null
+++ b/src/tokenizer.h
@@ -0,0 +1,220 @@
+/** \file tokenizer.h
+
+    A specialized tokenizer for tokenizing the fish language. In the
+    future, the tokenizer should be extended to support marks,
+    tokenizing multiple strings and disposing of unused string
+    segments.
+*/
+
+#ifndef FISH_TOKENIZER_H
+#define FISH_TOKENIZER_H
+
+#include <stddef.h>
+#include "common.h"
+
+/**
+   Token types
+*/
+enum token_type
+{
+    TOK_NONE, /**< Tokenizer not yet constructed */
+    TOK_ERROR, /**< Error reading token */
+    TOK_STRING,/**< String token */
+    TOK_PIPE,/**< Pipe token */
+    TOK_END,/**< End token (semicolon or newline, not literal end) */
+    TOK_REDIRECT_OUT, /**< redirection token */
+    TOK_REDIRECT_APPEND,/**< redirection append token */
+    TOK_REDIRECT_IN,/**< input redirection token */
+    TOK_REDIRECT_FD,/**< redirection to new fd token */
+    TOK_REDIRECT_NOCLOB, /**<? redirection token */
+    TOK_BACKGROUND,/**< send job to bg token */
+    TOK_COMMENT/**< comment token */
+};
+
+/**
+   Tokenizer error types
+*/
+enum tokenizer_error
+{
+    TOK_UNTERMINATED_QUOTE,
+    TOK_UNTERMINATED_SUBSHELL,
+    TOK_UNTERMINATED_ESCAPE,
+    TOK_OTHER
+}
+;
+
+
+/**
+   Flag telling the tokenizer to accept incomplete parameters,
+   i.e. parameters with mismatching paranthesis, etc. This is useful
+   for tab-completion.
+*/
+#define TOK_ACCEPT_UNFINISHED 1
+
+/**
+   Flag telling the tokenizer not to remove comments. Useful for
+   syntax highlighting.
+*/
+#define TOK_SHOW_COMMENTS 2
+
+/** Flag telling the tokenizer to not generate error messages, which we need to do when tokenizing off of the main thread (since wgettext is not thread safe).
+*/
+#define TOK_SQUASH_ERRORS 4
+
+/** Ordinarily, the tokenizer ignores newlines following a newline, or a semicolon.
+    This flag tells the tokenizer to return each of them as a separate END. */
+#define TOK_SHOW_BLANK_LINES 8
+
+typedef unsigned int tok_flags_t;
+
+/**
+   The tokenizer struct.
+*/
+struct tokenizer_t
+{
+    /** A pointer into the original string, showing where the next token begins */
+    const wchar_t *buff;
+    /** A copy of the original string */
+    const wchar_t *orig_buff;
+    /** The last token */
+    wcstring last_token;
+
+    /** Type of last token*/
+    enum token_type last_type;
+
+    /** Offset of last token*/
+    size_t last_pos;
+    /** Whether there are more tokens*/
+    bool has_next;
+    /** Whether incomplete tokens are accepted*/
+    bool accept_unfinished;
+    /** Whether comments should be returned*/
+    bool show_comments;
+    /** Whether all blank lines are returned */
+    bool show_blank_lines;
+    /** Type of last quote, can be either ' or ".*/
+    wchar_t last_quote;
+    /** Last error */
+    int error;
+    /* Whether we are squashing errors */
+    bool squash_errors;
+
+    /* Cached line number information */
+    size_t cached_lineno_offset;
+    int cached_lineno_count;
+
+    /* Whether to continue the previous line after the comment */
+    bool continue_line_after_comment;
+
+    /**
+      Constructor for a tokenizer. b is the string that is to be
+      tokenized. It is not copied, and should not be freed by the caller
+      until after the tokenizer is destroyed.
+
+      \param b The string to tokenize
+      \param flags Flags to the tokenizer. Setting TOK_ACCEPT_UNFINISHED will cause the tokenizer
+      to accept incomplete tokens, such as a subshell without a closing
+      parenthesis, as a valid token. Setting TOK_SHOW_COMMENTS will return comments as tokens
+
+    */
+    tokenizer_t(const wchar_t *b, tok_flags_t flags);
+};
+
+/**
+  Jump to the next token.
+*/
+void tok_next(tokenizer_t *tok);
+
+/**
+  Returns the type of the last token. Must be one of the values in the token_type enum.
+*/
+enum token_type tok_last_type(tokenizer_t *tok);
+
+/**
+  Returns the last token string. The string should not be freed by the caller. This returns nonsense results for some token types, like TOK_END.
+*/
+const wchar_t *tok_last(tokenizer_t *tok);
+
+/**
+  Returns true as long as there are more tokens left
+*/
+int tok_has_next(tokenizer_t *tok);
+
+/**
+  Returns the position of the beginning of the current token in the original string
+*/
+int tok_get_pos(const tokenizer_t *tok);
+
+/** Returns the extent of the current token */
+size_t tok_get_extent(const tokenizer_t *tok);
+
+/**
+   Returns only the first token from the specified string. This is a
+   convenience function, used to retrieve the first token of a
+   string. This can be useful for error messages, etc.
+
+   On failure, returns the empty string.
+*/
+wcstring tok_first(const wchar_t *str);
+
+/**
+   Indicates whether a character can be part of a string, or is a string separator.
+   Separators include newline, tab, |, ^, >, <, etc.
+
+   is_first should indicate whether this is the first character in a potential string.
+*/
+bool tok_is_string_character(wchar_t c, bool is_first);
+
+/**
+   Move tokenizer position
+*/
+void tok_set_pos(tokenizer_t *tok, int pos);
+
+/**
+   Returns a string description of the specified token type
+*/
+const wchar_t *tok_get_desc(int type);
+
+/**
+   Get tokenizer error type. Should only be called if tok_last_tope returns TOK_ERROR.
+*/
+int tok_get_error(tokenizer_t *tok);
+
+/* Helper function to determine redirection type from a string, or TOK_NONE if the redirection is invalid. Also returns the fd by reference. */
+enum token_type redirection_type_for_string(const wcstring &str, int *out_fd = NULL);
+
+/* Helper function to determine which fd is redirected by a pipe */
+int fd_redirected_by_pipe(const wcstring &str);
+
+/* Helper function to return oflags (as in open(2)) for a redirection type */
+int oflags_for_redirection_type(enum token_type type);
+
+enum move_word_style_t
+{
+    move_word_style_punctuation, //stop at punctuation
+    move_word_style_path_components, //stops at path components
+    move_word_style_whitespace // stops at whitespace
+};
+
+/* Our state machine that implements "one word" movement or erasure. */
+class move_word_state_machine_t
+{
+private:
+
+    bool consume_char_punctuation(wchar_t c);
+    bool consume_char_path_components(wchar_t c);
+    bool is_path_component_character(wchar_t c);
+    bool consume_char_whitespace(wchar_t c);
+
+    int state;
+    move_word_style_t style;
+
+public:
+
+    move_word_state_machine_t(move_word_style_t st);
+    bool consume_char(wchar_t c);
+    void reset();
+};
+
+
+#endif
diff --git a/src/utf8.cpp b/src/utf8.cpp
new file mode 100644
index 00000000..62453be4
--- /dev/null
+++ b/src/utf8.cpp
@@ -0,0 +1,513 @@
+/*
+ * Copyright (c) 2007 Alexey Vatchenko <av@bsdua.org>
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#include <sys/types.h>
+#include <stdint.h>
+
+#include "utf8.h"
+
+#include <string>
+#include <limits>
+#include <algorithm>
+
+#define _NXT	0x80
+#define _SEQ2	0xc0
+#define _SEQ3	0xe0
+#define _SEQ4	0xf0
+#define _SEQ5	0xf8
+#define _SEQ6	0xfc
+
+#define _BOM	0xfeff
+
+/* We can tweak the following typedef to allow us to simulate Windows-style 16 bit wchar's on Unix */
+typedef wchar_t utf8_wchar_t;
+#define UTF8_WCHAR_MAX ((size_t)std::numeric_limits<utf8_wchar_t>::max())
+
+bool is_wchar_ucs2()
+{
+    return UTF8_WCHAR_MAX <= 0xFFFF;
+}
+
+static size_t utf8_to_wchar_internal(const char *in, size_t insize, utf8_wchar_t *out, size_t outsize, int flags);
+static size_t wchar_to_utf8_internal(const utf8_wchar_t *in, size_t insize, char *out, size_t outsize, int flags);
+
+static bool safe_copy_wchar_to_utf8_wchar(const wchar_t *in, utf8_wchar_t *out, size_t count)
+{
+    bool result = true;
+    for (size_t i=0; i < count; i++)
+    {
+        wchar_t c = in[i];
+        if (c > UTF8_WCHAR_MAX)
+        {
+            result = false;
+            break;
+        }
+        out[i] = c;
+    }
+    return result;
+}
+
+bool utf8_to_wchar_string(const std::string &str, std::wstring *result)
+{
+    result->clear();
+    const size_t inlen = str.size();
+    if (inlen == 0)
+    {
+        return true;
+    }
+
+    bool success = false;
+    const char *input = str.c_str();
+    size_t outlen = utf8_to_wchar(input, inlen, NULL, 0, 0);
+    if (outlen > 0)
+    {
+        wchar_t *tmp = new wchar_t[outlen];
+        size_t outlen2 = utf8_to_wchar(input, inlen, tmp, outlen, 0);
+        if (outlen2 > 0)
+        {
+            result->assign(tmp, outlen2);
+            success = true;
+        }
+        delete[] tmp;
+    }
+    return success;
+}
+
+bool wchar_to_utf8_string(const std::wstring &str, std::string *result)
+{
+    result->clear();
+    const size_t inlen = str.size();
+    if (inlen == 0)
+    {
+        return true;
+    }
+
+    bool success = false;
+    const wchar_t *input = str.c_str();
+    size_t outlen = wchar_to_utf8(input, inlen, NULL, 0, 0);
+    if (outlen > 0)
+    {
+        char *tmp = new char[outlen];
+        size_t outlen2 = wchar_to_utf8(input, inlen, tmp, outlen, 0);
+        if (outlen2 > 0)
+        {
+            result->assign(tmp, outlen2);
+            success = true;
+        }
+        delete[] tmp;
+    }
+    return success;
+}
+
+size_t utf8_to_wchar(const char *in, size_t insize, wchar_t *out, size_t outsize, int flags)
+{
+    if (in == NULL || insize == 0 || (outsize == 0 && out != NULL))
+    {
+        return 0;
+    }
+
+    size_t result;
+    if (sizeof(wchar_t) == sizeof(utf8_wchar_t))
+    {
+        result = utf8_to_wchar_internal(in, insize, reinterpret_cast<utf8_wchar_t *>(out), outsize, flags);
+    }
+    else
+    {
+        // Allocate a temporary buffer to hold the output
+        // note: outsize may be 0
+        utf8_wchar_t *tmp_output = new utf8_wchar_t[outsize];
+
+        // Invoke the conversion with the temporary
+        result = utf8_to_wchar_internal(in, insize, tmp_output, outsize, flags);
+
+        // Copy back from tmp to the function's output, then clean it up
+        size_t amount_to_copy = std::min(result, outsize);
+        std::copy(tmp_output, tmp_output + amount_to_copy, out);
+        delete[] tmp_output;
+    }
+    return result;
+}
+
+size_t wchar_to_utf8(const wchar_t *in, size_t insize, char *out, size_t outsize, int flags)
+{
+    if (in == NULL || insize == 0 || (outsize == 0 && out != NULL))
+    {
+        return 0;
+    }
+
+    size_t result;
+    if (sizeof(wchar_t) == sizeof(utf8_wchar_t))
+    {
+        result = wchar_to_utf8_internal(reinterpret_cast<const utf8_wchar_t *>(in), insize, out, outsize, flags);
+    }
+    else
+    {
+        // Allocate a temporary buffer to hold the input
+        // the std::copy performs the size conversion
+        // note: insize may be 0
+        utf8_wchar_t *tmp_input = new utf8_wchar_t[insize];
+        if (! safe_copy_wchar_to_utf8_wchar(in, tmp_input, insize))
+        {
+            // our utf8_wchar_t is UCS-16 and there was an astral character
+            result = 0;
+        }
+        else
+        {
+            // Invoke the conversion with the temporary, then clean up the input
+            result = wchar_to_utf8_internal(tmp_input, insize, out, outsize, flags);
+        }
+        delete[] tmp_input;
+    }
+    return result;
+}
+
+
+static int __wchar_forbitten(utf8_wchar_t sym);
+static int __utf8_forbitten(unsigned char octet);
+
+static int
+__wchar_forbitten(utf8_wchar_t sym)
+{
+
+    /* Surrogate pairs */
+    if (sym >= 0xd800 && sym <= 0xdfff)
+        return (-1);
+
+    return (0);
+}
+
+static int
+__utf8_forbitten(unsigned char octet)
+{
+
+    switch (octet)
+    {
+        case 0xc0:
+        case 0xc1:
+        case 0xf5:
+        case 0xff:
+            return (-1);
+    }
+
+    return (0);
+}
+
+/*
+ * DESCRIPTION
+ *	This function translates UTF-8 string into UCS-2 or UCS-4 string (all symbols
+ *	will be in local machine byte order).
+ *
+ *	It takes the following arguments:
+ *	in	- input UTF-8 string. It can be null-terminated.
+ *	insize	- size of input string in bytes.
+ *	out	- result buffer for UCS-2/4 string. If out is NULL,
+ *		function returns size of result buffer.
+ *	outsize - size of out buffer in wide characters.
+ *
+ * RETURN VALUES
+ *	The function returns size of result buffer (in wide characters).
+ *	Zero is returned in case of error.
+ *
+ * CAVEATS
+ *	1. If UTF-8 string contains zero symbols, they will be translated
+ *	   as regular symbols.
+ *	2. If UTF8_IGNORE_ERROR or UTF8_SKIP_BOM flag is set, sizes may vary
+ *	   when `out' is NULL and not NULL. It's because of special UTF-8
+ *	   sequences which may result in forbitten (by RFC3629) UNICODE
+ *	   characters.  So, the caller must check return value every time and
+ *	   not prepare buffer in advance (\0 terminate) but after calling this
+ *	   function.
+ */
+static size_t utf8_to_wchar_internal(const char *in, size_t insize, utf8_wchar_t *out, size_t outsize, int flags)
+{
+    unsigned char *p, *lim;
+    utf8_wchar_t *wlim, high;
+    size_t n, total, i, n_bits;
+
+    if (in == NULL || insize == 0 || (outsize == 0 && out != NULL))
+        return (0);
+
+    total = 0;
+    p = (unsigned char *)in;
+    lim = p + insize;
+    wlim = out + outsize;
+
+    for (; p < lim; p += n)
+    {
+        if (__utf8_forbitten(*p) != 0 &&
+                (flags & UTF8_IGNORE_ERROR) == 0)
+            return (0);
+
+        /*
+         * Get number of bytes for one wide character.
+         */
+        n = 1;	/* default: 1 byte. Used when skipping bytes. */
+        if ((*p & 0x80) == 0)
+            high = (utf8_wchar_t)*p;
+        else if ((*p & 0xe0) == _SEQ2)
+        {
+            n = 2;
+            high = (utf8_wchar_t)(*p & 0x1f);
+        }
+        else if ((*p & 0xf0) == _SEQ3)
+        {
+            n = 3;
+            high = (utf8_wchar_t)(*p & 0x0f);
+        }
+        else if ((*p & 0xf8) == _SEQ4)
+        {
+            n = 4;
+            high = (utf8_wchar_t)(*p & 0x07);
+        }
+        else if ((*p & 0xfc) == _SEQ5)
+        {
+            n = 5;
+            high = (utf8_wchar_t)(*p & 0x03);
+        }
+        else if ((*p & 0xfe) == _SEQ6)
+        {
+            n = 6;
+            high = (utf8_wchar_t)(*p & 0x01);
+        }
+        else
+        {
+            if ((flags & UTF8_IGNORE_ERROR) == 0)
+                return (0);
+            continue;
+        }
+
+        /* does the sequence header tell us truth about length? */
+        if (lim - p <= n - 1)
+        {
+            if ((flags & UTF8_IGNORE_ERROR) == 0)
+                return (0);
+            n = 1;
+            continue;	/* skip */
+        }
+
+        /*
+         * Validate sequence.
+         * All symbols must have higher bits set to 10xxxxxx
+         */
+        if (n > 1)
+        {
+            for (i = 1; i < n; i++)
+            {
+                if ((p[i] & 0xc0) != _NXT)
+                    break;
+            }
+            if (i != n)
+            {
+                if ((flags & UTF8_IGNORE_ERROR) == 0)
+                    return (0);
+                n = 1;
+                continue;	/* skip */
+            }
+        }
+
+        total++;
+
+        if (out == NULL)
+            continue;
+
+        if (out >= wlim)
+            return (0);		/* no space left */
+
+        uint32_t out_val = 0;
+        *out = 0;
+        n_bits = 0;
+        for (i = 1; i < n; i++)
+        {
+            out_val |= (utf8_wchar_t)(p[n - i] & 0x3f) << n_bits;
+            n_bits += 6;		/* 6 low bits in every byte */
+        }
+        out_val |= high << n_bits;
+
+        bool skip = false;
+        if (__wchar_forbitten(out_val) != 0)
+        {
+            if ((flags & UTF8_IGNORE_ERROR) == 0)
+            {
+                return 0;	/* forbitten character */
+            }
+            else
+            {
+                skip = true;
+            }
+        }
+        else if (out_val == _BOM && (flags & UTF8_SKIP_BOM) != 0)
+        {
+            skip = true;
+        }
+
+        if (skip)
+        {
+            total--;
+        }
+        else if (out_val > UTF8_WCHAR_MAX)
+        {
+            // wchar_t is UCS-2, but the UTF-8 specified an astral character
+            return 0;
+        }
+        else
+        {
+            *out++ = out_val;
+        }
+    }
+
+    return (total);
+}
+
+/*
+ * DESCRIPTION
+ *	This function translates UCS-2/4 symbols (given in local machine
+ *	byte order) into UTF-8 string.
+ *
+ *	It takes the following arguments:
+ *	in	- input unicode string. It can be null-terminated.
+ *	insize	- size of input string in wide characters.
+ *	out	- result buffer for utf8 string. If out is NULL,
+ *		function returns size of result buffer.
+ *	outsize - size of result buffer.
+ *
+ * RETURN VALUES
+ *	The function returns size of result buffer (in bytes). Zero is returned
+ *	in case of error.
+ *
+ * CAVEATS
+ *	If UCS-4 string contains zero symbols, they will be translated
+ *	as regular symbols.
+ */
+static size_t wchar_to_utf8_internal(const utf8_wchar_t *in, size_t insize, char *out, size_t outsize, int flags)
+{
+    const utf8_wchar_t *w, *wlim;
+    unsigned char *p, *lim;
+    size_t total, n;
+
+    if (in == NULL || insize == 0 || (outsize == 0 && out != NULL))
+        return (0);
+
+    w = in;
+    wlim = w + insize;
+    p = (unsigned char *)out;
+    lim = p + outsize;
+    total = 0;
+    for (; w < wlim; w++)
+    {
+        if (__wchar_forbitten(*w) != 0)
+        {
+            if ((flags & UTF8_IGNORE_ERROR) == 0)
+                return (0);
+            else
+                continue;
+        }
+
+        if (*w == _BOM && (flags & UTF8_SKIP_BOM) != 0)
+            continue;
+
+        const int32_t w_wide = *w;
+        if (w_wide < 0)
+        {
+            if ((flags & UTF8_IGNORE_ERROR) == 0)
+                return (0);
+            continue;
+        }
+        else if (w_wide <= 0x0000007f)
+            n = 1;
+        else if (w_wide <= 0x000007ff)
+            n = 2;
+        else if (w_wide <= 0x0000ffff)
+            n = 3;
+        else if (w_wide <= 0x001fffff)
+            n = 4;
+        else if (w_wide <= 0x03ffffff)
+            n = 5;
+        else /* if (w_wide <= 0x7fffffff) */
+            n = 6;
+
+        total += n;
+
+        if (out == NULL)
+            continue;
+
+        if (lim - p <= n - 1)
+            return (0);		/* no space left */
+
+        /* extract the wchar_t as big-endian. If wchar_t is UCS-16, the first two bytes will be 0 */
+        unsigned char oc[4];
+        uint32_t w_tmp = *w;
+        oc[3] = w_tmp & 0xFF;
+        w_tmp >>= 8;
+        oc[2] = w_tmp & 0xFF;
+        w_tmp >>= 8;
+        oc[1] = w_tmp & 0xFF;
+        w_tmp >>= 8;
+        oc[0] = w_tmp & 0xFF;
+
+        switch (n)
+        {
+            case 1:
+                p[0] = oc[3];
+                break;
+
+            case 2:
+                p[1] = _NXT | (oc[3] & 0x3f);
+                p[0] = _SEQ2 | (oc[3] >> 6) | ((oc[2] & 0x07) << 2);
+                break;
+
+            case 3:
+                p[2] = _NXT | (oc[3] & 0x3f);
+                p[1] = _NXT | (oc[3] >> 6) | ((oc[2] & 0x0f) << 2);
+                p[0] = _SEQ3 | ((oc[2] & 0xf0) >> 4);
+                break;
+
+            case 4:
+                p[3] = _NXT | (oc[3] & 0x3f);
+                p[2] = _NXT | (oc[3] >> 6) | ((oc[2] & 0x0f) << 2);
+                p[1] = _NXT | ((oc[2] & 0xf0) >> 4) |
+                       ((oc[1] & 0x03) << 4);
+                p[0] = _SEQ4 | ((oc[1] & 0x1f) >> 2);
+                break;
+
+            case 5:
+                p[4] = _NXT | (oc[3] & 0x3f);
+                p[3] = _NXT | (oc[3] >> 6) | ((oc[2] & 0x0f) << 2);
+                p[2] = _NXT | ((oc[2] & 0xf0) >> 4) |
+                       ((oc[1] & 0x03) << 4);
+                p[1] = _NXT | (oc[1] >> 2);
+                p[0] = _SEQ5 | (oc[0] & 0x03);
+                break;
+
+            case 6:
+                p[5] = _NXT | (oc[3] & 0x3f);
+                p[4] = _NXT | (oc[3] >> 6) | ((oc[2] & 0x0f) << 2);
+                p[3] = _NXT | (oc[2] >> 4) | ((oc[1] & 0x03) << 4);
+                p[2] = _NXT | (oc[1] >> 2);
+                p[1] = _NXT | (oc[0] & 0x3f);
+                p[0] = _SEQ6 | ((oc[0] & 0x40) >> 6);
+                break;
+        }
+
+        /*
+         * NOTE: do not check here for forbitten UTF-8 characters.
+         * They cannot appear here because we do proper convertion.
+         */
+
+        p += n;
+    }
+
+    return (total);
+}
diff --git a/src/utf8.h b/src/utf8.h
new file mode 100644
index 00000000..1c9923db
--- /dev/null
+++ b/src/utf8.h
@@ -0,0 +1,40 @@
+/*
+ * Copyright (c) 2007 Alexey Vatchenko <av@bsdua.org>
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+/*
+ * utf8: implementation of UTF-8 charset encoding (RFC3629).
+ */
+#ifndef _UTF8_H_
+#define _UTF8_H_
+
+#include <stddef.h>
+
+#include <string>
+
+#define UTF8_IGNORE_ERROR		0x01
+#define UTF8_SKIP_BOM			0x02
+
+/* Convert a string between UTF8 and UCS-2/4 (depending on size of wchar_t). Returns true if successful, storing the result of the conversion in *result */
+bool utf8_to_wchar_string(const std::string &input, std::wstring *result);
+bool wchar_to_utf8_string(const std::wstring &input, std::string *result);
+
+/* Variants exposed for testing */
+size_t utf8_to_wchar(const char *in, size_t insize, wchar_t *out, size_t outsize, int flags);
+size_t wchar_to_utf8(const wchar_t *in, size_t insize, char *out, size_t outsize, int flags);
+
+bool is_wchar_ucs2();
+
+#endif /* !_UTF8_H_ */
diff --git a/src/util.cpp b/src/util.cpp
new file mode 100644
index 00000000..85b9d8d7
--- /dev/null
+++ b/src/util.cpp
@@ -0,0 +1,127 @@
+/** \file util.c
+  Generic utilities library.
+
+  Contains datastructures such as automatically growing array lists, priority queues, etc.
+*/
+
+#include "config.h"
+
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <wchar.h>
+#include <math.h>
+#include <sys/time.h>
+#include <stdarg.h>
+#include <string.h>
+#include <ctype.h>
+#include <wctype.h>
+#include <unistd.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <dirent.h>
+#include <errno.h>
+#include <assert.h>
+
+#include "fallback.h"
+#include "util.h"
+
+#include "common.h"
+#include "wutil.h"
+
+/**
+   Minimum allocated size for data structures. Used to avoid excessive
+   memory allocations for lists, hash tables, etc, which are nearly
+   empty.
+*/
+#define MIN_SIZE 32
+
+/**
+   Maximum number of characters that can be inserted using a single
+   call to sb_printf. This is needed since vswprintf doesn't tell us
+   what went wrong. We don't know if we ran out of space or something
+   else went wrong. We assume that any error is an out of memory-error
+   and try again until we reach this size.  After this size has been
+   reached, it is instead assumed that something was wrong with the
+   format string.
+*/
+#define SB_MAX_SIZE (128*1024*1024)
+
+int wcsfilecmp(const wchar_t *a, const wchar_t *b)
+{
+    CHECK(a, 0);
+    CHECK(b, 0);
+
+    if (*a==0)
+    {
+        if (*b==0)
+            return 0;
+        return -1;
+    }
+    if (*b==0)
+    {
+        return 1;
+    }
+
+    long secondary_diff=0;
+    if (iswdigit(*a) && iswdigit(*b))
+    {
+        wchar_t *aend, *bend;
+        long al;
+        long bl;
+        long diff;
+
+        errno = 0;
+        al = wcstol(a, &aend, 10);
+        bl = wcstol(b, &bend, 10);
+
+        if (errno)
+        {
+            /*
+              Huuuuuuuuge numbers - fall back to regular string comparison
+            */
+            return wcscmp(a, b);
+        }
+
+        diff = al - bl;
+        if (diff)
+            return diff > 0 ? 2 : -2;
+
+        secondary_diff = (aend-a) - (bend-b);
+
+        a=aend-1;
+        b=bend-1;
+    }
+    else
+    {
+        int diff = towlower(*a) - towlower(*b);
+        if (diff != 0)
+            return (diff>0)?2:-2;
+
+        secondary_diff = *a-*b;
+    }
+
+    int res = wcsfilecmp(a+1, b+1);
+
+    if (abs(res) < 2)
+    {
+        /*
+          No primary difference in rest of string.
+          Use secondary difference on this element if found.
+        */
+        if (secondary_diff)
+        {
+            return secondary_diff > 0 ? 1 :-1;
+        }
+    }
+
+    return res;
+}
+
+long long get_time()
+{
+    struct timeval time_struct;
+    gettimeofday(&time_struct, 0);
+    return 1000000ll*time_struct.tv_sec+time_struct.tv_usec;
+}
+
diff --git a/src/util.h b/src/util.h
new file mode 100644
index 00000000..ee7142be
--- /dev/null
+++ b/src/util.h
@@ -0,0 +1,71 @@
+/** \file util.h
+  Generic utilities library.
+*/
+
+#ifndef FISH_UTIL_H
+#define FISH_UTIL_H
+
+#include <wchar.h>
+#include <stdarg.h>
+#include <unistd.h>
+
+/**
+   Returns the larger of two ints
+*/
+template<typename T>
+static inline T maxi(T a, T b)
+{
+    return a>b?a:b;
+}
+
+/**
+   Returns the smaller of two ints
+ */
+template<typename T>
+static inline T mini(T a, T b)
+{
+    return a<b?a:b;
+}
+
+/**
+   Compares two wide character strings with an (arguably) intuitive
+   ordering.
+
+   This function tries to order strings in a way which is intuitive to
+   humans with regards to sorting strings containing numbers.
+
+   Most sorting functions would sort the strings 'file1.txt'
+   'file5.txt' and 'file12.txt' as:
+
+   file1.txt
+   file12.txt
+   file5.txt
+
+   This function regards any sequence of digits as a single entity
+   when performing comparisons, so the output is instead:
+
+   file1.txt
+   file5.txt
+   file12.txt
+
+   Which most people would find more intuitive.
+
+   This won't return the optimum results for numbers in bases higher
+   than ten, such as hexadecimal, but at least a stable sort order
+   will result.
+
+   This function performs a two-tiered sort, where difference in case
+   and in number of leading zeroes in numbers only have effect if no
+   other differences between strings are found. This way, a 'file1'
+   and 'File1' will not be considered identical, and hence their
+   internal sort order is not arbitrary, but the names 'file1',
+   'File2' and 'file3' will still be sorted in the order given above.
+*/
+int wcsfilecmp(const wchar_t *a, const wchar_t *b);
+
+/**
+   Get the current time in microseconds since Jan 1, 1970
+*/
+long long get_time();
+
+#endif
diff --git a/src/wcstringutil.cpp b/src/wcstringutil.cpp
new file mode 100644
index 00000000..e61e331b
--- /dev/null
+++ b/src/wcstringutil.cpp
@@ -0,0 +1,40 @@
+/** \file wcstringutil.cpp
+
+Helper functions for working with wcstring
+*/
+
+#include "config.h" // IWYU pragma: keep
+
+#include "wcstringutil.h"
+
+typedef wcstring::size_type size_type;
+
+wcstring_range wcstring_tok(wcstring& str, const wcstring &needle, wcstring_range last)
+{
+    size_type pos = last.second == wcstring::npos ? wcstring::npos : last.first;
+    if (pos != wcstring::npos && last.second != wcstring::npos) pos += last.second;
+    if (pos != wcstring::npos && pos != 0) ++pos;
+    if (pos == wcstring::npos || pos >= str.size())
+    {
+        return std::make_pair(wcstring::npos, wcstring::npos);
+    }
+
+    if (needle.empty())
+    {
+        return std::make_pair(pos, wcstring::npos);
+    }
+
+    pos = str.find_first_not_of(needle, pos);
+    if (pos == wcstring::npos) return std::make_pair(wcstring::npos, wcstring::npos);
+
+    size_type next_pos = str.find_first_of(needle, pos);
+    if (next_pos == wcstring::npos)
+    {
+        return std::make_pair(pos, wcstring::npos);
+    }
+    else
+    {
+        str[next_pos] = L'\0';
+        return std::make_pair(pos, next_pos - pos);
+    }
+}
diff --git a/src/wcstringutil.h b/src/wcstringutil.h
new file mode 100644
index 00000000..4d19fc0b
--- /dev/null
+++ b/src/wcstringutil.h
@@ -0,0 +1,30 @@
+/** \file wcstringutil.h
+
+Helper functions for working with wcstring
+*/
+
+#ifndef FISH_WCSTRINGUTIL_H
+#define FISH_WCSTRINGUTIL_H
+
+#include <string>
+#include <utility>
+#include "common.h"
+
+/**
+ typedef that represents a range in a wcstring.
+ The first element is the location, the second is the count.
+*/
+typedef std::pair<wcstring::size_type, wcstring::size_type> wcstring_range;
+
+/**
+   wcstring equivalent of wcstok(). Supports NUL.
+   For convenience and wcstok() compatibility, the first character of each
+   token separator is replaced with NUL.
+   Returns a pair of (pos, count).
+   Returns (npos, npos) when it's done.
+   Returns (pos, npos) when the token is already known to be the final token.
+   Note that the final token may not necessarily return (pos, npos).
+*/
+wcstring_range wcstring_tok(wcstring& str, const wcstring &needle, wcstring_range last = wcstring_range(0,0));
+
+#endif
diff --git a/src/wgetopt.cpp b/src/wgetopt.cpp
new file mode 100644
index 00000000..9cbe5418
--- /dev/null
+++ b/src/wgetopt.cpp
@@ -0,0 +1,604 @@
+/** \file wgetopt.c
+  A version of the getopt library for use with wide character strings.
+
+  This is simply the gnu getopt library, but converted for use with
+  wchar_t instead of char. This is not usually useful since the argv
+  array is always defined to be of type char**, but in fish, all
+  internal commands use wide characters and hence this library is
+  useful.
+
+  If you want to use this version of getopt in your program,
+  download the fish sourcecode, available at <a
+  href='http://fishshell.com'>the fish homepage</a>. Extract
+  the sourcode, copy wgetopt.c and wgetopt.h into your program
+  directory, include wgetopt.h in your program, and use all the
+  regular getopt functions, prefixing every function, global
+  variable and structure with a 'w', and use only wide character
+  strings. There are no other functional changes in this version of
+  getopt besides using wide character strings.
+
+  For examples of how to use wgetopt, see the fish builtin
+  functions, many of which are defined in builtin.c.
+
+*/
+
+
+/* Getopt for GNU.
+   NOTE: getopt is now part of the C library, so if you don't know what
+   "Keep this file name-space clean" means, talk to roland@gnu.ai.mit.edu
+   before changing it!
+
+   Copyright (C) 1987, 88, 89, 90, 91, 92, 93, 94
+   Free Software Foundation, Inc.
+
+   This file is part of the GNU C Library.  Its master source is NOT part of
+   the C library, however.  The master source lives in /gd/gnu/lib.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Library General Public License as
+   published by the Free Software Foundation; either version 2 of the
+   License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Library General Public License for more details.
+
+   You should have received a copy of the GNU Library General Public
+   License along with the GNU C Library; see the file COPYING.LIB.  If
+   not, write to the Free Software Foundation, Inc., 675 Mass Ave,
+   Cambridge, MA 02139, USA.  */
+
+#include "config.h"
+
+#if !defined (__STDC__) || !__STDC__
+/* This is a separate conditional since some stdc systems
+   reject `defined (const)'.  */
+#ifndef const
+#define const
+#endif
+#endif
+
+#include <stdio.h>
+#include <wchar.h>
+#include "common.h"
+
+/* This needs to come after some library #include
+   to get __GNU_LIBRARY__ defined.  */
+#ifdef  __GNU_LIBRARY__
+/* Don't include stdlib.h for non-GNU C libraries because some of them
+   contain conflicting prototypes for getopt.  */
+#include <stdlib.h>
+#endif  /* GNU C library.  */
+
+/* This version of `getopt' appears to the caller like standard Unix `getopt'
+   but it behaves differently for the user, since it allows the user
+   to intersperse the options with the other arguments.
+
+   As `getopt' works, it permutes the elements of ARGV so that,
+   when it is done, all the options precede everything else.  Thus
+   all application programs are extended to handle flexible argument order.
+
+   GNU application programs can use a third alternative mode in which
+   they can distinguish the relative order of options and other arguments.  */
+
+#include "wgetopt.h"
+#include "wutil.h"
+#include "fallback.h" // IWYU pragma: keep
+
+
+/**
+   Use translation functions if available
+*/
+#ifdef _
+#undef _
+#endif
+
+#ifdef HAVE_TRANSLATE_H
+#ifdef USE_GETTEXT
+#define _(string) wgettext(string)
+#else
+#define _(string) (string)
+#endif
+#else
+#define _(wstr) wstr
+#endif
+
+#ifdef  __GNU_LIBRARY__
+/* We want to avoid inclusion of string.h with non-GNU libraries
+   because there are many ways it can cause trouble.
+   On some systems, it contains special magic macros that don't work
+   in GCC.  */
+#include <string.h> // IWYU pragma: keep
+#define  my_index  wcschr
+#else
+
+/* Avoid depending on library functions or files
+   whose names are inconsistent.  */
+
+char *getenv();
+
+static wchar_t *
+my_index(const wchar_t *str, int chr)
+{
+    while (*str)
+    {
+        if (*str == chr)
+            return (wchar_t *) str;
+        str++;
+    }
+    return 0;
+}
+
+/* If using GCC, we can safely declare strlen this way.
+   If not using GCC, it is ok not to declare it.  */
+#ifdef __GNUC__
+/* Note that Motorola Delta 68k R3V7 comes with GCC but not stddef.h.
+   That was relevant to code that was here before.  */
+#if !defined (__STDC__) || !__STDC__
+/* gcc with -traditional declares the built-in strlen to return int,
+   and has done so at least since version 2.4.5. -- rms.  */
+extern int wcslen(const wchar_t *);
+#endif /* not __STDC__ */
+#endif /* __GNUC__ */
+
+#endif /* not __GNU_LIBRARY__ */
+
+
+/* Exchange two adjacent subsequences of ARGV.
+   One subsequence is elements [first_nonopt,last_nonopt)
+   which contains all the non-options that have been skipped so far.
+   The other is elements [last_nonopt,woptind), which contains all
+   the options processed since those non-options were skipped.
+
+   `first_nonopt' and `last_nonopt' are relocated so that they describe
+   the new indices of the non-options in ARGV after they are moved.  */
+
+void wgetopter_t::exchange(wchar_t **argv)
+{
+    int bottom = first_nonopt;
+    int middle = last_nonopt;
+    int top = woptind;
+    wchar_t *tem;
+
+    /* Exchange the shorter segment with the far end of the longer segment.
+       That puts the shorter segment into the right place.
+       It leaves the longer segment in the right place overall,
+       but it consists of two parts that need to be swapped next.  */
+
+    while (top > middle && middle > bottom)
+    {
+        if (top - middle > middle - bottom)
+        {
+            /* Bottom segment is the short one.  */
+            int len = middle - bottom;
+            int i;
+
+            /* Swap it with the top part of the top segment.  */
+            for (i = 0; i < len; i++)
+            {
+                tem = argv[bottom + i];
+                argv[bottom + i] = argv[top - (middle - bottom) + i];
+                argv[top - (middle - bottom) + i] = tem;
+            }
+            /* Exclude the moved bottom segment from further swapping.  */
+            top -= len;
+        }
+        else
+        {
+            /* Top segment is the short one.  */
+            int len = top - middle;
+            int i;
+
+            /* Swap it with the bottom part of the bottom segment.  */
+            for (i = 0; i < len; i++)
+            {
+                tem = argv[bottom + i];
+                argv[bottom + i] = argv[middle + i];
+                argv[middle + i] = tem;
+            }
+            /* Exclude the moved top segment from further swapping.  */
+            bottom += len;
+        }
+    }
+
+    /* Update records for the slots the non-options now occupy.  */
+
+    first_nonopt += (woptind - last_nonopt);
+    last_nonopt = woptind;
+}
+
+/* Initialize the internal data when the first call is made.  */
+
+const wchar_t * wgetopter_t::_wgetopt_initialize(const wchar_t *optstring)
+{
+    /* Start processing options with ARGV-element 1 (since ARGV-element 0
+       is the program name); the sequence of previously skipped
+       non-option ARGV-elements is empty.  */
+
+    first_nonopt = last_nonopt = woptind = 1;
+
+    nextchar = NULL;
+
+    /* Determine how to handle the ordering of options and nonoptions.  */
+
+    if (optstring[0] == '-')
+    {
+        ordering = RETURN_IN_ORDER;
+        ++optstring;
+    }
+    else if (optstring[0] == '+')
+    {
+        ordering = REQUIRE_ORDER;
+        ++optstring;
+    }
+    else
+        ordering = PERMUTE;
+
+    return optstring;
+}
+
+/* Scan elements of ARGV (whose length is ARGC) for option characters
+   given in OPTSTRING.
+
+   If an element of ARGV starts with '-', and is not exactly "-" or "--",
+   then it is an option element.  The characters of this element
+   (aside from the initial '-') are option characters.  If `getopt'
+   is called repeatedly, it returns successively each of the option characters
+   from each of the option elements.
+
+   If `getopt' finds another option character, it returns that character,
+   updating `woptind' and `nextchar' so that the next call to `getopt' can
+   resume the scan with the following option character or ARGV-element.
+
+   If there are no more option characters, `getopt' returns `EOF'.
+   Then `woptind' is the index in ARGV of the first ARGV-element
+   that is not an option.  (The ARGV-elements have been permuted
+   so that those that are not options now come last.)
+
+   OPTSTRING is a string containing the legitimate option characters.
+   If an option character is seen that is not listed in OPTSTRING,
+   return '?' after printing an error message.  If you set `wopterr' to
+   zero, the error message is suppressed but we still return '?'.
+
+   If a char in OPTSTRING is followed by a colon, that means it wants an arg,
+   so the following text in the same ARGV-element, or the text of the following
+   ARGV-element, is returned in `optarg'.  Two colons mean an option that
+   wants an optional arg; if there is text in the current ARGV-element,
+   it is returned in `w.woptarg', otherwise `w.woptarg' is set to zero.
+
+   If OPTSTRING starts with `-' or `+', it requests different methods of
+   handling the non-option ARGV-elements.
+   See the comments about RETURN_IN_ORDER and REQUIRE_ORDER, above.
+
+   Long-named options begin with `--' instead of `-'.
+   Their names may be abbreviated as long as the abbreviation is unique
+   or is an exact match for some defined option.  If they have an
+   argument, it follows the option name in the same ARGV-element, separated
+   from the option name by a `=', or else the in next ARGV-element.
+   When `getopt' finds a long-named option, it returns 0 if that option's
+   `flag' field is nonzero, the value of the option's `val' field
+   if the `flag' field is zero.
+
+   The elements of ARGV aren't really const, because we permute them.
+   But we pretend they're const in the prototype to be compatible
+   with other systems.
+
+   LONGOPTS is a vector of `struct option' terminated by an
+   element containing a name which is zero.
+
+   LONGIND returns the index in LONGOPT of the long-named option found.
+   It is only valid when a long-named option has been found by the most
+   recent call.
+
+   If LONG_ONLY is nonzero, '-' as well as '--' can introduce
+   long-named options.  */
+
+int wgetopter_t::_wgetopt_internal(int argc, wchar_t *const *argv, const wchar_t *optstring, const struct woption *longopts, int *longind, int long_only)
+{
+    woptarg = NULL;
+
+    if (woptind == 0)
+        optstring = _wgetopt_initialize(optstring);
+
+    if (nextchar == NULL || *nextchar == '\0')
+    {
+        /* Advance to the next ARGV-element.  */
+
+        if (ordering == PERMUTE)
+        {
+            /* If we have just processed some options following some non-options,
+               exchange them so that the options come first.  */
+
+            if (first_nonopt != last_nonopt && last_nonopt != woptind)
+                exchange((wchar_t **) argv);
+            else if (last_nonopt != woptind)
+                first_nonopt = woptind;
+
+            /* Skip any additional non-options
+               and extend the range of non-options previously skipped.  */
+
+            while (woptind < argc
+                    && (argv[woptind][0] != '-' || argv[woptind][1] == '\0'))
+                woptind++;
+            last_nonopt = woptind;
+        }
+
+        /* The special ARGV-element `--' means premature end of options.
+           Skip it like a null option,
+           then exchange with previous non-options as if it were an option,
+           then skip everything else like a non-option.  */
+
+        if (woptind != argc && !wcscmp(argv[woptind], L"--"))
+        {
+            woptind++;
+
+            if (first_nonopt != last_nonopt && last_nonopt != woptind)
+                exchange((wchar_t **) argv);
+            else if (first_nonopt == last_nonopt)
+                first_nonopt = woptind;
+            last_nonopt = argc;
+
+            woptind = argc;
+        }
+
+        /* If we have done all the ARGV-elements, stop the scan
+           and back over any non-options that we skipped and permuted.  */
+
+        if (woptind == argc)
+        {
+            /* Set the next-arg-index to point at the non-options
+               that we previously skipped, so the caller will digest them.  */
+            if (first_nonopt != last_nonopt)
+                woptind = first_nonopt;
+            return EOF;
+        }
+
+        /* If we have come to a non-option and did not permute it,
+           either stop the scan or describe it to the caller and pass it by.  */
+
+        if ((argv[woptind][0] != '-' || argv[woptind][1] == '\0'))
+        {
+            if (ordering == REQUIRE_ORDER)
+                return EOF;
+            woptarg = argv[woptind++];
+            return 1;
+        }
+
+        /* We have found another option-ARGV-element.
+           Skip the initial punctuation.  */
+
+        nextchar = (argv[woptind] + 1
+                    + (longopts != NULL && argv[woptind][1] == '-'));
+    }
+
+    /* Decode the current option-ARGV-element.  */
+
+    /* Check whether the ARGV-element is a long option.
+
+    If long_only and the ARGV-element has the form "-f", where f is
+    a valid short option, don't consider it an abbreviated form of
+    a long option that starts with f.  Otherwise there would be no
+    way to give the -f short option.
+
+    On the other hand, if there's a long option "fubar" and
+    the ARGV-element is "-fu", do consider that an abbreviation of
+    the long option, just like "--fu", and not "-f" with arg "u".
+
+    This distinction seems to be the most useful approach.  */
+
+    if (longopts != NULL
+            && (argv[woptind][1] == '-'
+                || (long_only && (argv[woptind][2] || !my_index(optstring, argv[woptind][1])))))
+    {
+        wchar_t *nameend;
+        const struct woption *p;
+        const struct woption *pfound = NULL;
+        int exact = 0;
+        int ambig = 0;
+        int indfound = 0; /* set to zero by Anton */
+        int option_index;
+
+        for (nameend = nextchar; *nameend && *nameend != '='; nameend++)
+            /* Do nothing.  */ ;
+
+        /* Test all long options for either exact match
+           or abbreviated matches.  */
+        for (p = longopts, option_index = 0; p->name; p++, option_index++)
+            if (!wcsncmp(p->name, nextchar, nameend - nextchar))
+            {
+                if ((unsigned int)(nameend - nextchar) == (unsigned int)wcslen(p->name))
+                {
+                    /* Exact match found.  */
+                    pfound = p;
+                    indfound = option_index;
+                    exact = 1;
+                    break;
+                }
+                else if (pfound == NULL)
+                {
+                    /* First nonexact match found.  */
+                    pfound = p;
+                    indfound = option_index;
+                }
+                else
+                    /* Second or later nonexact match found.  */
+                    ambig = 1;
+            }
+
+        if (ambig && !exact)
+        {
+            if (wopterr)
+                fwprintf(stderr, _(L"%ls: Option '%ls' is ambiguous\n"),
+                         argv[0], argv[woptind]);
+            nextchar += wcslen(nextchar);
+            woptind++;
+            return '?';
+        }
+
+        if (pfound != NULL)
+        {
+            option_index = indfound;
+            woptind++;
+            if (*nameend)
+            {
+                /* Don't test has_arg with >, because some C compilers don't
+                   allow it to be used on enums.  */
+                if (pfound->has_arg)
+                    woptarg = nameend + 1;
+                else
+                {
+                    if (wopterr)
+                    {
+                        if (argv[woptind - 1][1] == '-')
+                            /* --option */
+                            fwprintf(stderr,
+                                     _(L"%ls: Option '--%ls' doesn't allow an argument\n"),
+                                     argv[0], pfound->name);
+                        else
+                            /* +option or -option */
+                            fwprintf(stderr,
+                                     _(L"%ls: Option '%lc%ls' doesn't allow an argument\n"),
+                                     argv[0], argv[woptind - 1][0], pfound->name);
+                    }
+                    nextchar += wcslen(nextchar);
+                    return '?';
+                }
+            }
+            else if (pfound->has_arg == 1)
+            {
+                if (woptind < argc)
+                    woptarg = argv[woptind++];
+                else
+                {
+                    if (wopterr)
+                        fwprintf(stderr, _(L"%ls: Option '%ls' requires an argument\n"),
+                                 argv[0], argv[woptind - 1]);
+                    nextchar += wcslen(nextchar);
+                    return optstring[0] == ':' ? ':' : '?';
+                }
+            }
+            nextchar += wcslen(nextchar);
+            if (longind != NULL)
+                *longind = option_index;
+            if (pfound->flag)
+            {
+                *(pfound->flag) = pfound->val;
+                return 0;
+            }
+            return pfound->val;
+        }
+
+        /* Can't find it as a long option.  If this is not getopt_long_only,
+           or the option starts with '--' or is not a valid short
+           option, then it's an error.
+           Otherwise interpret it as a short option.  */
+        if (!long_only || argv[woptind][1] == '-'
+                || my_index(optstring, *nextchar) == NULL)
+        {
+            if (wopterr)
+            {
+                if (argv[woptind][1] == '-')
+                    /* --option */
+                    fwprintf(stderr, _(L"%ls: Unrecognized option '--%ls'\n"),
+                             argv[0], nextchar);
+                else
+                    /* +option or -option */
+                    fwprintf(stderr, _(L"%ls: Unrecognized option '%lc%ls'\n"),
+                             argv[0], argv[woptind][0], nextchar);
+            }
+            nextchar = (wchar_t *) L"";
+            woptind++;
+            return '?';
+        }
+    }
+
+    /* Look at and handle the next short option-character.  */
+
+    {
+        wchar_t c = *nextchar++;
+        wchar_t *temp = const_cast<wchar_t*>(my_index(optstring, c));
+
+        /* Increment `woptind' when we start to process its last character.  */
+        if (*nextchar == '\0')
+            ++woptind;
+
+        if (temp == NULL || c == ':')
+        {
+            if (wopterr)
+            {
+                fwprintf(stderr, _(L"%ls: Invalid option -- %lc\n"), argv[0], c);
+            }
+            woptopt = c;
+
+            if (*nextchar != '\0')
+                woptind++;
+
+            return '?';
+        }
+        if (temp[1] == ':')
+        {
+            if (temp[2] == ':')
+            {
+                /* This is an option that accepts an argument optionally.  */
+                if (*nextchar != '\0')
+                {
+                    woptarg = nextchar;
+                    woptind++;
+                }
+                else
+                    woptarg = NULL;
+                nextchar = NULL;
+            }
+            else
+            {
+                /* This is an option that requires an argument.  */
+                if (*nextchar != '\0')
+                {
+                    woptarg = nextchar;
+                    /* If we end this ARGV-element by taking the rest as an arg,
+                       we must advance to the next element now.  */
+                    woptind++;
+                }
+                else if (woptind == argc)
+                {
+                    if (wopterr)
+                    {
+                        /* 1003.2 specifies the format of this message.  */
+                        fwprintf(stderr, _(L"%ls: Option requires an argument -- %lc\n"),
+                                 argv[0], c);
+                    }
+                    woptopt = c;
+                    if (optstring[0] == ':')
+                        c = ':';
+                    else
+                        c = '?';
+                }
+                else
+                    /* We already incremented `woptind' once;
+                       increment it again when taking next ARGV-elt as argument.  */
+                    woptarg = argv[woptind++];
+                nextchar = NULL;
+            }
+        }
+        return c;
+    }
+}
+
+int wgetopter_t::wgetopt(int argc, wchar_t *const *argv, const wchar_t *optstring)
+{
+    return _wgetopt_internal(argc, argv, optstring,
+                             (const struct woption *) 0,
+                             (int *) 0,
+                             0);
+}
+
+int wgetopter_t::wgetopt_long(int argc, wchar_t *const *argv, const wchar_t *options, const struct woption *long_options, int *opt_index)
+{
+    return _wgetopt_internal(argc, argv, options, long_options, opt_index, 0);
+}
+
+int wgetopter_t::wgetopt_long_only(int argc, wchar_t *const *argv, const wchar_t *options, const struct woption *long_options, int *opt_index)
+{
+    return _wgetopt_internal(argc, argv, options, long_options, opt_index, 1);
+}
diff --git a/src/wgetopt.h b/src/wgetopt.h
new file mode 100644
index 00000000..91d50489
--- /dev/null
+++ b/src/wgetopt.h
@@ -0,0 +1,218 @@
+/** \file wgetopt.h
+  A version of the getopt library for use with wide character strings.
+
+  This is simply the gnu getopt library, but converted for use with
+  wchar_t instead of char. This is not usually useful since the argv
+  array is always defined to be of type char**, but in fish, all
+  internal commands use wide characters and hence this library is
+  useful.
+
+  If you want to use this version of getopt in your program,
+  download the fish sourcecode, available at <a
+  href='http://fishshell.com'>the fish homepage</a>. Extract
+  the sourcode, copy wgetopt.c and wgetopt.h into your program
+  directory, include wgetopt.h in your program, and use all the
+  regular getopt functions, prefixing every function, global
+  variable and structure with a 'w', and use only wide character
+  strings. There are no other functional changes in this version of
+  getopt besides using wide character strings.
+
+  For examples of how to use wgetopt, see the fish builtin
+  functions, many of which are defined in builtin.c.
+
+*/
+
+
+/* Declarations for getopt.
+   Copyright (C) 1989, 90, 91, 92, 93, 94 Free Software Foundation, Inc.
+
+This file is part of the GNU C Library.  Its master source is NOT part of
+the C library, however.  The master source lives in /gd/gnu/lib.
+
+The GNU C Library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Library General Public License as
+published by the Free Software Foundation; either version 2 of the
+License, or (at your option) any later version.
+
+The GNU C Library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+Library General Public License for more details.
+
+You should have received a copy of the GNU Library General Public
+License along with the GNU C Library; see the file COPYING.LIB.  If
+not, write to the Free Software Foundation, Inc., 675 Mass Ave,
+Cambridge, MA 02139, USA.  */
+
+#ifndef FISH_WGETOPT_H
+#define FISH_WGETOPT_H
+
+#include <wchar.h>
+
+class wgetopter_t
+{
+private:
+    void exchange(wchar_t **argv);
+    const wchar_t * _wgetopt_initialize(const wchar_t *optstring);
+    int _wgetopt_internal(int argc, wchar_t *const *argv, const wchar_t *optstring, const struct woption *longopts, int *longind, int long_only);
+    
+public:
+    /* For communication from `getopt' to the caller.
+     When `getopt' finds an option that takes an argument,
+     the argument value is returned here.
+     Also, when `ordering' is RETURN_IN_ORDER,
+     each non-option ARGV-element is returned here.  */
+    
+    wchar_t *woptarg;
+    
+    /* Index in ARGV of the next element to be scanned.
+     This is used for communication to and from the caller
+     and for communication between successive calls to `getopt'.
+     
+     On entry to `getopt', zero means this is the first call; initialize.
+     
+     When `getopt' returns EOF, this is the index of the first of the
+     non-option elements that the caller should itself scan.
+     
+     Otherwise, `woptind' communicates from one call to the next
+     how much of ARGV has been scanned so far.  */
+    
+    /* XXX 1003.2 says this must be 1 before any call.  */
+    int woptind;
+    
+    
+    /* The next char to be scanned in the option-element
+     in which the last option character we returned was found.
+     This allows us to pick up the scan where we left off.
+     
+     If this is zero, or a null string, it means resume the scan
+     by advancing to the next ARGV-element.  */
+    
+    wchar_t *nextchar;
+    
+    /* Callers store zero here to inhibit the error message
+     for unrecognized options.  */
+    
+    int wopterr;
+    
+    /* Set to an option character which was unrecognized.
+     This must be initialized on some systems to avoid linking in the
+     system's own getopt implementation.  */
+    
+    int woptopt;
+    
+    /* Describe how to deal with options that follow non-option ARGV-elements.
+     
+     If the caller did not specify anything,
+     the default is PERMUTE.
+     
+     REQUIRE_ORDER means don't recognize them as options;
+     stop option processing when the first non-option is seen.
+     This is what Unix does.
+     This mode of operation is selected by using `+' as the first
+     character of the list of option characters.
+     
+     PERMUTE is the default.  We permute the contents of ARGV as we scan,
+     so that eventually all the non-options are at the end.  This allows options
+     to be given in any order, even with programs that were not written to
+     expect this.
+     
+     RETURN_IN_ORDER is an option available to programs that were written
+     to expect options and other ARGV-elements in any order and that care about
+     the ordering of the two.  We describe each non-option ARGV-element
+     as if it were the argument of an option with character code 1.
+     Using `-' as the first character of the list of option characters
+     selects this mode of operation.
+     
+     The special argument `--' forces an end of option-scanning regardless
+     of the value of `ordering'.  In the case of RETURN_IN_ORDER, only
+     `--' can cause `getopt' to return EOF with `woptind' != ARGC.  */
+    
+    enum
+    {
+        REQUIRE_ORDER, PERMUTE, RETURN_IN_ORDER
+    } ordering;
+        
+    /* Handle permutation of arguments.  */
+    
+    /* Describe the part of ARGV that contains non-options that have
+     been skipped.  `first_nonopt' is the index in ARGV of the first of them;
+     `last_nonopt' is the index after the last of them.  */
+    
+    int first_nonopt;
+    int last_nonopt;
+    
+    
+    wgetopter_t() : woptarg(NULL), woptind(0), nextchar(0), wopterr(0), woptopt('?'), first_nonopt(0), last_nonopt(0)
+    {
+    }
+    
+    int wgetopt(int argc, wchar_t *const *argv, const wchar_t *optstring);
+    int wgetopt_long(int argc, wchar_t *const *argv, const wchar_t *options, const struct woption *long_options, int *opt_index);
+    int wgetopt_long_only(int argc, wchar_t *const *argv, const wchar_t *options, const struct woption *long_options, int *opt_index);
+};
+
+/** Describe the long-named options requested by the application.
+   The LONG_OPTIONS argument to getopt_long or getopt_long_only is a vector
+   of `struct option' terminated by an element containing a name which is
+   zero.
+
+   The field `has_arg' is:
+   no_argument    (or 0) if the option does not take an argument,
+   required_argument  (or 1) if the option requires an argument,
+   optional_argument   (or 2) if the option takes an optional argument.
+
+   If the field `flag' is not NULL, it points to a variable that is set
+   to the value given in the field `val' when the option is found, but
+   left unchanged if the option is not found.
+
+   To have a long-named option do something other than set an `int' to
+   a compiled-in constant, such as set a value from `optarg', set the
+   option's `flag' field to zero and its `val' field to a nonzero
+   value (the equivalent single-letter option character, if there is
+   one).  For long options that have a zero `flag' field, `getopt'
+   returns the contents of the `val' field.  */
+
+struct woption
+{
+    /**
+       long name for switch
+    */
+    const wchar_t *name;
+    /**
+      Must be one of no_argument, required_argument and
+      optional_argument.
+
+      has_arg can't be an enum because some compilers complain about
+      type mismatches in all the code that assumes it is an int.
+    */
+    int has_arg;
+
+    /**
+      If non-null, the flag whose value should be set if this switch is encountered
+    */
+    int *flag;
+
+    /**
+      If \c flag is non-null, this is the value that flag will be set
+      to. Otherwise, this is the return-value of the function call.
+    */
+    int val;
+};
+
+/* Names for the values of the `has_arg' field of `struct option'.  */
+
+/**
+   Specifies that a switch does not accept an argument
+*/
+#define  no_argument    0
+/**
+   Specifies that a switch requires an argument
+*/
+#define required_argument  1
+/**
+   Specifies that a switch accepts an optional argument
+*/
+#define optional_argument  2
+
+#endif /* FISH_WGETOPT_H */
diff --git a/src/wildcard.cpp b/src/wildcard.cpp
new file mode 100644
index 00000000..22672f60
--- /dev/null
+++ b/src/wildcard.cpp
@@ -0,0 +1,1010 @@
+/** \file wildcard.c
+
+Fish needs it's own globbing implementation to support
+tab-expansion of globbed parameters. Also provides recursive
+wildcards using **.
+
+*/
+
+#include "config.h" // IWYU pragma: keep
+#include <stdlib.h>
+#include <wchar.h>
+#include <unistd.h>
+#include <sys/stat.h>
+#include <dirent.h>
+#include <errno.h>
+#include <string.h>
+#include <set>
+#include <assert.h>
+#include <stddef.h>
+#include <wctype.h>
+#include <string>
+#include <utility>
+
+#include "fallback.h"
+#include "wutil.h"
+#include "common.h"
+#include "wildcard.h"
+#include "complete.h"
+#include "reader.h"
+#include "expand.h"
+#include <map>
+
+/**
+   This flag is set in the flags parameter of wildcard_expand if the
+   call is part of a recursiv wildcard search. It is used to make sure
+   that the contents of subdirectories are only searched once.
+*/
+#define WILDCARD_RECURSIVE 64
+
+/**
+   The maximum length of a filename token. This is a fallback value,
+   an attempt to find the true value using patchconf is always made.
+*/
+#define MAX_FILE_LENGTH 1024
+
+/**
+   Description for generic executable
+*/
+#define COMPLETE_EXEC_DESC _( L"Executable" )
+/**
+   Description for link to executable
+*/
+#define COMPLETE_EXEC_LINK_DESC _( L"Executable link" )
+
+/**
+   Description for regular file
+*/
+#define COMPLETE_FILE_DESC _( L"File" )
+/**
+   Description for character device
+*/
+#define COMPLETE_CHAR_DESC _( L"Character device" )
+/**
+   Description for block device
+*/
+#define COMPLETE_BLOCK_DESC _( L"Block device" )
+/**
+   Description for fifo buffer
+*/
+#define COMPLETE_FIFO_DESC _( L"Fifo" )
+/**
+   Description for symlink
+*/
+#define COMPLETE_SYMLINK_DESC _( L"Symbolic link" )
+/**
+   Description for symlink
+*/
+#define COMPLETE_DIRECTORY_SYMLINK_DESC _( L"Symbolic link to directory" )
+/**
+   Description for Rotten symlink
+*/
+#define COMPLETE_ROTTEN_SYMLINK_DESC _( L"Rotten symbolic link" )
+/**
+   Description for symlink loop
+*/
+#define COMPLETE_LOOP_SYMLINK_DESC _( L"Symbolic link loop" )
+/**
+   Description for socket files
+*/
+#define COMPLETE_SOCKET_DESC _( L"Socket" )
+/**
+   Description for directories
+*/
+#define COMPLETE_DIRECTORY_DESC _( L"Directory" )
+
+/** Hashtable containing all descriptions that describe an executable */
+static std::map<wcstring, wcstring> suffix_map;
+
+// Implementation of wildcard_has. Needs to take the length to handle embedded nulls (#1631)
+static bool wildcard_has_impl(const wchar_t *str, size_t len, bool internal)
+{
+    assert(str != NULL);
+    const wchar_t *end = str + len;
+    if (internal)
+    {
+        for (; str < end; str++)
+        {
+            if ((*str == ANY_CHAR) || (*str == ANY_STRING) || (*str == ANY_STRING_RECURSIVE))
+                return true;
+        }
+    }
+    else
+    {
+        wchar_t prev=0;
+        for (; str < end; str++)
+        {
+            if (((*str == L'*') || (*str == L'?')) && (prev != L'\\'))
+                return true;
+            prev = *str;
+        }
+    }
+
+    return false;
+}
+
+bool wildcard_has(const wchar_t *str, bool internal)
+{
+    assert(str != NULL);
+    return wildcard_has_impl(str, wcslen(str), internal);
+}
+
+bool wildcard_has(const wcstring &str, bool internal)
+{
+    return wildcard_has_impl(str.data(), str.size(), internal);
+}
+
+
+/**
+   Check whether the string str matches the wildcard string wc.
+
+   \param str String to be matched.
+   \param wc The wildcard.
+   \param is_first Whether files beginning with dots should not be matched against wildcards.
+*/
+static bool wildcard_match_internal(const wchar_t *str, const wchar_t *wc, bool leading_dots_fail_to_match, bool is_first)
+{
+    if (*str == 0 && *wc==0)
+        return true;
+
+    /* Hackish fix for https://github.com/fish-shell/fish-shell/issues/270 . Prevent wildcards from matching . or .., but we must still allow literal matches. */
+    if (leading_dots_fail_to_match && is_first && contains(str, L".", L".."))
+    {
+        /* The string is '.' or '..'. Return true if the wildcard exactly matches. */
+        return ! wcscmp(str, wc);
+    }
+
+    if (*wc == ANY_STRING || *wc == ANY_STRING_RECURSIVE)
+    {
+        /* Ignore hidden file */
+        if (leading_dots_fail_to_match && is_first && *str == L'.')
+        {
+            return false;
+        }
+
+        /* Try all submatches */
+        do
+        {
+            if (wildcard_match_internal(str, wc+1, leading_dots_fail_to_match, false))
+                return true;
+        }
+        while (*(str++) != 0);
+        return false;
+    }
+    else if (*str == 0)
+    {
+        /*
+          End of string, but not end of wildcard, and the next wildcard
+          element is not a '*', so this is not a match.
+        */
+        return false;
+    }
+
+    if (*wc == ANY_CHAR)
+    {
+        if (is_first && *str == L'.')
+        {
+            return false;
+        }
+
+        return wildcard_match_internal(str+1, wc+1, leading_dots_fail_to_match, false);
+    }
+
+    if (*wc == *str)
+        return wildcard_match_internal(str+1, wc+1, leading_dots_fail_to_match, false);
+
+    return false;
+}
+
+/**
+   Matches the string against the wildcard, and if the wildcard is a
+   possible completion of the string, the remainder of the string is
+   inserted into the out vector.
+*/
+static bool wildcard_complete_internal(const wcstring &orig,
+                                       const wchar_t *str,
+                                       const wchar_t *wc,
+                                       bool is_first,
+                                       const wchar_t *desc,
+                                       wcstring(*desc_func)(const wcstring &),
+                                       std::vector<completion_t> &out,
+                                       expand_flags_t expand_flags,
+                                       complete_flags_t flags)
+{
+    if (!wc || ! str || orig.empty())
+    {
+        debug(2, L"Got null string on line %d of file %s", __LINE__, __FILE__);
+        return 0;
+    }
+
+    bool has_match = false;
+    string_fuzzy_match_t fuzzy_match(fuzzy_match_exact);
+    const bool at_end_of_wildcard = (*wc == L'\0');
+    const wchar_t *completion_string = NULL;
+
+    // Hack hack hack
+    // Implement EXPAND_FUZZY_MATCH by short-circuiting everything if there are no remaining wildcards
+    if ((expand_flags & EXPAND_FUZZY_MATCH) && ! at_end_of_wildcard && ! wildcard_has(wc, true))
+    {
+        string_fuzzy_match_t local_fuzzy_match = string_fuzzy_match_string(wc, str);
+        if (local_fuzzy_match.type != fuzzy_match_none)
+        {
+            has_match = true;
+            fuzzy_match = local_fuzzy_match;
+
+            /* If we're not a prefix or exact match, then we need to replace the token. Note that in this case we're not going to call ourselves recursively, so these modified flags won't "leak" except into the completion. */
+            if (match_type_requires_full_replacement(local_fuzzy_match.type))
+            {
+                flags |= COMPLETE_REPLACES_TOKEN;
+                completion_string = orig.c_str();
+            }
+            else
+            {
+                /* Since we are not replacing the token, be careful to only store the part of the string after the wildcard */
+                size_t wc_len = wcslen(wc);
+                assert(wcslen(str) >= wc_len);
+                completion_string = str + wcslen(wc);
+            }
+        }
+    }
+
+    /* Maybe we satisfied the wildcard normally */
+    if (! has_match)
+    {
+        bool file_has_leading_dot = (is_first && str[0] == L'.');
+        if (at_end_of_wildcard && ! file_has_leading_dot)
+        {
+            has_match = true;
+            if (flags & COMPLETE_REPLACES_TOKEN)
+            {
+                completion_string = orig.c_str();
+            }
+            else
+            {
+                completion_string = str;
+            }
+        }
+    }
+
+    if (has_match)
+    {
+        /* Wildcard complete */
+        assert(completion_string != NULL);
+        wcstring out_completion = completion_string;
+        wcstring out_desc = (desc ? desc : L"");
+
+        size_t complete_sep_loc = out_completion.find(PROG_COMPLETE_SEP);
+        if (complete_sep_loc != wcstring::npos)
+        {
+            /* This completion has an embedded description, do not use the generic description */
+            out_desc.assign(out_completion, complete_sep_loc + 1, out_completion.size() - complete_sep_loc - 1);
+            out_completion.resize(complete_sep_loc);
+        }
+        else
+        {
+            if (desc_func && !(expand_flags & EXPAND_NO_DESCRIPTIONS))
+            {
+                /*
+                  A description generating function is specified, call
+                  it. If it returns something, use that as the
+                  description.
+                */
+                wcstring func_desc = desc_func(orig);
+                if (! func_desc.empty())
+                    out_desc = func_desc;
+            }
+
+        }
+
+        /* Note: out_completion may be empty if the completion really is empty, e.g. tab-completing 'foo' when a file 'foo' exists. */
+        append_completion(out, out_completion, out_desc, flags, fuzzy_match);
+        return true;
+    }
+
+    if (*wc == ANY_STRING)
+    {
+        bool res=false;
+
+        /* Ignore hidden file */
+        if (is_first && str[0] == L'.')
+            return false;
+
+        /* Try all submatches */
+        for (size_t i=0; str[i] != L'\0'; i++)
+        {
+            const size_t before_count = out.size();
+            if (wildcard_complete_internal(orig, str + i, wc+1, false, desc, desc_func, out, expand_flags, flags))
+            {
+                res = true;
+
+                /* #929: if the recursive call gives us a prefix match, just stop. This is sloppy - what we really want to do is say, once we've seen a match of a particular type, ignore all matches of that type further down the string, such that the wildcard produces the "minimal match." */
+                bool has_prefix_match = false;
+                const size_t after_count = out.size();
+                for (size_t j = before_count; j < after_count; j++)
+                {
+                    if (out[j].match.type <= fuzzy_match_prefix)
+                    {
+                        has_prefix_match = true;
+                        break;
+                    }
+                }
+                if (has_prefix_match)
+                    break;
+            }
+        }
+        return res;
+
+    }
+    else if (*wc == ANY_CHAR || *wc == *str)
+    {
+        return wildcard_complete_internal(orig, str+1, wc+1, false, desc, desc_func, out, expand_flags, flags);
+    }
+    else if (towlower(*wc) == towlower(*str))
+    {
+        return wildcard_complete_internal(orig, str+1, wc+1, false, desc, desc_func, out, expand_flags, flags | COMPLETE_REPLACES_TOKEN);
+    }
+    return false;
+}
+
+bool wildcard_complete(const wcstring &str,
+                       const wchar_t *wc,
+                       const wchar_t *desc,
+                       wcstring(*desc_func)(const wcstring &),
+                       std::vector<completion_t> &out,
+                       expand_flags_t expand_flags,
+                       complete_flags_t flags)
+{
+    bool res;
+    res =  wildcard_complete_internal(str, str.c_str(), wc, true, desc, desc_func, out, expand_flags, flags);
+    return res;
+}
+
+
+bool wildcard_match(const wcstring &str, const wcstring &wc, bool leading_dots_fail_to_match)
+{
+    return wildcard_match_internal(str.c_str(), wc.c_str(), leading_dots_fail_to_match, true /* first */);
+}
+
+/**
+   Creates a path from the specified directory and filename.
+*/
+static wcstring make_path(const wcstring &base_dir, const wcstring &name)
+{
+    return base_dir + name;
+}
+
+/**
+   Obtain a description string for the file specified by the filename.
+
+   The returned value is a string constant and should not be free'd.
+
+   \param filename The file for which to find a description string
+   \param lstat_res The result of calling lstat on the file
+   \param lbuf The struct buf output of calling lstat on the file
+   \param stat_res The result of calling stat on the file
+   \param buf The struct buf output of calling stat on the file
+   \param err The errno value after a failed stat call on the file.
+*/
+
+static wcstring file_get_desc(const wcstring &filename,
+                              int lstat_res,
+                              const struct stat &lbuf,
+                              int stat_res,
+                              struct stat buf,
+                              int err)
+{
+
+    if (!lstat_res)
+    {
+        if (S_ISLNK(lbuf.st_mode))
+        {
+            if (!stat_res)
+            {
+                if (S_ISDIR(buf.st_mode))
+                {
+                    return COMPLETE_DIRECTORY_SYMLINK_DESC;
+                }
+                else
+                {
+
+                    if ((buf.st_mode & S_IXUSR) ||
+                            (buf.st_mode & S_IXGRP) ||
+                            (buf.st_mode & S_IXOTH))
+                    {
+
+                        if (waccess(filename, X_OK) == 0)
+                        {
+                            /*
+                              Weird group permissions and other such
+                              issues make it non-trivial to find out
+                              if we can actually execute a file using
+                              the result from stat. It is much safer
+                              to use the access function, since it
+                              tells us exactly what we want to know.
+                            */
+                            return COMPLETE_EXEC_LINK_DESC;
+                        }
+                    }
+                }
+
+                return COMPLETE_SYMLINK_DESC;
+
+            }
+            else
+            {
+                switch (err)
+                {
+                    case ENOENT:
+                    {
+                        return COMPLETE_ROTTEN_SYMLINK_DESC;
+                    }
+
+                    case ELOOP:
+                    {
+                        return COMPLETE_LOOP_SYMLINK_DESC;
+                    }
+                }
+                /*
+                  On unknown errors we do nothing. The file will be
+                  given the default 'File' description or one based on the suffix.
+                */
+            }
+
+        }
+        else if (S_ISCHR(buf.st_mode))
+        {
+            return COMPLETE_CHAR_DESC;
+        }
+        else if (S_ISBLK(buf.st_mode))
+        {
+            return COMPLETE_BLOCK_DESC;
+        }
+        else if (S_ISFIFO(buf.st_mode))
+        {
+            return COMPLETE_FIFO_DESC;
+        }
+        else if (S_ISSOCK(buf.st_mode))
+        {
+            return COMPLETE_SOCKET_DESC;
+        }
+        else if (S_ISDIR(buf.st_mode))
+        {
+            return COMPLETE_DIRECTORY_DESC;
+        }
+        else
+        {
+            if ((buf.st_mode & S_IXUSR) ||
+                    (buf.st_mode & S_IXGRP) ||
+                    (buf.st_mode & S_IXOTH))
+            {
+
+                if (waccess(filename, X_OK) == 0)
+                {
+                    /*
+                      Weird group permissions and other such issues
+                      make it non-trivial to find out if we can
+                      actually execute a file using the result from
+                      stat. It is much safer to use the access
+                      function, since it tells us exactly what we want
+                      to know.
+                    */
+                    return COMPLETE_EXEC_DESC;
+                }
+            }
+        }
+    }
+
+    return COMPLETE_FILE_DESC ;
+}
+
+
+/**
+   Add the specified filename if it matches the specified wildcard.
+
+   If the filename matches, first get the description of the specified
+   filename. If this is a regular file, append the filesize to the
+   description.
+
+   \param list the list to add he completion to
+   \param fullname the full filename of the file
+   \param completion the completion part of the file name
+   \param wc the wildcard to match against
+   \param is_cmd whether we are performing command completion
+*/
+static void wildcard_completion_allocate(std::vector<completion_t> &list,
+        const wcstring &fullname,
+        const wcstring &completion,
+        const wchar_t *wc,
+        expand_flags_t expand_flags)
+{
+    struct stat buf, lbuf;
+    wcstring sb;
+    wcstring munged_completion;
+
+    int flags = 0;
+    int stat_res, lstat_res;
+    int stat_errno=0;
+
+    long long sz;
+
+    /*
+      If the file is a symlink, we need to stat both the file itself
+      _and_ the destination file. But we try to avoid this with
+      non-symlinks by first doing an lstat, and if the file is not a
+      link we copy the results over to the regular stat buffer.
+    */
+    if ((lstat_res = lwstat(fullname, &lbuf)))
+    {
+        /* lstat failed! */
+        sz=-1;
+        stat_res = lstat_res;
+    }
+    else
+    {
+        if (S_ISLNK(lbuf.st_mode))
+        {
+
+            if ((stat_res = wstat(fullname, &buf)))
+            {
+                sz=-1;
+            }
+            else
+            {
+                sz = (long long)buf.st_size;
+            }
+
+            /*
+              In order to differentiate between e.g. rotten symlinks
+              and symlink loops, we also need to know the error status of wstat.
+            */
+            stat_errno = errno;
+        }
+        else
+        {
+            stat_res = lstat_res;
+            memcpy(&buf, &lbuf, sizeof(struct stat));
+            sz = (long long)buf.st_size;
+        }
+    }
+
+
+    bool wants_desc = !(expand_flags & EXPAND_NO_DESCRIPTIONS);
+    wcstring desc;
+    if (wants_desc)
+        desc = file_get_desc(fullname, lstat_res, lbuf, stat_res, buf, stat_errno);
+
+    if (sz >= 0 && S_ISDIR(buf.st_mode))
+    {
+        flags |= COMPLETE_NO_SPACE;
+        munged_completion = completion;
+        munged_completion.push_back(L'/');
+        if (wants_desc)
+            sb.append(desc);
+    }
+    else
+    {
+        if (wants_desc)
+        {
+            if (! desc.empty())
+            {
+                sb.append(desc);
+                sb.append(L", ");
+            }
+            sb.append(format_size(sz));
+        }
+    }
+
+    const wcstring &completion_to_use = munged_completion.empty() ? completion : munged_completion;
+    wildcard_complete(completion_to_use, wc, sb.c_str(), NULL, list, expand_flags, flags);
+}
+
+/**
+  Test if the file specified by the given filename matches the
+  expansion flags specified. flags can be a combination of
+  EXECUTABLES_ONLY and DIRECTORIES_ONLY.
+*/
+static bool test_flags(const wchar_t *filename, expand_flags_t flags)
+{
+    if (flags & DIRECTORIES_ONLY)
+    {
+        struct stat buf;
+        if (wstat(filename, &buf) == -1)
+        {
+            return false;
+        }
+
+        if (!S_ISDIR(buf.st_mode))
+        {
+            return false;
+        }
+    }
+
+
+    if (flags & EXECUTABLES_ONLY)
+    {
+        if (waccess(filename, X_OK) != 0)
+            return false;
+        struct stat buf;
+        if (wstat(filename, &buf) == -1)
+        {
+            return false;
+        }
+
+        if (!S_ISREG(buf.st_mode))
+        {
+            return false;
+        }
+    }
+
+    return true;
+}
+
+/** Appends a completion to the completion list, if the string is missing from the set. */
+static void insert_completion_if_missing(const wcstring &str, std::vector<completion_t> &out, std::set<wcstring> &completion_set)
+{
+    if (completion_set.insert(str).second)
+        append_completion(out, str);
+}
+
+/**
+   The real implementation of wildcard expansion is in this
+   function. Other functions are just wrappers around this one.
+
+   This function traverses the relevant directory tree looking for
+   matches, and recurses when needed to handle wildcrards spanning
+   multiple components and recursive wildcards.
+
+   Because this function calls itself recursively with substrings,
+   it's important that the parameters be raw pointers instead of wcstring,
+   which would be too expensive to construct for all substrings.
+ */
+static int wildcard_expand_internal(const wchar_t *wc,
+                                    const wchar_t * const base_dir,
+                                    expand_flags_t flags,
+                                    std::vector<completion_t> &out,
+                                    std::set<wcstring> &completion_set,
+                                    std::set<file_id_t> &visited_files)
+{
+    /* Variables for traversing a directory */
+    DIR *dir;
+
+    /* The result returned */
+    int res = 0;
+
+
+    //  debug( 3, L"WILDCARD_EXPAND %ls in %ls", wc, base_dir );
+
+    if (is_main_thread() ? reader_interrupted() : reader_thread_job_is_stale())
+    {
+        return -1;
+    }
+
+    if (!wc || !base_dir)
+    {
+        debug(2, L"Got null string on line %d of file %s", __LINE__, __FILE__);
+        return 0;
+    }
+
+    const size_t base_dir_len = wcslen(base_dir);
+    const size_t wc_len = wcslen(wc);
+
+    if (flags & ACCEPT_INCOMPLETE)
+    {
+        /*
+           Avoid excessive number of returned matches for wc ending with a *
+        */
+        if (wc_len > 0 && (wc[wc_len-1]==ANY_STRING))
+        {
+            wchar_t * foo = wcsdup(wc);
+            foo[wc_len-1]=0;
+            int res = wildcard_expand_internal(foo, base_dir, flags, out, completion_set, visited_files);
+            free(foo);
+            return res;
+        }
+    }
+
+    /* Determine if we are the last segment */
+    const wchar_t * const next_slash = wcschr(wc,L'/');
+    const bool is_last_segment = (next_slash == NULL);
+    const size_t wc_segment_len = next_slash ? next_slash - wc : wc_len;
+    const wcstring wc_segment = wcstring(wc, wc_segment_len);
+    
+    /* Maybe this segment has no wildcards at all. If this is not the last segment, and it has no wildcards, then we don't need to match against the directory contents, and in fact we don't want to match since we may not be able to read it anyways (#2099). Don't even open the directory! */
+    const bool segment_has_wildcards = wildcard_has(wc_segment, true /* internal, i.e. look for ANY_CHAR instead of ? */);
+    if (! segment_has_wildcards && ! is_last_segment)
+    {
+        wcstring new_base_dir = make_path(base_dir, wc_segment);
+        new_base_dir.push_back(L'/');
+        
+        /* Skip multiple separators */
+        assert(next_slash != NULL);
+        const wchar_t *new_wc = next_slash;
+        while (*new_wc==L'/')
+        {
+            new_wc++;
+        }
+        /* Early out! */
+        return wildcard_expand_internal(new_wc, new_base_dir.c_str(), flags, out, completion_set, visited_files);
+    }
+    
+    /* Test for recursive match string in current segment */
+    const bool is_recursive = (wc_segment.find(ANY_STRING_RECURSIVE) != wcstring::npos);
+    
+
+    const wchar_t *base_dir_or_cwd = (base_dir[0] == L'\0') ? L"." : base_dir;
+    if (!(dir = wopendir(base_dir_or_cwd)))
+    {
+        return 0;
+    }
+    
+    /*
+      Is this segment of the wildcard the last?
+    */
+    if (is_last_segment)
+    {
+        /*
+          Wildcard segment is the last segment,
+
+          Insert all matching files/directories
+        */
+        if (wc[0]=='\0')
+        {
+            /*
+              The last wildcard segment is empty. Insert everything if
+              completing, the directory itself otherwise.
+            */
+            if (flags & ACCEPT_INCOMPLETE)
+            {
+                wcstring next;
+                while (wreaddir(dir, next))
+                {
+                    if (next[0] != L'.')
+                    {
+                        wcstring long_name = make_path(base_dir, next);
+
+                        if (test_flags(long_name.c_str(), flags))
+                        {
+                            wildcard_completion_allocate(out, long_name, next, L"", flags);
+                        }
+                    }
+                }
+            }
+            else
+            {
+                res = 1;
+                insert_completion_if_missing(base_dir, out, completion_set);
+            }
+        }
+        else
+        {
+            /* This is the last wildcard segment, and it is not empty. Match files/directories. */
+            wcstring name_str;
+            while (wreaddir(dir, name_str))
+            {
+                if (flags & ACCEPT_INCOMPLETE)
+                {
+
+                    const wcstring long_name = make_path(base_dir, name_str);
+
+                    /* Test for matches before stating file, so as to minimize the number of calls to the much slower stat function. The only expand flag we care about is EXPAND_FUZZY_MATCH; we have no complete flags. */
+                    std::vector<completion_t> test;
+                    if (wildcard_complete(name_str, wc, L"", NULL, test, flags & EXPAND_FUZZY_MATCH, 0))
+                    {
+                        if (test_flags(long_name.c_str(), flags))
+                        {
+                            wildcard_completion_allocate(out, long_name, name_str, wc, flags);
+
+                        }
+                    }
+                }
+                else
+                {
+                    if (wildcard_match(name_str, wc, true /* skip files with leading dots */))
+                    {
+                        const wcstring long_name = make_path(base_dir, name_str);
+                        int skip = 0;
+
+                        if (is_recursive)
+                        {
+                            /*
+                              In recursive mode, we are only
+                              interested in adding files -directories
+                              will be added in the next pass.
+                            */
+                            struct stat buf;
+                            if (!wstat(long_name, &buf))
+                            {
+                                skip = S_ISDIR(buf.st_mode);
+                            }
+                        }
+                        if (! skip)
+                        {
+                            insert_completion_if_missing(long_name, out, completion_set);
+                        }
+                        res = 1;
+                    }
+                }
+            }
+        }
+    }
+
+    if ((! is_last_segment) || is_recursive)
+    {
+        /*
+          Wilcard segment is not the last segment.  Recursively call
+          wildcard_expand for all matching subdirectories.
+        */
+
+        /*
+          In recursive mode, we look through the directory twice. If
+          so, this rewind is needed.
+        */
+        rewinddir(dir);
+
+        /* new_dir is a scratch area containing the full path to a file/directory we are iterating over */
+        wcstring new_dir = base_dir;
+
+        wcstring name_str;
+        while (wreaddir(dir, name_str))
+        {
+            /*
+              Test if the file/directory name matches the whole
+              wildcard element, i.e. regular matching.
+            */
+            bool whole_match = wildcard_match(name_str, wc_segment, true /* ignore leading dots */);
+            bool partial_match = false;
+
+            /*
+               If we are doing recursive matching, also check if this
+               directory matches the part up to the recusrive
+               wildcard, if so, then we can search all subdirectories
+               for matches.
+            */
+            if (is_recursive)
+            {
+                const wchar_t *end = wcschr(wc, ANY_STRING_RECURSIVE);
+                wchar_t *wc_sub = wcsndup(wc, end-wc+1);
+                partial_match = wildcard_match(name_str, wc_sub, true /* ignore leading dots */);
+                free(wc_sub);
+            }
+
+            if (whole_match || partial_match)
+            {
+                struct stat buf;
+                int new_res;
+
+                // new_dir is base_dir + some other path components
+                // Replace everything after base_dir with the new path component
+                new_dir.replace(base_dir_len, wcstring::npos, name_str);
+
+                int stat_res = wstat(new_dir, &buf);
+
+                if (!stat_res)
+                {
+                    // Insert a "file ID" into visited_files
+                    // If the insertion fails, we've already visited this file (i.e. a symlink loop)
+                    // If we're not recursive, insert anyways (in case we loop back around in a future recursive segment), but continue on; the idea being that literal path components should still work
+                    const file_id_t file_id = file_id_t::file_id_from_stat(&buf);
+                    if (S_ISDIR(buf.st_mode) && (visited_files.insert(file_id).second || ! is_recursive))
+                    {
+                        new_dir.push_back(L'/');
+
+                        /*
+                          Regular matching
+                        */
+                        if (whole_match)
+                        {
+                            const wchar_t *new_wc = L"";
+                            if (next_slash)
+                            {
+                                new_wc=next_slash+1;
+                                /*
+                                  Accept multiple '/' as a single directory separator
+                                */
+                                while (*new_wc==L'/')
+                                {
+                                    new_wc++;
+                                }
+                            }
+
+                            new_res = wildcard_expand_internal(new_wc,
+                                                               new_dir.c_str(),
+                                                               flags,
+                                                               out,
+                                                               completion_set,
+                                                               visited_files);
+
+                            if (new_res == -1)
+                            {
+                                res = -1;
+                                break;
+                            }
+                            res |= new_res;
+
+                        }
+
+                        /*
+                          Recursive matching
+                        */
+                        if (partial_match)
+                        {
+
+                            new_res = wildcard_expand_internal(wcschr(wc, ANY_STRING_RECURSIVE),
+                                                               new_dir.c_str(),
+                                                               flags | WILDCARD_RECURSIVE,
+                                                               out,
+                                                               completion_set,
+                                                               visited_files);
+
+                            if (new_res == -1)
+                            {
+                                res = -1;
+                                break;
+                            }
+                            res |= new_res;
+
+                        }
+                    }
+                }
+            }
+        }
+    }
+    closedir(dir);
+
+    return res;
+}
+
+
+int wildcard_expand(const wchar_t *wc,
+                    const wchar_t *base_dir,
+                    expand_flags_t flags,
+                    std::vector<completion_t> &out)
+{
+    size_t c = out.size();
+
+    /* Make a set of used completion strings so we can do fast membership tests inside wildcard_expand_internal. Otherwise wildcards like '**' are very slow, because we end up with an N^2 membership test.
+    */
+    std::set<wcstring> completion_set;
+    for (std::vector<completion_t>::const_iterator iter = out.begin(); iter != out.end(); ++iter)
+    {
+        completion_set.insert(iter->completion);
+    }
+
+    std::set<file_id_t> visited_files;
+    int res = wildcard_expand_internal(wc, base_dir, flags, out, completion_set, visited_files);
+
+    if (flags & ACCEPT_INCOMPLETE)
+    {
+        wcstring wc_base;
+        const wchar_t *wc_base_ptr = wcsrchr(wc, L'/');
+        if (wc_base_ptr)
+        {
+            wc_base = wcstring(wc, (wc_base_ptr-wc)+1);
+        }
+
+        for (size_t i=c; i<out.size(); i++)
+        {
+            completion_t &c = out.at(i);
+
+            if (c.flags & COMPLETE_REPLACES_TOKEN)
+            {
+                c.completion = format_string(L"%ls%ls%ls", base_dir, wc_base.c_str(), c.completion.c_str());
+            }
+        }
+    }
+    return res;
+}
+
+int wildcard_expand_string(const wcstring &wc, const wcstring &base_dir, expand_flags_t flags, std::vector<completion_t> &outputs)
+{
+    /* Hackish fix for 1631. We are about to call c_str(), which will produce a string truncated at any embedded nulls. We could fix this by passing around the size, etc. However embedded nulls are never allowed in a filename, so we just check for them and return 0 (no matches) if there is an embedded null. This isn't quite right, e.g. it will fail for \0?, but that is an edge case. */
+    if (wc.find(L'\0') != wcstring::npos)
+    {
+        return 0;
+    }
+    // PCA: not convinced this temporary variable is really necessary
+    std::vector<completion_t> lst;
+    int res = wildcard_expand(wc.c_str(), base_dir.c_str(), flags, lst);
+    outputs.insert(outputs.end(), lst.begin(), lst.end());
+    return res;
+}
diff --git a/src/wildcard.h b/src/wildcard.h
new file mode 100644
index 00000000..9e70f4aa
--- /dev/null
+++ b/src/wildcard.h
@@ -0,0 +1,94 @@
+/** \file wildcard.h
+
+    My own globbing implementation. Needed to implement this instead
+    of using libs globbing to support tab-expansion of globbed
+    paramaters.
+
+*/
+
+#ifndef FISH_WILDCARD_H
+/**
+   Header guard
+*/
+#define FISH_WILDCARD_H
+
+#include <vector>
+
+#include "common.h"
+#include "expand.h"
+#include "complete.h"
+
+/*
+  Use unencoded private-use keycodes for internal characters
+*/
+
+#define WILDCARD_RESERVED 0xf400
+
+/**
+   Enumeration of all wildcard types
+*/
+enum
+{
+    /** Character representing any character except '/' */
+    ANY_CHAR = WILDCARD_RESERVED,
+
+    /** Character representing any character string not containing '/' (A slash) */
+    ANY_STRING,
+
+    /** Character representing any character string */
+    ANY_STRING_RECURSIVE,
+}
+;
+
+/**
+    Expand the wildcard by matching against the filesystem.
+
+  New strings are allocated using malloc and should be freed by the caller.
+
+  wildcard_expand works by dividing the wildcard into segments at
+  each directory boundary. Each segment is processed separatly. All
+  except the last segment are handled by matching the wildcard
+  segment against all subdirectories of matching directories, and
+  recursively calling wildcard_expand for matches. On the last
+  segment, matching is made to any file, and all matches are
+  inserted to the list.
+
+  If wildcard_expand encounters any errors (such as insufficient
+  priviliges) during matching, no error messages will be printed and
+  wildcard_expand will continue the matching process.
+
+  \param wc The wildcard string
+  \param base_dir The base directory of the filesystem to perform the match against
+  \param flags flags for the search. Can be any combination of ACCEPT_INCOMPLETE and EXECUTABLES_ONLY
+  \param out The list in which to put the output
+
+  \return 1 if matches where found, 0 otherwise. Return -1 on abort (I.e. ^C was pressed).
+
+*/
+int wildcard_expand_string(const wcstring &wc, const wcstring &base_dir, expand_flags_t flags, std::vector<completion_t> &out);
+/**
+   Test whether the given wildcard matches the string. Does not perform any I/O.
+
+   \param str The string to test
+   \param wc The wildcard to test against
+   \param leading_dots_fail_to_match if set, strings with leading dots are assumed to be hidden files and are not matched
+   \return true if the wildcard matched
+*/
+bool wildcard_match(const wcstring &str, const wcstring &wc, bool leading_dots_fail_to_match = false);
+
+/** Check if the specified string contains wildcards */
+bool wildcard_has(const wcstring &, bool internal);
+bool wildcard_has(const wchar_t *, bool internal);
+
+/**
+   Test wildcard completion
+*/
+bool wildcard_complete(const wcstring &str,
+                       const wchar_t *wc,
+                       const wchar_t *desc,
+                       wcstring(*desc_func)(const wcstring &),
+                       std::vector<completion_t> &out,
+                       expand_flags_t expand_flags,
+                       complete_flags_t flags);
+
+#endif
diff --git a/src/wutil.cpp b/src/wutil.cpp
new file mode 100644
index 00000000..e5127720
--- /dev/null
+++ b/src/wutil.cpp
@@ -0,0 +1,611 @@
+/** \file wutil.c
+  Wide character equivalents of various standard unix
+  functions.
+*/
+#include "config.h"
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <unistd.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <wchar.h>
+#include <string.h>
+#include <dirent.h>
+#include <limits.h>
+#include <libgen.h>
+#include <pthread.h>
+#include <assert.h>
+#include <features.h>
+#include <string>
+#include <map>
+
+#include "fallback.h"
+
+#include "common.h"
+#include "wutil.h"
+
+typedef std::string cstring;
+
+const file_id_t kInvalidFileID = {(dev_t)-1LL, (ino_t)-1LL, (uint64_t)-1LL, -1, -1, (uint32_t)-1};
+
+/**
+   Minimum length of the internal covnersion buffers
+*/
+#define TMP_LEN_MIN 256
+
+#ifndef PATH_MAX
+#ifdef MAXPATHLEN
+#define PATH_MAX MAXPATHLEN
+#else
+/**
+   Fallback length of MAXPATHLEN. Just a hopefully sane value...
+*/
+#define PATH_MAX 4096
+#endif
+#endif
+
+/* Lock to protect wgettext */
+static pthread_mutex_t wgettext_lock;
+
+/* Maps string keys to (immortal) pointers to string values. */
+typedef std::map<wcstring, const wchar_t *> wgettext_map_t;
+static wgettext_map_t wgettext_map;
+
+void wutil_init()
+{
+}
+
+void wutil_destroy()
+{
+}
+
+bool wreaddir_resolving(DIR *dir, const std::wstring &dir_path, std::wstring &out_name, bool *out_is_dir)
+{
+    struct dirent *d = readdir(dir);
+    if (!d) return false;
+
+    out_name = str2wcstring(d->d_name);
+    if (out_is_dir)
+    {
+        /* The caller cares if this is a directory, so check */
+        bool is_dir = false;
+        
+        /* We may be able to skip stat, if the readdir can tell us the file type directly */
+        bool check_with_stat = true;
+#ifdef HAVE_STRUCT_DIRENT_D_TYPE
+        if (d->d_type == DT_DIR)
+        {
+            /* Known directory */
+            is_dir = true;
+            check_with_stat = false;
+        }
+        else if (d->d_type == DT_LNK || d->d_type == DT_UNKNOWN)
+        {
+            /* We want to treat symlinks to directories as directories. Use stat to resolve it. */
+            check_with_stat = true;
+        }
+        else
+        {
+            /* Regular file */
+            is_dir = false;
+            check_with_stat = false;
+        }
+#endif // HAVE_STRUCT_DIRENT_D_TYPE
+        if (check_with_stat)
+        {
+            /* We couldn't determine the file type from the dirent; check by stat'ing it */
+            cstring fullpath = wcs2string(dir_path);
+            fullpath.push_back('/');
+            fullpath.append(d->d_name);
+            struct stat buf;
+            if (stat(fullpath.c_str(), &buf) != 0)
+            {
+                is_dir = false;
+            }
+            else
+            {
+                is_dir = !!(S_ISDIR(buf.st_mode));
+            }
+        }
+        *out_is_dir = is_dir;
+    }
+    return true;
+}
+
+bool wreaddir(DIR *dir, std::wstring &out_name)
+{
+    struct dirent *d = readdir(dir);
+    if (!d) return false;
+
+    out_name = str2wcstring(d->d_name);
+    return true;
+}
+
+
+wchar_t *wgetcwd(wchar_t *buff, size_t sz)
+{
+    char *buffc = (char *)malloc(sz*MAX_UTF8_BYTES);
+    char *res;
+    wchar_t *ret = 0;
+
+    if (!buffc)
+    {
+        errno = ENOMEM;
+        return 0;
+    }
+
+    res = getcwd(buffc, sz*MAX_UTF8_BYTES);
+    if (res)
+    {
+        if ((size_t)-1 != mbstowcs(buff, buffc, sizeof(wchar_t) * sz))
+        {
+            ret = buff;
+        }
+    }
+
+    free(buffc);
+
+    return ret;
+}
+
+int wchdir(const wcstring &dir)
+{
+    cstring tmp = wcs2string(dir);
+    return chdir(tmp.c_str());
+}
+
+FILE *wfopen(const wcstring &path, const char *mode)
+{
+    int permissions = 0, options = 0;
+    size_t idx = 0;
+    switch (mode[idx++])
+    {
+        case 'r':
+            permissions = O_RDONLY;
+            break;
+        case 'w':
+            permissions = O_WRONLY;
+            options = O_CREAT | O_TRUNC;
+            break;
+        case 'a':
+            permissions = O_WRONLY;
+            options = O_CREAT | O_APPEND;
+            break;
+        default:
+            errno = EINVAL;
+            return NULL;
+            break;
+    }
+    /* Skip binary */
+    if (mode[idx] == 'b')
+        idx++;
+
+    /* Consider append option */
+    if (mode[idx] == '+')
+        permissions = O_RDWR;
+
+    int fd = wopen_cloexec(path, permissions | options, 0666);
+    if (fd < 0)
+        return NULL;
+    FILE *result = fdopen(fd, mode);
+    if (result == NULL)
+        close(fd);
+    return result;
+}
+
+bool set_cloexec(int fd)
+{
+    int flags = fcntl(fd, F_GETFD, 0);
+    if (flags < 0)
+    {
+        return false;
+    }
+    else if (flags & FD_CLOEXEC)
+    {
+        return true;
+    }
+    else
+    {
+        return fcntl(fd, F_SETFD, flags | FD_CLOEXEC) >= 0;
+    }
+}
+
+static int wopen_internal(const wcstring &pathname, int flags, mode_t mode, bool cloexec)
+{
+    ASSERT_IS_NOT_FORKED_CHILD();
+    cstring tmp = wcs2string(pathname);
+    /* Prefer to use O_CLOEXEC. It has to both be defined and nonzero. */
+#ifdef O_CLOEXEC
+    if (cloexec && (O_CLOEXEC != 0))
+    {
+        flags |= O_CLOEXEC;
+        cloexec = false;
+    }
+#endif
+    int fd = ::open(tmp.c_str(), flags, mode);
+    if (cloexec && fd >= 0 && ! set_cloexec(fd))
+    {
+        close(fd);
+        fd = -1;
+    }
+    return fd;
+
+}
+
+int wopen_cloexec(const wcstring &pathname, int flags, mode_t mode)
+{
+    return wopen_internal(pathname, flags, mode, true);
+}
+
+DIR *wopendir(const wcstring &name)
+{
+    const cstring tmp = wcs2string(name);
+    return opendir(tmp.c_str());
+}
+
+int wstat(const wcstring &file_name, struct stat *buf)
+{
+    const cstring tmp = wcs2string(file_name);
+    return stat(tmp.c_str(), buf);
+}
+
+int lwstat(const wcstring &file_name, struct stat *buf)
+{
+    const cstring tmp = wcs2string(file_name);
+    return lstat(tmp.c_str(), buf);
+}
+
+int waccess(const wcstring &file_name, int mode)
+{
+    const cstring tmp = wcs2string(file_name);
+    return access(tmp.c_str(), mode);
+}
+
+int wunlink(const wcstring &file_name)
+{
+    const cstring tmp = wcs2string(file_name);
+    return unlink(tmp.c_str());
+}
+
+void wperror(const wchar_t *s)
+{
+    int e = errno;
+    if (s[0] != L'\0')
+    {
+        fwprintf(stderr, L"%ls: ", s);
+    }
+    fwprintf(stderr, L"%s\n", strerror(e));
+}
+
+int make_fd_nonblocking(int fd)
+{
+    int flags = fcntl(fd, F_GETFL, 0);
+    int err = 0;
+    if (!(flags & O_NONBLOCK))
+    {
+        err = fcntl(fd, F_SETFL, flags | O_NONBLOCK);
+    }
+    return err == -1 ? errno : 0;
+}
+
+int make_fd_blocking(int fd)
+{
+    int flags = fcntl(fd, F_GETFL, 0);
+    int err = 0;
+    if (flags & O_NONBLOCK)
+    {
+        err = fcntl(fd, F_SETFL, flags & ~O_NONBLOCK);
+    }
+    return err == -1 ? errno : 0;
+}
+
+static inline void safe_append(char *buffer, const char *s, size_t buffsize)
+{
+    strncat(buffer, s, buffsize - strlen(buffer) - 1);
+}
+
+// In general, strerror is not async-safe, and therefore we cannot use it directly
+// So instead we have to grub through sys_nerr and sys_errlist directly
+// On GNU toolchain, this will produce a deprecation warning from the linker (!!),
+// which appears impossible to suppress!
+const char *safe_strerror(int err)
+{
+#if defined(__UCLIBC__)
+    // uClibc does not have sys_errlist, however, its strerror is believed to be async-safe
+    // See #808
+    return strerror(err);
+#elif defined(HAVE__SYS__ERRS) || defined(HAVE_SYS_ERRLIST)
+#ifdef HAVE_SYS_ERRLIST
+    if (err >= 0 && err < sys_nerr && sys_errlist[err] != NULL)
+    {
+        return sys_errlist[err];
+    }
+#elif defined(HAVE__SYS__ERRS)
+    extern const char _sys_errs[];
+    extern const int _sys_index[];
+    extern int _sys_num_err;
+
+    if (err >= 0 && err < _sys_num_err) {
+		return &_sys_errs[_sys_index[err]];
+    }
+#endif // either HAVE__SYS__ERRS or HAVE_SYS_ERRLIST
+    else
+#endif // defined(HAVE__SYS__ERRS) || defined(HAVE_SYS_ERRLIST)
+    {
+        int saved_err = errno;
+
+        /* Use a shared buffer for this case */
+        static char buff[384];
+        char errnum_buff[64];
+        format_long_safe(errnum_buff, err);
+
+        buff[0] = '\0';
+        safe_append(buff, "unknown error (errno was ", sizeof buff);
+        safe_append(buff, errnum_buff, sizeof buff);
+        safe_append(buff, ")", sizeof buff);
+
+        errno = saved_err;
+        return buff;
+    }
+}
+
+void safe_perror(const char *message)
+{
+    // Note we cannot use strerror, because on Linux it uses gettext, which is not safe
+    int err = errno;
+
+    char buff[384];
+    buff[0] = '\0';
+
+    if (message)
+    {
+        safe_append(buff, message, sizeof buff);
+        safe_append(buff, ": ", sizeof buff);
+    }
+    safe_append(buff, safe_strerror(err), sizeof buff);
+    safe_append(buff, "\n", sizeof buff);
+
+    write_ignore(STDERR_FILENO, buff, strlen(buff));
+    errno = err;
+}
+
+#ifdef HAVE_REALPATH_NULL
+
+wchar_t *wrealpath(const wcstring &pathname, wchar_t *resolved_path)
+{
+    cstring narrow_path = wcs2string(pathname);
+    char *narrow_res = realpath(narrow_path.c_str(), NULL);
+
+    if (!narrow_res)
+        return NULL;
+
+    wchar_t *res;
+    wcstring wide_res = str2wcstring(narrow_res);
+    if (resolved_path)
+    {
+        wcslcpy(resolved_path, wide_res.c_str(), PATH_MAX);
+        res = resolved_path;
+    }
+    else
+    {
+        res = wcsdup(wide_res.c_str());
+    }
+
+    free(narrow_res);
+
+    return res;
+}
+
+#else
+
+wchar_t *wrealpath(const wcstring &pathname, wchar_t *resolved_path)
+{
+    cstring tmp = wcs2string(pathname);
+    char narrow_buff[PATH_MAX];
+    char *narrow_res = realpath(tmp.c_str(), narrow_buff);
+    wchar_t *res;
+
+    if (!narrow_res)
+        return 0;
+
+    const wcstring wide_res = str2wcstring(narrow_res);
+    if (resolved_path)
+    {
+        wcslcpy(resolved_path, wide_res.c_str(), PATH_MAX);
+        res = resolved_path;
+    }
+    else
+    {
+        res = wcsdup(wide_res.c_str());
+    }
+    return res;
+}
+
+#endif
+
+
+wcstring wdirname(const wcstring &path)
+{
+    char *tmp = wcs2str(path.c_str());
+    char *narrow_res = dirname(tmp);
+    wcstring result = format_string(L"%s", narrow_res);
+    free(tmp);
+    return result;
+}
+
+wcstring wbasename(const wcstring &path)
+{
+    char *tmp = wcs2str(path.c_str());
+    char *narrow_res = basename(tmp);
+    wcstring result = format_string(L"%s", narrow_res);
+    free(tmp);
+    return result;
+}
+
+/* Really init wgettext */
+static void wgettext_really_init()
+{
+    pthread_mutex_init(&wgettext_lock, NULL);
+    fish_bindtextdomain(PACKAGE_NAME, LOCALEDIR);
+    fish_textdomain(PACKAGE_NAME);
+}
+
+/**
+   For wgettext: Internal init function. Automatically called when a translation is first requested.
+*/
+static void wgettext_init_if_necessary()
+{
+    static pthread_once_t once = PTHREAD_ONCE_INIT;
+    pthread_once(&once, wgettext_really_init);
+}
+
+const wchar_t *wgettext(const wchar_t *in)
+{
+    if (!in)
+        return in;
+
+    // preserve errno across this since this is often used in printing error messages
+    int err = errno;
+
+    wgettext_init_if_necessary();
+
+    wcstring key = in;
+    scoped_lock lock(wgettext_lock);
+
+    // Reference to pointer to string
+    const wchar_t *& val = wgettext_map[key];
+    if (val == NULL)
+    {
+        cstring mbs_in = wcs2string(key);
+        char *out = fish_gettext(mbs_in.c_str());
+        val = wcsdup(format_string(L"%s", out).c_str()); //note that this writes into the map!
+    }
+    errno = err;
+    return val; //looks dangerous but is safe, since the string is stored in the map
+}
+
+int wmkdir(const wcstring &name, int mode)
+{
+    cstring name_narrow = wcs2string(name);
+    return mkdir(name_narrow.c_str(), mode);
+}
+
+int wrename(const wcstring &old, const wcstring &newv)
+{
+    cstring old_narrow = wcs2string(old);
+    cstring new_narrow =wcs2string(newv);
+    return rename(old_narrow.c_str(), new_narrow.c_str());
+}
+
+int fish_wcstoi(const wchar_t *str, wchar_t ** endptr, int base)
+{
+    long ret = wcstol(str, endptr, base);
+    if (ret > INT_MAX)
+    {
+        ret = INT_MAX;
+        errno = ERANGE;
+    }
+    else if (ret < INT_MIN)
+    {
+        ret = INT_MIN;
+        errno = ERANGE;
+    }
+    return (int)ret;
+}
+
+file_id_t file_id_t::file_id_from_stat(const struct stat *buf)
+{
+    assert(buf != NULL);
+    
+    file_id_t result = {};
+    result.device = buf->st_dev;
+    result.inode = buf->st_ino;
+    result.size = buf->st_size;
+    result.change_seconds = buf->st_ctime;
+    
+#if STAT_HAVE_NSEC
+    result.change_nanoseconds = buf->st_ctime_nsec;
+#elif defined(__APPLE__)
+    result.change_nanoseconds = buf->st_ctimespec.tv_nsec;
+#elif defined(_BSD_SOURCE) || defined(_SVID_SOURCE) || defined(_XOPEN_SOURCE)
+    result.change_nanoseconds = buf->st_ctim.tv_nsec;
+#else
+    result.change_nanoseconds = 0;
+#endif
+
+#if defined(__APPLE__) || defined(__DragonFly__) || defined(__FreeBSD__) ||  defined(__OpenBSD__) || defined(__NetBSD__)
+    result.generation = buf->st_gen;
+#else
+    result.generation = 0;
+#endif
+    return result;
+}
+
+
+file_id_t file_id_for_fd(int fd)
+{
+    file_id_t result = kInvalidFileID;
+    struct stat buf = {};
+    if (0 == fstat(fd, &buf))
+    {
+        result = file_id_t::file_id_from_stat(&buf);
+    }
+    return result;
+}
+
+file_id_t file_id_for_path(const wcstring &path)
+{
+    file_id_t result = kInvalidFileID;
+    struct stat buf = {};
+    if (0 == wstat(path, &buf))
+    {
+        result = file_id_t::file_id_from_stat(&buf);
+    }
+    return result;
+
+}
+
+bool file_id_t::operator==(const file_id_t &rhs) const
+{
+    return device == rhs.device &&
+           inode == rhs.inode &&
+           size == rhs.size &&
+           change_seconds == rhs.change_seconds &&
+           change_nanoseconds == rhs.change_nanoseconds &&
+           generation == rhs.generation;
+}
+
+bool file_id_t::operator!=(const file_id_t &rhs) const
+{
+    return ! (*this == rhs);
+}
+
+template<typename T>
+int compare(T a, T b)
+{
+    if (a < b)
+    {
+        return -1;
+    }
+    else if (a > b)
+    {
+        return 1;
+    }
+    return 0;
+}
+
+bool file_id_t::operator<(const file_id_t &rhs) const
+{
+    /* Compare each field, stopping when we get to a non-equal field */
+    int ret = 0;
+    if (! ret) ret = compare(device, rhs.device);
+    if (! ret) ret = compare(inode, rhs.inode);
+    if (! ret) ret = compare(size, rhs.size);
+    if (! ret) ret = compare(generation, rhs.generation);
+    if (! ret) ret = compare(change_seconds, rhs.change_seconds);
+    if (! ret) ret = compare(change_nanoseconds, rhs.change_nanoseconds);
+    return ret < 0;
+}
diff --git a/src/wutil.h b/src/wutil.h
new file mode 100644
index 00000000..011f3477
--- /dev/null
+++ b/src/wutil.h
@@ -0,0 +1,166 @@
+/** \file wutil.h
+
+  Prototypes for wide character equivalents of various standard unix
+  functions.
+*/
+#ifndef FISH_WUTIL_H
+#define FISH_WUTIL_H
+
+#include <stdio.h>
+#include <dirent.h>
+#include <sys/types.h>
+#include <stddef.h>
+#include <time.h>
+#include <string>
+#include <stdint.h>
+#include "common.h"
+
+/**
+   Call this function on startup to create internal wutil
+   resources. This function doesn't do anything.
+*/
+void wutil_init();
+
+/**
+   Call this function on exit to free internal wutil resources
+*/
+void wutil_destroy();
+
+/**
+   Wide character version of fopen(). This sets CLO_EXEC.
+*/
+FILE *wfopen(const wcstring &path, const char *mode);
+
+/** Sets CLO_EXEC on a given fd */
+bool set_cloexec(int fd);
+
+/** Wide character version of open() that also sets the close-on-exec flag (atomically when possible). */
+int wopen_cloexec(const wcstring &pathname, int flags, mode_t mode = 0);
+
+/** Mark an fd as nonblocking; returns errno or 0 on success */
+int make_fd_nonblocking(int fd);
+
+/** Mark an fd as blocking; returns errno or 0 on success */
+int make_fd_blocking(int fd);
+
+/** Wide character version of opendir(). Note that opendir() is guaranteed to set close-on-exec by POSIX (hooray). */
+DIR *wopendir(const wcstring &name);
+
+/**
+   Wide character version of stat().
+*/
+int wstat(const wcstring &file_name, struct stat *buf);
+
+/**
+   Wide character version of lstat().
+*/
+int lwstat(const wcstring &file_name, struct stat *buf);
+
+/**
+   Wide character version of access().
+*/
+int waccess(const wcstring &pathname, int mode);
+
+/**
+   Wide character version of unlink().
+*/
+int wunlink(const wcstring &pathname);
+
+/**
+   Wide character version of perror().
+*/
+void wperror(const wchar_t *s);
+
+/**
+  Async-safe version of perror().
+*/
+void safe_perror(const char *message);
+
+/**
+  Async-safe version of strerror().
+*/
+const char *safe_strerror(int err);
+
+/**
+   Wide character version of getcwd().
+*/
+wchar_t *wgetcwd(wchar_t *buff, size_t sz);
+
+/**
+   Wide character version of chdir()
+*/
+int wchdir(const wcstring &dir);
+
+/**
+  Wide character version of realpath function. Just like the GNU
+  version of realpath, wrealpath will accept 0 as the value for the
+  second argument, in which case the result will be allocated using
+  malloc, and must be free'd by the user.
+*/
+wchar_t *wrealpath(const wcstring &pathname, wchar_t *resolved_path);
+
+/**
+   Wide character version of readdir()
+*/
+bool wreaddir(DIR *dir, std::wstring &out_name);
+bool wreaddir_resolving(DIR *dir, const std::wstring &dir_path, std::wstring &out_name, bool *out_is_dir);
+
+/**
+   Wide character version of dirname()
+*/
+std::wstring wdirname(const std::wstring &path);
+
+/**
+   Wide character version of basename()
+*/
+std::wstring wbasename(const std::wstring &path);
+
+/**
+   Wide character wrapper around the gettext function. For historic
+   reasons, unlike the real gettext function, wgettext takes care of
+   setting the correct domain, etc. using the textdomain and
+   bindtextdomain functions. This should probably be moved out of
+   wgettext, so that wgettext will be nothing more than a wrapper
+   around gettext, like all other functions in this file.
+*/
+const wchar_t *wgettext(const wchar_t *in);
+
+/**
+   Wide character version of mkdir
+*/
+int wmkdir(const wcstring &dir, int mode);
+
+/**
+   Wide character version of rename
+*/
+int wrename(const wcstring &oldName, const wcstring &newName);
+
+/** Like wcstol(), but fails on a value outside the range of an int */
+int fish_wcstoi(const wchar_t *str, wchar_t ** endptr, int base);
+
+/** Class for representing a file's inode. We use this to detect and avoid symlink loops, among other things. While an inode / dev pair is sufficient to distinguish co-existing files, Linux seems to aggressively re-use inodes, so it cannot determine if a file has been deleted (ABA problem). Therefore we include richer information. */
+struct file_id_t
+{
+    dev_t device;
+    ino_t inode;
+    uint64_t size;
+    time_t change_seconds;
+    long change_nanoseconds;
+    uint32_t generation;
+    
+    bool operator==(const file_id_t &rhs) const;
+    bool operator!=(const file_id_t &rhs) const;
+    
+    // Used to permit these as keys in std::map
+    bool operator<(const file_id_t &rhs) const;
+    
+    static file_id_t file_id_from_stat(const struct stat *buf);
+};
+
+file_id_t file_id_for_fd(int fd);
+file_id_t file_id_for_path(const wcstring &path);
+
+extern const file_id_t kInvalidFileID;
+
+
+#endif