aboutsummaryrefslogtreecommitdiffhomepage
diff options
context:
space:
mode:
Diffstat
-rw-r--r--Makefile.in4
-rw-r--r--builtin_set_color.cpp5
-rw-r--r--configure.ac11
-rw-r--r--doc_src/license.hdr16
-rw-r--r--env_universal_common.cpp306
-rw-r--r--fish.xcodeproj/project.pbxproj18
-rw-r--r--fish_tests.cpp307
-rw-r--r--highlight.cpp4
-rw-r--r--highlight.h5
-rw-r--r--osx/config.h2
-rw-r--r--parse_constants.h7
-rw-r--r--parse_execution.cpp5
-rw-r--r--parse_tree.cpp73
-rw-r--r--parse_tree.h33
-rw-r--r--utf8.cpp514
-rw-r--r--utf8.h41
16 files changed, 953 insertions, 398 deletions
diff --git a/Makefile.in b/Makefile.in
index 8d18ce1d..3063438c 100644
--- a/Makefile.in
+++ b/Makefile.in
@@ -91,7 +91,7 @@ FISH_OBJS := function.o builtin.o complete.o env.o exec.o expand.o \
signal.o io.o parse_util.o common.o screen.o path.o autoload.o \
parser_keywords.o iothread.o color.o postfork.o \
builtin_test.o parse_tree.o parse_productions.o parse_execution.cpp \
- pager.cpp
+ pager.cpp utf8.o
FISH_INDENT_OBJS := fish_indent.o print_help.o common.o \
parser_keywords.o wutil.o tokenizer.o
@@ -117,7 +117,7 @@ FISH_TESTS_OBJS := $(FISH_OBJS) fish_tests.o
#
FISHD_OBJS := fishd.o env_universal_common.o wutil.o print_help.o \
- common.o
+ common.o utf8.o
#
diff --git a/builtin_set_color.cpp b/builtin_set_color.cpp
index 6e8f55b0..0fdc1ac0 100644
--- a/builtin_set_color.cpp
+++ b/builtin_set_color.cpp
@@ -235,9 +235,8 @@ static int builtin_set_color(parser_t &parser, wchar_t **argv)
output_set_writer(saved_writer_func);
/* Output the collected string */
- std::string local_output;
- std::swap(builtin_set_color_output, local_output);
- stdout_buffer.append(str2wcstring(local_output));
+ stdout_buffer.append(str2wcstring(builtin_set_color_output));
+ builtin_set_color_output.clear();
return STATUS_BUILTIN_OK;
}
diff --git a/configure.ac b/configure.ac
index 34249566..91f4f88c 100644
--- a/configure.ac
+++ b/configure.ac
@@ -105,8 +105,6 @@ echo "CXXFLAGS: $CXXFLAGS"
#
# This mostly helps OS X users, since fink usually installs out of
# tree and doesn't update CXXFLAGS.
-#
-# It also helps FreeBSD which puts libiconv in /usr/local/lib
for i in /usr/pkg /sw /opt /opt/local /usr/local; do
@@ -151,9 +149,11 @@ AC_CONFIG_HEADERS(config.h)
AH_BOTTOM([#if __GNUC__ >= 3
#define __warn_unused __attribute__ ((warn_unused_result))
#define __sentinel __attribute__ ((sentinel))
+#define __packed __attribute__ ((packed))
#else
#define __warn_unused
#define __sentinel
+#define __packed
#endif])
@@ -407,7 +407,7 @@ AC_DEFINE(
AC_SEARCH_LIBS( connect, socket, , [AC_MSG_ERROR([Cannot find the socket library, needed to build this package.] )] )
AC_SEARCH_LIBS( nanosleep, rt, , [AC_MSG_ERROR([Cannot find the rt library, needed to build this package.] )] )
AC_SEARCH_LIBS( pthread_create, pthread, , [AC_MSG_ERROR([Cannot find the pthread library, needed to build this package.] )] )
-AC_SEARCH_LIBS( setupterm, [ncurses curses], , [AC_MSG_ERROR([Could not find a curses implementation, needed to build fish. If this is Linux, try running 'sudo apt-get install libncurses5-dev' or 'sudo yum install ncurses-devel'])] )
+AC_SEARCH_LIBS( setupterm, [ncurses tinfo curses], , [AC_MSG_ERROR([Could not find a curses implementation, needed to build fish. If this is Linux, try running 'sudo apt-get install libncurses5-dev' or 'sudo yum install ncurses-devel'])] )
AC_SEARCH_LIBS( [nan], [m], [AC_DEFINE( [HAVE_NAN], [1], [Define to 1 if you have the nan function])] )
if test x$local_gettext != xno; then
@@ -421,10 +421,6 @@ LIBS_SHARED=$LIBS
#
LIBS="$LIBS_SHARED"
-# Check for libiconv_open if we can't find iconv_open. Silly OS X does
-# weird macro magic for the sole purpose of amusing me.
-AC_SEARCH_LIBS( iconv_open, iconv, , [AC_SEARCH_LIBS( libiconv_open, iconv, , [AC_MSG_ERROR([Could not find an iconv implementation, needed to build fish])] )] )
-
LIBS_FISH=$LIBS
#
@@ -439,7 +435,6 @@ LIBS_FISH_INDENT=$LIBS
#
LIBS="$LIBS_SHARED"
-AC_SEARCH_LIBS( iconv_open, iconv, , [AC_SEARCH_LIBS( libiconv_open, iconv, , [AC_MSG_ERROR([Could not find an iconv implementation, needed to build fish])] )] )
LIBS_FISHD=$LIBS
#
diff --git a/doc_src/license.hdr b/doc_src/license.hdr
index c07a94ad..7f14383a 100644
--- a/doc_src/license.hdr
+++ b/doc_src/license.hdr
@@ -1402,4 +1402,20 @@ POSSIBILITY OF SUCH DAMAGES.
*/
+<h2>License for UTF8</h2>
+
+<p>Copyright (c) 2007 Alexey Vatchenko <av@bsdua.org>
+
+<p>Permission to use, copy, modify, and/or distribute this software for any
+purpose with or without fee is hereby granted, provided that the above
+copyright notice and this permission notice appear in all copies.
+
+<p>THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
\htmlonly </div> \endhtmlonly
diff --git a/env_universal_common.cpp b/env_universal_common.cpp
index dbf79c1a..048396af 100644
--- a/env_universal_common.cpp
+++ b/env_universal_common.cpp
@@ -21,7 +21,6 @@
#include <sys/stat.h>
#include <dirent.h>
#include <wctype.h>
-#include <iconv.h>
#include <errno.h>
#include <locale.h>
@@ -39,6 +38,7 @@
#include "common.h"
#include "wutil.h"
+#include "utf8.h"
#include "env_universal_common.h"
/**
@@ -116,304 +116,29 @@ static void (*callback)(fish_message_type_t type,
const wchar_t *key,
const wchar_t *val);
-/**
- List of names for the UTF-8 character set.
- */
-static const char *iconv_utf8_names[]=
-{
- "utf-8", "UTF-8",
- "utf8", "UTF8",
- 0
-}
-;
-
-/**
- List of wide character names, undefined byte length.
- */
-static const char *iconv_wide_names_unknown[]=
-{
- "wchar_t", "WCHAR_T",
- "wchar", "WCHAR",
- 0
-}
-;
-
-/**
- List of wide character names, 4 bytes long.
- */
-static const char *iconv_wide_names_4[]=
-{
- "wchar_t", "WCHAR_T",
- "wchar", "WCHAR",
- "ucs-4", "UCS-4",
- "ucs4", "UCS4",
- "utf-32", "UTF-32",
- "utf32", "UTF32",
- 0
-}
-;
-
-/**
- List of wide character names, 2 bytes long.
- */
-static const char *iconv_wide_names_2[]=
-{
- "wchar_t", "WCHAR_T",
- "wchar", "WCHAR",
- "ucs-2", "UCS-2",
- "ucs2", "UCS2",
- "utf-16", "UTF-16",
- "utf16", "UTF16",
- 0
-}
-;
-
-template<class T>
-class sloppy {};
-
-static size_t hack_iconv(iconv_t cd, const char * const* inbuf, size_t *inbytesleft, char **outbuf, size_t *outbytesleft)
+/* UTF <-> wchar conversions. These return a string allocated with malloc. These call sites could be cleaned up substantially to eliminate the dependence on malloc. */
+static wchar_t *utf2wcs(const char *input)
{
- /* FreeBSD has this prototype: size_t iconv (iconv_t, const char **...)
- OS X and Linux this one: size_t iconv (iconv_t, char **...)
- AFAIK there's no single type that can be passed as both char ** and const char **.
- Therefore, we let C++ figure it out, by providing a struct with an implicit conversion to both char** and const char **.
- */
- struct sloppy_char
- {
- const char * const * t;
- operator char** () const
- {
- return (char **)t;
- }
- operator const char** () const
- {
- return (const char**)t;
- }
- } slop_inbuf = {inbuf};
-
- return iconv(cd, slop_inbuf, inbytesleft, outbuf, outbytesleft);
-}
-
-/**
- Convert utf-8 string to wide string
- */
-static wchar_t *utf2wcs(const char *in)
-{
- iconv_t cd=(iconv_t) -1;
- int i,j;
-
- wchar_t *out;
-
- /*
- Try to convert to wchar_t. If that is not a valid character set,
- try various names for ucs-4. We can't be sure that ucs-4 is
- really the character set used by wchar_t, but it is the best
- assumption we can make.
- */
- const char **to_name=0;
-
- switch (sizeof(wchar_t))
- {
-
- case 2:
- to_name = iconv_wide_names_2;
- break;
-
- case 4:
- to_name = iconv_wide_names_4;
- break;
-
- default:
- to_name = iconv_wide_names_unknown;
- break;
- }
-
-
- /*
- The line protocol fish uses is always utf-8.
- */
- const char **from_name = iconv_utf8_names;
-
- size_t in_len = strlen(in);
- size_t out_len = sizeof(wchar_t)*(in_len+2);
- size_t nconv;
- char *nout;
-
- out = (wchar_t *)malloc(out_len);
- nout = (char *)out;
-
- if (!out)
- return 0;
-
- for (i=0; to_name[i]; i++)
- {
- for (j=0; from_name[j]; j++)
- {
- cd = iconv_open(to_name[i], from_name[j]);
-
- if (cd != (iconv_t) -1)
- {
- goto start_conversion;
-
- }
- }
- }
-
-start_conversion:
-
- if (cd == (iconv_t) -1)
- {
- /* Something went wrong. */
- debug(0, L"Could not perform utf-8 conversion");
- if (errno != EINVAL)
- wperror(L"iconv_open");
-
- /* Terminate the output string. */
- free(out);
- return 0;
- }
-
- /* FreeBSD has this prototype: size_t iconv (iconv_t, const char **...)
- OS X and Linux this one: size_t iconv (iconv_t, char **...)
- AFAIK there's no single type that can be passed as both char ** and const char **.
- Hence this hack.
- */
- nconv = hack_iconv(cd, &in, &in_len, &nout, &out_len);
-
- if (nconv == (size_t) -1)
- {
- debug(0, L"Error while converting from utf string");
- return 0;
- }
-
- *((wchar_t *) nout) = L'\0';
-
- /*
- Check for silly iconv behaviour inserting an bytemark in the output
- string.
- */
- if (*out == L'\xfeff' || *out == L'\xffef' || *out == L'\xefbbbf')
+ wchar_t *result = NULL;
+ wcstring converted;
+ if (utf8_to_wchar_string(input, &converted))
{
- wchar_t *out_old = out;
- out = wcsdup(out+1);
- if (! out)
- {
- debug(0, L"FNORD!!!!");
- free(out_old);
- return 0;
- }
- free(out_old);
+ result = wcsdup(converted.c_str());
}
-
-
- if (iconv_close(cd) != 0)
- wperror(L"iconv_close");
-
- return out;
+ return result;
}
-
-
-/**
- Convert wide string to utf-8
- */
-static char *wcs2utf(const wchar_t *in)
+static char *wcs2utf(const wchar_t *input)
{
- iconv_t cd=(iconv_t) -1;
- int i,j;
-
- char *char_in = (char *)in;
- char *out;
-
- /*
- Try to convert to wchar_t. If that is not a valid character set,
- try various names for ucs-4. We can't be sure that ucs-4 is
- really the character set used by wchar_t, but it is the best
- assumption we can make.
- */
- const char **from_name=0;
-
- switch (sizeof(wchar_t))
+ char *result = NULL;
+ std::string converted;
+ if (wchar_to_utf8_string(input, &converted))
{
-
- case 2:
- from_name = iconv_wide_names_2;
- break;
-
- case 4:
- from_name = iconv_wide_names_4;
- break;
-
- default:
- from_name = iconv_wide_names_unknown;
- break;
- }
-
- const char **to_name = iconv_utf8_names;
-
- size_t in_len = wcslen(in);
- size_t out_len = sizeof(char)*((MAX_UTF8_BYTES*in_len)+1);
- size_t nconv;
- char *nout;
-
- out = (char *)malloc(out_len);
- nout = (char *)out;
- in_len *= sizeof(wchar_t);
-
- if (!out)
- return 0;
-
- for (i=0; to_name[i]; i++)
- {
- for (j=0; from_name[j]; j++)
- {
- cd = iconv_open(to_name[i], from_name[j]);
-
- if (cd != (iconv_t) -1)
- {
- goto start_conversion;
-
- }
- }
+ result = strdup(converted.c_str());
}
-
-start_conversion:
-
- if (cd == (iconv_t) -1)
- {
- /* Something went wrong. */
- debug(0, L"Could not perform utf-8 conversion");
- if (errno != EINVAL)
- wperror(L"iconv_open");
-
- /* Terminate the output string. */
- free(out);
- return 0;
- }
-
- nconv = hack_iconv(cd, &char_in, &in_len, &nout, &out_len);
-
-
- if (nconv == (size_t) -1)
- {
- debug(0, L"%d %d", in_len, out_len);
- debug(0, L"Error while converting from to string");
-
- /* Terminate the output string. */
- free(out);
- return 0;
- }
-
- *nout = '\0';
-
- if (iconv_close(cd) != 0)
- wperror(L"iconv_close");
-
- return out;
+ return result;
}
-
-
void env_universal_common_init(void (*cb)(fish_message_type_t type, const wchar_t *key, const wchar_t *val))
{
callback = cb;
@@ -756,8 +481,9 @@ static wcstring full_escape(const wchar_t *in)
{
out.push_back(c);
}
- else if (c < 256)
+ else if (c <= ASCII_MAX)
{
+ // See #1225 for discussion of use of ASCII_MAX here
append_format(out, L"\\x%.2x", c);
}
else if (c < 65536)
diff --git a/fish.xcodeproj/project.pbxproj b/fish.xcodeproj/project.pbxproj
index d51f53e3..c7a6abae 100644
--- a/fish.xcodeproj/project.pbxproj
+++ b/fish.xcodeproj/project.pbxproj
@@ -113,10 +113,11 @@
D08A32B917B446B100F3A533 /* parse_productions.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0FE8EE7179FB75F008C9F21 /* parse_productions.cpp */; };
D08A32BA17B446B100F3A533 /* parse_tree.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0C52F351765284C00BFAB82 /* parse_tree.cpp */; };
D08A32BC17B4473B00F3A533 /* libncurses.dylib in Frameworks */ = {isa = PBXBuildFile; fileRef = D0D02A8C15983CFA008E62BD /* libncurses.dylib */; };
- D08A32BD17B4474000F3A533 /* libiconv.dylib in Frameworks */ = {isa = PBXBuildFile; fileRef = D0D02A8A15983CDF008E62BD /* libiconv.dylib */; };
D0A564FE168D23D800AF6161 /* man in CopyFiles */ = {isa = PBXBuildFile; fileRef = D0A564F1168D0BAB00AF6161 /* man */; };
D0A56501168D258300AF6161 /* man in Copy Files */ = {isa = PBXBuildFile; fileRef = D0A564F1168D0BAB00AF6161 /* man */; };
D0C52F371765284C00BFAB82 /* parse_tree.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0C52F351765284C00BFAB82 /* parse_tree.cpp */; };
+ D0C9733818DE5449002D7C81 /* utf8.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0C9733718DE5449002D7C81 /* utf8.cpp */; };
+ D0C9733918DE5449002D7C81 /* utf8.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0C9733718DE5449002D7C81 /* utf8.cpp */; };
D0CBD587159EF0E10024809C /* launch_fish.scpt in Resources */ = {isa = PBXBuildFile; fileRef = D0CBD586159EF0E10024809C /* launch_fish.scpt */; };
D0D02A67159837AD008E62BD /* complete.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0853713B3ACEE0099B651 /* complete.cpp */; };
D0D02A69159837B2008E62BD /* env.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0853A13B3ACEE0099B651 /* env.cpp */; };
@@ -152,7 +153,6 @@
D0D02A87159839D5008E62BD /* screen.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0855A13B3ACEE0099B651 /* screen.cpp */; };
D0D02A88159839D5008E62BD /* signal.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0855C13B3ACEE0099B651 /* signal.cpp */; };
D0D02A89159839DF008E62BD /* fish.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0854213B3ACEE0099B651 /* fish.cpp */; };
- D0D02A8B15983CDF008E62BD /* libiconv.dylib in Frameworks */ = {isa = PBXBuildFile; fileRef = D0D02A8A15983CDF008E62BD /* libiconv.dylib */; };
D0D02A8D15983CFA008E62BD /* libncurses.dylib in Frameworks */ = {isa = PBXBuildFile; fileRef = D0D02A8C15983CFA008E62BD /* libncurses.dylib */; };
D0D02A8F15983D8F008E62BD /* parser_keywords.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0855313B3ACEE0099B651 /* parser_keywords.cpp */; };
D0D02AC215985F3F008E62BD /* fishd.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0854313B3ACEE0099B651 /* fishd.cpp */; };
@@ -161,7 +161,6 @@
D0D02AC515985F5B008E62BD /* print_help.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0855613B3ACEE0099B651 /* print_help.cpp */; };
D0D02AC615985F65008E62BD /* common.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0853613B3ACEE0099B651 /* common.cpp */; };
D0D02AC715985F9D008E62BD /* libncurses.dylib in Frameworks */ = {isa = PBXBuildFile; fileRef = D0D02A8C15983CFA008E62BD /* libncurses.dylib */; };
- D0D02AC815985F9F008E62BD /* libiconv.dylib in Frameworks */ = {isa = PBXBuildFile; fileRef = D0D02A8A15983CDF008E62BD /* libiconv.dylib */; };
D0D02AD615986492008E62BD /* fish_indent.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0853F13B3ACEE0099B651 /* fish_indent.cpp */; };
D0D02AD715986498008E62BD /* print_help.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0855613B3ACEE0099B651 /* print_help.cpp */; };
D0D02AD81598649E008E62BD /* common.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0853613B3ACEE0099B651 /* common.cpp */; };
@@ -169,7 +168,6 @@
D0D02ADA159864AB008E62BD /* wutil.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0856113B3ACEE0099B651 /* wutil.cpp */; };
D0D02ADB159864C2008E62BD /* tokenizer.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0855D13B3ACEE0099B651 /* tokenizer.cpp */; };
D0D02ADC159864D5008E62BD /* libncurses.dylib in Frameworks */ = {isa = PBXBuildFile; fileRef = D0D02A8C15983CFA008E62BD /* libncurses.dylib */; };
- D0D02ADD159864D7008E62BD /* libiconv.dylib in Frameworks */ = {isa = PBXBuildFile; fileRef = D0D02A8A15983CDF008E62BD /* libiconv.dylib */; };
D0D2694915983772005D9B9C /* function.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0854413B3ACEE0099B651 /* function.cpp */; };
D0D2694A15983779005D9B9C /* builtin.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0853513B3ACEE0099B651 /* builtin.cpp */; };
D0F019F115A977140034B3B1 /* fish in CopyFiles */ = {isa = PBXBuildFile; fileRef = D0D2693C159835CA005D9B9C /* fish */; };
@@ -475,11 +473,12 @@
D0C6FCC914CFA4B0004CE8AD /* autoload.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = autoload.cpp; sourceTree = "<group>"; };
D0C6FCCB14CFA4B7004CE8AD /* autoload.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = autoload.h; sourceTree = "<group>"; };
D0C861EA16CC7054003B5A04 /* builtin_set_color.cpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; path = builtin_set_color.cpp; sourceTree = "<group>"; };
+ D0C9733718DE5449002D7C81 /* utf8.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = utf8.cpp; sourceTree = "<group>"; };
+ D0C9733A18DE5451002D7C81 /* utf8.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = utf8.h; sourceTree = "<group>"; };
D0CA63F316FC275F00093BD4 /* builtin_printf.cpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; path = builtin_printf.cpp; sourceTree = "<group>"; };
D0CBD580159EE48F0024809C /* config.fish */ = {isa = PBXFileReference; lastKnownFileType = text; name = config.fish; path = share/config.fish; sourceTree = "<group>"; };
D0CBD583159EEE010024809C /* Foundation.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = Foundation.framework; path = System/Library/Frameworks/Foundation.framework; sourceTree = SDKROOT; };
D0CBD586159EF0E10024809C /* launch_fish.scpt */ = {isa = PBXFileReference; lastKnownFileType = file; name = launch_fish.scpt; path = osx/launch_fish.scpt; sourceTree = "<group>"; };
- D0D02A8A15983CDF008E62BD /* libiconv.dylib */ = {isa = PBXFileReference; lastKnownFileType = "compiled.mach-o.dylib"; name = libiconv.dylib; path = usr/lib/libiconv.dylib; sourceTree = SDKROOT; };
D0D02A8C15983CFA008E62BD /* libncurses.dylib */ = {isa = PBXFileReference; lastKnownFileType = "compiled.mach-o.dylib"; name = libncurses.dylib; path = usr/lib/libncurses.dylib; sourceTree = SDKROOT; };
D0D02A9A15985A75008E62BD /* fish.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = fish.app; sourceTree = BUILT_PRODUCTS_DIR; };
D0D02AA915985C0C008E62BD /* Info.plist */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.plist.xml; name = Info.plist; path = osx/Info.plist; sourceTree = "<group>"; };
@@ -499,7 +498,6 @@
isa = PBXFrameworksBuildPhase;
buildActionMask = 2147483647;
files = (
- D08A32BD17B4474000F3A533 /* libiconv.dylib in Frameworks */,
D08A32BC17B4473B00F3A533 /* libncurses.dylib in Frameworks */,
);
runOnlyForDeploymentPostprocessing = 0;
@@ -509,7 +507,6 @@
buildActionMask = 2147483647;
files = (
D0D02AC715985F9D008E62BD /* libncurses.dylib in Frameworks */,
- D0D02AC815985F9F008E62BD /* libiconv.dylib in Frameworks */,
);
runOnlyForDeploymentPostprocessing = 0;
};
@@ -518,7 +515,6 @@
buildActionMask = 2147483647;
files = (
D0D02ADC159864D5008E62BD /* libncurses.dylib in Frameworks */,
- D0D02ADD159864D7008E62BD /* libiconv.dylib in Frameworks */,
);
runOnlyForDeploymentPostprocessing = 0;
};
@@ -527,7 +523,6 @@
buildActionMask = 2147483647;
files = (
D0D02A8D15983CFA008E62BD /* libncurses.dylib in Frameworks */,
- D0D02A8B15983CDF008E62BD /* libiconv.dylib in Frameworks */,
);
runOnlyForDeploymentPostprocessing = 0;
};
@@ -583,7 +578,6 @@
isa = PBXGroup;
children = (
D0D02A8C15983CFA008E62BD /* libncurses.dylib */,
- D0D02A8A15983CDF008E62BD /* libiconv.dylib */,
D0CBD583159EEE010024809C /* Foundation.framework */,
);
name = Libraries;
@@ -686,6 +680,8 @@
D0A0855C13B3ACEE0099B651 /* signal.cpp */,
D0A0852513B3ACEE0099B651 /* tokenizer.h */,
D0A0855D13B3ACEE0099B651 /* tokenizer.cpp */,
+ D0C9733A18DE5451002D7C81 /* utf8.h */,
+ D0C9733718DE5449002D7C81 /* utf8.cpp */,
D0A0852613B3ACEE0099B651 /* util.h */,
D0A0855E13B3ACEE0099B651 /* util.cpp */,
D0A0852713B3ACEE0099B651 /* wgetopt.h */,
@@ -1120,6 +1116,7 @@
files = (
D0D02AC215985F3F008E62BD /* fishd.cpp in Sources */,
D0D02AC315985F43008E62BD /* env_universal_common.cpp in Sources */,
+ D0C9733918DE5449002D7C81 /* utf8.cpp in Sources */,
D0D02AC415985F4D008E62BD /* wutil.cpp in Sources */,
D0D02AC515985F5B008E62BD /* print_help.cpp in Sources */,
D0D02AC615985F65008E62BD /* common.cpp in Sources */,
@@ -1157,6 +1154,7 @@
D0D02A86159839D5008E62BD /* postfork.cpp in Sources */,
D0D02A87159839D5008E62BD /* screen.cpp in Sources */,
D0D02A88159839D5008E62BD /* signal.cpp in Sources */,
+ D0C9733818DE5449002D7C81 /* utf8.cpp in Sources */,
D0D2694A15983779005D9B9C /* builtin.cpp in Sources */,
D0D2694915983772005D9B9C /* function.cpp in Sources */,
D0D02A67159837AD008E62BD /* complete.cpp in Sources */,
diff --git a/fish_tests.cpp b/fish_tests.cpp
index 605b21ed..3cca2a0c 100644
--- a/fish_tests.cpp
+++ b/fish_tests.cpp
@@ -62,6 +62,7 @@
#include "parse_util.h"
#include "pager.h"
#include "input.h"
+#include "utf8.h"
static const char * const * s_arguments;
static int s_test_run_count = 0;
@@ -140,17 +141,17 @@ static void err(const wchar_t *blah, ...)
va_list va;
va_start(va, blah);
err_count++;
-
+
// show errors in red
fputs("\x1b[31m", stdout);
wprintf(L"Error: ");
vwprintf(blah, va);
va_end(va);
-
+
// return to normal color
fputs("\x1b[0m", stdout);
-
+
wprintf(L"\n");
}
@@ -925,6 +926,260 @@ static void test_utils()
if (begin != a + wcslen(L"echo (echo (")) err(L"parse_util_cmdsubst_extent failed on line %ld", (long)__LINE__);
}
+/* UTF8 tests taken from Alexey Vatchenko's utf8 library. See http://www.bsdua.org/libbsdua.html */
+
+static void test_utf82wchar(const char *src, size_t slen, const wchar_t *dst, size_t dlen,
+ int flags, size_t res, const char *descr)
+{
+ size_t size;
+ wchar_t *mem = NULL;
+
+ /* Hack: if wchar is only UCS-2, and the UTF-8 input string contains astral characters, then tweak the expected size to 0 */
+ if (src != NULL && is_wchar_ucs2())
+ {
+ /* A UTF-8 code unit may represent an astral code point if it has 4 or more leading 1s */
+ const unsigned char astral_mask = 0xF0;
+ for (size_t i=0; i < slen; i++)
+ {
+ if ((src[i] & astral_mask) == astral_mask)
+ {
+ /* Astral char. We expect this conversion to just fail. */
+ res = 0;
+ break;
+ }
+ }
+ }
+
+ if (dst != NULL)
+ {
+ mem = (wchar_t *)malloc(dlen * sizeof(*mem));
+ if (mem == NULL)
+ {
+ err(L"u2w: %s: MALLOC FAILED\n", descr);
+ return;
+ }
+ }
+
+ do
+ {
+ size = utf8_to_wchar(src, slen, mem, dlen, flags);
+ if (res != size)
+ {
+ err(L"u2w: %s: FAILED (rv: %lu, must be %lu)", descr, size, res);
+ break;
+ }
+
+ if (mem == NULL)
+ break; /* OK */
+
+ if (memcmp(mem, dst, size * sizeof(*mem)) != 0)
+ {
+ err(L"u2w: %s: BROKEN", descr);
+ break;
+ }
+
+ }
+ while (0);
+
+ free(mem);
+}
+
+static void test_wchar2utf8(const wchar_t *src, size_t slen, const char *dst, size_t dlen,
+ int flags, size_t res, const char *descr)
+{
+ size_t size;
+ char *mem = NULL;
+
+ /* Hack: if wchar is simulating UCS-2, and the wchar_t input string contains astral characters, then tweak the expected size to 0 */
+ if (src != NULL && is_wchar_ucs2())
+ {
+ const uint32_t astral_mask = 0xFFFF0000U;
+ for (size_t i=0; i < slen; i++)
+ {
+ if ((src[i] & astral_mask) != 0)
+ {
+ /* astral char */
+ res = 0;
+ break;
+ }
+ }
+ }
+
+ if (dst != NULL)
+ {
+ mem = (char *)malloc(dlen);
+ if (mem == NULL)
+ {
+ err(L"w2u: %s: MALLOC FAILED", descr);
+ return;
+ }
+ }
+
+ do
+ {
+ size = wchar_to_utf8(src, slen, mem, dlen, flags);
+ if (res != size)
+ {
+ err(L"w2u: %s: FAILED (rv: %lu, must be %lu)", descr, size, res);
+ break;
+ }
+
+ if (mem == NULL)
+ break; /* OK */
+
+ if (memcmp(mem, dst, size) != 0)
+ {
+ err(L"w2u: %s: BROKEN", descr);
+ break;
+ }
+
+ }
+ while (0);
+
+ if (mem != NULL);
+ free(mem);
+}
+
+static void test_utf8()
+{
+ wchar_t w1[] = {0x54, 0x65, 0x73, 0x74};
+ wchar_t w2[] = {0x0422, 0x0435, 0x0441, 0x0442};
+ wchar_t w3[] = {0x800, 0x1e80, 0x98c4, 0x9910, 0xff00};
+ wchar_t w4[] = {0x15555, 0xf7777, 0xa};
+ wchar_t w5[] = {0x255555, 0x1fa04ff, 0xddfd04, 0xa};
+ wchar_t w6[] = {0xf255555, 0x1dfa04ff, 0x7fddfd04, 0xa};
+ wchar_t wb[] = {-2, 0xa, 0xffffffff, 0x0441};
+ wchar_t wm[] = {0x41, 0x0441, 0x3042, 0xff67, 0x9b0d, 0x2e05da67};
+ wchar_t wb1[] = {0xa, 0x0422};
+ wchar_t wb2[] = {0xd800, 0xda00, 0x41, 0xdfff, 0xa};
+ wchar_t wbom[] = {0xfeff, 0x41, 0xa};
+ wchar_t wbom2[] = {0x41, 0xa};
+ wchar_t wbom22[] = {0xfeff, 0x41, 0xa};
+ char u1[] = {0x54, 0x65, 0x73, 0x74};
+ char u2[] = {0xd0, 0xa2, 0xd0, 0xb5, 0xd1, 0x81, 0xd1, 0x82};
+ char u3[] = {0xe0, 0xa0, 0x80, 0xe1, 0xba, 0x80, 0xe9, 0xa3, 0x84,
+ 0xe9, 0xa4, 0x90, 0xef, 0xbc, 0x80
+ };
+ char u4[] = {0xf0, 0x95, 0x95, 0x95, 0xf3, 0xb7, 0x9d, 0xb7, 0xa};
+ char u5[] = {0xf8, 0x89, 0x95, 0x95, 0x95, 0xf9, 0xbe, 0xa0, 0x93,
+ 0xbf, 0xf8, 0xb7, 0x9f, 0xb4, 0x84, 0x0a
+ };
+ char u6[] = {0xfc, 0x8f, 0x89, 0x95, 0x95, 0x95, 0xfc, 0x9d, 0xbe,
+ 0xa0, 0x93, 0xbf, 0xfd, 0xbf, 0xb7, 0x9f, 0xb4, 0x84, 0x0a
+ };
+ char ub[] = {0xa, 0xd1, 0x81};
+ char um[] = {0x41, 0xd1, 0x81, 0xe3, 0x81, 0x82, 0xef, 0xbd, 0xa7,
+ 0xe9, 0xac, 0x8d, 0xfc, 0xae, 0x81, 0x9d, 0xa9, 0xa7
+ };
+ char ub1[] = {0xa, 0xff, 0xd0, 0xa2, 0xfe, 0x8f, 0xe0, 0x80};
+ char uc080[] = {0xc0, 0x80};
+ char ub2[] = {0xed, 0xa1, 0x8c, 0xed, 0xbe, 0xb4, 0xa};
+ char ubom[] = {0x41, 0xa};
+ char ubom2[] = {0xef, 0xbb, 0xbf, 0x41, 0xa};
+
+ /*
+ * UTF-8 -> UCS-4 string.
+ */
+ test_utf82wchar(ubom2, sizeof(ubom2), wbom2,
+ sizeof(wbom2) / sizeof(*wbom2), UTF8_SKIP_BOM,
+ sizeof(wbom2) / sizeof(*wbom2), "skip BOM");
+ test_utf82wchar(ubom2, sizeof(ubom2), wbom22,
+ sizeof(wbom22) / sizeof(*wbom22), 0,
+ sizeof(wbom22) / sizeof(*wbom22), "BOM");
+ test_utf82wchar(uc080, sizeof(uc080), NULL, 0, 0, 0,
+ "c0 80 - forbitten by rfc3629");
+ test_utf82wchar(ub2, sizeof(ub2), NULL, 0, 0, is_wchar_ucs2() ? 0 : 3,
+ "resulted in forbitten wchars (len)");
+ test_utf82wchar(ub2, sizeof(ub2), wb2, sizeof(wb2) / sizeof(*wb2), 0, 0,
+ "resulted in forbitten wchars");
+ test_utf82wchar(ub2, sizeof(ub2), L"\x0a", 1, UTF8_IGNORE_ERROR,
+ 1, "resulted in ignored forbitten wchars");
+ test_utf82wchar(u1, sizeof(u1), w1, sizeof(w1) / sizeof(*w1), 0,
+ sizeof(w1) / sizeof(*w1), "1 octet chars");
+ test_utf82wchar(u2, sizeof(u2), w2, sizeof(w2) / sizeof(*w2), 0,
+ sizeof(w2) / sizeof(*w2), "2 octets chars");
+ test_utf82wchar(u3, sizeof(u3), w3, sizeof(w3) / sizeof(*w3), 0,
+ sizeof(w3) / sizeof(*w3), "3 octets chars");
+ test_utf82wchar(u4, sizeof(u4), w4, sizeof(w4) / sizeof(*w4), 0,
+ sizeof(w4) / sizeof(*w4), "4 octets chars");
+ test_utf82wchar(u5, sizeof(u5), w5, sizeof(w5) / sizeof(*w5), 0,
+ sizeof(w5) / sizeof(*w5), "5 octets chars");
+ test_utf82wchar(u6, sizeof(u6), w6, sizeof(w6) / sizeof(*w6), 0,
+ sizeof(w6) / sizeof(*w6), "6 octets chars");
+ test_utf82wchar("\xff", 1, NULL, 0, 0, 0, "broken utf-8 0xff symbol");
+ test_utf82wchar("\xfe", 1, NULL, 0, 0, 0, "broken utf-8 0xfe symbol");
+ test_utf82wchar("\x8f", 1, NULL, 0, 0, 0,
+ "broken utf-8, start from 10 higher bits");
+ if (! is_wchar_ucs2()) test_utf82wchar(ub1, sizeof(ub1), wb1, sizeof(wb1) / sizeof(*wb1),
+ UTF8_IGNORE_ERROR, sizeof(wb1) / sizeof(*wb1), "ignore bad chars");
+ test_utf82wchar(um, sizeof(um), wm, sizeof(wm) / sizeof(*wm), 0,
+ sizeof(wm) / sizeof(*wm), "mixed languages");
+ test_utf82wchar(um, sizeof(um), wm, sizeof(wm) / sizeof(*wm) - 1, 0,
+ 0, "boundaries -1");
+ test_utf82wchar(um, sizeof(um), wm, sizeof(wm) / sizeof(*wm) + 1, 0,
+ sizeof(wm) / sizeof(*wm), "boundaries +1");
+ test_utf82wchar(um, sizeof(um), NULL, 0, 0,
+ sizeof(wm) / sizeof(*wm), "calculate length");
+ test_utf82wchar(ub1, sizeof(ub1), NULL, 0, 0,
+ 0, "calculate length of bad chars");
+ test_utf82wchar(ub1, sizeof(ub1), NULL, 0,
+ UTF8_IGNORE_ERROR, sizeof(wb1) / sizeof(*wb1),
+ "calculate length, ignore bad chars");
+ test_utf82wchar(NULL, 0, NULL, 0, 0, 0, "invalid params, all 0");
+ test_utf82wchar(u1, 0, NULL, 0, 0, 0,
+ "invalid params, src buf not NULL");
+ test_utf82wchar(NULL, 10, NULL, 0, 0, 0,
+ "invalid params, src length is not 0");
+ test_utf82wchar(u1, sizeof(u1), w1, 0, 0, 0,
+ "invalid params, dst is not NULL");
+
+ /*
+ * UCS-4 -> UTF-8 string.
+ */
+ test_wchar2utf8(wbom, sizeof(wbom) / sizeof(*wbom), ubom, sizeof(ubom),
+ UTF8_SKIP_BOM, sizeof(ubom), "BOM");
+ test_wchar2utf8(wb2, sizeof(wb2) / sizeof(*wb2), NULL, 0, 0,
+ 0, "prohibited wchars");
+ test_wchar2utf8(wb2, sizeof(wb2) / sizeof(*wb2), NULL, 0,
+ UTF8_IGNORE_ERROR, 2, "ignore prohibited wchars");
+ test_wchar2utf8(w1, sizeof(w1) / sizeof(*w1), u1, sizeof(u1), 0,
+ sizeof(u1), "1 octet chars");
+ test_wchar2utf8(w2, sizeof(w2) / sizeof(*w2), u2, sizeof(u2), 0,
+ sizeof(u2), "2 octets chars");
+ test_wchar2utf8(w3, sizeof(w3) / sizeof(*w3), u3, sizeof(u3), 0,
+ sizeof(u3), "3 octets chars");
+ test_wchar2utf8(w4, sizeof(w4) / sizeof(*w4), u4, sizeof(u4), 0,
+ sizeof(u4), "4 octets chars");
+ test_wchar2utf8(w5, sizeof(w5) / sizeof(*w5), u5, sizeof(u5), 0,
+ sizeof(u5), "5 octets chars");
+ test_wchar2utf8(w6, sizeof(w6) / sizeof(*w6), u6, sizeof(u6), 0,
+ sizeof(u6), "6 octets chars");
+ test_wchar2utf8(wb, sizeof(wb) / sizeof(*wb), ub, sizeof(ub), 0,
+ 0, "bad chars");
+ test_wchar2utf8(wb, sizeof(wb) / sizeof(*wb), ub, sizeof(ub),
+ UTF8_IGNORE_ERROR, sizeof(ub), "ignore bad chars");
+ test_wchar2utf8(wm, sizeof(wm) / sizeof(*wm), um, sizeof(um), 0,
+ sizeof(um), "mixed languages");
+ test_wchar2utf8(wm, sizeof(wm) / sizeof(*wm), um, sizeof(um) - 1, 0,
+ 0, "boundaries -1");
+ test_wchar2utf8(wm, sizeof(wm) / sizeof(*wm), um, sizeof(um) + 1, 0,
+ sizeof(um), "boundaries +1");
+ test_wchar2utf8(wm, sizeof(wm) / sizeof(*wm), NULL, 0, 0,
+ sizeof(um), "calculate length");
+ test_wchar2utf8(wb, sizeof(wb) / sizeof(*wb), NULL, 0, 0,
+ 0, "calculate length of bad chars");
+ test_wchar2utf8(wb, sizeof(wb) / sizeof(*wb), NULL, 0,
+ UTF8_IGNORE_ERROR, sizeof(ub),
+ "calculate length, ignore bad chars");
+ test_wchar2utf8(NULL, 0, NULL, 0, 0, 0, "invalid params, all 0");
+ test_wchar2utf8(w1, 0, NULL, 0, 0, 0,
+ "invalid params, src buf not NULL");
+ test_wchar2utf8(NULL, 10, NULL, 0, 0, 0,
+ "invalid params, src length is not 0");
+ test_wchar2utf8(w1, sizeof(w1) / sizeof(*w1), u1, 0, 0, 0,
+ "invalid params, dst is not NULL");
+}
+
static void test_escape_sequences(void)
{
say(L"Testing escape codes");
@@ -1178,9 +1433,9 @@ static void test_path()
static void test_pager_navigation()
{
say(L"Testing pager navigation");
-
+
/* Generate 19 strings of width 10. There's 2 spaces between completions, and our term size is 80; these can therefore fit into 6 columns (6 * 12 - 2 = 70) or 5 columns (58) but not 7 columns (7 * 12 - 2 = 82).
-
+
You can simulate this test by creating 19 files named "file00.txt" through "file_18.txt".
*/
completion_list_t completions;
@@ -1188,31 +1443,31 @@ static void test_pager_navigation()
{
append_completion(completions, L"abcdefghij");
}
-
+
pager_t pager;
pager.set_completions(completions);
pager.set_term_size(80, 24);
page_rendering_t render = pager.render();
-
+
if (render.term_width != 80)
err(L"Wrong term width");
if (render.term_height != 24)
err(L"Wrong term height");
-
+
size_t rows = 4, cols = 5;
-
+
/* We have 19 completions. We can fit into 6 columns with 4 rows or 5 columns with 4 rows; the second one is better and so is what we ought to have picked. */
if (render.rows != rows)
err(L"Wrong row count");
if (render.cols != cols)
err(L"Wrong column count");
-
+
/* Initially expect to have no completion index */
if (render.selected_completion_idx != (size_t)(-1))
{
err(L"Wrong initial selection");
}
-
+
/* Here are navigation directions and where we expect the selection to be */
const struct
{
@@ -1223,31 +1478,31 @@ static void test_pager_navigation()
{
/* Tab completion to get into the list */
{direction_next, 0},
-
+
/* Westward motion in upper left wraps along the top row */
{direction_west, 16},
{direction_east, 1},
-
+
/* "Next" motion goes down the column */
{direction_next, 2},
{direction_next, 3},
-
+
{direction_west, 18},
{direction_east, 3},
{direction_east, 7},
{direction_east, 11},
{direction_east, 15},
{direction_east, 3},
-
+
{direction_west, 18},
{direction_east, 3},
-
+
/* Eastward motion wraps along the bottom, westward goes to the prior column */
{direction_east, 7},
{direction_east, 11},
{direction_east, 15},
{direction_east, 3},
-
+
/* Column memory */
{direction_west, 18},
{direction_south, 15},
@@ -1265,7 +1520,7 @@ static void test_pager_navigation()
err(L"For command %lu, expected selection %lu, but found instead %lu\n", i, cmds[i].sel, render.selected_completion_idx);
}
}
-
+
}
enum word_motion_t
@@ -1604,14 +1859,14 @@ static void test_complete(void)
completions.clear();
complete(L"echo (builtin scuttlebut", completions, COMPLETION_REQUEST_DEFAULT);
do_test(completions.size() == 0);
-
+
/* Trailing spaces (#1261) */
complete_add(L"foobarbaz", false, 0, NULL, 0, NO_FILES, NULL, L"qux", NULL, COMPLETE_AUTO_SPACE);
completions.clear();
complete(L"foobarbaz ", completions, COMPLETION_REQUEST_DEFAULT);
do_test(completions.size() == 1);
do_test(completions.at(0).completion == L"qux");
-
+
/* Don't complete variable names in single quotes (#1023) */
completions.clear();
complete(L"echo '$Foo", completions, COMPLETION_REQUEST_DEFAULT);
@@ -1882,14 +2137,14 @@ static void test_input()
wcstring desired_binding = prefix_binding + L'a';
input_mapping_add(prefix_binding.c_str(), L"up-line");
input_mapping_add(desired_binding.c_str(), L"down-line");
-
+
/* Push the desired binding on the stack (backwards!) */
size_t idx = desired_binding.size();
while (idx--)
{
input_unreadch(desired_binding.at(idx));
}
-
+
/* Now test */
wint_t c = input_readch();
if (c != R_DOWN_LINE)
@@ -2816,7 +3071,7 @@ static void test_highlighting(void)
{L"'single_quote", highlight_spec_error},
{NULL, -1}
};
-
+
const highlight_component_t components11[] =
{
{L"echo", highlight_spec_command},
@@ -2829,7 +3084,7 @@ static void test_highlighting(void)
{L"]", highlight_spec_operator},
{NULL, -1}
};
-
+
const highlight_component_t components12[] =
{
{L"for", highlight_spec_command},
@@ -2935,6 +3190,7 @@ int main(int argc, char **argv)
if (should_test_function("cancellation")) test_cancellation();
if (should_test_function("indents")) test_indents();
if (should_test_function("utils")) test_utils();
+ if (should_test_function("utf8")) test_utf8();
if (should_test_function("escape_sequences")) test_escape_sequences();
if (should_test_function("lru")) test_lru();
if (should_test_function("expand")) test_expand();
@@ -2974,7 +3230,8 @@ int main(int argc, char **argv)
event_destroy();
proc_destroy();
- if(err_count != 0) {
+ if (err_count != 0)
+ {
return(1);
}
}
diff --git a/highlight.cpp b/highlight.cpp
index eaaec750..d6db4c46 100644
--- a/highlight.cpp
+++ b/highlight.cpp
@@ -44,8 +44,6 @@
*/
#define VAR_COUNT ( sizeof(highlight_var)/sizeof(wchar_t *) )
-static void highlight_universal_internal(const wcstring &buff, std::vector<highlight_spec_t> &color, size_t pos);
-
/** The environment variables used to specify the color of different tokens. This matches the order in highlight_spec_t */
static const wchar_t * const highlight_var[] =
{
@@ -350,7 +348,7 @@ bool plain_statement_get_expanded_command(const wcstring &src, const parse_node_
if (expand_one(cmd, EXPAND_SKIP_CMDSUBST | EXPAND_SKIP_VARIABLES | EXPAND_SKIP_JOBS))
{
/* Success, return the expanded string by reference */
- std::swap(cmd, *out_cmd);
+ out_cmd->swap(cmd);
result = true;
}
}
diff --git a/highlight.h b/highlight.h
index 6ef411f9..d411ae36 100644
--- a/highlight.h
+++ b/highlight.h
@@ -75,7 +75,6 @@ struct file_detection_context_t;
\param error a list in which a description of each error will be inserted. May be 0, in whcich case no error descriptions will be generated.
*/
void highlight_shell(const wcstring &buffstr, std::vector<highlight_spec_t> &color, size_t pos, wcstring_list_t *error, const env_vars_snapshot_t &vars);
-void highlight_shell_new_parser(const wcstring &buffstr, std::vector<highlight_spec_t> &color, size_t pos, wcstring_list_t *error, const env_vars_snapshot_t &vars);
/**
Perform syntax highlighting for the text in buff. Matching quotes and paranthesis are highlighted. The result is
@@ -125,9 +124,5 @@ enum
typedef unsigned int path_flags_t;
bool is_potential_path(const wcstring &const_path, const wcstring_list_t &directories, path_flags_t flags, wcstring *out_path = NULL);
-/* For testing */
-void highlight_shell_classic(const wcstring &buff, std::vector<highlight_spec_t> &color, size_t pos, wcstring_list_t *error, const env_vars_snapshot_t &vars);
-void highlight_shell_new_parser(const wcstring &buff, std::vector<highlight_spec_t> &color, size_t pos, wcstring_list_t *error, const env_vars_snapshot_t &vars);
-
#endif
diff --git a/osx/config.h b/osx/config.h
index 4968a78b..99f83742 100644
--- a/osx/config.h
+++ b/osx/config.h
@@ -218,7 +218,9 @@
#if __GNUC__ >= 3
#define __warn_unused __attribute__ ((warn_unused_result))
#define __sentinel __attribute__ ((sentinel))
+#define __packed __attribute__ ((packed))
#else
#define __warn_unused
#define __sentinel
+#define __packed
#endif
diff --git a/parse_constants.h b/parse_constants.h
index d464e396..31f07f60 100644
--- a/parse_constants.h
+++ b/parse_constants.h
@@ -6,10 +6,11 @@
#ifndef fish_parse_constants_h
#define fish_parse_constants_h
+#include "config.h"
+
#define PARSE_ASSERT(a) assert(a)
#define PARSER_DIE() do { fprintf(stderr, "Parser dying!\n"); exit_without_destructors(-1); } while (0)
-
enum parse_token_type_t
{
token_type_invalid,
@@ -70,7 +71,7 @@ enum parse_token_type_t
LAST_TOKEN_OR_SYMBOL = parse_token_type_terminate,
FIRST_PARSE_TOKEN_TYPE = parse_token_type_string
-};
+} __packed;
enum parse_keyword_t
{
@@ -93,7 +94,7 @@ enum parse_keyword_t
parse_keyword_exec,
LAST_KEYWORD = parse_keyword_exec
-};
+} __packed;
/* Statement decorations. This matches the order of productions in decorated_statement */
enum parse_statement_decoration_t
diff --git a/parse_execution.cpp b/parse_execution.cpp
index 6453ba37..649caaa8 100644
--- a/parse_execution.cpp
+++ b/parse_execution.cpp
@@ -72,7 +72,8 @@ node_offset_t parse_execution_context_t::get_offset(const parse_node_t &node) co
const parse_node_t *addr = &node;
const parse_node_t *base = &this->tree.at(0);
assert(addr >= base);
- node_offset_t offset = addr - base;
+ assert(addr - base < SOURCE_OFFSET_INVALID);
+ node_offset_t offset = static_cast<node_offset_t>(addr - base);
assert(offset < this->tree.size());
assert(&tree.at(offset) == &node);
return offset;
@@ -1100,7 +1101,7 @@ bool parse_execution_context_t::determine_io_chain(const parse_node_t &statement
if (out_chain && ! errored)
{
- std::swap(*out_chain, result);
+ out_chain->swap(result);
}
return ! errored;
}
diff --git a/parse_tree.cpp b/parse_tree.cpp
index 47fc68c5..ba742bcc 100644
--- a/parse_tree.cpp
+++ b/parse_tree.cpp
@@ -13,6 +13,13 @@ static bool production_is_empty(const production_t *production)
return (*production)[0] == token_type_invalid;
}
+void swap2(parse_node_tree_t &a, parse_node_tree_t &b)
+{
+ fprintf(stderr, "Swapping!\n");
+ // This uses the base vector implementation
+ a.swap(b);
+}
+
/** Returns a string description of this parse error */
wcstring parse_error_t::describe_with_prefix(const wcstring &src, const wcstring &prefix, bool is_interactive, bool skip_caret) const
{
@@ -422,7 +429,7 @@ static void dump_tree_recursive(const parse_node_tree_t &nodes, const wcstring &
result->push_back(L'\n');
++*line;
- for (size_t child_idx = node.child_start; child_idx < node.child_start + node.child_count; child_idx++)
+ for (node_offset_t child_idx = node.child_start; child_idx < node.child_start + node.child_count; child_idx++)
{
dump_tree_recursive(nodes, src, child_idx, indent + 1, result, line, inout_first_node_not_dumped);
}
@@ -546,28 +553,31 @@ class parse_ll_t
}
// Get the parent index. But we can't get the parent parse node yet, since it may be made invalid by adding children
- const size_t parent_node_idx = symbol_stack.back().node_idx;
+ const node_offset_t parent_node_idx = symbol_stack.back().node_idx;
// Add the children. Confusingly, we want our nodes to be in forwards order (last token last, so dumps look nice), but the symbols should be reverse order (last token first, so it's lowest on the stack)
- const size_t child_start = nodes.size();
- size_t child_count = 0;
+ const size_t child_start_big = nodes.size();
+ assert(child_start_big < NODE_OFFSET_INVALID);
+ node_offset_t child_start = static_cast<node_offset_t>(child_start_big);
+
+ // To avoid constructing multiple nodes, we push_back a single one that we modify
+ parse_node_t representative_child(token_type_invalid);
+ representative_child.parent = parent_node_idx;
+
+ node_offset_t child_count = 0;
for (size_t i=0; i < MAX_SYMBOLS_PER_PRODUCTION; i++)
{
production_element_t elem = (*production)[i];
- if (!production_element_is_valid(elem))
+ if (! production_element_is_valid(elem))
{
// All done, bail out
break;
}
- else
- {
- // Generate the parse node.
- parse_token_type_t child_type = production_element_type(elem);
- parse_node_t child = parse_node_t(child_type);
- child.parent = parent_node_idx;
- nodes.push_back(child);
- child_count++;
- }
+
+ // Append the parse node.
+ representative_child.type = production_element_type(elem);
+ nodes.push_back(representative_child);
+ child_count++;
}
// Update the parent
@@ -583,7 +593,7 @@ class parse_ll_t
// Replace the top of the stack with new stack elements corresponding to our new nodes. Note that these go in reverse order.
symbol_stack.pop_back();
symbol_stack.reserve(symbol_stack.size() + child_count);
- size_t idx = child_count;
+ node_offset_t idx = child_count;
while (idx--)
{
production_element_t elem = (*production)[idx];
@@ -669,18 +679,17 @@ void parse_ll_t::dump_stack(void) const
// Since children always appear after their parents, we can implement this very simply by walking backwards
void parse_ll_t::determine_node_ranges(void)
{
- const size_t source_start_invalid = -1;
size_t idx = nodes.size();
while (idx--)
{
- parse_node_t *parent = &nodes.at(idx);
+ parse_node_t *parent = &nodes[idx];
// Skip nodes that already have a source range. These are terminal nodes.
- if (parent->source_start != source_start_invalid)
+ if (parent->source_start != SOURCE_OFFSET_INVALID)
continue;
// Ok, this node needs a source range. Get all of its children, and then set its range.
- size_t min_start = source_start_invalid, max_end = 0; //note source_start_invalid is huge
+ source_offset_t min_start = SOURCE_OFFSET_INVALID, max_end = 0; //note SOURCE_OFFSET_INVALID is huge
for (node_offset_t i=0; i < parent->child_count; i++)
{
const parse_node_t &child = nodes.at(parent->child_offset(i));
@@ -691,7 +700,7 @@ void parse_ll_t::determine_node_ranges(void)
}
}
- if (min_start != source_start_invalid)
+ if (min_start != SOURCE_OFFSET_INVALID)
{
assert(max_end >= min_start);
parent->source_start = min_start;
@@ -704,13 +713,13 @@ void parse_ll_t::acquire_output(parse_node_tree_t *output, parse_error_list_t *e
{
if (output != NULL)
{
- std::swap(*output, this->nodes);
+ output->swap(this->nodes);
}
this->nodes.clear();
if (errors != NULL)
{
- std::swap(*errors, this->errors);
+ errors->swap(this->errors);
}
this->errors.clear();
this->symbol_stack.clear();
@@ -848,7 +857,7 @@ void parse_ll_t::parse_error(const wchar_t *expected, parse_token_t token)
void parse_ll_t::reset_symbols(enum parse_token_type_t goal)
{
/* Add a new goal node, and then reset our symbol list to point at it */
- node_offset_t where = nodes.size();
+ node_offset_t where = static_cast<node_offset_t>(nodes.size());
nodes.push_back(parse_node_t(goal));
symbol_stack.clear();
@@ -1064,14 +1073,10 @@ static parse_keyword_t keyword_for_token(token_type tok, const wchar_t *tok_txt)
}
/* Placeholder invalid token */
-static const parse_token_t kInvalidToken = {token_type_invalid,
-parse_keyword_none, false, false, static_cast<size_t>(-1),
- static_cast<size_t>(-1)};
+static const parse_token_t kInvalidToken = {token_type_invalid, parse_keyword_none, false, false, SOURCE_OFFSET_INVALID, 0};
/* Terminal token */
-static const parse_token_t kTerminalToken = {parse_token_type_terminate,
-parse_keyword_none, false, false, static_cast<size_t>(-1),
- static_cast<size_t>(-1)};
+static const parse_token_t kTerminalToken = {parse_token_type_terminate, parse_keyword_none, false, false, SOURCE_OFFSET_INVALID, 0};
static inline bool is_help_argument(const wchar_t *txt)
{
@@ -1099,8 +1104,8 @@ static inline parse_token_t next_parse_token(tokenizer_t *tok)
result.keyword = keyword_for_token(tok_type, tok_txt);
result.has_dash_prefix = (tok_txt[0] == L'-');
result.is_help_argument = result.has_dash_prefix && is_help_argument(tok_txt);
- result.source_start = (size_t)tok_start;
- result.source_length = tok_extent;
+ result.source_start = (source_offset_t)tok_start;
+ result.source_length = (source_offset_t)tok_extent;
tok_next(tok);
return result;
@@ -1212,7 +1217,7 @@ const parse_node_t *parse_node_tree_t::get_child(const parse_node_t &parent, nod
const parse_node_t &parse_node_tree_t::find_child(const parse_node_t &parent, parse_token_type_t type) const
{
- for (size_t i=0; i < parent.child_count; i++)
+ for (node_offset_t i=0; i < parent.child_count; i++)
{
const parse_node_t *child = this->get_child(parent, i);
if (child->type == type)
@@ -1258,7 +1263,7 @@ static void find_nodes_recursive(const parse_node_tree_t &tree, const parse_node
if (result->size() < max_count)
{
if (parent.type == type) result->push_back(&parent);
- for (size_t i=0; i < parent.child_count; i++)
+ for (node_offset_t i=0; i < parent.child_count; i++)
{
const parse_node_t *child = tree.get_child(parent, i);
assert(child != NULL);
@@ -1495,7 +1500,7 @@ const parse_node_t *parse_node_tree_t::next_node_in_node_list(const parse_node_t
const parse_node_t *next_cursor = NULL;
/* Walk through the children */
- for (size_t i=0; i < list_cursor->child_count; i++)
+ for (node_offset_t i=0; i < list_cursor->child_count; i++)
{
const parse_node_t *child = this->get_child(*list_cursor, i);
if (child->type == entry_type)
diff --git a/parse_tree.h b/parse_tree.h
index 706ee9c5..5b2f717e 100644
--- a/parse_tree.h
+++ b/parse_tree.h
@@ -18,9 +18,15 @@
class parse_node_t;
class parse_node_tree_t;
-typedef size_t node_offset_t;
+
+typedef uint32_t node_offset_t;
+
#define NODE_OFFSET_INVALID (static_cast<node_offset_t>(-1))
+typedef uint32_t source_offset_t;
+
+#define SOURCE_OFFSET_INVALID (static_cast<source_offset_t>(-1))
+
/* Returns a description of a list of parse errors */
wcstring parse_errors_description(const parse_error_list_t &errors, const wcstring &src, const wchar_t *prefix = NULL);
@@ -31,8 +37,8 @@ struct parse_token_t
enum parse_keyword_t keyword; // Any keyword represented by this token
bool has_dash_prefix; // Hackish: whether the source contains a dash prefix
bool is_help_argument; // Hackish: whether the source looks like '-h' or '--help'
- size_t source_start;
- size_t source_length;
+ source_offset_t source_start;
+ source_offset_t source_length;
wcstring describe() const;
wcstring user_presentable_description() const;
@@ -63,35 +69,36 @@ wcstring parse_dump_tree(const parse_node_tree_t &tree, const wcstring &src);
wcstring token_type_description(parse_token_type_t type);
wcstring keyword_description(parse_keyword_t type);
-/** Class for nodes of a parse tree */
+/** Class for nodes of a parse tree. Since there's a lot of these, the size and order of the fields is important. */
class parse_node_t
{
public:
-
- /* Type of the node */
- enum parse_token_type_t type;
-
/* Start in the source code */
- size_t source_start;
+ source_offset_t source_start;
/* Length of our range in the source code */
- size_t source_length;
+ source_offset_t source_length;
/* Parent */
node_offset_t parent;
/* Children */
node_offset_t child_start;
+
+ /* Number of children */
uint8_t child_count;
/* Which production was used */
uint8_t production_idx;
+
+ /* Type of the node */
+ enum parse_token_type_t type;
/* Description */
wcstring describe(void) const;
/* Constructor */
- explicit parse_node_t(parse_token_type_t ty) : type(ty), source_start(-1), source_length(0), parent(NODE_OFFSET_INVALID), child_start(0), child_count(0), production_idx(-1)
+ explicit parse_node_t(parse_token_type_t ty) : source_start(SOURCE_OFFSET_INVALID), source_length(0), parent(NODE_OFFSET_INVALID), child_start(0), child_count(0), production_idx(-1), type(ty)
{
}
@@ -104,7 +111,7 @@ public:
/* Indicate if this node has a range of source code associated with it */
bool has_source() const
{
- return source_start != (size_t)(-1);
+ return source_start != SOURCE_OFFSET_INVALID;
}
/* Gets source for the node, or the empty string if it has no source */
@@ -123,7 +130,6 @@ public:
}
};
-
/* The parse tree itself */
class parse_node_tree_t : public std::vector<parse_node_t>
{
@@ -180,6 +186,7 @@ public:
parse_node_list_t specific_statements_for_job(const parse_node_t &job) const;
};
+
/* The big entry point. Parse a string, attempting to produce a tree for the given goal type */
bool parse_tree_from_string(const wcstring &str, parse_tree_flags_t flags, parse_node_tree_t *output, parse_error_list_t *errors, parse_token_type_t goal = symbol_job_list);
diff --git a/utf8.cpp b/utf8.cpp
new file mode 100644
index 00000000..196b5268
--- /dev/null
+++ b/utf8.cpp
@@ -0,0 +1,514 @@
+/*
+ * Copyright (c) 2007 Alexey Vatchenko <av@bsdua.org>
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#include <sys/types.h>
+#include <stdint.h>
+#include <wchar.h>
+
+#include "utf8.h"
+
+#include <string>
+#include <vector>
+#include <limits>
+
+#define _NXT 0x80
+#define _SEQ2 0xc0
+#define _SEQ3 0xe0
+#define _SEQ4 0xf0
+#define _SEQ5 0xf8
+#define _SEQ6 0xfc
+
+#define _BOM 0xfeff
+
+/* We can tweak the following typedef to allow us to simulate Windows-style 16 bit wchar's on Unix */
+typedef wchar_t utf8_wchar_t;
+#define UTF8_WCHAR_MAX ((size_t)std::numeric_limits<utf8_wchar_t>::max())
+
+bool is_wchar_ucs2()
+{
+ return UTF8_WCHAR_MAX <= 0xFFFF;
+}
+
+static size_t utf8_to_wchar_internal(const char *in, size_t insize, utf8_wchar_t *out, size_t outsize, int flags);
+static size_t wchar_to_utf8_internal(const utf8_wchar_t *in, size_t insize, char *out, size_t outsize, int flags);
+
+static bool safe_copy_wchar_to_utf8_wchar(const wchar_t *in, utf8_wchar_t *out, size_t count)
+{
+ bool result = true;
+ for (size_t i=0; i < count; i++)
+ {
+ wchar_t c = in[i];
+ if (c > UTF8_WCHAR_MAX)
+ {
+ result = false;
+ break;
+ }
+ out[i] = c;
+ }
+ return result;
+}
+
+bool utf8_to_wchar_string(const std::string &str, std::wstring *result)
+{
+ result->clear();
+ const size_t inlen = str.size();
+ if (inlen == 0)
+ {
+ return true;
+ }
+
+ bool success = false;
+ const char *input = str.c_str();
+ size_t outlen = utf8_to_wchar(input, inlen, NULL, 0, 0);
+ if (outlen > 0)
+ {
+ wchar_t *tmp = new wchar_t[outlen];
+ size_t outlen2 = utf8_to_wchar(input, inlen, tmp, outlen, 0);
+ if (outlen2 > 0)
+ {
+ result->assign(tmp, outlen2);
+ success = true;
+ }
+ delete[] tmp;
+ }
+ return success;
+}
+
+bool wchar_to_utf8_string(const std::wstring &str, std::string *result)
+{
+ result->clear();
+ const size_t inlen = str.size();
+ if (inlen == 0)
+ {
+ return true;
+ }
+
+ bool success = false;
+ const wchar_t *input = str.c_str();
+ size_t outlen = wchar_to_utf8(input, inlen, NULL, 0, 0);
+ if (outlen > 0)
+ {
+ char *tmp = new char[outlen];
+ size_t outlen2 = wchar_to_utf8(input, inlen, tmp, outlen, 0);
+ if (outlen2 > 0)
+ {
+ result->assign(tmp, outlen2);
+ success = true;
+ }
+ delete[] tmp;
+ }
+ return success;
+}
+
+size_t utf8_to_wchar(const char *in, size_t insize, wchar_t *out, size_t outsize, int flags)
+{
+ if (in == NULL || insize == 0 || (outsize == 0 && out != NULL))
+ {
+ return 0;
+ }
+
+ size_t result;
+ if (sizeof(wchar_t) == sizeof(utf8_wchar_t))
+ {
+ result = utf8_to_wchar_internal(in, insize, reinterpret_cast<utf8_wchar_t *>(out), outsize, flags);
+ }
+ else
+ {
+ // Allocate a temporary buffer to hold the output
+ // note: outsize may be 0
+ utf8_wchar_t *tmp_output = new utf8_wchar_t[outsize];
+
+ // Invoke the conversion with the temporary
+ result = utf8_to_wchar_internal(in, insize, tmp_output, outsize, flags);
+
+ // Copy back from tmp to the function's output, then clean it up
+ size_t amount_to_copy = std::min(result, outsize);
+ std::copy(tmp_output, tmp_output + amount_to_copy, out);
+ delete[] tmp_output;
+ }
+ return result;
+}
+
+size_t wchar_to_utf8(const wchar_t *in, size_t insize, char *out, size_t outsize, int flags)
+{
+ if (in == NULL || insize == 0 || (outsize == 0 && out != NULL))
+ {
+ return 0;
+ }
+
+ size_t result;
+ if (sizeof(wchar_t) == sizeof(utf8_wchar_t))
+ {
+ result = wchar_to_utf8_internal(reinterpret_cast<const utf8_wchar_t *>(in), insize, out, outsize, flags);
+ }
+ else
+ {
+ // Allocate a temporary buffer to hold the input
+ // the std::copy performs the size conversion
+ // note: insize may be 0
+ utf8_wchar_t *tmp_input = new utf8_wchar_t[insize];
+ if (! safe_copy_wchar_to_utf8_wchar(in, tmp_input, insize))
+ {
+ // our utf8_wchar_t is UCS-16 and there was an astral character
+ result = 0;
+ }
+ else
+ {
+ // Invoke the conversion with the temporary, then clean up the input
+ result = wchar_to_utf8_internal(tmp_input, insize, out, outsize, flags);
+ }
+ delete[] tmp_input;
+ }
+ return result;
+}
+
+
+static int __wchar_forbitten(utf8_wchar_t sym);
+static int __utf8_forbitten(unsigned char octet);
+
+static int
+__wchar_forbitten(utf8_wchar_t sym)
+{
+
+ /* Surrogate pairs */
+ if (sym >= 0xd800 && sym <= 0xdfff)
+ return (-1);
+
+ return (0);
+}
+
+static int
+__utf8_forbitten(unsigned char octet)
+{
+
+ switch (octet)
+ {
+ case 0xc0:
+ case 0xc1:
+ case 0xf5:
+ case 0xff:
+ return (-1);
+ }
+
+ return (0);
+}
+
+/*
+ * DESCRIPTION
+ * This function translates UTF-8 string into UCS-2 or UCS-4 string (all symbols
+ * will be in local machine byte order).
+ *
+ * It takes the following arguments:
+ * in - input UTF-8 string. It can be null-terminated.
+ * insize - size of input string in bytes.
+ * out - result buffer for UCS-2/4 string. If out is NULL,
+ * function returns size of result buffer.
+ * outsize - size of out buffer in wide characters.
+ *
+ * RETURN VALUES
+ * The function returns size of result buffer (in wide characters).
+ * Zero is returned in case of error.
+ *
+ * CAVEATS
+ * 1. If UTF-8 string contains zero symbols, they will be translated
+ * as regular symbols.
+ * 2. If UTF8_IGNORE_ERROR or UTF8_SKIP_BOM flag is set, sizes may vary
+ * when `out' is NULL and not NULL. It's because of special UTF-8
+ * sequences which may result in forbitten (by RFC3629) UNICODE
+ * characters. So, the caller must check return value every time and
+ * not prepare buffer in advance (\0 terminate) but after calling this
+ * function.
+ */
+static size_t utf8_to_wchar_internal(const char *in, size_t insize, utf8_wchar_t *out, size_t outsize, int flags)
+{
+ unsigned char *p, *lim;
+ utf8_wchar_t *wlim, high;
+ size_t n, total, i, n_bits;
+
+ if (in == NULL || insize == 0 || (outsize == 0 && out != NULL))
+ return (0);
+
+ total = 0;
+ p = (unsigned char *)in;
+ lim = p + insize;
+ wlim = out + outsize;
+
+ for (; p < lim; p += n)
+ {
+ if (__utf8_forbitten(*p) != 0 &&
+ (flags & UTF8_IGNORE_ERROR) == 0)
+ return (0);
+
+ /*
+ * Get number of bytes for one wide character.
+ */
+ n = 1; /* default: 1 byte. Used when skipping bytes. */
+ if ((*p & 0x80) == 0)
+ high = (utf8_wchar_t)*p;
+ else if ((*p & 0xe0) == _SEQ2)
+ {
+ n = 2;
+ high = (utf8_wchar_t)(*p & 0x1f);
+ }
+ else if ((*p & 0xf0) == _SEQ3)
+ {
+ n = 3;
+ high = (utf8_wchar_t)(*p & 0x0f);
+ }
+ else if ((*p & 0xf8) == _SEQ4)
+ {
+ n = 4;
+ high = (utf8_wchar_t)(*p & 0x07);
+ }
+ else if ((*p & 0xfc) == _SEQ5)
+ {
+ n = 5;
+ high = (utf8_wchar_t)(*p & 0x03);
+ }
+ else if ((*p & 0xfe) == _SEQ6)
+ {
+ n = 6;
+ high = (utf8_wchar_t)(*p & 0x01);
+ }
+ else
+ {
+ if ((flags & UTF8_IGNORE_ERROR) == 0)
+ return (0);
+ continue;
+ }
+
+ /* does the sequence header tell us truth about length? */
+ if (lim - p <= n - 1)
+ {
+ if ((flags & UTF8_IGNORE_ERROR) == 0)
+ return (0);
+ n = 1;
+ continue; /* skip */
+ }
+
+ /*
+ * Validate sequence.
+ * All symbols must have higher bits set to 10xxxxxx
+ */
+ if (n > 1)
+ {
+ for (i = 1; i < n; i++)
+ {
+ if ((p[i] & 0xc0) != _NXT)
+ break;
+ }
+ if (i != n)
+ {
+ if ((flags & UTF8_IGNORE_ERROR) == 0)
+ return (0);
+ n = 1;
+ continue; /* skip */
+ }
+ }
+
+ total++;
+
+ if (out == NULL)
+ continue;
+
+ if (out >= wlim)
+ return (0); /* no space left */
+
+ uint32_t out_val = 0;
+ *out = 0;
+ n_bits = 0;
+ for (i = 1; i < n; i++)
+ {
+ out_val |= (utf8_wchar_t)(p[n - i] & 0x3f) << n_bits;
+ n_bits += 6; /* 6 low bits in every byte */
+ }
+ out_val |= high << n_bits;
+
+ bool skip = false;
+ if (__wchar_forbitten(out_val) != 0)
+ {
+ if ((flags & UTF8_IGNORE_ERROR) == 0)
+ {
+ return 0; /* forbitten character */
+ }
+ else
+ {
+ skip = true;
+ }
+ }
+ else if (out_val == _BOM && (flags & UTF8_SKIP_BOM) != 0)
+ {
+ skip = true;
+ }
+
+ if (skip)
+ {
+ total--;
+ }
+ else if (out_val > UTF8_WCHAR_MAX)
+ {
+ // wchar_t is UCS-2, but the UTF-8 specified an astral character
+ return 0;
+ }
+ else
+ {
+ *out++ = out_val;
+ }
+ }
+
+ return (total);
+}
+
+/*
+ * DESCRIPTION
+ * This function translates UCS-2/4 symbols (given in local machine
+ * byte order) into UTF-8 string.
+ *
+ * It takes the following arguments:
+ * in - input unicode string. It can be null-terminated.
+ * insize - size of input string in wide characters.
+ * out - result buffer for utf8 string. If out is NULL,
+ * function returns size of result buffer.
+ * outsize - size of result buffer.
+ *
+ * RETURN VALUES
+ * The function returns size of result buffer (in bytes). Zero is returned
+ * in case of error.
+ *
+ * CAVEATS
+ * If UCS-4 string contains zero symbols, they will be translated
+ * as regular symbols.
+ */
+static size_t wchar_to_utf8_internal(const utf8_wchar_t *in, size_t insize, char *out, size_t outsize, int flags)
+{
+ const utf8_wchar_t *w, *wlim;
+ unsigned char *p, *lim;
+ size_t total, n;
+
+ if (in == NULL || insize == 0 || (outsize == 0 && out != NULL))
+ return (0);
+
+ w = in;
+ wlim = w + insize;
+ p = (unsigned char *)out;
+ lim = p + outsize;
+ total = 0;
+ for (; w < wlim; w++)
+ {
+ if (__wchar_forbitten(*w) != 0)
+ {
+ if ((flags & UTF8_IGNORE_ERROR) == 0)
+ return (0);
+ else
+ continue;
+ }
+
+ if (*w == _BOM && (flags & UTF8_SKIP_BOM) != 0)
+ continue;
+
+ const int32_t w_wide = *w;
+ if (w_wide < 0)
+ {
+ if ((flags & UTF8_IGNORE_ERROR) == 0)
+ return (0);
+ continue;
+ }
+ else if (w_wide <= 0x0000007f)
+ n = 1;
+ else if (w_wide <= 0x000007ff)
+ n = 2;
+ else if (w_wide <= 0x0000ffff)
+ n = 3;
+ else if (w_wide <= 0x001fffff)
+ n = 4;
+ else if (w_wide <= 0x03ffffff)
+ n = 5;
+ else /* if (w_wide <= 0x7fffffff) */
+ n = 6;
+
+ total += n;
+
+ if (out == NULL)
+ continue;
+
+ if (lim - p <= n - 1)
+ return (0); /* no space left */
+
+ /* extract the wchar_t as big-endian. If wchar_t is UCS-16, the first two bytes will be 0 */
+ unsigned char oc[4];
+ uint32_t w_tmp = *w;
+ oc[3] = w_tmp & 0xFF;
+ w_tmp >>= 8;
+ oc[2] = w_tmp & 0xFF;
+ w_tmp >>= 8;
+ oc[1] = w_tmp & 0xFF;
+ w_tmp >>= 8;
+ oc[0] = w_tmp & 0xFF;
+
+ switch (n)
+ {
+ case 1:
+ p[0] = oc[3];
+ break;
+
+ case 2:
+ p[1] = _NXT | (oc[3] & 0x3f);
+ p[0] = _SEQ2 | (oc[3] >> 6) | ((oc[2] & 0x07) << 2);
+ break;
+
+ case 3:
+ p[2] = _NXT | (oc[3] & 0x3f);
+ p[1] = _NXT | (oc[3] >> 6) | ((oc[2] & 0x0f) << 2);
+ p[0] = _SEQ3 | ((oc[2] & 0xf0) >> 4);
+ break;
+
+ case 4:
+ p[3] = _NXT | (oc[3] & 0x3f);
+ p[2] = _NXT | (oc[3] >> 6) | ((oc[2] & 0x0f) << 2);
+ p[1] = _NXT | ((oc[2] & 0xf0) >> 4) |
+ ((oc[1] & 0x03) << 4);
+ p[0] = _SEQ4 | ((oc[1] & 0x1f) >> 2);
+ break;
+
+ case 5:
+ p[4] = _NXT | (oc[3] & 0x3f);
+ p[3] = _NXT | (oc[3] >> 6) | ((oc[2] & 0x0f) << 2);
+ p[2] = _NXT | ((oc[2] & 0xf0) >> 4) |
+ ((oc[1] & 0x03) << 4);
+ p[1] = _NXT | (oc[1] >> 2);
+ p[0] = _SEQ5 | (oc[0] & 0x03);
+ break;
+
+ case 6:
+ p[5] = _NXT | (oc[3] & 0x3f);
+ p[4] = _NXT | (oc[3] >> 6) | ((oc[2] & 0x0f) << 2);
+ p[3] = _NXT | (oc[2] >> 4) | ((oc[1] & 0x03) << 4);
+ p[2] = _NXT | (oc[1] >> 2);
+ p[1] = _NXT | (oc[0] & 0x3f);
+ p[0] = _SEQ6 | ((oc[0] & 0x40) >> 6);
+ break;
+ }
+
+ /*
+ * NOTE: do not check here for forbitten UTF-8 characters.
+ * They cannot appear here because we do proper convertion.
+ */
+
+ p += n;
+ }
+
+ return (total);
+}
diff --git a/utf8.h b/utf8.h
new file mode 100644
index 00000000..a1f50154
--- /dev/null
+++ b/utf8.h
@@ -0,0 +1,41 @@
+/*
+ * Copyright (c) 2007 Alexey Vatchenko <av@bsdua.org>
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+/*
+ * utf8: implementation of UTF-8 charset encoding (RFC3629).
+ */
+#ifndef _UTF8_H_
+#define _UTF8_H_
+
+#include <sys/types.h>
+
+#include <string>
+#include <wchar.h>
+
+#define UTF8_IGNORE_ERROR 0x01
+#define UTF8_SKIP_BOM 0x02
+
+/* Convert a string between UTF8 and UCS-2/4 (depending on size of wchar_t). Returns true if successful, storing the result of the conversion in *result */
+bool utf8_to_wchar_string(const std::string &input, std::wstring *result);
+bool wchar_to_utf8_string(const std::wstring &input, std::string *result);
+
+/* Variants exposed for testing */
+size_t utf8_to_wchar(const char *in, size_t insize, wchar_t *out, size_t outsize, int flags);
+size_t wchar_to_utf8(const wchar_t *in, size_t insize, char *out, size_t outsize, int flags);
+
+bool is_wchar_ucs2();
+
+#endif /* !_UTF8_H_ */
generated by cgit v1.2.3 (git 2.39.1) at 2024-06-23 02:16:11 +0000