utf-8 aware functions for basis. unit-testing.

4 files changed, 887 insertions, 0 deletions

diff --git a/tests/Makefile b/tests/Makefile
index ecf5557b..03e37e4b 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -28,3 +28,5 @@ simple::
 	./driver.sh fact
 	./driver.sh filter
 	./driver.sh jsbspace
+	./driver.sh utf8
+
diff --git a/tests/utf8.py b/tests/utf8.py
new file mode 100644
index 00000000..ff9b737a
--- /dev/null
+++ b/tests/utf8.py
@@ -0,0 +1,449 @@
+import unittest
+import base
+
+class Suite(base.Base):
+    def test_1(self):
+        """Test case: substring (1)"""
+        self.start('Utf8/substrings')
+
+        pre = self.xpath('pre[1]')
+        self.assertEqual('abc', pre.text)
+
+        pre = self.xpath('pre[2]')
+        self.assertEqual('bc', pre.text)
+
+        pre = self.xpath('pre[3]')
+        self.assertEqual('c', pre.text)
+
+        pre = self.xpath('pre[4]')
+        self.assertEqual('ábó', pre.text)
+
+        pre = self.xpath('pre[5]')
+        self.assertEqual('bó', pre.text)
+
+        pre = self.xpath('pre[6]')
+        self.assertEqual('ó', pre.text)
+        
+        pre = self.xpath('pre[7]')
+        self.assertEqual('çãó', pre.text)
+
+        pre = self.xpath('pre[8]')
+        self.assertEqual('ãó', pre.text)
+
+        pre = self.xpath('pre[9]')
+        self.assertEqual('ó', pre.text)
+
+        pre = self.xpath('pre[10]')
+        self.assertEqual('', pre.text)
+
+        pre = self.xpath('pre[11]')
+        self.assertEqual('', pre.text)
+
+        
+    def test_2(self):
+        """Test case: strlen (2)"""
+        self.start('Utf8/strlens')
+
+        pre = self.xpath('pre[1]')
+        self.assertEqual('3', pre.text)
+
+        pre = self.xpath('pre[2]')
+        self.assertEqual('3', pre.text)
+
+        pre = self.xpath('pre[3]')
+        self.assertEqual('3', pre.text)
+
+        pre = self.xpath('pre[4]')
+        self.assertEqual('3', pre.text)
+
+        pre = self.xpath('pre[5]')
+        self.assertEqual('1', pre.text)
+
+        pre = self.xpath('pre[6]')
+        self.assertEqual('1', pre.text)
+
+        pre = self.xpath('pre[7]')
+        self.assertEqual('0', pre.text)
+        
+        pre = self.xpath('pre[8]')
+        self.assertEqual('1', pre.text)
+        
+        pre = self.xpath('pre[9]')
+        self.assertEqual('1', pre.text)
+
+        pre = self.xpath('pre[10]')
+        self.assertEqual('1', pre.text)
+
+        pre = self.xpath('pre[11]')
+        self.assertEqual('6', pre.text)
+
+        pre = self.xpath('pre[12]')
+        self.assertEqual('2', pre.text)
+
+        pre = self.xpath('pre[13]')
+        self.assertEqual('14', pre.text)
+
+        
+    def test_3(self):
+        """Test case: strlenGe (3)"""
+        self.start('Utf8/strlenGens')
+        
+        pre = self.xpath('pre[1]')
+        self.assertEqual('False', pre.text)
+
+        pre = self.xpath('pre[2]')
+        self.assertEqual('True', pre.text)
+
+        pre = self.xpath('pre[3]')
+        self.assertEqual('False', pre.text)
+
+        pre = self.xpath('pre[4]')
+        self.assertEqual('True', pre.text)
+
+        pre = self.xpath('pre[5]')
+        self.assertEqual('True', pre.text)
+
+        pre = self.xpath('pre[6]')
+        self.assertEqual('False', pre.text)
+
+        pre = self.xpath('pre[7]')
+        self.assertEqual('True', pre.text)
+
+        pre = self.xpath('pre[8]')
+        self.assertEqual('True', pre.text)
+
+    def test_4(self):
+        """Test case: strcat (4)"""
+        self.start('Utf8/strcats')
+        
+        pre = self.xpath('pre[1]')
+        self.assertEqual('', pre.text)
+        
+        pre = self.xpath('pre[2]')
+        self.assertEqual('0', pre.text)
+
+        pre = self.xpath('pre[3]')
+        self.assertEqual('aabb', pre.text)
+        
+        pre = self.xpath('pre[4]')
+        self.assertEqual('4', pre.text)
+        
+        pre = self.xpath('pre[5]')
+        self.assertEqual('bb', pre.text)
+        
+        pre = self.xpath('pre[6]')
+        self.assertEqual('2', pre.text)
+        
+        pre = self.xpath('pre[7]')
+        self.assertEqual('aa', pre.text)
+        
+        pre = self.xpath('pre[8]')
+        self.assertEqual('2', pre.text)
+        
+        pre = self.xpath('pre[9]')
+        self.assertEqual('ààáá', pre.text)
+        
+        pre = self.xpath('pre[10]')
+        self.assertEqual('4', pre.text)
+        
+        pre = self.xpath('pre[11]')
+        self.assertEqual('áá', pre.text)
+        
+        pre = self.xpath('pre[12]')
+        self.assertEqual('2', pre.text)
+        
+        pre = self.xpath('pre[13]')
+        self.assertEqual('àà', pre.text)
+        
+        pre = self.xpath('pre[14]')
+        self.assertEqual('2', pre.text)
+
+    def test_5(self):
+        """Test case: strsub (5)"""
+        self.start('Utf8/strsubs')
+
+        pre = self.xpath('pre[1]')
+        self.assertEqual('a', pre.text)
+
+        pre = self.xpath('pre[2]')
+        self.assertEqual('b', pre.text)
+
+        pre = self.xpath('pre[3]')
+        self.assertEqual('à', pre.text)
+
+        pre = self.xpath('pre[4]')
+        self.assertEqual('ç', pre.text)
+
+    def test_6(self):
+        """Test case: strsuffix (6)"""
+        self.start('Utf8/strsuffixs')
+
+        pre = self.xpath('pre[1]')
+        self.assertEqual('abàç', pre.text)
+
+        pre = self.xpath('pre[2]')
+        self.assertEqual('bàç', pre.text)
+        
+        pre = self.xpath('pre[3]')
+        self.assertEqual('àç', pre.text)
+        
+        pre = self.xpath('pre[4]')
+        self.assertEqual('ç', pre.text)
+
+    def test_7(self):
+        """Test case: strchr (7)"""
+        self.start('Utf8/strchrs')
+
+        pre = self.xpath('pre[1]')
+        self.assertEqual('None', pre.text)
+
+        pre = self.xpath('pre[2]')
+        self.assertEqual('Some "bàç"', pre.text)
+        
+        pre = self.xpath('pre[3]')
+        self.assertEqual('Some "àç"', pre.text)
+
+        pre = self.xpath('pre[4]')
+        self.assertEqual('Some "ç"', pre.text)
+
+        pre = self.xpath('pre[5]')
+        self.assertEqual('Some ""', pre.text)
+        
+    def test_8(self):
+        """Test case: strindex (8)"""
+        self.start('Utf8/strindexs')
+        
+        pre = self.xpath('pre[1]')
+        self.assertEqual('None', pre.text)
+
+        pre = self.xpath('pre[2]')
+        self.assertEqual('Some 0', pre.text)
+        
+        pre = self.xpath('pre[3]')
+        self.assertEqual('Some 1', pre.text)
+
+        pre = self.xpath('pre[4]')
+        self.assertEqual('Some 2', pre.text)
+
+        pre = self.xpath('pre[5]')
+        self.assertEqual('Some 3', pre.text)
+
+    def test_9(self):
+        """Test case: strindex (9)"""
+        self.start('Utf8/strsindexs')
+
+        pre = self.xpath('pre[1]')
+        # behavior of strstr C function
+        self.assertEqual('Some 0', pre.text)
+        
+        pre = self.xpath('pre[2]')
+        self.assertEqual('Some 0', pre.text)
+        
+        pre = self.xpath('pre[3]')
+        self.assertEqual('None', pre.text)
+        
+        pre = self.xpath('pre[4]')
+        self.assertEqual('Some 1', pre.text)
+
+        pre = self.xpath('pre[5]')
+        self.assertEqual('None', pre.text)
+        
+        pre = self.xpath('pre[6]')
+        self.assertEqual('Some 2', pre.text)
+
+        pre = self.xpath('pre[7]')
+        self.assertEqual('None', pre.text)
+        
+        pre = self.xpath('pre[8]')
+        self.assertEqual('None', pre.text)
+
+        pre = self.xpath('pre[9]')
+        self.assertEqual('Some 3', pre.text)
+
+    def test_10(self):
+        """Test case: strcspn (10)"""
+        self.start('Utf8/strcspns')
+
+        pre = self.xpath('pre[1]')
+        self.assertEqual('4', pre.text)
+
+        pre = self.xpath('pre[2]')
+        self.assertEqual('0', pre.text)
+        
+        pre = self.xpath('pre[3]')
+        self.assertEqual('0', pre.text)
+        
+        pre = self.xpath('pre[4]')
+        self.assertEqual('1', pre.text)
+
+        pre = self.xpath('pre[5]')
+        self.assertEqual('2', pre.text)
+
+        pre = self.xpath('pre[6]')
+        self.assertEqual('3', pre.text)
+
+    def test_11(self):
+        """Test case: str1 (11)"""
+        self.start('Utf8/str1s')
+
+        pre = self.xpath('pre[1]')
+        self.assertEqual('a', pre.text)
+
+        pre = self.xpath('pre[2]')
+        self.assertEqual('à', pre.text)
+
+        pre = self.xpath('pre[3]')
+        self.assertEqual('á', pre.text)
+
+    def test_12(self):
+        """Test case: isalnum (12)"""
+        self.start('Utf8/isalnums')
+                               
+        for idx in range(1, 9):
+            pre = self.xpath('pre[' + str(idx) + ']')
+            self.assertEqual('True', pre.text, 'Failed isalnum: assert ' + str(idx))
+        
+    def test_13(self):
+        """Test case: isalpha (13)"""
+        self.start('Utf8/isalphas')
+                       
+        for idx in range(1, 9):
+            pre = self.xpath('pre[' + str(idx) + ']')
+            self.assertEqual('True', pre.text, 'Failed isalpha: assert ' + str(idx))
+        
+    def test_14(self):
+        """Test case: isblank (14)"""
+        self.start('Utf8/isblanks')
+               
+        for idx in range(1, 11):
+            pre = self.xpath('pre[' + str(idx) + ']')
+            self.assertEqual('True', pre.text, 'Failed isblank: assert ' + str(idx))
+
+    def test_15(self):
+        """Test case: iscntrl (15)"""
+        self.start('Utf8/iscntrls')
+               
+        for idx in range(1, 11):
+            pre = self.xpath('pre[' + str(idx) + ']')
+            self.assertEqual('True', pre.text, 'Failed iscntrl: assert ' + str(idx))
+        
+    def test_16(self):
+        """Test case: isdigit (16)"""
+        self.start('Utf8/isdigits')
+               
+        for idx in range(1, 11):
+            pre = self.xpath('pre[' + str(idx) + ']')
+            self.assertEqual('True', pre.text, 'Failed isdigit: assert ' + str(idx))
+
+        
+    def test_17(self):
+        """Test case: isgraph (17)"""
+        self.start('Utf8/isgraphs')
+        
+        for idx in range(1, 11):
+            pre = self.xpath('pre[' + str(idx) + ']')
+            self.assertEqual('True', pre.text, 'Failed isgraph: assert ' + str(idx))
+    
+    def test_18(self):
+        """Test case: islower (18)"""
+        self.start('Utf8/islowers')
+        
+        for idx in range(1, 11):
+            pre = self.xpath('pre[' + str(idx) + ']')
+            self.assertEqual('True', pre.text, 'Failed islower: assert ' + str(idx))
+        
+    def test_19(self):
+        """Test case: isprint (19)"""
+        self.start('Utf8/isprints')
+
+        for idx in range(1, 11):
+            pre = self.xpath('pre[' + str(idx) + ']')
+            self.assertEqual('True', pre.text, 'Failed isprint: assert ' + str(idx))
+        
+    def test_20(self):
+        """Test case: ispunct (20)"""
+        self.start('Utf8/ispuncts')
+
+        for idx in range(1, 11):
+            pre = self.xpath('pre[' + str(idx) + ']')
+            self.assertEqual('True', pre.text, 'Failed ispunct: assert ' + str(idx))
+        
+    def test_21(self):
+        """Test case: isspace (21)"""
+        self.start('Utf8/isspaces')
+
+        for idx in range(1, 11):
+            pre = self.xpath('pre[' + str(idx) + ']')
+            self.assertEqual('True', pre.text, 'Failed isspace: assert ' + str(idx))
+
+    def test_22(self):
+        """Test case: isupper (22)"""
+        self.start('Utf8/isuppers')
+
+        for idx in range(1, 11):
+            pre = self.xpath('pre[' + str(idx) + ']')
+            self.assertEqual('True', pre.text, 'Failed isupper: assert ' + str(idx))
+
+    def test_23(self):
+        """Test case: isxdigit (23)"""
+        self.start('Utf8/isxdigits')
+
+        for idx in range(1, 11):
+            pre = self.xpath('pre[' + str(idx) + ']')
+            self.assertEqual('True', pre.text, 'Failed isxdigit: assert ' + str(idx))
+
+    def test_24(self):
+        """Test case: toupper (24)"""
+        self.start('Utf8/touppers')
+
+        for idx in range(1, 6):
+            pre = self.xpath('pre[' + str(idx) + ']')
+            self.assertEqual('True', pre.text, 'Failed toupper: assert ' + str(idx))
+
+    def test_25(self):
+        """Test case: ord (25)"""
+        self.start('Utf8/ord_and_chrs')
+
+        for idx in range(1, 8):
+            pre = self.xpath('pre[' + str(idx) + ']')
+            self.assertEqual('True', pre.text, 'Failed ord: assert ' + str(idx))
+
+    def test_26 (self):
+        """Test case: test_db (26) """
+        self.start('Utf8/test_db')
+
+        pre = self.xpath('pre[1]')
+        self.assertEqual('abc', pre.text)
+
+        pre = self.xpath('pre[2]')
+        self.assertEqual('3', pre.text)
+
+        pre = self.xpath('pre[3]')
+        self.assertEqual('çãó', pre.text)
+
+        pre = self.xpath('pre[4]')
+        self.assertEqual('3', pre.text)
+
+        pre = self.xpath('pre[5]')
+        self.assertEqual('が', pre.text)
+
+        pre = self.xpath('pre[6]')
+        self.assertEqual('1', pre.text)
+
+        pre = self.xpath('pre[7]')
+        self.assertEqual('漢', pre.text)
+
+        pre = self.xpath('pre[8]')
+        self.assertEqual('1', pre.text)
+
+        pre = self.xpath('pre[9]')
+        self.assertEqual('カ', pre.text)
+
+        pre = self.xpath('pre[10]')
+        self.assertEqual('1', pre.text)
+
+        pre = self.xpath('pre[11]')
+        self.assertEqual('وظيفية', pre.text)
+
+        pre = self.xpath('pre[12]')
+        self.assertEqual('6', pre.text)
diff --git a/tests/utf8.ur b/tests/utf8.ur
new file mode 100644
index 00000000..0dedc726
--- /dev/null
+++ b/tests/utf8.ur
@@ -0,0 +1,431 @@
+fun substrings () : transaction page = return <xml>
+  <body>
+    <pre>{[substring "abc" 0 3]}</pre>   
+    <pre>{[substring "abc" 1 2]}</pre>
+    <pre>{[substring "abc" 2 1]}</pre>
+    <pre>{[substring "ábó" 0 3]}</pre>    
+    <pre>{[substring "ábó" 1 2]}</pre>
+    <pre>{[substring "ábó" 2 1]}</pre>    
+    <pre>{[substring "çãó" 0 3]}</pre>
+    <pre>{[substring "çãó" 1 2]}</pre>
+    <pre>{[substring "çãó" 2 1]}</pre>
+    <pre>{[substring "çãó" 2 0]}</pre>
+    <pre>{[substring "" 0 0]}</pre>
+  </body>
+</xml>
+
+fun strlens () : transaction page = return <xml>
+  <body>
+    <pre>{[strlen "abc"]}</pre>
+    <pre>{[strlen "çbc"]}</pre>
+    <pre>{[strlen "çãc"]}</pre>
+    <pre>{[strlen "çãó"]}</pre>
+    <pre>{[strlen "ç"]}</pre>
+    <pre>{[strlen "c"]}</pre>
+    <pre>{[strlen ""]}</pre>
+    <pre>{[strlen "が"]}</pre>
+    <pre>{[strlen "漢"]}</pre>
+    <pre>{[strlen "カ"]}</pre>
+    <pre>{[strlen "وظيفية"]}</pre>
+    <pre>{[strlen "函數"]}</pre>
+    <pre>{[strlen "Функциональное"]}</pre>
+  </body>
+  </xml>
+				       
+fun strlenGens () : transaction page = return <xml>
+  <body>
+    <pre>{[strlenGe "" 1]}</pre>
+    <pre>{[strlenGe "" 0]}</pre>
+    <pre>{[strlenGe "aba" 4]}</pre>
+    <pre>{[strlenGe "aba" 3]}</pre>
+    <pre>{[strlenGe "aba" 2]}</pre>
+    <pre>{[strlenGe "áçà" 4]}</pre>
+    <pre>{[strlenGe "áçà" 3]}</pre>
+    <pre>{[strlenGe "áçà" 2]}</pre>
+    
+  </body>
+  </xml>
+
+fun strcats () : transaction page =
+    let
+	fun catAndLen a b =
+	    <xml>
+	      <pre>{[strcat a b]}</pre>
+	      <pre>{[strlen (strcat a b)]}</pre>
+	    </xml>
+    in
+	return <xml>
+	  <body>
+	    {catAndLen "" ""}
+	    {catAndLen "aa" "bb"}
+	    {catAndLen "" "bb"}
+	    {catAndLen "aa" ""}
+	    {catAndLen "àà" "áá"}
+	    {catAndLen "" "áá"}
+	    {catAndLen "àà" ""}	    
+	  </body>
+	</xml>
+end
+
+fun strsubs () : transaction page =
+    return <xml>
+      <body>
+	<pre>{[strsub "abàç" 0]}</pre>
+	<pre>{[strsub "abàç" 1]}</pre>
+	<pre>{[strsub "abàç" 2]}</pre>
+	<pre>{[strsub "abàç" 3]}</pre>
+      </body>
+      </xml>
+
+fun strsuffixs () : transaction page =
+    return <xml>
+      <body>
+	<pre>{[strsuffix "abàç" 0]}</pre>
+	<pre>{[strsuffix "abàç" 1]}</pre>
+	<pre>{[strsuffix "abàç" 2]}</pre>
+	<pre>{[strsuffix "abàç" 3]}</pre>
+      </body>
+    </xml>
+
+fun strchrs () : transaction page =
+    let
+	fun optToStr ms =
+	    case ms of
+		None => "None"
+	      | Some s => "Some \"" ^ s ^ "\""
+
+    in
+	return <xml>
+	  <body>
+	    <pre>{[optToStr (strchr "abàç" #"c")]}</pre>
+	    <pre>{[optToStr (strchr "abàç" #"a")]}</pre>
+	    <pre>{[optToStr (strchr "abàç" #"b")]}</pre>
+	    <pre>{[optToStr (strchr "abàç" (strsub "à" 0))]}</pre>
+	    <pre>{[optToStr (strchr "abàç" (strsub "ç" 0))]}</pre>
+	  </body>
+	</xml>
+    end
+
+fun strindexs () : transaction page =
+    let
+	fun optToStr ms =
+	    case ms of
+		None => "None"
+	      | Some s => "Some " ^ (show s)
+
+    in
+	return <xml>
+	  <body>
+	    <pre>{[optToStr (strindex "abàç" #"c")]}</pre>
+	    <pre>{[optToStr (strindex "abàç" #"a")]}</pre>
+	    <pre>{[optToStr (strindex "abàç" #"b")]}</pre>
+	    <pre>{[optToStr (strindex "abàç" (strsub "à" 0))]}</pre>
+	    <pre>{[optToStr (strindex "abàç" (strsub "ç" 0))]}</pre>
+	  </body>
+	</xml>
+    end
+
+fun strsindexs () : transaction page =
+    let
+	fun optToStr ms =
+	    case ms of
+		None => "None"
+	      | Some s => "Some " ^ (show s)
+
+    in
+	return <xml>
+	  <body>
+	    <pre>{[optToStr (strsindex "abàç" "")]}</pre>
+	    <pre>{[optToStr (strsindex "abàç" "abàç")]}</pre>
+	    <pre>{[optToStr (strsindex "abàç" "abàc")]}</pre>
+	    <pre>{[optToStr (strsindex "abàç" "bàç")]}</pre>
+	    <pre>{[optToStr (strsindex "abàç" "bàc")]}</pre>
+	    <pre>{[optToStr (strsindex "abàç" "àç")]}</pre>
+	    <pre>{[optToStr (strsindex "abàç" "àc")]}</pre>
+	    <pre>{[optToStr (strsindex "abàç" "ac")]}</pre>
+	    <pre>{[optToStr (strsindex "abàç" "ç")]}</pre>
+	  </body>
+	</xml>
+    end
+	
+fun strcspns () : transaction page =
+    return <xml>
+      <body>
+	<pre>{[strcspn "abàç" ""]}</pre>
+	<pre>{[strcspn "abàç" "abàç"]}</pre>
+	<pre>{[strcspn "abàç" "a"]}</pre>
+	<pre>{[strcspn "abàç" "bàç"]}</pre>
+	<pre>{[strcspn "abàç" "àç"]}</pre>
+	<pre>{[strcspn "abàç" "ç"]}</pre>
+      </body>
+      </xml>
+
+fun str1s () : transaction page = return <xml>
+  <body>
+    <pre>{[str1 #"a"]}</pre>
+    <pre>{[str1 (strsub "à" 0)]}</pre>
+    <pre>{[str1 (strsub "aá" 1)]}</pre>
+  </body>
+  </xml>
+
+fun isalnums () : transaction page = return <xml>
+  <body>
+    <pre>{[isalnum #"a"]}</pre>
+    <pre>{[isalnum (strsub "à" 0)]}</pre>
+    <pre>{[isalnum #"A"]}</pre>
+    <pre>{[isalnum (strsub "À" 0)]}</pre>
+    <pre>{[isalnum #"1"]}</pre>
+    <pre>{[not (isalnum #"!")]}</pre>
+    <pre>{[not (isalnum #"#")]}</pre>
+    <pre>{[not (isalnum #" ")]}</pre>
+  </body>
+</xml>
+
+fun isalphas () : transaction page = return <xml>
+  <body>
+    <pre>{[isalpha #"a"]}</pre>
+    <pre>{[isalpha (strsub "à" 0)]}</pre>
+    <pre>{[isalpha #"A"]}</pre>
+    <pre>{[isalpha (strsub "À" 0)]}</pre>
+    <pre>{[not (isalpha #"1")]}</pre>
+    <pre>{[not (isalpha #"!")]}</pre>
+    <pre>{[not (isalpha #"#")]}</pre>
+    <pre>{[not (isalpha #" ")]}</pre>
+  </body>
+</xml>
+
+fun isblanks () : transaction page = 
+    return <xml>
+      <body>
+	<pre>{[not (isblank #"a")]}</pre>
+	<pre>{[not (isblank (strsub "à" 0))]}</pre>
+	<pre>{[not (isblank #"A")]}</pre>
+	<pre>{[not (isblank (strsub "À" 0))]}</pre>
+	<pre>{[not (isblank #"1")]}</pre>
+	<pre>{[not (isblank #"!")]}</pre>
+	<pre>{[not (isblank #"#")]}</pre>
+	<pre>{[isblank #" "]}</pre>
+	<pre>{[isblank #"\t"]}</pre>
+	<pre>{[not (isblank #"\n")]}</pre>
+      </body>
+    </xml>
+
+fun iscntrls () : transaction page =
+    return <xml>
+      <body>
+	<pre>{[not (iscntrl #"a")]}</pre>
+	<pre>{[not (iscntrl (strsub "à" 0))]}</pre>
+	<pre>{[not (iscntrl #"A")]}</pre>
+	<pre>{[not (iscntrl (strsub "À" 0))]}</pre>
+	<pre>{[not (iscntrl #"1")]}</pre>
+	<pre>{[not (iscntrl #"!")]}</pre>
+	<pre>{[not (iscntrl #"#")]}</pre>
+	<pre>{[not (iscntrl #" ")]}</pre>
+	<pre>{[iscntrl #"\t"]}</pre>
+	<pre>{[iscntrl #"\n"]}</pre>
+      </body>
+      </xml>
+
+fun isdigits () : transaction page =
+    return <xml>
+      <body>
+	<pre>{[not (isdigit #"a")]}</pre>
+	<pre>{[not (isdigit (strsub "à" 0))]}</pre>
+	<pre>{[not (isdigit #"A")]}</pre>
+	<pre>{[not (isdigit (strsub "À" 0))]}</pre>
+	<pre>{[isdigit #"1"]}</pre>
+	<pre>{[not (isdigit #"!")]}</pre>
+	<pre>{[not (isdigit #"#")]}</pre>
+	<pre>{[not (isdigit #" ")]}</pre>
+	<pre>{[not (isdigit #"\t")]}</pre>
+	<pre>{[not (isdigit #"\n")]}</pre>
+      </body>
+      </xml>
+
+fun isgraphs () : transaction page =
+    return <xml>
+      <body>
+	<pre>{[isgraph #"a"]}</pre>
+	<pre>{[isgraph (strsub "à" 0)]}</pre>
+	<pre>{[isgraph #"A"]}</pre>
+	<pre>{[isgraph (strsub "À" 0)]}</pre>
+	<pre>{[isgraph #"1"]}</pre>
+	<pre>{[isgraph #"!"]}</pre>
+	<pre>{[isgraph #"#"]}</pre>
+	<pre>{[not (isgraph #" ")]}</pre>
+	<pre>{[not (isgraph #"\t")]}</pre>
+	<pre>{[not (isdigit #"\n")]}</pre>
+      </body>
+      </xml>
+
+fun islowers () : transaction page =
+    return <xml>
+      <body>
+	<pre>{[islower #"a"]}</pre>
+	<pre>{[islower (strsub "à" 0)]}</pre>
+	<pre>{[not (islower #"A")]}</pre>
+	<pre>{[not (islower (strsub "À" 0))]}</pre>
+	<pre>{[not (islower #"1")]}</pre>
+	<pre>{[not (islower #"!")]}</pre>
+	<pre>{[not (islower #"#")]}</pre>
+	<pre>{[not (islower #" ")]}</pre>
+	<pre>{[not (islower #"\t")]}</pre>
+	<pre>{[not (islower #"\n")]}</pre>
+      </body>
+      </xml>
+
+fun isprints () : transaction page =
+    return <xml>
+      <body>
+	<pre>{[isprint #"a"]}</pre>
+	<pre>{[isprint (strsub "à" 0)]}</pre>
+	<pre>{[isprint #"A"]}</pre>
+	<pre>{[isprint (strsub "À" 0)]}</pre>
+	<pre>{[isprint #"1"]}</pre>
+	<pre>{[isprint #"!"]}</pre>
+	<pre>{[isprint #"#"]}</pre>
+	<pre>{[isprint #" "]}</pre>
+	<pre>{[not (isprint #"\t")]}</pre>
+	<pre>{[not (isprint #"\n")]}</pre>
+      </body>
+      </xml>
+
+fun ispuncts () : transaction page =
+    return <xml>
+      <body>
+	<pre>{[not (ispunct #"a")]}</pre>
+	<pre>{[not (ispunct (strsub "à" 0))]}</pre>
+	<pre>{[not (ispunct #"A")]}</pre>
+	<pre>{[not (ispunct (strsub "À" 0))]}</pre>
+	<pre>{[not (ispunct #"1")]}</pre>
+	<pre>{[ispunct #"!"]}</pre>
+	<pre>{[ispunct #"#"]}</pre>
+	<pre>{[not (ispunct #" ")]}</pre>
+	<pre>{[not (isprint #"\t")]}</pre>
+	<pre>{[not (isprint #"\n")]}</pre>
+      </body>
+      </xml>
+
+fun isspaces () : transaction page =
+    return <xml>
+      <body>
+	<pre>{[not (isspace #"a")]}</pre>
+	<pre>{[not (isspace (strsub "à" 0))]}</pre>
+	<pre>{[not (isspace #"A")]}</pre>
+	<pre>{[not (isspace (strsub "À" 0))]}</pre>
+	<pre>{[not (isspace #"1")]}</pre>
+	<pre>{[not (isspace #"!")]}</pre>
+	<pre>{[not (isspace #"#")]}</pre>
+	<pre>{[isspace #" "]}</pre>
+	<pre>{[isspace #"\t"]}</pre>
+	<pre>{[isspace #"\n"]}</pre>
+      </body>
+    </xml>
+    
+fun isuppers () : transaction page =
+    return <xml>
+      <body>
+	<pre>{[not (isupper #"a")]}</pre>
+	<pre>{[not (isupper (strsub "à" 0))]}</pre>
+	<pre>{[isupper #"A"]}</pre>
+	<pre>{[isupper (strsub "À" 0)]}</pre>
+	<pre>{[not (isupper #"1")]}</pre>
+	<pre>{[not (isupper #"!")]}</pre>
+	<pre>{[not (isupper #"#")]}</pre>
+	<pre>{[not (isupper #" ")]}</pre>
+	<pre>{[not (isupper #"\t")]}</pre>
+	<pre>{[not (isupper #"\n")]}</pre>
+      </body>
+      </xml>
+
+fun isxdigits () : transaction page =
+    return <xml>
+      <body>
+	<pre>{[isxdigit #"a"]}</pre>
+	<pre>{[not (isxdigit (strsub "à" 0))]}</pre>
+	<pre>{[isxdigit #"A"]}</pre>
+	<pre>{[not (isxdigit (strsub "À" 0))]}</pre>
+	<pre>{[isxdigit #"1"]}</pre>
+	<pre>{[not (isxdigit #"!")]}</pre>
+	<pre>{[not (isxdigit #"#")]}</pre>
+	<pre>{[not (isxdigit #" ")]}</pre>
+	<pre>{[not (isxdigit #"\t")]}</pre>
+	<pre>{[not (isxdigit #"\n")]}</pre>
+      </body>
+      </xml>
+
+fun tolowers () : transaction page =
+    return <xml>
+      <body>
+	<pre>{[tolower #"A" = #"a"]}</pre>
+	<pre>{[tolower #"a" = #"a"]}</pre>
+	<pre>{[tolower (strsub "á" 0) = (strsub "á" 0)]}</pre>
+	<pre>{[tolower (strsub "Á" 0) = (strsub "á" 0)]}</pre>
+	<pre>{[tolower #"1" = #"1"]}</pre>
+      </body>
+    </xml>
+    
+fun touppers () : transaction page =
+    return <xml>
+      <body>
+	<pre>{[toupper #"A" = #"A"]}</pre>
+	<pre>{[toupper #"a" = #"A"]}</pre>
+	<pre>{[toupper (strsub "á" 0) = (strsub "Á" 0)]}</pre>
+	<pre>{[toupper (strsub "Á" 0) = (strsub "Á" 0)]}</pre>
+	<pre>{[toupper #"1" = #"1"]}</pre>
+      </body>
+      </xml>
+
+fun ord_and_chrs () : transaction page =
+    return <xml>
+      <body>
+	<pre>{[chr (ord #"A") = #"A"]}</pre>
+	<pre>{[chr (ord #"a") = #"a"]}</pre>
+	<pre>{[chr (ord (strsub "á" 0)) = (strsub "á" 0)]}</pre>
+	<pre>{[chr (ord (strsub "Á" 0)) = (strsub "Á" 0)]}</pre>
+	<pre>{[chr (ord #"1") = #"1"]}</pre>
+	<pre>{[chr (ord #"\n") = #"\n"]}</pre>
+	<pre>{[chr (ord (strsub "が" 0)) = (strsub "が" 0)]}</pre>
+	<pre>{[chr (ord (strsub "漢" 0)) = (strsub "漢" 0)]}</pre>
+	<pre>{[chr (ord (strsub "カ" 0)) = (strsub "カ" 0)]}</pre>
+      </body>
+      </xml>
+
+table t : { Id : int, Text : string }
+
+
+fun test_db () : transaction page =
+    dml (INSERT INTO t (Id, Text) VALUES({[1]}, {["abc"]}));
+    t1 <- oneRow (SELECT t.Text FROM t WHERE t.Id = 1);
+
+    dml (INSERT INTO t (Id, Text) VALUES({[2]}, {["çãó"]}));
+    t2 <- oneRow (SELECT t.Text FROM t WHERE t.Id = 2);
+
+    dml (INSERT INTO t (Id, Text) VALUES({[3]}, {["が"]}));
+    t3 <- oneRow (SELECT t.Text FROM t WHERE t.Id = 3);
+
+    dml (INSERT INTO t (Id, Text) VALUES({[4]}, {["漢"]}));
+    t4 <- oneRow (SELECT t.Text FROM t WHERE t.Id = 4);
+
+    dml (INSERT INTO t (Id, Text) VALUES({[5]}, {["カ"]}));
+    t5 <- oneRow (SELECT t.Text FROM t WHERE t.Id = 5);
+
+    dml (INSERT INTO t (Id, Text) VALUES({[6]}, {["وظيفية"]}));
+    t6 <- oneRow (SELECT t.Text FROM t WHERE t.Id = 6);
+
+    return <xml>
+      <body>
+	<pre>{[t1.T.Text]}</pre>
+	<pre>{[strlen t1.T.Text]}</pre>
+	<pre>{[t2.T.Text]}</pre>
+	<pre>{[strlen t2.T.Text]}</pre>
+	<pre>{[t3.T.Text]}</pre>
+	<pre>{[strlen t3.T.Text]}</pre>
+	<pre>{[t4.T.Text]}</pre>
+	<pre>{[strlen t4.T.Text]}</pre>
+	<pre>{[t5.T.Text]}</pre>
+	<pre>{[strlen t5.T.Text]}</pre>
+	<pre>{[t6.T.Text]}</pre>
+	<pre>{[strlen t6.T.Text]}</pre>
+      </body>
+      </xml>
diff --git a/tests/utf8.urp b/tests/utf8.urp
new file mode 100644
index 00000000..9b3067af
--- /dev/null
+++ b/tests/utf8.urp
@@ -0,0 +1,5 @@
+database dbname=utf8
+sql utf8.sql
+safeGet Utf8/test_db
+
+utf8
\ No newline at end of file