From 954936dd180e34b79baca71e43d55a204dda9594 Mon Sep 17 00:00:00 2001 From: Adam Chlipala Date: Sat, 5 Nov 2011 15:05:13 -0400 Subject: Support the full set of XHTML character entities --- .hgignore | 3 + Makefile.am | 11 ++- Makefile.in | 11 ++- src/sources | 5 ++ src/urweb.lex | 20 +++-- src/utf8.sig | 32 +++++++ src/utf8.sml | 59 +++++++++++++ tests/entities.ur | 5 ++ xml/parse.sml | 77 ++++++++++++++++ xml/xhtml-lat1.ent | 196 +++++++++++++++++++++++++++++++++++++++++ xml/xhtml-special.ent | 80 +++++++++++++++++ xml/xhtml-symbol.ent | 237 ++++++++++++++++++++++++++++++++++++++++++++++++++ 12 files changed, 724 insertions(+), 12 deletions(-) create mode 100644 src/utf8.sig create mode 100644 src/utf8.sml create mode 100644 tests/entities.ur create mode 100644 xml/parse.sml create mode 100644 xml/xhtml-lat1.ent create mode 100644 xml/xhtml-special.ent create mode 100644 xml/xhtml-symbol.ent diff --git a/.hgignore b/.hgignore index cfb45564..401353b3 100644 --- a/.hgignore +++ b/.hgignore @@ -48,6 +48,9 @@ doc/*.html Makefile.coq *.vo +xml/parse +xml/entities.sml + syntax: regexp ^Makefile$ diff --git a/Makefile.am b/Makefile.am index 45e7d7e0..42e5d4d7 100644 --- a/Makefile.am +++ b/Makefile.am @@ -21,7 +21,7 @@ mlton: bin/urweb clean-local: rm -f src/*.mlton.grm.* src/*.mlton.lex.* \ - src/urweb.cm src/urweb.mlb + src/urweb.cm src/urweb.mlb xml/parse xml/entities.sml rm -rf .cm src/.cm src/urweb.cm: src/prefix.cm src/sources @@ -55,11 +55,18 @@ MLTON := mlton # MLTON += -profile $(PROFILE) #endif -bin/urweb: src/compiler.mlb src/urweb.mlb src/*.sig src/*.sml \ +bin/urweb: xml/entities.sml \ + src/compiler.mlb src/urweb.mlb src/*.sig src/*.sml \ src/urweb.mlton.lex.sml \ src/urweb.mlton.grm.sig src/urweb.mlton.grm.sml $(MLTON) -output $@ src/compiler.mlb +xml/entities.sml: xml/parse xml/xhtml-lat1.ent xml/xhtml-special.ent xml/xhtml-symbol.ent + xml/parse >xml/entities.sml + +xml/parse: xml/parse.sml + $(MLTON) xml/parse.sml + install-exec-emacs: if USE_EMACS mkdir -p $(DESTDIR)$(SITELISP) diff --git a/Makefile.in b/Makefile.in index f8a7dfd9..c758310d 100644 --- a/Makefile.in +++ b/Makefile.in @@ -750,7 +750,7 @@ mlton: bin/urweb clean-local: rm -f src/*.mlton.grm.* src/*.mlton.lex.* \ - src/urweb.cm src/urweb.mlb + src/urweb.cm src/urweb.mlb xml/parse xml/entities.sml rm -rf .cm src/.cm src/urweb.cm: src/prefix.cm src/sources @@ -782,11 +782,18 @@ src/urweb.mlton.grm.sig src/urweb.mlton.grm.sml: src/urweb.mlton.grm # MLTON += -profile $(PROFILE) #endif -bin/urweb: src/compiler.mlb src/urweb.mlb src/*.sig src/*.sml \ +bin/urweb: xml/entities.sml \ + src/compiler.mlb src/urweb.mlb src/*.sig src/*.sml \ src/urweb.mlton.lex.sml \ src/urweb.mlton.grm.sig src/urweb.mlton.grm.sml $(MLTON) -output $@ src/compiler.mlb +xml/entities.sml: xml/parse xml/xhtml-lat1.ent xml/xhtml-special.ent xml/xhtml-symbol.ent + xml/parse >xml/entities.sml + +xml/parse: xml/parse.sml + $(MLTON) xml/parse.sml + install-exec-emacs: @USE_EMACS_TRUE@ mkdir -p $(DESTDIR)$(SITELISP) @USE_EMACS_TRUE@ cp src/elisp/*.el $(DESTDIR)$(SITELISP)/ diff --git a/src/sources b/src/sources index ebc2ab13..862845d5 100644 --- a/src/sources +++ b/src/sources @@ -47,6 +47,11 @@ export.sml source.sml +utf8.sig +utf8.sml + +../xml/entities.sml + urweb.grm urweb.lex diff --git a/src/urweb.lex b/src/urweb.lex index 19f28738..a989d933 100644 --- a/src/urweb.lex +++ b/src/urweb.lex @@ -113,6 +113,14 @@ fun initialize () = (xmlTag := []; xmlString := false) +structure StringMap = BinaryMapFn(struct + type ord_key = string + val compare = String.compare + end) + +val entities = foldl (fn ((key, value), entities) => StringMap.insert (entities, key, value)) + StringMap.empty Entities.all + fun unescape loc s = let fun process (s, acc) = @@ -141,20 +149,16 @@ fun unescape loc s = let val code = Substring.string (Substring.slice (code, 1, NONE)) in - Option.map chr (Int.fromString code) + Option.map Utf8.encode (Int.fromString code) end - else case Substring.string code of - "amp" => SOME #"&" - | "lt" => SOME #"<" - | "gt" => SOME #">" - | "quot" => SOME #"\"" - | _ => NONE + else + Option.map Utf8.encode (StringMap.find (entities, Substring.string code)) in case special of NONE => (ErrorMsg.errorAt' loc ("Unsupported XML character entity " ^ Substring.string code); "") - | SOME ch => process (s, Substring.full (String.str ch) :: pre :: acc) + | SOME sp => process (s, Substring.full sp :: pre :: acc) end end end diff --git a/src/utf8.sig b/src/utf8.sig new file mode 100644 index 00000000..4198f604 --- /dev/null +++ b/src/utf8.sig @@ -0,0 +1,32 @@ +(* Copyright (c) 2011, Adam Chlipala + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * - Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * - Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * - The names of contributors may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + *) + +(* UTF-8 conversion *) + +signature UTF8 = sig + val encode : int -> string +end diff --git a/src/utf8.sml b/src/utf8.sml new file mode 100644 index 00000000..cbd2fa5c --- /dev/null +++ b/src/utf8.sml @@ -0,0 +1,59 @@ +(* Copyright (c) 2011, Adam Chlipala + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * - Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * - Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * - The names of contributors may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + *) + +(* UTF-8 conversion *) + +structure Utf8 :> UTF8 = struct + +fun byte n = str (chr (Word.toInt n)) + +fun encode n = + if n <= 0 then + raise Fail "Invalid character to UTF-8-encode" + else if n <= 0x7F then + str (chr n) + else if n <= 0x7FF then + let + val w = Word.fromInt n + val b1 = Word.orb (Word.fromInt (128 + 64), Word.>> (w, Word.fromInt 6)) + val b2 = Word.orb (Word.fromInt 128, Word.andb (w, Word.fromInt 63)) + in + byte b1 ^ byte b2 + end + else if n <= 0xFFFF then + let + val w = Word.fromInt n + val b1 = Word.orb (Word.fromInt (128 + 64 + 32), Word.>> (w, Word.fromInt 12)) + val b2 = Word.orb (Word.fromInt 128, Word.andb (Word.>> (w, Word.fromInt 6), Word.fromInt 63)) + val b3 = Word.orb (Word.fromInt 128, Word.andb (w, Word.fromInt 63)) + in + byte b1 ^ byte b2 ^ byte b3 + end + else + raise Fail "Exceeded supported range for UTF-8 characters" + +end diff --git a/tests/entities.ur b/tests/entities.ur new file mode 100644 index 00000000..8b78edbc --- /dev/null +++ b/tests/entities.ur @@ -0,0 +1,5 @@ +fun main () : transaction page = return + Hello world! & so on, © me today (8 €)
+ ♠ ♣ ♥ ♦
+ † DANGER † + diff --git a/xml/parse.sml b/xml/parse.sml new file mode 100644 index 00000000..86ff3682 --- /dev/null +++ b/xml/parse.sml @@ -0,0 +1,77 @@ +(* Copyright (c) 2011, Adam Chlipala + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * - Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * - Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * - The names of contributors may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + *) + +(* Building SML code from XML entity tables *) + +fun main () = + let + fun doFile fname = + let + val inf = TextIO.openIn fname + + fun loop () = + case TextIO.inputLine inf of + NONE => TextIO.closeIn inf + | SOME line => + if String.isPrefix " Char.isSpace ch orelse ch = #">") line of + " + let + val exp = if String.isPrefix "\"&#" exp andalso String.isSuffix ";\"" exp then + let + val middle = String.substring (exp, 3, size exp - 5) + in + if CharVector.all Char.isDigit middle then + middle + else if String.isPrefix "38;#" middle then + String.extract (middle, 4, NONE) + else + raise Fail "Bad entity expression [1]" + end + else + raise Fail "Bad entity expansion [2]" + in + print ("\t\t(\"" ^ ent ^ "\", " ^ exp ^ "),\n"); + loop () + end + | _ => raise Fail "Bad ENTITY line" + else + loop () + in + loop () + end + in + print "structure Entities = struct\n"; + print "\tval all = [\n"; + doFile "xml/xhtml-lat1.ent"; + doFile "xml/xhtml-special.ent"; + doFile "xml/xhtml-symbol.ent"; + print "\t(\"\", 0)]\n"; + print "end\n" + end + +val () = main () diff --git a/xml/xhtml-lat1.ent b/xml/xhtml-lat1.ent new file mode 100644 index 00000000..ffee223e --- /dev/null +++ b/xml/xhtml-lat1.ent @@ -0,0 +1,196 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/xml/xhtml-special.ent b/xml/xhtml-special.ent new file mode 100644 index 00000000..ca358b2f --- /dev/null +++ b/xml/xhtml-special.ent @@ -0,0 +1,80 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/xml/xhtml-symbol.ent b/xml/xhtml-symbol.ent new file mode 100644 index 00000000..63c2abfa --- /dev/null +++ b/xml/xhtml-symbol.ent @@ -0,0 +1,237 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + -- cgit v1.2.3