Merge pull request #82 from berdario/unicode-categories

Use Unicode categories for lexing JS identifiers

Author: achudnov

language-ecmascript.cabal

@@ -17,7 +17,7 @@ Category:       Language
 Build-Type:     Simple
 Synopsis:       JavaScript parser and pretty-printer library
 Description:
-  Tools for working with ECMAScript 3 (popularly known as JavaScript). 
+  Tools for working with ECMAScript 3 (popularly known as JavaScript).
   Includes a parser, pretty-printer, tools for working with source tree
   annotations and an arbitrary instance. See CHANGELOG for a summary of
   changes. The package follows the Haskell Package Versioning Policy since version 0.17.0.1.
@@ -45,12 +45,13 @@ Library
     QuickCheck >= 2.5 && < 3,
     template-haskell >= 2.7 && < 3,
     Diff == 0.3.*,
-    testing-feat >= 0.4.0.2 && < 0.5
+    testing-feat >= 0.4.0.2 && < 0.5,
+    charset >= 0.3
   ghc-options:
     -fwarn-incomplete-patterns
   Exposed-Modules:
-    Language.ECMAScript3 
-    Language.ECMAScript3.Lexer 
+    Language.ECMAScript3
+    Language.ECMAScript3.Lexer
     Language.ECMAScript3.Parser
     Language.ECMAScript3.PrettyPrint
     Language.ECMAScript3.Syntax

src/Language/ECMAScript3/Lexer.hs

@@ -13,6 +13,10 @@ module Language.ECMAScript3.Lexer(lexeme,identifier,reserved,operator,reservedOp
                                  ,hexIntLit,decIntLit, decDigits, decDigitsOpt, exponentPart, decLit) where
 
 import Prelude hiding (lex)
+import Data.Char
+import Data.Monoid ((<>), mconcat)
+import qualified Data.CharSet                  as Set
+import qualified Data.CharSet.Unicode.Category as Set
 import Text.Parsec
 import qualified Text.Parsec.Token as T
 import Language.ECMAScript3.Parser.State
@@ -21,8 +25,33 @@ import Control.Monad.Identity
 import Control.Applicative ((<$>), (<*>))
 import Data.Maybe (isNothing)
 
+identifierStartCharSet :: Set.CharSet
+identifierStartCharSet =
+  mconcat
+    [ Set.fromDistinctAscList "$_"
+    , Set.lowercaseLetter
+    , Set.uppercaseLetter
+    , Set.titlecaseLetter
+    , Set.modifierLetter
+    , Set.otherLetter
+    , Set.letterNumber
+    ]
+
+identifierRestCharSet :: Set.CharSet
+identifierRestCharSet =
+  identifierStartCharSet
+    <> mconcat
+         [ Set.nonSpacingMark
+         , Set.spacingCombiningMark
+         , Set.decimalNumber
+         , Set.connectorPunctuation
+         ]
+
 identifierStart :: Stream s Identity Char => Parser s Char
-identifierStart = letter <|> oneOf "$_"
+identifierStart = satisfy (flip Set.member identifierStartCharSet) <?> "letter, '$', '_'"
+
+identifierRest :: Stream s Identity Char => Parser s Char
+identifierRest = satisfy (flip Set.member identifierRestCharSet) <?> "letter, digits, '$', '_' ..."
 
 javascriptDef :: Stream s Identity Char =>T.GenLanguageDef s ParserState Identity
 javascriptDef =
@@ -31,7 +60,7 @@ javascriptDef =
                 "//"
                 False -- no nested comments
                 identifierStart
-                (alphaNum <|> oneOf "$_") -- identifier rest
+                identifierRest
                 (oneOf "{}<>()~.,?:|&^=!+-*/%!") -- operator start
                 (oneOf "=<>|&+") -- operator rest
                 ["break", "case", "catch", "const", "continue", "debugger",