@@ -21,28 +21,30 @@ module Text.Parsing.Parser.Token
21
21
)
22
22
where
23
23
24
- import Data.Array as Array
25
- import Data.Char.Unicode as Unicode
26
- import Data.List as List
24
+ import Prelude hiding (when ,between )
25
+
27
26
import Control.Lazy (fix )
28
- import Control.Monad.State (modify , gets )
27
+ import Control.Monad.State (gets , modify_ )
29
28
import Control.MonadPlus (guard , (<|>))
29
+ import Data.Array as Array
30
30
import Data.Char (fromCharCode , toCharCode )
31
31
import Data.Char.Unicode (digitToInt , isAlpha , isAlphaNum , isDigit , isHexDigit , isOctDigit , isSpace , isUpper )
32
+ import Data.Char.Unicode as Unicode
32
33
import Data.Either (Either (..))
33
34
import Data.Foldable (foldl , foldr )
34
35
import Data.Identity (Identity )
35
36
import Data.Int (toNumber )
36
37
import Data.List (List (..))
38
+ import Data.List as List
37
39
import Data.Maybe (Maybe (..), maybe )
38
- import Data.String (toCharArray , null , toLower , fromCharArray , singleton , uncons )
40
+ import Data.String (null , toLower )
41
+ import Data.String.CodeUnits as SCU
39
42
import Data.Tuple (Tuple (..))
40
43
import Math (pow )
41
44
import Text.Parsing.Parser (ParseState (..), ParserT , fail )
42
45
import Text.Parsing.Parser.Combinators (skipMany1 , try , tryRethrow , skipMany , notFollowedBy , option , choice , between , sepBy1 , sepBy , (<?>), (<??>))
43
46
import Text.Parsing.Parser.Pos (Position )
44
47
import Text.Parsing.Parser.String (satisfy , oneOf , noneOf , string , char )
45
- import Prelude hiding (when ,between )
46
48
47
49
-- | Create a parser which Returns the first token in the stream.
48
50
token :: forall m a . Monad m => (a -> Position ) -> ParserT (List a ) m a
@@ -51,7 +53,7 @@ token tokpos = do
51
53
case List .uncons input of
52
54
Nothing -> fail " Unexpected EOF"
53
55
Just { head, tail } -> do
54
- modify \(ParseState _ position _) ->
56
+ modify_ \(ParseState _ position _) ->
55
57
ParseState tail (tokpos head) true
56
58
pure head
57
59
@@ -397,7 +399,7 @@ makeTokenParser (LanguageDef languageDef)
397
399
go :: ParserT String m String
398
400
go = do
399
401
maybeChars <- between (char ' "' ) (char ' "' <?> " end of string" ) (List .many stringChar)
400
- pure $ fromCharArray $ List .toUnfoldable $ foldr folder Nil maybeChars
402
+ pure $ SCU . fromCharArray $ List .toUnfoldable $ foldr folder Nil maybeChars
401
403
402
404
folder :: Maybe Char -> List Char -> List Char
403
405
folder Nothing chars = chars
@@ -432,7 +434,9 @@ makeTokenParser (LanguageDef languageDef)
432
434
charControl = do
433
435
_ <- char ' ^'
434
436
code <- upper
435
- pure <<< fromCharCode $ toCharCode code - toCharCode ' A' + 1
437
+ case fromCharCode (toCharCode code - toCharCode ' A' + 1 ) of
438
+ Just c -> pure c
439
+ Nothing -> fail " invalid character code (should not happen)"
436
440
437
441
charNum :: ParserT String m Char
438
442
charNum = do
@@ -441,7 +445,9 @@ makeTokenParser (LanguageDef languageDef)
441
445
<|> ( char ' x' *> number 16 hexDigit )
442
446
if code > 0x10FFFF
443
447
then fail " invalid escape sequence"
444
- else pure $ fromCharCode code
448
+ else case fromCharCode code of
449
+ Just c -> pure c
450
+ Nothing -> fail " invalid character code (should not happen)"
445
451
446
452
charEsc :: ParserT String m Char
447
453
charEsc = choice (map parseEsc escMap)
@@ -567,8 +573,8 @@ makeTokenParser (LanguageDef languageDef)
567
573
568
574
sign :: forall a . (Ring a ) => ParserT String m (a -> a )
569
575
sign = (char ' -' $> negate)
570
- <|> (char ' +' $> id )
571
- <|> pure id
576
+ <|> (char ' +' $> identity )
577
+ <|> pure identity
572
578
573
579
nat :: ParserT String m Int
574
580
nat = zeroNumber <|> decimal
@@ -624,7 +630,7 @@ makeTokenParser (LanguageDef languageDef)
624
630
go = do
625
631
c <- languageDef.opStart
626
632
cs <- Array .many languageDef.opLetter
627
- pure $ singleton c <> fromCharArray cs
633
+ pure $ SCU . singleton c <> SCU . fromCharArray cs
628
634
629
635
isReservedOp :: String -> Boolean
630
636
isReservedOp name = isReserved (Array .sort languageDef.reservedOpNames) name
@@ -645,7 +651,7 @@ makeTokenParser (LanguageDef languageDef)
645
651
| otherwise = walk name $> name
646
652
where
647
653
walk :: String -> ParserT String m Unit
648
- walk name' = case uncons name' of
654
+ walk name' = case SCU . uncons name' of
649
655
Nothing -> pure unit
650
656
Just { head: c, tail: cs } -> (caseChar c <?> msg) *> walk cs
651
657
@@ -675,7 +681,7 @@ makeTokenParser (LanguageDef languageDef)
675
681
go = do
676
682
c <- languageDef.identStart
677
683
cs <- Array .many languageDef.identLetter
678
- pure $ singleton c <> fromCharArray cs
684
+ pure $ SCU . singleton c <> SCU . fromCharArray cs
679
685
680
686
681
687
-- ---------------------------------------------------------
@@ -757,7 +763,7 @@ inCommentMulti langDef@(LanguageDef languageDef) =
757
763
<?> " end of comment"
758
764
where
759
765
startEnd :: Array Char
760
- startEnd = toCharArray languageDef.commentEnd <> toCharArray languageDef.commentStart
766
+ startEnd = SCU . toCharArray languageDef.commentEnd <> SCU . toCharArray languageDef.commentStart
761
767
762
768
inCommentSingle :: forall m . Monad m => GenLanguageDef String m -> ParserT String m Unit
763
769
inCommentSingle (LanguageDef languageDef) =
@@ -767,7 +773,7 @@ inCommentSingle (LanguageDef languageDef) =
767
773
<?> " end of comment"
768
774
where
769
775
startEnd :: Array Char
770
- startEnd = toCharArray languageDef.commentEnd <> toCharArray languageDef.commentStart
776
+ startEnd = SCU . toCharArray languageDef.commentEnd <> SCU . toCharArray languageDef.commentStart
771
777
772
778
-- -----------------------------------------------------------------------
773
779
-- Helper functions that should maybe go in Text.Parsing.Parser.String --
0 commit comments