Add support for async function specifier and await keyword

.gitignore

@@ -2,22 +2,12 @@
 *.hi
 *.o
 *~
-/dist/*
-/parse.txt
-/src/Language/JavaScript/Parser/Grammar5.hs
-/src/Language/JavaScript/Parser/Lexer.info
-/unicode/uc-ll.htm
-/unicode/uc-lm.htm
-/unicode/uc-lo.htm
-/unicode/uc-lt.htm
-/unicode/uc-lu.htm
-/unicode/uc-mc.htm
-/unicode/uc-mn.htm
-/unicode/uc-nd.htm
-/unicode/uc-nl.htm
-/unicode/uc-nl.htm
-/unicode/uc-pc.htm
-cabal.sandbox.config
+dist/
+dist-newstyle/
+parse.txt
+src/Language/JavaScript/Parser/Grammar5.hs
+src/Language/JavaScript/Parser/Lexer.info
+unicode/*.htm
 
 # stack
-/.stack-work/
+.stack-work/

.travis.yml

@@ -10,7 +10,7 @@ env:
  - GHCVER=8.2.2
  - GHCVER=8.4.4
  - GHCVER=8.6.5
- - GHCVER=8.8.1
+ - GHCVER=8.8.3
 
 before_install:
  - sudo add-apt-repository -y ppa:hvr/ghc

ChangeLog.md

@@ -1,5 +1,8 @@
 # ChangeLog for `language-javascript`
 
+## 0.7.1.0 -- 2020-03-22
++ Add support `async` function specifiers and `await` keyword.
+
 ## 0.7.0.0 -- 2019-10-10
 
 + Add support for (Ryan Hendrickson):

cabal.project

@@ -0,0 +1,6 @@
+packages:
+  ./
+
+test-show-details: direct
+
+tests: True

language-javascript.cabal

@@ -1,5 +1,5 @@
 Name:                language-javascript
-Version:             0.7.0.0
+Version:             0.7.1.0
 Synopsis:            Parser for JavaScript
 Description:         Parses Javascript into an Abstract Syntax Tree (AST).  Initially intended as frontend to hjsmin.
                      .
@@ -19,7 +19,6 @@ bug-reports:         https://github.com/erikd/language-javascript/issues
 Extra-source-files:  README.md
                      ChangeLog.md
                      .ghci
-                     buildall.sh
                      test/Unicode.js
                      test/k.js
                      test/unicode.txt

src/Language/JavaScript/Parser/AST.hs

@@ -146,6 +146,7 @@ data JSStatement
     | JSForConstOf !JSAnnot !JSAnnot !JSAnnot !JSExpression !JSBinOp !JSExpression !JSAnnot !JSStatement -- ^for,lb,var,vardecl,in,expr,rb,stmt
     | JSForOf !JSAnnot !JSAnnot !JSExpression !JSBinOp !JSExpression !JSAnnot !JSStatement -- ^for,lb,expr,in,expr,rb,stmt
     | JSForVarOf !JSAnnot !JSAnnot !JSAnnot !JSExpression !JSBinOp !JSExpression !JSAnnot !JSStatement -- ^for,lb,var,vardecl,in,expr,rb,stmt
+    | JSAsyncFunction !JSAnnot !JSAnnot !JSIdent !JSAnnot !(JSCommaList JSExpression) !JSAnnot !JSBlock !JSSemi  -- ^fn,name, lb,parameter list,rb,block,autosemi
     | JSFunction !JSAnnot !JSIdent !JSAnnot !(JSCommaList JSExpression) !JSAnnot !JSBlock !JSSemi  -- ^fn,name, lb,parameter list,rb,block,autosemi
     | JSGenerator !JSAnnot !JSAnnot !JSIdent !JSAnnot !(JSCommaList JSExpression) !JSAnnot !JSBlock !JSSemi  -- ^fn,*,name, lb,parameter list,rb,block,autosemi
     | JSIf !JSAnnot !JSAnnot !JSExpression !JSAnnot !JSStatement -- ^if,(,expr,),stmt
@@ -177,6 +178,7 @@ data JSExpression
     -- | Non Terminals
     | JSArrayLiteral !JSAnnot ![JSArrayElement] !JSAnnot -- ^lb, contents, rb
     | JSAssignExpression !JSExpression !JSAssignOp !JSExpression -- ^lhs, assignop, rhs
+    | JSAwaitExpression !JSAnnot !JSExpression -- ^await, expr
     | JSCallExpression !JSExpression !JSAnnot !(JSCommaList JSExpression) !JSAnnot  -- ^expr, bl, args, rb
     | JSCallExpressionDot !JSExpression !JSAnnot !JSExpression  -- ^expr, dot, expr
     | JSCallExpressionSquare !JSExpression !JSAnnot !JSExpression !JSAnnot  -- ^expr, [, expr, ]
@@ -391,6 +393,7 @@ instance ShowStripped JSStatement where
     ss (JSForOf _ _lb x1s _i x2 _rb x3) = "JSForOf " ++ ss x1s ++ " (" ++ ss x2 ++ ") (" ++ ss x3 ++ ")"
     ss (JSForVarOf _ _lb _v x1 _i x2 _rb x3) = "JSForVarOf (" ++ ss x1 ++ ") (" ++ ss x2 ++ ") (" ++ ss x3 ++ ")"
     ss (JSFunction _ n _lb pl _rb x3 _) = "JSFunction " ++ ssid n ++ " " ++ ss pl ++ " (" ++ ss x3 ++ ")"
+    ss (JSAsyncFunction _ _ n _lb pl _rb x3 _) = "JSAsyncFunction " ++ ssid n ++ " " ++ ss pl ++ " (" ++ ss x3 ++ ")"
     ss (JSGenerator _ _ n _lb pl _rb x3 _) = "JSGenerator " ++ ssid n ++ " " ++ ss pl ++ " (" ++ ss x3 ++ ")"
     ss (JSIf _ _lb x1 _rb x2) = "JSIf (" ++ ss x1 ++ ") (" ++ ss x2 ++ ")"
     ss (JSIfElse _ _lb x1 _rb x2 _e x3) = "JSIfElse (" ++ ss x1 ++ ") (" ++ ss x2 ++ ") (" ++ ss x3 ++ ")"
@@ -412,6 +415,7 @@ instance ShowStripped JSStatement where
 instance ShowStripped JSExpression where
     ss (JSArrayLiteral _lb xs _rb) = "JSArrayLiteral " ++ ss xs
     ss (JSAssignExpression lhs op rhs) = "JSOpAssign (" ++ ss op ++ "," ++ ss lhs ++ "," ++ ss rhs ++ ")"
+    ss (JSAwaitExpression _ e) = "JSAwaitExpresson " ++ ss e
     ss (JSCallExpression ex _ xs _) = "JSCallExpression ("++ ss ex ++ ",JSArguments " ++ ss xs ++ ")"
     ss (JSCallExpressionDot ex _os xs) = "JSCallExpressionDot (" ++ ss ex ++ "," ++ ss xs ++ ")"
     ss (JSCallExpressionSquare ex _os xs _cs) = "JSCallExpressionSquare (" ++ ss ex ++ "," ++ ss xs ++ ")"

src/Language/JavaScript/Parser/Grammar7.y

@@ -86,6 +86,8 @@ import qualified Language.JavaScript.Parser.AST as AST
 
      'as'         { AsToken {} }
      'autosemi'   { AutoSemiToken {} }
+     'async'      { AsyncToken {} }
+     'await'      { AwaitToken {} }
      'break'      { BreakToken {} }
      'case'       { CaseToken {} }
      'catch'      { CatchToken {} }
@@ -331,6 +333,8 @@ OpAssign : '*='     { AST.JSTimesAssign  (mkJSAnnot $1) }
 -- TODO: make this include any reserved word too, including future ones
 IdentifierName :: { AST.JSExpression }
 IdentifierName : Identifier {$1}
+             | 'async'      { AST.JSIdentifier (mkJSAnnot $1) "async" }
+             | 'await'      { AST.JSIdentifier (mkJSAnnot $1) "await" }
              | 'break'      { AST.JSIdentifier (mkJSAnnot $1) "break" }
              | 'case'       { AST.JSIdentifier (mkJSAnnot $1) "case" }
              | 'catch'      { AST.JSIdentifier (mkJSAnnot $1) "catch" }
@@ -407,6 +411,12 @@ For : 'for' { mkJSAnnot $1 }
 Continue :: { AST.JSAnnot }
 Continue : 'continue' { mkJSAnnot $1 }
 
+Async :: { AST.JSAnnot }
+Async : 'async' { mkJSAnnot $1 }
+
+Await :: { AST.JSAnnot }
+Await : 'await' { mkJSAnnot $1 }
+
 Break :: { AST.JSAnnot }
 Break : 'break' { mkJSAnnot $1 }
 
@@ -657,6 +667,10 @@ NewExpression :: { AST.JSExpression }
 NewExpression : MemberExpression    { $1                        {- 'NewExpression1' -} }
               | New NewExpression   { AST.JSNewExpression $1 $2 {- 'NewExpression2' -} }
 
+AwaitExpression :: { AST.JSExpression }
+AwaitExpression
+  : Await Expression { AST.JSAwaitExpression $1 $2 }
+
 -- CallExpression :                                             See 11.2
 --        MemberExpression Arguments
 --        CallExpression Arguments
@@ -696,6 +710,7 @@ ArgumentList : AssignmentExpression                    { AST.JSLOne $1
 LeftHandSideExpression :: { AST.JSExpression }
 LeftHandSideExpression : NewExpression  { $1 {- 'LeftHandSideExpression1' -} }
                        | CallExpression { $1 {- 'LeftHandSideExpression12' -} }
+                       | AwaitExpression { $1 {- 'LeftHandSideExpression13' -} }
 
 -- PostfixExpression :                                          See 11.3
 --        LeftHandSideExpression
@@ -976,20 +991,23 @@ Statement : StatementNoEmpty   { $1 {- 'Statement1' -} }
           | EmptyStatement     { $1 {- 'Statement2' -} }
 
 StatementNoEmpty :: { AST.JSStatement }
-StatementNoEmpty : StatementBlock      { $1 {- 'StatementNoEmpty1' -} }
-                 | VariableStatement   { $1 {- 'StatementNoEmpty2' -} }
-                 | ExpressionStatement { $1 {- 'StatementNoEmpty4' -} }
-                 | IfStatement         { $1 {- 'StatementNoEmpty5' -} }
-                 | IterationStatement  { $1 {- 'StatementNoEmpty6' -} }
-                 | ContinueStatement   { $1 {- 'StatementNoEmpty7' -} }
-                 | BreakStatement      { $1 {- 'StatementNoEmpty8' -} }
-                 | ReturnStatement     { $1 {- 'StatementNoEmpty9' -} }
-                 | WithStatement       { $1 {- 'StatementNoEmpty10' -} }
-                 | LabelledStatement   { $1 {- 'StatementNoEmpty11' -} }
-                 | SwitchStatement     { $1 {- 'StatementNoEmpty12' -} }
-                 | ThrowStatement      { $1 {- 'StatementNoEmpty13' -} }
-                 | TryStatement        { $1 {- 'StatementNoEmpty14' -} }
-                 | DebuggerStatement   { $1 {- 'StatementNoEmpty15' -} }
+StatementNoEmpty
+  : IfStatement             { $1 {- 'StatementNoEmpty5' -} }
+  | ContinueStatement       { $1 {- 'StatementNoEmpty7' -} }
+  | BreakStatement          { $1 {- 'StatementNoEmpty8' -} }
+  | ReturnStatement         { $1 {- 'StatementNoEmpty9' -} }
+  | WithStatement           { $1 {- 'StatementNoEmpty10' -} }
+  | LabelledStatement       { $1 {- 'StatementNoEmpty11' -} }
+  | SwitchStatement         { $1 {- 'StatementNoEmpty12' -} }
+  | ThrowStatement          { $1 {- 'StatementNoEmpty13' -} }
+  | TryStatement            { $1 {- 'StatementNoEmpty14' -} }
+  | StatementBlock          { $1 {- 'StatementNoEmpty1' -} }
+  | VariableStatement       { $1 {- 'StatementNoEmpty2' -} }
+  | IterationStatement      { $1 {- 'StatementNoEmpty6' -} }
+  | ExpressionStatement     { $1 {- 'StatementNoEmpty4' -} }
+  | AsyncFunctionStatement  { $1 {- 'StatementNoEmpty15' -} }
+  | DebuggerStatement       { $1 {- 'StatementNoEmpty15' -} }
+
 
 
 StatementBlock :: { AST.JSStatement }
@@ -1208,7 +1226,10 @@ DebuggerStatement : 'debugger' MaybeSemi { AST.JSExpressionStatement (AST.JSLite
 -- FunctionDeclaration :                                                      See clause 13
 --        function Identifier ( FormalParameterListopt ) { FunctionBody }
 FunctionDeclaration :: { AST.JSStatement }
-FunctionDeclaration : NamedFunctionExpression MaybeSemi  {  expressionToStatement $1 $2              {- 'FunctionDeclaration1' -} }
+FunctionDeclaration : NamedFunctionExpression MaybeSemi { expressionToStatement $1 $2                {- 'FunctionDeclaration1' -} }
+
+AsyncFunctionStatement :: { AST.JSStatement }
+AsyncFunctionStatement : Async NamedFunctionExpression MaybeSemi {  expressionToAsyncFunction $1 $2 $3  {- 'AsyncFunctionStatement1' -} }
 
 -- FunctionExpression :                                                       See clause 13
 --        function Identifieropt ( FormalParameterListopt ) { FunctionBody }
@@ -1499,6 +1520,9 @@ expressionToStatement (AST.JSMemberExpression e l a r) s = AST.JSMethodCall e l
 expressionToStatement (AST.JSClassExpression a b@(AST.JSIdentName{}) c d e f) s = AST.JSClass a b c d e f s
 expressionToStatement exp s = AST.JSExpressionStatement exp s
 
+expressionToAsyncFunction :: AST.JSAnnot -> AST.JSExpression -> AST.JSSemi -> AST.JSStatement
+expressionToAsyncFunction aa (AST.JSFunctionExpression a b@(AST.JSIdentName{}) c d e f) s = AST.JSAsyncFunction aa a b c d e f s
+expressionToAsyncFunction _aa _exp _s = error "Bad async function."
 
 mkJSCallExpression :: AST.JSExpression -> JSArguments -> AST.JSExpression
 mkJSCallExpression e (JSArguments l arglist r) = AST.JSCallExpression e l arglist r

src/Language/JavaScript/Parser/Lexer.x

@@ -529,7 +529,9 @@ keywords = Map.fromList keywordNames
 
 keywordNames :: [(String, TokenPosn -> String -> [CommentAnnotation] -> Token)]
 keywordNames =
-    [ ( "break", BreakToken )
+    [ ( "async", AsyncToken )
+    , ( "await", AwaitToken )
+    , ( "break", BreakToken )
     , ( "case", CaseToken )
     , ( "catch", CatchToken )
 

src/Language/JavaScript/Parser/Token.hs

@@ -56,6 +56,8 @@ data Token
     -- ^ Literal: Regular Expression
 
     -- Keywords
+    | AsyncToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation]  }
+    | AwaitToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation]  }
     | BreakToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation]  }
     | CaseToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation]  }
     | CatchToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation]  }

src/Language/JavaScript/Pretty/Printer.hs

@@ -1,5 +1,5 @@
 
-{-# LANGUAGE FlexibleInstances, NoOverloadedStrings, TypeSynonymInstances #-}
+{-# LANGUAGE CPP, FlexibleInstances, NoOverloadedStrings, TypeSynonymInstances #-}
 
 module Language.JavaScript.Pretty.Printer
     ( -- * Printing
@@ -10,8 +10,10 @@ module Language.JavaScript.Pretty.Printer
 
 import Blaze.ByteString.Builder (Builder, toLazyByteString)
 import Data.List
+#if ! MIN_VERSION_base(4,13,0)
 import Data.Monoid (mempty)
 import Data.Semigroup ((<>))
+#endif
 import Data.Text.Lazy (Text)
 import Language.JavaScript.Parser.AST
 import Language.JavaScript.Parser.SrcLocation
@@ -74,6 +76,7 @@ instance RenderJS JSExpression where
     (|>) pacc (JSArrayLiteral         als xs ars)             = pacc |> als |> "[" |> xs |> ars |> "]"
     (|>) pacc (JSArrowExpression      xs a x)                 = pacc |> xs |> a |> "=>" |> x
     (|>) pacc (JSAssignExpression     lhs op rhs)             = pacc |> lhs |> op |> rhs
+    (|>) pacc (JSAwaitExpression      a e)                    = pacc |> a |> "await" |> e
     (|>) pacc (JSCallExpression       ex lb xs rb)            = pacc |> ex |> lb |> "(" |> xs |> rb |> ")"
     (|>) pacc (JSCallExpressionDot    ex os xs)               = pacc |> ex |> os |> "." |> xs
     (|>) pacc (JSCallExpressionSquare ex als xs ars)          = pacc |> ex |> als |> "[" |> xs |> ars |> "]"
@@ -242,6 +245,7 @@ instance RenderJS JSStatement where
     (|>) pacc (JSForConstOf af alb v x1 i x2 arb x3)         = pacc |> af |> "for" |> alb |> "(" |> "const" |> v |> x1 |> i |> x2 |> arb |> ")" |> x3
     (|>) pacc (JSForOf af alb x1s i x2 arb x3)             = pacc |> af |> "for" |> alb |> "(" |> x1s |> i |> x2 |> arb |> ")" |> x3
     (|>) pacc (JSForVarOf af alb v x1 i x2 arb x3)         = pacc |> af |> "for" |> alb |> "(" |> "var" |> v |> x1 |> i |> x2 |> arb |> ")" |> x3
+    (|>) pacc (JSAsyncFunction aa af n alb x2s arb x3 s)   = pacc |> aa |> "async" |> af |> "function" |> n |> alb |> "(" |> x2s |> arb |> ")" |> x3 |> s
     (|>) pacc (JSFunction af n alb x2s arb x3 s)           = pacc |> af |> "function" |> n |> alb |> "(" |> x2s |> arb |> ")" |> x3 |> s
     (|>) pacc (JSGenerator af as n alb x2s arb x3 s)       = pacc |> af |> "function" |> as |> "*" |> n |> alb |> "(" |> x2s |> arb |> ")" |> x3 |> s
     (|>) pacc (JSIf annot alb x1 arb x2s)                  = pacc |> annot |> "if" |> alb |> "(" |> x1 |> arb |> ")" |> x2s