bpo-27494: Fix 2to3 handling of trailing comma after a generator expression

While this is a valid Python 2 and Python 3 syntax lib2to3 would choke
on it:

    set(x for x in [],)

This patch changes the grammar definition used by lib2to3 so that the
actual Python syntax is supported now and backwards compatibility is
preserved.

Author: jstasiak

Lib/lib2to3/Grammar.txt

@@ -130,8 +130,8 @@ atom: ('(' [yield_expr|testlist_gexp] ')' |
        '{' [dictsetmaker] '}' |
        '`' testlist1 '`' |
        NAME | NUMBER | STRING+ | '.' '.' '.')
-listmaker: (test|star_expr) ( comp_for | (',' (test|star_expr))* [','] )
-testlist_gexp: (test|star_expr) ( comp_for | (',' (test|star_expr))* [','] )
+listmaker: (test|star_expr) ( old_comp_for | (',' (test|star_expr))* [','] )
+testlist_gexp: (test|star_expr) ( old_comp_for | (',' (test|star_expr))* [','] )
 lambdef: 'lambda' [varargslist] ':' test
 trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME
 subscriptlist: subscript (',' subscript)* [',']
@@ -161,9 +161,28 @@ argument: ( test [comp_for] |
 	    star_expr )
 
 comp_iter: comp_for | comp_if
-comp_for: [ASYNC] 'for' exprlist 'in' testlist_safe [comp_iter]
+comp_for: [ASYNC] 'for' exprlist 'in' or_test [comp_iter]
 comp_if: 'if' old_test [comp_iter]
 
+# See note above regarding testlist_safe to see why testlist_safe is
+# necessary. Unfortunately we can't use it indiscriminately in all derivations
+# using comp_for-like pattern because testlist_safe derivation contains
+# comma which clashes with trailing comma in arglist.
+#
+# The above was an issue because the parser would not follow the correct derivation
+# when parsing syntactically valid Python code. Since testlist_safe was created
+# specifically to handle list comprehensions and generator expressions enclosed
+# with parentheses it's safe to only use it in those. That avoids the issue
+# mentioned above and we can parse code like set(x for x in [],).
+#
+# Note that the syntax supported by this set of rules is not a valid Python 3
+# syntax, hence the prefix "old".
+#
+# See the bug report: http://bugs.python.org/issue27494
+old_comp_iter: old_comp_for | old_comp_if
+old_comp_for: [ASYNC] 'for' exprlist 'in' testlist_safe [old_comp_iter]
+old_comp_if: 'if' old_test [old_comp_iter]
+
 testlist1: test (',' test)*
 
 # not used in grammar, but may appear in "node" passed from Parser to Compiler

Lib/lib2to3/fixer_util.py

@@ -101,8 +101,8 @@ def ListComp(xp, fp, it, test=None):
         test.prefix = " "
         if_leaf = Leaf(token.NAME, "if")
         if_leaf.prefix = " "
-        inner_args.append(Node(syms.comp_if, [if_leaf, test]))
-    inner = Node(syms.listmaker, [xp, Node(syms.comp_for, inner_args)])
+        inner_args.append(Node(syms.old_comp_if, [if_leaf, test]))
+    inner = Node(syms.listmaker, [xp, Node(syms.old_comp_for, inner_args)])
     return Node(syms.atom,
                        [Leaf(token.LBRACE, "["),
                         inner,
@@ -208,6 +208,7 @@ def attr_chain(obj, attr):
         next = getattr(next, attr)
 
 p0 = """for_stmt< 'for' any 'in' node=any ':' any* >
+        | old_comp_for< 'for' any 'in' node=any any* >
         | comp_for< 'for' any 'in' node=any any* >
      """
 p1 = """

Lib/lib2to3/fixes/fix_dict.py

@@ -83,6 +83,7 @@ def transform(self, node, results):
     p1 = patcomp.compile_pattern(P1)
 
     P2 = """for_stmt< 'for' any 'in' node=any ':' any* >
+            | old_comp_for< 'for' any 'in' node=any any* >
             | comp_for< 'for' any 'in' node=any any* >
          """
     p2 = patcomp.compile_pattern(P2)

Lib/lib2to3/fixes/fix_paren.py

@@ -15,7 +15,7 @@ class FixParen(fixer_base.BaseFix):
     PATTERN = """
         atom< ('[' | '(')
             (listmaker< any
-                comp_for<
+                old_comp_for<
                     'for' NAME 'in'
                     target=testlist_safe< any (',' any)+ [',']
                      >
@@ -24,7 +24,7 @@ class FixParen(fixer_base.BaseFix):
             >
             |
             testlist_gexp< any
-                comp_for<
+                old_comp_for<
                     'for' NAME 'in'
                     target=testlist_safe< any (',' any)+ [',']
                      >

Lib/lib2to3/fixes/fix_xrange.py

@@ -55,6 +55,7 @@ def transform_range(self, node, results):
     p1 = patcomp.compile_pattern(P1)
 
     P2 = """for_stmt< 'for' any 'in' node=any ':' any* >
+            | old_comp_for< 'for' any 'in' node=any any* >
             | comp_for< 'for' any 'in' node=any any* >
             | comparison< any 'in' node=any any*>
          """

Lib/lib2to3/tests/test_parser.py

@@ -459,6 +459,13 @@ def test_multiline_str_literals(self):
         self.validate(s)
 
 
+class TestGeneratorExpressions(GrammarTest):
+
+    def test_trailing_comma_after_generator_expression_argument_works(self):
+        # BPO issue 27494
+        self.validate("set(x for x in [],)")
+
+
 def diff(fn, result):
     try:
         with open('@', 'w') as f:

Misc/NEWS.d/next/Library/2017-09-26-01-43-17.bpo-27494.37QnaT.rst

@@ -0,0 +1,2 @@
+2to3 doesn't choke on a trailing comma following a generator expression
+anymore.