Merge remote-tracking branch 'origin/master' into bpo_1635741_1

* origin/master: (1373 commits)
  bpo-1635741: Port mashal module to multi-phase init (python#22149)
  bpo-1635741: Port _string module to multi-phase init (pythonGH-22148)
  bpo-1635741: Convert _sha256 types to heap types (pythonGH-22134)
  bpo-1635741: Port the termios to multi-phase init (PEP 489) (pythonGH-22139)
  bpo-41732: add iterator to memoryview (pythonGH-22119)
  bpo-40744: Drop support for SQLite pre 3.7.3 (pythonGH-20909)
  bpo-41316: Make tarfile follow specs for FNAME (pythonGH-21511)
  bpo-41720: Add "return NotImplemented" in turtle.Vec2D.__rmul__(). (pythonGH-22092)
  bpo-1635741 port _curses_panel to multi-phase init (PEP 489) (pythonGH-21986)
  bpo-1635741: Port _overlapped module to multi-phase init (pythonGH-22051)
  bpo-1635741: Port _opcode module to multi-phase init (PEP 489) (pythonGH-22050)
  bpo-1635741 port zlib module to multi-phase init (pythonGH-21995)
  [doc] Add link to Generic in typing (pythonGH-22125)
  bpo-41513: Expand comments and add references for a better understanding (pythonGH-22123)
  bpo-1635741: Port _sha1, _sha512, _md5 to multiphase init (pythonGH-21818)
  closes bpo-41723: Fix an error in the py_compile documentation. (pythonGH-22110)
  [doc] Fix padding in some typing definitions (pythonGH-22114)
  Fix documented Python version for venv --upgrade-deps (pythonGH-22113)
  bpo-40318: Migrate to SQLite3 trace v2 API (pythonGH-19581)
  bpo-41687: Fix sendfile implementation to work with Solaris (python#22040)
  ...

Author: shihai1991

.azure-pipelines/ci.yml

@@ -1,18 +1,14 @@
 variables:
-  manylinux: false
   coverage: false
 
-resources:
-  containers:
-  - container: manylinux1
-    image: pyca/cryptography-manylinux1:x86_64
+trigger: ['master', '3.9', '3.8', '3.7']
 
 jobs:
 - job: Prebuild
   displayName: Pre-build checks
 
   pool:
-    vmImage: ubuntu-16.04
+    vmImage: ubuntu-18.04
 
   steps:
   - template: ./prebuild-checks.yml
@@ -24,7 +20,7 @@ jobs:
   condition: and(succeeded(), eq(dependencies.Prebuild.outputs['docs.run'], 'true'))
 
   pool:
-    vmImage: ubuntu-16.04
+    vmImage: ubuntu-18.04
 
   steps:
   - template: ./docs-steps.yml
@@ -56,50 +52,19 @@ jobs:
   condition: and(succeeded(), eq(dependencies.Prebuild.outputs['tests.run'], 'true'))
 
   pool:
-    vmImage: ubuntu-16.04
+    vmImage: ubuntu-18.04
 
   variables:
     testRunTitle: '$(build.sourceBranchName)-linux'
     testRunPlatform: linux
-    openssl_version: 1.1.1d
+    openssl_version: 1.1.1g
 
   steps:
   - template: ./posix-steps.yml
     parameters:
       dependencies: apt
 
 
-- job: ManyLinux1_CI_Tests
-  displayName: ManyLinux1 CI Tests
-  dependsOn: Prebuild
-  condition: |
-    and(
-        and(
-            succeeded(),
-            eq(variables['manylinux'], 'true')
-        ),
-        eq(dependencies.Prebuild.outputs['tests.run'], 'true')
-    )
-
-  pool:
-    vmImage: ubuntu-16.04
-
-  container: manylinux1
-
-  variables:
-    testRunTitle: '$(build.sourceBranchName)-manylinux1'
-    testRunPlatform: manylinux1
-    openssl_version: ''
-
-  steps:
-  - template: ./posix-steps.yml
-    parameters:
-      dependencies: yum
-      sudo_dependencies: ''
-      xvfb: false
-      patchcheck: false
-
-
 - job: Ubuntu_Coverage_CI_Tests
   displayName: Ubuntu CI Tests (coverage)
   dependsOn: Prebuild
@@ -113,12 +78,12 @@ jobs:
     )
 
   pool:
-    vmImage: ubuntu-16.04
+    vmImage: ubuntu-18.04
 
   variables:
     testRunTitle: '$(Build.SourceBranchName)-linux-coverage'
     testRunPlatform: linux-coverage
-    openssl_version: 1.1.1d
+    openssl_version: 1.1.1g
 
   steps:
   - template: ./posix-steps.yml

.azure-pipelines/pr.yml

@@ -1,18 +1,14 @@
 variables:
-  manylinux: false
   coverage: false
 
-resources:
-  containers:
-  - container: manylinux1
-    image: pyca/cryptography-manylinux1:x86_64
+pr: ['master', '3.9', '3.8', '3.7']
 
 jobs:
 - job: Prebuild
   displayName: Pre-build checks
 
   pool:
-    vmImage: ubuntu-16.04
+    vmImage: ubuntu-18.04
 
   steps:
   - template: ./prebuild-checks.yml
@@ -24,7 +20,7 @@ jobs:
   condition: and(succeeded(), eq(dependencies.Prebuild.outputs['docs.run'], 'true'))
 
   pool:
-    vmImage: ubuntu-16.04
+    vmImage: ubuntu-18.04
 
   steps:
   - template: ./docs-steps.yml
@@ -56,50 +52,19 @@ jobs:
   condition: and(succeeded(), eq(dependencies.Prebuild.outputs['tests.run'], 'true'))
 
   pool:
-    vmImage: ubuntu-16.04
+    vmImage: ubuntu-18.04
 
   variables:
     testRunTitle: '$(system.pullRequest.TargetBranch)-linux'
     testRunPlatform: linux
-    openssl_version: 1.1.1d
+    openssl_version: 1.1.1g
 
   steps:
   - template: ./posix-steps.yml
     parameters:
       dependencies: apt
 
 
-- job: ManyLinux1_PR_Tests
-  displayName: ManyLinux1 PR Tests
-  dependsOn: Prebuild
-  condition: |
-    and(
-        and(
-            succeeded(),
-            eq(variables['manylinux'], 'true')
-        ),
-        eq(dependencies.Prebuild.outputs['tests.run'], 'true')
-    )
-
-  pool:
-    vmImage: ubuntu-16.04
-
-  container: manylinux1
-
-  variables:
-    testRunTitle: '$(system.pullRequest.TargetBranch)-manylinux1'
-    testRunPlatform: manylinux1
-    openssl_version: ''
-
-  steps:
-  - template: ./posix-steps.yml
-    parameters:
-      dependencies: yum
-      sudo_dependencies: ''
-      xvfb: false
-      patchcheck: false
-
-
 - job: Ubuntu_Coverage_PR_Tests
   displayName: Ubuntu PR Tests (coverage)
   dependsOn: Prebuild
@@ -113,12 +78,12 @@ jobs:
     )
 
   pool:
-    vmImage: ubuntu-16.04
+    vmImage: ubuntu-18.04
 
   variables:
     testRunTitle: '$(Build.SourceBranchName)-linux-coverage'
     testRunPlatform: linux-coverage
-    openssl_version: 1.1.1d
+    openssl_version: 1.1.1g
 
   steps:
   - template: ./posix-steps.yml

.azure-pipelines/windows-release/msi-steps.yml

@@ -1,6 +1,12 @@
 steps:
   - template: ./checkout.yml
 
+  - powershell: |
+      $d = (.\PCbuild\build.bat -V) | %{ if($_ -match '\s+(\w+):\s*(.+)\s*$') { @{$Matches[1] = $Matches[2];} }};
+      Write-Host "##vso[task.setvariable variable=SigningDescription]Python $($d.PythonVersion)"
+    displayName: 'Update signing description'
+    condition: and(succeeded(), not(variables['SigningDescription']))
+
   - task: DownloadPipelineArtifact@1
     displayName: 'Download artifact: doc'
     inputs:

.azure-pipelines/windows-release/stage-pack-msix.yml

@@ -105,9 +105,15 @@ jobs:
     clean: all
 
   steps:
-  - checkout: none
+  - template: ./checkout.yml
   - template: ./find-sdk.yml
 
+  - powershell: |
+      $d = (.\PCbuild\build.bat -V) | %{ if($_ -match '\s+(\w+):\s*(.+)\s*$') { @{$Matches[1] = $Matches[2];} }};
+      Write-Host "##vso[task.setvariable variable=SigningDescription]Python $($d.PythonVersion)"
+    displayName: 'Update signing description'
+    condition: and(succeeded(), not(variables['SigningDescription']))
+
   - task: DownloadBuildArtifacts@0
     displayName: 'Download Artifact: unsigned_msix'
     inputs:

.azure-pipelines/windows-release/stage-sign.yml

@@ -26,6 +26,12 @@ jobs:
   - template: ./checkout.yml
   - template: ./find-sdk.yml
 
+  - powershell: |
+      $d = (.\PCbuild\build.bat -V) | %{ if($_ -match '\s+(\w+):\s*(.+)\s*$') { @{$Matches[1] = $Matches[2];} }};
+      Write-Host "##vso[task.setvariable variable=SigningDescription]Python $($d.PythonVersion)"
+    displayName: 'Update signing description'
+    condition: and(succeeded(), not(variables['SigningDescription']))
+
   - powershell: |
       Write-Host "##vso[build.addbuildtag]signed"
     displayName: 'Add build tags'

.gitattributes

@@ -41,8 +41,6 @@ PC/readme.txt text eol=crlf
 
 # Generated files
 # https://github.com/github/linguist#generated-code
-Include/graminit.h          linguist-generated=true
-Python/graminit.h           linguist-generated=true
 Modules/clinic/*.h          linguist-generated=true
 Objects/clinic/*.h          linguist-generated=true
 PC/clinic/*.h               linguist-generated=true

.github/CODEOWNERS

@@ -72,8 +72,10 @@ Include/pytime.h              @pganssle @abalkin
 /Modules/gcmodule.c           @pablogsal
 /Doc/library/gc.rst           @pablogsal
 
-# Parser/Pgen
-/Parser/pgen/                 @pablogsal
+# Parser
+/Parser/                      @pablogsal @lysnikolaou
+/Tools/peg_generator/         @pablogsal @lysnikolaou
+/Lib/test/test_peg_generator/ @pablogsal @lysnikolaou
 
 # SQLite 3
 **/*sqlite*                   @berkerpeksag

.github/CONTRIBUTING.rst

@@ -8,17 +8,17 @@ Build Status
 
   + `Stable buildbots <http://buildbot.python.org/3.x.stable/>`_
 
-- 3.7
+- 3.9
 
-  + `Stable buildbots <http://buildbot.python.org/3.7.stable/>`_
+  + `Stable buildbots <http://buildbot.python.org/3.9.stable/>`_
 
-- 3.6
+- 3.8
 
-  + `Stable buildbots <http://buildbot.python.org/3.6.stable/>`_
+  + `Stable buildbots <http://buildbot.python.org/3.8.stable/>`_
 
-- 2.7
+- 3.7
 
-  + `Stable buildbots <http://buildbot.python.org/2.7.stable/>`_
+  + `Stable buildbots <http://buildbot.python.org/3.7.stable/>`_
 
 
 Thank You

.github/problem-matchers/gcc.json

@@ -0,0 +1,18 @@
+{
+    "__comment": "Taken from vscode-cpptools's Extension/package.json gcc rule",
+    "problemMatcher": [
+        {
+            "owner": "gcc-problem-matcher",
+            "pattern": [
+                {
+                    "regexp": "^(.*):(\\d+):(\\d+):\\s+(?:fatal\\s+)?(warning|error):\\s+(.*)$",
+                    "file": 1,
+                    "line": 2,
+                    "column": 3,
+                    "severity": 4,
+                    "message": 5
+                }
+            ]
+        }
+    ]
+}

.github/problem-matchers/sphinx.json

@@ -0,0 +1,40 @@
+{
+    "problemMatcher": [
+        {
+            "owner": "sphinx-problem-matcher",
+            "pattern": [
+                {
+                    "regexp": "^(.*):(\\d+):\\s+(\\w*):\\s+(.*)$",
+                    "file": 1,
+                    "line": 2,
+                    "severity": 3,
+                    "message": 4
+                }
+            ]
+        },
+        {
+            "owner": "sphinx-problem-matcher-loose",
+            "pattern": [
+                {
+                    "_comment": "A bit of a looser pattern, doesn't look for line numbers, just looks for file names relying on them to start with / and end with .rst",
+                    "regexp": "(\/.*\\.rst):\\s+(\\w*):\\s+(.*)$",
+                    "file": 1,
+                    "severity": 2,
+                    "message": 3
+                }
+            ]
+        },
+        {
+            "owner": "sphinx-problem-matcher-loose-no-severity",
+            "pattern": [
+                {
+                    "_comment": "Looks for file names ending with .rst and line numbers but without severity",
+                    "regexp": "^(.*\\.rst):(\\d+):(.*)$",
+                    "file": 1,
+                    "line": 2,
+                    "message": 3
+                }
+            ]           
+        }
+    ]
+}