Merge branch 'main' into struct-name-repeats

Author: encukou

Doc/conf.py

@@ -105,6 +105,14 @@
 # Short title used e.g. for <title> HTML tags.
 html_short_title = '%s Documentation' % release
 
+# Deployment preview information, from Netlify
+# (See netlify.toml and https://docs.netlify.com/configure-builds/environment-variables/#git-metadata)
+html_context = {
+    "is_deployment_preview": os.getenv("IS_DEPLOYMENT_PREVIEW"),
+    "repository_url": os.getenv("REPOSITORY_URL"),
+    "pr_id": os.getenv("REVIEW_ID")
+}
+
 # If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
 # using the given strftime format.
 html_last_updated_fmt = '%b %d, %Y'

Doc/tools/templates/layout.html

@@ -8,6 +8,19 @@
     <a href="/3/{{ pagename }}{{ file_suffix }}">{% trans %} Python documentation for the current stable release{% endtrans %}</a>.
 </div>
 {%- endif %}
+
+{%- if is_deployment_preview %}
+<div id="deployment-preview-warning" style="padding: .5em; text-align: center; background-color: #fff2ba; color: #6a580e;">
+  <div style="float: right; margin-top: -10px; margin-left: 10px;">
+    <a href="https://www.netlify.com">
+      <img src="https://www.netlify.com/img/global/badges/netlify-color-accent.svg" alt="Deploys by Netlify" />
+    </a>
+  </div>
+  {% trans %}This is a deploy preview created from a <a href="{{ repository_url }}/pull/{{ pr_id }}">pull request</a>.
+  For authoritative documentation, see the {% endtrans %}
+  <a href="https://docs.python.org/3/{{ pagename }}{{ file_suffix }}">{% trans %} the current stable release{% endtrans %}</a>.
+</div>
+{%- endif %}
 {% endblock %}
 
 {% block rootrellink %}

Doc/using/configure.rst

@@ -24,6 +24,7 @@ Features required to build CPython:
 
 .. versionchanged:: 3.11
    C11 compiler, IEEE 754 and NaN support are now required.
+   On Windows, Visual Studio 2017 or later is required.
 
 .. versionchanged:: 3.10
    OpenSSL 1.1.1 is now required.

Lib/test/test_frame.py

@@ -2,6 +2,7 @@
 import re
 import sys
 import textwrap
+import threading
 import types
 import unittest
 import weakref
@@ -11,6 +12,7 @@
     _testcapi = None
 
 from test import support
+from test.support import threading_helper
 from test.support.script_helper import assert_python_ok
 
 
@@ -329,6 +331,46 @@ def f():
             if old_enabled:
                 gc.enable()
 
+    @support.cpython_only
+    @threading_helper.requires_working_threading()
+    def test_sneaky_frame_object_teardown(self):
+
+        class SneakyDel:
+            def __del__(self):
+                """
+                Stash a reference to the entire stack for walking later.
+
+                It may look crazy, but you'd be surprised how common this is
+                when using a test runner (like pytest). The typical recipe is:
+                ResourceWarning + -Werror + a custom sys.unraisablehook.
+                """
+                nonlocal sneaky_frame_object
+                sneaky_frame_object = sys._getframe()
+
+        class SneakyThread(threading.Thread):
+            """
+            A separate thread isn't needed to make this code crash, but it does
+            make crashes more consistent, since it means sneaky_frame_object is
+            backed by freed memory after the thread completes!
+            """
+
+            def run(self):
+                """Run SneakyDel.__del__ as this frame is popped."""
+                ref = SneakyDel()
+
+        sneaky_frame_object = None
+        t = SneakyThread()
+        t.start()
+        t.join()
+        # sneaky_frame_object can be anything, really, but it's crucial that
+        # SneakyThread.run's frame isn't anywhere on the stack while it's being
+        # torn down:
+        self.assertIsNotNone(sneaky_frame_object)
+        while sneaky_frame_object is not None:
+            self.assertIsNot(
+                sneaky_frame_object.f_code, SneakyThread.run.__code__
+            )
+            sneaky_frame_object = sneaky_frame_object.f_back
 
 @unittest.skipIf(_testcapi is None, 'need _testcapi')
 class TestCAPI(unittest.TestCase):

Misc/NEWS.d/next/Core and Builtins/2022-11-26-04-00-41.gh-issue-99729.A3ovwQ.rst

@@ -0,0 +1,3 @@
+Fix an issue that could cause frames to be visible to Python code as they
+are being torn down, possibly leading to memory corruption or hard crashes
+of the interpreter.

Python/bytecodes.c

@@ -619,7 +619,10 @@ dummy_func(
             DTRACE_FUNCTION_EXIT();
             _Py_LeaveRecursiveCallPy(tstate);
             assert(frame != &entry_frame);
-            frame = cframe.current_frame = pop_frame(tstate, frame);
+            // GH-99729: We need to unlink the frame *before* clearing it:
+            _PyInterpreterFrame *dying = frame;
+            frame = cframe.current_frame = dying->previous;
+            _PyEvalFrameClearAndPop(tstate, dying);
             _PyFrame_StackPush(frame, retval);
             goto resume_frame;
         }

Python/ceval.c

@@ -1009,14 +1009,6 @@ trace_function_exit(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject
     return 0;
 }
 
-static _PyInterpreterFrame *
-pop_frame(PyThreadState *tstate, _PyInterpreterFrame *frame)
-{
-    _PyInterpreterFrame *prev_frame = frame->previous;
-    _PyEvalFrameClearAndPop(tstate, frame);
-    return prev_frame;
-}
-
 
 int _Py_CheckRecursiveCallPy(
     PyThreadState *tstate)
@@ -1432,7 +1424,10 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int
     assert(_PyErr_Occurred(tstate));
     _Py_LeaveRecursiveCallPy(tstate);
     assert(frame != &entry_frame);
-    frame = cframe.current_frame = pop_frame(tstate, frame);
+    // GH-99729: We need to unlink the frame *before* clearing it:
+    _PyInterpreterFrame *dying = frame;
+    frame = cframe.current_frame = dying->previous;
+    _PyEvalFrameClearAndPop(tstate, dying);
     if (frame == &entry_frame) {
         /* Restore previous cframe and exit */
         tstate->cframe = cframe.previous;

Python/frame.c

@@ -127,6 +127,9 @@ _PyFrame_Clear(_PyInterpreterFrame *frame)
      * to have cleared the enclosing generator, if any. */
     assert(frame->owner != FRAME_OWNED_BY_GENERATOR ||
         _PyFrame_GetGenerator(frame)->gi_frame_state == FRAME_CLEARED);
+    // GH-99729: Clearing this frame can expose the stack (via finalizers). It's
+    // crucial that this frame has been unlinked, and is no longer visible:
+    assert(_PyThreadState_GET()->cframe->current_frame != frame);
     if (frame->frame_obj) {
         PyFrameObject *f = frame->frame_obj;
         frame->frame_obj = NULL;

Python/generated_cases.c.h

@@ -184,6 +184,7 @@ def gather_stats(input):
                     key = key.strip()
                     value = int(value)
                     stats[key] += value
+            stats['__nfiles__'] += 1
         return stats
     else:
         raise ValueError(f"{input:r} is not a file or directory path")
@@ -561,6 +562,9 @@ def output_single_stats(stats):
     emit_specialization_overview(opcode_stats, total)
     emit_call_stats(stats)
     emit_object_stats(stats)
+    with Section("Meta stats", summary="Meta statistics"):
+        emit_table(("", "Count:"), [('Number of data files', stats['__nfiles__'])])
+
 
 def output_comparative_stats(base_stats, head_stats):
     base_opcode_stats = extract_opcode_stats(base_stats)

Tools/scripts/summarize_stats.py

@@ -0,0 +1,8 @@
+[build]
+    base = "Doc/"
+    command = "make html"
+    publish = "build/html"
+
+[build.environment]
+  PYTHON_VERSION = "3.8"
+  IS_DEPLOYMENT_PREVIEW = "true"