Merge branch 'main' into fetch-restore-objects

Author: iritkatriel

Doc/library/asyncio-eventloop.rst

@@ -524,8 +524,8 @@ Opening network connections
       When a server's IPv4 path and protocol are working, but the server's
       IPv6 path and protocol are not working, a dual-stack client
       application experiences significant connection delay compared to an
-      IPv4-only client.  This is undesirable because it causes the dual-
-      stack client to have a worse user experience.  This document
+      IPv4-only client.  This is undesirable because it causes the
+      dual-stack client to have a worse user experience.  This document
       specifies requirements for algorithms that reduce this user-visible
       delay and provides an algorithm.
 

Doc/library/decimal.rst

@@ -11,7 +11,7 @@
 __all__ = ["version", "bootstrap"]
 _PACKAGE_NAMES = ('setuptools', 'pip')
 _SETUPTOOLS_VERSION = "65.5.0"
-_PIP_VERSION = "23.0"
+_PIP_VERSION = "23.0.1"
 _PROJECTS = [
     ("setuptools", _SETUPTOOLS_VERSION, "py3"),
     ("pip", _PIP_VERSION, "py3"),

Lib/ensurepip/__init__.py

@@ -7,6 +7,9 @@
 from test.support import check_free_after_iterating, ALWAYS_EQ, NEVER_EQ
 import pickle
 import collections.abc
+import functools
+import contextlib
+import builtins
 
 # Test result of triple loop (too big to inline)
 TRIPLETS = [(0, 0, 0), (0, 0, 1), (0, 0, 2),
@@ -91,6 +94,12 @@ def __call__(self):
             raise IndexError # Emergency stop
         return i
 
+class EmptyIterClass:
+    def __len__(self):
+        return 0
+    def __getitem__(self, i):
+        raise StopIteration
+
 # Main test suite
 
 class TestCase(unittest.TestCase):
@@ -238,6 +247,78 @@ def test_mutating_seq_class_exhausted_iter(self):
         self.assertEqual(list(empit), [5, 6])
         self.assertEqual(list(a), [0, 1, 2, 3, 4, 5, 6])
 
+    def test_reduce_mutating_builtins_iter(self):
+        # This is a reproducer of issue #101765
+        # where iter `__reduce__` calls could lead to a segfault or SystemError
+        # depending on the order of C argument evaluation, which is undefined
+
+        # Backup builtins
+        builtins_dict = builtins.__dict__
+        orig = {"iter": iter, "reversed": reversed}
+
+        def run(builtin_name, item, sentinel=None):
+            it = iter(item) if sentinel is None else iter(item, sentinel)
+
+            class CustomStr:
+                def __init__(self, name, iterator):
+                    self.name = name
+                    self.iterator = iterator
+                def __hash__(self):
+                    return hash(self.name)
+                def __eq__(self, other):
+                    # Here we exhaust our iterator, possibly changing
+                    # its `it_seq` pointer to NULL
+                    # The `__reduce__` call should correctly get
+                    # the pointers after this call
+                    list(self.iterator)
+                    return other == self.name
+
+            # del is required here
+            # to not prematurely call __eq__ from
+            # the hash collision with the old key
+            del builtins_dict[builtin_name]
+            builtins_dict[CustomStr(builtin_name, it)] = orig[builtin_name]
+
+            return it.__reduce__()
+
+        types = [
+            (EmptyIterClass(),),
+            (bytes(8),),
+            (bytearray(8),),
+            ((1, 2, 3),),
+            (lambda: 0, 0),
+            (tuple[int],)  # GenericAlias
+        ]
+
+        try:
+            run_iter = functools.partial(run, "iter")
+            # The returned value of `__reduce__` should not only be valid
+            # but also *empty*, as `it` was exhausted during `__eq__`
+            # i.e "xyz" returns (iter, ("",))
+            self.assertEqual(run_iter("xyz"), (orig["iter"], ("",)))
+            self.assertEqual(run_iter([1, 2, 3]), (orig["iter"], ([],)))
+
+            # _PyEval_GetBuiltin is also called for `reversed` in a branch of
+            # listiter_reduce_general
+            self.assertEqual(
+                run("reversed", orig["reversed"](list(range(8)))),
+                (iter, ([],))
+            )
+
+            for case in types:
+                self.assertEqual(run_iter(*case), (orig["iter"], ((),)))
+        finally:
+            # Restore original builtins
+            for key, func in orig.items():
+                # need to suppress KeyErrors in case
+                # a failed test deletes the key without setting anything
+                with contextlib.suppress(KeyError):
+                    # del is required here
+                    # to not invoke our custom __eq__ from
+                    # the hash collision with the old key
+                    del builtins_dict[key]
+                builtins_dict[key] = func
+
     # Test a new_style class with __iter__ but no next() method
     def test_new_style_iter_class(self):
         class IterClass(object):

Lib/ensurepip/_bundled/pip-23.0-py3-none-any.whl renamed to Lib/ensurepip/_bundled/pip-23.0.1-py3-none-any.whl

@@ -225,18 +225,19 @@ def test_add_dir_getmember(self):
         self.add_dir_and_getmember('bar')
         self.add_dir_and_getmember('a'*101)
 
-    @unittest.skipIf(
-        (hasattr(os, 'getuid') and os.getuid() > 0o777_7777) or
-        (hasattr(os, 'getgid') and os.getgid() > 0o777_7777),
-        "uid or gid too high for USTAR format."
-    )
+    @unittest.skipUnless(hasattr(os, "getuid") and hasattr(os, "getgid"),
+                         "Missing getuid or getgid implementation")
     def add_dir_and_getmember(self, name):
+        def filter(tarinfo):
+            tarinfo.uid = tarinfo.gid = 100
+            return tarinfo
+
         with os_helper.temp_cwd():
             with tarfile.open(tmpname, 'w') as tar:
                 tar.format = tarfile.USTAR_FORMAT
                 try:
                     os.mkdir(name)
-                    tar.add(name)
+                    tar.add(name, filter=filter)
                 finally:
                     os.rmdir(name)
             with tarfile.open(tmpname) as tar:

Lib/test/test_iter.py

@@ -330,7 +330,8 @@ def test_joinpath_constant_time(self):
         # Check the file iterated all items
         assert entries.count == self.HUGE_ZIPFILE_NUM_ENTRIES
 
-    @set_timeout(3)
+    # timeout disabled due to #102209
+    # @set_timeout(3)
     def test_implied_dirs_performance(self):
         data = ['/'.join(string.ascii_lowercase + str(n)) for n in range(10000)]
         zipfile.CompleteDirs._implied_dirs(data)

Lib/test/test_tarfile.py

@@ -1308,6 +1308,7 @@ Jon Oberheide
 Milan Oberkirch
 Pascal Oberndoerfer
 Géry Ogam
+Seonkyo Ok
 Jeffrey Ollie
 Adam Olsen
 Bryan Olson

Lib/test/test_zipfile/test_path.py

@@ -0,0 +1 @@
+Fix SystemError / segmentation fault in iter ``__reduce__`` when internal access of ``builtins.__dict__`` keys mutates the iter object.

Misc/ACKS

@@ -0,0 +1 @@
+Fix deadlock at shutdown when clearing thread states if any finalizer tries to acquire the runtime head lock. Patch by Kumar Aditya.

Misc/NEWS.d/next/Core and Builtins/2023-02-10-07-21-47.gh-issue-101765.MO5LlC.rst

@@ -0,0 +1 @@
+Upgrade pip wheel bundled with ensurepip (pip 23.0.1)

Misc/NEWS.d/next/Core and Builtins/2023-02-24-17-59-39.gh-issue-102126.HTT8Vc.rst

@@ -79,7 +79,7 @@ abc_data_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
         return NULL;
     }
 
-    state = PyType_GetModuleState(type);
+    state = _PyType_GetModuleState(type);
     if (state == NULL) {
         Py_DECREF(self);
         return NULL;

Misc/NEWS.d/next/Library/2023-02-17-18-44-27.gh-issue-101997.A6_blD.rst

@@ -76,7 +76,7 @@ get_asyncio_state(PyObject *mod)
 static inline asyncio_state *
 get_asyncio_state_by_cls(PyTypeObject *cls)
 {
-    asyncio_state *state = (asyncio_state *)PyType_GetModuleState(cls);
+    asyncio_state *state = (asyncio_state *)_PyType_GetModuleState(cls);
     assert(state != NULL);
     return state;
 }
@@ -1422,7 +1422,6 @@ _asyncio_Future__make_cancelled_error_impl(FutureObj *self)
 static void
 FutureObj_finalize(FutureObj *fut)
 {
-    PyObject *error_type, *error_value, *error_traceback;
     PyObject *context;
     PyObject *message = NULL;
     PyObject *func;
@@ -1434,7 +1433,7 @@ FutureObj_finalize(FutureObj *fut)
     fut->fut_log_tb = 0;
 
     /* Save the current exception, if any. */
-    PyErr_Fetch(&error_type, &error_value, &error_traceback);
+    PyObject *exc = PyErr_GetRaisedException();
 
     context = PyDict_New();
     if (context == NULL) {
@@ -1476,7 +1475,7 @@ FutureObj_finalize(FutureObj *fut)
     Py_XDECREF(message);
 
     /* Restore the saved exception. */
-    PyErr_Restore(error_type, error_value, error_traceback);
+    PyErr_SetRaisedException(exc);
 }
 
 static PyMethodDef FutureType_methods[] = {
@@ -2491,14 +2490,13 @@ TaskObj_finalize(TaskObj *task)
     PyObject *context;
     PyObject *message = NULL;
     PyObject *func;
-    PyObject *error_type, *error_value, *error_traceback;
 
     if (task->task_state != STATE_PENDING || !task->task_log_destroy_pending) {
         goto done;
     }
 
     /* Save the current exception, if any. */
-    PyErr_Fetch(&error_type, &error_value, &error_traceback);
+    PyObject *exc = PyErr_GetRaisedException();
 
     context = PyDict_New();
     if (context == NULL) {
@@ -2541,7 +2539,7 @@ TaskObj_finalize(TaskObj *task)
     Py_XDECREF(message);
 
     /* Restore the saved exception. */
-    PyErr_Restore(error_type, error_value, error_traceback);
+    PyErr_SetRaisedException(exc);
 
 done:
     FutureObj_finalize((FutureObj*)task);
@@ -2766,8 +2764,6 @@ task_step_impl(asyncio_state *state, TaskObj *task, PyObject *exc)
     }
 
     if (gen_status == PYGEN_RETURN || gen_status == PYGEN_ERROR) {
-        PyObject *et, *ev, *tb;
-
         if (result != NULL) {
             /* The error is StopIteration and that means that
                the underlying coroutine has resolved */
@@ -2794,52 +2790,39 @@ task_step_impl(asyncio_state *state, TaskObj *task, PyObject *exc)
 
         if (PyErr_ExceptionMatches(state->asyncio_CancelledError)) {
             /* CancelledError */
-            PyErr_Fetch(&et, &ev, &tb);
-            assert(et);
-            PyErr_NormalizeException(&et, &ev, &tb);
-            if (tb != NULL) {
-                PyException_SetTraceback(ev, tb);
-                Py_DECREF(tb);
-            }
-            Py_XDECREF(et);
+
+            PyObject *exc = PyErr_GetRaisedException();
+            assert(exc);
 
             FutureObj *fut = (FutureObj*)task;
             /* transfer ownership */
-            fut->fut_cancelled_exc = ev;
+            fut->fut_cancelled_exc = exc;
 
             return future_cancel(state, fut, NULL);
         }
 
         /* Some other exception; pop it and call Task.set_exception() */
-        PyErr_Fetch(&et, &ev, &tb);
-        assert(et);
-        PyErr_NormalizeException(&et, &ev, &tb);
-        if (tb != NULL) {
-            PyException_SetTraceback(ev, tb);
-        }
+        PyObject *exc = PyErr_GetRaisedException();
+        assert(exc);
 
-        o = future_set_exception(state, (FutureObj*)task, ev);
+        o = future_set_exception(state, (FutureObj*)task, exc);
         if (!o) {
             /* An exception in Task.set_exception() */
-            Py_DECREF(et);
-            Py_XDECREF(tb);
-            Py_XDECREF(ev);
+            Py_DECREF(exc);
             goto fail;
         }
         assert(o == Py_None);
         Py_DECREF(o);
 
-        if (PyErr_GivenExceptionMatches(et, PyExc_KeyboardInterrupt) ||
-            PyErr_GivenExceptionMatches(et, PyExc_SystemExit))
+        if (PyErr_GivenExceptionMatches(exc, PyExc_KeyboardInterrupt) ||
+            PyErr_GivenExceptionMatches(exc, PyExc_SystemExit))
         {
             /* We've got a KeyboardInterrupt or a SystemError; re-raise it */
-            PyErr_Restore(et, ev, tb);
+            PyErr_SetRaisedException(exc);
             goto fail;
         }
 
-        Py_DECREF(et);
-        Py_XDECREF(tb);
-        Py_XDECREF(ev);
+        Py_DECREF(exc);
 
         Py_RETURN_NONE;
     }
@@ -3059,10 +3042,9 @@ task_step(asyncio_state *state, TaskObj *task, PyObject *exc)
     res = task_step_impl(state, task, exc);
 
     if (res == NULL) {
-        PyObject *et, *ev, *tb;
-        PyErr_Fetch(&et, &ev, &tb);
+        PyObject *exc = PyErr_GetRaisedException();
         leave_task(state, task->task_loop, (PyObject*)task);
-        _PyErr_ChainExceptions(et, ev, tb); /* Normalizes (et, ev, tb) */
+        _PyErr_ChainExceptions1(exc);
         return NULL;
     }
     else {
@@ -3079,7 +3061,6 @@ task_step(asyncio_state *state, TaskObj *task, PyObject *exc)
 static PyObject *
 task_wakeup(TaskObj *task, PyObject *o)
 {
-    PyObject *et, *ev, *tb;
     PyObject *result;
     assert(o);
 
@@ -3111,18 +3092,12 @@ task_wakeup(TaskObj *task, PyObject *o)
         /* exception raised */
     }
 
-    PyErr_Fetch(&et, &ev, &tb);
-    assert(et);
-    PyErr_NormalizeException(&et, &ev, &tb);
-    if (tb != NULL) {
-        PyException_SetTraceback(ev, tb);
-    }
+    PyObject *exc = PyErr_GetRaisedException();
+    assert(exc);
 
-    result = task_step(state, task, ev);
+    result = task_step(state, task, exc);
 
-    Py_DECREF(et);
-    Py_XDECREF(tb);
-    Py_XDECREF(ev);
+    Py_DECREF(exc);
 
     return result;
 }

Modules/_abc.c

@@ -437,10 +437,9 @@ _io_open_impl(PyObject *module, PyObject *file, const char *mode,
 
   error:
     if (result != NULL) {
-        PyObject *exc, *val, *tb, *close_result;
-        PyErr_Fetch(&exc, &val, &tb);
-        close_result = PyObject_CallMethodNoArgs(result, &_Py_ID(close));
-        _PyErr_ChainExceptions(exc, val, tb);
+        PyObject *exc = PyErr_GetRaisedException();
+        PyObject *close_result = PyObject_CallMethodNoArgs(result, &_Py_ID(close));
+        _PyErr_ChainExceptions1(exc);
         Py_XDECREF(close_result);
         Py_DECREF(result);
     }