Merge in the main branch

Author: encukou

.github/workflows/build.yml

@@ -250,7 +250,7 @@ jobs:
 
   build_ubuntu_ssltests:
     name: 'Ubuntu SSL tests with OpenSSL'
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     timeout-minutes: 60
     needs: check_source
     if: needs.check_source.outputs.run_tests == 'true'
@@ -316,7 +316,7 @@ jobs:
 
   test_hypothesis:
     name: "Hypothesis tests on Ubuntu"
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     timeout-minutes: 60
     needs: check_source
     if: needs.check_source.outputs.run_tests == 'true' && needs.check_source.outputs.run_hypothesis == 'true'
@@ -429,7 +429,7 @@ jobs:
 
   build_asan:
     name: 'Address sanitizer'
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     timeout-minutes: 60
     needs: check_source
     if: needs.check_source.outputs.run_tests == 'true'

.github/workflows/jit.yml

@@ -75,14 +75,10 @@ jobs:
             architecture: aarch64
             runner: ubuntu-latest
             compiler: gcc
-            # These fail because of emulation, not because of the JIT:
-            exclude: test_pathlib test_posixpath test_unix_events test_init test_process_pool test_shutdown test_multiprocessing_fork test_cmd_line test_faulthandler test_os test_perf_profiler test_posix test_signal test_socket test_subprocess test_threading test_venv test_external_inspection
           - target: aarch64-unknown-linux-gnu/clang
             architecture: aarch64
             runner: ubuntu-latest
             compiler: clang
-            # These fail because of emulation, not because of the JIT:
-            exclude: test_pathlib test_posixpath test_unix_events test_init test_process_pool test_shutdown test_multiprocessing_fork test_cmd_line test_faulthandler test_os test_perf_profiler test_posix test_signal test_socket test_subprocess test_threading test_venv test_external_inspection
     env:
       CC: ${{ matrix.compiler }}
     steps:
@@ -97,7 +93,7 @@ jobs:
           choco upgrade llvm -y
           choco install llvm --allow-downgrade --no-progress --version ${{ matrix.llvm }}
           ./PCbuild/build.bat --experimental-jit ${{ matrix.debug && '-d' || '--pgo' }} -p ${{ matrix.architecture }}
-          ./PCbuild/rt.bat ${{ matrix.debug && '-d' }} -p ${{ matrix.architecture }} -q --exclude ${{ matrix.exclude }} --multiprocess 0 --timeout 4500 --verbose2 --verbose3
+          ./PCbuild/rt.bat ${{ matrix.debug && '-d' || '' }} -p ${{ matrix.architecture }} -q --multiprocess 0 --timeout 4500 --verbose2 --verbose3
 
       # No PGO or tests (yet):
       - name: Emulated Windows
@@ -115,7 +111,7 @@ jobs:
           SDKROOT="$(xcrun --show-sdk-path)" \
             ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations --with-lto' }}
           make all --jobs 4
-          ./python.exe -m test --exclude ${{ matrix.exclude }} --multiprocess 0 --timeout 4500 --verbose2 --verbose3
+          ./python.exe -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3
 
       # --with-lto has been removed temporarily as a result of an open issue in LLVM 18 (see https://github.com/llvm/llvm-project/issues/87553)
       - name: Native Linux
@@ -125,11 +121,12 @@ jobs:
           export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH"
           ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations' }}
           make all --jobs 4
-          ./python -m test --exclude ${{ matrix.exclude }} --multiprocess 0 --timeout 4500 --verbose2 --verbose3
+          ./python -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3
 
       # --with-lto has been removed temporarily as a result of an open issue in LLVM 18 (see https://github.com/llvm/llvm-project/issues/87553)
       - name: Emulated Linux
         if: runner.os == 'Linux' && matrix.architecture != 'x86_64'
+        # The --ignorefile on ./python -m test is used to exclude tests known to fail when running on an emulated Linux.
         run: |
           sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }}
           export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH"
@@ -146,4 +143,4 @@ jobs:
             HOSTRUNNER=qemu-${{ matrix.architecture }} \
             ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations ' }} --build=x86_64-linux-gnu --host="$HOST" --with-build-python=../build/bin/python3 --with-pkg-config=no ac_cv_buggy_getaddrinfo=no ac_cv_file__dev_ptc=no ac_cv_file__dev_ptmx=yes
           make all --jobs 4
-          ./python -m test --exclude ${{ matrix.exclude }} --multiprocess 0 --timeout 4500 --verbose2 --verbose3
+          ./python -m test --ignorefile=Tools/jit/ignore-tests-emulated-linux.txt --multiprocess 0 --timeout 4500 --verbose2 --verbose3

.github/workflows/mypy.yml

@@ -8,6 +8,7 @@ on:
   pull_request:
     paths:
       - ".github/workflows/mypy.yml"
+      - "Lib/_pyrepl/**"
       - "Lib/test/libregrtest/**"
       - "Tools/build/generate_sbom.py"
       - "Tools/cases_generator/**"
@@ -35,8 +36,9 @@ jobs:
     strategy:
       matrix:
         target: [
+          "Lib/_pyrepl",
           "Lib/test/libregrtest",
-          "Tools/build/",
+          "Tools/build",
           "Tools/cases_generator",
           "Tools/clinic",
           "Tools/jit",

.github/workflows/reusable-macos.yml

@@ -22,6 +22,7 @@ jobs:
       HOMEBREW_NO_INSTALL_CLEANUP: 1
       HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK: 1
       PYTHONSTRICTEXTENSIONBUILD: 1
+      TERM: linux
     strategy:
       fail-fast: false
       matrix:

.github/workflows/reusable-ubuntu.yml

@@ -12,11 +12,12 @@ jobs:
   build_ubuntu_reusable:
     name: 'build and test'
     timeout-minutes: 60
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     env:
       FORCE_COLOR: 1
       OPENSSL_VER: 3.0.13
       PYTHONSTRICTEXTENSIONBUILD: 1
+      TERM: linux
     steps:
     - uses: actions/checkout@v4
     - name: Register gcc problem matcher

.github/workflows/reusable-wasi.yml

@@ -9,7 +9,7 @@ jobs:
   build_wasi_reusable:
     name: 'build and test'
     timeout-minutes: 60
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     env:
       WASMTIME_VERSION: 18.0.3
       WASI_SDK_VERSION: 21

.pre-commit-config.yaml

@@ -11,6 +11,14 @@ repos:
         args: [--exit-non-zero-on-fix, --config=Tools/clinic/.ruff.toml]
         files: ^Tools/clinic/|Lib/test/test_clinic.py
 
+  - repo: https://github.com/psf/black-pre-commit-mirror
+    rev: 24.4.2
+    hooks:
+      - id: black
+        name: Run Black on Tools/jit/
+        files: ^Tools/jit/
+        language_version: python3.12
+
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v4.5.0
     hooks:

Doc/c-api/frame.rst

@@ -120,12 +120,19 @@ See also :ref:`Reflection <reflection>`.
 
 .. c:function:: PyObject* PyFrame_GetLocals(PyFrameObject *frame)
 
-   Get the *frame*'s :attr:`~frame.f_locals` attribute (:class:`dict`).
+   Get the *frame*'s :attr:`~frame.f_locals` attribute.
+   If the frame refers to a function or comprehension, this returns
+   a write-through proxy object that allows modifying the locals.
+   In all other cases (classes, modules) it returns the :class:`dict`
+   representing the frame locals directly.
 
    Return a :term:`strong reference`.
 
    .. versionadded:: 3.11
 
+   .. versionchanged:: 3.13
+      Return a proxy object for functions and comprehensions.
+
 
 .. c:function:: int PyFrame_GetLineNumber(PyFrameObject *frame)
 

Doc/c-api/index.rst

@@ -25,3 +25,4 @@ document the API functions in detail.
    memory.rst
    objimpl.rst
    apiabiversion.rst
+   monitoring.rst

Doc/c-api/init.rst

@@ -1904,6 +1904,58 @@ Python-level trace functions in previous versions.
 
 .. versionadded:: 3.12
 
+Reference tracing
+=================
+
+.. versionadded:: 3.13
+
+.. c:type:: int (*PyRefTracer)(PyObject *, int event, void* data)
+
+   The type of the trace function registered using :c:func:`PyRefTracer_SetTracer`.
+   The first parameter is a Python object that has been just created (when **event**
+   is set to :c:data:`PyRefTracer_CREATE`) or about to be destroyed (when **event**
+   is set to :c:data:`PyRefTracer_DESTROY`). The **data** argument is the opaque pointer
+   that was provided when :c:func:`PyRefTracer_SetTracer` was called.
+
+.. versionadded:: 3.13
+
+.. c:var:: int PyRefTracer_CREATE
+
+   The value for the *event* parameter to :c:type:`PyRefTracer` functions when a Python
+   object has been created.
+
+.. c:var:: int PyRefTracer_DESTROY
+
+   The value for the *event* parameter to :c:type:`PyRefTracer` functions when a Python
+   object has been destroyed.
+
+.. c:function:: int PyRefTracer_SetTracer(PyRefTracer tracer, void *data)
+
+   Register a reference tracer function. The function will be called when a new
+   Python has been created or when an object is going to be destroyed. If
+   **data** is provided it must be an opaque pointer that will be provided when
+   the tracer function is called. Return ``0`` on success. Set an exception and
+   return ``-1`` on error.
+
+   Not that tracer functions **must not** create Python objects inside or
+   otherwise the call will be re-entrant. The tracer also **must not** clear
+   any existing exception or set an exception.  The GIL will be held every time
+   the tracer function is called.
+
+   The GIL must be held when calling this function.
+
+.. versionadded:: 3.13
+
+.. c:function:: PyRefTracer PyRefTracer_GetTracer(void** data)
+
+   Get the registered reference tracer function and the value of the opaque data
+   pointer that was registered when :c:func:`PyRefTracer_SetTracer` was called.
+   If no tracer was registered this function will return NULL and will set the
+   **data** pointer to NULL.
+
+   The GIL must be held when calling this function.
+
+.. versionadded:: 3.13
 
 .. _advanced-debugging:
 

Doc/c-api/init_config.rst

@@ -1251,7 +1251,10 @@ PyConfig
       for more information.
 
       Set by :option:`-X perf <-X>` command line option and by the
-      :envvar:`PYTHONPERFSUPPORT` environment variable.
+      :envvar:`PYTHONPERFSUPPORT` environment variable for perf support
+      with stack pointers and :option:`-X perfjit <-X>` command line option
+      and by the :envvar:`PYTHONPERFJITSUPPORT` environment variable for perf
+      support with DWARF JIT information.
 
       Default: ``-1``.
 

Doc/c-api/long.rst

@@ -390,7 +390,7 @@ distinguished from a number.  Use :c:func:`PyErr_Occurred` to disambiguate.
    Usage example::
 
       int32_t value;
-      Py_ssize_t bytes = PyLong_AsNativeBits(pylong, &value, sizeof(value), -1);
+      Py_ssize_t bytes = PyLong_AsNativeBytes(pylong, &value, sizeof(value), -1);
       if (bytes < 0) {
           // Failed. A Python exception was set with the reason.
           return NULL;
@@ -418,7 +418,7 @@ distinguished from a number.  Use :c:func:`PyErr_Occurred` to disambiguate.
    called twice: first to determine the buffer size, then to fill it::
 
       // Ask how much space we need.
-      Py_ssize_t expected = PyLong_AsNativeBits(pylong, NULL, 0, -1);
+      Py_ssize_t expected = PyLong_AsNativeBytes(pylong, NULL, 0, -1);
       if (expected < 0) {
           // Failed. A Python exception was set with the reason.
           return NULL;
@@ -430,7 +430,7 @@ distinguished from a number.  Use :c:func:`PyErr_Occurred` to disambiguate.
           return NULL;
       }
       // Safely get the entire value.
-      Py_ssize_t bytes = PyLong_AsNativeBits(pylong, bignum, expected, -1);
+      Py_ssize_t bytes = PyLong_AsNativeBytes(pylong, bignum, expected, -1);
       if (bytes < 0) {  // Exception has been set.
           free(bignum);
           return NULL;

Doc/c-api/module.rst

@@ -411,6 +411,31 @@ The available slot types are:
 
    .. versionadded:: 3.12
 
+.. c:macro:: Py_mod_gil
+
+   Specifies one of the following values:
+
+   .. c:macro:: Py_MOD_GIL_USED
+
+      The module depends on the presence of the global interpreter lock (GIL),
+      and may access global state without synchronization.
+
+   .. c:macro:: Py_MOD_GIL_NOT_USED
+
+      The module is safe to run without an active GIL.
+
+   This slot is ignored by Python builds not configured with
+   :option:`--disable-gil`.  Otherwise, it determines whether or not importing
+   this module will cause the GIL to be automatically enabled. See
+   :envvar:`PYTHON_GIL` and :option:`-X gil <-X>` for more detail.
+
+   Multiple ``Py_mod_gil`` slots may not be specified in one module definition.
+
+   If ``Py_mod_gil`` is not specified, the import machinery defaults to
+   ``Py_MOD_GIL_USED``.
+
+   .. versionadded: 3.13
+
 See :PEP:`489` for more details on multi-phase initialization.
 
 Low-level module creation functions
@@ -609,6 +634,19 @@ state:
 
    .. versionadded:: 3.9
 
+.. c:function:: int PyModule_ExperimentalSetGIL(PyObject *module, void *gil)
+
+   Indicate that *module* does or does not support running without the global
+   interpreter lock (GIL), using one of the values from
+   :c:macro:`Py_mod_gil`. It must be called during *module*'s initialization
+   function. If this function is not called during module initialization, the
+   import machinery assumes the module does not support running without the
+   GIL. This function is only available in Python builds configured with
+   :option:`--disable-gil`.
+   Return ``-1`` on error, ``0`` on success.
+
+   .. versionadded:: 3.13
+
 
 Module lookup
 ^^^^^^^^^^^^^