gh-94673: Add Per-Interpreter tp_weaklist for Static Builtin Types (#95302)

* Store tp_weaklist on the interpreter state for static builtin types.

* Factor out _PyStaticType_GET_WEAKREFS_LISTPTR().

* Add _PyStaticType_ClearWeakRefs().

* Add a comment about how _PyStaticType_ClearWeakRefs() loops.

* Document the change.

* Update Doc/whatsnew/3.12.rst

* Fix a typo.

Author: ericsnowcurrently

Doc/c-api/typeobj.rst

@@ -1942,6 +1942,13 @@ and :c:type:`PyType_Type` effectively act as defaults.)
    Weak reference list head, for weak references to this type object.  Not
    inherited.  Internal use only.
 
+   .. versionchanged:: 3.12
+
+      Internals detail: For the static builtin types this is always ``NULL``,
+      even if weakrefs are added.  Instead, the weakrefs for each are stored
+      on ``PyInterpreterState``.  Use the public C-API or the internal
+      ``_PyObject_GET_WEAKREFS_LISTPTR()`` macro to avoid the distinction.
+
    **Inheritance:**
 
    This field is not inherited.

Doc/whatsnew/3.12.rst

@@ -413,6 +413,13 @@ Porting to Python 3.12
   ``Py_UNICODE*`` based format (e.g. ``u``, ``Z``) anymore. Please migrate
   to other formats for Unicode like ``s``, ``z``, ``es``, and ``U``.
 
+* ``tp_weaklist`` for all static builtin types is always ``NULL``.
+  This is an internal-only field on ``PyTypeObject``
+  but we're pointing out the change in case someone happens to be
+  accessing the field directly anyway.  To avoid breakage, consider
+  using the existing public C-API instead, or, if necessary, the
+  (internal-only) ``_PyObject_GET_WEAKREFS_LISTPTR()`` macro.
+
 Deprecated
 ----------
 

Include/cpython/object.h

@@ -219,7 +219,7 @@ struct _typeobject {
     PyObject *tp_mro; /* method resolution order */
     PyObject *tp_cache;
     PyObject *tp_subclasses;
-    PyObject *tp_weaklist;
+    PyObject *tp_weaklist; /* not used for static builtin types */
     destructor tp_del;
 
     /* Type attribute cache version tag. Added in version 2.6 */

Include/internal/pycore_object.h

@@ -220,6 +220,12 @@ extern void _Py_PrintReferenceAddresses(FILE *);
 static inline PyObject **
 _PyObject_GET_WEAKREFS_LISTPTR(PyObject *op)
 {
+    if (PyType_Check(op) &&
+            ((PyTypeObject *)op)->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) {
+        static_builtin_state *state = _PyStaticType_GetState(
+                                                        (PyTypeObject *)op);
+        return _PyStaticType_GET_WEAKREFS_LISTPTR(state);
+    }
     Py_ssize_t offset = Py_TYPE(op)->tp_weaklistoffset;
     return (PyObject **)((char *)op + offset);
 }

Include/internal/pycore_typeobject.h

@@ -45,8 +45,20 @@ struct type_cache {
 
 typedef struct {
     PyTypeObject *type;
+    /* We never clean up weakrefs for static builtin types since
+       they will effectively never get triggered.  However, there
+       are also some diagnostic uses for the list of weakrefs,
+       so we still keep it. */
+    PyObject *tp_weaklist;
 } static_builtin_state;
 
+static inline PyObject **
+_PyStaticType_GET_WEAKREFS_LISTPTR(static_builtin_state *state)
+{
+    assert(state != NULL);
+    return &state->tp_weaklist;
+}
+
 struct types_state {
     struct type_cache type_cache;
     size_t num_builtins_initialized;
@@ -58,6 +70,7 @@ extern PyStatus _PyTypes_InitSlotDefs(void);
 
 extern int _PyStaticType_InitBuiltin(PyTypeObject *type);
 extern static_builtin_state * _PyStaticType_GetState(PyTypeObject *);
+extern void _PyStaticType_ClearWeakRefs(PyTypeObject *type);
 extern void _PyStaticType_Dealloc(PyTypeObject *type);
 
 

Objects/typeobject.c

@@ -127,6 +127,8 @@ static_builtin_state_init(PyTypeObject *self)
 
     static_builtin_state *state = static_builtin_state_get(interp, self);
     state->type = self;
+    /* state->tp_weaklist is left NULL until insert_head() or insert_after()
+       (in weakrefobject.c) sets it. */
 }
 
 static void
@@ -138,6 +140,7 @@ static_builtin_state_clear(PyTypeObject *self)
 
     static_builtin_state *state = static_builtin_state_get(interp, self);
     state->type = NULL;
+    assert(state->tp_weaklist == NULL);  // It was already cleared out.
     static_builtin_index_clear(self);
 
     assert(interp->types.num_builtins_initialized > 0);
@@ -502,6 +505,8 @@ static PyMemberDef type_members[] = {
     {"__basicsize__", T_PYSSIZET, offsetof(PyTypeObject,tp_basicsize),READONLY},
     {"__itemsize__", T_PYSSIZET, offsetof(PyTypeObject, tp_itemsize), READONLY},
     {"__flags__", T_ULONG, offsetof(PyTypeObject, tp_flags), READONLY},
+    /* Note that this value is misleading for static builtin types,
+       since the memory at this offset will always be NULL. */
     {"__weakrefoffset__", T_PYSSIZET,
      offsetof(PyTypeObject, tp_weaklistoffset), READONLY},
     {"__base__", T_OBJECT, offsetof(PyTypeObject, tp_base), READONLY},
@@ -4353,6 +4358,7 @@ _PyStaticType_Dealloc(PyTypeObject *type)
     type->tp_flags &= ~Py_TPFLAGS_READY;
 
     if (type->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) {
+        _PyStaticType_ClearWeakRefs(type);
         static_builtin_state_clear(type);
         /* We leave _Py_TPFLAGS_STATIC_BUILTIN set on tp_flags. */
     }

Objects/weakrefobject.c

@@ -1019,3 +1019,22 @@ PyObject_ClearWeakRefs(PyObject *object)
         PyErr_Restore(err_type, err_value, err_tb);
     }
 }
+
+/* This function is called by _PyStaticType_Dealloc() to clear weak references.
+ *
+ * This is called at the end of runtime finalization, so we can just
+ * wipe out the type's weaklist.  We don't bother with callbacks
+ * or anything else.
+ */
+void
+_PyStaticType_ClearWeakRefs(PyTypeObject *type)
+{
+    static_builtin_state *state = _PyStaticType_GetState(type);
+    PyObject **list = _PyStaticType_GET_WEAKREFS_LISTPTR(state);
+    while (*list != NULL) {
+        /* Note that clear_weakref() pops the first ref off the type's
+           weaklist before clearing its wr_object and wr_callback.
+           That is how we're able to loop over the list. */
+        clear_weakref((PyWeakReference *)*list);
+    }
+}