bpo-46417: Finalize structseq types at exit (GH-30645)

Add _PyStructSequence_FiniType() and _PyStaticType_Dealloc()
functions to finalize a structseq static type in Py_Finalize().
Currrently, these functions do nothing if Python is built in release
mode.

Clear static types:

* AsyncGenHooksType: sys.set_asyncgen_hooks()
* FlagsType: sys.flags
* FloatInfoType: sys.float_info
* Hash_InfoType: sys.hash_info
* Int_InfoType: sys.int_info
* ThreadInfoType: sys.thread_info
* UnraisableHookArgsType: sys.unraisablehook
* VersionInfoType: sys.version
* WindowsVersionType: sys.getwindowsversion()

Author: vstinner

Include/internal/pycore_floatobject.h

@@ -14,6 +14,7 @@ extern "C" {
 extern void _PyFloat_InitState(PyInterpreterState *);
 extern PyStatus _PyFloat_InitTypes(PyInterpreterState *);
 extern void _PyFloat_Fini(PyInterpreterState *);
+extern void _PyFloat_FiniType(PyInterpreterState *);
 
 
 /* other API */

Include/internal/pycore_long.h

@@ -15,6 +15,7 @@ extern "C" {
 /* runtime lifecycle */
 
 extern PyStatus _PyLong_InitTypes(PyInterpreterState *);
+extern void _PyLong_FiniTypes(PyInterpreterState *interp);
 
 
 /* other API */

Include/internal/pycore_pyerrors.h

@@ -12,6 +12,7 @@ extern "C" {
 /* runtime lifecycle */
 
 extern PyStatus _PyErr_InitTypes(PyInterpreterState *);
+extern void _PyErr_FiniTypes(PyInterpreterState *);
 
 
 /* other API */

Include/internal/pycore_pylifecycle.h

@@ -58,6 +58,7 @@ extern PyStatus _PySys_Create(
 extern PyStatus _PySys_ReadPreinitWarnOptions(PyWideStringList *options);
 extern PyStatus _PySys_ReadPreinitXOptions(PyConfig *config);
 extern int _PySys_UpdateConfig(PyThreadState *tstate);
+extern void _PySys_Fini(PyInterpreterState *interp);
 extern PyStatus _PyBuiltins_AddExceptions(PyObject * bltinmod);
 extern PyStatus _Py_HashRandomization_Init(const PyConfig *);
 
@@ -81,6 +82,7 @@ extern void _PyTraceMalloc_Fini(void);
 extern void _PyWarnings_Fini(PyInterpreterState *interp);
 extern void _PyAST_Fini(PyInterpreterState *interp);
 extern void _PyAtExit_Fini(PyInterpreterState *interp);
+extern void _PyThread_FiniType(PyInterpreterState *interp);
 
 extern PyStatus _PyGILState_Init(_PyRuntimeState *runtime);
 extern PyStatus _PyGILState_SetTstate(PyThreadState *tstate);

Include/internal/pycore_typeobject.h

@@ -40,6 +40,8 @@ struct type_cache {
 
 extern PyStatus _PyTypes_InitSlotDefs(void);
 
+extern void _PyStaticType_Dealloc(PyTypeObject *type);
+
 
 #ifdef __cplusplus
 }

Include/structseq.h

@@ -27,6 +27,9 @@ PyAPI_FUNC(void) PyStructSequence_InitType(PyTypeObject *type,
 PyAPI_FUNC(int) PyStructSequence_InitType2(PyTypeObject *type,
                                            PyStructSequence_Desc *desc);
 #endif
+#ifdef Py_BUILD_CORE
+PyAPI_FUNC(void) _PyStructSequence_FiniType(PyTypeObject *type);
+#endif
 PyAPI_FUNC(PyTypeObject*) PyStructSequence_NewType(PyStructSequence_Desc *desc);
 
 PyAPI_FUNC(PyObject *) PyStructSequence_New(PyTypeObject* type);

Lib/test/_test_embed_structseq.py

@@ -0,0 +1,55 @@
+import sys
+import types
+import unittest
+
+
+# bpo-46417: Test that structseq types used by the sys module are still
+# valid when Py_Finalize()/Py_Initialize() are called multiple times.
+class TestStructSeq(unittest.TestCase):
+    # test PyTypeObject members
+    def check_structseq(self, obj_type):
+        # ob_refcnt
+        self.assertGreaterEqual(sys.getrefcount(obj_type), 1)
+        # tp_base
+        self.assertTrue(issubclass(obj_type, tuple))
+        # tp_bases
+        self.assertEqual(obj_type.__bases__, (tuple,))
+        # tp_dict
+        self.assertIsInstance(obj_type.__dict__, types.MappingProxyType)
+        # tp_mro
+        self.assertEqual(obj_type.__mro__, (obj_type, tuple, object))
+        # tp_name
+        self.assertIsInstance(type.__name__, str)
+        # tp_subclasses
+        self.assertEqual(obj_type.__subclasses__(), [])
+
+    def test_sys_attrs(self):
+        for attr_name in (
+            'flags',          # FlagsType
+            'float_info',     # FloatInfoType
+            'hash_info',      # Hash_InfoType
+            'int_info',       # Int_InfoType
+            'thread_info',    # ThreadInfoType
+            'version_info',   # VersionInfoType
+        ):
+            with self.subTest(attr=attr_name):
+                attr = getattr(sys, attr_name)
+                self.check_structseq(type(attr))
+
+    def test_sys_funcs(self):
+        func_names = ['get_asyncgen_hooks']  # AsyncGenHooksType
+        if hasattr(sys, 'getwindowsversion'):
+            func_names.append('getwindowsversion')  # WindowsVersionType
+        for func_name in func_names:
+            with self.subTest(func=func_name):
+                func = getattr(sys, func_name)
+                obj = func()
+                self.check_structseq(type(obj))
+
+
+try:
+    unittest.main()
+except SystemExit as exc:
+    if exc.args[0] != 0:
+        raise
+print("Tests passed")

Lib/test/test_embed.py

@@ -329,6 +329,18 @@ def test_run_main_loop(self):
         self.assertEqual(out, "Py_RunMain(): sys.argv=['-c', 'arg2']\n" * nloop)
         self.assertEqual(err, '')
 
+    def test_finalize_structseq(self):
+        # bpo-46417: Py_Finalize() clears structseq static types. Check that
+        # sys attributes using struct types still work when
+        # Py_Finalize()/Py_Initialize() is called multiple times.
+        # print() calls type->tp_repr(instance) and so checks that the types
+        # are still working properly.
+        script = support.findfile('_test_embed_structseq.py')
+        with open(script, encoding="utf-8") as fp:
+            code = fp.read()
+        out, err = self.run_embedded_interpreter("test_repeated_init_exec", code)
+        self.assertEqual(out, 'Tests passed\n' * INIT_LOOPS)
+
 
 class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
     maxDiff = 4096

Objects/floatobject.c

@@ -2082,6 +2082,14 @@ _PyFloat_Fini(PyInterpreterState *interp)
 #endif
 }
 
+void
+_PyFloat_FiniType(PyInterpreterState *interp)
+{
+    if (_Py_IsMainInterpreter(interp)) {
+        _PyStructSequence_FiniType(&FloatInfoType);
+    }
+}
+
 /* Print summary info about the state of the optimized allocator */
 void
 _PyFloat_DebugMallocStats(FILE *out)

Objects/longobject.c

@@ -5949,3 +5949,14 @@ _PyLong_InitTypes(PyInterpreterState *interp)
 
     return _PyStatus_OK();
 }
+
+
+void
+_PyLong_FiniTypes(PyInterpreterState *interp)
+{
+    if (!_Py_IsMainInterpreter(interp)) {
+        return;
+    }
+
+    _PyStructSequence_FiniType(&Int_InfoType);
+}

Objects/structseq.c

@@ -532,6 +532,36 @@ PyStructSequence_InitType(PyTypeObject *type, PyStructSequence_Desc *desc)
     (void)PyStructSequence_InitType2(type, desc);
 }
 
+
+void
+_PyStructSequence_FiniType(PyTypeObject *type)
+{
+    // Ensure that the type is initialized
+    assert(type->tp_name != NULL);
+    assert(type->tp_base == &PyTuple_Type);
+
+    // Cannot delete a type if it still has subclasses
+    if (type->tp_subclasses != NULL) {
+        return;
+    }
+
+    // Undo PyStructSequence_NewType()
+    type->tp_name = NULL;
+    PyMem_Free(type->tp_members);
+
+    _PyStaticType_Dealloc(type);
+    assert(Py_REFCNT(type) == 1);
+    // Undo Py_INCREF(type) of _PyStructSequence_InitType().
+    // Don't use Py_DECREF(): static type must not be deallocated
+    Py_SET_REFCNT(type, 0);
+
+    // Make sure that _PyStructSequence_InitType() will initialize
+    // the type again
+    assert(Py_REFCNT(type) == 0);
+    assert(type->tp_name == NULL);
+}
+
+
 PyTypeObject *
 PyStructSequence_NewType(PyStructSequence_Desc *desc)
 {

Objects/typeobject.c

@@ -4070,10 +4070,27 @@ type_setattro(PyTypeObject *type, PyObject *name, PyObject *value)
 extern void
 _PyDictKeys_DecRef(PyDictKeysObject *keys);
 
+
+void
+_PyStaticType_Dealloc(PyTypeObject *type)
+{
+    // _PyStaticType_Dealloc() must not be called if a type has subtypes.
+    // A subtype can inherit attributes and methods of its parent type,
+    // and a type must no longer be used once it's deallocated.
+    assert(type->tp_subclasses == NULL);
+
+    Py_CLEAR(type->tp_dict);
+    Py_CLEAR(type->tp_bases);
+    Py_CLEAR(type->tp_mro);
+    Py_CLEAR(type->tp_cache);
+    Py_CLEAR(type->tp_subclasses);
+    type->tp_flags &= ~Py_TPFLAGS_READY;
+}
+
+
 static void
 type_dealloc(PyTypeObject *type)
 {
-    PyHeapTypeObject *et;
     PyObject *tp, *val, *tb;
 
     /* Assert this is a heap-allocated type object */
@@ -4082,8 +4099,8 @@ type_dealloc(PyTypeObject *type)
     PyErr_Fetch(&tp, &val, &tb);
     remove_all_subclasses(type, type->tp_bases);
     PyErr_Restore(tp, val, tb);
+
     PyObject_ClearWeakRefs((PyObject *)type);
-    et = (PyHeapTypeObject *)type;
     Py_XDECREF(type->tp_base);
     Py_XDECREF(type->tp_dict);
     Py_XDECREF(type->tp_bases);
@@ -4094,6 +4111,8 @@ type_dealloc(PyTypeObject *type)
      * of most other objects.  It's okay to cast it to char *.
      */
     PyObject_Free((char *)type->tp_doc);
+
+    PyHeapTypeObject *et = (PyHeapTypeObject *)type;
     Py_XDECREF(et->ht_name);
     Py_XDECREF(et->ht_qualname);
     Py_XDECREF(et->ht_slots);

Programs/_testembed.c

@@ -15,12 +15,18 @@
 #include <stdlib.h>               // putenv()
 #include <wchar.h>
 
+int main_argc;
+char **main_argv;
+
 /*********************************************************
  * Embedded interpreter tests that need a custom exe
  *
  * Executed via 'EmbeddingTests' in Lib/test/test_capi.py
  *********************************************************/
 
+// Use to display the usage
+#define PROGRAM "test_embed"
+
 /* Use path starting with "./" avoids a search along the PATH */
 #define PROGRAM_NAME L"./_testembed"
 
@@ -113,6 +119,36 @@ PyInit_embedded_ext(void)
     return PyModule_Create(&embedded_ext);
 }
 
+/****************************************************************************
+ * Call Py_Initialize()/Py_Finalize() multiple times and execute Python code
+ ***************************************************************************/
+
+// Used by bpo-46417 to test that structseq types used by the sys module are
+// cleared properly and initialized again properly when Python is finalized
+// multiple times.
+static int test_repeated_init_exec(void)
+{
+    if (main_argc < 3) {
+        fprintf(stderr, "usage: %s test_repeated_init_exec CODE\n", PROGRAM);
+        exit(1);
+    }
+    const char *code = main_argv[2];
+
+    for (int i=1; i <= INIT_LOOPS; i++) {
+        fprintf(stderr, "--- Loop #%d ---\n", i);
+        fflush(stderr);
+
+        _testembed_Py_Initialize();
+        int err = PyRun_SimpleString(code);
+        Py_Finalize();
+        if (err) {
+            return 1;
+        }
+    }
+    return 0;
+}
+
+
 /*****************************************************
  * Test forcing a particular IO encoding
  *****************************************************/
@@ -1880,6 +1916,7 @@ struct TestCase
 
 static struct TestCase TestCases[] = {
     // Python initialization
+    {"test_repeated_init_exec", test_repeated_init_exec},
     {"test_forced_io_encoding", test_forced_io_encoding},
     {"test_repeated_init_and_subinterpreters", test_repeated_init_and_subinterpreters},
     {"test_repeated_init_and_inittab", test_repeated_init_and_inittab},
@@ -1946,6 +1983,9 @@ static struct TestCase TestCases[] = {
 
 int main(int argc, char *argv[])
 {
+    main_argc = argc;
+    main_argv = argv;
+
     if (argc > 1) {
         for (struct TestCase *tc = TestCases; tc && tc->name; tc++) {
             if (strcmp(argv[1], tc->name) == 0)

Python/errors.c

@@ -1241,6 +1241,17 @@ _PyErr_InitTypes(PyInterpreterState *interp)
 }
 
 
+void
+_PyErr_FiniTypes(PyInterpreterState *interp)
+{
+    if (!_Py_IsMainInterpreter(interp)) {
+        return;
+    }
+
+    _PyStructSequence_FiniType(&UnraisableHookArgsType);
+}
+
+
 static PyObject *
 make_unraisable_hook_args(PyThreadState *tstate, PyObject *exc_type,
                           PyObject *exc_value, PyObject *exc_tb,

Python/pylifecycle.c

@@ -1666,11 +1666,17 @@ flush_std_files(void)
 static void
 finalize_interp_types(PyInterpreterState *interp)
 {
+    _PySys_Fini(interp);
     _PyExc_Fini(interp);
     _PyFrame_Fini(interp);
     _PyAsyncGen_Fini(interp);
     _PyContext_Fini(interp);
+    _PyFloat_FiniType(interp);
+    _PyLong_FiniTypes(interp);
+    _PyThread_FiniType(interp);
+    _PyErr_FiniTypes(interp);
     _PyTypes_Fini(interp);
+
     // Call _PyUnicode_ClearInterned() before _PyDict_Fini() since it uses
     // a dict internally.
     _PyUnicode_ClearInterned(interp);

Python/sysmodule.c

@@ -3102,6 +3102,21 @@ _PySys_Create(PyThreadState *tstate, PyObject **sysmod_p)
 }
 
 
+void
+_PySys_Fini(PyInterpreterState *interp)
+{
+    if (_Py_IsMainInterpreter(interp)) {
+        _PyStructSequence_FiniType(&VersionInfoType);
+        _PyStructSequence_FiniType(&FlagsType);
+#if defined(MS_WINDOWS)
+        _PyStructSequence_FiniType(&WindowsVersionType);
+#endif
+        _PyStructSequence_FiniType(&Hash_InfoType);
+        _PyStructSequence_FiniType(&AsyncGenHooksType);
+    }
+}
+
+
 static PyObject *
 makepathobject(const wchar_t *path, wchar_t delim)
 {

Python/thread.c

@@ -243,3 +243,14 @@ PyThread_GetInfo(void)
     PyStructSequence_SET_ITEM(threadinfo, pos++, value);
     return threadinfo;
 }
+
+
+void
+_PyThread_FiniType(PyInterpreterState *interp)
+{
+    if (!_Py_IsMainInterpreter(interp)) {
+        return;
+    }
+
+    _PyStructSequence_FiniType(&ThreadInfoType);
+}