From f2b70a2c38fc62a0aad2a878913fb504edf01c23 Mon Sep 17 00:00:00 2001 From: Carl Meyer Date: Tue, 4 Oct 2022 16:41:41 -0700 Subject: [PATCH 01/11] gh-91051: allow setting a callback hook on PyType_Modified --- Include/internal/pycore_interp.h | 1 + Include/object.h | 3 ++ Modules/_testcapimodule.c | 87 ++++++++++++++++++++++++++++++++ Objects/typeobject.c | 20 ++++++++ 4 files changed, 111 insertions(+) diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index b21708a388b339..70e74564cccc2f 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -157,6 +157,7 @@ struct _is { struct atexit_state atexit; PyObject *audit_hooks; + void *type_modified_callback; struct _Py_unicode_state unicode; struct _Py_float_state float_state; diff --git a/Include/object.h b/Include/object.h index 75624fe8c77a51..9bb0ef2df27608 100644 --- a/Include/object.h +++ b/Include/object.h @@ -288,6 +288,9 @@ PyAPI_FUNC(PyObject *) PyType_GenericNew(PyTypeObject *, PyObject *, PyObject *); PyAPI_FUNC(unsigned int) PyType_ClearCache(void); PyAPI_FUNC(void) PyType_Modified(PyTypeObject *); +typedef void(*PyType_ModifiedCallback)(PyTypeObject *); +PyAPI_FUNC(void) PyType_SetModifiedCallback(PyType_ModifiedCallback callback); +PyAPI_FUNC(PyType_ModifiedCallback) PyType_GetModifiedCallback(void); /* Generic operations on objects */ PyAPI_FUNC(PyObject *) PyObject_Repr(PyObject *); diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 3d6535f50be957..ddf45b48c91c19 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -5486,6 +5486,92 @@ test_macros(PyObject *self, PyObject *Py_UNUSED(args)) } +static PyObject *g_type_modified_events; +static PyType_ModifiedCallback g_prev_type_modified_callback; + +static void +type_modified_callback(PyTypeObject *type) +{ + assert(PyList_Check(g_type_modified_events)); + PyList_Append(g_type_modified_events, (PyObject *)type); + if (g_prev_type_modified_callback) { + g_prev_type_modified_callback(type); + } +} + +static int +type_modified_assert(Py_ssize_t expected_num_events, + PyTypeObject *expected_last_type) +{ + char buf[512]; + Py_ssize_t actual_num_events = PyList_Size(g_type_modified_events); + if (expected_num_events != actual_num_events) { + snprintf(buf, + 512, + "got %d type modified events, expected %d", + (int)actual_num_events, + (int)expected_num_events); + raiseTestError("test_type_modified_callback", (const char *)&buf); + return -1; + } + PyObject *last_obj = PyList_GetItem(g_type_modified_events, + actual_num_events - 1); + if (!PyType_Check(last_obj)) { + raiseTestError("test_type_modified_callback", "non-type in event list"); + return -1; + } + PyTypeObject *last_type = (PyTypeObject *)last_obj; + if (last_type != expected_last_type) { + snprintf(buf, + 512, + "last type is '%s', expected '%s'", + last_type->tp_name, + expected_last_type->tp_name); + raiseTestError("test_type_modified_callback", (const char *)&buf); + return -1; + } + return 0; +} + + +static PyObject * +test_type_modified_callback(PyObject *self, PyObject *Py_UNUSED(args)) +{ + g_type_modified_events = PyList_New(0); + g_prev_type_modified_callback = PyType_GetModifiedCallback(); + + PyType_Slot type_slots[] = {{0, 0},}; + PyType_Spec type_spec = {"_testcapimodule.test_type", + sizeof(PyObject), + 0, + Py_TPFLAGS_DEFAULT, + type_slots}; + PyTypeObject *type = (PyTypeObject *)PyType_FromSpec(&type_spec); + + if (!type) { + return NULL; + } + + PyType_SetModifiedCallback(type_modified_callback); + if (PyType_GetModifiedCallback() != type_modified_callback) { + raiseTestError("test_type_modified_callback", + "type modified callback is not what we just set it to"); + return NULL; + } + + + PyType_Modified(type); + + if (type_modified_assert(1, type)) { + return NULL; + } + + PyType_SetModifiedCallback(g_prev_type_modified_callback); + Py_CLEAR(g_type_modified_events); + Py_RETURN_NONE; +} + + static PyObject *test_buildvalue_issue38913(PyObject *, PyObject *); static PyObject *getargs_s_hash_int(PyObject *, PyObject *, PyObject*); static PyObject *getargs_s_hash_int2(PyObject *, PyObject *, PyObject*); @@ -5762,6 +5848,7 @@ static PyMethodDef TestMethods[] = { {"settrace_to_record", settrace_to_record, METH_O, NULL}, {"test_macros", test_macros, METH_NOARGS, NULL}, {"clear_managed_dict", clear_managed_dict, METH_O, NULL}, + {"test_type_modified_callback", test_type_modified_callback, METH_NOARGS, NULL}, {NULL, NULL} /* sentinel */ }; diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 196a6aee4993b8..49b0f2bd12e9e4 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -372,6 +372,20 @@ _PyTypes_Fini(PyInterpreterState *interp) static PyObject * lookup_subclasses(PyTypeObject *); +void +PyType_SetModifiedCallback(PyType_ModifiedCallback callback) +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); + interp->type_modified_callback = (void *)callback; +} + +PyType_ModifiedCallback +PyType_GetModifiedCallback(void) +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); + return (PyType_ModifiedCallback)interp->type_modified_callback; +} + void PyType_Modified(PyTypeObject *type) { @@ -390,6 +404,12 @@ PyType_Modified(PyTypeObject *type) We don't assign new version tags eagerly, but only as needed. */ + PyInterpreterState *interp = _PyInterpreterState_GET(); + PyType_ModifiedCallback cb = (PyType_ModifiedCallback)interp->type_modified_callback; + if (cb) { + cb(type); + } + if (!_PyType_HasFeature(type, Py_TPFLAGS_VALID_VERSION_TAG)) { return; } From 1a28a4595ecff52916ff86391802e0517042ce54 Mon Sep 17 00:00:00 2001 From: Carl Meyer Date: Wed, 5 Oct 2022 10:43:45 -0700 Subject: [PATCH 02/11] Add NEWS blurb. --- .../next/C API/2022-10-05-10-43-32.gh-issue-91051.ODDRsQ.rst | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 Misc/NEWS.d/next/C API/2022-10-05-10-43-32.gh-issue-91051.ODDRsQ.rst diff --git a/Misc/NEWS.d/next/C API/2022-10-05-10-43-32.gh-issue-91051.ODDRsQ.rst b/Misc/NEWS.d/next/C API/2022-10-05-10-43-32.gh-issue-91051.ODDRsQ.rst new file mode 100644 index 00000000000000..b39855e3a024dd --- /dev/null +++ b/Misc/NEWS.d/next/C API/2022-10-05-10-43-32.gh-issue-91051.ODDRsQ.rst @@ -0,0 +1,2 @@ +Add :c:func:`PyType_SetModifiedCallback` to allow registering a callback to +be called whenever :c:func:`PyType_Modified` is called. From 95e49eeae3abc750f20455be8af4fae751475f44 Mon Sep 17 00:00:00 2001 From: Carl Meyer Date: Wed, 5 Oct 2022 10:46:52 -0700 Subject: [PATCH 03/11] Move new API to Include/cpython so it's not in limited API --- Include/cpython/object.h | 4 ++++ Include/object.h | 3 --- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/Include/cpython/object.h b/Include/cpython/object.h index c80fc1df0e0ba4..a7c268e79f8892 100644 --- a/Include/cpython/object.h +++ b/Include/cpython/object.h @@ -510,3 +510,7 @@ Py_DEPRECATED(3.11) typedef int UsingDeprecatedTrashcanMacro; PyAPI_FUNC(int) _PyObject_VisitManagedDict(PyObject *obj, visitproc visit, void *arg); PyAPI_FUNC(void) _PyObject_ClearManagedDict(PyObject *obj); + +typedef void(*PyType_ModifiedCallback)(PyTypeObject *); +PyAPI_FUNC(void) PyType_SetModifiedCallback(PyType_ModifiedCallback callback); +PyAPI_FUNC(PyType_ModifiedCallback) PyType_GetModifiedCallback(void); diff --git a/Include/object.h b/Include/object.h index 9bb0ef2df27608..75624fe8c77a51 100644 --- a/Include/object.h +++ b/Include/object.h @@ -288,9 +288,6 @@ PyAPI_FUNC(PyObject *) PyType_GenericNew(PyTypeObject *, PyObject *, PyObject *); PyAPI_FUNC(unsigned int) PyType_ClearCache(void); PyAPI_FUNC(void) PyType_Modified(PyTypeObject *); -typedef void(*PyType_ModifiedCallback)(PyTypeObject *); -PyAPI_FUNC(void) PyType_SetModifiedCallback(PyType_ModifiedCallback callback); -PyAPI_FUNC(PyType_ModifiedCallback) PyType_GetModifiedCallback(void); /* Generic operations on objects */ PyAPI_FUNC(PyObject *) PyObject_Repr(PyObject *); From cb71ae92603e36c71490f62e4897b4f5490a0942 Mon Sep 17 00:00:00 2001 From: Carl Meyer Date: Sat, 8 Oct 2022 11:31:53 -0700 Subject: [PATCH 04/11] Multiple watchers and per-type watching --- Doc/c-api/type.rst | 42 +++++++++ Doc/whatsnew/3.12.rst | 6 ++ Include/cpython/object.h | 13 ++- Include/internal/pycore_interp.h | 2 +- Lib/test/test_capi.py | 133 ++++++++++++++++++++++++++++ Modules/_testcapimodule.c | 144 ++++++++++++++++++++----------- Objects/typeobject.c | 102 ++++++++++++++++++---- 7 files changed, 371 insertions(+), 71 deletions(-) diff --git a/Doc/c-api/type.rst b/Doc/c-api/type.rst index 1dc05001adfa37..105562cfc1b185 100644 --- a/Doc/c-api/type.rst +++ b/Doc/c-api/type.rst @@ -57,6 +57,48 @@ Type Objects modification of the attributes or base classes of the type. +.. c:function:: int PyType_AddWatcher(PyType_WatchCallback callback) + + Register *callback* as a type watcher. Return a non-negative integer ID + which must be passed to future calls to :c:func:`PyType_Watch`. In case of + error (e.g. no more watcher IDs available), return ``-1`` and set an + exception. + + .. versionadded:: 3.12 + + +.. c:function:: int PyType_ClearWatcher(int watcher_id) + + Clear watcher identified by *watcher_id* (previously returned from + :c:func:`PyType_AddWatcher`). Return ``0`` on success, ``-1`` on error (e.g. + if *watcher_id* was never registered.) + + .. versionadded:: 3.12 + + +.. c:function:: int PyType_Watch(int watcher_id, PyObject *type) + + Mark *type* as watched. The callback granted *watcher_id* by + :c:func:`PyType_AddWatcher` will be called whenever + :c:func:`PyType_Modified` reports a change to *type*. (The callback may be + called only once for a series of consecutive modifications to *type*, if + :c:func:`PyType_Lookup` is not called on *type* between the modifications; + this is an implementation detail and subject to change.) + + .. versionadded:: 3.12 + + +.. c:function:: int (*PyType_WatchCallback)(PyObject *type) + + Type of a type-watcher callback function. + + The callback must not modify *type* or cause :c:func:`PyType_Modified` to be + called on *type* or any type in its MRO; violating this rule could cause + infinite recursion. + + .. versionadded:: 3.12 + + .. c:function:: int PyType_HasFeature(PyTypeObject *o, int feature) Return non-zero if the type object *o* sets the feature *feature*. diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 341e85103a3cf7..fb0d94da42857e 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -557,6 +557,12 @@ New Features :c:func:`PyDict_AddWatch` and related APIs to be called whenever a dictionary is modified. This is intended for use by optimizing interpreters, JIT compilers, or debuggers. + (Contributed by Carl Meyer in :gh:`31787`.) + +* Added :c:func:`PyType_AddWatcher` and :c:func:`PyType_Watch` API to register + callbacks to receive notification on changes to a type. + (Contributed by Carl Meyer in :gh:`97875`.) + Porting to Python 3.12 ---------------------- diff --git a/Include/cpython/object.h b/Include/cpython/object.h index a7c268e79f8892..900b52321dff9f 100644 --- a/Include/cpython/object.h +++ b/Include/cpython/object.h @@ -224,6 +224,9 @@ struct _typeobject { destructor tp_finalize; vectorcallfunc tp_vectorcall; + + /* bitset of which type-watchers care about this type */ + char tp_watched; }; /* This struct is used by the specializer @@ -511,6 +514,10 @@ Py_DEPRECATED(3.11) typedef int UsingDeprecatedTrashcanMacro; PyAPI_FUNC(int) _PyObject_VisitManagedDict(PyObject *obj, visitproc visit, void *arg); PyAPI_FUNC(void) _PyObject_ClearManagedDict(PyObject *obj); -typedef void(*PyType_ModifiedCallback)(PyTypeObject *); -PyAPI_FUNC(void) PyType_SetModifiedCallback(PyType_ModifiedCallback callback); -PyAPI_FUNC(PyType_ModifiedCallback) PyType_GetModifiedCallback(void); +#define TYPE_MAX_WATCHERS 8 + +typedef int(*PyType_WatchCallback)(PyTypeObject *); +PyAPI_FUNC(int) PyType_AddWatcher(PyType_WatchCallback callback); +PyAPI_FUNC(int) PyType_ClearWatcher(int watcher_id); +PyAPI_FUNC(int) PyType_Watch(int watcher_id, PyObject *type); +PyAPI_FUNC(int) PyType_Unwatch(int watcher_id, PyObject *type); diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index 0dae6a019145fe..8a1ae27b3dd045 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -161,7 +161,7 @@ struct _is { struct atexit_state atexit; PyObject *audit_hooks; - void *type_modified_callback; + PyType_WatchCallback type_watchers[TYPE_MAX_WATCHERS]; struct _Py_unicode_state unicode; struct _Py_float_state float_state; diff --git a/Lib/test/test_capi.py b/Lib/test/test_capi.py index 17425050ce00c0..6ebc4190559a24 100644 --- a/Lib/test/test_capi.py +++ b/Lib/test/test_capi.py @@ -1556,5 +1556,138 @@ def test_clear_unassigned_watcher_id(self): self.clear_watcher(1) +class TestTypeWatchers(unittest.TestCase): + # types of watchers testcapimodule can add: + TYPES = 0 # appends modified types to global event list + ERROR = 1 # unconditionally sets and signals a RuntimeException + WRAP = 2 # appends modified type wrapped in list to global event list + + def add_watcher(self, kind=TYPES): + return _testcapi.add_type_watcher(kind) + + def clear_watcher(self, watcher_id): + _testcapi.clear_type_watcher(watcher_id) + + @contextmanager + def watcher(self, kind=TYPES): + wid = self.add_watcher(kind) + try: + yield wid + finally: + self.clear_watcher(wid) + + def assert_events(self, expected): + actual = _testcapi.get_type_modified_events() + self.assertEqual(actual, expected) + + def watch(self, wid, t): + _testcapi.watch_type(wid, t) + + def unwatch(self, wid, t): + _testcapi.unwatch_type(wid, t) + + def test_watch_type(self): + class C: pass + with self.watcher() as wid: + self.watch(wid, C) + C.foo = "bar" + self.assert_events([C]) + + def test_event_aggregation(self): + class C: pass + with self.watcher() as wid: + self.watch(wid, C) + C.foo = "bar" + C.bar = "baz" + # only one event registered for both modifications + self.assert_events([C]) + + def test_lookup_resets_aggregation(self): + class C: pass + with self.watcher() as wid: + self.watch(wid, C) + C.foo = "bar" + # lookup resets type version tag + self.assertEqual(C.foo, "bar") + C.bar = "baz" + # both events registered + self.assert_events([C, C]) + + def test_unwatch_type(self): + class C: pass + with self.watcher() as wid: + self.watch(wid, C) + C.foo = "bar" + self.assertEqual(C.foo, "bar") + self.assert_events([C]) + self.unwatch(wid, C) + C.bar = "baz" + self.assert_events([C]) + + def test_watch_type_subclass(self): + class C: pass + class D(C): pass + with self.watcher() as wid: + self.watch(wid, D) + C.foo = "bar" + self.assert_events([D]) + + def test_error(self): + class C: pass + with self.watcher(kind=self.ERROR) as wid: + self.watch(wid, C) + with catch_unraisable_exception() as cm: + C.foo = "bar" + self.assertIs(cm.unraisable.object, C) + self.assertEqual(str(cm.unraisable.exc_value), "boom!") + self.assert_events([]) + + def test_two_watchers(self): + class C1: pass + class C2: pass + with self.watcher() as wid1: + with self.watcher(kind=self.WRAP) as wid2: + self.watch(wid1, C1) + self.watch(wid2, C2) + C1.foo = "bar" + C2.hmm = "baz" + self.assert_events([C1, [C2]]) + + def test_watch_non_type(self): + with self.watcher() as wid: + with self.assertRaisesRegex(ValueError, r"Cannot watch non-type"): + self.watch(wid, 1) + + def test_watch_out_of_range_watcher_id(self): + class C: pass + with self.assertRaisesRegex(ValueError, r"Invalid type watcher ID -1"): + self.watch(-1, C) + with self.assertRaisesRegex(ValueError, r"Invalid type watcher ID 8"): + self.watch(8, C) # TYPE_MAX_WATCHERS = 8 + + def test_watch_unassigned_watcher_id(self): + class C: pass + with self.assertRaisesRegex(ValueError, r"No type watcher set for ID 1"): + self.watch(1, C) + + def test_unwatch_non_type(self): + with self.watcher() as wid: + with self.assertRaisesRegex(ValueError, r"Cannot watch non-type"): + self.unwatch(wid, 1) + + def test_unwatch_out_of_range_watcher_id(self): + class C: pass + with self.assertRaisesRegex(ValueError, r"Invalid type watcher ID -1"): + self.unwatch(-1, C) + with self.assertRaisesRegex(ValueError, r"Invalid type watcher ID 8"): + self.unwatch(8, C) # TYPE_MAX_WATCHERS = 8 + + def test_unwatch_unassigned_watcher_id(self): + class C: pass + with self.assertRaisesRegex(ValueError, r"No type watcher set for ID 1"): + self.unwatch(1, C) + + + if __name__ == "__main__": unittest.main() diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index ad01a058aa6827..5ab231cfd4398f 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -5639,88 +5639,124 @@ test_macros(PyObject *self, PyObject *Py_UNUSED(args)) } +// type watchers + static PyObject *g_type_modified_events; -static PyType_ModifiedCallback g_prev_type_modified_callback; +static int g_type_watchers_installed; -static void +static int type_modified_callback(PyTypeObject *type) { assert(PyList_Check(g_type_modified_events)); - PyList_Append(g_type_modified_events, (PyObject *)type); - if (g_prev_type_modified_callback) { - g_prev_type_modified_callback(type); + if(PyList_Append(g_type_modified_events, (PyObject *)type) < 0) { + return -1; } + return 0; } static int -type_modified_assert(Py_ssize_t expected_num_events, - PyTypeObject *expected_last_type) -{ - char buf[512]; - Py_ssize_t actual_num_events = PyList_Size(g_type_modified_events); - if (expected_num_events != actual_num_events) { - snprintf(buf, - 512, - "got %d type modified events, expected %d", - (int)actual_num_events, - (int)expected_num_events); - raiseTestError("test_type_modified_callback", (const char *)&buf); +type_modified_callback_wrap(PyTypeObject *type) +{ + assert(PyList_Check(g_type_modified_events)); + PyObject *list = PyList_New(0); + if (!list) { return -1; } - PyObject *last_obj = PyList_GetItem(g_type_modified_events, - actual_num_events - 1); - if (!PyType_Check(last_obj)) { - raiseTestError("test_type_modified_callback", "non-type in event list"); + if (PyList_Append(list, (PyObject *)type) < 0) { + Py_DECREF(list); return -1; } - PyTypeObject *last_type = (PyTypeObject *)last_obj; - if (last_type != expected_last_type) { - snprintf(buf, - 512, - "last type is '%s', expected '%s'", - last_type->tp_name, - expected_last_type->tp_name); - raiseTestError("test_type_modified_callback", (const char *)&buf); + if (PyList_Append(g_type_modified_events, list) < 0) { + Py_DECREF(list); return -1; } + Py_DECREF(list); return 0; } +static int +type_modified_callback_error(PyTypeObject *type) +{ + PyErr_SetString(PyExc_RuntimeError, "boom!"); + return -1; +} static PyObject * -test_type_modified_callback(PyObject *self, PyObject *Py_UNUSED(args)) +add_type_watcher(PyObject *self, PyObject *kind) { - g_type_modified_events = PyList_New(0); - g_prev_type_modified_callback = PyType_GetModifiedCallback(); - - PyType_Slot type_slots[] = {{0, 0},}; - PyType_Spec type_spec = {"_testcapimodule.test_type", - sizeof(PyObject), - 0, - Py_TPFLAGS_DEFAULT, - type_slots}; - PyTypeObject *type = (PyTypeObject *)PyType_FromSpec(&type_spec); - - if (!type) { + int watcher_id; + assert(PyLong_Check(kind)); + long kind_l = PyLong_AsLong(kind); + if (kind_l == 2) { + watcher_id = PyType_AddWatcher(type_modified_callback_wrap); + } else if (kind_l == 1) { + watcher_id = PyType_AddWatcher(type_modified_callback_error); + } else { + watcher_id = PyType_AddWatcher(type_modified_callback); + } + if (watcher_id < 0) { return NULL; } + if (!g_type_watchers_installed) { + assert(!g_type_modified_events); + if (!(g_type_modified_events = PyList_New(0))) { + return NULL; + } + } + g_type_watchers_installed++; + return PyLong_FromLong(watcher_id); +} - PyType_SetModifiedCallback(type_modified_callback); - if (PyType_GetModifiedCallback() != type_modified_callback) { - raiseTestError("test_type_modified_callback", - "type modified callback is not what we just set it to"); +static PyObject * +clear_type_watcher(PyObject *self, PyObject *watcher_id) +{ + if (PyType_ClearWatcher(PyLong_AsLong(watcher_id))) { return NULL; } + g_type_watchers_installed--; + if (!g_type_watchers_installed) { + assert(g_type_modified_events); + Py_CLEAR(g_type_modified_events); + } + Py_RETURN_NONE; +} +static PyObject * +get_type_modified_events(PyObject *self, PyObject *Py_UNUSED(args)) +{ + if (!g_type_modified_events) { + PyErr_SetString(PyExc_RuntimeError, "no watchers active"); + return NULL; + } + Py_INCREF(g_type_modified_events); + return g_type_modified_events; +} - PyType_Modified(type); - - if (type_modified_assert(1, type)) { +static PyObject * +watch_type(PyObject *self, PyObject *args) +{ + PyObject *type; + int watcher_id; + if (!PyArg_ParseTuple(args, "iO", &watcher_id, &type)) { return NULL; } + if (PyType_Watch(watcher_id, type)) { + return NULL; + } + Py_RETURN_NONE; +} - PyType_SetModifiedCallback(g_prev_type_modified_callback); - Py_CLEAR(g_type_modified_events); +static PyObject * +unwatch_type(PyObject *self, PyObject *args) +{ + PyObject *type; + int watcher_id; + if (!PyArg_ParseTuple(args, "iO", &watcher_id, &type)) { + return NULL; + } + if (PyType_Unwatch(watcher_id, type)) { + return NULL; + } Py_RETURN_NONE; } @@ -6006,7 +6042,11 @@ static PyMethodDef TestMethods[] = { {"watch_dict", watch_dict, METH_VARARGS, NULL}, {"unwatch_dict", unwatch_dict, METH_VARARGS, NULL}, {"get_dict_watcher_events", get_dict_watcher_events, METH_NOARGS, NULL}, - {"test_type_modified_callback", test_type_modified_callback, METH_NOARGS, NULL}, + {"add_type_watcher", add_type_watcher, METH_O, NULL}, + {"clear_type_watcher", clear_type_watcher, METH_O, NULL}, + {"watch_type", watch_type, METH_VARARGS, NULL}, + {"unwatch_type", unwatch_type, METH_VARARGS, NULL}, + {"get_type_modified_events", get_type_modified_events, METH_NOARGS, NULL}, {NULL, NULL} /* sentinel */ }; diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 49b0f2bd12e9e4..1b0c70dea79970 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -372,18 +372,81 @@ _PyTypes_Fini(PyInterpreterState *interp) static PyObject * lookup_subclasses(PyTypeObject *); -void -PyType_SetModifiedCallback(PyType_ModifiedCallback callback) +int +PyType_AddWatcher(PyType_WatchCallback callback) +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); + + for (int i = 0; i < TYPE_MAX_WATCHERS; i++) { + if (!interp->type_watchers[i]) { + interp->type_watchers[i] = callback; + return i; + } + } + + PyErr_SetString(PyExc_RuntimeError, "no more type watcher IDs available"); + return -1; +} + +static inline int +validate_watcher_id(PyInterpreterState *interp, int watcher_id) +{ + if (watcher_id < 0 || watcher_id >= TYPE_MAX_WATCHERS) { + PyErr_Format(PyExc_ValueError, "Invalid type watcher ID %d", watcher_id); + return -1; + } + if (!interp->type_watchers[watcher_id]) { + PyErr_Format(PyExc_ValueError, "No type watcher set for ID %d", watcher_id); + return -1; + } + return 0; +} + +int +PyType_ClearWatcher(int watcher_id) +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); + if (validate_watcher_id(interp, watcher_id)) { + return -1; + } + interp->type_watchers[watcher_id] = NULL; + return 0; +} + +static int assign_version_tag(PyTypeObject *type); + +int +PyType_Watch(int watcher_id, PyObject* obj) { + if (!PyType_Check(obj)) { + PyErr_SetString(PyExc_ValueError, "Cannot watch non-type"); + return -1; + } + PyTypeObject *type = (PyTypeObject *)obj; PyInterpreterState *interp = _PyInterpreterState_GET(); - interp->type_modified_callback = (void *)callback; + if (validate_watcher_id(interp, watcher_id)) { + return -1; + } + // ensure we will get a callback on the next modification + assign_version_tag(type); + type->tp_watched |= (1 << watcher_id); + return 0; } -PyType_ModifiedCallback -PyType_GetModifiedCallback(void) +int +PyType_Unwatch(int watcher_id, PyObject* obj) { + if (!PyType_Check(obj)) { + PyErr_SetString(PyExc_ValueError, "Cannot watch non-type"); + return -1; + } + PyTypeObject *type = (PyTypeObject *)obj; PyInterpreterState *interp = _PyInterpreterState_GET(); - return (PyType_ModifiedCallback)interp->type_modified_callback; + if (validate_watcher_id(interp, watcher_id)) { + return -1; + } + type->tp_watched &= ~(1 << watcher_id); + return 0; } void @@ -404,12 +467,6 @@ PyType_Modified(PyTypeObject *type) We don't assign new version tags eagerly, but only as needed. */ - PyInterpreterState *interp = _PyInterpreterState_GET(); - PyType_ModifiedCallback cb = (PyType_ModifiedCallback)interp->type_modified_callback; - if (cb) { - cb(type); - } - if (!_PyType_HasFeature(type, Py_TPFLAGS_VALID_VERSION_TAG)) { return; } @@ -429,6 +486,21 @@ PyType_Modified(PyTypeObject *type) } } + if (type->tp_watched) { + PyInterpreterState *interp = _PyInterpreterState_GET(); + char bits = type->tp_watched; + for (int i = 0; i < TYPE_MAX_WATCHERS; i++) { + if (bits & 1) { + PyType_WatchCallback cb = interp->type_watchers[i]; + if (cb && (cb(type) < 0)) { + PyErr_WriteUnraisable((PyObject *)type); + } + } + bits >>= 1; + } + } + + type->tp_flags &= ~Py_TPFLAGS_VALID_VERSION_TAG; type->tp_version_tag = 0; /* 0 is not a valid version tag */ } @@ -487,7 +559,7 @@ type_mro_modified(PyTypeObject *type, PyObject *bases) { } static int -assign_version_tag(struct type_cache *cache, PyTypeObject *type) +assign_version_tag(PyTypeObject *type) { /* Ensure that the tp_version_tag is valid and set Py_TPFLAGS_VALID_VERSION_TAG. To respect the invariant, this @@ -512,7 +584,7 @@ assign_version_tag(struct type_cache *cache, PyTypeObject *type) Py_ssize_t n = PyTuple_GET_SIZE(bases); for (Py_ssize_t i = 0; i < n; i++) { PyObject *b = PyTuple_GET_ITEM(bases, i); - if (!assign_version_tag(cache, _PyType_CAST(b))) + if (!assign_version_tag(_PyType_CAST(b))) return 0; } type->tp_flags |= Py_TPFLAGS_VALID_VERSION_TAG; @@ -4131,7 +4203,7 @@ _PyType_Lookup(PyTypeObject *type, PyObject *name) return NULL; } - if (MCACHE_CACHEABLE_NAME(name) && assign_version_tag(cache, type)) { + if (MCACHE_CACHEABLE_NAME(name) && assign_version_tag(type)) { h = MCACHE_HASH_METHOD(type, name); struct type_cache_entry *entry = &cache->hashtable[h]; entry->version = type->tp_version_tag; From ab1606c454f0ca7215387b82c2b260da145669a8 Mon Sep 17 00:00:00 2001 From: Carl Meyer Date: Sat, 8 Oct 2022 11:36:29 -0700 Subject: [PATCH 05/11] Update NEWS blurb --- .../next/C API/2022-10-05-10-43-32.gh-issue-91051.ODDRsQ.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Misc/NEWS.d/next/C API/2022-10-05-10-43-32.gh-issue-91051.ODDRsQ.rst b/Misc/NEWS.d/next/C API/2022-10-05-10-43-32.gh-issue-91051.ODDRsQ.rst index b39855e3a024dd..c18e2d61f3978b 100644 --- a/Misc/NEWS.d/next/C API/2022-10-05-10-43-32.gh-issue-91051.ODDRsQ.rst +++ b/Misc/NEWS.d/next/C API/2022-10-05-10-43-32.gh-issue-91051.ODDRsQ.rst @@ -1,2 +1,2 @@ -Add :c:func:`PyType_SetModifiedCallback` to allow registering a callback to -be called whenever :c:func:`PyType_Modified` is called. +Add :c:func:`PyType_Watch` and related APIs to allow callbacks on +:c:func:`PyType_Modified`. From 26ddee2fc861b6c849048821d1f03865b8928e8d Mon Sep 17 00:00:00 2001 From: Carl Meyer Date: Sat, 8 Oct 2022 11:48:42 -0700 Subject: [PATCH 06/11] Fix doc build issues --- Doc/c-api/type.rst | 2 +- Doc/whatsnew/3.12.rst | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/Doc/c-api/type.rst b/Doc/c-api/type.rst index 105562cfc1b185..dea6db0be37bd0 100644 --- a/Doc/c-api/type.rst +++ b/Doc/c-api/type.rst @@ -88,7 +88,7 @@ Type Objects .. versionadded:: 3.12 -.. c:function:: int (*PyType_WatchCallback)(PyObject *type) +.. c:type:: int (*PyType_WatchCallback)(PyObject *type) Type of a type-watcher callback function. diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index fb0d94da42857e..5a33ad7cdf9c95 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -557,7 +557,6 @@ New Features :c:func:`PyDict_AddWatch` and related APIs to be called whenever a dictionary is modified. This is intended for use by optimizing interpreters, JIT compilers, or debuggers. - (Contributed by Carl Meyer in :gh:`31787`.) * Added :c:func:`PyType_AddWatcher` and :c:func:`PyType_Watch` API to register callbacks to receive notification on changes to a type. From 4783e07be563b17daacddb79b41a53563e577711 Mon Sep 17 00:00:00 2001 From: Carl Meyer Date: Sat, 8 Oct 2022 12:38:11 -0700 Subject: [PATCH 07/11] Fix typeobject size test --- Lib/test/test_sys.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index 41482734872e06..9184e9a42f1941 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -1521,7 +1521,7 @@ def delx(self): del self.__x check((1,2,3), vsize('') + 3*self.P) # type # static type: PyTypeObject - fmt = 'P2nPI13Pl4Pn9Pn12PIP' + fmt = 'P2nPI13Pl4Pn9Pn12PIPc' s = vsize('2P' + fmt) check(int, s) # class From 21f42ee9a029ffb9a1b51d867136426e05620f9c Mon Sep 17 00:00:00 2001 From: Carl Meyer Date: Sun, 9 Oct 2022 11:47:58 +0200 Subject: [PATCH 08/11] Fix issue number in whatsnew (and add for dict watchers too) --- Doc/whatsnew/3.12.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 8f337704c793c8..391f7243f48f61 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -575,10 +575,11 @@ New Features :c:func:`PyDict_AddWatch` and related APIs to be called whenever a dictionary is modified. This is intended for use by optimizing interpreters, JIT compilers, or debuggers. + (Contributed by Carl Meyer in :gh:`91052`.) * Added :c:func:`PyType_AddWatcher` and :c:func:`PyType_Watch` API to register callbacks to receive notification on changes to a type. - (Contributed by Carl Meyer in :gh:`97875`.) + (Contributed by Carl Meyer in :gh:`91051`.) Porting to Python 3.12 From 2b42bfe2bbc59553f43548c7e7c5c377ff2fa13f Mon Sep 17 00:00:00 2001 From: Carl Meyer Date: Sat, 15 Oct 2022 01:31:39 +0400 Subject: [PATCH 09/11] Type watcher review comments --- Doc/c-api/type.rst | 7 +++++++ Lib/test/test_capi.py | 14 ++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/Doc/c-api/type.rst b/Doc/c-api/type.rst index dea6db0be37bd0..7b5d1fac40ed87 100644 --- a/Doc/c-api/type.rst +++ b/Doc/c-api/type.rst @@ -73,6 +73,10 @@ Type Objects :c:func:`PyType_AddWatcher`). Return ``0`` on success, ``-1`` on error (e.g. if *watcher_id* was never registered.) + An extension should never call ``PyType_ClearWatcher`` with a *watcher_id* + that was not returned to it by a previous call to + :c:func:`PyType_AddWatcher`. + .. versionadded:: 3.12 @@ -85,6 +89,9 @@ Type Objects :c:func:`PyType_Lookup` is not called on *type* between the modifications; this is an implementation detail and subject to change.) + An extension should never call ``PyType_Watch`` with a *watcher_id* that was + not returned to it by a previous call to :c:func:`PyType_AddWatcher`. + .. versionadded:: 3.12 diff --git a/Lib/test/test_capi.py b/Lib/test/test_capi.py index 6ebc4190559a24..f3b3232c4866b3 100644 --- a/Lib/test/test_capi.py +++ b/Lib/test/test_capi.py @@ -1624,6 +1624,19 @@ class C: pass C.bar = "baz" self.assert_events([C]) + def test_clear_watcher(self): + class C: pass + # outer watcher is unused, it's just to keep events list alive + with self.watcher() as _: + with self.watcher() as wid: + self.watch(wid, C) + C.foo = "bar" + self.assertEqual(C.foo, "bar") + self.assert_events([C]) + C.bar = "baz" + # Watcher on C has been cleared, no new event + self.assert_events([C]) + def test_watch_type_subclass(self): class C: pass class D(C): pass @@ -1647,6 +1660,7 @@ class C1: pass class C2: pass with self.watcher() as wid1: with self.watcher(kind=self.WRAP) as wid2: + self.assertNotEqual(wid1, wid2) self.watch(wid1, C1) self.watch(wid2, C2) C1.foo = "bar" From 4329743afb637c4a32d19c0289a9de9092889002 Mon Sep 17 00:00:00 2001 From: Carl Meyer Date: Mon, 17 Oct 2022 14:15:57 -0600 Subject: [PATCH 10/11] Review comments --- Lib/test/test_capi.py | 26 +++++++++++++++++++++++--- Objects/typeobject.c | 4 ++-- 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_capi.py b/Lib/test/test_capi.py index 64b92b0c52fa0c..364c607b3c18b3 100644 --- a/Lib/test/test_capi.py +++ b/Lib/test/test_capi.py @@ -2,7 +2,7 @@ # these are all functions _testcapi exports whose name begins with 'test_'. from collections import OrderedDict -from contextlib import contextmanager +from contextlib import contextmanager, ExitStack import _thread import importlib.machinery import importlib.util @@ -1612,6 +1612,9 @@ class TestTypeWatchers(unittest.TestCase): ERROR = 1 # unconditionally sets and signals a RuntimeException WRAP = 2 # appends modified type wrapped in list to global event list + # duplicating the C constant + TYPE_MAX_WATCHERS = 8 + def add_watcher(self, kind=TYPES): return _testcapi.add_type_watcher(kind) @@ -1727,7 +1730,7 @@ class C: pass with self.assertRaisesRegex(ValueError, r"Invalid type watcher ID -1"): self.watch(-1, C) with self.assertRaisesRegex(ValueError, r"Invalid type watcher ID 8"): - self.watch(8, C) # TYPE_MAX_WATCHERS = 8 + self.watch(self.TYPE_MAX_WATCHERS, C) def test_watch_unassigned_watcher_id(self): class C: pass @@ -1744,13 +1747,30 @@ class C: pass with self.assertRaisesRegex(ValueError, r"Invalid type watcher ID -1"): self.unwatch(-1, C) with self.assertRaisesRegex(ValueError, r"Invalid type watcher ID 8"): - self.unwatch(8, C) # TYPE_MAX_WATCHERS = 8 + self.unwatch(self.TYPE_MAX_WATCHERS, C) def test_unwatch_unassigned_watcher_id(self): class C: pass with self.assertRaisesRegex(ValueError, r"No type watcher set for ID 1"): self.unwatch(1, C) + def test_clear_out_of_range_watcher_id(self): + with self.assertRaisesRegex(ValueError, r"Invalid type watcher ID -1"): + self.clear_watcher(-1) + with self.assertRaisesRegex(ValueError, r"Invalid type watcher ID 8"): + self.clear_watcher(self.TYPE_MAX_WATCHERS) + + def test_clear_unassigned_watcher_id(self): + with self.assertRaisesRegex(ValueError, r"No type watcher set for ID 1"): + self.clear_watcher(1) + + def test_no_more_ids_available(self): + contexts = [self.watcher() for i in range(self.TYPE_MAX_WATCHERS)] + with ExitStack() as stack: + for ctx in contexts: + stack.enter_context(ctx) + with self.assertRaisesRegex(RuntimeError, r"no more type watcher IDs"): + self.add_watcher() if __name__ == "__main__": diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 1b0c70dea79970..b105d405f25417 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -406,7 +406,7 @@ int PyType_ClearWatcher(int watcher_id) { PyInterpreterState *interp = _PyInterpreterState_GET(); - if (validate_watcher_id(interp, watcher_id)) { + if (validate_watcher_id(interp, watcher_id) < 0) { return -1; } interp->type_watchers[watcher_id] = NULL; @@ -424,7 +424,7 @@ PyType_Watch(int watcher_id, PyObject* obj) } PyTypeObject *type = (PyTypeObject *)obj; PyInterpreterState *interp = _PyInterpreterState_GET(); - if (validate_watcher_id(interp, watcher_id)) { + if (validate_watcher_id(interp, watcher_id) < 0) { return -1; } // ensure we will get a callback on the next modification From 22d190b0adced2a90130624e6e172224eb1a7741 Mon Sep 17 00:00:00 2001 From: Carl Meyer Date: Thu, 20 Oct 2022 10:27:09 -0600 Subject: [PATCH 11/11] Short circuit watched bits check if no more bits are set --- Objects/typeobject.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Objects/typeobject.c b/Objects/typeobject.c index b105d405f25417..7f8f2c7846eb01 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -488,14 +488,16 @@ PyType_Modified(PyTypeObject *type) if (type->tp_watched) { PyInterpreterState *interp = _PyInterpreterState_GET(); - char bits = type->tp_watched; - for (int i = 0; i < TYPE_MAX_WATCHERS; i++) { + int bits = type->tp_watched; + int i = 0; + while(bits && i < TYPE_MAX_WATCHERS) { if (bits & 1) { PyType_WatchCallback cb = interp->type_watchers[i]; if (cb && (cb(type) < 0)) { PyErr_WriteUnraisable((PyObject *)type); } } + i += 1; bits >>= 1; } }