bpo-43244: Remove the pyarena.h header (GH-25007)

Remove the pyarena.h header file with functions:

* PyArena_New()
* PyArena_Free()
* PyArena_Malloc()
* PyArena_AddPyObject()

These functions were undocumented, excluded from the limited C API,
and were only used internally by the compiler.

Add pycore_pyarena.h header. Rename functions:

* PyArena_New() => _PyArena_New()
* PyArena_Free() => _PyArena_Free()
* PyArena_Malloc() => _PyArena_Malloc()
* PyArena_AddPyObject() => _PyArena_AddPyObject()

Author: vstinner

Doc/whatsnew/3.10.rst

@@ -1419,3 +1419,14 @@ Removed
 
   These functions were undocumented and excluded from the limited C API.
   (Contributed by Victor Stinner in :issue:`43244`.)
+
+* Remove the ``pyarena.h`` header file with functions:
+
+  * ``PyArena_New()``
+  * ``PyArena_Free()``
+  * ``PyArena_Malloc()``
+  * ``PyArena_AddPyObject()``
+
+  These functions were undocumented, excluded from the limited C API, and were
+  only used internally by the compiler.
+  (Contributed by Victor Stinner in :issue:`43244`.)

Include/Python.h

@@ -137,7 +137,6 @@
 #include "pystate.h"
 #include "context.h"
 
-#include "cpython/pyarena.h"
 #include "modsupport.h"
 #include "compile.h"
 #include "pythonrun.h"

Include/cpython/pyarena.h

@@ -8,6 +8,8 @@ extern "C" {
 #  error "this header requires Py_BUILD_CORE define"
 #endif
 
+#include "pycore_pyarena.h"       // _PyArena_Malloc()
+
 typedef PyObject * identifier;
 typedef PyObject * string;
 typedef PyObject * object;
@@ -65,7 +67,7 @@ asdl_ ## NAME ## _seq *_Py_asdl_ ## NAME ## _seq_new(Py_ssize_t size, PyArena *a
         return NULL; \
     } \
     n += sizeof(asdl_ ## NAME ## _seq); \
-    seq = (asdl_ ## NAME ## _seq *)PyArena_Malloc(arena, n); \
+    seq = (asdl_ ## NAME ## _seq *)_PyArena_Malloc(arena, n); \
     if (!seq) { \
         PyErr_NoMemory(); \
         return NULL; \

Include/internal/pycore_asdl.h

@@ -8,15 +8,16 @@ extern "C" {
 #  error "this header requires Py_BUILD_CORE define"
 #endif
 
-struct _mod;   // Type defined in pycore_ast.h
+struct _arena;   // Type defined in pycore_pyarena.h
+struct _mod;     // Type defined in pycore_ast.h
 
 // Export the symbol for test_peg_generator (built as a library)
 PyAPI_FUNC(PyCodeObject*) _PyAST_Compile(
     struct _mod *mod,
     PyObject *filename,
     PyCompilerFlags *flags,
     int optimize,
-    PyArena *arena);
+    struct _arena *arena);
 extern PyFutureFeatures* _PyFuture_FromAST(
     struct _mod * mod,
     PyObject *filename
@@ -31,7 +32,7 @@ typedef struct {
 
 extern int _PyAST_Optimize(
     struct _mod *,
-    PyArena *arena,
+    struct _arena *arena,
     _PyASTOptimizeState *state);
 
 #ifdef __cplusplus

Include/internal/pycore_compile.h

@@ -0,0 +1,64 @@
+/* An arena-like memory interface for the compiler.
+ */
+
+#ifndef Py_INTERNAL_PYARENA_H
+#define Py_INTERNAL_PYARENA_H
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifndef Py_BUILD_CORE
+#  error "this header requires Py_BUILD_CORE define"
+#endif
+
+typedef struct _arena PyArena;
+
+/* _PyArena_New() and _PyArena_Free() create a new arena and free it,
+   respectively.  Once an arena has been created, it can be used
+   to allocate memory via _PyArena_Malloc().  Pointers to PyObject can
+   also be registered with the arena via _PyArena_AddPyObject(), and the
+   arena will ensure that the PyObjects stay alive at least until
+   _PyArena_Free() is called.  When an arena is freed, all the memory it
+   allocated is freed, the arena releases internal references to registered
+   PyObject*, and none of its pointers are valid.
+   XXX (tim) What does "none of its pointers are valid" mean?  Does it
+   XXX mean that pointers previously obtained via _PyArena_Malloc() are
+   XXX no longer valid?  (That's clearly true, but not sure that's what
+   XXX the text is trying to say.)
+
+   _PyArena_New() returns an arena pointer.  On error, it
+   returns a negative number and sets an exception.
+   XXX (tim):  Not true.  On error, _PyArena_New() actually returns NULL,
+   XXX and looks like it may or may not set an exception (e.g., if the
+   XXX internal PyList_New(0) returns NULL, _PyArena_New() passes that on
+   XXX and an exception is set; OTOH, if the internal
+   XXX block_new(DEFAULT_BLOCK_SIZE) returns NULL, that's passed on but
+   XXX an exception is not set in that case).
+*/
+PyAPI_FUNC(PyArena*) _PyArena_New(void);
+PyAPI_FUNC(void) _PyArena_Free(PyArena *);
+
+/* Mostly like malloc(), return the address of a block of memory spanning
+ * `size` bytes, or return NULL (without setting an exception) if enough
+ * new memory can't be obtained.  Unlike malloc(0), _PyArena_Malloc() with
+ * size=0 does not guarantee to return a unique pointer (the pointer
+ * returned may equal one or more other pointers obtained from
+ * _PyArena_Malloc()).
+ * Note that pointers obtained via _PyArena_Malloc() must never be passed to
+ * the system free() or realloc(), or to any of Python's similar memory-
+ * management functions.  _PyArena_Malloc()-obtained pointers remain valid
+ * until _PyArena_Free(ar) is called, at which point all pointers obtained
+ * from the arena `ar` become invalid simultaneously.
+ */
+PyAPI_FUNC(void*) _PyArena_Malloc(PyArena *, size_t size);
+
+/* This routine isn't a proper arena allocation routine.  It takes
+ * a PyObject* and records it so that it can be DECREFed when the
+ * arena is freed.
+ */
+PyAPI_FUNC(int) _PyArena_AddPyObject(PyArena *, PyObject *);
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* !Py_INTERNAL_PYARENA_H */

Include/internal/pycore_pyarena.h

@@ -1119,7 +1119,6 @@ PYTHON_HEADERS= \
 		$(srcdir)/Include/cpython/objimpl.h \
 		$(srcdir)/Include/cpython/odictobject.h \
 		$(srcdir)/Include/cpython/picklebufobject.h \
-		$(srcdir)/Include/cpython/pyarena.h \
 		$(srcdir)/Include/cpython/pyctype.h \
 		$(srcdir)/Include/cpython/pydebug.h \
 		$(srcdir)/Include/cpython/pyerrors.h \
@@ -1163,6 +1162,7 @@ PYTHON_HEADERS= \
 		$(srcdir)/Include/internal/pycore_long.h \
 		$(srcdir)/Include/internal/pycore_object.h \
 		$(srcdir)/Include/internal/pycore_pathconfig.h \
+		$(srcdir)/Include/internal/pycore_pyarena.h \
 		$(srcdir)/Include/internal/pycore_pyerrors.h \
 		$(srcdir)/Include/internal/pycore_pyhash.h \
 		$(srcdir)/Include/internal/pycore_pylifecycle.h \

Makefile.pre.in

@@ -0,0 +1,10 @@
+Remove the ``pyarena.h`` header file with functions:
+
+* ``PyArena_New()``
+* ``PyArena_Free()``
+* ``PyArena_Malloc()``
+* ``PyArena_AddPyObject()``
+
+These functions were undocumented, excluded from the limited C API, and were
+only used internally by the compiler.
+Patch by Victor Stinner.

Misc/NEWS.d/next/C API/2021-03-24-01-22-14.bpo-43244.31-97x.rst

@@ -201,6 +201,7 @@
     <ClInclude Include="..\Include\internal\pycore_long.h" />
     <ClInclude Include="..\Include\internal\pycore_object.h" />
     <ClInclude Include="..\Include\internal\pycore_pathconfig.h" />
+    <ClInclude Include="..\Include\internal\pycore_pyarena.h" />
     <ClInclude Include="..\Include\internal\pycore_pyerrors.h" />
     <ClInclude Include="..\Include\internal\pycore_pyhash.h" />
     <ClInclude Include="..\Include\internal\pycore_pylifecycle.h" />

PCbuild/pythoncore.vcxproj

@@ -564,6 +564,9 @@
     <ClInclude Include="..\Include\internal\pycore_pathconfig.h">
       <Filter>Include\internal</Filter>
     </ClInclude>
+    <ClInclude Include="..\Include\internal\pycore_pyarena.h">
+      <Filter>Include\internal</Filter>
+    </ClInclude>
     <ClInclude Include="..\Include\internal\pycore_pyerrors.h">
       <Filter>Include\internal</Filter>
     </ClInclude>

PCbuild/pythoncore.vcxproj.filters

@@ -362,7 +362,7 @@ def emit(s, depth=0, reflow=True):
                 emit('return NULL;', 2)
                 emit('}', 1)
 
-        emit("p = (%s)PyArena_Malloc(arena, sizeof(*p));" % ctype, 1);
+        emit("p = (%s)_PyArena_Malloc(arena, sizeof(*p));" % ctype, 1);
         emit("if (!p)", 1)
         emit("return NULL;", 2)
         if union:
@@ -946,7 +946,7 @@ def visitModule(self, mod):
     if (obj == Py_None)
         obj = NULL;
     if (obj) {
-        if (PyArena_AddPyObject(arena, obj) < 0) {
+        if (_PyArena_AddPyObject(arena, obj) < 0) {
             *out = NULL;
             return -1;
         }
@@ -958,7 +958,7 @@ def visitModule(self, mod):
 
 static int obj2ast_constant(struct ast_state *Py_UNUSED(state), PyObject* obj, PyObject** out, PyArena* arena)
 {
-    if (PyArena_AddPyObject(arena, obj) < 0) {
+    if (_PyArena_AddPyObject(arena, obj) < 0) {
         *out = NULL;
         return -1;
     }