GH-107956: install a static build description file

.gitignore

@@ -127,6 +127,8 @@ Tools/unicode/data/
 /.ccache
 /cross-build/
 /jit_stencils.h
+/install-details.toml
+/build-details.json
 /platform
 /profile-clean-stamp
 /profile-run-stamp

Doc/library/sysconfig.rst

@@ -429,6 +429,8 @@ Other functions
 
 .. _sysconfig-cli:
 
+.. _sysconfig-commandline:
+
 Using :mod:`sysconfig` as a script
 ----------------------------------
 

Doc/using/index.rst

@@ -19,3 +19,4 @@ interpreter and things that make working with Python easier.
    windows.rst
    mac.rst
    editors.rst
+   introspection.rst

Doc/using/introspection.rst

@@ -0,0 +1,153 @@
+.. _introspection:
+
+***********************************
+Introspecting a Python installation
+***********************************
+
+The mechanisms available to introspect a Python installation are highly
+dependent on the installation you have. Their availability may depend on the
+Python implementation in question, the build format, the target platform, or
+several other details.
+
+This document aims to give an overview of these mechanisms and provide the user
+a general guide on how to approach this task, but users should still refer to
+the documentation of their installation for more accurate information regarding
+implementation details and availability.
+
+
+Introduction
+============
+
+Generally, the most portable way to do this is by running some Python code to
+extract information from the target interpreter, though this can often be
+automated with the use of built-in helpers (see
+:ref:`introspection-stdlib-clis`, :ref:`introspection-python-config-script`).
+
+However, when introspecting a Python installation, running code is often
+undesirable or impossible. For this purpose, Python defines a standard
+:ref:`static install description file <introspection-install-details>` (please
+see the :ref:`introspection-install-details-format` section for the format
+specification), which may be provided together with your interpreter (please
+see the :ref:`introspection-install-details-location` section for more details).
+When available, this file can be used to lookup certain details from the
+installation in question without having to run the interpreter.
+
+
+Reference
+=========
+
+
+.. _introspection-install-details:
+
+Install details file
+--------------------
+
+
+.. _introspection-install-details-format:
+
+Format
+~~~~~~
+
+The details file is JSON file and the top-level object is a dictionary
+containing the data described in :ref:`introspection-install-details-fields`.
+
+
+.. _introspection-install-details-fields:
+
+Fields
+~~~~~~
+
+This section documents some of the data fields that may be present. It is not
+extensive — different implementations, platforms etc. may include additional
+fields.
+
+Dictionaries represent sub-sections, and their names are separated with
+dots (``.``) in this documentation.
+
+``python.version``
+++++++++++++++++++
+
+:Type: ``str``
+:Description: A string representation of the Python version.
+
+``python.version_parts``
+++++++++++++++++++++++++
+
+:Type: ``dict[str, int | str]``
+:Description: A dictionary containing the different components of the Python
+              version, as found in :py:data:`sys.version_info`.
+
+``python.executable``
++++++++++++++++++++++
+
+:Type: ``str``
+:Description: An absolute path pointing to an interpreter from the installation.
+:Availability: May be omitted in situations where it is unavailable, there isn't
+               a reliable path that can be specified, etc.
+
+``python.stdlib``
++++++++++++++++++++++
+
+:Type: ``str``
+:Description: An absolute path pointing to the directory where the standard
+              library is installed.
+:Availability: May be omitted in situations where it is unavailable, there isn't
+               a reliable path that can be specified, etc.
+
+
+.. _introspection-install-details-location:
+
+Location
+~~~~~~~~
+
+On standard CPython installations, when available, the details file will be
+installed in the same directory as the standard library, with the name
+``build-details.json``.
+
+Different implementations may place it in a different path, choose to provide
+the file via a different mechanism, or choose not to include it at all.
+
+.. note:: For Python implementation maintainers
+
+   It would be good to try to align the install location between
+   implementations, when possible, to help with compatibility.
+
+   We just do not choose a installation path for everyone because different
+   implementations may work differently, and it should be up to each project to
+   decide what makes the most sense for them.
+
+
+.. _introspection-stdlib-clis:
+
+Standard library modules with a CLI
+-----------------------------------
+
+Some standard library modules include a CLI that exposes information, which can
+be helpful for introspection.
+
+- :mod:`sysconfig` (see :ref:`sysconfig-commandline`)
+- :mod:`site` (see :ref:`site-commandline`)
+
+
+.. _introspection-python-config-script:
+
+The ``python-config`` script
+----------------------------
+
+.. TODO: Currently, we don't have any documentation covering python-config, but
+         if/when we add some, we refer to it here, instead.
+
+.. availability:: POSIX
+
+A ``python-config`` script may be available alongside the interpreter
+executable. This script exposes information especially relevant when building C
+extensions.
+
+When using it via ``PATH``, you should be careful with your environment, and
+make sure that the first ``python-config`` entry does in fact belong to the
+correct interpreter.
+
+Additionally, the current implementation does not need to run the interpreter,
+so this script may be helpful in situtation where that is not possible, or
+undesirable. Though, please keep in mind that this is an implementation detail
+and no guarantees are made regarding this aspect of the implementation.

Lib/test/test_install_details_file.py

@@ -0,0 +1,127 @@
+import json
+import os
+import sys
+import sysconfig
+import string
+import unittest
+
+
+class FormatTestsBase:
+    @property
+    def contents(self):
+        """Install details file contents. Should be overriden by subclasses."""
+        raise NotImplementedError
+
+    @property
+    def data(self):
+        """Parsed install details file data, as a Python object."""
+        return json.loads(self.contents)
+
+    def key(self, name):
+        """Helper to fetch subsection entries.
+
+        It takes the entry name, allowing the usage of a dot to separate the
+        different subsection names (eg. specifying 'a.b.c' as the key will
+        return the value of self.data['a']['b']['c']).
+        """
+        value = self.data
+        for part in name.split('.'):
+            value = value[part]
+        return value
+
+    def test_parse(self):
+        self.data
+
+    def test_top_level_container(self):
+        self.assertIsInstance(self.data, dict)
+        for key, value in self.data.items():
+            with self.subTest(section=key):
+                self.assertIsInstance(value, dict)
+
+    def test_python_version(self):
+        allowed_characters = string.digits + string.ascii_letters + '.'
+        value = self.key('python.version')
+
+        self.assertLessEqual(set(value), set(allowed_characters))
+        self.assertTrue(sys.version.startswith(value))
+
+    def test_python_version_parts(self):
+        value = self.key('python.version_parts')
+
+        self.assertEqual(len(value), sys.version_info.n_fields)
+        for part_name, part_value in value.items():
+            with self.subTest(part=part_name):
+                self.assertEqual(part_value, getattr(sys.version_info, part_name))
+
+    def test_python_executable(self):
+        """Test the python.executable entry.
+
+        The generic test wants the key to be missing. If your implementation
+        provides a value for it, you should override this test.
+        """
+        with self.assertRaises(KeyError):
+            self.key('python.executable')
+
+    def test_python_stdlib(self):
+        """Test the python.stdlib entry.
+
+        The generic test wants the key to be missing. If your implementation
+        provides a value for it, you should override this test.
+        """
+        with self.assertRaises(KeyError):
+            self.key('python.stdlib')
+
+
+needs_installed_python = unittest.skipIf(
+    sysconfig.is_python_build(),
+    'This test can only run in an installed Python',
+)
+
+
+@unittest.skipIf(os.name != 'posix', 'Feature only implemented on POSIX right now')
+class CPythonInstallDetailsFileTests(unittest.TestCase, FormatTestsBase):
+    """Test CPython's install details file implementation."""
+
+    @property
+    def location(self):
+        if sysconfig.is_python_build():
+            dirname = sysconfig.get_config_var('projectbase')
+        else:
+            dirname = sysconfig.get_path('stdlib')
+        return os.path.join(dirname, 'build-details.json')
+
+    @property
+    def contents(self):
+        with open(self.location, 'r') as f:
+            return f.read()
+
+    def setUp(self):
+        if sys.platform in ('wasi', 'emscripten') and not os.path.isfile(self.location):
+            self.skipTest(f'{sys.platform} build without a build-details.json file')
+
+    @needs_installed_python
+    def test_location(self):
+        self.assertTrue(os.path.isfile(self.location))
+
+    # Override generic format tests with tests for our specific implemenation.
+
+    @needs_installed_python
+    def test_python_executable(self):
+        value = self.key('python.executable')
+
+        self.assertEqual(os.path.realpath(value), os.path.realpath(sys.executable))
+
+    @needs_installed_python
+    def test_python_stdlib(self):
+        value = self.key('python.stdlib')
+
+        try:
+            stdlib = os.path.dirname(sysconfig.__file__)
+        except AttributeError as exc:
+            self.skipTest(str(exc))
+
+        self.assertEqual(os.path.realpath(value), os.path.realpath(stdlib))
+
+
+if __name__ == "__main__":
+    unittest.main()

Makefile.pre.in

@@ -654,7 +654,7 @@ all:		@DEF_MAKE_ALL_RULE@
 
 .PHONY: build_all
 build_all:	check-clean-src $(BUILDPYTHON) platform sharedmods \
-		gdbhooks Programs/_testembed scripts checksharedmods rundsymutil
+		gdbhooks Programs/_testembed scripts checksharedmods rundsymutil build-details.json
 
 .PHONY: build_wasm
 build_wasm: check-clean-src $(BUILDPYTHON) platform sharedmods \
@@ -831,6 +831,10 @@ $(BUILDPYTHON):	Programs/python.o $(LINK_PYTHON_DEPS)
 platform: $(PYTHON_FOR_BUILD_DEPS) pybuilddir.txt
 	$(RUNSHARED) $(PYTHON_FOR_BUILD) -c 'import sys ; from sysconfig import get_platform ; print("%s-%d.%d" % (get_platform(), *sys.version_info[:2]))' >platform
 
+build-details.json: $(PYTHON_FOR_BUILD_DEPS) pybuilddir.txt $(srcdir)/Tools/build/generate_install_details_file.py
+	$(RUNSHARED) $(PYTHON_FOR_BUILD) $(srcdir)/Tools/build/generate_build_details_file.py ./build-details.json \
+		--executable $(BINDIR)/python3$(EXE)
+
 # Create build directory and generate the sysconfig build-time data there.
 # pybuilddir.txt contains the name of the build dir and is used for
 # sys.path fixup -- see Modules/getpath.c.
@@ -2042,7 +2046,7 @@ altinstall: commoninstall
 .PHONY: commoninstall
 commoninstall:  check-clean-src @FRAMEWORKALTINSTALLFIRST@ \
 		altbininstall libinstall inclinstall libainstall \
-		sharedinstall altmaninstall @FRAMEWORKALTINSTALLLAST@
+		sharedinstall altmaninstall descfileinstall @FRAMEWORKALTINSTALLLAST@
 
 # Install shared libraries enabled by Setup
 DESTDIRS=	$(exec_prefix) $(LIBDIR) $(BINLIBDEST) $(DESTSHARED)
@@ -2671,6 +2675,11 @@ frameworkaltinstallunixtools:
 frameworkinstallextras:
 	cd Mac && $(MAKE) installextras DESTDIR="$(DESTDIR)"
 
+# Install the
+.PHONY: descfileinstall
+descfileinstall: libinstall
+	$(INSTALL) -m $(FILEMODE) ./build-details.json $(DESTDIR)$(LIBDEST)/build-details.json
+
 # Build the toplevel Makefile
 Makefile.pre: $(srcdir)/Makefile.pre.in config.status
 	CONFIG_FILES=Makefile.pre CONFIG_HEADERS= ./config.status
@@ -2787,6 +2796,7 @@ clean-retain-profile: pycremoval
 	find build -name '*.py' -exec rm -f {} ';' || true
 	find build -name '*.py[co]' -exec rm -f {} ';' || true
 	-rm -f pybuilddir.txt
+	-rm -f build-details.json
 	-rm -f _bootstrap_python
 	-rm -f python.html python*.js python.data python*.symbols python*.map
 	-rm -f $(WASM_STDLIB)

Misc/NEWS.d/next/Build/2023-10-05-18-56-06.gh-issue-107956.ZC5KRw.rst

@@ -0,0 +1,3 @@
+A ``build-details.json`` file containing information regarding the Python
+build/installation is now installed in the same directory as the standard
+library.