diff --git a/.gitignore b/.gitignore index 6ed7197e3ab626..44adade58ef189 100644 --- a/.gitignore +++ b/.gitignore @@ -127,6 +127,8 @@ Tools/unicode/data/ /.ccache /cross-build/ /jit_stencils.h +/install-details.toml +/build-details.json /platform /profile-clean-stamp /profile-run-stamp diff --git a/Doc/library/sysconfig.rst b/Doc/library/sysconfig.rst index 2faab212e46eff..9b3ecedd2588a1 100644 --- a/Doc/library/sysconfig.rst +++ b/Doc/library/sysconfig.rst @@ -429,6 +429,8 @@ Other functions .. _sysconfig-cli: +.. _sysconfig-commandline: + Using :mod:`sysconfig` as a script ---------------------------------- diff --git a/Doc/using/index.rst b/Doc/using/index.rst index e1a3111f36a44f..ff63b5f8e7cb8c 100644 --- a/Doc/using/index.rst +++ b/Doc/using/index.rst @@ -19,3 +19,4 @@ interpreter and things that make working with Python easier. windows.rst mac.rst editors.rst + introspection.rst diff --git a/Doc/using/introspection.rst b/Doc/using/introspection.rst new file mode 100644 index 00000000000000..5f49c88df40413 --- /dev/null +++ b/Doc/using/introspection.rst @@ -0,0 +1,153 @@ +.. _introspection: + +*********************************** +Introspecting a Python installation +*********************************** + +The mechanisms available to introspect a Python installation are highly +dependent on the installation you have. Their availability may depend on the +Python implementation in question, the build format, the target platform, or +several other details. + +This document aims to give an overview of these mechanisms and provide the user +a general guide on how to approach this task, but users should still refer to +the documentation of their installation for more accurate information regarding +implementation details and availability. + + +Introduction +============ + +Generally, the most portable way to do this is by running some Python code to +extract information from the target interpreter, though this can often be +automated with the use of built-in helpers (see +:ref:`introspection-stdlib-clis`, :ref:`introspection-python-config-script`). + +However, when introspecting a Python installation, running code is often +undesirable or impossible. For this purpose, Python defines a standard +:ref:`static install description file ` (please +see the :ref:`introspection-install-details-format` section for the format +specification), which may be provided together with your interpreter (please +see the :ref:`introspection-install-details-location` section for more details). +When available, this file can be used to lookup certain details from the +installation in question without having to run the interpreter. + + +Reference +========= + + +.. _introspection-install-details: + +Install details file +-------------------- + + +.. _introspection-install-details-format: + +Format +~~~~~~ + +The details file is JSON file and the top-level object is a dictionary +containing the data described in :ref:`introspection-install-details-fields`. + + +.. _introspection-install-details-fields: + +Fields +~~~~~~ + +This section documents some of the data fields that may be present. It is not +extensive — different implementations, platforms etc. may include additional +fields. + +Dictionaries represent sub-sections, and their names are separated with +dots (``.``) in this documentation. + +``python.version`` +++++++++++++++++++ + +:Type: ``str`` +:Description: A string representation of the Python version. + +``python.version_parts`` +++++++++++++++++++++++++ + +:Type: ``dict[str, int | str]`` +:Description: A dictionary containing the different components of the Python + version, as found in :py:data:`sys.version_info`. + +``python.executable`` ++++++++++++++++++++++ + +:Type: ``str`` +:Description: An absolute path pointing to an interpreter from the installation. +:Availability: May be omitted in situations where it is unavailable, there isn't + a reliable path that can be specified, etc. + +``python.stdlib`` ++++++++++++++++++++++ + +:Type: ``str`` +:Description: An absolute path pointing to the directory where the standard + library is installed. +:Availability: May be omitted in situations where it is unavailable, there isn't + a reliable path that can be specified, etc. + + +.. _introspection-install-details-location: + +Location +~~~~~~~~ + +On standard CPython installations, when available, the details file will be +installed in the same directory as the standard library, with the name +``build-details.json``. + +Different implementations may place it in a different path, choose to provide +the file via a different mechanism, or choose not to include it at all. + +.. note:: For Python implementation maintainers + + It would be good to try to align the install location between + implementations, when possible, to help with compatibility. + + We just do not choose a installation path for everyone because different + implementations may work differently, and it should be up to each project to + decide what makes the most sense for them. + + +.. _introspection-stdlib-clis: + +Standard library modules with a CLI +----------------------------------- + +Some standard library modules include a CLI that exposes information, which can +be helpful for introspection. + +- :mod:`sysconfig` (see :ref:`sysconfig-commandline`) +- :mod:`site` (see :ref:`site-commandline`) + + +.. _introspection-python-config-script: + +The ``python-config`` script +---------------------------- + +.. TODO: Currently, we don't have any documentation covering python-config, but + if/when we add some, we refer to it here, instead. + +.. availability:: POSIX + +A ``python-config`` script may be available alongside the interpreter +executable. This script exposes information especially relevant when building C +extensions. + +When using it via ``PATH``, you should be careful with your environment, and +make sure that the first ``python-config`` entry does in fact belong to the +correct interpreter. + +Additionally, the current implementation does not need to run the interpreter, +so this script may be helpful in situtation where that is not possible, or +undesirable. Though, please keep in mind that this is an implementation detail +and no guarantees are made regarding this aspect of the implementation. diff --git a/Lib/test/test_install_details_file.py b/Lib/test/test_install_details_file.py new file mode 100644 index 00000000000000..82781c617b3623 --- /dev/null +++ b/Lib/test/test_install_details_file.py @@ -0,0 +1,127 @@ +import json +import os +import sys +import sysconfig +import string +import unittest + + +class FormatTestsBase: + @property + def contents(self): + """Install details file contents. Should be overriden by subclasses.""" + raise NotImplementedError + + @property + def data(self): + """Parsed install details file data, as a Python object.""" + return json.loads(self.contents) + + def key(self, name): + """Helper to fetch subsection entries. + + It takes the entry name, allowing the usage of a dot to separate the + different subsection names (eg. specifying 'a.b.c' as the key will + return the value of self.data['a']['b']['c']). + """ + value = self.data + for part in name.split('.'): + value = value[part] + return value + + def test_parse(self): + self.data + + def test_top_level_container(self): + self.assertIsInstance(self.data, dict) + for key, value in self.data.items(): + with self.subTest(section=key): + self.assertIsInstance(value, dict) + + def test_python_version(self): + allowed_characters = string.digits + string.ascii_letters + '.' + value = self.key('python.version') + + self.assertLessEqual(set(value), set(allowed_characters)) + self.assertTrue(sys.version.startswith(value)) + + def test_python_version_parts(self): + value = self.key('python.version_parts') + + self.assertEqual(len(value), sys.version_info.n_fields) + for part_name, part_value in value.items(): + with self.subTest(part=part_name): + self.assertEqual(part_value, getattr(sys.version_info, part_name)) + + def test_python_executable(self): + """Test the python.executable entry. + + The generic test wants the key to be missing. If your implementation + provides a value for it, you should override this test. + """ + with self.assertRaises(KeyError): + self.key('python.executable') + + def test_python_stdlib(self): + """Test the python.stdlib entry. + + The generic test wants the key to be missing. If your implementation + provides a value for it, you should override this test. + """ + with self.assertRaises(KeyError): + self.key('python.stdlib') + + +needs_installed_python = unittest.skipIf( + sysconfig.is_python_build(), + 'This test can only run in an installed Python', +) + + +@unittest.skipIf(os.name != 'posix', 'Feature only implemented on POSIX right now') +class CPythonInstallDetailsFileTests(unittest.TestCase, FormatTestsBase): + """Test CPython's install details file implementation.""" + + @property + def location(self): + if sysconfig.is_python_build(): + dirname = sysconfig.get_config_var('projectbase') + else: + dirname = sysconfig.get_path('stdlib') + return os.path.join(dirname, 'build-details.json') + + @property + def contents(self): + with open(self.location, 'r') as f: + return f.read() + + def setUp(self): + if sys.platform in ('wasi', 'emscripten') and not os.path.isfile(self.location): + self.skipTest(f'{sys.platform} build without a build-details.json file') + + @needs_installed_python + def test_location(self): + self.assertTrue(os.path.isfile(self.location)) + + # Override generic format tests with tests for our specific implemenation. + + @needs_installed_python + def test_python_executable(self): + value = self.key('python.executable') + + self.assertEqual(os.path.realpath(value), os.path.realpath(sys.executable)) + + @needs_installed_python + def test_python_stdlib(self): + value = self.key('python.stdlib') + + try: + stdlib = os.path.dirname(sysconfig.__file__) + except AttributeError as exc: + self.skipTest(str(exc)) + + self.assertEqual(os.path.realpath(value), os.path.realpath(stdlib)) + + +if __name__ == "__main__": + unittest.main() diff --git a/Makefile.pre.in b/Makefile.pre.in index 4c1a18602b2d0b..8efbc947eed144 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -654,7 +654,7 @@ all: @DEF_MAKE_ALL_RULE@ .PHONY: build_all build_all: check-clean-src $(BUILDPYTHON) platform sharedmods \ - gdbhooks Programs/_testembed scripts checksharedmods rundsymutil + gdbhooks Programs/_testembed scripts checksharedmods rundsymutil build-details.json .PHONY: build_wasm build_wasm: check-clean-src $(BUILDPYTHON) platform sharedmods \ @@ -831,6 +831,10 @@ $(BUILDPYTHON): Programs/python.o $(LINK_PYTHON_DEPS) platform: $(PYTHON_FOR_BUILD_DEPS) pybuilddir.txt $(RUNSHARED) $(PYTHON_FOR_BUILD) -c 'import sys ; from sysconfig import get_platform ; print("%s-%d.%d" % (get_platform(), *sys.version_info[:2]))' >platform +build-details.json: $(PYTHON_FOR_BUILD_DEPS) pybuilddir.txt $(srcdir)/Tools/build/generate_install_details_file.py + $(RUNSHARED) $(PYTHON_FOR_BUILD) $(srcdir)/Tools/build/generate_build_details_file.py ./build-details.json \ + --executable $(BINDIR)/python3$(EXE) + # Create build directory and generate the sysconfig build-time data there. # pybuilddir.txt contains the name of the build dir and is used for # sys.path fixup -- see Modules/getpath.c. @@ -2042,7 +2046,7 @@ altinstall: commoninstall .PHONY: commoninstall commoninstall: check-clean-src @FRAMEWORKALTINSTALLFIRST@ \ altbininstall libinstall inclinstall libainstall \ - sharedinstall altmaninstall @FRAMEWORKALTINSTALLLAST@ + sharedinstall altmaninstall descfileinstall @FRAMEWORKALTINSTALLLAST@ # Install shared libraries enabled by Setup DESTDIRS= $(exec_prefix) $(LIBDIR) $(BINLIBDEST) $(DESTSHARED) @@ -2671,6 +2675,11 @@ frameworkaltinstallunixtools: frameworkinstallextras: cd Mac && $(MAKE) installextras DESTDIR="$(DESTDIR)" +# Install the +.PHONY: descfileinstall +descfileinstall: libinstall + $(INSTALL) -m $(FILEMODE) ./build-details.json $(DESTDIR)$(LIBDEST)/build-details.json + # Build the toplevel Makefile Makefile.pre: $(srcdir)/Makefile.pre.in config.status CONFIG_FILES=Makefile.pre CONFIG_HEADERS= ./config.status @@ -2787,6 +2796,7 @@ clean-retain-profile: pycremoval find build -name '*.py' -exec rm -f {} ';' || true find build -name '*.py[co]' -exec rm -f {} ';' || true -rm -f pybuilddir.txt + -rm -f build-details.json -rm -f _bootstrap_python -rm -f python.html python*.js python.data python*.symbols python*.map -rm -f $(WASM_STDLIB) diff --git a/Misc/NEWS.d/next/Build/2023-10-05-18-56-06.gh-issue-107956.ZC5KRw.rst b/Misc/NEWS.d/next/Build/2023-10-05-18-56-06.gh-issue-107956.ZC5KRw.rst new file mode 100644 index 00000000000000..7015a52422e49c --- /dev/null +++ b/Misc/NEWS.d/next/Build/2023-10-05-18-56-06.gh-issue-107956.ZC5KRw.rst @@ -0,0 +1,3 @@ +A ``build-details.json`` file containing information regarding the Python +build/installation is now installed in the same directory as the standard +library. diff --git a/Tools/build/generate_build_details_file.py b/Tools/build/generate_build_details_file.py new file mode 100644 index 00000000000000..b76a2f39732918 --- /dev/null +++ b/Tools/build/generate_build_details_file.py @@ -0,0 +1,42 @@ +import argparse +import json +import os +import sys +import sysconfig + + +BasicConfigItem = str | int +ConfigItem = BasicConfigItem | list[BasicConfigItem] +SectionData = dict[str, 'ConfigItem | SectionData'] +ConfigData = dict[str, SectionData] + + +def generic_info(executable: str) -> ConfigData: + return { + 'python': { + 'version': sys.version.split(' ')[0], + 'version_parts': { + field: getattr(sys.version_info, field) + for field in ('major', 'minor', 'micro', 'releaselevel', 'serial') + }, + 'executable': executable, + 'stdlib': sysconfig.get_path('stdlib'), + }, + } + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument('output_path', metavar='FILE') + parser.add_argument('--executable', help='The executable path on the system.') + + args = parser.parse_args() + + config = generic_info(args.executable) + + with open(args.output_path, 'w') as f: + json.dump(config, f) + + +if __name__ == '__main__': + main()