From 5031753c048177d361b9598799f0ac5902be4ce4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filipe=20La=C3=ADns?= Date: Fri, 25 Aug 2023 15:15:10 +0100 Subject: [PATCH 01/15] GH-107956: install a static description file MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Filipe Laíns --- .gitignore | 1 + Makefile.pre.in | 13 +++- Tools/build/generate_install_details_file.py | 65 ++++++++++++++++++++ 3 files changed, 77 insertions(+), 2 deletions(-) create mode 100644 Tools/build/generate_install_details_file.py diff --git a/.gitignore b/.gitignore index 6ed7197e3ab626..33b1971e9d1e67 100644 --- a/.gitignore +++ b/.gitignore @@ -127,6 +127,7 @@ Tools/unicode/data/ /.ccache /cross-build/ /jit_stencils.h +/install-details.toml /platform /profile-clean-stamp /profile-run-stamp diff --git a/Makefile.pre.in b/Makefile.pre.in index 4c1a18602b2d0b..ef7e4360be8492 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -654,7 +654,7 @@ all: @DEF_MAKE_ALL_RULE@ .PHONY: build_all build_all: check-clean-src $(BUILDPYTHON) platform sharedmods \ - gdbhooks Programs/_testembed scripts checksharedmods rundsymutil + gdbhooks Programs/_testembed scripts checksharedmods rundsymutil install-details.toml .PHONY: build_wasm build_wasm: check-clean-src $(BUILDPYTHON) platform sharedmods \ @@ -831,6 +831,9 @@ $(BUILDPYTHON): Programs/python.o $(LINK_PYTHON_DEPS) platform: $(PYTHON_FOR_BUILD_DEPS) pybuilddir.txt $(RUNSHARED) $(PYTHON_FOR_BUILD) -c 'import sys ; from sysconfig import get_platform ; print("%s-%d.%d" % (get_platform(), *sys.version_info[:2]))' >platform +install-details.toml: $(PYTHON_FOR_BUILD_DEPS) pybuilddir.txt $(srcdir)/Tools/build/generate_install_details_file.py + $(RUNSHARED) $(PYTHON_FOR_BUILD) $(srcdir)/Tools/build/generate_install_details_file.py ./install-details.toml + # Create build directory and generate the sysconfig build-time data there. # pybuilddir.txt contains the name of the build dir and is used for # sys.path fixup -- see Modules/getpath.c. @@ -2042,7 +2045,7 @@ altinstall: commoninstall .PHONY: commoninstall commoninstall: check-clean-src @FRAMEWORKALTINSTALLFIRST@ \ altbininstall libinstall inclinstall libainstall \ - sharedinstall altmaninstall @FRAMEWORKALTINSTALLLAST@ + sharedinstall altmaninstall descfileinstall @FRAMEWORKALTINSTALLLAST@ # Install shared libraries enabled by Setup DESTDIRS= $(exec_prefix) $(LIBDIR) $(BINLIBDEST) $(DESTSHARED) @@ -2671,6 +2674,11 @@ frameworkaltinstallunixtools: frameworkinstallextras: cd Mac && $(MAKE) installextras DESTDIR="$(DESTDIR)" +# Install the +.PHONY: descfileinstall +descfileinstall: + $(INSTALL) -d -m $(FILEMODE) ./install-details.toml $(DESTDIR)$(LIBDEST)/install-details.toml + # Build the toplevel Makefile Makefile.pre: $(srcdir)/Makefile.pre.in config.status CONFIG_FILES=Makefile.pre CONFIG_HEADERS= ./config.status @@ -2787,6 +2795,7 @@ clean-retain-profile: pycremoval find build -name '*.py' -exec rm -f {} ';' || true find build -name '*.py[co]' -exec rm -f {} ';' || true -rm -f pybuilddir.txt + -rm -f install-details.toml -rm -f _bootstrap_python -rm -f python.html python*.js python.data python*.symbols python*.map -rm -f $(WASM_STDLIB) diff --git a/Tools/build/generate_install_details_file.py b/Tools/build/generate_install_details_file.py new file mode 100644 index 00000000000000..fae37169246f22 --- /dev/null +++ b/Tools/build/generate_install_details_file.py @@ -0,0 +1,65 @@ +import argparse +import io +import os +import sys +import sysconfig +import tomllib + +from typing import Any, Callable + + +ConfigItem = str | int | list[str | int] +ConfigData = dict[str, dict[str, ConfigItem]] + + +def generic_info() -> ConfigData: + return { + 'python': { + 'version': sys.version.split(' ')[0], + 'version_parts': { + field: getattr(sys.version_info, field) + for field in ('major', 'minor', 'micro', 'releaselevel', 'serial') + }, + 'executable': os.path.join( + sysconfig.get_path('scripts'), + os.path.basename(sys.executable), + ), + 'stdlib': sysconfig.get_path('stdlib'), + }, + } + + +def toml_dump(fd: io.TextIOBase, data: ConfigData) -> None: + """**Very** basic TOML writer. It only implements what is necessary for this use-case.""" + def toml_repr(obj: object) -> str: + if isinstance(obj, dict): + return '{ ' + ', '.join(f'{k} = {toml_repr(v)}' for k, v in obj.items()) + ' }' + else: + return repr(obj) + + for section, entries in data.items(): + print(f'[{section}]', file=fd) + for name, value in entries.items(): + print(f'{name} = {toml_repr(value)}', file=fd) + fd.write('\n') + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument('output_path', metavar='FILE') + + args = parser.parse_args() + + config = generic_info() + + # write TOML with our very basic writer + with open(args.output_path, 'w') as f: + toml_dump(f, config) + + # try to load the data as a sanity check to verify our writer outputed, at least, valid TOML + with open(args.output_path, 'rb') as f: + parsed = tomllib.load(f) + + +if __name__ == '__main__': + main() From 4da59963511898dfb69d86a96adafc6e8a0199a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filipe=20La=C3=ADns?= Date: Mon, 4 Sep 2023 22:33:58 +0100 Subject: [PATCH 02/15] escape strings correctly MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Filipe Laíns --- Tools/build/generate_install_details_file.py | 70 +++++++++++++++----- 1 file changed, 53 insertions(+), 17 deletions(-) diff --git a/Tools/build/generate_install_details_file.py b/Tools/build/generate_install_details_file.py index fae37169246f22..20aa0469f0c93e 100644 --- a/Tools/build/generate_install_details_file.py +++ b/Tools/build/generate_install_details_file.py @@ -5,11 +5,61 @@ import sysconfig import tomllib -from typing import Any, Callable +BasicConfigItem = str | int +ConfigItem = BasicConfigItem | list[BasicConfigItem] +SectionData = dict[str, 'ConfigItem | SectionData'] +ConfigData = dict[str, SectionData] + + +# From https://toml.io/en/v1.0.0#string: +# Any Unicode character may be used except those that must be escaped: +# quotation mark, backslash, and the control characters other than tab +# (U+0000 to U+0008, U+000A to U+001F, U+007F). +_needs_escaping = {*range(0x0008 + 1), *range(0x000A, 0x001F + 1), 0x007F} +toml_string_escape_table = {i: f'\\u{i:04}' for i in _needs_escaping} +toml_string_escape_table.update({ + # characters with a custom escape sequence + 0x0008: r'\b', + 0x0009: r'\t', + 0x000A: r'\n', + 0x000C: r'\f', + 0x000D: r'\r', + 0x0022: r'\"', + 0x005C: r'\\\\', +}) + + +def toml_dump(fd: io.TextIOBase, data: ConfigData, outer_section: str | None = None) -> None: + """**Very** basic TOML writer. It only implements what is necessary for our use-case.""" -ConfigItem = str | int | list[str | int] -ConfigData = dict[str, dict[str, ConfigItem]] + def toml_repr(obj: object) -> str: + if isinstance(obj, str): + return f'"{obj.translate(toml_string_escape_table)}"' + else: + return repr(obj) + + def dump_section(name: str, data: SectionData) -> None: + subsections: dict[str, SectionData] = {} + + # don't emit a newline before the first section + if fd.tell() != 0: + fd.write('\n') + + # write section + print(f'[{name}]', file=fd) + for key, value in data.items(): + if isinstance(value, dict): + subsections[f'{name}.{key}'] = value + else: + print(f'{key} = {toml_repr(value)}', file=fd) + + # write subsections + for subsection_name, subsection_data in subsections.items(): + dump_section(subsection_name, subsection_data) + + for name, section_data in data.items(): + dump_section(name, section_data) def generic_info() -> ConfigData: @@ -29,20 +79,6 @@ def generic_info() -> ConfigData: } -def toml_dump(fd: io.TextIOBase, data: ConfigData) -> None: - """**Very** basic TOML writer. It only implements what is necessary for this use-case.""" - def toml_repr(obj: object) -> str: - if isinstance(obj, dict): - return '{ ' + ', '.join(f'{k} = {toml_repr(v)}' for k, v in obj.items()) + ' }' - else: - return repr(obj) - - for section, entries in data.items(): - print(f'[{section}]', file=fd) - for name, value in entries.items(): - print(f'{name} = {toml_repr(value)}', file=fd) - fd.write('\n') - def main() -> None: parser = argparse.ArgumentParser() From 4047d8697c039b99a4e905d16dfcb45146a16f34 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filipe=20La=C3=ADns?= Date: Mon, 4 Sep 2023 22:34:21 +0100 Subject: [PATCH 03/15] compare the data loaded by tomllib to what we wrote in the sanity check MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Filipe Laíns --- Tools/build/generate_install_details_file.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Tools/build/generate_install_details_file.py b/Tools/build/generate_install_details_file.py index 20aa0469f0c93e..5cb3ac4e2b2451 100644 --- a/Tools/build/generate_install_details_file.py +++ b/Tools/build/generate_install_details_file.py @@ -95,6 +95,7 @@ def main() -> None: # try to load the data as a sanity check to verify our writer outputed, at least, valid TOML with open(args.output_path, 'rb') as f: parsed = tomllib.load(f) + assert parsed == config if __name__ == '__main__': From b71fab245f7d0356dee40df94184c76e1f866ac4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filipe=20La=C3=ADns?= Date: Thu, 5 Oct 2023 01:51:01 +0100 Subject: [PATCH 04/15] change the format to JSON MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Filipe Laíns --- .gitignore | 1 + Doc/library/sysconfig.rst | 2 + Doc/using/index.rst | 1 + Doc/using/introspection.rst | 153 +++++++++++++++++++ Makefile.pre.in | 10 +- Tools/build/generate_install_details_file.py | 65 +------- 6 files changed, 167 insertions(+), 65 deletions(-) create mode 100644 Doc/using/introspection.rst diff --git a/.gitignore b/.gitignore index 33b1971e9d1e67..32fabad39fbc69 100644 --- a/.gitignore +++ b/.gitignore @@ -128,6 +128,7 @@ Tools/unicode/data/ /cross-build/ /jit_stencils.h /install-details.toml +/install-details.json /platform /profile-clean-stamp /profile-run-stamp diff --git a/Doc/library/sysconfig.rst b/Doc/library/sysconfig.rst index 2faab212e46eff..9b3ecedd2588a1 100644 --- a/Doc/library/sysconfig.rst +++ b/Doc/library/sysconfig.rst @@ -429,6 +429,8 @@ Other functions .. _sysconfig-cli: +.. _sysconfig-commandline: + Using :mod:`sysconfig` as a script ---------------------------------- diff --git a/Doc/using/index.rst b/Doc/using/index.rst index e1a3111f36a44f..ff63b5f8e7cb8c 100644 --- a/Doc/using/index.rst +++ b/Doc/using/index.rst @@ -19,3 +19,4 @@ interpreter and things that make working with Python easier. windows.rst mac.rst editors.rst + introspection.rst diff --git a/Doc/using/introspection.rst b/Doc/using/introspection.rst new file mode 100644 index 00000000000000..ece4c698203cd9 --- /dev/null +++ b/Doc/using/introspection.rst @@ -0,0 +1,153 @@ +.. _introspection: + +*********************************** +Introspecting a Python installation +*********************************** + +The mechanisms available to introspect a Python installation are highly +dependent on the installation you have. Their availability may depend on the +Python implementation in question, the build format, the target platform, or +several other details. + +This document aims to give an overview of these mechanisms and provide the user +a general guide on how to approach this task, but users should still refer to +the documentation of their installation for more accurate information regarding +implementation details and availability. + + +Introduction +============ + +Generally, the most portable way to do this is by running some Python code to +extract information from the target interpreter, though this can often be +automated with the use of built-in helpers (see +:ref:`introspection-stdlib-clis`, :ref:`introspection-python-config-script`). + +However, when introspecting a Python installation, running code is often +undesirable or impossible. For this purpose, Python defines a standard +:ref:`static install description file ` (please +see the :ref:`introspection-install-details-format` section for the format +specification), which may be provided together with your interpreter (please +see the :ref:`introspection-install-details-location` section for more details). +When available, this file can be used to lookup certain details from the +installation in question without having to run the interpreter. + + +Reference +========= + + +.. _introspection-install-details: + +Install details file +-------------------- + + +.. _introspection-install-details-format: + +Format +~~~~~~ + +The details file is JSON file and the top-level object is a dictionary +containing the data described in :ref:`introspection-install-details-fields`. + + +.. _introspection-install-details-fields: + +Fields +~~~~~~ + +This section documents some of the data fields that may be present. It is not +extensive — different implementations, platforms etc. may include additional +fields. + +Dictionaries represent sub-sections, and their names are separated with +dots (``.``) in this documentation. + +``python.version`` +++++++++++++++++++ + +:Type: ``str`` +:Description: A string representation of the Python version. + +``python.version_parts`` +++++++++++++++++++++++++ + +:Type: ``dict[str, int | str]`` +:Description: A dictionary containing the different components of the Python + version, as found in :py:data:`sys.version_info`. + +``python.executable`` ++++++++++++++++++++++ + +:Type: ``str`` +:Description: An absolute path pointing to an interpreter from the installation. +:Availability: May be omitted in situations where is is unavailable, there isn't + a reliable path that can be specified, etc. + +``python.stdlib`` ++++++++++++++++++++++ + +:Type: ``str`` +:Description: An absolute path pointing to the directory where the standard + library is installed. +:Availability: May be omitted in situations where is is unavailable, there isn't + a reliable path that can be specified, etc. + + +.. _introspection-install-details-location: + +Location +~~~~~~~~ + +On standard CPython installations, when available, the details file will be +installed in the same directory as the standard library, with the name +``install-details.json``. + +Different implementations may place it in a different path, choose to provide +the file via a different mechanism, or choose not to include it at all. + +.. note:: For Python implementation maintainers + + It would be good to try to align the install location between + implementations, when possible, to help with compatibility. + + We just do not choose a installation path for everyone because different + implementations may work differently, and it should be up to each project to + decide what makes the most sense for them. + + +.. _introspection-stdlib-clis: + +Standard library modules with a CLI +----------------------------------- + +Some standard library modules include a CLI that exposes information, which can +be helpful for introspection. + +- :mod:`sysconfig` (see :ref:`sysconfig-commandline`) +- :mod:`site` (see :ref:`site-commandline`) + + +.. _introspection-python-config-script: + +The ``python-config`` script +---------------------------- + +.. TODO: Currently, we don't have any documentation covering python-config, but + if/when we add some, we refer to it here, instead. + +.. availability:: POSIX + +A ``python-config`` script may be available alongside the interpreter +executable. This script exposes information especially relevant when building C +extensions. + +When using it via ``PATH``, you should be careful with your environment, and +make sure that the first ``python-config`` entry does in fact belong to the +correct interpreter. + +Additionally, the current implementation does not need to run the interpreter, +so this script may be helpful in situtation where that is not possible, or +undesirable. Though, please keep in mind that this is an implementation detail +and no guarantees are made regarding this aspect of the implementation. diff --git a/Makefile.pre.in b/Makefile.pre.in index ef7e4360be8492..4eab681411aa79 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -654,7 +654,7 @@ all: @DEF_MAKE_ALL_RULE@ .PHONY: build_all build_all: check-clean-src $(BUILDPYTHON) platform sharedmods \ - gdbhooks Programs/_testembed scripts checksharedmods rundsymutil install-details.toml + gdbhooks Programs/_testembed scripts checksharedmods rundsymutil install-details.json .PHONY: build_wasm build_wasm: check-clean-src $(BUILDPYTHON) platform sharedmods \ @@ -831,8 +831,8 @@ $(BUILDPYTHON): Programs/python.o $(LINK_PYTHON_DEPS) platform: $(PYTHON_FOR_BUILD_DEPS) pybuilddir.txt $(RUNSHARED) $(PYTHON_FOR_BUILD) -c 'import sys ; from sysconfig import get_platform ; print("%s-%d.%d" % (get_platform(), *sys.version_info[:2]))' >platform -install-details.toml: $(PYTHON_FOR_BUILD_DEPS) pybuilddir.txt $(srcdir)/Tools/build/generate_install_details_file.py - $(RUNSHARED) $(PYTHON_FOR_BUILD) $(srcdir)/Tools/build/generate_install_details_file.py ./install-details.toml +install-details.json: $(PYTHON_FOR_BUILD_DEPS) pybuilddir.txt $(srcdir)/Tools/build/generate_install_details_file.py + $(RUNSHARED) $(PYTHON_FOR_BUILD) $(srcdir)/Tools/build/generate_install_details_file.py ./install-details.json # Create build directory and generate the sysconfig build-time data there. # pybuilddir.txt contains the name of the build dir and is used for @@ -2677,7 +2677,7 @@ frameworkinstallextras: # Install the .PHONY: descfileinstall descfileinstall: - $(INSTALL) -d -m $(FILEMODE) ./install-details.toml $(DESTDIR)$(LIBDEST)/install-details.toml + $(INSTALL) -d -m $(FILEMODE) ./install-details.json $(DESTDIR)$(LIBDEST)/install-details.json # Build the toplevel Makefile Makefile.pre: $(srcdir)/Makefile.pre.in config.status @@ -2795,7 +2795,7 @@ clean-retain-profile: pycremoval find build -name '*.py' -exec rm -f {} ';' || true find build -name '*.py[co]' -exec rm -f {} ';' || true -rm -f pybuilddir.txt - -rm -f install-details.toml + -rm -f install-details.json -rm -f _bootstrap_python -rm -f python.html python*.js python.data python*.symbols python*.map -rm -f $(WASM_STDLIB) diff --git a/Tools/build/generate_install_details_file.py b/Tools/build/generate_install_details_file.py index 5cb3ac4e2b2451..15aa0614e75ab8 100644 --- a/Tools/build/generate_install_details_file.py +++ b/Tools/build/generate_install_details_file.py @@ -1,9 +1,8 @@ import argparse -import io +import json import os import sys import sysconfig -import tomllib BasicConfigItem = str | int @@ -12,56 +11,6 @@ ConfigData = dict[str, SectionData] -# From https://toml.io/en/v1.0.0#string: -# Any Unicode character may be used except those that must be escaped: -# quotation mark, backslash, and the control characters other than tab -# (U+0000 to U+0008, U+000A to U+001F, U+007F). -_needs_escaping = {*range(0x0008 + 1), *range(0x000A, 0x001F + 1), 0x007F} -toml_string_escape_table = {i: f'\\u{i:04}' for i in _needs_escaping} -toml_string_escape_table.update({ - # characters with a custom escape sequence - 0x0008: r'\b', - 0x0009: r'\t', - 0x000A: r'\n', - 0x000C: r'\f', - 0x000D: r'\r', - 0x0022: r'\"', - 0x005C: r'\\\\', -}) - - -def toml_dump(fd: io.TextIOBase, data: ConfigData, outer_section: str | None = None) -> None: - """**Very** basic TOML writer. It only implements what is necessary for our use-case.""" - - def toml_repr(obj: object) -> str: - if isinstance(obj, str): - return f'"{obj.translate(toml_string_escape_table)}"' - else: - return repr(obj) - - def dump_section(name: str, data: SectionData) -> None: - subsections: dict[str, SectionData] = {} - - # don't emit a newline before the first section - if fd.tell() != 0: - fd.write('\n') - - # write section - print(f'[{name}]', file=fd) - for key, value in data.items(): - if isinstance(value, dict): - subsections[f'{name}.{key}'] = value - else: - print(f'{key} = {toml_repr(value)}', file=fd) - - # write subsections - for subsection_name, subsection_data in subsections.items(): - dump_section(subsection_name, subsection_data) - - for name, section_data in data.items(): - dump_section(name, section_data) - - def generic_info() -> ConfigData: return { 'python': { @@ -79,7 +28,6 @@ def generic_info() -> ConfigData: } - def main() -> None: parser = argparse.ArgumentParser() parser.add_argument('output_path', metavar='FILE') @@ -88,14 +36,11 @@ def main() -> None: config = generic_info() - # write TOML with our very basic writer - with open(args.output_path, 'w') as f: - toml_dump(f, config) + # move this to tests afterwards + assert len(config['python']['version_parts']) == sys.version_info.n_fields - # try to load the data as a sanity check to verify our writer outputed, at least, valid TOML - with open(args.output_path, 'rb') as f: - parsed = tomllib.load(f) - assert parsed == config + with open(args.output_path, 'w') as f: + json.dump(config, f) if __name__ == '__main__': From de5729e877d26184645e4c6e9a2b0757312a98e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filipe=20La=C3=ADns?= Date: Thu, 5 Oct 2023 18:52:23 +0100 Subject: [PATCH 05/15] add tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Filipe Laíns --- Lib/test/test_install_details_file.py | 123 +++++++++++++++++++ Tools/build/generate_install_details_file.py | 3 - 2 files changed, 123 insertions(+), 3 deletions(-) create mode 100644 Lib/test/test_install_details_file.py diff --git a/Lib/test/test_install_details_file.py b/Lib/test/test_install_details_file.py new file mode 100644 index 00000000000000..6c478c3d82d86f --- /dev/null +++ b/Lib/test/test_install_details_file.py @@ -0,0 +1,123 @@ +import json +import os +import sys +import sysconfig +import string +import unittest + + +class FormatTestsBase: + @property + def contents(self): + """Install details file contents. Should be overriden by subclasses.""" + raise NotImplementedError + + @property + def data(self): + """Parsed install details file data, as a Python object.""" + return json.loads(self.contents) + + def key(self, name): + """Helper to fetch subsection entries. + + It takes the entry name, allowing the usage of a dot to separate the + different subsection names (eg. specifying 'a.b.c' as the key will + return the value of self.data['a']['b']['c']). + """ + value = self.data + for part in name.split('.'): + value = value[part] + return value + + def test_parse(self): + self.data + + def test_top_level_container(self): + self.assertIsInstance(self.data, dict) + for key, value in self.data.items(): + with self.subTest(section=key): + self.assertIsInstance(value, dict) + + def test_python_version(self): + allowed_characters = string.digits + string.ascii_letters + '.' + value = self.key('python.version') + + self.assertLessEqual(set(value), set(allowed_characters)) + self.assertTrue(sys.version.startswith(value)) + + def test_python_version_parts(self): + value = self.key('python.version_parts') + + self.assertEqual(len(value), sys.version_info.n_fields) + for part_name, part_value in value.items(): + with self.subTest(part=part_name): + self.assertEqual(part_value, getattr(sys.version_info, part_name)) + + def test_python_executable(self): + """Test the python.executable entry. + + The generic test wants the key to be missing. If your implementation + provides a value for it, you should override this test. + """ + with self.assertRaises(KeyError): + self.key('python.executable') + + def test_python_stdlib(self): + """Test the python.stdlib entry. + + The generic test wants the key to be missing. If your implementation + provides a value for it, you should override this test. + """ + with self.assertRaises(KeyError): + self.key('python.stdlib') + + +needs_installed_python = unittest.skipIf( + sysconfig.is_python_build(), + 'This test can only run in an installed Python', +) + + +@unittest.skipIf(os.name == 'nt', 'Feature only implemented on POSIX right now') +class CPythonInstallDetailsFileTests(unittest.TestCase, FormatTestsBase): + """Test CPython's install details file implementation.""" + + @property + def location(self): + if sysconfig.is_python_build(): + dirname = sysconfig.get_config_var('projectbase') + else: + dirname = sysconfig.get_path('stdlib') + return os.path.join(dirname, 'install-details.json') + + @property + def contents(self): + with open(self.location, 'r') as f: + return f.read() + + @needs_installed_python + def test_location(self): + self.assertTrue(os.path.isfile(self.location)) + + # Override generic format tests with tests for our specific implemenation. + + @needs_installed_python + def test_python_executable(self): + value = self.key('python.executable') + + self.assertEqual(os.path.realpath(value), os.path.realpath(sys.executable)) + + @needs_installed_python + def test_python_stdlib(self): + value = self.key('python.stdlib') + + try: + stdlib = os.path.dirname(unittest.__path__) + except AttributeError as exc: + self.skipTest(str(exc)) + + self.assertEqual(os.path.realpath(value), os.path.realpath(stdlib)) + + +if __name__ == "__main__": + unittest.main() diff --git a/Tools/build/generate_install_details_file.py b/Tools/build/generate_install_details_file.py index 15aa0614e75ab8..dd879aa5adf89a 100644 --- a/Tools/build/generate_install_details_file.py +++ b/Tools/build/generate_install_details_file.py @@ -36,9 +36,6 @@ def main() -> None: config = generic_info() - # move this to tests afterwards - assert len(config['python']['version_parts']) == sys.version_info.n_fields - with open(args.output_path, 'w') as f: json.dump(config, f) From 4155f2ce858b96914fc9c5377a15daec3cbb8eac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filipe=20La=C3=ADns?= Date: Thu, 5 Oct 2023 18:56:11 +0100 Subject: [PATCH 06/15] add news MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Filipe Laíns --- .../next/Build/2023-10-05-18-56-06.gh-issue-107956.ZC5KRw.rst | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 Misc/NEWS.d/next/Build/2023-10-05-18-56-06.gh-issue-107956.ZC5KRw.rst diff --git a/Misc/NEWS.d/next/Build/2023-10-05-18-56-06.gh-issue-107956.ZC5KRw.rst b/Misc/NEWS.d/next/Build/2023-10-05-18-56-06.gh-issue-107956.ZC5KRw.rst new file mode 100644 index 00000000000000..02f0664bd9f653 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2023-10-05-18-56-06.gh-issue-107956.ZC5KRw.rst @@ -0,0 +1,3 @@ +A ``install-details.json`` file containing information regarding the Python +build/installation is now installed in the same directory as the standard +library. From 4adb942dda4f802a2befa245c475f67f187e0b9f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filipe=20La=C3=ADns?= Date: Fri, 6 Oct 2023 00:03:14 +0100 Subject: [PATCH 07/15] fix test_python_stdlib MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Filipe Laíns --- Lib/test/test_install_details_file.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_install_details_file.py b/Lib/test/test_install_details_file.py index 6c478c3d82d86f..8d0b73d5d5bfd2 100644 --- a/Lib/test/test_install_details_file.py +++ b/Lib/test/test_install_details_file.py @@ -112,7 +112,7 @@ def test_python_stdlib(self): value = self.key('python.stdlib') try: - stdlib = os.path.dirname(unittest.__path__) + stdlib = os.path.dirname(sysconfig.__file__) except AttributeError as exc: self.skipTest(str(exc)) From d1eaf101e74312dbc697f72fbc9bbcdf0c18b53b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filipe=20La=C3=ADns?= Date: Fri, 6 Oct 2023 00:03:44 +0100 Subject: [PATCH 08/15] fix the install-details.json installation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Filipe Laíns --- Makefile.pre.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile.pre.in b/Makefile.pre.in index 4eab681411aa79..6edfc0cd8a2bbc 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -2677,7 +2677,7 @@ frameworkinstallextras: # Install the .PHONY: descfileinstall descfileinstall: - $(INSTALL) -d -m $(FILEMODE) ./install-details.json $(DESTDIR)$(LIBDEST)/install-details.json + $(INSTALL) -m $(FILEMODE) ./install-details.json $(DESTDIR)$(LIBDEST)/install-details.json # Build the toplevel Makefile Makefile.pre: $(srcdir)/Makefile.pre.in config.status From 5b3fe211c602268b64ec6f6fc7a550729bf9bbdc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filipe=20La=C3=ADns?= Date: Fri, 6 Oct 2023 00:03:59 +0100 Subject: [PATCH 09/15] get the interpreter path from the build system MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Filipe Laíns --- Makefile.pre.in | 3 ++- Tools/build/generate_install_details_file.py | 10 ++++------ 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/Makefile.pre.in b/Makefile.pre.in index 6edfc0cd8a2bbc..8286f8bc7680f2 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -832,7 +832,8 @@ platform: $(PYTHON_FOR_BUILD_DEPS) pybuilddir.txt $(RUNSHARED) $(PYTHON_FOR_BUILD) -c 'import sys ; from sysconfig import get_platform ; print("%s-%d.%d" % (get_platform(), *sys.version_info[:2]))' >platform install-details.json: $(PYTHON_FOR_BUILD_DEPS) pybuilddir.txt $(srcdir)/Tools/build/generate_install_details_file.py - $(RUNSHARED) $(PYTHON_FOR_BUILD) $(srcdir)/Tools/build/generate_install_details_file.py ./install-details.json + $(RUNSHARED) $(PYTHON_FOR_BUILD) $(srcdir)/Tools/build/generate_install_details_file.py ./install-details.json \ + --executable $(BINDIR)/python3$(EXE) # Create build directory and generate the sysconfig build-time data there. # pybuilddir.txt contains the name of the build dir and is used for diff --git a/Tools/build/generate_install_details_file.py b/Tools/build/generate_install_details_file.py index dd879aa5adf89a..b76a2f39732918 100644 --- a/Tools/build/generate_install_details_file.py +++ b/Tools/build/generate_install_details_file.py @@ -11,7 +11,7 @@ ConfigData = dict[str, SectionData] -def generic_info() -> ConfigData: +def generic_info(executable: str) -> ConfigData: return { 'python': { 'version': sys.version.split(' ')[0], @@ -19,10 +19,7 @@ def generic_info() -> ConfigData: field: getattr(sys.version_info, field) for field in ('major', 'minor', 'micro', 'releaselevel', 'serial') }, - 'executable': os.path.join( - sysconfig.get_path('scripts'), - os.path.basename(sys.executable), - ), + 'executable': executable, 'stdlib': sysconfig.get_path('stdlib'), }, } @@ -31,10 +28,11 @@ def generic_info() -> ConfigData: def main() -> None: parser = argparse.ArgumentParser() parser.add_argument('output_path', metavar='FILE') + parser.add_argument('--executable', help='The executable path on the system.') args = parser.parse_args() - config = generic_info() + config = generic_info(args.executable) with open(args.output_path, 'w') as f: json.dump(config, f) From 7e7c59a619ce4514cb42712cae319a593d71fa22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filipe=20La=C3=ADns?= Date: Fri, 6 Oct 2023 02:16:07 +0100 Subject: [PATCH 10/15] fix WASM tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Filipe Laíns --- Lib/test/test_install_details_file.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_install_details_file.py b/Lib/test/test_install_details_file.py index 8d0b73d5d5bfd2..51559d5683ede0 100644 --- a/Lib/test/test_install_details_file.py +++ b/Lib/test/test_install_details_file.py @@ -78,7 +78,7 @@ def test_python_stdlib(self): ) -@unittest.skipIf(os.name == 'nt', 'Feature only implemented on POSIX right now') +@unittest.skipIf(os.name != 'posix', 'Feature only implemented on POSIX right now') class CPythonInstallDetailsFileTests(unittest.TestCase, FormatTestsBase): """Test CPython's install details file implementation.""" From f792b6426bf71c9505aef6ffdcdc0a102467b08b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filipe=20La=C3=ADns?= Date: Fri, 6 Oct 2023 02:16:22 +0100 Subject: [PATCH 11/15] add missing dependency to the desciprion file install target MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Filipe Laíns --- Makefile.pre.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile.pre.in b/Makefile.pre.in index 8286f8bc7680f2..cb230d79f3e661 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -2677,7 +2677,7 @@ frameworkinstallextras: # Install the .PHONY: descfileinstall -descfileinstall: +descfileinstall: install $(INSTALL) -m $(FILEMODE) ./install-details.json $(DESTDIR)$(LIBDEST)/install-details.json # Build the toplevel Makefile From b8d6aed27dc950b05521b643ee427ad68839ce62 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filipe=20La=C3=ADns?= Date: Fri, 6 Oct 2023 14:40:51 +0100 Subject: [PATCH 12/15] fix descfileinstall target dependency MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Filipe Laíns --- Makefile.pre.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile.pre.in b/Makefile.pre.in index cb230d79f3e661..bbcea8bf476904 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -2677,7 +2677,7 @@ frameworkinstallextras: # Install the .PHONY: descfileinstall -descfileinstall: install +descfileinstall: libinstall $(INSTALL) -m $(FILEMODE) ./install-details.json $(DESTDIR)$(LIBDEST)/install-details.json # Build the toplevel Makefile From 15f8a1a6d6940a97a7aef920231c8f1f0abab4bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filipe=20La=C3=ADns?= Date: Fri, 6 Oct 2023 14:59:52 +0100 Subject: [PATCH 13/15] skip install-details.json test on wasi and emscripten when not available MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Filipe Laíns --- Lib/test/test_install_details_file.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Lib/test/test_install_details_file.py b/Lib/test/test_install_details_file.py index 51559d5683ede0..67bbb9262075a7 100644 --- a/Lib/test/test_install_details_file.py +++ b/Lib/test/test_install_details_file.py @@ -95,6 +95,10 @@ def contents(self): with open(self.location, 'r') as f: return f.read() + def setUp(self): + if sys.platform in ('wasi', 'emscripten') and not os.path.isfile(self.location): + self.skipTest(f'{sys.platform} build without a install-details.json file') + @needs_installed_python def test_location(self): self.assertTrue(os.path.isfile(self.location)) From c96e447f14f658cc3820ec9f4d08bb71a829f47d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filipe=20La=C3=ADns?= Date: Thu, 12 Oct 2023 18:52:04 +0100 Subject: [PATCH 14/15] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Steve Dower Signed-off-by: Filipe Laíns --- Doc/using/introspection.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/using/introspection.rst b/Doc/using/introspection.rst index ece4c698203cd9..514116be948bf1 100644 --- a/Doc/using/introspection.rst +++ b/Doc/using/introspection.rst @@ -82,7 +82,7 @@ dots (``.``) in this documentation. :Type: ``str`` :Description: An absolute path pointing to an interpreter from the installation. -:Availability: May be omitted in situations where is is unavailable, there isn't +:Availability: May be omitted in situations where it is unavailable, there isn't a reliable path that can be specified, etc. ``python.stdlib`` @@ -91,7 +91,7 @@ dots (``.``) in this documentation. :Type: ``str`` :Description: An absolute path pointing to the directory where the standard library is installed. -:Availability: May be omitted in situations where is is unavailable, there isn't +:Availability: May be omitted in situations where it is unavailable, there isn't a reliable path that can be specified, etc. From 217682bdd428cf54068be90b47107ed25bfcc507 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filipe=20La=C3=ADns?= Date: Sat, 24 Feb 2024 16:27:42 +0000 Subject: [PATCH 15/15] rename file to build-details.json MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Filipe Laíns --- .gitignore | 2 +- Doc/using/introspection.rst | 2 +- Lib/test/test_install_details_file.py | 4 ++-- Makefile.pre.in | 10 +++++----- .../2023-10-05-18-56-06.gh-issue-107956.ZC5KRw.rst | 2 +- ..._details_file.py => generate_build_details_file.py} | 0 6 files changed, 10 insertions(+), 10 deletions(-) rename Tools/build/{generate_install_details_file.py => generate_build_details_file.py} (100%) diff --git a/.gitignore b/.gitignore index 32fabad39fbc69..44adade58ef189 100644 --- a/.gitignore +++ b/.gitignore @@ -128,7 +128,7 @@ Tools/unicode/data/ /cross-build/ /jit_stencils.h /install-details.toml -/install-details.json +/build-details.json /platform /profile-clean-stamp /profile-run-stamp diff --git a/Doc/using/introspection.rst b/Doc/using/introspection.rst index 514116be948bf1..5f49c88df40413 100644 --- a/Doc/using/introspection.rst +++ b/Doc/using/introspection.rst @@ -102,7 +102,7 @@ Location On standard CPython installations, when available, the details file will be installed in the same directory as the standard library, with the name -``install-details.json``. +``build-details.json``. Different implementations may place it in a different path, choose to provide the file via a different mechanism, or choose not to include it at all. diff --git a/Lib/test/test_install_details_file.py b/Lib/test/test_install_details_file.py index 67bbb9262075a7..82781c617b3623 100644 --- a/Lib/test/test_install_details_file.py +++ b/Lib/test/test_install_details_file.py @@ -88,7 +88,7 @@ def location(self): dirname = sysconfig.get_config_var('projectbase') else: dirname = sysconfig.get_path('stdlib') - return os.path.join(dirname, 'install-details.json') + return os.path.join(dirname, 'build-details.json') @property def contents(self): @@ -97,7 +97,7 @@ def contents(self): def setUp(self): if sys.platform in ('wasi', 'emscripten') and not os.path.isfile(self.location): - self.skipTest(f'{sys.platform} build without a install-details.json file') + self.skipTest(f'{sys.platform} build without a build-details.json file') @needs_installed_python def test_location(self): diff --git a/Makefile.pre.in b/Makefile.pre.in index bbcea8bf476904..8efbc947eed144 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -654,7 +654,7 @@ all: @DEF_MAKE_ALL_RULE@ .PHONY: build_all build_all: check-clean-src $(BUILDPYTHON) platform sharedmods \ - gdbhooks Programs/_testembed scripts checksharedmods rundsymutil install-details.json + gdbhooks Programs/_testembed scripts checksharedmods rundsymutil build-details.json .PHONY: build_wasm build_wasm: check-clean-src $(BUILDPYTHON) platform sharedmods \ @@ -831,8 +831,8 @@ $(BUILDPYTHON): Programs/python.o $(LINK_PYTHON_DEPS) platform: $(PYTHON_FOR_BUILD_DEPS) pybuilddir.txt $(RUNSHARED) $(PYTHON_FOR_BUILD) -c 'import sys ; from sysconfig import get_platform ; print("%s-%d.%d" % (get_platform(), *sys.version_info[:2]))' >platform -install-details.json: $(PYTHON_FOR_BUILD_DEPS) pybuilddir.txt $(srcdir)/Tools/build/generate_install_details_file.py - $(RUNSHARED) $(PYTHON_FOR_BUILD) $(srcdir)/Tools/build/generate_install_details_file.py ./install-details.json \ +build-details.json: $(PYTHON_FOR_BUILD_DEPS) pybuilddir.txt $(srcdir)/Tools/build/generate_install_details_file.py + $(RUNSHARED) $(PYTHON_FOR_BUILD) $(srcdir)/Tools/build/generate_build_details_file.py ./build-details.json \ --executable $(BINDIR)/python3$(EXE) # Create build directory and generate the sysconfig build-time data there. @@ -2678,7 +2678,7 @@ frameworkinstallextras: # Install the .PHONY: descfileinstall descfileinstall: libinstall - $(INSTALL) -m $(FILEMODE) ./install-details.json $(DESTDIR)$(LIBDEST)/install-details.json + $(INSTALL) -m $(FILEMODE) ./build-details.json $(DESTDIR)$(LIBDEST)/build-details.json # Build the toplevel Makefile Makefile.pre: $(srcdir)/Makefile.pre.in config.status @@ -2796,7 +2796,7 @@ clean-retain-profile: pycremoval find build -name '*.py' -exec rm -f {} ';' || true find build -name '*.py[co]' -exec rm -f {} ';' || true -rm -f pybuilddir.txt - -rm -f install-details.json + -rm -f build-details.json -rm -f _bootstrap_python -rm -f python.html python*.js python.data python*.symbols python*.map -rm -f $(WASM_STDLIB) diff --git a/Misc/NEWS.d/next/Build/2023-10-05-18-56-06.gh-issue-107956.ZC5KRw.rst b/Misc/NEWS.d/next/Build/2023-10-05-18-56-06.gh-issue-107956.ZC5KRw.rst index 02f0664bd9f653..7015a52422e49c 100644 --- a/Misc/NEWS.d/next/Build/2023-10-05-18-56-06.gh-issue-107956.ZC5KRw.rst +++ b/Misc/NEWS.d/next/Build/2023-10-05-18-56-06.gh-issue-107956.ZC5KRw.rst @@ -1,3 +1,3 @@ -A ``install-details.json`` file containing information regarding the Python +A ``build-details.json`` file containing information regarding the Python build/installation is now installed in the same directory as the standard library. diff --git a/Tools/build/generate_install_details_file.py b/Tools/build/generate_build_details_file.py similarity index 100% rename from Tools/build/generate_install_details_file.py rename to Tools/build/generate_build_details_file.py