bpo-41282: add vendor config

Signed-off-by: Filipe Laíns <[email protected]>

Author: FFY00

Doc/library/site.rst

@@ -32,14 +32,20 @@ It starts by constructing up to four directories from a head and a tail part.
 For the head part, it uses ``sys.prefix`` and ``sys.exec_prefix``; empty heads
 are skipped.  For the tail part, it uses the empty string and then
 :file:`lib/site-packages` (on Windows) or
-:file:`lib/python{X.Y}/site-packages` (on Unix and Macintosh).  For each
+:file:`lib/python{X.Y}/site-packages` (on Unix and Macintosh), and finally
+the ``purelib`` and ``platlib`` paths for each scheme specified in the
+``EXTRA_SITE_INSTALL_SCHEMES`` variable of the vendor config.  For each
 of the distinct head-tail combinations, it sees if it refers to an existing
 directory, and if so, adds it to ``sys.path`` and also inspects the newly
 added path for configuration files.
 
 .. versionchanged:: 3.5
    Support for the "site-python" directory has been removed.
 
+.. versionchanged:: 3.10
+   Extra site install schemes specified in the vendor config
+   (``--with-vendor-config`` configure option) will also be loaded.
+
 If a file named "pyvenv.cfg" exists one directory above sys.executable,
 sys.prefix and sys.exec_prefix are set to that directory and
 it is also checked for site-packages (sys.base_prefix and

Doc/library/sysconfig.rst

@@ -72,7 +72,7 @@ Every new component that is installed using :mod:`distutils` or a
 Distutils-based system will follow the same scheme to copy its file in the right
 places.
 
-Python currently supports seven schemes:
+Python currently supports seven mandatory schemes:
 
 - *posix_prefix*: scheme for POSIX platforms like Linux or Mac OS X.  This is
   the default scheme used when Python or a component is installed.
@@ -85,6 +85,10 @@ Python currently supports seven schemes:
 - *nt*: scheme for NT platforms like Windows.
 - *nt_user*: scheme for NT platforms, when the *user* option is used.
 
+Additionally to these, Python also supports vendor schemes specified in the
+``EXTRA_SITE_INSTALL_SCHEMES`` variable of the vendor config
+(``--with-vendor-config`` configure option).
+
 Each scheme is itself composed of a series of paths and each path has a unique
 identifier.  Python currently uses eight paths:
 

Doc/using/configure.rst

@@ -116,6 +116,21 @@ General Options
 
    .. versionadded:: 3.10
 
+.. cmdoption:: --with-vendor-config=config.py
+
+   Path to the vendor config (none by default).
+
+   The vendor config is a Python file that allows configuring some aspects of
+   the Python distribution.
+
+   A `EXTRA_SITE_INSTALL_SCHEMES` variable can be specified in the config to
+   add extra site install schemes. These schemes will be picked up by the
+   :mod:`sysconfig` module, and will be activated in the :mod:`site` module
+   initialization. This options allow Python distributors to define custom
+   locations to use for their Python packages.
+
+   .. versionadded:: 3.10
+
 
 Install Options
 ---------------

Lib/site.py

@@ -74,6 +74,7 @@
 import builtins
 import _sitebuiltins
 import io
+import sysconfig
 
 # Prefixes for site-packages; add additional prefixes like /usr/local here
 PREFIXES = [sys.prefix, sys.exec_prefix]
@@ -87,6 +88,8 @@
 USER_SITE = None
 USER_BASE = None
 
+_VENDOR_SCHEMES = None
+
 
 def _trace(message):
     if sys.flags.verbose:
@@ -350,6 +353,7 @@ def getsitepackages(prefixes=None):
     this function will find its `site-packages` subdirectory depending on the
     system environment, and will return a list of full paths.
     """
+    global _VENDOR_SCHEMES
     sitepackages = []
     seen = set()
 
@@ -377,6 +381,23 @@ def getsitepackages(prefixes=None):
             for libdir in libdirs:
                 path = os.path.join(prefix, libdir, "site-packages")
                 sitepackages.append(path)
+
+    if _VENDOR_SCHEMES is None:  # delayed execution
+        try:
+            import _vendor_config
+
+
+            _VENDOR_SCHEMES = _vendor_config.EXTRA_SITE_INSTALL_SCHEMES.keys()
+        except (ModuleNotFoundError, AttributeError):
+            _VENDOR_SCHEMES = []
+
+    # vendor site schemes
+    for scheme in _VENDOR_SCHEMES:
+        sitepackages += list({
+            sysconfig.get_path('purelib', scheme),
+            sysconfig.get_path('platlib', scheme),
+        })
+
     return sitepackages
 
 def addsitepackages(known_paths, prefixes=None):

Lib/sysconfig.py

@@ -59,6 +59,23 @@
     }
 
 
+def _load_vendor_schemes():
+    # add vendor defined schemes
+    try:
+        import _vendor_config
+
+        _INSTALL_SCHEMES.update({
+            name: scheme
+            for name, scheme in _vendor_config.EXTRA_SITE_INSTALL_SCHEMES.items()
+            if name not in _INSTALL_SCHEMES
+        })
+    except (ModuleNotFoundError, AttributeError):
+        pass
+
+
+_load_vendor_schemes()
+
+
 # NOTE: site.py has copy of this function.
 # Sync it when modify this function.
 def _getuserbase():

Lib/test/test_site.py

@@ -277,6 +277,11 @@ def test_getusersitepackages(self):
         self.assertEqual(site.USER_BASE, site.getuserbase())
 
     def test_getsitepackages(self):
+        sys.path.append(os.path.abspath(os.path.join(__file__, '..', 'vendor_config')))
+        # force re-load of vendor schemes with the patched sys.path
+        site._VENDOR_SCHEMES = None
+        sysconfig._load_vendor_schemes()
+
         site.PREFIXES = ['xoxo']
         dirs = site.getsitepackages()
         if os.sep == '/':
@@ -288,17 +293,20 @@ def test_getsitepackages(self):
                                       'site-packages')
                 self.assertEqual(dirs[0], wanted)
             else:
-                self.assertEqual(len(dirs), 1)
+                self.assertEqual(len(dirs), 3)
             wanted = os.path.join('xoxo', 'lib',
                                   'python%d.%d' % sys.version_info[:2],
                                   'site-packages')
-            self.assertEqual(dirs[-1], wanted)
+            self.assertEqual(dirs[-3], wanted)
+            self.assertEqual(sorted(dirs[-2:]), ['vendor-plat-packages', 'vendor-pure-packages'])
         else:
             # other platforms
             self.assertEqual(len(dirs), 2)
             self.assertEqual(dirs[0], 'xoxo')
             wanted = os.path.join('xoxo', 'lib', 'site-packages')
             self.assertEqual(dirs[1], wanted)
+            self.assertEqual(dirs[2], 'vendor-pure-packages')
+            self.assertEqual(dirs[3], 'vendor-plat-packages')
 
     @unittest.skipUnless(HAS_USER_SITE, 'need user site')
     def test_no_home_directory(self):

Lib/test/test_sysconfig.py

@@ -263,10 +263,14 @@ def test_get_config_h_filename(self):
         self.assertTrue(os.path.isfile(config_h), config_h)
 
     def test_get_scheme_names(self):
-        wanted = ['nt', 'posix_home', 'posix_prefix']
+        sys.path.append(os.path.abspath(os.path.join(__file__, '..', 'vendor_config')))
+        # force re-load of vendor schemes with the patched sys.path
+        sysconfig._load_vendor_schemes()
+
+        wanted = ['nt', 'posix_home', 'posix_prefix', 'some_vendor']
         if HAS_USER_BASE:
             wanted.extend(['nt_user', 'osx_framework_user', 'posix_user'])
-        self.assertEqual(get_scheme_names(), tuple(sorted(wanted)))
+        self.assertEqual(sysconfig.get_scheme_names(), tuple(sorted(wanted)))
 
     @skip_unless_symlink
     def test_symlink(self): # Issue 7880

Lib/test/vendor_config/_vendor_config.py

@@ -0,0 +1,14 @@
+EXTRA_SITE_INSTALL_SCHEMES = {
+    'some_vendor': {
+        'stdlib': '{installed_base}/{platlibdir}/python{py_version_short}',
+        'platstdlib': '{platbase}/{platlibdir}/python{py_version_short}',
+        'include':
+            '{installed_base}/include/python{py_version_short}{abiflags}',
+        'platinclude':
+            '{installed_platbase}/include/python{py_version_short}{abiflags}',
+        'purelib': 'vendor-pure-packages',
+        'platlib': 'vendor-plat-packages',
+        'scripts': 'vendor-scripts',
+        'data': 'vendor-data',
+    },
+}

Makefile.pre.in

@@ -1508,6 +1508,7 @@ TESTSUBDIRS=	ctypes/test \
 		unittest/test unittest/test/testmock
 
 TEST_MODULES=@TEST_MODULES@
+VENDOR_CONFIG='@VENDOR_CONFIG@'
 libinstall:	build_all $(srcdir)/Modules/xxmodule.c
 	@for i in $(SCRIPTDIR) $(LIBDEST); \
 	do \
@@ -1543,6 +1544,9 @@ libinstall:	build_all $(srcdir)/Modules/xxmodule.c
 			echo $(INSTALL_DATA) $$i $(LIBDEST); \
 		fi; \
 	done
+	@if test ! -z "$(VENDOR_CONFIG)"; then \
+		$(INSTALL_SCRIPT) $(VENDOR_CONFIG) $(DESTDIR)$(LIBDEST)/_vendor_config.py \
+	fi
 	@if test "$(TEST_MODULES)" = yes; then \
 		subdirs="$(LIBSUBDIRS) $(TESTSUBDIRS)"; \
 	else \

Misc/NEWS.d/next/Build/2021-04-28-21-57-32.bpo-41282.xgQ6Cn.rst

@@ -0,0 +1,2 @@
+Introduced support for Python distributors to specify a vendor config, via
+--with-vendor-config, which allows them to add custom install schemes.

configure.ac

@@ -5941,6 +5941,27 @@ else
 fi
 AC_SUBST(TEST_MODULES)
 
+# --with-vendor-config
+VENDOR_CONFIG=''
+AC_MSG_CHECKING(for --with-vendor-config)
+AC_ARG_WITH(vendor-config,
+            AS_HELP_STRING([--with-vendor-config=<config.py>]
+                           [use a vendor config to customize the certain details of the Python installation]),
+[
+    AC_CHECK_FILE("$withval",
+    [
+        if ( echo "$withval" | grep '.*\.py$' > /dev/null); then
+            VENDOR_CONFIG="$withval"
+        else
+            AC_MSG_ERROR([--with-vendor-config requires a Python file])
+        fi
+    ],
+    [
+        AC_MSG_ERROR([--with-vendor-config requires a valid file])
+    ])
+],
+[])
+AC_SUBST(VENDOR_CONFIG)
 
 # generate output files
 AC_CONFIG_FILES(Makefile.pre Misc/python.pc Misc/python-embed.pc Misc/python-config.sh)