Skip to content

[POC] gh-85454: Extract and inject distutils from setuptools whl #95254

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 2 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions Lib/test/support/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -1827,6 +1827,7 @@ def missing_compiler_executable(cmd_names=[]):

"""
# TODO (PEP 632): alternate check without using distutils
# Remove with Lib/distutils/tests
from distutils import ccompiler, sysconfig, spawn, errors
compiler = ccompiler.new_compiler()
sysconfig.customize_compiler(compiler)
Expand Down
43 changes: 43 additions & 0 deletions Lib/test/support/import_helper.py
Original file line number Diff line number Diff line change
@@ -1,14 +1,20 @@
import tempfile
import contextlib
import ensurepip
import _imp
import importlib
import importlib.util
import importlib.resources
import os
import site
import shutil
import sys
import unittest
import warnings
import zipfile

from .os_helper import unlink
from . import requires_zlib


@contextlib.contextmanager
Expand Down Expand Up @@ -246,3 +252,40 @@ def modules_cleanup(oldmodules):
# do currently). Implicitly imported *real* modules should be left alone
# (see issue 10556).
sys.modules.update(oldmodules)


@requires_zlib()
@contextlib.contextmanager
def inject_setuptools():
sud = "SETUPTOOLS_USE_DISTUTILS"
package = ensurepip._get_packages()["setuptools"]
if package.wheel_name:
# Use bundled wheel package
ensurepip_res = importlib.resources.files("ensurepip")
wheel_path = ensurepip_res / "_bundled" / package.wheel_name
else:
wheel_path = package.wheel_path
orig_path = sys.path[:]
if sud in os.environ:
orig_sud = os.environ[sud]
else:
orig_sud = None
os.environ[sud] = "local"
tmpdir = tempfile.mkdtemp()
try:
zf = zipfile.ZipFile(wheel_path)
zf.extractall(tmpdir)
site.addsitedir(tmpdir)
import setuptools._distutils
sys.modules["distutils"] = setuptools._distutils
yield
finally:
sys.path[:] = orig_path
if orig_sud is not None:
os.environ[sud] = orig_sud
else:
os.environ.pop(sud)
shutil.rmtree(tmpdir)
for name in list(sys.modules):
if name.startswith(("setuptools", "distutils", "pkg_resources")):
forget(name)
36 changes: 35 additions & 1 deletion Lib/test/test_peg_generator/test_c_parser.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@

from test import test_tools
from test import support
from test.support import import_helper
from test.support import os_helper
from test.support.script_helper import assert_python_ok

Expand Down Expand Up @@ -88,10 +89,11 @@ def setUpClass(cls):
# runtime overhead of spawning compiler processes.
cls.library_dir = tempfile.mkdtemp(dir=cls.tmp_base)
cls.addClassCleanup(shutil.rmtree, cls.library_dir)
cls.enterClassContext(import_helper.inject_setuptools())

def setUp(self):
self._backup_config_vars = dict(sysconfig._CONFIG_VARS)
cmd = support.missing_compiler_executable()
cmd = self.missing_compiler_executable()
if cmd is not None:
self.skipTest("The %r command is not found" % cmd)
self.old_cwd = os.getcwd()
Expand All @@ -104,6 +106,38 @@ def tearDown(self):
sysconfig._CONFIG_VARS.clear()
sysconfig._CONFIG_VARS.update(self._backup_config_vars)

def missing_compiler_executable(self, cmd_names=()):
"""Check if the compiler components used to build the interpreter exist.

Check for the existence of the compiler executables whose names are listed
in 'cmd_names' or all the compiler executables when 'cmd_names' is empty
and return the first missing executable or None when none is found
missing.

"""
# TODO (PEP 632): alternate check without using distutils
# uses distutils from setuptools
from distutils import ccompiler, sysconfig, spawn, errors
compiler = ccompiler.new_compiler()
sysconfig.customize_compiler(compiler)
if compiler.compiler_type == "msvc":
# MSVC has no executables, so check whether initialization succeeds
try:
compiler.initialize()
except errors.DistutilsPlatformError:
return "msvc"
for name in compiler.executables:
if cmd_names and name not in cmd_names:
continue
cmd = getattr(compiler, name)
if cmd_names:
assert cmd is not None, \
"the '%s' executable is not configured" % name
elif not cmd:
continue
if spawn.find_executable(cmd[0]) is None:
return cmd[0]

def build_extension(self, grammar_source):
grammar = parse_string(grammar_source, GrammarParser)
# Because setUp() already changes the current directory to the
Expand Down
36 changes: 35 additions & 1 deletion Tools/peg_generator/pegen/build.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import itertools
import os
import pathlib
import sys
import sysconfig
Expand All @@ -19,6 +20,40 @@
TokenDefinitions = Tuple[Dict[int, str], Dict[str, int], Set[str]]


def fixup_build_ext(cmd) -> None:
"""Function needed to make build_ext tests pass.
When Python was built with --enable-shared on Unix, -L. is not enough to
find libpython<blah>.so, because regrtest runs in a tempdir, not in the
source directory where the .so lives.
When Python was built with in debug mode on Windows, build_ext commands
need their debug attribute set, and it is not done automatically for
some reason.
This function handles both of these things. Example use:
cmd = build_ext(dist)
support.fixup_build_ext(cmd)
cmd.ensure_finalized()
Unlike most other Unix platforms, Mac OS X embeds absolute paths
to shared libraries into executables, so the fixup is not needed there.

Copied from distutils.tests.support.
"""
if os.name == 'nt':
cmd.debug = sys.executable.endswith('_d.exe')
elif sysconfig.get_config_var('Py_ENABLE_SHARED'):
# To further add to the shared builds fun on Unix, we can't just add
# library_dirs to the Extension() instance because that doesn't get
# plumbed through to the final compiler command.
runshared = sysconfig.get_config_var('RUNSHARED')
if runshared is None:
cmd.library_dirs = ['.']
else:
if sys.platform == 'darwin':
cmd.library_dirs = []
else:
name, equals, value = runshared.partition('=')
cmd.library_dirs = [d for d in value.split(os.pathsep) if d]


def get_extra_flags(compiler_flags: str, compiler_py_flags_nodist: str) -> List[str]:
flags = sysconfig.get_config_var(compiler_flags)
py_flags_nodist = sysconfig.get_config_var(compiler_py_flags_nodist)
Expand Down Expand Up @@ -51,7 +86,6 @@ def compile_c_extension(
"""
import distutils.log
from distutils.core import Distribution, Extension
from distutils.tests.support import fixup_build_ext # type: ignore

from distutils.ccompiler import new_compiler
from distutils.dep_util import newer_group
Expand Down