diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index dd9a10764b075a..1d360958064c38 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -117,6 +117,7 @@ def __init__(self, ns: Namespace): self.python_cmd: tuple[str] = tuple(ns.python) else: self.python_cmd = None + print("main process python_cmd:", self.python_cmd) self.coverage: bool = ns.trace self.coverage_dir: StrPath | None = ns.coverdir self.tmp_dir: StrPath | None = ns.tempdir @@ -406,7 +407,7 @@ def create_run_tests(self, tests: TestTuple): python_cmd=self.python_cmd, randomize=self.randomize, random_seed=self.random_seed, - json_fd=None, + json_file=None, ) def _run_tests(self, selected: TestTuple, tests: TestList | None) -> int: @@ -507,5 +508,10 @@ def main(self, tests: TestList | None = None): def main(tests=None, **kwargs): """Run the Python suite.""" + + print("main process start sys.platform:", sys.platform) + print("main process start is_emscripten:", support.is_emscripten) + print("main process start is_wasi:", support.is_wasi) + ns = _parse_args(sys.argv[1:], **kwargs) Regrtest(ns).main(tests=tests) diff --git a/Lib/test/libregrtest/run_workers.py b/Lib/test/libregrtest/run_workers.py index eaca0af17ea13a..5dfa9677b8b96c 100644 --- a/Lib/test/libregrtest/run_workers.py +++ b/Lib/test/libregrtest/run_workers.py @@ -17,10 +17,10 @@ from .logger import Logger from .result import TestResult, State from .results import TestResults -from .runtests import RunTests +from .runtests import RunTests, JsonFileType from .single import PROGRESS_MIN_TIME from .utils import ( - StrPath, StrJSON, TestName, MS_WINDOWS, + StrPath, StrJSON, TestName, MS_WINDOWS, TMP_PREFIX, format_duration, print_warning, count, plural) from .worker import create_worker_process, USE_PROCESS_GROUP @@ -155,10 +155,11 @@ def mp_result_error( ) -> MultiprocessResult: return MultiprocessResult(test_result, stdout, err_msg) - def _run_process(self, runtests: RunTests, output_fd: int, json_fd: int, + def _run_process(self, runtests: RunTests, output_fd: int, + json_file: JsonFileType, tmp_dir: StrPath | None = None) -> int: try: - popen = create_worker_process(runtests, output_fd, json_fd, + popen = create_worker_process(runtests, output_fd, json_file, tmp_dir) self._killed = False @@ -226,21 +227,38 @@ def _runtest(self, test_name: TestName) -> MultiprocessResult: match_tests = None err_msg = None + stdout_file = tempfile.TemporaryFile('w+', encoding=encoding) + + json_file_use_filename = self.runtests.json_file_use_filename() + print("main process: json_file_use_filename?", json_file_use_filename) + if json_file_use_filename: + prefix = TMP_PREFIX + 'json_' + json_tmpfile = tempfile.NamedTemporaryFile('w+', encoding='utf8', + prefix=prefix) + print("main process: create NamedTemporaryFile", json_tmpfile.name) + else: + json_tmpfile = tempfile.TemporaryFile('w+', encoding='utf8') + print("main process: create TemporaryFile") + # gh-94026: Write stdout+stderr to a tempfile as workaround for # non-blocking pipes on Emscripten with NodeJS. - with (tempfile.TemporaryFile('w+', encoding=encoding) as stdout_file, - tempfile.TemporaryFile('w+', encoding='utf8') as json_file): + with (stdout_file, json_tmpfile): stdout_fd = stdout_file.fileno() - json_fd = json_file.fileno() - if MS_WINDOWS: - json_fd = msvcrt.get_osfhandle(json_fd) + if json_file_use_filename: + json_file = json_tmpfile.name + else: + json_file = json_tmpfile.fileno() + if MS_WINDOWS: + json_file = msvcrt.get_osfhandle(json_file) + print("main process json_file type:", type(json_file)) + print("main process json_file:", json_file) kwargs = {} if match_tests: kwargs['match_tests'] = match_tests worker_runtests = self.runtests.copy( tests=tests, - json_fd=json_fd, + json_file=json_file, **kwargs) # gh-93353: Check for leaked temporary files in the parent process, @@ -254,13 +272,13 @@ def _runtest(self, test_name: TestName) -> MultiprocessResult: tmp_dir = os.path.abspath(tmp_dir) try: retcode = self._run_process(worker_runtests, - stdout_fd, json_fd, tmp_dir) + stdout_fd, json_file, tmp_dir) finally: tmp_files = os.listdir(tmp_dir) os_helper.rmtree(tmp_dir) else: retcode = self._run_process(worker_runtests, - stdout_fd, json_fd) + stdout_fd, json_file) tmp_files = () stdout_file.seek(0) @@ -275,8 +293,8 @@ def _runtest(self, test_name: TestName) -> MultiprocessResult: try: # deserialize run_tests_worker() output - json_file.seek(0) - worker_json: StrJSON = json_file.read() + json_tmpfile.seek(0) + worker_json: StrJSON = json_tmpfile.read() if worker_json: result = TestResult.from_json(worker_json) else: diff --git a/Lib/test/libregrtest/runtests.py b/Lib/test/libregrtest/runtests.py index 5c68df126e2a8f..c61ef5390f0552 100644 --- a/Lib/test/libregrtest/runtests.py +++ b/Lib/test/libregrtest/runtests.py @@ -2,10 +2,20 @@ import json from typing import Any +from test import support + from .utils import ( StrPath, StrJSON, TestTuple, FilterTuple, FilterDict) +# See RunTests.json_file_use_filename() +JsonFileType = int | StrPath + +import os +print(os.getpid(), "JsonFileType:", JsonFileType) + + + @dataclasses.dataclass(slots=True, frozen=True) class HuntRefleak: warmups: int @@ -38,9 +48,7 @@ class RunTests: python_cmd: tuple[str] | None randomize: bool random_seed: int | None - # On Unix, it's a file descriptor. - # On Windows, it's a handle. - json_fd: int | None + json_file: JsonFileType | None def copy(self, **override): state = dataclasses.asdict(self) @@ -74,6 +82,18 @@ def as_json(self) -> StrJSON: def from_json(worker_json: StrJSON) -> 'RunTests': return json.loads(worker_json, object_hook=_decode_runtests) + def json_file_use_filename(self): + # On Unix, it's a file descriptor. + # On Windows, it's a handle. + # On Emscripten/WASI, it's a filename. Passing a file descriptor to a + # worker process fails with "OSError: [Errno 8] Bad file descriptor" in the + # worker process. + return ( + self.python_cmd + or support.is_emscripten + or support.is_wasi + ) + class _EncodeRunTests(json.JSONEncoder): def default(self, o: Any) -> dict[str, Any]: diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py index 03c27b9fe17053..ce7342aabfffbe 100644 --- a/Lib/test/libregrtest/utils.py +++ b/Lib/test/libregrtest/utils.py @@ -17,8 +17,12 @@ MS_WINDOWS = (sys.platform == 'win32') -WORK_DIR_PREFIX = 'test_python_' -WORKER_WORK_DIR_PREFIX = f'{WORK_DIR_PREFIX}worker_' + +# All temporary files and temporary directories created by libregrtest should +# use TMP_PREFIX so cleanup_temp_dir() can remove them all. +TMP_PREFIX = 'test_python_' +WORK_DIR_PREFIX = TMP_PREFIX +WORKER_WORK_DIR_PREFIX = WORK_DIR_PREFIX + 'worker_' # bpo-38203: Maximum delay in seconds to exit Python (call Py_Finalize()). # Used to protect against threading._shutdown() hang. @@ -387,7 +391,7 @@ def get_work_dir(parent_dir: StrPath, worker: bool = False) -> StrPath: # testing (see the -j option). # Emscripten and WASI have stubbed getpid(), Emscripten has only # milisecond clock resolution. Use randint() instead. - if sys.platform in {"emscripten", "wasi"}: + if support.is_emscripten or support.is_wasi: nounce = random.randint(0, 1_000_000) else: nounce = os.getpid() @@ -580,7 +584,7 @@ def display_header(): def cleanup_temp_dir(tmp_dir: StrPath): import glob - path = os.path.join(glob.escape(tmp_dir), WORK_DIR_PREFIX + '*') + path = os.path.join(glob.escape(tmp_dir), TMP_PREFIX + '*') print("Cleanup %s directory" % tmp_dir) for name in glob.glob(path): if os.path.isdir(name): diff --git a/Lib/test/libregrtest/worker.py b/Lib/test/libregrtest/worker.py index 0963faa2e4d2a1..f88e6dbc16f539 100644 --- a/Lib/test/libregrtest/worker.py +++ b/Lib/test/libregrtest/worker.py @@ -7,7 +7,7 @@ from test.support import os_helper from .setup import setup_process, setup_test_dir -from .runtests import RunTests +from .runtests import RunTests, JsonFileType from .single import run_single_test from .utils import ( StrPath, StrJSON, FilterTuple, MS_WINDOWS, @@ -18,7 +18,7 @@ def create_worker_process(runtests: RunTests, - output_fd: int, json_fd: int, + output_fd: int, json_file: JsonFileType, tmp_dir: StrPath | None = None) -> subprocess.Popen: python_cmd = runtests.python_cmd worker_json = runtests.as_json() @@ -27,10 +27,12 @@ def create_worker_process(runtests: RunTests, executable = python_cmd else: executable = [sys.executable] + print("main process executable:", executable) cmd = [*executable, *support.args_from_interpreter_flags(), '-u', # Unbuffered stdout and stderr '-m', 'test.libregrtest.worker', worker_json] + print("main process worker cmd:", cmd) env = dict(os.environ) if tmp_dir is not None: @@ -55,33 +57,50 @@ def create_worker_process(runtests: RunTests, close_fds=True, cwd=work_dir, ) - if not MS_WINDOWS: - kwargs['pass_fds'] = [json_fd] - else: + + # Pass json_file to the worker process + if isinstance(json_file, str): + # Filename: nothing to do to + print("create_worker_process() json_file: filename") + pass + elif MS_WINDOWS: + # Windows handle + print("create_worker_process() json_file: Windows handle") startupinfo = subprocess.STARTUPINFO() - startupinfo.lpAttributeList = {"handle_list": [json_fd]} + startupinfo.lpAttributeList = {"handle_list": [json_file]} kwargs['startupinfo'] = startupinfo + else: + # Unix file descriptor + print("create_worker_process() json_file: Unix fd") + kwargs['pass_fds'] = [json_file] + if USE_PROCESS_GROUP: kwargs['start_new_session'] = True if MS_WINDOWS: - os.set_handle_inheritable(json_fd, True) + os.set_handle_inheritable(json_file, True) try: return subprocess.Popen(cmd, **kwargs) finally: if MS_WINDOWS: - os.set_handle_inheritable(json_fd, False) + os.set_handle_inheritable(json_file, False) def worker_process(worker_json: StrJSON) -> NoReturn: runtests = RunTests.from_json(worker_json) test_name = runtests.tests[0] match_tests: FilterTuple | None = runtests.match_tests - json_fd: int = runtests.json_fd + # On Unix, it's a file descriptor. + # On Windows, it's a handle. + # On Emscripten/WASI, it's a filename. + json_file: JsonFileType = runtests.json_file + print("worker: json_file type:", type(json_file)) + print("worker: json_file:", json_file) if MS_WINDOWS: import msvcrt - json_fd = msvcrt.open_osfhandle(json_fd, os.O_WRONLY) + # Create a file descriptor from the handle + json_file = msvcrt.open_osfhandle(json_file, os.O_WRONLY) setup_test_dir(runtests.test_dir) @@ -96,8 +115,8 @@ def worker_process(worker_json: StrJSON) -> NoReturn: result = run_single_test(test_name, runtests) - with open(json_fd, 'w', encoding='utf-8') as json_file: - result.write_json_into(json_file) + with open(json_file, 'w', encoding='utf-8') as fp: + result.write_json_into(fp) sys.exit(0) @@ -111,6 +130,10 @@ def main(): tmp_dir = get_temp_dir() work_dir = get_work_dir(tmp_dir, worker=True) + print("worker process sys.platform:", sys.platform) + print("worker process is_emscripten:", support.is_emscripten) + print("worker process is_wasi:", support.is_wasi) + with exit_timeout(): with os_helper.temp_cwd(work_dir, quiet=True): worker_process(worker_json)