Skip to content

gh-109276: libregrtest: WASM use filename for JSON #109326

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 5 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 7 additions & 1 deletion Lib/test/libregrtest/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -117,6 +117,7 @@ def __init__(self, ns: Namespace):
self.python_cmd: tuple[str] = tuple(ns.python)
else:
self.python_cmd = None
print("main process python_cmd:", self.python_cmd)
self.coverage: bool = ns.trace
self.coverage_dir: StrPath | None = ns.coverdir
self.tmp_dir: StrPath | None = ns.tempdir
Expand Down Expand Up @@ -406,7 +407,7 @@ def create_run_tests(self, tests: TestTuple):
python_cmd=self.python_cmd,
randomize=self.randomize,
random_seed=self.random_seed,
json_fd=None,
json_file=None,
)

def _run_tests(self, selected: TestTuple, tests: TestList | None) -> int:
Expand Down Expand Up @@ -507,5 +508,10 @@ def main(self, tests: TestList | None = None):

def main(tests=None, **kwargs):
"""Run the Python suite."""

print("main process start sys.platform:", sys.platform)
print("main process start is_emscripten:", support.is_emscripten)
print("main process start is_wasi:", support.is_wasi)

ns = _parse_args(sys.argv[1:], **kwargs)
Regrtest(ns).main(tests=tests)
46 changes: 32 additions & 14 deletions Lib/test/libregrtest/run_workers.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,10 @@
from .logger import Logger
from .result import TestResult, State
from .results import TestResults
from .runtests import RunTests
from .runtests import RunTests, JsonFileType
from .single import PROGRESS_MIN_TIME
from .utils import (
StrPath, StrJSON, TestName, MS_WINDOWS,
StrPath, StrJSON, TestName, MS_WINDOWS, TMP_PREFIX,
format_duration, print_warning, count, plural)
from .worker import create_worker_process, USE_PROCESS_GROUP

Expand Down Expand Up @@ -155,10 +155,11 @@ def mp_result_error(
) -> MultiprocessResult:
return MultiprocessResult(test_result, stdout, err_msg)

def _run_process(self, runtests: RunTests, output_fd: int, json_fd: int,
def _run_process(self, runtests: RunTests, output_fd: int,
json_file: JsonFileType,
tmp_dir: StrPath | None = None) -> int:
try:
popen = create_worker_process(runtests, output_fd, json_fd,
popen = create_worker_process(runtests, output_fd, json_file,
tmp_dir)

self._killed = False
Expand Down Expand Up @@ -226,21 +227,38 @@ def _runtest(self, test_name: TestName) -> MultiprocessResult:
match_tests = None
err_msg = None

stdout_file = tempfile.TemporaryFile('w+', encoding=encoding)

json_file_use_filename = self.runtests.json_file_use_filename()
print("main process: json_file_use_filename?", json_file_use_filename)
if json_file_use_filename:
prefix = TMP_PREFIX + 'json_'
json_tmpfile = tempfile.NamedTemporaryFile('w+', encoding='utf8',
prefix=prefix)
print("main process: create NamedTemporaryFile", json_tmpfile.name)
else:
json_tmpfile = tempfile.TemporaryFile('w+', encoding='utf8')
print("main process: create TemporaryFile")

# gh-94026: Write stdout+stderr to a tempfile as workaround for
# non-blocking pipes on Emscripten with NodeJS.
with (tempfile.TemporaryFile('w+', encoding=encoding) as stdout_file,
tempfile.TemporaryFile('w+', encoding='utf8') as json_file):
with (stdout_file, json_tmpfile):
stdout_fd = stdout_file.fileno()
json_fd = json_file.fileno()
if MS_WINDOWS:
json_fd = msvcrt.get_osfhandle(json_fd)
if json_file_use_filename:
json_file = json_tmpfile.name
else:
json_file = json_tmpfile.fileno()
if MS_WINDOWS:
json_file = msvcrt.get_osfhandle(json_file)
print("main process json_file type:", type(json_file))
print("main process json_file:", json_file)

kwargs = {}
if match_tests:
kwargs['match_tests'] = match_tests
worker_runtests = self.runtests.copy(
tests=tests,
json_fd=json_fd,
json_file=json_file,
**kwargs)

# gh-93353: Check for leaked temporary files in the parent process,
Expand All @@ -254,13 +272,13 @@ def _runtest(self, test_name: TestName) -> MultiprocessResult:
tmp_dir = os.path.abspath(tmp_dir)
try:
retcode = self._run_process(worker_runtests,
stdout_fd, json_fd, tmp_dir)
stdout_fd, json_file, tmp_dir)
finally:
tmp_files = os.listdir(tmp_dir)
os_helper.rmtree(tmp_dir)
else:
retcode = self._run_process(worker_runtests,
stdout_fd, json_fd)
stdout_fd, json_file)
tmp_files = ()
stdout_file.seek(0)

Expand All @@ -275,8 +293,8 @@ def _runtest(self, test_name: TestName) -> MultiprocessResult:

try:
# deserialize run_tests_worker() output
json_file.seek(0)
worker_json: StrJSON = json_file.read()
json_tmpfile.seek(0)
worker_json: StrJSON = json_tmpfile.read()
if worker_json:
result = TestResult.from_json(worker_json)
else:
Expand Down
26 changes: 23 additions & 3 deletions Lib/test/libregrtest/runtests.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,20 @@
import json
from typing import Any

from test import support

from .utils import (
StrPath, StrJSON, TestTuple, FilterTuple, FilterDict)


# See RunTests.json_file_use_filename()
JsonFileType = int | StrPath

import os
print(os.getpid(), "JsonFileType:", JsonFileType)



@dataclasses.dataclass(slots=True, frozen=True)
class HuntRefleak:
warmups: int
Expand Down Expand Up @@ -38,9 +48,7 @@ class RunTests:
python_cmd: tuple[str] | None
randomize: bool
random_seed: int | None
# On Unix, it's a file descriptor.
# On Windows, it's a handle.
json_fd: int | None
json_file: JsonFileType | None

def copy(self, **override):
state = dataclasses.asdict(self)
Expand Down Expand Up @@ -74,6 +82,18 @@ def as_json(self) -> StrJSON:
def from_json(worker_json: StrJSON) -> 'RunTests':
return json.loads(worker_json, object_hook=_decode_runtests)

def json_file_use_filename(self):
# On Unix, it's a file descriptor.
# On Windows, it's a handle.
# On Emscripten/WASI, it's a filename. Passing a file descriptor to a
# worker process fails with "OSError: [Errno 8] Bad file descriptor" in the
# worker process.
return (
self.python_cmd
or support.is_emscripten
or support.is_wasi
)


class _EncodeRunTests(json.JSONEncoder):
def default(self, o: Any) -> dict[str, Any]:
Expand Down
12 changes: 8 additions & 4 deletions Lib/test/libregrtest/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,12 @@


MS_WINDOWS = (sys.platform == 'win32')
WORK_DIR_PREFIX = 'test_python_'
WORKER_WORK_DIR_PREFIX = f'{WORK_DIR_PREFIX}worker_'

# All temporary files and temporary directories created by libregrtest should
# use TMP_PREFIX so cleanup_temp_dir() can remove them all.
TMP_PREFIX = 'test_python_'
WORK_DIR_PREFIX = TMP_PREFIX
WORKER_WORK_DIR_PREFIX = WORK_DIR_PREFIX + 'worker_'

# bpo-38203: Maximum delay in seconds to exit Python (call Py_Finalize()).
# Used to protect against threading._shutdown() hang.
Expand Down Expand Up @@ -387,7 +391,7 @@ def get_work_dir(parent_dir: StrPath, worker: bool = False) -> StrPath:
# testing (see the -j option).
# Emscripten and WASI have stubbed getpid(), Emscripten has only
# milisecond clock resolution. Use randint() instead.
if sys.platform in {"emscripten", "wasi"}:
if support.is_emscripten or support.is_wasi:
nounce = random.randint(0, 1_000_000)
else:
nounce = os.getpid()
Expand Down Expand Up @@ -580,7 +584,7 @@ def display_header():
def cleanup_temp_dir(tmp_dir: StrPath):
import glob

path = os.path.join(glob.escape(tmp_dir), WORK_DIR_PREFIX + '*')
path = os.path.join(glob.escape(tmp_dir), TMP_PREFIX + '*')
print("Cleanup %s directory" % tmp_dir)
for name in glob.glob(path):
if os.path.isdir(name):
Expand Down
47 changes: 35 additions & 12 deletions Lib/test/libregrtest/worker.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
from test.support import os_helper

from .setup import setup_process, setup_test_dir
from .runtests import RunTests
from .runtests import RunTests, JsonFileType
from .single import run_single_test
from .utils import (
StrPath, StrJSON, FilterTuple, MS_WINDOWS,
Expand All @@ -18,7 +18,7 @@


def create_worker_process(runtests: RunTests,
output_fd: int, json_fd: int,
output_fd: int, json_file: JsonFileType,
tmp_dir: StrPath | None = None) -> subprocess.Popen:
python_cmd = runtests.python_cmd
worker_json = runtests.as_json()
Expand All @@ -27,10 +27,12 @@ def create_worker_process(runtests: RunTests,
executable = python_cmd
else:
executable = [sys.executable]
print("main process executable:", executable)
cmd = [*executable, *support.args_from_interpreter_flags(),
'-u', # Unbuffered stdout and stderr
'-m', 'test.libregrtest.worker',
worker_json]
print("main process worker cmd:", cmd)

env = dict(os.environ)
if tmp_dir is not None:
Expand All @@ -55,33 +57,50 @@ def create_worker_process(runtests: RunTests,
close_fds=True,
cwd=work_dir,
)
if not MS_WINDOWS:
kwargs['pass_fds'] = [json_fd]
else:

# Pass json_file to the worker process
if isinstance(json_file, str):
# Filename: nothing to do to
print("create_worker_process() json_file: filename")
pass
elif MS_WINDOWS:
# Windows handle
print("create_worker_process() json_file: Windows handle")
startupinfo = subprocess.STARTUPINFO()
startupinfo.lpAttributeList = {"handle_list": [json_fd]}
startupinfo.lpAttributeList = {"handle_list": [json_file]}
kwargs['startupinfo'] = startupinfo
else:
# Unix file descriptor
print("create_worker_process() json_file: Unix fd")
kwargs['pass_fds'] = [json_file]

if USE_PROCESS_GROUP:
kwargs['start_new_session'] = True

if MS_WINDOWS:
os.set_handle_inheritable(json_fd, True)
os.set_handle_inheritable(json_file, True)
try:
return subprocess.Popen(cmd, **kwargs)
finally:
if MS_WINDOWS:
os.set_handle_inheritable(json_fd, False)
os.set_handle_inheritable(json_file, False)


def worker_process(worker_json: StrJSON) -> NoReturn:
runtests = RunTests.from_json(worker_json)
test_name = runtests.tests[0]
match_tests: FilterTuple | None = runtests.match_tests
json_fd: int = runtests.json_fd
# On Unix, it's a file descriptor.
# On Windows, it's a handle.
# On Emscripten/WASI, it's a filename.
json_file: JsonFileType = runtests.json_file
print("worker: json_file type:", type(json_file))
print("worker: json_file:", json_file)

if MS_WINDOWS:
import msvcrt
json_fd = msvcrt.open_osfhandle(json_fd, os.O_WRONLY)
# Create a file descriptor from the handle
json_file = msvcrt.open_osfhandle(json_file, os.O_WRONLY)


setup_test_dir(runtests.test_dir)
Expand All @@ -96,8 +115,8 @@ def worker_process(worker_json: StrJSON) -> NoReturn:

result = run_single_test(test_name, runtests)

with open(json_fd, 'w', encoding='utf-8') as json_file:
result.write_json_into(json_file)
with open(json_file, 'w', encoding='utf-8') as fp:
result.write_json_into(fp)

sys.exit(0)

Expand All @@ -111,6 +130,10 @@ def main():
tmp_dir = get_temp_dir()
work_dir = get_work_dir(tmp_dir, worker=True)

print("worker process sys.platform:", sys.platform)
print("worker process is_emscripten:", support.is_emscripten)
print("worker process is_wasi:", support.is_wasi)

with exit_timeout():
with os_helper.temp_cwd(work_dir, quiet=True):
worker_process(worker_json)
Expand Down