diff --git a/runtests.py b/runtests.py new file mode 100644 index 000000000..3161963e0 --- /dev/null +++ b/runtests.py @@ -0,0 +1,10 @@ +#!/usr/bin/env python + +import runpy +import os +import sdc +import numba + + +if __name__ == "__main__": + runpy.run_module('sdc.runtests', run_name='__main__') diff --git a/sdc/runtests.py b/sdc/runtests.py index aa0fc14fc..fa39bfb8f 100644 --- a/sdc/runtests.py +++ b/sdc/runtests.py @@ -1,68 +1,8 @@ -# ***************************************************************************** -# Copyright (c) 2019-2020, Intel Corporation All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# -# Redistributions of source code must retain the above copyright notice, -# this list of conditions and the following disclaimer. -# -# Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, -# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; -# OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, -# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR -# OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, -# EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# ***************************************************************************** - - -import os -import unittest -import sdc.tests -from sdc.tests.test_basic import get_rank - -""" - Every test in suite can be executed specified times using - desired value for SDC_REPEAT_TEST_NUMBER environment variable. - This can be used to locate scpecific failures occured - on next execution of affected test. - - loadTestsFromModule returns TestSuite obj with _tests member - which contains further TestSuite instanses for each found testCase: - hpat_tests = TestSuite(sdc.tests) - TestSuite(sdc.tests)._tests = [TestSuite(sdc.tests.TestBasic), TestSuite(sdc.tests.TestDataFrame), ...] - TestSuite(sdc.tests.TestBasic)._tests = [TestBasic testMethod=test_array_reduce, ...] -""" - - -def load_tests(loader, tests, pattern): - suite = unittest.TestSuite() - hpat_tests = loader.loadTestsFromModule(sdc.tests) - repeat_test_number = int(os.getenv('SDC_REPEAT_TEST_NUMBER', '1')) - - if repeat_test_number > 1: - for i, test_case in enumerate(hpat_tests): - extended_tests = [] - for test in test_case: - for _ in range(repeat_test_number): - extended_tests.append(test) - hpat_tests._tests[i]._tests = extended_tests - - suite.addTests(hpat_tests) - return suite - +from sdc.testing._runtests import _main if __name__ == '__main__': - # initialize MPI to avoid "Attempting to use an MPI routine before initializing MPICH" in any pipeline - get_rank() - - unittest.main() + import sys + # For parallel testing under Windows + from multiprocessing import freeze_support + freeze_support() + sys.exit(0 if _main(sys.argv) else 1) diff --git a/sdc/testing/__init__.py b/sdc/testing/__init__.py new file mode 100644 index 000000000..d1bb2cd4d --- /dev/null +++ b/sdc/testing/__init__.py @@ -0,0 +1,61 @@ +import os +import sys +import functools +import unittest +import traceback +from fnmatch import fnmatch +from os.path import join, isfile, relpath, normpath, splitext, abspath +import numba + +from numba.testing.main import NumbaTestProgram, SerialSuite, make_tag_decorator + + +def load_testsuite(loader, dir): + """Find tests in 'dir'.""" + try: + suite = unittest.TestSuite() + files = [] + for f in os.listdir(dir): + path = join(dir, f) + if isfile(path) and fnmatch(f, 'test_*.py'): + files.append(f) + elif isfile(join(path, '__init__.py')): + suite.addTests(loader.discover(path, top_level_dir=str(abspath(join(path, "../../.."))))) + for f in files: + # turn 'f' into a filename relative to the toplevel dir... + f = relpath(join(dir, f), loader._top_level_dir) + # ...and translate it to a module name. + f = splitext(normpath(f.replace(os.path.sep, '.')))[0] + suite.addTests(loader.loadTestsFromName(f)) + return suite + except Exception: + traceback.print_exc(file=sys.stderr) + sys.exit(-1) + + +def run_tests(argv=None, defaultTest=None, topleveldir=None, + xmloutput=None, verbosity=1, nomultiproc=False): + """ + args + ---- + - xmloutput [str or None] + Path of XML output directory (optional) + - verbosity [int] + Verbosity level of tests output + + Returns the TestResult object after running the test *suite*. + """ + + if xmloutput is not None: + import xmlrunner + runner = xmlrunner.XMLTestRunner(output=xmloutput) + else: + runner = None + prog = NumbaTestProgram(argv=argv, + module=None, + defaultTest=defaultTest, + topleveldir=topleveldir, + testRunner=runner, exit=False, + verbosity=verbosity, + nomultiproc=nomultiproc) + return prog.result diff --git a/sdc/testing/_runtests.py b/sdc/testing/_runtests.py new file mode 100644 index 000000000..a0634c899 --- /dev/null +++ b/sdc/testing/_runtests.py @@ -0,0 +1,114 @@ +import json +import re +import logging + + +def _main(argv, **kwds): + from sdc.testing import run_tests + # This helper function assumes the first element of argv + # is the name of the calling program. + # The 'main' API function is invoked in-process, and thus + # will synthesize that name. + + if '--log' in argv: + logging.basicConfig(level=logging.DEBUG) + argv.remove('--log') + + if '--failed-first' in argv: + # Failed first + argv.remove('--failed-first') + return _FailedFirstRunner().main(argv, kwds) + elif '--last-failed' in argv: + argv.remove('--last-failed') + return _FailedFirstRunner(last_failed=True).main(argv, kwds) + else: + return run_tests(argv, defaultTest='sdc.tests', + **kwds).wasSuccessful() + + +def main(*argv, **kwds): + """keyword arguments are accepted for backward compatibility only. + See `numba.testing.run_tests()` documentation for details.""" + return _main(['
'] + list(argv), **kwds) + + +class _FailedFirstRunner(object): + """ + Test Runner to handle the failed-first (--failed-first) option. + """ + cache_filename = '.runtests_lastfailed' + + def __init__(self, last_failed=False): + self.last_failed = last_failed + + def main(self, argv, kwds): + from sdc.testing import run_tests + prog = argv[0] + argv = argv[1:] + flags = [a for a in argv if a.startswith('-')] + + all_tests, failed_tests = self.find_last_failed(argv) + # Prepare tests to run + if failed_tests: + ft = "There were {} previously failed tests" + print(ft.format(len(failed_tests))) + remaing_tests = [t for t in all_tests + if t not in failed_tests] + if self.last_failed: + tests = list(failed_tests) + else: + tests = failed_tests + remaing_tests + else: + if self.last_failed: + tests = [] + else: + tests = list(all_tests) + + if not tests: + print("No tests to run") + return True + # Run the testsuite + print("Running {} tests".format(len(tests))) + print('Flags', flags) + result = run_tests([prog] + flags + tests, **kwds) + # Update failed tests records only if we have run the all the tests + # last failed. + if len(tests) == result.testsRun: + self.save_failed_tests(result, all_tests) + return result.wasSuccessful() + + def save_failed_tests(self, result, all_tests): + print("Saving failed tests to {}".format(self.cache_filename)) + cache = [] + # Find failed tests + failed = set() + for case in result.errors + result.failures: + failed.add(case[0].id()) + # Build cache + for t in all_tests: + if t in failed: + cache.append(t) + # Write cache + with open(self.cache_filename, 'w') as fout: + json.dump(cache, fout) + + def find_last_failed(self, argv): + from numba.tests.support import captured_output + + # Find all tests + listargv = ['-l'] + [a for a in argv if not a.startswith('-')] + with captured_output("stdout") as stream: + main(*listargv) + + pat = re.compile(r"^(\w+\.)+\w+$") + lines = stream.getvalue().splitlines() + all_tests = [x for x in lines if pat.match(x) is not None] + + try: + fobj = open(self.cache_filename) + except IOError: + failed_tests = [] + else: + with fobj as fin: + failed_tests = json.load(fin) + return all_tests, failed_tests diff --git a/sdc/tests/__init__.py b/sdc/tests/__init__.py index eaba6a8a9..76f3ca60c 100644 --- a/sdc/tests/__init__.py +++ b/sdc/tests/__init__.py @@ -1,54 +1,10 @@ -# ***************************************************************************** -# Copyright (c) 2019-2020, Intel Corporation All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# -# Redistributions of source code must retain the above copyright notice, -# this list of conditions and the following disclaimer. -# -# Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, -# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; -# OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, -# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR -# OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, -# EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# ***************************************************************************** +from os.path import dirname +from unittest.suite import TestSuite +from sdc.testing import load_testsuite -from sdc.tests.test_basic import * -from sdc.tests.test_series import * -from sdc.tests.test_dataframe import * -from sdc.tests.test_hiframes import * -from .categorical import * - -# from sdc.tests.test_d4p import * -from sdc.tests.test_date import * -from sdc.tests.test_strings import * - -from sdc.tests.test_groupby import * -from sdc.tests.test_join import * -from sdc.tests.test_rolling import * - -from sdc.tests.test_ml import * - -from sdc.tests.test_io import * - -from sdc.tests.test_hpat_jit import * -from sdc.tests.test_indexes import * - -from sdc.tests.test_sdc_numpy import * -from sdc.tests.test_prange_utils import * -from sdc.tests.test_compile_time import * - -# performance tests -import sdc.tests.tests_perf +def load_tests(loader, tests, pattern): + suite = TestSuite() + suite.addTests(load_testsuite(loader, dirname(__file__))) + return suite + \ No newline at end of file