From a3db0c7ec7191133f55e302ca72f278c277a5dc8 Mon Sep 17 00:00:00 2001
From: Ryan Gonzalez <rymg19@gmail.com>
Date: Sun, 12 Jun 2016 17:30:46 -0500
Subject: [PATCH 01/20] Rename test-requirements.txt ->
 requirements-testing.txt to avoid pytest trying to run it

---
 .travis.yml                                       | 2 +-
 test-requirements.txt => requirements-testing.txt | 0
 2 files changed, 1 insertion(+), 1 deletion(-)
 rename test-requirements.txt => requirements-testing.txt (100%)

diff --git a/.travis.yml b/.travis.yml
index 4110653853bb..43498026bd0c 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -9,7 +9,7 @@ python:
   # - "pypy3"
 
 install:
-  - pip install -r test-requirements.txt
+  - pip install -r requirements-testing.txt
   - python setup.py install
 
 script:
diff --git a/test-requirements.txt b/requirements-testing.txt
similarity index 100%
rename from test-requirements.txt
rename to requirements-testing.txt

From 8b29ba0784cfe12270aa8b3f039ad613079853db Mon Sep 17 00:00:00 2001
From: Ryan Gonzalez <rymg19@gmail.com>
Date: Tue, 14 Jun 2016 17:18:37 -0500
Subject: [PATCH 02/20] Move most of testcheck over to pytest

---
 conftest.py                               |  3 ++
 mypy/test/data.py                         | 24 ++++++------
 mypy/test/{testcheck.py => test_check.py} | 48 ++++++++++++++++-------
 pytest.ini                                |  3 ++
 4 files changed, 52 insertions(+), 26 deletions(-)
 create mode 100644 conftest.py
 rename mypy/test/{testcheck.py => test_check.py} (84%)
 create mode 100644 pytest.ini

diff --git a/conftest.py b/conftest.py
new file mode 100644
index 000000000000..193b2795179e
--- /dev/null
+++ b/conftest.py
@@ -0,0 +1,3 @@
+def pytest_addoption(parser):
+    parser.addoption('--update-testcases', action='store_true',
+                     dest='UPDATE_TESTCASES')
diff --git a/mypy/test/data.py b/mypy/test/data.py
index bc5b7c40fc0d..a1ed7276491f 100644
--- a/mypy/test/data.py
+++ b/mypy/test/data.py
@@ -6,14 +6,14 @@
 from os import remove, rmdir
 import shutil
 
-from typing import Callable, List, Tuple
+from typing import Callable, List, Tuple, Any
 
 from mypy.myunit import TestCase, SkipTestCaseException
 
 
 def parse_test_cases(
         path: str,
-        perform: Callable[['DataDrivenTestCase'], None],
+        perform: Callable[[Any, 'DataDrivenTestCase'], None],
         base_path: str = '.',
         optional_out: bool = False,
         include_path: str = None,
@@ -88,23 +88,26 @@ def parse_test_cases(
     return out
 
 
-class DataDrivenTestCase(TestCase):
+class DataDrivenTestCase:
+    name = None # type: str
     input = None  # type: List[str]
     output = None  # type: List[str]
 
     file = ''
     line = 0
 
-    perform = None  # type: Callable[['DataDrivenTestCase'], None]
+    perform = None  # type: Callable[[Any, 'DataDrivenTestCase'], None]
 
     # (file path, file content) tuples
     files = None  # type: List[Tuple[str, str]]
 
     clean_up = None  # type: List[Tuple[bool, str]]
 
+    is_skip = False # type: bool
+
     def __init__(self, name, input, output, file, line, lastline,
                  perform, files):
-        super().__init__(name)
+        self.name = name
         self.input = input
         self.output = output
         self.lastline = lastline
@@ -112,9 +115,9 @@ def __init__(self, name, input, output, file, line, lastline,
         self.line = line
         self.perform = perform
         self.files = files
+        self.is_skip = self.name.endswith('-skip')
 
     def set_up(self) -> None:
-        super().set_up()
         self.clean_up = []
         for path, content in self.files:
             dir = os.path.dirname(path)
@@ -137,11 +140,9 @@ def add_dirs(self, dir: str) -> List[str]:
             os.mkdir(dir)
             return dirs
 
-    def run(self):
-        if self.name.endswith('-skip'):
-            raise SkipTestCaseException()
-        else:
-            self.perform(self)
+    def run(self, obj: Any):
+        assert not self.is_skip
+        self.perform(obj, self)
 
     def tear_down(self) -> None:
         # First remove files.
@@ -172,7 +173,6 @@ def tear_down(self) -> None:
                     if path.startswith('tmp/') and os.path.isdir(path):
                         shutil.rmtree(path)
                     raise
-        super().tear_down()
 
 
 class TestItem:
diff --git a/mypy/test/testcheck.py b/mypy/test/test_check.py
similarity index 84%
rename from mypy/test/testcheck.py
rename to mypy/test/test_check.py
index d64f3a4dec36..e1eff51a59b5 100644
--- a/mypy/test/testcheck.py
+++ b/mypy/test/test_check.py
@@ -4,18 +4,16 @@
 import re
 import shutil
 import sys
+import pytest
 
 from typing import Tuple, List, Dict, Set
 
 from mypy import build
-import mypy.myunit  # for mutable globals (ick!)
 from mypy.build import BuildSource, find_module_clear_caches
-from mypy.myunit import Suite, AssertionFailure
 from mypy.test.config import test_temp_dir, test_data_prefix
 from mypy.test.data import parse_test_cases, DataDrivenTestCase
 from mypy.test.helpers import (
-    assert_string_arrays_equal, normalize_error_messages,
-    testcase_pyversion, update_testcase_output,
+    normalize_error_messages, testcase_pyversion, update_testcase_output,
 )
 from mypy.errors import CompileError
 
@@ -61,14 +59,35 @@
 ]
 
 
-class TypeCheckSuite(Suite):
+@pytest.fixture(scope='function')
+def test(request):
+    test = request.function.test
+    test.set_up()
+    request.addfinalizer(test.tear_down)
+    return test
 
-    def cases(self) -> List[DataDrivenTestCase]:
+
+class TestTypeCheck:
+    @classmethod
+    def setup_tests(cls):
         c = []  # type: List[DataDrivenTestCase]
         for f in files:
             c += parse_test_cases(os.path.join(test_data_prefix, f),
-                                  self.run_test, test_temp_dir, True)
-        return c
+                                  cls.run_test, test_temp_dir, True)
+        for test in c:
+            def func(self, test):
+                test.run(self)
+            if test.is_skip:
+                func = pytest.mark.skip(reason='Test ends with -skip')(func)
+            if 'FastParse' in test.name and not test.is_skip:
+                try:
+                    import mypy.fastparse
+                except SystemExit:
+                    func = pytest.mark.skip(
+                        reason='You must install the typed_ast package in ' \
+                               'order to run this test')(func)
+            func.test = test
+            setattr(cls, test.name, func)
 
     def run_test(self, testcase: DataDrivenTestCase) -> None:
         incremental = 'Incremental' in testcase.name.lower() or 'incremental' in testcase.file
@@ -122,13 +141,12 @@ def run_test_once(self, testcase: DataDrivenTestCase, incremental=0) -> None:
             a = e.messages
         a = normalize_error_messages(a)
 
-        if output != a and mypy.myunit.UPDATE_TESTCASES:
+        if output != a and pytest.config.getoption('UPDATE_TESTCASES'):
             update_testcase_output(testcase, a, mypy.myunit.APPEND_TESTCASES)
 
-        assert_string_arrays_equal(
-            output, a,
+        assert output == a, \
             'Invalid type checker output ({}, line {})'.format(
-                testcase.file, testcase.line))
+                testcase.file, testcase.line)
 
         if incremental and res:
             self.verify_cache(module_name, program_name, a, res.manager)
@@ -146,8 +164,8 @@ def verify_cache(self, module_name: str, program_name: str, a: List[str],
         modules.update({module_name: program_name})
         missing_paths = self.find_missing_cache_files(modules, manager)
         if missing_paths != error_paths:
-            raise AssertionFailure("cache data discrepancy %s != %s" %
-                                   (missing_paths, error_paths))
+            assert False, "cache data discrepancy %s != %s" % \
+                            (missing_paths, error_paths)
 
     def find_error_paths(self, a: List[str]) -> Set[str]:
         hits = set()
@@ -209,3 +227,5 @@ def parse_flags(self, program_text: str) -> List[str]:
             return m.group(1).split()
         else:
             return []
+
+TestTypeCheck.setup_tests()
diff --git a/pytest.ini b/pytest.ini
new file mode 100644
index 000000000000..0ebacaef0bbe
--- /dev/null
+++ b/pytest.ini
@@ -0,0 +1,3 @@
+[pytest]
+norecursedirs=stdlib-samples lib-typing pinfer mypy/codec
+; python_functions=test_*

From 6435c002719485067ad6159e7ee13d1a98eecc96 Mon Sep 17 00:00:00 2001
From: Ryan Gonzalez <rymg19@gmail.com>
Date: Tue, 14 Jun 2016 17:29:04 -0500
Subject: [PATCH 03/20] Tweaks

---
 mypy/test/test_check.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/mypy/test/test_check.py b/mypy/test/test_check.py
index e1eff51a59b5..923c88bc2de8 100644
--- a/mypy/test/test_check.py
+++ b/mypy/test/test_check.py
@@ -163,9 +163,7 @@ def verify_cache(self, module_name: str, program_name: str, a: List[str],
         modules = self.find_module_files()
         modules.update({module_name: program_name})
         missing_paths = self.find_missing_cache_files(modules, manager)
-        if missing_paths != error_paths:
-            assert False, "cache data discrepancy %s != %s" % \
-                            (missing_paths, error_paths)
+        assert missing_paths == error_paths, 'cache data discrepancy'
 
     def find_error_paths(self, a: List[str]) -> Set[str]:
         hits = set()

From a57a633d9897724ea1ae3634f97219bfa4f3cf3a Mon Sep 17 00:00:00 2001
From: Ryan Gonzalez <rymg19@gmail.com>
Date: Tue, 14 Jun 2016 18:10:39 -0500
Subject: [PATCH 04/20] Update runtests.py for pytest

---
 runtests.py | 24 +++++++++++++++++++-----
 1 file changed, 19 insertions(+), 5 deletions(-)

diff --git a/runtests.py b/runtests.py
index 82b42c3f58a4..b001e61ba29b 100755
--- a/runtests.py
+++ b/runtests.py
@@ -39,11 +39,12 @@ def get_versions():  # type: () -> typing.List[str]
 class Driver:
 
     def __init__(self, whitelist: List[str], blacklist: List[str],
-            arglist: List[str], verbosity: int, parallel_limit: int,
-            xfail: List[str]) -> None:
+            arglist: List[str], testarglist: List[str], verbosity: int,
+            parallel_limit: int, xfail: List[str]) -> None:
         self.whitelist = whitelist
         self.blacklist = blacklist
         self.arglist = arglist
+        self.testarglist = testarglist
         self.verbosity = verbosity
         self.waiter = Waiter(verbosity=verbosity, limit=parallel_limit, xfail=xfail)
         self.versions = get_versions()
@@ -188,6 +189,8 @@ def add_imports(driver: Driver) -> None:
             driver.add_python_string('import %s' % mod, 'import %s' % mod)
         driver.add_flake8('module %s' % mod, f)
 
+def add_pytest(driver: Driver) -> None:
+    driver.add_python_mod('pytest', 'pytest', *driver.testarglist)
 
 def add_myunit(driver: Driver) -> None:
     for f in find_files('mypy', prefix='test', suffix='.py'):
@@ -265,7 +268,8 @@ def usage(status: int) -> None:
     print('Run mypy tests. If given no arguments, run all tests.')
     print()
     print('Examples:')
-    print('  %s unit-test  (run unit tests only)' % sys.argv[0])
+    print('  %s pytest     (run pytest only)')
+    print('  %s unit-test  (run unit tests and pytest only)' % sys.argv[0])
     print('  %s unit-test -a "*tuple*"' % sys.argv[0])
     print('       (run all unit tests with "tuple" in test name)')
     print()
@@ -276,6 +280,7 @@ def usage(status: int) -> None:
     print('  -jN                    run N tasks at once (default: one per CPU)')
     print('  -a, --argument ARG     pass an argument to myunit tasks')
     print('                         (-v: verbose; glob pattern: filter by test name)')
+    print('  -t, --test-arg ARG     pass an argument to pytest')
     print('  -l, --list             list included tasks (after filtering) and exit')
     print('  FILTER                 include tasks matching FILTER')
     print('  -x, --exclude FILTER   exclude tasks matching FILTER')
@@ -306,13 +311,15 @@ def main() -> None:
     whitelist = []  # type: List[str]
     blacklist = []  # type: List[str]
     arglist = []  # type: List[str]
+    testarglist = [] # type: List[str]
     list_only = False
     dirty_stubs = False
 
     allow_opts = True
     curlist = whitelist
     for a in sys.argv[1:]:
-        if curlist is not arglist and allow_opts and a.startswith('-'):
+        if (curlist is not arglist and curlist is not testarglist and
+            allow_opts and a.startswith('-')):
             if curlist is not whitelist:
                 break
             if a == '--':
@@ -330,6 +337,8 @@ def main() -> None:
                 curlist = blacklist
             elif a == '-a' or a == '--argument':
                 curlist = arglist
+            elif a == '-t' or a == '--test-args':
+                curlist = testarglist
             elif a == '-l' or a == '--list':
                 list_only = True
             elif a == '-f' or a == '--dirty-stubs':
@@ -349,8 +358,12 @@ def main() -> None:
     if not whitelist:
         whitelist.append('')
 
+    if 'unit-test' in whitelist and 'pytest' not in whitelist:
+        whitelist.append('pytest')
+
     driver = Driver(whitelist=whitelist, blacklist=blacklist, arglist=arglist,
-            verbosity=verbosity, parallel_limit=parallel_limit, xfail=[])
+            testarglist=testarglist, verbosity=verbosity,
+            parallel_limit=parallel_limit, xfail=[])
 
     if not dirty_stubs:
         git.verify_git_integrity_or_abort(driver.cwd)
@@ -365,6 +378,7 @@ def main() -> None:
     add_basic(driver)
     add_selftypecheck(driver)
     add_myunit(driver)
+    add_pytest(driver)
     add_imports(driver)
     add_stubs(driver)
     add_stdlibsamples(driver)

From 1bf37e7246c9d19815108f5c1947e6159503ad56 Mon Sep 17 00:00:00 2001
From: Ryan Gonzalez <rymg19@gmail.com>
Date: Wed, 15 Jun 2016 13:14:55 -0500
Subject: [PATCH 05/20] Move test_cmdline tests to pytest and reorg some stuff

---
 mypy/test/helpers.py                          | 30 +++++++++++++++++++
 mypy/test/test_check.py                       | 29 +++---------------
 mypy/test/{testcmdline.py => test_cmdline.py} | 17 ++++++-----
 runtests.py                                   | 19 +++++-------
 4 files changed, 50 insertions(+), 45 deletions(-)
 rename mypy/test/{testcmdline.py => test_cmdline.py} (82%)

diff --git a/mypy/test/helpers.py b/mypy/test/helpers.py
index d1b5a65a02c9..e81e291d8f7e 100644
--- a/mypy/test/helpers.py
+++ b/mypy/test/helpers.py
@@ -1,6 +1,7 @@
 import sys
 import re
 import os
+import pytest
 
 from typing import List, Dict, Tuple
 
@@ -284,3 +285,32 @@ def normalize_error_messages(messages: List[str]) -> List[str]:
     for m in messages:
         a.append(m.replace(os.sep, '/'))
     return a
+
+
+@pytest.fixture(scope='function')
+def test(request):
+    test = request.function.test
+    test.set_up()
+    request.addfinalizer(test.tear_down)
+    return test
+
+
+class PytestSuite:
+    """Assists in setting up data-driven test cases for pytest."""
+    @classmethod
+    def setup_tests(cls):
+        c = cls.cases()  # type: List[DataDrivenTestCase]
+        for test in c:
+            def func(self, test):
+                test.run(self)
+            if test.is_skip:
+                func = pytest.mark.skip(reason='Test ends with -skip')(func)
+            if 'FastParse' in test.name and not test.is_skip:
+                try:
+                    import mypy.fastparse
+                except SystemExit:
+                    func = pytest.mark.skip(
+                        reason='You must install the typed_ast package in ' \
+                               'order to run this test')(func)
+            func.test = test
+            setattr(cls, test.name, func)
diff --git a/mypy/test/test_check.py b/mypy/test/test_check.py
index 923c88bc2de8..d12893f9de79 100644
--- a/mypy/test/test_check.py
+++ b/mypy/test/test_check.py
@@ -4,7 +4,6 @@
 import re
 import shutil
 import sys
-import pytest
 
 from typing import Tuple, List, Dict, Set
 
@@ -14,6 +13,7 @@
 from mypy.test.data import parse_test_cases, DataDrivenTestCase
 from mypy.test.helpers import (
     normalize_error_messages, testcase_pyversion, update_testcase_output,
+    PytestSuite, test
 )
 from mypy.errors import CompileError
 
@@ -59,35 +59,14 @@
 ]
 
 
-@pytest.fixture(scope='function')
-def test(request):
-    test = request.function.test
-    test.set_up()
-    request.addfinalizer(test.tear_down)
-    return test
-
-
-class TestTypeCheck:
+class TestTypeCheck(PytestSuite):
     @classmethod
-    def setup_tests(cls):
+    def cases(cls):
         c = []  # type: List[DataDrivenTestCase]
         for f in files:
             c += parse_test_cases(os.path.join(test_data_prefix, f),
                                   cls.run_test, test_temp_dir, True)
-        for test in c:
-            def func(self, test):
-                test.run(self)
-            if test.is_skip:
-                func = pytest.mark.skip(reason='Test ends with -skip')(func)
-            if 'FastParse' in test.name and not test.is_skip:
-                try:
-                    import mypy.fastparse
-                except SystemExit:
-                    func = pytest.mark.skip(
-                        reason='You must install the typed_ast package in ' \
-                               'order to run this test')(func)
-            func.test = test
-            setattr(cls, test.name, func)
+        return c
 
     def run_test(self, testcase: DataDrivenTestCase) -> None:
         incremental = 'Incremental' in testcase.name.lower() or 'incremental' in testcase.file
diff --git a/mypy/test/testcmdline.py b/mypy/test/test_cmdline.py
similarity index 82%
rename from mypy/test/testcmdline.py
rename to mypy/test/test_cmdline.py
index a78cbe265a79..30e4bcfa9000 100644
--- a/mypy/test/testcmdline.py
+++ b/mypy/test/test_cmdline.py
@@ -14,7 +14,7 @@
 from mypy.myunit import Suite, SkipTestCaseException
 from mypy.test.config import test_data_prefix, test_temp_dir
 from mypy.test.data import parse_test_cases, DataDrivenTestCase
-from mypy.test.helpers import assert_string_arrays_equal
+from mypy.test.helpers import PytestSuite, test
 
 # Path to Python 3 interpreter
 python3_path = sys.executable
@@ -23,9 +23,9 @@
 cmdline_files = ['cmdline.test']
 
 
-class PythonEvaluationSuite(Suite):
-
-    def cases(self) -> List[DataDrivenTestCase]:
+class TestPythonEvaluation(PytestSuite):
+    @classmethod
+    def cases(cls) -> List[DataDrivenTestCase]:
         c = []  # type: List[DataDrivenTestCase]
         for f in cmdline_files:
             c += parse_test_cases(os.path.join(test_data_prefix, f),
@@ -36,7 +36,7 @@ def cases(self) -> List[DataDrivenTestCase]:
         return c
 
 
-def test_python_evaluation(testcase: DataDrivenTestCase) -> None:
+def test_python_evaluation(obj: None, testcase: DataDrivenTestCase) -> None:
     # Write the program to a file.
     program = '_program.py'
     program_path = os.path.join(test_temp_dir, program)
@@ -58,9 +58,8 @@ def test_python_evaluation(testcase: DataDrivenTestCase) -> None:
     # Remove temp file.
     os.remove(program_path)
     # Compare actual output to expected.
-    assert_string_arrays_equal(testcase.output, out,
-                               'Invalid output ({}, line {})'.format(
-                                   testcase.file, testcase.line))
+    assert testcase.output == out, 'Invalid output ({}, line {})'.format(
+                                        testcase.file, testcase.line)
 
 
 def parse_args(line: str) -> List[str]:
@@ -78,3 +77,5 @@ def parse_args(line: str) -> List[str]:
     if not m:
         return []  # No args; mypy will spit out an error.
     return m.group(1).split()
+
+TestPythonEvaluation.setup_tests() # MUST come after def of test_python_evaluation.
diff --git a/runtests.py b/runtests.py
index b001e61ba29b..d7fa07d059df 100755
--- a/runtests.py
+++ b/runtests.py
@@ -189,9 +189,11 @@ def add_imports(driver: Driver) -> None:
             driver.add_python_string('import %s' % mod, 'import %s' % mod)
         driver.add_flake8('module %s' % mod, f)
 
+
 def add_pytest(driver: Driver) -> None:
     driver.add_python_mod('pytest', 'pytest', *driver.testarglist)
 
+
 def add_myunit(driver: Driver) -> None:
     for f in find_files('mypy', prefix='test', suffix='.py'):
         mod = file_to_module(f)
@@ -199,10 +201,9 @@ def add_myunit(driver: Driver) -> None:
             # myunit is Python3 only.
             driver.add_python_mod('unittest %s' % mod, 'unittest', mod)
             driver.add_python2('unittest %s' % mod, '-m', 'unittest', mod)
-        elif mod in ('mypy.test.testpythoneval', 'mypy.test.testcmdline'):
-            # Run Python evaluation integration tests and command-line
-            # parsing tests separately since they are much slower than
-            # proper unit tests.
+        elif mod == 'mypy.test.testpythoneval':
+            # Run Python evaluation integration tests separately since they are #
+            # much slower than proper unit tests.
             pass
         else:
             driver.add_python_mod('unit-test %s' % mod, 'mypy.myunit', '-m', mod, *driver.arglist)
@@ -213,11 +214,6 @@ def add_pythoneval(driver: Driver) -> None:
                           '-m', 'mypy.test.testpythoneval', *driver.arglist)
 
 
-def add_cmdline(driver: Driver) -> None:
-    driver.add_python_mod('cmdline-test', 'mypy.myunit',
-                          '-m', 'mypy.test.testcmdline', *driver.arglist)
-
-
 def add_stubs(driver: Driver) -> None:
     # We only test each module in the one version mypy prefers to find.
     # TODO: test stubs for other versions, especially Python 2 stubs.
@@ -311,7 +307,7 @@ def main() -> None:
     whitelist = []  # type: List[str]
     blacklist = []  # type: List[str]
     arglist = []  # type: List[str]
-    testarglist = [] # type: List[str]
+    testarglist = []  # type: List[str]
     list_only = False
     dirty_stubs = False
 
@@ -319,7 +315,7 @@ def main() -> None:
     curlist = whitelist
     for a in sys.argv[1:]:
         if (curlist is not arglist and curlist is not testarglist and
-            allow_opts and a.startswith('-')):
+                allow_opts and a.startswith('-')):
             if curlist is not whitelist:
                 break
             if a == '--':
@@ -374,7 +370,6 @@ def main() -> None:
     driver.prepend_path('PYTHONPATH', [join(driver.cwd, 'lib-typing', v) for v in driver.versions])
 
     add_pythoneval(driver)
-    add_cmdline(driver)
     add_basic(driver)
     add_selftypecheck(driver)
     add_myunit(driver)

From c2a3f9c18b16e22f49bccd171f1f45d3c86ae2f7 Mon Sep 17 00:00:00 2001
From: Ryan Gonzalez <rymg19@gmail.com>
Date: Wed, 15 Jun 2016 13:15:43 -0500
Subject: [PATCH 06/20] Cleanup

---
 mypy/test/data.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/mypy/test/data.py b/mypy/test/data.py
index a1ed7276491f..08b07f937f22 100644
--- a/mypy/test/data.py
+++ b/mypy/test/data.py
@@ -8,8 +8,6 @@
 
 from typing import Callable, List, Tuple, Any
 
-from mypy.myunit import TestCase, SkipTestCaseException
-
 
 def parse_test_cases(
         path: str,

From 30089db3e7137b14395a142c52ac538821940e02 Mon Sep 17 00:00:00 2001
From: Ryan Gonzalez <rymg19@gmail.com>
Date: Wed, 15 Jun 2016 13:21:07 -0500
Subject: [PATCH 07/20] Move over test_semanal

---
 mypy/test/{testsemanal.py => test_semanal.py} | 35 +++++++++++--------
 1 file changed, 20 insertions(+), 15 deletions(-)
 rename mypy/test/{testsemanal.py => test_semanal.py} (91%)

diff --git a/mypy/test/testsemanal.py b/mypy/test/test_semanal.py
similarity index 91%
rename from mypy/test/testsemanal.py
rename to mypy/test/test_semanal.py
index 9611284e793e..b12e3c62394d 100644
--- a/mypy/test/testsemanal.py
+++ b/mypy/test/test_semanal.py
@@ -6,9 +6,9 @@
 
 from mypy import build
 from mypy.build import BuildSource
-from mypy.myunit import Suite
 from mypy.test.helpers import (
     assert_string_arrays_equal, normalize_error_messages, testfile_pyversion,
+    PytestSuite
 )
 from mypy.test.data import parse_test_cases
 from mypy.test.config import test_data_prefix, test_temp_dir
@@ -31,19 +31,20 @@
                  'semanal-python2.test']
 
 
-class SemAnalSuite(Suite):
-    def cases(self):
+class SemAnalSuite(PytestSuite):
+    @classmethod
+    def cases(cls):
         c = []
         for f in semanal_files:
             c += parse_test_cases(os.path.join(test_data_prefix, f),
-                                  test_semanal,
+                                  semanal_tests,
                                   base_path=test_temp_dir,
                                   optional_out=True,
                                   native_sep=True)
         return c
 
 
-def test_semanal(testcase):
+def semanal_tests(obj, testcase):
     """Perform a semantic analysis test case.
 
     The testcase argument contains a description of the test case
@@ -88,17 +89,19 @@ def test_semanal(testcase):
 semanal_error_files = ['semanal-errors.test']
 
 
-class SemAnalErrorSuite(Suite):
-    def cases(self):
+class SemAnalErrorSuite(PytestSuite):
+    @classmethod
+    def cases(cls):
         # Read test cases from test case description files.
         c = []
         for f in semanal_error_files:
             c += parse_test_cases(os.path.join(test_data_prefix, f),
-                                  test_semanal_error, test_temp_dir, optional_out=True)
+                                  semanal_error_tests, test_temp_dir,
+                                  optional_out=True)
         return c
 
 
-def test_semanal_error(testcase):
+def semanal_error_tests(obj, testcase):
     """Perform a test case."""
 
     try:
@@ -124,12 +127,13 @@ def test_semanal_error(testcase):
 semanal_symtable_files = ['semanal-symtable.test']
 
 
-class SemAnalSymtableSuite(Suite):
-    def cases(self):
+class SemAnalSymtableSuite(PytestSuite):
+    @classmethod
+    def cases(cls):
         c = []
         for f in semanal_symtable_files:
             c += parse_test_cases(os.path.join(test_data_prefix, f),
-                                  self.run_test, test_temp_dir)
+                                  cls.run_test, test_temp_dir)
         return c
 
     def run_test(self, testcase):
@@ -163,13 +167,14 @@ def run_test(self, testcase):
 semanal_typeinfo_files = ['semanal-typeinfo.test']
 
 
-class SemAnalTypeInfoSuite(Suite):
-    def cases(self):
+class SemAnalTypeInfoSuite(PytestSuite):
+    @classmethod
+    def cases(cls):
         """Test case descriptions"""
         c = []
         for f in semanal_typeinfo_files:
             c += parse_test_cases(os.path.join(test_data_prefix, f),
-                                  self.run_test, test_temp_dir)
+                                  cls.run_test, test_temp_dir)
         return c
 
     def run_test(self, testcase):

From 49ccce1fce0516641b7a7ad3557e87c7c68db2ed Mon Sep 17 00:00:00 2001
From: Ryan Gonzalez <rymg19@gmail.com>
Date: Thu, 16 Jun 2016 15:08:03 -0500
Subject: [PATCH 08/20] Fix myunit tests

---
 mypy/test/data.py | 22 ++++++++++++++++++----
 runtests.py       |  3 ++-
 2 files changed, 20 insertions(+), 5 deletions(-)

diff --git a/mypy/test/data.py b/mypy/test/data.py
index 08b07f937f22..48f2d6d4dc24 100644
--- a/mypy/test/data.py
+++ b/mypy/test/data.py
@@ -8,6 +8,7 @@
 
 from typing import Callable, List, Tuple, Any
 
+from mypy.myunit import TestCase, SkipTestCaseException
 
 def parse_test_cases(
         path: str,
@@ -86,7 +87,7 @@ def parse_test_cases(
     return out
 
 
-class DataDrivenTestCase:
+class DataDrivenTestCase(TestCase):
     name = None # type: str
     input = None  # type: List[str]
     output = None  # type: List[str]
@@ -105,6 +106,7 @@ class DataDrivenTestCase:
 
     def __init__(self, name, input, output, file, line, lastline,
                  perform, files):
+        super().__init__(name)
         self.name = name
         self.input = input
         self.output = output
@@ -116,6 +118,7 @@ def __init__(self, name, input, output, file, line, lastline,
         self.is_skip = self.name.endswith('-skip')
 
     def set_up(self) -> None:
+        super().set_up()
         self.clean_up = []
         for path, content in self.files:
             dir = os.path.dirname(path)
@@ -138,9 +141,19 @@ def add_dirs(self, dir: str) -> List[str]:
             os.mkdir(dir)
             return dirs
 
-    def run(self, obj: Any):
-        assert not self.is_skip
-        self.perform(obj, self)
+    def run(self, obj: Any = None):
+        if obj is None:
+            # XXX: The unit tests are being converted over to pytest. Due to
+            # modifications requires to make BOTH run at the moment, this branch
+            # is necessary. It should be removed once all the tests relying on
+            # DataDrivenTestCase are converted to pytest.
+            if self.is_skip:
+                raise SkipTestCaseException()
+            else:
+                self.perform(self)
+        else:
+            assert not self.is_skip
+            self.perform(obj, self)
 
     def tear_down(self) -> None:
         # First remove files.
@@ -171,6 +184,7 @@ def tear_down(self) -> None:
                     if path.startswith('tmp/') and os.path.isdir(path):
                         shutil.rmtree(path)
                     raise
+        super().tear_down()
 
 
 class TestItem:
diff --git a/runtests.py b/runtests.py
index d7fa07d059df..3f26fcfa8d75 100755
--- a/runtests.py
+++ b/runtests.py
@@ -205,7 +205,8 @@ def add_myunit(driver: Driver) -> None:
             # Run Python evaluation integration tests separately since they are #
             # much slower than proper unit tests.
             pass
-        else:
+        elif not 'test_' in mod:
+            # Modules containing `test_` have been ported to pytest.
             driver.add_python_mod('unit-test %s' % mod, 'mypy.myunit', '-m', mod, *driver.arglist)
 
 

From e0bdfaca695c625f2d0dc4c72a6930d82caed577 Mon Sep 17 00:00:00 2001
From: Ryan Gonzalez <rymg19@gmail.com>
Date: Thu, 16 Jun 2016 15:09:36 -0500
Subject: [PATCH 09/20] Explanation

---
 mypy/test/helpers.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/mypy/test/helpers.py b/mypy/test/helpers.py
index e81e291d8f7e..a9666cc6de9f 100644
--- a/mypy/test/helpers.py
+++ b/mypy/test/helpers.py
@@ -299,6 +299,12 @@ class PytestSuite:
     """Assists in setting up data-driven test cases for pytest."""
     @classmethod
     def setup_tests(cls):
+        """
+        Sets up the child class's test case. The child must have a method
+        `cases` that returns a list of `DataDrivenTestCase`s. This method will
+        load the data-driven test cases and use setattr to assign it to the
+        class, which will allow pytest to recognize the test.
+        """
         c = cls.cases()  # type: List[DataDrivenTestCase]
         for test in c:
             def func(self, test):

From 7bf2f279ff663dcf0940fed877f374e7f2df9a4c Mon Sep 17 00:00:00 2001
From: Ryan Gonzalez <rymg19@gmail.com>
Date: Thu, 16 Jun 2016 15:16:07 -0500
Subject: [PATCH 10/20] Test fixes

---
 mypy/test/data.py         |  4 +++-
 mypy/test/test_semanal.py | 40 +++++++++++++++++++++------------------
 2 files changed, 25 insertions(+), 19 deletions(-)

diff --git a/mypy/test/data.py b/mypy/test/data.py
index 48f2d6d4dc24..3dafeba70281 100644
--- a/mypy/test/data.py
+++ b/mypy/test/data.py
@@ -158,10 +158,12 @@ def run(self, obj: Any = None):
     def tear_down(self) -> None:
         # First remove files.
         for is_dir, path in reversed(self.clean_up):
-            if not is_dir:
+            if not is_dir and os.path.exists(path):
                 remove(path)
         # Then remove directories.
         for is_dir, path in reversed(self.clean_up):
+            if not os.path.exists(path):
+                continue
             if is_dir:
                 pycache = os.path.join(path, '__pycache__')
                 if os.path.isdir(pycache):
diff --git a/mypy/test/test_semanal.py b/mypy/test/test_semanal.py
index b12e3c62394d..e3de09cfc382 100644
--- a/mypy/test/test_semanal.py
+++ b/mypy/test/test_semanal.py
@@ -7,8 +7,7 @@
 from mypy import build
 from mypy.build import BuildSource
 from mypy.test.helpers import (
-    assert_string_arrays_equal, normalize_error_messages, testfile_pyversion,
-    PytestSuite
+    normalize_error_messages, testfile_pyversion, PytestSuite, test
 )
 from mypy.test.data import parse_test_cases
 from mypy.test.config import test_data_prefix, test_temp_dir
@@ -31,7 +30,7 @@
                  'semanal-python2.test']
 
 
-class SemAnalSuite(PytestSuite):
+class TestSemAnal(PytestSuite):
     @classmethod
     def cases(cls):
         c = []
@@ -78,10 +77,11 @@ def semanal_tests(obj, testcase):
                 a += str(f).split('\n')
     except CompileError as e:
         a = e.messages
-    assert_string_arrays_equal(
-        testcase.output, a,
+    assert testcase.output == a, \
         'Invalid semantic analyzer output ({}, line {})'.format(testcase.file,
-                                                                testcase.line))
+                                                                testcase.line)
+
+TestSemAnal.setup_tests()
 
 # Semantic analyzer error test cases
 
@@ -89,7 +89,7 @@ def semanal_tests(obj, testcase):
 semanal_error_files = ['semanal-errors.test']
 
 
-class SemAnalErrorSuite(PytestSuite):
+class TestSemAnalError(PytestSuite):
     @classmethod
     def cases(cls):
         # Read test cases from test case description files.
@@ -116,9 +116,11 @@ def semanal_error_tests(obj, testcase):
         # Verify that there was a compile error and that the error messages
         # are equivalent.
         a = e.messages
-    assert_string_arrays_equal(
-        testcase.output, normalize_error_messages(a),
-        'Invalid compiler output ({}, line {})'.format(testcase.file, testcase.line))
+    assert testcase.output == normalize_error_messages(a), \
+        'Invalid compiler output ({}, line {})'.format(testcase.file,
+                                                       testcase.line)
+
+TestSemAnalError.setup_tests()
 
 
 # SymbolNode table export test cases
@@ -127,7 +129,7 @@ def semanal_error_tests(obj, testcase):
 semanal_symtable_files = ['semanal-symtable.test']
 
 
-class SemAnalSymtableSuite(PytestSuite):
+class TestSemAnalSymtable(PytestSuite):
     @classmethod
     def cases(cls):
         c = []
@@ -156,10 +158,11 @@ def run_test(self, testcase):
                         a.append('  ' + s)
         except CompileError as e:
             a = e.messages
-        assert_string_arrays_equal(
-            testcase.output, a,
+        assert testcase.output == a, \
             'Invalid semantic analyzer output ({}, line {})'.format(
-                testcase.file, testcase.line))
+                testcase.file, testcase.line)
+
+TestSemAnalSymtable.setup_tests()
 
 
 # Type info export test cases
@@ -167,7 +170,7 @@ def run_test(self, testcase):
 semanal_typeinfo_files = ['semanal-typeinfo.test']
 
 
-class SemAnalTypeInfoSuite(PytestSuite):
+class TestSemAnalTypeInfo(PytestSuite):
     @classmethod
     def cases(cls):
         """Test case descriptions"""
@@ -201,10 +204,11 @@ def run_test(self, testcase):
             a = str(typeinfos).split('\n')
         except CompileError as e:
             a = e.messages
-        assert_string_arrays_equal(
-            testcase.output, a,
+        assert testcase.output == a, \
             'Invalid semantic analyzer output ({}, line {})'.format(
-                testcase.file, testcase.line))
+                testcase.file, testcase.line)
+
+TestSemAnalTypeInfo.setup_tests()
 
 
 class TypeInfoMap(Dict[str, TypeInfo]):

From 8257ee83ddde536255065fd87223feffe8ce4deb Mon Sep 17 00:00:00 2001
From: Ryan Gonzalez <rymg19@gmail.com>
Date: Thu, 16 Jun 2016 15:30:21 -0500
Subject: [PATCH 11/20] Prevent picking up stuff that isn't actually a test

---
 mypy/test/helpers.py | 2 +-
 pytest.ini           | 3 ++-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/mypy/test/helpers.py b/mypy/test/helpers.py
index a9666cc6de9f..28e2b9c6df00 100644
--- a/mypy/test/helpers.py
+++ b/mypy/test/helpers.py
@@ -319,4 +319,4 @@ def func(self, test):
                         reason='You must install the typed_ast package in ' \
                                'order to run this test')(func)
             func.test = test
-            setattr(cls, test.name, func)
+            setattr(cls, test.name.replace('test', 'test_', 1), func)
diff --git a/pytest.ini b/pytest.ini
index 0ebacaef0bbe..09ab6d11baf0 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -1,3 +1,4 @@
 [pytest]
 norecursedirs=stdlib-samples lib-typing pinfer mypy/codec
-; python_functions=test_*
+; This is to prevent picking up non-test functions.
+python_functions=test_*

From 1efadc722c7be137debc6fc679ef94f4853953bb Mon Sep 17 00:00:00 2001
From: Ryan Gonzalez <rymg19@gmail.com>
Date: Sat, 18 Jun 2016 16:12:22 -0500
Subject: [PATCH 12/20] Don't attempt to run test helpers

---
 conftest.py               | 19 +++++++++++++++++++
 mypy/test/helpers.py      |  4 +++-
 mypy/test/test_cmdline.py |  4 ++--
 pytest.ini                |  3 +--
 4 files changed, 25 insertions(+), 5 deletions(-)

diff --git a/conftest.py b/conftest.py
index 193b2795179e..aa12df17bd97 100644
--- a/conftest.py
+++ b/conftest.py
@@ -1,3 +1,22 @@
+import pytest
+
 def pytest_addoption(parser):
     parser.addoption('--update-testcases', action='store_true',
                      dest='UPDATE_TESTCASES')
+
+# mypy.test.helpers defines several top-level utility functions that
+# pytest will pick up as tests. This removes them.
+def pytest_collection_modifyitems(items):
+    to_remove = []
+
+    for i, item in enumerate(items):
+        if (isinstance(item, pytest.Function) and
+                item.function.__module__ == 'mypy.test.helpers' and
+                # This is to prevent removing data-driven tests, which are
+                # defined by mypy.test.helpers.PytestSuite.
+                not getattr(item.function, 'is_test_attr', False)):
+            to_remove.append(i)
+
+    # reversed is to prevent changing indexes that haven't been removed yet.
+    for index in reversed(to_remove):
+        items.pop(index)
diff --git a/mypy/test/helpers.py b/mypy/test/helpers.py
index 28e2b9c6df00..56cdbcd5af56 100644
--- a/mypy/test/helpers.py
+++ b/mypy/test/helpers.py
@@ -309,6 +309,7 @@ def setup_tests(cls):
         for test in c:
             def func(self, test):
                 test.run(self)
+            func.is_test_attr = True
             if test.is_skip:
                 func = pytest.mark.skip(reason='Test ends with -skip')(func)
             if 'FastParse' in test.name and not test.is_skip:
@@ -319,4 +320,5 @@ def func(self, test):
                         reason='You must install the typed_ast package in ' \
                                'order to run this test')(func)
             func.test = test
-            setattr(cls, test.name.replace('test', 'test_', 1), func)
+            setattr(cls, test.name, func)
+            # setattr(cls, test.name.replace('test', 'test_', 1), func)
diff --git a/mypy/test/test_cmdline.py b/mypy/test/test_cmdline.py
index 30e4bcfa9000..0b8e3e5029ed 100644
--- a/mypy/test/test_cmdline.py
+++ b/mypy/test/test_cmdline.py
@@ -29,14 +29,14 @@ def cases(cls) -> List[DataDrivenTestCase]:
         c = []  # type: List[DataDrivenTestCase]
         for f in cmdline_files:
             c += parse_test_cases(os.path.join(test_data_prefix, f),
-                                  test_python_evaluation,
+                                  python_evaluation_test,
                                   base_path=test_temp_dir,
                                   optional_out=True,
                                   native_sep=True)
         return c
 
 
-def test_python_evaluation(obj: None, testcase: DataDrivenTestCase) -> None:
+def python_evaluation_test(obj: None, testcase: DataDrivenTestCase) -> None:
     # Write the program to a file.
     program = '_program.py'
     program_path = os.path.join(test_temp_dir, program)
diff --git a/pytest.ini b/pytest.ini
index 09ab6d11baf0..5327f691d30a 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -1,4 +1,3 @@
 [pytest]
 norecursedirs=stdlib-samples lib-typing pinfer mypy/codec
-; This is to prevent picking up non-test functions.
-python_functions=test_*
+addopts=--ignore=mypy/test/helpers.py

From d7095227af5d0d2a862ee9a4da3273809899aade Mon Sep 17 00:00:00 2001
From: Ryan Gonzalez <rymg19@gmail.com>
Date: Sat, 18 Jun 2016 16:37:35 -0500
Subject: [PATCH 13/20] Fix flake8

---
 mypy/test/data.py         | 5 +++--
 mypy/test/helpers.py      | 5 ++++-
 mypy/test/test_check.py   | 2 ++
 mypy/test/test_cmdline.py | 4 ++--
 runtests.py               | 2 +-
 5 files changed, 12 insertions(+), 6 deletions(-)

diff --git a/mypy/test/data.py b/mypy/test/data.py
index 3dafeba70281..be41b8ed55f9 100644
--- a/mypy/test/data.py
+++ b/mypy/test/data.py
@@ -10,6 +10,7 @@
 
 from mypy.myunit import TestCase, SkipTestCaseException
 
+
 def parse_test_cases(
         path: str,
         perform: Callable[[Any, 'DataDrivenTestCase'], None],
@@ -88,7 +89,7 @@ def parse_test_cases(
 
 
 class DataDrivenTestCase(TestCase):
-    name = None # type: str
+    name = None  # type: str
     input = None  # type: List[str]
     output = None  # type: List[str]
 
@@ -102,7 +103,7 @@ class DataDrivenTestCase(TestCase):
 
     clean_up = None  # type: List[Tuple[bool, str]]
 
-    is_skip = False # type: bool
+    is_skip = False
 
     def __init__(self, name, input, output, file, line, lastline,
                  perform, files):
diff --git a/mypy/test/helpers.py b/mypy/test/helpers.py
index 56cdbcd5af56..49d0b9b1faae 100644
--- a/mypy/test/helpers.py
+++ b/mypy/test/helpers.py
@@ -304,6 +304,9 @@ def setup_tests(cls):
         `cases` that returns a list of `DataDrivenTestCase`s. This method will
         load the data-driven test cases and use setattr to assign it to the
         class, which will allow pytest to recognize the test.
+
+        Note that this method **must** be run after the definition of any
+        functions used by PytestSuite.cases.
         """
         c = cls.cases()  # type: List[DataDrivenTestCase]
         for test in c:
@@ -317,7 +320,7 @@ def func(self, test):
                     import mypy.fastparse
                 except SystemExit:
                     func = pytest.mark.skip(
-                        reason='You must install the typed_ast package in ' \
+                        reason='You must install the typed_ast package in '
                                'order to run this test')(func)
             func.test = test
             setattr(cls, test.name, func)
diff --git a/mypy/test/test_check.py b/mypy/test/test_check.py
index d12893f9de79..013355f57ff8 100644
--- a/mypy/test/test_check.py
+++ b/mypy/test/test_check.py
@@ -4,6 +4,8 @@
 import re
 import shutil
 import sys
+import mypy
+import pytest
 
 from typing import Tuple, List, Dict, Set
 
diff --git a/mypy/test/test_cmdline.py b/mypy/test/test_cmdline.py
index 0b8e3e5029ed..c7fc4960fde4 100644
--- a/mypy/test/test_cmdline.py
+++ b/mypy/test/test_cmdline.py
@@ -59,7 +59,7 @@ def python_evaluation_test(obj: None, testcase: DataDrivenTestCase) -> None:
     os.remove(program_path)
     # Compare actual output to expected.
     assert testcase.output == out, 'Invalid output ({}, line {})'.format(
-                                        testcase.file, testcase.line)
+        testcase.file, testcase.line)
 
 
 def parse_args(line: str) -> List[str]:
@@ -78,4 +78,4 @@ def parse_args(line: str) -> List[str]:
         return []  # No args; mypy will spit out an error.
     return m.group(1).split()
 
-TestPythonEvaluation.setup_tests() # MUST come after def of test_python_evaluation.
+TestPythonEvaluation.setup_tests()
diff --git a/runtests.py b/runtests.py
index 3f26fcfa8d75..16ae6a8ddf28 100755
--- a/runtests.py
+++ b/runtests.py
@@ -205,7 +205,7 @@ def add_myunit(driver: Driver) -> None:
             # Run Python evaluation integration tests separately since they are #
             # much slower than proper unit tests.
             pass
-        elif not 'test_' in mod:
+        elif 'test_' not in mod:
             # Modules containing `test_` have been ported to pytest.
             driver.add_python_mod('unit-test %s' % mod, 'mypy.myunit', '-m', mod, *driver.arglist)
 

From d42933b3ea40e9dfaf1f744f58afe9c59e5ece6a Mon Sep 17 00:00:00 2001
From: Ryan Gonzalez <rymg19@gmail.com>
Date: Sat, 18 Jun 2016 16:56:20 -0500
Subject: [PATCH 14/20] Fix typing

---
 mypy/test/data.py       | 2 +-
 mypy/test/helpers.py    | 2 +-
 mypy/test/test_check.py | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/mypy/test/data.py b/mypy/test/data.py
index be41b8ed55f9..203c8b6d4c16 100644
--- a/mypy/test/data.py
+++ b/mypy/test/data.py
@@ -96,7 +96,7 @@ class DataDrivenTestCase(TestCase):
     file = ''
     line = 0
 
-    perform = None  # type: Callable[[Any, 'DataDrivenTestCase'], None]
+    perform = None  # type: Callable[..., None]
 
     # (file path, file content) tuples
     files = None  # type: List[Tuple[str, str]]
diff --git a/mypy/test/helpers.py b/mypy/test/helpers.py
index 49d0b9b1faae..1dee9be0f997 100644
--- a/mypy/test/helpers.py
+++ b/mypy/test/helpers.py
@@ -1,7 +1,7 @@
 import sys
 import re
 import os
-import pytest
+import pytest  # type: ignore
 
 from typing import List, Dict, Tuple
 
diff --git a/mypy/test/test_check.py b/mypy/test/test_check.py
index 013355f57ff8..5aeb35e17d13 100644
--- a/mypy/test/test_check.py
+++ b/mypy/test/test_check.py
@@ -5,7 +5,7 @@
 import shutil
 import sys
 import mypy
-import pytest
+import pytest  # type: ignore
 
 from typing import Tuple, List, Dict, Set
 

From e4beb6d8e1caa20abe9c4742a6e4ff467e931d45 Mon Sep 17 00:00:00 2001
From: Ryan Gonzalez <rymg19@gmail.com>
Date: Sat, 18 Jun 2016 17:13:03 -0500
Subject: [PATCH 15/20] Update requirements

---
 requirements-testing.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/requirements-testing.txt b/requirements-testing.txt
index 47744fe21aac..bb240784ac4d 100644
--- a/requirements-testing.txt
+++ b/requirements-testing.txt
@@ -1,2 +1,3 @@
 flake8
 typed-ast
+pytest

From e7045455f22c741e4d1fc226f4a485eecadb040e Mon Sep 17 00:00:00 2001
From: Ryan Gonzalez <rymg19@gmail.com>
Date: Sun, 19 Jun 2016 16:35:33 -0500
Subject: [PATCH 16/20] Update docs

---
 README.md | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/README.md b/README.md
index d056fada1e35..af5d829bb137 100644
--- a/README.md
+++ b/README.md
@@ -193,6 +193,13 @@ pass inferior arguments via `-a`:
     $ PYTHONPATH=$PWD scripts/myunit -m mypy.test.testlex -v '*backslash*'
     $ ./runtests.py mypy.test.testlex -a -v -a '*backslash*'
 
+Mypy is currently in the process of converting its tests from myunit to pytest.
+Some of the tests, such as `test_check`, have already been converted. To run
+these, you must use pytest instead, and use `-k` instead of `-a`:
+
+    $ ./runtests.py pytest -t -k -t NestedListAssignment
+    $ py.test -k NestedListAssignment
+
 You can also run the type checker for manual testing without
 installing anything by setting up the Python module search path
 suitably (the lib-typing/3.2 path entry is not needed for Python 3.5

From 624b5f2b06d3efdc48f3e2218d9965738f34aad8 Mon Sep 17 00:00:00 2001
From: Ryan Gonzalez <rymg19@gmail.com>
Date: Thu, 30 Jun 2016 13:34:03 -0500
Subject: [PATCH 17/20] Fix README

---
 README.md | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index af5d829bb137..b48e7e7f4f6d 100644
--- a/README.md
+++ b/README.md
@@ -195,7 +195,8 @@ pass inferior arguments via `-a`:
 
 Mypy is currently in the process of converting its tests from myunit to pytest.
 Some of the tests, such as `test_check`, have already been converted. To run
-these, you must use pytest instead, and use `-k` instead of `-a`:
+these individually, you can either use `./runtests.py` but pass options with
+`-t` instead of `-a`, or you can just run `py.test` itself:
 
     $ ./runtests.py pytest -t -k -t NestedListAssignment
     $ py.test -k NestedListAssignment

From 464057de7cf3cf0c14259d7107b4a35b869843db Mon Sep 17 00:00:00 2001
From: Ryan Gonzalez <rymg19@gmail.com>
Date: Thu, 30 Jun 2016 13:37:03 -0500
Subject: [PATCH 18/20] Rename some functions and clean up

---
 conftest.py                | 17 -----------------
 mypy/test/helpers.py       |  7 +++----
 mypy/test/test_check.py    |  4 ++--
 mypy/test/test_semanal.py  |  4 ++--
 mypy/test/testtransform.py |  4 ++--
 5 files changed, 9 insertions(+), 27 deletions(-)

diff --git a/conftest.py b/conftest.py
index aa12df17bd97..7e991da021b9 100644
--- a/conftest.py
+++ b/conftest.py
@@ -3,20 +3,3 @@
 def pytest_addoption(parser):
     parser.addoption('--update-testcases', action='store_true',
                      dest='UPDATE_TESTCASES')
-
-# mypy.test.helpers defines several top-level utility functions that
-# pytest will pick up as tests. This removes them.
-def pytest_collection_modifyitems(items):
-    to_remove = []
-
-    for i, item in enumerate(items):
-        if (isinstance(item, pytest.Function) and
-                item.function.__module__ == 'mypy.test.helpers' and
-                # This is to prevent removing data-driven tests, which are
-                # defined by mypy.test.helpers.PytestSuite.
-                not getattr(item.function, 'is_test_attr', False)):
-            to_remove.append(i)
-
-    # reversed is to prevent changing indexes that haven't been removed yet.
-    for index in reversed(to_remove):
-        items.pop(index)
diff --git a/mypy/test/helpers.py b/mypy/test/helpers.py
index 1dee9be0f997..c6e403423c84 100644
--- a/mypy/test/helpers.py
+++ b/mypy/test/helpers.py
@@ -264,18 +264,18 @@ def num_skipped_suffix_lines(a1: List[str], a2: List[str]) -> int:
     return max(0, num_eq - 4)
 
 
-def testfile_pyversion(path: str) -> Tuple[int, int]:
+def pyversion_testfile(path: str) -> Tuple[int, int]:
     if path.endswith('python2.test'):
         return defaults.PYTHON2_VERSION
     else:
         return defaults.PYTHON3_VERSION
 
 
-def testcase_pyversion(path: str, testcase_name: str) -> Tuple[int, int]:
+def pyversion_testcase(path: str, testcase_name: str) -> Tuple[int, int]:
     if testcase_name.endswith('python2'):
         return defaults.PYTHON2_VERSION
     else:
-        return testfile_pyversion(path)
+        return pyversion_testfile(path)
 
 
 def normalize_error_messages(messages: List[str]) -> List[str]:
@@ -312,7 +312,6 @@ def setup_tests(cls):
         for test in c:
             def func(self, test):
                 test.run(self)
-            func.is_test_attr = True
             if test.is_skip:
                 func = pytest.mark.skip(reason='Test ends with -skip')(func)
             if 'FastParse' in test.name and not test.is_skip:
diff --git a/mypy/test/test_check.py b/mypy/test/test_check.py
index 5aeb35e17d13..732a964a846f 100644
--- a/mypy/test/test_check.py
+++ b/mypy/test/test_check.py
@@ -14,7 +14,7 @@
 from mypy.test.config import test_temp_dir, test_data_prefix
 from mypy.test.data import parse_test_cases, DataDrivenTestCase
 from mypy.test.helpers import (
-    normalize_error_messages, testcase_pyversion, update_testcase_output,
+    normalize_error_messages, pyversion_testcase, update_testcase_output,
     PytestSuite, test
 )
 from mypy.errors import CompileError
@@ -88,7 +88,7 @@ def clear_cache(self) -> None:
 
     def run_test_once(self, testcase: DataDrivenTestCase, incremental=0) -> None:
         find_module_clear_caches()
-        pyversion = testcase_pyversion(testcase.file, testcase.name)
+        pyversion = pyversion_testcase(testcase.file, testcase.name)
         program_text = '\n'.join(testcase.input)
         module_name, program_name, program_text = self.parse_options(program_text)
         flags = self.parse_flags(program_text)
diff --git a/mypy/test/test_semanal.py b/mypy/test/test_semanal.py
index e3de09cfc382..ba081da09b20 100644
--- a/mypy/test/test_semanal.py
+++ b/mypy/test/test_semanal.py
@@ -7,7 +7,7 @@
 from mypy import build
 from mypy.build import BuildSource
 from mypy.test.helpers import (
-    normalize_error_messages, testfile_pyversion, PytestSuite, test
+    normalize_error_messages, pyversion_testfile, PytestSuite, test
 )
 from mypy.test.data import parse_test_cases
 from mypy.test.config import test_data_prefix, test_temp_dir
@@ -54,7 +54,7 @@ def semanal_tests(obj, testcase):
         src = '\n'.join(testcase.input)
         result = build.build(target=build.SEMANTIC_ANALYSIS,
                              sources=[BuildSource('main', None, src)],
-                             pyversion=testfile_pyversion(testcase.file),
+                             pyversion=pyversion_testfile(testcase.file),
                              flags=[build.TEST_BUILTINS],
                              alt_lib_path=test_temp_dir)
         a = result.errors
diff --git a/mypy/test/testtransform.py b/mypy/test/testtransform.py
index 4b17d2ef8e59..b09929953a72 100644
--- a/mypy/test/testtransform.py
+++ b/mypy/test/testtransform.py
@@ -7,7 +7,7 @@
 from mypy import build
 from mypy.build import BuildSource
 from mypy.myunit import Suite
-from mypy.test.helpers import assert_string_arrays_equal, testfile_pyversion
+from mypy.test.helpers import assert_string_arrays_equal, pyversion_testfile
 from mypy.test.data import parse_test_cases
 from mypy.test.config import test_data_prefix, test_temp_dir
 from mypy.errors import CompileError
@@ -43,7 +43,7 @@ def test_transform(testcase):
         src = '\n'.join(testcase.input)
         result = build.build(target=build.SEMANTIC_ANALYSIS,
                              sources=[BuildSource('main', None, src)],
-                             pyversion=testfile_pyversion(testcase.file),
+                             pyversion=pyversion_testfile(testcase.file),
                              flags=[build.TEST_BUILTINS],
                              alt_lib_path=test_temp_dir)
         a = result.errors

From 1506da508cc23fb828396827c863670a4d2a5cfb Mon Sep 17 00:00:00 2001
From: Ryan Gonzalez <rymg19@gmail.com>
Date: Thu, 30 Jun 2016 13:43:07 -0500
Subject: [PATCH 19/20] Rename is_skip -> marked_skip and add some informative
 comments

---
 mypy/test/data.py    | 16 +++++++++++-----
 mypy/test/helpers.py |  2 +-
 2 files changed, 12 insertions(+), 6 deletions(-)

diff --git a/mypy/test/data.py b/mypy/test/data.py
index 203c8b6d4c16..ae9fd265518c 100644
--- a/mypy/test/data.py
+++ b/mypy/test/data.py
@@ -13,7 +13,7 @@
 
 def parse_test_cases(
         path: str,
-        perform: Callable[[Any, 'DataDrivenTestCase'], None],
+        perform: Callable[..., None],
         base_path: str = '.',
         optional_out: bool = False,
         include_path: str = None,
@@ -96,6 +96,7 @@ class DataDrivenTestCase(TestCase):
     file = ''
     line = 0
 
+    # NOTE: For info on the ..., see `run`.
     perform = None  # type: Callable[..., None]
 
     # (file path, file content) tuples
@@ -103,7 +104,7 @@ class DataDrivenTestCase(TestCase):
 
     clean_up = None  # type: List[Tuple[bool, str]]
 
-    is_skip = False
+    marked_skip = False
 
     def __init__(self, name, input, output, file, line, lastline,
                  perform, files):
@@ -116,7 +117,7 @@ def __init__(self, name, input, output, file, line, lastline,
         self.line = line
         self.perform = perform
         self.files = files
-        self.is_skip = self.name.endswith('-skip')
+        self.marked_skip = self.name.endswith('-skip')
 
     def set_up(self) -> None:
         super().set_up()
@@ -148,12 +149,17 @@ def run(self, obj: Any = None):
             # modifications requires to make BOTH run at the moment, this branch
             # is necessary. It should be removed once all the tests relying on
             # DataDrivenTestCase are converted to pytest.
-            if self.is_skip:
+            if self.marked_skip:
                 raise SkipTestCaseException()
             else:
                 self.perform(self)
         else:
-            assert not self.is_skip
+            assert not self.marked_skip
+            # Because perform is an unbound method, it needs to be passed it
+            # own self, which is obj. In the future, after all tests are moved
+            # over to pytest, this whole class should probably be generic, to
+            # allow annotating obj. In the mean time, there isn't a cleaner way
+            # to handle this...
             self.perform(obj, self)
 
     def tear_down(self) -> None:
diff --git a/mypy/test/helpers.py b/mypy/test/helpers.py
index c6e403423c84..e59d5802e384 100644
--- a/mypy/test/helpers.py
+++ b/mypy/test/helpers.py
@@ -312,7 +312,7 @@ def setup_tests(cls):
         for test in c:
             def func(self, test):
                 test.run(self)
-            if test.is_skip:
+            if test.marked_skip:
                 func = pytest.mark.skip(reason='Test ends with -skip')(func)
             if 'FastParse' in test.name and not test.is_skip:
                 try:

From 5b7aba1754c3244c2fa51f0993159c3bcbc2a090 Mon Sep 17 00:00:00 2001
From: Ryan Gonzalez <rymg19@gmail.com>
Date: Thu, 30 Jun 2016 13:54:30 -0500
Subject: [PATCH 20/20] Avoid explicit collect_tests/setup_tests calls

---
 conftest.py               | 8 ++++++++
 mypy/test/helpers.py      | 8 ++++----
 mypy/test/test_check.py   | 2 --
 mypy/test/test_cmdline.py | 2 --
 mypy/test/test_semanal.py | 8 --------
 5 files changed, 12 insertions(+), 16 deletions(-)

diff --git a/conftest.py b/conftest.py
index 7e991da021b9..a6f1824ef3d7 100644
--- a/conftest.py
+++ b/conftest.py
@@ -1,5 +1,13 @@
+from mypy.test.helpers import PytestSuite
+import inspect
 import pytest
 
 def pytest_addoption(parser):
     parser.addoption('--update-testcases', action='store_true',
                      dest='UPDATE_TESTCASES')
+
+def pytest_pycollect_makeitem(collector, name, obj):
+     if (inspect.isclass(obj) and issubclass(obj, PytestSuite) and
+            obj is not PytestSuite):
+        print(name)
+        obj.collect_tests()
diff --git a/mypy/test/helpers.py b/mypy/test/helpers.py
index e59d5802e384..559fe98510ac 100644
--- a/mypy/test/helpers.py
+++ b/mypy/test/helpers.py
@@ -298,15 +298,15 @@ def test(request):
 class PytestSuite:
     """Assists in setting up data-driven test cases for pytest."""
     @classmethod
-    def setup_tests(cls):
+    def collect_tests(cls):
         """
         Sets up the child class's test case. The child must have a method
         `cases` that returns a list of `DataDrivenTestCase`s. This method will
         load the data-driven test cases and use setattr to assign it to the
         class, which will allow pytest to recognize the test.
 
-        Note that this method **must** be run after the definition of any
-        functions used by PytestSuite.cases.
+        This will be called during test collection (see conftest.py in the root
+        of the repository).
         """
         c = cls.cases()  # type: List[DataDrivenTestCase]
         for test in c:
@@ -314,7 +314,7 @@ def func(self, test):
                 test.run(self)
             if test.marked_skip:
                 func = pytest.mark.skip(reason='Test ends with -skip')(func)
-            if 'FastParse' in test.name and not test.is_skip:
+            if 'FastParse' in test.name and not test.marked_skip:
                 try:
                     import mypy.fastparse
                 except SystemExit:
diff --git a/mypy/test/test_check.py b/mypy/test/test_check.py
index 732a964a846f..09e625290ab9 100644
--- a/mypy/test/test_check.py
+++ b/mypy/test/test_check.py
@@ -206,5 +206,3 @@ def parse_flags(self, program_text: str) -> List[str]:
             return m.group(1).split()
         else:
             return []
-
-TestTypeCheck.setup_tests()
diff --git a/mypy/test/test_cmdline.py b/mypy/test/test_cmdline.py
index c7fc4960fde4..549561298ff3 100644
--- a/mypy/test/test_cmdline.py
+++ b/mypy/test/test_cmdline.py
@@ -77,5 +77,3 @@ def parse_args(line: str) -> List[str]:
     if not m:
         return []  # No args; mypy will spit out an error.
     return m.group(1).split()
-
-TestPythonEvaluation.setup_tests()
diff --git a/mypy/test/test_semanal.py b/mypy/test/test_semanal.py
index ba081da09b20..a1b6191a2a35 100644
--- a/mypy/test/test_semanal.py
+++ b/mypy/test/test_semanal.py
@@ -81,8 +81,6 @@ def semanal_tests(obj, testcase):
         'Invalid semantic analyzer output ({}, line {})'.format(testcase.file,
                                                                 testcase.line)
 
-TestSemAnal.setup_tests()
-
 # Semantic analyzer error test cases
 
 # Paths to files containing test case descriptions.
@@ -120,8 +118,6 @@ def semanal_error_tests(obj, testcase):
         'Invalid compiler output ({}, line {})'.format(testcase.file,
                                                        testcase.line)
 
-TestSemAnalError.setup_tests()
-
 
 # SymbolNode table export test cases
 
@@ -162,8 +158,6 @@ def run_test(self, testcase):
             'Invalid semantic analyzer output ({}, line {})'.format(
                 testcase.file, testcase.line)
 
-TestSemAnalSymtable.setup_tests()
-
 
 # Type info export test cases
 
@@ -208,8 +202,6 @@ def run_test(self, testcase):
             'Invalid semantic analyzer output ({}, line {})'.format(
                 testcase.file, testcase.line)
 
-TestSemAnalTypeInfo.setup_tests()
-
 
 class TypeInfoMap(Dict[str, TypeInfo]):
     def __str__(self) -> str: