Handle error instead of crash on file with bad encoding (#1619)

Fixes #1521

* Handle LookupError generated by decoding a file with an invalid encoding
- Re-raise the LookupError as a slightly more helpful DecodeError
- Catch the DecodeError and raise as CompileError so the BuildManager will
handle it instead of crashing.

* Add unit test to check for crashing on files with invalid encoding
- In cmdline.test so the file parsing happens in build.py. The tests in
parse-errors.test call the parse() method in parse.py directly.

Author: margaret

mypy/build.py

@@ -28,7 +28,7 @@
                         SymbolTableNode, MODULE_REF)
 from mypy.semanal import FirstPass, SemanticAnalyzer, ThirdPass
 from mypy.checker import TypeChecker
-from mypy.errors import Errors, CompileError, report_internal_error
+from mypy.errors import Errors, CompileError, DecodeError, report_internal_error
 from mypy import fixup
 from mypy.report import Reports
 from mypy import defaults
@@ -683,6 +683,10 @@ def read_with_python_encoding(path: str, pyversion: Tuple[int, int]) -> str:
                 encoding = _encoding
 
         source_bytearray.extend(f.read())
+        try:
+            source_bytearray.decode(encoding)
+        except LookupError as lookuperr:
+            raise DecodeError(str(lookuperr))
         return source_bytearray.decode(encoding)
 
 
@@ -1215,7 +1219,7 @@ def parse_file(self) -> None:
                 except IOError as ioerr:
                     raise CompileError([
                         "mypy: can't read file '{}': {}".format(self.path, ioerr.strerror)])
-                except UnicodeDecodeError as decodeerr:
+                except (UnicodeDecodeError, DecodeError) as decodeerr:
                     raise CompileError([
                         "mypy: can't decode file '{}': {}".format(self.path, str(decodeerr))])
             self.tree = manager.parse_file(self.id, self.xpath, source)

mypy/errors.py

@@ -346,6 +346,13 @@ def __init__(self, messages: List[str], use_stdout: bool = False) -> None:
         self.use_stdout = use_stdout
 
 
+class DecodeError(Exception):
+    """Exception raised when a file cannot be decoded due to an unknown encoding type.
+
+    Essentially a wrapper for the LookupError raised by `bytearray.decode`
+    """
+
+
 def remove_path_prefix(path: str, prefix: str) -> str:
     """If path starts with prefix, return copy of path with the prefix removed.
     Otherwise, return path. If path is None, return None.

mypy/test/data/cmdline.test

@@ -74,3 +74,10 @@ undef
 [out]
 dir/subpkg/a.py:1: error: Name 'undef' is not defined
 dir/a.py:1: error: Name 'undef' is not defined
+
+[case testBadFileEncoding]
+# cmd: mypy a.py
+[file a.py]
+# coding: uft-8
+[out]
+mypy: can't decode file 'a.py': unknown encoding: uft-8