From 390233ec8cb8bb0e690623360d08ec1a01f7d05c Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Sun, 13 Jun 2021 23:12:45 +0100 Subject: [PATCH 1/2] bpo-44409: Fix error location in tokenizer errors that happen during initialization --- Lib/test/test_exceptions.py | 1 + .../Core and Builtins/2021-06-13-23-12-18.bpo-44409.eW4LS-.rst | 2 ++ Parser/pegen.c | 2 ++ 3 files changed, 5 insertions(+) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2021-06-13-23-12-18.bpo-44409.eW4LS-.rst diff --git a/Lib/test/test_exceptions.py b/Lib/test/test_exceptions.py index 9cb5466a674d13..d444a122af9be7 100644 --- a/Lib/test/test_exceptions.py +++ b/Lib/test/test_exceptions.py @@ -246,6 +246,7 @@ def baz(): check("pass\npass\npass\n(1+)\npass\npass\npass", 4, 4) check("(1+)", 1, 4) check("[interesting\nfoo()\n", 1, 1) + check(b"\xef\xbb\xbf#coding: utf8\nprint('\xe6\x88\x91')\n", 0, -1) # Errors thrown by symtable.c check('x = [(yield i) for i in range(3)]', 1, 5) diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-06-13-23-12-18.bpo-44409.eW4LS-.rst b/Misc/NEWS.d/next/Core and Builtins/2021-06-13-23-12-18.bpo-44409.eW4LS-.rst new file mode 100644 index 00000000000000..0f204ed812b27a --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2021-06-13-23-12-18.bpo-44409.eW4LS-.rst @@ -0,0 +1,2 @@ +Fix error location information for tokenizer errors raised on initialization +of the tokenizer. Patch by Pablo Galindo. diff --git a/Parser/pegen.c b/Parser/pegen.c index 19412446b118ab..431f11792ccf92 100644 --- a/Parser/pegen.c +++ b/Parser/pegen.c @@ -283,6 +283,8 @@ static void raise_tokenizer_init_error(PyObject *filename) { if (!(PyErr_ExceptionMatches(PyExc_LookupError) + || PyErr_ExceptionMatches(PyExc_SyntaxError) + || PyErr_ExceptionMatches(PyExc_UnicodeDecodeError) || PyErr_ExceptionMatches(PyExc_ValueError) || PyErr_ExceptionMatches(PyExc_UnicodeDecodeError))) { return; From d522aacf51db89a5c6d72149ff6fca2f81d43516 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Mon, 14 Jun 2021 16:59:42 +0100 Subject: [PATCH 2/2] Update Parser/pegen.c --- Parser/pegen.c | 1 - 1 file changed, 1 deletion(-) diff --git a/Parser/pegen.c b/Parser/pegen.c index 431f11792ccf92..615047c1b6a007 100644 --- a/Parser/pegen.c +++ b/Parser/pegen.c @@ -284,7 +284,6 @@ raise_tokenizer_init_error(PyObject *filename) { if (!(PyErr_ExceptionMatches(PyExc_LookupError) || PyErr_ExceptionMatches(PyExc_SyntaxError) - || PyErr_ExceptionMatches(PyExc_UnicodeDecodeError) || PyErr_ExceptionMatches(PyExc_ValueError) || PyErr_ExceptionMatches(PyExc_UnicodeDecodeError))) { return;