Skip to content

Commit 19dca04

Browse files
serhiy-storchakaserhiy-storchaka
authored
gh-91760: Deprecate group names and numbers which will be invalid in future (GH-91794)
Only sequence of ASCII digits will be accepted as a numerical reference. The group name in bytes patterns and replacement strings could only contain ASCII letters and digits and underscore.
1 parent 6d0d547 commit 19dca04

File tree

5 files changed

+112
-7
lines changed

5 files changed

+112
-7
lines changed

‎Doc/library/re.rst

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -417,6 +417,9 @@ The special characters are:
417417
| | * ``\1`` |
418418
+---------------------------------------+----------------------------------+
419419

420+
.. deprecated:: 3.11
421+
Group names containing non-ASCII characters in bytes patterns.
422+
420423
.. index:: single: (?P=; in regular expressions
421424

422425
``(?P=name)``
@@ -486,6 +489,9 @@ The special characters are:
486489
will match with ``'<[email protected]>'`` as well as ``'[email protected]'``, but
487490
not with ``'<[email protected]'`` nor ``'[email protected]>'``.
488491

492+
.. deprecated:: 3.11
493+
Group *id* containing anything except ASCII digits.
494+
489495

490496
The special sequences consist of ``'\'`` and a character from the list below.
491497
If the ordinary character is not an ASCII digit or an ASCII letter, then the
@@ -995,6 +1001,10 @@ form.
9951001
Empty matches for the pattern are replaced when adjacent to a previous
9961002
non-empty match.
9971003

1004+
.. deprecated:: 3.11
1005+
Group *id* containing anything except ASCII digits.
1006+
Group names containing non-ASCII characters in bytes replacement strings.
1007+
9981008

9991009
.. function:: subn(pattern, repl, string, count=0, flags=0)
10001010

‎Doc/whatsnew/3.11.rst

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1151,6 +1151,14 @@ Deprecated
11511151
(Contributed by Brett Cannon in :issue:`47061` and Victor Stinner in
11521152
:gh:`68966`.)
11531153

1154+
* More strict rules will be applied now applied for numerical group references
1155+
and group names in regular expressions in future Python versions.
1156+
Only sequence of ASCII digits will be now accepted as a numerical reference.
1157+
The group name in bytes patterns and replacement strings could only
1158+
contain ASCII letters and digits and underscore.
1159+
For now, a deprecation warning is raised for such syntax.
1160+
(Contributed by Serhiy Storchaka in :gh:`91760`.)
1161+
11541162

11551163
Removed
11561164
=======

‎Lib/re/_parser.py

Lines changed: 34 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -287,8 +287,22 @@ def seek(self, index):
287287
self.__next()
288288

289289
def error(self, msg, offset=0):
290+
if not self.istext:
291+
msg = msg.encode('ascii', 'backslashreplace').decode('ascii')
290292
return error(msg, self.string, self.tell() - offset)
291293

294+
def checkgroupname(self, name, offset, nested):
295+
if not name.isidentifier():
296+
msg = "bad character in group name %r" % name
297+
raise self.error(msg, len(name) + offset)
298+
if not (self.istext or name.isascii()):
299+
import warnings
300+
warnings.warn(
301+
"bad character in group name %a at position %d" %
302+
(name, self.tell() - len(name) - offset),
303+
DeprecationWarning, stacklevel=nested + 7
304+
)
305+
292306
def _class_escape(source, escape):
293307
# handle escape code inside character class
294308
code = ESCAPES.get(escape)
@@ -703,15 +717,11 @@ def _parse(source, state, verbose, nested, first=False):
703717
if sourcematch("<"):
704718
# named group: skip forward to end of name
705719
name = source.getuntil(">", "group name")
706-
if not name.isidentifier():
707-
msg = "bad character in group name %r" % name
708-
raise source.error(msg, len(name) + 1)
720+
source.checkgroupname(name, 1, nested)
709721
elif sourcematch("="):
710722
# named backreference
711723
name = source.getuntil(")", "group name")
712-
if not name.isidentifier():
713-
msg = "bad character in group name %r" % name
714-
raise source.error(msg, len(name) + 1)
724+
source.checkgroupname(name, 1, nested)
715725
gid = state.groupdict.get(name)
716726
if gid is None:
717727
msg = "unknown group name %r" % name
@@ -773,6 +783,7 @@ def _parse(source, state, verbose, nested, first=False):
773783
# conditional backreference group
774784
condname = source.getuntil(")", "group name")
775785
if condname.isidentifier():
786+
source.checkgroupname(condname, 1, nested)
776787
condgroup = state.groupdict.get(condname)
777788
if condgroup is None:
778789
msg = "unknown group name %r" % condname
@@ -795,6 +806,14 @@ def _parse(source, state, verbose, nested, first=False):
795806
state.grouprefpos[condgroup] = (
796807
source.tell() - len(condname) - 1
797808
)
809+
if not (condname.isdecimal() and condname.isascii()):
810+
import warnings
811+
warnings.warn(
812+
"bad character in group name %s at position %d" %
813+
(repr(condname) if source.istext else ascii(condname),
814+
source.tell() - len(condname) - 1),
815+
DeprecationWarning, stacklevel=nested + 6
816+
)
798817
state.checklookbehindgroup(condgroup, source)
799818
item_yes = _parse(source, state, verbose, nested + 1)
800819
if source.match("|"):
@@ -1000,11 +1019,11 @@ def addgroup(index, pos):
10001019
# group
10011020
c = this[1]
10021021
if c == "g":
1003-
name = ""
10041022
if not s.match("<"):
10051023
raise s.error("missing <")
10061024
name = s.getuntil(">", "group name")
10071025
if name.isidentifier():
1026+
s.checkgroupname(name, 1, -1)
10081027
try:
10091028
index = groupindex[name]
10101029
except KeyError:
@@ -1020,6 +1039,14 @@ def addgroup(index, pos):
10201039
if index >= MAXGROUPS:
10211040
raise s.error("invalid group reference %d" % index,
10221041
len(name) + 1)
1042+
if not (name.isdecimal() and name.isascii()):
1043+
import warnings
1044+
warnings.warn(
1045+
"bad character in group name %s at position %d" %
1046+
(repr(name) if s.istext else ascii(name),
1047+
s.tell() - len(name) - 1),
1048+
DeprecationWarning, stacklevel=5
1049+
)
10231050
addgroup(index, len(name) + 1)
10241051
elif c == "0":
10251052
if s.next in OCTDIGITS:

‎Lib/test/test_re.py

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -135,6 +135,7 @@ def test_basic_re_sub(self):
135135
self.assertEqual(re.sub('(?P<a>x)', r'\g<a>\g<1>', 'xx'), 'xxxx')
136136
self.assertEqual(re.sub('(?P<unk>x)', r'\g<unk>\g<unk>', 'xx'), 'xxxx')
137137
self.assertEqual(re.sub('(?P<unk>x)', r'\g<1>\g<1>', 'xx'), 'xxxx')
138+
self.assertEqual(re.sub('()x', r'\g<0>\g<0>', 'xx'), 'xxxx')
138139

139140
self.assertEqual(re.sub('a', r'\t\n\v\r\f\a\b', 'a'), '\t\n\v\r\f\a\b')
140141
self.assertEqual(re.sub('a', '\t\n\v\r\f\a\b', 'a'), '\t\n\v\r\f\a\b')
@@ -274,6 +275,21 @@ def test_symbolic_groups_errors(self):
274275
self.checkPatternError('(?P<©>x)', "bad character in group name '©'", 4)
275276
self.checkPatternError('(?P=©)', "bad character in group name '©'", 4)
276277
self.checkPatternError('(?(©)y)', "bad character in group name '©'", 3)
278+
with self.assertWarnsRegex(DeprecationWarning,
279+
r"bad character in group name '\\xc2\\xb5' "
280+
r"at position 4") as w:
281+
re.compile(b'(?P<\xc2\xb5>x)')
282+
self.assertEqual(w.filename, __file__)
283+
with self.assertWarnsRegex(DeprecationWarning,
284+
r"bad character in group name '\\xc2\\xb5' "
285+
r"at position 4"):
286+
self.checkPatternError(b'(?P=\xc2\xb5)',
287+
r"unknown group name '\xc2\xb5'", 4)
288+
with self.assertWarnsRegex(DeprecationWarning,
289+
r"bad character in group name '\\xc2\\xb5' "
290+
r"at position 3"):
291+
self.checkPatternError(b'(?(\xc2\xb5)y)',
292+
r"unknown group name '\xc2\xb5'", 3)
277293

278294
def test_symbolic_refs(self):
279295
self.assertEqual(re.sub('(?P<a>x)|(?P<b>y)', r'\g<b>', 'xx'), '')
@@ -306,12 +322,35 @@ def test_symbolic_refs_errors(self):
306322
re.sub('(?P<a>x)', r'\g<ab>', 'xx')
307323
self.checkTemplateError('(?P<a>x)', r'\g<-1>', 'xx',
308324
"bad character in group name '-1'", 3)
325+
with self.assertWarnsRegex(DeprecationWarning,
326+
r"bad character in group name '\+1' "
327+
r"at position 3") as w:
328+
re.sub('(?P<a>x)', r'\g<+1>', 'xx')
329+
self.assertEqual(w.filename, __file__)
330+
with self.assertWarnsRegex(DeprecationWarning,
331+
r"bad character in group name '1_0' "
332+
r"at position 3"):
333+
re.sub('()'*10, r'\g<1_0>', 'xx')
334+
with self.assertWarnsRegex(DeprecationWarning,
335+
r"bad character in group name ' 1 ' "
336+
r"at position 3"):
337+
re.sub('(?P<a>x)', r'\g< 1 >', 'xx')
309338
self.checkTemplateError('(?P<a>x)', r'\g<©>', 'xx',
310339
"bad character in group name '©'", 3)
340+
with self.assertWarnsRegex(DeprecationWarning,
341+
r"bad character in group name '\\xc2\\xb5' "
342+
r"at position 3") as w:
343+
with self.assertRaisesRegex(IndexError, "unknown group name '\xc2\xb5'"):
344+
re.sub(b'(?P<a>x)', b'\\g<\xc2\xb5>', b'xx')
345+
self.assertEqual(w.filename, __file__)
311346
self.checkTemplateError('(?P<a>x)', r'\g<㊀>', 'xx',
312347
"bad character in group name '㊀'", 3)
313348
self.checkTemplateError('(?P<a>x)', r'\g<¹>', 'xx',
314349
"bad character in group name '¹'", 3)
350+
with self.assertWarnsRegex(DeprecationWarning,
351+
r"bad character in group name '१' "
352+
r"at position 3"):
353+
re.sub('(?P<a>x)', r'\g<१>', 'xx')
315354

316355
def test_re_subn(self):
317356
self.assertEqual(re.subn("(?i)b+", "x", "bbbb BBBB"), ('x x', 2))
@@ -577,10 +616,27 @@ def test_re_groupref_exists_errors(self):
577616
self.checkPatternError(r'(?P<a>)(?(0)a|b)', 'bad group number', 10)
578617
self.checkPatternError(r'()(?(-1)a|b)',
579618
"bad character in group name '-1'", 5)
619+
with self.assertWarnsRegex(DeprecationWarning,
620+
r"bad character in group name '\+1' "
621+
r"at position 5") as w:
622+
re.compile(r'()(?(+1)a|b)')
623+
self.assertEqual(w.filename, __file__)
624+
with self.assertWarnsRegex(DeprecationWarning,
625+
r"bad character in group name '1_0' "
626+
r"at position 23"):
627+
re.compile(r'()'*10 + r'(?(1_0)a|b)')
628+
with self.assertWarnsRegex(DeprecationWarning,
629+
r"bad character in group name ' 1 ' "
630+
r"at position 5"):
631+
re.compile(r'()(?( 1 )a|b)')
580632
self.checkPatternError(r'()(?(㊀)a|b)',
581633
"bad character in group name '㊀'", 5)
582634
self.checkPatternError(r'()(?(¹)a|b)',
583635
"bad character in group name '¹'", 5)
636+
with self.assertWarnsRegex(DeprecationWarning,
637+
r"bad character in group name '१' "
638+
r"at position 5"):
639+
re.compile(r'()(?(१)a|b)')
584640
self.checkPatternError(r'()(?(1',
585641
"missing ), unterminated name", 5)
586642
self.checkPatternError(r'()(?(1)a',

‎Misc/NEWS.d/next/Library/2022-04-21-19-46-03.gh-issue-91760.zDtv1E.rst

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
More strict rules will be applied for numerical group references and group
2+
names in regular expressions. For now, a deprecation warning is emitted for
3+
group references and group names which will be errors in future Python
4+
versions.

0 commit comments

Comments
 (0)