diff --git a/Lib/lib2to3/pgen2/driver.py b/Lib/lib2to3/pgen2/driver.py index ce601bb04f9589..e8d10e47dba078 100644 --- a/Lib/lib2to3/pgen2/driver.py +++ b/Lib/lib2to3/pgen2/driver.py @@ -19,6 +19,7 @@ import codecs import os import logging +import pkgutil import StringIO import sys @@ -143,6 +144,26 @@ def _newer(a, b): return os.path.getmtime(a) >= os.path.getmtime(b) +def load_packaged_grammar(package, grammar_source): + """Normally, loads a pickled grammar by doing + pkgutil.get_data(package, pickled_grammar) + where *pickled_grammar* is computed from *grammar_source* by adding the + Python version and using a ``.pickle`` extension. + + However, if *grammar_source* is an extant file, load_grammar(grammar_source) + is called instead. This facilities using a packaged grammar file when needed + but preserves load_grammar's automatic regeneration behavior when possible. + + """ + if os.path.isfile(grammar_source): + return load_grammar(grammar_source) + pickled_name = _generate_pickle_name(os.path.basename(grammar_source)) + data = pkgutil.get_data(package, pickled_name) + g = grammar.Grammar() + g.loads(data) + return g + + def main(*args): """Main program, when run as a script: produce grammar pickle files. diff --git a/Lib/lib2to3/pgen2/grammar.py b/Lib/lib2to3/pgen2/grammar.py index 75255e9c013827..0b6d86b67997b4 100644 --- a/Lib/lib2to3/pgen2/grammar.py +++ b/Lib/lib2to3/pgen2/grammar.py @@ -109,6 +109,10 @@ def load(self, filename): f.close() self.__dict__.update(d) + def loads(self, pkl): + """Load the grammar tables from a pickle bytes object.""" + self.__dict__.update(pickle.loads(pkl)) + def copy(self): """ Copy the grammar. diff --git a/Lib/lib2to3/pygram.py b/Lib/lib2to3/pygram.py index 621ff24c95471b..7e67e4a8678841 100644 --- a/Lib/lib2to3/pygram.py +++ b/Lib/lib2to3/pygram.py @@ -29,12 +29,12 @@ def __init__(self, grammar): setattr(self, name, symbol) -python_grammar = driver.load_grammar(_GRAMMAR_FILE) +python_grammar = driver.load_packaged_grammar("lib2to3", _GRAMMAR_FILE) python_symbols = Symbols(python_grammar) python_grammar_no_print_statement = python_grammar.copy() del python_grammar_no_print_statement.keywords["print"] -pattern_grammar = driver.load_grammar(_PATTERN_GRAMMAR_FILE) +pattern_grammar = driver.load_packaged_grammar("lib2to3", _PATTERN_GRAMMAR_FILE) pattern_symbols = Symbols(pattern_grammar) diff --git a/Lib/lib2to3/tests/test_parser.py b/Lib/lib2to3/tests/test_parser.py index ebf84418fe6034..d2254f13e09e6a 100644 --- a/Lib/lib2to3/tests/test_parser.py +++ b/Lib/lib2to3/tests/test_parser.py @@ -11,11 +11,14 @@ from .support import driver, test_dir # Python imports +import operator import os +import pickle import shutil import subprocess import sys import tempfile +import types import unittest # Local imports @@ -97,6 +100,18 @@ def test_load_grammar_from_subprocess(self): finally: shutil.rmtree(tmpdir) + def test_load_packaged_grammar(self): + modname = __name__ + '.load_test' + class MyLoader: + def get_data(self, where): + return pickle.dumps({'elephant': 19}) + class MyModule(types.ModuleType): + __file__ = 'parsertestmodule' + __loader__ = MyLoader() + sys.modules[modname] = MyModule(modname) + self.addCleanup(operator.delitem, sys.modules, modname) + g = pgen2_driver.load_packaged_grammar(modname, 'Grammar.txt') + self.assertEqual(g.elephant, 19) class GrammarTest(support.TestCase): diff --git a/Misc/NEWS.d/next/Tools-Demos/2017-12-22-09-25-51.bpo-24960.TGdAgO.rst b/Misc/NEWS.d/next/Tools-Demos/2017-12-22-09-25-51.bpo-24960.TGdAgO.rst new file mode 100644 index 00000000000000..6f9d83c5772d73 --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2017-12-22-09-25-51.bpo-24960.TGdAgO.rst @@ -0,0 +1,3 @@ +2to3 and lib2to3 can now read pickled grammar files using pkgutil.get_data() +rather than probing the filesystem. This lets 2to3 and lib2to3 work when run +from a zipfile.