[2.7] bpo-24960: use pkgutil.get_data in lib2to3 to read pickled grammar files (GH-4977) (#4980)
This is more complicated than it should be because we need to preserve the
useful mtime-based regeneration feature that lib2to3.pgen2.driver.load_grammar
has. We only look for the pickled grammar file with pkgutil.get_data and only if
the source file does not exist..
(cherry picked from commit 8a5877165e993afb2633cd48da5222326d3f6e0e)
diff --git a/Lib/lib2to3/pgen2/driver.py b/Lib/lib2to3/pgen2/driver.py
index ce601bb..e8d10e4 100644
--- a/Lib/lib2to3/pgen2/driver.py
+++ b/Lib/lib2to3/pgen2/driver.py
@@ -19,6 +19,7 @@
import codecs
import os
import logging
+import pkgutil
import StringIO
import sys
@@ -143,6 +144,26 @@
return os.path.getmtime(a) >= os.path.getmtime(b)
+def load_packaged_grammar(package, grammar_source):
+ """Normally, loads a pickled grammar by doing
+ pkgutil.get_data(package, pickled_grammar)
+ where *pickled_grammar* is computed from *grammar_source* by adding the
+ Python version and using a ``.pickle`` extension.
+
+ However, if *grammar_source* is an extant file, load_grammar(grammar_source)
+ is called instead. This facilities using a packaged grammar file when needed
+ but preserves load_grammar's automatic regeneration behavior when possible.
+
+ """
+ if os.path.isfile(grammar_source):
+ return load_grammar(grammar_source)
+ pickled_name = _generate_pickle_name(os.path.basename(grammar_source))
+ data = pkgutil.get_data(package, pickled_name)
+ g = grammar.Grammar()
+ g.loads(data)
+ return g
+
+
def main(*args):
"""Main program, when run as a script: produce grammar pickle files.