commit 145fa86598b850e9ee0efed413dec9ec5e4431b1
author Pablo Galindo <Pablogsal@gmail.com> Mon Jun 15 04:31:05 2020 +0100
committer GitHub <noreply@github.com> Mon Jun 15 04:31:05 2020 +0100
tree f6102dae64cf5caa63a665946dce856f6f0627af
parent 8a3469047c3c7b68f434ed244ef3ae4292dd8cbc

[3.9] Include soft keywords in keyword.py (GH-20877). (GH-20880)

(cherry picked from commit 78319e373d57cd4da67660f888aa7092efbd6f24)

Co-authored-by: Pablo Galindo <Pablogsal@gmail.com>

Lib/keyword.py

diff --git a/Lib/keyword.py b/Lib/keyword.py
index a4db67e..59fcfb0 100644
--- a/Lib/keyword.py
+++ b/Lib/keyword.py
@@ -13,7 +13,7 @@
 Alternatively, you can run 'make regen-keyword'.
 """
 
-__all__ = ["iskeyword", "kwlist"]
+__all__ = ["iskeyword", "issoftkeyword", "kwlist", "softkwlist"]
 
 kwlist = [
     'False',
@@ -54,4 +54,9 @@
     'yield'
 ]
 
+softkwlist = [
+
+]
+
 iskeyword = frozenset(kwlist).__contains__
+issoftkeyword = frozenset(softkwlist).__contains__

Tools/peg_generator/pegen/c_generator.py

diff --git a/Tools/peg_generator/pegen/c_generator.py b/Tools/peg_generator/pegen/c_generator.py
index ce1d6bb..58a44fb 100644
--- a/Tools/peg_generator/pegen/c_generator.py
+++ b/Tools/peg_generator/pegen/c_generator.py
@@ -105,6 +105,7 @@
         self.non_exact_tokens = non_exact_tokens
         self.cache: Dict[Any, FunctionCall] = {}
         self.keyword_cache: Dict[str, int] = {}
+        self.soft_keywords: Set[str] = set()
 
     def keyword_helper(self, keyword: str) -> FunctionCall:
         if keyword not in self.keyword_cache:
@@ -119,6 +120,7 @@
         )
 
     def soft_keyword_helper(self, value: str) -> FunctionCall:
+        self.soft_keywords.add(value.replace('"', ""))
         return FunctionCall(
             assigned_variable="_keyword",
             function="_PyPegen_expect_soft_keyword",

Tools/peg_generator/pegen/keywordgen.py

diff --git a/Tools/peg_generator/pegen/keywordgen.py b/Tools/peg_generator/pegen/keywordgen.py
index 279c34b..fa57770 100644
--- a/Tools/peg_generator/pegen/keywordgen.py
+++ b/Tools/peg_generator/pegen/keywordgen.py
@@ -21,13 +21,18 @@
 Alternatively, you can run 'make regen-keyword'.
 """
 
-__all__ = ["iskeyword", "kwlist"]
+__all__ = ["iskeyword", "issoftkeyword", "kwlist", "softkwlist"]
 
 kwlist = [
-    {keywords}
+{keywords}
+]
+
+softkwlist = [
+{soft_keywords}
 ]
 
 iskeyword = frozenset(kwlist).__contains__
+issoftkeyword = frozenset(softkwlist).__contains__
 '''.lstrip()
 
 EXTRA_KEYWORDS = ["async", "await"]
@@ -61,12 +66,12 @@
     gen.collect_todo()
 
     with args.keyword_file as thefile:
-        all_keywords = sorted(
-            list(gen.callmakervisitor.keyword_cache.keys()) + EXTRA_KEYWORDS
-        )
+        all_keywords = sorted(list(gen.callmakervisitor.keyword_cache.keys()) + EXTRA_KEYWORDS)
+        all_soft_keywords = sorted(gen.callmakervisitor.soft_keywords)
 
-        keywords = ",\n    ".join(map(repr, all_keywords))
-        thefile.write(TEMPLATE.format(keywords=keywords))
+        keywords = "" if not all_keywords else "    " + ",\n    ".join(map(repr, all_keywords))
+        soft_keywords = "" if not all_soft_keywords else "    " + ",\n    ".join(map(repr, all_soft_keywords))
+        thefile.write(TEMPLATE.format(keywords=keywords, soft_keywords=soft_keywords))
 
 
 if __name__ == "__main__":