[Python-checkins] Include soft keywords in keyword.py (GH-20877)

Sun Jun 14 22:55:31 EDT 2020

https://github.com/python/cpython/commit/78319e373d57cd4da67660f888aa7092efbd6f24
commit: 78319e373d57cd4da67660f888aa7092efbd6f24
branch: master
author: Pablo Galindo <Pablogsal at gmail.com>
committer: GitHub <noreply at github.com>
date: 2020年06月15日T03:55:15+01:00
summary:
Include soft keywords in keyword.py (GH-20877)
files:
M Lib/keyword.py
M Tools/peg_generator/pegen/c_generator.py
M Tools/peg_generator/pegen/keywordgen.py

diff --git a/Lib/keyword.py b/Lib/keyword.py
index b6a9982570211..ccc951500f6d8 100644
--- a/Lib/keyword.py
+++ b/Lib/keyword.py
@@ -13,7 +13,7 @@
 Alternatively, you can run 'make regen-keyword'.
 """
 
-__all__ = ["iskeyword", "kwlist"]
+__all__ = ["iskeyword", "issoftkeyword", "kwlist", "softkwlist"]
 
 kwlist = [
 'False',
@@ -53,4 +53,9 @@
 'yield'
 ]
 
+softkwlist = [
+
+]
+
 iskeyword = frozenset(kwlist).__contains__
+issoftkeyword = frozenset(softkwlist).__contains__
diff --git a/Tools/peg_generator/pegen/c_generator.py b/Tools/peg_generator/pegen/c_generator.py
index ce1d6bb7bf355..58a44fbe67e8b 100644
--- a/Tools/peg_generator/pegen/c_generator.py
+++ b/Tools/peg_generator/pegen/c_generator.py
@@ -105,6 +105,7 @@ def __init__(
 self.non_exact_tokens = non_exact_tokens
 self.cache: Dict[Any, FunctionCall] = {}
 self.keyword_cache: Dict[str, int] = {}
+ self.soft_keywords: Set[str] = set()
 
 def keyword_helper(self, keyword: str) -> FunctionCall:
 if keyword not in self.keyword_cache:
@@ -119,6 +120,7 @@ def keyword_helper(self, keyword: str) -> FunctionCall:
 )
 
 def soft_keyword_helper(self, value: str) -> FunctionCall:
+ self.soft_keywords.add(value.replace('"', ""))
 return FunctionCall(
 assigned_variable="_keyword",
 function="_PyPegen_expect_soft_keyword",
diff --git a/Tools/peg_generator/pegen/keywordgen.py b/Tools/peg_generator/pegen/keywordgen.py
index 8684944096654..639f01bf2373e 100644
--- a/Tools/peg_generator/pegen/keywordgen.py
+++ b/Tools/peg_generator/pegen/keywordgen.py
@@ -21,13 +21,18 @@
 Alternatively, you can run 'make regen-keyword'.
 """
 
-__all__ = ["iskeyword", "kwlist"]
+__all__ = ["iskeyword", "issoftkeyword", "kwlist", "softkwlist"]
 
 kwlist = [
- {keywords}
+{keywords}
+]
+
+softkwlist = [
+{soft_keywords}
 ]
 
 iskeyword = frozenset(kwlist).__contains__
+issoftkeyword = frozenset(softkwlist).__contains__
 '''.lstrip()
 
 EXTRA_KEYWORDS = ["async", "await"]
@@ -60,9 +65,11 @@ def main():
 
 with args.keyword_file as thefile:
 all_keywords = sorted(list(gen.callmakervisitor.keyword_cache.keys()) + EXTRA_KEYWORDS)
+ all_soft_keywords = sorted(gen.callmakervisitor.soft_keywords)
 
- keywords = ",\n ".join(map(repr, all_keywords))
- thefile.write(TEMPLATE.format(keywords=keywords))
+ keywords = " " + ",\n ".join(map(repr, all_keywords))
+ soft_keywords = " " + ",\n ".join(map(repr, all_soft_keywords))
+ thefile.write(TEMPLATE.format(keywords=keywords, soft_keywords=soft_keywords))
 
 
 if __name__ == "__main__":
