Improved PythonEvaluator when dealing with concept class

src/parsers/BaseParser.py

@@ -141,15 +141,22 @@ class BaseParser:
             body=self.error_sink if self.has_error else tree,
             try_parsed=try_parse)
 
-    def get_input_as_text(self, parser_input, custom_switcher=None):
+    def get_input_as_text(self, parser_input, custom_switcher=None, tracker=None):
+        """
+        Recreate back the source code from parser_input
+        :param parser_input: list of Tokens
+        :param custom_switcher: map of [TokenKind, overridden values]
+        :param tracker: keep track of the value overridden by custom_switcher
+        :return:
+        """
         if isinstance(parser_input, list):
-            return self.get_text_from_tokens(parser_input, custom_switcher)
+            return self.get_text_from_tokens(parser_input, custom_switcher, tracker)
 
         if isinstance(parser_input, ParserResultConcept):
             parser_input = parser_input.source
 
         if "c:" in parser_input:
-            return self.get_text_from_tokens(list(Tokenizer(parser_input)), custom_switcher)
+            return self.get_text_from_tokens(list(Tokenizer(parser_input)), custom_switcher, tracker)
 
         return parser_input
 
@@ -194,7 +201,14 @@ class BaseParser:
         return lst
 
     @staticmethod
-    def get_text_from_tokens(tokens, custom_switcher=None):
+    def get_text_from_tokens(tokens, custom_switcher=None, tracker=None):
+        """
+        Create the source code, from the list of token
+        :param tokens: list of tokens
+        :param custom_switcher: to override the behaviour (the return value) of some token
+        :param tracker: keep track of the original token value when custom switched
+        :return:
+        """
         if tokens is None:
             return ""
         res = ""
@@ -213,6 +227,8 @@ class BaseParser:
         for token in tokens:
             value = switcher.get(token.type, lambda t: t.value)(token)
             res += value
+            if tracker is not None and token.type in custom_switcher:
+                tracker[value] = token.value
         return res
 
     @staticmethod

src/parsers/PythonParser.py

@@ -72,11 +72,12 @@ class PythonParser(BaseParser):
         tree = None
 
         python_switcher = {
-            TokenKind.CONCEPT: lambda t: core.utils.encode_concept(t.value, True)
+            TokenKind.CONCEPT: lambda t: core.utils.encode_concept(t.value)
         }
 
         try:
-            source = self.get_input_as_text(parser_input, python_switcher)
+            tracker = {}
+            source = self.get_input_as_text(parser_input, python_switcher, tracker)
             source = source.strip()
             parser_input = parser_input if isinstance(parser_input, str) else source
 
@@ -108,7 +109,7 @@ class PythonParser(BaseParser):
                     BuiltinConcepts.PARSER_RESULT,
                     parser=self,
                     source=parser_input,
-                    body=PythonNode(parser_input, tree),
+                    body=PythonNode(parser_input, tree, tracker),
                     try_parsed=None))
 
         self.log_result(context, parser_input, ret)