Added simple form of concept composition

parsers/PythonParser.py

@@ -5,6 +5,8 @@ from dataclasses import dataclass, field
 import ast
 import logging
 
+from parsers.ConceptLexerParser import ConceptNode
+
 log = logging.getLogger(__name__)
 
 
@@ -22,7 +24,7 @@ class PythonNode(Node):
     def __init__(self, source, ast_, concepts=None):
         self.source = source
         self.ast_ = ast_
-        self.concepts = concepts or {}
+        self.concepts = concepts or {}  # when concepts are recognized in the expression
 
     # def __repr__(self):
     #     return "PythonNode(source='" + self.source + "', ast=" + self.get_dump(self.ast_) + ")"
@@ -133,3 +135,80 @@ class PythonGetNamesVisitor(ast.NodeVisitor):
 
     def visit_Name(self, node):
         self.names.add(node.id)
+
+class LexerNodeParserHelperForPython:
+    """Helper class to parse mix of concepts and Python"""
+
+    def __init__(self):
+        self.identifiers = {}  # cache for already created identifier (the key is id(concept))
+        self.identifiers_key = {}  # number of identifiers with the same root (prefix)
+
+    def _get_identifier(self, concept):
+        """
+        Get an identifier for a concept.
+        Make sure to return the same identifier if the same concept
+        Make sure to return a different identifier if same name but different concept
+
+        Internal function because I don't want identifiers, identifiers_key and python_ids_mappings
+        to be instance variables
+        I would like to keep this parser as stateless as possible
+        :param concept:
+        :return:
+        """
+        if id(concept) in self.identifiers:
+            return self.identifiers[id(concept)]
+
+        identifier = "__C__" + self._sanitize(concept.key or concept.name)
+        if concept.id:
+            identifier += "__" + concept.id
+
+        if identifier in self.identifiers_key:
+            self.identifiers_key[identifier] += 1
+            identifier += f"_{self.identifiers_key[identifier]}"
+        else:
+            self.identifiers_key[identifier] = 0
+
+        identifier += "__C__"
+
+        self.identifiers[id(concept)] = identifier
+        return identifier
+
+    @staticmethod
+    def _sanitize(identifier):
+        res = ""
+        for c in identifier:
+            res += c if c.isalnum() else "0"
+        return res
+
+    def parse(self, context, nodes):
+        source = ""
+        to_parse = ""
+
+        concepts = {}  # the key is the Python identifier
+
+        for node in nodes:
+            if isinstance(node, ConceptNode):
+                source += node.source
+                if to_parse:
+                    to_parse += " "
+                concept = node.concept
+                python_id = self._get_identifier(concept)
+                to_parse += python_id
+                concepts[python_id] = concept
+            else:
+                source += node.source
+                to_parse += node.source
+
+        with context.push(self, desc="Trying Python for '" + to_parse + "'") as sub_context:
+            sub_context.add_inputs(to_parse=to_parse)
+            python_parser = PythonParser()
+            result = python_parser.parse(sub_context, to_parse)
+            sub_context.add_values(return_values=result)
+
+        if result.status:
+            python_node = result.body.body
+            python_node.source = source
+            python_node.concepts = concepts
+            return python_node
+
+        return result.body  # the error