Added simple form of concept composition

parsers/ConceptLexerParser.py

@@ -6,7 +6,8 @@
 #       Arpeggio: A flexible PEG parser for Python,
 #       Knowledge-Based Systems, 2016, 95, 71 - 74, doi:10.1016/j.knosys.2015.12.004
 #####################################################################################################
-from dataclasses import field, dataclass
+from collections import namedtuple
+from dataclasses import dataclass
 from collections import defaultdict
 from core.builtin_concepts import BuiltinConcepts
 from core.concept import Concept, ConceptParts, DoNotResolve
@@ -15,23 +16,6 @@ from parsers.BaseParser import BaseParser, Node, ErrorNode
 import core.utils
 
 
-def flatten(iterable):
-    if iterable is None:
-        return []
-
-    result = []
-    for e in iterable:
-        if e.parsing_expression.rule_name is not None and e.parsing_expression.rule_name != "":
-            if hasattr(e, "children"):
-                e.children = flatten(e.children)
-            result.append(e)
-        elif hasattr(e, "children"):
-            result.extend(flatten(e.children))
-        else:
-            result.append(e)
-    return result
-
-
 @dataclass()
 class LexerNode(Node):
     start: int  # starting index in the tokens list
@@ -68,10 +52,10 @@ class UnrecognizedTokensNode(LexerNode):
         return not (len(self.tokens) == 1 and self.tokens[0].type in (TokenKind.WHITESPACE, TokenKind.NEWLINE))
 
     def __eq__(self, other):
-        if isinstance(other, tuple):
-            if len(other) != 3:
-                return False
-            return self.start == other[0] and self.end == other[1] and self.source == other[2]
+        if isinstance(other, utnode):
+            return self.start == other.start and \
+                   self.end == other.end and \
+                   self.source == other.source
 
         if not isinstance(other, UnrecognizedTokensNode):
             return False
@@ -80,6 +64,9 @@ class UnrecognizedTokensNode(LexerNode):
                self.end == other.end and \
                self.source == other.source
 
+    def __hash__(self):
+        return hash((self.start, self.end, self.source))
+
     def __repr__(self):
         return f"UnrecognizedTokensNode(start={self.start}, end={self.end}, source='{self.source}')"
 
@@ -99,17 +86,14 @@ class ConceptNode(LexerNode):
             self.source = BaseParser.get_text_from_tokens(self.tokens)
 
     def __eq__(self, other):
-        if isinstance(other, tuple):
-            if len(other) == 2:
-                return self.concept.key == other[0] and self.source == other[1]
-            else:
-                return self.concept.key == other[0] and \
-                       self.start == other[1] and \
-                       self.end == other[2] and \
-                       self.source == other[3]
+        if isinstance(other, cnode):
+            return self.concept.key == other.concept_key and \
+                   self.start == other.start and \
+                   self.end == other.end and \
+                   self.source == other.source
 
-        # if not super().__eq__(other):
-        #     return False
+        if isinstance(other, short_cnode):
+            return self.concept.key == other.concept_key and self.source == other.source
 
         if not isinstance(other, ConceptNode):
             return False
@@ -127,6 +111,42 @@ class ConceptNode(LexerNode):
         return f"ConceptNode(concept='{self.concept}', start={self.start}, end={self.end}, source='{self.source}')"
 
 
+class SourceCodeNode(LexerNode):
+    """
+    Returned when some source code (like Python source code is recognized)
+    """
+
+    def __init__(self, node, start, end, tokens=None, source=None):
+        super().__init__(start, end, tokens, source)
+        self.node = node  # The PythonNode (or whatever language node) that is found
+
+    def __eq__(self, other):
+        if isinstance(other, scnode):
+            return self.start == other.start and \
+                   self.end == other.end and \
+                   self.source == other.source
+
+        if not isinstance(other, SourceCodeNode):
+            return False
+
+        return self.node == other.node and \
+               self.start == other.start and \
+               self.end == other.end and \
+               self.source == other.source
+
+    def __hash__(self):
+        return hash((self.start, self.end, self.source))
+
+    def __repr__(self):
+        return f"SourceCodeNode(start={self.start}, end={self.end}, source='{self.source}')"
+
+
+cnode = namedtuple("ConceptNode", "concept_key start end source")
+short_cnode = namedtuple("ConceptNode", "concept_key source")
+utnode = namedtuple("UnrecognizedTokensNode", "start end source")
+scnode = namedtuple("SourceCodeNode", "start end source")
+
+
 class NonTerminalNode(LexerNode):
     """
     Returned by the ConceptLexerParser
@@ -146,9 +166,6 @@ class NonTerminalNode(LexerNode):
         return name + sub_names
 
     def __eq__(self, other):
-        # if not super().__eq__(other):
-        #     return False
-
         if not isinstance(other, NonTerminalNode):
             return False
 
@@ -176,9 +193,6 @@ class TerminalNode(LexerNode):
         return name + f"'{self.value}'"
 
     def __eq__(self, other):
-        # if not super().__eq__(other):
-        #     return False
-
         if not isinstance(other, TerminalNode):
             return False