Implemented some enhancement request and fixed some bugs

Fixed #2 : Variables are not recognized when inside a rule token
Fixed #15 : Rule: rete attributes are lost when a new ontology is created
Fixed #14 : ReteNetwork: Format rules must not be added to Rete network
Fixed #16 : DefConcept: Variables are not recognized when they are keyword arguments
Fixed #4 : Comparison are not correctly set when comparison property is a concept
Fixed #14 : Parser: merge FunctionParser.NamesNode and ExpressionParser.NamesNode
Fixed #18 : Parser: Add SourceCodeNode test to UnrecognizedNodeParser
Fixed #20 : At startup Number concept is saved in db a numerous number of time
Fixed #21 : CacheManager: I can remove all elements from a ListIfNeededCache and fill it again
Fixed #22 : CacheManager: I can remove all elements from a SetCache and fill it again
Fixed #23 : HistoryManager: history() no longer works
Fixed #24 : HistoryManager: history() no longer works after creating an exec rule
Fixed #25 : SheerkaMemory: Use MemoryObject instead of sheerka.local
Fixed #26 : Debugger: add the list all available services..
Fixed #27 : CONCEPTS_GRAMMARS_ENTRY does not seems to be in use any more
Fixed #28 : Give order to services

src/parsers/BaseNodeParser.py

@@ -231,8 +231,7 @@ class ConceptNode(LexerNode):
 
     def clone(self):
         # do we need to clone the concept as well ?
-        clone = ConceptNode(self.concept, self.start, self.end, self.tokens, self.source, self.underlying)
-        return clone
+        return ConceptNode(self.concept, self.start, self.end, self.tokens, self.source, self.underlying)
 
     def as_bag(self):
         """
@@ -305,6 +304,16 @@ class SourceCodeNode(LexerNode):
     def get_source_to_parse(self):
         return self.python_node.source
 
+    def clone(self):
+        clone = SourceCodeNode(
+            self.start,
+            self.end,
+            self.tokens,
+            self.source,
+            self.python_node,
+            self.return_value)
+        return clone
+
 
 class SourceCodeWithConceptNode(LexerNode):
     """
@@ -413,6 +422,8 @@ class SourceCodeWithConceptNode(LexerNode):
 
     def clone(self):
         clone = SourceCodeWithConceptNode(self.first, self.last, self.nodes.copy(), self.has_unrecognized)
+        clone.python_node = self.python_node
+        clone.return_value = self.return_value
         return clone
 
     def to_short_str(self):

src/parsers/BnfNodeParser.py

@@ -1189,7 +1189,7 @@ class BnfNodeParser(BaseNodeParser):
 
         if 'sheerka' in kwargs:
             sheerka = kwargs.get("sheerka")
-            self.concepts_grammars = sheerka.concepts_grammars
+            self.concepts_grammars = sheerka.get_concepts_bnf_definitions()
             self.sheerka = sheerka
         else:
             self.concepts_grammars = Cache()

src/parsers/DefConceptParser.py

@@ -211,12 +211,12 @@ class DefConceptParser(BaseCustomGrammarParser):
             return None
 
         source = self.sheerka.services[SheerkaExecute.NAME].get_parser_input(None, tokens[1:])
-        parsed = core.builtin_helpers.parse_unrecognized(self.context,
-                                                         source,
-                                                         parsers="all",
-                                                         who=self.name,
-                                                         prop=keyword,
-                                                         filter_func=core.builtin_helpers.expect_one)
+        parsed = self.sheerka.parse_unrecognized(self.context,
+                                                 source,
+                                                 parsers="all",
+                                                 who=self.name,
+                                                 prop=keyword,
+                                                 filter_func=core.builtin_helpers.expect_one)
 
         if not parsed.status:
             self.add_error(parsed.value)

src/parsers/ExpressionParser.py

@@ -1,8 +1,7 @@
 from itertools import product
 
 from core.builtin_concepts import BuiltinConcepts
-from core.builtin_helpers import only_successful, parse_unrecognized, get_inner_body, parse_python, \
-    get_lexer_nodes_using_positions
+from core.builtin_helpers import only_successful, get_inner_body, get_lexer_nodes_using_positions
 from core.sheerka.services.SheerkaExecute import ParserInput
 from core.sheerka.services.sheerka_service import FailedToCompileError
 from core.tokenizer import TokenKind, Tokenizer, Keywords
@@ -247,7 +246,7 @@ class ExpressionParser(BaseParser):
         for conjunction in conjunctions:
             # try to recognize conjunction, one by one
             # negative conjunction can be a concept starting with 'not'
-            parsed_ret = parse_unrecognized(
+            parsed_ret = context.sheerka.parse_unrecognized(
                 context,
                 conjunction.get_value(),  # we remove the 'NOT' part when needed to ease the recognition
                 parsers="all",
@@ -280,7 +279,7 @@ class ExpressionParser(BaseParser):
                 return_values.append(recognized_conjunctions[0])
             elif len(recognized_conjunctions) == 1 and recognized_conjunctions[0].who == "parsers.Python":
                 # it is a negated python Node. Need to parse again
-                ret = parse_python(context, source=str(conjunctions[0]))
+                ret = context.sheerka.parse_python(context, source=str(conjunctions[0]))
                 if ret.status:
                     return_values.append(ret)
                 else:

src/parsers/FunctionParser.py

@@ -1,22 +1,20 @@
 from dataclasses import dataclass
-from typing import List
 
 from core.builtin_concepts import BuiltinConcepts
 from core.builtin_helpers import get_lexer_nodes_from_unrecognized, update_compiled
 from core.concept import Concept
 from core.sheerka.services.SheerkaExecute import ParserInput
-from core.tokenizer import TokenKind, Token
+from core.tokenizer import TokenKind
 from core.utils import get_n_clones
-from parsers.SequenceNodeParser import SequenceNodeParser
 from parsers.BaseNodeParser import SourceCodeNode, SourceCodeWithConceptNode, UnrecognizedTokensNode
 from parsers.BaseParser import BaseParser, UnexpectedTokenParsingError, UnexpectedEofParsingError, Node
 from parsers.BnfNodeParser import BnfNodeParser
 from parsers.PythonWithConceptsParser import PythonWithConceptsParser
 from parsers.RuleParser import RuleParser
+from parsers.SequenceNodeParser import SequenceNodeParser
 from parsers.SyaNodeParser import SyaNodeParser
+from parsers.expressions import NameExprNode
 
-# No need to check for Python code as the source code node will resolve to python code anyway
-# I only look for concepts, so
 PARSERS = [RuleParser.NAME,
            SequenceNodeParser.NAME,
            BnfNodeParser.NAME,
@@ -28,49 +26,16 @@ class FunctionParserNode(Node):
     pass
 
 
-@dataclass()
-class NamesNode(FunctionParserNode):
-    start: int  # index of the first token
-    end: int  # index of the last token
-    tokens: List[Token]
-
-    def __repr__(self):
-        return f"NameNode('{self.str_value()}')"
-
-    def str_value(self):
-        if self.tokens is None:
-            return None
-
-        return "".join([t.str_value for t in self.tokens])
-
-    def to_unrecognized(self):
-        """
-        UnrecognizedTokensNode with all tokens
-        """
-        return UnrecognizedTokensNode(self.start, self.end, self.tokens).fix_source()
-
-    def to_str_unrecognized(self):
-        """
-        UnrecognizedTokensNode with one token, which is a string token of all the tokens
-        """
-        token = Token(TokenKind.STRING,
-                      "'" + self.str_value() + "'",
-                      self.tokens[0].index,
-                      self.tokens[0].line,
-                      self.tokens[0].column)
-        return UnrecognizedTokensNode(self.start, self.end, [token]).fix_source()
-
-
 @dataclass()
 class FunctionParameter:
     """
     class the represent result of the parameter parsing
     """
-    value: NamesNode  # value parsed
-    separator: NamesNode = None  # holds the value and the position of the separator
+    value: NameExprNode  # value parsed
+    separator: NameExprNode = None  # holds the value and the position of the separator
 
     def add_sep(self, start, end, tokens):
-        self.separator = NamesNode(start, end, tokens)
+        self.separator = NameExprNode(start, end, tokens)
 
     def value_to_unrecognized(self):
         return UnrecognizedTokensNode(self.value.start, self.value.end, self.value.tokens).fix_source()
@@ -83,8 +48,8 @@ class FunctionParameter:
 
 @dataclass
 class FunctionNode(FunctionParserNode):
-    first: NamesNode  # beginning of the function (it should represent the name of the function)
-    last: NamesNode  # last part of the function (it should be the trailing parenthesis)
+    first: NameExprNode  # beginning of the function (it should represent the name of the function)
+    last: NameExprNode  # last part of the function (it should be the trailing parenthesis)
     parameters: list
 
 
@@ -95,11 +60,11 @@ class FN(FunctionNode):
 
     Thereby,
     FN("first", "last", ["param1," ...]) can be compared to
-    FunctionNode(NamesNode("first"), NamesNode("second"),  [FunctionParameter(NamesNodes("param1"), NamesNodes(", ")])
+    FunctionNode(NameExprNode("first"), NameExprNode("second"),  [FunctionParameter(NamesNodes("param1"), NamesNodes(", ")])
 
     Note that FunctionParameter can easily be defined with a single string
-        * "param" -> FunctionParameter(NamesNode("param"), None)
-        * "param, " -> FunctionParameter(NamesNode("param"), NamesNode(", "))
+        * "param" -> FunctionParameter(NameExprNode("param"), None)
+        * "param, " -> FunctionParameter(NameExprNode("param"), NameExprNode(", "))
     For more complicated situations, you can use a tuple (value, sep) to define the value part and the separator part
     """
 
@@ -123,14 +88,13 @@ class FN(FunctionNode):
             return self.first == other.first and self.last == other.last and self.parameters == other.parameters
 
         if isinstance(other, FunctionNode):
-            if self.first != other.first.str_value() or self.last != other.last.str_value():
+            if self.first != other.first.value or self.last != other.last.value:
                 return False
             if len(self.parameters) != len(other.parameters):
                 return False
             for self_parameter, other_parameter in zip(self.parameters, other.parameters):
-                value = other_parameter.value.str_value() if isinstance(self_parameter[0],
-                                                                        str) else other_parameter.value
-                sep = other_parameter.separator.str_value() if other_parameter.separator else None
+                value = other_parameter.value.value if isinstance(self_parameter[0], str) else other_parameter.value
+                sep = other_parameter.separator.value if other_parameter.separator else None
                 if self_parameter[0] != value or self_parameter[1] != sep:
                     return False
 
@@ -244,7 +208,7 @@ class FunctionParser(BaseParser):
                                                        [TokenKind.LPAR]))
             return None
 
-        start_node = NamesNode(start, start + 1, self.parser_input.tokens[start:start + 2])
+        start_node = NameExprNode(start, start + 1, self.parser_input.tokens[start:start + 2])
         if not self.parser_input.next_token():
             self.add_error(UnexpectedEofParsingError(f"Unexpected EOF after left parenthesis"))
             return FunctionNode(start_node, None, None)
@@ -261,7 +225,7 @@ class FunctionParser(BaseParser):
             return FunctionNode(start_node, None, params)
 
         return FunctionNode(start_node,
-                            NamesNode(self.parser_input.pos, self.parser_input.pos, [token]),
+                            NameExprNode(self.parser_input.pos, self.parser_input.pos, [token]),
                             params)
 
     def parse_parameters(self):
@@ -319,7 +283,7 @@ class FunctionParser(BaseParser):
             if not self.parser_input.next_token(skip_whitespace=False):
                 break
 
-        return NamesNode(start_pos, self.parser_input.pos - 1, tokens) if len(tokens) else None
+        return NameExprNode(start_pos, self.parser_input.pos - 1, tokens) if len(tokens) else None
 
     def to_source_code_node(self, function_node: FunctionNode):
         python_parser = PythonWithConceptsParser()
@@ -350,7 +314,7 @@ class FunctionParser(BaseParser):
 
         # try to recognize every parameter, one by one
         for param in function_node.parameters:
-            if isinstance(param.value, NamesNode):
+            if isinstance(param.value, NameExprNode):
                 # try to recognize concepts
                 unrecognized = param.value.to_unrecognized()
                 nodes_sequences = get_lexer_nodes_from_unrecognized(self.context,

src/parsers/PythonWithConceptsParser.py

@@ -1,5 +1,5 @@
 from core.builtin_concepts import BuiltinConcepts
-from core.builtin_helpers import parse_python, CreateObjectIdentifiers
+from core.builtin_helpers import CreateObjectIdentifiers
 from parsers.BaseNodeParser import ConceptNode, RuleNode
 from parsers.BaseNodeParser import SourceCodeWithConceptNode
 from parsers.BaseParser import BaseParser
@@ -71,7 +71,7 @@ class PythonWithConceptsParser(BaseParser):
                 if hasattr(node, "get_python_node"):
                     python_ids_mappings.update(node.get_python_node().objects)
 
-        result = parse_python(context, to_parse, "Trying Python for '" + to_parse + "'")
+        result = context.sheerka.parse_python(context, to_parse, "Trying Python for '" + to_parse + "'")
 
         if result.status:
             python_node = result.body.body

src/parsers/SyaNodeParser.py

@@ -5,7 +5,6 @@ from typing import List
 
 from core import builtin_helpers
 from core.builtin_concepts import BuiltinConcepts
-from core.builtin_helpers import parse_function
 from core.concept import Concept, DEFINITION_TYPE_BNF
 from core.global_symbols import CONCEPT_COMPARISON_CONTEXT
 from core.sheerka.services.SheerkaComparisonManager import SheerkaComparisonManager
@@ -833,10 +832,10 @@ class InFixToPostFix:
 
             if self.unrecognized_tokens.parenthesis_count == 0:
                 self.unrecognized_tokens.fix_source()
-                res = parse_function(self.context,
-                                     self.unrecognized_tokens.source,
-                                     self.unrecognized_tokens.tokens[:],
-                                     self.unrecognized_tokens.start)
+                res = self.context.sheerka.parse_function(self.context,
+                                                          self.unrecognized_tokens.source,
+                                                          self.unrecognized_tokens.tokens[:],
+                                                          self.unrecognized_tokens.start)
 
                 instances = get_n_clones(self, len(res))
                 self.forked.extend(instances[1:])

src/parsers/UnrecognizedNodeParser.py

@@ -2,11 +2,11 @@ from dataclasses import dataclass
 
 import core.utils
 from core.builtin_concepts import BuiltinConcepts
-from core.builtin_helpers import only_successful, parse_unrecognized, get_lexer_nodes, update_compiled
-from parsers.SequenceNodeParser import SequenceNodeParser
+from core.builtin_helpers import only_successful, get_lexer_nodes, update_compiled
 from parsers.BaseNodeParser import ConceptNode, UnrecognizedTokensNode, SourceCodeNode, SourceCodeWithConceptNode
 from parsers.BaseParser import BaseParser, ParsingError
 from parsers.BnfNodeParser import BnfNodeParser
+from parsers.SequenceNodeParser import SequenceNodeParser
 from parsers.SyaNodeParser import SyaNodeParser
 
 PARSERS = ["EmptyString",
@@ -56,7 +56,7 @@ class UnrecognizedNodeParser(BaseParser):
                     sequences_found = core.utils.sheerka_product(sequences_found, [res.body])
 
             elif isinstance(node, UnrecognizedTokensNode):
-                res = parse_unrecognized(context, node.source, PARSERS)
+                res = context.sheerka.parse_unrecognized(context, node.source, PARSERS)
                 res = only_successful(context, res)
                 if res.status:
                     lexer_nodes = get_lexer_nodes(res.body.body, node.start, node.tokens)

src/parsers/expressions.py

@@ -3,6 +3,7 @@ from typing import List, Tuple
 
 from core.tokenizer import Token, TokenKind, Tokenizer
 from core.utils import tokens_are_matching
+from parsers.BaseNodeParser import UnrecognizedTokensNode
 from parsers.BaseParser import Node, ParsingError
 
 
@@ -86,6 +87,23 @@ class NameExprNode(ExprNode):
     def __hash__(self):
         return super().__hash__()
 
+    def to_unrecognized(self):
+        """
+        UnrecognizedTokensNode with all tokens
+        """
+        return UnrecognizedTokensNode(self.start, self.end, self.tokens).fix_source()
+
+    def to_str_unrecognized(self):
+        """
+        UnrecognizedTokensNode with one token, which is a string token of all the tokens
+        """
+        token = Token(TokenKind.STRING,
+                      "'" + self.str_value() + "'",
+                      self.tokens[0].index,
+                      self.tokens[0].line,
+                      self.tokens[0].column)
+        return UnrecognizedTokensNode(self.start, self.end, [token]).fix_source()
+
 
 @dataclass(init=False)
 class AndNode(ExprNode):