Fixed some bugs

tests/BaseTest.py

@@ -80,6 +80,13 @@ class BaseTest:
 
     @staticmethod
     def get_concept_instance(sheerka, concept, **kwargs):
+        """
+        Use to instantiate concept with default variables already set
+        :param sheerka:
+        :param concept:
+        :param kwargs:
+        :return:
+        """
         instance = sheerka.new(concept.key if isinstance(concept, Concept) else concept)
         for i, var in enumerate(instance.metadata.variables):
             if var[0] in kwargs:

tests/cache/test_cache.py

@@ -1,4 +1,5 @@
 import pytest
+from cache.BaseCache import MAX_INITIALIZED_KEY
 from cache.Cache import Cache
 from cache.CacheManager import CacheManager
 from cache.DictionaryCache import DictionaryCache
@@ -55,6 +56,14 @@ class TestCache(TestUsingMemoryBasedSheerka):
         assert cache.get("key") == "key_not_found"
         assert "key" in cache  # default callable are put in cache
 
+    def test_i_dont_ask_the_remote_repository_twice(self):
+        nb_request = []
+
+        cache = Cache(default=lambda key: nb_request.append("requested"))
+        assert cache.get("key") is None
+        assert cache.get("key") is None
+        assert len(nb_request) == 1
+
     def test_i_can_put_and_retrieve_value_from_list_cache(self):
         cache = ListCache()
 
@@ -532,3 +541,27 @@ class TestCache(TestUsingMemoryBasedSheerka):
         cache.delete("key")
         assert cache.get("value") is None
         assert cache.to_remove == {"key"}
+
+    def test_initialized_key_is_removed_when_the_entry_is_found(self):
+        caches = [Cache(), ListCache(), ListIfNeededCache(), SetCache()]
+
+        for cache in caches:
+            cache.put("key", "value")
+            cache.get("key")
+
+            assert len(cache._initialized_keys) == 0
+
+        cache = IncCache()
+        cache.put("key", 10)
+        cache.get("key")
+        assert len(cache._initialized_keys) == 0
+
+    def test_initialized_keys_are_reset_when_max_length_is_reached(self):
+        cache = Cache()
+        for i in range(MAX_INITIALIZED_KEY):
+            cache.get(str(i))
+
+        assert len(cache._initialized_keys) == MAX_INITIALIZED_KEY
+
+        cache.get(str(MAX_INITIALIZED_KEY + 1))
+        assert len(cache._initialized_keys) == 1

tests/core/test_SheerkaSetsManager.py

@@ -273,7 +273,7 @@ class TestSheerkaSetsManager(TestUsingMemoryBasedSheerka):
         # update number
         sheerka.set_isa(context, sheerka.new("one"), number)
 
-        assert twenties.bnf.elements[1].recurse_id == "1002#number#1003"
+        assert twenties.bnf.elements[1].recurse_id == "1003#1002(number)"
 
     def test_concepts_in_group_cache_is_updated(self):
         sheerka, context, one, two, number = self.init_concepts("one", "two", "number")

tests/evaluators/test_LexerNodeEvaluator.py

@@ -5,8 +5,8 @@ from core.builtin_concepts import ReturnValueConcept, ParserResultConcept, Built
 from core.concept import Concept, ConceptParts, DoNotResolve
 from core.sheerka.services.SheerkaExecute import ParserInput
 from evaluators.LexerNodeEvaluator import LexerNodeEvaluator
-from parsers.BaseNodeParser import SourceCodeNode
-from parsers.BnfNodeParser import ConceptNode, BnfNodeParser, UnrecognizedTokensNode
+from parsers.BaseNodeParser import SourceCodeNode, ConceptNode, UnrecognizedTokensNode
+from parsers.BnfNodeParser import BnfNodeParser
 from parsers.PythonParser import PythonNode
 
 from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka

tests/non_reg/test_sheerka_non_reg.py

@@ -1091,6 +1091,39 @@ as:
         assert res[0].status
         assert res[0].body
 
+    def test_i_can_evaluate_source_code_with_concept(self):
+        init = [
+            "def concept the a ret a",
+        ]
+
+        sheerka = self.init_scenario(init)
+        res = sheerka.evaluate_user_input("desc(the a)")
+
+        assert len(res) == 1
+        assert res[0].status
+
+    def test_i_can_parse_concept_with_variables_using_short_name(self):
+        init = [
+            "def concept foo from a foo b where a,b",
+            "def concept bar from bar a where a",
+            "def concept baz from a baz where a",
+        ]
+
+        sheerka = self.init_scenario(init)
+        res = sheerka.evaluate_user_input("desc(foo)")
+        assert len(res) == 1
+        assert res[0].status
+
+        sheerka = self.init_scenario(init)
+        res = sheerka.evaluate_user_input("desc(bar)")
+        assert len(res) == 1
+        assert res[0].status
+
+        sheerka = self.init_scenario(init)
+        res = sheerka.evaluate_user_input("desc(baz)")
+        assert len(res) == 1
+        assert res[0].status
+
 
 class TestSheerkaNonRegFile(TestUsingFileBasedSheerka):
     def test_i_can_def_several_concepts(self):

tests/parsers/parsers_utils.py

@@ -87,10 +87,13 @@ def get_node(
             return sub_expr
 
     if isinstance(sub_expr, SCWC):
-        first = get_node(concepts_map, expression_as_tokens, sub_expr.first, sya=sya)
-        last = get_node(concepts_map, expression_as_tokens, sub_expr.last, sya=sya)
-        content = [get_node(concepts_map, expression_as_tokens, c, sya=sya) for c in sub_expr.content]
-        return SourceCodeWithConceptNode(first, last, content).pseudo_fix_source()
+        sub_expr.first = get_node(concepts_map, expression_as_tokens, sub_expr.first, sya=sya)
+        sub_expr.last = get_node(concepts_map, expression_as_tokens, sub_expr.last, sya=sya)
+        sub_expr.content = [get_node(concepts_map, expression_as_tokens, c, sya=sya) for c in sub_expr.content]
+        sub_expr.fix_pos(sub_expr.first)
+        sub_expr.fix_pos(sub_expr.last)
+        return sub_expr
+        #return SourceCodeWithConceptNode(first, last, content).pseudo_fix_source()
 
     if isinstance(sub_expr, SCN):
         node = get_node(concepts_map, expression_as_tokens, sub_expr.source, sya=sya)

tests/parsers/test_BnfNodeParser.py

@@ -3,8 +3,8 @@ from core.builtin_concepts import BuiltinConcepts
 from core.concept import Concept, ConceptParts, DoNotResolve, CC, DEFINITION_TYPE_BNF
 from core.sheerka.services.SheerkaExecute import ParserInput
 from parsers.BaseNodeParser import CNC, UTN, CN
-from parsers.BnfNodeParser import BnfNodeParser, StrMatch, TerminalNode, NonTerminalNode, Sequence, OrderedChoice, \
-    Optional, ZeroOrMore, OneOrMore, ConceptExpression, UnOrderedChoice
+from parsers.BnfNodeParser import StrMatch, TerminalNode, NonTerminalNode, Sequence, OrderedChoice, \
+    Optional, ZeroOrMore, OneOrMore, ConceptExpression, UnOrderedChoice, BnfNodeParser
 from parsers.BnfParser import BnfParser
 
 import tests.parsers.parsers_utils
@@ -969,7 +969,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
         assert ConceptExpression(my_map["one"], rule_name="one") in number_nodes[0].nodes
         assert ConceptExpression(my_map["twenty"], rule_name="twenty") in number_nodes[0].nodes
 
-    def test_i_can_get_parsing_expression_when_sequence_of_concept(self):
+    def test_i_can_get_parsing_expression_when_sequence_of_concepts(self):
         my_map = {
             "one": Concept("one"),
             "two_ones": self.bnf_concept("two_ones", Sequence(ConceptExpression("one"), ConceptExpression("one")))
@@ -1203,6 +1203,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
 
     def test_i_can_parse_hundreds_like_expression(self):
         sheerka, context, parser = self.init_parser(init_from_sheerka=True)
+        sheerka.concepts_grammars.clear()
 
         text = "three hundred and thirty two"
         three = CC("three", body=DoNotResolve("three"))
@@ -1226,7 +1227,9 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
                              thirties=thirty_two))
 
         expected_array = compute_expected_array(cmap, text, [expected])
+
         res = parser.parse(context, ParserInput(text))
+
         parser_result = res.value
         concepts_nodes = res.value.value
 
@@ -1400,6 +1403,15 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
         assert parser.parse(context, ParserInput("foo foo foo bar")).status
         assert not parser.parse(context, ParserInput("foo baz")).status
 
+    def test_i_only_get_the_requested_parsing_expression(self):
+        sheerka, context, parser = self.init_parser(init_from_sheerka=True)
+        parser.context = context
+        parser.sheerka = sheerka
+        sheerka.concepts_grammars.clear()  # to simulate restart
+
+        parser.get_parsing_expression(context, sheerka.resolve("thirties"))
+        assert len(parser.concepts_grammars) == 9  # requested concept + concepts that do not contains UnorderedChoice
+
     @pytest.mark.parametrize("name, expected", [
         (None, []),
         ("", []),

tests/parsers/test_BnfParser.py

@@ -5,8 +5,9 @@ from core.sheerka.services.SheerkaExecute import ParserInput
 from core.tokenizer import Tokenizer, TokenKind, LexerError, Token
 from parsers.BaseNodeParser import cnode
 from parsers.BaseParser import UnexpectedTokenErrorNode
-from parsers.BnfNodeParser import StrMatch, Optional, ZeroOrMore, OrderedChoice, Sequence, OneOrMore, \
-    BnfNodeParser, ConceptExpression
+from parsers.BnfNodeParser import StrMatch, Optional, ZeroOrMore, OrderedChoice, Sequence, \
+    OneOrMore, ConceptExpression
+from parsers.BnfNodeParser import BnfNodeParser
 from parsers.BnfParser import BnfParser, UnexpectedEndOfFileError
 
 from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
@@ -235,4 +236,4 @@ class TestBnfParser(TestUsingMemoryBasedSheerka):
         assert res.status
         pexpression = res.value.value
         assert pexpression == Sequence(StrMatch('twenty'), ConceptExpression(number, "n1"))
-        assert pexpression.elements[1].recurse_id == "1003#n1#1004"
+        assert pexpression.elements[1].recurse_id == "1004#1003(n1)"

tests/parsers/test_SyaNodeParser.py

@@ -5,7 +5,7 @@ from core.sheerka.services.SheerkaComparisonManager import SheerkaComparisonMana
 from core.sheerka.services.SheerkaExecute import ParserInput
 from core.tokenizer import Tokenizer
 from parsers.BaseNodeParser import utnode, ConceptNode, cnode, short_cnode, UnrecognizedTokensNode, \
-    SCWC, CNC, UTN
+    SCWC, CNC, UTN, SourceCodeWithConceptNode
 from parsers.PythonParser import PythonNode
 from parsers.SyaNodeParser import SyaNodeParser, SyaConceptParserHelper, SyaAssociativity, \
     NoneAssociativeSequenceErrorNode, TooManyParametersFound
@@ -56,10 +56,12 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
         cmap["minus"].set_prop(BuiltinConcepts.ASSOCIATIVITY, "right")
         TestSyaNodeParser.sheerka.services[SheerkaComparisonManager.NAME].set_is_greater_than(context,
                                                                                               BuiltinConcepts.PRECEDENCE,
-                                                                                              cmap["mult"], cmap["plus"])
+                                                                                              cmap["mult"],
+                                                                                              cmap["plus"])
         TestSyaNodeParser.sheerka.services[SheerkaComparisonManager.NAME].set_is_greater_than(context,
                                                                                               BuiltinConcepts.PRECEDENCE,
-                                                                                              cmap["mult"], cmap["minus"])
+                                                                                              cmap["mult"],
+                                                                                              cmap["minus"])
 
         # TestSyaNodeParser.sheerka.force_sya_def(context, [
         #     (cmap["plus"].id, 5, SyaAssociativity.Right),
@@ -716,6 +718,9 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
             ["one", "two", "three", "if", SCWC(" function(", ")", "x$!#")]]),
         ("one prefixed function(two)", [["one", "prefixed", SCWC(" function(", ")", "two")]]),
         ("suffixed one function(two)", [["one", "suffixed", SCWC(" function(", ")", "two")]]),
+        (
+            "func1(suffixed one func2(two))",
+            [[SCWC("func1(", (")", 1), "one", "suffixed", SCWC(" func2(", ")", "two"))]]),
     ])
     def test_i_can_post_fix_when_parenthesis_and_unknown(self, expression, expected_sequences):
         sheerka, context, parser = self.init_parser()
@@ -728,19 +733,19 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
             assert res_i.out == expected_array
 
     @pytest.mark.parametrize("expression, expected", [
-        # ("(", ("(", 0)),
-        # ("one plus ( 1 + ", ("(", 4)),
-        # ("one( 1 + ", ("(", 1)),
-        # ("one ( 1 + ", ("(", 2)),
-        # ("function( 1 + ", ("(", 1)),
-        # ("function ( 1 + ", ("(", 2)),
-        # ("one plus ) 1 + ", (")", 4)),
-        # ("one ) 1 + ", (")", 2)),
-        # ("function ) 1 + ", (")", 2)),
-        # ("one ? ( : two", ("(", 4)),
-        # ("one ? one plus ( : two", ("(", 8)),
-        # ("one ? ) : two", (")", 4)),
-        # ("one ? one plus ) : two", (")", 8)),
+        ("(", ("(", 0)),
+        ("one plus ( 1 + ", ("(", 4)),
+        ("one( 1 + ", ("(", 1)),
+        ("one ( 1 + ", ("(", 2)),
+        ("function( 1 + ", ("(", 1)),
+        ("function ( 1 + ", ("(", 2)),
+        ("one plus ) 1 + ", (")", 4)),
+        ("one ) 1 + ", (")", 2)),
+        ("function ) 1 + ", (")", 2)),
+        ("one ? ( : two", ("(", 4)),
+        ("one ? one plus ( : two", ("(", 8)),
+        ("one ? ) : two", (")", 4)),
+        ("one ? one plus ) : two", (")", 8)),
         ("(one plus ( 1 + )", ("(", 0)),
     ])
     def test_i_can_detect_parenthesis_mismatch_error_when_post_fixing(self, expression, expected):
@@ -797,6 +802,29 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
         assert len(res) == 1
         assert res[0].out == expected_array
 
+    def test_i_cannot_post_fix_using_concept_short_name(self):
+        concepts_map = {
+            "infixed": self.from_def_concept("infixed", "a infixed b", ["a", "b"]),
+            "suffixed": self.from_def_concept("suffixed", "suffixed a", ["a"]),
+            "prefixed": self.from_def_concept("prefixed", "a prefixed", ["a"]),
+        }
+        sheerka, context, parser = self.init_parser(concepts_map)
+
+        res = parser.infix_to_postfix(context, ParserInput("desc(infixed)"))
+        assert len(res) == 1
+        assert isinstance(res[0].out[0], SourceCodeWithConceptNode)
+        assert res[0].out[0].nodes[0].error == 'Not enough prefix parameters'
+
+        res = parser.infix_to_postfix(context, ParserInput("desc(suffixed)"))
+        assert len(res) == 1
+        assert isinstance(res[0].out[0], SourceCodeWithConceptNode)
+        assert res[0].out[0].nodes[0].error == 'Not enough suffix parameters'
+
+        res = parser.infix_to_postfix(context, ParserInput("desc(prefixed)"))
+        assert len(res) == 1
+        assert isinstance(res[0].out[0], SourceCodeWithConceptNode)
+        assert res[0].out[0].nodes[0].error == 'Not enough prefix parameters'
+
     @pytest.mark.parametrize("expression", [
         "one ? two : three",
         "one?two:three",
@@ -1117,6 +1145,26 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
         assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
         assert lexer_nodes == expected
 
+    def test_i_cannot_parse_function_using_short_name(self):
+        concepts_map = {
+            "infixed": self.from_def_concept("infixed", "a infixed b", ["a", "b"]),
+            "suffixed": self.from_def_concept("suffixed", "suffixed a", ["a"]),
+            "prefixed": self.from_def_concept("prefixed", "a prefixed", ["a"]),
+        }
+        sheerka, context, parser = self.init_parser(concepts_map)
+
+        res = parser.parse(context, ParserInput("desc(infixed)"))
+        assert not res.status
+        assert sheerka.isinstance(res.body, BuiltinConcepts.NOT_FOR_ME)
+
+        res = parser.parse(context, ParserInput("desc(suffixed)"))
+        assert not res.status
+        assert sheerka.isinstance(res.body, BuiltinConcepts.NOT_FOR_ME)
+
+        res = parser.parse(context, ParserInput("desc(prefixed)"))
+        assert not res.status
+        assert sheerka.isinstance(res.body, BuiltinConcepts.NOT_FOR_ME)
+
     @pytest.mark.parametrize("text", [
         "one",
         "1 + 1",
@@ -1146,4 +1194,3 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
 
         assert not res.status
         assert sheerka.isinstance(res.body, BuiltinConcepts.IS_EMPTY)
-

tests/parsers/test_UnrecognizedNodeParser.py

@@ -2,7 +2,7 @@ from core.builtin_concepts import ParserResultConcept, BuiltinConcepts
 from core.concept import Concept, CC
 from core.tokenizer import Tokenizer, TokenKind
 from parsers.BaseNodeParser import ConceptNode, UnrecognizedTokensNode, scnode, cnode, \
-    utnode, SyaAssociativity, CN, CNC, UTN
+    utnode, SyaAssociativity, CN, CNC, UTN, SourceCodeWithConceptNode, SCWC, SourceCodeNode
 from parsers.UnrecognizedNodeParser import UnrecognizedNodeParser
 
 from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
@@ -24,11 +24,20 @@ def get_input_nodes_from(my_concepts_map, full_expr, *args):
             concept = n.concept if hasattr(n, "concept") and n.concept else \
                 Concept().update_from(my_concepts_map[n.concept_key])
             tokens = full_expr_as_tokens[n.start: n.end + 1]
-            if hasattr(node, "compiled"):
+            if hasattr(n, "compiled"):
                 for k, v in n.compiled.items():
                     concept.compiled[k] = _get_real_node(v)
             return ConceptNode(concept, n.start, n.end, tokens)
 
+        if isinstance(n, SCWC):
+            n.first = _get_real_node(n.first)
+            n.last = _get_real_node(n.first)
+            n.content = tuple(_get_real_node(nn) for nn in n.content)
+            return SourceCodeWithConceptNode(n.first, n.last, list(n.content))
+
+        if isinstance(n, (UnrecognizedTokensNode, ConceptNode, SourceCodeNode, SourceCodeWithConceptNode)):
+            return n
+
         raise NotImplementedError()
 
     res = []
@@ -307,6 +316,41 @@ class TestUnrecognizedNodeParser(TestUsingMemoryBasedSheerka):
             exclude_body=True)
         assert actual_nodes == expected_array
 
+    def test_i_can_parse_unrecognized_source_code_with_concept_node(self):
+        sheerka, context, parser = self.init_parser()
+
+        expression = "desc(a plus b)"
+        source_code_concepts = SCWC("desc(", ")", CNC("plus", a=UTN("a"), b=UTN("b")))
+        nodes = get_input_nodes_from(concepts_map, expression, source_code_concepts)
+        parser_input = ParserResultConcept("parsers.xxx", source=expression, value=nodes)
+
+        res = parser.parse(context, parser_input)
+        parser_result = res.body
+        actual_nodes = res.body.body
+
+        assert not res.status # status is False to let PythonWithConceptParser validate the code
+        assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT)
+        assert len(actual_nodes) == 1
+        assert actual_nodes[0].nodes[0].concept.metadata.is_evaluated  # 'a plus b' is recognized as concept definition
+
+    def test_i_can_parse_unrecognized_source_code_with_concept_node_when_var_in_short_term_memory(self):
+        sheerka, context, parser = self.init_parser()
+
+        expression = "desc(a plus b)"
+        source_code_concepts = SCWC("desc(", ")", CNC("plus", a=UTN("a"), b=UTN("b")))
+        nodes = get_input_nodes_from(concepts_map, expression, source_code_concepts)
+        parser_input = ParserResultConcept("parsers.xxx", source=expression, value=nodes)
+
+        context.add_to_short_term_memory("a", 1)
+        res = parser.parse(context, parser_input)
+        parser_result = res.body
+        actual_nodes = res.body.body
+
+        assert not res.status  # status is False to let PythonWithConceptParser validate the code
+        assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT)
+        assert len(actual_nodes) == 1
+        assert not actual_nodes[0].nodes[0].concept.metadata.is_evaluated  # 'a plus b' need to be evaluated
+
     def test_i_can_parse_sequences(self):
         sheerka, context, parser = self.init_parser()