Fixed variable recognition when it is a concept

src/core/sheerka/Sheerka.py

@@ -691,6 +691,9 @@ class Sheerka(Concept):
             return None
         return new_instances(concept) if return_new else concept
 
+    def fast_get_by_name(self, name):
+        return self.cache_manager.get(self.CONCEPTS_BY_NAME_ENTRY, name)
+
     def has_id(self, concept_id):
         """
         Returns True if a concept with this id exists in cache

src/core/sheerka/services/SheerkaCreateNewConcept.py

@@ -21,6 +21,7 @@ class SheerkaCreateNewConcept(BaseService):
 
     def initialize(self):
         self.sheerka.bind_service_method(self.create_new_concept, True)
+        self.sheerka.bind_service_method(self.not_is_variable, False, visible=False)
 
     def create_new_concept(self, context, concept: Concept):
         """
@@ -108,3 +109,11 @@ class SheerkaCreateNewConcept(BaseService):
                 refs.add(concept.id)
 
         return refs
+
+    def not_is_variable(self, name):
+        """
+        Given a name tells if it refers to a variable name
+        :param name:
+        :return:
+        """
+        return not self.sheerka.cache_manager.get(self.sheerka.CONCEPTS_BY_NAME_ENTRY, name)

src/evaluators/DefConceptEvaluator.py

@@ -8,7 +8,7 @@ from evaluators.BaseEvaluator import OneReturnValueEvaluator
 from parsers.BaseParser import NotInitializedNode
 from parsers.BnfNodeParser import ParsingExpression, ParsingExpressionVisitor
 from parsers.DefConceptParser import DefConceptNode, NameNode
-from parsers.PythonParser import PythonNode
+from parsers.PythonParser import PythonNode, get_python_node
 
 
 class ConceptOrRuleNameVisitor(ParsingExpressionVisitor):
@@ -129,15 +129,13 @@ class DefConceptEvaluator(OneReturnValueEvaluator):
         This function can only be a draft, as there may be tons of different situations
         I guess that it can only be complete when will we have access to Sheerka memory
         """
-
         #
         # Case of NameNode
         #
         if isinstance(ret_value, NameNode):
             names = [str(t.value) for t in ret_value.tokens if t.type in (
                 TokenKind.IDENTIFIER, TokenKind.STRING, TokenKind.KEYWORD)]
-            variables = filter(lambda x: x in concept_name, names)
+            return set(filter(lambda x: x in concept_name and context.sheerka.not_is_variable(x), names))
-            return set(variables)
 
         #
         # case of BNF
@@ -150,13 +148,13 @@ class DefConceptEvaluator(OneReturnValueEvaluator):
         #
         # Case of python code
         #
-        if isinstance(ret_value.value, ParserResultConcept) and isinstance(ret_value.value.value, PythonNode):
+        if (python_node := get_python_node(ret_value.value.value)) is not None:
             if len(concept_name) > 1:
-                python_node = ret_value.value.value
                 visitor = UnreferencedVariablesVisitor(context)
                 names = visitor.get_names(python_node.ast_)
-                variables = filter(lambda x: x in concept_name, names)
+                return set(filter(lambda x: x in concept_name and context.sheerka.not_is_variable(x), names))
-                return set(variables)
+            else:
+                return set()
 
         #
         # Concept
@@ -172,4 +170,4 @@ class DefConceptEvaluator(OneReturnValueEvaluator):
                     variables.add(identifier)
             return variables
 
-        return []
+        return set()

src/evaluators/PythonEvaluator.py

@@ -12,7 +12,7 @@ from core.rule import Rule
 from core.sheerka.ExecutionContext import ExecutionContext
 from core.tokenizer import Token, TokenKind
 from evaluators.BaseEvaluator import OneReturnValueEvaluator
-from parsers.PythonParser import PythonNode
+from parsers.PythonParser import PythonNode, get_python_node
 
 TO_DISABLED = ["breakpoint", "callable", "compile", "delattr", "eval", "exec", "exit", "input", "locals", "open",
                "print", "quit", "setattr"]
@@ -88,8 +88,7 @@ class PythonEvaluator(OneReturnValueEvaluator):
 
     def eval(self, context, return_value):
         sheerka = context.sheerka
-        node = return_value.value.value if isinstance(return_value.value.value, PythonNode) else \
+        node = get_python_node(return_value.value.value)
-            return_value.value.value.python_node
 
         debugger = context.get_debugger(PythonEvaluator.NAME, "eval")
         debugger.debug_entering(node=node)

src/parsers/BaseNodeParser.py

@@ -18,6 +18,12 @@ class ChickenAndEggError(Exception):
     concepts: Set[str]
 
 
+@dataclass
+class NoFirstTokenError(ErrorNode):
+    concept: Concept
+    key: str
+
+
 @dataclass()
 class LexerNode(Node):
     start: int  # starting index in the tokens list
@@ -891,7 +897,7 @@ class BaseNodeParser(BaseParser):
 
             if keywords is None:
                 # no first token found for a concept ?
-                return sheerka.ret(sheerka.name, False, concept)
+                return sheerka.ret(sheerka.name, False, NoFirstTokenError(concept, concept.key))
 
             for keyword in keywords:
                 res.setdefault(keyword, []).append(concept.id)

src/parsers/PythonParser.py

@@ -11,6 +11,14 @@ from parsers.BaseParser import BaseParser, Node, ErrorNode
 log = logging.getLogger(__name__)
 
 
+def get_python_node(obj):
+    if isinstance(obj, PythonNode):
+        return obj
+    if hasattr(obj, "python_node"):
+        return obj.python_node
+    return None
+
+
 @dataclass()
 class PythonErrorNode(ErrorNode):
     source: str

tests/evaluators/test_DefConceptEvaluator.py

@@ -6,8 +6,8 @@ from core.concept import VARIABLE_PREFIX, Concept, DEFINITION_TYPE_BNF, DEFINITI
 from core.tokenizer import Tokenizer
 from evaluators.DefConceptEvaluator import DefConceptEvaluator
 from parsers.BaseParser import BaseParser
-from parsers.BnfNodeParser import Sequence, StrMatch, ZeroOrMore, ConceptExpression
 from parsers.BnfDefinitionParser import BnfDefinitionParser
+from parsers.BnfNodeParser import Sequence, StrMatch, ZeroOrMore, ConceptExpression
 from parsers.DefConceptParser import DefConceptNode, NameNode
 from parsers.PythonParser import PythonNode, PythonParser
 
@@ -76,6 +76,10 @@ class TestDefConceptEvaluator(TestUsingMemoryBasedSheerka):
 
         return ReturnValueConcept(BaseParser.PREFIX + "some_name", True, ParserResultConcept(value=def_concept))
 
+    @staticmethod
+    def get_def_concept_node_from_name_only(name):
+        return DefConceptNode([], name=NameNode(list(Tokenizer(name))))
+
     @pytest.mark.parametrize("ret_val, expected", [
         (ReturnValueConcept(BaseParser.PREFIX + "some_name", True, ParserResultConcept(value=DefConceptNode([]))),
          True),
@@ -88,7 +92,7 @@ class TestDefConceptEvaluator(TestUsingMemoryBasedSheerka):
         context = self.get_context()
         assert DefConceptEvaluator().matches(context, ret_val) == expected
 
-    def test_that_the_source_is_correctly_set_for_bnf_concept(self):
+    def test_that_the_source_is_correctly_set_for_bnf_concepts(self):
         context = self.get_context()
         def_concept_return_value = self.get_def_concept(
             name="hello a",
@@ -114,22 +118,7 @@ class TestDefConceptEvaluator(TestUsingMemoryBasedSheerka):
         assert created_concept.get_metadata().definition == "hello a"
         assert created_concept.get_metadata().definition_type == "bnf"
 
-    def test_i_can_add_concept_with_the_correct_variables_when_referencing_other_concepts(self):
+    def test_that_the_source_is_correctly_set_for_concepts_with_simple_definition(self):
-        context = self.get_context()
-        def_concept_return_value = self.get_def_concept(
-            name="x plus y",
-            where=self.pretval(Concept("u is a v").def_var("u").def_var("v"), source="x is a number"),
-            body=self.pretval(Concept("add a b").def_var("a").def_var("b"), source="add x y"), )
-
-        evaluated = DefConceptEvaluator().eval(context, def_concept_return_value)
-
-        assert evaluated.status
-        assert context.sheerka.isinstance(evaluated.body, BuiltinConcepts.NEW_CONCEPT)
-
-        created_concept = evaluated.body.body
-        assert created_concept.get_metadata().variables == [("x", None), ("y", None)]
-
-    def test_that_the_source_is_correctly_set_for_concept_with_simple_definition(self):
         context = self.get_context()
         def_concept_return_value = self.get_def_concept(
             name="greetings",
@@ -153,6 +142,30 @@ class TestDefConceptEvaluator(TestUsingMemoryBasedSheerka):
         assert created_concept.get_metadata().definition == "hello a"
         assert created_concept.get_metadata().definition_type == "def"
 
+    def test_i_can_add_concept_with_the_correct_variables_when_referencing_other_concepts(self):
+        context = self.get_context()
+        def_concept_return_value = self.get_def_concept(
+            name="x plus y",
+            where=self.pretval(Concept("u is a v").def_var("u").def_var("v"), source="x is a number"),
+            body=self.pretval(Concept("add a b").def_var("a").def_var("b"), source="add x y"), )
+
+        evaluated = DefConceptEvaluator().eval(context, def_concept_return_value)
+
+        assert evaluated.status
+        assert context.sheerka.isinstance(evaluated.body, BuiltinConcepts.NEW_CONCEPT)
+
+        created_concept = evaluated.body.body
+        assert created_concept.get_metadata().variables == [("x", None), ("y", None)]
+
+    def test_other_concepts_are_not_variables(self):
+        sheerka, context, *concepts = self.init_concepts("little", "size", create_new=True)
+
+        def_concept_node = self.get_def_concept_node_from_name_only("little x")
+        name_to_use = DefConceptEvaluator.get_name_to_use(def_concept_node)
+        concept_part = self.get_concept_part("set_attr(x, size, little)")
+
+        assert DefConceptEvaluator.get_variables(context, concept_part, name_to_use) == {"x"}
+
     def test_that_the_new_concept_is_correctly_saved_in_db(self):
         context = self.get_context()
         def_concept_return_value = self.get_def_concept(
@@ -195,7 +208,7 @@ class TestDefConceptEvaluator(TestUsingMemoryBasedSheerka):
         ret_val = self.get_concept_part(expression)
         context = self.get_context()
 
-        assert DefConceptEvaluator.get_variables(context.sheerka, ret_val, name.split()) == expected
+        assert DefConceptEvaluator.get_variables(context, ret_val, name.split()) == expected
 
     def test_i_can_get_variables_when_keywords(self):
         sheerka, context = self.init_concepts()
@@ -204,20 +217,20 @@ class TestDefConceptEvaluator(TestUsingMemoryBasedSheerka):
         name_to_use = DefConceptEvaluator.get_name_to_use(def_concept)
         concept_part = self.get_concept_part("pre")
 
-        assert DefConceptEvaluator.get_variables(context.sheerka, concept_part, name_to_use) == {"pre"}
+        assert DefConceptEvaluator.get_variables(context, concept_part, name_to_use) == {"pre"}
 
     def test_i_cannot_get_variables_from_python_node_when_name_has_only_one_token(self):
         ret_val = self.get_concept_part("isinstance(a, str)")
         context = self.get_context()
 
-        assert DefConceptEvaluator.get_variables(context.sheerka, ret_val, ["a"]) == []
+        assert DefConceptEvaluator.get_variables(context, ret_val, ["a"]) == set()
 
     def test_i_can_get_variables_from_definition(self):
         parsing_expression = Sequence(ConceptExpression('mult'),
                                       ZeroOrMore(Sequence(StrMatch("+"), ConceptExpression("add"))))
         ret_val = self.get_return_value("mult (('+'|'-') add)?", parsing_expression)
 
-        assert DefConceptEvaluator.get_variables(self.get_sheerka(), ret_val, []) == {"add", "mult"}
+        assert DefConceptEvaluator.get_variables(self.get_context(), ret_val, []) == {"add", "mult"}
 
     def test_concept_that_references_itself_is_correctly_created(self):
         context = self.get_context()

tests/parsers/test_BaseNodeParser.py

@@ -1,6 +1,6 @@
 import pytest
 from core.concept import Concept
-from parsers.BaseNodeParser import BaseNodeParser
+from parsers.BaseNodeParser import BaseNodeParser, NoFirstTokenError
 from parsers.BnfNodeParser import StrMatch, Sequence, OrderedChoice, Optional, ZeroOrMore, OneOrMore, ConceptExpression
 
 from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
@@ -115,6 +115,13 @@ class TestBaseNodeParser(TestUsingMemoryBasedSheerka):
             "qux": ["1005"],
         }
 
+    def test_i_cannot_get_concept_by_first_keyword_when_no_first_keyword(self):
+        sheerka, context, foo = self.init_concepts(Concept("x y", body="x y").def_var("x").def_var("y"))
+        res = BaseNodeParser.get_concepts_by_first_token(context, [foo])
+
+        assert not res.status
+        assert res.body == NoFirstTokenError(foo, foo.key)
+
     def test_i_can_resolve_concepts_by_first_keyword(self):
         sheerka, context, *updated = self.init_concepts(
             "one",