ExactConceptParser can now recognize concepts by their names

src/core/builtin_helpers.py

@@ -470,3 +470,19 @@ def remove_from_ret_val(sheerka, return_values, concept_key):
         return_values.remove(item)
 
     return return_values
+
+
+def set_is_evaluated(concepts):
+    """
+    set is_evaluated to True
+    :param concepts:
+    :return:
+    """
+    if concepts is None:
+        return
+
+    if hasattr(concepts, "__iter__"):
+        for c in concepts:
+            c.metadata.is_evaluated = True
+    else:
+        concepts.metadata.is_evaluated = True

src/core/concept.py

@@ -115,10 +115,7 @@ class Concept:
         if isinstance(other, simplec):
             return self.name == other.name and self.body == other.body
 
-        if isinstance(other, CC):
+        if isinstance(other, (CC, CB, CMV)):
-            return other == self
-
-        if isinstance(other, CB):
             return other == self
 
         if not isinstance(other, Concept):
@@ -601,5 +598,60 @@ class CB:
     def __hash__(self):
         return hash((self.concept, self.body))
 
+    def __repr__(self):
+        return f"CB({self.body})"
+
+
+class CMV:
+    """
+    Concept with metadata variables
+    CMV stands for Concept Metadata Variables
+    Test class that only compare the key and the metadata variables
+    """
+
+    def __init__(self, concept, **kwargs):
+        self.concept_key = concept.key if isinstance(concept, Concept) else concept
+        self.concept = concept if isinstance(concept, Concept) else None
+        self.variables = kwargs
+
+    def __eq__(self, other):
+        if id(self) == id(other):
+            return True
+
+        if isinstance(other, Concept):
+            if other.key != self.concept_key:
+                return False
+
+            if len(other.metadata.variables) != len(self.variables):
+                return False
+
+            for name, value in other.metadata.variables:
+                if self.variables[name] != value:
+                    return False
+            return True
+
+        if not isinstance(other, CMV):
+            return False
+
+        if self.concept_key != other.concept_key:
+            return False
+
+        return self.variables == other.variables
+
+    def __hash__(self):
+        if self.concept:
+            return hash(self.concept)
+        return hash(self.concept_key)
+
+    def __repr__(self):
+        if self.concept:
+            txt = f"CMV(concept='{self.concept}'"
+        else:
+            txt = f"CMV(concept_key='{self.concept_key}'"
+
+        for k, v in self.variables.items():
+            txt += f", {k}='{v}'"
+        return txt + ")"
+
 
 simplec = namedtuple("concept", "name body")  # for simple concept (tests purposes only)

src/core/sheerka/Sheerka.py

@@ -491,8 +491,14 @@ class Sheerka(Concept):
         return self._get_unknown(metadata)
 
     def resolve(self, concept):
+
+        def new_instances(concepts):
+            if hasattr(concepts, "__iter__"):
+                return [self.new_from_template(c, c.key) for c in concepts]
+            return self.new_from_template(concepts, concepts.key)
+
         if concept is None:
-            return concept
+            return None
 
         # if the entry is a concept token, use its values.
         if isinstance(concept, Token):
@@ -500,24 +506,34 @@ class Sheerka(Concept):
                 return None
             concept = concept.value
 
+        if isinstance(concept, str) and \
+            concept.startswith("c:") and \
+            (tmp := core.utils.unstr_concept(concept)) != (None, None):
+            concept = tmp
+
         # if the entry is a tuple
         # concept[0] is the name
         # concept[1] is the id
         if isinstance(concept, tuple):
             if concept[1]:
                 if self.is_known(found := self.get_by_id(concept[1])):
-                    return found
+                    instance = self.new_from_template(found, found.key)
+                    instance.metadata.is_evaluated = True
+                    return instance
             elif concept[0]:
-                return found if self.is_known(found := self.get_by_name(concept[0])) else None
+                if self.is_known(found := self.get_by_name(concept[0])):
+                    instances = new_instances(found)
+                    core.builtin_helpers.set_is_evaluated(instances)
+                    return instances
             else:
                 return None
 
         # otherwise search in db
         if isinstance(concept, str):
-            if self.is_known(found := self.get_by_id(concept)):
-                return found
             if self.is_known(found := self.get_by_name(concept)):
-                return found
+                instances = new_instances(found)
+                core.builtin_helpers.set_is_evaluated(instances)
+                return instances
 
         return None
 

src/core/sheerka/services/SheerkaEvaluateConcept.py

@@ -2,6 +2,7 @@ from core.builtin_concepts import BuiltinConcepts
 from core.builtin_helpers import expect_one, only_successful
 from core.concept import Concept, DoNotResolve, ConceptParts, InfiniteRecursionResolved
 from core.sheerka.services.sheerka_service import BaseService
+from core.utils import unstr_concept
 
 CONCEPT_EVALUATION_STEPS = [
     BuiltinConcepts.BEFORE_EVALUATION,
@@ -73,7 +74,6 @@ class SheerkaEvaluateConcept(BaseService):
         Basically, it runs the parsers on all parts
         :param concept:
         :param context:
-        :param logger:
         :return:
         """
 
@@ -82,6 +82,11 @@ class SheerkaEvaluateConcept(BaseService):
             return context.sheerka.isinstance(r, BuiltinConcepts.RETURN_VALUE) and \
                    context.sheerka.isinstance(r.body, BuiltinConcepts.ONLY_SUCCESSFUL)
 
+        def parse_token_concept(s):
+            if s.startswith("c:") and (identifier := unstr_concept(s)) != (None, None):
+                return self.sheerka.resolve(identifier)
+            return None
+
         steps = [BuiltinConcepts.BEFORE_PARSING, BuiltinConcepts.PARSING, BuiltinConcepts.AFTER_PARSING]
         for part_key in ConceptParts:
             if part_key in concept.compiled:
@@ -93,6 +98,11 @@ class SheerkaEvaluateConcept(BaseService):
 
             if source.strip() == "":
                 concept.compiled[part_key] = DoNotResolve(source)
+            else:
+                # first case, when the metadata references another concept via c:xxx: keyword
+                if concept_found := parse_token_concept(source):
+                    context.log(f"Recognized concept '{concept_found}'", self.NAME)
+                    concept.compiled[part_key] = concept_found
                 else:
                     with context.push(desc=f"Initializing *compiled* for {part_key}") as sub_context:
                         sub_context.add_inputs(source=source)
@@ -112,6 +122,11 @@ class SheerkaEvaluateConcept(BaseService):
 
             if default_value.strip() == "":
                 concept.compiled[var_name] = DoNotResolve(default_value)
+            else:
+                # first case, when the metadata references another concept via c:xxx: keyword
+                if concept_found := parse_token_concept(default_value):
+                    context.log(f"Recognized concept '{concept_found}'", self.NAME)
+                    concept.compiled[var_name] = concept_found
                 else:
                     with context.push(desc=f"Initializing *compiled* for property {var_name}") as sub_context:
                         sub_context.add_inputs(source=default_value)

src/core/utils.py

@@ -288,7 +288,7 @@ def decode_enum(enum_repr: str):
         return None
 
 
-def str_concept(t, skip_key=None):
+def str_concept(t, drop_name=None):
     """
     The key,id identifiers of a concept are stored in a tuple
     we want to return the key and the id, separated by a pipe
@@ -298,21 +298,21 @@ def str_concept(t, skip_key=None):
     >>> assert str_concept((None, "id")) == "c:|id:"
     >>> assert str_concept(("key", None)) == "c:key:"
     >>> assert str_concept((None, None)) == ""
-    >>> assert str_concept(Concept(key="foo", id="bar")) == "c:foo|bar:"
+    >>> assert str_concept(Concept(name="foo", id="bar")) == "c:foo|bar:"
-    >>> assert str_concept(Concept(key="foo", id="bar"), skip_key=True) == "c:|bar:"
+    >>> assert str_concept(Concept(name="foo", id="bar"), drop_name=True) == "c:|bar:"
     :param t:
-    :param skip_key: True if we only want the id (and not the key)
+    :param drop_name: True if we only want the id (and not the key)
     :return:
     """
     if isinstance(t, tuple):
-        key, id_ = t[0], t[1]
+        name, id_ = t[0], t[1]
     else:
-        key, id_ = t.key, t.id
+        name, id_ = t.key, t.id
 
-    if key is None and id_ is None:
+    if name is None and id_ is None:
         return ""
 
-    result = 'c:' if (key is None or skip_key) else "c:" + key
+    result = 'c:' if (name is None or drop_name) else "c:" + name
     if id_:
         result += "|" + id_
     return result + ":"

src/parsers/BaseParser.py

@@ -255,6 +255,26 @@ class BaseParser:
 
         return start, end
 
+    @staticmethod
+    def merge_concepts(list_a, b):
+        if not b:
+            return list_a
+
+        list_b = b if isinstance(b, list) else [b]
+
+        if not list_a:
+            return list_b
+
+        by_ids = {c.id for c in list_b}
+        for c in list_b:
+            if c.id in by_ids:  # and c.metadata.is_evaluated == by_ids[c.id].metadata.is_evaluated:
+                continue
+
+            list_a.append(c)
+            by_ids.add(c.id)
+
+        return list_a
+
 
 class BaseTokenizerIterParser(BaseParser):
 

src/parsers/BnfNodeParser.py

@@ -455,7 +455,7 @@ class BnfNodeFirstTokenVisitor(ParsingExpressionVisitor):
     def visit_ConceptExpression(self, pe):
         concept = self.sheerka.get_by_key(pe.concept) if isinstance(pe.concept, str) else pe.concept
         if self.sheerka.is_known(concept):
-            self.add_first_token(core.utils.str_concept(concept, skip_key=True))
+            self.add_first_token(core.utils.str_concept(concept, drop_name=True))
         return self.STOP
 
     def visit_StrMatch(self, pe):

src/parsers/ExactConceptParser.py

@@ -3,7 +3,9 @@ import logging
 from core.builtin_concepts import ReturnValueConcept, BuiltinConcepts
 from core.concept import VARIABLE_PREFIX
 from core.tokenizer import Keywords, TokenKind, LexerError
+from core.utils import str_concept
 from parsers.BaseParser import BaseParser
+import core.builtin_helpers
 
 
 class ExactConceptParser(BaseParser):
@@ -26,8 +28,8 @@ class ExactConceptParser(BaseParser):
         """
 
         context.log(f"Parsing '{parser_input}'", self.name)
-        res = []
         sheerka = context.sheerka
+
         try:
             words = self.get_words(parser_input)
         except LexerError as e:
@@ -40,7 +42,7 @@ class ExactConceptParser(BaseParser):
             body = sheerka.new(BuiltinConcepts.NOT_FOR_ME, body=parser_input, reason=too_long)
             return sheerka.ret(self.name, False, body)
 
-        recognized = []  # keep track of the concepts founds
+        already_recognized = []  # keep track of the concepts founds
         for combination in self.combinations(words):
 
             concept_key = " ".join(combination)
@@ -52,7 +54,7 @@ class ExactConceptParser(BaseParser):
             concepts = result if isinstance(result, list) else [result]
 
             for concept in concepts:
-                if concept.id in recognized:
+                if concept in already_recognized:
                     context.log(f"Recognized concept {concept} again. Skipping.", self.name)
                     # example
                     # if the input is foo a and a concept is defined as foo a
@@ -65,37 +67,32 @@ class ExactConceptParser(BaseParser):
                 for i, token in enumerate(combination):
                     if token.startswith(VARIABLE_PREFIX):
                         index = int(token[len(VARIABLE_PREFIX):])
-                        concept.def_var_by_index(index, words[i])
+                        value = words[i]
+                        concept.def_var_by_index(index, str_concept(value) if isinstance(value, tuple) else value)
                         concept.metadata.need_validation = True
                         if self.verbose_log.isEnabledFor(logging.DEBUG):
                             prop_name = concept.metadata.variables[index][0]
                             context.log(
-                                f"Added property {index}: {prop_name}='{words[i]}'.",
+                                f"Added variable {index}: {prop_name}='{words[i]}'.",
                                 self.name)
 
-                res.append(ReturnValueConcept(
+                already_recognized.append(concept)
-                    self.name,
-                    True,
-                    context.sheerka.new(
-                        BuiltinConcepts.PARSER_RESULT,
-                        parser=self,
-                        source=parser_input if isinstance(parser_input, str) else self.get_text_from_tokens(
-                            parser_input),
-                        body=concept,
-                        try_parsed=concept)))
-                recognized.append(concept.id)
 
-        if len(recognized) > 0:
+        by_name = sheerka.resolve(self.get_input_as_text(parser_input))
+        core.builtin_helpers.set_is_evaluated(by_name)
+        recognized = self.merge_concepts(already_recognized, by_name)
+
+        if len(recognized) == 0:
+            ret = sheerka.ret(self.name, False, sheerka.new(BuiltinConcepts.UNKNOWN_CONCEPT, body=parser_input))
+            self.log_result(context, parser_input, ret)
+            return ret
+        else:
+            res = [self.as_return_value(context, parser_input, c) for c in recognized]
             if len(res) == 1:
                 self.log_result(context, parser_input, res[0])
             else:
                 self.log_multiple_results(context, parser_input, res)
             return res
-            return res
-
-        ret = sheerka.ret(self.name, False, sheerka.new(BuiltinConcepts.UNKNOWN_CONCEPT, body=parser_input))
-        self.log_result(context, parser_input, ret)
-        return ret
 
     def get_words(self, text):
         tokens = self.get_input_as_tokens(text)
@@ -138,7 +135,17 @@ class ExactConceptParser(BaseParser):
                     indices[j] = indices[j - 1] + 1
                 res.add(self.get_tuple(pool, indices))
 
-        return res
+        # remove all result that contains a token concepts
+        # They are not valid entries, since a token concept MUST be replaced by a variable
+        filtered = set()
+        for combination in res:
+            for entry in combination:
+                if isinstance(entry, tuple):
+                    break
+            else:
+                filtered.add(combination)
+
+        return filtered
 
     @staticmethod
     def get_tuple(pool, indices):
@@ -158,3 +165,14 @@ class ExactConceptParser(BaseParser):
             value = pool[i]
             res.append(vars[value] if value in vars else value)
         return tuple(res)
+
+    def as_return_value(self, context, parser_input, concept):
+        return ReturnValueConcept(
+            self.name,
+            True,
+            context.sheerka.new(
+                BuiltinConcepts.PARSER_RESULT,
+                parser=self,
+                source=parser_input if isinstance(parser_input, str) else self.get_text_from_tokens(parser_input),
+                body=concept,
+                try_parsed=concept))

tests/BaseTest.py

@@ -98,7 +98,15 @@ class BaseTest:
                                 try_parsed=concept))
 
     @staticmethod
-    def create_concept_lite(sheerka, name, variables=None, bnf=None):
+    def create_and_add_in_cache_concept(sheerka, name, variables=None, bnf=None):
+        """
+        Create a concept using parameters and add it in cache
+        :param sheerka:
+        :param name:
+        :param variables:
+        :param bnf:
+        :return:
+        """
         concept = Concept(name) if isinstance(name, str) else name
         if variables:
             for v in variables:
@@ -124,7 +132,7 @@ class BaseTest:
         return concept
 
     @staticmethod
-    def def_concept(name, definition, variables=None, **kwargs):
+    def from_def_concept(name, definition, variables=None, **kwargs):
         concept = Concept(name=name, definition=definition, definition_type=DEFINITION_TYPE_DEF)
         if variables:
             for v in variables:

tests/core/test_SheerkaCreateNewConcept.py

@@ -142,7 +142,7 @@ class TestSheerkaCreateNewConcept(TestUsingMemoryBasedSheerka):
     def test_i_can_get_by_name_when_created_with_def_definition(self):
         sheerka = self.get_sheerka(cache_only=False)
         context = self.get_context(sheerka)
-        concept = self.def_concept("plus", "a plus b", ["a", "b"])
+        concept = self.from_def_concept("plus", "a plus b", ["a", "b"])
 
         res = sheerka.create_new_concept(context, concept)
 

tests/core/test_SheerkaEvaluateConcept.py

@@ -96,7 +96,7 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
         assert evaluated.body == "do not resolve"
         assert evaluated.metadata.is_evaluated
 
-    def test_i_can_evaluate_property_using_do_not_resolve(self):
+    def test_i_can_evaluate_variable_using_do_not_resolve(self):
         sheerka, context, concept = self.init_concepts(Concept("foo").def_var("a"), eval_body=True)
         concept.compiled["a"] = DoNotResolve("do not resolve")
 
@@ -181,7 +181,7 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
         assert sheerka.objvalue(evaluated) == CB("a", None)
         assert evaluated.metadata.is_evaluated
 
-    def test_i_can_evaluate_concept_when_properties_reference_others_concepts(self):
+    def test_i_can_evaluate_concept_when_variables_reference_others_concepts(self):
         sheerka, context, concept_a, concept = self.init_concepts(
             Concept("a"),
             Concept("foo", body="a").def_var("a", "a"),
@@ -194,7 +194,7 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
         assert evaluated.key == concept.key
         assert evaluated.body == concept_a
 
-    def test_i_can_evaluate_concept_when_properties_reference_others_concepts_2(self):
+    def test_i_can_evaluate_concept_when_variables_reference_others_concepts_2(self):
         """
         Same test,
         but the name of the property and the name of the concept are different
@@ -208,7 +208,7 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
         assert evaluated.key == concept.key
         assert evaluated.body == concept_a
 
-    def test_i_can_evaluate_concept_when_properties_reference_others_concepts_with_body(self):
+    def test_i_can_evaluate_concept_when_variables_reference_others_concepts_with_body(self):
         sheerka, context, *concepts = self.init_concepts(
             Concept(name="a", body="1"),
             Concept(name="b", body="2"),
@@ -221,7 +221,7 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
         assert evaluated.key == concept.key
         assert evaluated.body == 3
 
-    def test_i_can_evaluate_concept_when_properties_is_a_concept(self):
+    def test_i_can_evaluate_concept_when_variables_is_a_concept(self):
         sheerka, context, concept_a = self.init_concepts(Concept(name="a", body="'a'"), eval_body=True)
 
         concept = Concept("foo").def_var("a")
@@ -231,7 +231,29 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
         assert evaluated.key == concept.key
         assert evaluated.get_value("a") == CB("a", "a")
 
-    def test_i_can_evaluate_when_property_asts_is_a_list(self):
+    def test_i_can_evaluate_concept_when_variable_is_a_concept_token(self):
+        sheerka, context, concept_a, concept_b = self.init_concepts(
+            "a",
+            Concept("b").def_var("var_name", "c:a:"),  # c:a: means concept 'a'
+            eval_body=True)
+
+        evaluated = sheerka.evaluate_concept(context, concept_b)
+
+        assert evaluated.key == concept_b.key
+        assert evaluated.get_value("var_name") == concept_a
+
+    def test_i_can_evaluate_concept_when_body_isa_concept_token(self):
+        sheerka, context, concept_a, concept_b = self.init_concepts(
+            "a",
+            Concept("b", body="c:a:"),  # c:a: means concept 'a'
+            eval_body=True)
+
+        evaluated = sheerka.evaluate_concept(context, concept_b)
+
+        assert evaluated.key == concept_b.key
+        assert evaluated.body == concept_a
+
+    def test_i_can_evaluate_when_variable_asts_is_a_list(self):
         sheerka = self.get_sheerka()
         foo = Concept("foo", body="1")
 
@@ -270,7 +292,7 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
         assert evaluated.key == concept.key
         assert evaluated.body == sheerka.test()
 
-    def test_properties_values_takes_precedence_over_the_outside_world(self):
+    def test_variables_values_takes_precedence_over_the_outside_world(self):
         sheerka, context, concept_a, concept_b = self.init_concepts(
             Concept(name="a", body="'concept_a'"),
             Concept(name="b", body="'concept_b'"),
@@ -294,7 +316,7 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
         assert evaluated.key == concept.key
         assert evaluated.body == CB("b", "concept_b")
 
-    def test_properties_values_takes_precedence(self):
+    def test_variables_values_takes_precedence(self):
         sheerka, context, concept_a, concept_b = self.init_concepts(
             Concept(name="a", body="'concept_a'"),
             Concept(name="b", body="'concept_b'"),
@@ -306,7 +328,7 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
         assert evaluated.key == concept.key
         assert evaluated.body == 'prop_aconcept_b'
 
-    def test_i_can_reference_sub_property_of_a_property(self):
+    def test_i_can_reference_sub_property_of_a_variable(self):
         sheerka, context, concept_a = self.init_concepts(
             Concept(name="concept_a").def_var("subProp", "'sub_a'"),
             eval_body=True
@@ -316,7 +338,7 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
         evaluated = sheerka.evaluate_concept(context, concept)
         assert evaluated == CB(concept.key, "sub_a")
 
-    def test_i_cannot_evaluate_concept_if_property_is_in_error(self):
+    def test_i_cannot_evaluate_concept_if_variable_is_in_error(self):
         sheerka = self.get_sheerka()
 
         concept = Concept(name="concept_a").def_var("subProp", "undef_concept")

tests/core/test_sheerka.py

@@ -279,7 +279,6 @@ class TestSheerkaUsingMemoryBasedSheerka(TestUsingMemoryBasedSheerka):
         (None, None),
         ("foo", ["foo", "foo2"]),
         ("bar", "bar"),
-        ("1001", "foo"),  # by id take precedence over by name
         ("plus", "plus"),
         ("a mult b", "mult"),
         ("unknown", None),
@@ -298,13 +297,17 @@ class TestSheerkaUsingMemoryBasedSheerka(TestUsingMemoryBasedSheerka):
         (Token(TokenKind.CONCEPT, (None, None), 0, 0, 0), None),
         (Token(TokenKind.CONCEPT, ("foo", None), 0, 0, 0), ["foo", "foo2"]),
 
+        # by concept token str
+        ("c:foo:", ["foo", "foo2"]),
+        ("c:unknown:", None),
+
     ])
     def test_i_can_resolve_concept(self, concept, expected):
         sheerka, context, *concepts = self.init_concepts(
             "foo",
             Concept("foo", body="another one"),
             "bar",
-            self.def_concept("plus", "a plus b", ["a", "b"]),
+            self.from_def_concept("plus", "a plus b", ["a", "b"]),
             Concept("a mult b").def_var("a").def_var("b"),
             Concept("1001"),
         )
@@ -319,7 +322,7 @@ class TestSheerkaUsingMemoryBasedSheerka(TestUsingMemoryBasedSheerka):
 
     def test_i_can_resolve_when_searching_by_definition(self):
         sheerka, context, plus = self.init_concepts(
-            self.def_concept("plus", "a plus b", ["a", "b"]),
+            self.from_def_concept("plus", "a plus b", ["a", "b"]),
             create_new=True
         )
 

tests/core/test_utils.py

@@ -170,14 +170,14 @@ def test_i_can_str_concept():
     assert core.utils.str_concept((None, "id")) == "c:|id:"
     assert core.utils.str_concept(("key", None)) == "c:key:"
     assert core.utils.str_concept((None, None)) == ""
-    assert core.utils.str_concept(("key", "id"), skip_key=True) == "c:|id:"
+    assert core.utils.str_concept(("key", "id"), drop_name=True) == "c:|id:"
 
     concept = Concept("foo").init_key()
     assert core.utils.str_concept(concept) == "c:foo:"
 
     concept.metadata.id = "1001"
     assert core.utils.str_concept(concept) == "c:foo|1001:"
-    assert core.utils.str_concept(concept, skip_key=True) == "c:|1001:"
+    assert core.utils.str_concept(concept, drop_name=True) == "c:|1001:"
 
 
 @pytest.mark.parametrize("text, expected", [

tests/evaluators/test_PythonEvaluator.py

@@ -137,8 +137,8 @@ class TestPythonEvaluator(TestUsingMemoryBasedSheerka):
 
     def test_i_can_call_function_with_complex_concepts(self):
         sheerka, context, plus, mult = self.init_concepts(
-            self.def_concept("plus", "a plus b", ["a", "b"]),
+            self.from_def_concept("plus", "a plus b", ["a", "b"]),
-            self.def_concept("mult", "a mult b", ["a", "b"]),
+            self.from_def_concept("mult", "a mult b", ["a", "b"]),
         )
 
         parsed = PythonParser().parse(context, "is_greater_than(BuiltinConcepts.PRECEDENCE, mult, plus)")

tests/non_reg/test_sheerka_non_reg.py

@@ -1,6 +1,6 @@
 import pytest
 from core.builtin_concepts import BuiltinConcepts
-from core.concept import Concept, PROPERTIES_TO_SERIALIZE, simplec
+from core.concept import Concept, PROPERTIES_TO_SERIALIZE, simplec, CMV
 from evaluators.MutipleSameSuccessEvaluator import MultipleSameSuccessEvaluator
 from parsers.BaseNodeParser import SyaAssociativity
 from parsers.BnfNodeParser import Sequence, StrMatch, OrderedChoice, Optional, ConceptExpression
@@ -226,9 +226,9 @@ as:
 
     def test_i_can_recognize_duplicate_concepts_with_same_value(self):
         sheerka = self.get_sheerka()
-        self.create_concept_lite(sheerka, Concept(name="hello a", body="'hello ' + a"), variables=["a"])
+        self.create_and_add_in_cache_concept(sheerka, Concept(name="hello a", body="'hello ' + a"), variables=["a"])
-        self.create_concept_lite(sheerka, Concept(name="hello foo", body="'hello foo'"))
+        self.create_and_add_in_cache_concept(sheerka, Concept(name="hello foo", body="'hello foo'"))
-        self.create_concept_lite(sheerka, Concept(name="foo", body="'foo'"))
+        self.create_and_add_in_cache_concept(sheerka, Concept(name="foo", body="'foo'"))
 
         res = sheerka.evaluate_user_input("hello foo")
         assert len(res) == 1
@@ -238,9 +238,9 @@ as:
 
     def test_i_cannot_manage_duplicate_concepts_when_the_values_are_different(self):
         sheerka = self.get_sheerka()
-        self.create_concept_lite(sheerka, Concept(name="hello a", body="'hello ' + a"), variables=["a"])
+        self.create_and_add_in_cache_concept(sheerka, Concept(name="hello a", body="'hello ' + a"), variables=["a"])
-        self.create_concept_lite(sheerka, Concept(name="hello foo", body="'hello foo'"))
+        self.create_and_add_in_cache_concept(sheerka, Concept(name="hello foo", body="'hello foo'"))
-        self.create_concept_lite(sheerka, Concept(name="foo", body="'another value'"))
+        self.create_and_add_in_cache_concept(sheerka, Concept(name="foo", body="'another value'"))
 
         res = sheerka.evaluate_user_input("hello foo")
         assert len(res) == 1
@@ -894,9 +894,31 @@ as:
         res = sheerka.evaluate_user_input("eval four > three")
         assert res[0].status
 
-        res = sheerka.evaluate_user_input("get_concepts_weights('some_prop')")
+        assert sheerka.get_concepts_weights("some_prop") == {'1001': 1, '1002': 2, '1003': 3, '1004': 4}
+
+    def test_i_can_evaluate_expression_when_using_token_concept(self):
+        sheerka, context, one, two, plus = self.init_concepts(
+            Concept("one", body="1"),
+            Concept("two", body="2"),
+            self.from_def_concept("<", "a < b", ["a", "b"], body="is_less_than('some_prop', a, b)")
+        )
+
+        expression = "c:one: < c:two:"
+        res = sheerka.evaluate_user_input(expression)
         assert res[0].status
-        assert res[0].body == {'1001': 1, '1002': 2, '1003': 3, '1004': 4}
+        assert res[0].body == CMV(plus, a="c:one:", b="c:two:")
+        assert res[0].body.a is None  # concept is not evaluated
+        assert res[0].body.b is None  # concept is not evaluated
+
+        expression = "eval c:one: < c:two:"
+        res = sheerka.evaluate_user_input(expression)
+        assert res[0].status
+        assert sheerka.isinstance(res[0].body, BuiltinConcepts.SUCCESS)
+        assert sheerka.get_concepts_weights("some_prop") == {'1001': 1, '1002': 2}
+
+        expression = "eval one < two"
+        res = sheerka.evaluate_user_input(expression)
+        assert not res[0].status
 
 
 class TestSheerkaNonRegFile(TestUsingFileBasedSheerka):

tests/parsers/test_BaseNodeParser.py

@@ -163,10 +163,10 @@ class TestBaseNodeParser(TestUsingMemoryBasedSheerka):
     def test_concepts_are_defined_once(self):
         sheerka = self.get_sheerka()
         context = self.get_context(sheerka)
-        good = self.create_concept_lite(sheerka, "good")
+        good = self.create_and_add_in_cache_concept(sheerka, "good")
-        foo = self.create_concept_lite(sheerka, "foo", bnf=ConceptExpression("good"))
+        foo = self.create_and_add_in_cache_concept(sheerka, "foo", bnf=ConceptExpression("good"))
-        bar = self.create_concept_lite(sheerka, "bar", bnf=ConceptExpression("good"))
+        bar = self.create_and_add_in_cache_concept(sheerka, "bar", bnf=ConceptExpression("good"))
-        baz = self.create_concept_lite(sheerka, "baz", bnf=OrderedChoice(
+        baz = self.create_and_add_in_cache_concept(sheerka, "baz", bnf=OrderedChoice(
             ConceptExpression("foo"),
             ConceptExpression("bar")))
 
@@ -183,8 +183,8 @@ class TestBaseNodeParser(TestUsingMemoryBasedSheerka):
         sheerka = self.get_sheerka()
         context = self.get_context(sheerka)
 
-        a = self.create_concept_lite(sheerka, "a", bnf=Sequence("one", "two"))
+        a = self.create_and_add_in_cache_concept(sheerka, "a", bnf=Sequence("one", "two"))
-        b = self.create_concept_lite(sheerka, "b", bnf=Sequence(ConceptExpression("a"), "two"))
+        b = self.create_and_add_in_cache_concept(sheerka, "b", bnf=Sequence(ConceptExpression("a"), "two"))
 
         concepts_by_first_keywords = BaseNodeParser.get_concepts_by_first_keyword(
             context, [a, b]).body

tests/parsers/test_ExactConceptParser.py

@@ -1,5 +1,5 @@
 from core.builtin_concepts import BuiltinConcepts
-from core.concept import Concept
+from core.concept import Concept, CMV
 from core.tokenizer import Tokenizer
 from parsers.ExactConceptParser import ExactConceptParser
 
@@ -49,10 +49,10 @@ class TestExactConceptParser(TestUsingMemoryBasedSheerka):
                        ('__var__1', '__var__0', '__var__1')}
         # TODO: the last tuple is not possible, so the algo can be improved
 
-    def test_i_can_recognize_a_simple_concept(self):
+    def test_i_can_parse_a_simple_concept(self):
         sheerka = self.get_sheerka(singleton=True)
         context = self.get_context(sheerka)
-        concept = self.create_concept_lite(sheerka, "hello world")
+        concept = self.create_and_add_in_cache_concept(sheerka, "hello world")
 
         source = "hello world"
         results = ExactConceptParser().parse(context, source)
@@ -62,12 +62,13 @@ class TestExactConceptParser(TestUsingMemoryBasedSheerka):
         assert results[0].status
         assert concept_found == concept
         assert not concept_found.metadata.need_validation
+        assert not concept_found.metadata.is_evaluated
 
-    def test_i_can_recognize_concepts_defined_several_times(self):
+    def test_i_can_parse_concepts_defined_several_times(self):
         sheerka = self.get_sheerka(singleton=True)
         context = self.get_context(sheerka)
-        self.create_concept_lite(sheerka, "hello world")
+        self.create_and_add_in_cache_concept(sheerka, "hello world")
-        self.create_concept_lite(sheerka, "hello a", variables=["a"])
+        self.create_and_add_in_cache_concept(sheerka, "hello a", variables=["a"])
 
         source = "hello world"
         results = ExactConceptParser().parse(context, source)
@@ -84,10 +85,10 @@ class TestExactConceptParser(TestUsingMemoryBasedSheerka):
         assert results[1].value.value.name == "hello world"
         assert not results[1].value.value.metadata.need_validation
 
-    def test_i_can_recognize_a_concept_with_variables(self):
+    def test_i_can_parse_a_concept_with_variables(self):
         sheerka = self.get_sheerka(singleton=True)
         context = self.get_context(sheerka)
-        concept = self.create_concept_lite(sheerka, "a + b", ["a", "b"])
+        concept = self.create_and_add_in_cache_concept(sheerka, "a + b", ["a", "b"])
 
         source = "10 + 5"
         results = ExactConceptParser().parse(context, source)
@@ -96,15 +97,14 @@ class TestExactConceptParser(TestUsingMemoryBasedSheerka):
         assert results[0].status
 
         concept_found = results[0].value.value
-        assert concept_found.key == concept.key
+        assert concept_found == CMV(concept, a="10", b="5")
-        assert variable_def(concept_found, "a") == "10"
-        assert variable_def(concept_found, "b") == "5"
         assert concept_found.metadata.need_validation
+        assert not concept_found.metadata.is_evaluated
 
-    def test_i_can_recognize_a_concept_with_duplicate_variables(self):
+    def test_i_can_parse_a_concept_with_duplicate_variables(self):
         sheerka = self.get_sheerka(singleton=True)
         context = self.get_context(sheerka)
-        concept = self.create_concept_lite(sheerka, "a + b + a", ["a", "b"])
+        concept = self.create_and_add_in_cache_concept(sheerka, "a + b + a", ["a", "b"])
 
         source = "10 + 5 + 10"
         results = ExactConceptParser(max_word_size=5).parse(context, source)
@@ -113,11 +113,54 @@ class TestExactConceptParser(TestUsingMemoryBasedSheerka):
         assert results[0].status
 
         concept_found = results[0].value.value
-        assert concept_found.key == concept.key
+        assert concept_found == CMV(concept, a="10", b="5")
-        assert variable_def(concept_found, "a") == "10"
-        assert variable_def(concept_found, "b") == "5"
         assert concept_found.metadata.need_validation
 
+    def test_i_can_parse_concept_when_defined_using_from_def(self):
+        sheerka, context, plus = self.init_concepts(
+            self.from_def_concept("+", "a + b", ["a", "b"])
+        )
+
+        source = "10 + 5"
+        results = ExactConceptParser().parse(context, source)
+        concept_found = results[0].value.value
+
+        assert len(results) == 1
+        assert results[0].status
+        assert concept_found == CMV(plus, a="10", b="5")
+        assert concept_found.metadata.need_validation
+        assert not concept_found.metadata.is_evaluated
+
+    def test_i_can_parse_concept_token(self):
+        sheerka, context, foo = self.init_concepts("foo")
+
+        source = "c:foo:"
+        results = ExactConceptParser().parse(context, source)
+        concept_found = results[0].value.value
+
+        assert len(results) == 1
+        assert results[0].status
+        assert concept_found == foo
+        assert not concept_found.metadata.need_validation
+        assert concept_found.metadata.is_evaluated
+
+    def test_i_can_parse_concept_with_concept_tokens(self):
+        sheerka, context, one, two, plus = self.init_concepts(
+            "one",
+            "two",
+            self.from_def_concept("plus", "a plus b", ["a", "b"])
+        )
+
+        source = "c:one: plus c:two:"
+        results = ExactConceptParser().parse(context, source)
+        concept_found = results[0].value.value
+
+        assert len(results) == 1
+        assert results[0].status
+        assert concept_found == CMV(plus, a="c:one:", b="c:two:")
+        assert concept_found.metadata.need_validation
+        assert not concept_found.metadata.is_evaluated
+
     def test_i_can_manage_unknown_concept(self):
         context = self.get_context(self.get_sheerka(singleton=True))
         source = "def concept hello"  # this is not a concept by itself