Improved PythonEvaluator when dealing with concept class

src/core/concept.py

@@ -248,7 +248,7 @@ class Concept:
             if token.value in variables:
                 key += VARIABLE_PREFIX + str(variables.index(token.value))
             else:
-                #value = token.value[1:-1] if token.type == TokenKind.STRING else token.value
+                # value = token.value[1:-1] if token.type == TokenKind.STRING else token.value
                 key += token.value
             first = False
 
@@ -490,6 +490,16 @@ class InfiniteRecursionResolved:
         return self.value
 
 
+def ensure_concept(*concepts):
+    if hasattr(concepts, "__iter__"):
+        for concept in concepts:
+            if not isinstance(concept, Concept):
+                raise TypeError(f"'{concept}' must be a concept")
+    else:
+        if not isinstance(concepts, Concept):
+            raise TypeError(f"'{concepts}' must be a concept")
+
+
 # ################################
 #
 # Class created for tests purpose

src/core/sheerka/Sheerka.py

@@ -13,6 +13,7 @@ from core.builtin_concepts import BuiltinConcepts, ErrorConcept, ReturnValueConc
 from core.concept import Concept, ConceptParts, PROPERTIES_FOR_NEW
 from core.sheerka.ExecutionContext import ExecutionContext
 from core.sheerka_logger import console_handler
+from core.tokenizer import Token, TokenKind
 from printer.SheerkaPrinter import SheerkaPrinter
 from sdp.sheerkaDataProvider import SheerkaDataProvider, Event
 
@@ -489,6 +490,37 @@ class Sheerka(Concept):
         metadata = [(index_name, key), ("id", concept_id)] if concept_id else (index_name, key)
         return self._get_unknown(metadata)
 
+    def resolve(self, concept):
+        if concept is None:
+            return concept
+
+        # if the entry is a concept token, use its values.
+        if isinstance(concept, Token):
+            if concept.type != TokenKind.CONCEPT:
+                return None
+            concept = concept.value
+
+        # if the entry is a tuple
+        # concept[0] is the name
+        # concept[1] is the id
+        if isinstance(concept, tuple):
+            if concept[1]:
+                if self.is_known(found := self.get_by_id(concept[1])):
+                    return found
+            elif concept[0]:
+                return found if self.is_known(found := self.get_by_name(concept[0])) else None
+            else:
+                return None
+
+        # otherwise search in db
+        if isinstance(concept, str):
+            if self.is_known(found := self.get_by_id(concept)):
+                return found
+            if self.is_known(found := self.get_by_name(concept)):
+                return found
+
+        return None
+
     def has_id(self, concept_id):
         """
         Returns True if a concept with this id exists in cache

src/core/sheerka/services/SheerkaComparisonManager.py

@@ -3,6 +3,7 @@ from dataclasses import dataclass
 from cache.Cache import Cache
 from cache.ListCache import ListCache
 from core.builtin_concepts import BuiltinConcepts
+from core.concept import ensure_concept
 from core.sheerka.services.sheerka_service import ServiceObj, BaseService
 
 
@@ -105,6 +106,7 @@ class SheerkaComparisonManager(BaseService):
         :return:
         """
         context.log(f"Setting concept {concept_a} is greater than {concept_b}", who=self.NAME)
+        ensure_concept(concept_a, concept_b)
 
         event_digest = context.event.get_digest()
         comparison_obj = ComparisonObj(event_digest, prop_name, concept_a.id, concept_b.id, ">", comparison_context)
@@ -121,6 +123,7 @@ class SheerkaComparisonManager(BaseService):
         :return:
         """
         context.log(f"Setting concept {concept_a} is less than {concept_b}", who=self.NAME)
+        ensure_concept(concept_a, concept_b)
 
         event_digest = context.event.get_digest()
         comparison_obj = ComparisonObj(event_digest, prop_name, concept_a.id, concept_b.id, "<", comparison_context)

src/core/sheerka/services/SheerkaCreateNewConcept.py

@@ -1,6 +1,6 @@
 import core.utils
 from core.builtin_concepts import BuiltinConcepts, ErrorConcept
-from core.concept import Concept
+from core.concept import Concept, DEFINITION_TYPE_DEF, ensure_concept
 from core.sheerka.services.sheerka_service import BaseService
 from sdp.sheerkaDataProvider import SheerkaDataProviderDuplicateKeyError
 
@@ -30,6 +30,8 @@ class SheerkaCreateNewConcept(BaseService):
         :return: digest of the new concept
         """
 
+        ensure_concept(concept)
+
         sheerka = self.sheerka
 
         concept.init_key()
@@ -49,24 +51,29 @@ class SheerkaCreateNewConcept(BaseService):
         # set id before saving in db
         sheerka.set_id_if_needed(concept, False)
 
-        # update the dictionary of concepts by first key
+        # compute new concepts_by_first_keyword
         init_ret_value = self.bnp.get_concepts_by_first_keyword(context, [concept], True)
         if not init_ret_value.status:
             return sheerka.ret(self.NAME, False, ErrorConcept(init_ret_value.value))
         concepts_by_first_keyword = init_ret_value.body
 
-        # update resolved dictionary
+        # computes resolved concepts_by_first_keyword
         init_ret_value = self.bnp.resolve_concepts_by_first_keyword(context, concepts_by_first_keyword)
         if not init_ret_value.status:
             return sheerka.ret(self.NAME, False, ErrorConcept(init_ret_value.value))
         resolved_concepts_by_first_keyword = init_ret_value.body
 
-        concept.freeze_definition_hash()
+        # if everything is fine
 
+        concept.freeze_definition_hash()
         cache_manager.add_concept(concept)
         cache_manager.put(sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, False, concepts_by_first_keyword)
         cache_manager.put(sheerka.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY, False, resolved_concepts_by_first_keyword)
 
+        if concept.metadata.definition_type == DEFINITION_TYPE_DEF and concept.metadata.definition != concept.name:
+            # allow search by definition when definition relevant
+            cache_manager.put(self.sheerka.CONCEPTS_BY_NAME_ENTRY, concept.metadata.definition, concept)
+
         if concept.bnf and init_bnf_ret_value is not None and init_bnf_ret_value.status:
             sheerka.cache_manager.clear(sheerka.CONCEPTS_GRAMMARS_ENTRY)
 

src/core/sheerka/services/SheerkaModifyConcept.py

@@ -41,6 +41,7 @@ class SheerkaModifyConcept(BaseService):
         # TODO : update concept by first keyword
         # TODO : update resolved by first keyword
         # TODO : update concepts grammars
+        # TODO : update when definition_type = DEFINITION_TYPE_DEF
 
         ret = self.sheerka.ret(self.NAME, True, self.sheerka.new(BuiltinConcepts.NEW_CONCEPT, body=concept))
         return ret

src/core/sheerka/services/SheerkaSetsManager.py

@@ -2,7 +2,7 @@ import core.builtin_helpers
 from cache.SetCache import SetCache
 from core.ast.nodes import python_to_concept
 from core.builtin_concepts import BuiltinConcepts
-from core.concept import Concept, ConceptParts
+from core.concept import Concept, ConceptParts, ensure_concept
 from core.sheerka.services.sheerka_service import BaseService
 
 GROUP_PREFIX = 'All_'
@@ -36,6 +36,7 @@ class SheerkaSetsManager(BaseService):
         """
 
         context.log(f"Setting concept {concept} is a {concept_set}", who=self.NAME)
+        ensure_concept(concept, concept_set)
 
         if BuiltinConcepts.ISA in concept.metadata.props and concept_set in concept.metadata.props[BuiltinConcepts.ISA]:
             return self.sheerka.ret(
@@ -61,9 +62,7 @@ class SheerkaSetsManager(BaseService):
         """
 
         context.log(f"Adding concept {concept} to set {concept_set}", who=self.NAME)
-
-        assert concept.id
-        assert concept_set.id
+        ensure_concept(concept, concept_set)
 
         set_elements = self.sheerka.cache_manager.get(self.CONCEPTS_GROUPS_ENTRY, concept_set.id)
         if set_elements and concept.id in set_elements:
@@ -79,6 +78,7 @@ class SheerkaSetsManager(BaseService):
         """Adding multiple concepts at the same time"""
 
         context.log(f"Adding concepts {concepts} to set {concept_set}", who=self.NAME)
+        ensure_concept(concept_set)
         already_in_set = []
         for concept in concepts:
             res = self.add_concept_to_set(context, concept, concept_set)
@@ -103,6 +103,8 @@ class SheerkaSetsManager(BaseService):
         :return:
         """
 
+        ensure_concept(concept)
+
         def _get_set_elements(sub_concept):
             if not self.isaset(context, sub_concept):
                 return self.sheerka.new(BuiltinConcepts.NOT_A_SET, body=concept)
@@ -151,8 +153,9 @@ class SheerkaSetsManager(BaseService):
         if isinstance(a, BuiltinConcepts):  # common KSI error ;-)
             raise SyntaxError("Remember that the first parameter of isinstance MUST be a concept")
 
-        if not (isinstance(a, Concept) and isinstance(b, Concept)):
-            return False
+        ensure_concept(a, b)
+        # if not (isinstance(a, Concept) and isinstance(b, Concept)):
+        #     return False
 
         # TODO, first check the 'isa' property of a
         if not (a.id and b.id):
@@ -163,6 +166,7 @@ class SheerkaSetsManager(BaseService):
 
     def isa(self, a, b):
 
+        ensure_concept(a, b)
         if BuiltinConcepts.ISA not in a.metadata.props:
             return False
 

src/core/utils.py

@@ -363,7 +363,7 @@ def unstr_concept(concept_repr):
     return key if key != "" else None, id if id != "" else None
 
 
-def encode_concept(t, use_concept=False):
+def encode_concept(t):
     """
     Given a tuple of concept id, concept id
     Create a valid Python identifier that can be parsed back
@@ -371,15 +371,13 @@ def encode_concept(t, use_concept=False):
     >>> assert encode_concept(("key", "id")) == "__C__KEY_key__ID_id__C__"
     >>> assert encode_concept((None, "id")) == "__C__KEY_00None00__ID_id__C__"
     >>> assert encode_concept(("key", None)) == "__C__KEY_key__ID_00None00__C__"
-    >>> assert encode_concept(("key", "id"), True) == "__C__USE_CONCEPT__KEY_key__ID_id__C__"
 
     :param t:
-    :param use_concept:
     :return:
     """
 
     key, id_ = (t[0], t[1]) if isinstance(t, tuple) else (t.key, t.id)
-    prefix = "__C__USE_CONCEPT" if use_concept else "__C"
+    prefix = "__C"
     sanitized_key = "".join(c if c.isalnum() else "0" for c in key) if key else "00None00"
     return prefix + f"__KEY_{sanitized_key}__ID_{id_ or '00None00'}__C__"
 
@@ -393,15 +391,14 @@ def decode_concept(text):
     :param text:
     :return:
     """
-    use_concept = text.startswith("__C__USE_CONCEPT")
     m = decode_regex.search(text)
     lookup = {"00None00": None}
     if m:
         key = lookup.get(m.group(1), m.group(1))
         id_ = lookup.get(m.group(2), m.group(2))
-        return key, id_, use_concept
+        return key, id_
 
-    return None, None, None
+    return None, None
 
 
 def tokens_index(tokens, sub_tokens, skip=0):

src/evaluators/PythonEvaluator.py

@@ -57,8 +57,8 @@ class PythonEvaluator(OneReturnValueEvaluator):
 
             # Do not evaluate if the ast refers to a concept (leave it to ConceptEvaluator)
             if isinstance(node.ast_, ast.Expression) and isinstance(node.ast_.body, ast.Name):
-                c = context.sheerka.get_by_key(node.ast_.body.id)
-                if not context.sheerka.isinstance(c, BuiltinConcepts.UNKNOWN_CONCEPT):
+                c = context.sheerka.resolve(node.ast_.body.id)
+                if c is not None:
                     context.log("It's a simple concept. Not for me.", self.name)
                     not_for_me = context.sheerka.new(BuiltinConcepts.NOT_FOR_ME, body=node)
                     return sheerka.ret(self.name, False, not_for_me, parents=[return_value])
@@ -91,54 +91,11 @@ class PythonEvaluator(OneReturnValueEvaluator):
             "BuiltinConcepts": core.builtin_concepts.BuiltinConcepts,
         }
 
+        # has to tbe the first, to allow override
         method_from_sheerka = self.update_globals_with_sheerka_methods(my_locals, context)
 
-        if context.obj:
-            context.log(f"Concept '{context.obj}' is in context. Adding it and its properties to locals.", self.name)
-
-            for prop_name in context.obj.variables():
-                prop_value = context.obj.get_value(prop_name)
-                if isinstance(prop_value, Concept):
-                    my_locals[prop_name] = context.sheerka.objvalue(prop_value)
-                else:
-                    my_locals[prop_name] = prop_value
-
-            my_locals["self"] = context.obj.body
-
-        node_concept = core.ast.nodes.python_to_concept(node.ast_)
-        unreferenced_names_visitor = UnreferencedNamesVisitor(context.sheerka)
-        unreferenced_names_visitor.visit(node_concept)
-
-        for name in unreferenced_names_visitor.names:
-            context.log(f"Resolving '{name}'.", self.name)
-
-            if name in node.concepts:
-                context.log(f"Using value from node.", self.name)
-                concept = node.concepts[name]
-                return_concept = False
-
-            else:
-                c_key, c_id, return_concept = self.resolve_name(name)
-
-                if c_key in my_locals:
-                    context.log(f"Using value from property.", self.name)
-                    continue
-
-                context.log(f"Instantiating new concept with {c_key=}, {c_id=}.", self.name)
-                new = context.sheerka.new
-                concept = new((None, c_id)) if c_id else new(c_key)
-                if context.sheerka.isinstance(concept, BuiltinConcepts.UNKNOWN_CONCEPT):
-                    context.log(f"({c_key=}, {c_id=}) is not a concept. Skipping.", self.name)
-                    continue
-
-            context.log(f"Evaluating '{concept}'", self.name)
-            with context.push(self.name, desc=f"Evaluating '{concept}'", obj=concept) as sub_context:
-                sub_context.local_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
-                evaluated = context.sheerka.evaluate_concept(sub_context, concept)
-                sub_context.add_values(return_values=evaluated)
-
-                if evaluated.key == concept.key:
-                    my_locals[name] = evaluated if return_concept else context.sheerka.objvalue(evaluated)
+        self.update_globals_with_context(my_locals, context)
+        self.update_globals_with_node(my_locals, context, node)
 
         if self.locals:  # when exta values are given. Add them
             my_locals.update(self.locals)
@@ -163,6 +120,77 @@ class PythonEvaluator(OneReturnValueEvaluator):
 
         return methods_from_sheerka  # to allow access using prefix "sheerka."
 
+    def update_globals_with_context(self, my_locals, context):
+        if context.obj:
+            context.log(f"Concept '{context.obj}' is in context. Adding it and its properties to locals.", self.name)
+
+            for prop_name in context.obj.variables():
+                prop_value = context.obj.get_value(prop_name)
+                if isinstance(prop_value, Concept):
+                    my_locals[prop_name] = context.sheerka.objvalue(prop_value)
+                else:
+                    my_locals[prop_name] = prop_value
+
+            my_locals["self"] = context.obj.body
+
+    def update_globals_with_node(self, my_locals, context, node):
+        node_concept = core.ast.nodes.python_to_concept(node.ast_)
+        unreferenced_names_visitor = UnreferencedNamesVisitor(context.sheerka)
+        unreferenced_names_visitor.visit(node_concept)
+
+        for name in unreferenced_names_visitor.names:
+            context.log(f"Resolving '{name}'.", self.name)
+
+            if name in node.concepts:
+                context.log(f"Using value from node.", self.name)
+                concept = self.resolve_concept(context, node.concepts[name])
+
+            elif name in my_locals:
+                context.log(f"Using value from property.", self.name)
+                continue
+
+            else:
+                context.log(f"Instantiating new concept with {name}.", self.name)
+                concept = self.resolve_concept(context, name)
+
+            if concept is None:
+                context.log(f"Concept '{name}' is not found or cannot be instantiated. Skipping.", self.name)
+                continue
+
+            if concept.metadata.is_evaluated:
+                context.log(f"Concept {name} is already evaluated.", self.name)
+
+            else:
+                context.log(f"Evaluating '{concept}'", self.name)
+                with context.push(self.name, desc=f"Evaluating '{concept}'", obj=concept) as sub_context:
+                    sub_context.local_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
+                    evaluated = context.sheerka.evaluate_concept(sub_context, concept)
+                    sub_context.add_values(return_values=evaluated)
+
+                    if evaluated.key != concept.key:
+                        context.log(f"Error while evaluating '{name}'. Skipping.", self.name)
+                        continue
+                    concept = evaluated
+
+            my_locals[name] = context.sheerka.objvalue(concept)
+
+    @staticmethod
+    def resolve_concept(context, concept_hint):
+        if isinstance(concept_hint, Concept):
+            return concept_hint
+
+        concept = context.sheerka.resolve(concept_hint)
+        if concept is None:
+            return None
+        new_instance = context.sheerka.new_from_template(concept, concept.key)
+        if isinstance(concept_hint, tuple):
+            # It's means that it comes from PythonParser which have found a concept token (c:xxx:)
+            # So a concept was explicitly required, not its value
+            # We mark the concept as already evaluated, so it's body will not be evaluated
+            new_instance.metadata.is_evaluated = True
+
+        return new_instance
+
     @staticmethod
     def resolve_name(to_resolve):
         """

src/parsers/BaseParser.py

@@ -141,15 +141,22 @@ class BaseParser:
             body=self.error_sink if self.has_error else tree,
             try_parsed=try_parse)
 
-    def get_input_as_text(self, parser_input, custom_switcher=None):
+    def get_input_as_text(self, parser_input, custom_switcher=None, tracker=None):
+        """
+        Recreate back the source code from parser_input
+        :param parser_input: list of Tokens
+        :param custom_switcher: map of [TokenKind, overridden values]
+        :param tracker: keep track of the value overridden by custom_switcher
+        :return:
+        """
         if isinstance(parser_input, list):
-            return self.get_text_from_tokens(parser_input, custom_switcher)
+            return self.get_text_from_tokens(parser_input, custom_switcher, tracker)
 
         if isinstance(parser_input, ParserResultConcept):
             parser_input = parser_input.source
 
         if "c:" in parser_input:
-            return self.get_text_from_tokens(list(Tokenizer(parser_input)), custom_switcher)
+            return self.get_text_from_tokens(list(Tokenizer(parser_input)), custom_switcher, tracker)
 
         return parser_input
 
@@ -194,7 +201,14 @@ class BaseParser:
         return lst
 
     @staticmethod
-    def get_text_from_tokens(tokens, custom_switcher=None):
+    def get_text_from_tokens(tokens, custom_switcher=None, tracker=None):
+        """
+        Create the source code, from the list of token
+        :param tokens: list of tokens
+        :param custom_switcher: to override the behaviour (the return value) of some token
+        :param tracker: keep track of the original token value when custom switched
+        :return:
+        """
         if tokens is None:
             return ""
         res = ""
@@ -213,6 +227,8 @@ class BaseParser:
         for token in tokens:
             value = switcher.get(token.type, lambda t: t.value)(token)
             res += value
+            if tracker is not None and token.type in custom_switcher:
+                tracker[value] = token.value
         return res
 
     @staticmethod

src/parsers/PythonParser.py

@@ -72,11 +72,12 @@ class PythonParser(BaseParser):
         tree = None
 
         python_switcher = {
-            TokenKind.CONCEPT: lambda t: core.utils.encode_concept(t.value, True)
+            TokenKind.CONCEPT: lambda t: core.utils.encode_concept(t.value)
         }
 
         try:
-            source = self.get_input_as_text(parser_input, python_switcher)
+            tracker = {}
+            source = self.get_input_as_text(parser_input, python_switcher, tracker)
             source = source.strip()
             parser_input = parser_input if isinstance(parser_input, str) else source
 
@@ -108,7 +109,7 @@ class PythonParser(BaseParser):
                     BuiltinConcepts.PARSER_RESULT,
                     parser=self,
                     source=parser_input,
-                    body=PythonNode(parser_input, tree),
+                    body=PythonNode(parser_input, tree, tracker),
                     try_parsed=None))
 
         self.log_result(context, parser_input, ret)