Fixed memory() and RET usage

src/core/sheerka/ExecutionContext.py

@@ -4,7 +4,7 @@ import time
 from core.builtin_concepts import BuiltinConcepts, ParserResultConcept
 from core.concept import Concept
 from core.sheerka.services.SheerkaExecute import NO_MATCH
-from core.sheerka.services.SheerkaShortTermMemory import SheerkaShortTermMemory
+from core.sheerka.services.SheerkaMemory import SheerkaMemory
 from core.sheerka_logger import get_logger
 from sdp.sheerkaDataProvider import Event
 
@@ -130,7 +130,7 @@ class ExecutionContext:
 
     def __exit__(self, exc_type, exc_val, exc_tb):
         if self.stm:
-            self.sheerka.services[SheerkaShortTermMemory.NAME].remove_context(self)
+            self.sheerka.services[SheerkaMemory.NAME].remove_context(self)
 
         self._stop = time.time_ns()
         if self._show_stats:

src/core/sheerka/Sheerka.py

@@ -64,6 +64,7 @@ class Sheerka(Concept):
     USER_CONCEPTS_KEYS = "User_Concepts"  # sequential key for user defined concepts
 
     MAX_EXECUTION_HISTORY = 100
+    MAX_RETURN_VALUES_HISTORY = 100
 
     def __init__(self, cache_only=False, debug=False, loggers=None):
         self.init_logging(debug, loggers)
@@ -113,6 +114,7 @@ class Sheerka(Concept):
         self.locals = {}
 
         self.last_executions = []
+        self.last_return_values = []
 
     @property
     def resolved_concepts_by_first_keyword(self):
@@ -138,21 +140,23 @@ class Sheerka(Concept):
     def chicken_and_eggs(self):
         return self.cache_manager.caches[self.CHICKEN_AND_EGG_CONCEPTS_ENTRY].cache
 
-    def bind_service_method(self, bound_method, has_side_effect, as_name=None):
+    def bind_service_method(self, bound_method, has_side_effect, as_name=None, visible=True):
         """
         Bind service method to sheerka instance for ease of use ?
         :param bound_method:
         :param has_side_effect: False if the method is safe
-        :param as_name:
+        :param as_name: give another name to the method
+        :param visible: make the method visible to Sheerka
         :return:
         """
         if as_name is None:
             as_name = bound_method.__name__
 
-        signature = inspect.signature(bound_method)
-        if len(signature.parameters) > 0 and list(signature.parameters.keys())[0] == "context":
-            self.methods_with_context.add(as_name)
-        self.sheerka_methods[as_name] = SheerkaMethod(bound_method, has_side_effect)
+        if visible:
+            signature = inspect.signature(bound_method)
+            if len(signature.parameters) > 0 and list(signature.parameters.keys())[0] == "context":
+                self.methods_with_context.add(as_name)
+            self.sheerka_methods[as_name] = SheerkaMethod(bound_method, has_side_effect)
 
         setattr(self, as_name, bound_method)
 
@@ -437,6 +441,10 @@ class Sheerka(Concept):
             del self.last_executions[0]
         self.last_executions.append(execution_context)
 
+        if len(self.last_return_values) == self.MAX_RETURN_VALUES_HISTORY:
+            del self.last_return_values[0]
+        self.last_return_values.append(ret)
+
         return ret
 
     def print(self, result, instructions=None):

src/core/sheerka/services/SheerkaAdmin.py

@@ -3,9 +3,10 @@ import time
 from core.builtin_concepts import BuiltinConcepts
 from core.sheerka.services.sheerka_service import BaseService
 
-CONCEPTS_FILE = "_concepts_lite.txt"
+CONCEPTS_FILE_LITE = "_concepts_lite.txt"
 CONCEPTS_FILE_ALL_CONCEPTS = "_concepts.txt"
-CONCEPTS_FILE_TO_USE = CONCEPTS_FILE
+CONCEPTS_FILE_TO_USE = CONCEPTS_FILE_ALL_CONCEPTS
+
 
 class SheerkaAdmin(BaseService):
     NAME = "Admin"
@@ -19,6 +20,7 @@ class SheerkaAdmin(BaseService):
         self.sheerka.bind_service_method(self.restore, True)
         self.sheerka.bind_service_method(self.concepts, False)
         self.sheerka.bind_service_method(self.last_created_concept, False)
+        self.sheerka.bind_service_method(self.last_ret, False)
 
     def caches_names(self):
         """
@@ -27,16 +29,20 @@ class SheerkaAdmin(BaseService):
         """
         return list(self.sheerka.cache_manager.caches.keys())
 
-    def cache(self, name):
+    def cache(self, name, *keys):
         """
         Returns the content of a cache
         :param name:
+        :param keys: look for a specific key. May ask to sdp if the key is not in cache
         :return:
         """
         if name not in self.sheerka.cache_manager.caches:
             return self.sheerka.new(BuiltinConcepts.NOT_FOUND, body={"cache": name})
 
-        return self.sheerka.cache_manager.caches[name].cache.copy()
+        if not keys:
+            return self.sheerka.cache_manager.caches[name].cache.copy()
+
+        return {key: self.sheerka.cache_manager.get(name, key) for key in keys}
 
     def restore(self, concept_file=CONCEPTS_FILE_TO_USE):
         """
@@ -53,6 +59,7 @@ class SheerkaAdmin(BaseService):
         try:
             start = time.time_ns()
             nb_lines = 0
+            nb_lines_in_error = 0
             self.sheerka.during_restore = True
             with open(concept_file, "r") as f:
                 for line in f.readlines():
@@ -63,14 +70,19 @@ class SheerkaAdmin(BaseService):
                     self.sheerka.log.info(line)
                     res = self.sheerka.evaluate_user_input(line)
                     if len(res) > 1 or not res[0].status:
-                        self.sheerka.log.error("Error detected !")
+                        nb_lines_in_error += 1
+                        self.sheerka.log.error("\u001b[31mError detected !\u001b[0m")
             self.sheerka.during_restore = False
             stop = time.time_ns()
 
             nano_sec = stop - start
             dt = nano_sec / 1e6
             elapsed = f"{dt} ms" if dt < 1000 else f"{dt / 1000} s"
-            print(f"Imported {nb_lines} line(s) in {elapsed}.")
+            self.sheerka.log.info(f"Imported {nb_lines} line(s) in {elapsed}.")
+            if nb_lines_in_error > 0:
+                self.sheerka.log.info(f"\u001b[31m{nb_lines_in_error} errors(s) found.\u001b[0m")
+            else:
+                self.sheerka.log.info(f"No error.")
         except IOError:
             pass
 
@@ -89,3 +101,5 @@ class SheerkaAdmin(BaseService):
 
         return self.sheerka.new(BuiltinConcepts.NOT_FOUND)
 
+    def last_ret(self, context, index=-1):
+        return self.sheerka.last_return_values[index]

src/core/sheerka/services/SheerkaEvaluateConcept.py

@@ -2,7 +2,7 @@ from dataclasses import dataclass
 
 from core.builtin_concepts import BuiltinConcepts
 from core.builtin_helpers import expect_one, only_successful, parse_unrecognized, evaluate
-from core.concept import Concept, DoNotResolve, ConceptParts, InfiniteRecursionResolved, NotInit
+from core.concept import Concept, DoNotResolve, ConceptParts, InfiniteRecursionResolved, NotInit, ensure_concept
 from core.sheerka.services.SheerkaExecute import ParserInput
 from core.sheerka.services.sheerka_service import BaseService
 from core.tokenizer import Tokenizer
@@ -32,6 +32,7 @@ class SheerkaEvaluateConcept(BaseService):
 
     def initialize(self):
         self.sheerka.bind_service_method(self.evaluate_concept, True)
+        self.sheerka.bind_service_method(self.set_auto_eval, True)
 
     @staticmethod
     def infinite_recursion_detected(context, concept):
@@ -229,9 +230,12 @@ class SheerkaEvaluateConcept(BaseService):
                 continue
 
             source = getattr(concept.metadata, part_key.value)
-            if source is None or not isinstance(source, str):
+            if source is None:  # or not isinstance(source, str):
                 continue
 
+            if not isinstance(source, str):
+                raise Exception("Invalid concept init. metadata must be a string")
+
             if source.strip() == "":
                 concept.compiled[part_key] = DoNotResolve(source)
             else:
@@ -251,9 +255,12 @@ class SheerkaEvaluateConcept(BaseService):
             if var_name in concept.compiled:
                 continue
 
-            if default_value is None or not isinstance(default_value, str):
+            if default_value is None:
                 continue
 
+            if not isinstance(default_value, str):
+                raise Exception("Invalid concept init. variable metadata must be a string")
+
             if default_value.strip() == "":
                 concept.compiled[var_name] = DoNotResolve(default_value)
             else:
@@ -332,12 +339,13 @@ class SheerkaEvaluateConcept(BaseService):
             # when it's a concept, evaluate it
             if isinstance(to_resolve, Concept) and \
                 not context.sheerka.isinstance(to_resolve, BuiltinConcepts.RETURN_VALUE):
+
                 evaluated = self.evaluate_concept(sub_context, to_resolve)
                 sub_context.add_values(return_values=evaluated)
-                if evaluated.key == to_resolve.key:  # quicker (and dirtier) than sheerka.is_success()
-                    return self.apply_ret(evaluated)
-                else:
+                if not context.sheerka.is_success(evaluated) and evaluated.key != to_resolve.key:
                     error = evaluated
+                else:
+                    return evaluated
 
             # otherwise, execute all return values to find out what is the value
             else:
@@ -449,7 +457,7 @@ class SheerkaEvaluateConcept(BaseService):
                 sub_context.protected_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
 
             # auto evaluate commands
-            if context.sheerka.isa(concept, context.sheerka.new(BuiltinConcepts.COMMAND)):
+            if context.sheerka.isa(concept, context.sheerka.new(BuiltinConcepts.AUTO_EVAL)):
                 sub_context.protected_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
 
             self.initialize_concept_asts(sub_context, concept)
@@ -531,12 +539,19 @@ class SheerkaEvaluateConcept(BaseService):
             if "body" in all_metadata_to_eval:
                 concept.metadata.is_evaluated = True
 
-                # # update the cache for concepts with no variables
-                # Cannot use cache. See the comment at the beginning of this method
-                # if len(concept.metadata.variables) == 0:
-                #     self.sheerka.cache_manager.put(self.sheerka.CONCEPTS_BY_ID_ENTRY, concept.id, concept)
+            # # update the cache for concepts with no variables
+            # Cannot use cache. See the comment at the beginning of this method
+            # if len(concept.metadata.variables) == 0:
+            #     self.sheerka.cache_manager.put(self.sheerka.CONCEPTS_BY_ID_ENTRY, concept.id, concept)
 
-            return concept
+            if not concept.metadata.is_builtin:
+                self.sheerka.register_object(sub_context, concept.name, concept)
+
+            # manage RET metadata
+            if sub_context.in_context(BuiltinConcepts.EVAL_BODY_REQUESTED) and ConceptParts.RET in concept.values:
+                return concept.get_value(ConceptParts.RET)
+            else:
+                return concept
 
     def compute_metadata_to_eval(self, context, concept):
         to_eval = []
@@ -553,10 +568,11 @@ class SheerkaEvaluateConcept(BaseService):
             body |= b
             to_eval.extend(needed)
 
-        needed, v, b = self.get_needed_metadata(concept, ConceptParts.RET, not variables, not body)
-        variables |= v
-        body |= b
-        to_eval.extend(needed)
+        if context.in_context(BuiltinConcepts.EVAL_BODY_REQUESTED):
+            needed, v, b = self.get_needed_metadata(concept, ConceptParts.RET, not variables, not body)
+            variables |= v
+            body |= b
+            to_eval.extend(needed)
 
         needed, v, b = self.get_needed_metadata(concept, ConceptParts.POST, not variables, not body)
         variables |= v
@@ -571,3 +587,13 @@ class SheerkaEvaluateConcept(BaseService):
                 to_eval.append("body")
 
         return to_eval
+
+    def set_auto_eval(self, context, concept):
+        """
+        add AUTO_EVAL to ISA
+        :param context:
+        :param concept:
+        :return:
+        """
+        ensure_concept(concept)
+        return self.sheerka.set_isa(context, concept, self.sheerka.new(BuiltinConcepts.AUTO_EVAL))

src/core/sheerka/services/SheerkaMemory.py

@@ -0,0 +1,132 @@
+from dataclasses import dataclass
+
+from cache.ListIfNeededCache import ListIfNeededCache
+from core.builtin_concepts import BuiltinConcepts
+from core.concept import Concept
+from core.sheerka.services.sheerka_service import BaseService, ServiceObj
+
+
+@dataclass
+class MemoryObject(ServiceObj):
+    obj: object
+
+
+class SheerkaMemory(BaseService):
+    NAME = "Memory"
+
+    SHORT_TERM_OBJECTS_ENTRY = "Memory:ShortTermMemoryObjects"
+    OBJECTS_ENTRY = "Memory:Objects"
+
+    def __init__(self, sheerka):
+        super().__init__(sheerka)
+        self.short_term_objects = ListIfNeededCache()
+        self.objects = ListIfNeededCache(default=lambda k: self.sheerka.sdp.get(self.OBJECTS_ENTRY, k))
+        self.registration = {}
+
+    def initialize(self):
+        self.sheerka.bind_service_method(self.get_from_short_term_memory, False, visible=False)
+        self.sheerka.bind_service_method(self.add_to_short_term_memory, True, visible=False)
+        self.sheerka.bind_service_method(self.add_to_memory, True, visible=False)
+        self.sheerka.bind_service_method(self.get_from_memory, False)
+        self.sheerka.bind_service_method(self.register_object, True, visible=False)
+        self.sheerka.bind_service_method(self.unregister_object, True, visible=False)
+        self.sheerka.bind_service_method(self.add_registered_objects, True, visible=False)
+        self.sheerka.bind_service_method(self.memory, False)
+
+        self.sheerka.cache_manager.register_cache(self.SHORT_TERM_OBJECTS_ENTRY, self.short_term_objects, persist=False)
+        self.sheerka.cache_manager.register_cache(self.OBJECTS_ENTRY, self.objects, persist=True, use_ref=True)
+
+    def get_from_short_term_memory(self, context, key):
+        while True:
+            key_to_use = (str(context.id) if context else "") + ":" + key
+            if (obj := self.sheerka.cache_manager.get(self.SHORT_TERM_OBJECTS_ENTRY, key_to_use)) is not None:
+                return obj
+
+            if context is None:
+                return None
+
+            context = context.get_parent()
+
+    def add_to_short_term_memory(self, context, key, concept):
+        if context:
+            context.stm = True
+        key_to_use = (str(context.id) if context else "") + ":" + key
+        return self.sheerka.cache_manager.put(self.SHORT_TERM_OBJECTS_ENTRY, key_to_use, concept)
+
+    def remove_context(self, context):
+        self.short_term_objects.evict_by_key(lambda k: k.startswith(str(context.id) + ":"))
+
+    def add_to_memory(self, context, key, concept):
+        """
+        Adds an object to memory
+        :param context:
+        :param key:
+        :param concept:
+        :return:
+        """
+        self.objects.put(key, MemoryObject(context.event.get_digest(), concept))
+
+    def get_from_memory(self, context, key):
+        """"
+        """
+        return self.objects.get(key)
+
+    def register_object(self, context, key, concept):
+        """
+        Before adding objects to memory, they first need to be registered
+        :param context:
+        :param key:
+        :param concept:
+        :return:
+        """
+        self.registration[key] = concept
+
+    def unregister_object(self, context, key):
+        """
+        To indicate that key is no longer to be remembered
+        :param context:
+        :param key:
+        :return:
+        """
+        try:
+            del self.registration[key]
+        except KeyError:
+            pass
+
+    def add_registered_objects(self, context):
+        """
+        Adds all registered objects
+        :param context:
+        :return:
+        """
+        for k, v in self.registration.items():
+            self.add_to_memory(context, k, v)
+        self.registration.clear()
+
+    def memory(self, context, name=None):
+        """
+        Get the list of all objects in memory
+        :param context:
+        :param name:
+        :return:
+        """
+        if name:
+            name_to_use = name.name if isinstance(name, Concept) else name
+            self.unregister_object(context, name_to_use)
+            obj = self.get_from_memory(context, name_to_use)
+            if obj is None:
+                return self.sheerka.new(BuiltinConcepts.NOT_FOUND, body={"#name": name})
+
+            if isinstance(obj, list):
+                obj = obj[-1]
+
+            return obj.obj
+
+        res = {}
+        for k in self.objects:
+            obj = self.objects.get(k)
+            if isinstance(obj, list):
+                obj = obj[-1]
+            res[k] = obj.obj
+
+        return res

src/core/sheerka/services/SheerkaModifyConcept.py

@@ -1,18 +1,27 @@
 from core.builtin_concepts import BuiltinConcepts
+from core.concept import ensure_concept
 from core.sheerka.services.sheerka_service import BaseService
 from parsers.BnfParser import BnfParser
 
 
 class SheerkaModifyConcept(BaseService):
     NAME = "ModifyConcept"
-    
+
     def __init__(self, sheerka):
         super().__init__(sheerka)
 
     def initialize(self):
         self.sheerka.bind_service_method(self.modify_concept, True)
+        self.sheerka.bind_service_method(self.set_attr, True)
+        self.sheerka.bind_service_method(self.get_attr, False)
 
     def modify_concept(self, context, concept):
+        """
+        Modify the definition of a concept
+        :param context:
+        :param concept:
+        :return:
+        """
         old_version = self.sheerka.get_by_id(concept.id)
 
         if old_version is None:
@@ -38,16 +47,12 @@ class SheerkaModifyConcept(BaseService):
                     BuiltinConcepts.ALREADY_DEFINED,
                     body=concept))
 
-        old_references = self.sheerka.cache_manager.get(self.sheerka.CONCEPTS_REFERENCES_ENTRY, concept.id)
-        if old_references:
-            old_references = old_references.copy()
-
         self.sheerka.cache_manager.update_concept(old_version, concept)
 
-        # TODO : update concept by first keyword
-        # TODO : update resolved by first keyword
-        # TODO : update concepts grammars
-        # TODO : update when definition_type = DEFINITION_TYPE_DEF
+        # TODO : update concept by first keyword : have a look at update_references() below
+        # TODO : update resolved by first keyword : have a look at update_references() below
+        # TODO : update when definition_type = DEFINITION_TYPE_DEF : have a look at update_references() below
+        # TODO : Update concepts grammars : have a look at update_references() below
 
         ret = self.sheerka.ret(self.NAME, True, self.sheerka.new(BuiltinConcepts.NEW_CONCEPT, body=concept))
         return ret
@@ -69,3 +74,30 @@ class SheerkaModifyConcept(BaseService):
 
             if concept.bnf is not None:
                 BnfParser.update_recurse_id(context, concept_id, concept.bnf)
+
+            # remove the grammar entry so that it can be recreated
+            self.sheerka.cache_manager.delete(self.sheerka.CONCEPTS_GRAMMARS_ENTRY, concept_id)
+
+    def set_attr(self, concept, attribute, value):
+        """
+        Modifies an attribute of a concept (concept.values)
+        :param context:
+        :param concept:
+        :param attribute:
+        :param value:
+        :return:
+        """
+        ensure_concept(concept)
+        concept.set_value(attribute, value)
+        return self.sheerka.ret(self.NAME, True, self.sheerka.new(BuiltinConcepts.SUCCESS))
+
+    def get_attr(self, concept, attribute):
+        """
+        Returns the attribute of a concept
+        :param context:
+        :param concept:
+        :param attribute:
+        :return:
+        """
+        ensure_concept()
+        return concept.get_value(attribute)

src/core/sheerka/services/SheerkaSetsManager.py

@@ -59,10 +59,6 @@ class SheerkaSetsManager(BaseService):
 
         res = self.add_concept_to_set(context, concept, concept_set)
 
-        # update concept_set references
-        self.sheerka.services[SheerkaModifyConcept.NAME].update_references(context, concept_set)
-        self.concepts_in_set.delete(concept_set.id)
-
         return res
 
     def add_concept_to_set(self, context, concept, concept_set):
@@ -85,11 +81,21 @@ class SheerkaSetsManager(BaseService):
                 self.sheerka.new(BuiltinConcepts.CONCEPT_ALREADY_IN_SET, body=concept, concept_set=concept_set))
 
         self.sets.put(concept_set.id, concept.id)
+
+        # invalidate the cache of what contains concept_set
         self.concepts_in_set.delete(concept_set.id)
+
+        # update concept_set references
+        self.sheerka.services[SheerkaModifyConcept.NAME].update_references(context, concept_set)
+
         return self.sheerka.ret(self.NAME, True, self.sheerka.new(BuiltinConcepts.SUCCESS))
 
     def add_concepts_to_set(self, context, concepts, concept_set):
-        """Adding multiple concepts at the same time"""
+        """
+        Adding multiple concepts at the same time
+        ******** THIS METHOD IS FOR TEST ONLY *************
+        As it is not optimized. It needs to be rewritten in case of production usage
+        """
 
         context.log(f"Adding concepts {concepts} to set {concept_set}", who=self.NAME)
         ensure_concept(concept_set)

src/core/sheerka/services/SheerkaShortTermMemory.py

@@ -1,37 +0,0 @@
-from cache.ListIfNeededCache import ListIfNeededCache
-from core.sheerka.services.sheerka_service import BaseService
-
-
-class SheerkaShortTermMemory(BaseService):
-    NAME = "ShortTermMemory"
-
-    SHORT_TERM_MEMORY_ENTRY = "ShortTermMemory:Objects"
-
-    def __init__(self, sheerka):
-        super().__init__(sheerka)
-        self.objects = ListIfNeededCache()
-
-    def initialize(self):
-        self.sheerka.bind_service_method(self.get_from_short_term_memory, False)
-        self.sheerka.bind_service_method(self.add_to_short_term_memory, True)
-        self.sheerka.cache_manager.register_cache(self.SHORT_TERM_MEMORY_ENTRY, self.objects, persist=False)
-
-    def get_from_short_term_memory(self, context, key):
-        while True:
-            key_to_use = (str(context.id) if context else "") + ":" + key
-            if (obj := self.sheerka.cache_manager.get(self.SHORT_TERM_MEMORY_ENTRY, key_to_use)) is not None:
-                return obj
-
-            if context is None:
-                return None
-
-            context = context.get_parent()
-
-    def add_to_short_term_memory(self, context, key, concept):
-        if context:
-            context.stm = True
-        key_to_use = (str(context.id) if context else "") + ":" + key
-        return self.sheerka.cache_manager.put(self.SHORT_TERM_MEMORY_ENTRY, key_to_use, concept)
-
-    def remove_context(self, context):
-        self.objects.evict_by_key(lambda k: k.startswith(str(context.id) + ":"))