Fixed memory() and RET usage

src/core/builtin_concepts.py

@@ -15,9 +15,9 @@ class BuiltinConcepts(Enum):
     """
     SHEERKA = "sheerka"
 
-    # processing instructions during sheerka.execute()
+    # processing instructions during sheerka.execute() or sheerka.evaluate_concept()
     # The instruction may alter how the actions work
-    DEBUG = "debug" # activate all debug information
+    DEBUG = "debug"  # activate all debug information
     EVAL_BODY_REQUESTED = "eval body"  # to evaluate the body
     EVAL_WHERE_REQUESTED = "eval where"  # to evaluate the where clause
     RETURN_BODY_REQUESTED = "return body"  # returns the body of the concept instead of the concept itself
@@ -53,7 +53,7 @@ class BuiltinConcepts(Enum):
 
     # builtin attributes
     ISA = "is a"  # when a concept is an instance of another one
-    COMMAND = "command"  # when the concept must be auto evaluated
+    AUTO_EVAL = "auto eval"  # when the concept must be auto evaluated
 
     # object
     USER_INPUT = "user input concept"  # represent an input from an user
@@ -155,7 +155,7 @@ BuiltinUnique = [
     BuiltinConcepts.TESTING,
 
     BuiltinConcepts.ISA,
-    BuiltinConcepts.COMMAND,
+    BuiltinConcepts.AUTO_EVAL,
 
     BuiltinConcepts.INVALID_LESSER_OPERATION,
     BuiltinConcepts.INVALID_GREATEST_OPERATION,

src/core/builtin_helpers.py

@@ -23,7 +23,7 @@ def is_same_success(context, return_values):
     Returns True if all returns values are successful and have the same value
     :param context:
     :param return_values:
-    :return:
+    :return: True False or None (None if the concept is not evaluated)
     """
     assert isinstance(return_values, list)
 
@@ -31,17 +31,10 @@ def is_same_success(context, return_values):
         if not ret_val.status:
             raise Exception("Status is false")
 
-        if isinstance(ret_val.body, Concept):
-            if not ret_val.body.metadata.is_evaluated:
-                evaluated = context.sheerka.evaluate_concept(context, ret_val.body, eval_body=True)
-                if not context.sheerka.is_success(evaluated):
-                    raise Exception("Failed to evaluate evaluate")
+        if isinstance(ret_val.body, Concept) and not ret_val.body.metadata.is_evaluated:
+            raise Exception("Concept is not evaluated")
 
-                return context.sheerka.objvalue(evaluated)
-            else:
-                return context.sheerka.objvalue(ret_val.body)
-        else:
-            return context.sheerka.objvalue(ret_val)
+        return context.sheerka.objvalue(ret_val)
 
     try:
         reference = _get_value(return_values[0])
@@ -53,7 +46,7 @@ def is_same_success(context, return_values):
 
     except Exception as ex:
         context.log_error(ex)
-        return False
+        return None
 
     return True
 

src/core/sheerka/ExecutionContext.py

@@ -4,7 +4,7 @@ import time
 from core.builtin_concepts import BuiltinConcepts, ParserResultConcept
 from core.concept import Concept
 from core.sheerka.services.SheerkaExecute import NO_MATCH
-from core.sheerka.services.SheerkaShortTermMemory import SheerkaShortTermMemory
+from core.sheerka.services.SheerkaMemory import SheerkaMemory
 from core.sheerka_logger import get_logger
 from sdp.sheerkaDataProvider import Event
 
@@ -130,7 +130,7 @@ class ExecutionContext:
 
     def __exit__(self, exc_type, exc_val, exc_tb):
         if self.stm:
-            self.sheerka.services[SheerkaShortTermMemory.NAME].remove_context(self)
+            self.sheerka.services[SheerkaMemory.NAME].remove_context(self)
 
         self._stop = time.time_ns()
         if self._show_stats:

src/core/sheerka/Sheerka.py

@@ -64,6 +64,7 @@ class Sheerka(Concept):
     USER_CONCEPTS_KEYS = "User_Concepts"  # sequential key for user defined concepts
 
     MAX_EXECUTION_HISTORY = 100
+    MAX_RETURN_VALUES_HISTORY = 100
 
     def __init__(self, cache_only=False, debug=False, loggers=None):
         self.init_logging(debug, loggers)
@@ -113,6 +114,7 @@ class Sheerka(Concept):
         self.locals = {}
 
         self.last_executions = []
+        self.last_return_values = []
 
     @property
     def resolved_concepts_by_first_keyword(self):
@@ -138,21 +140,23 @@ class Sheerka(Concept):
     def chicken_and_eggs(self):
         return self.cache_manager.caches[self.CHICKEN_AND_EGG_CONCEPTS_ENTRY].cache
 
-    def bind_service_method(self, bound_method, has_side_effect, as_name=None):
+    def bind_service_method(self, bound_method, has_side_effect, as_name=None, visible=True):
         """
         Bind service method to sheerka instance for ease of use ?
         :param bound_method:
         :param has_side_effect: False if the method is safe
-        :param as_name:
+        :param as_name: give another name to the method
+        :param visible: make the method visible to Sheerka
         :return:
         """
         if as_name is None:
             as_name = bound_method.__name__
 
-        signature = inspect.signature(bound_method)
-        if len(signature.parameters) > 0 and list(signature.parameters.keys())[0] == "context":
-            self.methods_with_context.add(as_name)
-        self.sheerka_methods[as_name] = SheerkaMethod(bound_method, has_side_effect)
+        if visible:
+            signature = inspect.signature(bound_method)
+            if len(signature.parameters) > 0 and list(signature.parameters.keys())[0] == "context":
+                self.methods_with_context.add(as_name)
+            self.sheerka_methods[as_name] = SheerkaMethod(bound_method, has_side_effect)
 
         setattr(self, as_name, bound_method)
 
@@ -437,6 +441,10 @@ class Sheerka(Concept):
             del self.last_executions[0]
         self.last_executions.append(execution_context)
 
+        if len(self.last_return_values) == self.MAX_RETURN_VALUES_HISTORY:
+            del self.last_return_values[0]
+        self.last_return_values.append(ret)
+
         return ret
 
     def print(self, result, instructions=None):

src/core/sheerka/services/SheerkaAdmin.py

@@ -3,9 +3,10 @@ import time
 from core.builtin_concepts import BuiltinConcepts
 from core.sheerka.services.sheerka_service import BaseService
 
-CONCEPTS_FILE = "_concepts_lite.txt"
+CONCEPTS_FILE_LITE = "_concepts_lite.txt"
 CONCEPTS_FILE_ALL_CONCEPTS = "_concepts.txt"
-CONCEPTS_FILE_TO_USE = CONCEPTS_FILE
+CONCEPTS_FILE_TO_USE = CONCEPTS_FILE_ALL_CONCEPTS
+
 
 class SheerkaAdmin(BaseService):
     NAME = "Admin"
@@ -19,6 +20,7 @@ class SheerkaAdmin(BaseService):
         self.sheerka.bind_service_method(self.restore, True)
         self.sheerka.bind_service_method(self.concepts, False)
         self.sheerka.bind_service_method(self.last_created_concept, False)
+        self.sheerka.bind_service_method(self.last_ret, False)
 
     def caches_names(self):
         """
@@ -27,16 +29,20 @@ class SheerkaAdmin(BaseService):
         """
         return list(self.sheerka.cache_manager.caches.keys())
 
-    def cache(self, name):
+    def cache(self, name, *keys):
         """
         Returns the content of a cache
         :param name:
+        :param keys: look for a specific key. May ask to sdp if the key is not in cache
         :return:
         """
         if name not in self.sheerka.cache_manager.caches:
             return self.sheerka.new(BuiltinConcepts.NOT_FOUND, body={"cache": name})
 
-        return self.sheerka.cache_manager.caches[name].cache.copy()
+        if not keys:
+            return self.sheerka.cache_manager.caches[name].cache.copy()
+
+        return {key: self.sheerka.cache_manager.get(name, key) for key in keys}
 
     def restore(self, concept_file=CONCEPTS_FILE_TO_USE):
         """
@@ -53,6 +59,7 @@ class SheerkaAdmin(BaseService):
         try:
             start = time.time_ns()
             nb_lines = 0
+            nb_lines_in_error = 0
             self.sheerka.during_restore = True
             with open(concept_file, "r") as f:
                 for line in f.readlines():
@@ -63,14 +70,19 @@ class SheerkaAdmin(BaseService):
                     self.sheerka.log.info(line)
                     res = self.sheerka.evaluate_user_input(line)
                     if len(res) > 1 or not res[0].status:
-                        self.sheerka.log.error("Error detected !")
+                        nb_lines_in_error += 1
+                        self.sheerka.log.error("\u001b[31mError detected !\u001b[0m")
             self.sheerka.during_restore = False
             stop = time.time_ns()
 
             nano_sec = stop - start
             dt = nano_sec / 1e6
             elapsed = f"{dt} ms" if dt < 1000 else f"{dt / 1000} s"
-            print(f"Imported {nb_lines} line(s) in {elapsed}.")
+            self.sheerka.log.info(f"Imported {nb_lines} line(s) in {elapsed}.")
+            if nb_lines_in_error > 0:
+                self.sheerka.log.info(f"\u001b[31m{nb_lines_in_error} errors(s) found.\u001b[0m")
+            else:
+                self.sheerka.log.info(f"No error.")
         except IOError:
             pass
 
@@ -89,3 +101,5 @@ class SheerkaAdmin(BaseService):
 
         return self.sheerka.new(BuiltinConcepts.NOT_FOUND)
 
+    def last_ret(self, context, index=-1):
+        return self.sheerka.last_return_values[index]

src/core/sheerka/services/SheerkaEvaluateConcept.py

@@ -2,7 +2,7 @@ from dataclasses import dataclass
 
 from core.builtin_concepts import BuiltinConcepts
 from core.builtin_helpers import expect_one, only_successful, parse_unrecognized, evaluate
-from core.concept import Concept, DoNotResolve, ConceptParts, InfiniteRecursionResolved, NotInit
+from core.concept import Concept, DoNotResolve, ConceptParts, InfiniteRecursionResolved, NotInit, ensure_concept
 from core.sheerka.services.SheerkaExecute import ParserInput
 from core.sheerka.services.sheerka_service import BaseService
 from core.tokenizer import Tokenizer
@@ -32,6 +32,7 @@ class SheerkaEvaluateConcept(BaseService):
 
     def initialize(self):
         self.sheerka.bind_service_method(self.evaluate_concept, True)
+        self.sheerka.bind_service_method(self.set_auto_eval, True)
 
     @staticmethod
     def infinite_recursion_detected(context, concept):
@@ -229,9 +230,12 @@ class SheerkaEvaluateConcept(BaseService):
                 continue
 
             source = getattr(concept.metadata, part_key.value)
-            if source is None or not isinstance(source, str):
+            if source is None:  # or not isinstance(source, str):
                 continue
 
+            if not isinstance(source, str):
+                raise Exception("Invalid concept init. metadata must be a string")
+
             if source.strip() == "":
                 concept.compiled[part_key] = DoNotResolve(source)
             else:
@@ -251,9 +255,12 @@ class SheerkaEvaluateConcept(BaseService):
             if var_name in concept.compiled:
                 continue
 
-            if default_value is None or not isinstance(default_value, str):
+            if default_value is None:
                 continue
 
+            if not isinstance(default_value, str):
+                raise Exception("Invalid concept init. variable metadata must be a string")
+
             if default_value.strip() == "":
                 concept.compiled[var_name] = DoNotResolve(default_value)
             else:
@@ -332,12 +339,13 @@ class SheerkaEvaluateConcept(BaseService):
             # when it's a concept, evaluate it
             if isinstance(to_resolve, Concept) and \
                 not context.sheerka.isinstance(to_resolve, BuiltinConcepts.RETURN_VALUE):
+
                 evaluated = self.evaluate_concept(sub_context, to_resolve)
                 sub_context.add_values(return_values=evaluated)
-                if evaluated.key == to_resolve.key:  # quicker (and dirtier) than sheerka.is_success()
-                    return self.apply_ret(evaluated)
-                else:
+                if not context.sheerka.is_success(evaluated) and evaluated.key != to_resolve.key:
                     error = evaluated
+                else:
+                    return evaluated
 
             # otherwise, execute all return values to find out what is the value
             else:
@@ -449,7 +457,7 @@ class SheerkaEvaluateConcept(BaseService):
                 sub_context.protected_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
 
             # auto evaluate commands
-            if context.sheerka.isa(concept, context.sheerka.new(BuiltinConcepts.COMMAND)):
+            if context.sheerka.isa(concept, context.sheerka.new(BuiltinConcepts.AUTO_EVAL)):
                 sub_context.protected_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
 
             self.initialize_concept_asts(sub_context, concept)
@@ -531,12 +539,19 @@ class SheerkaEvaluateConcept(BaseService):
             if "body" in all_metadata_to_eval:
                 concept.metadata.is_evaluated = True
 
-                # # update the cache for concepts with no variables
-                # Cannot use cache. See the comment at the beginning of this method
-                # if len(concept.metadata.variables) == 0:
-                #     self.sheerka.cache_manager.put(self.sheerka.CONCEPTS_BY_ID_ENTRY, concept.id, concept)
+            # # update the cache for concepts with no variables
+            # Cannot use cache. See the comment at the beginning of this method
+            # if len(concept.metadata.variables) == 0:
+            #     self.sheerka.cache_manager.put(self.sheerka.CONCEPTS_BY_ID_ENTRY, concept.id, concept)
 
-            return concept
+            if not concept.metadata.is_builtin:
+                self.sheerka.register_object(sub_context, concept.name, concept)
+
+            # manage RET metadata
+            if sub_context.in_context(BuiltinConcepts.EVAL_BODY_REQUESTED) and ConceptParts.RET in concept.values:
+                return concept.get_value(ConceptParts.RET)
+            else:
+                return concept
 
     def compute_metadata_to_eval(self, context, concept):
         to_eval = []
@@ -553,10 +568,11 @@ class SheerkaEvaluateConcept(BaseService):
             body |= b
             to_eval.extend(needed)
 
-        needed, v, b = self.get_needed_metadata(concept, ConceptParts.RET, not variables, not body)
-        variables |= v
-        body |= b
-        to_eval.extend(needed)
+        if context.in_context(BuiltinConcepts.EVAL_BODY_REQUESTED):
+            needed, v, b = self.get_needed_metadata(concept, ConceptParts.RET, not variables, not body)
+            variables |= v
+            body |= b
+            to_eval.extend(needed)
 
         needed, v, b = self.get_needed_metadata(concept, ConceptParts.POST, not variables, not body)
         variables |= v
@@ -571,3 +587,13 @@ class SheerkaEvaluateConcept(BaseService):
                 to_eval.append("body")
 
         return to_eval
+
+    def set_auto_eval(self, context, concept):
+        """
+        add AUTO_EVAL to ISA
+        :param context:
+        :param concept:
+        :return:
+        """
+        ensure_concept(concept)
+        return self.sheerka.set_isa(context, concept, self.sheerka.new(BuiltinConcepts.AUTO_EVAL))

src/core/sheerka/services/SheerkaMemory.py

@@ -0,0 +1,132 @@
+from dataclasses import dataclass
+
+from cache.ListIfNeededCache import ListIfNeededCache
+from core.builtin_concepts import BuiltinConcepts
+from core.concept import Concept
+from core.sheerka.services.sheerka_service import BaseService, ServiceObj
+
+
+@dataclass
+class MemoryObject(ServiceObj):
+    obj: object
+
+
+class SheerkaMemory(BaseService):
+    NAME = "Memory"
+
+    SHORT_TERM_OBJECTS_ENTRY = "Memory:ShortTermMemoryObjects"
+    OBJECTS_ENTRY = "Memory:Objects"
+
+    def __init__(self, sheerka):
+        super().__init__(sheerka)
+        self.short_term_objects = ListIfNeededCache()
+        self.objects = ListIfNeededCache(default=lambda k: self.sheerka.sdp.get(self.OBJECTS_ENTRY, k))
+        self.registration = {}
+
+    def initialize(self):
+        self.sheerka.bind_service_method(self.get_from_short_term_memory, False, visible=False)
+        self.sheerka.bind_service_method(self.add_to_short_term_memory, True, visible=False)
+        self.sheerka.bind_service_method(self.add_to_memory, True, visible=False)
+        self.sheerka.bind_service_method(self.get_from_memory, False)
+        self.sheerka.bind_service_method(self.register_object, True, visible=False)
+        self.sheerka.bind_service_method(self.unregister_object, True, visible=False)
+        self.sheerka.bind_service_method(self.add_registered_objects, True, visible=False)
+        self.sheerka.bind_service_method(self.memory, False)
+
+        self.sheerka.cache_manager.register_cache(self.SHORT_TERM_OBJECTS_ENTRY, self.short_term_objects, persist=False)
+        self.sheerka.cache_manager.register_cache(self.OBJECTS_ENTRY, self.objects, persist=True, use_ref=True)
+
+    def get_from_short_term_memory(self, context, key):
+        while True:
+            key_to_use = (str(context.id) if context else "") + ":" + key
+            if (obj := self.sheerka.cache_manager.get(self.SHORT_TERM_OBJECTS_ENTRY, key_to_use)) is not None:
+                return obj
+
+            if context is None:
+                return None
+
+            context = context.get_parent()
+
+    def add_to_short_term_memory(self, context, key, concept):
+        if context:
+            context.stm = True
+        key_to_use = (str(context.id) if context else "") + ":" + key
+        return self.sheerka.cache_manager.put(self.SHORT_TERM_OBJECTS_ENTRY, key_to_use, concept)
+
+    def remove_context(self, context):
+        self.short_term_objects.evict_by_key(lambda k: k.startswith(str(context.id) + ":"))
+
+    def add_to_memory(self, context, key, concept):
+        """
+        Adds an object to memory
+        :param context:
+        :param key:
+        :param concept:
+        :return:
+        """
+        self.objects.put(key, MemoryObject(context.event.get_digest(), concept))
+
+    def get_from_memory(self, context, key):
+        """"
+        """
+        return self.objects.get(key)
+
+    def register_object(self, context, key, concept):
+        """
+        Before adding objects to memory, they first need to be registered
+        :param context:
+        :param key:
+        :param concept:
+        :return:
+        """
+        self.registration[key] = concept
+
+    def unregister_object(self, context, key):
+        """
+        To indicate that key is no longer to be remembered
+        :param context:
+        :param key:
+        :return:
+        """
+        try:
+            del self.registration[key]
+        except KeyError:
+            pass
+
+    def add_registered_objects(self, context):
+        """
+        Adds all registered objects
+        :param context:
+        :return:
+        """
+        for k, v in self.registration.items():
+            self.add_to_memory(context, k, v)
+        self.registration.clear()
+
+    def memory(self, context, name=None):
+        """
+        Get the list of all objects in memory
+        :param context:
+        :param name:
+        :return:
+        """
+        if name:
+            name_to_use = name.name if isinstance(name, Concept) else name
+            self.unregister_object(context, name_to_use)
+            obj = self.get_from_memory(context, name_to_use)
+            if obj is None:
+                return self.sheerka.new(BuiltinConcepts.NOT_FOUND, body={"#name": name})
+
+            if isinstance(obj, list):
+                obj = obj[-1]
+
+            return obj.obj
+
+        res = {}
+        for k in self.objects:
+            obj = self.objects.get(k)
+            if isinstance(obj, list):
+                obj = obj[-1]
+            res[k] = obj.obj
+
+        return res

src/core/sheerka/services/SheerkaModifyConcept.py

@@ -1,18 +1,27 @@
 from core.builtin_concepts import BuiltinConcepts
+from core.concept import ensure_concept
 from core.sheerka.services.sheerka_service import BaseService
 from parsers.BnfParser import BnfParser
 
 
 class SheerkaModifyConcept(BaseService):
     NAME = "ModifyConcept"
-    
+
     def __init__(self, sheerka):
         super().__init__(sheerka)
 
     def initialize(self):
         self.sheerka.bind_service_method(self.modify_concept, True)
+        self.sheerka.bind_service_method(self.set_attr, True)
+        self.sheerka.bind_service_method(self.get_attr, False)
 
     def modify_concept(self, context, concept):
+        """
+        Modify the definition of a concept
+        :param context:
+        :param concept:
+        :return:
+        """
         old_version = self.sheerka.get_by_id(concept.id)
 
         if old_version is None:
@@ -38,16 +47,12 @@ class SheerkaModifyConcept(BaseService):
                     BuiltinConcepts.ALREADY_DEFINED,
                     body=concept))
 
-        old_references = self.sheerka.cache_manager.get(self.sheerka.CONCEPTS_REFERENCES_ENTRY, concept.id)
-        if old_references:
-            old_references = old_references.copy()
-
         self.sheerka.cache_manager.update_concept(old_version, concept)
 
-        # TODO : update concept by first keyword
-        # TODO : update resolved by first keyword
-        # TODO : update concepts grammars
-        # TODO : update when definition_type = DEFINITION_TYPE_DEF
+        # TODO : update concept by first keyword : have a look at update_references() below
+        # TODO : update resolved by first keyword : have a look at update_references() below
+        # TODO : update when definition_type = DEFINITION_TYPE_DEF : have a look at update_references() below
+        # TODO : Update concepts grammars : have a look at update_references() below
 
         ret = self.sheerka.ret(self.NAME, True, self.sheerka.new(BuiltinConcepts.NEW_CONCEPT, body=concept))
         return ret
@@ -69,3 +74,30 @@ class SheerkaModifyConcept(BaseService):
 
             if concept.bnf is not None:
                 BnfParser.update_recurse_id(context, concept_id, concept.bnf)
+
+            # remove the grammar entry so that it can be recreated
+            self.sheerka.cache_manager.delete(self.sheerka.CONCEPTS_GRAMMARS_ENTRY, concept_id)
+
+    def set_attr(self, concept, attribute, value):
+        """
+        Modifies an attribute of a concept (concept.values)
+        :param context:
+        :param concept:
+        :param attribute:
+        :param value:
+        :return:
+        """
+        ensure_concept(concept)
+        concept.set_value(attribute, value)
+        return self.sheerka.ret(self.NAME, True, self.sheerka.new(BuiltinConcepts.SUCCESS))
+
+    def get_attr(self, concept, attribute):
+        """
+        Returns the attribute of a concept
+        :param context:
+        :param concept:
+        :param attribute:
+        :return:
+        """
+        ensure_concept()
+        return concept.get_value(attribute)

src/core/sheerka/services/SheerkaSetsManager.py

@@ -59,10 +59,6 @@ class SheerkaSetsManager(BaseService):
 
         res = self.add_concept_to_set(context, concept, concept_set)
 
-        # update concept_set references
-        self.sheerka.services[SheerkaModifyConcept.NAME].update_references(context, concept_set)
-        self.concepts_in_set.delete(concept_set.id)
-
         return res
 
     def add_concept_to_set(self, context, concept, concept_set):
@@ -85,11 +81,21 @@ class SheerkaSetsManager(BaseService):
                 self.sheerka.new(BuiltinConcepts.CONCEPT_ALREADY_IN_SET, body=concept, concept_set=concept_set))
 
         self.sets.put(concept_set.id, concept.id)
+
+        # invalidate the cache of what contains concept_set
         self.concepts_in_set.delete(concept_set.id)
+
+        # update concept_set references
+        self.sheerka.services[SheerkaModifyConcept.NAME].update_references(context, concept_set)
+
         return self.sheerka.ret(self.NAME, True, self.sheerka.new(BuiltinConcepts.SUCCESS))
 
     def add_concepts_to_set(self, context, concepts, concept_set):
-        """Adding multiple concepts at the same time"""
+        """
+        Adding multiple concepts at the same time
+        ******** THIS METHOD IS FOR TEST ONLY *************
+        As it is not optimized. It needs to be rewritten in case of production usage
+        """
 
         context.log(f"Adding concepts {concepts} to set {concept_set}", who=self.NAME)
         ensure_concept(concept_set)

src/core/sheerka/services/SheerkaShortTermMemory.py

@@ -1,37 +0,0 @@
-from cache.ListIfNeededCache import ListIfNeededCache
-from core.sheerka.services.sheerka_service import BaseService
-
-
-class SheerkaShortTermMemory(BaseService):
-    NAME = "ShortTermMemory"
-
-    SHORT_TERM_MEMORY_ENTRY = "ShortTermMemory:Objects"
-
-    def __init__(self, sheerka):
-        super().__init__(sheerka)
-        self.objects = ListIfNeededCache()
-
-    def initialize(self):
-        self.sheerka.bind_service_method(self.get_from_short_term_memory, False)
-        self.sheerka.bind_service_method(self.add_to_short_term_memory, True)
-        self.sheerka.cache_manager.register_cache(self.SHORT_TERM_MEMORY_ENTRY, self.objects, persist=False)
-
-    def get_from_short_term_memory(self, context, key):
-        while True:
-            key_to_use = (str(context.id) if context else "") + ":" + key
-            if (obj := self.sheerka.cache_manager.get(self.SHORT_TERM_MEMORY_ENTRY, key_to_use)) is not None:
-                return obj
-
-            if context is None:
-                return None
-
-            context = context.get_parent()
-
-    def add_to_short_term_memory(self, context, key, concept):
-        if context:
-            context.stm = True
-        key_to_use = (str(context.id) if context else "") + ":" + key
-        return self.sheerka.cache_manager.put(self.SHORT_TERM_MEMORY_ENTRY, key_to_use, concept)
-
-    def remove_context(self, context):
-        self.objects.evict_by_key(lambda k: k.startswith(str(context.id) + ":"))

src/evaluators/AddToMemoryEvaluator.py

@@ -0,0 +1,27 @@
+from core.builtin_concepts import BuiltinConcepts
+from evaluators.BaseEvaluator import OneReturnValueEvaluator
+
+
+class AddToMemoryEvaluator(OneReturnValueEvaluator):
+    """
+    Last chance to alter the return_value
+    This evaluator is supposed to be a generic evaluator for all rules that must be executed just before
+        the aggregations
+    """
+
+    NAME = "AddToMemory"
+
+    def __init__(self):
+        super().__init__(self.NAME, [BuiltinConcepts.AFTER_EVALUATION], 10)
+
+    def matches(self, context, return_value):
+        evaluation_parents = context.get_parents(lambda c: c.action == BuiltinConcepts.PROCESSING)
+        if len(evaluation_parents) > 1:
+            return False  # It must be executed only when the top level context
+
+        from core.sheerka.services.SheerkaMemory import SheerkaMemory
+        return len(context.sheerka.services[SheerkaMemory.NAME].registration) > 0
+
+    def eval(self, context, return_value):
+        context.sheerka.add_registered_objects(context)
+        return None  # no need to have a second pass

src/evaluators/ConceptEvaluator.py

@@ -53,8 +53,9 @@ class ConceptEvaluator(OneReturnValueEvaluator):
 
         evaluated = sheerka.evaluate_concept(context, concept)
 
-        if evaluated.key != concept.key:
+        if not sheerka.is_success(evaluated) and evaluated.key != concept.key:
             # evaluated.key != concept.key means that we have transformed the concept
+            # not sheerka.is_success(evaluated) means that it was transformed into an error
             # When you successfully evaluate an error, the status should not be false
             return sheerka.ret(
                 self.name,

src/evaluators/PostExecutionEvaluator.py

@@ -22,7 +22,7 @@ class PostExecutionEvaluator(OneReturnValueEvaluator):
 
         # only support the rule for the COMMANDS
         value = return_value.body
-        return isinstance(value, Concept) and context.sheerka.isa(value, context.sheerka.new(BuiltinConcepts.COMMAND))
+        return isinstance(value, Concept) and context.sheerka.isa(value, context.sheerka.new(BuiltinConcepts.AUTO_EVAL))
 
     def eval(self, context, return_value):
         # only support the rule for the COMMANDS

src/evaluators/PythonEvaluator.py

@@ -85,6 +85,9 @@ class PythonEvaluator(OneReturnValueEvaluator):
                 not_for_me = context.sheerka.new(BuiltinConcepts.NOT_FOR_ME, body=node)
                 return sheerka.ret(self.name, False, not_for_me, parents=[return_value])
 
+        # If we evaluate a Concept metadata which is NOT the body ex (pre, post, where...)
+        # We need to disable the function that may alter the state
+        # It's a poor way to have source code security check
         attr_under_eval = context.get_parents(lambda ec: ec.action == BuiltinConcepts.EVALUATING_ATTRIBUTE)
         if attr_under_eval:
             attr_under_eval = attr_under_eval[0]
@@ -256,7 +259,7 @@ class PythonEvaluator(OneReturnValueEvaluator):
             else:
                 context.log(f"Evaluating '{concept}'", self.name)
                 evaluated = context.sheerka.evaluate_concept(context, concept, eval_body=True)
-                if evaluated.key != concept.key:
+                if not context.sheerka.is_success(evaluated) and evaluated.key != concept.key:
                     context.log(f"Error while evaluating '{name}'. Skipping.", self.name)
                     continue
                 concept = evaluated

src/evaluators/RetEvaluator.py

@@ -1,40 +1,41 @@
-from core.builtin_concepts import BuiltinConcepts
-from core.builtin_helpers import ensure_evaluated
-from core.concept import Concept, ConceptParts
-from evaluators.BaseEvaluator import OneReturnValueEvaluator
-
-
-class RetEvaluator(OneReturnValueEvaluator):
-    """
-    The evaluator transforms the concept, using the RET metadata value
-    """
-
-    NAME = "Ret"
-
-    def __init__(self):
-        super().__init__(self.NAME, [BuiltinConcepts.EVALUATION], 10)
-
-    def matches(self, context, return_value):
-        return return_value.status and \
-               isinstance(return_value.value, Concept) and \
-               return_value.value.metadata.ret is not None
-
-    def eval(self, context, return_value):
-        sheerka = context.sheerka
-        concept = return_value.value
-        context.log(f"Processing ret value for concept {concept}.", self.name)
-
-        if not concept.metadata.is_evaluated:
-            evaluated = ensure_evaluated(context, concept)
-            if evaluated.key != concept.key:
-                context.log(f"Failed to evaluate concept '{concept}'")
-                return None
-            ret = evaluated.get_value(ConceptParts.RET)
-        else:
-            ret = concept.get_value(ConceptParts.RET)
-
-        if isinstance(ret, Concept) and sheerka.is_known(ret):
-            return sheerka.ret(self.name, True, ret, parents=[return_value])
-
-        context.log(f"ret '{ret}' is not a concept!")
-        return None
+# from core.builtin_concepts import BuiltinConcepts
+# from core.builtin_helpers import ensure_evaluated
+# from core.concept import Concept, ConceptParts
+# from evaluators.BaseEvaluator import OneReturnValueEvaluator
+#
+#
+# class RetEvaluator(OneReturnValueEvaluator):
+#     """
+#     The evaluator transforms the concept, using the RET metadata value
+#     """
+#
+#     NAME = "Ret"
+#
+#     def __init__(self):
+#         super().__init__(self.NAME, [BuiltinConcepts.EVALUATION], 10)
+#         self.enabled = False
+#
+#     def matches(self, context, return_value):
+#         return return_value.status and \
+#                isinstance(return_value.value, Concept) and \
+#                return_value.value.metadata.ret is not None
+#
+#     def eval(self, context, return_value):
+#         sheerka = context.sheerka
+#         concept = return_value.value
+#         context.log(f"Processing ret value for concept {concept}.", self.name)
+#
+#         if not concept.metadata.is_evaluated:
+#             evaluated = ensure_evaluated(context, concept)
+#             if evaluated.key != concept.key:
+#                 context.log(f"Failed to evaluate concept '{concept}'")
+#                 return None
+#             ret = evaluated.get_value(ConceptParts.RET)
+#         else:
+#             ret = concept.get_value(ConceptParts.RET)
+#
+#         if isinstance(ret, Concept) and sheerka.is_known(ret):
+#             return sheerka.ret(self.name, True, ret, parents=[return_value])
+#
+#         context.log(f"ret '{ret}' is not a concept!")
+#         return None

src/evaluators/TooManySuccessEvaluator.py

@@ -1,8 +1,8 @@
 import logging
 
-from core.builtin_concepts import BuiltinConcepts
 import core.builtin_helpers
-from evaluators.BaseEvaluator import AllReturnValuesEvaluator, BaseEvaluator
+from core.builtin_concepts import BuiltinConcepts
+from evaluators.BaseEvaluator import AllReturnValuesEvaluator
 from parsers.BaseParser import BaseParser
 
 
@@ -44,7 +44,11 @@ class TooManySuccessEvaluator(AllReturnValuesEvaluator):
                 context.log(s, self.name)
                 context.log(f"value={sheerka.value(s.value)}", self.name)
 
-        if not core.builtin_helpers.is_same_success(context, self.success):
+        same_success = core.builtin_helpers.is_same_success(context, self.success)
+        if same_success is None:
+            return None
+
+        if not same_success:
             context.log(f"Values are different. Raising {BuiltinConcepts.TOO_MANY_SUCCESS}.", self.name)
             too_many_success = sheerka.new(BuiltinConcepts.TOO_MANY_SUCCESS, body=self.success)
             return sheerka.ret(self.name, False, too_many_success, parents=self.eaten)

src/parsers/BaseNodeParser.py

@@ -772,8 +772,6 @@ class BaseNodeParser(BaseParser):
 
         if token.type == TokenKind.STRING:
             name = token.value[1:-1] if strip_quotes else token.value
-        elif token.type == TokenKind.KEYWORD:
-            name = token.value.value
         else:
             name = token.value
 

src/parsers/ExactConceptParser.py

@@ -48,7 +48,7 @@ class ExactConceptParser(BaseParser):
         for combination in self.combinations(words):
 
             concept_key = " ".join(combination)
-            result = sheerka.new(concept_key)  # use new(), not get() because we need a new instance
+            result = sheerka.get_by_key(concept_key)  # use new(), not get() because we need a new instance
 
             if sheerka.isinstance(result, BuiltinConcepts.UNKNOWN_CONCEPT):
                 continue
@@ -66,6 +66,9 @@ class ExactConceptParser(BaseParser):
                     continue
 
                 context.log(f"Recognized concept {concept}.", self.name)
+                # We can ask for a new instance
+                concept = sheerka.new_from_template(concept, concept.key)
+
                 # update the properties if needed
                 for i, token in enumerate(combination):
                     if token.startswith(VARIABLE_PREFIX):

src/parsers/FunctionParser.py

@@ -38,6 +38,14 @@ class NamesNode(FunctionParserNode):
     def to_unrecognized(self):
         return UnrecognizedTokensNode(self.start, self.end, self.tokens).fix_source()
 
+    def to_str_unrecognized(self):
+        token = Token(TokenKind.STRING,
+                      "'" + self.str_value() + "'",
+                      self.tokens[0].index,
+                      self.tokens[0].line,
+                      self.tokens[0].column)
+        return UnrecognizedTokensNode(self.start, self.end, [token]).fix_source()
+
 
 @dataclass()
 class FunctionParameter:
@@ -322,10 +330,13 @@ class FunctionParser(BaseParser):
                 scn.add_node(sep.to_unrecognized())
 
         res = [SourceCodeWithConceptNode(function_node.first.to_unrecognized(), function_node.last.to_unrecognized())]
+
+        function_name = function_node.first.str_value()
+
         for param in function_node.parameters:
             if isinstance(param.value, NamesNode):
-                unrecognized = param.value.to_unrecognized()
                 # try to recognize concepts
+                unrecognized = param.value.to_unrecognized()
                 nodes_sequences = get_lexer_nodes_from_unrecognized(self.context,
                                                                     unrecognized,
                                                                     PARSERS)

src/parsers/SyaNodeParser.py

@@ -1219,7 +1219,8 @@ class SyaNodeParser(BaseNodeParser):
                 start = inner_item.start
             if inner_item.end > end:
                 end = inner_item.end
-            has_unrecognized |= isinstance(inner_item, (UnrecognizedTokensNode, SourceCodeWithConceptNode))
+            has_unrecognized |= isinstance(inner_item, (UnrecognizedTokensNode, SourceCodeWithConceptNode)) or \
+                hasattr(inner_item, "has_unrecognized") and inner_item.has_unrecognized
 
             param_name = concept.metadata.variables[param_index][0]
             param_value = inner_item.concept if hasattr(inner_item, "concept") else \

src/sdp/readme.md

@@ -14,6 +14,7 @@
 - D : concept definitions (no history management)
 - R : executionContext ('R' stands for Result or ReturnValue, no history management)
 - O : ServiceObj (from pickle)
+- M : MemoryObject (using SheerkaPickle)
 
 ## How concepts are serialized ?
 - get the id of the concept

src/sdp/sheerkaSerializer.py

@@ -60,6 +60,7 @@ class Serializer:
         self.register(ConceptSerializer())
         self.register(DictionarySerializer())
         self.register(ExecutionContextSerializer())
+        self.register(MemoryObjectSerializer()) # before ServiceObjSerializer
         self.register(ServiceObjSerializer())
 
     def register(self, serializer):
@@ -229,13 +230,14 @@ class StateSerializer(PickleSerializer):
             1)
 
 
-class ConceptSerializer(BaseSerializer):
+class SheerkaPickleSerializer(BaseSerializer):
 
-    def __init__(self):
-        BaseSerializer.__init__(self, "C", 1)
+    def __init__(self, predicate, name, version):
+        BaseSerializer.__init__(self, name, version)
+        self.predicate = predicate
 
     def matches(self, obj):
-        return isinstance(obj, Concept)
+        return self.predicate(obj)
 
     def dump(self, stream, obj, context):
         stream.write(sheerkapickle.encode(context.sheerka, obj).encode("utf-8"))
@@ -248,6 +250,12 @@ class ConceptSerializer(BaseSerializer):
         return obj
 
 
+class ConceptSerializer(SheerkaPickleSerializer):
+
+    def __init__(self):
+        super().__init__(lambda obj: isinstance(obj, Concept), "C", 1)
+
+
 class DictionarySerializer(BaseSerializer):
     def __init__(self):
         super().__init__("D", 1)
@@ -267,25 +275,11 @@ class DictionarySerializer(BaseSerializer):
         return obj
 
 
-class ExecutionContextSerializer(BaseSerializer):
+class ExecutionContextSerializer(SheerkaPickleSerializer):
     CLASS_NAME = "core.sheerka.ExecutionContext.ExecutionContext"
 
     def __init__(self):
-        BaseSerializer.__init__(self, "R", 1)
-
-    def matches(self, obj):
-        return get_full_qualified_name(obj) == self.CLASS_NAME
-
-    def dump(self, stream, obj, context):
-        stream.write(sheerkapickle.encode(context.sheerka, obj).encode("utf-8"))
-        stream.seek(0)
-        return stream
-
-    def load(self, stream, context):
-        json_stream = stream.read().decode("utf-8")
-        obj = sheerkapickle.decode(context.sheerka, json_stream)
-        # json_message = json.loads(json_stream)
-        return obj
+        super().__init__(lambda obj: get_full_qualified_name(obj) == self.CLASS_NAME, "R", 1)
 
 
 class ServiceObjSerializer(PickleSerializer):
@@ -297,7 +291,9 @@ class ServiceObjSerializer(PickleSerializer):
             "O",
             1)
 
-#
-# class SheerkaSerializer(ObjectSerializer):
-#     def __init__(self):
-#         ObjectSerializer.__init__(self, "core.sheerka.Sheerka", "C", 1)
+
+class MemoryObjectSerializer(SheerkaPickleSerializer):
+    CLASS_NAME = "core.sheerka.services.SheerkaMemory.MemoryObject"
+
+    def __init__(self):
+        super().__init__(lambda obj: get_full_qualified_name(obj) == self.CLASS_NAME, "R", 1)