Added set of set handling (thru concept ISA)

src/core/ast/nodes.py

@@ -139,7 +139,7 @@ def concept_to_python(concept_node):
             value = node.get_prop(field)
             if isinstance(value, list) or isinstance(value, Concept) and value.key == str(BuiltinConcepts.LIST):
                 lst = []
-                for i in value:
+                for i in value.body:
                     lst.append(_transform(i))
                 setattr(ast_object, field, lst)
             elif isinstance(value, NodeConcept):

src/core/ast/visitors.py

@@ -22,7 +22,7 @@ class ConceptNodeVisitor:
         """Called if no explicit visitor function exists for a node."""
         for field, value in iter_props(node):
             if isinstance(value, ListConcept):
-                for item in value:
+                for item in value.body:
                     if isinstance(item, NodeConcept):
                         self.visit(item)
             elif isinstance(value, NodeConcept):
@@ -97,8 +97,6 @@ class ExtractPredicateVisitor(ConceptNodeVisitor):
         self.variable_name = variable_name
 
 
-
-
 def get_parents(node):
     if node.parent is None:
         return []

src/core/builtin_concepts.py

@@ -52,6 +52,7 @@ class BuiltinConcepts(Enum):
     NOT_A_SET = "not a set"  # the concept has no entry in sets
     WHERE_CLAUSE_FAILED = "where clause failed"  # failed to validate where clause during evaluation
     CHICKEN_AND_EGG = "chicken and egg"  # infinite recursion when declaring concept
+    ISA = "is a"  # builtin concept to express that a concept is an instance of another one
 
     NODE = "node"
     GENERIC_NODE = "generic node"
@@ -340,8 +341,8 @@ class EnumerationConcept(Concept):
         self.set_metadata_value(ConceptParts.BODY, iteration)
         self.metadata.is_evaluated = True
 
-    def __iter__(self):
-        return iter(self.body)
+    # def __iter__(self):
+    #     return iter(self.body)
 
 
 class ListConcept(Concept):
@@ -353,20 +354,20 @@ class ListConcept(Concept):
     def append(self, obj):
         self.body.append(obj)
 
-    def __len__(self):
-        return len(self.body)
-
-    def __getitem__(self, key):
-        return self.body[key]
-
-    def __setitem__(self, key, value):
-        self.body[key] = value
-
-    def __iter__(self):
-        return iter(self.body)
-
-    def __contains__(self, item):
-        return item in self.body
+    # def __len__(self):
+    #     return len(self.body)
+    #
+    # def __getitem__(self, key):
+    #     return self.body[key]
+    #
+    # def __setitem__(self, key, value):
+    #     self.body[key] = value
+    #
+    # def __iter__(self):
+    #     return iter(self.body)
+    #
+    # def __contains__(self, item):
+    #     return item in self.body
 
 
 class ConceptAlreadyInSet(Concept):

src/core/builtin_helpers.py

@@ -7,7 +7,6 @@ from core.ast.visitors import UnreferencedNamesVisitor
 from core.builtin_concepts import BuiltinConcepts
 
 
-
 def is_same_success(sheerka, return_values):
     """
     Returns True if all returns values are successful and have the same value
@@ -206,7 +205,7 @@ def _extract_predicates(sheerka, node, variables_to_include, variables_to_exclud
     elif node.node_type == "BoolOp":
         all_op = True
         temp_res = []
-        for op in node.get_prop("values"):
+        for op in node.get_prop("values").body:
             res = _extract_predicates(sheerka, op, variables_to_include, variables_to_exclude)
             if len(res) == 0:
                 all_op = False

src/core/concept.py

@@ -47,6 +47,7 @@ class ConceptMetadata:
     id: str  # unique identifier for a concept. The id will never be modified (but the key can)
     props: list  # list properties, with their default values
     is_evaluated: bool = False  # True is the concept is evaluated by sheerka.eval_concept()
+    full_serialization: bool = False  # If True, the full object will be serialized, rather than just the diff
 
 
 simplec = namedtuple("concept", "name body")  # for simple concept (tests purposes only)
@@ -163,10 +164,10 @@ class Concept:
         name = self.name if 'metadata' in vars(self) else 'Concept'
         raise AttributeError(f"'{name}' concept has no attribute '{item}'")
 
-    def def_prop(self, prop_name: str, default_value=None):
+    def def_prop(self, prop_name, default_value=None):
         """
         Adds a property to the metadata
-        :param prop_name:
+        :param prop_name: name or concept
         :param default_value:
         :return:
         """
@@ -242,25 +243,6 @@ class Concept:
     def body(self):
         return self.values[ConceptParts.BODY] if ConceptParts.BODY in self.values else None
 
-    # def add_codes(self, codes):
-    #     """
-    #     Gets the ASTs for 'where', 'pre', 'post' and 'body'
-    #     There ASTs are know when the concept is freshly parsed.
-    #     So the values are kept in cache.
-    #
-    #     For concepts loaded from sdp, these ASTs must be created again
-    #     TODO : Seems to be a service method. Can be put somewhere else
-    #     :param codes:
-    #     :return:
-    #     """
-    #     if codes is None:
-    #         return
-    #
-    #     for key in codes:
-    #         self.compiled[key] = codes[key]
-    #
-    #     return self
-
     def get_digest(self):
         """
         Returns the digest of the event
@@ -321,12 +303,22 @@ class Concept:
 
         return self
 
-    def set_prop(self, prop_name: str, prop_value):
-        """Directly sets a value to a property"""
+    def set_prop(self, prop_name, prop_value):
+        """
+        Set the value of a property (not the metadata)
+        :param prop_name: Name the property or another concept
+        :param prop_value:
+        :return:
+        """
         self.props[prop_name] = Property(prop_name, prop_value)
         return self
 
     def get_prop(self, prop_name: str):
+        """
+        Gets the value of a property
+        :param prop_name: name or concept
+        :return:
+        """
         return self.props[prop_name].value
 
     def set_metadata_value(self, metadata: ConceptParts, value):

src/core/sheerka/Services/SheerkaCreateNewConcept.py

@@ -59,6 +59,7 @@ class SheerkaCreateNewConcept:
                     return self.sheerka.ret(self.logger_name, False, ErrorConcept(init_ret_value.value))
 
         # save the new concept in sdp
+        concept.metadata.full_serialization = True
         try:
             # TODO : needs to make these calls atomic (or at least one single call)
             # save the new concept
@@ -90,6 +91,7 @@ class SheerkaCreateNewConcept:
                 error.args[0])
 
         # Updates the caches
+        concept.metadata.full_serialization = False
         self.sheerka.cache_by_key[concept.key] = self.sheerka.sdp.get_safe(self.sheerka.CONCEPTS_ENTRY, concept.key)
         self.sheerka.cache_by_id[concept.id] = concept
         if init_ret_value is not None and init_ret_value.status:

src/core/sheerka/Services/SheerkaDump.py

@@ -31,6 +31,8 @@ class SheerkaDump:
 
     def dump_desc(self, *concept_names, eval=False):
         first = True
+        event = Event(f"Dumping description", "")
+        context = ExecutionContext("dump_desc", event, self.sheerka)
         for concept_name in concept_names:
             if isinstance(concept_name, Concept):
                 concepts = concept_name
@@ -45,8 +47,6 @@ class SheerkaDump:
 
             for c in concepts:
                 if eval:
-                    event = Event(f"Evaluating {c}", "")
-                    context = ExecutionContext("dump_desc", event, self.sheerka)
                     evaluated = self.sheerka.evaluate_concept(context, c)
                     value = evaluated.body if evaluated.key == c.key else evaluated
 
@@ -60,8 +60,8 @@ class SheerkaDump:
                     self.sheerka.log.info(f"value  : {value}")
                 self.sheerka.log.info(f"digest : {c.get_digest()}")
 
-                if self.sheerka.isaset(c):
-                    items = self.sheerka.get_set_elements(c)
+                if self.sheerka.isaset(context, c):
+                    items = self.sheerka.get_set_elements(context, c)
                     self.sheerka.log.info(f"elements  : {items}")
 
                 first = False

src/core/sheerka/Services/SheerkaEvaluateConcept.py

@@ -235,6 +235,12 @@ class SheerkaEvaluateConcept:
                         concept.set_prop(prop_name, resolved)
             else:
                 part_key = ConceptParts(metadata_to_eval)
+
+                # do not evaluate where when the body is a set
+                # Indeed, the way that the where clause is expressed is not a valid python or concept code
+                if part_key == ConceptParts.WHERE and self.sheerka.isaset(context, concept.body):
+                    continue
+
                 if part_key in concept.compiled and concept.compiled[part_key] is not None:
                     metadata_ast = concept.compiled[part_key]
                     resolved = self.resolve(context, metadata_ast, part_key, concept, logger)
@@ -244,7 +250,7 @@ class SheerkaEvaluateConcept:
                         concept.values[part_key] = self.get_infinite_recursion_resolution(resolved) or resolved
 
         # validate where clause
-        if concept.metadata.where is not None:
+        if ConceptParts.WHERE in concept.values:
             where_value = concept.values[ConceptParts.WHERE]
             if not (where_value is None or self.sheerka.value(where_value) is True):
                 return self.sheerka.new(BuiltinConcepts.WHERE_CLAUSE_FAILED, body=concept)

src/core/sheerka/Services/SheerkaHistoryManager.py

@@ -68,7 +68,7 @@ class SheerkaHistoryManager:
         events = list(self.sheerka.sdp.load_events(depth_or_digest, start))
         for event in events:
             try:
-                result = self.sheerka.sdp.load_result(self.sheerka, event.get_digest())
+                result = self.sheerka.sdp.load_result(event.get_digest())
             except (IOError, KeyError):
                 result = None
             yield History(event, result)

src/core/sheerka/Services/SheerkaModifyConcept.py

@@ -0,0 +1,15 @@
+from core.builtin_concepts import BuiltinConcepts
+
+
+class SheerkaModifyConcept:
+    def __init__(self, sheerka):
+        self.sheerka = sheerka
+        self.logger_name = self.modify_concept.__name__
+
+    def modify_concept(self, context, concept, logger=None):
+        logger = logger or self.sheerka.log
+
+        self.sheerka.sdp.modify(context.event.get_digest(), self.sheerka.CONCEPTS_ENTRY, concept.key, concept)
+
+        ret = self.sheerka.ret(self.logger_name, True, self.sheerka.new(BuiltinConcepts.NEW_CONCEPT, body=concept))
+        return ret

src/core/sheerka/Services/SheerkaSetsManager.py

@@ -1,5 +1,7 @@
+from core.ast.nodes import python_to_concept
 from core.builtin_concepts import BuiltinConcepts, ErrorConcept
-from core.concept import Concept
+from core.concept import Concept, ConceptParts
+import core.builtin_helpers
 
 GROUP_PREFIX = 'All_'
 
@@ -9,6 +11,30 @@ class SheerkaSetsManager:
         self.sheerka = sheerka
         self.logger_name = self.add_concept_to_set.__name__
 
+    def set_isa(self, context, concept, concept_set, logger=None):
+        """
+        Defines that concept a is b is another concept
+        :param context:
+        :param concept:
+        :param concept_set:
+        :param logger:
+        :return:
+        """
+
+        logger = logger or self.sheerka.log
+        context.log(logger, f"Setting that concept {concept} is a {concept_set}", who=self.logger_name)
+
+        isa = [] if BuiltinConcepts.ISA not in concept.props else concept.get_prop(BuiltinConcepts.ISA)
+        if concept_set not in isa:
+            isa.append(concept_set)
+
+        concept.set_prop(BuiltinConcepts.ISA, isa)
+        res = self.sheerka.modify_concept(context, concept, logger)
+        if not res.status:
+            return res
+
+        return self.add_concept_to_set(context, concept, concept_set, logger)
+
     def add_concept_to_set(self, context, concept, concept_set, logger=None):
         """
         Add an entry in sdp to tell that concept isa concept_set
@@ -53,24 +79,51 @@ class SheerkaSetsManager:
             context.log_error(logger, "Failed to add to set.", who=self.logger_name)
             return self.sheerka.ret(self.logger_name, False, ErrorConcept(error), error.args[0])
 
-    def get_set_elements(self, concept):
+    def get_set_elements(self, context, concept, logger=None):
         """
         Concept is supposed to be a set
         Returns all elements if the set
+        :param context:
         :param concept:
+        :param logger:
         :return:
         """
 
-        assert concept.id
+        logger = logger or self.sheerka.log
 
-        ids = self.sheerka.sdp.get_safe(GROUP_PREFIX + concept.id)
-        if ids is None:
-            return self.sheerka.new(BuiltinConcepts.NOT_A_SET, body=concept)
+        # noinspection PyShadowingNames
+        def _get_set_elements(context, concept, sub_concept):
+            if not (isinstance(sub_concept, Concept) and sub_concept.id):
+                return self.sheerka.new(BuiltinConcepts.NOT_A_SET, body=concept)
 
-        elements = [self.sheerka.get_by_id(element_id) for element_id in ids]
-        return elements
+            ids = self.sheerka.sdp.get_safe(GROUP_PREFIX + sub_concept.id)
+            if ids:
+                if concept.metadata.where:
+                    new_condition = self._validate_where_clause(concept)
+                    if not new_condition:
+                        return self.sheerka.new(BuiltinConcepts.WHERE_CLAUSE_FAILED, body=concept)
+                    else:
+                        # This methods sucks, but I don't have enough tools (like proper AST manipulation functions)
+                        # to do it properly now. It will be enhanced later
+                        concepts = self._get_concepts(context, ids, True, logger)
+                        globals_ = {"xx__concepts__xx": concepts, "sheerka": self.sheerka}
+                        locals_ = {}
+                        exec(new_condition, globals_, locals_)
+                        return locals_["result"]
+                else:
+                    return self._get_concepts(context, ids, False, logger)
 
-    def isa(self, a, b):
+            # it may be a concept that references a set
+            if not sub_concept.metadata.is_evaluated:
+                with context.push(desc=f"Evaluating concept {sub_concept}") as sub_context:
+                    evaluated = self.sheerka.evaluate_concept(sub_context, sub_concept)
+                    if evaluated.key != concept.key:
+                        return False
+            return _get_set_elements(context, concept, sub_concept.body)
+
+        return _get_set_elements(context, concept, concept)
+
+    def isinset(self, a, b):
         """
         return true if the concept a is a b
         Will handle when the keyword isa will be implemented
@@ -82,17 +135,101 @@ class SheerkaSetsManager:
         if isinstance(a, BuiltinConcepts):  # common KSI error ;-)
             raise SyntaxError("Remember that the first parameter of isinstance MUST be a concept")
 
-        assert isinstance(a, Concept)
-        assert isinstance(b, Concept)
+        if not (isinstance(a, Concept) and isinstance(b, Concept)):
+            return False
 
         # TODO, first check the 'isa' property of a
+        if not (a.id and b.id):
+            return False
 
-        return self.sheerka.sdp.exists(GROUP_PREFIX + b.id, a.id)
+        if self.sheerka.sdp.exists(GROUP_PREFIX + b.id, a.id):
+            return True
 
-    def isaset(self, concept):
-        """True if exists All_<concept_id> in sdp"""
-        if not concept.id:
+        return False
+
+    def isa(self, a, b):
+
+        if BuiltinConcepts.ISA not in a.props:
+            return False
+
+        for c in a.get_prop(BuiltinConcepts.ISA):
+            if c == b:
+                return True
+            if self.isa(c, b):
+                return True
+
+        return False
+
+    def isaset(self, context, concept, logger=None):
+        """
+        True if exists All_<concept_id> in sdp or if concept references to a concept that has all_<concept_id>
+        :param context:
+        :param concept:
+        :param logger:
+        :return:
+        """
+        """"""
+
+        logger = logger or self.sheerka.log
+
+        if not (isinstance(concept, Concept) and concept.id):
             return None
 
+        # it may be a concept that references a set
+        if not concept.metadata.is_evaluated:
+            with context.push(desc=f"Evaluating concept {concept}") as sub_context:
+                evaluated = self.sheerka.evaluate_concept(sub_context, concept, logger)
+                if evaluated.key != concept.key:
+                    return False
+
+        if concept.body:
+            return self.isaset(context, concept.body, logger)
+
         res = self.sheerka.sdp.get_safe(GROUP_PREFIX + concept.id)
         return res is not None
+
+    def _validate_where_clause(self, concept):
+        python_parser_result = [r for r in concept.compiled[ConceptParts.WHERE] if r.who == "parsers.Python"]
+        if not python_parser_result or not python_parser_result[0].status:
+            return None
+
+        ast_ = python_parser_result[0].body.body.ast_
+        ast_as_concepts = python_to_concept(ast_)
+        names = core.builtin_helpers.get_names(self.sheerka, ast_as_concepts)
+        if len(names) != 1 or names[0] != concept.metadata.body:
+            return None
+
+        condition = concept.metadata.where.replace(concept.metadata.body, "sheerka.value(x)")
+        expression = f"""
+result=[]
+for x in xx__concepts__xx:
+    try:
+        if {condition}:
+            result.append(x)
+    except Exception:
+        pass
+"""
+        return expression
+
+    def _get_concepts(self, context, ids, evaluate, logger):
+        """
+        Gets concepts from a list of concepts ids
+        :param ids:
+        :param evaluate: if True, all the elements are evaluated before returned
+        :return:
+        """
+
+        if not ids:
+            return []
+
+        if not evaluate:
+            return [self.sheerka.get_by_id(element_id) for element_id in ids]
+
+        result = []
+        with context.push(desc=f"Evaluating concepts of a set") as sub_context:
+            for element_id in ids:
+                concept = self.sheerka.get_by_id(element_id)
+                evaluated = self.sheerka.evaluate_concept(context, concept, logger)
+                result.append(evaluated)
+
+        return result

src/core/sheerka/Sheerka.py

@@ -7,6 +7,7 @@ from core.sheerka.Services.SheerkaDump import SheerkaDump
 from core.sheerka.Services.SheerkaEvaluateConcept import SheerkaEvaluateConcept
 from core.sheerka.Services.SheerkaExecute import SheerkaExecute
 from core.sheerka.Services.SheerkaHistoryManager import SheerkaHistoryManager
+from core.sheerka.Services.SheerkaModifyConcept import SheerkaModifyConcept
 from core.sheerka.Services.SheerkaSetsManager import SheerkaSetsManager
 from sdp.sheerkaDataProvider import SheerkaDataProvider, Event
 import core.utils
@@ -81,12 +82,14 @@ class Sheerka(Concept):
 
         self.execute_handler = SheerkaExecute(self)
         self.create_new_concept_handler = SheerkaCreateNewConcept(self)
+        self.modify_concept_handler = SheerkaModifyConcept(self)
         self.dump_handler = SheerkaDump(self)
         self.sets_handler = SheerkaSetsManager(self)
         self.evaluate_concept_handler = SheerkaEvaluateConcept(self)
         self.history_handler = SheerkaHistoryManager(self)
 
         self.during_restore = False
+        self._builtins_classes_cache = None
 
     def initialize(self, root_folder: str = None):
         """
@@ -101,7 +104,7 @@ class Sheerka(Concept):
             from sheerkapickle.sheerka_handlers import initialize_pickle_handlers
             initialize_pickle_handlers()
 
-            self.sdp = SheerkaDataProvider(root_folder)
+            self.sdp = SheerkaDataProvider(root_folder, self)
             if self.sdp.first_time:
                 self.sdp.set_key(self.USER_CONCEPTS_KEYS, 1000)
 
@@ -117,7 +120,7 @@ class Sheerka(Concept):
 
             exec_context.add_values(return_values=res)
             if not self.skip_builtins_in_db:
-                self.sdp.save_result(self, exec_context)
+                self.sdp.save_result(exec_context)
 
         except IOError as e:
             res = ReturnValueConcept(self, False, self.get(BuiltinConcepts.ERROR), e)
@@ -151,6 +154,7 @@ class Sheerka(Concept):
                 if from_db is None:
                     self.init_log.debug(f"'{concept.name}' concept is not found in db. Adding.")
                     self.set_id_if_needed(concept, True)
+                    concept.metadata.full_serialization = True
                     self.sdp.add("init", self.CONCEPTS_ENTRY, concept, use_ref=True)
                 else:
                     self.init_log.debug(f"Found concept '{from_db}' in db. Updating.")
@@ -247,9 +251,9 @@ class Sheerka(Concept):
             execution_context.add_values(return_values=ret)
 
             if not self.skip_builtins_in_db:
-                self.sdp.save_result(self, execution_context)
+                self.sdp.save_result(execution_context)
 
-        # hack to save valid concept definition
+        # # hack to save valid concept definition
         # if not self.during_restore:
         #     if len(ret) == 1 and ret[0].status and self.isinstance(ret[0].value, BuiltinConcepts.NEW_CONCEPT):
         #         with open(CONCEPTS_FILE, "a") as f:
@@ -294,6 +298,9 @@ class Sheerka(Concept):
 
         return self.create_new_concept_handler.create_new_concept(context, concept, logger)
 
+    def modify_concept(self, context, concept: Concept, logger):
+        return self.modify_concept_handler.modify_concept(context, concept, logger)
+
     def add_concept_to_set(self, context, concept, concept_set, logger=None):
         """
         Add an entry in sdp to tell that concept isa concept_set
@@ -305,15 +312,27 @@ class Sheerka(Concept):
         """
         return self.sets_handler.add_concept_to_set(context, concept, concept_set, logger)
 
-    def get_set_elements(self, concept):
+    def set_isa(self, context, concept, concept_set, logger=None):
+        """
+
+        :param context:
+        :param concept:
+        :param concept_set:
+        :param logger:
+        :return:
+        """
+        return self.sets_handler.set_isa(context, concept, concept_set, logger)
+
+    def get_set_elements(self, context, concept):
         """
         Concept is supposed to be a set
         Returns all elements if the set
+        :param context:
         :param concept:
         :return:
         """
 
-        return self.sets_handler.get_set_elements(concept)
+        return self.sets_handler.get_set_elements(context, concept)
 
     def evaluate_concept(self, context, concept: Concept, logger=None):
         """
@@ -524,7 +543,10 @@ class Sheerka(Concept):
                 self.isinstance(objs, BuiltinConcepts.ENUMERATION)):
             objs = [objs]
 
-        return (self.value(obj) for obj in objs)
+        if isinstance(objs, list):
+            return (self.value(obj) for obj in objs)
+
+        return (self.value(obj) for obj in objs.body)
 
     def is_success(self, obj):
         if isinstance(obj, bool):  # quick win
@@ -562,11 +584,14 @@ class Sheerka(Concept):
 
         return a.key == b_key
 
+    def isinset(self, a, b):
+        return self.sets_handler.isinset(a, b)
+
     def isa(self, a, b):
         return self.sets_handler.isa(a, b)
 
-    def isaset(self, concept):
-        return self.sets_handler.isaset(concept)
+    def isaset(self, context, concept):
+        return self.sets_handler.isaset(context, concept)
 
     def get_evaluator_name(self, name):
         if self.evaluators_prefix is None:

src/evaluators/AddConceptEvaluator.py

@@ -61,7 +61,7 @@ class AddConceptEvaluator(OneReturnValueEvaluator):
             if not isinstance(part_ret_val, ReturnValueConcept) or not part_ret_val.status:
                 continue  # Nothing to do is not initialized
 
-            # update the parts
+            # update the metadata
             source = self.get_source(part_ret_val)
             setattr(concept.metadata, prop, source)
 

src/evaluators/EvalEvaluator.py

@@ -29,13 +29,21 @@ class EvalEvaluator(AllReturnValuesEvaluator):
 
         for ret_val in return_values:
             if ret_val.status and isinstance(ret_val.body, Concept) and ret_val.body.body:
-                context.log(self.verbose_log, f"Evaluating {ret_val}", who=self)
+                context.log(self.verbose_log, f"Evaluating {ret_val.body}", who=self)
                 result.append(sheerka.ret(self.name, True, ret_val.body.body, parents=[ret_val, self.eval_requested]))
+            elif ret_val.status and sheerka.isaset(context, ret_val.body):
+                context.log(self.verbose_log, f"Evaluating set {ret_val.body}", who=self)
+                result.append(sheerka.ret(
+                    self.name,
+                    True,
+                    sheerka.get_set_elements(context, ret_val.body),
+                    parents=[ret_val, self.eval_requested]))
 
         if len(result) > 0:
             return result
         else:
             # suppress the successful BuiltinConcepts.CONCEPT_EVAL_REQUESTED
+            # status of CONCEPT_EVAL_REQUESTED is now set to False
             return sheerka.ret(
                 self.name,
                 False,

src/evaluators/PythonEvaluator.py

@@ -42,17 +42,18 @@ class PythonEvaluator(OneReturnValueEvaluator):
                     return sheerka.ret(self.name, False, not_for_me, parents=[return_value])
 
             # get locals
-            my_locals = self.get_locals(context, node)
-            context.log(self.verbose_log, f"locals={my_locals}", self.name)
+            my_globals = self.get_globals(context, node)
+            my_locals = {}
+            context.log(self.verbose_log, f"globals={my_globals}", self.name)
 
             # eval
             if isinstance(node.ast_, ast.Expression):
                 context.log(self.verbose_log, "Evaluating using 'eval'.", self.name)
                 compiled = compile(node.ast_, "<string>", "eval")
-                evaluated = eval(compiled, {}, my_locals)
+                evaluated = eval(compiled, my_globals, my_locals)
             else:
                 context.log(self.verbose_log, "Evaluating using 'exec'.", self.name)
-                evaluated = self.exec_with_return(node.ast_, my_locals)
+                evaluated = self.exec_with_return(node.ast_, my_globals, my_locals)
 
             context.log(self.verbose_log, f"{evaluated=}", self.name)
             return sheerka.ret(self.name, True, evaluated, parents=[return_value])
@@ -62,7 +63,7 @@ class PythonEvaluator(OneReturnValueEvaluator):
             error = sheerka.new(BuiltinConcepts.ERROR, body=error)
             return sheerka.ret(self.name, False, error, parents=[return_value])
 
-    def get_locals(self, context, node):
+    def get_globals(self, context, node):
         my_locals = {
             "sheerka": context.sheerka,
             "desc": context.sheerka.dump_handler.dump_desc,
@@ -70,6 +71,7 @@ class PythonEvaluator(OneReturnValueEvaluator):
             "definitions": context.sheerka.dump_handler.dump_definitions,
             "history": context.sheerka.dump_handler.dump_history,
             "state": context.sheerka.dump_handler.dump_state,
+            "Concept": core.concept.Concept
         }
         if context.obj:
             context.log(self.verbose_log,
@@ -189,7 +191,7 @@ class PythonEvaluator(OneReturnValueEvaluator):
 
         return result
 
-    def exec_with_return(self, code_ast, my_locals):
+    def exec_with_return(self, code_ast, my_globals, my_locals):
 
         init_ast = copy.deepcopy(code_ast)
         init_ast.body = code_ast.body[:-1]
@@ -199,6 +201,6 @@ class PythonEvaluator(OneReturnValueEvaluator):
 
         exec(compile(init_ast, "<ast>", "exec"), {}, my_locals)
         if type(last_ast.body[0]) == ast.Expr:
-            return eval(compile(self.expr_to_expression(last_ast.body[0]), "<ast>", "eval"), {}, my_locals)
+            return eval(compile(self.expr_to_expression(last_ast.body[0]), "<ast>", "eval"), my_globals, my_locals)
         else:
-            exec(compile(last_ast, "<ast>", "exec"), {}, my_locals)
+            exec(compile(last_ast, "<ast>", "exec"), my_globals, my_locals)

src/parsers/BnfParser.py

@@ -236,7 +236,8 @@ class BnfParser(BaseParser):
         if token.type == TokenKind.CONCEPT:
             self.next_token()
             concept = self.sheerka.new((token.value[0], token.value[1]))
-            expr = ConceptGroupExpression(concept) if self.sheerka.isaset(concept) else ConceptExpression(concept)
+            expr = ConceptGroupExpression(concept) if self.sheerka.isaset(self.context, concept) \
+                else ConceptExpression(concept)
             return self.eat_rule_name_if_needed(expr)
 
         if token.type == TokenKind.IDENTIFIER:
@@ -260,7 +261,8 @@ class BnfParser(BaseParser):
                                      body=("key", concept_name)))
                 return None
             else:
-                expr = ConceptGroupExpression(concept) if self.sheerka.isaset(concept) else ConceptExpression(concept)
+                expr = ConceptGroupExpression(concept) if self.sheerka.isaset(self.context, concept) \
+                    else ConceptExpression(concept)
                 expr.rule_name = concept.name
                 return expr
 

src/parsers/ConceptLexerParser.py

@@ -318,7 +318,7 @@ class ConceptGroupExpression(ConceptExpression):
         self.concept = to_match  # Memoize
 
         if to_match not in parser.concepts_grammars:
-            concepts_in_group = parser.sheerka.get_set_elements(self.concept)
+            concepts_in_group = parser.sheerka.get_set_elements(parser.context, self.concept)
             nodes = [ConceptExpression(c, rule_name=c.name) for c in concepts_in_group]
             expr = OrderedChoice(nodes)
             expr.nodes = nodes
@@ -697,7 +697,7 @@ class ConceptLexerParser(BaseParser):
         # A copy must be created
         def inner_get_model(expression):
             if isinstance(expression, Concept):
-                if self.sheerka.isaset(expression):
+                if self.sheerka.isaset(self.context, expression):
                     ret = ConceptGroupExpression(expression, rule_name=expression.name)
                 else:
                     ret = ConceptExpression(expression, rule_name=expression.name)

src/parsers/DefaultParser.py

@@ -246,7 +246,7 @@ class DefaultParser(BaseParser):
 
         # the definition of a concept consists of several parts
         # Keywords.CONCEPT to get the name of the concept
-        # Keywords.FROM [Keywords.REGEX] to get the definition of the concept
+        # Keywords.FROM [Keywords.BNF] to get the definition of the concept
         # Keywords.AS to get the body
         # Keywords.WHERE to get the conditions to recognize for the variables
         # Keywords.PRE to know if the conditions to evaluate the concept

src/sdp/sheerkaDataProvider.py

@@ -294,16 +294,18 @@ class SheerkaDataProvider:
     KeysFile = "keys"
     REF_PREFIX = "##REF##:"
 
-    def __init__(self, root=None):
+    def __init__(self, root=None, sheerka=None):
         self.log = get_logger(__name__)
         self.init_log = get_logger("init." + __name__)
         self.init_log.debug("Initializing sdp.")
 
+        self.sheerka = sheerka
         self.io = SheerkaDataProviderIO.get(root)
         self.first_time = self.io.first_time
 
         self.serializer = Serializer()
 
+
     @staticmethod
     def get_obj_key(obj):
         """
@@ -729,13 +731,12 @@ class SheerkaDataProvider:
             digest = event.parents[0]
             count += 1
 
-    def save_result(self, sheerka, execution_context):
+    def save_result(self, execution_context):
         """
         Save the execution context associated with an event
         To make a long story short,
         for every single user input, there is an event (which is the first thing that is created)
         and a result (the ExecutionContext created by sheerka.evaluate_user_input()
-        :param sheerka:
         :param execution_context:
         :return:
         """
@@ -745,15 +746,15 @@ class SheerkaDataProvider:
         if self.io.exists(target_path):
             return digest
 
-        context = SerializerContext(sheerka=sheerka)
+        context = SerializerContext(sheerka=self.sheerka)
         self.io.write_binary(target_path, self.serializer.serialize(execution_context, context).read())
         return digest
 
-    def load_result(self, sheerka, digest):
+    def load_result(self, digest):
         target_path = self.io.get_obj_path(SheerkaDataProvider.EventFolder, digest) + "_result"
 
         with self.io.open(target_path, "rb") as f:
-            context = SerializerContext(sheerka=sheerka)
+            context = SerializerContext(sheerka=self.sheerka)
             return self.serializer.deserialize(f, context)
 
     def save_state(self, state: State):
@@ -763,7 +764,8 @@ class SheerkaDataProvider:
         if self.io.exists(target_path):
             return digest
 
-        self.io.write_binary(target_path, self.serializer.serialize(state, None).read())
+        context = SerializerContext(sheerka=self.sheerka)
+        self.io.write_binary(target_path, self.serializer.serialize(state, context).read())
         return digest
 
     def load_state(self, digest):
@@ -772,11 +774,13 @@ class SheerkaDataProvider:
 
         target_path = self.io.get_obj_path(SheerkaDataProvider.StateFolder, digest)
         with self.io.open(target_path, "rb") as f:
-            return self.serializer.deserialize(f, None)
+            context = SerializerContext(sheerka=self.sheerka)
+            return self.serializer.deserialize(f, context)
 
     def save_obj(self, obj):
         self.log.debug(f"Saving '{obj}' as reference...")
-        stream = self.serializer.serialize(obj, SerializerContext(user_name="kodjo"))
+        context = SerializerContext(user_name="kodjo", sheerka=self.sheerka)
+        stream = self.serializer.serialize(obj, context)
         digest = obj.get_digest() if hasattr(obj, "get_digest") else self.get_stream_digest(stream)
 
         target_path = self.io.get_obj_path(SheerkaDataProvider.ObjectsFolder, digest)
@@ -798,7 +802,8 @@ class SheerkaDataProvider:
             return None
 
         with self.io.open(target_path, "rb") as f:
-            obj = self.serializer.deserialize(f, SerializerContext(origin=digest))
+            context = SerializerContext(origin=digest, sheerka=self.sheerka)
+            obj = self.serializer.deserialize(f, context)
 
         # set the origin of the object
         if add_origin:

src/sdp/sheerkaSerializer.py

@@ -231,13 +231,24 @@ class StateSerializer(PickleSerializer):
             1)
 
 
-class ConceptSerializer(JsonSerializer):
+class ConceptSerializer(BaseSerializer):
+
     def __init__(self):
-        JsonSerializer.__init__(self, "core.concept.Concept", "C", 1)
+        BaseSerializer.__init__(self, "C", 1)
 
     def matches(self, obj):
         return isinstance(obj, Concept)
 
+    def dump(self, stream, obj, context):
+        stream.write(sheerkapickle.encode(context.sheerka, obj).encode("utf-8"))
+        stream.seek(0)
+        return stream
+
+    def load(self, stream, context):
+        json_stream = stream.read().decode("utf-8")
+        obj = sheerkapickle.decode(context.sheerka, json_stream)
+        return obj
+
 
 class DictionarySerializer(BaseSerializer):
     def __init__(self):
@@ -275,7 +286,7 @@ class ExecutionContextSerializer(BaseSerializer):
     def load(self, stream, context):
         json_stream = stream.read().decode("utf-8")
         obj = sheerkapickle.decode(context.sheerka, json_stream)
-        #json_message = json.loads(json_stream)
+        # json_message = json.loads(json_stream)
         return obj
 
 #

src/sheerkapickle/sheerka_handlers.py

@@ -16,11 +16,11 @@ class ConceptHandler(BaseHandler):
         pickler = self.context
         sheerka = self.sheerka
 
-        if obj.id:
+        if obj.metadata.full_serialization:
+            ref = default_concept
+        else:
             ref = sheerka.get_by_id(obj.id)
             data[CONCEPT_ID] = (obj.key, obj.id)
-        else:
-            ref = default_concept
 
         # transform metadata
         for prop in CONCEPT_PROPERTIES_TO_SERIALIZE:
@@ -41,7 +41,7 @@ class ConceptHandler(BaseHandler):
             if prop not in ref.props or value != ref.props[prop].value:
                 if "props" not in data:
                     data["props"] = []
-                data["props"].append((prop, pickler.flatten(value)))
+                data["props"].append((pickler.flatten(prop), pickler.flatten(value)))
 
         return data
 
@@ -77,7 +77,7 @@ class ConceptHandler(BaseHandler):
         return instance
 
 
-class UserInputHandler(BaseHandler):
+class UserInputHandler(ConceptHandler):
 
     def flatten(self, obj: UserInputConcept, data):
         data[CONCEPT_ID] = (obj.key, obj.id)
@@ -86,12 +86,12 @@ class UserInputHandler(BaseHandler):
         return data
 
     def new(self, data):
-        sheerka = self.sheerka
-
-        instance = sheerka.new(tuple(data[CONCEPT_ID]), body=data["text"], user_name=data["user_name"])
-        return instance
+        return UserInputConcept.__new__(UserInputConcept)
 
     def restore(self, data, instance):
+        instance.__init__(data["text"], data["user_name"])
+        instance.metadata.key = data[CONCEPT_ID][0]
+        instance.metadata.id = data[CONCEPT_ID][1]
         return instance
 
 
@@ -100,6 +100,7 @@ class ReturnValueHandler(BaseHandler):
     def flatten(self, obj: ReturnValueConcept, data):
         pickler = self.context
 
+        data[CONCEPT_ID] = (obj.key, obj.id)
         data["who"] = f"c:{obj.who.id}:" if isinstance(obj.who, Concept) else \
             obj.who.name if isinstance(obj.who, (BaseParser, BaseEvaluator)) else \
                 obj.who
@@ -110,38 +111,30 @@ class ReturnValueHandler(BaseHandler):
         return data
 
     def new(self, data):
-        sheerka = self.sheerka
-        instance = sheerka.ret(data["who"], data["status"], None)
-        return instance
+        return ReturnValueConcept.__new__(ReturnValueConcept)
 
     def restore(self, data, instance):
         pickler = self.context
 
-        instance.value = pickler.restore(data["value"])
+        instance.__init__(data["who"], data["status"], pickler.restore(data["value"]))
         if "parents" in data:
             instance.parents = pickler.restore(data["parents"])
+
+        instance.metadata.key = data[CONCEPT_ID][0]
+        instance.metadata.id = data[CONCEPT_ID][1]
         return instance
 
 
-# class BuiltinConceptsHandler(BaseHandler):
-#
-#     def flatten(self, obj: BuiltinConcepts, data):
-#         return data
-#
-#     def restore(self, obj):
-#         pass
+class SheerkaHandler(ConceptHandler):
 
-
-class SheerkaHandler(BaseHandler):
-
-    def flatten(self, obj: BuiltinConcepts, data):
-        return data
+    # def flatten(self, obj: BuiltinConcepts, data):
+    #     return ConceptHandler().flatten()data
 
     def new(self, data):
         return self.sheerka
 
-    def restore(self, data, instance):
-        return instance
+    # def restore(self, data, instance):
+    #     return instance
 
 
 class ExecutionContextHandler(BaseHandler):