Added set of set handling (thru concept ISA)

src/core/sheerka/Services/SheerkaCreateNewConcept.py

@@ -59,6 +59,7 @@ class SheerkaCreateNewConcept:
                     return self.sheerka.ret(self.logger_name, False, ErrorConcept(init_ret_value.value))
 
         # save the new concept in sdp
+        concept.metadata.full_serialization = True
         try:
             # TODO : needs to make these calls atomic (or at least one single call)
             # save the new concept
@@ -90,6 +91,7 @@ class SheerkaCreateNewConcept:
                 error.args[0])
 
         # Updates the caches
+        concept.metadata.full_serialization = False
         self.sheerka.cache_by_key[concept.key] = self.sheerka.sdp.get_safe(self.sheerka.CONCEPTS_ENTRY, concept.key)
         self.sheerka.cache_by_id[concept.id] = concept
         if init_ret_value is not None and init_ret_value.status:

src/core/sheerka/Services/SheerkaDump.py

@@ -31,6 +31,8 @@ class SheerkaDump:
 
     def dump_desc(self, *concept_names, eval=False):
         first = True
+        event = Event(f"Dumping description", "")
+        context = ExecutionContext("dump_desc", event, self.sheerka)
         for concept_name in concept_names:
             if isinstance(concept_name, Concept):
                 concepts = concept_name
@@ -45,8 +47,6 @@ class SheerkaDump:
 
             for c in concepts:
                 if eval:
-                    event = Event(f"Evaluating {c}", "")
-                    context = ExecutionContext("dump_desc", event, self.sheerka)
                     evaluated = self.sheerka.evaluate_concept(context, c)
                     value = evaluated.body if evaluated.key == c.key else evaluated
 
@@ -60,8 +60,8 @@ class SheerkaDump:
                     self.sheerka.log.info(f"value  : {value}")
                 self.sheerka.log.info(f"digest : {c.get_digest()}")
 
-                if self.sheerka.isaset(c):
-                    items = self.sheerka.get_set_elements(c)
+                if self.sheerka.isaset(context, c):
+                    items = self.sheerka.get_set_elements(context, c)
                     self.sheerka.log.info(f"elements  : {items}")
 
                 first = False

src/core/sheerka/Services/SheerkaEvaluateConcept.py

@@ -235,6 +235,12 @@ class SheerkaEvaluateConcept:
                         concept.set_prop(prop_name, resolved)
             else:
                 part_key = ConceptParts(metadata_to_eval)
+
+                # do not evaluate where when the body is a set
+                # Indeed, the way that the where clause is expressed is not a valid python or concept code
+                if part_key == ConceptParts.WHERE and self.sheerka.isaset(context, concept.body):
+                    continue
+
                 if part_key in concept.compiled and concept.compiled[part_key] is not None:
                     metadata_ast = concept.compiled[part_key]
                     resolved = self.resolve(context, metadata_ast, part_key, concept, logger)
@@ -244,7 +250,7 @@ class SheerkaEvaluateConcept:
                         concept.values[part_key] = self.get_infinite_recursion_resolution(resolved) or resolved
 
         # validate where clause
-        if concept.metadata.where is not None:
+        if ConceptParts.WHERE in concept.values:
             where_value = concept.values[ConceptParts.WHERE]
             if not (where_value is None or self.sheerka.value(where_value) is True):
                 return self.sheerka.new(BuiltinConcepts.WHERE_CLAUSE_FAILED, body=concept)

src/core/sheerka/Services/SheerkaHistoryManager.py

@@ -68,7 +68,7 @@ class SheerkaHistoryManager:
         events = list(self.sheerka.sdp.load_events(depth_or_digest, start))
         for event in events:
             try:
-                result = self.sheerka.sdp.load_result(self.sheerka, event.get_digest())
+                result = self.sheerka.sdp.load_result(event.get_digest())
             except (IOError, KeyError):
                 result = None
             yield History(event, result)

src/core/sheerka/Services/SheerkaModifyConcept.py

@@ -0,0 +1,15 @@
+from core.builtin_concepts import BuiltinConcepts
+
+
+class SheerkaModifyConcept:
+    def __init__(self, sheerka):
+        self.sheerka = sheerka
+        self.logger_name = self.modify_concept.__name__
+
+    def modify_concept(self, context, concept, logger=None):
+        logger = logger or self.sheerka.log
+
+        self.sheerka.sdp.modify(context.event.get_digest(), self.sheerka.CONCEPTS_ENTRY, concept.key, concept)
+
+        ret = self.sheerka.ret(self.logger_name, True, self.sheerka.new(BuiltinConcepts.NEW_CONCEPT, body=concept))
+        return ret

src/core/sheerka/Services/SheerkaSetsManager.py

@@ -1,5 +1,7 @@
+from core.ast.nodes import python_to_concept
 from core.builtin_concepts import BuiltinConcepts, ErrorConcept
-from core.concept import Concept
+from core.concept import Concept, ConceptParts
+import core.builtin_helpers
 
 GROUP_PREFIX = 'All_'
 
@@ -9,6 +11,30 @@ class SheerkaSetsManager:
         self.sheerka = sheerka
         self.logger_name = self.add_concept_to_set.__name__
 
+    def set_isa(self, context, concept, concept_set, logger=None):
+        """
+        Defines that concept a is b is another concept
+        :param context:
+        :param concept:
+        :param concept_set:
+        :param logger:
+        :return:
+        """
+
+        logger = logger or self.sheerka.log
+        context.log(logger, f"Setting that concept {concept} is a {concept_set}", who=self.logger_name)
+
+        isa = [] if BuiltinConcepts.ISA not in concept.props else concept.get_prop(BuiltinConcepts.ISA)
+        if concept_set not in isa:
+            isa.append(concept_set)
+
+        concept.set_prop(BuiltinConcepts.ISA, isa)
+        res = self.sheerka.modify_concept(context, concept, logger)
+        if not res.status:
+            return res
+
+        return self.add_concept_to_set(context, concept, concept_set, logger)
+
     def add_concept_to_set(self, context, concept, concept_set, logger=None):
         """
         Add an entry in sdp to tell that concept isa concept_set
@@ -53,24 +79,51 @@ class SheerkaSetsManager:
             context.log_error(logger, "Failed to add to set.", who=self.logger_name)
             return self.sheerka.ret(self.logger_name, False, ErrorConcept(error), error.args[0])
 
-    def get_set_elements(self, concept):
+    def get_set_elements(self, context, concept, logger=None):
         """
         Concept is supposed to be a set
         Returns all elements if the set
+        :param context:
         :param concept:
+        :param logger:
         :return:
         """
 
-        assert concept.id
+        logger = logger or self.sheerka.log
 
-        ids = self.sheerka.sdp.get_safe(GROUP_PREFIX + concept.id)
-        if ids is None:
-            return self.sheerka.new(BuiltinConcepts.NOT_A_SET, body=concept)
+        # noinspection PyShadowingNames
+        def _get_set_elements(context, concept, sub_concept):
+            if not (isinstance(sub_concept, Concept) and sub_concept.id):
+                return self.sheerka.new(BuiltinConcepts.NOT_A_SET, body=concept)
 
-        elements = [self.sheerka.get_by_id(element_id) for element_id in ids]
-        return elements
+            ids = self.sheerka.sdp.get_safe(GROUP_PREFIX + sub_concept.id)
+            if ids:
+                if concept.metadata.where:
+                    new_condition = self._validate_where_clause(concept)
+                    if not new_condition:
+                        return self.sheerka.new(BuiltinConcepts.WHERE_CLAUSE_FAILED, body=concept)
+                    else:
+                        # This methods sucks, but I don't have enough tools (like proper AST manipulation functions)
+                        # to do it properly now. It will be enhanced later
+                        concepts = self._get_concepts(context, ids, True, logger)
+                        globals_ = {"xx__concepts__xx": concepts, "sheerka": self.sheerka}
+                        locals_ = {}
+                        exec(new_condition, globals_, locals_)
+                        return locals_["result"]
+                else:
+                    return self._get_concepts(context, ids, False, logger)
 
-    def isa(self, a, b):
+            # it may be a concept that references a set
+            if not sub_concept.metadata.is_evaluated:
+                with context.push(desc=f"Evaluating concept {sub_concept}") as sub_context:
+                    evaluated = self.sheerka.evaluate_concept(sub_context, sub_concept)
+                    if evaluated.key != concept.key:
+                        return False
+            return _get_set_elements(context, concept, sub_concept.body)
+
+        return _get_set_elements(context, concept, concept)
+
+    def isinset(self, a, b):
         """
         return true if the concept a is a b
         Will handle when the keyword isa will be implemented
@@ -82,17 +135,101 @@ class SheerkaSetsManager:
         if isinstance(a, BuiltinConcepts):  # common KSI error ;-)
             raise SyntaxError("Remember that the first parameter of isinstance MUST be a concept")
 
-        assert isinstance(a, Concept)
-        assert isinstance(b, Concept)
+        if not (isinstance(a, Concept) and isinstance(b, Concept)):
+            return False
 
         # TODO, first check the 'isa' property of a
+        if not (a.id and b.id):
+            return False
 
-        return self.sheerka.sdp.exists(GROUP_PREFIX + b.id, a.id)
+        if self.sheerka.sdp.exists(GROUP_PREFIX + b.id, a.id):
+            return True
 
-    def isaset(self, concept):
-        """True if exists All_<concept_id> in sdp"""
-        if not concept.id:
+        return False
+
+    def isa(self, a, b):
+
+        if BuiltinConcepts.ISA not in a.props:
+            return False
+
+        for c in a.get_prop(BuiltinConcepts.ISA):
+            if c == b:
+                return True
+            if self.isa(c, b):
+                return True
+
+        return False
+
+    def isaset(self, context, concept, logger=None):
+        """
+        True if exists All_<concept_id> in sdp or if concept references to a concept that has all_<concept_id>
+        :param context:
+        :param concept:
+        :param logger:
+        :return:
+        """
+        """"""
+
+        logger = logger or self.sheerka.log
+
+        if not (isinstance(concept, Concept) and concept.id):
             return None
 
+        # it may be a concept that references a set
+        if not concept.metadata.is_evaluated:
+            with context.push(desc=f"Evaluating concept {concept}") as sub_context:
+                evaluated = self.sheerka.evaluate_concept(sub_context, concept, logger)
+                if evaluated.key != concept.key:
+                    return False
+
+        if concept.body:
+            return self.isaset(context, concept.body, logger)
+
         res = self.sheerka.sdp.get_safe(GROUP_PREFIX + concept.id)
         return res is not None
+
+    def _validate_where_clause(self, concept):
+        python_parser_result = [r for r in concept.compiled[ConceptParts.WHERE] if r.who == "parsers.Python"]
+        if not python_parser_result or not python_parser_result[0].status:
+            return None
+
+        ast_ = python_parser_result[0].body.body.ast_
+        ast_as_concepts = python_to_concept(ast_)
+        names = core.builtin_helpers.get_names(self.sheerka, ast_as_concepts)
+        if len(names) != 1 or names[0] != concept.metadata.body:
+            return None
+
+        condition = concept.metadata.where.replace(concept.metadata.body, "sheerka.value(x)")
+        expression = f"""
+result=[]
+for x in xx__concepts__xx:
+    try:
+        if {condition}:
+            result.append(x)
+    except Exception:
+        pass
+"""
+        return expression
+
+    def _get_concepts(self, context, ids, evaluate, logger):
+        """
+        Gets concepts from a list of concepts ids
+        :param ids:
+        :param evaluate: if True, all the elements are evaluated before returned
+        :return:
+        """
+
+        if not ids:
+            return []
+
+        if not evaluate:
+            return [self.sheerka.get_by_id(element_id) for element_id in ids]
+
+        result = []
+        with context.push(desc=f"Evaluating concepts of a set") as sub_context:
+            for element_id in ids:
+                concept = self.sheerka.get_by_id(element_id)
+                evaluated = self.sheerka.evaluate_concept(context, concept, logger)
+                result.append(evaluated)
+
+        return result

src/core/sheerka/Sheerka.py

@@ -7,6 +7,7 @@ from core.sheerka.Services.SheerkaDump import SheerkaDump
 from core.sheerka.Services.SheerkaEvaluateConcept import SheerkaEvaluateConcept
 from core.sheerka.Services.SheerkaExecute import SheerkaExecute
 from core.sheerka.Services.SheerkaHistoryManager import SheerkaHistoryManager
+from core.sheerka.Services.SheerkaModifyConcept import SheerkaModifyConcept
 from core.sheerka.Services.SheerkaSetsManager import SheerkaSetsManager
 from sdp.sheerkaDataProvider import SheerkaDataProvider, Event
 import core.utils
@@ -81,12 +82,14 @@ class Sheerka(Concept):
 
         self.execute_handler = SheerkaExecute(self)
         self.create_new_concept_handler = SheerkaCreateNewConcept(self)
+        self.modify_concept_handler = SheerkaModifyConcept(self)
         self.dump_handler = SheerkaDump(self)
         self.sets_handler = SheerkaSetsManager(self)
         self.evaluate_concept_handler = SheerkaEvaluateConcept(self)
         self.history_handler = SheerkaHistoryManager(self)
 
         self.during_restore = False
+        self._builtins_classes_cache = None
 
     def initialize(self, root_folder: str = None):
         """
@@ -101,7 +104,7 @@ class Sheerka(Concept):
             from sheerkapickle.sheerka_handlers import initialize_pickle_handlers
             initialize_pickle_handlers()
 
-            self.sdp = SheerkaDataProvider(root_folder)
+            self.sdp = SheerkaDataProvider(root_folder, self)
             if self.sdp.first_time:
                 self.sdp.set_key(self.USER_CONCEPTS_KEYS, 1000)
 
@@ -117,7 +120,7 @@ class Sheerka(Concept):
 
             exec_context.add_values(return_values=res)
             if not self.skip_builtins_in_db:
-                self.sdp.save_result(self, exec_context)
+                self.sdp.save_result(exec_context)
 
         except IOError as e:
             res = ReturnValueConcept(self, False, self.get(BuiltinConcepts.ERROR), e)
@@ -151,6 +154,7 @@ class Sheerka(Concept):
                 if from_db is None:
                     self.init_log.debug(f"'{concept.name}' concept is not found in db. Adding.")
                     self.set_id_if_needed(concept, True)
+                    concept.metadata.full_serialization = True
                     self.sdp.add("init", self.CONCEPTS_ENTRY, concept, use_ref=True)
                 else:
                     self.init_log.debug(f"Found concept '{from_db}' in db. Updating.")
@@ -247,9 +251,9 @@ class Sheerka(Concept):
             execution_context.add_values(return_values=ret)
 
             if not self.skip_builtins_in_db:
-                self.sdp.save_result(self, execution_context)
+                self.sdp.save_result(execution_context)
 
-        # hack to save valid concept definition
+        # # hack to save valid concept definition
         # if not self.during_restore:
         #     if len(ret) == 1 and ret[0].status and self.isinstance(ret[0].value, BuiltinConcepts.NEW_CONCEPT):
         #         with open(CONCEPTS_FILE, "a") as f:
@@ -294,6 +298,9 @@ class Sheerka(Concept):
 
         return self.create_new_concept_handler.create_new_concept(context, concept, logger)
 
+    def modify_concept(self, context, concept: Concept, logger):
+        return self.modify_concept_handler.modify_concept(context, concept, logger)
+
     def add_concept_to_set(self, context, concept, concept_set, logger=None):
         """
         Add an entry in sdp to tell that concept isa concept_set
@@ -305,15 +312,27 @@ class Sheerka(Concept):
         """
         return self.sets_handler.add_concept_to_set(context, concept, concept_set, logger)
 
-    def get_set_elements(self, concept):
+    def set_isa(self, context, concept, concept_set, logger=None):
+        """
+
+        :param context:
+        :param concept:
+        :param concept_set:
+        :param logger:
+        :return:
+        """
+        return self.sets_handler.set_isa(context, concept, concept_set, logger)
+
+    def get_set_elements(self, context, concept):
         """
         Concept is supposed to be a set
         Returns all elements if the set
+        :param context:
         :param concept:
         :return:
         """
 
-        return self.sets_handler.get_set_elements(concept)
+        return self.sets_handler.get_set_elements(context, concept)
 
     def evaluate_concept(self, context, concept: Concept, logger=None):
         """
@@ -524,7 +543,10 @@ class Sheerka(Concept):
                 self.isinstance(objs, BuiltinConcepts.ENUMERATION)):
             objs = [objs]
 
-        return (self.value(obj) for obj in objs)
+        if isinstance(objs, list):
+            return (self.value(obj) for obj in objs)
+
+        return (self.value(obj) for obj in objs.body)
 
     def is_success(self, obj):
         if isinstance(obj, bool):  # quick win
@@ -562,11 +584,14 @@ class Sheerka(Concept):
 
         return a.key == b_key
 
+    def isinset(self, a, b):
+        return self.sets_handler.isinset(a, b)
+
     def isa(self, a, b):
         return self.sets_handler.isa(a, b)
 
-    def isaset(self, concept):
-        return self.sets_handler.isaset(concept)
+    def isaset(self, context, concept):
+        return self.sets_handler.isaset(context, concept)
 
     def get_evaluator_name(self, name):
         if self.evaluators_prefix is None: