Added set of set handling (thru concept ISA)

src/core/ast/nodes.py

@@ -139,7 +139,7 @@ def concept_to_python(concept_node):
             value = node.get_prop(field)
             if isinstance(value, list) or isinstance(value, Concept) and value.key == str(BuiltinConcepts.LIST):
                 lst = []
-                for i in value:
+                for i in value.body:
                     lst.append(_transform(i))
                 setattr(ast_object, field, lst)
             elif isinstance(value, NodeConcept):

src/core/ast/visitors.py

@@ -22,7 +22,7 @@ class ConceptNodeVisitor:
         """Called if no explicit visitor function exists for a node."""
         for field, value in iter_props(node):
             if isinstance(value, ListConcept):
-                for item in value:
+                for item in value.body:
                     if isinstance(item, NodeConcept):
                         self.visit(item)
             elif isinstance(value, NodeConcept):
@@ -97,8 +97,6 @@ class ExtractPredicateVisitor(ConceptNodeVisitor):
         self.variable_name = variable_name
 
 
-
-
 def get_parents(node):
     if node.parent is None:
         return []

src/core/builtin_concepts.py

@@ -52,6 +52,7 @@ class BuiltinConcepts(Enum):
     NOT_A_SET = "not a set"  # the concept has no entry in sets
     WHERE_CLAUSE_FAILED = "where clause failed"  # failed to validate where clause during evaluation
     CHICKEN_AND_EGG = "chicken and egg"  # infinite recursion when declaring concept
+    ISA = "is a"  # builtin concept to express that a concept is an instance of another one
 
     NODE = "node"
     GENERIC_NODE = "generic node"
@@ -340,8 +341,8 @@ class EnumerationConcept(Concept):
         self.set_metadata_value(ConceptParts.BODY, iteration)
         self.metadata.is_evaluated = True
 
-    def __iter__(self):
-        return iter(self.body)
+    # def __iter__(self):
+    #     return iter(self.body)
 
 
 class ListConcept(Concept):
@@ -353,20 +354,20 @@ class ListConcept(Concept):
     def append(self, obj):
         self.body.append(obj)
 
-    def __len__(self):
-        return len(self.body)
-
-    def __getitem__(self, key):
-        return self.body[key]
-
-    def __setitem__(self, key, value):
-        self.body[key] = value
-
-    def __iter__(self):
-        return iter(self.body)
-
-    def __contains__(self, item):
-        return item in self.body
+    # def __len__(self):
+    #     return len(self.body)
+    #
+    # def __getitem__(self, key):
+    #     return self.body[key]
+    #
+    # def __setitem__(self, key, value):
+    #     self.body[key] = value
+    #
+    # def __iter__(self):
+    #     return iter(self.body)
+    #
+    # def __contains__(self, item):
+    #     return item in self.body
 
 
 class ConceptAlreadyInSet(Concept):

src/core/builtin_helpers.py

@@ -7,7 +7,6 @@ from core.ast.visitors import UnreferencedNamesVisitor
 from core.builtin_concepts import BuiltinConcepts
 
 
-
 def is_same_success(sheerka, return_values):
     """
     Returns True if all returns values are successful and have the same value
@@ -206,7 +205,7 @@ def _extract_predicates(sheerka, node, variables_to_include, variables_to_exclud
     elif node.node_type == "BoolOp":
         all_op = True
         temp_res = []
-        for op in node.get_prop("values"):
+        for op in node.get_prop("values").body:
             res = _extract_predicates(sheerka, op, variables_to_include, variables_to_exclude)
             if len(res) == 0:
                 all_op = False

src/core/concept.py

@@ -47,6 +47,7 @@ class ConceptMetadata:
     id: str  # unique identifier for a concept. The id will never be modified (but the key can)
     props: list  # list properties, with their default values
     is_evaluated: bool = False  # True is the concept is evaluated by sheerka.eval_concept()
+    full_serialization: bool = False  # If True, the full object will be serialized, rather than just the diff
 
 
 simplec = namedtuple("concept", "name body")  # for simple concept (tests purposes only)
@@ -163,10 +164,10 @@ class Concept:
         name = self.name if 'metadata' in vars(self) else 'Concept'
         raise AttributeError(f"'{name}' concept has no attribute '{item}'")
 
-    def def_prop(self, prop_name: str, default_value=None):
+    def def_prop(self, prop_name, default_value=None):
         """
         Adds a property to the metadata
-        :param prop_name:
+        :param prop_name: name or concept
         :param default_value:
         :return:
         """
@@ -242,25 +243,6 @@ class Concept:
     def body(self):
         return self.values[ConceptParts.BODY] if ConceptParts.BODY in self.values else None
 
-    # def add_codes(self, codes):
-    #     """
-    #     Gets the ASTs for 'where', 'pre', 'post' and 'body'
-    #     There ASTs are know when the concept is freshly parsed.
-    #     So the values are kept in cache.
-    #
-    #     For concepts loaded from sdp, these ASTs must be created again
-    #     TODO : Seems to be a service method. Can be put somewhere else
-    #     :param codes:
-    #     :return:
-    #     """
-    #     if codes is None:
-    #         return
-    #
-    #     for key in codes:
-    #         self.compiled[key] = codes[key]
-    #
-    #     return self
-
     def get_digest(self):
         """
         Returns the digest of the event
@@ -321,12 +303,22 @@ class Concept:
 
         return self
 
-    def set_prop(self, prop_name: str, prop_value):
-        """Directly sets a value to a property"""
+    def set_prop(self, prop_name, prop_value):
+        """
+        Set the value of a property (not the metadata)
+        :param prop_name: Name the property or another concept
+        :param prop_value:
+        :return:
+        """
         self.props[prop_name] = Property(prop_name, prop_value)
         return self
 
     def get_prop(self, prop_name: str):
+        """
+        Gets the value of a property
+        :param prop_name: name or concept
+        :return:
+        """
         return self.props[prop_name].value
 
     def set_metadata_value(self, metadata: ConceptParts, value):

src/core/sheerka/Services/SheerkaCreateNewConcept.py

@@ -59,6 +59,7 @@ class SheerkaCreateNewConcept:
                     return self.sheerka.ret(self.logger_name, False, ErrorConcept(init_ret_value.value))
 
         # save the new concept in sdp
+        concept.metadata.full_serialization = True
         try:
             # TODO : needs to make these calls atomic (or at least one single call)
             # save the new concept
@@ -90,6 +91,7 @@ class SheerkaCreateNewConcept:
                 error.args[0])
 
         # Updates the caches
+        concept.metadata.full_serialization = False
         self.sheerka.cache_by_key[concept.key] = self.sheerka.sdp.get_safe(self.sheerka.CONCEPTS_ENTRY, concept.key)
         self.sheerka.cache_by_id[concept.id] = concept
         if init_ret_value is not None and init_ret_value.status:

src/core/sheerka/Services/SheerkaDump.py

@@ -31,6 +31,8 @@ class SheerkaDump:
 
     def dump_desc(self, *concept_names, eval=False):
         first = True
+        event = Event(f"Dumping description", "")
+        context = ExecutionContext("dump_desc", event, self.sheerka)
         for concept_name in concept_names:
             if isinstance(concept_name, Concept):
                 concepts = concept_name
@@ -45,8 +47,6 @@ class SheerkaDump:
 
             for c in concepts:
                 if eval:
-                    event = Event(f"Evaluating {c}", "")
-                    context = ExecutionContext("dump_desc", event, self.sheerka)
                     evaluated = self.sheerka.evaluate_concept(context, c)
                     value = evaluated.body if evaluated.key == c.key else evaluated
 
@@ -60,8 +60,8 @@ class SheerkaDump:
                     self.sheerka.log.info(f"value  : {value}")
                 self.sheerka.log.info(f"digest : {c.get_digest()}")
 
-                if self.sheerka.isaset(c):
-                    items = self.sheerka.get_set_elements(c)
+                if self.sheerka.isaset(context, c):
+                    items = self.sheerka.get_set_elements(context, c)
                     self.sheerka.log.info(f"elements  : {items}")
 
                 first = False

src/core/sheerka/Services/SheerkaEvaluateConcept.py

@@ -235,6 +235,12 @@ class SheerkaEvaluateConcept:
                         concept.set_prop(prop_name, resolved)
             else:
                 part_key = ConceptParts(metadata_to_eval)
+
+                # do not evaluate where when the body is a set
+                # Indeed, the way that the where clause is expressed is not a valid python or concept code
+                if part_key == ConceptParts.WHERE and self.sheerka.isaset(context, concept.body):
+                    continue
+
                 if part_key in concept.compiled and concept.compiled[part_key] is not None:
                     metadata_ast = concept.compiled[part_key]
                     resolved = self.resolve(context, metadata_ast, part_key, concept, logger)
@@ -244,7 +250,7 @@ class SheerkaEvaluateConcept:
                         concept.values[part_key] = self.get_infinite_recursion_resolution(resolved) or resolved
 
         # validate where clause
-        if concept.metadata.where is not None:
+        if ConceptParts.WHERE in concept.values:
             where_value = concept.values[ConceptParts.WHERE]
             if not (where_value is None or self.sheerka.value(where_value) is True):
                 return self.sheerka.new(BuiltinConcepts.WHERE_CLAUSE_FAILED, body=concept)

src/core/sheerka/Services/SheerkaHistoryManager.py

@@ -68,7 +68,7 @@ class SheerkaHistoryManager:
         events = list(self.sheerka.sdp.load_events(depth_or_digest, start))
         for event in events:
             try:
-                result = self.sheerka.sdp.load_result(self.sheerka, event.get_digest())
+                result = self.sheerka.sdp.load_result(event.get_digest())
             except (IOError, KeyError):
                 result = None
             yield History(event, result)

src/core/sheerka/Services/SheerkaModifyConcept.py

@@ -0,0 +1,15 @@
+from core.builtin_concepts import BuiltinConcepts
+
+
+class SheerkaModifyConcept:
+    def __init__(self, sheerka):
+        self.sheerka = sheerka
+        self.logger_name = self.modify_concept.__name__
+
+    def modify_concept(self, context, concept, logger=None):
+        logger = logger or self.sheerka.log
+
+        self.sheerka.sdp.modify(context.event.get_digest(), self.sheerka.CONCEPTS_ENTRY, concept.key, concept)
+
+        ret = self.sheerka.ret(self.logger_name, True, self.sheerka.new(BuiltinConcepts.NEW_CONCEPT, body=concept))
+        return ret

src/core/sheerka/Services/SheerkaSetsManager.py

@@ -1,5 +1,7 @@
+from core.ast.nodes import python_to_concept
 from core.builtin_concepts import BuiltinConcepts, ErrorConcept
-from core.concept import Concept
+from core.concept import Concept, ConceptParts
+import core.builtin_helpers
 
 GROUP_PREFIX = 'All_'
 
@@ -9,6 +11,30 @@ class SheerkaSetsManager:
         self.sheerka = sheerka
         self.logger_name = self.add_concept_to_set.__name__
 
+    def set_isa(self, context, concept, concept_set, logger=None):
+        """
+        Defines that concept a is b is another concept
+        :param context:
+        :param concept:
+        :param concept_set:
+        :param logger:
+        :return:
+        """
+
+        logger = logger or self.sheerka.log
+        context.log(logger, f"Setting that concept {concept} is a {concept_set}", who=self.logger_name)
+
+        isa = [] if BuiltinConcepts.ISA not in concept.props else concept.get_prop(BuiltinConcepts.ISA)
+        if concept_set not in isa:
+            isa.append(concept_set)
+
+        concept.set_prop(BuiltinConcepts.ISA, isa)
+        res = self.sheerka.modify_concept(context, concept, logger)
+        if not res.status:
+            return res
+
+        return self.add_concept_to_set(context, concept, concept_set, logger)
+
     def add_concept_to_set(self, context, concept, concept_set, logger=None):
         """
         Add an entry in sdp to tell that concept isa concept_set
@@ -53,24 +79,51 @@ class SheerkaSetsManager:
             context.log_error(logger, "Failed to add to set.", who=self.logger_name)
             return self.sheerka.ret(self.logger_name, False, ErrorConcept(error), error.args[0])
 
-    def get_set_elements(self, concept):
+    def get_set_elements(self, context, concept, logger=None):
         """
         Concept is supposed to be a set
         Returns all elements if the set
+        :param context:
         :param concept:
+        :param logger:
         :return:
         """
 
-        assert concept.id
+        logger = logger or self.sheerka.log
 
-        ids = self.sheerka.sdp.get_safe(GROUP_PREFIX + concept.id)
-        if ids is None:
-            return self.sheerka.new(BuiltinConcepts.NOT_A_SET, body=concept)
+        # noinspection PyShadowingNames
+        def _get_set_elements(context, concept, sub_concept):
+            if not (isinstance(sub_concept, Concept) and sub_concept.id):
+                return self.sheerka.new(BuiltinConcepts.NOT_A_SET, body=concept)
 
-        elements = [self.sheerka.get_by_id(element_id) for element_id in ids]
-        return elements
+            ids = self.sheerka.sdp.get_safe(GROUP_PREFIX + sub_concept.id)
+            if ids:
+                if concept.metadata.where:
+                    new_condition = self._validate_where_clause(concept)
+                    if not new_condition:
+                        return self.sheerka.new(BuiltinConcepts.WHERE_CLAUSE_FAILED, body=concept)
+                    else:
+                        # This methods sucks, but I don't have enough tools (like proper AST manipulation functions)
+                        # to do it properly now. It will be enhanced later
+                        concepts = self._get_concepts(context, ids, True, logger)
+                        globals_ = {"xx__concepts__xx": concepts, "sheerka": self.sheerka}
+                        locals_ = {}
+                        exec(new_condition, globals_, locals_)
+                        return locals_["result"]
+                else:
+                    return self._get_concepts(context, ids, False, logger)
 
-    def isa(self, a, b):
+            # it may be a concept that references a set
+            if not sub_concept.metadata.is_evaluated:
+                with context.push(desc=f"Evaluating concept {sub_concept}") as sub_context:
+                    evaluated = self.sheerka.evaluate_concept(sub_context, sub_concept)
+                    if evaluated.key != concept.key:
+                        return False
+            return _get_set_elements(context, concept, sub_concept.body)
+
+        return _get_set_elements(context, concept, concept)
+
+    def isinset(self, a, b):
         """
         return true if the concept a is a b
         Will handle when the keyword isa will be implemented
@@ -82,17 +135,101 @@ class SheerkaSetsManager:
         if isinstance(a, BuiltinConcepts):  # common KSI error ;-)
             raise SyntaxError("Remember that the first parameter of isinstance MUST be a concept")
 
-        assert isinstance(a, Concept)
-        assert isinstance(b, Concept)
+        if not (isinstance(a, Concept) and isinstance(b, Concept)):
+            return False
 
         # TODO, first check the 'isa' property of a
+        if not (a.id and b.id):
+            return False
 
-        return self.sheerka.sdp.exists(GROUP_PREFIX + b.id, a.id)
+        if self.sheerka.sdp.exists(GROUP_PREFIX + b.id, a.id):
+            return True
 
-    def isaset(self, concept):
-        """True if exists All_<concept_id> in sdp"""
-        if not concept.id:
+        return False
+
+    def isa(self, a, b):
+
+        if BuiltinConcepts.ISA not in a.props:
+            return False
+
+        for c in a.get_prop(BuiltinConcepts.ISA):
+            if c == b:
+                return True
+            if self.isa(c, b):
+                return True
+
+        return False
+
+    def isaset(self, context, concept, logger=None):
+        """
+        True if exists All_<concept_id> in sdp or if concept references to a concept that has all_<concept_id>
+        :param context:
+        :param concept:
+        :param logger:
+        :return:
+        """
+        """"""
+
+        logger = logger or self.sheerka.log
+
+        if not (isinstance(concept, Concept) and concept.id):
             return None
 
+        # it may be a concept that references a set
+        if not concept.metadata.is_evaluated:
+            with context.push(desc=f"Evaluating concept {concept}") as sub_context:
+                evaluated = self.sheerka.evaluate_concept(sub_context, concept, logger)
+                if evaluated.key != concept.key:
+                    return False
+
+        if concept.body:
+            return self.isaset(context, concept.body, logger)
+
         res = self.sheerka.sdp.get_safe(GROUP_PREFIX + concept.id)
         return res is not None
+
+    def _validate_where_clause(self, concept):
+        python_parser_result = [r for r in concept.compiled[ConceptParts.WHERE] if r.who == "parsers.Python"]
+        if not python_parser_result or not python_parser_result[0].status:
+            return None
+
+        ast_ = python_parser_result[0].body.body.ast_
+        ast_as_concepts = python_to_concept(ast_)
+        names = core.builtin_helpers.get_names(self.sheerka, ast_as_concepts)
+        if len(names) != 1 or names[0] != concept.metadata.body:
+            return None
+
+        condition = concept.metadata.where.replace(concept.metadata.body, "sheerka.value(x)")
+        expression = f"""
+result=[]
+for x in xx__concepts__xx:
+    try:
+        if {condition}:
+            result.append(x)
+    except Exception:
+        pass
+"""
+        return expression
+
+    def _get_concepts(self, context, ids, evaluate, logger):
+        """
+        Gets concepts from a list of concepts ids
+        :param ids:
+        :param evaluate: if True, all the elements are evaluated before returned
+        :return:
+        """
+
+        if not ids:
+            return []
+
+        if not evaluate:
+            return [self.sheerka.get_by_id(element_id) for element_id in ids]
+
+        result = []
+        with context.push(desc=f"Evaluating concepts of a set") as sub_context:
+            for element_id in ids:
+                concept = self.sheerka.get_by_id(element_id)
+                evaluated = self.sheerka.evaluate_concept(context, concept, logger)
+                result.append(evaluated)
+
+        return result

src/core/sheerka/Sheerka.py

@@ -7,6 +7,7 @@ from core.sheerka.Services.SheerkaDump import SheerkaDump
 from core.sheerka.Services.SheerkaEvaluateConcept import SheerkaEvaluateConcept
 from core.sheerka.Services.SheerkaExecute import SheerkaExecute
 from core.sheerka.Services.SheerkaHistoryManager import SheerkaHistoryManager
+from core.sheerka.Services.SheerkaModifyConcept import SheerkaModifyConcept
 from core.sheerka.Services.SheerkaSetsManager import SheerkaSetsManager
 from sdp.sheerkaDataProvider import SheerkaDataProvider, Event
 import core.utils
@@ -81,12 +82,14 @@ class Sheerka(Concept):
 
         self.execute_handler = SheerkaExecute(self)
         self.create_new_concept_handler = SheerkaCreateNewConcept(self)
+        self.modify_concept_handler = SheerkaModifyConcept(self)
         self.dump_handler = SheerkaDump(self)
         self.sets_handler = SheerkaSetsManager(self)
         self.evaluate_concept_handler = SheerkaEvaluateConcept(self)
         self.history_handler = SheerkaHistoryManager(self)
 
         self.during_restore = False
+        self._builtins_classes_cache = None
 
     def initialize(self, root_folder: str = None):
         """
@@ -101,7 +104,7 @@ class Sheerka(Concept):
             from sheerkapickle.sheerka_handlers import initialize_pickle_handlers
             initialize_pickle_handlers()
 
-            self.sdp = SheerkaDataProvider(root_folder)
+            self.sdp = SheerkaDataProvider(root_folder, self)
             if self.sdp.first_time:
                 self.sdp.set_key(self.USER_CONCEPTS_KEYS, 1000)
 
@@ -117,7 +120,7 @@ class Sheerka(Concept):
 
             exec_context.add_values(return_values=res)
             if not self.skip_builtins_in_db:
-                self.sdp.save_result(self, exec_context)
+                self.sdp.save_result(exec_context)
 
         except IOError as e:
             res = ReturnValueConcept(self, False, self.get(BuiltinConcepts.ERROR), e)
@@ -151,6 +154,7 @@ class Sheerka(Concept):
                 if from_db is None:
                     self.init_log.debug(f"'{concept.name}' concept is not found in db. Adding.")
                     self.set_id_if_needed(concept, True)
+                    concept.metadata.full_serialization = True
                     self.sdp.add("init", self.CONCEPTS_ENTRY, concept, use_ref=True)
                 else:
                     self.init_log.debug(f"Found concept '{from_db}' in db. Updating.")
@@ -247,9 +251,9 @@ class Sheerka(Concept):
             execution_context.add_values(return_values=ret)
 
             if not self.skip_builtins_in_db:
-                self.sdp.save_result(self, execution_context)
+                self.sdp.save_result(execution_context)
 
-        # hack to save valid concept definition
+        # # hack to save valid concept definition
         # if not self.during_restore:
         #     if len(ret) == 1 and ret[0].status and self.isinstance(ret[0].value, BuiltinConcepts.NEW_CONCEPT):
         #         with open(CONCEPTS_FILE, "a") as f:
@@ -294,6 +298,9 @@ class Sheerka(Concept):
 
         return self.create_new_concept_handler.create_new_concept(context, concept, logger)
 
+    def modify_concept(self, context, concept: Concept, logger):
+        return self.modify_concept_handler.modify_concept(context, concept, logger)
+
     def add_concept_to_set(self, context, concept, concept_set, logger=None):
         """
         Add an entry in sdp to tell that concept isa concept_set
@@ -305,15 +312,27 @@ class Sheerka(Concept):
         """
         return self.sets_handler.add_concept_to_set(context, concept, concept_set, logger)
 
-    def get_set_elements(self, concept):
+    def set_isa(self, context, concept, concept_set, logger=None):
+        """
+
+        :param context:
+        :param concept:
+        :param concept_set:
+        :param logger:
+        :return:
+        """
+        return self.sets_handler.set_isa(context, concept, concept_set, logger)
+
+    def get_set_elements(self, context, concept):
         """
         Concept is supposed to be a set
         Returns all elements if the set
+        :param context:
         :param concept:
         :return:
         """
 
-        return self.sets_handler.get_set_elements(concept)
+        return self.sets_handler.get_set_elements(context, concept)
 
     def evaluate_concept(self, context, concept: Concept, logger=None):
         """
@@ -524,7 +543,10 @@ class Sheerka(Concept):
                 self.isinstance(objs, BuiltinConcepts.ENUMERATION)):
             objs = [objs]
 
-        return (self.value(obj) for obj in objs)
+        if isinstance(objs, list):
+            return (self.value(obj) for obj in objs)
+
+        return (self.value(obj) for obj in objs.body)
 
     def is_success(self, obj):
         if isinstance(obj, bool):  # quick win
@@ -562,11 +584,14 @@ class Sheerka(Concept):
 
         return a.key == b_key
 
+    def isinset(self, a, b):
+        return self.sets_handler.isinset(a, b)
+
     def isa(self, a, b):
         return self.sets_handler.isa(a, b)
 
-    def isaset(self, concept):
-        return self.sets_handler.isaset(concept)
+    def isaset(self, context, concept):
+        return self.sets_handler.isaset(context, concept)
 
     def get_evaluator_name(self, name):
         if self.evaluators_prefix is None: