Reimplemented explain feature

src/core/sheerka/services/SheerkaComparisonManager.py

@@ -24,8 +24,8 @@ class SheerkaComparisonManager(BaseService):
     Manage partitioning of concepts
     """
     NAME = "ComparisonManager"
-    COMPARISON_ENTRY = "Comparison"
-    RESOLVED_COMPARISON_ENTRY = "Resolved_Comparison"
+    COMPARISON_ENTRY = "ComparisonManager:Comparison"
+    RESOLVED_COMPARISON_ENTRY = "ComparisonManager:Resolved_Comparison"
 
     def __init__(self, sheerka):
         super().__init__(sheerka)

src/core/sheerka/services/SheerkaExecute.py

@@ -137,16 +137,15 @@ class SheerkaExecute(BaseService):
     """
 
     NAME = "Execute"
-    PARSERS_INPUTS_ENTRY = "ParserInput"  # entry for admin or internal variables
+    PARSERS_INPUTS_ENTRY = "Execute:ParserInput"  # entry for admin or internal variables
 
     def __init__(self, sheerka):
         super().__init__(sheerka)
-        self.pi_cache = None
+        self.pi_cache = Cache(default=lambda key: ParserInput(key), max_size=20)
 
     def initialize(self):
         self.sheerka.bind_service_method(self.execute)
 
-        self.pi_cache = Cache(default=lambda key: ParserInput(key), max_size=20)
         self.sheerka.cache_manager.register_cache(self.PARSERS_INPUTS_ENTRY, self.pi_cache, False)
 
     def get_parser_input(self, text, tokens=None):

src/core/sheerka/services/SheerkaFilter.py

@@ -0,0 +1,434 @@
+# the principle and the Pipe class are taken from
+# https://github.com/JulienPalard/Pipe
+#
+import builtins
+import functools
+import inspect
+import itertools
+import sys
+from collections import deque
+
+from cache.Cache import Cache
+from core.builtin_concepts import BuiltinConcepts
+from core.concept import Concept, ConceptParts
+from core.sheerka.services.sheerka_service import BaseService
+from core.utils import as_bag
+from printer.FormatInstructions import FormatInstructions
+from sheerkapickle.utils import is_primitive
+
+
+class PropDesc:
+    def __init__(self, class_name, props):
+        self.class_name = class_name
+        self.props = props
+
+    def __repr__(self):
+        return f"({self.class_name}{self.props})"
+
+    def __eq__(self, other):
+        if id(other) == id(self):
+            return True
+
+        if not isinstance(other, PropDesc):
+            return False
+
+        return self.class_name == other.class_name and sorted(self.props) == sorted(other.props)
+
+
+class Pipe:
+    """
+    Represent a Pipeable Element :
+    Described as :
+    first = Pipe(lambda iterable: next(iter(iterable)))
+    and used as :
+    print [1, 2, 3] | first
+    printing 1
+    Or represent a Pipeable Function :
+    It's a function returning a Pipe
+    Described as :
+    select = Pipe(lambda iterable, pred: (pred(x) for x in iterable))
+    and used as :
+    print [1, 2, 3] | select(lambda x: x * 2)
+    # 2, 4, 6
+    """
+
+    def __init__(self, function, context=None):
+        self.context = context
+
+        if isinstance(function, Pipe):
+            self.function = function.function
+            self.need_context = function.need_context
+        else:
+            signature = inspect.signature(function)
+            if len(signature.parameters) > 0 and list(signature.parameters.keys())[0] == "context":
+                self.need_context = True
+                self.function = (lambda x: function(context, x)) if len(signature.parameters) == 2 else function
+            else:
+                self.need_context = False
+                self.function = function
+
+        functools.update_wrapper(self, function)
+
+    def __ror__(self, other):
+        if isinstance(other, Concept) and other.key == str(BuiltinConcepts.EXPLANATION):
+            other.set_value(ConceptParts.BODY, self.function(other.body))
+            return other
+        return self.function(other)
+
+    def __call__(self, *args, **kwargs):
+        if self.need_context:
+            return Pipe(lambda x: self.function(self.context, x, *args, **kwargs), self.context)
+        else:
+            return Pipe(lambda x: self.function(x, *args, **kwargs), self.context)
+
+
+class SheerkaFilter(BaseService):
+    NAME = "Filter"
+    PREDICATES_ENTRY = "Filter:Predicates"
+
+    def __init__(self, sheerka):
+        super().__init__(sheerka)
+        self.cache = Cache(max_size=30)
+
+    def initialize(self):
+        # For a weird reason, when the attribute @Pipe is directly added to the function
+        # all following instances have the context property null
+        for k, v in SheerkaFilter.__dict__.items():
+            if k.startswith("pipe_"):
+                if isinstance(v, staticmethod):
+                    self.sheerka.add_pipeable(k[5:], v.__func__)
+                else:
+                    self.sheerka.add_pipeable(k[5:], v.__get__(self, self.__class__))
+
+        self.sheerka.cache_manager.register_cache(self.PREDICATES_ENTRY, self.cache, False, False)
+
+    def get_compiled(self, file_name, predicate):
+        """
+        Returns the compiled version of the predicate
+        :param file_name:
+        :param predicate:
+        :return:
+        """
+        compiled = self.cache.get(predicate)
+        if compiled is not None:
+            return compiled
+
+        compiled = compile(predicate, f"<{file_name}>", "eval")
+        self.cache.put(predicate, compiled)
+        return compiled
+
+    @staticmethod
+    def pipe_first(iterable):
+        """
+        Return the first element of the list
+        :param iterable:
+        :return:
+        """
+        return next(iter(iterable))
+
+    @staticmethod
+    def pipe_take(iterable, n):
+        """
+        Take the n first element of a list
+        :param iterable:
+        :param n:
+        :return:
+        """
+        for item in iterable:
+            if n > 0:
+                n -= 1
+                yield item
+            else:
+                return
+
+    @staticmethod
+    def pipe_props(iterable):
+        """
+        Return the list of available properties of the iterable
+        :return:
+        """
+        for item in iterable:
+            yield PropDesc(type(item).__name__, list(as_bag(item).keys()))
+
+    @staticmethod
+    def pipe_tail(iterable, qte):
+        "Yield qte of elements in the given iterable."
+        return deque(iterable, maxlen=qte)
+
+    @staticmethod
+    def pipe_skip(iterable, qte):
+        "Skip qte elements in the given iterable, then yield others."
+        for item in iterable:
+            if qte == 0:
+                yield item
+            else:
+                qte -= 1
+
+    @staticmethod
+    def pipe_dedup(iterable, key=lambda x: x):
+        """Only yield unique items. Use a set to keep track of duplicate data."""
+        seen = set()
+        for item in iterable:
+            dupkey = key(item)
+            if dupkey not in seen:
+                seen.add(dupkey)
+                yield item
+
+    @staticmethod
+    def pipe_uniq(iterable, key=lambda x: x):
+        """Deduplicate consecutive duplicate values."""
+        iterator = iter(iterable)
+        try:
+            prev = next(iterator)
+        except StopIteration:
+            return
+        yield prev
+        prevkey = key(prev)
+        for item in iterator:
+            itemkey = key(item)
+            if itemkey != prevkey:
+                yield item
+            prevkey = itemkey
+
+    @staticmethod
+    def pipe_all(iterable, pred):
+        """Returns True if ALL elements in the given iterable are true for the
+        given pred function"""
+        return builtins.all(pred(x) for x in iterable)
+
+    @staticmethod
+    def pipe_any(iterable, pred):
+        """Returns True if ANY element in the given iterable is True for the
+        given pred function"""
+        return builtins.any(pred(x) for x in iterable)
+
+    @staticmethod
+    def pipe_average(iterable):
+        """Build the average for the given iterable, starting with 0.0 as seed
+        Will try a division by 0 if the iterable is empty...
+        """
+        # warnings.warn(
+        #     "pipe.average is deprecated, use statistics.mean instead.",
+        #     DeprecationWarning,
+        #     stacklevel=4,
+        # )
+        total = 0.0
+        qte = 0
+        for element in iterable:
+            total += element
+            qte += 1
+        return total / qte
+
+    @staticmethod
+    def pipe_count(iterable):
+        "Count the size of the given iterable, walking thrue it."
+        # warnings.warn(
+        #     "pipe.count is deprecated, use the builtin len() instead.",
+        #     DeprecationWarning,
+        #     stacklevel=4,
+        # )
+        count = 0
+        for element in iterable:
+            count += 1
+        return count
+
+    @staticmethod
+    def pipe_max(iterable, **kwargs):
+        # warnings.warn(
+        #     "pipe.max is deprecated, use the builtin max() instead.",
+        #     DeprecationWarning,
+        #     stacklevel=4,
+        # )
+        return builtins.max(iterable, **kwargs)
+
+    @staticmethod
+    def pipe_min(iterable, **kwargs):
+        # warnings.warn(
+        #     "pipe.min is deprecated, use the builtin min() instead.",
+        #     DeprecationWarning,
+        #     stacklevel=4,
+        # )
+        return builtins.min(iterable, **kwargs)
+
+    @staticmethod
+    def pipe_as_dict(iterable):
+        # warnings.warn(
+        #     "pipe.as_dict is deprecated, use dict(your | pipe) instead.",
+        #     DeprecationWarning,
+        #     stacklevel=4,
+        # )
+        return dict(iterable)
+
+    @staticmethod
+    def pipe_as_set(iterable):
+        # warnings.warn(
+        #     "pipe.as_set is deprecated, use set(your | pipe) instead.",
+        #     DeprecationWarning,
+        #     stacklevel=4,
+        # )
+        return set(iterable)
+
+    @staticmethod
+    def pipe_permutations(iterable, r=None):
+        # permutations('ABCD', 2) --> AB AC AD BA BC BD CA CB CD DA DB DC
+        # permutations(range(3)) --> 012 021 102 120 201 210
+        for x in itertools.permutations(iterable, r):
+            yield x
+
+    # @staticmethod
+    # def pipe_netcat(to_send, host, port):
+    #     with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s:
+    #         s.connect((host, port))
+    #         for data in to_send | traverse:
+    #             s.send(data)
+    #         while 1:
+    #             data = s.recv(4096)
+    #             if not data:
+    #                 break
+    #             yield data
+    #
+    # @staticmethod
+    # def pipe_netwrite(to_send, host, port):
+    #     warnings.warn("pipe.netwite is deprecated.", DeprecationWarning, stacklevel=4)
+    #     with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s:
+    #         s.connect((host, port))
+    #         for data in to_send | SheerkaFilter.pipe_traverse:
+    #             s.send(data)
+
+    @staticmethod
+    def pipe_traverse(args):
+        for arg in args:
+            try:
+                if isinstance(arg, str):
+                    yield arg
+                else:
+                    for i in arg | SheerkaFilter.pipe_traverse:
+                        yield i
+            except TypeError:
+                # not iterable --- output leaf
+                yield arg
+
+    @staticmethod
+    def pipe_concat(iterable, separator=", "):
+        # warnings.warn(
+        #     "pipe.concat is deprecated, use ', '.join(your | pipe) instead.",
+        #     DeprecationWarning,
+        #     stacklevel=4,
+        # )
+        return separator.join(builtins.map(str, iterable))
+
+    @staticmethod
+    def pipe_as_list(iterable):
+        # warnings.warn(
+        #     "pipe.as_list is deprecated, use list(your | pipe) instead.",
+        #     DeprecationWarning,
+        #     stacklevel=4,
+        # )
+        return list(iterable)
+
+    @staticmethod
+    def pipe_as_tuple(iterable):
+        # warnings.warn(
+        #     "pipe.as_tuple is deprecated, use tuple(your | pipe) instead.",
+        #     DeprecationWarning,
+        #     stacklevel=4,
+        # )
+        return tuple(iterable)
+
+    @staticmethod
+    def pipe_tee(iterable):
+        for item in iterable:
+            sys.stdout.write(str(item) + "\n")
+            yield item
+
+    @staticmethod
+    def pipe_write(iterable, fname, glue="\n"):
+        with open(fname, "w") as f:
+            for item in iterable:
+                f.write(str(item) + glue)
+
+    @staticmethod
+    def pipe_add(x):
+        # warnings.warn(
+        #     "pipe.add is deprecated, use sum(your | pipe) instead.",
+        #     DeprecationWarning,
+        #     stacklevel=4,
+        # )
+        return sum(x)
+
+    @staticmethod
+    def pipe_select(iterable, selector):
+        return builtins.map(selector, iterable)
+
+    @staticmethod
+    def pipe_format_l(iterable, template, when=None):
+        """
+        Define a formatting when printing a list of items
+        :param iterable:
+        :param template:
+        :param when: format_l is set when the condition is verified
+        :return:
+        """
+        for item in iterable:
+            if hasattr(item, "get_format_instructions"):
+                instructions = item.get_format_instructions() or FormatInstructions()
+                instructions.set_format_l(item, template)
+                item.set_format_instructions(instructions)
+            yield item
+
+    @staticmethod
+    def pipe_format_d(iterable, *props, when=None, **format_l):
+        """
+        Define a formatting when printing the detail of an item
+        :param iterable:
+        :param props: list of properties to display
+        :param when: format_d is set when the condition is verified
+        :param format_l: custom formatting when printing the value of a property
+        :return:
+        """
+
+        template = dict((p, "{" + p + "}") for p in props)
+        for k, v in format_l.items():
+            template[k] = v
+
+        for item in iterable:
+            if hasattr(item, "get_format_instructions"):
+
+                if len(template) == 0:
+                    bag = as_bag(item)
+                    template = dict((p, "{" + p + "}") for p in bag)
+
+                instructions = item.get_format_instructions() or FormatInstructions()
+                instructions.set_format_d(item, template)
+                item.set_format_instructions(instructions)
+            yield item
+
+    @staticmethod
+    def pipe_recurse(iterable, depth, prop_name="children", when=None):
+        """
+        When printing an object that has sub properties,
+        indicate the depth of recursion to apply to a specific properties
+        Quick and dirty version because the prop name is not taken from the item (but set to 'children' by default)
+        :param iterable:
+        :param depth:
+        :param prop_name:
+        :param when: recurse is set when the condition is verified
+        :return:
+        """
+        for item in iterable:
+            if hasattr(item, "get_format_instructions"):
+                instructions = item.get_format_instructions() or FormatInstructions()
+                instructions.set_recurse(prop_name, depth)
+                item.set_format_instructions(instructions)
+            yield item
+
+    def pipe_filter(self, iterable, predicate):
+        compiled = self.get_compiled("filter", predicate)
+        for item in iterable:
+            try:
+                context = {} if is_primitive(item) else as_bag(item)
+                context["self"] = item
+                if eval(compiled, context):
+                    yield item
+            except NameError:
+                pass

src/core/sheerka/services/SheerkaHistoryManager.py

@@ -11,6 +11,7 @@ class History:
         self.event = event
         self.result = result
         self._status = None
+        self._format_instructions = None
 
     def __str__(self):
         msg = f"{self.event.get_digest()} {self.event.date.strftime('%d/%m/%Y %H:%M:%S')} : {self.event.message}"
@@ -42,6 +43,12 @@ class History:
         self._status = self.result.get_status() if self.result else None
         return self._status
 
+    def get_format_instructions(self):
+        return self._format_instructions
+
+    def set_format_instructions(self, instructions):
+        self._format_instructions = instructions
+
 
 class SheerkaHistoryManager(BaseService):
     NAME = "History"

src/core/sheerka/services/SheerkaResultManager.py

@@ -0,0 +1,125 @@
+from core.builtin_concepts import BuiltinConcepts
+from core.sheerka.services.sheerka_service import BaseService
+
+
+class SheerkaResultConcept(BaseService):
+    NAME = "Result"
+
+    def __init__(self, sheerka, page_size=30):
+        super().__init__(sheerka)
+        self.page_size = page_size
+
+    def initialize(self):
+        self.sheerka.bind_service_method(self.get_results_by_digest)
+        self.sheerka.bind_service_method(self.get_results_by_command)
+        self.sheerka.bind_service_method(self.get_last_results)
+        self.sheerka.bind_service_method(self.get_results)
+
+    def get_results_by_digest(self, context, digest, record_digest=True):
+        """
+        Gets the entire execution tree for the given event digest
+        :param context:
+        :param digest:
+        :param record_digest:
+        :return:
+        """
+        if digest is None:
+            return None
+
+        try:
+            result = self.sheerka.sdp.load_result(digest)
+            event = self.sheerka.sdp.load_event(digest)
+
+            if record_digest:
+                context.log(f"Recording digest '{digest}'")
+                self.sheerka.record(context, self.NAME, "digest", digest)
+
+            return self.sheerka.new(BuiltinConcepts.EXPLANATION,
+                                    digest=event.get_digest(),
+                                    command=event.message,
+                                    body=self.as_list(result))
+        except FileNotFoundError as ex:
+            context.log_error(f"Digest {digest} is not found.", self.NAME, ex)
+            return self.sheerka.new(BuiltinConcepts.NOT_FOUND, body={"digest": digest})
+
+    def get_results_by_command(self, context, command, record_digest=True):
+        """
+        Get the result of the command that starts with command
+        :param context:
+        :param command:
+        :param record_digest:
+        :return:
+        """
+        if command is None:
+            return None
+
+        start = 0
+        consumed = 0
+        while True:
+            for event in self.sheerka.sdp.load_events(self.page_size, start):
+                consumed += 1
+                if event.message.startswith(command):
+                    return self.get_results_by_digest(context, event.get_digest(), record_digest)
+
+            if consumed < self.page_size:
+                break
+
+            start += self.page_size
+            consumed = 0
+
+        return self.sheerka.new(BuiltinConcepts.NOT_FOUND, body={"command": command})
+
+    def get_last_results(self, context, record_digest=True):
+        """
+        Gets the results of the last command
+        :param context:
+        :param record_digest:
+        :return:
+        """
+
+        start = 0
+        page_size = 2
+        consumed = 0
+        while True:
+            for event in self.sheerka.sdp.load_events(page_size, start):
+                consumed += 1
+                if self.sheerka.sdp.has_result(event.get_digest()):
+                    return self.get_results_by_digest(context, event.get_digest(), record_digest)
+
+            if consumed < page_size:
+                break
+
+            if page_size < 100:
+                page_size *= 2
+
+            start += page_size
+            consumed = 0
+
+        return self.sheerka.new(BuiltinConcepts.NOT_FOUND, body={"query": "last"})
+
+    def get_results(self, context):
+        """
+        Use the last digest saved to get the execution results
+        :param context:
+        :return:
+        """
+
+        digest = self.sheerka.load(self.NAME, "digest")
+        if digest is None:
+            context.log("No recorded digest found.")
+            return None
+
+        return self.get_results_by_digest(context, digest, False)
+
+    @staticmethod
+    def as_list(execution_context):
+
+        def _yield_result(lst):
+
+            for e in lst:
+                yield e
+
+                if e.children:
+                    yield from _yield_result(e.children)
+
+        return _yield_result([execution_context])

src/core/sheerka/services/SheerkaSetsManager.py

@@ -10,7 +10,7 @@ GROUP_PREFIX = 'All_'
 
 class SheerkaSetsManager(BaseService):
     NAME = "SetsManager"
-    CONCEPTS_GROUPS_ENTRY = "Concepts_Groups"
+    CONCEPTS_GROUPS_ENTRY = "SetsManager:Concepts_Groups"
 
     def __init__(self, sheerka):
         super().__init__(sheerka)

src/core/sheerka/services/SheerkaVariableManager.py

@@ -21,7 +21,7 @@ class Variable(ServiceObj):
 
 class SheerkaVariableManager(BaseService):
     NAME = "VariableManager"
-    VARIABLES_ENTRY = "Variables"  # entry for admin or internal variables
+    VARIABLES_ENTRY = "VariableManager:Variables"  # entry for admin or internal variables
 
     def __init__(self, sheerka):
         super().__init__(sheerka)