Reimplemented explain feature

src/core/sheerka/ExecutionContext.py

@@ -50,9 +50,10 @@ class ExecutionContext:
         self._parent = None
         self._id = ExecutionContext.get_id(event.get_digest()) if event else None
         self._tab = ""
-        self._bag = {}  # other variables
-        self._start = 0
-        self._stop = 0
+        self._bag = {}  # context variables
+        self._start = 0  # when the execution starts (to measure elapsed time)
+        self._stop = 0  # when the execution stops (to measure elapses time)
+        self._format_instructions = None # how to print the execution context
 
         self.who = who  # who is asking
         self.event = event  # what was the (original) trigger
@@ -65,11 +66,13 @@ class ExecutionContext:
         self.global_hints = set() if global_hints is None else global_hints
         self.global_errors = [] if global_errors is None else global_errors
 
+
         self.inputs = {}  # what was the parameters of the execution context
         self.values = {}  # what was produced by the execution context
 
-        self.obj = kwargs.pop("obj", None)
-        self.concepts = kwargs.pop("concepts", {})
+        self.obj = kwargs.pop("obj", None)  # current obj we are working on
+        self.concepts = kwargs.pop("concepts", {})  # known concepts specific to this context
+
         # update the other elements
         for k, v in kwargs.items():
             self._bag[k] = v
@@ -313,7 +316,7 @@ class ExecutionContext:
                 return None
             return ret_val.status
 
-    def to_bag(self):
+    def as_bag(self):
         """
         Creates a dictionary with the useful properties of the concept
         It quicker to implement than creating the actual property mechanism with @property
@@ -338,3 +341,9 @@ class ExecutionContext:
             value = value[:47] + "..."
         to_str = f"ReturnValue(who={r.who}, status={r.status}, value={value})"
         return to_str
+
+    def get_format_instructions(self):
+        return self._format_instructions
+
+    def set_format_instructions(self, instructions):
+        self._format_instructions = instructions

src/core/sheerka/Sheerka.py

@@ -83,6 +83,7 @@ class Sheerka(Concept):
             "test": self.test,
             "test_using_context": self.test_using_context
         }
+        self.sheerka_pipeables = {}
 
     @property
     def resolved_concepts_by_first_keyword(self):
@@ -121,6 +122,15 @@ class Sheerka(Concept):
 
         setattr(self, as_name, bound_method)
 
+    def add_pipeable(self, func_name, function):
+        """
+        Adds a function that can bu used with pipe '|'
+        :param func_name:
+        :param function:
+        :return:
+        """
+        self.sheerka_pipeables[func_name] = function
+
     def initialize(self, root_folder: str = None, save_execution_context=True):
         """
         Starting Sheerka
@@ -360,8 +370,11 @@ class Sheerka(Concept):
             if self.cache_manager.is_dirty:
                 self.cache_manager.commit(execution_context)
 
-            if self.save_execution_context and self.load(self.name, "save_execution_context"):
-                self.sdp.save_result(execution_context)
+            try:
+                if self.save_execution_context and self.load(self.name, "save_execution_context"):
+                    self.sdp.save_result(execution_context)
+            except Exception as ex:
+                self.log.error(f"Failed to save execution context. Reason: {ex}")
 
         # # hack to save valid concept definition
         # if not self.during_restore:
@@ -760,6 +773,9 @@ class Sheerka(Concept):
         except IOError:
             pass
 
+    def get_last_execution(self):
+        return self._last_execution
+
     def test(self):
         return f"I have access to Sheerka !"
 
@@ -841,6 +857,28 @@ class Sheerka(Concept):
 
     @staticmethod
     def init_logging(debug, loggers):
+        def add_coloring_to_emit_ansi(fn):
+            # add methods we need to the class
+            def new(*args):
+                levelno = args[1].levelno
+                if levelno >= 50:
+                    color = '\x1b[31m'  # red
+                elif levelno >= 40:
+                    color = '\x1b[31m'  # red
+                elif levelno >= 30:
+                    color = '\x1b[33m'  # yellow
+                elif levelno >= 20:
+                    color = '\x1b[32m'  # green
+                elif levelno >= 10:
+                    color = '\x1b[35m'  # pink
+                else:
+                    color = '\x1b[0m'  # normal
+                args[1].msg = color + str(args[1].msg) + '\x1b[0m'  # normal
+                # print "after"
+                return fn(*args)
+
+            return new
+
         core.sheerka_logger.init_config(loggers)
         if debug:
             log_format = "%(asctime)s"
@@ -853,3 +891,6 @@ class Sheerka(Concept):
             log_level = logging.INFO
 
         logging.basicConfig(format=log_format, level=log_level, handlers=[console_handler])
+        logging.addLevelName(logging.ERROR, "\033[1;41m%s\033[1;0m" % logging.getLevelName(logging.ERROR))
+        # uncomment the following line to enable colors
+        #logging.StreamHandler.emit = add_coloring_to_emit_ansi(logging.StreamHandler.emit)

src/core/sheerka/services/SheerkaComparisonManager.py

@@ -24,8 +24,8 @@ class SheerkaComparisonManager(BaseService):
     Manage partitioning of concepts
     """
     NAME = "ComparisonManager"
-    COMPARISON_ENTRY = "Comparison"
-    RESOLVED_COMPARISON_ENTRY = "Resolved_Comparison"
+    COMPARISON_ENTRY = "ComparisonManager:Comparison"
+    RESOLVED_COMPARISON_ENTRY = "ComparisonManager:Resolved_Comparison"
 
     def __init__(self, sheerka):
         super().__init__(sheerka)

src/core/sheerka/services/SheerkaExecute.py

@@ -137,16 +137,15 @@ class SheerkaExecute(BaseService):
     """
 
     NAME = "Execute"
-    PARSERS_INPUTS_ENTRY = "ParserInput"  # entry for admin or internal variables
+    PARSERS_INPUTS_ENTRY = "Execute:ParserInput"  # entry for admin or internal variables
 
     def __init__(self, sheerka):
         super().__init__(sheerka)
-        self.pi_cache = None
+        self.pi_cache = Cache(default=lambda key: ParserInput(key), max_size=20)
 
     def initialize(self):
         self.sheerka.bind_service_method(self.execute)
 
-        self.pi_cache = Cache(default=lambda key: ParserInput(key), max_size=20)
         self.sheerka.cache_manager.register_cache(self.PARSERS_INPUTS_ENTRY, self.pi_cache, False)
 
     def get_parser_input(self, text, tokens=None):

src/core/sheerka/services/SheerkaFilter.py

@@ -0,0 +1,434 @@
+# the principle and the Pipe class are taken from
+# https://github.com/JulienPalard/Pipe
+#
+import builtins
+import functools
+import inspect
+import itertools
+import sys
+from collections import deque
+
+from cache.Cache import Cache
+from core.builtin_concepts import BuiltinConcepts
+from core.concept import Concept, ConceptParts
+from core.sheerka.services.sheerka_service import BaseService
+from core.utils import as_bag
+from printer.FormatInstructions import FormatInstructions
+from sheerkapickle.utils import is_primitive
+
+
+class PropDesc:
+    def __init__(self, class_name, props):
+        self.class_name = class_name
+        self.props = props
+
+    def __repr__(self):
+        return f"({self.class_name}{self.props})"
+
+    def __eq__(self, other):
+        if id(other) == id(self):
+            return True
+
+        if not isinstance(other, PropDesc):
+            return False
+
+        return self.class_name == other.class_name and sorted(self.props) == sorted(other.props)
+
+
+class Pipe:
+    """
+    Represent a Pipeable Element :
+    Described as :
+    first = Pipe(lambda iterable: next(iter(iterable)))
+    and used as :
+    print [1, 2, 3] | first
+    printing 1
+    Or represent a Pipeable Function :
+    It's a function returning a Pipe
+    Described as :
+    select = Pipe(lambda iterable, pred: (pred(x) for x in iterable))
+    and used as :
+    print [1, 2, 3] | select(lambda x: x * 2)
+    # 2, 4, 6
+    """
+
+    def __init__(self, function, context=None):
+        self.context = context
+
+        if isinstance(function, Pipe):
+            self.function = function.function
+            self.need_context = function.need_context
+        else:
+            signature = inspect.signature(function)
+            if len(signature.parameters) > 0 and list(signature.parameters.keys())[0] == "context":
+                self.need_context = True
+                self.function = (lambda x: function(context, x)) if len(signature.parameters) == 2 else function
+            else:
+                self.need_context = False
+                self.function = function
+
+        functools.update_wrapper(self, function)
+
+    def __ror__(self, other):
+        if isinstance(other, Concept) and other.key == str(BuiltinConcepts.EXPLANATION):
+            other.set_value(ConceptParts.BODY, self.function(other.body))
+            return other
+        return self.function(other)
+
+    def __call__(self, *args, **kwargs):
+        if self.need_context:
+            return Pipe(lambda x: self.function(self.context, x, *args, **kwargs), self.context)
+        else:
+            return Pipe(lambda x: self.function(x, *args, **kwargs), self.context)
+
+
+class SheerkaFilter(BaseService):
+    NAME = "Filter"
+    PREDICATES_ENTRY = "Filter:Predicates"
+
+    def __init__(self, sheerka):
+        super().__init__(sheerka)
+        self.cache = Cache(max_size=30)
+
+    def initialize(self):
+        # For a weird reason, when the attribute @Pipe is directly added to the function
+        # all following instances have the context property null
+        for k, v in SheerkaFilter.__dict__.items():
+            if k.startswith("pipe_"):
+                if isinstance(v, staticmethod):
+                    self.sheerka.add_pipeable(k[5:], v.__func__)
+                else:
+                    self.sheerka.add_pipeable(k[5:], v.__get__(self, self.__class__))
+
+        self.sheerka.cache_manager.register_cache(self.PREDICATES_ENTRY, self.cache, False, False)
+
+    def get_compiled(self, file_name, predicate):
+        """
+        Returns the compiled version of the predicate
+        :param file_name:
+        :param predicate:
+        :return:
+        """
+        compiled = self.cache.get(predicate)
+        if compiled is not None:
+            return compiled
+
+        compiled = compile(predicate, f"<{file_name}>", "eval")
+        self.cache.put(predicate, compiled)
+        return compiled
+
+    @staticmethod
+    def pipe_first(iterable):
+        """
+        Return the first element of the list
+        :param iterable:
+        :return:
+        """
+        return next(iter(iterable))
+
+    @staticmethod
+    def pipe_take(iterable, n):
+        """
+        Take the n first element of a list
+        :param iterable:
+        :param n:
+        :return:
+        """
+        for item in iterable:
+            if n > 0:
+                n -= 1
+                yield item
+            else:
+                return
+
+    @staticmethod
+    def pipe_props(iterable):
+        """
+        Return the list of available properties of the iterable
+        :return:
+        """
+        for item in iterable:
+            yield PropDesc(type(item).__name__, list(as_bag(item).keys()))
+
+    @staticmethod
+    def pipe_tail(iterable, qte):
+        "Yield qte of elements in the given iterable."
+        return deque(iterable, maxlen=qte)
+
+    @staticmethod
+    def pipe_skip(iterable, qte):
+        "Skip qte elements in the given iterable, then yield others."
+        for item in iterable:
+            if qte == 0:
+                yield item
+            else:
+                qte -= 1
+
+    @staticmethod
+    def pipe_dedup(iterable, key=lambda x: x):
+        """Only yield unique items. Use a set to keep track of duplicate data."""
+        seen = set()
+        for item in iterable:
+            dupkey = key(item)
+            if dupkey not in seen:
+                seen.add(dupkey)
+                yield item
+
+    @staticmethod
+    def pipe_uniq(iterable, key=lambda x: x):
+        """Deduplicate consecutive duplicate values."""
+        iterator = iter(iterable)
+        try:
+            prev = next(iterator)
+        except StopIteration:
+            return
+        yield prev
+        prevkey = key(prev)
+        for item in iterator:
+            itemkey = key(item)
+            if itemkey != prevkey:
+                yield item
+            prevkey = itemkey
+
+    @staticmethod
+    def pipe_all(iterable, pred):
+        """Returns True if ALL elements in the given iterable are true for the
+        given pred function"""
+        return builtins.all(pred(x) for x in iterable)
+
+    @staticmethod
+    def pipe_any(iterable, pred):
+        """Returns True if ANY element in the given iterable is True for the
+        given pred function"""
+        return builtins.any(pred(x) for x in iterable)
+
+    @staticmethod
+    def pipe_average(iterable):
+        """Build the average for the given iterable, starting with 0.0 as seed
+        Will try a division by 0 if the iterable is empty...
+        """
+        # warnings.warn(
+        #     "pipe.average is deprecated, use statistics.mean instead.",
+        #     DeprecationWarning,
+        #     stacklevel=4,
+        # )
+        total = 0.0
+        qte = 0
+        for element in iterable:
+            total += element
+            qte += 1
+        return total / qte
+
+    @staticmethod
+    def pipe_count(iterable):
+        "Count the size of the given iterable, walking thrue it."
+        # warnings.warn(
+        #     "pipe.count is deprecated, use the builtin len() instead.",
+        #     DeprecationWarning,
+        #     stacklevel=4,
+        # )
+        count = 0
+        for element in iterable:
+            count += 1
+        return count
+
+    @staticmethod
+    def pipe_max(iterable, **kwargs):
+        # warnings.warn(
+        #     "pipe.max is deprecated, use the builtin max() instead.",
+        #     DeprecationWarning,
+        #     stacklevel=4,
+        # )
+        return builtins.max(iterable, **kwargs)
+
+    @staticmethod
+    def pipe_min(iterable, **kwargs):
+        # warnings.warn(
+        #     "pipe.min is deprecated, use the builtin min() instead.",
+        #     DeprecationWarning,
+        #     stacklevel=4,
+        # )
+        return builtins.min(iterable, **kwargs)
+
+    @staticmethod
+    def pipe_as_dict(iterable):
+        # warnings.warn(
+        #     "pipe.as_dict is deprecated, use dict(your | pipe) instead.",
+        #     DeprecationWarning,
+        #     stacklevel=4,
+        # )
+        return dict(iterable)
+
+    @staticmethod
+    def pipe_as_set(iterable):
+        # warnings.warn(
+        #     "pipe.as_set is deprecated, use set(your | pipe) instead.",
+        #     DeprecationWarning,
+        #     stacklevel=4,
+        # )
+        return set(iterable)
+
+    @staticmethod
+    def pipe_permutations(iterable, r=None):
+        # permutations('ABCD', 2) --> AB AC AD BA BC BD CA CB CD DA DB DC
+        # permutations(range(3)) --> 012 021 102 120 201 210
+        for x in itertools.permutations(iterable, r):
+            yield x
+
+    # @staticmethod
+    # def pipe_netcat(to_send, host, port):
+    #     with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s:
+    #         s.connect((host, port))
+    #         for data in to_send | traverse:
+    #             s.send(data)
+    #         while 1:
+    #             data = s.recv(4096)
+    #             if not data:
+    #                 break
+    #             yield data
+    #
+    # @staticmethod
+    # def pipe_netwrite(to_send, host, port):
+    #     warnings.warn("pipe.netwite is deprecated.", DeprecationWarning, stacklevel=4)
+    #     with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s:
+    #         s.connect((host, port))
+    #         for data in to_send | SheerkaFilter.pipe_traverse:
+    #             s.send(data)
+
+    @staticmethod
+    def pipe_traverse(args):
+        for arg in args:
+            try:
+                if isinstance(arg, str):
+                    yield arg
+                else:
+                    for i in arg | SheerkaFilter.pipe_traverse:
+                        yield i
+            except TypeError:
+                # not iterable --- output leaf
+                yield arg
+
+    @staticmethod
+    def pipe_concat(iterable, separator=", "):
+        # warnings.warn(
+        #     "pipe.concat is deprecated, use ', '.join(your | pipe) instead.",
+        #     DeprecationWarning,
+        #     stacklevel=4,
+        # )
+        return separator.join(builtins.map(str, iterable))
+
+    @staticmethod
+    def pipe_as_list(iterable):
+        # warnings.warn(
+        #     "pipe.as_list is deprecated, use list(your | pipe) instead.",
+        #     DeprecationWarning,
+        #     stacklevel=4,
+        # )
+        return list(iterable)
+
+    @staticmethod
+    def pipe_as_tuple(iterable):
+        # warnings.warn(
+        #     "pipe.as_tuple is deprecated, use tuple(your | pipe) instead.",
+        #     DeprecationWarning,
+        #     stacklevel=4,
+        # )
+        return tuple(iterable)
+
+    @staticmethod
+    def pipe_tee(iterable):
+        for item in iterable:
+            sys.stdout.write(str(item) + "\n")
+            yield item
+
+    @staticmethod
+    def pipe_write(iterable, fname, glue="\n"):
+        with open(fname, "w") as f:
+            for item in iterable:
+                f.write(str(item) + glue)
+
+    @staticmethod
+    def pipe_add(x):
+        # warnings.warn(
+        #     "pipe.add is deprecated, use sum(your | pipe) instead.",
+        #     DeprecationWarning,
+        #     stacklevel=4,
+        # )
+        return sum(x)
+
+    @staticmethod
+    def pipe_select(iterable, selector):
+        return builtins.map(selector, iterable)
+
+    @staticmethod
+    def pipe_format_l(iterable, template, when=None):
+        """
+        Define a formatting when printing a list of items
+        :param iterable:
+        :param template:
+        :param when: format_l is set when the condition is verified
+        :return:
+        """
+        for item in iterable:
+            if hasattr(item, "get_format_instructions"):
+                instructions = item.get_format_instructions() or FormatInstructions()
+                instructions.set_format_l(item, template)
+                item.set_format_instructions(instructions)
+            yield item
+
+    @staticmethod
+    def pipe_format_d(iterable, *props, when=None, **format_l):
+        """
+        Define a formatting when printing the detail of an item
+        :param iterable:
+        :param props: list of properties to display
+        :param when: format_d is set when the condition is verified
+        :param format_l: custom formatting when printing the value of a property
+        :return:
+        """
+
+        template = dict((p, "{" + p + "}") for p in props)
+        for k, v in format_l.items():
+            template[k] = v
+
+        for item in iterable:
+            if hasattr(item, "get_format_instructions"):
+
+                if len(template) == 0:
+                    bag = as_bag(item)
+                    template = dict((p, "{" + p + "}") for p in bag)
+
+                instructions = item.get_format_instructions() or FormatInstructions()
+                instructions.set_format_d(item, template)
+                item.set_format_instructions(instructions)
+            yield item
+
+    @staticmethod
+    def pipe_recurse(iterable, depth, prop_name="children", when=None):
+        """
+        When printing an object that has sub properties,
+        indicate the depth of recursion to apply to a specific properties
+        Quick and dirty version because the prop name is not taken from the item (but set to 'children' by default)
+        :param iterable:
+        :param depth:
+        :param prop_name:
+        :param when: recurse is set when the condition is verified
+        :return:
+        """
+        for item in iterable:
+            if hasattr(item, "get_format_instructions"):
+                instructions = item.get_format_instructions() or FormatInstructions()
+                instructions.set_recurse(prop_name, depth)
+                item.set_format_instructions(instructions)
+            yield item
+
+    def pipe_filter(self, iterable, predicate):
+        compiled = self.get_compiled("filter", predicate)
+        for item in iterable:
+            try:
+                context = {} if is_primitive(item) else as_bag(item)
+                context["self"] = item
+                if eval(compiled, context):
+                    yield item
+            except NameError:
+                pass

src/core/sheerka/services/SheerkaHistoryManager.py

@@ -11,6 +11,7 @@ class History:
         self.event = event
         self.result = result
         self._status = None
+        self._format_instructions = None
 
     def __str__(self):
         msg = f"{self.event.get_digest()} {self.event.date.strftime('%d/%m/%Y %H:%M:%S')} : {self.event.message}"
@@ -42,6 +43,12 @@ class History:
         self._status = self.result.get_status() if self.result else None
         return self._status
 
+    def get_format_instructions(self):
+        return self._format_instructions
+
+    def set_format_instructions(self, instructions):
+        self._format_instructions = instructions
+
 
 class SheerkaHistoryManager(BaseService):
     NAME = "History"

src/core/sheerka/services/SheerkaResultManager.py

@@ -0,0 +1,125 @@
+from core.builtin_concepts import BuiltinConcepts
+from core.sheerka.services.sheerka_service import BaseService
+
+
+class SheerkaResultConcept(BaseService):
+    NAME = "Result"
+
+    def __init__(self, sheerka, page_size=30):
+        super().__init__(sheerka)
+        self.page_size = page_size
+
+    def initialize(self):
+        self.sheerka.bind_service_method(self.get_results_by_digest)
+        self.sheerka.bind_service_method(self.get_results_by_command)
+        self.sheerka.bind_service_method(self.get_last_results)
+        self.sheerka.bind_service_method(self.get_results)
+
+    def get_results_by_digest(self, context, digest, record_digest=True):
+        """
+        Gets the entire execution tree for the given event digest
+        :param context:
+        :param digest:
+        :param record_digest:
+        :return:
+        """
+        if digest is None:
+            return None
+
+        try:
+            result = self.sheerka.sdp.load_result(digest)
+            event = self.sheerka.sdp.load_event(digest)
+
+            if record_digest:
+                context.log(f"Recording digest '{digest}'")
+                self.sheerka.record(context, self.NAME, "digest", digest)
+
+            return self.sheerka.new(BuiltinConcepts.EXPLANATION,
+                                    digest=event.get_digest(),
+                                    command=event.message,
+                                    body=self.as_list(result))
+        except FileNotFoundError as ex:
+            context.log_error(f"Digest {digest} is not found.", self.NAME, ex)
+            return self.sheerka.new(BuiltinConcepts.NOT_FOUND, body={"digest": digest})
+
+    def get_results_by_command(self, context, command, record_digest=True):
+        """
+        Get the result of the command that starts with command
+        :param context:
+        :param command:
+        :param record_digest:
+        :return:
+        """
+        if command is None:
+            return None
+
+        start = 0
+        consumed = 0
+        while True:
+            for event in self.sheerka.sdp.load_events(self.page_size, start):
+                consumed += 1
+                if event.message.startswith(command):
+                    return self.get_results_by_digest(context, event.get_digest(), record_digest)
+
+            if consumed < self.page_size:
+                break
+
+            start += self.page_size
+            consumed = 0
+
+        return self.sheerka.new(BuiltinConcepts.NOT_FOUND, body={"command": command})
+
+    def get_last_results(self, context, record_digest=True):
+        """
+        Gets the results of the last command
+        :param context:
+        :param record_digest:
+        :return:
+        """
+
+        start = 0
+        page_size = 2
+        consumed = 0
+        while True:
+            for event in self.sheerka.sdp.load_events(page_size, start):
+                consumed += 1
+                if self.sheerka.sdp.has_result(event.get_digest()):
+                    return self.get_results_by_digest(context, event.get_digest(), record_digest)
+
+            if consumed < page_size:
+                break
+
+            if page_size < 100:
+                page_size *= 2
+
+            start += page_size
+            consumed = 0
+
+        return self.sheerka.new(BuiltinConcepts.NOT_FOUND, body={"query": "last"})
+
+    def get_results(self, context):
+        """
+        Use the last digest saved to get the execution results
+        :param context:
+        :return:
+        """
+
+        digest = self.sheerka.load(self.NAME, "digest")
+        if digest is None:
+            context.log("No recorded digest found.")
+            return None
+
+        return self.get_results_by_digest(context, digest, False)
+
+    @staticmethod
+    def as_list(execution_context):
+
+        def _yield_result(lst):
+
+            for e in lst:
+                yield e
+
+                if e.children:
+                    yield from _yield_result(e.children)
+
+        return _yield_result([execution_context])

src/core/sheerka/services/SheerkaSetsManager.py

@@ -10,7 +10,7 @@ GROUP_PREFIX = 'All_'
 
 class SheerkaSetsManager(BaseService):
     NAME = "SetsManager"
-    CONCEPTS_GROUPS_ENTRY = "Concepts_Groups"
+    CONCEPTS_GROUPS_ENTRY = "SetsManager:Concepts_Groups"
 
     def __init__(self, sheerka):
         super().__init__(sheerka)

src/core/sheerka/services/SheerkaVariableManager.py

@@ -21,7 +21,7 @@ class Variable(ServiceObj):
 
 class SheerkaVariableManager(BaseService):
     NAME = "VariableManager"
-    VARIABLES_ENTRY = "Variables"  # entry for admin or internal variables
+    VARIABLES_ENTRY = "VariableManager:Variables"  # entry for admin or internal variables
 
     def __init__(self, sheerka):
         super().__init__(sheerka)