From d7573f095fbaa71a984d0d0f531bae7d64f03fce Mon Sep 17 00:00:00 2001 From: Kodjo Sossouvi Date: Thu, 4 Jun 2020 18:43:15 +0200 Subject: [PATCH] Reimplemented explain feature --- docs/source/blog/blog.rst | 1 + docs/source/blog/parsers.rst | 8 + docs/source/blog/rules.rst | 100 ++++ src/core/builtin_concepts.py | 20 +- src/core/concept.py | 18 +- src/core/sheerka/ExecutionContext.py | 21 +- src/core/sheerka/Sheerka.py | 45 +- .../services/SheerkaComparisonManager.py | 4 +- src/core/sheerka/services/SheerkaExecute.py | 5 +- src/core/sheerka/services/SheerkaFilter.py | 434 ++++++++++++++++++ .../sheerka/services/SheerkaHistoryManager.py | 7 + .../sheerka/services/SheerkaResultManager.py | 125 +++++ .../sheerka/services/SheerkaSetsManager.py | 2 +- .../services/SheerkaVariableManager.py | 2 +- src/core/utils.py | 15 + src/evaluators/ExplainEvaluator.py | 150 ------ src/evaluators/PythonEvaluator.py | 24 +- src/parsers/ExplainParser.py | 361 --------------- src/printer/FormatInstructions.py | 39 +- src/printer/Formatter.py | 193 +++++++- src/printer/SheerkaPrinter.py | 72 ++- src/sdp/sheerkaDataProviderIO.py | 3 + tests/core/test_SheerkaFilter.py | 213 +++++++++ tests/core/test_sheerkaResultManager.py | 147 ++++++ tests/core/test_sheerka_printer.py | 303 ++++++++++-- tests/evaluators/test_ExplainEvaluator.py | 317 ------------- tests/parsers/test_ExplainParser.py | 205 --------- 27 files changed, 1673 insertions(+), 1161 deletions(-) create mode 100644 docs/source/blog/rules.rst create mode 100644 src/core/sheerka/services/SheerkaFilter.py create mode 100644 src/core/sheerka/services/SheerkaResultManager.py delete mode 100644 src/evaluators/ExplainEvaluator.py delete mode 100644 src/parsers/ExplainParser.py create mode 100644 tests/core/test_SheerkaFilter.py create mode 100644 tests/core/test_sheerkaResultManager.py delete mode 100644 tests/evaluators/test_ExplainEvaluator.py delete mode 100644 tests/parsers/test_ExplainParser.py diff --git a/docs/source/blog/blog.rst b/docs/source/blog/blog.rst index ee284be..ba9a474 100644 --- a/docs/source/blog/blog.rst +++ b/docs/source/blog/blog.rst @@ -2,6 +2,7 @@ :maxdepth: 1 concepts + rules parsers persistence diff --git a/docs/source/blog/parsers.rst b/docs/source/blog/parsers.rst index e69de29..e2c7452 100644 --- a/docs/source/blog/parsers.rst +++ b/docs/source/blog/parsers.rst @@ -0,0 +1,8 @@ +Concepts +======== + + + +Basic definition +**************** +To define a new concept diff --git a/docs/source/blog/rules.rst b/docs/source/blog/rules.rst new file mode 100644 index 0000000..5bd8b31 --- /dev/null +++ b/docs/source/blog/rules.rst @@ -0,0 +1,100 @@ +Rules +======== + + + +Basic definition +**************** +To define a new rule + + +:: + + > when then + +Rules can have name, so you can also use the syntax + +:: + + > def rule as when then + + +Existing rule engines +********************* + +I am not quite sure yet about the implementation. I have started to search on the net to see if +I can found some interesting implementation that I can use. + +I found: + +* Durable Rules Engine : https://github.com/jruizgit/rules + + Python implementation, with the rule engine written in C (or C++) to be faster. A good candidate + +* PyKE : http://pyke.sourceforge.net/knowledge_bases/rule_bases.html + + Another Python implementation of the rule engine + +* Business-rules : https://github.com/venmo/business-rules + +* Intellect : https://github.com/nemonik/Intellect + +* CLIPS : http://www.clipsrules.net/ + + A standard. Run on a separate server. I need to check how it can be embedded, or dockerized + +* And of course drools : https://www.drools.org/ + + Another standard + +I am not an expert in rule engine. So I guess that the best way to figure out what engine to use it to list what are the feature that I need. + + +Use cases +********* + +I see the rules engine like the caching service or the logging service. It can be used anywhere in the code. +It's not just a global feature of Sheerka. It's another way of achieving common task. + +For example, in the print service, I want to print all the failed ``ReturnValueConcept`` in red. +Doing it in an imperative way (ie coding this functionality) is + +1. Intrusive in the code. I need to understand what code and where to put it +2. Not straightforward : if I want to that all successful ``ReturnValueConcept`` in green, chances are that I will have to rewrite some code + +So It has to be declarative. With an engine that takes these declarations and correctly paint the outputs. +And a declarative system that accepts conditions is (I guess) a rule engine. + +So let's try something like: + +:: + + > when action==Print and obj==ReturnValueConcept and obj.status then print_the_status_in_red() + +We immediately see that the rule engine will have to be aware of the current system. +So the chosen rule engine will have to manage state or facts. I haven't checked all the listed one, but I am quite sure that they all do, +as it's the minimum requirement for a rule engine. + +I also need two types of rules. + + * permanent rules + It will be triggered as long as the system allows it + + * one use rule: + it will be triggered only once + +If I take my example to color the status of the ``ReturnValueConcept``, it may be a permanent rule, +that will apply to any output, or it can be something that is specific to the current execution context. + + +In the predicate part, I need to control how expression are evaluated. + For example in the expression ``action==Print``, Print can be a string ('Print'), a builtin concept (``BuiltinConcepts.PRINT``) or even another concept + + +In the predicate part, as well as in the action part, I must be able to used other concepts + +:: + + > def concept status is not ok as + > def concept paint in red as + > when status is not ok then paint in red diff --git a/src/core/builtin_concepts.py b/src/core/builtin_concepts.py index 7091205..39b2c3d 100644 --- a/src/core/builtin_concepts.py +++ b/src/core/builtin_concepts.py @@ -62,6 +62,8 @@ class BuiltinConcepts(Enum): PRECEDENCE = "precedence" # use to set priority among concepts when parsing ASSOCIATIVITY = "associativity" # use to set priority among concepts when parsing NOT_INITIALIZED = "not initialized" + NOT_FOUND = "not found" # when the wanted resource is not found + FORMAT_INSTRUCTIONS = "format instructions" # to express how to print the concept NODE = "node" GENERIC_NODE = "generic node" @@ -73,6 +75,21 @@ class BuiltinConcepts(Enum): def __str__(self): return "__" + self.name + def __eq__(self, other): + if id(self) == id(other): + return True + + if isinstance(other, str): + return str(self) == other + + if not isinstance(other, BuiltinConcepts): + return False + + return self.value == other.value + + def __hash__(self): + return hash(self.value) + BuiltinUnique = [ BuiltinConcepts.BEFORE_PARSING, @@ -106,7 +123,8 @@ BuiltinErrors = [str(e) for e in { BuiltinConcepts.NOT_A_SET, BuiltinConcepts.WHERE_CLAUSE_FAILED, BuiltinConcepts.CHICKEN_AND_EGG, - BuiltinConcepts.NOT_INITIALIZED + BuiltinConcepts.NOT_INITIALIZED, + BuiltinConcepts.NOT_FOUND }] """ diff --git a/src/core/concept.py b/src/core/concept.py index 3840088..fa24993 100644 --- a/src/core/concept.py +++ b/src/core/concept.py @@ -278,10 +278,10 @@ class Concept: def to_dict(self, props_to_use=None): """ Returns a dict representing 'self' - to_dict() is used for serializing the definition of the concept - You will not that it does not dump the actual values of the properties, nor the body + to_dict() is used for serializing the **definition** of the concept - If you need a dictionary version of the Concept, use to_bag() + It does not dump the actual values of the properties, nor the body + If you need a dictionary version of the Concept, use as_bag() :return: """ @@ -369,7 +369,7 @@ class Concept: :param concept_key: name of the behaviour :return: """ - return self.metadata.props[concept_key] if concept_key in self.metadata.props else None + return self.metadata.props.get(concept_key, None) def set_value(self, name, value): """ @@ -426,7 +426,7 @@ class Concept: def get_original_definition_hash(self): return self.original_definition_hash - def to_bag(self): + def as_bag(self): """ Creates a dictionary with the useful properties of the concept It quicker to implement than creating the actual property mechanism with @property @@ -441,6 +441,14 @@ class Concept: bag[prop] = getattr(self, prop) return bag + def get_format_instructions(self): + from core.builtin_concepts import BuiltinConcepts + return self.get_prop(BuiltinConcepts.FORMAT_INSTRUCTIONS) + + def set_format_instructions(self, instructions): + from core.builtin_concepts import BuiltinConcepts + self.set_prop(BuiltinConcepts.FORMAT_INSTRUCTIONS, instructions) + class Property: """ diff --git a/src/core/sheerka/ExecutionContext.py b/src/core/sheerka/ExecutionContext.py index 73eef3a..f5ac46e 100644 --- a/src/core/sheerka/ExecutionContext.py +++ b/src/core/sheerka/ExecutionContext.py @@ -50,9 +50,10 @@ class ExecutionContext: self._parent = None self._id = ExecutionContext.get_id(event.get_digest()) if event else None self._tab = "" - self._bag = {} # other variables - self._start = 0 - self._stop = 0 + self._bag = {} # context variables + self._start = 0 # when the execution starts (to measure elapsed time) + self._stop = 0 # when the execution stops (to measure elapses time) + self._format_instructions = None # how to print the execution context self.who = who # who is asking self.event = event # what was the (original) trigger @@ -65,11 +66,13 @@ class ExecutionContext: self.global_hints = set() if global_hints is None else global_hints self.global_errors = [] if global_errors is None else global_errors + self.inputs = {} # what was the parameters of the execution context self.values = {} # what was produced by the execution context - self.obj = kwargs.pop("obj", None) - self.concepts = kwargs.pop("concepts", {}) + self.obj = kwargs.pop("obj", None) # current obj we are working on + self.concepts = kwargs.pop("concepts", {}) # known concepts specific to this context + # update the other elements for k, v in kwargs.items(): self._bag[k] = v @@ -313,7 +316,7 @@ class ExecutionContext: return None return ret_val.status - def to_bag(self): + def as_bag(self): """ Creates a dictionary with the useful properties of the concept It quicker to implement than creating the actual property mechanism with @property @@ -338,3 +341,9 @@ class ExecutionContext: value = value[:47] + "..." to_str = f"ReturnValue(who={r.who}, status={r.status}, value={value})" return to_str + + def get_format_instructions(self): + return self._format_instructions + + def set_format_instructions(self, instructions): + self._format_instructions = instructions diff --git a/src/core/sheerka/Sheerka.py b/src/core/sheerka/Sheerka.py index 2ac6fa7..dc5e3e3 100644 --- a/src/core/sheerka/Sheerka.py +++ b/src/core/sheerka/Sheerka.py @@ -83,6 +83,7 @@ class Sheerka(Concept): "test": self.test, "test_using_context": self.test_using_context } + self.sheerka_pipeables = {} @property def resolved_concepts_by_first_keyword(self): @@ -121,6 +122,15 @@ class Sheerka(Concept): setattr(self, as_name, bound_method) + def add_pipeable(self, func_name, function): + """ + Adds a function that can bu used with pipe '|' + :param func_name: + :param function: + :return: + """ + self.sheerka_pipeables[func_name] = function + def initialize(self, root_folder: str = None, save_execution_context=True): """ Starting Sheerka @@ -360,8 +370,11 @@ class Sheerka(Concept): if self.cache_manager.is_dirty: self.cache_manager.commit(execution_context) - if self.save_execution_context and self.load(self.name, "save_execution_context"): - self.sdp.save_result(execution_context) + try: + if self.save_execution_context and self.load(self.name, "save_execution_context"): + self.sdp.save_result(execution_context) + except Exception as ex: + self.log.error(f"Failed to save execution context. Reason: {ex}") # # hack to save valid concept definition # if not self.during_restore: @@ -760,6 +773,9 @@ class Sheerka(Concept): except IOError: pass + def get_last_execution(self): + return self._last_execution + def test(self): return f"I have access to Sheerka !" @@ -841,6 +857,28 @@ class Sheerka(Concept): @staticmethod def init_logging(debug, loggers): + def add_coloring_to_emit_ansi(fn): + # add methods we need to the class + def new(*args): + levelno = args[1].levelno + if levelno >= 50: + color = '\x1b[31m' # red + elif levelno >= 40: + color = '\x1b[31m' # red + elif levelno >= 30: + color = '\x1b[33m' # yellow + elif levelno >= 20: + color = '\x1b[32m' # green + elif levelno >= 10: + color = '\x1b[35m' # pink + else: + color = '\x1b[0m' # normal + args[1].msg = color + str(args[1].msg) + '\x1b[0m' # normal + # print "after" + return fn(*args) + + return new + core.sheerka_logger.init_config(loggers) if debug: log_format = "%(asctime)s" @@ -853,3 +891,6 @@ class Sheerka(Concept): log_level = logging.INFO logging.basicConfig(format=log_format, level=log_level, handlers=[console_handler]) + logging.addLevelName(logging.ERROR, "\033[1;41m%s\033[1;0m" % logging.getLevelName(logging.ERROR)) + # uncomment the following line to enable colors + #logging.StreamHandler.emit = add_coloring_to_emit_ansi(logging.StreamHandler.emit) diff --git a/src/core/sheerka/services/SheerkaComparisonManager.py b/src/core/sheerka/services/SheerkaComparisonManager.py index a46cc27..5853225 100644 --- a/src/core/sheerka/services/SheerkaComparisonManager.py +++ b/src/core/sheerka/services/SheerkaComparisonManager.py @@ -24,8 +24,8 @@ class SheerkaComparisonManager(BaseService): Manage partitioning of concepts """ NAME = "ComparisonManager" - COMPARISON_ENTRY = "Comparison" - RESOLVED_COMPARISON_ENTRY = "Resolved_Comparison" + COMPARISON_ENTRY = "ComparisonManager:Comparison" + RESOLVED_COMPARISON_ENTRY = "ComparisonManager:Resolved_Comparison" def __init__(self, sheerka): super().__init__(sheerka) diff --git a/src/core/sheerka/services/SheerkaExecute.py b/src/core/sheerka/services/SheerkaExecute.py index c6e0659..87d34dc 100644 --- a/src/core/sheerka/services/SheerkaExecute.py +++ b/src/core/sheerka/services/SheerkaExecute.py @@ -137,16 +137,15 @@ class SheerkaExecute(BaseService): """ NAME = "Execute" - PARSERS_INPUTS_ENTRY = "ParserInput" # entry for admin or internal variables + PARSERS_INPUTS_ENTRY = "Execute:ParserInput" # entry for admin or internal variables def __init__(self, sheerka): super().__init__(sheerka) - self.pi_cache = None + self.pi_cache = Cache(default=lambda key: ParserInput(key), max_size=20) def initialize(self): self.sheerka.bind_service_method(self.execute) - self.pi_cache = Cache(default=lambda key: ParserInput(key), max_size=20) self.sheerka.cache_manager.register_cache(self.PARSERS_INPUTS_ENTRY, self.pi_cache, False) def get_parser_input(self, text, tokens=None): diff --git a/src/core/sheerka/services/SheerkaFilter.py b/src/core/sheerka/services/SheerkaFilter.py new file mode 100644 index 0000000..5853d0d --- /dev/null +++ b/src/core/sheerka/services/SheerkaFilter.py @@ -0,0 +1,434 @@ +# the principle and the Pipe class are taken from +# https://github.com/JulienPalard/Pipe +# +import builtins +import functools +import inspect +import itertools +import sys +from collections import deque + +from cache.Cache import Cache +from core.builtin_concepts import BuiltinConcepts +from core.concept import Concept, ConceptParts +from core.sheerka.services.sheerka_service import BaseService +from core.utils import as_bag +from printer.FormatInstructions import FormatInstructions +from sheerkapickle.utils import is_primitive + + +class PropDesc: + def __init__(self, class_name, props): + self.class_name = class_name + self.props = props + + def __repr__(self): + return f"({self.class_name}{self.props})" + + def __eq__(self, other): + if id(other) == id(self): + return True + + if not isinstance(other, PropDesc): + return False + + return self.class_name == other.class_name and sorted(self.props) == sorted(other.props) + + +class Pipe: + """ + Represent a Pipeable Element : + Described as : + first = Pipe(lambda iterable: next(iter(iterable))) + and used as : + print [1, 2, 3] | first + printing 1 + Or represent a Pipeable Function : + It's a function returning a Pipe + Described as : + select = Pipe(lambda iterable, pred: (pred(x) for x in iterable)) + and used as : + print [1, 2, 3] | select(lambda x: x * 2) + # 2, 4, 6 + """ + + def __init__(self, function, context=None): + self.context = context + + if isinstance(function, Pipe): + self.function = function.function + self.need_context = function.need_context + else: + signature = inspect.signature(function) + if len(signature.parameters) > 0 and list(signature.parameters.keys())[0] == "context": + self.need_context = True + self.function = (lambda x: function(context, x)) if len(signature.parameters) == 2 else function + else: + self.need_context = False + self.function = function + + functools.update_wrapper(self, function) + + def __ror__(self, other): + if isinstance(other, Concept) and other.key == str(BuiltinConcepts.EXPLANATION): + other.set_value(ConceptParts.BODY, self.function(other.body)) + return other + return self.function(other) + + def __call__(self, *args, **kwargs): + if self.need_context: + return Pipe(lambda x: self.function(self.context, x, *args, **kwargs), self.context) + else: + return Pipe(lambda x: self.function(x, *args, **kwargs), self.context) + + +class SheerkaFilter(BaseService): + NAME = "Filter" + PREDICATES_ENTRY = "Filter:Predicates" + + def __init__(self, sheerka): + super().__init__(sheerka) + self.cache = Cache(max_size=30) + + def initialize(self): + # For a weird reason, when the attribute @Pipe is directly added to the function + # all following instances have the context property null + for k, v in SheerkaFilter.__dict__.items(): + if k.startswith("pipe_"): + if isinstance(v, staticmethod): + self.sheerka.add_pipeable(k[5:], v.__func__) + else: + self.sheerka.add_pipeable(k[5:], v.__get__(self, self.__class__)) + + self.sheerka.cache_manager.register_cache(self.PREDICATES_ENTRY, self.cache, False, False) + + def get_compiled(self, file_name, predicate): + """ + Returns the compiled version of the predicate + :param file_name: + :param predicate: + :return: + """ + compiled = self.cache.get(predicate) + if compiled is not None: + return compiled + + compiled = compile(predicate, f"<{file_name}>", "eval") + self.cache.put(predicate, compiled) + return compiled + + @staticmethod + def pipe_first(iterable): + """ + Return the first element of the list + :param iterable: + :return: + """ + return next(iter(iterable)) + + @staticmethod + def pipe_take(iterable, n): + """ + Take the n first element of a list + :param iterable: + :param n: + :return: + """ + for item in iterable: + if n > 0: + n -= 1 + yield item + else: + return + + @staticmethod + def pipe_props(iterable): + """ + Return the list of available properties of the iterable + :return: + """ + for item in iterable: + yield PropDesc(type(item).__name__, list(as_bag(item).keys())) + + @staticmethod + def pipe_tail(iterable, qte): + "Yield qte of elements in the given iterable." + return deque(iterable, maxlen=qte) + + @staticmethod + def pipe_skip(iterable, qte): + "Skip qte elements in the given iterable, then yield others." + for item in iterable: + if qte == 0: + yield item + else: + qte -= 1 + + @staticmethod + def pipe_dedup(iterable, key=lambda x: x): + """Only yield unique items. Use a set to keep track of duplicate data.""" + seen = set() + for item in iterable: + dupkey = key(item) + if dupkey not in seen: + seen.add(dupkey) + yield item + + @staticmethod + def pipe_uniq(iterable, key=lambda x: x): + """Deduplicate consecutive duplicate values.""" + iterator = iter(iterable) + try: + prev = next(iterator) + except StopIteration: + return + yield prev + prevkey = key(prev) + for item in iterator: + itemkey = key(item) + if itemkey != prevkey: + yield item + prevkey = itemkey + + @staticmethod + def pipe_all(iterable, pred): + """Returns True if ALL elements in the given iterable are true for the + given pred function""" + return builtins.all(pred(x) for x in iterable) + + @staticmethod + def pipe_any(iterable, pred): + """Returns True if ANY element in the given iterable is True for the + given pred function""" + return builtins.any(pred(x) for x in iterable) + + @staticmethod + def pipe_average(iterable): + """Build the average for the given iterable, starting with 0.0 as seed + Will try a division by 0 if the iterable is empty... + """ + # warnings.warn( + # "pipe.average is deprecated, use statistics.mean instead.", + # DeprecationWarning, + # stacklevel=4, + # ) + total = 0.0 + qte = 0 + for element in iterable: + total += element + qte += 1 + return total / qte + + @staticmethod + def pipe_count(iterable): + "Count the size of the given iterable, walking thrue it." + # warnings.warn( + # "pipe.count is deprecated, use the builtin len() instead.", + # DeprecationWarning, + # stacklevel=4, + # ) + count = 0 + for element in iterable: + count += 1 + return count + + @staticmethod + def pipe_max(iterable, **kwargs): + # warnings.warn( + # "pipe.max is deprecated, use the builtin max() instead.", + # DeprecationWarning, + # stacklevel=4, + # ) + return builtins.max(iterable, **kwargs) + + @staticmethod + def pipe_min(iterable, **kwargs): + # warnings.warn( + # "pipe.min is deprecated, use the builtin min() instead.", + # DeprecationWarning, + # stacklevel=4, + # ) + return builtins.min(iterable, **kwargs) + + @staticmethod + def pipe_as_dict(iterable): + # warnings.warn( + # "pipe.as_dict is deprecated, use dict(your | pipe) instead.", + # DeprecationWarning, + # stacklevel=4, + # ) + return dict(iterable) + + @staticmethod + def pipe_as_set(iterable): + # warnings.warn( + # "pipe.as_set is deprecated, use set(your | pipe) instead.", + # DeprecationWarning, + # stacklevel=4, + # ) + return set(iterable) + + @staticmethod + def pipe_permutations(iterable, r=None): + # permutations('ABCD', 2) --> AB AC AD BA BC BD CA CB CD DA DB DC + # permutations(range(3)) --> 012 021 102 120 201 210 + for x in itertools.permutations(iterable, r): + yield x + + # @staticmethod + # def pipe_netcat(to_send, host, port): + # with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s: + # s.connect((host, port)) + # for data in to_send | traverse: + # s.send(data) + # while 1: + # data = s.recv(4096) + # if not data: + # break + # yield data + # + # @staticmethod + # def pipe_netwrite(to_send, host, port): + # warnings.warn("pipe.netwite is deprecated.", DeprecationWarning, stacklevel=4) + # with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s: + # s.connect((host, port)) + # for data in to_send | SheerkaFilter.pipe_traverse: + # s.send(data) + + @staticmethod + def pipe_traverse(args): + for arg in args: + try: + if isinstance(arg, str): + yield arg + else: + for i in arg | SheerkaFilter.pipe_traverse: + yield i + except TypeError: + # not iterable --- output leaf + yield arg + + @staticmethod + def pipe_concat(iterable, separator=", "): + # warnings.warn( + # "pipe.concat is deprecated, use ', '.join(your | pipe) instead.", + # DeprecationWarning, + # stacklevel=4, + # ) + return separator.join(builtins.map(str, iterable)) + + @staticmethod + def pipe_as_list(iterable): + # warnings.warn( + # "pipe.as_list is deprecated, use list(your | pipe) instead.", + # DeprecationWarning, + # stacklevel=4, + # ) + return list(iterable) + + @staticmethod + def pipe_as_tuple(iterable): + # warnings.warn( + # "pipe.as_tuple is deprecated, use tuple(your | pipe) instead.", + # DeprecationWarning, + # stacklevel=4, + # ) + return tuple(iterable) + + @staticmethod + def pipe_tee(iterable): + for item in iterable: + sys.stdout.write(str(item) + "\n") + yield item + + @staticmethod + def pipe_write(iterable, fname, glue="\n"): + with open(fname, "w") as f: + for item in iterable: + f.write(str(item) + glue) + + @staticmethod + def pipe_add(x): + # warnings.warn( + # "pipe.add is deprecated, use sum(your | pipe) instead.", + # DeprecationWarning, + # stacklevel=4, + # ) + return sum(x) + + @staticmethod + def pipe_select(iterable, selector): + return builtins.map(selector, iterable) + + @staticmethod + def pipe_format_l(iterable, template, when=None): + """ + Define a formatting when printing a list of items + :param iterable: + :param template: + :param when: format_l is set when the condition is verified + :return: + """ + for item in iterable: + if hasattr(item, "get_format_instructions"): + instructions = item.get_format_instructions() or FormatInstructions() + instructions.set_format_l(item, template) + item.set_format_instructions(instructions) + yield item + + @staticmethod + def pipe_format_d(iterable, *props, when=None, **format_l): + """ + Define a formatting when printing the detail of an item + :param iterable: + :param props: list of properties to display + :param when: format_d is set when the condition is verified + :param format_l: custom formatting when printing the value of a property + :return: + """ + + template = dict((p, "{" + p + "}") for p in props) + for k, v in format_l.items(): + template[k] = v + + for item in iterable: + if hasattr(item, "get_format_instructions"): + + if len(template) == 0: + bag = as_bag(item) + template = dict((p, "{" + p + "}") for p in bag) + + instructions = item.get_format_instructions() or FormatInstructions() + instructions.set_format_d(item, template) + item.set_format_instructions(instructions) + yield item + + @staticmethod + def pipe_recurse(iterable, depth, prop_name="children", when=None): + """ + When printing an object that has sub properties, + indicate the depth of recursion to apply to a specific properties + Quick and dirty version because the prop name is not taken from the item (but set to 'children' by default) + :param iterable: + :param depth: + :param prop_name: + :param when: recurse is set when the condition is verified + :return: + """ + for item in iterable: + if hasattr(item, "get_format_instructions"): + instructions = item.get_format_instructions() or FormatInstructions() + instructions.set_recurse(prop_name, depth) + item.set_format_instructions(instructions) + yield item + + def pipe_filter(self, iterable, predicate): + compiled = self.get_compiled("filter", predicate) + for item in iterable: + try: + context = {} if is_primitive(item) else as_bag(item) + context["self"] = item + if eval(compiled, context): + yield item + except NameError: + pass diff --git a/src/core/sheerka/services/SheerkaHistoryManager.py b/src/core/sheerka/services/SheerkaHistoryManager.py index a643d8b..c140eff 100644 --- a/src/core/sheerka/services/SheerkaHistoryManager.py +++ b/src/core/sheerka/services/SheerkaHistoryManager.py @@ -11,6 +11,7 @@ class History: self.event = event self.result = result self._status = None + self._format_instructions = None def __str__(self): msg = f"{self.event.get_digest()} {self.event.date.strftime('%d/%m/%Y %H:%M:%S')} : {self.event.message}" @@ -42,6 +43,12 @@ class History: self._status = self.result.get_status() if self.result else None return self._status + def get_format_instructions(self): + return self._format_instructions + + def set_format_instructions(self, instructions): + self._format_instructions = instructions + class SheerkaHistoryManager(BaseService): NAME = "History" diff --git a/src/core/sheerka/services/SheerkaResultManager.py b/src/core/sheerka/services/SheerkaResultManager.py new file mode 100644 index 0000000..22de478 --- /dev/null +++ b/src/core/sheerka/services/SheerkaResultManager.py @@ -0,0 +1,125 @@ +from core.builtin_concepts import BuiltinConcepts +from core.sheerka.services.sheerka_service import BaseService + + +class SheerkaResultConcept(BaseService): + NAME = "Result" + + def __init__(self, sheerka, page_size=30): + super().__init__(sheerka) + self.page_size = page_size + + def initialize(self): + self.sheerka.bind_service_method(self.get_results_by_digest) + self.sheerka.bind_service_method(self.get_results_by_command) + self.sheerka.bind_service_method(self.get_last_results) + self.sheerka.bind_service_method(self.get_results) + + def get_results_by_digest(self, context, digest, record_digest=True): + """ + Gets the entire execution tree for the given event digest + :param context: + :param digest: + :param record_digest: + :return: + """ + if digest is None: + return None + + try: + result = self.sheerka.sdp.load_result(digest) + event = self.sheerka.sdp.load_event(digest) + + if record_digest: + context.log(f"Recording digest '{digest}'") + self.sheerka.record(context, self.NAME, "digest", digest) + + return self.sheerka.new(BuiltinConcepts.EXPLANATION, + digest=event.get_digest(), + command=event.message, + body=self.as_list(result)) + except FileNotFoundError as ex: + context.log_error(f"Digest {digest} is not found.", self.NAME, ex) + return self.sheerka.new(BuiltinConcepts.NOT_FOUND, body={"digest": digest}) + + def get_results_by_command(self, context, command, record_digest=True): + """ + Get the result of the command that starts with command + :param context: + :param command: + :param record_digest: + :return: + """ + if command is None: + return None + + start = 0 + consumed = 0 + while True: + for event in self.sheerka.sdp.load_events(self.page_size, start): + consumed += 1 + if event.message.startswith(command): + return self.get_results_by_digest(context, event.get_digest(), record_digest) + + if consumed < self.page_size: + break + + start += self.page_size + consumed = 0 + + return self.sheerka.new(BuiltinConcepts.NOT_FOUND, body={"command": command}) + + def get_last_results(self, context, record_digest=True): + """ + Gets the results of the last command + :param context: + :param record_digest: + :return: + """ + + start = 0 + page_size = 2 + consumed = 0 + while True: + for event in self.sheerka.sdp.load_events(page_size, start): + consumed += 1 + if self.sheerka.sdp.has_result(event.get_digest()): + return self.get_results_by_digest(context, event.get_digest(), record_digest) + + if consumed < page_size: + break + + if page_size < 100: + page_size *= 2 + + start += page_size + consumed = 0 + + return self.sheerka.new(BuiltinConcepts.NOT_FOUND, body={"query": "last"}) + + def get_results(self, context): + """ + Use the last digest saved to get the execution results + :param context: + :return: + """ + + digest = self.sheerka.load(self.NAME, "digest") + if digest is None: + context.log("No recorded digest found.") + return None + + return self.get_results_by_digest(context, digest, False) + + @staticmethod + def as_list(execution_context): + + def _yield_result(lst): + + for e in lst: + yield e + + if e.children: + yield from _yield_result(e.children) + + return _yield_result([execution_context]) diff --git a/src/core/sheerka/services/SheerkaSetsManager.py b/src/core/sheerka/services/SheerkaSetsManager.py index 25e02fc..d92786d 100644 --- a/src/core/sheerka/services/SheerkaSetsManager.py +++ b/src/core/sheerka/services/SheerkaSetsManager.py @@ -10,7 +10,7 @@ GROUP_PREFIX = 'All_' class SheerkaSetsManager(BaseService): NAME = "SetsManager" - CONCEPTS_GROUPS_ENTRY = "Concepts_Groups" + CONCEPTS_GROUPS_ENTRY = "SetsManager:Concepts_Groups" def __init__(self, sheerka): super().__init__(sheerka) diff --git a/src/core/sheerka/services/SheerkaVariableManager.py b/src/core/sheerka/services/SheerkaVariableManager.py index c6224ad..200cf07 100644 --- a/src/core/sheerka/services/SheerkaVariableManager.py +++ b/src/core/sheerka/services/SheerkaVariableManager.py @@ -21,7 +21,7 @@ class Variable(ServiceObj): class SheerkaVariableManager(BaseService): NAME = "VariableManager" - VARIABLES_ENTRY = "Variables" # entry for admin or internal variables + VARIABLES_ENTRY = "VariableManager:Variables" # entry for admin or internal variables def __init__(self, sheerka): super().__init__(sheerka) diff --git a/src/core/utils.py b/src/core/utils.py index 01b496c..6086354 100644 --- a/src/core/utils.py +++ b/src/core/utils.py @@ -424,3 +424,18 @@ def tokens_index(tokens, sub_tokens, skip=0): skip -= 1 raise ValueError(f"sub tokens '{sub_tokens}' not found") + + +def as_bag(obj): + """ + Get the properties of an object (static and dynamic) + :param obj: + :return: + """ + if hasattr(obj, "as_bag"): + bag = obj.as_bag() + else: + bag = {prop: getattr(obj, prop) for prop in dir(obj) if not prop.startswith("_")} + + bag["self"] = obj + return bag diff --git a/src/evaluators/ExplainEvaluator.py b/src/evaluators/ExplainEvaluator.py deleted file mode 100644 index 93362aa..0000000 --- a/src/evaluators/ExplainEvaluator.py +++ /dev/null @@ -1,150 +0,0 @@ -from typing import List - -from core.builtin_concepts import BuiltinConcepts, ParserResultConcept -from core.sheerka.ExecutionContext import ExecutionContext -from evaluators.BaseEvaluator import OneReturnValueEvaluator -from parsers.ExplainParser import ExplanationNode, FilterNode, RecurseDefNode, FormatLNode, FormatDNode -from parsers.ExpressionParser import ExpressionVisitor, IsaNode -from printer.SheerkaPrinter import FormatInstructions - - -class ExplainExpressionVisitor(ExpressionVisitor): - def __init__(self): - self.instructions = FormatInstructions() - - def visit_RecurseDefNode(self, expr_node): - self.instructions.set_recurse("children", expr_node.depth) - - def visit_FormatLNode(self, expr_node): - self.instructions.set_format_l(ExecutionContext, expr_node.template) - - -class ExplainEvaluator(OneReturnValueEvaluator): - NAME = "Explain" - - def __init__(self): - super().__init__(self.NAME, [BuiltinConcepts.EVALUATION], 60) - - def get_event_digest(self, sheerka, explanation_node): - if explanation_node.digest and sheerka.sdp.has_result(explanation_node.digest): - return explanation_node.digest - - if not explanation_node.digest and not explanation_node.record_digest: - # use a previous digest if found - digest = sheerka.load(self.name, "digest") - if digest is not None: - return digest - - start = 0 - while True: - events = list(sheerka.sdp.load_events(5, start)) - if not events: - break - - for event in events: - if not sheerka.sdp.has_result(event.get_digest()): - continue - if not explanation_node.digest or explanation_node.digest == event.message: - # maybe explanation_node.digest is not a real digest, but the command we want to explain - return event.get_digest() - - start += 5 - if start > 20: - break - - return None - - @staticmethod - def get_execution_result(sheerka, digest): - if digest is None: - # the test is done here to ease the unit tests - return None - return [sheerka.sdp.load_result(digest)] - - @staticmethod - def get_instructions(filter_node: FilterNode): - instructions = FormatInstructions() - for directive in filter_node.directives: - if isinstance(directive, RecurseDefNode): - instructions.set_recurse("children", directive.depth) - elif isinstance(directive, FormatLNode): - instructions.set_format_l(ExecutionContext, directive.template) - elif isinstance(directive, FormatDNode): - instructions.add_format_d(IsaNode(ExecutionContext), directive.properties) - return instructions - - @staticmethod - def get_title(filter_node): - return "" - - def matches(self, context, return_value): - if not return_value.status: - return False - - if not isinstance(return_value.value, ParserResultConcept): - return False - - return isinstance(return_value.value.value, ExplanationNode) - - def eval(self, context, return_value): - sheerka = context.sheerka - explanation_node = return_value.value.value - - if explanation_node.digest and not explanation_node.record_digest: - context.log(f"Deleting recorded digest") - sheerka.delete(context, self.name, "digest") - - digest = self.get_event_digest(sheerka, explanation_node) - executions_results = self.get_execution_result(sheerka, digest) - if executions_results is None and not digest: - res = sheerka.new(BuiltinConcepts.ERROR, body=f"No result found (digest={explanation_node.digest})") - - else: - # record the digest if needed - if explanation_node.record_digest: - context.log(f"Recording digest '{digest}'") - sheerka.record(context, self.name, "digest", digest) - - filter_nodes = explanation_node.expr.filters - global_instructions = self.get_instructions(filter_nodes[0]) - if len(filter_nodes) == 1: - filtered = [[]] - self.filter(executions_results, filter_nodes, filtered) - res = sheerka.new(BuiltinConcepts.EXPLANATION, - digest=digest, - command=explanation_node.command, - title="<all>", - body=filtered[0], - instructions=global_instructions) - else: - res = [] - filter_nodes = filter_nodes[1:] # remove the first filter_node (which always returns True) - filtered = [] - for i in range(len(filter_nodes)): - filtered.append([]) - self.filter(executions_results, filter_nodes, filtered) - for i, filter_node in enumerate(filter_nodes): - instructions = global_instructions.clone().merge(self.get_instructions(filter_node)) - res.append(sheerka.new(BuiltinConcepts.EXPLANATION, - digest=digest, - command=explanation_node.command, - title=self.get_title(filter_node), - body=filtered[i], - instructions=instructions)) - - if len(res) == 1: - res = res[0] - - return sheerka.ret(self.name, not sheerka.isinstance(res, BuiltinConcepts.ERROR), res, parents=[return_value]) - - def filter(self, executions_results, filter_nodes: List[FilterNode], res): - - for execution_result in executions_results: - for i, filter_node in enumerate(filter_nodes): - if filter_node.expr.eval(execution_result): - res[i].append(execution_result) - - if execution_result.children: - self.filter(execution_result.children, filter_nodes, res) - - return res diff --git a/src/evaluators/PythonEvaluator.py b/src/evaluators/PythonEvaluator.py index c615e2a..8b7cefb 100644 --- a/src/evaluators/PythonEvaluator.py +++ b/src/evaluators/PythonEvaluator.py @@ -1,8 +1,10 @@ import ast import copy import traceback +from functools import partial, update_wrapper import core.ast.nodes +from core.sheerka.services.SheerkaFilter import Pipe import core.utils from core.ast.visitors import UnreferencedNamesVisitor from core.builtin_concepts import BuiltinConcepts, ParserResultConcept @@ -86,22 +88,22 @@ class PythonEvaluator(OneReturnValueEvaluator): return sheerka.ret(self.name, False, error, parents=[return_value]) def get_globals(self, context, node): - my_locals = { + my_globals = { "Concept": core.concept.Concept, "BuiltinConcepts": core.builtin_concepts.BuiltinConcepts, } - # has to tbe the first, to allow override - method_from_sheerka = self.update_globals_with_sheerka_methods(my_locals, context) + # has to be the first, to allow override + method_from_sheerka = self.update_globals_with_sheerka_methods(my_globals, context) - self.update_globals_with_context(my_locals, context) - self.update_globals_with_node(my_locals, context, node) + self.update_globals_with_context(my_globals, context) + self.update_globals_with_node(my_globals, context, node) - if self.locals: # when exta values are given. Add them - my_locals.update(self.locals) + if self.locals: # when extra values are given. Add them + my_globals.update(self.locals) - my_locals["sheerka"] = Expando(method_from_sheerka) # it's the last, so I cannot be overridden - return my_locals + my_globals["sheerka"] = Expando(method_from_sheerka) # it's the last, so I cannot be overridden + return my_globals @staticmethod def update_globals_with_sheerka_methods(my_locals, context): @@ -118,6 +120,10 @@ class PythonEvaluator(OneReturnValueEvaluator): for method_name, method in methods_from_sheerka.items(): my_locals[method_name] = method + # Add pipeable functions + for func_name, function in context.sheerka.sheerka_pipeables.items(): + my_locals[func_name] = Pipe(function, context) + return methods_from_sheerka # to allow access using prefix "sheerka." def update_globals_with_context(self, my_locals, context): diff --git a/src/parsers/ExplainParser.py b/src/parsers/ExplainParser.py deleted file mode 100644 index 88b1d24..0000000 --- a/src/parsers/ExplainParser.py +++ /dev/null @@ -1,361 +0,0 @@ -from dataclasses import dataclass, field -from typing import List, Dict - -from core.builtin_concepts import BuiltinConcepts -from core.tokenizer import LexerError, Token -from parsers.BaseParser import Node, UnexpectedTokenErrorNode, BaseSplitIterParser, UnexpectedEof, ErrorNode -from parsers.ExpressionParser import ExprNode, TrueNode, PropertyEqualsNode, PropertyContainsNode, OrNode, AndNode - - -@dataclass() -class ValueErrorNode(ErrorNode): - """ - When the value parse has an incorrect type or value - """ - message: str - token: Token # token when the error is detected - - -@dataclass() -class MultipleDigestError(ErrorNode): - message: str - token: Token - - -@dataclass() -class ExplanationNode(Node): - digest: str # digest of the event to explain - command: str # original explain command - expr: ExprNode = None - record_digest: bool = False - - -@dataclass -class FilterNode(ExprNode): - """ - Wraps predicates - """ - expr: ExprNode - directives: List[ExprNode] = field(default_factory=list) - - def eval(self, obj): - return self.expr.eval(obj) - - -@dataclass -class RecurseDefNode(ExprNode): - """ - It is used to defined the depth of the recursion - """ - depth: int - - -@dataclass -class FormatLNode(ExprNode): - """ - Define the template to use for ExecutionContext when printed in line - """ - template: str - - -@dataclass -class FormatDNode(ExprNode): - """ - Defines the properties to display, and their format - """ - properties: Dict[str, str] - - -@dataclass -class UnionNode(ExprNode): - """ - Define the template to use for ExecutionContext when printed in line - """ - filters: List[FilterNode] - - def eval(self, obj): - if len(self.filters) == 0: - return False - - if len(self.filters) == 0: - return self.filters[0].eval(obj) - - res = False - for f in self.filters[1:]: - res |= f.eval(obj) - return res - - -class ExplainParser(BaseSplitIterParser): - def __init__(self, **kwargs): - super().__init__("Explain", 81, none_on_eof=True) - - def parse_explain(self): - - token = self.get_token() - if token is None: - return BuiltinConcepts.IS_EMPTY - - if token.value != 'explain': - self.add_error(UnexpectedTokenErrorNode("", token, ["explain"])) - return BuiltinConcepts.NOT_FOR_ME - - digest = "" - record_digest = False - expr_node = UnionNode([FilterNode(TrueNode(), [])]) - self.next_token() - while True: - # no need to continue when error - if self.has_error: - return None - - token = self.get_token() - if token is None: - break - - if token.value == "-f" or token.value == "--filter": - self.next_token() - expr_node.filters.append(self.parse_filter()) - elif token.value in ("-r", "--recurse"): - self.next_token() - expr_node.filters[-1].directives.append(self.parse_recurse()) - elif token.value == "--format_l": - self.next_token() - expr_node.filters[-1].directives.append(self.parse_format_l()) - elif token.value == "--format_d": - self.next_token() - expr_node.filters[-1].directives.append(self.parse_format_d()) - elif token.value in ("-d", "--digest"): - self.next_token() - digest = self.parse_digest(digest) - record_digest = True - elif token.value.startswith("-"): - self.add_error(UnexpectedTokenErrorNode("", token, [])) - else: - digest = self.parse_digest(digest) - - return ExplanationNode(digest, self.text, expr=expr_node, record_digest=record_digest) - - def parse_digest(self, digest): - token = self.get_token() - if token is None or token.value.startswith("-"): - return "" - - if digest != "": - self.add_error(MultipleDigestError("Too many digest", token)) - return None - - digest = token.value - self.next_token() - return digest - - def parse_filter(self): - node = self.parse_or() - if node is None: - return None - return FilterNode(node) - - def parse_or(self): - parts = [] - - node = self.parse_and() - if node is None: - return None - - parts.append(node) - - while True: - token = self.get_token() - if token is None or token.value != "or": - break - - self.next_token() - node = self.parse_and() - if node is None: - return None - else: - parts.append(node) - - return parts[0] if len(parts) == 1 else OrNode(*parts) - - def parse_and(self): - parts = [] - - node = self.parse_predicate() - if node is None: - return None - - parts.append(node) - - while True: - token = self.get_token() - if token is None or token.value != "and": - break - - self.next_token() - node = self.parse_predicate() - if node is None: - return None - else: - parts.append(node) - - return parts[0] if len(parts) == 1 else AndNode(*parts) - - def parse_predicate(self): - token = self.get_token() - if token is None: - self.add_error(UnexpectedEof("Unexpected EOF while parsing filter")) - return None - - if token.value == "(": - self.next_token() - expr = self.parse_or() - - token = self.get_token() - if token is None: - self.add_error(UnexpectedEof("Missing right parenthesis")) - return None - if token.value != ")": - self.add_error(UnexpectedTokenErrorNode("Parenthesis mismatch", token, [")"])) - return None - self.next_token() - - else: - expr = self.parse_property_predicate() - - return expr - - def parse_recurse(self): - token = self.get_token() - if token is None: - self.add_error(UnexpectedEof("Unexpected EOF while parsing recurse")) - return None - - try: - depth = int(token.value) - self.next_token() - return RecurseDefNode(depth) - except ValueError: - self.add_error(ValueErrorNode(f"'{token.value}' is not an integer", token)) - return None - - def parse_format_l(self): - token = self.get_token() - if token is None: - self.add_error(UnexpectedEof("Unexpected EOF while parsing format_l")) - return None - - if token.value.startswith("-"): - self.add_error(UnexpectedTokenErrorNode("parsing format_l", token, ["<property name>"])) - return None - - template = token.value - self.next_token() - return FormatLNode(template) - - def parse_format_d(self): - props = {} - - while TrueNode: - token = self.get_token() - if token is None: - self.add_error(UnexpectedEof("Unexpected EOF while parsing format_d")) - return None - - if token.value.startswith("-"): - self.add_error(UnexpectedTokenErrorNode("parsing format_d", token, ["<property name>"])) - return None - - parts = token.value.split(':') - if len(parts) == 1: - props[token.value] = "{" + token.value + "}" - else: - props[parts[0]] = parts[1] - - self.next_token() - token = self.get_token() - - if token is None or token.value.startswith("-"): - break - elif token.value == ",": - self.next_token() - else: - self.add_error(UnexpectedTokenErrorNode("parsing format_d", token, ["<eof>", ","])) - - return FormatDNode(props) - - def parse_property_predicate(self): - token = self.get_token() - if token is None: - self.add_error(UnexpectedEof("Unexpected EOF while parsing predicate")) - return None - prop_name = token.value - if prop_name.startswith("-"): - self.add_error(UnexpectedTokenErrorNode("while parsing predicate", token, ["<property_name>"])) - return None - self.next_token() - - token = self.get_token() - if token is None: - self.add_error(UnexpectedEof("Unexpected EOF while parsing predicate")) - return None - operand = token.value - - if operand not in ("=", "=="): - self.add_error(UnexpectedTokenErrorNode("Unexpected token when parsing predicate", token, ['=', "=="])) - return None - self.next_token() - - token = self.get_token() - if token is None: - self.add_error(UnexpectedEof("Unexpected EOF while parsing filter")) - return None - self.next_token() - prop_value = token.value - - return PropertyEqualsNode(prop_name, prop_value) if operand == "==" else \ - PropertyContainsNode(prop_name, prop_value) - - def parse(self, context, parser_input): - """ - parser_input can be string, but text can also be an list of tokens - :param context: - :param parser_input: - :return: - """ - - context.log(f"Parsing '{parser_input}'", self.name) - sheerka = context.sheerka - - if not isinstance(parser_input, str): - return sheerka.ret(self.name, False, sheerka.new(BuiltinConcepts.NOT_FOR_ME, reason=parser_input)) - - explanation_node = None - try: - self.reset_parser(context, parser_input) - self.next_token() - explanation_node = self.parse_explain() - except LexerError as e: - self.add_error(e, False) - - if self.has_error or not isinstance(explanation_node, ExplanationNode): - if explanation_node in (BuiltinConcepts.NOT_FOR_ME, BuiltinConcepts.IS_EMPTY): - error_body = sheerka.new( - BuiltinConcepts.NOT_FOR_ME, - body=parser_input, - reason=self.error_sink if self.has_error else BuiltinConcepts.IS_EMPTY) - else: - error_body = sheerka.new( - BuiltinConcepts.ERROR, - body=self.error_sink) - ret = sheerka.ret(self.name, False, error_body) - else: - ret = sheerka.ret(self.name, True, - sheerka.new( - BuiltinConcepts.PARSER_RESULT, - parser=self, - source=parser_input, - body=explanation_node)) - - self.log_result(context, parser_input, ret) - return ret diff --git a/src/printer/FormatInstructions.py b/src/printer/FormatInstructions.py index da9a07e..7e0c903 100644 --- a/src/printer/FormatInstructions.py +++ b/src/printer/FormatInstructions.py @@ -5,7 +5,6 @@ from typing import Dict from core.concept import Concept from core.utils import get_full_qualified_name -from parsers.ExpressionParser import ExprNode class FormatDetailType(Enum): @@ -18,12 +17,21 @@ class FormatDetailDesc: """ class that describes how to print the details """ - predicate: ExprNode # the detail will be printed if the predicate is matched format_type: FormatDetailType properties: Dict[str, str] # name of the property, format to use + def __post_init__(self): + if isinstance(self.properties, list): + self.properties = {p: "{" + p + "}" for p in self.properties} + class FormatInstructions: + """ + This class instructs how a concept should be displayed + In order to be processed, it has to be placed under the property FORMAT_INSTRUCTION of the concept + Note that the format instructions can be modified by rules (when there will be rules !) + """ + def __init__(self, tab_indent=None, tab=None, no_color=None): self._tab_indent = 2 self._tab = "" @@ -31,12 +39,13 @@ class FormatInstructions: self.recursive_props = {} # which property that does in recursion and what depth self.format_l = {} # what format to use when printing obj line by line - self.format_d = [] # list of FormatDetailDesc + self.format_d = {} # What to use when printing obj details - # keep track of the modifications + # keep track of the modifications in order to merge self.modified = set() self.recursive_props_modified = set() self.format_l_modified = set() + self.format_d_modified = set() if tab_indent is not None: self.tab_indent = tab_indent @@ -89,10 +98,16 @@ class FormatInstructions: self.format_l_modified.add(key) return self - def add_format_d(self, predicate, properties, format_type=FormatDetailType.Props_In_Line): - if isinstance(properties, list): - properties = dict((p, "{" + p + " }") for p in properties) - self.format_d.append(FormatDetailDesc(predicate, format_type, properties)) + def set_format_d(self, obj, properties, format_type=FormatDetailType.Props_In_Line): + """ + Defines how to print the detail of an object + :param properties: + :param format_type: + :return: + """ + key = self.get_obj_key(obj) + self.format_d[key] = FormatDetailDesc(format_type, properties) + self.format_d_modified.add(key) return self def clone(self): @@ -100,6 +115,9 @@ class FormatInstructions: return clone def merge(self, other): + if other is None: + return self + for prop in other.modified: setattr(self, prop, getattr(other, prop)) @@ -109,12 +127,13 @@ class FormatInstructions: for key in other.format_l_modified: self.set_format_l(key, other.format_l[key]) - self.format_d.extend(other.format_d) + for key in other.format_d_modified: + self.set_format_d(key, other.format_d[key].properties, other.format_d[key].format_type) return self @staticmethod def get_obj_key(obj): - return obj.id if isinstance(obj, Concept) else \ + return f"c:{obj.id}:" if isinstance(obj, Concept) else \ obj if isinstance(obj, str) else \ get_full_qualified_name(obj) diff --git a/src/printer/Formatter.py b/src/printer/Formatter.py index 25404fb..da6e087 100644 --- a/src/printer/Formatter.py +++ b/src/printer/Formatter.py @@ -1,6 +1,16 @@ +from dataclasses import dataclass + +from core.utils import as_bag from printer.FormatInstructions import FormatDetailDesc, FormatDetailType, FormatInstructions +@dataclass +class BraceToken: + start: int + end: int + colon: int + + class Formatter: def __init__(self): @@ -10,22 +20,34 @@ class Formatter: def reset_formats(self): self.custom_l_formats = {} - self.custom_d_formats = [] + self.custom_d_formats = {} def register_format_l(self, obj, template): key = FormatInstructions.get_obj_key(obj) self.custom_l_formats[key] = template return self - def register_format_d(self, predicate, properties, format_type=FormatDetailType.Props_In_Line): - if isinstance(properties, list): - properties = dict([(p, "{" + p + "}") for p in properties]) - self.custom_d_formats.append(FormatDetailDesc(predicate, format_type, properties)) + def register_format_d(self, obj, properties=None, format_type=FormatDetailType.Props_In_Line): + key = FormatInstructions.get_obj_key(obj) + + if properties is None: + if isinstance(obj, str): + raise Exception("I need the instance of the obj to compute the properties") + bag = as_bag(obj) + properties = dict((p, "{" + p + "}") for p in bag) + + self.custom_d_formats[key] = FormatDetailDesc(format_type, properties) return self - def compute_format_l(self, custom_formats_override, key): - if custom_formats_override and key in custom_formats_override: - custom_template = custom_formats_override[key] + def compute_format_l(self, format_l_override, key): + """ + merge format_l_override and default format_l + :param format_l_override: + :param key: + :return: + """ + if format_l_override and key in format_l_override: + custom_template = format_l_override[key] if custom_template in ("+", "\\+", "+\\"): return custom_template elif custom_template.startswith("+"): @@ -45,28 +67,50 @@ class Formatter: else: return None - def compute_format_d(self, custom_formats_override): - if custom_formats_override and not self.custom_d_formats: - return custom_formats_override - if self.custom_d_formats and not custom_formats_override: + def compute_format_d(self, format_d_override, key): + """ + merge format_d_override and default format_d + :param format_d_override: + :param key: + :return: + """ + if format_d_override and key in format_d_override: + return format_d_override + elif self.custom_d_formats and key in self.custom_d_formats: return self.custom_d_formats - if self.custom_d_formats and custom_formats_override: - return self.custom_d_formats + custom_formats_override - return [] + else: + return None def format_l(self, obj, custom_formats_override=None): + """ + Get the one-line representation of obj + :param obj: + :param custom_formats_override: + :return: + """ key = FormatInstructions.get_obj_key(obj) format_l = self.compute_format_l(custom_formats_override, key) - return self.to_string(obj, format_l) if format_l else str(obj) + return self.to_string_l(obj, format_l) if format_l else str(obj) - def format_d(self, obj, format_d_desc: FormatDetailDesc): + def format_d(self, obj, format_d_override=None): + """ + Get the detail representation of an object + :param obj: object to format + :param format_d_override: dictionary of format_d templates + :return: Formatted representation or None + """ + key = FormatInstructions.get_obj_key(obj) + format_d = self.compute_format_d(format_d_override, key) + if not format_d: + return None + + format_d_desc = format_d[key] max_prop_length = self.get_properties_max_length(format_d_desc.properties.keys()) res = "" for prop, template in format_d_desc.properties.items(): if res: res += "\n" - #value = getattr(obj, prop) if hasattr(obj, prop) else "*Undefined*" - res += prop.ljust(max_prop_length) + ": " + self.to_string(obj, template) + res += prop.ljust(max_prop_length) + ": " + self.to_string_d(obj, template, max_prop_length + 2) return res @@ -75,9 +119,110 @@ class Formatter: return max((len(p) for p in properties)) @staticmethod - def to_string(obj, template): + def to_string_l(obj, template): + try: - bag = obj.to_bag() if hasattr(obj, "to_bag") else obj.__dict__ - return template.format(**bag) - except KeyError: - return "*Undefined*" + bag = as_bag(obj) + return eval("f'" + template + "'", bag) + except KeyError as kerr: + return f"*Undefined {kerr}*" + except NameError as nerr: + return f"*{nerr}*" + + def to_string_d(self, obj, template, tab): + def format_obj(o, _tab, override=None): + if o == obj: + return o + + res = self.format_d(o, override) + if res is not None: + return res + + if isinstance(o, str): + return f'"{o}"' if "'" in o else f"'{o}'" + + if isinstance(o, dict): + if len(o) == 0: + return "{}" + res = "{" + max_prop_length = self.get_properties_max_length([f"'{k}'" for k in o.keys()]) + for prop in o: + if res != "{": + res += "\n" + (" " * (_tab + 1)) + details = str(format_obj(o[prop], _tab + max_prop_length + 3, override)) + res += f"'{prop}'".ljust(max_prop_length) + ": " + details + return res + "}" + + if hasattr(o, "__iter__"): + res = "[" if isinstance(o, list) else "{" if isinstance(o, set) else "(" + for item in o: + if res not in ("[", "(", "{"): + res += "," + "\n" + (" " * (_tab + 1)) + res += str(format_obj(item, _tab + 1, override)) + return res + ("]" if isinstance(o, list) else "}" if isinstance(o, set) else ")") + + return o + + template = self.inject_format_obj(template, tab) + try: + + bag = as_bag(obj) + bag["format_obj"] = format_obj + return eval("f'" + template + "'", bag) + except KeyError as kerr: + return f"*Undefined {kerr}*" + except NameError as nerr: + return f"*{nerr}*" + + @staticmethod + def braces(template): + if template is None: + return + + i = 0 + start = -1 + colon = -1 + while i < len(template): + c = template[i] + if c == "{": + if i + 1 < len(template) and template[i + 1] == "{": + i += 1 + else: + start = i + elif start != -1 and c == ":": + colon = i + elif start != -1 and c == "}": + if i + 1 < len(template) and template[i + 1] == "}": + i += 1 + else: + yield BraceToken(start, i, colon) + start = -1 + colon = -1 + i += 1 + + @staticmethod + def inject_format_obj(template, tab): + if template is None: + return None + + if template == "": + return "" + + res = "" + previous = 0 + for brace_token in iter(Formatter.braces(template)): + res += template[previous:brace_token.start] + res += "{format_obj(" + if brace_token.colon != -1: + res += template[brace_token.start + 1: brace_token.colon] + res += f", {tab})" + res += template[brace_token.colon: brace_token.end + 1] + else: + res += template[brace_token.start + 1: brace_token.end] + res += f", {tab})" + res += template[brace_token.end] + previous = brace_token.end + 1 + + res += template[previous: len(template)] + + return res diff --git a/src/printer/SheerkaPrinter.py b/src/printer/SheerkaPrinter.py index 69992eb..94f376d 100644 --- a/src/printer/SheerkaPrinter.py +++ b/src/printer/SheerkaPrinter.py @@ -1,4 +1,5 @@ -import click +import types + from core.builtin_concepts import BuiltinConcepts from core.concept import Concept from printer.FormatInstructions import FormatInstructions, FormatDetailType @@ -28,7 +29,6 @@ class SheerkaPrinter: def __init__(self, sheerka): self.sheerka = sheerka self.formatter = Formatter() - self.formatter.register_format_l(EXECUTION_CONTEXT_CLASS, "[{id:3}] %tab%{desc} ({status})") self.custom_concepts_printers = None self.reset() @@ -38,6 +38,10 @@ class SheerkaPrinter: str(BuiltinConcepts.RETURN_VALUE): self.print_return_value, } self.formatter.reset_formats() + self.formatter.register_format_l(EXECUTION_CONTEXT_CLASS, "[{id:3}] %tab%{desc} ({status})") + self.formatter.register_format_l(SyntaxError, + '%red%{self.__class__.__name__}: {msg}\\n{text}\\n{"^": >{offset}}%reset%') + self.formatter.register_format_l(Exception, "%red%{self}%reset%") def register_custom_printer(self, concept, custom_format): key = concept.key if isinstance(concept, Concept) else concept @@ -47,12 +51,22 @@ class SheerkaPrinter: def register_format_l(self, obj, template): self.formatter.register_format_l(obj, template) - def register_format_d(self, predicate, properties, format_type=FormatDetailType.Props_In_Line): - self.formatter.register_format_d(predicate, properties, format_type) + def register_format_d(self, obj, properties=None, format_type=FormatDetailType.Props_In_Line): + self.formatter.register_format_d(obj, properties, format_type) def print(self, to_print, instructions=None): + """ + Print using SheerkaPrinter.out + :param to_print: + :param instructions: FormatInstructions + :return: + """ instructions = instructions or FormatInstructions() - self.fp(instructions, to_print) + try: + self.fp(instructions, to_print) + except Exception as ex: + self.fp(instructions, ex) + return def fp(self, instructions, item): """ @@ -61,11 +75,12 @@ class SheerkaPrinter: :param item: :return: """ - if isinstance(item, (list, tuple)): - for i in item: - self.fp(instructions, i) - return - elif isinstance(item, str): + + # first, get the merged instructions + instructions = self.merge_instructions(instructions, item) + + # We can only print string + if isinstance(item, str): for color in COLORS: item = item.replace("%" + color + "%", "" if instructions.no_color else COLORS[color]) if "%tab%" in item: @@ -74,23 +89,48 @@ class SheerkaPrinter: self.out(instructions.tab + item) return + # if list or generator, print one by one + elif hasattr(item, "__iter__"): + for i in item: + self.fp(instructions, i) + return + + # Custom print required elif isinstance(item, Concept) and item.key in self.custom_concepts_printers: self.custom_concepts_printers[item.key](self, instructions, item) + + # get the format per line and print else: self.fp(instructions, self.formatter.format_l(item, instructions.format_l)) # print details - format_d = self.formatter.compute_format_d(instructions.format_d) - for format_d_desc in reversed(format_d): - if format_d_desc.predicate.eval(item): - self.fp(instructions, self.formatter.format_d(item, format_d_desc)) - break + format_d = self.formatter.format_d(item, instructions.format_d) + if format_d: + self.fp(instructions, format_d) if instructions.recursive_props: for k, v in instructions.recursive_props.items(): if hasattr(item, k) and v > 0 and (value := getattr(item, k)) != BuiltinConcepts.NOT_INITIALIZED: self.fp(instructions.recurse(k), value) + @staticmethod + def merge_instructions(instructions, obj): + """ + Merge the format instruction coming from the context with the one from obj + Note that if obj is not a Concept, there is not instruction to merge + :param instructions: + :param obj: + :return: + """ + if not hasattr(obj, "get_format_instructions"): + return instructions + + obj_instructions = obj.get_format_instructions() + if obj_instructions is None: + return instructions + + return instructions.clone().merge(obj_instructions) + @staticmethod def print_explanation(printer, instructions, item): explanation_instructions = instructions.clone().merge(item.instructions) @@ -102,7 +142,7 @@ class SheerkaPrinter: if printer.sheerka.isinstance(item.body, BuiltinConcepts.EXPLANATION): return printer.fp(instructions, item.body) - if isinstance(item.body, (list, tuple)): + if isinstance(item.body, (list, tuple, types.GeneratorType)): return printer.fp(instructions, item.body) status = item.status diff --git a/src/sdp/sheerkaDataProviderIO.py b/src/sdp/sheerkaDataProviderIO.py index 1e2cbf6..6c87041 100644 --- a/src/sdp/sheerkaDataProviderIO.py +++ b/src/sdp/sheerkaDataProviderIO.py @@ -174,6 +174,9 @@ class SheerkaDataProviderDictionaryIO(SheerkaDataProviderIO): stream.close = on_close(self, file_path, stream)(stream.close) return stream + if file_path not in self.cache: + raise FileNotFoundError(file_path) + return io.BytesIO(self.cache[file_path]) if "b" in mode else io.StringIO(self.cache[file_path]) def reset(self): diff --git a/tests/core/test_SheerkaFilter.py b/tests/core/test_SheerkaFilter.py new file mode 100644 index 0000000..4130b8d --- /dev/null +++ b/tests/core/test_SheerkaFilter.py @@ -0,0 +1,213 @@ +from dataclasses import dataclass + +import pytest +from core.builtin_concepts import BuiltinConcepts +from core.sheerka.services.SheerkaFilter import Pipe, SheerkaFilter +from printer.FormatInstructions import FormatInstructions, FormatDetailDesc, FormatDetailType + +from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka + + +@dataclass +class Obj: + prop1: str + prop2: str + + +@dataclass +class ObjWithAsBag: + prop1: str + prop2: str + + def as_bag(self): + return { + "first_prop": self.prop1, + "second_prop": self.prop2, + } + + +class TestSheerkaFilter(TestUsingMemoryBasedSheerka): + + def test_i_can_pipe_using_decorator(self): + @Pipe + def is_ok_with_decorator(iterable): + for item in iterable: + yield item + " ok" + + def exclamation(iterable): + for item in iterable: + yield item + "!" + + res = ["one", "two", "three"] | is_ok_with_decorator | Pipe(exclamation) + assert list(res) == ["one ok!", "two ok!", "three ok!"] + + def test_i_can_pipe_function_with_context_as_first_parameter(self): + def func_with_context(context, iterable, var_name): + for item in iterable: + yield f"{context.desc}: {var_name}={item}" + + sheerka, context = self.init_concepts() + context.desc = "desc" + pipeable = Pipe(func_with_context, context) + + assert pipeable.need_context + assert list(["one", "two", "three"] | pipeable("var")) == ['desc: var=one', 'desc: var=two', 'desc: var=three'] + + def test_i_can_pipe_function_with_context_as_only_parameter(self): + # This time, func_with_context does not have other parameter than context and iterable + def func_with_context(context, iterable): + for item in iterable: + yield f"{context.desc}: var={item}" + + sheerka, context = self.init_concepts() + context.desc = "desc" + pipeable = Pipe(func_with_context, context) + + assert pipeable.need_context + assert list(["one", "two", "three"] | pipeable) == ['desc: var=one', 'desc: var=two', 'desc: var=three'] + + def test_i_can_pipe_explanation_concept(self): + sheerka, context = self.init_concepts() + execution_contexts = [context.push(desc=f"desc_{i}") for i in range(4)] + explanation_node = sheerka.new(BuiltinConcepts.EXPLANATION, body=execution_contexts) + + @Pipe + def get_desc(iterable): + for item in iterable: + yield item.desc + + res = explanation_node | get_desc + + assert sheerka.isinstance(res, BuiltinConcepts.EXPLANATION) + assert list(res.body) == ["desc_0", "desc_1", "desc_2", "desc_3"] # body is modified + + @pytest.mark.parametrize("predicate, expected", [ + ("True", ["one", "two", "three"]), + ("self == 'two'", ["two"]) + ]) + def test_i_can_filter(self, predicate, expected): + filter_service = SheerkaFilter(None) + + res = ["one", "two", "three"] | Pipe(filter_service.pipe_filter)(predicate) + + assert list(res) == expected + + def test_i_can_filter_obj(self): + filter_service = SheerkaFilter(None) + + lst = [Obj("a", "b"), Obj("c", "d")] + predicate = "prop2 == 'd'" + res = lst | Pipe(filter_service.pipe_filter)(predicate) + + assert list(res) == [Obj("c", "d")] + + def test_i_can_filter_obj_implementing_as_bag(self): + filter_service = SheerkaFilter(None) + + lst = [ObjWithAsBag("a", "b"), ObjWithAsBag("c", "d")] + predicate = "second_prop == 'd'" + res = lst | Pipe(filter_service.pipe_filter)(predicate) + + assert list(res) == [ObjWithAsBag("c", "d")] + + def test_i_can_manage_name_error(self): + filter_service = SheerkaFilter(None) + + lst = [Obj("a", "b"), Obj("c", "d"), ObjWithAsBag("a", "b"), ObjWithAsBag("c", "d")] + predicate = "second_prop == 'd'" # 'second_prop' does not exist in Obj + res = lst | Pipe(filter_service.pipe_filter)(predicate) + + assert list(res) == [ObjWithAsBag("c", "d")] + + def test_i_cannot_filter_if_the_predicate_is_incorrect(self): + filter_service = SheerkaFilter(None) + + lst = [Obj("a", "b"), Obj("c", "d")] + predicate = "prop2 ==" + + with pytest.raises(SyntaxError): + res = lst | Pipe(filter_service.pipe_filter)(predicate) + list(res) + + def test_i_can_format_l(self): + sheerka, context, foo, bar = self.init_concepts("foo", "bar") + lst = [foo, bar] + + res = lst | Pipe(SheerkaFilter.pipe_format_l)("my_format") + res = list(res) + + assert len(res) == 2 + format_instructions = res[0].get_prop(BuiltinConcepts.FORMAT_INSTRUCTIONS) + assert isinstance(format_instructions, FormatInstructions) + assert format_instructions.format_l[f"c:{foo.id}:"] == "my_format" + + format_instructions = res[1].get_prop(BuiltinConcepts.FORMAT_INSTRUCTIONS) + assert isinstance(format_instructions, FormatInstructions) + assert format_instructions.format_l[f"c:{bar.id}:"] == "my_format" + + def test_i_can_format_d(self): + sheerka, context, foo, bar = self.init_concepts("foo", "bar") + lst = [foo, bar] + + res = lst | Pipe(SheerkaFilter.pipe_format_d)("id", "name", "body", id="%red%{id}%reset%") + res = list(res) + + expected_props = { + "id": "%red%{id}%reset%", + "name": "{name}", + "body": "{body}" + } + + assert len(res) == 2 + format_instructions = res[0].get_prop(BuiltinConcepts.FORMAT_INSTRUCTIONS) + assert isinstance(format_instructions, FormatInstructions) + assert format_instructions.format_d[f"c:{foo.id}:"] == FormatDetailDesc(FormatDetailType.Props_In_Line, expected_props) + + format_instructions = res[1].get_prop(BuiltinConcepts.FORMAT_INSTRUCTIONS) + assert isinstance(format_instructions, FormatInstructions) + assert format_instructions.format_d[f"c:{bar.id}:"] == FormatDetailDesc(FormatDetailType.Props_In_Line, expected_props) + + def test_i_can_format_d_all_properties(self): + sheerka, context, foo, bar = self.init_concepts("foo", "bar") + lst = [foo, bar] + + res = lst | Pipe(SheerkaFilter.pipe_format_d)() + res = list(res) + + expected_props = { + 'id': '{id}', + 'name': '{name}', + 'key': '{key}', + 'body': '{body}', + 'self': '{self}' + } + + assert len(res) == 2 + format_instructions = res[0].get_prop(BuiltinConcepts.FORMAT_INSTRUCTIONS) + assert isinstance(format_instructions, FormatInstructions) + assert format_instructions.format_d[f"c:{foo.id}:"] == FormatDetailDesc(FormatDetailType.Props_In_Line, expected_props) + + def test_i_can_set_recurse(self): + sheerka, context, foo, bar = self.init_concepts("foo", "bar") + lst = [foo, bar] + + res = lst | Pipe(SheerkaFilter.pipe_recurse)(10) + res = list(res) + + assert len(res) == 2 + format_instructions = res[0].get_prop(BuiltinConcepts.FORMAT_INSTRUCTIONS) + assert isinstance(format_instructions, FormatInstructions) + assert format_instructions.recursive_props["children"] == 10 + + format_instructions = res[1].get_prop(BuiltinConcepts.FORMAT_INSTRUCTIONS) + assert isinstance(format_instructions, FormatInstructions) + assert format_instructions.recursive_props["children"] == 10 + + res = lst | Pipe(SheerkaFilter.pipe_recurse)(15, "other_prop") + res = list(res) + + assert len(res) == 2 + format_instructions = res[0].get_prop(BuiltinConcepts.FORMAT_INSTRUCTIONS) + assert isinstance(format_instructions, FormatInstructions) + assert format_instructions.recursive_props["children"] == 10 + assert format_instructions.recursive_props["other_prop"] == 15 diff --git a/tests/core/test_sheerkaResultManager.py b/tests/core/test_sheerkaResultManager.py new file mode 100644 index 0000000..0cf48c6 --- /dev/null +++ b/tests/core/test_sheerkaResultManager.py @@ -0,0 +1,147 @@ +import types + +from core.builtin_concepts import BuiltinConcepts +from core.sheerka.services.SheerkaResultManager import SheerkaResultConcept +from sdp.sheerkaDataProvider import Event + +from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka + + +class TestSheerkaResultManager(TestUsingMemoryBasedSheerka): + + @classmethod + def setup_class(cls): + sheerka = cls().get_sheerka() + sheerka.save_execution_context = True + + @classmethod + def teardown_class(cls): + sheerka = cls().get_sheerka() + sheerka.save_execution_context = False + + def test_i_can_get_the_result_by_digest(self): + sheerka, context = self.init_concepts() + + sheerka.evaluate_user_input("def concept one as 1") + digest = sheerka.get_last_execution().event.get_digest() + + res = sheerka.get_results_by_digest(context, digest) + + assert sheerka.isinstance(res, BuiltinConcepts.EXPLANATION) + assert res.command == "def concept one as 1" + assert res.digest == digest + assert isinstance(res.body, types.GeneratorType) + + assert sheerka.load(SheerkaResultConcept.NAME, "digest") == digest + + previous_results = list(res.body) + + # Second test, + # I can get the result from the recorded digest + res = sheerka.get_results(context) + assert sheerka.isinstance(res, BuiltinConcepts.EXPLANATION) + assert res.command == "def concept one as 1" + assert res.digest == digest + assert isinstance(res.body, types.GeneratorType) + + assert list(res.body) == previous_results + + def test_i_cannot_get_result_by_digest_if_the_digest_does_not_exist(self): + sheerka, context = self.init_concepts() + + res = sheerka.get_results_by_digest(context, "fake digest") + assert sheerka.isinstance(res, BuiltinConcepts.NOT_FOUND) + assert res.body == {'digest': 'fake digest'} + + def test_i_cannot_get_results_if_no_previous_digest(self): + sheerka, context = self.init_concepts() + assert sheerka.get_results(context) is None + + def test_i_can_get_the_result_by_command_name(self): + sheerka, context = self.init_concepts() + + sheerka.evaluate_user_input("def concept one as 1") + digest = sheerka.get_last_execution().event.get_digest() + + sheerka.evaluate_user_input("one") # another command + + res = sheerka.get_results_by_command(context, "def concept") + assert sheerka.isinstance(res, BuiltinConcepts.EXPLANATION) + assert res.command == "def concept one as 1" + assert res.digest == digest + assert isinstance(res.body, types.GeneratorType) + + def test_i_can_get_the_result_by_command_when_not_in_the_same_page_size(self): + sheerka, context = self.init_concepts() + + sheerka.evaluate_user_input("def concept one as 1") + sheerka.evaluate_user_input("one") + sheerka.evaluate_user_input("one") + sheerka.evaluate_user_input("one") + + service = SheerkaResultConcept(sheerka, 2) + res = service.get_results_by_command(context, "def concept") + assert sheerka.isinstance(res, BuiltinConcepts.EXPLANATION) + assert res.command == "def concept one as 1" + + def test_i_cannot_get_results_from_command_if_the_command_does_not_exist(self): + sheerka, context = self.init_concepts() + + res = sheerka.get_results_by_command(context, "def concept") + assert sheerka.isinstance(res, BuiltinConcepts.NOT_FOUND) + assert res.body == {'command': 'def concept'} + + def test_i_cannot_get_result_from_command_if_the_command_does_not_exists_multiple_pages(self): + sheerka, context = self.init_concepts() + + sheerka.evaluate_user_input("def concept one as 1") + sheerka.evaluate_user_input("one") + sheerka.evaluate_user_input("one") + sheerka.evaluate_user_input("one") + + service = SheerkaResultConcept(sheerka, 2) + res = service.get_results_by_command(context, "fake command") + assert sheerka.isinstance(res, BuiltinConcepts.NOT_FOUND) + assert res.body == {'command': 'fake command'} + + def test_i_can_get_last_results(self): + sheerka, context = self.init_concepts() + + sheerka.evaluate_user_input("def concept one as 1") + sheerka.evaluate_user_input("one") + + res = sheerka.get_last_results(context) + assert sheerka.isinstance(res, BuiltinConcepts.EXPLANATION) + assert res.command == "one" + + def test_i_can_get_last_results_when_event_with_no_result(self): + sheerka, context = self.init_concepts() + + sheerka.sdp.save_event(Event("event 1")) + sheerka.sdp.save_event(Event("event 2")) + sheerka.sdp.save_event(Event("event 3")) + sheerka.evaluate_user_input("def concept one as 1") + + res = sheerka.get_last_results(context) + assert sheerka.isinstance(res, BuiltinConcepts.EXPLANATION) + assert res.command == "def concept one as 1" + + def test_i_cannot_get_last_results_when_no_result(self): + sheerka, context = self.init_concepts() + + res = sheerka.get_last_results(context) + assert sheerka.isinstance(res, BuiltinConcepts.NOT_FOUND) + assert res.body == {'query': 'last'} + + def test_i_cannot_get_last_results_when_only_events(self): + sheerka, context = self.init_concepts() + + sheerka.sdp.save_event(Event("event 1")) + sheerka.sdp.save_event(Event("event 2")) + sheerka.sdp.save_event(Event("event 3")) + + res = sheerka.get_last_results(context) + assert sheerka.isinstance(res, BuiltinConcepts.NOT_FOUND) + assert res.body == {'query': 'last'} + + diff --git a/tests/core/test_sheerka_printer.py b/tests/core/test_sheerka_printer.py index 5411e87..85770c5 100644 --- a/tests/core/test_sheerka_printer.py +++ b/tests/core/test_sheerka_printer.py @@ -3,7 +3,8 @@ from dataclasses import dataclass import pytest from core.builtin_concepts import BuiltinConcepts from core.concept import Concept, ConceptParts -from parsers.ExpressionParser import TrueNode, LambdaNode +from core.sheerka.services.SheerkaFilter import Pipe, SheerkaFilter +from printer.Formatter import Formatter, BraceToken from printer.SheerkaPrinter import FormatInstructions from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka @@ -149,7 +150,43 @@ class TestSheerkaPrinter(TestUsingMemoryBasedSheerka): (None)level33 """ - def test_i_can_format_concept(self, capsys): + def test_i_can_format_l_concepts_using_default_format_l_definition(self, capsys): + # default format_l definition + # for all obj of a given type + + sheerka = self.get_sheerka() + foo = Concept("foo a b").def_var("a").def_var("b").init_key() + foo.set_value("a", "value a").set_value("b", "value b") + foo.set_value(ConceptParts.BODY, "body") + sheerka.set_id_if_needed(foo, False) + + sheerka.printer_handler.register_format_l(foo, "DEFAULT:{id}-{name}-{key}-{body}-{a}-{b}") + + sheerka.print(foo) + captured = capsys.readouterr() + assert captured.out == "DEFAULT:1001-foo a b-foo __var__0 __var__1-body-value a-value b\n" + + def test_i_can_format_l_concepts_using_context_format_l_definition(self, capsys): + # context format_l definition + # for all obj of a given type in the current print call + + sheerka = self.get_sheerka() + foo = Concept("foo a b").def_var("a").def_var("b").init_key() + foo.set_value("a", "value a").set_value("b", "value b") + foo.set_value(ConceptParts.BODY, "body") + sheerka.set_id_if_needed(foo, False) + + sheerka.printer_handler.register_format_l(foo, "DEFAULT:{id}-{name}-{key}-{body}-{a}-{b}") + context_instructions = FormatInstructions().set_format_l(foo, "CONTEXT:{id}-{name}-{key}-{body}-{a}-{b}") + + sheerka.print(foo, context_instructions) + captured = capsys.readouterr() + assert captured.out == "CONTEXT:1001-foo a b-foo __var__0 __var__1-body-value a-value b\n" + + def test_i_can_format_l_concepts_using_item_format_l_definition(self, capsys): + # item format_l definition + # for the item only + sheerka = self.get_sheerka() foo = Concept("foo a b").def_var("a").def_var("b").init_key() foo.set_value("a", "value a").set_value("b", "value b") @@ -157,12 +194,16 @@ class TestSheerkaPrinter(TestUsingMemoryBasedSheerka): sheerka.set_id_if_needed(foo, False) sheerka.printer_handler.register_format_l(foo, "{id}-{name}-{key}-{body}-{a}-{b}") + context_instructions = FormatInstructions().set_format_l(foo, "CONTEXT:{id}-{name}-{key}-{body}-{a}-{b}") + item_instructions = FormatInstructions().set_format_l(foo, "ITEM:{id}-{name}-{key}-{body}-{a}-{b}") - sheerka.print(foo) + foo.set_prop(BuiltinConcepts.FORMAT_INSTRUCTIONS, item_instructions) + + sheerka.print(foo, context_instructions) captured = capsys.readouterr() - assert captured.out == "1001-foo a b-foo __var__0 __var__1-body-value a-value b\n" + assert captured.out == "ITEM:1001-foo a b-foo __var__0 __var__1-body-value a-value b\n" - def test_i_can_format_object(self, capsys): + def test_i_can_format_l_objects(self, capsys): sheerka = self.get_sheerka() foo = Obj("value a", "value b") @@ -172,6 +213,21 @@ class TestSheerkaPrinter(TestUsingMemoryBasedSheerka): captured = capsys.readouterr() assert captured.out == "value a-value b\n" + def test_i_can_format_l_execution_context(self, capsys): + # In this test, no format_l definition is provided + # The system knows how to format ExecutionContext + sheerka = self.get_sheerka() + context = self.get_context(sheerka) + + execution_context = context.push("test_sheerka_printer", "Testing Execution Context Printing") + ret_val = sheerka.ret("test_sheerka_printer", True, sheerka.new(BuiltinConcepts.SUCCESS)) + execution_context.add_values(return_value=ret_val) + ec = execution_context.as_bag() + + sheerka.print(execution_context) + captured = capsys.readouterr() + assert captured.out == f"[{ec['id']:3}] {ec['desc']} ({ec['status']})\n" + def test_i_can_register_a_custom_format_by_its_name(self, capsys): sheerka = self.get_sheerka() foo = Obj("value a", "value b") @@ -182,80 +238,196 @@ class TestSheerkaPrinter(TestUsingMemoryBasedSheerka): captured = capsys.readouterr() assert captured.out == "value a-value b\n" - def test_i_can_define_format_in_print_instruction(self, capsys): - sheerka = self.get_sheerka() - foo = Obj("value a", "value b") + def test_i_can_format_d_concepts_using_default_definition(self, capsys): + sheerka, context, foo = self.init_concepts(Concept("foo a b").def_var("a").def_var("b")) + foo_1 = sheerka.new(foo.key, a="value a", b="value b") + foo_2 = sheerka.new(foo.key, a="value c", b="value d") + lst = [foo_1, foo_2] - instructions = FormatInstructions().set_format_l("tests.core.test_sheerka_printer.Obj", "{a}-{b}") - - sheerka.print(foo, instructions) + sheerka.printer_handler.register_format_d(foo, {"a": "DEFAULT:{a}", "b": "DEFAULT:{b}"}) + sheerka.print(lst) captured = capsys.readouterr() - assert captured.out == "value a-value b\n" + assert captured.out == """(1001)foo a b +a: DEFAULT:'value a' +b: DEFAULT:'value b' +(1001)foo a b +a: DEFAULT:'value c' +b: DEFAULT:'value d' +""" - def test_format_print_instruction_override_register_format(self, capsys): - sheerka = self.get_sheerka() - foo = Obj("value a", "value b") + def test_i_can_format_d_concepts_using_context_definition(self, capsys): + sheerka, context, foo = self.init_concepts(Concept("foo a b").def_var("a").def_var("b")) + foo_1 = sheerka.new(foo.key, a="value a", b="value b") + foo_2 = sheerka.new(foo.key, a="value c", b="value d") + lst = [foo_1, foo_2] - sheerka.printer_handler.register_format_l("tests.core.test_sheerka_printer.Obj", "{a}-{b}") - instructions = FormatInstructions().set_format_l("tests.core.test_sheerka_printer.Obj", "a={a} <> b={b}") - - sheerka.print(foo, instructions) + sheerka.printer_handler.register_format_d(foo, {"a": "DEFAULT:{a}", "b": "DEFAULT:{b}"}) + context_instructions = FormatInstructions().set_format_d(foo, {"a": "CONTEXT:{a}", "b": "CONTEXT:{b}"}) + sheerka.print(lst, context_instructions) captured = capsys.readouterr() - assert captured.out == "a=value a <> b=value b\n" + assert captured.out == """(1001)foo a b +a: CONTEXT:'value a' +b: CONTEXT:'value b' +(1001)foo a b +a: CONTEXT:'value c' +b: CONTEXT:'value d' +""" - def test_i_can_format_d(self, capsys): + def test_i_can_format_d_concepts_using_item_definition(self, capsys): + sheerka, context, foo = self.init_concepts(Concept("foo a b").def_var("a").def_var("b")) + item_instructions = FormatInstructions().set_format_d(foo, {"a": "ITEM:{a}", "b": "ITEM:{b}"}) + foo.set_format_instructions(item_instructions) + foo_1 = sheerka.new(foo.key, a="value a", b="value b") + foo_2 = sheerka.new(foo.key, a="value c", b="value d") + lst = [foo_1, foo_2] + + sheerka.printer_handler.register_format_d(foo, {"a": "DEFAULT:{a}", "b": "DEFAULT:{b}"}) + context_instructions = FormatInstructions().set_format_d(foo, {"a": "CONTEXT:{a}", "b": "CONTEXT:{b}"}) + # Note that this time, foo has a format instruction defined + + sheerka.print(lst, context_instructions) + captured = capsys.readouterr() + assert captured.out == """(1001)foo a b +a: ITEM:'value a' +b: ITEM:'value b' +(1001)foo a b +a: ITEM:'value c' +b: ITEM:'value d' +""" + + def test_i_can_format_d_objects(self, capsys): sheerka = self.get_sheerka() foo = [Obj("value a", "value b"), Obj("value c", "value d")] - sheerka.printer_handler.register_format_d(TrueNode(), ["a", "b"]) + sheerka.printer_handler.register_format_d(Obj, {"a": "CUSTOM:{a}", "b": "CUSTOM:{b}"}) sheerka.print(foo) captured = capsys.readouterr() assert captured.out == """Obj(a='value a', b='value b') -a: value a -b: value b +a: CUSTOM:'value a' +b: CUSTOM:'value b' Obj(a='value c', b='value d') -a: value c -b: value d +a: CUSTOM:'value c' +b: CUSTOM:'value d' +""" + + def test_i_can_format_d_with_only_a_list_of_properties(self, capsys): + sheerka = self.get_sheerka() + foo = [Obj("value a", "value b"), Obj("value c", "value d")] + + sheerka.printer_handler.register_format_d(Obj, ["a", "b"]) + sheerka.print(foo) + captured = capsys.readouterr() + assert captured.out == """Obj(a='value a', b='value b') +a: 'value a' +b: 'value b' +Obj(a='value c', b='value d') +a: 'value c' +b: 'value d' """ def test_i_can_format_d_and_align_properties(self, capsys): sheerka = self.get_sheerka() foo = [ObjLongProp("value a", "value b"), ObjLongProp("value c", "value d")] - sheerka.printer_handler.register_format_d(TrueNode(), ["first_property_name", "second"]) + sheerka.printer_handler.register_format_d(ObjLongProp, ["first_property_name", "second"]) sheerka.print(foo) captured = capsys.readouterr() assert captured.out == """ObjLongProp(first_property_name='value a', second='value b') -first_property_name: value a -second : value b +first_property_name: 'value a' +second : 'value b' ObjLongProp(first_property_name='value c', second='value d') -first_property_name: value c -second : value d +first_property_name: 'value c' +second : 'value d' +""" + + def test_i_can_format_d_all_properties_of_a_concept(self, capsys): + sheerka, context, foo = self.init_concepts(Concept("foo a b").def_var("a").def_var("b")) + foo_1 = sheerka.new(foo.key, a="value a", b="value b") + foo_2 = sheerka.new(foo.key, a="value c", b="value d") + lst = [foo_1, foo_2] + + sheerka.printer_handler.register_format_d(foo) + sheerka.print(lst) + captured = capsys.readouterr() + assert captured.out == """(1001)foo a b +a : 'value a' +var.a: *name 'var' is not defined* +b : 'value b' +var.b: *name 'var' is not defined* +id : '1001' +name : 'foo a b' +key : 'foo __var__0 __var__1' +body : __NOT_INITIALIZED +self : (1001)foo a b +(1001)foo a b +a : 'value c' +var.a: *name 'var' is not defined* +b : 'value d' +var.b: *name 'var' is not defined* +id : '1001' +name : 'foo a b' +key : 'foo __var__0 __var__1' +body : __NOT_INITIALIZED +self : (1001)foo a b +""" + + def test_i_can_format_d_when_dictionary(self, capsys): + sheerka, context, foo = self.init_concepts(Concept("foo a b").def_var("a").def_var("b")) + dict_value = { + "a": "value a", + "beta": {"b1": 10, "b2": Obj("10", 15), "b3": ["items", "in", "a", "list"]}, + "gamma": {"list": ["'quoted string'", '"double" \'single\'', "c3"], "empty": []}, + "epsilon": ["a", "b", "c"], + "g": {"tuple": ("tuple-a", "tuple-b", "tuple-b"), "empty": tuple()}, + "h": {"set": {"set-a"}, "empty": set()}, + } + foo_1 = sheerka.new(foo.key, a="value a", b=dict_value) + lst = [foo_1] + + sheerka.printer_handler.register_format_d(foo) + sheerka.print(lst) + captured = capsys.readouterr() + assert captured.out == """(1001)foo a b +a : 'value a' +var.a: *name 'var' is not defined* +b : {'a' : 'value a' + 'beta' : {'b1': 10 + 'b2': Obj(a='10', b=15) + 'b3': ['items', + 'in', + 'a', + 'list']} + 'gamma' : {'list' : ["'quoted string'", + ""double" 'single'", + 'c3'] + 'empty': []} + 'epsilon': ['a', + 'b', + 'c'] + 'g' : {'tuple': ('tuple-a', + 'tuple-b', + 'tuple-b') + 'empty': ()} + 'h' : {'set' : {'set-a'} + 'empty': {}}} +var.b: *name 'var' is not defined* +id : '1001' +name : 'foo a b' +key : 'foo __var__0 __var__1' +body : __NOT_INITIALIZED +self : (1001)foo a b """ def test_i_can_manage_when_property_does_not_exist(self, capsys): sheerka = self.get_sheerka() foo = Obj("value a", "value b") - sheerka.printer_handler.register_format_d(TrueNode(), ["foo", "bar"]) + sheerka.printer_handler.register_format_d(Obj, ["foo", "bar"]) sheerka.print(foo) captured = capsys.readouterr() assert captured.out == """Obj(a='value a', b='value b') -foo: *Undefined* -bar: *Undefined* -""" - - def test_i_can_select_the_object_to_format_d(self, capsys): - sheerka = self.get_sheerka() - foo = [Obj("value a", "value b"), ObjLongProp("value c", "value d")] - - sheerka.printer_handler.register_format_d(LambdaNode(lambda o: isinstance(o, Obj)), ["a", "b"]) - sheerka.print(foo) - captured = capsys.readouterr() - assert captured.out == """Obj(a='value a', b='value b') -a: value a -b: value b -ObjLongProp(first_property_name='value c', second='value d') +foo: *name 'foo' is not defined* +bar: *name 'bar' is not defined* """ @pytest.mark.parametrize("template, expected", [ @@ -265,7 +437,6 @@ ObjLongProp(first_property_name='value c', second='value d') ("{b}\\+", "value b+\n"), ("+", "+\n"), ("\\+", "\\+\n"), - ("+\\", "+\\\n"), ]) def test_i_can_concat_print_instruction_and_register_format(self, capsys, template, expected): sheerka = self.get_sheerka() @@ -277,3 +448,39 @@ ObjLongProp(first_property_name='value c', second='value d') sheerka.print(foo, instructions) captured = capsys.readouterr() assert captured.out == expected + + def test_i_can_manage_exception_when_printing(self, capsys): + sheerka = self.get_sheerka() + filter_service = SheerkaFilter(sheerka) + predicate = "self='two'" # it should be self=='two' + items = ["one", "two", "three"] | Pipe(filter_service.pipe_filter)(predicate) + + sheerka.print(items) + captured = capsys.readouterr() + assert captured.out == "\x1b[31mSyntaxError: invalid syntax\nself='two'\n ^\x1b[0m\n" + + @pytest.mark.parametrize("template, expected", [ + (None, []), + ("", []), + ("foo", []), + ("{foo}", [BraceToken(0, 4, -1)]), + ("xxx{foo}yyy", [BraceToken(3, 7, -1)]), + ("{foo}{bar}-{baz}", [BraceToken(0, 4, -1), BraceToken(5, 9, -1), BraceToken(11, 15, -1)]), + ("xxx{{foo}}yyy", []), + ("xxx{{foo}yyy", []), + ("xxx{yyy", []), + ("xxx{", []), + ]) + def test_i_can_get_braces(self, template, expected): + assert list(Formatter.braces(template)) == expected + + @pytest.mark.parametrize("template, expected", [ + (None, None), + ("", ""), + ("foo", "foo"), + ("{foo}", "{format_obj(foo, 0)}"), + ("{foo : >16}", "{format_obj(foo , 0): >16}"), + ("{foo : >16}{bar}xxx{baz}", "{format_obj(foo , 0): >16}{format_obj(bar, 0)}xxx{format_obj(baz, 0)}"), + ]) + def test_i_can_inject_format_obj(self, template, expected): + assert Formatter.inject_format_obj(template, 0) == expected diff --git a/tests/evaluators/test_ExplainEvaluator.py b/tests/evaluators/test_ExplainEvaluator.py deleted file mode 100644 index 7f8aef8..0000000 --- a/tests/evaluators/test_ExplainEvaluator.py +++ /dev/null @@ -1,317 +0,0 @@ -import os - -import pytest -from core.builtin_concepts import ParserResultConcept, ReturnValueConcept, BuiltinConcepts -from core.concept import Concept -from core.sheerka.ExecutionContext import ExecutionContext -from evaluators.ExplainEvaluator import ExplainEvaluator -from parsers.ExplainParser import ExplanationNode, RecurseDefNode, FormatLNode, UnionNode, FilterNode, FormatDNode -from parsers.ExpressionParser import PropertyEqualsNode, PropertyEqualsSequenceNode, TrueNode, IsaNode -from printer.FormatInstructions import FormatDetailDesc, FormatDetailType -from pytest import fixture -from sdp.sheerkaDataProvider import Event -from sdp.sheerkaSerializer import Serializer, SerializerContext - -from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka - - -@fixture(scope="module") -def serializer(): - """ - Return a :class:`sdp.sheerkaSerializer.Serializer` instance for the module - """ - return Serializer() - - -class EC: - """ - Helper to create execution context (AKA execution result) - """ - - def __init__(self, children=None, **props): - self.props = props - self.children = children - - -def get_return_value(expr): - if isinstance(expr, ExplanationNode): - value = expr - else: - value = ExplanationNode("xxx_test_explain_evaluator_xxx", "", expr=expr) - - return ReturnValueConcept( - "TestEvaluator", - True, - ParserResultConcept(parser="parser", value=value)) - - -def create_executions_results(context, list_of_ecs): - def update(execution_context, ec): - for prop_name, pro_value in ec.props.items(): - setattr(execution_context, prop_name, pro_value) - - if ec.children: - for child_ec in ec.children: - child_execution_context = execution_context.push("TestEvaluator") - update(child_execution_context, child_ec) - - res = [] - for ec in list_of_ecs: - execution_context = ExecutionContext("TestEvaluator", context.event, context.sheerka) - update(execution_context, ec) - res.append(execution_context) - - return res - - -def get_execution_result_from_file(sheerka, digest, serializer): - target_path = os.path.join("../_fixture/", digest) + "_result" - with open(target_path, "rb") as f: - context = SerializerContext(sheerka=sheerka) - return serializer.deserialize(f, context) - - -def get_execution_result_from_list(executions_result): - return executions_result - - -class TestExplainEvaluator(TestUsingMemoryBasedSheerka): - - @staticmethod - def init_evaluator_with_file(self, serializer): - sheerka = self.get_sheerka() - context = self.get_context(sheerka) - evaluator = ExplainEvaluator() - evaluator.get_execution_result = lambda s, d: get_execution_result_from_file(s, d, serializer) - - return sheerka, context, evaluator - - def init_evaluator_with_list(self, list_of_ecs): - sheerka = self.get_sheerka() - context = self.get_context(sheerka) - evaluator = ExplainEvaluator() - - executions_result = create_executions_results(context, list_of_ecs) - evaluator.get_execution_result = lambda s, d: get_execution_result_from_list(executions_result) - - return sheerka, context, evaluator, executions_result - - @pytest.mark.parametrize("ret_val, expected", [ - (ReturnValueConcept("some_name", True, ParserResultConcept(value=ExplanationNode("", ""))), True), - (ReturnValueConcept("some_name", True, ParserResultConcept(value="other thing")), False), - (ReturnValueConcept("some_name", False, "not relevant"), False), - (ReturnValueConcept("some_name", True, Concept()), False) - ]) - def test_i_can_match(self, ret_val, expected): - context = self.get_context() - assert ExplainEvaluator().matches(context, ret_val) == expected - - def test_i_can_eval_in_list(self, serializer): - sheerka, context, evaluator, execution_results = self.init_evaluator_with_list( - [ - EC(desc="correct desc"), - EC(desc="wrong desc"), - ] - ) - ret_val = get_return_value(UnionNode( - [ - FilterNode(TrueNode()), - FilterNode(PropertyEqualsNode("desc", "correct desc")), - ])) - - res = evaluator.eval(context, ret_val) - assert res.status - assert sheerka.isinstance(res.body, BuiltinConcepts.EXPLANATION) - - filtered = res.body.body - assert filtered == [execution_results[0]] - - def test_i_can_eval_in_children(self): - sheerka, context, evaluator, execution_results = self.init_evaluator_with_list( - [ - EC(desc="wrong desc", children=[EC(desc="wrong sub"), EC(desc="good sub")]), - EC(desc="wrong desc", children=[EC(desc="good sub")]), - ] - ) - ret_val = get_return_value(UnionNode( - [ - FilterNode(TrueNode()), - FilterNode(PropertyEqualsNode("desc", "good sub")), - ])) - - res = evaluator.eval(context, ret_val) - assert res.status - assert sheerka.isinstance(res.body, BuiltinConcepts.EXPLANATION) - - filtered = res.body.body - assert filtered == [ - execution_results[0].children[1], - execution_results[1].children[0], - ] - - def test_i_can_evaluate_multiple_filter_node(self): - sheerka, context, evaluator, execution_results = self.init_evaluator_with_list( - [ - EC(desc="parent1", _id=1, children=[EC(desc="wrong sub"), EC(desc="good sub")]), - EC(desc="parent2", children=[EC(desc="wrong sub"), EC(desc="good sub")]), - EC(desc="good sub") - ]) - ret_val = get_return_value(UnionNode( - [ - FilterNode(TrueNode()), - FilterNode(PropertyEqualsNode("id", "1")), - FilterNode(PropertyEqualsNode("desc", "good sub")), - ])) - - res = evaluator.eval(context, ret_val) - assert res.status - assert len(res.body) == 2 - - assert sheerka.isinstance(res.body[0], BuiltinConcepts.EXPLANATION) - assert sheerka.isinstance(res.body[1], BuiltinConcepts.EXPLANATION) - - assert res.body[0].body == [execution_results[0]] - assert res.body[1].body == [ - execution_results[0].children[1], - execution_results[1].children[1], - execution_results[2] - ] - - def test_i_can_eval_parent_and_child(self): - sheerka, context, evaluator, execution_results = self.init_evaluator_with_list( - [ - EC(desc="parent1", children=[EC(desc="wrong sub"), EC(desc="good sub")]), - EC(desc="parent2", children=[EC(desc="wrong sub"), EC(desc="good sub")]), - EC(desc="good sub") - ] - ) - ret_val = get_return_value(UnionNode( - [ - FilterNode(TrueNode()), - FilterNode(PropertyEqualsSequenceNode(["desc", "desc"], ["parent1", "good sub"])), - ])) - - res = evaluator.eval(context, ret_val) - assert res.status - assert sheerka.isinstance(res.body, BuiltinConcepts.EXPLANATION) - - filtered = res.body.body - assert filtered == [ - execution_results[0].children[1], - ] - - def test_i_correctly_create_format_instructions(self): - sheerka, context, evaluator, execution_results = self.init_evaluator_with_list([]) - ret_val = get_return_value(UnionNode( - [ - FilterNode(TrueNode(), [ - RecurseDefNode(2), - FormatLNode("abc"), - FormatDNode({"a": "{a}", "b": "{b}"}) - ]), - ])) - - res = evaluator.eval(context, ret_val) - assert res.status - assert sheerka.isinstance(res.body, BuiltinConcepts.EXPLANATION) - - instructions = res.body.instructions - assert instructions.recursive_props == {"children": 2} - assert instructions.format_l == {'core.sheerka.ExecutionContext.ExecutionContext': 'abc'} - assert instructions.format_d == [FormatDetailDesc( - IsaNode(ExecutionContext), - FormatDetailType.Props_In_Line, - {"a": "{a}", "b": "{b}"})] - - def test_i_correctly_create_format_instructions_with_filtering(self): - sheerka, context, evaluator, execution_results = self.init_evaluator_with_list([]) - - ret_val = get_return_value(UnionNode( - [ - FilterNode(TrueNode()), - FilterNode(PropertyEqualsNode("id", "1"), [RecurseDefNode(2), FormatLNode("abc")]), - ])) - res = evaluator.eval(context, ret_val) - assert res.status - assert sheerka.isinstance(res.body, BuiltinConcepts.EXPLANATION) - - instructions = res.body.instructions - assert instructions.format_l == {'core.sheerka.ExecutionContext.ExecutionContext': 'abc'} - assert instructions.recursive_props == {"children": 2} - - def test_i_can_have_different_instructions_for_different_filtering(self): - sheerka, context, evaluator, execution_results = self.init_evaluator_with_list([]) - ret_val = get_return_value(UnionNode( - [ - FilterNode(TrueNode()), - FilterNode(PropertyEqualsNode("id", "1"), [RecurseDefNode(2)]), - FilterNode(PropertyEqualsNode("desc", "good sub"), [FormatLNode("abc")]), - ])) - - res = evaluator.eval(context, ret_val) - assert res.status - assert len(res.body) == 2 - - assert res.body[0].instructions.recursive_props == {"children": 2} - assert res.body[1].instructions.format_l == {'core.sheerka.ExecutionContext.ExecutionContext': 'abc'} - - def test_filtering_instructions_inherit_from_the_first_filtering_node(self): - sheerka, context, evaluator, execution_results = self.init_evaluator_with_list([]) - ret_val = get_return_value(UnionNode( - [ - FilterNode(TrueNode(), [RecurseDefNode(2)]), - FilterNode(PropertyEqualsNode("id", "1"), [RecurseDefNode(1)]), - FilterNode(PropertyEqualsNode("desc", "good sub"), [FormatLNode("abc")]), - ])) - - res = evaluator.eval(context, ret_val) - assert res.status - assert len(res.body) == 2 - - assert res.body[0].instructions.recursive_props == {"children": 1} # overridden - - assert res.body[1].instructions.format_l == {'core.sheerka.ExecutionContext.ExecutionContext': 'abc'} - assert res.body[1].instructions.recursive_props == {"children": 2} - - def test_i_can_reuse_a_recorded_digest(self): - sheerka, context, evaluator, execution_results = self.init_evaluator_with_list([]) - expr = UnionNode([FilterNode(TrueNode(), [RecurseDefNode(2)])]) - - # need a valid result to test this feature - event = Event("fake message") - execution_context = ExecutionContext("TestExplainEvaluator", event, sheerka) - sheerka.sdp.save_result(execution_context) - - # save another result - event2 = Event("fake message") - execution_context = ExecutionContext("TestExplainEvaluator", event2, sheerka) - sheerka.sdp.save_result(execution_context) - - # digest is recorded during the first call - explanation_node = ExplanationNode(event.get_digest(), "", expr=expr, record_digest=True) - ret_val = get_return_value(explanation_node) - evaluator.eval(context, ret_val) - - # the next call to get_event_digest will load the recorded digest - explanation_node = ExplanationNode("", "", expr=expr, record_digest=False) # digest is not provided - digest = evaluator.get_event_digest(sheerka, explanation_node) - assert digest == event.get_digest() - - # test I can record another digest - explanation_node = ExplanationNode(event2.get_digest(), "", expr=expr, record_digest=True) - ret_val = get_return_value(explanation_node) - evaluator.eval(context, ret_val) - - explanation_node = ExplanationNode("", "", expr=expr, record_digest=False) # digest is not provided - digest = evaluator.get_event_digest(sheerka, explanation_node) - assert digest == event2.get_digest() - - # test can now reset the recorded digest - # (a digest is provided, but record_digest is set to False) - explanation_node = ExplanationNode(event.get_digest(), "", expr=expr, record_digest=False) - ret_val = get_return_value(explanation_node) - evaluator.eval(context, ret_val) - - explanation_node = ExplanationNode("", "", expr=expr, record_digest=False) # digest is not provided - digest = evaluator.get_event_digest(sheerka, explanation_node) - assert digest is None diff --git a/tests/parsers/test_ExplainParser.py b/tests/parsers/test_ExplainParser.py deleted file mode 100644 index 478c496..0000000 --- a/tests/parsers/test_ExplainParser.py +++ /dev/null @@ -1,205 +0,0 @@ -import pytest -from core.builtin_concepts import BuiltinConcepts -from parsers.BaseParser import UnexpectedTokenErrorNode, UnexpectedEof -from parsers.ExplainParser import ExplainParser, ExplanationNode, MultipleDigestError, ValueErrorNode, \ - RecurseDefNode, FormatLNode, UnionNode, FilterNode, FormatDNode -from parsers.ExpressionParser import PropertyContainsNode, PropertyEqualsNode, TrueNode, AndNode, OrNode - -from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka - - -class TestExplainParser(TestUsingMemoryBasedSheerka): - def init_parser(self, **kwargs): - sheerka = self.get_sheerka(singleton=True, **kwargs) - context = self.get_context(sheerka) - parser = ExplainParser() - return sheerka, context, parser - - def test_i_cannot_parse_empty_string(self): - sheerka, context, parser = self.init_parser() - - res = parser.parse(context, "") - - assert not res.status - assert sheerka.isinstance(res.body, BuiltinConcepts.NOT_FOR_ME) - - def test_i_cannot_parse_if_not_for_me(self): - sheerka, context, parser = self.init_parser() - - text = "foo" - res = parser.parse(context, text) - not_for_me = res.body - - assert not res.status - assert sheerka.isinstance(not_for_me, BuiltinConcepts.NOT_FOR_ME) - assert not_for_me.body == text - assert isinstance(not_for_me.reason[0], UnexpectedTokenErrorNode) - - @pytest.mark.parametrize("text, digest, command, directives", [ - # ("explain", "", "explain", []), - ("explain digest", "digest", "explain digest", []), - ("explain -r 3", "", "explain -r 3", [RecurseDefNode(3)]), - ("explain digest -r 3", "digest", "explain digest -r 3", [RecurseDefNode(3)]), - ]) - def test_i_can_parse_explain_without_filter(self, text, digest, command, directives): - sheerka, context, parser = self.init_parser() - - res = parser.parse(context, text) - parser_result = res.body - explanation_node = res.body.body - - assert res.status - assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT) - assert parser_result.parser.name == "parsers.Explain" - assert parser_result.source == text - - assert explanation_node.digest == digest - assert explanation_node.command == command - assert explanation_node.expr == UnionNode([FilterNode(TrueNode(), directives)]) - - def test_i_can_parse_using_filter(self): - sheerka, context, parser = self.init_parser() - - text = "explain -f a=b" - res = parser.parse(context, text) - parser_result = res.body - explanation_node = res.body.body - - assert res.status - assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT) - assert parser_result.parser.name == "parsers.Explain" - assert parser_result.source == text - - assert explanation_node.expr == UnionNode([ - FilterNode(TrueNode()), - FilterNode(PropertyContainsNode("a", "b"))]) - - @pytest.mark.parametrize("text, expected", [ - ("-f a==b", PropertyEqualsNode("a", "b")), - ("--filter a==b", PropertyEqualsNode("a", "b")), - ("-f a==b and c=d", AndNode(PropertyEqualsNode("a", "b"), PropertyContainsNode("c", "d"))), - ("-f a==b or c=d", OrNode(PropertyEqualsNode("a", "b"), PropertyContainsNode("c", "d"))), - ("-f a==b or c==d and e==f", OrNode( - PropertyEqualsNode("a", "b"), - AndNode(PropertyEqualsNode("c", "d"), PropertyEqualsNode("e", "f")))), - ("-f a==b and c==d or e==f", OrNode( - AndNode(PropertyEqualsNode("a", "b"), PropertyEqualsNode("c", "d")), - PropertyEqualsNode("e", "f"))), - ("-f (a==b or c==d) and e==f", AndNode( - OrNode(PropertyEqualsNode("a", "b"), PropertyEqualsNode("c", "d")), - PropertyEqualsNode("e", "f"))), - ]) - def test_i_can_parse_filter_expressions(self, text, expected): - sheerka, context, parser = self.init_parser() - - res = parser.parse(context, "explain " + text) - parser_result = res.body - explanation_node = res.body.body - expr_node = explanation_node.expr.filters[-1].expr - - assert res.status - assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT) - assert isinstance(explanation_node, ExplanationNode) - - assert expr_node == expected - - @pytest.mark.parametrize("text, expected", [ - ("-r 2", [ - FilterNode(TrueNode(), [RecurseDefNode(2)]) - ]), - ("--format_l 'abc'", [ - FilterNode(TrueNode(), [FormatLNode('abc')]) - ]), - ("--format_d 'abc'", [ - FilterNode(TrueNode(), [FormatDNode({"abc": "{abc}"})]) - ]), - ("--format_d a,b,c", [ - FilterNode(TrueNode(), [FormatDNode({"a": "{a}", "b": "{b}", "c": "{c}"})]) - ]), - ("--format_d a , b , c", [ - FilterNode(TrueNode(), [FormatDNode({"a": "{a}", "b": "{b}", "c": "{c}"})]) - ]), - ("-r 2 --format_l 'abc'", [ - FilterNode(TrueNode(), [RecurseDefNode(2), FormatLNode('abc')]) - ]), - ("--format_d a, b -r 2", [ - FilterNode(TrueNode(), [FormatDNode({"a": "{a}", "b": "{b}"}), RecurseDefNode(2)]) - ]), - ("-f a==b -r 3", [ - FilterNode(TrueNode()), - FilterNode(PropertyEqualsNode("a", "b"), [RecurseDefNode(3)]), - ]), - ("-f a==b --format_l 'abc'", [ - FilterNode(TrueNode()), - FilterNode(PropertyEqualsNode("a", "b"), [FormatLNode("abc")]), - ]), - ("-r 3 -f a==b", [ - FilterNode(TrueNode(), [RecurseDefNode(3)]), - FilterNode(PropertyEqualsNode("a", "b"), []), - ]), - ("--format_l 'abc' -f a==b", [ - FilterNode(TrueNode(), [FormatLNode("abc")]), - FilterNode(PropertyEqualsNode("a", "b"), []), - ]), - ("-f a==b -f c==d", [ - FilterNode(TrueNode()), - FilterNode(PropertyEqualsNode("a", "b")), - FilterNode(PropertyEqualsNode("c", "d")) - ]), - ("-r 1 -f a==b -r 2 -f c==d -r 3", [ - FilterNode(TrueNode(), [RecurseDefNode(1)]), - FilterNode(PropertyEqualsNode("a", "b"), [RecurseDefNode(2)]), - FilterNode(PropertyEqualsNode("c", "d"), [RecurseDefNode(3)]) - ]), - ]) - def test_i_can_parse_other_directives(self, text, expected): - sheerka, context, parser = self.init_parser() - - res = parser.parse(context, "explain " + text) - parser_result = res.body - explanation_node = res.body.body - expr_node = explanation_node.expr - - assert res.status - assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT) - assert isinstance(explanation_node, ExplanationNode) - - assert expr_node.filters == expected - - @pytest.mark.parametrize("text, expected", [ - ("explain -d digest", "digest"), - ("explain -d", ""), - ("explain -d -f a=b", "") - ]) - def test_i_can_parse_record_digest(self, text, expected): - sheerka, context, parser = self.init_parser() - - res = parser.parse(context, text) - explanation_node = res.body.body - - assert explanation_node.digest == expected - assert explanation_node.record_digest - - @pytest.mark.parametrize("text, expected_error_type", [ - ("explain digest1 digest2", MultipleDigestError), - ("explain -r", UnexpectedEof), - ("explain -r foo", ValueErrorNode), - ("explain -r 1.2", ValueErrorNode), - ("explain -f -r 1.2", UnexpectedTokenErrorNode), - ("explain -f", UnexpectedEof), - ("explain --format_d", UnexpectedEof), - ("explain --format_l", UnexpectedEof), - ("explain --format_l -r foo", UnexpectedTokenErrorNode), - ("explain --format_d -r foo", UnexpectedTokenErrorNode), - ]) - def test_i_cannot_parse(self, text, expected_error_type): - sheerka, context, parser = self.init_parser() - - res = parser.parse(context, text) - error = res.body - errors = res.body.body - - assert not res.status - assert sheerka.isinstance(error, BuiltinConcepts.ERROR) - assert len(errors) == 1 - assert isinstance(errors[0], expected_error_type)