From 8b86998225d2d2137ed6a57c6844bd9a988fd6f4 Mon Sep 17 00:00:00 2001 From: Kodjo Sossouvi Date: Thu, 3 Dec 2020 21:50:48 +0100 Subject: [PATCH] First implementation of Debugger for SyaNodeParser --- _concepts_admin.txt | 29 +- _concepts_default.txt | 27 + _concepts_full.txt | 99 +- _concepts_lite.txt | 4 +- _concepts_numbers.txt | 92 ++ _concepts_test-adjective.txt | 9 + src/core/builtin_concepts.py | 21 + src/core/concept.py | 23 +- src/core/global_symbols.py | 8 +- src/core/sheerka/ExecutionContext.py | 15 +- src/core/sheerka/Sheerka.py | 64 +- src/core/sheerka/services/SheerkaAdmin.py | 41 - .../services/SheerkaComparisonManager.py | 6 +- .../services/SheerkaCreateNewConcept.py | 4 + .../sheerka/services/SheerkaDebugManager.py | 413 ++++--- src/core/sheerka/services/SheerkaDump.py | 5 +- src/core/sheerka/services/SheerkaMemory.py | 28 +- .../sheerka/services/SheerkaResultManager.py | 175 ++- .../sheerka/services/SheerkaRuleManager.py | 17 +- .../sheerka/services/SheerkaSetsManager.py | 1 + src/core/simple_debug.py | 46 + src/core/utils.py | 137 ++- src/evaluators/AddToMemoryEvaluator.py | 6 + src/evaluators/MutipleSameSuccessEvaluator.py | 4 +- src/out/AsStrVisitor.py | 29 +- src/out/ConsoleVisistor.py | 10 +- src/out/DeveloperVisitor.py | 18 +- src/parsers/BaseNodeParser.py | 2 +- src/parsers/BnfNodeParser.py | 2 + src/parsers/PythonParser.py | 3 - src/parsers/SyaNodeParser.py | 208 +++- src/sdp/sheerkaDataProvider.py | 3 + src/sdp/sheerkaDataProvider_Old.py | 1087 ----------------- src/sdp/sheerkaSerializer.py | 3 +- src/sheerkapickle/SheerkaPickler.py | 6 +- tests/core/test_SheerkaAdmin.py | 112 +- tests/core/test_SheerkaComparisonManager.py | 6 +- tests/core/test_SheerkaDebugManager.py | 348 +++++- tests/core/test_SheerkaMemory.py | 7 +- tests/core/test_SheerkaRuleManager.py | 16 +- tests/core/test_sheerkaResultManager.py | 149 ++- tests/core/test_utils.py | 79 ++ tests/non_reg/test_sheerka_display.py | 16 + tests/non_reg/test_sheerka_non_reg.py | 2 +- tests/out/test_AsStrVisitor.py | 14 +- tests/out/test_DeveloperVisitor.py | 40 + tests/out/test_SheerkaOut.py | 48 + tests/parsers/test_SyaNodeParser.py | 94 +- 48 files changed, 1781 insertions(+), 1795 deletions(-) create mode 100644 _concepts_default.txt create mode 100644 _concepts_numbers.txt create mode 100644 _concepts_test-adjective.txt create mode 100644 src/core/simple_debug.py delete mode 100644 src/sdp/sheerkaDataProvider_Old.py create mode 100644 tests/non_reg/test_sheerka_display.py create mode 100644 tests/out/test_DeveloperVisitor.py diff --git a/_concepts_admin.txt b/_concepts_admin.txt index 264c1d0..f7ba879 100644 --- a/_concepts_admin.txt +++ b/_concepts_admin.txt @@ -8,39 +8,20 @@ set_isa(c:explain last:, __AUTO_EVAL) def concept explain x as get_results(id=x, depth=3) set_isa(c:explain x:, __AUTO_EVAL) -def concept precedence a > precedence b as set_is_greater_than(__PRECEDENCE, a, b) +def concept precedence a > precedence b as set_is_greater_than(__PRECEDENCE, a, b, 'Sya') set_isa(c:precedence a > precedence b:, __AUTO_EVAL) def concept x is a command as set_auto_eval(x, __AUTO_EVAL) set_auto_eval(c:x is a command:) -def concept q from q ? as question(q) pre is_question() -set_is_lesser(__PRECEDENCE, q) -set_auto_eval(c:q:) - -def concept "x is a concept" as isinstance(x, Concept) pre is_question() - -def concept x is a y as set_isa(x, y) -set_auto_eval(c:x is a y:) -def concept x is an y as set_isa(x, y) -set_auto_eval(c:x is an y:) -def concept x is a y as isa(x,y) pre is_question() -# no need to auto eval as it's a question -def concept x is an y as isa(x,y) pre is_question() -# no need to auto eval as it's a question -def concept x has a y as set_hasa(x, y) -set_auto_eval(c:x has a y:) -def concept x has an y as set_hasa(x, y) -set_auto_eval(c:x has an y:) -def concept x has a y as hasa(x,y) pre is_question() -# no need to auto eval as it's a question -def concept x has an y as hasa(x,y) pre is_question() -# no need to auto eval as it's a question - def concept activate debug as set_debug(True) set_auto_eval(c:activate debug:) def concept deactivate debug as set_debug(False) set_auto_eval(c:deactivate debug:) +def concept debug on as set_debug(True) +set_auto_eval(c:debug on:) +def concept debug off as set_debug(False) +set_auto_eval(c:debug off:) def concept activate debug on x as debug_var(x) set_auto_eval(c:activate debug on x:) diff --git a/_concepts_default.txt b/_concepts_default.txt new file mode 100644 index 0000000..8e1e3ad --- /dev/null +++ b/_concepts_default.txt @@ -0,0 +1,27 @@ +# question +def concept q from q ? as question(q) pre is_question() +set_is_lesser(__PRECEDENCE, q, 'Sya') +set_auto_eval(c:q:) + +def concept "x is a concept" as isinstance(x, Concept) pre is_question() + +# is a +def concept x is a y as set_isa(x, y) +set_auto_eval(c:x is a y:) +def concept x is an y as set_isa(x, y) +set_auto_eval(c:x is an y:) +def concept x is a y as isa(x,y) pre is_question() +# no need to auto eval as it's a question +def concept x is an y as isa(x,y) pre is_question() +# no need to auto eval as it's a question + + +# has a +def concept x has a y as set_hasa(x, y) +set_auto_eval(c:x has a y:) +def concept x has an y as set_hasa(x, y) +set_auto_eval(c:x has an y:) +def concept x has a y as hasa(x,y) pre is_question() +# no need to auto eval as it's a question +def concept x has an y as hasa(x,y) pre is_question() +# no need to auto eval as it's a question diff --git a/_concepts_full.txt b/_concepts_full.txt index 3ecd19f..50cf0e9 100644 --- a/_concepts_full.txt +++ b/_concepts_full.txt @@ -1,98 +1,7 @@ #import admin - -# define numbers -def concept one as 1 -def concept two as 2 -def concept three as 3 -def concept four as 4 -def concept five as 5 -def concept six as 6 -def concept seven as 7 -def concept eight as 8 -def concept nine as 9 -def concept ten as 10 -def concept eleven as 11 -def concept twelve as 12 -def concept thirteen as 13 -def concept fourteen as 14 -def concept fifteen as 15 -def concept sixteen as 16 -def concept seventeen as 17 -def concept eighteen as 18 -def concept nineteen as 19 -def concept twenty as 20 -def concept number -set_isa(one, number) -set_isa(two, number) -set_isa(three, number) -set_isa(four, number) -set_isa(five, number) -set_isa(six, number) -set_isa(seven, number) -set_isa(eight, number) -set_isa(nine, number) -set_isa(ten, number) -set_isa(eleven, number) -set_isa(twelve, number) -set_isa(thirteen, number) -set_isa(fourteen, number) -set_isa(fifteen, number) -set_isa(sixteen, number) -set_isa(seventeen, number) -set_isa(eighteen, number) -set_isa(nineteen, number) -set_isa(twenty, number) -def concept twenties from bnf twenty number where number < 10 as twenty + number -set_isa(twenties, number) -def concept thirty as 30 -set_isa(thirty, number) -def concept thirties from bnf thirty number where number < 10 as thirty + number -set_isa(thirties, number) -def concept forty as 40 -set_isa(forty, number) -def concept forties from bnf forty number where number < 10 as forty + number -set_isa(forties, number) -def concept fifty as 50 -set_isa(fifty, number) -def concept fifties from bnf fifty number where number < 10 as fifty + number -set_isa(fifties, number) -def concept sixty as 60 -set_isa(sixty, number) -def concept sixties from bnf sixty number where number < 10 as sixty + number -set_isa(sixties, number) -def concept seventy as 70 -set_isa(seventy, number) -def concept seventies from bnf seventy number where number < 10 as seventy + number -set_isa(seventies, number) -def concept eighty as 80 -set_isa(eighty, number) -def concept eighties from bnf eighty number where number < 10 as eighty + number -set_isa(eighties, number) -def concept ninety as 90 -set_isa(ninety, number) -def concept nineties from bnf ninety number where number < 10 as ninety + number -set_isa(nineties, number) -def concept one hundred as 100 -set_isa(one hundred, number) -def concept hundreds from bnf number=n1 'hundred' 'and' number=n2 where n1 < 10 and n2 < 100 as n1 * 100 + n2 -set_isa(hundreds, number) -def concept hundreds from bnf number 'hundred' where number < 10 as number * 100 -set_isa(last_created_concept(), number) -def concept thousands from bnf number 'thousand' where number < 1000 as number * 1000 -set_isa(thousands, number) -def concept thousands from bnf number=n1 'thousand' 'and' number=n2 as n1 * 1000 + n2 where n1 < 1000 and n2 < 1000 -set_isa(last_created_concept(), number) - -# define basic operations on numbers -def concept plus from a plus b as a + b -def concept minus from a minus b as a - b -def concept multiplied from a multiplied by b as a * b -def concept divided from a divided by b as a * b -set_is_greater_than(__PRECEDENCE, multiplied, plus) -set_is_greater_than(__PRECEDENCE, divided, plus) -set_is_greater_than(__PRECEDENCE, multiplied, minus) -set_is_greater_than(__PRECEDENCE, divided, minus) - -activate return values processing +#import default +#import numbers + + diff --git a/_concepts_lite.txt b/_concepts_lite.txt index 2c4ba3f..3c7c1ab 100644 --- a/_concepts_lite.txt +++ b/_concepts_lite.txt @@ -1,4 +1,6 @@ #import admin +#import default + def concept one as 1 def concept two as 2 def concept number @@ -6,5 +8,5 @@ def concept apple def concept table def concept location def concept x is on y as set_attr(x, location, y) -activate return values processing + diff --git a/_concepts_numbers.txt b/_concepts_numbers.txt new file mode 100644 index 0000000..204e7c0 --- /dev/null +++ b/_concepts_numbers.txt @@ -0,0 +1,92 @@ +# define numbers +def concept one as 1 +def concept two as 2 +def concept three as 3 +def concept four as 4 +def concept five as 5 +def concept six as 6 +def concept seven as 7 +def concept eight as 8 +def concept nine as 9 +def concept ten as 10 +def concept eleven as 11 +def concept twelve as 12 +def concept thirteen as 13 +def concept fourteen as 14 +def concept fifteen as 15 +def concept sixteen as 16 +def concept seventeen as 17 +def concept eighteen as 18 +def concept nineteen as 19 +def concept twenty as 20 +def concept number +set_isa(one, number) +set_isa(two, number) +set_isa(three, number) +set_isa(four, number) +set_isa(five, number) +set_isa(six, number) +set_isa(seven, number) +set_isa(eight, number) +set_isa(nine, number) +set_isa(ten, number) +set_isa(eleven, number) +set_isa(twelve, number) +set_isa(thirteen, number) +set_isa(fourteen, number) +set_isa(fifteen, number) +set_isa(sixteen, number) +set_isa(seventeen, number) +set_isa(eighteen, number) +set_isa(nineteen, number) +set_isa(twenty, number) +def concept twenties from bnf twenty number where number < 10 as twenty + number +set_isa(twenties, number) +def concept thirty as 30 +set_isa(thirty, number) +def concept thirties from bnf thirty number where number < 10 as thirty + number +set_isa(thirties, number) +def concept forty as 40 +set_isa(forty, number) +def concept forties from bnf forty number where number < 10 as forty + number +set_isa(forties, number) +def concept fifty as 50 +set_isa(fifty, number) +def concept fifties from bnf fifty number where number < 10 as fifty + number +set_isa(fifties, number) +def concept sixty as 60 +set_isa(sixty, number) +def concept sixties from bnf sixty number where number < 10 as sixty + number +set_isa(sixties, number) +def concept seventy as 70 +set_isa(seventy, number) +def concept seventies from bnf seventy number where number < 10 as seventy + number +set_isa(seventies, number) +def concept eighty as 80 +set_isa(eighty, number) +def concept eighties from bnf eighty number where number < 10 as eighty + number +set_isa(eighties, number) +def concept ninety as 90 +set_isa(ninety, number) +def concept nineties from bnf ninety number where number < 10 as ninety + number +set_isa(nineties, number) +def concept one hundred as 100 +set_isa(one hundred, number) +def concept hundreds from bnf number=n1 'hundred' 'and' number=n2 where n1 < 10 and n2 < 100 as n1 * 100 + n2 +set_isa(hundreds, number) +def concept hundreds from bnf number 'hundred' where number < 10 as number * 100 +set_isa(last_created_concept(), number) +def concept thousands from bnf number 'thousand' where number < 1000 as number * 1000 +set_isa(thousands, number) +def concept thousands from bnf number=n1 'thousand' 'and' number=n2 as n1 * 1000 + n2 where n1 < 1000 and n2 < 1000 +set_isa(last_created_concept(), number) + +# define basic operations on numbers +def concept plus from a plus b as a + b +def concept minus from a minus b as a - b +def concept multiplied from a multiplied by b as a * b +def concept divided from a divided by b as a * b +set_is_greater_than(__PRECEDENCE, multiplied, plus) +set_is_greater_than(__PRECEDENCE, divided, plus) +set_is_greater_than(__PRECEDENCE, multiplied, minus) +set_is_greater_than(__PRECEDENCE, divided, minus) \ No newline at end of file diff --git a/_concepts_test-adjective.txt b/_concepts_test-adjective.txt new file mode 100644 index 0000000..0fd02a1 --- /dev/null +++ b/_concepts_test-adjective.txt @@ -0,0 +1,9 @@ +#import full +def concept size +def concept little +def concept girl +def concept little x as set_attr(x, size, little) ret x +def concept the x ret memory(x) +def concept how is x as get_attr(x, size) + + diff --git a/src/core/builtin_concepts.py b/src/core/builtin_concepts.py index 6afdf2e..cedd395 100644 --- a/src/core/builtin_concepts.py +++ b/src/core/builtin_concepts.py @@ -565,3 +565,24 @@ class ToListConcept(Concept): self.set_value("recurse_on", recurse_on) # which sub items should we display self.set_value("tab", tab) # customise tab (content and length) self._metadata.is_evaluated = True + + +class NewConceptConcept(Concept): + ALL_ATTRIBUTES = ["concept"] + + def __init__(self, concept=None): + Concept.__init__(self, + BuiltinConcepts.NEW_CONCEPT, + True, + False, + BuiltinConcepts.NEW_CONCEPT, + bound_body="concept") + + self.set_value("concept", concept) + self._metadata.is_evaluated = True + + def __repr__(self): + if self.concept: + return f"NewConcept(concept={self.concept}, key='{self.concept.key}')" + else: + return super().__repr__() diff --git a/src/core/concept.py b/src/core/concept.py index 483994d..ab5e637 100644 --- a/src/core/concept.py +++ b/src/core/concept.py @@ -393,7 +393,7 @@ class Concept: for name, value in other.get_metadata().variables: self.def_var(name, value) elif prop == "props": - self._metadata.props = deepcopy(other.get_metadata().props) + self._metadata.props = core.utils.sheerka_deepcopy(other.get_metadata().props) else: setattr(self._metadata, prop, getattr(other.get_metadata(), prop)) @@ -534,6 +534,27 @@ class Concept: bag[prop] = getattr(self, prop) return bag + def as_debug_bag(self, new_obj, recurse): + + bag = {"id": self.id, "name": self.name, "key": self.key} + + # add variable metadata + for k, v in [(k, v) for k, v in self._metadata.variables if v]: + bag[f"meta.{k}"] = f"'{v}'" + + # add compiled info + for k, v in [(k, v) for k, v in self._compiled.items() if isinstance(v, Concept)]: + bag[f"compiled.{k}"] = new_obj(v) if recurse else v + + # add values + for prop in [p for p in self.__dict__ if not p.startswith("_") and not p.startswith("##")]: + v = self.get_value(prop) + if v == NotInit: + continue + bag[f"value.{prop}"] = new_obj(v) if (isinstance(v, Concept) and recurse) else core.utils.escape_str(v) + + return bag + def get_format_instructions(self): from core.builtin_concepts import BuiltinConcepts return self.get_prop(BuiltinConcepts.FORMAT_INSTRUCTIONS) diff --git a/src/core/global_symbols.py b/src/core/global_symbols.py index b09d317..96772b0 100644 --- a/src/core/global_symbols.py +++ b/src/core/global_symbols.py @@ -1,7 +1,9 @@ # events -CONCEPT_PRECEDENCE_MODIFIED = "cpm" -RULE_PRECEDENCE_MODIFIED = "rpm" -CONTEXT_DISPOSED = "cd" +EVENT_CONCEPT_PRECEDENCE_MODIFIED = "evt_cpm" +EVENT_RULE_PRECEDENCE_MODIFIED = "evt_rpm" +EVENT_CONTEXT_DISPOSED = "evt_cd" +EVENT_USER_INPUT_EVALUATED = "evt_uie" +EVENT_CONCEPT_CREATED = "evt_cc" # comparison context RULE_COMPARISON_CONTEXT = "Rule" diff --git a/src/core/sheerka/ExecutionContext.py b/src/core/sheerka/ExecutionContext.py index 5abd0d8..ae43241 100644 --- a/src/core/sheerka/ExecutionContext.py +++ b/src/core/sheerka/ExecutionContext.py @@ -5,18 +5,13 @@ import time from core.builtin_concepts import BuiltinConcepts, ParserResultConcept from core.concept import Concept, get_concept_attrs -from core.global_symbols import CONTEXT_DISPOSED +from core.global_symbols import EVENT_CONTEXT_DISPOSED from core.sheerka.services.SheerkaExecute import NO_MATCH from core.sheerka.services.SheerkaMemory import SheerkaMemory -from core.utils import CONSOLE_COLORS_MAP as CCM +from core.utils import CONSOLE_COLORS_MAP as CCM, CONSOLE_COLUMNS from sdp.sheerkaDataProvider import Event -try: - rows, columns = os.popen('stty size', 'r').read().split() -except ValueError: - rows, columns = 50, 80 - -pp = pprint.PrettyPrinter(indent=2, width=columns) +pp = pprint.PrettyPrinter(indent=2, width=CONSOLE_COLUMNS) DEBUG_TAB_SIZE = 4 @@ -134,7 +129,7 @@ class ExecutionContext: return if self.stm: - self.sheerka.publish(self, CONTEXT_DISPOSED) + self.sheerka.publish(self, EVENT_CONTEXT_DISPOSED) self._stop = time.time_ns() @@ -207,7 +202,7 @@ class ExecutionContext: def activate_push(self): if self._push: if self._push.stm: - self.sheerka.publish(self._push, CONTEXT_DISPOSED) + self.sheerka.publish(self._push, EVENT_CONTEXT_DISPOSED) self._push._stop = time.time_ns() self._push = None diff --git a/src/core/sheerka/Sheerka.py b/src/core/sheerka/Sheerka.py index 68a805b..c50f343 100644 --- a/src/core/sheerka/Sheerka.py +++ b/src/core/sheerka/Sheerka.py @@ -14,9 +14,11 @@ from core.builtin_concepts import BuiltinConcepts, ErrorConcept, ReturnValueConc UnknownConcept, AllBuiltinConcepts from core.concept import Concept, ConceptParts, NotInit, get_concept_attrs from core.error import ErrorObj +from core.global_symbols import EVENT_USER_INPUT_EVALUATED from core.profiling import profile from core.sheerka.ExecutionContext import ExecutionContext from core.sheerka_logger import console_handler +from core.simple_debug import my_debug from core.tokenizer import Token, TokenKind from printer.SheerkaPrinter import SheerkaPrinter from sdp.sheerkaDataProvider import SheerkaDataProvider, Event @@ -81,15 +83,6 @@ class Sheerka(Concept): self.return_value_concept_id = None self.error_concept_id = None - # a concept can be instantiated - # ex: File is a concept, but File('foo.txt') is an instance - # TODO: manage contexts - self.instances = [] - - # List of the known rules by the system - # ex: hello => say('hello') - self.rules = [] - self.sdp: SheerkaDataProvider = None self.cache_manager = CacheManager(cache_only) @@ -105,10 +98,11 @@ class Sheerka(Concept): self.printer_handler = SheerkaPrinter(self) self.during_restore = False + self.during_initialisation = False self._builtins_classes_cache = None self.save_execution_context = True - self.enable_process_return_values = False + self.enable_process_return_values = True self.methods_with_context = {"test_using_context"} # only the names, the method is defined in sheerka_methods self.sheerka_methods = { @@ -120,10 +114,6 @@ class Sheerka(Concept): self.locals = {} - self.last_executions = [] - self.last_return_values = [] - self.execution_count = 0 - @property def resolved_concepts_by_first_keyword(self): """ @@ -196,6 +186,7 @@ class Sheerka(Concept): self.enable_process_return_values = enable_process_return_values try: + self.during_initialisation = True from sheerkapickle.sheerka_handlers import initialize_pickle_handlers initialize_pickle_handlers() @@ -235,6 +226,9 @@ class Sheerka(Concept): except IOError as e: res = ReturnValueConcept(self, False, self.get(BuiltinConcepts.ERROR), e) + finally: + self.during_initialisation = False + return res def initialize_caching(self): @@ -445,6 +439,7 @@ class Sheerka(Concept): :return: """ # self.log.debug(f"Processing user input '{text}', {user_name=}.") + my_debug(f"****************** Processing user input '{text}', {user_name=}.***********************************") event = Event(text, user_name) self.sdp.save_event(event) @@ -456,6 +451,7 @@ class Sheerka(Concept): desc=f"Evaluating '{text}'") as execution_context: user_input = self.ret(self.name, True, self.new(BuiltinConcepts.USER_INPUT, body=text, user_name=user_name)) + execution_context.add_inputs(user_input=user_input) # TODO. Must be a context hint, not a return value reduce_requested = self.ret(self.name, True, self.new(BuiltinConcepts.REDUCE_REQUESTED)) @@ -466,32 +462,12 @@ class Sheerka(Concept): if self.cache_manager.is_dirty: self.cache_manager.commit(execution_context) - # exec_count = ExecutionContext.ids[execution_context.event.get_digest()] - # print("Execution Context Count:", exec_count) - if self.save_execution_context: - try: - # if exec_count > 3400: - # print("Saving result. digest=", execution_context.event.get_digest()) - self.sdp.save_result(execution_context) - except Exception as ex: - print(f"Failed to save execution context. Reason: {ex}") - pass - # self.log.error(f"Failed to save execution context. Reason: {ex}") + self.publish(execution_context, EVENT_USER_INPUT_EVALUATED) # Do not save execution contexts from process_return_values if self.enable_process_return_values: self.process_return_values(execution_context, ret) - self.execution_count += 1 - self._last_execution = execution_context - if len(self.last_executions) == self.MAX_EXECUTION_HISTORY: - del self.last_executions[0] - self.last_executions.append(execution_context) - - if len(self.last_return_values) == self.MAX_RETURN_VALUES_HISTORY: - del self.last_return_values[0] - self.last_return_values.append(ret) - return ret def print(self, result, instructions=None): @@ -887,24 +863,6 @@ class Sheerka(Concept): return self.parsers_prefix + name - # def concepts(self): - # """ - # List of all known concepts (look up in sdp) - # :return: - # """ - # res = [] - # lst = self.sdp.list(self.CONCEPTS_BY_ID_ENTRY) - # for item in lst: - # if isinstance(item, list): - # res.extend(item) - # else: - # res.append(item) - # - # return sorted(res, key=lambda i: int(i.id)) - - def get_last_execution(self): - return self._last_execution - def test(self): return f"I have access to Sheerka !" diff --git a/src/core/sheerka/services/SheerkaAdmin.py b/src/core/sheerka/services/SheerkaAdmin.py index 3087b77..1bf93e8 100644 --- a/src/core/sheerka/services/SheerkaAdmin.py +++ b/src/core/sheerka/services/SheerkaAdmin.py @@ -24,9 +24,6 @@ class SheerkaAdmin(BaseService): self.sheerka.bind_service_method(self.restore, True) self.sheerka.bind_service_method(self.concepts, False) self.sheerka.bind_service_method(self.desc, False) - self.sheerka.bind_service_method(self.last_created_concept, False) - self.sheerka.bind_service_method(self.last_ret, False) - self.sheerka.bind_service_method(self.last_error_ret, False) self.sheerka.bind_service_method(self.extended_isinstance, False) self.sheerka.bind_service_method(self.is_container, False) self.sheerka.bind_service_method(self.format_rules, False) @@ -149,44 +146,6 @@ class SheerkaAdmin(BaseService): def format_rules(self): return self.sheerka.new(BuiltinConcepts.TO_LIST, items=self.sheerka.get_format_rules()) - def last_created_concept(self, use_history=False): - for exec_result in reversed(self.sheerka.last_executions): - return_values = exec_result.values["return_values"] - for ret in return_values: - if ret.status and self.sheerka.isinstance(ret.value, BuiltinConcepts.NEW_CONCEPT): - return ret.value.body - - if use_history: - return self.sheerka.new(BuiltinConcepts.ERROR, body="Not yet implement") - - return self.sheerka.new(BuiltinConcepts.NOT_FOUND) - - def last_ret(self, context, index=-1): - try: - last = self.sheerka.last_return_values[index] - return last[0] if isinstance(last, list) and len(last) == 1 else last - except IndexError: - return None - - def last_error_ret(self, context, index=-1): - while index >= -len(self.sheerka.last_return_values): - last = self.sheerka.last_return_values[index] - last = [last] if not hasattr(last, "__iter__") else last - last = [ret_val for ret_val in last if not ret_val.status] - if len(last) == 0: - index -= 1 - continue - - if len(last) > 1: - return context.sheerka.ret(SheerkaAdmin.NAME, - False, - context.sheerka.new(BuiltinConcepts.TOO_MANY_ERRORS, body=last)) - - return last[0] - - return context.sheerka.ret(SheerkaAdmin.NAME, - False, - context.sheerka.new(BuiltinConcepts.NOT_FOUND)) def extended_isinstance(self, a, b): """ diff --git a/src/core/sheerka/services/SheerkaComparisonManager.py b/src/core/sheerka/services/SheerkaComparisonManager.py index 6d0074a..7bd804b 100644 --- a/src/core/sheerka/services/SheerkaComparisonManager.py +++ b/src/core/sheerka/services/SheerkaComparisonManager.py @@ -3,7 +3,7 @@ from dataclasses import dataclass from cache.Cache import Cache from cache.ListCache import ListCache from core.builtin_concepts import BuiltinConcepts -from core.global_symbols import CONCEPT_PRECEDENCE_MODIFIED, RULE_PRECEDENCE_MODIFIED, RULE_COMPARISON_CONTEXT, \ +from core.global_symbols import EVENT_CONCEPT_PRECEDENCE_MODIFIED, EVENT_RULE_PRECEDENCE_MODIFIED, RULE_COMPARISON_CONTEXT, \ CONCEPT_COMPARISON_CONTEXT from core.builtin_helpers import ensure_concept_or_rule from core.concept import Concept @@ -183,9 +183,9 @@ class SheerkaComparisonManager(BaseService): if comparison_obj.property == BuiltinConcepts.PRECEDENCE: if comparison_obj.context == CONCEPT_COMPARISON_CONTEXT: - self.sheerka.publish(context, CONCEPT_PRECEDENCE_MODIFIED) + self.sheerka.publish(context, EVENT_CONCEPT_PRECEDENCE_MODIFIED) elif comparison_obj.context == RULE_COMPARISON_CONTEXT: - self.sheerka.publish(context, RULE_PRECEDENCE_MODIFIED) + self.sheerka.publish(context, EVENT_RULE_PRECEDENCE_MODIFIED) return self.sheerka.ret(self.NAME, True, self.sheerka.new(BuiltinConcepts.SUCCESS)) diff --git a/src/core/sheerka/services/SheerkaCreateNewConcept.py b/src/core/sheerka/services/SheerkaCreateNewConcept.py index e3eb1b3..90c29af 100644 --- a/src/core/sheerka/services/SheerkaCreateNewConcept.py +++ b/src/core/sheerka/services/SheerkaCreateNewConcept.py @@ -2,6 +2,7 @@ import core.utils from core.builtin_concepts import BuiltinConcepts, ErrorConcept from core.builtin_helpers import ensure_concept from core.concept import Concept, DEFINITION_TYPE_DEF, DEFINITION_TYPE_BNF, freeze_concept_attrs +from core.global_symbols import EVENT_CONCEPT_CREATED from core.sheerka.services.sheerka_service import BaseService from sdp.sheerkaDataProvider import SheerkaDataProviderDuplicateKeyError @@ -85,6 +86,9 @@ class SheerkaCreateNewConcept(BaseService): if concept.get_bnf() and init_bnf_ret_value is not None and init_bnf_ret_value.status: sheerka.cache_manager.clear(sheerka.CONCEPTS_GRAMMARS_ENTRY) + # publish the new concept + sheerka.publish(context, EVENT_CONCEPT_CREATED, concept) + # process the return if needed ret = sheerka.ret(self.NAME, True, sheerka.new(BuiltinConcepts.NEW_CONCEPT, body=concept)) return ret diff --git a/src/core/sheerka/services/SheerkaDebugManager.py b/src/core/sheerka/services/SheerkaDebugManager.py index f6f3c3b..c33e7c6 100644 --- a/src/core/sheerka/services/SheerkaDebugManager.py +++ b/src/core/sheerka/services/SheerkaDebugManager.py @@ -1,20 +1,57 @@ -import os import pprint import re from dataclasses import dataclass from core.builtin_concepts import BuiltinConcepts +from core.concept import Concept, NotInit from core.sheerka.ExecutionContext import ExecutionContext from core.sheerka.services.sheerka_service import BaseService -from core.utils import CONSOLE_COLORS_MAP as CCM +from core.utils import CONSOLE_COLORS_MAP as CCM, CONSOLE_COLUMNS, PRIMITIVES_TYPES from core.utils import evaluate_expression, as_bag -try: - rows, columns = os.popen('stty size', 'r').read().split() -except ValueError: - rows, columns = 50, 80 +pp = pprint.PrettyPrinter(indent=2, width=CONSOLE_COLUMNS) -pp = pprint.PrettyPrinter(indent=2, width=columns) +NotFound = "** Not Found **" + + +class ConceptDebugObj: + def __init__(self, concept): + self.concept = concept + self.attrs = concept.as_debug_bag(lambda x: ConceptDebugObj(x), True) + + def __repr__(self): + return f"({self.concept_repr()})" + + def __eq__(self, other): + if not isinstance(other, ConceptDebugObj): + return False + + return self.attrs == other.attrs + + def __hash__(self): + return hash(self.concept) + + def concept_repr(self): + res = f":{self.concept.name}|{self.concept.id}:" + + # print metadata + first = True + for k, v in [(k, v) for k, v in self.attrs.items() if k not in ("id", "name", "key")]: + if not first: + res += ", " + res += f"{k}={v}" + first = False + + return res + + +class ConceptNodeDebugObj: + def __init__(self, concept_node): + self.concept_node = concept_node + self.concept_debug = ConceptDebugObj(concept_node.concept) + + def __repr__(self): + return f"ConceptNode({self.concept_debug.concept_repr()})" class BaseDebugLogger: @@ -49,6 +86,9 @@ class BaseDebugLogger: def is_enabled(self): pass + def get_enabled_vars(self): + pass + class NullDebugLogger(BaseDebugLogger): def __init__(self): @@ -57,6 +97,9 @@ class NullDebugLogger(BaseDebugLogger): def is_enabled(self): return False + def get_enabled_vars(self): + pass + class ConsoleDebugLogger(BaseDebugLogger): @@ -70,9 +113,29 @@ class ConsoleDebugLogger(BaseDebugLogger): self.is_highlighted = "" def is_enabled(self): + """ + True if the debug is activated for the current service, method and context + :return: + """ return True + def get_enabled_vars(self): + """ + Returns the list of all enabled variables for this console debugger + :return: + """ + return self.debug_manager.get_enabled_items("vars", + self.context, + self.service_name, + self.method_name, + self.debug_id) + def debug_entering(self, **kwargs): + """ + Log that we start debugging a method (for a specified service and context) + :param kwargs: + :return: + """ super().debug_entering(**kwargs) str_text = f"{CCM['blue']}Entering {self.service_name}.{self.method_name} with {CCM['reset']}" @@ -84,10 +147,24 @@ class ConsoleDebugLogger(BaseDebugLogger): self.debug_manager.debug(self.prefix() + str_vars) def debug_log(self, text, is_error=False): + """ + Prints a debug information (not related to a specific variable, concept or rule) + :param text: + :param is_error: + :return: + """ color = 'red' if is_error else 'blue' self.debug_manager.debug(self.prefix() + f"{CCM[color]}..{text}{CCM['reset']}") def debug_var(self, name, value, is_error=False, hint=None): + """ + Prints the value of a variable + :param name: + :param value: + :param is_error: + :param hint: + :return: + """ enabled = is_error or self.debug_manager.compute_debug_var(self.context, self.service_name, self.method_name, @@ -103,6 +180,12 @@ class ConsoleDebugLogger(BaseDebugLogger): self.debug(str_text, str_vars) def debug_rule(self, rule, results): + """ + Prints debug information related to a specific rule id + :param rule: + :param results: + :return: + """ if not self.debug_manager.compute_debug_rule(self.context, self.service_name, self.method_name, @@ -115,6 +198,13 @@ class ConsoleDebugLogger(BaseDebugLogger): self.debug(str_text, str_vars) def debug_concept(self, concept, text=None, **kwargs): + """ + Prints debug information related to a specific concept + :param concept: + :param text: + :param kwargs: + :return: + """ raw = kwargs.pop('raw', None) if not self.debug_manager.compute_debug_concept(self.context, self.service_name, @@ -283,27 +373,6 @@ class SheerkaDebugManager(BaseService): return debug_for_self, debug_for_children - def inspect(self, context, context_id, *props): - """ - Print - :param context: - :param context_id: - :return: - """ - to_inspect = self.sheerka.get_execution_item(context, context_id) - if not isinstance(to_inspect, ExecutionContext): - return to_inspect - - if not props: - props = ["inputs", "values.return_values"] - - bag = as_bag(to_inspect) - res = {} - for prop in props: - res[prop] = evaluate_expression(prop, bag) - - return self.sheerka.new(BuiltinConcepts.TO_DICT, body=res) - def debug(self, *args, **kwargs): print(*args, **kwargs) @@ -428,6 +497,24 @@ class SheerkaDebugManager(BaseService): return res + def get_enabled_items(self, item_type, context, service_name, method_name, debug_id): + if not self.activated: + return False + + selected = set() + for setting in getattr(self, self.container_name(item_type)): + if not setting.enabled or setting.item is None: + continue + + if (setting.service_name is None or setting.service_name == service_name) and \ + (setting.method_name is None or setting.method_name == method_name) and \ + (setting.context_id is None or setting.context_id == context.id or ( + setting.context_children and context.has_parent(setting.context_id))) and \ + (setting.debug_id is None or setting.debug_id == debug_id): + selected.add(setting.item) + + return selected + def reset_debug(self, context): for item_type in ["vars", "rules", "concepts"]: setting_name = self.container_name(item_type) @@ -487,6 +574,123 @@ class SheerkaDebugManager(BaseService): def compute_debug_rule(self, context, service_name, method_name, item, debug_id): return self.compute_debug_item("rules", context, service_name, method_name, item, debug_id) + def inspect(self, context, *args, **kwargs): + """ + Print + :param context: + :param args: 1st parameter is what to display, the other are the properties to display + :param kwargs: how to display the result + :return: + """ + + if len(args) == 0: + return + + forced_prop, props = None, None + if isinstance(args[0], int): + to_inspect = self.sheerka.get_execution_item(context, int(args[0])) + if isinstance(to_inspect, ExecutionContext): + if len(args) == 1: + props = ["inputs", "values.return_values"] + else: + props = args[1:] + else: + props = None + else: + to_inspect = args[0] + if isinstance(to_inspect, Concept): + if len(args) > 1: + forced_prop = args[1:] + props = forced_prop + else: + props = args[1:] + + bag = self.as_debug_bag(to_inspect, False, forced_prop) + props = props or list(bag.keys()) + + res = self.inspect_object(bag, props, False, **kwargs) + + # Attributes that are not found are probably directly requested by the user + # Let's try for the full names of these attributes + not_found = {k: v for k, v in [(k, v) for k, v in res.items() if v == NotFound]} + if len(not_found) > 0: + to_add = {} + to_remove = [] + + for k, v in not_found.items(): + alternate_props = ["meta." + k, "compiled." + k, "value." + k] + res2 = self.inspect_object(bag, alternate_props, True, **kwargs) + if len(res2) > 0: + to_add.update(res2) + to_remove.append(k) + + res.update(to_add) + for k in to_remove: + del res[k] + + return self.sheerka.new(BuiltinConcepts.TO_DICT, body=res) + + def inspect_object(self, bag, props, discard_not_found, **kwargs): + values_required = kwargs.get("values", False) + as_bag_required = kwargs.get("as_bag", False) + + res = {} + for prop in props: + if prop in res: + # discard duplicates + continue + + try: + value = bag[prop] if prop.startswith("#") else evaluate_expression(prop, bag) + except NameError: + if discard_not_found: + continue + else: + value = NotFound + + if callable(value): + # discard methods + continue + + if values_required: + value = self.get_inner_values(value, **kwargs) + elif as_bag_required: + value = self.get_debug_repr(value, **kwargs) + + res[prop] = value + return res + + def get_inner_values(self, obj, **kwargs): + if obj is None: + return None + + if isinstance(obj, list): + return [self.get_inner_values(item, **kwargs) for item in obj] + + if hasattr(obj, "get_obj_value"): + return obj.get_obj_value() + + if not isinstance(obj, Concept): + return self.get_debug_repr(obj, **kwargs) + + if obj.body is NotInit: + return self.get_debug_repr(obj, **kwargs) + + return self.get_inner_values(obj.body, **kwargs) + + def get_debug_repr(self, obj, **kwargs): + if kwargs.get("as_bag", False): + forced_props = self.get_concept_forced_props(obj) if isinstance(obj, Concept) else None + return self.as_debug_bag(obj, True, forced_props) + + from parsers.BaseNodeParser import ConceptNode + if isinstance(obj, Concept): + return ConceptDebugObj(obj) + elif isinstance(obj, ConceptNode): + return ConceptNodeDebugObj(obj) + else: + return obj + @staticmethod def container_name(item_type): return f"debug_{item_type}_settings" @@ -505,7 +709,7 @@ class SheerkaDebugManager(BaseService): if len(parts) > 1: method_name = None if parts[1] == "*" else parts[1] if len(parts) > 2: - item = parts[2] + item = ".".join(parts[2:]) if len(args) > 1: context_part = args[1] @@ -535,139 +739,22 @@ class SheerkaDebugManager(BaseService): return item, service, method_name, context_id, context_children, debug_id, enabled - # def debug_rule(self, context, rule=None, context_id=None, debug_id=None, enabled=True): - # """ - # Add a debug rule request - # :param context: - # :param rule: - # :param context_id: - # :param debug_id: - # :param enabled: - # :return: - # """ - # rule = str(rule) if rule is not None else None - # for setting in self.debug_rules_settings: - # if setting.rule_id == rule and \ - # setting.context_id == context_id and \ - # setting.debug_id == debug_id: - # setting.enabled = enabled - # break - # else: - # self.debug_rules_settings.append(DebugRuleSetting(rule, - # context_id, - # debug_id, - # enabled)) - # - # self.sheerka.record_var(context, self.NAME, "debug_rules_settings", self.debug_rules_settings) - # return self.sheerka.ret(SheerkaDebugManager.NAME, True, self.sheerka.new(BuiltinConcepts.SUCCESS)) - # - # def debug_concept(self, context, concept=None, context_id=None, debug_id=None, enabled=True): - # """ - # Add a debug rule request - # :param context: - # :param concept: - # :param context_id: - # :param debug_id: - # :param enabled: - # :return: - # """ - # concept_id = concept.id if isinstance(concept, Concept) else str(concept) if concept is not None else None - # for setting in self.debug_concepts_settings: - # if setting.concept_id == concept_id and \ - # setting.context_id == context_id and \ - # setting.debug_id == debug_id: - # setting.enabled = enabled - # break - # else: - # self.debug_concepts_settings.append(DebugConceptSetting(concept_id, - # context_id, - # debug_id, - # enabled)) - # - # self.sheerka.record_var(context, self.NAME, "debug_concepts_settings", self.debug_concepts_settings) - # return self.sheerka.ret(SheerkaDebugManager.NAME, True, self.sheerka.new(BuiltinConcepts.SUCCESS)) - # + @staticmethod + def get_concept_forced_props(concept): + return ["id", "name", "key"] + [p for p in concept.__dict__ if not p.startswith("_")] - # + @staticmethod + def as_debug_bag(obj, recurse=True, forced_props=None): + if type(obj) in PRIMITIVES_TYPES: + return obj - # def compute_debug_var(self, service_name, method_name, context_id, variable_name, debug_id): - # if not self.activated: - # return False - # - # selected = [] - # for setting in self.debug_vars_settings: - # if setting.variable_name is None and setting.debug_id is None: - # continue - # - # if (setting.service_name is None or setting.service_name == service_name) and \ - # (setting.method_name is None or setting.method_name == method_name) and \ - # (setting.context_id is None or setting.context_id == context_id) and \ - # (setting.variable_name is None or - # setting.variable_name == "*" or - # setting.variable_name == variable_name) and \ - # (setting.debug_id is None or setting.debug_id == debug_id): - # selected.append(setting.enabled) - # - # if len(selected) == 0: - # return False - # - # res = selected[0] - # for enabled in selected[1:]: - # if res == False or enabled == False: - # return False - # - # if isinstance(res, str): - # continue - # - # res = enabled - # - # return res - # - # def compute_debug_rule(self, rule_id, context_id, debug_id): - # if not self.activated: - # return False - # - # selected = [] - # for setting in self.debug_rules_settings: - # if (setting.rule_id is None or setting.rule_id == rule_id) and \ - # (setting.context_id is None or setting.context_id == context_id) and \ - # (setting.debug_id is None or setting.debug_id == debug_id): - # selected.append(setting.enabled) - # - # if len(selected) == 0: - # return False - # - # res = selected[0] - # for enabled in selected[1:]: - # res &= enabled - # - # return res - # - # def compute_debug_concept(self, concept_id, context_id, debug_id): - # if not self.activated: - # return False - # - # selected = [] - # for setting in self.debug_concepts_settings: - # if (setting.concept_id is None or setting.concept_id == concept_id) and \ - # (setting.context_id is None or setting.context_id == context_id) and \ - # (setting.debug_id is None or setting.debug_id == debug_id): - # selected.append(setting.enabled) - # - # if len(selected) == 0: - # return False - # - # res = selected[0] - # for enabled in selected[1:]: - # res &= enabled - # - # return res - # - # def reset_debug_rules(self, context): - # self.debug_rules_settings.clear() - # self.sheerka.record_var(context, self.NAME, "debug_rules_settings", self.debug_rules_settings) - # return self.sheerka.ret(SheerkaDebugManager.NAME, True, self.sheerka.new(BuiltinConcepts.SUCCESS)) - # - # def get_debug_settings(self): - # lst = self.debug_vars_settings + self.debug_concepts_settings + self.debug_rules_settings - # return self.sheerka.new(BuiltinConcepts.TO_LIST, body=lst) + res = {'#type#': type(obj).__name__} + if isinstance(obj, Concept) and not obj.get_metadata().is_builtin: + res.update(obj.as_debug_bag(SheerkaDebugManager.as_debug_bag, recurse)) + + else: + forced_props_to_use = [p for p in forced_props if p != "#type#"] if forced_props else None + res.update(as_bag(obj, forced_props_to_use)) + del res["self"] + + return res diff --git a/src/core/sheerka/services/SheerkaDump.py b/src/core/sheerka/services/SheerkaDump.py index 129cbef..bfd1ce1 100644 --- a/src/core/sheerka/services/SheerkaDump.py +++ b/src/core/sheerka/services/SheerkaDump.py @@ -1,16 +1,15 @@ -import os import pprint from core.builtin_concepts import BuiltinConcepts from core.concept import Concept from core.sheerka.ExecutionContext import ExecutionContext from core.sheerka.services.sheerka_service import BaseService +from core.utils import CONSOLE_COLUMNS from sdp.sheerkaDataProvider import SheerkaDataProvider, Event def get_pp(): - rows, columns = os.popen('stty size', 'r').read().split() - pp = pprint.PrettyPrinter(width=columns, compact=True) + pp = pprint.PrettyPrinter(width=CONSOLE_COLUMNS, compact=True) return pp diff --git a/src/core/sheerka/services/SheerkaMemory.py b/src/core/sheerka/services/SheerkaMemory.py index 05fa6d8..5e60c71 100644 --- a/src/core/sheerka/services/SheerkaMemory.py +++ b/src/core/sheerka/services/SheerkaMemory.py @@ -4,7 +4,7 @@ from cache.FastCache import FastCache from cache.ListIfNeededCache import ListIfNeededCache from core.builtin_concepts import BuiltinConcepts from core.concept import Concept -from core.global_symbols import CONTEXT_DISPOSED +from core.global_symbols import EVENT_CONTEXT_DISPOSED from core.sheerka.services.sheerka_service import BaseService, ServiceObj @@ -23,7 +23,7 @@ class SheerkaMemory(BaseService): def __init__(self, sheerka): super().__init__(sheerka) self.short_term_objects = FastCache() - self.objects = ListIfNeededCache(default=lambda k: self.sheerka.sdp.get(self.OBJECTS_ENTRY, k)) + self.memory_objects = ListIfNeededCache(default=lambda k: self.sheerka.sdp.get(self.OBJECTS_ENTRY, k)) self.registration = {} def initialize(self): @@ -37,14 +37,16 @@ class SheerkaMemory(BaseService): self.sheerka.bind_service_method(self.unregister_object, True, visible=False) self.sheerka.bind_service_method(self.add_registered_objects, True, visible=False) self.sheerka.bind_service_method(self.memory, False) + self.sheerka.bind_service_method(self.mem, False) - self.sheerka.cache_manager.register_cache(self.OBJECTS_ENTRY, self.objects, persist=True, use_ref=True) + self.sheerka.cache_manager.register_cache(self.OBJECTS_ENTRY, self.memory_objects, persist=True, use_ref=True) def reset(self): self.short_term_objects.clear() + self.memory_objects.clear() def initialize_deferred(self, context, is_first_time): - self.sheerka.subscribe(CONTEXT_DISPOSED, self.remove_context) + self.sheerka.subscribe(EVENT_CONTEXT_DISPOSED, self.remove_context) def get_from_short_term_memory(self, context, key): while True: @@ -90,16 +92,16 @@ class SheerkaMemory(BaseService): :param concept: :return: """ - self.objects.put(key, MemoryObject(context.event.get_digest(), concept)) + self.memory_objects.put(key, MemoryObject(context.event.get_digest(), concept)) def get_from_memory(self, context, key): """" """ - return self.objects.get(key) + return self.memory_objects.get(key) def register_object(self, context, key, concept): """ - Before adding objects to memory, they first need to be registered + Before adding memory_objects to memory, they first need to be registered More: We don't want to add all evaluated concept into memory (because some of them may be ref to concept already in memory) @@ -126,7 +128,7 @@ class SheerkaMemory(BaseService): def add_registered_objects(self, context): """ - Adds all registered objects + Adds all registered memory_objects :param context: :return: """ @@ -136,7 +138,7 @@ class SheerkaMemory(BaseService): def memory(self, context, name=None): """ - Get the list of all objects in memory + Get the list of all memory_objects in memory :param context: :param name: :return: @@ -154,10 +156,14 @@ class SheerkaMemory(BaseService): return obj.obj res = {} - for k in self.objects: - obj = self.objects.get(k) + for k in self.memory_objects: + obj = self.memory_objects.get(k) if isinstance(obj, list): obj = obj[-1] res[k] = obj.obj return res + + def mem(self): + keys = sorted([k for k in self.memory_objects]) + return {"keys": keys, "len": len(keys)} diff --git a/src/core/sheerka/services/SheerkaResultManager.py b/src/core/sheerka/services/SheerkaResultManager.py index 0a473b6..0f0afaf 100644 --- a/src/core/sheerka/services/SheerkaResultManager.py +++ b/src/core/sheerka/services/SheerkaResultManager.py @@ -1,9 +1,14 @@ import ast +from cache.Cache import Cache from core.builtin_concepts import BuiltinConcepts +from core.global_symbols import EVENT_USER_INPUT_EVALUATED, EVENT_CONCEPT_CREATED from core.sheerka.services.sheerka_service import BaseService +from core.utils import CONSOLE_COLORS_MAP as CCM from core.utils import as_bag +MAX_EXECUTION_HISTORY = 100 + class SheerkaResultConcept(BaseService): NAME = "Result" @@ -11,6 +16,10 @@ class SheerkaResultConcept(BaseService): def __init__(self, sheerka, page_size=30): super().__init__(sheerka) self.page_size = page_size + self.executions_contexts_cache = Cache(MAX_EXECUTION_HISTORY) + self.last_execution = None + self.last_created_concept = None + self.last_created_concept_id = None def initialize(self): self.sheerka.bind_service_method(self.get_results_by_digest, True) # digest is recorded @@ -18,6 +27,19 @@ class SheerkaResultConcept(BaseService): self.sheerka.bind_service_method(self.get_last_results, True) # digest is recorded self.sheerka.bind_service_method(self.get_results, False) self.sheerka.bind_service_method(self.get_execution_item, False) + self.sheerka.bind_service_method(self.get_last_ret, False, as_name="last_ret") + self.sheerka.bind_service_method(self.get_last_created_concept, False, as_name="last_created_concept") + + def initialize_deferred(self, context, is_first_time): + self.restore_values("last_created_concept_id") + self.sheerka.subscribe(EVENT_USER_INPUT_EVALUATED, self.user_input_evaluated) + self.sheerka.subscribe(EVENT_CONCEPT_CREATED, self.new_concept_created) + + def reset(self): + self.executions_contexts_cache.clear() + self.last_execution = None + self.last_created_concept = None + self.last_created_concept_id = None @staticmethod def get_predicate(**kwargs): @@ -38,12 +60,24 @@ class SheerkaResultConcept(BaseService): predicate = " and ".join(res) return compile(ast.parse(predicate, mode="eval"), "", mode="eval") + @staticmethod + def as_list(execution_context, predicate): + def _yield_result(lst): + for e in lst: + if predicate is None or eval(predicate, as_bag(e)): + yield e + + if e._children: + yield from _yield_result(e._children) + + return _yield_result([execution_context]) + def get_results_by_digest(self, context, digest, filter=None, record_digest=True, **kwargs): """ Gets the entire execution tree for the given event digest - :param filter: :param context: :param digest: + :param filter: :param record_digest: :return: """ @@ -54,8 +88,12 @@ class SheerkaResultConcept(BaseService): kwargs["filter"] = filter try: - result = self.sheerka.sdp.load_result(digest) - event = self.sheerka.sdp.load_event(digest) + if digest in self.executions_contexts_cache: + result = self.executions_contexts_cache.get(digest) + event = result.event + else: + result = self.sheerka.sdp.load_result(digest) + event = self.sheerka.sdp.load_event(digest) # there is no real need for a cache of the events if record_digest: context.log(f"Recording digest '{digest}'") @@ -89,7 +127,18 @@ class SheerkaResultConcept(BaseService): if command is None: return None - start = 0 + # first, search in cache + for event_id in self.executions_contexts_cache: + execution_context = self.executions_contexts_cache.get(event_id) + if execution_context.event.message.startswith(command): + return self.get_results_by_digest(context, + execution_context.event.get_digest(), + filter, + record_digest, + **kwargs) + + # not found, search in db + start = len(self.executions_contexts_cache) consumed = 0 while True: for event in self.sheerka.sdp.load_events(self.page_size, start): @@ -103,6 +152,7 @@ class SheerkaResultConcept(BaseService): start += self.page_size consumed = 0 + # not found, return error return self.sheerka.new(BuiltinConcepts.NOT_FOUND, body={"command": command}) def get_last_results(self, context, filter=None, record_digest=True, **kwargs): @@ -114,23 +164,16 @@ class SheerkaResultConcept(BaseService): :return: """ - start = 0 - page_size = 2 - consumed = 0 - while True: - for event in self.sheerka.sdp.load_events(page_size, start): - consumed += 1 - if self.sheerka.sdp.has_result(event.get_digest()): - return self.get_results_by_digest(context, event.get_digest(), filter, record_digest, **kwargs) + if self.last_execution: + return self.get_results_by_digest(context, + self.last_execution.event.get_digest(), + filter, + record_digest, + **kwargs) - if consumed < page_size: - break - - if page_size < 100: - page_size *= 2 - - start += page_size - consumed = 0 + event_id = self._get_last_execution_result_event_id_from_db() + if event_id is not None: + return self.get_results_by_digest(context, event_id, filter, record_digest, **kwargs) return self.sheerka.new(BuiltinConcepts.NOT_FOUND, body={"query": "last"}) @@ -150,12 +193,21 @@ class SheerkaResultConcept(BaseService): return self.get_results_by_digest(context, digest, filter, False, **kwargs) def get_execution_item(self, context, item_id): + """ + Return the item_id'th element of the execution result under investigation + :param context: + :param item_id: + :return: + """ digest = self.sheerka.load_var(self.NAME, "digest") if digest is None: return self.sheerka.new(BuiltinConcepts.NOT_FOUND, body="no digest") try: - result = self.sheerka.sdp.load_result(digest) + if digest in self.executions_contexts_cache: + result = self.executions_contexts_cache.get(digest) + else: + result = self.sheerka.sdp.load_result(digest) items = list(self.as_list(result, self.get_predicate(id=item_id))) if len(items) == 0: @@ -167,14 +219,77 @@ class SheerkaResultConcept(BaseService): context.log_error(f"Digest {digest} is not found.", self.NAME, ex) return self.sheerka.new(BuiltinConcepts.NOT_FOUND, body={"digest": digest}) - @staticmethod - def as_list(execution_context, predicate): - def _yield_result(lst): - for e in lst: - if predicate is None or eval(predicate, as_bag(e)): - yield e + def user_input_evaluated(self, execution_context): + """ + Callback that updates the cache of execution contexts + :param execution_context: + :return: + """ + if self.sheerka.save_execution_context: + try: + self.sheerka.sdp.save_result(execution_context) + except Exception as ex: + print(f"{CCM['red']}Failed to save execution context. Reason: {ex}{CCM['reset']}") + pass + # self.log.error(f"Failed to save execution context. Reason: {ex}") - if e._children: - yield from _yield_result(e._children) + self.executions_contexts_cache.put(execution_context.event.get_digest(), execution_context) + self.last_execution = execution_context - return _yield_result([execution_context]) + def get_last_ret(self, context): + """ + Return the last return value(s) + :return: + """ + if self.last_execution: + return self.last_execution.values["return_values"] + + event_id = self._get_last_execution_result_event_id_from_db() + if event_id is not None: + try: + + execution_result = self.sheerka.sdp.load_result(event_id) + return execution_result.values["return_values"] + + except FileNotFoundError as ex: + context.log_error(f"Digest {event_id} is not found.", self.NAME, ex) + return self.sheerka.new(BuiltinConcepts.NOT_FOUND, body={"digest": event_id}) + + return self.sheerka.new(BuiltinConcepts.NOT_FOUND, body={"query": "last"}) + + def new_concept_created(self, context, concept): + self.last_created_concept = concept + self.last_created_concept_id = concept.id + + self.sheerka.record_var(context, self.NAME, "last_created_concept_id", concept.id) + + def get_last_created_concept(self, context): + if self.last_created_concept: + return self.last_created_concept + + if self.last_created_concept_id: + self.last_created_concept = self.sheerka.new((None, self.last_created_concept_id)) + return self.last_created_concept + + return self.sheerka.new(BuiltinConcepts.NOT_FOUND, body={"query": "last_created_concept"}) + + def _get_last_execution_result_event_id_from_db(self): + start = 0 + page_size = 2 + consumed = 0 + while True: + for event in self.sheerka.sdp.load_events(page_size, start): + consumed += 1 + if self.sheerka.sdp.has_result(event.get_digest()): + return event.get_digest() + + if consumed < page_size: + break + + if page_size < 100: + page_size *= 2 + + start += page_size + consumed = 0 + + return None diff --git a/src/core/sheerka/services/SheerkaRuleManager.py b/src/core/sheerka/services/SheerkaRuleManager.py index 17b4a0e..99d5b83 100644 --- a/src/core/sheerka/services/SheerkaRuleManager.py +++ b/src/core/sheerka/services/SheerkaRuleManager.py @@ -7,7 +7,7 @@ from cache.Cache import Cache from core.builtin_concepts import BuiltinConcepts, ReturnValueConcept from core.builtin_helpers import parse_unrecognized, only_successful, ensure_rule from core.concept import Concept -from core.global_symbols import RULE_PRECEDENCE_MODIFIED, RULE_COMPARISON_CONTEXT +from core.global_symbols import EVENT_RULE_PRECEDENCE_MODIFIED, RULE_COMPARISON_CONTEXT from core.rule import Rule from core.sheerka.services.sheerka_service import BaseService from core.tokenizer import Keywords, TokenKind, Token, IterParser @@ -112,13 +112,14 @@ class FormatAstList(FormatAstNode): prefix: str = None suffix: str = None show_index: bool = False + index: object = None items: object = None def clone(self, **kwargs): return super().clone( FormatAstList(self.variable), - ("items_prop", "recurse_on", "recursion_depth", "debug", "prefix", "suffix", "show_index", "items"), + ("items_prop", "recurse_on", "recursion_depth", "debug", "prefix", "suffix", "show_index", "index", "items"), **kwargs) @@ -496,10 +497,12 @@ class SheerkaRuleManager(BaseService): if is_first_time: # add builtin rules if it's the first initialization of Sheerka self.init_builtin_rules(context) - - # adds the other rules (when it's not the first time) - self.format_rule_cache.populate(lambda: self.sheerka.sdp.list(self.FORMAT_RULE_ENTRY), lambda rule: rule.id) - self.exec_rule_cache.populate(lambda: self.sheerka.sdp.list(self.EXEC_RULE_ENTRY), lambda rule: rule.id) + else: + # adds the other rules (when it's not the first time) + self.format_rule_cache.populate(lambda: self.sheerka.sdp.list(self.FORMAT_RULE_ENTRY), lambda rule: rule.id) + self.exec_rule_cache.populate(lambda: self.sheerka.sdp.list(self.EXEC_RULE_ENTRY), lambda rule: rule.id) + self.format_rule_cache.reset_events() + self.exec_rule_cache.reset_events() # compile all the rules for rule_id in self.format_rule_cache: @@ -508,7 +511,7 @@ class SheerkaRuleManager(BaseService): # update rules priorities self.update_rules_priorities(context) - self.sheerka.subscribe(RULE_PRECEDENCE_MODIFIED, self.update_rules_priorities) + self.sheerka.subscribe(EVENT_RULE_PRECEDENCE_MODIFIED, self.update_rules_priorities) def update_rules_priorities(self, context): """ diff --git a/src/core/sheerka/services/SheerkaSetsManager.py b/src/core/sheerka/services/SheerkaSetsManager.py index e9ccea3..84e9b03 100644 --- a/src/core/sheerka/services/SheerkaSetsManager.py +++ b/src/core/sheerka/services/SheerkaSetsManager.py @@ -43,6 +43,7 @@ class SheerkaSetsManager(BaseService): context.log(f"Setting concept {concept} is a {concept_set}", who=self.NAME) core.builtin_helpers.ensure_concept(concept, concept_set) + if BuiltinConcepts.ISA in concept.get_metadata().props and concept_set in concept.get_metadata().props[ BuiltinConcepts.ISA]: return self.sheerka.ret( diff --git a/src/core/simple_debug.py b/src/core/simple_debug.py new file mode 100644 index 0000000..773c4cf --- /dev/null +++ b/src/core/simple_debug.py @@ -0,0 +1,46 @@ +default_debug_name = "*default*" +debug_activated = set() + + +def my_debug(*args, check_started=None): + """ + Write one line per arg in 'debug.txt' + :param args: + :param check_started: + True : first check if start_debug() was called + : first check if start_debug(name) was called + list of : first check if start_debug() is called for all names + :return: + """ + if check_started and default_debug_name not in debug_activated: + return + + if isinstance(check_started, str) and check_started not in debug_activated: + return + + if isinstance(check_started, list): + for debug_name in check_started: + if debug_name not in debug_activated: + return + + # with open("debug.txt", "a") as f: + # for arg in args: + # if isinstance(arg, list): + # for item in arg: + # f.write(f"{item}\n") + # else: + # f.write(f"{arg}\n") + + +def start_debug(debug_name=default_debug_name, msg=None): + debug_activated.add(debug_name) + if msg: + with open("debug.txt", "a") as f: + f.write(f"{msg}\n") + + +def stop_debug(debug_name=default_debug_name, msg=None): + if msg: + with open("debug.txt", "a") as f: + f.write(f"{msg}\n") + debug_activated.remove(debug_name) diff --git a/src/core/utils.py b/src/core/utils.py index b70e0a1..3162a6f 100644 --- a/src/core/utils.py +++ b/src/core/utils.py @@ -1,16 +1,15 @@ import ast import importlib import inspect +import os import pkgutil +from copy import deepcopy from cache.Cache import Cache from core.ast_helpers import ast_to_props from core.tokenizer import TokenKind, Tokenizer from pyparsing import * -default_debug_name = "*default*" -debug_activated = set() - COLORS = { "black", "red", @@ -43,55 +42,17 @@ integer = Word(nums) escapeSeq = Combine(ESC + '[' + Optional(delimitedList(integer, ';')) + oneOf(list(alphas))) +try: + CONSOLE_ROWS, CONSOLE_COLUMNS = os.popen('stty size', 'r').read().split() + CONSOLE_ROWS, CONSOLE_COLUMNS = int(CONSOLE_ROWS), int(CONSOLE_COLUMNS) +except ValueError: + CONSOLE_ROWS, CONSOLE_COLUMNS = 50, 80 + def no_color_str(text): return Suppress(escapeSeq).transformString(str(text)) -def my_debug(*args, check_started=None): - """ - Write one line per arg in 'debug.txt' - :param args: - :param check_started: - True : first check if start_debug() was called - : first check if start_debug(name) was called - list of : first check if start_debug() is called for all names - :return: - """ - if check_started and default_debug_name not in debug_activated: - return - - if isinstance(check_started, str) and check_started not in debug_activated: - return - - if isinstance(check_started, list): - for debug_name in check_started: - if debug_name not in debug_activated: - return - - with open("debug.txt", "a") as f: - for arg in args: - if isinstance(arg, list): - for item in arg: - f.write(f"{item}\n") - else: - f.write(f"{arg}\n") - - -def start_debug(debug_name=default_debug_name, msg=None): - debug_activated.add(debug_name) - if msg: - with open("debug.txt", "a") as f: - f.write(f"{msg}\n") - - -def stop_debug(debug_name=default_debug_name, msg=None): - if msg: - with open("debug.txt", "a") as f: - f.write(f"{msg}\n") - debug_activated.remove(debug_name) - - def sysarg_to_string(argv): """ Transform a list of strings into a single string @@ -606,13 +567,17 @@ def tokens_index(tokens, sub_tokens, skip=0): raise ValueError(f"sub tokens '{sub_tokens}' not found") -def as_bag(obj): +def as_bag(obj, forced_properties=None): """ Get the properties of an object (static and dynamic) :param obj: + :param forced_properties: :return: """ - if hasattr(obj, "as_bag"): + + if forced_properties: + bag = {p: getattr(obj, p) for p in forced_properties} + elif hasattr(obj, "as_bag"): bag = obj.as_bag() else: bag = {} if type(obj) in PRIMITIVES_TYPES else {prop: getattr(obj, prop) @@ -726,3 +691,77 @@ def dump_ast(node): for to_remove in [", ctx=Load()", ", kind=None", ", type_ignores=[]"]: dump = dump.replace(to_remove, "") return dump + + +def sheerka_deepcopy(obj): + """ + Internal implementation of deepcopy that can handle Concept circular references + :param obj: + :return: + """ + already_seen = {} + + def copy_concept(c): + id_c = id(c) + if id_c in already_seen: + ref = already_seen[id_c] + if ref == '_##_REF_##_': + raise Exception("Circular Ref not managed yet!") + else: + return ref + + already_seen[id_c] = '_##_REF_##_' + + cls = type(c) + instance = cls() + # update the metadata + for prop_name, prop_value in vars(c.get_metadata()).items(): + if prop_name != "props": + setattr(instance.get_metadata(), prop_name, prop_value) + else: + setattr(instance.get_metadata(), prop_name, sheerka_deepcopy(prop_value)) + + # update the values + for prop_name, prop_value in c.values().items(): + setattr(instance, prop_name, prop_value) + + already_seen[id_c] = instance + return instance + + from core.concept import Concept + if isinstance(obj, dict): + res = {sheerka_deepcopy(k): sheerka_deepcopy(v) for k, v in obj.items()} + return res + elif isinstance(obj, list): + return [sheerka_deepcopy(item) for item in obj] + elif isinstance(obj, set): + return {sheerka_deepcopy(item) for item in obj} + elif isinstance(obj, tuple): + return tuple((sheerka_deepcopy(item) for item in obj)) + elif isinstance(obj, Concept): + return copy_concept(obj) + else: + return deepcopy(obj) + + +def escape_str(x): + """ + Returns a string representation that look like what would produce a debugger + :param x: + :return: + """ + if isinstance(x, str): + return f"'{x}'" + return x + + +class NextIdManager: + """ + solely return the next integer + """ + def __init__(self): + self.id = -1 + + def get_next_id(self): + self.id += 1 + return self.id diff --git a/src/evaluators/AddToMemoryEvaluator.py b/src/evaluators/AddToMemoryEvaluator.py index 263d4e9..c9b23a0 100644 --- a/src/evaluators/AddToMemoryEvaluator.py +++ b/src/evaluators/AddToMemoryEvaluator.py @@ -23,5 +23,11 @@ class AddToMemoryEvaluator(OneReturnValueEvaluator): return len(context.sheerka.services[SheerkaMemory.NAME].registration) > 0 def eval(self, context, return_value): + if context.sheerka.during_initialisation: + from core.sheerka.services.SheerkaMemory import SheerkaMemory + service = context.sheerka.services[SheerkaMemory.NAME] + service.registration.clear() + return None + context.sheerka.add_registered_objects(context) return None # no need to have a second pass diff --git a/src/evaluators/MutipleSameSuccessEvaluator.py b/src/evaluators/MutipleSameSuccessEvaluator.py index baf5e9f..8cec1ff 100644 --- a/src/evaluators/MutipleSameSuccessEvaluator.py +++ b/src/evaluators/MutipleSameSuccessEvaluator.py @@ -36,10 +36,10 @@ class MultipleSameSuccessEvaluator(AllReturnValuesEvaluator): to_process = True self.eaten.append(ret) elif ret.who.startswith(BaseEvaluator.PREFIX): + self.eaten.append(ret) if ret.status: nb_successful_evaluators += 1 self.success.append(ret) - self.eaten.append(ret) elif ret.who.startswith(BaseParser.PREFIX): self.eaten.append(ret) if ret.status: @@ -63,7 +63,7 @@ class MultipleSameSuccessEvaluator(AllReturnValuesEvaluator): # I gave a random order to the other # # I guess that we need a proper algorithm to elect which return value to use if they have the same result - # I guts feeling is that, it will depend on the intent of the user + # My guts feeling is that, it will depend on the intent of the user # So it depends on the context # try to return a concept if possible diff --git a/src/out/AsStrVisitor.py b/src/out/AsStrVisitor.py index 4db5b19..52a4dbb 100644 --- a/src/out/AsStrVisitor.py +++ b/src/out/AsStrVisitor.py @@ -1,16 +1,17 @@ import re from core.sheerka.services.SheerkaRuleManager import FormatAstNode -from core.utils import CONSOLE_COLORS_MAP as CCM, no_color_str +from core.utils import CONSOLE_COLORS_MAP as CCM, no_color_str, CONSOLE_COLUMNS from out.OutVisitor import OutVisitor -get_start = re.compile(r"^([\(\{\[]\w*)") +get_starting_brace_bracket_parent = re.compile(r"^([\(\{\[]\w*)") class AsStrVisitor(OutVisitor): - def __init__(self, expand=False): + def __init__(self, expand=None, width=None): self.expand = expand + self.width = width def visit_FormatAstRawText(self, format_ast): return str(format_ast.text) @@ -58,6 +59,7 @@ class AsStrVisitor(OutVisitor): def visit_FormatAstDict(self, format_ast): first = True result = "" + expand = self.expand or False keys_values = [] max_len = 0 @@ -71,23 +73,36 @@ class AsStrVisitor(OutVisitor): if format_ast.prefix: result += format_ast.prefix - sep = ",\n" if self.expand else ", " if format_ast.debug else "\n" + sep = ",\n" if expand else ", " if format_ast.debug else "\n" for i, (k, v) in enumerate(format_ast.items): start = "" if first else sep - key = f"{keys_values[i]:<{max_len}}" if (self.expand or not format_ast.debug) else keys_values[i] + key = f"{keys_values[i]:<{max_len}}" if (expand or not format_ast.debug) else keys_values[i] colon = ": " indent = len(no_color_str(key)) + len(colon) + value = self.visit(v) - if self.expand: - if m := get_start.match(no_color_str(value)): + if self.debug_activated(v) and self.width and len(no_color_str(value)) >= self.width: + value = forced_expanded_visitor.visit(v) + expand = True + + if expand: + if m := get_starting_brace_bracket_parent.match(no_color_str(value)): indent += len(m.group(1)) value = value.replace("\n", "\n" + " " * indent) first = False result += start + key + colon + value + expand = self.expand or False # reset expand if format_ast.suffix: result += format_ast.suffix return result + + @staticmethod + def debug_activated(node): + return isinstance(node, FormatAstNode) and hasattr(node, "debug") and node.debug + + +forced_expanded_visitor = AsStrVisitor(expand=True, width=CONSOLE_COLUMNS) diff --git a/src/out/ConsoleVisistor.py b/src/out/ConsoleVisistor.py index e1c7a41..4a78006 100644 --- a/src/out/ConsoleVisistor.py +++ b/src/out/ConsoleVisistor.py @@ -1,3 +1,4 @@ +from core.utils import CONSOLE_COLUMNS from out.AsStrVisitor import AsStrVisitor @@ -6,7 +7,7 @@ class ConsoleVisitor(AsStrVisitor): Prints to the console """ - def __init__(self, expand_mode="auto"): + def __init__(self, expand_mode="auto", console_width=None): """ expand_mode: auto: not the first dict, the sub ones depends of the width @@ -18,6 +19,7 @@ class ConsoleVisitor(AsStrVisitor): super().__init__() self.out = print self.expand_mode = expand_mode + self.console_width = console_width or CONSOLE_COLUMNS def visit_FormatAstRawText(self, format_ast): self.out(super().visit_FormatAstRawText(format_ast)) @@ -32,11 +34,11 @@ class ConsoleVisitor(AsStrVisitor): self.out(super().visit_FormatAstColor(format_ast)) def visit_FormatAstSequence(self, format_ast): - visitor = AsStrVisitor() + visitor = AsStrVisitor(width=self.console_width) self.out(visitor.visit_FormatAstSequence(format_ast)) def visit_FormatAstList(self, format_ast): - visitor = AsStrVisitor() + visitor = AsStrVisitor(width=self.console_width) res = visitor.visit_FormatAstList(format_ast) self.out(res) @@ -46,6 +48,6 @@ class ConsoleVisitor(AsStrVisitor): else: expand = False - visitor = AsStrVisitor(expand) + visitor = AsStrVisitor(expand, width=self.console_width) res = visitor.visit_FormatAstDict(format_ast) self.out(res) diff --git a/src/out/DeveloperVisitor.py b/src/out/DeveloperVisitor.py index cfe99fe..0065f04 100644 --- a/src/out/DeveloperVisitor.py +++ b/src/out/DeveloperVisitor.py @@ -100,10 +100,20 @@ class DeveloperVisitor: for i, item in enumerate(items): bag["__item"] = item sub_visitor = DeveloperVisitor(self.sheerka_out, self.debugger, set(), self.list_recursion_depth) - result.append(sub_visitor.visit(context, FormatAstVariable("__item", - debug=format_ast.debug, - value=item, - index=i), bag)) + to_visit = FormatAstDict("__item", debug=True, prefix='{', suffix='}') if isinstance(item, dict) else \ + FormatAstList("__item", debug=True, index=i, prefix='[', suffix=']') if isinstance(item, list) else \ + FormatAstVariable("__item", value=item, index=i, debug=format_ast.debug) + result.append(sub_visitor.visit(context, to_visit, bag)) + # if isinstance(item, list): + # format_ast_item = sub_visitor.visit(context, + # FormatAstList("__item", debug=format_ast.debug, index=i), + # bag) + # else: + # format_ast_item = sub_visitor.visit(context, + # FormatAstVariable("__item", debug=format_ast.debug, value=item, + # index=i), + # bag) + # result.append(format_ast_item) # recursion management recursion_depth, recurse_on = self.get_recurse_info(item, recursion_depth, recurse_on) diff --git a/src/parsers/BaseNodeParser.py b/src/parsers/BaseNodeParser.py index ce3c0a4..4877801 100644 --- a/src/parsers/BaseNodeParser.py +++ b/src/parsers/BaseNodeParser.py @@ -873,7 +873,7 @@ class BaseNodeParser(BaseParser): if not to_keep(concept): continue - concept = to_map(self, concept) if to_map else concept + concept = to_map(concept, self, self.sheerka) if to_map else concept result.append(concept) return core.utils.make_unique(result + custom_concepts, diff --git a/src/parsers/BnfNodeParser.py b/src/parsers/BnfNodeParser.py index 6fdf940..e29e63c 100644 --- a/src/parsers/BnfNodeParser.py +++ b/src/parsers/BnfNodeParser.py @@ -1296,12 +1296,14 @@ class BnfNodeParser(BaseNodeParser): if not concepts: if debugger.is_enabled(): debugger.debug_log(debug_prefix + ", no concept found.") + for concept_parser in not_locked: concept_parser.eat_unrecognized(token) continue if debugger.is_enabled(): debugger.debug_log(debug_prefix + f", concept(s) found={concepts}") + if len(concepts) == 1: for concept_parser in not_locked: concept_parser.eat_concept(concepts[0], token) diff --git a/src/parsers/PythonParser.py b/src/parsers/PythonParser.py index b683eb7..43b80f0 100644 --- a/src/parsers/PythonParser.py +++ b/src/parsers/PythonParser.py @@ -53,9 +53,6 @@ class PythonNode(Node): self.compiled = compile(self.ast_, "", "eval") return self.compiled - # def __repr__(self): - # return "PythonNode(parser_input='" + self.parser_input + "', ast=" + self.get_dump(self.ast_) + ")" - def __repr__(self): ast_type = "expr" if isinstance(self.ast_, ast.Expression) else "module" return "PythonNode(" + ast_type + "='" + self.source + "')" diff --git a/src/parsers/SyaNodeParser.py b/src/parsers/SyaNodeParser.py index 597065b..60290fa 100644 --- a/src/parsers/SyaNodeParser.py +++ b/src/parsers/SyaNodeParser.py @@ -11,7 +11,7 @@ from core.global_symbols import CONCEPT_COMPARISON_CONTEXT from core.sheerka.services.SheerkaComparisonManager import SheerkaComparisonManager from core.sheerka.services.SheerkaExecute import ParserInput from core.tokenizer import Token, TokenKind, Tokenizer -from core.utils import get_n_clones +from core.utils import get_n_clones, get_text_from_tokens, NextIdManager from parsers.BaseNodeParser import UnrecognizedTokensNode, ConceptNode, SourceCodeNode, SyaAssociativity, \ SourceCodeWithConceptNode, BaseNodeParser from parsers.BaseParser import ErrorNode @@ -25,6 +25,7 @@ DEBUG_PUSH_UNREC = "PUSH_UNREC" DEBUG_POP = "POP" DEBUG_EAT = "EAT" DEBUG_RECOG = "RECOG" +DEBUG_CAN_POP = "CAN_POP" @dataclass() @@ -42,12 +43,13 @@ class DebugInfo: token: Token = None # current token concept: Concept = None # current concept if ay action: str = None # action taken + level: str = None def __repr__(self): token_repr = self.token.repr_value if isinstance(self.token, Token) else self.token msg = f"{self.pos:3}:{token_repr}" if self.pos != -1 else " _:" if self.concept: - msg += f"({self.concept})" + msg += f" {self.concept.short_repr()}" return msg + f" => {self.action}" @@ -118,6 +120,36 @@ class SyaConceptDef: precedence: int = SheerkaComparisonManager.DEFAULT_COMPARISON_VALUE associativity: SyaAssociativity = SyaAssociativity.Right + @staticmethod + def get_sya_concept_def(concept, parser, sheerka): + sya_concept_def = SyaConceptDef(concept) + + # first, try to look in the parser + # it is where to find the data during the unit tests + if parser and concept.id in parser.sya_definitions: + # Manage when precedence and associativity are given in the unit tests + sya_def = parser.sya_definitions.get(concept.id) + if sya_def[0] is not None: + sya_concept_def.precedence = sya_def[0] + if sya_def[1] is not None: + sya_concept_def.associativity = sya_def[1] + + # otherwise, use sheerka + if sheerka: + concept_weight = parser.sheerka.get_concepts_weights(BuiltinConcepts.PRECEDENCE, CONCEPT_COMPARISON_CONTEXT) + if concept.str_id in concept_weight: + sya_concept_def.precedence = concept_weight[concept.str_id] + + # in the case of Sheerka, the associativity is managed by the concept itself + # There is no conflict with the settings of the unit test, as I don't use the props in the unit tests + if associativity := concept.get_prop(BuiltinConcepts.ASSOCIATIVITY): + sya_concept_def.associativity = SyaAssociativity(associativity) + + return sya_concept_def + + def short_repr(self): + return f"({self.concept}, prio={self.precedence}, assoc={self.associativity})" + @dataclass() class SyaConceptParserHelper: @@ -248,9 +280,19 @@ class SyaConceptParserHelper: class InFixToPostFix: - def __init__(self, context, debug_enabled=False): + def __init__(self, context, next_id_manager, debugger=None): self.context = context - self.debug_enabled = debug_enabled + + self.next_id_manager = next_id_manager + self.id = self.next_id_manager.get_next_id() + + self.debugger = debugger + if debugger: + self.debug_enabled = debugger.is_enabled() + self.enabled_debug_levels = debugger.get_enabled_vars() + else: + self.debug_enabled = False + self.enabled_debug_levels = None self.is_locked = False # when locked, cannot process input @@ -284,9 +326,15 @@ class InFixToPostFix: def _add_error(self, error): if self.debug_enabled: - self.debug.append(DebugInfo(action=f"=> ERROR {error}")) + self._add_debug(DebugInfo(action=f"=> ERROR {error}")) self.errors.append(error) + def _add_debug(self, debug_info: DebugInfo): + if debug_info.level is None or (self.enabled_debug_levels and + (f"#{self.id}.{debug_info.level}" in self.enabled_debug_levels or + "*" in self.enabled_debug_levels)): + self.debug.append(debug_info) + def _is_lpar(self, token): """ True if the token is a left parenthesis '(' @@ -337,10 +385,10 @@ class InFixToPostFix: else: item.error = f"token '{item.expected[0].strip_quote}' not found" if self.debug_enabled: - self.debug.append(DebugInfo(action=f"ERROR {item.error}")) + self._add_debug(DebugInfo(action=f"ERROR {item.error}")) if self.debug_enabled: - self.debug.append(DebugInfo(action=f"{DEBUG_POP} {item}")) + self._add_debug(DebugInfo(action=f"{DEBUG_POP} {item}")) if isinstance(item, SyaConceptParserHelper) and item.potential_pos != -1: self.out.insert(item.potential_pos, item) else: @@ -402,6 +450,11 @@ class InFixToPostFix: self.debug.pop() def _debug_nodes(self, nodes_sequences): + """ + Returns a debug representation of a sequence of LexerNodes + :param nodes_sequences: + :return: + """ res = "[" first = True for sequence in nodes_sequences: @@ -520,7 +573,7 @@ class InFixToPostFix: # There are more than one solution found # In the case, we create a new InfixToPostfix for each new possibility if self.debug_enabled: - self.debug.append(DebugInfo(action=f"{DEBUG_RECOG} {self._debug_nodes(nodes_sequences)}")) + self._add_debug(DebugInfo(action=f"{DEBUG_RECOG} {self._debug_nodes(nodes_sequences)}")) if len(nodes_sequences) > 1: for node_sequence in nodes_sequences[1:]: clone = self.clone() @@ -599,33 +652,52 @@ class InFixToPostFix: self.stack.pop() self._put_to_out(item) - def i_can_pop(self, concept_node): + def i_can_pop(self, sya_parser_helper): """ Validate the Shunting Yard Algorithm conditions to pop out from the stack Note that it's a custom implementation as I need to manage UnrecognizedTokensNode - :param concept_node: + :param sya_parser_helper: :return: """ if len(self.stack) == 0: + if self.debug_enabled: + self._add_debug(DebugInfo(action=f"No stack. {DEBUG_CAN_POP} false.", level="can_pop")) return False stack_head = self.stack[-1] if not isinstance(stack_head, SyaConceptParserHelper): # mostly left parenthesis + if self.debug_enabled: + self._add_debug(DebugInfo(action=f"No concept. {DEBUG_CAN_POP} false.", level="can_pop")) return False - current = concept_node.concept + current = sya_parser_helper.concept stack = stack_head.concept if stack.associativity == SyaAssociativity.No and current.associativity == SyaAssociativity.No: - self._add_error(NoneAssociativeSequenceErrorNode(current.concept, stack_head.start, concept_node.start)) + self._add_error( + NoneAssociativeSequenceErrorNode(current.concept, stack_head.start, sya_parser_helper.start)) if current.associativity == SyaAssociativity.Left and current.precedence <= stack.precedence: + if self.debug_enabled: + current_debug = f"{current.concept.id}({current.precedence})" + stack_debug = f"{stack.concept.id}({stack.precedence})" + self._add_debug( + DebugInfo(action=f"assoc=Left and {current_debug} <= {stack_debug}. {DEBUG_CAN_POP} True.", + level="can_pop")) return True if current.associativity == SyaAssociativity.Right and current.precedence < stack.precedence: + if self.debug_enabled: + current_debug = f"{current.concept.id}({current.precedence})" + stack_debug = f"{stack.concept.id}({stack.precedence})" + self._add_debug( + DebugInfo(action=f"assoc=Right and {current_debug} < {stack_debug}. {DEBUG_CAN_POP} True.", + level="can_pop")) return True + if self.debug_enabled: + self._add_debug(DebugInfo(action=f"No rule. {DEBUG_CAN_POP} False.", level="can_pop")) return False def handle_expected_token(self, token, pos): @@ -693,7 +765,7 @@ class InFixToPostFix: current_concept.end = pos if self.debug_enabled: - self.debug.append(DebugInfo(pos, token, None, "??")) + self._add_debug(DebugInfo(pos, token, None, "??")) self.manage_unrecognized() # manage that some clones may have been forked for forked in self.forked: @@ -755,7 +827,7 @@ class InFixToPostFix: if self.parsing_function: if self.debug_enabled: - self.debug.append(DebugInfo(pos, token, None, DEBUG_PUSH_UNREC)) + self._add_debug(DebugInfo(pos, token, None, DEBUG_PUSH_UNREC)) self.unrecognized_tokens.add_token(token, pos) @@ -790,13 +862,13 @@ class InFixToPostFix: # if the token 'bar' is found, it has to be considered as part of the concept foo if self.debug_enabled: self._remove_debug_info_if_needed() - self.debug.append(DebugInfo(pos, token, None, DEBUG_EAT)) + self._add_debug(DebugInfo(pos, token, None, DEBUG_EAT)) return True elif self._is_lpar(token): if self.debug_enabled: - self.debug.append(DebugInfo(pos, token, None, DEBUG_PUSH_UNREC)) + self._add_debug(DebugInfo(pos, token, None, DEBUG_PUSH_UNREC)) if self.unrecognized_tokens.is_empty() or self.unrecognized_tokens.is_whitespace(): @@ -865,7 +937,7 @@ class InFixToPostFix: elif self._is_rpar(token): if self.debug_enabled: - self.debug.append(DebugInfo(pos, token, None, DEBUG_EAT)) + self._add_debug(DebugInfo(pos, token, None, DEBUG_EAT)) # first, remove what was in the buffer self.manage_unrecognized() @@ -933,7 +1005,7 @@ class InFixToPostFix: if first_pass: if self.debug_enabled: - self.debug.append(DebugInfo(pos, token, sya_concept_def, "??")) + self._add_debug(DebugInfo(pos, token, sya_concept_def, "??")) if self.unrecognized_tokens.last_token_type() == TokenKind.WHITESPACE: parser_helper.remember_whitespace = self.unrecognized_tokens.tokens[-1] @@ -970,7 +1042,7 @@ class InFixToPostFix: else: if self.debug_enabled: self._remove_debug_info_if_needed() - self.debug.append(DebugInfo(pos, token, sya_concept_def, DEBUG_PUSH)) + self._add_debug(DebugInfo(pos, token, sya_concept_def, DEBUG_PUSH)) self.stack.append(parser_helper) self.manage_parameters_when_new_concept(parser_helper) @@ -985,7 +1057,7 @@ class InFixToPostFix: return if self.debug_enabled: - self.debug.append(DebugInfo(pos, token, None, DEBUG_PUSH_UNREC)) + self._add_debug(DebugInfo(pos, token, None, DEBUG_PUSH_UNREC)) self.unrecognized_tokens.add_token(token, pos) @@ -1005,7 +1077,7 @@ class InFixToPostFix: return # no need to pop the buffer, as no concept is found if self.debug_enabled: - self.debug.append(DebugInfo(pos, "", None, "??")) + self._add_debug(DebugInfo(pos, "", None, "??")) while len(self.stack) > 0: parser_helper = self.stack[-1] @@ -1036,7 +1108,7 @@ class InFixToPostFix: forked.finalize(pos) def clone(self): - clone = InFixToPostFix(self.context, self.debug_enabled) + clone = InFixToPostFix(self.context, self.next_id_manager, self.debugger) clone.is_locked = self.is_locked clone.out = self.out[:] clone.stack = [i.clone() if hasattr(i, "clone") else i for i in self.stack] @@ -1054,6 +1126,7 @@ class PostFixToItem: start: int end: int has_unrecognized: bool + source: str class SyaNodeParser(BaseNodeParser): @@ -1069,14 +1142,6 @@ class SyaNodeParser(BaseNodeParser): self.concepts_by_first_keyword = {} self.sya_definitions = {} - # self.token = None - # self.pos = -1 - # self.tokens = None - # - # self.context: ExecutionContext = None - # self.text = None - # self.sheerka = None - def init_from_concepts(self, context, concepts, **kwargs): super().init_from_concepts(context, concepts) @@ -1093,27 +1158,8 @@ class SyaNodeParser(BaseNodeParser): """ # We only concepts that has parameter (refuse atoms) # Bnf definitions are not supposed to be managed by this parser either - return len(concept.get_metadata().variables) > 0 and concept.get_metadata().definition_type != DEFINITION_TYPE_BNF - - def _get_sya_concept_def(self, parser, concept): - sya_concept_def = SyaConceptDef(concept) - if concept.id in parser.sya_definitions: - # Manage when precedence and associativity are given in the unit tests - sya_def = parser.sya_definitions.get(concept.id) - if sya_def[0] is not None: - sya_concept_def.precedence = sya_def[0] - if sya_def[1] is not None: - sya_concept_def.associativity = sya_def[1] - - if parser.sheerka: - concept_weight = parser.sheerka.get_concepts_weights(BuiltinConcepts.PRECEDENCE, CONCEPT_COMPARISON_CONTEXT) - if concept.str_id in concept_weight: - sya_concept_def.precedence = concept_weight[concept.str_id] - - if associativity := concept.get_prop(BuiltinConcepts.ASSOCIATIVITY): - sya_concept_def.associativity = SyaAssociativity(associativity) - - return sya_concept_def + return len( + concept.get_metadata().variables) > 0 and concept.get_metadata().definition_type != DEFINITION_TYPE_BNF def infix_to_postfix(self, context, parser_input: ParserInput): """ @@ -1126,6 +1172,9 @@ class SyaNodeParser(BaseNodeParser): if not self.reset_parser(context, parser_input): return None + debugger = context.get_debugger(self.NAME, "parse") + debugger.debug_entering(source=self.parser_input.as_text()) + forked = [] def _add_forked_to_res(): @@ -1138,16 +1187,21 @@ class SyaNodeParser(BaseNodeParser): res.extend(forked) forked.clear() - res = [InFixToPostFix(context, context.debug_enabled)] + res = [InFixToPostFix(context, NextIdManager(), debugger)] while self.parser_input.next_token(False): for infix_to_postfix in res: infix_to_postfix.reset() token = self.parser_input.token + if debugger.is_enabled(): + debug_prefix = f"pos={self.parser_input.pos}, {token=}, {len(res)} parser(s)" try: if token.type in (TokenKind.LPAR, TokenKind.RPAR): # little optim, no need to lock, unlock or get the concept when parenthesis + if debugger.is_enabled(): + debugger.debug_log(debug_prefix + ", eat token.") + for infix_to_postfix in res: infix_to_postfix.eat_token(token, self.parser_input.pos) continue @@ -1156,21 +1210,34 @@ class SyaNodeParser(BaseNodeParser): if infix_to_postfix.eat_token(token, self.parser_input.pos): infix_to_postfix.lock() - concepts = self.get_concepts(token, self._is_eligible, to_map=self._get_sya_concept_def) - if not concepts: + nb_locked = len([itp for itp in res if itp.is_locked]) + if nb_locked == len(res): + if debugger.is_enabled(): + debugger.debug_log(debug_prefix + f", all parsers are locked") + continue + + concepts_def = self.get_concepts(token, self._is_eligible, to_map=SyaConceptDef.get_sya_concept_def) + if not concepts_def: + if debugger.is_enabled(): + debugger.debug_log(debug_prefix + f", no concept found") + for infix_to_postfix in res: infix_to_postfix.eat_unrecognized(token, self.parser_input.pos) continue - if len(concepts) == 1: + if debugger.is_enabled(): + found = [cd.short_repr() for cd in concepts_def] + debugger.debug_log(debug_prefix + f", concept(s) found={found}") + + if len(concepts_def) == 1: for infix_to_postfix in res: - infix_to_postfix.eat_concept(concepts[0], token, self.parser_input.pos) + infix_to_postfix.eat_concept(concepts_def[0], token, self.parser_input.pos) continue # make the cartesian product temp_res = [] for infix_to_postfix in res: - for concept in concepts: + for concept in concepts_def: clone = infix_to_postfix.clone() temp_res.append(clone) clone.eat_concept(concept, token, self.parser_input.pos) @@ -1185,13 +1252,13 @@ class SyaNodeParser(BaseNodeParser): infix_to_postfix.finalize(self.parser_input.pos) _add_forked_to_res() - if context.debug_enabled: - context.debug(self.name, "infix_to_postfix", None, f"Parsing {parser_input}") - context.debug(self.name, "infix_to_postfix", "nb_found", f"{len(res)} InfixToPostFix(s) found") - for i, r in enumerate(res): - context.debug(self.name, "infix_to_postfix", "infix_to_postfix", f"#{i}") + if debugger.is_enabled(): + for r in res: for line in r.debug: - context.debug(self.name, "infix_to_postfix", "infix_to_postfix", line) + if line.level: + debugger.debug_var(f"#{r.id}.{line.level}", line) + else: + debugger.debug_var(f"#{r.id}", line) return res @@ -1222,6 +1289,7 @@ class SyaNodeParser(BaseNodeParser): end = item.end has_unrecognized = False concept = sheerka.new_from_template(item.concept, item.concept.key) + concept_metadata = [] for param_index in reversed(range(len(concept.get_metadata().variables))): inner_item = self.postfix_to_item(sheerka, postfixed) if inner_item.start < start: @@ -1237,8 +1305,20 @@ class SyaNodeParser(BaseNodeParser): inner_item concept.get_compiled()[param_name] = param_value + concept_metadata.append((param_name, inner_item.source)) - return PostFixToItem(concept, start, end, has_unrecognized) + # update the metadata + concept_metadata.reverse() + + # ---- Sanity check. To remove at some point + assert len(concept_metadata) == len(concept.get_metadata().variables) + for meta_orig, meta_new in zip(concept.get_metadata().variables, concept_metadata): + assert meta_orig[0] == meta_new[0] + # ---- Sanity check. To remove at some point + concept.get_metadata().variables = concept_metadata + + source = get_text_from_tokens(self.parser_input.tokens[start:end + 1]) + return PostFixToItem(concept, start, end, has_unrecognized, source) def parse(self, context, parser_input: ParserInput): """ diff --git a/src/sdp/sheerkaDataProvider.py b/src/sdp/sheerkaDataProvider.py index 487048e..b007188 100644 --- a/src/sdp/sheerkaDataProvider.py +++ b/src/sdp/sheerkaDataProvider.py @@ -232,6 +232,7 @@ class SheerkaDataProvider: """ with self.lock: + self.log.debug(f"getting {entry=}, {key=}, {default=}, {load_origin=}") if entry not in self.state.data: return default @@ -464,6 +465,8 @@ class SheerkaDataProvider: def load_ref_if_needed_ex(self, item, load_origin): """ + New version of the function. + The old one must be replaced at some point Make sure we return the real object, even inside a collection :param item: :param load_origin: diff --git a/src/sdp/sheerkaDataProvider_Old.py b/src/sdp/sheerkaDataProvider_Old.py deleted file mode 100644 index dd7080c..0000000 --- a/src/sdp/sheerkaDataProvider_Old.py +++ /dev/null @@ -1,1087 +0,0 @@ -# import hashlib -# import json -# import zlib -# import time -# from dataclasses import dataclass -# from datetime import datetime, date -# -# from core.sheerka_logger import get_logger -# from sdp.sheerkaDataProviderIO import SheerkaDataProviderIO -# from sdp.sheerkaSerializer import Serializer, SerializerContext -# -# -# def json_default_converter(o): -# """ -# Default formatter for json -# It's used when the json serializer does not know -# how to serialise a type -# :param o: -# :return: -# """ -# if isinstance(o, (date, datetime)): -# return o.isoformat() -# -# if isinstance(o, SheerkaDataProviderRef): -# return f"##XREF##:{o.target}" -# -# -# class Event(object): -# """ -# Class that represents something that modifies the state of the system -# """ -# -# def __init__(self, message="", user="", date=datetime.now(), parents=None): -# self.version = 1 -# self.user = user -# self.date = date -# self.message = message -# self.parents = parents -# self._digest = None -# -# def __str__(self): -# return f"{self.date.strftime('%d/%m/%Y %H:%M:%S')} {self.message}" -# -# def __repr__(self): -# return f"{self.get_digest()[:12]} {self.message}" -# -# def get_digest(self): -# """ -# Returns the digest of the event -# :return: hexa form of the sha256 -# """ -# -# if self._digest: -# return self._digest -# -# if self.message == "" and self.user == "": -# self._digest = "xxx" # to speed unit tests -# return self._digest -# -# if not isinstance(self.message, str): -# raise NotImplementedError -# -# to_hash = f"Event:{self.user}{self.date}{self.message}{self.parents}".encode("utf-8") -# self._digest = hashlib.sha256(to_hash).hexdigest() -# return self._digest -# -# def to_dict(self): -# return self.__dict__ -# -# def from_dict(self, as_dict): -# self.user = as_dict["user"] -# self.date = datetime.fromisoformat(as_dict["date"]) -# self.message = as_dict["message"] -# self.parents = as_dict["parents"] -# self._digest = as_dict["_digest"] # freeze the digest -# -# -# class ObjToUpdate: -# """ -# Internal key value class; -# You give it an obj, and it tries to figure out what is the key of the obj -# Note that you can force the key if you want -# It was first create to make the difference between an object that has a key and {key, value} -# """ -# -# def __init__(self, obj, key=None, digest=None): -# self.obj = obj -# self.has_key = None -# self.has_digest = None -# self._key = None -# self._digest = None -# if key is not None: -# self.set_key(key) -# if digest is not None: -# self.set_digest(digest) -# -# def get_key(self): -# if self.has_key is None: -# key = SheerkaDataProvider.get_obj_key(self.obj) -# if key is None: -# self.has_key = False -# return None -# else: -# self.has_key = True -# self._key = key -# return key -# elif not self.has_key: -# return None -# else: -# return self._key -# -# def get_digest(self): -# if self.has_digest is None: -# digest = SheerkaDataProvider.get_obj_digest(self.obj) -# if digest is None: -# self.has_digest = False -# return None -# else: -# self.has_digest = True -# self._digest = digest -# return digest -# elif not self.has_digest: -# return None -# else: -# return self._digest -# -# def set_digest(self, digest): -# self.has_digest = True -# self._digest = digest -# -# def set_key(self, key): -# self.has_key = True -# self._key = key -# -# -# class State: -# """ -# Class that represents the state of the system (dictionary of all known entries) -# """ -# -# def __init__(self): -# self.version = 1 -# self.date = None -# self.parents = [] -# self.events = [] -# self.data = {} -# -# @staticmethod -# def check_duplicate(items, obj: ObjToUpdate, key): -# digest = obj.get_digest() -# if digest is None: -# return -# -# if not hasattr(items, "__iter__"): -# items = [items] -# -# for item in items: -# item_digest = SheerkaDataProvider.get_obj_digest(item) -# if item_digest == digest: -# raise SheerkaDataProviderDuplicateKeyError(key, obj.obj) -# -# def update(self, entry, obj: ObjToUpdate, append=True): -# """ -# adds obj to entry -# :param entry: -# :param obj: -# :param append: if True, duplicate keys will create lists -# :return: -# """ -# obj_to_use = {obj.get_key(): obj.obj} if obj.has_key else obj.obj -# -# if entry not in self.data: -# self.data[entry] = obj_to_use -# -# elif not append: -# if isinstance(obj_to_use, dict): -# self.data[entry].update(obj_to_use) -# else: -# self.data[entry] = obj_to_use -# -# elif isinstance(self.data[entry], list): -# self.check_duplicate(self.data[entry], obj, entry) -# self.data[entry].append(obj.obj) -# -# elif isinstance(obj_to_use, dict): -# for k in obj_to_use: -# if k not in self.data[entry]: -# self.data[entry][k] = obj_to_use[k] -# elif isinstance(self.data[entry][k], list): -# self.check_duplicate(self.data[entry][k], obj, entry + "." + k) -# self.data[entry][k].append(obj_to_use[k]) -# else: -# self.check_duplicate(self.data[entry][k], obj, entry + "." + k) -# self.data[entry][k] = [self.data[entry][k], obj_to_use[k]] -# -# elif isinstance(self.data[entry], dict): -# raise SheerkaDataProviderError(f"Cannot found key on '{obj.obj}' while all other elements have.", obj.obj) -# -# else: -# self.check_duplicate(self.data[entry], obj, entry) -# self.data[entry] = [self.data[entry], obj_to_use] -# -# def modify(self, entry, key, obj, obj_key): -# # if the key changes, make sure to remove the previous entry -# append = False -# if obj_key != key: -# self.remove(entry, lambda k, o: k == key) # modify from on object to another -# append = True -# -# self.update(entry, ObjToUpdate(obj, obj_key), append=append) -# -# def modify_in_list(self, entry, key, obj, obj_key, obj_origin, load_ref_if_needed, save_ref_if_needed): -# found = False -# to_remove = None -# new_digest = None -# -# def _get_item_origin(o): -# if hasattr(o, Serializer.ORIGIN): -# return getattr(o, Serializer.ORIGIN) -# -# if isinstance(o, dict) and Serializer.ORIGIN in o: -# return o[Serializer.ORIGIN] -# -# if hasattr(o, "get_digest"): -# return o.get_digest() -# -# if isinstance(o, str): -# return o -# -# return None -# -# for i in range(len(self.data[entry][key])): -# item, is_ref = load_ref_if_needed(self.data[entry][key][i]) -# item_origin = _get_item_origin(item) -# if item_origin is None: -# continue -# if item_origin == obj_origin: -# obj = save_ref_if_needed(is_ref, obj) -# if is_ref: -# new_digest = obj[len(SheerkaDataProvider.REF_PREFIX):] -# if obj_key == key: -# self.data[entry][key][i] = obj -# else: -# to_remove = i -# self.update(entry, ObjToUpdate(obj, obj_key), append=True) -# found = True -# break -# -# if not found: -# raise (SheerkaDataProviderError(f"Cannot modify '{entry}.{key}'. Item '{obj_origin}' not found.", obj)) -# -# if to_remove is not None: -# del self.data[entry][key][to_remove] -# -# return new_digest -# -# def remove(self, entry, filter): -# if filter is None: -# del (self.data[entry]) -# -# elif isinstance(self.data[entry], dict): -# keys_to_remove = [] -# for key, element in self.data[entry].items(): -# if filter(key, element): -# keys_to_remove.append(key) -# for key in keys_to_remove: -# del (self.data[entry][key]) -# -# elif not isinstance(self.data[entry], list): -# if filter(self.data[entry]): -# del (self.data[entry]) -# -# else: -# for element in self.data[entry]: -# if filter(element): -# self.data[entry].remove(element) -# -# def get_digest(self): -# as_json = json.dumps(self.__dict__, default=json_default_converter) -# return hashlib.sha256(as_json.encode("utf-8")).hexdigest() -# -# def contains(self, entry, key): -# """ -# if key is None, returns True if entry exists -# if key has a value -# returns True if entry is an dict and contains key -# :param entry: -# :param key: -# :return: -# """ -# if entry not in self.data: -# return False -# if key is None: -# return entry in self.data -# if not isinstance(self.data[entry], dict): -# return False -# return key in self.data[entry] -# -# -# class SheerkaDataProviderError(Exception): -# def __init__(self, message, obj): -# Exception.__init__(self, message) -# self.obj = obj -# -# -# class SheerkaDataProviderDuplicateKeyError(Exception): -# def __init__(self, key, obj): -# Exception.__init__(self, "Duplicate object.") -# self.key = key -# self.obj = obj -# -# -# @dataclass -# class SheerkaDataProviderResult: -# """ -# Object that is returned after adding, setting or modifying an entry -# """ -# obj: object # obj that was given to store/modify -# entry: str # entry where the object is put -# key: str # key to use to retrieve the object -# digest: str # digest used to store the reference -# already_exists: bool = False # the same object was already persisted -# -# -# @dataclass -# class SheerkaDataProviderRef: -# """ -# Object that tells where an object is store (target is the digest of the reference) -# """ -# key: str # key of the object -# target: str # digest of the reference -# original_target: str = None # when the object is modified, previous digest -# -# def get_digest(self): -# return self.original_target -# -# def get_key(self): -# return self.key -# -# -# class SheerkaDataProvider: -# """Manages the state of the system""" -# -# EventFolder = "events" -# StateFolder = "state" -# ObjectsFolder = "objects" -# CacheFolder = "cache" -# HeadFile = "HEAD" -# LastEventFile = "LAST_EVENT" -# KeysFile = "keys" -# REF_PREFIX = "##REF##:" -# -# def __init__(self, root=None, sheerka=None): -# self.log = get_logger(__name__) -# self.init_log = get_logger("init." + __name__) -# self.init_log.debug("Initializing sdp.") -# -# self.sheerka = sheerka -# self.io = SheerkaDataProviderIO.get(root) -# self.first_time = self.io.first_time -# -# self.serializer = Serializer() -# -# @staticmethod -# def get_obj_key(obj): -# """ -# Tries to find the key of an object -# Look for .key, .get_key() -# :param obj: -# :return: String version of that is found, None otherwise -# """ -# return str(obj.get_key()) if hasattr(obj, "get_key") \ -# else str(obj.key) if hasattr(obj, "key") \ -# else None -# -# @staticmethod -# def get_obj_digest(obj): -# """ -# Tries to find the key of an object -# Look for .digest, .get_digest() -# :param obj: -# :return: digest, None otherwise -# """ -# if isinstance(obj, str) and obj.startswith(SheerkaDataProvider.REF_PREFIX): -# return obj[len(SheerkaDataProvider.REF_PREFIX):] -# -# return obj.digest if hasattr(obj, "digest") \ -# else obj.get_digest() if hasattr(obj, "get_digest") \ -# else None -# -# @staticmethod -# def get_obj_origin(obj): -# """ -# Get the digest used to save obj if set -# """ -# if isinstance(obj, dict) and Serializer.ORIGIN in obj: -# return obj[Serializer.ORIGIN] -# -# if hasattr(obj, Serializer.ORIGIN): -# return getattr(obj, Serializer.ORIGIN) -# -# if isinstance(obj, SheerkaDataProviderRef): -# return obj.original_target -# -# return None -# -# @staticmethod -# def get_stream_digest(stream): -# sha256_hash = hashlib.sha256() -# for byte_block in iter(lambda: stream.read(4096), b""): -# sha256_hash.update(byte_block) -# -# stream.seek(0) -# return sha256_hash.hexdigest() -# -# @staticmethod -# def is_reference(obj): -# return isinstance(obj, str) and obj.startswith(SheerkaDataProvider.REF_PREFIX) -# -# def reset(self): -# self.first_time = self.io.first_time -# if hasattr(self.io, "reset"): -# self.io.reset() -# -# def add(self, event_digest: str, entry, obj, allow_multiple=True, use_ref=False): -# """ -# Adds obj to the entry 'entry' -# :param event_digest: digest of the event that triggers the modification of the state -# :param entry: entry of the state to update -# :param obj: obj to insert or add -# :param allow_multiple: if set to true, the same key can be added several times. -# All entries will be put in a list -# :param use_ref: if True the actual object is saved under 'objects' folder, -# only a reference is saved in the state -# :return: (entry, key) to retrieve the object -# """ -# -# original_obj = obj.copy() if isinstance(obj, dict) else obj -# -# snapshot = self.get_snapshot(SheerkaDataProvider.HeadFile) -# state = self.load_state(snapshot) -# -# self.log.debug(f"Adding obj '{obj}' in entry '{entry}' (allow_multiple={allow_multiple}, use_ref={use_ref})") -# -# if not isinstance(obj, ObjToUpdate): -# obj = ObjToUpdate(obj) -# -# # check uniqueness, cannot add the same key twice if allow_multiple == False -# key = obj.get_key() -# self.log.debug(f"key found : '{key}'") if key else self.log.debug("No key found") -# if not allow_multiple: -# if isinstance(obj.obj, dict): -# for k in obj.obj: -# if state.contains(entry, k): -# raise IndexError(f"{entry}.{k}") -# else: -# if state.contains(entry, key): -# raise IndexError(f"{entry}.{key}" if key else entry) -# -# state.parents = [] if snapshot is None else [snapshot] -# state.events = [event_digest] -# state.date = datetime.now() -# -# if use_ref: -# obj.set_digest(self.save_obj(obj.obj)) -# obj.obj = self.REF_PREFIX + obj.get_digest() -# -# state.update(entry, obj) -# -# new_snapshot = self.save_state(state) -# self.set_snapshot(SheerkaDataProvider.HeadFile, new_snapshot) -# return SheerkaDataProviderResult(original_obj, entry, obj.get_key(), obj.get_digest()) -# -# def add_with_auto_key(self, event_digest: str, entry, obj): -# """ -# Add obj to entry. An autogenerated key created for obj -# :param event_digest: -# :param entry: -# :param obj: -# :return: -# """ -# -# original_obj = obj.copy() if isinstance(obj, dict) else obj -# -# next_key = self.get_next_key(entry) -# if hasattr(obj, "set_key"): -# obj.set_key(next_key) -# res = self.add(event_digest, entry, ObjToUpdate(obj, next_key)) -# return SheerkaDataProviderResult(original_obj, res.entry, res.key, res.digest) -# -# def add_unique(self, event_digest: str, entry, obj): -# """Add an entry and make sure it's unique""" -# -# original_obj = obj.copy() if isinstance(obj, dict) else obj -# -# snapshot = self.get_snapshot(SheerkaDataProvider.HeadFile) -# state = self.load_state(snapshot) -# -# state.parents = [] if snapshot is None else [snapshot] -# state.events = [event_digest] -# state.date = datetime.now() -# if entry not in state.data: -# state.data[entry] = {obj} -# already_exist = False -# else: -# already_exist = obj in state.data[entry] -# if not already_exist: -# state.data[entry].add(obj) -# -# new_snapshot = self.save_state(state) -# self.set_snapshot(SheerkaDataProvider.HeadFile, new_snapshot) -# return SheerkaDataProviderResult( -# original_obj, -# entry, -# None, -# None, -# already_exist) -# -# def set(self, event_digest, entry, obj, use_ref=False, is_ref=False): -# """ -# Add or replace an entry. The entry is reinitialized. -# If the previous value was dict, all keys are lost -# :param event_digest: -# :param entry: -# :param obj: -# :param use_ref: Do not save obj in State (save it under objects), use_ref in State -# :param is_ref: obj is supposed to be a reference -# :return: -# """ -# -# original_obj = obj.copy() if isinstance(obj, dict) else obj -# -# if use_ref and is_ref: -# raise SheerkaDataProviderError("Cannot use use_ref and is_ref at the same time", None) -# -# if is_ref and not isinstance(obj, dict): -# raise SheerkaDataProviderError("is_ref can only be used with dictionaries", obj) -# -# snapshot = self.get_snapshot(SheerkaDataProvider.HeadFile) -# state = self.load_state(snapshot) -# -# state.parents = [] if snapshot is None else [snapshot] -# state.events = [event_digest] -# state.date = datetime.now() -# -# key = self.get_obj_key(obj) -# obj = self.save_ref_if_needed(use_ref, obj) -# -# if is_ref: -# for k, v in obj.items(): -# obj[k] = self.REF_PREFIX + v -# -# state.data[entry] = obj if key is None else {key: obj} -# -# new_snapshot = self.save_state(state) -# self.set_snapshot(SheerkaDataProvider.HeadFile, new_snapshot) -# return SheerkaDataProviderResult(original_obj, entry, key, self.get_obj_digest(obj)) -# -# def modify(self, event_digest, entry, key, obj): -# """ -# Replace an element -# If the key is not provided, has the same effect than set eg, the entry is reset -# :param event_digest: -# :param entry: -# :param key: key of the object to update -# :param obj: new data -# :return: -# """ -# -# original_obj = obj.copy() if isinstance(obj, dict) else obj -# -# if key is None: -# raise SheerkaDataProviderError("Key is mandatory.", None) -# -# snapshot = self.get_snapshot(SheerkaDataProvider.HeadFile) -# state = self.load_state(snapshot) -# -# if entry not in state.data: -# raise IndexError(entry) -# -# if key is not None and key not in state.data[entry]: -# raise IndexError(f"{entry}.{key}") -# -# state.parents = [] if snapshot is None else [snapshot] -# state.events = [event_digest] -# state.date = datetime.now() -# -# # Gets obj original key, it will help to know if the key has changed -# obj_key = self.get_obj_key(obj) or key -# digest = None -# -# if isinstance(state.data[entry][key], list): -# obj_origin = self.get_obj_origin(obj) -# if obj_origin is None: -# raise (SheerkaDataProviderError(f"Multiple entries under '{entry}.{key}'", obj)) -# -# digest = state.modify_in_list( -# entry, -# key, -# obj, -# obj_key, -# obj_origin, -# self.load_ref_if_needed, -# self.save_ref_if_needed) -# -# else: -# was_saved_as_reference = self.is_reference(state.data[entry][key]) -# if was_saved_as_reference: -# obj = self.save_ref_if_needed(True, obj) -# digest = self.get_obj_digest(obj) -# -# state.modify(entry, key, obj, obj_key) -# -# new_snapshot = self.save_state(state) -# self.set_snapshot(SheerkaDataProvider.HeadFile, new_snapshot) -# return SheerkaDataProviderResult(original_obj, entry, obj_key, digest) -# -# def list(self, entry, filter=None): -# """ -# Lists elements of entry 'entry' -# :param entry: name of the entry to list -# :param filter: filter to use -# :return: list of elements -# """ -# snapshot = self.get_snapshot(SheerkaDataProvider.HeadFile) -# state = self.load_state(snapshot) -# if entry not in state.data: -# return [] -# -# elements = state.data[entry] -# -# if isinstance(elements, dict): -# # manage when elements have a key -# filter_to_use = (lambda k, o: True) if filter is None else filter -# for key, element in elements.items(): -# if filter_to_use(key, element): -# if isinstance(element, list): -# yield [self.load_ref_if_needed(e)[0] for e in element] -# else: -# yield self.load_ref_if_needed(element)[0] -# else: -# # manage when no key is defined for the elements -# if not isinstance(elements, list) and not isinstance(elements, set): -# elements = [elements] -# -# filter_to_use = (lambda o: True) if filter is None else filter -# for element in elements: -# if filter_to_use(element): -# yield self.load_ref_if_needed(element)[0] -# -# def remove(self, event_digest, entry, filter=None, silent_remove=True): -# """ -# Removes elements under the entry 'entry' -# :param event_digest: event that triggers the deletion -# :param entry: -# :param filter: filter to use -# :param silent_remove: Do not throw exception if entry does not exist -# :return: new sha256 of the state -# TODO: Remove by key -# """ -# snapshot = self.get_snapshot(SheerkaDataProvider.HeadFile) -# state = self.load_state(snapshot) -# -# if entry not in state.data: -# if silent_remove: -# return snapshot -# else: -# raise IndexError(entry) -# -# state.parents = [] if snapshot is None else [snapshot] -# state.events = [event_digest] -# state.date = datetime.now() -# state.remove(entry, filter) -# -# new_snapshot = self.save_state(state) -# self.set_snapshot(SheerkaDataProvider.HeadFile, new_snapshot) -# return new_snapshot -# -# def get(self, entry, key=None, load_origin=True): -# """ -# Retrieve an element by its key -# :param entry: -# :param key: -# :param load_origin: if True, adds the origin (parent digest) to the object -# :return: -# """ -# snapshot = self.get_snapshot(SheerkaDataProvider.HeadFile) -# state = self.load_state(snapshot) -# -# if entry not in state.data: -# raise IndexError(entry) -# -# if key is not None and key not in state.data[entry]: -# raise IndexError(f"{entry}.{key}") -# -# item = state.data[entry] if key is None else state.data[entry][key] -# if isinstance(item, list): -# return [self.load_ref_if_needed(i, load_origin)[0] for i in item] -# -# return self.load_ref_if_needed(item, load_origin)[0] -# -# def get_safe(self, entry, key=None, load_origin=True): -# """ -# Retrieve an element by its key. Return None if the element does not exist -# :param entry: -# :param key: -# :param load_origin: if True, adds the origin (parent digest) to the object -# :return: -# """ -# snapshot = self.get_snapshot(SheerkaDataProvider.HeadFile) -# state = self.load_state(snapshot) -# -# if entry not in state.data: -# return None -# -# if key is not None and key not in state.data[entry]: -# return None -# -# item = state.data[entry] if key is None else state.data[entry][key] -# if isinstance(item, list): -# return [self.load_ref_if_needed(i, load_origin)[0] for i in item] -# -# return self.load_ref_if_needed(item, load_origin)[0] -# -# def get_ref(self, entry, key=None): -# """ -# Returns the reference of an object if the object exists -# This function allows to retrieve obj.##origin## without loading the object -# :param entry: -# :param key: -# :return: -# """ -# -# snapshot = self.get_snapshot(SheerkaDataProvider.HeadFile) -# state = self.load_state(snapshot) -# -# if entry not in state.data: -# raise IndexError(entry) -# -# if key is not None and key not in state.data[entry]: -# raise IndexError(f"{entry}.{key}") -# -# item = state.data[entry] if key is None else state.data[entry][key] -# if isinstance(item, list): -# res = [] -# for element in item: -# if not self.is_reference(element): -# raise SheerkaDataProviderError("Not a reference", f"{entry}.{key}") -# res.append(self.get_obj_digest(element)) -# return res -# -# if not self.is_reference(item): -# raise SheerkaDataProviderError("Not a reference", f"{entry}.{key}") -# -# return self.get_obj_digest(item) -# -# def exists(self, entry, key=None, digest=None): -# """ -# Returns true if the entry is defined -# :param key: -# :param entry: -# :param digest: digest of the object, when several entries share the same key -# :return: -# """ -# snapshot = self.get_snapshot(SheerkaDataProvider.HeadFile) -# state = self.load_state(snapshot) -# exist = entry in state.data -# -# if not exist or key is None: -# return exist -# -# items = state.data[entry] -# exist = key in items -# if not exist or digest is None: -# return exist -# -# items = items[key] -# if not isinstance(items, list): -# items = [items] -# -# for item in items: -# item_digest = SheerkaDataProvider.get_obj_digest(item) -# if item_digest == digest: -# return True -# -# return False -# -# def save_event(self, event: Event): -# """ -# return an event, given its digest -# :param event: -# :return: digest of the event -# """ -# parent = self.get_snapshot(SheerkaDataProvider.LastEventFile) -# event.parents = [parent] if parent else None -# digest = event.get_digest() # must be call after setting the parents -# -# target_path = self.io.get_obj_path(SheerkaDataProvider.EventFolder, digest) -# if self.io.exists(target_path): -# return digest -# -# self.io.write_binary(target_path, self.serializer.serialize(event, None).read()) -# self.set_snapshot(SheerkaDataProvider.LastEventFile, digest) -# -# return digest -# -# def load_event(self, digest=None): -# """ -# return an event, given its digest -# :param digest: -# :return: -# """ -# digest = digest or self.get_snapshot(SheerkaDataProvider.LastEventFile) -# if digest is None: -# return None -# -# target_path = self.io.get_obj_path(SheerkaDataProvider.EventFolder, digest) -# -# with self.io.open(target_path, "rb") as f: -# return self.serializer.deserialize(f, None) -# -# def load_events(self, page_size, start=0): -# """ -# Load multiple events in the same command -# :param start: -# :param page_size: -# :return: -# """ -# -# digest = None -# if start: -# for i in range(start): -# event = self.load_event(digest) -# if event is None or event.parents is None: -# return -# digest = event.parents[0] -# -# count = 0 -# while count < page_size or page_size <= 0: -# event = self.load_event(digest) -# if event is None: -# return -# -# yield event -# -# if event.parents is None: -# return -# -# digest = event.parents[0] -# count += 1 -# -# def get_result_file_path(self, digest, is_admin): -# ext = "_admin_result" if is_admin else "_result" -# return self.io.get_obj_path(SheerkaDataProvider.EventFolder, digest) + ext -# -# def has_result(self, digest, is_admin=False): -# """ -# Check is a result file was created for a specific event -# :param digest: -# :param is_admin: True is the result is an internal admin result file -# :return: -# """ -# target_path = self.get_result_file_path(digest, is_admin) -# return self.io.exists(target_path) -# -# def save_result(self, execution_context, is_admin=False): -# """ -# Save the execution context associated with an event -# To make a long story short, -# for every single user input, there is an event (which is the first thing that is created) -# and a result (the ExecutionContext created by sheerka.evaluate_user_input() -# :param execution_context: -# :param is_admin: True is the result is an internal admin result file -# :return: -# """ -# start = time.time() -# message = execution_context.event.message -# digest = execution_context.event.get_digest() -# self.log.debug(f"Saving execution context. digest={digest}, message={message}") -# target_path = self.get_result_file_path(digest, is_admin) -# if self.io.exists(target_path): -# return digest -# -# context = SerializerContext(sheerka=self.sheerka) -# length = self.io.write_binary(target_path, self.serializer.serialize(execution_context, context).read()) -# elapsed = time.time() - start -# self.log.debug(f"Saved execution context. message={message}, length={length}, elapsed={elapsed}") -# return digest -# -# def load_result(self, digest, is_admin=False): -# """ -# Load and deserialize a result file -# :param digest: -# :param is_admin: True is the result is an internal admin result file -# :return: -# """ -# target_path = self.get_result_file_path(digest, is_admin) -# -# with self.io.open(target_path, "rb") as f: -# context = SerializerContext(sheerka=self.sheerka) -# return self.serializer.deserialize(f, context) -# -# def save_state(self, state: State): -# digest = state.get_digest() -# self.log.debug(f"Saving new state. digest={digest}") -# target_path = self.io.get_obj_path(SheerkaDataProvider.StateFolder, digest) -# if self.io.exists(target_path): -# return digest -# -# context = SerializerContext(sheerka=self.sheerka) -# self.io.write_binary(target_path, self.serializer.serialize(state, context).read()) -# return digest -# -# def load_state(self, digest): -# if digest is None: -# return State() -# -# target_path = self.io.get_obj_path(SheerkaDataProvider.StateFolder, digest) -# with self.io.open(target_path, "rb") as f: -# context = SerializerContext(sheerka=self.sheerka) -# return self.serializer.deserialize(f, context) -# -# def save_obj(self, obj): -# self.log.debug(f"Saving '{obj}' as reference...") -# context = SerializerContext(user_name="kodjo", sheerka=self.sheerka) -# stream = self.serializer.serialize(obj, context) -# digest = obj.get_digest() if hasattr(obj, "get_digest") else self.get_stream_digest(stream) -# -# target_path = self.io.get_obj_path(SheerkaDataProvider.ObjectsFolder, digest) -# if self.io.exists(target_path): -# self.log.debug(f"...already saved. digest is {digest}") -# return digest -# -# self.io.write_binary(target_path, stream.read()) -# -# self.log.debug(f"...digest={digest}.") -# return digest -# -# def load_obj(self, digest, add_origin=True): -# if digest is None: -# return None -# -# target_path = self.io.get_obj_path(SheerkaDataProvider.ObjectsFolder, digest) -# if not self.io.exists(target_path): -# return None -# -# with self.io.open(target_path, "rb") as f: -# context = SerializerContext(origin=digest, sheerka=self.sheerka) -# obj = self.serializer.deserialize(f, context) -# -# # set the origin of the object -# if add_origin: -# if isinstance(obj, dict): -# obj[Serializer.ORIGIN] = digest -# elif not isinstance(obj, str): -# setattr(obj, Serializer.ORIGIN, digest) -# return obj -# -# def load_ref_if_needed(self, obj, load_origin=True): -# if isinstance(obj, SheerkaDataProviderRef): -# resolved = self.load_obj(obj.target, load_origin) -# return resolved, False -# -# if not isinstance(obj, str) or not obj.startswith(SheerkaDataProvider.REF_PREFIX): -# return obj, False -# -# resolved = self.load_obj(obj[len(SheerkaDataProvider.REF_PREFIX):], load_origin) -# return (obj, False) if resolved is None else (resolved, True) -# -# def save_ref_if_needed(self, save_ref, obj): -# if not save_ref: -# return obj -# -# digest = self.save_obj(obj) -# return self.REF_PREFIX + digest -# -# def get_cache_params(self, category, key): -# digest = hashlib.sha3_256(f"{category}:{key}".encode("utf-8")).hexdigest() -# cache_path = self.io.get_obj_path(SheerkaDataProvider.CacheFolder, digest) -# return digest, cache_path -# -# def add_to_cache(self, category, key, obj, update=False): -# """ -# Save obj in the internal cache system -# :param category: -# :param key: -# :param obj: -# :param update: -# :return: -# """ -# digest, cache_path = self.get_cache_params(category, key) -# -# if self.io.exists(cache_path) and not update: -# return digest -# -# self.io.write_binary(cache_path, zlib.compress(obj.encode("utf-8"), 9)) -# return digest -# -# def load_from_cache(self, category, key): -# """ -# Reload a compress object from the cache -# :param category: -# :param key: -# :return: -# """ -# digest, cache_path = self.get_cache_params(category, key) -# -# if not self.io.exists(cache_path): -# raise IndexError(f"{category}.{key}") -# -# with self.io.open(cache_path, "rb") as f: -# return zlib.decompress(f.read()).decode("utf-8") -# -# def remove_from_cache(self, category, key): -# """ -# -# :param category: -# :param key: -# :return: -# """ -# digest, cache_path = self.get_cache_params(category, key) -# if self.io.exists(cache_path): -# self.io.remove(cache_path) -# -# return digest -# -# def in_cache(self, category, key): -# """ -# Returns true if the key is in cache -# :param category: -# :param key: -# :return: -# """ -# digest, cache_path = self.get_cache_params(category, key) -# return self.io.exists(cache_path) -# -# def get_snapshot(self, file): -# head_file = self.io.path_join(file) -# if not self.io.exists(head_file): -# return None -# return self.io.read_text(head_file) -# # with open(head_file, "r") as f: -# # return f.read() -# -# def set_snapshot(self, file, digest): -# head_file = self.io.path_join(file) -# return self.io.write_text(head_file, digest) -# # with open(head_file, "w") as f: -# # return f.write(digest) -# -# def load_keys(self): -# keys_file = self.io.path_join(SheerkaDataProvider.KeysFile) -# if not self.io.exists(keys_file): -# keys = {} -# else: -# with self.io.open(keys_file, "r") as f: -# keys = json.load(f) -# return keys -# -# def save_keys(self, keys): -# keys_file = self.io.path_join(SheerkaDataProvider.KeysFile) -# with self.io.open(keys_file, "w") as f: -# json.dump(keys, f) -# -# def get_next_key(self, entry): -# keys = self.load_keys() -# -# next_key = keys.get(entry, 0) + 1 -# keys[entry] = next_key -# -# self.save_keys(keys) -# return str(next_key) -# -# def set_key(self, entry, value): -# keys = self.load_keys() -# keys[entry] = value -# self.save_keys(keys) -# return str(value) -# -# def dump_state(self, digest=None): -# digest = digest or self.get_snapshot(SheerkaDataProvider.HeadFile) -# state = self.load_state(digest) -# print(json.dumps(state.data, sort_keys=True, default=json_default_converter, indent=True)) -# -# def dump_obj(self, digest): -# obj = self.load_obj(digest) -# print(json.dumps(obj.__dict__, sort_keys=True, default=json_default_converter, indent=True)) diff --git a/src/sdp/sheerkaSerializer.py b/src/sdp/sheerkaSerializer.py index 4de20a0..9804ae0 100644 --- a/src/sdp/sheerkaSerializer.py +++ b/src/sdp/sheerkaSerializer.py @@ -109,6 +109,7 @@ class Serializer: raise TypeError(f"Don't know how serializer name={header[0]}, version={header[1]}") serializer = serializers[0] + self.log.debug(f"deserializing using '{serializer}'") return serializer.load(stream, context) @@ -298,7 +299,7 @@ class MemoryObjectSerializer(SheerkaPickleSerializer): CLASS_NAME = "core.sheerka.services.SheerkaMemory.MemoryObject" def __init__(self): - super().__init__(lambda obj: get_full_qualified_name(obj) == self.CLASS_NAME, "R", 1) + super().__init__(lambda obj: get_full_qualified_name(obj) == self.CLASS_NAME, "M", 1) class RuleSerializer(SheerkaPickleSerializer): diff --git a/src/sheerkapickle/SheerkaPickler.py b/src/sheerkapickle/SheerkaPickler.py index 5d16a8c..804ea63 100644 --- a/src/sheerkapickle/SheerkaPickler.py +++ b/src/sheerkapickle/SheerkaPickler.py @@ -4,11 +4,15 @@ from logging import Logger import core.utils from core.concept import Concept, NotInitialized from core.sheerka.services.SheerkaExecute import ParserInput +from core.simple_debug import my_debug from sheerkapickle import utils, tags, handlers def encode(sheerka, obj): - return json.dumps(SheerkaPickler(sheerka).flatten(obj)) + pickler = SheerkaPickler(sheerka) + flatten = pickler.flatten(obj) + my_debug(f"{obj} ids={len(pickler.ids)}, objs={len(pickler.objs)}") + return json.dumps(flatten) class ToReduce: diff --git a/tests/core/test_SheerkaAdmin.py b/tests/core/test_SheerkaAdmin.py index 8820110..4a62ae8 100644 --- a/tests/core/test_SheerkaAdmin.py +++ b/tests/core/test_SheerkaAdmin.py @@ -7,59 +7,59 @@ class TestSheerkaAdmin(TestUsingMemoryBasedSheerka): def test_i_can_get_last_ret(self): pass - def test_i_can_get_last_error_ret(self): - sheerka, context = self.init_concepts() - - # nothing in history - last_error_ret = sheerka.last_error_ret(context) - assert sheerka.isinstance(last_error_ret, BuiltinConcepts.RETURN_VALUE) - assert not last_error_ret.status - assert sheerka.isinstance(last_error_ret.body, BuiltinConcepts.NOT_FOUND) - - # only success in history - r_success = sheerka.ret("Test", True, "success") - sheerka.last_return_values.append([r_success]) - last_error_ret = sheerka.last_error_ret(context) - assert sheerka.isinstance(last_error_ret, BuiltinConcepts.RETURN_VALUE) - assert not last_error_ret.status - assert sheerka.isinstance(last_error_ret.body, BuiltinConcepts.NOT_FOUND) - - # at least on error - r_failure = sheerka.ret("Test", False, "failure") - sheerka.last_return_values.append([r_failure]) - assert sheerka.last_error_ret(context) == r_failure - - # add another success and make sure we get the same error - sheerka.last_return_values.append([r_success]) - assert sheerka.last_error_ret(context) == r_failure - - # and I only get the last failure - r_failure2 = sheerka.ret("Test", False, "another failure") - sheerka.last_return_values.append([r_failure2]) - assert sheerka.last_error_ret(context) == r_failure2 - - # but I still can get the previous error - assert sheerka.last_error_ret(context, -2) == r_failure - - def test_i_can_get_last_error_ret_when_only_one_ret_has_failed(self): - sheerka, context = self.init_concepts() - - r_success = sheerka.ret("Test", True, "success") - r_failure = sheerka.ret("Test", False, "False1") - - sheerka.last_return_values.append([r_success, r_failure]) - assert sheerka.last_error_ret(context) == r_failure - - def test_i_cannot_get_last_error_ret_when_too_many_errors(self): - sheerka, context = self.init_concepts() - - r1 = sheerka.ret("Test", True, "success") - r2 = sheerka.ret("Test", False, "False1") - r3 = sheerka.ret("Test", False, "False2") - - sheerka.last_return_values.append([r1, r2, r3]) - last_error_ret = sheerka.last_error_ret(context) - assert sheerka.isinstance(last_error_ret, BuiltinConcepts.RETURN_VALUE) - assert not last_error_ret.status - assert sheerka.isinstance(last_error_ret.body, BuiltinConcepts.TOO_MANY_ERRORS) - assert last_error_ret.body.body == [r2, r3] + # def test_i_can_get_last_error_ret(self): + # sheerka, context = self.init_concepts() + # + # # nothing in history + # last_error_ret = sheerka.last_error_ret(context) + # assert sheerka.isinstance(last_error_ret, BuiltinConcepts.RETURN_VALUE) + # assert not last_error_ret.status + # assert sheerka.isinstance(last_error_ret.body, BuiltinConcepts.NOT_FOUND) + # + # # only success in history + # r_success = sheerka.ret("Test", True, "success") + # sheerka.last_return_values.append([r_success]) + # last_error_ret = sheerka.last_error_ret(context) + # assert sheerka.isinstance(last_error_ret, BuiltinConcepts.RETURN_VALUE) + # assert not last_error_ret.status + # assert sheerka.isinstance(last_error_ret.body, BuiltinConcepts.NOT_FOUND) + # + # # at least on error + # r_failure = sheerka.ret("Test", False, "failure") + # sheerka.last_return_values.append([r_failure]) + # assert sheerka.last_error_ret(context) == r_failure + # + # # add another success and make sure we get the same error + # sheerka.last_return_values.append([r_success]) + # assert sheerka.last_error_ret(context) == r_failure + # + # # and I only get the last failure + # r_failure2 = sheerka.ret("Test", False, "another failure") + # sheerka.last_return_values.append([r_failure2]) + # assert sheerka.last_error_ret(context) == r_failure2 + # + # # but I still can get the previous error + # assert sheerka.last_error_ret(context, -2) == r_failure + # + # def test_i_can_get_last_error_ret_when_only_one_ret_has_failed(self): + # sheerka, context = self.init_concepts() + # + # r_success = sheerka.ret("Test", True, "success") + # r_failure = sheerka.ret("Test", False, "False1") + # + # sheerka.last_return_values.append([r_success, r_failure]) + # assert sheerka.last_error_ret(context) == r_failure + # + # def test_i_cannot_get_last_error_ret_when_too_many_errors(self): + # sheerka, context = self.init_concepts() + # + # r1 = sheerka.ret("Test", True, "success") + # r2 = sheerka.ret("Test", False, "False1") + # r3 = sheerka.ret("Test", False, "False2") + # + # sheerka.last_return_values.append([r1, r2, r3]) + # last_error_ret = sheerka.last_error_ret(context) + # assert sheerka.isinstance(last_error_ret, BuiltinConcepts.RETURN_VALUE) + # assert not last_error_ret.status + # assert sheerka.isinstance(last_error_ret.body, BuiltinConcepts.TOO_MANY_ERRORS) + # assert last_error_ret.body.body == [r2, r3] diff --git a/tests/core/test_SheerkaComparisonManager.py b/tests/core/test_SheerkaComparisonManager.py index 0e29a5a..9e6aa98 100644 --- a/tests/core/test_SheerkaComparisonManager.py +++ b/tests/core/test_SheerkaComparisonManager.py @@ -1,7 +1,7 @@ import pytest from core.builtin_concepts import BuiltinConcepts from core.concept import Concept -from core.global_symbols import CONCEPT_PRECEDENCE_MODIFIED, CONCEPT_COMPARISON_CONTEXT, RULE_PRECEDENCE_MODIFIED, \ +from core.global_symbols import EVENT_CONCEPT_PRECEDENCE_MODIFIED, CONCEPT_COMPARISON_CONTEXT, EVENT_RULE_PRECEDENCE_MODIFIED, \ RULE_COMPARISON_CONTEXT from core.sheerka.services.SheerkaComparisonManager import SheerkaComparisonManager, ComparisonObj @@ -446,7 +446,7 @@ class TestSheerkaGreaterThanManager(TestUsingMemoryBasedSheerka): nonlocal event_received event_received = True - sheerka.subscribe(CONCEPT_PRECEDENCE_MODIFIED, receive_event) + sheerka.subscribe(EVENT_CONCEPT_PRECEDENCE_MODIFIED, receive_event) sheerka.set_is_greater_than(context, foo, one, two) assert not event_received @@ -471,7 +471,7 @@ class TestSheerkaGreaterThanManager(TestUsingMemoryBasedSheerka): nonlocal event_received event_received = True - sheerka.subscribe(RULE_PRECEDENCE_MODIFIED, receive_event) + sheerka.subscribe(EVENT_RULE_PRECEDENCE_MODIFIED, receive_event) sheerka.set_is_greater_than(context, foo, r1, r2) assert not event_received diff --git a/tests/core/test_SheerkaDebugManager.py b/tests/core/test_SheerkaDebugManager.py index c618606..048b4d5 100644 --- a/tests/core/test_SheerkaDebugManager.py +++ b/tests/core/test_SheerkaDebugManager.py @@ -1,12 +1,23 @@ import pytest from core.builtin_concepts import BuiltinConcepts +from core.concept import Concept, NotInit from core.sheerka.ExecutionContext import ExecutionContext -from core.sheerka.services.SheerkaDebugManager import SheerkaDebugManager, DebugItem +from core.sheerka.services.SheerkaDebugManager import SheerkaDebugManager, DebugItem, ConceptDebugObj +from parsers.PythonParser import PythonNode from sdp.sheerkaDataProvider import Event from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka +class DummyObj: + def __init__(self, a, b): + self.a = a + self.b = b + + def __repr__(self): + return f"DummyObj(a={self.a}, b={self.b})" + + class TestSheerkaDebugManager(TestUsingMemoryBasedSheerka): def test_i_can_activate_debug(self): @@ -478,12 +489,85 @@ class TestSheerkaDebugManager(TestUsingMemoryBasedSheerka): assert len(service.debug_rules_settings) == 0 assert len(service.debug_concepts_settings) == 0 + @pytest.mark.parametrize("settings, expected", [ + ({"service": "my_service", "item": "var_name"}, {"var_name"}), + ({"method": "my_method", "item": "var_name"}, {"var_name"}), + ({"debug_id": 0, "item": "var_name"}, {"var_name"}), + ({"service": "my_service", "item": "*"}, {"*"}), + ({"method": "my_method", "item": "*"}, {"*"}), + ({"debug_id": 0, "item": "*"}, {"*"}), + ({"service": "my_service"}, set()), + ({"method": "my_method"}, set()), + ({"debug_id": 0}, set()), + ]) + def test_i_can_get_enabled_items(self, settings, expected): + sheerka, context = self.init_concepts() + service = sheerka.services[SheerkaDebugManager.NAME] + service.set_debug(context, True) + + service.add_or_update_debug_item(context, "vars", **settings) + + assert service.get_enabled_items("vars", context, "my_service", "my_method", 0) == expected + + def test_i_can_get_enabled_items_for_context(self): + sheerka, context = self.init_concepts() + another_context = context.push(BuiltinConcepts.TESTING, None) + service = sheerka.services[SheerkaDebugManager.NAME] + service.set_debug(context, True) + + service.add_or_update_debug_item(context, "rules", context_id=context.id, item="item", enabled=True) + + assert service.get_enabled_items("rules", context, "my_service", "my_method", 10) == {"item"} + assert service.get_enabled_items("rules", another_context, "my_service", "my_method", 10) == set() + + def test_i_can_get_enabled_items_for_sub_context(self): + sheerka, context = self.init_concepts() + sub_context = context.push(BuiltinConcepts.TESTING, None) + another_context = self.get_context(sheerka) + service = sheerka.services[SheerkaDebugManager.NAME] + service.set_debug(context, True) + + service.add_or_update_debug_item(context, "rules", context_id=context.id, item="item", context_children=True) + + assert service.get_enabled_items("rules", context, "my_service", "my_method", 10) == {"item"} + assert service.get_enabled_items("rules", sub_context, "my_service", "my_method", 10) == {"item"} + assert service.get_enabled_items("rules", another_context, "my_service", "my_method", 10) == set() + + def test_i_can_get_all_enabled_items(self): + sheerka, context = self.init_concepts() + service = sheerka.services[SheerkaDebugManager.NAME] + service.set_debug(context, True) + + service.add_or_update_debug_item(context, "vars", service="s1", item="v11") + service.add_or_update_debug_item(context, "vars", service="s2", item="v21") + service.add_or_update_debug_item(context, "vars", method="m1", item="v12") + service.add_or_update_debug_item(context, "vars", method="m2", item="v22") + service.add_or_update_debug_item(context, "vars", debug_id=1, item="v13") + service.add_or_update_debug_item(context, "vars", debug_id=2, item="v23") + service.add_or_update_debug_item(context, "vars", service="s1", method="m1", item="v111") + + assert service.get_enabled_items("vars", context, "s1", "m1", 1) == {"v11", "v12", "v13", "v111"} + + def test_i_can_manage_duplicates_when_get_enabled_items(self): + sheerka, context = self.init_concepts() + service = sheerka.services[SheerkaDebugManager.NAME] + service.set_debug(context, True) + + service.add_or_update_debug_item(context, "vars", service="s1", item="var_name") + service.add_or_update_debug_item(context, "vars", method="m1", item="*") + service.add_or_update_debug_item(context, "vars", debug_id=1, item="var_name") + service.add_or_update_debug_item(context, "vars", service="s1", method="m1", item="*") + + assert service.get_enabled_items("vars", context, "s1", "m1", 1) == {"var_name", "*"} + @pytest.mark.parametrize("args, kwargs, expected", [ (["my_service.my_method.my_var"], {}, ("my_var", "my_service", "my_method", None, False, None, True)), (["*.*.my_var"], {}, ("my_var", None, None, None, False, None, True)), (["my_service"], {}, (None, "my_service", None, None, False, None, True)), (["my_service.my_method"], {}, (None, "my_service", "my_method", None, False, None, True)), (["*.*.*"], {}, ("*", None, None, None, False, None, True)), + (["*.*.*.*.*"], {}, ("*.*.*", None, None, None, False, None, True)), + (["s.m.var.in.multi.parts"], {}, ("var.in.multi.parts", "s", "m", None, False, None, True)), ([1], {}, ("1", None, None, None, False, None, True)), (["", 1], {}, (None, None, None, 1, False, None, True)), ([None, 1], {}, (None, None, None, 1, False, None, True)), @@ -566,3 +650,265 @@ class TestSheerkaDebugManager(TestUsingMemoryBasedSheerka): DebugItem('1', None, None, None, False, None, False, True)] assert another_service.debug_concepts_settings == [ DebugItem('1001', None, None, None, False, None, False, True)] + + def test_i_can_inspect_concept_all_attributes(self): + sheerka, context, foo = self.init_concepts("foo") + foo.values() # freeze known attributes + foo.set_value("new_var", "var_value") + + res = sheerka.inspect(context, foo) + assert res.body == {"#type#": "Concept", + "id": foo.id, + "name": foo.name, + "key": foo.key, + "value.new_var": "'var_value'"} + + def test_i_can_inspect_concept_specified_attributes(self): + sheerka, context, foo = self.init_concepts("foo") + foo.values() # freeze known attributes + foo.set_value("new_var", "var_value") + + res = sheerka.inspect(context, foo, "id", "value.new_var") + assert res.body == {"id": foo.id, "value.new_var": "'var_value'"} + + def test_i_can_inspect_concept_specified_attributes_using_short_name(self): + sheerka, context, foo, bar = self.init_concepts(Concept("foo").def_var("var_name", "default_value"), "bar") + foo.set_value("var_name", "var_value") + foo.get_compiled()["var_name"] = bar + + res = sheerka.inspect(context, foo, "id", "var_name") + assert res.body == {"id": foo.id, + "meta.var_name": "'default_value'", + "compiled.var_name": bar, + "value.var_name": "'var_value'"} + + def test_i_can_inspect_object_all_attributes(self): + sheerka, context = self.init_concepts() + python_node = PythonNode("one + 1").init_ast() + + res = sheerka.inspect(context, python_node) + assert set(res.body.keys()) == {"#type#", 'ast_', 'ast_str', 'compiled', 'objects', 'source'} + + def test_i_can_inspect_object_specified_attributes(self): + sheerka, context = self.init_concepts() + python_node = PythonNode("one + 1").init_ast() + + res = sheerka.inspect(context, python_node, "#type#", "source", "ast_str") + assert res.body == { + "#type#": "PythonNode", + 'ast_str': "Expression(body=BinOp(left=Name(id='one'), op=Add(), right=Constant(value=1)))", + 'source': 'one + 1'} + + def test_i_can_inspect_object_with_as_bag_all_attributes(self): + sheerka, context = self.init_concepts() + + res = sheerka.inspect(context, context) + assert res.body == {'#type#': 'ExecutionContext', + '_children': [], + 'action': '__TESTING', + 'concepts': None, + 'context': None, + 'desc': None, + 'digest': 'xxx', + 'elapsed': 0, + 'elapsed_str': '0.0 ms', + 'id': context.id, + 'inputs': {}, + 'obj': None, + 'status': None, + 'values': {}, + 'who': 'test'} + + def test_i_can_inspect_object_with_as_bag_specified_attributes(self): + sheerka, context = self.init_concepts() + + res = sheerka.inspect(context, context, "who", "_children") + assert res.body == {'_children': [], + 'who': 'test'} + + def test_i_can_inspect_object_with_concept(self): + sheerka, context, foo = self.init_concepts("foo") + foo.values() # freeze known attributes + sheerka.set_attr(foo, "new_var", "var_value") + + dummy = DummyObj(foo, "value") + res = sheerka.inspect(context, dummy) + assert res.body == {'#type#': 'DummyObj', 'a': foo, 'b': 'value'} + + def test_i_can_inspect_object_with_concept_when_as_bag(self): + sheerka, context, foo = self.init_concepts("foo") + foo.values() # freeze known attributes + sheerka.set_attr(foo, "new_var", "var_value") + + dummy = DummyObj(foo, "value") + res = sheerka.inspect(context, dummy, as_bag=True) + assert res.body == {'#type#': 'DummyObj', + 'a': {'#type#': 'Concept', + 'id': '1001', + 'key': 'foo', + 'name': 'foo', + 'value.new_var': "'var_value'"}, + 'b': 'value'} + + def test_i_can_inspect_object_with_concept_when_values(self): + sheerka, context, foo = self.init_concepts("foo") + foo.values() # freeze known attributes + sheerka.set_attr(foo, "new_var", "var_value") + + dummy = DummyObj(foo, "value") + res = sheerka.inspect(context, dummy, values=True) + assert res.body == {'#type#': 'DummyObj', 'a': ConceptDebugObj(foo), 'b': 'value'} + + def test_i_can_inspect_concept_with_concept(self): + sheerka, context, foo, bar = self.init_concepts(Concept("foo").def_var("var_name"), "bar") + foo.set_value("var_name", bar) + + res = sheerka.inspect(context, foo) + assert res.body == {'#type#': 'Concept', + 'id': '1001', + 'key': 'foo', + 'name': 'foo', + 'value.var_name': bar} + + def test_i_can_inspect_concept_with_concept_when_as_bag(self): + sheerka, context, foo, bar = self.init_concepts(Concept("foo").def_var("var_name"), "bar") + foo.set_value("var_name", bar) + + res = sheerka.inspect(context, foo, as_bag=True) + assert res.body == {'#type#': 'Concept', + 'id': '1001', + 'key': 'foo', + 'name': 'foo', + 'value.var_name': {'#type#': 'Concept', + 'id': '1002', + 'key': 'bar', + 'name': 'bar'}} + + def test_i_can_inspect_concept_with_concept_when_values(self): + sheerka, context, foo, bar = self.init_concepts(Concept("foo").def_var("var_name"), "bar") + foo.set_value("var_name", bar) + + res = sheerka.inspect(context, foo, values=True) + assert res.body == {'#type#': 'Concept', + 'id': '1001', + 'key': 'foo', + 'name': 'foo', + 'value.var_name': ConceptDebugObj(bar)} + + def test_i_can_inspect_execution_context_item(self): + sheerka, context = self.init_concepts() + ExecutionContext.ids.clear() + + sheerka.evaluate_user_input("def concept one as 1") + results = list(sheerka.get_last_results(context).body) + user_input_ret_val = results[0].inputs["user_input"] + return_values = results[0].values["return_values"] + + res = sheerka.inspect(context, 0) + assert res.body == {'inputs': {'user_input': user_input_ret_val}, + 'values.return_values': return_values} + + def test_i_can_inspect_execution_context_values(self): + sheerka, context = self.init_concepts() + ExecutionContext.ids.clear() + + sheerka.evaluate_user_input("def concept one as 1") + results = list(sheerka.get_last_results(context).body) + user_input_ret_val = results[0].inputs["user_input"] + return_values = results[0].values["return_values"] + + res = sheerka.inspect(context, 0, values=True) + + assert res.body == {'inputs': {'user_input': user_input_ret_val}, + 'values.return_values': [ConceptDebugObj(return_values[0].body.body)]} + + def test_i_can_inspect_when_a_property_does_not_exist(self): + sheerka, context, foo = self.init_concepts("foo") + foo.values() # freeze known attributes + sheerka.set_attr(foo, "new_var", "var_value") + + dummy = DummyObj(foo, "value") + res = sheerka.inspect(context, dummy, "#type#", "fake", "a", "b") + assert res.body == {'#type#': 'DummyObj', + 'fake': "** Not Found **", + 'a': foo, + 'b': 'value'} + + def test_i_can_inspect_when_properties_are_specified_several_times(self): + sheerka, context, foo = self.init_concepts("foo") + foo.values() # freeze known attributes + sheerka.set_attr(foo, "new_var", "var_value") + + dummy = DummyObj(foo, "value") + res = sheerka.inspect(context, dummy, "#type#", "a", "b", "a") + assert res.body == {'#type#': 'DummyObj', + 'a': foo, + 'b': 'value'} + + def test_i_cannot_inspect_execution_context_item_if_no_last_item(self): + sheerka, context = self.init_concepts() + + res = sheerka.inspect(context, 0) + + assert res.body == {'#type#': 'NotFound', + 'id': '70', + 'key': '__NOT_FOUND', + 'name': '__NOT_FOUND', + 'body': 'no digest'} + + def test_i_can_inspect_values(self): + sheerka, context, table, how, little = self.init_concepts( + "table", + Concept("how is x").def_var("x"), + Concept("little x").def_var("x"), + create_new=True + ) + + return_values = sheerka.evaluate_user_input("how is little table") + + res = sheerka.inspect(context, return_values[0], values=True) + concept_debug_obj = ConceptDebugObj(return_values[0].body) + assert res.body == { + 'body': concept_debug_obj, + '#type#': 'ReturnValueConcept', + 'id': '43', + 'key': '__RETURN_VALUE', + 'message': None, + 'name': '__RETURN_VALUE', + 'parents': [concept_debug_obj], + 'status': True, + 'value': concept_debug_obj, + 'who': 'evaluators.OneSuccess'} + + # I also can print it using bag + res = sheerka.inspect(context, return_values[0], '#type#', "who", "status", "value", values=True, as_bag=True) + assert res.body == {'#type#': 'ReturnValueConcept', + 'who': 'evaluators.OneSuccess', + 'status': True, + 'value': {'#type#': 'Concept', + 'compiled.x': {'#type#': 'Concept', + 'compiled.x': {'#type#': 'Concept', + 'id': '1001', + 'key': 'table', + 'name': 'table'}, + 'id': '1003', + 'key': 'little __var__0', + 'meta.x': "'table'", + 'name': 'little x'}, + 'id': '1002', + 'key': 'how is __var__0', + 'meta.x': "'little table'", + 'name': 'how is x'}} + + def test_i_can_display_meta_and_compile_attributes_using_concept_debug_obj(self): + foo = Concept("foo", id=1001, key="foo_key").def_var("x", "x_meta").def_var("y", "y_meta") + foo.get_compiled()["x"] = Concept("bar", id=1002).def_var("a", "a_meta").set_value("a", "a_value") + foo.set_value("x", "x_value") + foo.set_value("y", NotInit) + + foo.values() # freeze attributes + + foo.set_value("z", "extra_value") + + assert str(ConceptDebugObj(foo)) == \ + "(:foo|1001:meta.x='x_meta', meta.y='y_meta', compiled.x=(:bar|1002:meta.a='a_meta', value.a='a_value'), value.x='x_value', value.z='extra_value')" diff --git a/tests/core/test_SheerkaMemory.py b/tests/core/test_SheerkaMemory.py index b8695c9..15a6f5a 100644 --- a/tests/core/test_SheerkaMemory.py +++ b/tests/core/test_SheerkaMemory.py @@ -85,7 +85,7 @@ class TestSheerkaMemory(TestUsingMemoryBasedSheerka): foo = Concept("foo") sheerka.add_to_memory(context, "a", foo) - assert service.objects.copy() == {"a": MemoryObject(context.event.get_digest(), foo)} + assert service.memory_objects.copy() == {"a": MemoryObject(context.event.get_digest(), foo)} assert id(sheerka.get_from_memory(context, "a").obj) == id(foo) def test_i_can_use_memory_to_get_the_list_of_all_objects(self): @@ -134,6 +134,11 @@ class TestSheerkaMemory(TestUsingMemoryBasedSheerka): assert sheerka.memory(context, "item") == bar + def test_object_are_not_added_in_memory_during_the_initialisation(self): + sheerka, context = self.init_concepts() + + assert len(sheerka.memory(context)) == 0 + class TestSheerkaMemoryUsingFileBase(TestUsingFileBasedSheerka): def test_i_can_record_memory_objects(self): diff --git a/tests/core/test_SheerkaRuleManager.py b/tests/core/test_SheerkaRuleManager.py index 32372d6..416f1c9 100644 --- a/tests/core/test_SheerkaRuleManager.py +++ b/tests/core/test_SheerkaRuleManager.py @@ -2,7 +2,7 @@ import ast import pytest from core.builtin_concepts import BuiltinConcepts -from core.concept import Concept +from core.concept import Concept, CMV from core.global_symbols import RULE_COMPARISON_CONTEXT from core.rule import Rule from core.sheerka.services.SheerkaRuleManager import SheerkaRuleManager, FormatRuleParser, \ @@ -227,8 +227,8 @@ class TestSheerkaRuleManager(TestUsingMemoryBasedSheerka): assert res[0].concept == expected @pytest.mark.parametrize("text, expected_variables", [ - ("a cat is an animal", ["cat", "animal"]), - ("a cat is an b", ["a", "animal"]), + ("a cat is an animal", ["a cat", "animal"]), + ("a cat is an b", ["a cat", "b"]), ]) def test_i_can_compile_predicate_when_sya_node_parser(self, text, expected_variables): sheerka, context, *concepts = self.init_concepts( @@ -238,7 +238,7 @@ class TestSheerkaRuleManager(TestUsingMemoryBasedSheerka): create_new=True ) service = sheerka.services[SheerkaRuleManager.NAME] - expected = concepts[0] + expected = CMV(concepts[0], x=expected_variables[0], y=expected_variables[1]) res = service.compile_when(context, "test", text) @@ -281,14 +281,14 @@ class TestSheerkaRuleManager(TestUsingMemoryBasedSheerka): assert isinstance(res[0], RulePredicate) assert res[0].evaluator == CONCEPT_EVALUATOR_NAME assert sheerka.isinstance(res[0].predicate, BuiltinConcepts.RETURN_VALUE) - assert sheerka.objvalue(res[0].predicate)[0].concept == concepts[0] - assert res[0].concept == concepts[0] + assert sheerka.objvalue(res[0].predicate)[0].concept == CMV(concepts[0], x="a", y="b") + assert res[0].concept == CMV(concepts[0], x="a", y="b") assert isinstance(res[1], RulePredicate) assert res[1].evaluator == CONCEPT_EVALUATOR_NAME assert sheerka.isinstance(res[1].predicate, BuiltinConcepts.RETURN_VALUE) - assert sheerka.objvalue(res[1].predicate)[0].concept == concepts[1] - assert res[1].concept == concepts[1] + assert sheerka.objvalue(res[1].predicate)[0].concept == CMV(concepts[1], x="a", y="b") + assert res[1].concept == CMV(concepts[1], x="a", y="b") # @pytest.mark.skip # @pytest.mark.parametrize("text, expected", [ diff --git a/tests/core/test_sheerkaResultManager.py b/tests/core/test_sheerkaResultManager.py index d3f8886..a4ceaca 100644 --- a/tests/core/test_sheerkaResultManager.py +++ b/tests/core/test_sheerkaResultManager.py @@ -14,10 +14,8 @@ class TestSheerkaResultManager(TestUsingMemoryBasedSheerka): @classmethod def setup_class(cls): - sheerka = cls().get_sheerka() + sheerka = cls().get_sheerka(cache_only=False) sheerka.save_execution_context = True - sheerka.evaluate_user_input("def concept one as 1") - cls.io_cache = sheerka.sdp.io.cache.copy() @classmethod def teardown_class(cls): @@ -26,21 +24,73 @@ class TestSheerkaResultManager(TestUsingMemoryBasedSheerka): def init_test(self): sheerka, context = self.init_concepts() - sheerka.sdp.io.cache = self.io_cache.copy() - return sheerka, context + service = sheerka.services[SheerkaResultConcept.NAME] - def test_i_can_get_the_result_by_digest(self): - sheerka, context = self.init_test() + return sheerka, context, service - digest = sheerka.get_last_execution().event.get_digest() + def test_i_can_record_execution_contexts(self): + sheerka, context, service = self.init_test() + + sheerka.evaluate_user_input("foo") + + executions_contexts_in_cache = service.executions_contexts_cache.copy() + assert len(executions_contexts_in_cache) == 1 + event_id = list(executions_contexts_in_cache.keys())[0] + execution_context = list(executions_contexts_in_cache.values())[0] + + assert execution_context.desc == "Evaluating 'foo'" + + executions_contexts_in_db = sheerka.sdp.load_result(event_id) + assert executions_contexts_in_db is not None + assert executions_contexts_in_db.desc == "Evaluating 'foo'" + + assert service.last_execution is not None + assert service.last_execution.desc == "Evaluating 'foo'" + + def test_i_can_get_the_result_by_digest_using_cache(self): + sheerka, context, service = self.init_test() + sheerka.evaluate_user_input("def concept one as 1") + + digest = service.last_execution.event.get_digest() res = sheerka.get_results_by_digest(context, digest) + # we get the result assert sheerka.isinstance(res, BuiltinConcepts.EXPLANATION) assert res.command == "def concept one as 1" assert res.digest == digest assert isinstance(res.body, types.GeneratorType) + # the digest is correctly recorded + assert sheerka.load_var(SheerkaResultConcept.NAME, "digest") == digest + + previous_results = list(res.body) + + # Second test, + # I can get the result from the recorded digest + res = sheerka.get_results(context) + assert sheerka.isinstance(res, BuiltinConcepts.EXPLANATION) + assert res.command == "def concept one as 1" + assert res.digest == digest + assert isinstance(res.body, types.GeneratorType) + + assert list(res.body) == previous_results + + def test_i_can_get_result_by_digest_using_db(self): + sheerka, context, service = self.init_test() + sheerka.evaluate_user_input("def concept one as 1") + digest = service.last_execution.event.get_digest() + service.reset() + + res = sheerka.get_results_by_digest(context, digest) + + # we get the result + assert sheerka.isinstance(res, BuiltinConcepts.EXPLANATION) + assert res.command == "def concept one as 1" + assert res.digest == digest + assert isinstance(res.body, types.GeneratorType) + + # the digest is correctly recorded assert sheerka.load_var(SheerkaResultConcept.NAME, "digest") == digest previous_results = list(res.body) @@ -67,9 +117,10 @@ class TestSheerkaResultManager(TestUsingMemoryBasedSheerka): assert sheerka.get_results(context) is None def test_i_can_get_the_result_by_command_name(self): - sheerka, context = self.init_test() + sheerka, context, service = self.init_test() + sheerka.evaluate_user_input("def concept one as 1") - digest = sheerka.get_last_execution().event.get_digest() + digest = service.last_execution.event.get_digest() sheerka.evaluate_user_input("one") # another command res = sheerka.get_results_by_command(context, "def concept") @@ -78,8 +129,18 @@ class TestSheerkaResultManager(TestUsingMemoryBasedSheerka): assert res.digest == digest assert isinstance(res.body, types.GeneratorType) + def test_i_can_get_the_result_by_command_name_using_db(self): + sheerka, context, service = self.init_test() + sheerka.evaluate_user_input("def concept one as 1") + sheerka.evaluate_user_input("one") # another command + service.reset() + + res = sheerka.get_results_by_command(context, "def concept") + assert res.command == "def concept one as 1" + def test_i_can_get_the_result_by_command_when_not_in_the_same_page_size(self): - sheerka, context = self.init_test() + sheerka, context, service = self.init_test() + sheerka.evaluate_user_input("def concept one as 1") sheerka.evaluate_user_input("one") sheerka.evaluate_user_input("one") @@ -98,7 +159,8 @@ class TestSheerkaResultManager(TestUsingMemoryBasedSheerka): assert res.body == {'command': 'def concept'} def test_i_cannot_get_result_from_command_if_the_command_does_not_exists_multiple_pages(self): - sheerka, context = self.init_test() + sheerka, context, service = self.init_test() + sheerka.evaluate_user_input("def concept one as 1") sheerka.evaluate_user_input("one") sheerka.evaluate_user_input("one") @@ -110,7 +172,8 @@ class TestSheerkaResultManager(TestUsingMemoryBasedSheerka): assert res.body == {'command': 'fake command'} def test_i_can_get_last_results(self): - sheerka, context = self.init_test() + sheerka, context, service = self.init_test() + sheerka.evaluate_user_input("def concept one as 1") sheerka.evaluate_user_input("one") @@ -118,6 +181,16 @@ class TestSheerkaResultManager(TestUsingMemoryBasedSheerka): assert sheerka.isinstance(res, BuiltinConcepts.EXPLANATION) assert res.command == "one" + def test_i_can_get_last_results_using_db(self): + sheerka, context, service = self.init_test() + sheerka.evaluate_user_input("def concept one as 1") + sheerka.evaluate_user_input("one") + service.reset() + + res = sheerka.get_last_results(context) + assert sheerka.isinstance(res, BuiltinConcepts.EXPLANATION) + assert res.command == "one" + def test_i_can_get_last_results_when_event_with_no_result(self): sheerka, context = self.init_concepts() @@ -154,7 +227,8 @@ class TestSheerkaResultManager(TestUsingMemoryBasedSheerka): {"desc": "Evaluating 'def concept one as 1'", "id": 0} ]) def test_i_can_get_last_results_using_kwarg(self, kwargs): - sheerka, context = self.init_test() + sheerka, context, service = self.init_test() + sheerka.evaluate_user_input("def concept one as 1") ExecutionContext.ids.clear() res = sheerka.get_last_results(context, **kwargs) @@ -170,7 +244,8 @@ class TestSheerkaResultManager(TestUsingMemoryBasedSheerka): ("'def concept one as 1' in desc and id == 0", {"desc": "Evaluating 'def concept one as 1'", "id": 0}) ]) def test_i_can_get_last_results_using_filter(self, predicate, expected): - sheerka, context = self.init_test() + sheerka, context, service = self.init_test() + sheerka.evaluate_user_input("def concept one as 1") ExecutionContext.ids.clear() res = sheerka.get_last_results(context, filter=predicate) @@ -186,7 +261,8 @@ class TestSheerkaResultManager(TestUsingMemoryBasedSheerka): ("'def concept one as 1' in desc and id == 0", {"desc": "Evaluating 'def concept one as 1'", "id": 0}) ]) def test_i_can_get_last_results_using_the_first_argument_to_filter(self, predicate, expected): - sheerka, context = self.init_test() + sheerka, context, service = self.init_test() + sheerka.evaluate_user_input("def concept one as 1") ExecutionContext.ids.clear() res = sheerka.get_last_results(context, predicate) @@ -202,7 +278,8 @@ class TestSheerkaResultManager(TestUsingMemoryBasedSheerka): {"desc": "Evaluating 'def concept one as 1'", "id": 0} ]) def test_i_can_get_results_using_kwarg(self, kwargs): - sheerka, context = self.init_test() + sheerka, context, service = self.init_test() + sheerka.evaluate_user_input("def concept one as 1") ExecutionContext.ids.clear() sheerka.get_last_results(context) @@ -219,7 +296,8 @@ class TestSheerkaResultManager(TestUsingMemoryBasedSheerka): ("'def concept one as 1' in desc and id == 0", {"desc": "Evaluating 'def concept one as 1'", "id": 0}) ]) def test_i_can_get_results_using_filter(self, predicate, expected): - sheerka, context = self.init_test() + sheerka, context, service = self.init_test() + sheerka.evaluate_user_input("def concept one as 1") ExecutionContext.ids.clear() sheerka.get_last_results(context) @@ -236,7 +314,8 @@ class TestSheerkaResultManager(TestUsingMemoryBasedSheerka): ("'def concept one as 1' in desc and id == 0", {"desc": "Evaluating 'def concept one as 1'", "id": 0}) ]) def test_i_can_get_results_using_the_first_argument_to_filter(self, predicate, expected): - sheerka, context = self.init_test() + sheerka, context, service = self.init_test() + sheerka.evaluate_user_input("def concept one as 1") ExecutionContext.ids.clear() sheerka.get_last_results(context) @@ -251,3 +330,35 @@ class TestSheerkaResultManager(TestUsingMemoryBasedSheerka): predicate = {"filter": "a b c"} with pytest.raises(SyntaxError): SheerkaResultConcept.get_predicate(**predicate) + + def test_i_can_get_last_return_value(self): + sheerka, context, service = self.init_test() + + sheerka.evaluate_user_input("def concept one as 1") + ret = sheerka.last_ret(context) + assert sheerka.isinstance(ret[0].body, BuiltinConcepts.NEW_CONCEPT) + + sheerka.evaluate_user_input("eval one") + ret = sheerka.last_ret(context) + assert ret[0].body == 1 + + def test_i_can_track_new_concept(self): + sheerka, context, service = self.init_test() + + res = sheerka.evaluate_user_input("def concept one as 1") + new_concept = res[0].body.body + + assert sheerka.last_created_concept(context) == new_concept + assert service.last_created_concept_id == new_concept.id + + def test_last_created_concept_is_recorded(self): + sheerka, context, service = self.init_test() + res = sheerka.evaluate_user_input("def concept one as 1") + new_concept = res[0].body.body + service.reset() + + service.initialize_deferred(context, False) + + assert service.last_created_concept_id == new_concept.id + assert sheerka.last_created_concept(context) == new_concept + diff --git a/tests/core/test_utils.py b/tests/core/test_utils.py index 911e46b..f46c494 100644 --- a/tests/core/test_utils.py +++ b/tests/core/test_utils.py @@ -2,6 +2,7 @@ from dataclasses import dataclass import core.utils import pytest +from core.builtin_concepts import BuiltinConcepts from core.concept import Concept from core.tokenizer import Token, TokenKind, Tokenizer, Keywords @@ -338,3 +339,81 @@ def test_i_can_get_text_from_tokens(text, expected_text): def test_i_can_get_text_from_tokens_with_custom_switcher(text, custom, expected_text): tokens = list(Tokenizer(text)) assert core.utils.get_text_from_tokens(tokens, custom) == expected_text + + +def test_i_can_deep_copy_a_concept(): + def check_are_the_same(actual, expected): + assert id(actual) != id(expected) + + for k, v in vars(expected.get_metadata()).items(): + assert getattr(actual.get_metadata(), k) == v + + # test the values + for k, v in expected.values().items(): + assert getattr(actual, k) == v + + concept1 = Concept(name="concept1_name", + is_builtin=True, + is_unique=True, + key="concept1_key", + body="concept1_body", + where='concept1_where', + pre="concept1_pre", + post="concept1_post", + ret="concept1_ret", + definition="concept1_definition", + definition_type="concept1_definition_type", + desc="concept1_desc", + id="concept1_ids", + props="concept1_props", + variables=[], + bound_body=None) + + concept2 = Concept(name="concept2_name", + is_builtin=True, + is_unique=True, + key="concept2_key", + body="concept2_body", + where='concept2_where', + pre="concept2_pre", + post="concept2_post", + ret="concept2_ret", + definition="concept2_definition", + definition_type="concept2_definition_type", + desc="concept2_desc", + id="concept2_ids", + props={"prop_name": concept1}, + variables=[("var1", "default_value1"), ("var2", "default_value2")], + bound_body="var1") + + concept = Concept(name="my_name", + is_builtin=True, + is_unique=True, + key="my_key", + body="my_body", + where='my_where', + pre="my_pre", + post="my_post", + ret="my_ret", + definition="my_definition", + definition_type="my_definition_type", + desc="my_desc", + id="my_ids", + props={ + BuiltinConcepts.ISA: {concept1, concept2}, + "prop2": ["value1, value2"], + "prop3": {"a": 1, "b": 2}, + "prop4": "a simple value"}, + variables=[("var1", "default_value1"), ("var2", "default_value2")]) + concept.set_value("var1", "string_value") + concept.set_value("var2", 10) + concept.set_value("var3", concept1) + + copied = core.utils.sheerka_deepcopy(concept) + + check_are_the_same(copied, concept) + + copied_props = sorted(list(copied.get_prop(BuiltinConcepts.ISA)), key=lambda o: o.id) + concept_props = sorted(list(concept.get_prop(BuiltinConcepts.ISA)), key=lambda o: o.id) + for copied_prop, concept_prop in zip(copied_props, concept_props): + check_are_the_same(copied_prop, concept_prop) diff --git a/tests/non_reg/test_sheerka_display.py b/tests/non_reg/test_sheerka_display.py new file mode 100644 index 0000000..88a3eac --- /dev/null +++ b/tests/non_reg/test_sheerka_display.py @@ -0,0 +1,16 @@ +from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka + + +class TestSheerkaNonRegDisplay(TestUsingMemoryBasedSheerka): + + def test_i_can_display_results_when_return_values_processing_is_on(self, capsys): + init = [ + "def concept one as 1", + ] + + sheerka = self.init_scenario(init) + sheerka.enable_process_return_values = True + sheerka.evaluate_user_input("one") + + captured = capsys.readouterr() + assert captured.out == "ReturnValue(who=evaluators.OneSuccess, status=True, value=(1001)one)\n" diff --git a/tests/non_reg/test_sheerka_non_reg.py b/tests/non_reg/test_sheerka_non_reg.py index d1beced..f25c7f1 100644 --- a/tests/non_reg/test_sheerka_non_reg.py +++ b/tests/non_reg/test_sheerka_non_reg.py @@ -1092,7 +1092,7 @@ as: ] sheerka = self.init_scenario(init) - res = sheerka.evaluate_user_input("desc(the a)") + res = sheerka.evaluate_user_input("desc(c:the a:)") assert len(res) == 1 assert res[0].status diff --git a/tests/out/test_AsStrVisitor.py b/tests/out/test_AsStrVisitor.py index 8070674..41150d8 100644 --- a/tests/out/test_AsStrVisitor.py +++ b/tests/out/test_AsStrVisitor.py @@ -142,7 +142,7 @@ class TestAsStrVisitor(TestUsingMemoryBasedSheerka): (FormatAstVariable('__key', index=0, value="key1", debug=True), FormatAstVariable('__value', index="key1", value=1, debug=True)), - (FormatAstVariable('__key', index=0, value="key2", debug=True), + (FormatAstVariable('__key', index=1, value="key2", debug=True), FormatAstDict("__value", debug=True, prefix="{", suffix="}", items=[ (FormatAstVariable('__key', index=0, value="sub_key1", debug=True), FormatAstVariable('__value', index="sub_key1", value=1, debug=True)), @@ -155,8 +155,14 @@ class TestAsStrVisitor(TestUsingMemoryBasedSheerka): ])), ])), - (FormatAstVariable('__key', index=1, value="long_key3", debug=True), + (FormatAstVariable('__key', index=2, value="long_key3", debug=True), FormatAstVariable('__value', index="key2", value="value2", debug=True)), + + (FormatAstVariable('__key', index=3, value="key3", debug=True), + FormatAstList("__value", debug=True, prefix="[", suffix="]", items=[ + FormatAstVariable('__item', index=0, value="first element", debug=True), + FormatAstVariable('__item', index=1, value="second element", debug=True), + ])), ]) res = visitor.visit(bag) @@ -165,4 +171,6 @@ class TestAsStrVisitor(TestUsingMemoryBasedSheerka): 'key2' : {'sub_key1' : 1, 'sub_long_key2': {'sub_sub_key1': 1, 'sub_sub_key2': 'sub_sub_value'}}, -'long_key3': 'value2'}""" +'long_key3': 'value2', +'key3' : ['first element', + 'second element']}""" diff --git a/tests/out/test_DeveloperVisitor.py b/tests/out/test_DeveloperVisitor.py new file mode 100644 index 0000000..12c8a58 --- /dev/null +++ b/tests/out/test_DeveloperVisitor.py @@ -0,0 +1,40 @@ +from core.sheerka.services.SheerkaDebugManager import NullDebugLogger +from core.sheerka.services.SheerkaOut import SheerkaOut +from core.sheerka.services.SheerkaRuleManager import FormatAstList, FormatAstVariable +from out.DeveloperVisitor import DeveloperVisitor + +from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka + + +class TestDeveloperVisitor(TestUsingMemoryBasedSheerka): + def test_i_can_develop_list(self): + sheerka, context = self.init_concepts() + service_out = sheerka.services[SheerkaOut.NAME] + dev_visitor = DeveloperVisitor(service_out, NullDebugLogger(), set(), 0) + + bag = {"a": ["a", "b", "c"]} + + res = dev_visitor.visit(context, FormatAstList("a"), bag) + assert res == FormatAstList(variable="a", items=[ + FormatAstVariable(name="__item", index=0, value="a"), + FormatAstVariable(name="__item", index=1, value="b"), + FormatAstVariable(name="__item", index=2, value="c"), + ]) + + def test_i_can_develop_list_of_list(self): + sheerka, context = self.init_concepts() + service_out = sheerka.services[SheerkaOut.NAME] + dev_visitor = DeveloperVisitor(service_out, NullDebugLogger(), set(), 0) + + bag = {"a": [["a1", "a2"], ["b1"]]} + + res = dev_visitor.visit(context, FormatAstList("a"), bag) + assert res == FormatAstList(variable="a", items=[ + FormatAstList(variable="__item", index=0, debug=True, prefix='[', suffix=']', items=[ + FormatAstVariable(name="__item", index=0, debug=True, value="a1"), + FormatAstVariable(name="__item", index=1, debug=True, value="a2"), + ]), + FormatAstList(variable="__item", index=1, debug=True, prefix='[', suffix=']', items=[ + FormatAstVariable(name="__item", index=0, debug=True, value="b1"), + ]), + ]) diff --git a/tests/out/test_SheerkaOut.py b/tests/out/test_SheerkaOut.py index c5ff3c8..ce55a3b 100644 --- a/tests/out/test_SheerkaOut.py +++ b/tests/out/test_SheerkaOut.py @@ -449,3 +449,51 @@ key2: value2 captured = capsys.readouterr() assert captured.out == """{'\x1b[32mkey1\x1b[0m': 'value1', 'key2': 1, 'key3': DummyObj(prop_1=3.15, prop_2='a string'), 'key4': ['alpha', 0]} """ + + def test_i_can_print_out_dict_sub_items(self, capsys): + sheerka, context, service, *rules = self.init_service_with_rules( + ("isinstance(__obj, dict)", "dict(__obj)"), + ("__key=='key1'", "green(__key)") + ) + obj = { + "key1": "value1", + "key2": 1, + "key3": DummyObj(prop_1=3.15, prop_2='a string'), + "key4": {"a": 1, "b": "value"}, + "key5": ["alpha", 0] + } + service.process_return_values(context, obj) + captured = capsys.readouterr() + assert captured.out == """\x1b[32mkey1\x1b[0m: value1 +key2: 1 +key3: DummyObj(prop_1=3.15, prop_2='a string') +key4: {'a': 1, 'b': 'value'} +key5: ['alpha', 0] +""" + + def test_i_can_print_out_dict_with_expanded_sub_items(self, capsys): + sheerka, context, service, *rules = self.init_service_with_rules( + ("isinstance(__obj, dict)", "dict(__obj)"), + ("__key=='key1'", "green(__key)") + ) + obj = { + "key1": "value1", + "key2": 1, + "key3": DummyObj(prop_1=3.15, prop_2='a string'), + "key4": {"a": 1, "b": "value"}, + "key5": ["alpha", 0] + } + + old_value = service.out_visitors[0].console_width + service.out_visitors[0].console_width = 5 + service.process_return_values(context, obj) + captured = capsys.readouterr() + assert captured.out == """\x1b[32mkey1\x1b[0m: value1 +key2: 1 +key3: DummyObj(prop_1=3.15, prop_2='a string') +key4: {'a': 1, + 'b': 'value'} +key5: ['alpha', + 0] +""" + service.out_visitors[0].console_width = old_value diff --git a/tests/parsers/test_SyaNodeParser.py b/tests/parsers/test_SyaNodeParser.py index 86e90e5..a6a2721 100644 --- a/tests/parsers/test_SyaNodeParser.py +++ b/tests/parsers/test_SyaNodeParser.py @@ -1,10 +1,11 @@ import pytest from core.builtin_concepts import BuiltinConcepts -from core.concept import Concept, CIO, ALL_ATTRIBUTES +from core.concept import Concept, CIO, ALL_ATTRIBUTES, CMV from core.global_symbols import CONCEPT_COMPARISON_CONTEXT from core.sheerka.services.SheerkaExecute import ParserInput from core.tokenizer import Tokenizer -from parsers.BaseNodeParser import utnode, ConceptNode, cnode, short_cnode, UnrecognizedTokensNode, \ +from core.utils import NextIdManager +from parsers.BaseNodeParser import utnode, cnode, short_cnode, UnrecognizedTokensNode, \ SCWC, CNC, UTN, SCN, CN from parsers.PythonParser import PythonNode from parsers.SyaNodeParser import SyaNodeParser, SyaConceptParserHelper, SyaAssociativity, \ @@ -916,21 +917,21 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka): @pytest.mark.parametrize("expression, expected_debugs", [ ("one", [[" 0:one => PUSH_UNREC"]]), - ("one plus two", [[ - ' 0:one => PUSH_UNREC', - ' 1: => PUSH_UNREC', - ' 2:plus(SyaConceptDef(concept=(1005)a plus b, precedence=1, associativity=right)) => ??', - " _: => RECOG [[CN((1001)one)]]", - " _: => POP ConceptNode(concept='(1001)one', source='one', start=0, end=0)", - ' 2:plus(SyaConceptDef(concept=(1005)a plus b, precedence=1, associativity=right)) => PUSH', - ' 3: => EAT', - ' 4:two => PUSH_UNREC', - ' 5: => ??', - " _: => RECOG [[CN((1002)two)]]", - " _: => POP ConceptNode(concept='(1002)two', source='two', start=4, end=4)", - ' _: => POP SyaConceptParserHelper(concept=(1005)a plus b, start=2, error=None)']]), + ("one plus two", [[' 0:one => PUSH_UNREC', + ' 1: => PUSH_UNREC', + ' 2:plus ((1005)a plus b, prio=1, assoc=SyaAssociativity.Right) => ??', + ' _: => RECOG [[CN((1001)one)]]', + " _: => POP ConceptNode(concept='(1001)one', source='one', start=0, end=0)", + ' 2:plus ((1005)a plus b, prio=1, assoc=SyaAssociativity.Right) => PUSH', + ' 3: => EAT', + ' 4:two => PUSH_UNREC', + ' 5: => ??', + ' _: => RECOG [[CN((1002)two)]]', + " _: => POP ConceptNode(concept='(1002)two', source='two', start=4, end=4)", + ' _: => POP SyaConceptParserHelper(concept=(1005)a plus b, start=2, ' + 'error=None)']]), ("suffixed one", [[ - ' 0:suffixed(SyaConceptDef(concept=(1009)suffixed a, precedence=1, associativity=right)) => PUSH', + ' 0:suffixed ((1009)suffixed a, prio=1, assoc=SyaAssociativity.Right) => PUSH', ' 1: => EAT', ' 2:one => PUSH_UNREC', ' 3: => ??', @@ -941,10 +942,10 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka): ("one ? twenty one : three", [[ ' 0:one => PUSH_UNREC', ' 1: => PUSH_UNREC', - ' 2:?(SyaConceptDef(concept=(1011)a ? b : c, precedence=1, associativity=right)) => ??', - " _: => RECOG [[CN((1001)one)]]", + ' 2:? ((1011)a ? b : c, prio=1, assoc=SyaAssociativity.Right) => ??', + ' _: => RECOG [[CN((1001)one)]]', " _: => POP ConceptNode(concept='(1001)one', source='one', start=0, end=0)", - ' 2:?(SyaConceptDef(concept=(1011)a ? b : c, precedence=1, associativity=right)) => PUSH', + ' 2:? ((1011)a ? b : c, prio=1, assoc=SyaAssociativity.Right) => PUSH', ' 3: => EAT', ' 4:twenty => PUSH_UNREC', ' 5: => PUSH_UNREC', @@ -955,14 +956,13 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka): " _: => POP UnrecognizedTokensNode(source='twenty ', start=4, end=5)", " _: => POP ConceptNode(concept='(1001)one', source='one', start=6, end=6)", " _: => => ERROR Too many parameters found for '(1011)a ? b : c' before token 'Token(:)'", - ' 8:: => EAT', - ], [ + ' 8:: => EAT'], [ ' 0:one => PUSH_UNREC', ' 1: => PUSH_UNREC', - ' 2:?(SyaConceptDef(concept=(1011)a ? b : c, precedence=1, associativity=right)) => ??', + ' 2:? ((1011)a ? b : c, prio=1, assoc=SyaAssociativity.Right) => ??', ' _: => RECOG [[CN((1001)one)]]', " _: => POP ConceptNode(concept='(1001)one', source='one', start=0, end=0)", - ' 2:?(SyaConceptDef(concept=(1011)a ? b : c, precedence=1, associativity=right)) => PUSH', + ' 2:? ((1011)a ? b : c, prio=1, assoc=SyaAssociativity.Right) => PUSH', ' 3: => EAT', ' 4:twenty => PUSH_UNREC', ' 5: => PUSH_UNREC', @@ -976,12 +976,12 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka): ' 11: => ??', ' _: => RECOG [[CN((1003)three)]]', " _: => POP ConceptNode(concept='(1003)three', source='three', start=10, end=10)", - ' _: => POP SyaConceptParserHelper(concept=(1011)a ? b : c, start=2, error=None)' - ]]), + ' _: => POP SyaConceptParserHelper(concept=(1011)a ? b : c, start=2, error=None)']]), ]) def test_i_can_debug(self, expression, expected_debugs): sheerka, context, parser = self.init_parser() - context.debug_enabled = True + sheerka.set_debug(context, True) + sheerka.debug_var(context, "Sya") res = parser.infix_to_postfix(context, ParserInput(expression)) assert len(res) == len(expected_debugs) @@ -989,6 +989,19 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka): actual_debug = [str(di) for di in res_i.debug] assert actual_debug == expected_debug + @pytest.mark.parametrize("settings", [ + "Sya.*.*", + "Sya.*.#0.can_pop" + ]) + def test_i_can_debug_can_pop_using_star(self, settings): + sheerka, context, parser = self.init_parser() + sheerka.set_debug(context, True) + sheerka.debug_var(context, settings) + res = parser.infix_to_postfix(context, ParserInput("one plus two mult three")) + + debug = [str(di) for di in res[0].debug] + assert debug[5] == ' _: => No stack. CAN_POP false.' + def test_i_can_parse_when_concept_atom_only(self): sheerka, context, parser = self.init_parser() @@ -999,15 +1012,20 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka): assert res.status assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT) - assert lexer_nodes == [ConceptNode(cmap["plus"], 0, 8, source=text)] + assert lexer_nodes == [CN(cmap["plus"], 0, 8, source=text)] # check the compiled expected_concept = lexer_nodes[0].concept assert expected_concept.get_compiled()["a"] == cmap["one"] - assert expected_concept.get_compiled()["b"] == cmap["mult"] + assert expected_concept.get_compiled()["b"] == CMV(cmap["mult"], a="two", b="three") assert expected_concept.get_compiled()["b"].get_compiled()["a"] == cmap["two"] assert expected_concept.get_compiled()["b"].get_compiled()["b"] == cmap["three"] + # check the metadata + expected_concept = lexer_nodes[0].concept + assert expected_concept.get_metadata().variables == [("a", "one"), ("b", "two mult three")] + assert expected_concept.get_compiled()["b"].get_metadata().variables == [("a", "two"), ("b", "three")] + def test_i_can_parse_when_python_code(self): sheerka, context, parser = self.init_parser() @@ -1018,7 +1036,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka): assert res.status assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT) - assert lexer_nodes == [ConceptNode(cmap["suffixed"], 0, 6, source=text)] + assert lexer_nodes == [CN(cmap["suffixed"], 0, 6, source=text)] # check the compiled expected_concept = lexer_nodes[0].concept @@ -1031,6 +1049,9 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka): assert return_value_a.body.source == "1 + 1" assert isinstance(return_value_a.body.body, PythonNode) + # check metadata + assert expected_concept.get_metadata().variables == [("a", "1 + 1")] + def test_i_can_parse_when_bnf_concept(self): sheerka, context, parser = self.init_parser() @@ -1043,13 +1064,16 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka): lexer_nodes = res[1].body.body assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT) - assert lexer_nodes == [ConceptNode(cmap["suffixed"], 0, 4, source=text)] + assert lexer_nodes == [CN(cmap["suffixed"], 0, 4, source=text)] # check the compiled expected_concept = lexer_nodes[0].concept assert sheerka.isinstance(expected_concept.get_compiled()["a"], "twenties") assert expected_concept.get_compiled()["a"].get_compiled()["unit"] == cmap["one"] + # check metadata + assert expected_concept.get_metadata().variables == [("a", "twenty one")] + def test_i_can_parse_sequences(self): sheerka, context, parser = self.init_parser() @@ -1061,8 +1085,8 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka): assert res.status assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT) assert lexer_nodes == [ - ConceptNode(cmap["plus"], 0, 9, source="one plus 1 + 1 "), - ConceptNode(cmap["suffixed"], 10, 12, source="suffixed two")] + CN(cmap["plus"], 0, 9, source="one plus 1 + 1 "), + CN(cmap["suffixed"], 10, 12, source="suffixed two")] # check the compiled concept_plus_a = lexer_nodes[0].concept.get_compiled()["a"] @@ -1198,7 +1222,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka): assert not res.status assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT) - assert lexer_nodes == [ConceptNode(cmap[expected_concept], 0, expected_end, source=text)] + assert lexer_nodes == [CN(cmap[expected_concept], 0, expected_end, source=text)] concept_found = lexer_nodes[0].concept for unrecognized in expected_unrecognized: @@ -1278,7 +1302,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka): ]) def test_i_can_get_functions_names_from_unrecognized(self, expression, expected): sheerka, context, parser = self.init_parser() - infix_to_postfix = InFixToPostFix(context) + infix_to_postfix = InFixToPostFix(context, NextIdManager()) tokens = list(Tokenizer(expression, yield_eof=False)) for pos, token in enumerate(tokens[:-1]): @@ -1302,7 +1326,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka): ]) def test_i_can_get_functions_names_from_unrecognized_when_multiple_results(self, expression, expected_list): sheerka, context, parser = self.init_parser() - infix_to_postfix = InFixToPostFix(context) + infix_to_postfix = InFixToPostFix(context, NextIdManager()) tokens = list(Tokenizer(expression, yield_eof=False)) for pos, token in enumerate(tokens[:-1]):