From 315f8ea09b0ca062ced0212476a5dcc05b96bbbd Mon Sep 17 00:00:00 2001 From: Kodjo Sossouvi Date: Fri, 20 Nov 2020 13:41:45 +0100 Subject: [PATCH] Added first version of DebugManager. Implemented draft of the rule engine --- .gitignore | 5 +- _concepts_admin.txt | 41 +- _concepts.txt => _concepts_full.txt | 0 _concepts_lite.txt | 1 + docs/source/README.txt | 7 + docs/source/blog/blog.rst | 2 +- docs/source/blog/format_rule.rst | 66 ++ docs/source/blog_old.rst | 2 +- main.py | 3 +- src/cache/BaseCache.py | 37 +- src/cache/CacheManager.py | 35 +- src/cache/FastCache.py | 43 ++ src/cache/SetCache.py | 6 + src/core/ast/nodes.py | 152 ----- src/core/ast/visitors.py | 128 ---- src/core/ast_helpers.py | 129 ++++ src/core/builtin_concepts.py | 592 ++++++++-------- src/core/builtin_helpers.py | 245 +++---- src/core/concept.py | 297 ++++---- src/core/error.py | 6 + src/core/global_symbols.py | 8 + src/core/profiling.py | 34 +- src/core/rule.py | 84 +++ src/core/sheerka/ExecutionContext.py | 162 +++-- src/core/sheerka/Sheerka.py | 248 ++++--- src/core/sheerka/services/SheerkaAdmin.py | 101 ++- .../services/SheerkaComparisonManager.py | 107 +-- .../services/SheerkaConceptsAlgebra.py | 43 +- .../services/SheerkaCreateNewConcept.py | 17 +- .../sheerka/services/SheerkaDebugManager.py | 440 ++++++++++++ src/core/sheerka/services/SheerkaDump.py | 45 +- .../services/SheerkaEvaluateConcept.py | 107 +-- .../sheerka/services/SheerkaEvaluateRules.py | 114 ++++ .../sheerka/services/SheerkaEventManager.py | 60 ++ src/core/sheerka/services/SheerkaExecute.py | 259 ++++--- .../sheerka/services/SheerkaHasAManager.py | 10 +- src/core/sheerka/services/SheerkaMemory.py | 55 +- .../sheerka/services/SheerkaModifyConcept.py | 20 +- src/core/sheerka/services/SheerkaOut.py | 100 +++ .../sheerka/services/SheerkaResultManager.py | 89 ++- .../sheerka/services/SheerkaRuleManager.py | 642 ++++++++++++++++++ .../sheerka/services/SheerkaSetsManager.py | 43 +- .../services/SheerkaVariableManager.py | 52 +- src/core/sheerka/services/sheerka_service.py | 11 + src/core/tokenizer.py | 124 +++- src/core/utils.py | 200 +++++- src/evaluators/BaseEvaluator.py | 12 +- src/evaluators/ConceptEvaluator.py | 4 +- ...eptEvaluator.py => DefConceptEvaluator.py} | 32 +- src/evaluators/FormatRuleEvaluator.py | 46 ++ src/evaluators/LexerNodeEvaluator.py | 4 + src/evaluators/MultipleErrorsEvaluator.py | 6 +- src/evaluators/MutipleSameSuccessEvaluator.py | 4 + src/evaluators/OneErrorEvaluator.py | 4 + src/evaluators/OneSuccessEvaluator.py | 9 +- src/evaluators/PostExecutionEvaluator.py | 30 +- src/evaluators/PrepareEvalBodyEvaluator.py | 3 + .../PrepareEvalQuestionEvaluator.py | 3 + src/evaluators/PythonEvaluator.py | 293 ++++---- src/evaluators/ResolveAmbiguityEvaluator.py | 4 + src/evaluators/RetEvaluator.py | 4 +- src/evaluators/ReturnBodyEvaluator.py | 7 +- src/evaluators/RuleEvaluator.py | 47 ++ src/evaluators/TooManySuccessEvaluator.py | 14 +- .../UpdateFunctionsParametersEvaluator.py | 19 +- src/out/ConsoleVisistor.py | 55 ++ src/out/DeveloperVisitor.py | 140 ++++ src/out/OutVisitor.py | 10 + src/{core/ast => out}/__init__.py | 0 src/parsers/BaseCustomGrammarParser.py | 5 +- src/parsers/BaseNodeParser.py | 131 +++- src/parsers/BaseParser.py | 99 +-- .../{BnfParser.py => BnfDefinitionParser.py} | 16 +- src/parsers/BnfNodeParser.py | 39 +- src/parsers/DefConceptParser.py | 4 +- src/parsers/ExactConceptParser.py | 12 +- src/parsers/FormatRuleParser.py | 44 +- src/parsers/FunctionParser.py | 14 +- src/parsers/PythonParser.py | 27 +- src/parsers/PythonWithConceptsParser.py | 24 +- src/parsers/RuleParser.py | 88 +++ ...tomNodeParser.py => SequenceNodeParser.py} | 20 +- src/parsers/SyaNodeParser.py | 41 +- src/parsers/UnrecognizedNodeParser.py | 13 +- src/printer/SheerkaPrinter.py | 13 +- src/repl/SheerkaPrompt.py | 10 +- src/sdp/readme.md | 1 + src/sdp/sheerkaDataProvider.py | 72 +- src/sdp/sheerkaSerializer.py | 9 +- src/sheerkapickle/SheerkaPickler.py | 10 +- src/sheerkapickle/SheerkaUnpickler.py | 8 +- src/sheerkapickle/sheerka_handlers.py | 87 ++- src/sheerkapickle/utils.py | 24 +- tests/BaseTest.py | 77 ++- tests/TestUsingFileBasedSheerka.py | 9 +- tests/TestUsingMemoryBasedSheerka.py | 10 +- tests/cache/test_FastCache.py | 67 ++ tests/cache/test_cache.py | 32 + tests/core/test_ExecutionContext.py | 268 +++++--- tests/core/test_SheerkaAdmin.py | 65 ++ tests/core/test_SheerkaComparisonManager.py | 238 +++++-- tests/core/test_SheerkaConceptAlgebra.py | 12 +- tests/core/test_SheerkaCreateNewConcept.py | 12 +- tests/core/test_SheerkaDebugManager.py | 425 ++++++++++++ tests/core/test_SheerkaEvaluateConcept.py | 166 ++--- tests/core/test_SheerkaEvaluateRules.py | 74 ++ tests/core/test_SheerkaEventManager.py | 66 ++ tests/core/test_SheerkaHistoryManager.py | 3 +- tests/core/test_SheerkaMemory.py | 34 +- tests/core/test_SheerkaModifyConcept.py | 33 +- tests/core/test_SheerkaRuleManager.py | 355 ++++++++++ tests/core/test_SheerkaSetsManager.py | 18 +- tests/core/test_SheerkaVariableManager.py | 51 +- tests/core/test_ast.py | 174 ----- tests/core/test_ast_helper.py | 44 ++ tests/core/test_builtin_helpers.py | 76 +-- tests/core/test_concept.py | 63 +- tests/core/test_sheerka.py | 72 +- tests/core/test_sheerkaResultManager.py | 130 +++- tests/core/test_sheerka_call_evaluators.py | 114 +++- tests/core/test_sheerka_printer.py | 86 ++- tests/core/test_tokenizer.py | 33 +- tests/core/test_utils.py | 104 ++- ...aluator.py => test_DefConceptEvaluator.py} | 96 +-- tests/evaluators/test_FormatRuleEvaluator.py | 51 ++ tests/evaluators/test_LexerNodeEvaluator.py | 4 +- tests/evaluators/test_OneSuccessEvaluator.py | 12 +- tests/evaluators/test_PythonEvaluator.py | 130 ++-- tests/evaluators/test_ReturnBodyEvaluator.py | 50 ++ tests/evaluators/test_RuleEvaluator.py | 70 ++ tests/non_reg/test_sheerka_non_reg.py | 67 +- tests/out/__init__.py | 0 tests/out/test_SheerkaOut.py | 381 +++++++++++ tests/parsers/parsers_utils.py | 12 +- tests/parsers/test_AtomsParser.py | 14 +- tests/parsers/test_BaseCustomGrammarParser.py | 20 +- tests/parsers/test_BaseNodeParser.py | 6 +- tests/parsers/test_BaseParser.py | 20 - tests/parsers/test_BnfNodeParser.py | 78 +-- tests/parsers/test_BnfParser.py | 12 +- tests/parsers/test_DefConceptParser.py | 8 +- tests/parsers/test_ExactConceptParser.py | 38 +- tests/parsers/test_ExpressionParser.py | 16 +- tests/parsers/test_FormatRuleParser.py | 74 +- tests/parsers/test_FunctionParser.py | 78 ++- tests/parsers/test_PythonParser.py | 58 +- .../parsers/test_PythonWithConceptsParser.py | 60 +- tests/parsers/test_RuleParser.py | 101 +++ tests/parsers/test_SyaNodeParser.py | 51 +- tests/parsers/test_UnrecognizedNodeParser.py | 111 +-- tests/repl/test_SheerkaPromptCompleter.py | 11 +- tests/sdp/test_sheerkaDataProvider.py | 9 +- tests/sheerkapickle/test_SheerkaPickler.py | 30 +- tests/sheerkapickle/test_sheerka_handlers.py | 67 +- utils/sheerka.rebuild.sh | 42 ++ utils/sheerka.reset.sh | 19 + 156 files changed, 8388 insertions(+), 2852 deletions(-) rename _concepts.txt => _concepts_full.txt (100%) create mode 100644 docs/source/README.txt create mode 100644 docs/source/blog/format_rule.rst create mode 100644 src/cache/FastCache.py delete mode 100644 src/core/ast/nodes.py delete mode 100644 src/core/ast/visitors.py create mode 100644 src/core/ast_helpers.py create mode 100644 src/core/error.py create mode 100644 src/core/global_symbols.py create mode 100644 src/core/rule.py create mode 100644 src/core/sheerka/services/SheerkaDebugManager.py create mode 100644 src/core/sheerka/services/SheerkaEvaluateRules.py create mode 100644 src/core/sheerka/services/SheerkaEventManager.py create mode 100644 src/core/sheerka/services/SheerkaOut.py create mode 100644 src/core/sheerka/services/SheerkaRuleManager.py rename src/evaluators/{AddConceptEvaluator.py => DefConceptEvaluator.py} (85%) create mode 100644 src/evaluators/FormatRuleEvaluator.py create mode 100644 src/evaluators/RuleEvaluator.py create mode 100644 src/out/ConsoleVisistor.py create mode 100644 src/out/DeveloperVisitor.py create mode 100644 src/out/OutVisitor.py rename src/{core/ast => out}/__init__.py (100%) rename src/parsers/{BnfParser.py => BnfDefinitionParser.py} (96%) create mode 100644 src/parsers/RuleParser.py rename src/parsers/{AtomNodeParser.py => SequenceNodeParser.py} (95%) create mode 100644 tests/cache/test_FastCache.py create mode 100644 tests/core/test_SheerkaAdmin.py create mode 100644 tests/core/test_SheerkaDebugManager.py create mode 100644 tests/core/test_SheerkaEvaluateRules.py create mode 100644 tests/core/test_SheerkaEventManager.py create mode 100644 tests/core/test_SheerkaRuleManager.py delete mode 100644 tests/core/test_ast.py create mode 100644 tests/core/test_ast_helper.py rename tests/evaluators/{test_AddConceptEvaluator.py => test_DefConceptEvaluator.py} (70%) create mode 100644 tests/evaluators/test_FormatRuleEvaluator.py create mode 100644 tests/evaluators/test_ReturnBodyEvaluator.py create mode 100644 tests/evaluators/test_RuleEvaluator.py create mode 100644 tests/out/__init__.py create mode 100644 tests/out/test_SheerkaOut.py create mode 100644 tests/parsers/test_RuleParser.py create mode 100755 utils/sheerka.rebuild.sh create mode 100755 utils/sheerka.reset.sh diff --git a/.gitignore b/.gitignore index 2be4d2d..c38cba9 100644 --- a/.gitignore +++ b/.gitignore @@ -9,4 +9,7 @@ _build prof log.txt tests/_concepts.txt -tests/**/*result_test \ No newline at end of file +tests/**/*result_test +testingPython.ipynb +profile*.txt +*.prof \ No newline at end of file diff --git a/_concepts_admin.txt b/_concepts_admin.txt index 1af4bbc..264c1d0 100644 --- a/_concepts_admin.txt +++ b/_concepts_admin.txt @@ -1,16 +1,13 @@ # admin helpers -def concept explain as get_results() | filter("id == 0") | recurse(2) +def concept explain as get_results(id=0, depth=2) set_isa(c:explain:, __AUTO_EVAL) -def concept explain last as get_last_results() | filter("id == 0") | recurse(2) +def concept explain last as get_last_results(id=0, depth=2) set_isa(c:explain last:, __AUTO_EVAL) -def concept explain x as get_results() | filter(f"id == {x}") | recurse(3) where isinstance(x, int) +def concept explain x as get_results(id=x, depth=3) set_isa(c:explain x:, __AUTO_EVAL) -def concept explain x values where isinstance(x, int) as get_results() | filter(f"id=={x}") | format_d -set_isa(c:explain x values:, __AUTO_EVAL) - def concept precedence a > precedence b as set_is_greater_than(__PRECEDENCE, a, b) set_isa(c:precedence a > precedence b:, __AUTO_EVAL) @@ -28,11 +25,41 @@ set_auto_eval(c:x is a y:) def concept x is an y as set_isa(x, y) set_auto_eval(c:x is an y:) def concept x is a y as isa(x,y) pre is_question() +# no need to auto eval as it's a question def concept x is an y as isa(x,y) pre is_question() - +# no need to auto eval as it's a question def concept x has a y as set_hasa(x, y) set_auto_eval(c:x has a y:) def concept x has an y as set_hasa(x, y) set_auto_eval(c:x has an y:) def concept x has a y as hasa(x,y) pre is_question() +# no need to auto eval as it's a question def concept x has an y as hasa(x,y) pre is_question() +# no need to auto eval as it's a question + +def concept activate debug as set_debug(True) +set_auto_eval(c:activate debug:) +def concept deactivate debug as set_debug(False) +set_auto_eval(c:deactivate debug:) + +def concept activate debug on x as debug_var(x) +set_auto_eval(c:activate debug on x:) +def concept debug x as debug_var(x) +set_auto_eval(c:debug x:) + +def concept debug var x as debug_var(variable=x) +set_auto_eval(c:debug var x:) +def concept debug variable x as debug_var(variable=x) +set_auto_eval(c:debug variable x:) +def concept debug method x as debug_var(method=x) +set_auto_eval(c:debug method x:) + +set_auto_eval(c:activate debug on x:) +def concept deactivate debug on x as debug_var(x, enabled=False) where x +set_auto_eval(c:deactivate debug on x:) + +def concept activate return values processing as set_var("sheerka.enable_process_return_values", True) +def concept deactivate return values processing as set_var("sheerka.enable_process_return_values", False) +set_auto_eval(c:activate return values processing:) +set_auto_eval(c:deactivate return values processing:) + diff --git a/_concepts.txt b/_concepts_full.txt similarity index 100% rename from _concepts.txt rename to _concepts_full.txt diff --git a/_concepts_lite.txt b/_concepts_lite.txt index 2de29a9..2c4ba3f 100644 --- a/_concepts_lite.txt +++ b/_concepts_lite.txt @@ -6,4 +6,5 @@ def concept apple def concept table def concept location def concept x is on y as set_attr(x, location, y) +activate return values processing diff --git a/docs/source/README.txt b/docs/source/README.txt new file mode 100644 index 0000000..49d2ccc --- /dev/null +++ b/docs/source/README.txt @@ -0,0 +1,7 @@ +The documentation is build using Sphinx +I prefer using VSCode for the documentation +install the following extensions: +* LiveServer (from Ritwick Dey) +* reStructuredText (from Lextudio Inc) +doc page for reStructuredText : https://docs.restructuredtext.net/ +shortcuts : https://docs.restructuredtext.net/articles/shortcuts.html \ No newline at end of file diff --git a/docs/source/blog/blog.rst b/docs/source/blog/blog.rst index ba9a474..1570160 100644 --- a/docs/source/blog/blog.rst +++ b/docs/source/blog/blog.rst @@ -661,7 +661,7 @@ Some example : and so on... -So when a concept is defined using its bnf definition, I use the **BnfParser** to create the grammar, and then +So when a concept is defined using its bnf definition, I use the **BnfDefinitionParser** to create the grammar, and then I use the **BnfNodeParser** to recognize the concepts The current implementation to recognize a concept is not very efficient. All the definitions are in a dictionary diff --git a/docs/source/blog/format_rule.rst b/docs/source/blog/format_rule.rst new file mode 100644 index 0000000..91fceda --- /dev/null +++ b/docs/source/blog/format_rule.rst @@ -0,0 +1,66 @@ +Format Rules +============ + + +Basic definition +**************** + +Format rules are rules that define how the result of the execution must be printed. +I guess that it will be ultimately be replaced / transformed into the program that can speak + + +:: + + > when print + +where: + + predicate is a valid sheerka syntax + + action is a custom language defined below + + +Action +******* + +:: + + ACTION := FUNCTION | VARIABLE | RAW + FUNCTION := IDENT \( PARAM* (, PARAM)* \) + PARAM := + VARIABLE := \{PREFIX:IDENT \} + PREFIX := + RAW := + +Some examples + +:: + + blue(digest) : {command}\nlist(body) + + +Predicate +********* +The predicate indicates the conditions to trigger the print / formatting +You can use the whole Sheerka language (methods and Concepts) + +In addition to the regular function, you have also access to local variables + ++-----------------+-------------------------------------------------------------------------------------------------------------------------+ +| Name | Definition | ++=================+=========================================================================================================================+ +| __obj | The current object being processed | ++-----------------+-------------------------------------------------------------------------------------------------------------------------+ +| __ret | current object if it is a ``ReturnValueConcept`` | ++-----------------+-------------------------------------------------------------------------------------------------------------------------+ +| __ret_value | | +| | * current value's body if the current value is a ``ReturnValueConcept`` | +| | * current value's body's body if the current value is a ``ReturnValueConcept`` and the body is a container concept | +| | * current value otherwise | ++-----------------+-------------------------------------------------------------------------------------------------------------------------+ +| __ret_container | When the current value is a ``ReturnValueConcept`` and its body is a container concept such as | +| | * ``ParserResultConcept`` | +| | * ``ErrorConcept`` | +| | * ``OnlySuccessfulConcept`` | +| | * ... | ++-----------------+-------------------------------------------------------------------------------------------------------------------------+ diff --git a/docs/source/blog_old.rst b/docs/source/blog_old.rst index d544e3f..8239bf9 100644 --- a/docs/source/blog_old.rst +++ b/docs/source/blog_old.rst @@ -652,7 +652,7 @@ Some example : and so on... -So when a concept is defined using its bnf definition, I use the **BnfParser** to create the grammar, and then +So when a concept is defined using its bnf definition, I use the **BnfDefinitionParser** to create the grammar, and then I use the **BnfNodeParser** to recognize the concepts The current implementation to recognize a concept is not very efficient. All the definitions are in a dictionary diff --git a/main.py b/main.py index 56dc41d..864f205 100644 --- a/main.py +++ b/main.py @@ -43,7 +43,8 @@ def main(argv): else: _in = core.utils.sysarg_to_string(args) result = sheerka.evaluate_user_input(_in) - sheerka.print(result) + if not sheerka.enable_process_return_values: + sheerka.print(result) return result[-1].status if len(result) > 0 else 1 diff --git a/src/cache/BaseCache.py b/src/cache/BaseCache.py index 254dcd4..152e853 100644 --- a/src/cache/BaseCache.py +++ b/src/cache/BaseCache.py @@ -86,6 +86,15 @@ class BaseCache: with self._lock: return self._get(key) + def get_all(self): + """ + Retrieve all items already in cache + This method does not fetch in the remoter repository + :return: + """ + with self._lock: + return self._cache.values() + def inner_get(self, key): return self._cache[key] @@ -108,6 +117,17 @@ class BaseCache: except KeyError: pass + def populate(self, populate_function, get_key_function): + """ + Initialise the cache with a bunch of data + :param populate_function: + :param get_key_function: + :return: + """ + with self._lock: + for item in populate_function(): + self.put(get_key_function(item), item) + def has(self, key): """ Return True if the key is in the cache @@ -194,7 +214,7 @@ class BaseCache: with self._lock: return self._cache.copy() - def init_from(self, dump): + def init_from_dump(self, dump): with self._lock: self._current_size = dump["current_size"] self._cache = dump["cache"].copy() @@ -259,18 +279,3 @@ class BaseCache: def _delete(self, key, value): raise NotImplementedError() - - # def _put(self, key, value): - # self._cache[key] = value - # self._add_to_add(key) - # return True - # - - # - # def _update(self, old_key, old_value, new_key, new_value): - # self._cache[new_key] = new_value - # self._add_to_add(new_key) - # - # if new_key != old_key: - # del (self._cache[old_key]) - # self._add_to_remove(old_key) diff --git a/src/cache/CacheManager.py b/src/cache/CacheManager.py index 7e9cab9..3356634 100644 --- a/src/cache/CacheManager.py +++ b/src/cache/CacheManager.py @@ -2,7 +2,7 @@ from dataclasses import dataclass, field from threading import RLock from typing import Callable -from cache.Cache import Cache +from cache.BaseCache import BaseCache from core.concept import Concept @@ -18,7 +18,7 @@ class MultipleEntryError(Exception): @dataclass class CacheDefinition: - cache: Cache + cache: BaseCache use_ref: bool get_key: Callable[[Concept], str] = field(repr=False) persist: bool = True @@ -137,6 +137,15 @@ class CacheManager: with self._lock: return self.caches[cache_name].cache.get(key) + def get_cache(self, cache_name): + """ + Return the BaseCache object + :param cache_name: + :return: + """ + with self._lock: + return self.caches[cache_name].cache + def copy(self, cache_name): """ get a copy the content of the whole cache as a dictionary @@ -170,6 +179,17 @@ class CacheManager: self.caches[cache_name].cache.delete(key, value) self.is_dirty = True + def populate(self, cache_name, populate_function, get_key_function): + """ + Populate a specific cache with a bunch of items + :param cache_name: + :param populate_function: how to get the items + :param get_key_function: how to get the key, out of an item + :return: + """ + with self._lock: + self.caches[cache_name].cache.init(populate_function, get_key_function) + def has(self, cache_name, key): """ True if the value is in cache only. Never try to look in a remote repository @@ -210,7 +230,7 @@ class CacheManager: for values in items.values(): update_full_serialisation(values, value) elif isinstance(items, Concept): - items.metadata.full_serialization = value + items.get_metadata().full_serialization = value if self.cache_only: return @@ -245,6 +265,10 @@ class CacheManager: cache_def.cache.clear() def dump(self): + """ + For test purpose, dumps the whole content of the cache manager + :return: + """ with self._lock: res = {} for cache_name, cache_def in self.caches.items(): @@ -252,11 +276,11 @@ class CacheManager: return res - def init_from(self, dump): + def init_from_dump(self, dump): with self._lock: for cache_name, content in dump.items(): if cache_name in self.caches: - self.caches[cache_name].cache.init_from(content) + self.caches[cache_name].cache.init_from_dump(content) return self @@ -267,4 +291,3 @@ class CacheManager: self.caches.clear() self.concept_caches.clear() self.is_dirty = False - diff --git a/src/cache/FastCache.py b/src/cache/FastCache.py new file mode 100644 index 0000000..f370c32 --- /dev/null +++ b/src/cache/FastCache.py @@ -0,0 +1,43 @@ +class FastCache: + """ + Simplest LRU cache + """ + + def __init__(self, max_size=256): + self.max_size = max_size + self.cache = {} + self.lru = [] + + def put(self, key, value): + if len(self.cache) == self.max_size: + del self.cache[self.lru.pop(0)] + + if key in self.cache: + self.lru.remove(key) + + self.cache[key] = value + self.lru.append(key) + + def get(self, key): + try: + return self.cache[key] + except KeyError: + return None + + def evict_by_key(self, predicate): + to_remove = [] + + for k, v in self.cache.items(): + if predicate(k): + to_remove.append(k) + + for k in to_remove: + self.lru.remove(k) + del self.cache[k] + + def copy(self): + return self.cache.copy() + + def clear(self): + self.cache.clear() + self.lru.clear() diff --git a/src/cache/SetCache.py b/src/cache/SetCache.py index 3e3ba28..95071c1 100644 --- a/src/cache/SetCache.py +++ b/src/cache/SetCache.py @@ -5,8 +5,14 @@ class SetCache(BaseCache): """ An in memory FIFO cache object When the max_size is reach the first element that was put is removed + You can use the same key multiple times, but the elements under this key will be unique When there are multiple elements, a python set is used + + >> self.put('key', 'value1') + >> assert {'value1'} == self.get('key') + >> self.put('key', 'value2') + >> assert {'value1', 'value2'} == self.get('key') """ def _put(self, key, value): diff --git a/src/core/ast/nodes.py b/src/core/ast/nodes.py deleted file mode 100644 index 71846fd..0000000 --- a/src/core/ast/nodes.py +++ /dev/null @@ -1,152 +0,0 @@ -from core.builtin_concepts import BuiltinConcepts, ListConcept -from core.concept import Concept, ConceptParts -import ast -import core.utils - -import logging - -log = logging.getLogger(__name__) - - -class NodeParent: - """ - Class that represent the ancestor of a Node - For example, the 'For' nodes has three fields (target, iter and body) - So, for a node under For.iter - node -> For - field -> iter - """ - - def __init__(self, node, field): - self.node = node - self.field = field - - def __repr__(self): - if self.node is None: - return None - - if self.field is None: - return self.node.get_node_type() - - return self.node.get_node_type() + "." + self.field - - def __eq__(self, other): - # I can compare with type for simplification - if isinstance(other, tuple): - return self.node.get_node_type() == other[0] and self.field == other[1] - - # normal equals implementation - if not isinstance(other, NodeParent): - return False - - return self.node.get_node_type() == other.node.get_node_type() and self.field == other.field - - def __hash__(self): - return hash((self.node.get_node_type(), self.field)) - - -class NodeConcept(Concept): - def __init__(self, key, node_type, parent: NodeParent): - super().__init__(key, True, False, key) - self.parent = parent - self.node_type = node_type - - def get_node_type(self): - return self.node_type - - -class GenericNodeConcept(NodeConcept): - def __init__(self, node_type, parent): - super().__init__(BuiltinConcepts.GENERIC_NODE, node_type, parent) - - def __repr__(self): - return "Generic:" + self.node_type - - def get_node_type(self): - return self.node_type - - def get_obj_value(self): - if self.node_type == "Name": - return self.get_value("id") - - if self.node_type == "arg": - return self.get_value("arg") - - return self.body - - -class IdentifierNodeConcept(NodeConcept): - def __init__(self, parent, name): - super().__init__(BuiltinConcepts.IDENTIFIER_NODE, "Name", parent) - self.set_value(ConceptParts.BODY, name) - - -class CallNodeConcept(NodeConcept): - def __init__(self, parent=None): - super().__init__(BuiltinConcepts.IDENTIFIER_NODE, "Call", parent) - - def get_args_names(self, sheerka): - return sheerka.objvalues(self.get_value("args")) - - -def python_to_concept(python_node): - """ - Transform Python AST node into concept nodes - for better usage - :param python_node: - :return: - """ - - def _transform(node, parent): - node_type = node.__class__.__name__ - concept = GenericNodeConcept(node_type, parent).init_key() - for field in node._fields: - if not hasattr(node, field): - continue - - value = getattr(node, field) - concept.def_var(field) - if isinstance(value, list): - lst = ListConcept().init_key() - for i in value: - lst.append(_transform(i, NodeParent(concept, field))) - concept.set_value(field, lst) - elif isinstance(value, ast.AST): - concept.set_value(field, _transform(value, NodeParent(concept, field))) - else: - concept.set_value(field, value) - - concept.metadata.is_evaluated = True - return concept - - return _transform(python_node, None) - - -def concept_to_python(concept_node): - """ - Transform back concept_node to Python AST node - :param concept_node: - :return: - """ - - def _transform(node): - node_type = node.get_node_type() - ast_object = core.utils.new_object("_ast." + node_type) - for field in node.values: - if field not in ast_object._fields: - continue - - value = node.get_value(field) - if isinstance(value, list) or isinstance(value, Concept) and value.key == str(BuiltinConcepts.LIST): - lst = [] - for i in value.body: - lst.append(_transform(i)) - setattr(ast_object, field, lst) - elif isinstance(value, NodeConcept): - setattr(ast_object, field, _transform(value)) - else: - setattr(ast_object, field, value) - return ast_object - - res = _transform(concept_node) - return res diff --git a/src/core/ast/visitors.py b/src/core/ast/visitors.py deleted file mode 100644 index 2a94f42..0000000 --- a/src/core/ast/visitors.py +++ /dev/null @@ -1,128 +0,0 @@ -from core.ast.nodes import GenericNodeConcept, NodeConcept -from core.builtin_concepts import ListConcept - - -class ConceptNodeVisitor: - """ - Base class to visit NodeConcept - It is insolently inspired by python AST.Visitor class - """ - - def visit(self, node): - - """Visit a node.""" - name = node.node_type if isinstance(node, GenericNodeConcept) else node.name - name = str(name).capitalize() - - method = 'visit_' + name - visitor = getattr(self, method, self.generic_visit) - return visitor(node) - - def generic_visit(self, node): - """Called if no explicit visitor function exists for a node.""" - for field, value in iter_props(node): - if isinstance(value, ListConcept): - for item in value.body: - if isinstance(item, NodeConcept): - self.visit(item) - elif isinstance(value, NodeConcept): - self.visit(value) - - def visit_Constant(self, node): - value = node.get_value("value") - type_name = _const_node_type_names.get(type(value)) - if type_name is None: - for cls, name in _const_node_type_names.items(): - if isinstance(value, cls): - type_name = name - break - if type_name is not None: - method = 'visit_' + type_name - try: - visitor = getattr(self, method) - except AttributeError: - pass - else: - import warnings - warnings.warn(f"{method} is deprecated; add visit_Constant", - PendingDeprecationWarning, 2) - return visitor(node) - return self.generic_visit(node) - - -class UnreferencedNamesVisitor(ConceptNodeVisitor): - def __init__(self, sheerka): - self.names = set() - self.sheerka = sheerka - - def visit_Name(self, node): - parents = get_parents(node) - if ("For", "target") in parents: # variable used by the 'for' iteration - return - - if ("Call", "func") in parents: # name of the function - return - - # if ("Assign", "targets") in parents: # variable which is assigned - # return - - if self.can_be_discarded(self.sheerka.objvalue(node), parents): - return - - self.names.add(self.sheerka.objvalue(node)) - - def can_be_discarded(self, variable_name, parents): - - for node in (parent.node for parent in parents): - if node is None: - return False - - if node.get_node_type() == "For" and self.sheerka.objvalue(node.get_value("target")) == variable_name: - # variable used by the loop - return True - - if node.get_node_type() == "FunctionDef": - # variable defined as a function parameter - args = node.get_value("args") - args_values = list(self.sheerka.objvalues(args.get_value("args"))) - if variable_name in args_values: - return True - - return False - - -class ExtractPredicateVisitor(ConceptNodeVisitor): - def __init__(self, variable_name): - self.predicates = [] - self.variable_name = variable_name - - -def get_parents(node): - if node.parent is None: - return [] - - res = [] - while True: - if node.parent is None: - break - res.append(node.parent) - node = node.parent.node - - return res - - -def iter_props(node): - for p in [p for p in node.values if isinstance(p, str)]: - yield p, node.get_value(p) - - -_const_node_type_names = { - bool: 'NameConstant', # should be before int - type(None): 'NameConstant', - int: 'Num', - float: 'Num', - complex: 'Num', - str: 'Str', - bytes: 'Bytes', - type(...): 'Ellipsis', -} diff --git a/src/core/ast_helpers.py b/src/core/ast_helpers.py new file mode 100644 index 0000000..36623fe --- /dev/null +++ b/src/core/ast_helpers.py @@ -0,0 +1,129 @@ +import ast +from dataclasses import dataclass + +from cache.FastCache import FastCache + + +@dataclass +class PropDef: + """ + Helper class when parsing simple expression + """ + prop: str # name of the property + index: object # indexing if any + + +def ast_to_props(res, _ast, _index): + """ + + :param res: list where to put the results + :param _ast: ast to parse (must start by an ast.Attribute, ast.Name or ast.subScript) + :param _index: index in the array/dictionary or None + :return: + """ + + def get_index(_slice): + if not isinstance(_slice, ast.Index): + raise NotImplementedError(f"ast_to_prop: {_slice}") + + if isinstance(_slice.value, ast.Name): + return _slice.value.id + elif isinstance(_slice.value, ast.Constant): + return _slice.value.value + else: + raise NotImplementedError(f"ast_to_prop: {_slice.value}") + + if isinstance(_ast, ast.Attribute): + res.append(PropDef(_ast.attr, _index)) + ast_to_props(res, _ast.value, None) + elif isinstance(_ast, ast.Name): + res.append(PropDef(_ast.id, _index)) + elif isinstance(_ast, ast.Subscript): + index = get_index(_ast.slice) + ast_to_props(res, _ast.value, index) + + +class UnreferencedNamesVisitor(ast.NodeVisitor): + """ + Try to find variables (names) that will be requested by the ast + """ + + cache = FastCache() + + def __init__(self, context): + self.context = context + self.names = set() + + def get_names(self, node): + names = UnreferencedNamesVisitor.cache.get(node) + if names is not None: + return names + + self.visit(node) + UnreferencedNamesVisitor.cache.put(node, self.names) + return self.names + + def visit_Name(self, node): + self.names.add(node.id) + + def visit_For(self, node: ast.For): + self.visit_selected(node, ["body", "orelse"]) + + def visit_selected(self, node, to_visit): + """Called if no explicit visitor function exists for a node.""" + for field in to_visit: + value = getattr(node, field) + if isinstance(value, list): + for item in value: + if isinstance(item, ast.AST): + self.visit(item) + elif isinstance(value, ast.AST): + self.visit(value) + + +class UnreferencedVariablesVisitor(UnreferencedNamesVisitor): + """ + Try to find variables (names) that will be requested by the ast + """ + + def visit_Call(self, node: ast.Call): + self.visit_selected(node, ["args"]) + + +class NamesWithAttributesVisitor(ast.NodeVisitor): + + def __init__(self): + self.sequences = [] + self.temp = [] + self.to_lookup = None + + def get_sequences(self, ast_, to_lookup): + self.to_lookup = to_lookup + self.visit(ast_) + return self.sequences + + def visit_Attribute(self, node: ast.Attribute): + self.temp.append(node.attr) + if isinstance(node.value, ast.Attribute): + self.visit_Attribute(node.value) + if isinstance(node.value, ast.Subscript): + self.visit_Subscript(node.value) + elif isinstance(node.value, ast.Name): + self.visit_Name(node.value) + + def visit_Subscript(self, node: ast.Subscript): + # TODO manage the index when it will be needed + # using node.slice + if isinstance(node.value, ast.Attribute): + self.visit_Attribute(node.value) + if isinstance(node.value, ast.Subscript): + self.visit_Subscript(node.value) + elif isinstance(node.value, ast.Name): + self.visit_Name(node.value) + + def visit_Name(self, node: ast.Name): + if node.id == self.to_lookup: + self.temp.append(node.id) + self.temp.reverse() + self.sequences.append(self.temp.copy()) + self.temp.clear() diff --git a/src/core/builtin_concepts.py b/src/core/builtin_concepts.py index e363c45..8d2e4e5 100644 --- a/src/core/builtin_concepts.py +++ b/src/core/builtin_concepts.py @@ -1,9 +1,8 @@ -from enum import Enum - from core.concept import Concept, ConceptParts +from core.error import ErrorObj -class BuiltinConcepts(Enum): +class BuiltinConcepts: """ List of builtin concepts that do no need any specific implementation Please note that the value of the enum is informal. It is not used in the system @@ -13,116 +12,102 @@ class BuiltinConcepts(Enum): The values of the enum is not used the code """ - SHEERKA = "sheerka" + SHEERKA = "__SHEERKA" # processing instructions during sheerka.execute() or sheerka.evaluate_concept() - # The instruction may alter how the actions work - DEBUG = "debug" # activate all debug information - EVAL_BODY_REQUESTED = "eval body" # to evaluate the body - EVAL_WHERE_REQUESTED = "eval where" # to evaluate the where clause - RETURN_BODY_REQUESTED = "return body" # returns the body of the concept instead of the concept itself - REDUCE_REQUESTED = "reduce" # remove meaningless error when possible - EVAL_UNTIL_SUCCESS_REQUESTED = "eval until success" # PythonEvaluator tries combination until True is found - EVAL_QUESTION_REQUESTED = "question" # the user input must be treated as question + # The instructions may alter how the actions work + DEBUG = "__DEBUG" # activate all debug information + EVAL_BODY_REQUESTED = "__EVAL_BODY_REQUESTED" # to evaluate the body + EVAL_WHERE_REQUESTED = "__EVAL_WHERE_REQUESTED" # to evaluate the where clause + RETURN_BODY_REQUESTED = "__RETURN_BODY_REQUESTED" # returns the body of the concept instead of the concept itself + REDUCE_REQUESTED = "__REDUCE_REQUESTED" # remove meaningless error when possible + EVAL_UNTIL_SUCCESS_REQUESTED = "__EVAL_UNTIL_SUCCESS_REQUESTED" # PythonEvaluator tries combination until True is found + EVAL_QUESTION_REQUESTED = "__EVAL_QUESTION_REQUESTED" # the user input must be treated as question - # possible actions during sheerka.execute() - INIT_SHEERKA = "init sheerka" # - PROCESS_INPUT = "process input" # Processing user input or other input - PROCESSING = "processing input" # Processing user input or other input - BEFORE_PARSING = "before parsing" # activated before evaluation by the parsers - PARSING = "parsing" # activated during the parsing. It contains the text to parse - AFTER_PARSING = "after parsing" # after parsing - BEFORE_EVALUATION = "before evaluation" # before evaluation - EVALUATION = "evaluation" # activated when the parsing process seems to be finished - AFTER_EVALUATION = "after evaluation" # activated when the parsing process seems to be finished - BEFORE_RENDERING = "before rendering" # activate before the output is rendered - RENDERING = "rendering" # rendering the response from sheerka - AFTER_RENDERING = "after rendering" # rendering the response from sheerka - EVALUATE_SOURCE = "evaluate source" # - EVALUATE_CONCEPT = "evaluate concept" # a concept will be evaluated - EVALUATING_CONCEPT = "evaluating concept" # a concept will be evaluated - EVALUATING_ATTRIBUTE = "evaluating concept attribute" # - VALIDATE_CONCEPT = "validate concept" - VALIDATING_CONCEPT = "validating concept" - INIT_COMPILED = "initializing concept compiled" - INIT_BNF = "initialize bnf" - MANAGE_INFINITE_RECURSION = "manage infinite recursion" - PARSE_CODE = "execute source code" - EXEC_CODE = "execute source code" # to use when executing Python or other language compiled code - TESTING = "testing" + # possible actions during sheerka.execute() or sheerka.evaluate_rules() + INIT_SHEERKA = "__INIT_SHEERKA" # + PROCESS_INPUT = "__PROCESS_INPUT" # Processing user input or other input + PROCESSING = "__PROCESSING" # Processing user input or other input + BEFORE_PARSING = "__BEFORE_PARSING" # activated before evaluation by the parsers + PARSING = "__PARSING" # activated during the parsing. It contains the text to parse + AFTER_PARSING = "__AFTER_PARSING" # after parsing + BEFORE_EVALUATION = "__BEFORE_EVALUATION" # before evaluation + EVALUATION = "__EVALUATION" # activated when the parsing process seems to be finished + AFTER_EVALUATION = "__AFTER_EVALUATION" # activated when the parsing process seems to be finished + BEFORE_RENDERING = "__BEFORE_RENDERING" # activate before the output is rendered + RENDERING = "__RENDERING" # rendering the response from sheerka + AFTER_RENDERING = "__AFTER_RENDERING" # rendering the response from sheerka + EVALUATE_SOURCE = "__EVALUATE_SOURCE" # + EVALUATE_CONCEPT = "__EVALUATE_CONCEPT" # a concept will be evaluated + EVALUATING_CONCEPT = "__EVALUATING_CONCEPT" # a concept will be evaluated + EVALUATING_ATTRIBUTE = "__EVALUATING_ATTRIBUTE" # + VALIDATE_CONCEPT = "__VALIDATE_CONCEPT" + VALIDATING_CONCEPT = "__VALIDATING_CONCEPT" + INIT_COMPILED = "__INIT_COMPILED" + INIT_BNF = "__INIT_BNF" + MANAGE_INFINITE_RECURSION = "__MANAGE_INFINITE_RECURSION" + PARSE_CODE = "__PARSE_CODE" + EXEC_CODE = "__EXEC_CODE" # to use when executing Python or other language compiled code + TESTING = "__TESTING" + EVALUATOR_PRE_PROCESS = "__EVALUATOR_PRE_PROCESS" # used modify / tweak behaviour of evaluators + EVALUATING_RULES = "__EVALUATING_RULES" # builtin attributes - ISA = "is a" # when a concept is an instance of another one - HASA = "has a" # when a concept has/owns another concept - AUTO_EVAL = "auto eval" # when the concept must be auto evaluated + ISA = "__ISA" # when a concept is an instance of another one + HASA = "__HASA" # when a concept has/owns another concept + AUTO_EVAL = "__AUTO_EVAL" # when the concept must be auto evaluated # object - USER_INPUT = "user input concept" # represent an input from an user - SUCCESS = "success concept" - ERROR = "error concept" - UNKNOWN_CONCEPT = "unknown concept" # the request concept is not recognized - CANNOT_RESOLVE_CONCEPT = "cannot resolve concept" # when too many concepts with the same name - RETURN_VALUE = "return value concept" # a value is returned - CONCEPT_TOO_LONG = "concept too long concept" # concept cannot be processed by exactConcept parser - NEW_CONCEPT = "new concept" # when a new concept is added - UNKNOWN_PROPERTY = "unknown property" # when requesting for a unknown property - PARSER_RESULT = "parser result" - TOO_MANY_SUCCESS = "too many success" # when expecting a limited number of successful return value - TOO_MANY_ERRORS = "too many errors" # when expecting a limited number of successful return value - ONLY_SUCCESSFUL = "only successful" # filter the result, only keep successful ones - MULTIPLE_ERRORS = "multiple errors" # filter the result, only keep evaluator in error - NOT_FOR_ME = "not for me" # a parser recognize that the entry is not meant for it - IS_EMPTY = "is empty" # when a set is empty - NO_RESULT = "no result" # no return value returned - INVALID_RETURN_VALUE = "invalid return value" # the return value of an evaluator is not correct - CONCEPT_ALREADY_DEFINED = "concept already defined" # when you try to add the same object twice (a concept or whatever) - PROPERTY_ALREADY_DEFINED = "property already defined" # When you try to add the same element in a property - NOP = "no operation" # no operation concept. Does nothing - CONCEPT_EVAL_ERROR = "concept evaluation error" # cannot evaluate a property or metadata of a concept - ENUMERATION = "enum" # represents a list or a set - LIST = "list" # represents a list - FILTERED = "filtered" # represents the result of a filtering - CONCEPT_ALREADY_IN_SET = "concept already in set" - EVALUATOR_PRE_PROCESS = "evaluator pre process" # used modify / tweak behaviour of evaluators - NOT_A_SET = "not a set" # the concept has no entry in sets - CONDITION_FAILED = "where clause failed" # failed to validate where clause during evaluation - CHICKEN_AND_EGG = "chicken and egg" # infinite recursion when declaring concept - EXPLANATION = "explanation" - PRECEDENCE = "precedence" # use to set priority among concepts when parsing - ASSOCIATIVITY = "associativity" # use to set priority among concepts when parsing - NOT_INITIALIZED = "not initialized" - NOT_FOUND = "not found" # when the wanted resource is not found - FORMAT_INSTRUCTIONS = "format instructions" # to express how to print the concept - NOT_IMPLEMENTED = "not implemented" # instead of raise an error - PYTHON_SECURITY_ERROR = "security error" # when trying to execute statement when only expression is allowed - INVALID_LESSER_OPERATION = "Invalid lesser operation" - INVALID_GREATEST_OPERATION = "Invalid greatest operation" + USER_INPUT = "__USER_INPUT" # represent an input from an user + SUCCESS = "__SUCCESS" + ERROR = "__ERROR" + UNKNOWN_CONCEPT = "__UNKNOWN_CONCEPT" # the request concept is not recognized + CANNOT_RESOLVE_CONCEPT = "__CANNOT_RESOLVE_CONCEPT" # when too many concepts with the same name + RETURN_VALUE = "__RETURN_VALUE" # a value is returned + CONCEPT_TOO_LONG = "__CONCEPT_TOO_LONG" # concept cannot be processed by exactConcept parser + NEW_CONCEPT = "__NEW_CONCEPT" # when a new concept is added + UNKNOWN_PROPERTY = "__UNKNOWN_PROPERTY" # when requesting for a unknown property + PARSER_RESULT = "__PARSER_RESULT" + TOO_MANY_SUCCESS = "__TOO_MANY_SUCCESS" # when expecting a limited number of successful return value + TOO_MANY_ERRORS = "__TOO_MANY_ERRORS" # when expecting a limited number of successful return value + ONLY_SUCCESSFUL = "__ONLY_SUCCESSFUL" # filter the result, only keep successful ones + MULTIPLE_ERRORS = "__MULTIPLE_ERRORS" # filter the result, only keep evaluator in error + NOT_FOR_ME = "__NOT_FOR_ME" # a parser recognize that the entry is not meant for it + IS_EMPTY = "__IS_EMPTY" # when a set is empty + NO_RESULT = "__NO_RESULT" # no return value returned + INVALID_RETURN_VALUE = "__INVALID_RETURN_VALUE" # the return value of an evaluator is not correct + CONCEPT_ALREADY_DEFINED = "__CONCEPT_ALREADY_DEFINED" # when you try to add the same object twice (a concept or whatever) + PROPERTY_ALREADY_DEFINED = "__PROPERTY_ALREADY_DEFINED" # When you try to add the same element in a property + NOP = "__NOP" # no operation concept. Does nothing + CONCEPT_EVAL_ERROR = "__CONCEPT_EVAL_ERROR" # cannot evaluate a property or metadata of a concept + ENUMERATION = "__ENUMERATION" # represents a list or a set + LIST = "__LIST" # represents a list + FILTERED = "__FILTERED" # represents the result of a filtering + CONCEPT_ALREADY_IN_SET = "__CONCEPT_ALREADY_IN_SET" + NOT_A_SET = "__NOT_A_SET" # the concept has no entry in sets + CONDITION_FAILED = "__CONDITION_FAILED" # failed to validate where clause during evaluation + CHICKEN_AND_EGG = "__CHICKEN_AND_EGG" # infinite recursion when declaring concept + EXPLANATION = "__EXPLANATION" + PRECEDENCE = "__PRECEDENCE" # use to set priority among concepts when parsing + ASSOCIATIVITY = "__ASSOCIATIVITY" # use to set priority among concepts when parsing + NOT_FOUND = "__NOT_FOUND" # when the wanted resource is not found + FORMAT_INSTRUCTIONS = "__FORMAT_INSTRUCTIONS" # to express how to print the concept + NOT_IMPLEMENTED = "__NOT_IMPLEMENTED" # instead of raise an error + PYTHON_SECURITY_ERROR = "__PYTHON_SECURITY_ERROR" # when trying to execute statement when only expression is allowed + INVALID_LESSER_OPERATION = "__INVALID_LESSER_OPERATION" + INVALID_GREATEST_OPERATION = "__INVALID_GREATEST_OPERATION" + NEW_RULE = "__NEW_RULE" + UNKNOWN_RULE = "__UNKNOWN_RULE" - NODE = "node" - GENERIC_NODE = "generic node" - IDENTIFIER_NODE = "identifier node" + NODE = "__NODE" + GENERIC_NODE = "__GENERIC_NODE" + IDENTIFIER_NODE = "__IDENTIFIER_NODE" - def __repr__(self): - return "__" + self.name + # formatting + TO_LIST = "__TO_LIST" - def __str__(self): - return "__" + self.name - - def __eq__(self, other): - if id(self) == id(other): - return True - - if isinstance(other, str): - return str(self) == other - - if not isinstance(other, BuiltinConcepts): - return False - - return self.value == other.value - - def __hash__(self): - return hash(self.value) +AllBuiltinConcepts = [v for n, v in BuiltinConcepts.__dict__.items() if not n.startswith("__")] BuiltinUnique = [ BuiltinConcepts.EVAL_BODY_REQUESTED, @@ -163,7 +148,7 @@ BuiltinUnique = [ BuiltinConcepts.INVALID_GREATEST_OPERATION, ] -BuiltinErrors = [str(e) for e in { +BuiltinErrors = [ BuiltinConcepts.ERROR, BuiltinConcepts.UNKNOWN_CONCEPT, BuiltinConcepts.CANNOT_RESOLVE_CONCEPT, @@ -180,12 +165,18 @@ BuiltinErrors = [str(e) for e in { BuiltinConcepts.NOT_A_SET, BuiltinConcepts.CONDITION_FAILED, BuiltinConcepts.CHICKEN_AND_EGG, - BuiltinConcepts.NOT_INITIALIZED, BuiltinConcepts.NOT_FOUND, BuiltinConcepts.INVALID_LESSER_OPERATION, BuiltinConcepts.INVALID_GREATEST_OPERATION, - # DO NOT PUT NOT_INITIALIZED. It's not an error -}] +] + +BuiltinContainers = [ + BuiltinConcepts.PARSER_RESULT, + BuiltinConcepts.ONLY_SUCCESSFUL, + BuiltinConcepts.FILTERED, + BuiltinConcepts.EXPLANATION, + BuiltinConcepts.TO_LIST, +] """ Some concepts have a specific implementation @@ -194,39 +185,51 @@ It's mainly to ease the usage class UserInputConcept(Concept): + ALL_ATTRIBUTES = ["text", "user_name"] + def __init__(self, text=None, user_name=None): - super().__init__(BuiltinConcepts.USER_INPUT, True, False, BuiltinConcepts.USER_INPUT) - self.set_value(ConceptParts.BODY, text) + Concept.__init__(self, + BuiltinConcepts.USER_INPUT, + True, + False, + BuiltinConcepts.USER_INPUT, bound_body="text") + self.set_value("text", text) self.set_value("user_name", user_name) - self.metadata.is_evaluated = True - - @property - def text(self): - return self.body - - @property - def user_name(self): - return self.get_value("user_name") + self._metadata.is_evaluated = True def __repr__(self): return f"({self.id}){self.name}: '{self.body}'" -class ErrorConcept(Concept): - def __init__(self, error=None): - super().__init__(BuiltinConcepts.ERROR, True, False, BuiltinConcepts.ERROR) - self.set_value(ConceptParts.BODY, error) - self.metadata.is_evaluated = True +class ErrorConcept(Concept, ErrorObj): + ALL_ATTRIBUTES = ["error"] + + def __init__(self, error=None, concept_id=None): + Concept.__init__(self, + BuiltinConcepts.ERROR, + True, + False, + BuiltinConcepts.ERROR, + id=concept_id, + bound_body="error") + self.set_value("error", error) + self._metadata.is_evaluated = True def __repr__(self): return f"({self.id}){self.name}: {self.body}" -class UnknownConcept(Concept): - def __init__(self, metadata=None): - super().__init__(BuiltinConcepts.UNKNOWN_CONCEPT, True, False, BuiltinConcepts.UNKNOWN_CONCEPT) - self.set_value(ConceptParts.BODY, metadata) - self.metadata.is_evaluated = True +class UnknownConcept(Concept, ErrorObj): + ALL_ATTRIBUTES = ["concept_ref"] + + def __init__(self, concept_ref=None): + Concept.__init__(self, + BuiltinConcepts.UNKNOWN_CONCEPT, + True, + False, + BuiltinConcepts.UNKNOWN_CONCEPT, bound_body="concept_ref") + self.set_value("concept_ref", concept_ref) + self._metadata.is_evaluated = True def __repr__(self): return f"({self.id}){self.name}: {self.body}" @@ -238,67 +241,36 @@ class ReturnValueConcept(Concept): It's the main input for the evaluators """ - def __init__(self, who=None, status=None, value=None, message=None, parents=None, concept_id=None): - super().__init__(BuiltinConcepts.RETURN_VALUE, True, False, BuiltinConcepts.RETURN_VALUE) - self.set_value(ConceptParts.BODY, value) + ALL_ATTRIBUTES = ["who", "status", "value", "parents", "message"] + + def __init__(self, who=None, status=None, value=None, parents=None, message=None, concept_id=None): + Concept.__init__(self, + BuiltinConcepts.RETURN_VALUE, + True, + False, + BuiltinConcepts.RETURN_VALUE, + id=concept_id, + bound_body="value") self.set_value("who", who) self.set_value("status", status) - self.set_value("message", message) + self.set_value("value", value) self.set_value("parents", parents) - self.metadata.is_evaluated = True - self.metadata.id = concept_id - - @property - def who(self): - return self.get_value("who") - - @who.setter - def who(self, value): - self.set_value("who", value) - - @property - def status(self): - return self.get_value("status") - - @status.setter - def status(self, value): - self.set_value("status", value) - - @property - def value(self): - return self.body - - @value.setter - def value(self, value): - self.set_value(ConceptParts.BODY, value) - - @property - def message(self): - return self.get_value("message") - - @message.setter - def message(self, value): - self.set_value("message", value) - - @property - def parents(self): - return self.get_value("parents") - - @parents.setter - def parents(self, value): - self.set_value("parents", value) + self.set_value("message", message) + self._metadata.is_evaluated = True def __repr__(self): return f"ReturnValue(who={self.who}, status={self.status}, value={self.value}, message={self.message})" def __eq__(self, other): + if id(self) == id(other): + return True + if not isinstance(other, ReturnValueConcept): return False return self.who == other.who and \ self.status == other.status and \ - self.value == other.value and \ - self.message == other.message + self.value == other.value def __hash__(self): if hasattr(self.value, "__iter__") and not isinstance(self.value, str): @@ -309,47 +281,52 @@ class ReturnValueConcept(Concept): return hash((self.who, self.status, value_hash)) -class UnknownPropertyConcept(Concept): +class UnknownPropertyConcept(Concept, ErrorObj): """ This error is raised when, during sheerka.new(), an unknown property is asked """ + ALL_ATTRIBUTES = ["property_name", "concept"] def __init__(self, property_name=None, concept=None): - super().__init__(BuiltinConcepts.UNKNOWN_PROPERTY, True, False, BuiltinConcepts.UNKNOWN_PROPERTY) - self.set_value(ConceptParts.BODY, property_name) + Concept.__init__(self, + BuiltinConcepts.UNKNOWN_PROPERTY, + True, + False, + BuiltinConcepts.UNKNOWN_PROPERTY, + bound_body="property_name") + self.set_value("property_name", property_name) self.set_value("concept", concept) - self.metadata.is_evaluated = True + self._metadata.is_evaluated = True def __repr__(self): return f"UnknownProperty(property={self.property_name}, concept={self.concept})" - @property - def concept(self): - return self.get_value("concept") - - @property - def property_name(self): - return self.body - class ParserResultConcept(Concept): """ Result of a parsing """ - def __init__(self, parser=None, source=None, tokens=None, value=None, try_parsed=None): - super().__init__(BuiltinConcepts.PARSER_RESULT, True, False, BuiltinConcepts.PARSER_RESULT) - self.set_value(ConceptParts.BODY, value) + ALL_ATTRIBUTES = ["parser", "source", "tokens", "value", "try_parsed"] + + def __init__(self, parser=None, source=None, tokens=None, value=None, try_parsed=None, concept_id=None): + Concept.__init__(self, + BuiltinConcepts.PARSER_RESULT, + True, + False, + BuiltinConcepts.PARSER_RESULT, + id=concept_id, + bound_body="value") self.set_value("parser", parser) self.set_value("source", source) self.set_value("tokens", tokens) - self.set_value("try_parsed", try_parsed) # in case of error, what was found before the error - self.metadata.is_evaluated = True + self.set_value("value", value) + self.set_value("try_parsed", try_parsed) + self._metadata.is_evaluated = True def __repr__(self): - text = f"ParserResult(parser={self.get_value('parser')}" - source = self.get_value('source') - text += f", source='{source}')" if source else f", body='{self.body}')" + text = f"ParserResult(parser={self.parser}" + text += f", source='{self.source}')" if self.source else f", body='{self.value}')" return text def __eq__(self, other): @@ -361,27 +338,10 @@ class ParserResultConcept(Concept): return self.source == other.source and \ self_parser_name == other_parser_name and \ - self.body == other.body and \ - self.try_parsed == other.try_parsed + self.value == other.value def __hash__(self): - return hash(self.metadata.name) - - @property - def value(self): - return self.body - - @property - def try_parsed(self): - return self.get_value("try_parsed") - - @property - def source(self): - return self.get_value("source") - - @property - def parser(self): - return self.get_value("parser") + return hash(self._metadata.name) @staticmethod def get_parser_name(parser): @@ -389,191 +349,217 @@ class ParserResultConcept(Concept): return parser.name if isinstance(parser, BaseParser) else str(parser) -class InvalidReturnValueConcept(Concept): +class InvalidReturnValueConcept(Concept, ErrorObj): """ Error returned when an evaluator is not correctly coded The accepted return value are ReturnValueConcept, list of ReturnValueConcept or None """ + ALL_ATTRIBUTES = ["return_value", "evaluator"] + def __init__(self, return_value=None, evaluator=None): super().__init__( BuiltinConcepts.INVALID_RETURN_VALUE, True, False, - BuiltinConcepts.INVALID_RETURN_VALUE) - self.set_value(ConceptParts.BODY, return_value) + BuiltinConcepts.INVALID_RETURN_VALUE, + bound_body="return_value") + self.set_value("return_value", return_value) self.set_value("evaluator", evaluator) - self.metadata.is_evaluated = True + self._metadata.is_evaluated = True -class ConceptEvalError(Concept): +class ConceptEvalError(Concept, ErrorObj): + ALL_ATTRIBUTES = ["error", "concept", "property_name"] + def __init__(self, error=None, concept=None, property_name=None): super().__init__(BuiltinConcepts.CONCEPT_EVAL_ERROR, True, False, - BuiltinConcepts.CONCEPT_EVAL_ERROR) - self.set_value(ConceptParts.BODY, error) + BuiltinConcepts.CONCEPT_EVAL_ERROR, + bound_body="error") + self.set_value("error", error) self.set_value("concept", concept) self.set_value("property_name", property_name) - self.metadata.is_evaluated = True + self._metadata.is_evaluated = True def __repr__(self): return f"ConceptEvalError(error={self.error}, concept={self.concept}, property={self.property_name})" - @property - def error(self): - return self.body - - @property - def concept(self): - return self.get_value("concept") - - @property - def property_name(self): - return self.get_value("property_name") - - -class EnumerationConcept(Concept): - def __init__(self, iteration=None): - super().__init__(BuiltinConcepts.ENUMERATION, True, False, BuiltinConcepts.ENUMERATION) - self.set_value(ConceptParts.BODY, iteration) - self.metadata.is_evaluated = True - class ListConcept(Concept): + ALL_ATTRIBUTES = ["items"] + def __init__(self, items=None): - super().__init__(BuiltinConcepts.LIST, True, False, BuiltinConcepts.LIST) - self.set_value(ConceptParts.BODY, items or []) - self.metadata.is_evaluated = True + Concept.__init__(self, + BuiltinConcepts.LIST, + True, + False, + BuiltinConcepts.LIST, + bound_body="items") + self.set_value("items", items or []) + self._metadata.is_evaluated = True def append(self, obj): self.body.append(obj) class FilteredConcept(Concept): + ALL_ATTRIBUTES = ["filtered", "iterable", "predicate"] + def __init__(self, filtered=None, iterable=None, predicate=None): - super().__init__(BuiltinConcepts.FILTERED, True, False, BuiltinConcepts.FILTERED) - self.set_value(ConceptParts.BODY, filtered) - self.set_value("iterable", iterable) - self.set_value("predicate", predicate) - self.metadata.is_evaluated = True - - -class ConceptAlreadyInSet(Concept): - def __init__(self, concept=None, concept_set=None): - super().__init__(BuiltinConcepts.CONCEPT_ALREADY_IN_SET, + Concept.__init__(self, + BuiltinConcepts.FILTERED, True, False, - BuiltinConcepts.CONCEPT_ALREADY_IN_SET) - self.set_value(ConceptParts.BODY, concept) + BuiltinConcepts.FILTERED, + bound_body="filtered") + self.set_value("filtered", filtered) + self.set_value("iterable", iterable) + self.set_value("predicate", predicate) + self._metadata.is_evaluated = True + + +class ConceptAlreadyInSet(Concept, ErrorObj): + ALL_ATTRIBUTES = ["concept", "concept_set"] + + def __init__(self, concept=None, concept_set=None): + Concept.__init__(self, + BuiltinConcepts.CONCEPT_ALREADY_IN_SET, + True, + False, + BuiltinConcepts.CONCEPT_ALREADY_IN_SET, + bound_body="concept") + self.set_value("concept", concept) self.set_value("concept_set", concept_set) - self.metadata.is_evaluated = True + self._metadata.is_evaluated = True def __repr__(self): return f"ConceptAlreadyInSet(concept={self.concept}, concept_set={self.concept_set})" - @property - def concept(self): - return self.body - @property - def concept_set(self): - return self.get_value("concept_set") +class PropertyAlreadyDefined(Concept, ErrorObj): + ALL_ATTRIBUTES = ["property_name", "property_value", "concept"] - -class PropertyAlreadyDefined(Concept): def __init__(self, property_name=None, property_value=None, concept=None): - super().__init__(BuiltinConcepts.PROPERTY_ALREADY_DEFINED, + Concept.__init__(self, + BuiltinConcepts.PROPERTY_ALREADY_DEFINED, True, False, - BuiltinConcepts.PROPERTY_ALREADY_DEFINED) - self.set_value(ConceptParts.BODY, property_name) + BuiltinConcepts.PROPERTY_ALREADY_DEFINED, + bound_body="property_name") + self.set_value("property_name", property_name) self.set_value("property_value", property_value) self.set_value("concept", concept) - self.metadata.is_evaluated = True + self._metadata.is_evaluated = True def __repr__(self): return f"PropertyAlreadyDefined(property={self.property_name}, value={self.property_value}, concept={self.concept})" - @property - def property_name(self): - return self.body - @property - def property_value(self): - return self.get_value("property_value") +class ConditionFailed(Concept, ErrorObj): + ALL_ATTRIBUTES = ["condition", "concept", "prop", "reason"] - @property - def concept(self): - return self.get_value("concept") - - -class ConditionFailed(Concept): - def __init__(self, condition=None, concept=None, prop=None): - super().__init__(BuiltinConcepts.CONDITION_FAILED, + def __init__(self, condition=None, concept=None, prop=None, reason=None): + Concept.__init__(self, + BuiltinConcepts.CONDITION_FAILED, True, False, - BuiltinConcepts.CONDITION_FAILED) - self.set_value(ConceptParts.BODY, condition) + BuiltinConcepts.CONDITION_FAILED, + bound_body="condition") + self.set_value("condition", condition) self.set_value("concept", concept) self.set_value("prop", prop) - self.metadata.is_evaluated = True + self.set_value("reason", reason) + self._metadata.is_evaluated = True def __repr__(self): return f"ConditionFailed(condition='{self.body}', concept='{self.concept}', prop='{self.prop}')" -class NotForMeConcept(Concept): +class NotForMeConcept(Concept): # Not considered as an error ? + ALL_ATTRIBUTES = ["source", "reason"] + def __init__(self, source=None, reason=None): - super().__init__(BuiltinConcepts.NOT_FOR_ME, + Concept.__init__(self, + BuiltinConcepts.NOT_FOR_ME, True, False, - BuiltinConcepts.NOT_FOR_ME) - self.set_value(ConceptParts.BODY, source) + BuiltinConcepts.NOT_FOR_ME, + bound_body="source") + self.set_value("source", source) self.set_value("reason", reason) - self.metadata.is_evaluated = True + self._metadata.is_evaluated = True def __repr__(self): return f"NotForMeConcept(source={self.body}, reason={self.get_value('reason')})" class ExplanationConcept(Concept): + ALL_ATTRIBUTES = ["digest", "command", "title", "instructions", "execution_result"] + def __init__(self, digest=None, command=None, title=None, instructions=None, execution_result=None): - super().__init__(BuiltinConcepts.EXPLANATION, + Concept.__init__(self, + BuiltinConcepts.EXPLANATION, True, False, - BuiltinConcepts.EXPLANATION) + BuiltinConcepts.EXPLANATION, + bound_body="execution_result") self.set_value("digest", digest) # event digest self.set_value("command", command) # explain command parameters self.set_value("title", title) # a title to the explanation self.set_value("instructions", instructions) # instructions for SheerkaPrint - self.set_value(ConceptParts.BODY, execution_result) # list of results - self.metadata.is_evaluated = True + self.set_value("execution_result", execution_result) # list of results + self._metadata.is_evaluated = True -class PythonSecurityError(Concept): +class PythonSecurityError(Concept, ErrorObj): + ALL_ATTRIBUTES = ["prop", "source_code", "source", "line", "column"] + def __init__(self, prop=None, source_code=None, source=None, line=None, column=None): - super().__init__(BuiltinConcepts.PYTHON_SECURITY_ERROR, + Concept.__init__(self, + BuiltinConcepts.PYTHON_SECURITY_ERROR, True, False, - BuiltinConcepts.PYTHON_SECURITY_ERROR) + BuiltinConcepts.PYTHON_SECURITY_ERROR, + bound_body="source_code") self.set_value("prop", prop) # property or variable that was evaluated self.set_value("source", source) # origin of the source code (eg. file name) self.set_value("line", line) # line number self.set_value("column", column) # column number - self.set_value(ConceptParts.BODY, source_code) # code being executed - self.metadata.is_evaluated = True + self.set_value("source_code", source_code) # code being executed + self._metadata.is_evaluated = True -class NotFound(Concept): +class NotFound(Concept, ErrorObj): + ALL_ATTRIBUTES = [] + def __init__(self, body=None): - super().__init__(BuiltinConcepts.NOT_FOUND, + Concept.__init__(self, + BuiltinConcepts.NOT_FOUND, True, False, BuiltinConcepts.NOT_FOUND) self.set_value(ConceptParts.BODY, body) def __repr__(self): - return f"({self.metadata.id}){self.metadata.name}, body={self.get_value(ConceptParts.BODY)}" + return f"({self._metadata.id}){self._metadata.name}, body={self.get_value(ConceptParts.BODY)}" + + +class ToListConcept(Concept): + ALL_ATTRIBUTES = ["items", "recursion_depth", "recurse_on", "tab"] + + def __init__(self, items=None, recursion_depth=None, recurse_on=None, tab=None): + Concept.__init__(self, + BuiltinConcepts.TO_LIST, + True, + False, + BuiltinConcepts.TO_LIST, + bound_body="items") + self.set_value("items", items) # items to display + self.set_value("recursion_depth", recursion_depth) # recursion depth when showing children + self.set_value("recurse_on", recurse_on) # which sub items should we display + self.set_value("tab", tab) # customise tab (content and length) + self._metadata.is_evaluated = True diff --git a/src/core/builtin_helpers.py b/src/core/builtin_helpers.py index fec7131..9695a25 100644 --- a/src/core/builtin_helpers.py +++ b/src/core/builtin_helpers.py @@ -1,21 +1,18 @@ -import ast import logging -import core.ast.nodes -from core.ast.nodes import CallNodeConcept -from core.ast.visitors import UnreferencedNamesVisitor from core.builtin_concepts import BuiltinConcepts -from core.concept import Concept, NotInit, ConceptParts +from core.concept import Concept, NotInit, ConceptParts, DEFINITION_TYPE_BNF, concept_part_value +from core.rule import Rule from core.sheerka.services.SheerkaExecute import SheerkaExecute from core.tokenizer import Keywords -# from evaluators.BaseEvaluator import BaseEvaluator -from parsers.BaseNodeParser import SourceCodeNode, ConceptNode, UnrecognizedTokensNode, SourceCodeWithConceptNode +from parsers.BaseNodeParser import SourceCodeNode, ConceptNode, UnrecognizedTokensNode, SourceCodeWithConceptNode, \ + RuleNode from parsers.BaseParser import BaseParser, ErrorNode PARSE_STEPS = [BuiltinConcepts.BEFORE_PARSING, BuiltinConcepts.PARSING, BuiltinConcepts.AFTER_PARSING] EVAL_STEPS = PARSE_STEPS + [BuiltinConcepts.BEFORE_EVALUATION, BuiltinConcepts.EVALUATION, BuiltinConcepts.AFTER_EVALUATION] -PARSERS = ["EmptyString", "ShortTermMemory", "AtomNode", "BnfNode", "SyaNode", "Python"] +PARSERS = ["EmptyString", "ShortTermMemory", "Sequence", "Bnf", "Sya", "Python"] def is_same_success(context, return_values): @@ -31,7 +28,7 @@ def is_same_success(context, return_values): if not ret_val.status: raise Exception("Status is false") - if isinstance(ret_val.body, Concept) and not ret_val.body.metadata.is_evaluated: + if isinstance(ret_val.body, Concept) and not ret_val.body.get_metadata().is_evaluated: raise Exception("Concept is not evaluated") return context.sheerka.objvalue(ret_val) @@ -185,7 +182,7 @@ def resolve_ambiguity(context, concepts): # the concept matches the context by_complexity = {} for c in concepts: - by_complexity.setdefault(get_condition_complexity(c, "pre"), []).append(c) + by_complexity.setdefault(get_condition_complexity(c, concept_part_value(ConceptParts.PRE)), []).append(c) remaining_concepts = [] for complexity in sorted(by_complexity.keys(), reverse=True): @@ -193,7 +190,7 @@ def resolve_ambiguity(context, concepts): remaining_concepts.extend(by_complexity[complexity]) else: for c in by_complexity[complexity]: - evaluated = context.sheerka.evaluate_concept(context, c, metadata=["pre"]) + evaluated = context.sheerka.evaluate_concept(context, c, metadata=[ConceptParts.PRE]) if context.sheerka.is_success(evaluated) or evaluated.key == c.key: remaining_concepts.append(c) @@ -208,21 +205,21 @@ def resolve_ambiguity(context, concepts): # when the input is "hello world" by_number_of_vars = {} for c in remaining_concepts: - by_number_of_vars.setdefault(len(c.metadata.variables), []).append(c) + by_number_of_vars.setdefault(len(c.get_metadata().variables), []).append(c) return by_number_of_vars[min(by_number_of_vars.keys())] def get_condition_complexity(concept, concept_part_str): """ - Need to find a proper algorithm to compute the complexity of a concept - So far, the concept is considered as complex if it has pre + Need to find a proper algorithm to compute the complexity of a concept metadata + So far, the concept is considered as complex if it has concept_part_str (so far with concept_part_str='pre') :param concept: :param concept_part_str: :return: """ - concept_part_value = getattr(concept.metadata, concept_part_str) - if concept_part_value is None or concept_part_value.strip() == 0: + value = getattr(concept.get_metadata(), concept_part_str) + if value is None or value.strip() == 0: return 0 return 1 # no real computing as of now @@ -270,7 +267,8 @@ def only_parsers_results(context, return_values): return sheerka.ret( context.who, False, - sheerka.new(BuiltinConcepts.TOO_MANY_ERRORS, body=return_values), + sheerka.new(BuiltinConcepts.TOO_MANY_ERRORS if len(return_values) > 1 else BuiltinConcepts.ERROR, + body=return_values), parents=return_values) return sheerka.ret( @@ -291,7 +289,7 @@ def parse_unrecognized(context, source, parsers, who=None, prop=None, filter_fun :param parsers: :param who: who is asking the parsing ? :param prop: Extra info, when parsing a property - :param filter_func: filter function to call is provided + :param filter_func: Once the result are found, call this function to filter them :return: """ sheerka = context.sheerka @@ -306,11 +304,12 @@ def parse_unrecognized(context, source, parsers, who=None, prop=None, filter_fun with context.push(BuiltinConcepts.PARSING, action_context, who=who, desc=desc) as sub_context: # disable all parsers but the requested ones if parsers != "all": - sub_context.add_preprocess(BaseParser.PREFIX + "*", enabled=False) - for parser in parsers: - sub_context.add_preprocess(BaseParser.PREFIX + parser, enabled=True) + sub_context.preprocess_parsers = [BaseParser.PREFIX + parser for parser in parsers] + # sub_context.add_preprocess(BaseParser.PREFIX + "*", enabled=False) + # for parser in parsers: + # sub_context.add_preprocess(BaseParser.PREFIX + parser, enabled=True) - if prop in (Keywords.WHERE, Keywords.PRE, ConceptParts.WHERE, ConceptParts.PRE): + if prop in (Keywords.WHERE, Keywords.PRE, ConceptParts.WHERE, ConceptParts.PRE, Keywords.WHEN): sub_context.protected_hints.add(BuiltinConcepts.EVAL_QUESTION_REQUESTED) sub_context.add_inputs(source=source) @@ -323,25 +322,7 @@ def parse_unrecognized(context, source, parsers, who=None, prop=None, filter_fun res = filter_func(sub_context, res) sub_context.add_values(return_values=res) - if not hasattr(res, "__iter__"): - return res - - # discard Python response if accepted by AtomNode - is_concept = False - for r in res: - if r.status and r.who == "parsers.AtomNode": - is_concept = True - - if not is_concept: - return res - - no_python = [] - for r in res: - if r.who == "parsers.Python": - continue - no_python.append(r) - - return no_python + return res def parse_function(context, source, tokens=None, start=0): @@ -465,7 +446,7 @@ def get_lexer_nodes(return_values, start, tokens): for concept in concepts: lexer_nodes.append([ConceptNode(concept, start, end, tokens, ret_val.body.source)]) - elif ret_val.who in ("parsers.BnfNode", "parsers.SyaNode", "parsers.AtomNode"): + elif ret_val.who in ("parsers.Bnf", "parsers.Sya", "parsers.Sequence"): nodes = [node for node in ret_val.body.body] for node in nodes: node.start += start @@ -474,6 +455,12 @@ def get_lexer_nodes(return_values, start, tokens): # but append the whole sequence if when it's a sequence lexer_nodes.append(nodes) + elif ret_val.who == "parsers.Rule": + rules = ret_val.body.body if hasattr(ret_val.body.body, "__iter__") else [ret_val.body.body] + end = start + len(tokens) - 1 + for rule in rules: + lexer_nodes.append([RuleNode(rule, start, end, tokens, ret_val.body.source)]) + else: raise NotImplementedError() @@ -488,16 +475,16 @@ def ensure_evaluated(context, concept, eval_body=True): :param eval_body: :return: """ - if concept.metadata.is_evaluated: + if concept.get_metadata().is_evaluated: return concept # do not try to evaluate concept that are not fully initialized - for var in concept.metadata.variables: - # to code - if var[1] is None and \ - var[0] not in concept.compiled and \ - (var[0] not in concept.values or concept.get_value(var[0]) == NotInit): - return concept + if concept.get_metadata().definition_type != DEFINITION_TYPE_BNF: + for var in concept.get_metadata().variables: + if var[1] is None and \ + var[0] not in concept.get_compiled() and \ + (var[0] not in concept.values() or concept.get_value(var[0]) == NotInit): + return concept evaluated = context.sheerka.evaluate_concept(context, concept, eval_body=eval_body) return evaluated @@ -523,7 +510,7 @@ def get_lexer_nodes_from_unrecognized(context, unrecognized_tokens_node, parsers def update_compiled(context, concept, errors, parsers=None): """ - recursively iterate thru concept.compiled to replace LexerNode into concepts or list of ReturnValueConcept + recursively iterate thru concept.get_compiled() to replace LexerNode into concepts or list of ReturnValueConcept When parsing using a LexerNodeParser (SyaNodeParser, BnfNodeParser...) the result will be a LexerNode. In the specific case of a ConceptNode, the compiled variables will also be LexerNode (UnrecognizedTokensNode...) @@ -534,7 +521,6 @@ def update_compiled(context, concept, errors, parsers=None): :param parsers: to customize the parsers to use :return: """ - sheerka = context.sheerka parsers = parsers or PARSERS @@ -544,7 +530,7 @@ def update_compiled(context, concept, errors, parsers=None): :param c: :return: """ - for k, v in c.compiled.items(): + for k, v in c.get_compiled().items(): if isinstance(v, Concept): _validate_concept(v) @@ -553,7 +539,7 @@ def update_compiled(context, concept, errors, parsers=None): parser_helper = PythonWithConceptsParser() res = parser_helper.parse_nodes(context, v.get_all_nodes()) if res.status: - c.compiled[k] = [res] + c.get_compiled()[k] = [res] else: errors.append(sheerka.new(BuiltinConcepts.ERROR, body=f"Cannot parse '{v.source}'")) @@ -561,7 +547,7 @@ def update_compiled(context, concept, errors, parsers=None): res = parse_unrecognized(context, v.source, parsers) res = only_successful(context, res) # only key successful parsers if res.status: - c.compiled[k] = res.body.body + c.get_compiled()[k] = res.body.body else: errors.append(sheerka.new(BuiltinConcepts.ERROR, body=f"Cannot parse '{v.source}'")) @@ -588,117 +574,12 @@ def update_compiled(context, concept, errors, parsers=None): # and the user has entered 'a plus b' # Chances are that we are talking about the concept itself, and not an instantiation (like '10 plus 2') # This means that 'a' and 'b' don't have any real value - if len(concept.metadata.variables) > 0: - for name, value in concept.metadata.variables: - if _get_source(concept.compiled, name) != name: + if len(concept.get_metadata().variables) > 0: + for name, value in concept.get_metadata().variables: + if _get_source(concept.get_compiled(), name) != name: break else: - concept.metadata.is_evaluated = True - - -def get_names(sheerka, concept_node): - """ - Finds all the names referenced by the concept_node - :param sheerka: - :param concept_node: - :return: - """ - unreferenced_names_visitor = UnreferencedNamesVisitor(sheerka) - unreferenced_names_visitor.visit(concept_node) - return list(unreferenced_names_visitor.names) - - -def extract_predicates(sheerka, expression, variables_to_include, variables_to_exclude): - """ - from a given expression and a variable (or list of variables) - tries to find out all the predicates referencing the(se) variable(s), and the(se) variable(s) solely - for example - exp : isinstance(a, int) and isinstance(b, str) - will return 'isinstance(a, int)' if variable_name == 'a' - :param sheerka: - :param expression: - :param variables_to_include: - :param variables_to_exclude: - :return: list of predicates - """ - - if len(variables_to_include) == 0: - return [] - - def _get_predicates(_nodes): - _predicates = [] - for _node in _nodes: - python_node = ast.Expression(body=core.ast.nodes.concept_to_python(_node)) - python_node = ast.fix_missing_locations(python_node) - _predicates.append(python_node) - return _predicates - - if isinstance(expression, str): - node = ast.parse(expression, mode="eval") - else: - return NotImplementedError() - - concept_node = core.ast.nodes.python_to_concept(node) - main_op = concept_node.get_value("body") - - return _get_predicates(_extract_predicates(sheerka, main_op, variables_to_include, variables_to_exclude)) - - -def _extract_predicates(sheerka, node, variables_to_include, variables_to_exclude): - predicates = [] - - def _matches(_names, to_include, to_exclude): - _res = None - for n in _names: - if n in to_include and _res is None: - _res = True - if n in to_exclude: - _res = False - return _res - - if node.node_type == "Compare": - if node.get_value("left").node_type == "Name": - """Simple case of one comparison""" - comparison_name = sheerka.objvalue(node.get_value("left")) - if comparison_name in variables_to_include and comparison_name not in variables_to_exclude: - predicates.append(node) - else: - """The left part is an expression""" - res = _extract_predicates(sheerka, node.get_value("left"), variables_to_include, variables_to_exclude) - if len(res) > 0: - predicates.append(node) - elif node.node_type == "Call": - """Simple case predicate""" - call_node = node if isinstance(node, CallNodeConcept) else CallNodeConcept().update_from(node) - args = list(call_node.get_args_names(sheerka)) - if _matches(args, variables_to_include, variables_to_exclude): - predicates.append(node) - elif node.node_type == "UnaryOp" and node.get_value("op").node_type == "Not": - """Simple case of negation""" - res = _extract_predicates(sheerka, node.get_value("operand"), variables_to_include, variables_to_exclude) - if len(res) > 0: - predicates.append(node) - elif node.node_type == "BinOp": - names = get_names(sheerka, node) - if _matches(names, variables_to_include, variables_to_exclude): - predicates.append(node) - elif node.node_type == "BoolOp": - all_op = True - temp_res = [] - for op in node.get_value("values").body: - res = _extract_predicates(sheerka, op, variables_to_include, variables_to_exclude) - if len(res) == 0: - all_op = False - else: - temp_res.extend(res) - - if all_op: - predicates.append(node) - else: - for res in temp_res: - predicates.append(res) - - return predicates + concept.get_metadata().is_evaluated = True def add_to_ret_val(sheerka, context, return_values, concept_key): @@ -732,8 +613,38 @@ def set_is_evaluated(concepts, check_nb_variables=False): if hasattr(concepts, "__iter__"): for c in concepts: - if not check_nb_variables or check_nb_variables and len(c.metadata.variables) > 0: - c.metadata.is_evaluated = True + if not check_nb_variables or check_nb_variables and len(c.get_metadata().variables) > 0: + c.get_metadata().is_evaluated = True else: - if not check_nb_variables or check_nb_variables and len(concepts.metadata.variables) > 0: - concepts.metadata.is_evaluated = True + if not check_nb_variables or check_nb_variables and len(concepts.get_metadata().variables) > 0: + concepts.get_metadata().is_evaluated = True + + +def ensure_concept(*concepts): + if hasattr(concepts, "__iter__"): + for concept in concepts: + if not isinstance(concept, Concept): + raise TypeError(f"'{concept}' must be a concept") + else: + if not isinstance(concepts, Concept): + raise TypeError(f"'{concepts}' must be a concept") + + +def ensure_rule(*rules): + if hasattr(rules, "__iter__"): + for rule in rules: + if not isinstance(rule, Rule): + raise TypeError(f"'{rule}' must be a rule") + else: + if not isinstance(rules, Rule): + raise TypeError(f"'{rules}' must be a rule") + + +def ensure_concept_or_rule(*items): + if hasattr(items, "__iter__"): + for item in items: + if not isinstance(item, (Concept, Rule)): + raise TypeError(f"'{item}' must be a concept or rule") + else: + if not isinstance(items, (Concept, Rule)): + raise TypeError(f"'{items}' must be a concept or rule") diff --git a/src/core/concept.py b/src/core/concept.py index eb84024..483994d 100644 --- a/src/core/concept.py +++ b/src/core/concept.py @@ -2,11 +2,9 @@ import hashlib from collections import namedtuple from copy import deepcopy from dataclasses import dataclass -from enum import Enum from typing import Union import core.utils -from core.sheerka_logger import get_logger from core.tokenizer import Tokenizer, TokenKind PROPERTIES_FOR_DIGEST = ("name", "key", @@ -28,23 +26,29 @@ class NotInitialized: def __repr__(self): return self.value + def __eq__(self, other): + return isinstance(other, NotInitialized) + NotInit = NotInitialized() -class ConceptParts(Enum): +class ConceptParts: """ Lists metadata that can contains some code """ - WHERE = "where" - PRE = "pre" - POST = "post" - BODY = "body" - RET = "ret" + WHERE = "#where#" + PRE = "#pre#" + POST = "#post#" + BODY = "#body#" + RET = "#ret#" - @staticmethod - def get_parts(): - return set(item.value for item in ConceptParts) + +AllConceptParts = [v for k, v in ConceptParts.__dict__.items() if not k.startswith("__")] + + +def concept_part_value(c): + return c[1:-1] @dataclass @@ -69,6 +73,28 @@ class ConceptMetadata: full_serialization: bool = False # If True, the full object will be serialized, rather than just the diff +ALL_ATTRIBUTES = {} + + +def get_concept_attrs(concept): + if concept.ALL_ATTRIBUTES is not None: + return concept.ALL_ATTRIBUTES + + try: + return ALL_ATTRIBUTES[concept.id] + except KeyError: + pass + + all_attributes = [k for k in concept.__dict__ if k[0] != "_" and k[0] != "#"] + if concept.id: + ALL_ATTRIBUTES[concept.id] = all_attributes + return all_attributes + + +def freeze_concept_attrs(concept): + ALL_ATTRIBUTES[concept.id] = [k for k in concept.__dict__ if k[0] != "_" and k[0] != "#"] + + class Concept: """ Default concept object @@ -76,6 +102,8 @@ class Concept: Everything is a concept """ + ALL_ATTRIBUTES = None + def __init__(self, name=None, is_builtin=False, is_unique=False, @@ -90,13 +118,14 @@ class Concept: desc=None, id=None, props=None, - variables=None): + variables=None, + bound_body=None): metadata = ConceptMetadata( - str(name) if name else None, + name if name else None, is_builtin, is_unique, - str(key) if key else None, + key if key else None, body, where, pre, @@ -110,17 +139,16 @@ class Concept: variables or [] ) - self.metadata = metadata - self.compiled = {} # cached ast for the where, pre, post and body parts and variables - self.values = {} # resolved values. As compiled, it's used both for metadata and variables - self.bnf = None # parsing expression - self.log = get_logger("core." + self.__class__.__name__) - self.init_log = get_logger("init.core." + self.__class__.__name__) - self.original_definition_hash = None # concept hash before any alteration of the metadata + self._metadata = metadata + self._bound_body = bound_body + self._compiled = {} # cached ast for the where, pre, post and body parts and variables + self._bnf = None # parsing expression + self._original_definition_hash = None # concept hash before any alteration of the metadata + self._format = None # how to print the concept def __repr__(self): - text = f"({self.metadata.id}){self.metadata.name}" - return text + " (" + self.metadata.pre + ")" if self.metadata.pre else text + text = f"({self._metadata.id}){self._metadata.name}" + return text + " (" + self._metadata.pre + ")" if self._metadata.pre else text def __eq__(self, other): @@ -139,18 +167,18 @@ class Concept: # check the metadata for prop in PROPERTIES_TO_SERIALIZE: # print(prop) # use full to know which id does not match - my_value = getattr(self.metadata, prop) - other_value = getattr(other.metadata, prop) + my_value = getattr(self._metadata, prop) + other_value = getattr(other._metadata, prop) if isinstance(my_value, Concept) and isinstance(other_value, Concept): # need to check if circular references if id(self) == id(other): continue - sub_value = getattr(other_value.metadata, prop) + sub_value = getattr(other_value._metadata, prop) while isinstance(sub_value, Concept): if id(self) == id(sub_value): return False # circular reference - sub_value = getattr(sub_value.metadata, prop) + sub_value = getattr(sub_value._metadata, prop) if my_value != other_value: return False @@ -160,26 +188,28 @@ class Concept: return False # checks the values - if len(self.values) != len(other.values): + self_values = self.values() + other_values = other.values() + if len(self_values) != len(other_values): return False - for name in self.values: - if self.get_value(name) != other.get_value(name): + for name, value in self_values.items(): + if value != other.get_value(name): return False return True def __hash__(self): - return hash(self.metadata.name) + return hash(self._metadata.name) - def __getattr__(self, item): - # I have this complicated implementation because of the usage of Pickle - - if 'values' in vars(self) and item in self.values: - return self.get_value(item) - - name = self.name if 'metadata' in vars(self) else 'Concept' - raise AttributeError(f"'{name}' concept has no attribute '{item}'") + # def __getattr__(self, item): + # # I have this complicated implementation because of the usage of Pickle + # + # if 'values' in vars(self) and item in self.values: + # return self.get_value(item) + # + # name = self.name if 'metadata' in vars(self) else 'Concept' + # raise AttributeError(f"'{name}' concept has no attribute '{item}'") def def_var(self, var_name, default_value=None): """ @@ -196,7 +226,7 @@ class Concept: # - list of concepts is used by ISA assert default_value is None or isinstance(default_value, str) - self.metadata.variables.append((var_name, default_value)) + self._metadata.variables.append((var_name, default_value)) self.set_value(var_name, NotInit) # do not set the default value # why not setting variables to the default values ? @@ -212,21 +242,40 @@ class Concept: :return: """ assert value is None or isinstance(value, str) # default properties will have to be evaluated - var_name = self.metadata.variables[index] - self.metadata.variables[index] = (var_name[0], value) # change the default value + var_name = self._metadata.variables[index] + self._metadata.variables[index] = (var_name[0], value) # change the default value return self + def get_metadata(self): + return self._metadata + + def get_compiled(self): + return self._compiled + + def set_compiled(self, compiled): + self._compiled = compiled + + def get_bnf(self): + return self._bnf + + def set_bnf(self, value): + self._bnf = value + @property def name(self): - return self.metadata.name + return self._metadata.name @property def id(self): - return self.metadata.id + return self._metadata.id + + @property + def str_id(self): + return core.utils.str_concept(self) @property def key(self): - return self.metadata.key + return self._metadata.key def init_key(self, tokens=None): """ @@ -237,16 +286,16 @@ class Concept: :param tokens: :return: """ - if self.metadata.key is not None: + if self._metadata.key is not None: return self if tokens is None: - if self.metadata.definition_type == DEFINITION_TYPE_DEF: - tokens = list(Tokenizer(self.metadata.definition)) + if self._metadata.definition_type == DEFINITION_TYPE_DEF: + tokens = list(Tokenizer(self._metadata.definition)) else: - tokens = list(Tokenizer(self.metadata.name)) + tokens = list(Tokenizer(self._metadata.name)) - variables = [p[0] for p in self.metadata.variables] if len(core.utils.strip_tokens(tokens, True)) > 1 else [] + variables = [p[0] for p in self._metadata.variables] if len(core.utils.strip_tokens(tokens, True)) > 1 else [] key = "" first = True @@ -264,7 +313,7 @@ class Concept: key += token.value.value if token.type == TokenKind.KEYWORD else token.value first = False - self.metadata.key = key + self._metadata.key = key return self @property @@ -304,9 +353,9 @@ class Concept: props_as_dict = {} for prop in props_to_use: if prop == "props": # no need to copy variables as the ref won't be used in from_dict - props_as_dict[prop] = deepcopy(getattr(self.metadata, prop)) + props_as_dict[prop] = deepcopy(getattr(self._metadata, prop)) else: - props_as_dict[prop] = getattr(self.metadata, prop) + props_as_dict[prop] = getattr(self._metadata, prop) return props_as_dict def from_dict(self, as_dict): @@ -321,7 +370,7 @@ class Concept: for name, value in as_dict[prop]: self.def_var(name, value) else: - setattr(self.metadata, prop, as_dict[prop]) + setattr(self._metadata, prop, as_dict[prop]) return self def update_from(self, other, update_value=True): @@ -339,16 +388,25 @@ class Concept: if id(other) == id(self): return self - # update metadata - self.from_dict(other.to_dict()) + for prop in PROPERTIES_TO_SERIALIZE: + if prop == "variables": + for name, value in other.get_metadata().variables: + self.def_var(name, value) + elif prop == "props": + self._metadata.props = deepcopy(other.get_metadata().props) + else: + setattr(self._metadata, prop, getattr(other.get_metadata(), prop)) + + # # update metadata + # self.from_dict(other.to_dict()) # update values if update_value: - for k in other.values: - self.set_value(k, other.get_value(k)) + for k, v in other.values().items(): + self.set_value(k, v) # update bnf definition - self.bnf = other.bnf + self._bnf = other.get_bnf() # origin from sdp.sheerkaSerializer import Serializer @@ -365,10 +423,10 @@ class Concept: :param value: :return: """ - if property_name in self.metadata.props: - self.metadata.props[property_name].add(value) + if property_name in self._metadata.props: + self._metadata.props[property_name].add(value) else: - self.metadata.props[property_name] = {value} # a set + self._metadata.props[property_name] = {value} # a set return self def set_prop(self, property_name, value): @@ -379,7 +437,7 @@ class Concept: :param value: :return: """ - self.metadata.props[property_name] = value + self._metadata.props[property_name] = value def get_prop(self, concept_key): """ @@ -387,7 +445,7 @@ class Concept: :param concept_key: name of the behaviour :return: """ - return self.metadata.props.get(concept_key, None) + return self._metadata.props.get(concept_key, None) def set_value(self, name, value): """ @@ -396,10 +454,14 @@ class Concept: :param value: :return: """ - if name in self.values: - self.values[name].value = value - else: - self.values[name] = Property(name, value) + try: + setattr(self, name, value) + if name == self._bound_body: + setattr(self, ConceptParts.BODY, value) + elif self._bound_body and name == ConceptParts.BODY: + setattr(self, self._bound_body, value) + except AttributeError: + print(f"Cannot set {name}") return self def get_value(self, prop_name): @@ -408,13 +470,29 @@ class Concept: :param prop_name: :return: """ - if prop_name not in self.values: + try: + return getattr(self, prop_name) + except AttributeError: from core.builtin_concepts import BuiltinConcepts - return BuiltinConcepts.NOT_INITIALIZED - return self.values[prop_name].value + return NotInit + + def values(self): + try: + values = {k: getattr(self, k) for k in get_concept_attrs(self)} + except AttributeError as err: + print(f"{err}, {self=}") + raise err + + for prop_name in AllConceptParts: + try: + values[prop_name] = getattr(self, prop_name) + except AttributeError: + pass + return values def variables(self): - return dict([(k, v) for k, v in self.values.items() if isinstance(k, str)]) + return {k: v for k, v in self.values().items() if not k[0] == "#"} + # return dict([(k, v) for k, v in self.values.items() if isinstance(k, str)]) def auto_init(self): """ @@ -424,25 +502,25 @@ class Concept: :return: """ - if self.metadata.is_evaluated: + if self._metadata.is_evaluated: return self - for metadata in ConceptParts: - value = getattr(self.metadata, metadata.value) + for metadata in AllConceptParts: + value = getattr(self._metadata, concept_part_value(metadata)) if value is not None: self.set_value(metadata, value) - for var, value in self.metadata.variables: + for var, value in self._metadata.variables: self.set_value(var, value) - self.metadata.is_evaluated = True + self._metadata.is_evaluated = True return self def freeze_definition_hash(self): - self.original_definition_hash = self.get_definition_hash() + self._original_definition_hash = self.get_definition_hash() def get_original_definition_hash(self): - return self.original_definition_hash + return self._original_definition_hash def as_bag(self): """ @@ -450,11 +528,8 @@ class Concept: It quicker to implement than creating the actual property mechanism with @property And it removes the visibility from the other attributes/methods """ - bag = {} - for var in self.values: - if isinstance(var, str): - bag[var] = self.get_value(var) - bag["var." + var] = self.get_value(var) + bag = self.variables() + for prop in ("id", "name", "key", "body"): bag[prop] = getattr(self, prop) return bag @@ -467,29 +542,14 @@ class Concept: from core.builtin_concepts import BuiltinConcepts self.set_prop(BuiltinConcepts.FORMAT_INSTRUCTIONS, instructions) + def set_format_instr(self, **kwargs): + self._format = kwargs -class Property: - """ - Defines the variables of a concept - It as its specific class, because from experience, - property management is more complex than a key/value pair - """ + def get_format_instr(self, key): + if self._format is None: + return None - def __init__(self, name, value): - self.name = name - self.value = value - - def __repr__(self): - return f"{self.name}={self.value}" - - def __eq__(self, other): - if not isinstance(other, Property): - return False - - return self.name == other.name and self.value == other.value - - def __hash__(self): - return hash((self.name, self.value)) + return self._format.get(key, None) @dataclass() @@ -500,7 +560,7 @@ class DoNotResolve: For example, if you want to set a value to the BODY that will not change when when the concept will be evaluated, - set concept.compiled[BODY] to DoNotResolve(value) + set concept._compiled[BODY] to DoNotResolve(value) """ value: object @@ -514,16 +574,6 @@ class InfiniteRecursionResolved: return self.value -def ensure_concept(*concepts): - if hasattr(concepts, "__iter__"): - for concept in concepts: - if not isinstance(concept, Concept): - raise TypeError(f"'{concept}' must be a concept") - else: - if not isinstance(concepts, Concept): - raise TypeError(f"'{concepts}' must be a concept") - - # ################################ # # Class created for tests purpose @@ -563,9 +613,9 @@ class CC: if other.key != self.concept_key: return False if self.exclude_body: - to_compare = {k: v for k, v in other.compiled.items() if k != ConceptParts.BODY} + to_compare = {k: v for k, v in other.get_compiled().items() if k != ConceptParts.BODY} else: - to_compare = other.compiled + to_compare = other.get_compiled() if self.compiled == to_compare: return True else: @@ -647,10 +697,9 @@ class CV: self.concept = concept if isinstance(concept, Concept) else None self.values = {} for k, v in kwargs.items(): - try: - concept_part = ConceptParts(k) - self.values[concept_part] = v - except ValueError: + if f"#{k}#" in AllConceptParts: + self.values[f"#{k}#"] = v + else: self.values[k] = v def __eq__(self, other): @@ -694,10 +743,10 @@ class CMV: if other.key != self.concept_key: return False - if len(other.metadata.variables) != len(self.variables): + if len(other._metadata.variables) != len(self.variables): return False - for name, value in other.metadata.variables: + for name, value in other._metadata.variables: if self.variables[name] != value: return False return True diff --git a/src/core/error.py b/src/core/error.py new file mode 100644 index 0000000..a1b0489 --- /dev/null +++ b/src/core/error.py @@ -0,0 +1,6 @@ +class ErrorObj: + """ + To indicate that somehow, the underlying object is (or has) an error + """ + pass + diff --git a/src/core/global_symbols.py b/src/core/global_symbols.py new file mode 100644 index 0000000..b09d317 --- /dev/null +++ b/src/core/global_symbols.py @@ -0,0 +1,8 @@ +# events +CONCEPT_PRECEDENCE_MODIFIED = "cpm" +RULE_PRECEDENCE_MODIFIED = "rpm" +CONTEXT_DISPOSED = "cd" + +# comparison context +RULE_COMPARISON_CONTEXT = "Rule" +CONCEPT_COMPARISON_CONTEXT = "Sya" diff --git a/src/core/profiling.py b/src/core/profiling.py index fe9ba5c..145bfd9 100644 --- a/src/core/profiling.py +++ b/src/core/profiling.py @@ -6,19 +6,41 @@ import pstats from cProfile import Profile -def profile(sort_args=None, print_args=None): - sort_args = sort_args or ['cumulative'] - print_args = print_args or [20] +# sort by +# 'calls' : call count +# 'cumulative' : cumulative time +# 'cumtime' : cumulative time +# 'file' : file name +# 'filename' : file name +# 'module' : file name +# 'ncalls' : call count +# 'pcalls' : primitive call count +# 'line' : line number +# 'name' : function name +# 'nfl' : name / file / line +# 'stdname' : standard name +# 'time' : internal time +# 'tottime' : internal time + +def profile(sort_args=None, print_args=None, filename=None): + sort_args = sort_args or ["cumulative"] + print_args = print_args or [2000] profiler = Profile() def decorator(fn): def inner(*args, **kwargs): - result = None try: result = profiler.runcall(fn, *args, **kwargs) finally: - stats = pstats.Stats(profiler) - stats.strip_dirs().sort_stats(*sort_args).print_stats(*print_args) + if filename: + with open(filename + ".txt", "w") as out: + stats = pstats.Stats(profiler, stream=out) + stats.strip_dirs().sort_stats(*sort_args).print_stats(*print_args) + profiler.dump_stats(filename + ".prof") + else: + stats = pstats.Stats(profiler) + stats.strip_dirs().sort_stats(*sort_args).print_stats(*print_args) + return result return inner diff --git a/src/core/rule.py b/src/core/rule.py new file mode 100644 index 0000000..4960bb8 --- /dev/null +++ b/src/core/rule.py @@ -0,0 +1,84 @@ +from dataclasses import dataclass +from typing import Union + +import core.utils + +ACTION_TYPE_PRINT = "print" +ACTION_TYPE_EXEC = "exec" +ACTION_TYPE_DEFERRED = "deferred" + + +@dataclass +class RuleMetadata: + action_type: str # print, exec, deferred + name: Union[str, None] + predicate: str + action: str + + id: str = None + is_compiled: bool = False + is_enabled: bool = False + + +class Rule: + def __init__(self, + action_type=ACTION_TYPE_EXEC, + name=None, + predicate=None, + action=None, + priority=None, + is_enabled=None): + self.metadata = RuleMetadata(action_type, name, predicate, action, is_enabled=is_enabled) + self.compiled_predicate = None + self.compiled_action = None + from core.sheerka.services.SheerkaComparisonManager import SheerkaComparisonManager + self.priority = priority if priority is not None else SheerkaComparisonManager.DEFAULT_COMPARISON_VALUE + self.error_sink = None + + def __repr__(self): + return f"Rule(#{self.metadata.id}, when '{self.metadata.predicate}' {self.metadata.action_type} '{self.metadata.action}', priority={self.priority})" + + def __eq__(self, other): + if id(other) == id(self): + return True + + if not isinstance(other, Rule): + return False + + for p in ["name", "predicate", "action_type", "action", "id"]: + if getattr(self.metadata, p) != getattr(other.metadata, p): + return False + + return True + + def __hash__(self): + return hash((self.metadata.name, + self.metadata.predicate, + self.metadata.action_type, + self.metadata.action)) + + def set_id(self, rule_id): + self.metadata.id = rule_id + return self + + def to_tuple_id(self): + return self.metadata.name, self.id + + @property + def id(self): + return self.metadata.id + + @property + def name(self): + return self.metadata.name + + @property + def key(self): + return self.metadata.name + + @property + def str_id(self): + return core.utils.str_concept(self, drop_name=True, prefix="r:") + + def short_str(self): + return f"Rule(#{self.metadata.id}, '{self.metadata.predicate}', priority={self.priority})" diff --git a/src/core/sheerka/ExecutionContext.py b/src/core/sheerka/ExecutionContext.py index 9faeab1..5abd0d8 100644 --- a/src/core/sheerka/ExecutionContext.py +++ b/src/core/sheerka/ExecutionContext.py @@ -1,17 +1,26 @@ import logging +import os +import pprint import time from core.builtin_concepts import BuiltinConcepts, ParserResultConcept -from core.concept import Concept +from core.concept import Concept, get_concept_attrs +from core.global_symbols import CONTEXT_DISPOSED from core.sheerka.services.SheerkaExecute import NO_MATCH from core.sheerka.services.SheerkaMemory import SheerkaMemory -from core.sheerka_logger import get_logger +from core.utils import CONSOLE_COLORS_MAP as CCM from sdp.sheerkaDataProvider import Event +try: + rows, columns = os.popen('stty size', 'r').read().split() +except ValueError: + rows, columns = 50, 80 + +pp = pprint.PrettyPrinter(indent=2, width=columns) + DEBUG_TAB_SIZE = 4 PROPERTIES_TO_SERIALIZE = ("_id", - "_bag", "_children", "_start", "_stop", @@ -50,19 +59,17 @@ class ExecutionContext: logger=None, global_hints=None, errors=None, - **kwargs): + obj=None, + concepts=None): self._id = ExecutionContext.get_id(event.get_digest()) if event else None self._parent = None self._children = [] - self._tab = "" - self._bag = {} # context variables self._start = 0 # when the execution starts (to measure elapsed time) self._stop = 0 # when the execution stops (to measure elapses time) self._logger = logger self._format_instructions = None # how to print the execution context - self._stat_log = get_logger("stats") - self._show_stats = False + self._push = None self.who = who # who is asking self.event = event # what was the (original) trigger @@ -70,6 +77,8 @@ class ExecutionContext: self.action = action self.action_context = action_context self.desc = desc # human description of what is going on + self.preprocess_parsers = None + self.preprocess_evaluators = None self.preprocess = None self.stm = False # True if the context has short term memory entries @@ -80,13 +89,11 @@ class ExecutionContext: self.inputs = {} # what were the parameters of the execution context self.values = {} # what was produced by the execution context - self.obj = kwargs.pop("obj", None) # current obj we are working on + self.obj = obj + self.concepts = concepts - self.concepts = kwargs.pop("concepts", {}) # known concepts specific to this context - - # update the other elements - for k, v in kwargs.items(): - self._bag[k] = v + self_debug, self.debug_mode = sheerka.get_context_debug_mode(self.id) + self.debug_enabled = self_debug is not None @property def elapsed(self): @@ -117,24 +124,19 @@ class ExecutionContext: """ return self._children - def __getattr__(self, item): - if item in self._bag: - return self._bag[item] - - raise AttributeError(f"'ExecutionContext' object has no attribute '{item}'") - def __enter__(self): self._start = time.time_ns() - self.log_new() + # self.log_new() return self def __exit__(self, exc_type, exc_val, exc_tb): + if self._push: + return + if self.stm: - self.sheerka.services[SheerkaMemory.NAME].remove_context(self) + self.sheerka.publish(self, CONTEXT_DISPOSED) self._stop = time.time_ns() - if self._show_stats: - self._stat_log.debug(f"[{self._id:2}]" + self._tab + "Execution time: " + self.elapsed_str) def __repr__(self): msg = f"ExecutionContext(who={self.who}, id={self._id}, action={self.action}, context={self.action_context}" @@ -143,11 +145,6 @@ class ExecutionContext: msg += ")" return msg - # def __str__(self): - # msg = self.desc or "New Context" - # msg += f", who={self.who}, id={self.id}" - # return msg - def __eq__(self, other): if id(self) == id(other): return True @@ -168,12 +165,12 @@ class ExecutionContext: return True - def push(self, action: BuiltinConcepts, action_context, who=None, desc=None, logger=None, **kwargs): + def push(self, action: BuiltinConcepts, action_context, who=None, desc=None, logger=None, obj=None, concepts=None): + if self._push: + return self._push + who = who or self.who logger = logger or self._logger - _kwargs = {"obj": self.obj, "concepts": self.concepts} - _kwargs.update(self._bag) - _kwargs.update(kwargs) new = ExecutionContext( who, self.event, @@ -184,19 +181,40 @@ class ExecutionContext: logger, self.global_hints, self.errors, - **_kwargs) + obj or self.obj, + concepts or self.concepts) new._parent = self - new._tab = self._tab + " " * DEBUG_TAB_SIZE new.preprocess = self.preprocess + new.preprocess_parsers = self.preprocess_parsers + new.preprocess_evaluators = self.preprocess_evaluators new.protected_hints.update(self.protected_hints) + if new.debug_mode is None and self.debug_mode == "protected": + new.debug_mode = "protected" + new.debug_enabled = True + self._children.append(new) return new + def deactivate_push(self): + self._push = self.push(BuiltinConcepts.NOP, None) + self._push._push = self._push + if self.stm: + bag = self.sheerka.services[SheerkaMemory.NAME].get_all_short_term_memory(self) + self.sheerka.add_many_to_short_term_memory(self._push, bag) + + def activate_push(self): + if self._push: + if self._push.stm: + self.sheerka.publish(self._push, CONTEXT_DISPOSED) + self._push._stop = time.time_ns() + + self._push = None + def add_preprocess(self, name, **kwargs): preprocess = self.sheerka.new(BuiltinConcepts.EVALUATOR_PRE_PROCESS) - preprocess.set_value("name", name) + preprocess.set_value("preprocess_name", name) for k, v in kwargs.items(): preprocess.set_value(k, v) @@ -206,13 +224,17 @@ class ExecutionContext: return self def add_inputs(self, **kwargs): - for k, v in kwargs.items(): - self.inputs[k] = v + if self._push: + return + + self.inputs.update(kwargs) return self def add_values(self, **kwargs): - for k, v in kwargs.items(): - self.values[k] = v + if self._push: + return + + self.values.update(kwargs) return self def add_to_short_term_memory(self, key, concept): @@ -224,6 +246,9 @@ class ExecutionContext: """ self.sheerka.add_to_short_term_memory(self, key, concept) + def clear_short_term_memory(self): + self.sheerka.clear_short_term_memory(self) + def get_from_short_term_memory(self, key): """ @@ -237,11 +262,10 @@ class ExecutionContext: if isinstance(self.obj, Concept): if self.obj.key == key: return self.obj - for var_name in self.obj.values: - if var_name == key: - value = self.obj.get_value(var_name) - if isinstance(value, Concept): - return value + if key in get_concept_attrs(self.obj): + value = self.obj.get_value(key) + if isinstance(value, Concept): + return value # search in concepts if self.concepts: @@ -296,8 +320,34 @@ class ExecutionContext: to_str = self.return_value_to_str(r) self._logger.debug(f"[{self._id:2}]" + self._tab + "-> " + to_str) - def debug(self, text): - print(text) + def get_debugger(self, who, method_name): + return self.sheerka.get_debugger(self, who, method_name) + + def debug(self, who, method_name, variable_name, text, is_error=False): + activated = self.sheerka.debug_activated_for(who) + if activated: + str_text = pp.pformat(text) + color = 'red' if is_error else 'green' + if "\n" not in str(str_text): + self.sheerka.debug( + f"[{self._id:3}] {CCM[color]}{who}.{method_name}.{variable_name}: {CCM['reset']}{str_text}") + else: + self.sheerka.debug(f"[{self._id:3}] {CCM[color]}{who}.{method_name}.{variable_name}: {CCM['reset']}") + self.sheerka.debug(str_text) + + def debug_entering(self, who, method_name, **kwargs): + if self.sheerka.debug_activated_for(who): + str_text = pp.pformat(kwargs) + if "\n" not in str(str_text): + self.sheerka.debug( + f"[{self._id:3}] {CCM['blue']}Entering {who}.{method_name} with {CCM['reset']}{str_text}") + else: + self.sheerka.debug(f"[{self._id:3}] {CCM['blue']}Entering {who}.{method_name}:{CCM['reset']}") + self.sheerka.debug(f"[{self._id:3}] {str_text}") + + def debug_log(self, who, text): + if self.sheerka.debug_activated_for(who): + self.sheerka.debug(f"[{self._id:3}] {CCM['blue']}{text}{CCM['reset']}") def get_parent(self): return self._parent @@ -384,9 +434,6 @@ class ExecutionContext: And it removes the visibility from the other attributes/methods """ bag = {} - for k, v in self._bag.items(): - bag[k] = v - bag["bag." + k] = v for prop in ("id", "who", "action", "desc", "obj", "inputs", "values", "concepts"): bag[prop] = getattr(self, prop) bag["context"] = self.action_context @@ -396,6 +443,7 @@ class ExecutionContext: bag["elapsed"] = self.elapsed bag["elapsed_str"] = self.elapsed_str bag["digest"] = self.event.get_digest() if self.event else None + bag["_children"] = self._children return bag @staticmethod @@ -438,3 +486,17 @@ class ExecutionContext: break current = current._parent + + def has_parent(self, context_id): + current = self + + while current._parent: + current = current._parent + if current.id == context_id: + return True + if current.id < context_id: + return False + + return False + + diff --git a/src/core/sheerka/Sheerka.py b/src/core/sheerka/Sheerka.py index 4c15f87..6458ff2 100644 --- a/src/core/sheerka/Sheerka.py +++ b/src/core/sheerka/Sheerka.py @@ -11,8 +11,10 @@ from cache.IncCache import IncCache from cache.ListIfNeededCache import ListIfNeededCache from cache.SetCache import SetCache from core.builtin_concepts import BuiltinConcepts, ErrorConcept, ReturnValueConcept, BuiltinErrors, BuiltinUnique, \ - UnknownConcept -from core.concept import Concept, ConceptParts, PROPERTIES_FOR_NEW + UnknownConcept, AllBuiltinConcepts +from core.concept import Concept, ConceptParts, NotInit, get_concept_attrs +from core.error import ErrorObj +from core.profiling import profile from core.sheerka.ExecutionContext import ExecutionContext from core.sheerka_logger import console_handler from core.tokenizer import Token, TokenKind @@ -66,15 +68,18 @@ class Sheerka(Concept): MAX_EXECUTION_HISTORY = 100 MAX_RETURN_VALUES_HISTORY = 100 + ALL_ATTRIBUTES = [] + def __init__(self, cache_only=False, debug=False, loggers=None): self.init_logging(debug, loggers) self.loggers = loggers super().__init__(BuiltinConcepts.SHEERKA, True, True, BuiltinConcepts.SHEERKA) - self.log.debug("Starting Sheerka.") + # self.log.debug("Starting Sheerka.") self.bnp = None # reference to the BaseNodeParser class (to compute first keyword token) self.return_value_concept_id = None + self.error_concept_id = None # a concept can be instantiated # ex: File is a concept, but File('foo.txt') is an instance @@ -85,7 +90,7 @@ class Sheerka(Concept): # ex: hello => say('hello') self.rules = [] - self.sdp: SheerkaDataProvider = None # SheerkaDataProvider + self.sdp: SheerkaDataProvider = None self.cache_manager = CacheManager(cache_only) self.services = {} # sheerka plugins @@ -103,6 +108,7 @@ class Sheerka(Concept): self._builtins_classes_cache = None self.save_execution_context = True + self.enable_process_return_values = False self.methods_with_context = {"test_using_context"} # only the names, the method is defined in sheerka_methods self.sheerka_methods = { @@ -171,17 +177,22 @@ class Sheerka(Concept): """ self.sheerka_pipeables[func_name] = SheerkaMethod(function, has_side_effect) - def initialize(self, root_folder: str = None, save_execution_context=True): + def initialize(self, root_folder: str = None, save_execution_context=None, enable_process_return_values=None): """ Starting Sheerka Loads the current configuration Notes that when it's the first time, it also create the needed working folders :param root_folder: root configuration folder :param save_execution_context: + :param enable_process_return_values: :return: ReturnValue(Success or Error) """ - self.save_execution_context = save_execution_context + if save_execution_context is not None: + self.save_execution_context = save_execution_context + + if enable_process_return_values is not None: + self.enable_process_return_values = enable_process_return_values try: from sheerkapickle.sheerka_handlers import initialize_pickle_handlers @@ -189,7 +200,10 @@ class Sheerka(Concept): self.sdp = SheerkaDataProvider(root_folder, self) self.initialize_caching() + self.get_builtin_parsers() + self.get_builtin_evaluators() self.initialize_services() + self.initialize_builtin_evaluators() event = Event("Initializing Sheerka.", user_id=self.name) self.sdp.save_event(event) @@ -198,25 +212,24 @@ class Sheerka(Concept): self, BuiltinConcepts.INIT_SHEERKA, None, - desc="Initializing Sheerka.", - logger=self.init_log) as exec_context: + desc="Initializing Sheerka.") as exec_context: if self.sdp.first_time: self.first_time_initialisation(exec_context) - self.initialize_builtin_parsers() - self.initialize_builtin_evaluators() self.initialize_builtin_concepts() self.initialize_concept_node_parsing(exec_context) - res = ReturnValueConcept(self, True, self) + self.initialize_services_deferred(exec_context, self.sdp.first_time) + + res = ReturnValueConcept(self, True, self) exec_context.add_values(return_values=res) if self.cache_manager.is_dirty: self.cache_manager.commit(exec_context) - if save_execution_context: + if self.save_execution_context: self.sdp.save_result(exec_context, is_admin=True) - self.init_log.debug(f"Sheerka successfully initialized") + # self.init_log.debug(f"Sheerka successfully initialized") except IOError as e: res = ReturnValueConcept(self, False, self.get(BuiltinConcepts.ERROR), e) @@ -276,7 +289,7 @@ class Sheerka(Concept): Introspect to find services and bind them :return: """ - self.init_log.debug("Initializing services") + # self.init_log.debug("Initializing services") core.utils.import_module_and_sub_module('core.sheerka.services') base_class = "core.sheerka.services.sheerka_service.BaseService" @@ -286,49 +299,63 @@ class Sheerka(Concept): instance.initialize() self.services[service.NAME] = instance + def initialize_services_deferred(self, context, is_first_time): + """ + Initialize part of services that may takes some time or that need the execution context + TODO: Create a separate thread for these initialisations as they may take time + :return: + """ + # self.init_log.debug("Initializing services (deferred)") + + for service in self.services.values(): + if hasattr(service, "initialize_deferred"): + service.initialize_deferred(context, is_first_time) + def first_time_initialisation(self, context): self.cache_manager.put(self.CONCEPTS_KEYS_ENTRY, self.USER_CONCEPTS_KEYS, 1000) - self.record(context, self.name, "save_execution_context", True) + self.record_var(context, self.name, "save_execution_context", True) def initialize_builtin_concepts(self): """ Initializes the builtin concepts :return: None """ - self.init_log.debug("Initializing builtin concepts") + # self.init_log.debug("Initializing builtin concepts") builtins_classes = self.get_builtins_classes_as_dict() # this all initialization of the builtins seems to be little bit complicated # why do we need to update it from DB ? - for key in BuiltinConcepts: + for key in AllBuiltinConcepts: concept = self if key == BuiltinConcepts.SHEERKA \ else builtins_classes[str(key)]() if str(key) in builtins_classes \ else Concept(key, True, False, key) if key in BuiltinUnique: - concept.metadata.is_unique = True - concept.metadata.is_evaluated = True + concept._metadata.is_unique = True + concept._metadata.is_evaluated = True - if not concept.metadata.is_unique and str(key) in builtins_classes: + if not concept._metadata.is_unique and str(key) in builtins_classes: self.builtin_cache[key] = builtins_classes[str(key)] - from_db = self.cache_manager.get(self.CONCEPTS_BY_KEY_ENTRY, concept.metadata.key) + from_db = self.cache_manager.get(self.CONCEPTS_BY_KEY_ENTRY, concept._metadata.key) if from_db is None: - self.init_log.debug(f"'{concept.name}' concept is not found in db. Adding.") + # self.init_log.debug(f"'{concept.name}' concept is not found in db. Adding.") self.set_id_if_needed(concept, True) self.cache_manager.add_concept(concept) if key == BuiltinConcepts.RETURN_VALUE: self.return_value_concept_id = concept.id + elif key == BuiltinConcepts.ERROR: + self.error_concept_id = concept.id else: - self.init_log.debug(f"Found concept '{from_db}' in db. Updating.") + # self.init_log.debug(f"Found concept '{from_db}' in db. Updating.") concept.update_from(from_db) return - def initialize_builtin_parsers(self): + def get_builtin_parsers(self): """ Init the parsers :return: @@ -343,7 +370,7 @@ class Sheerka(Concept): continue qualified_name = core.utils.get_full_qualified_name(parser) - self.init_log.debug(f"Adding builtin parser '{qualified_name}'") + # self.init_log.debug(f"Adding builtin parser '{qualified_name}'") temp_result[qualified_name] = parser # keep a reference to base_node_parser @@ -361,22 +388,29 @@ class Sheerka(Concept): self.parsers[name] = temp_result[name] + def get_builtin_evaluators(self): + """ + get all evaluators + :return: + """ + core.utils.import_module_and_sub_module("evaluators") + evaluators = core.utils.get_sub_classes("evaluators", "evaluators.BaseEvaluator.OneReturnValueEvaluator") + evaluators.extend(core.utils.get_sub_classes("evaluators", "evaluators.BaseEvaluator.AllReturnValuesEvaluator")) + + for evaluator in evaluators: + self.evaluators.append(evaluator) + def initialize_builtin_evaluators(self): """ Init the evaluators :return: """ - core.utils.import_module_and_sub_module("evaluators") - for evaluator in core.utils.get_sub_classes("evaluators", "evaluators.BaseEvaluator.OneReturnValueEvaluator"): - self.init_log.debug(f"Adding builtin evaluator '{evaluator.__name__}'") - self.evaluators.append(evaluator) - - for evaluator in core.utils.get_sub_classes("evaluators", "evaluators.BaseEvaluator.AllReturnValuesEvaluator"): - self.init_log.debug(f"Adding builtin evaluator '{evaluator.__name__}'") - self.evaluators.append(evaluator) + for evaluator in self.evaluators: + if hasattr(evaluator, "initialize"): + evaluator.initialize(self) def initialize_concept_node_parsing(self, context): - self.init_log.debug("siInitializing concepts by first keyword.") + # self.init_log.debug("Initializing concepts by first keyword.") concepts_by_first_keyword = self.cache_manager.copy(self.CONCEPTS_BY_FIRST_KEYWORD_ENTRY) res = self.bnp.resolve_concepts_by_first_keyword(context, concepts_by_first_keyword) @@ -391,11 +425,16 @@ class Sheerka(Concept): service.initialize() else: self.cache_manager.clear() + + for service in self.services.values(): + if hasattr(service, "reset"): + service.reset() + self.printer_handler.reset() self.sdp.reset() self.locals = {} - # @profile() + # @profile(filename="profile_80") def evaluate_user_input(self, text: str, user_name="kodjo"): """ Note to KSI: If you try to add execution context to this function, @@ -404,19 +443,20 @@ class Sheerka(Concept): :param user_name: :return: """ - self.log.debug(f"Processing user input '{text}', {user_name=}.") + # self.log.debug(f"Processing user input '{text}', {user_name=}.") event = Event(text, user_name) - evt_digest = self.sdp.save_event(event) - self.log.debug(f"{evt_digest=}") + self.sdp.save_event(event) with ExecutionContext(self.key, event, self, BuiltinConcepts.PROCESS_INPUT, text, - desc=f"Evaluating '{text}'", - logger=self.log) as execution_context: + desc=f"Evaluating '{text}'") as execution_context: + user_input = self.ret(self.name, True, self.new(BuiltinConcepts.USER_INPUT, body=text, user_name=user_name)) + + # TODO. Must be a context hint, not a return value reduce_requested = self.ret(self.name, True, self.new(BuiltinConcepts.REDUCE_REQUESTED)) ret = self.execute(execution_context, [user_input, reduce_requested], EXECUTE_STEPS) @@ -425,17 +465,21 @@ class Sheerka(Concept): if self.cache_manager.is_dirty: self.cache_manager.commit(execution_context) - try: - if self.save_execution_context and self.load(self.name, "save_execution_context"): + # exec_count = ExecutionContext.ids[execution_context.event.get_digest()] + # print("Execution Context Count:", exec_count) + if self.save_execution_context: + try: + # if exec_count > 3400: + # print("Saving result. digest=", execution_context.event.get_digest()) self.sdp.save_result(execution_context) - except Exception as ex: - self.log.error(f"Failed to save execution context. Reason: {ex}") + except Exception as ex: + print(f"Failed to save execution context. Reason: {ex}") + pass + # self.log.error(f"Failed to save execution context. Reason: {ex}") - # # hack to save valid concept definition - # if not self.during_restore: - # if len(ret) == 1 and ret[0].status and self.isinstance(ret[0].value, BuiltinConcepts.NEW_CONCEPT): - # with open(CONCEPTS_FILE, "a") as f: - # f.write(text + "\n") + # Do not save execution contexts from process_return_values + if self.enable_process_return_values: + self.process_return_values(execution_context, ret) self.execution_count += 1 self._last_execution = execution_context @@ -461,17 +505,16 @@ class Sheerka(Concept): def set_id_if_needed(self, obj: Concept, is_builtin: bool): """ Set the key for the concept if needed - For test purpose only !!!!! :param obj: :param is_builtin: :return: """ - if obj.metadata.id is not None: + if obj._metadata.id is not None: return key = self.BUILTIN_CONCEPTS_KEYS if is_builtin else self.USER_CONCEPTS_KEYS - obj.metadata.id = str(self.cache_manager.get(self.CONCEPTS_KEYS_ENTRY, key)) - self.log.debug(f"Setting id '{obj.metadata.id}' to concept '{obj.metadata.name}'.") + obj._metadata.id = str(self.cache_manager.get(self.CONCEPTS_KEYS_ENTRY, key)) + # self.log.debug(f"Setting id '{obj.metadata.id}' to concept '{obj.metadata.name}'.") def force_sya_def(self, context, list_of_def): """ @@ -588,9 +631,9 @@ class Sheerka(Concept): # ############## # if the entry is a concept token, use its values. if isinstance(concept, Token): - if concept.type != TokenKind.CONCEPT: + if concept.type == TokenKind.RULE: # do not recognize rules !!! return None - concept = concept.value + concept = concept.value # concept is now a tuple if isinstance(concept, str) and \ concept.startswith("c:") and \ @@ -607,7 +650,7 @@ class Sheerka(Concept): if concept[1]: if self.is_known(found := self.get_by_id(concept[1])): instance = self.new_from_template(found, found.key) - instance.metadata.is_evaluated = True + instance._metadata.is_evaluated = True return instance elif concept[0]: if self.is_known(found := self.get_by_name(concept[0])): @@ -626,6 +669,28 @@ class Sheerka(Concept): return None + def fast_resolve(self, key, return_new=True): + def new_instances(concepts): + if hasattr(concepts, "__iter__"): + return [self.new_from_template(c, c.key) for c in concepts] + return self.new_from_template(concepts, concepts.key) + + if isinstance(key, Token): + if key.type == TokenKind.RULE: # do not recognize rules !!! + return None + + if key.value[1]: + concept = self.cache_manager.get(self.CONCEPTS_BY_ID_ENTRY, key.value[1]) + else: + concept = self.cache_manager.get(self.CONCEPTS_BY_NAME_ENTRY, key.value[0]) + + else: + concept = self.cache_manager.get(self.CONCEPTS_BY_NAME_ENTRY, key) + + if concept is None: + return None + return new_instances(concept) if return_new else concept + def has_id(self, concept_id): """ Returns True if a concept with this id exists in cache @@ -694,13 +759,13 @@ class Sheerka(Concept): def new_from_template(self, template, key, **kwargs): # core.utils.my_debug(f"Created {template}, {key=}, {kwargs=}") # manage singleton - if template.metadata.is_unique: + if template.get_metadata().is_unique: return template # otherwise, create another instance concept = self.builtin_cache[key]() if key in self.builtin_cache else Concept() concept.update_from(template, update_value=False) - concept.freeze_definition_hash() + # concept.freeze_definition_hash() if len(kwargs) == 0: return concept @@ -708,17 +773,17 @@ class Sheerka(Concept): # update the properties, values, attributes # Not quite sure that this is the correct process order for k, v in kwargs.items(): - if k in concept.values: + if k in get_concept_attrs(concept): concept.set_value(k, v) - elif k in PROPERTIES_FOR_NEW: - concept.set_value(ConceptParts(k), v) + elif k == "body": + concept.set_value(ConceptParts.BODY, v) elif hasattr(concept, k): setattr(concept, k, v) else: return self.new(BuiltinConcepts.UNKNOWN_PROPERTY, body=k, concept=concept) # TODO : add the concept to the list of known concepts (self.instances) - concept.metadata.is_evaluated = True # because we have manually set the variables + concept._metadata.is_evaluated = True # because we have manually set the variables return concept def ret(self, who: str, status: bool, value, message=None, parents=None): @@ -732,22 +797,16 @@ class Sheerka(Concept): :return: """ - # 1 second saved every twenty seconds in unit tests return ReturnValueConcept( who=who, status=status, value=value, - message=message, parents=parents, concept_id=self.return_value_concept_id ) - # return self.new( - # BuiltinConcepts.RETURN_VALUE, - # who=who, - # status=status, - # value=value, - # message=message, - # parents=parents) + + def err(self, body): + return ErrorConcept(body, self.error_concept_id) def objvalue(self, obj, reduce_simple_list=False): if obj is None: @@ -759,7 +818,7 @@ class Sheerka(Concept): if not isinstance(obj, Concept): return obj - if obj.body is BuiltinConcepts.NOT_INITIALIZED: + if obj.body is NotInit: return obj if reduce_simple_list and (isinstance(obj.body, list) or isinstance(obj.body, set)) and len(obj.body) == 1: @@ -796,7 +855,7 @@ class Sheerka(Concept): return self.value_by_concept(obj.body, concept) def get_error(self, obj): - if isinstance(obj, Concept) and obj.metadata.is_builtin and obj.key in BuiltinErrors: + if isinstance(obj, Concept) and obj._metadata.is_builtin and obj.key in BuiltinErrors: return obj if isinstance(obj, (list, set, tuple)): @@ -848,9 +907,9 @@ class Sheerka(Concept): def test(self): return f"I have access to Sheerka !" - def test_using_context(self, context, param1, param2): + def test_using_context(self, context, param): event = context.event.get_digest() - return f"I have access to Sheerka ! {param1=}, {param2=}, {event=}." + return f"I have access to Sheerka ! {param=}, {event=}." def test_error(self): raise Exception("I can raise an error") @@ -863,14 +922,17 @@ class Sheerka(Concept): if isinstance(obj, ReturnValueConcept): return obj.status + if isinstance(obj, ErrorObj): + return False + # other cases ? # ... # manage internal errors - if isinstance(obj, Concept) and obj.metadata.is_builtin and obj.key in BuiltinErrors: + if isinstance(obj, Concept) and obj._metadata.is_builtin and obj.key in BuiltinErrors: return False - return obj + return bool(obj) @staticmethod def is_known(obj): @@ -914,9 +976,7 @@ class Sheerka(Concept): unknown_concept = UnknownConcept() # don't use new() for prevent circular reference unknown_concept.set_value(ConceptParts.BODY, metadata) - for meta in (metadata if isinstance(metadata, list) else [metadata]): - unknown_concept.set_value(meta[0], meta[1]) - unknown_concept.metadata.is_evaluated = True + unknown_concept._metadata.is_evaluated = True return unknown_concept @staticmethod @@ -924,7 +984,7 @@ class Sheerka(Concept): res = {} for c in core.utils.get_classes("core.builtin_concepts"): if issubclass(c, Concept) and c != Concept: - res[c().metadata.key] = c + res[c()._metadata.key] = c return res @@ -967,3 +1027,29 @@ class Sheerka(Concept): logging.addLevelName(logging.ERROR, "\033[1;41m%s\033[1;0m" % logging.getLevelName(logging.ERROR)) # uncomment the following line to enable colors # logging.StreamHandler.emit = add_coloring_to_emit_ansi(logging.StreamHandler.emit) + + +def to_profile(): + sheerka = Sheerka() + sheerka.initialize(save_execution_context=False, enable_process_return_values=False) + event = Event("test", "kodjoko") + execution_context = ExecutionContext(sheerka.name, + event, + sheerka, + BuiltinConcepts.PROCESS_INPUT, + None) + + profile_push(execution_context) + + +@profile(filename="profile_push") +def profile_push(execution_context): + for i in range(177942): + execution_context.push(BuiltinConcepts.NOP, + {"action": "fake"}, + execution_context.sheerka.name, + desc="a proper description") + + +if __name__ == '__main__': + to_profile() diff --git a/src/core/sheerka/services/SheerkaAdmin.py b/src/core/sheerka/services/SheerkaAdmin.py index dcf4da7..e762704 100644 --- a/src/core/sheerka/services/SheerkaAdmin.py +++ b/src/core/sheerka/services/SheerkaAdmin.py @@ -1,12 +1,14 @@ +import sys import time from os import path -from core.builtin_concepts import BuiltinConcepts +from core.builtin_concepts import BuiltinConcepts, BuiltinContainers +from core.concept import Concept from core.sheerka.services.sheerka_service import BaseService CONCEPTS_FILE_LITE = "_concepts_lite.txt" -CONCEPTS_FILE_ALL_CONCEPTS = "_concepts.txt" -CONCEPTS_FILE_TO_USE = CONCEPTS_FILE_ALL_CONCEPTS +CONCEPTS_FILE_FULL = "_concepts_full.txt" +CONCEPTS_FILE_TO_USE = CONCEPTS_FILE_FULL class SheerkaAdmin(BaseService): @@ -22,6 +24,10 @@ class SheerkaAdmin(BaseService): self.sheerka.bind_service_method(self.concepts, False) self.sheerka.bind_service_method(self.last_created_concept, False) self.sheerka.bind_service_method(self.last_ret, False) + self.sheerka.bind_service_method(self.last_error_ret, False) + self.sheerka.bind_service_method(self.extended_isinstance, False) + self.sheerka.bind_service_method(self.is_container, False) + self.sheerka.bind_service_method(self.format_rules, False) def caches_names(self): """ @@ -53,18 +59,19 @@ class SheerkaAdmin(BaseService): def restore_from_file(file_name): _nb_lines, _nb_instructions, _nb_lines_in_error = 0, 0, 0 - if not path.exists(file_name): - self.sheerka.log.error(f"\u001b[31mFile '{file_name}' is not found !\u001b[0m") + file_path = path.join(path.dirname(sys.argv[0]), file_name) + if not path.exists(file_path): + print(f"\u001b[31mFile '{file_path}' is not found !\u001b[0m") return 0, 0, 1 - with open(file_name, "r") as f: + with open(file_path, "r") as f: for line in f.readlines(): _nb_lines += 1 line = line.strip() if line.startswith("#import "): to_import = "_concepts_" + line[8:] + ".txt" - self.sheerka.log.info(f"Importing {to_import}") + print(f"Importing {to_import}") res = restore_from_file(to_import) _nb_lines += res[0] _nb_instructions += res[1] @@ -74,42 +81,49 @@ class SheerkaAdmin(BaseService): if line == "" or line.startswith("#"): continue - self.sheerka.log.info(line) + print(line) _nb_instructions += 1 res = self.sheerka.evaluate_user_input(line) if len(res) > 1 or not res[0].status: _nb_lines_in_error += 1 - self.sheerka.log.error("\u001b[31mError detected !\u001b[0m") + print("\u001b[31mError detected !\u001b[0m") return _nb_lines, _nb_instructions, _nb_lines_in_error - if concept_file == "full": - concept_file = CONCEPTS_FILE_ALL_CONCEPTS - - elif not concept_file.startswith("_concepts"): + if not concept_file.startswith("_concepts"): concept_file = f"_concepts_{concept_file}.txt" try: start = time.time_ns() self.sheerka.during_restore = True + self.sheerka.save_execution_context = False + enable_process_return_values_previous_value = self.sheerka.enable_process_return_values + self.sheerka.enable_process_return_values = False + nb_lines, nb_instructions, nb_lines_in_error = restore_from_file(concept_file) + + self.sheerka.enable_process_return_values = enable_process_return_values_previous_value + self.sheerka.save_execution_context = True self.sheerka.during_restore = False stop = time.time_ns() nano_sec = stop - start dt = nano_sec / 1e6 elapsed = f"{dt} ms" if dt < 1000 else f"{dt / 1000} s" - self.sheerka.log.info(f"Imported {nb_lines} line(s) in {elapsed}.") - self.sheerka.log.info(f"{nb_instructions} instruction(s).") + print(f"Imported {nb_lines} line(s) in {elapsed}.") + print(f"{nb_instructions} instruction(s).") if nb_lines_in_error > 0: - self.sheerka.log.info(f"\u001b[31m{nb_lines_in_error} errors(s) found.\u001b[0m") + print(f"\u001b[31m{nb_lines_in_error} errors(s) found.\u001b[0m") else: - self.sheerka.log.info(f"No error.") + print(f"No error.") except IOError as e: raise e def concepts(self): - return self.sheerka.sdp.list(self.sheerka.CONCEPTS_BY_ID_ENTRY) + return self.sheerka.new(BuiltinConcepts.TO_LIST, body=self.sheerka.sdp.list(self.sheerka.CONCEPTS_BY_ID_ENTRY)) + + def format_rules(self): + return self.sheerka.new(BuiltinConcepts.TO_LIST, items=self.sheerka.get_format_rules()) def last_created_concept(self, use_history=False): for exec_result in reversed(self.sheerka.last_executions): @@ -124,4 +138,53 @@ class SheerkaAdmin(BaseService): return self.sheerka.new(BuiltinConcepts.NOT_FOUND) def last_ret(self, context, index=-1): - return self.sheerka.last_return_values[index] + try: + last = self.sheerka.last_return_values[index] + return last[0] if isinstance(last, list) and len(last) == 1 else last + except IndexError: + return None + + def last_error_ret(self, context, index=-1): + while index >= -len(self.sheerka.last_return_values): + last = self.sheerka.last_return_values[index] + last = [last] if not hasattr(last, "__iter__") else last + last = [ret_val for ret_val in last if not ret_val.status] + if len(last) == 0: + index -= 1 + continue + + if len(last) > 1: + return context.sheerka.ret(SheerkaAdmin.NAME, + False, + context.sheerka.new(BuiltinConcepts.TOO_MANY_ERRORS, body=last)) + + return last[0] + + return context.sheerka.ret(SheerkaAdmin.NAME, + False, + context.sheerka.new(BuiltinConcepts.NOT_FOUND)) + + def extended_isinstance(self, a, b): + """ + switch between sheerka.isinstance and builtin.isinstance + :param a: + :param b: + :return: + """ + + if isinstance(b, (type, tuple)): + return isinstance(a, b) + + return self.sheerka.isinstance(a, b) + + @staticmethod + def is_container(obj): + """ + A container concept is a builtin concept that embed a result + :param obj: + :return: + """ + if not isinstance(obj, Concept): + return False + + return obj.key in BuiltinContainers diff --git a/src/core/sheerka/services/SheerkaComparisonManager.py b/src/core/sheerka/services/SheerkaComparisonManager.py index b18ae46..6d0074a 100644 --- a/src/core/sheerka/services/SheerkaComparisonManager.py +++ b/src/core/sheerka/services/SheerkaComparisonManager.py @@ -3,7 +3,11 @@ from dataclasses import dataclass from cache.Cache import Cache from cache.ListCache import ListCache from core.builtin_concepts import BuiltinConcepts -from core.concept import ensure_concept, Concept +from core.global_symbols import CONCEPT_PRECEDENCE_MODIFIED, RULE_PRECEDENCE_MODIFIED, RULE_COMPARISON_CONTEXT, \ + CONCEPT_COMPARISON_CONTEXT +from core.builtin_helpers import ensure_concept_or_rule +from core.concept import Concept +from core.sheerka.services.SheerkaRuleManager import SheerkaRuleManager from core.sheerka.services.sheerka_service import ServiceObj, BaseService @@ -43,7 +47,7 @@ class SheerkaComparisonManager(BaseService): :return: """ if isinstance(prop_name, Concept): - prefix = prop_name.key if prop_name.metadata.is_builtin else prop_name.id + prefix = prop_name.key if prop_name.get_metadata().is_builtin else prop_name.id else: prefix = prop_name @@ -66,9 +70,11 @@ class SheerkaComparisonManager(BaseService): for _ in range(len(comparison_objs)): for comparison_obj in comparison_objs: if comparison_obj.op == ">": - values[comparison_obj.a] = values[comparison_obj.b] + 1 + if values[comparison_obj.a] <= values[comparison_obj.b]: + values[comparison_obj.a] = values[comparison_obj.b] + 1 else: - values[comparison_obj.b] = values[comparison_obj.a] + 1 + if values[comparison_obj.b] <= values[comparison_obj.a]: + values[comparison_obj.b] = values[comparison_obj.a] + 1 return values @@ -128,17 +134,17 @@ class SheerkaComparisonManager(BaseService): res.setdefault(v, []).append(k) return res - def _add_comparison(self, comparison_obj): + def _add_comparison(self, context, comparison_obj): key = self._compute_key(comparison_obj.property, comparison_obj.context) previous = self.sheerka.cache_manager.get(self.COMPARISON_ENTRY, key) new = previous.copy() if previous else [] for co in new: if co.property == comparison_obj.property and \ - co.a == comparison_obj.a and \ - co.b == comparison_obj.b and \ - co.op == comparison_obj.op and \ - co.context == comparison_obj.context: + co.a == comparison_obj.a and \ + co.b == comparison_obj.b and \ + co.op == comparison_obj.op and \ + co.context == comparison_obj.context: return self.sheerka.ret(self.NAME, False, self.sheerka.new(BuiltinConcepts.CONCEPT_ALREADY_DEFINED)) new.append(comparison_obj) @@ -166,7 +172,7 @@ class SheerkaComparisonManager(BaseService): cycles = self.detect_cycles(new) if cycles: - concepts_in_cycle = [self.sheerka.get_by_id(c) for c in cycles] + concepts_in_cycle = [self.sheerka.resolve(c) for c in cycles] chicken_an_egg = self.sheerka.new(BuiltinConcepts.CHICKEN_AND_EGG, body=concepts_in_cycle) return self.sheerka.ret(self.NAME, False, chicken_an_egg) @@ -175,6 +181,12 @@ class SheerkaComparisonManager(BaseService): lesser_objs_ids, greatest_objs_ids)) + if comparison_obj.property == BuiltinConcepts.PRECEDENCE: + if comparison_obj.context == CONCEPT_COMPARISON_CONTEXT: + self.sheerka.publish(context, CONCEPT_PRECEDENCE_MODIFIED) + elif comparison_obj.context == RULE_COMPARISON_CONTEXT: + self.sheerka.publish(context, RULE_PRECEDENCE_MODIFIED) + return self.sheerka.ret(self.NAME, True, self.sheerka.new(BuiltinConcepts.SUCCESS)) def initialize(self): @@ -191,41 +203,51 @@ class SheerkaComparisonManager(BaseService): self.sheerka.bind_service_method(self.get_partition, False) self.sheerka.bind_service_method(self.get_concepts_weights, False) - def set_is_greater_than(self, context, prop_name, concept_a, concept_b, comparison_context="#"): + def set_is_greater_than(self, context, prop_name, item_a, item_b, comparison_context="#"): """ Records that the property of concept a is greater than concept b's one :param context: :param prop_name: - :param concept_a: - :param concept_b: + :param item_a: + :param item_b: :param comparison_context: :return: """ - context.log(f"Setting concept {concept_a} is greater than {concept_b}", who=self.NAME) - ensure_concept(concept_a, concept_b) + context.log(f"Setting item {item_a} is greater than {item_b}", who=self.NAME) + ensure_concept_or_rule(item_a, item_b) event_digest = context.event.get_digest() - comparison_obj = ComparisonObj(event_digest, prop_name, concept_a.id, concept_b.id, ">", comparison_context) - return self._add_comparison(comparison_obj) + comparison_obj = ComparisonObj(event_digest, + prop_name, + item_a.str_id, + item_b.str_id, + ">", + comparison_context) + return self._add_comparison(context, comparison_obj) - def set_is_less_than(self, context, prop_name, concept_a, concept_b, comparison_context="#"): + def set_is_less_than(self, context, prop_name, item_a, item_b, comparison_context="#"): """ Records that the property of concept a is lesser than concept b's one :param context: :param prop_name: - :param concept_a: - :param concept_b: + :param item_a: + :param item_b: :param comparison_context: :return: """ - context.log(f"Setting concept {concept_a} is less than {concept_b}", who=self.NAME) - ensure_concept(concept_a, concept_b) + context.log(f"Setting item {item_a} is less than {item_b}", who=self.NAME) + ensure_concept_or_rule(item_a, item_b) event_digest = context.event.get_digest() - comparison_obj = ComparisonObj(event_digest, prop_name, concept_a.id, concept_b.id, "<", comparison_context) - return self._add_comparison(comparison_obj) + comparison_obj = ComparisonObj(event_digest, + prop_name, + item_a.str_id, + item_b.str_id, + "<", + comparison_context) + return self._add_comparison(context, comparison_obj) - def set_is_lesser(self, context, prop_name, concept, comparison_context="#"): + def set_is_lesser(self, context, prop_name, item, comparison_context="#"): """ Records that the concept is less than any other concept if no direct comparison is given @@ -235,18 +257,23 @@ class SheerkaComparisonManager(BaseService): * All lesser concepts that have no comparison directive are greater than the others (and share the same weight) :param context: :param prop_name: - :param concept: + :param item: :param comparison_context: :return: """ - context.log(f"Setting concept {concept} is lesser", who=self.NAME) - ensure_concept(concept) + context.log(f"Setting item {item} is lesser", who=self.NAME) + ensure_concept_or_rule(item) event_digest = context.event.get_digest() - comparison_obj = ComparisonObj(event_digest, prop_name, concept.id, None, "<<", comparison_context) - return self._add_comparison(comparison_obj) + comparison_obj = ComparisonObj(event_digest, + prop_name, + item.str_id, + None, + "<<", + comparison_context) + return self._add_comparison(context, comparison_obj) - def set_is_greatest(self, context, prop_name, concept, comparison_context="#"): + def set_is_greatest(self, context, prop_name, item, comparison_context="#"): """ Records that the concept is greater than any other concept if no direct comparison is given @@ -256,16 +283,21 @@ class SheerkaComparisonManager(BaseService): * All greatest concepts that have no comparison directive are less than the others (and share the same weight) :param context: :param prop_name: - :param concept: + :param item: :param comparison_context: :return: """ - context.log(f"Setting concept {concept} is greatest", who=self.NAME) - ensure_concept(concept) + context.log(f"Setting item {item} is greatest", who=self.NAME) + ensure_concept_or_rule(item) event_digest = context.event.get_digest() - comparison_obj = ComparisonObj(event_digest, prop_name, concept.id, None, ">>", comparison_context) - return self._add_comparison(comparison_obj) + comparison_obj = ComparisonObj(event_digest, + prop_name, + item.str_id, + None, + ">>", + comparison_context) + return self._add_comparison(context, comparison_obj) def set_are_equivalent(self, context, prop_name, concept_a, concept_b, comparison_context="#"): """ @@ -281,9 +313,6 @@ class SheerkaComparisonManager(BaseService): """ pass - def set_are_equiv(self, context, prop_name, concept_a, concept_b, comparison_context="#"): - pass - def get_partition(self, prop_name, comparison_context="#"): """ Returns the equivalent classes for the property, using the comparison_context diff --git a/src/core/sheerka/services/SheerkaConceptsAlgebra.py b/src/core/sheerka/services/SheerkaConceptsAlgebra.py index 3dee820..46458fd 100644 --- a/src/core/sheerka/services/SheerkaConceptsAlgebra.py +++ b/src/core/sheerka/services/SheerkaConceptsAlgebra.py @@ -2,7 +2,8 @@ from dataclasses import dataclass from operator import attrgetter from core.builtin_concepts import BuiltinConcepts -from core.concept import Concept, ensure_concept +from core.builtin_helpers import ensure_concept +from core.concept import Concept from core.sheerka.Sheerka import Sheerka from core.sheerka.services.sheerka_service import BaseService @@ -81,13 +82,13 @@ class SheerkaConceptsAlgebra(BaseService): :param key: :return: """ - if key not in source.metadata.props: + if key not in source.get_metadata().props: return - if key in destination.metadata.props: - destination.metadata.props[key].update(source.metadata.props[key]) + if key in destination.get_metadata().props: + destination.get_metadata().props[key].update(source.get_metadata().props[key]) else: - destination.metadata.props[key] = source.metadata.props[key].copy() + destination.get_metadata().props[key] = source.get_metadata().props[key].copy() def sub_props(self, destination, source, key): """ @@ -97,11 +98,11 @@ class SheerkaConceptsAlgebra(BaseService): :param key: :return: """ - if key not in source.metadata.props or key not in destination.metadata.props: + if key not in source.get_metadata().props or key not in destination.get_metadata().props: return - for item in source.metadata.props[key]: - destination.metadata.props[key].discard(item) + for item in source.get_metadata().props[key]: + destination.get_metadata().props[key].discard(item) def recognize(self, concept, all_scores=False): """ @@ -118,7 +119,7 @@ class SheerkaConceptsAlgebra(BaseService): return res all_concepts = self.sheerka.cache_manager.copy(Sheerka.CONCEPTS_BY_ID_ENTRY).values() \ - if self.sheerka.cache_manager.cache_only else self.sheerka.concepts() + if self.sheerka.cache_manager.cache_only else self.sheerka.sdp.list(self.sheerka.CONCEPTS_BY_ID_ENTRY) for c in all_concepts: score = self._compute_score(c, concept, step_b=round(1 / nb_props, 2)) @@ -127,8 +128,8 @@ class SheerkaConceptsAlgebra(BaseService): if len(res) == 0: props = [] - for p in [p for p in PROPERTIES_TO_COMPUTE if p in concept.metadata.props]: - props.append((p, concept.metadata.props[p])) + for p in [p for p in PROPERTIES_TO_COMPUTE if p in concept.get_metadata().props]: + props.append((p, concept.get_metadata().props[p])) return self.sheerka.get_unknown(props) res.sort(key=attrgetter('score'), reverse=True) @@ -158,9 +159,9 @@ class SheerkaConceptsAlgebra(BaseService): # adds step_b for every property that are in both a and b for prop in PROPERTIES_TO_COMPUTE: - if prop in b.metadata.props and prop in a.metadata.props: - for prop_value in b.metadata.props[prop]: - if prop_value in a.metadata.props[prop]: + if prop in b.get_metadata().props and prop in a.get_metadata().props: + for prop_value in b.get_metadata().props[prop]: + if prop_value in a.get_metadata().props[prop]: score += step_b if not step_a: @@ -171,11 +172,11 @@ class SheerkaConceptsAlgebra(BaseService): # remove step_a for every property that is in a, but not in b for prop in PROPERTIES_TO_COMPUTE: - if prop in a.metadata.props and prop not in a.metadata.props: - score += step_a * len(a.metadata.props) - elif prop in a.metadata.props and prop in a.metadata.props: - for prop_value in a.metadata.props[prop]: - if prop_value not in b.metadata.props[prop]: + if prop in a.get_metadata().props and prop not in a.get_metadata().props: + score += step_a * len(a.get_metadata().props) + elif prop in a.get_metadata().props and prop in a.get_metadata().props: + for prop_value in a.get_metadata().props[prop]: + if prop_value not in b.get_metadata().props[prop]: score -= step_b return score @@ -189,6 +190,6 @@ class SheerkaConceptsAlgebra(BaseService): """ nb_props = 0 for prop in PROPERTIES_TO_COMPUTE: - if prop in concept.metadata.props: - nb_props += len(concept.metadata.props[prop]) + if prop in concept.get_metadata().props: + nb_props += len(concept.get_metadata().props[prop]) return nb_props diff --git a/src/core/sheerka/services/SheerkaCreateNewConcept.py b/src/core/sheerka/services/SheerkaCreateNewConcept.py index 9e5544a..2154943 100644 --- a/src/core/sheerka/services/SheerkaCreateNewConcept.py +++ b/src/core/sheerka/services/SheerkaCreateNewConcept.py @@ -1,6 +1,7 @@ import core.utils from core.builtin_concepts import BuiltinConcepts, ErrorConcept -from core.concept import Concept, DEFINITION_TYPE_DEF, ensure_concept, DEFINITION_TYPE_BNF +from core.builtin_helpers import ensure_concept +from core.concept import Concept, DEFINITION_TYPE_DEF, DEFINITION_TYPE_BNF, freeze_concept_attrs from core.sheerka.services.sheerka_service import BaseService from sdp.sheerkaDataProvider import SheerkaDataProviderDuplicateKeyError @@ -28,7 +29,6 @@ class SheerkaCreateNewConcept(BaseService): :param concept: DefConceptNode :return: digest of the new concept """ - ensure_concept(concept) sheerka = self.sheerka @@ -50,6 +50,9 @@ class SheerkaCreateNewConcept(BaseService): # set id before saving in db sheerka.set_id_if_needed(concept, False) + # freeze attributes + freeze_concept_attrs(concept) + # compute new concepts_by_first_keyword init_ret_value = self.bnp.get_concepts_by_first_token(context, [concept], True) if not init_ret_value.status: @@ -68,9 +71,9 @@ class SheerkaCreateNewConcept(BaseService): cache_manager.put(sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, False, concepts_by_first_keyword) cache_manager.put(sheerka.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY, False, resolved_concepts_by_first_keyword) - if concept.metadata.definition_type == DEFINITION_TYPE_DEF and concept.metadata.definition != concept.name: + if concept.get_metadata().definition_type == DEFINITION_TYPE_DEF and concept.get_metadata().definition != concept.name: # allow search by definition when definition relevant - cache_manager.put(self.sheerka.CONCEPTS_BY_NAME_ENTRY, concept.metadata.definition, concept) + cache_manager.put(self.sheerka.CONCEPTS_BY_NAME_ENTRY, concept.get_metadata().definition, concept) # update references for ref in self.compute_references(concept): @@ -78,7 +81,7 @@ class SheerkaCreateNewConcept(BaseService): # TODO : this line seems to be useless # The grammar is never reset - if concept.bnf and init_bnf_ret_value is not None and init_bnf_ret_value.status: + if concept.get_bnf() and init_bnf_ret_value is not None and init_bnf_ret_value.status: sheerka.cache_manager.clear(sheerka.CONCEPTS_GRAMMARS_ENTRY) # process the return if needed @@ -94,10 +97,10 @@ class SheerkaCreateNewConcept(BaseService): """ refs = set() - if concept.metadata.definition_type == DEFINITION_TYPE_BNF: + if concept.get_metadata().definition_type == DEFINITION_TYPE_BNF: from parsers.BnfNodeParser import BnfNodeConceptExpressionVisitor other_concepts_visitor = BnfNodeConceptExpressionVisitor() - other_concepts_visitor.visit(concept.bnf) + other_concepts_visitor.visit(concept.get_bnf()) for concept in other_concepts_visitor.references: if isinstance(concept, str): diff --git a/src/core/sheerka/services/SheerkaDebugManager.py b/src/core/sheerka/services/SheerkaDebugManager.py new file mode 100644 index 0000000..b6e905f --- /dev/null +++ b/src/core/sheerka/services/SheerkaDebugManager.py @@ -0,0 +1,440 @@ +import os +import pprint +import re +from dataclasses import dataclass + +from core.builtin_concepts import BuiltinConcepts +from core.sheerka.ExecutionContext import ExecutionContext +from core.sheerka.services.sheerka_service import BaseService +from core.utils import CONSOLE_COLORS_MAP as CCM +from core.utils import evaluate_expression, as_bag + +try: + rows, columns = os.popen('stty size', 'r').read().split() +except ValueError: + rows, columns = 50, 80 + +pp = pprint.PrettyPrinter(indent=2, width=columns) + + +class BaseDebugLogger: + ids = {} + + @staticmethod + def next_id(hint): + if hint in BaseDebugLogger.ids: + BaseDebugLogger.ids[hint] += 1 + else: + BaseDebugLogger.ids[hint] = 0 + return BaseDebugLogger.ids[hint] + + def __init__(self, debug_manager, who, method_name, context_id, debug_id): + pass + + def debug_entering(self, **kwargs): + pass + + def debug_var(self, name, value, is_error=False): + pass + + def debug_rule(self, rule, results): + pass + + def debug_log(self, text): + pass + + +class NullDebugLogger(BaseDebugLogger): + def __init__(self): + pass + + +class ConsoleDebugLogger(BaseDebugLogger): + + def __init__(self, debug_manager, service_name, method_name, context_id, debug_id): + BaseDebugLogger.__init__(self, debug_manager, service_name, method_name, context_id, debug_id) + self.debug_manager = debug_manager + self.service_name = service_name + self.method_name = method_name + self.context_id = context_id + self.debug_id = debug_id + self.is_highlighted = "" + + def debug_entering(self, **kwargs): + super().debug_entering(**kwargs) + + str_text = f"{CCM['blue']}Entering {self.service_name}.{self.method_name} with {CCM['reset']}" + str_vars = pp.pformat(kwargs) + if "\n" not in str(str_vars): + self.debug_manager.debug(self.prefix() + str_text + str_vars) + else: + self.debug_manager.debug(self.prefix() + str_text) + self.debug_manager.debug(self.prefix() + str_vars) + + def debug_var(self, name, value, is_error=False): + enabled = self.debug_manager.compute_var_debug(self.service_name, + self.method_name, + self.context_id, + name, + self.context_id) + if enabled == False: + return + + color = 'red' if is_error else 'green' + str_text = f"{CCM[color]}..{name}={CCM['reset']}" + str_vars = "" if isinstance(enabled, str) else pp.pformat(value) + if "\n" not in str(str_vars): + self.debug_manager.debug(self.prefix() + str_text + str_vars) + else: + self.debug_manager.debug(self.prefix() + str_text) + self.debug_manager.debug(self.prefix() + str_vars) + + def debug_rule(self, rule, results): + if not self.debug_manager.compute_debug_rule(rule.id, self.context_id, self.debug_id): + return + + str_text = f"{CCM['green']}..results({rule.id})={CCM['reset']}" + str_vars = pp.pformat(results) + if "\n" not in str(str_vars): + self.debug_manager.debug(self.prefix() + str_text + str_vars) + else: + self.debug_manager.debug(self.prefix() + str_text) + self.debug_manager.debug(self.prefix() + str_vars) + + def debug_log(self, text): + self.debug_manager.debug(self.prefix() + f"{CCM['blue']}..{text}{CCM['reset']}") + + def prefix(self): + return f"[{self.context_id:2}][{self.debug_id:2}] {self.is_highlighted}" + + +@dataclass +class DebugVarSetting: + service_name: str + method_name: str + variable_name: str + context_id: int + context_children: bool + debug_id: int + debug_children: bool + + enabled: bool + + +@dataclass +class DebugRuleSetting: + rule_id: str + context_id: int + debug_id: int + + enabled: bool + + +class SheerkaDebugManager(BaseService): + NAME = "Debug" + PREFIX = "debug." + + children_activation_regex = re.compile(r"(\d+)\+") + + def __init__(self, sheerka): + super().__init__(sheerka) + self.activated = False # is debug activated + self.explicit = False # No need to activate context debug when debug mode is on + self.context_cache = set() # debug for specific context + self.variable_cache = set() # debug for specific variable + self.debug_vars_settings = [] + self.debug_rules_settings = [] + + def initialize(self): + self.sheerka.bind_service_method(self.set_debug, True) + self.sheerka.bind_service_method(self.set_explicit, True) + self.sheerka.bind_service_method(self.activate_debug_for, True) + self.sheerka.bind_service_method(self.deactivate_debug_for, True) + self.sheerka.bind_service_method(self.debug_activated, False) + self.sheerka.bind_service_method(self.debug_activated_for, False) + self.sheerka.bind_service_method(self.get_context_debug_mode, False) + self.sheerka.bind_service_method(self.debug_rule, True) + self.sheerka.bind_service_method(self.debug_rule_activated, False) + self.sheerka.bind_service_method(self.inspect, False) + self.sheerka.bind_service_method(self.debug, False, visible=False) + self.sheerka.bind_service_method(self.get_debugger, False) + self.sheerka.bind_service_method(self.debug_var, False) + self.sheerka.bind_service_method(self.reset_debug, False) + self.sheerka.bind_service_method(self.get_debug_settings, False, as_name="debug_settings") + + def initialize_deferred(self, context, is_first_time): + self.restore_values("activated", + "explicit", + "context_cache", + "variable_cache", + "debug_vars_settings", + "debug_rules_settings") + + def reset(self): + """ + For test purpose + :return: + """ + self.activated = False + self.context_cache.clear() + self.variable_cache.clear() + self.debug_vars_settings.clear() + self.debug_rules_settings.clear() + + def set_debug(self, context, value=True): + self.activated = value + self.sheerka.record_var(context, self.NAME, "activated", self.activated) + return self.sheerka.ret(SheerkaDebugManager.NAME, True, self.sheerka.new(BuiltinConcepts.SUCCESS)) + + def set_explicit(self, context, value=True): + self.explicit = value + self.sheerka.record_var(context, self.NAME, "explicit", self.explicit) + return self.sheerka.ret(SheerkaDebugManager.NAME, True, self.sheerka.new(BuiltinConcepts.SUCCESS)) + + def activate_debug_for(self, context, debug_id, children=False): + """ + + :param context: + :param debug_id: if debug_id is str, activate variable cache, context_cache otherwise + :param children: + :return: + """ + # preprocess + if isinstance(debug_id, str) and (m := self.children_activation_regex.match(debug_id)): + debug_id = int(m.group(1)) + children = True + + if isinstance(debug_id, str): + self.variable_cache.add(debug_id) + self.sheerka.record_var(context, self.NAME, "variable_cache", self.variable_cache) + else: + self.context_cache.add(debug_id) + if children: + self.context_cache.add(str(debug_id) + "+") + self.sheerka.record_var(context, self.NAME, "context_cache", self.context_cache) + return self.sheerka.ret(SheerkaDebugManager.NAME, True, self.sheerka.new(BuiltinConcepts.SUCCESS)) + + def deactivate_debug_for(self, context, debug_id, children=False): + if isinstance(debug_id, str): + self.variable_cache.discard(debug_id) + self.sheerka.record_var(context, self.NAME, "variable_cache", self.variable_cache) + else: + self.context_cache.discard(debug_id) + if children: + self.context_cache.discard(str(debug_id) + "+") + self.sheerka.record_var(context, self.NAME, "context_cache", self.context_cache) + return self.sheerka.ret(SheerkaDebugManager.NAME, True, self.sheerka.new(BuiltinConcepts.SUCCESS)) + + def debug_activated(self): + return self.activated + + def debug_activated_for(self, debug_id): + if not self.activated: + return None + + return debug_id in self.variable_cache + + def debug_rule_activated(self, rule_id, context_id): + """ + + :param rule_id: + :param context_id: + :return: + """ + key = f"{rule_id}|{context_id}" + return key in self.rules_cache + + def get_context_debug_mode(self, context_id): + if not self.activated: + return None, None + + debug_for_children = "protected" if str(context_id) + "+" in self.context_cache else None + debug_for_self = "private" if not self.explicit or context_id in self.context_cache else None + + return debug_for_self, debug_for_children + + def inspect(self, context, context_id, *props): + """ + Print + :param context: + :param context_id: + :return: + """ + to_inspect = self.sheerka.get_execution_item(context, context_id) + if not isinstance(to_inspect, ExecutionContext): + return to_inspect + + if not props: + props = ["inputs", "values.return_values"] + + bag = as_bag(to_inspect) + res = {} + for prop in props: + res[prop] = evaluate_expression(prop, bag) + # res = { + # "return_values": to_inspect.values.get("return_values", None) + # } + + pp.pprint(res) + return None + + def debug(self, *args, **kwargs): + print(*args, **kwargs) + + def get_debugger(self, context, who, method_name): + if self.compute_debug(who, method_name, context): + debug_id = ConsoleDebugLogger.next_id(context.event.get_digest() + str(context.id)) + return ConsoleDebugLogger(self, who, method_name, context.id, debug_id) + + return NullDebugLogger() + + def debug_var(self, context, + service=None, + method=None, + variable=None, + context_id=None, + context_children=False, + debug_id=None, + debug_children=False, + enabled=True): + + for setting in self.debug_vars_settings: + if setting.service_name == service and \ + setting.method_name == method and \ + setting.variable_name == variable and \ + setting.context_id == context_id and \ + setting.context_children == context_children and \ + setting.debug_id == debug_id and \ + setting.debug_children == debug_children: + setting.enabled = enabled + break + else: + self.debug_vars_settings.append(DebugVarSetting(service, + method, + variable, + context_id, + context_children, + debug_id, + debug_children, + enabled)) + + self.sheerka.record_var(context, self.NAME, "debug_vars_settings", self.debug_vars_settings) + return self.sheerka.ret(SheerkaDebugManager.NAME, True, self.sheerka.new(BuiltinConcepts.SUCCESS)) + + def reset_debug(self, context): + self.debug_vars_settings.clear() + self.debug_rules_settings.clear() + self.sheerka.record_var(context, self.NAME, "debug_vars_settings", self.debug_vars_settings) + self.sheerka.record_var(context, self.NAME, "debug_rules_settings", self.debug_vars_settings) + return self.sheerka.ret(SheerkaDebugManager.NAME, True, self.sheerka.new(BuiltinConcepts.SUCCESS)) + + def compute_debug(self, service_name, method_name, context): + if not self.activated: + return False + + selected = [] + for setting in self.debug_vars_settings: + if setting.service_name is None and setting.method_name is None and setting.context_id is None: + continue + + if (setting.service_name is None or setting.service_name == service_name) and \ + (setting.method_name is None or setting.method_name == method_name) and \ + (setting.context_id is None or setting.context_id == context.id or ( + setting.context_children and context.has_parent(setting.context_id))): + selected.append(setting.enabled) + + if len(selected) == 0: + return False + + res = selected[0] + for enabled in selected[1:]: + res &= enabled + + return res + + def compute_var_debug(self, service_name, method_name, context_id, variable_name, debug_id): + if not self.activated: + return False + + selected = [] + for setting in self.debug_vars_settings: + if setting.variable_name is None and setting.debug_id is None: + continue + + if (setting.service_name is None or setting.service_name == service_name) and \ + (setting.method_name is None or setting.method_name == method_name) and \ + (setting.context_id is None or setting.context_id == context_id) and \ + (setting.variable_name is None or + setting.variable_name == "*" or + setting.variable_name == variable_name) and \ + (setting.debug_id is None or setting.debug_id == debug_id): + selected.append(setting.enabled) + + if len(selected) == 0: + return False + + res = selected[0] + for enabled in selected[1:]: + if res == False or enabled == False: + return False + + if isinstance(res, str): + continue + + res = enabled + + return res + + def debug_rule(self, context, rule=None, context_id=None, debug_id=None, enabled=True): + """ + Add a debug rule request + :param context: + :param rule: + :param context_id: + :param debug_id: + :param enabled: + :return: + """ + rule = str(rule) if rule is not None else None + for setting in self.debug_rules_settings: + if setting.rule_id == rule and \ + setting.context_id == context_id and \ + setting.debug_id == debug_id: + setting.enabled = enabled + break + else: + self.debug_rules_settings.append(DebugRuleSetting(rule, + context_id, + debug_id, + enabled)) + + self.sheerka.record_var(context, self.NAME, "debug_rules_settings", self.debug_rules_settings) + return self.sheerka.ret(SheerkaDebugManager.NAME, True, self.sheerka.new(BuiltinConcepts.SUCCESS)) + + def compute_debug_rule(self, rule_id, context_id, debug_id): + if not self.activated: + return False + + selected = [] + for setting in self.debug_rules_settings: + if (setting.rule_id is None or setting.rule_id == rule_id) and \ + (setting.context_id is None or setting.context_id == context_id) and \ + (setting.debug_id is None or setting.debug_id == debug_id): + selected.append(setting.enabled) + + if len(selected) == 0: + return False + + res = selected[0] + for enabled in selected[1:]: + res &= enabled + + return res + + def reset_debug_rules(self, context): + self.debug_rules_settings.clear() + self.sheerka.record_var(context, self.NAME, "debug_rules_settings", self.debug_rules_settings) + return self.sheerka.ret(SheerkaDebugManager.NAME, True, self.sheerka.new(BuiltinConcepts.SUCCESS)) + + def get_debug_settings(self): + return self.debug_vars_settings diff --git a/src/core/sheerka/services/SheerkaDump.py b/src/core/sheerka/services/SheerkaDump.py index 3ffb02c..946ad5e 100644 --- a/src/core/sheerka/services/SheerkaDump.py +++ b/src/core/sheerka/services/SheerkaDump.py @@ -21,7 +21,7 @@ class SheerkaDump(BaseService): super().__init__(sheerka) def initialize(self): - self.sheerka.bind_service_method(self.dump_desc, True, "desc") # because concept is evaluated + self.sheerka.bind_service_method(self.dump_desc, True, "desc") # has_side_effect 'cause concept is evaluated self.sheerka.bind_service_method(self.dump_sdp, False, "dump_sdp") def dump_desc(self, *concept_names, eval=False): @@ -34,7 +34,7 @@ class SheerkaDump(BaseService): else: concepts = self.sheerka.get_by_key(concept_name) if self.sheerka.isinstance(concepts, BuiltinConcepts.UNKNOWN_CONCEPT): - self.sheerka.log.error(f"Concept '{concept_name}' is unknown") + print(f"Concept '{concept_name}' is unknown") return False if not hasattr(concepts, "__iter__"): @@ -46,36 +46,35 @@ class SheerkaDump(BaseService): value = evaluated.body if evaluated.key == c.key else evaluated if not first: - self.sheerka.log.info("") - self.sheerka.log.info(f"id : {c.id}") - self.sheerka.log.info(f"name : {c.name}") - self.sheerka.log.info(f"key : {c.key}") - self.sheerka.log.info(f"definition : {c.metadata.definition}") - self.sheerka.log.info(f"type : {c.metadata.definition_type}") - self.sheerka.log.info(f"body : {c.metadata.body}") - self.sheerka.log.info(f"where : {c.metadata.where}") - self.sheerka.log.info(f"pre : {c.metadata.pre}") - self.sheerka.log.info(f"post : {c.metadata.post}") - self.sheerka.log.info(f"ret : {c.metadata.ret}") - self.sheerka.log.info(f"vars : {c.metadata.variables}") - self.sheerka.log.info(f"props : {c.metadata.props}") + print("") + print(f"id : {c.id}") + print(f"name : {c.name}") + print(f"key : {c.key}") + print(f"definition : {c.get_metadata().definition}") + print(f"type : {c.get_metadata().definition_type}") + print(f"body : {c.get_metadata().body}") + print(f"where : {c.get_metadata().where}") + print(f"pre : {c.get_metadata().pre}") + print(f"post : {c.get_metadata().post}") + print(f"ret : {c.get_metadata().ret}") + print(f"vars : {c.get_metadata().variables}") + print(f"props : {c.get_metadata().props}") if eval: - self.sheerka.log.info(f"value : {value}") - if c.values: - for v in c.values: - self.sheerka.log.info(f"{v}: {c.get_value(v)}") + print(f"value : {value}") + for v in c.values(): + print(f"{v}: {c.get_value(v)}") else: - self.sheerka.log.info("No variable") + print("No variable") - self.sheerka.log.info(f"digest : {c.get_origin()}") + print(f"digest : {c.get_origin()}") if self.sheerka.isaset(context, c): items = self.sheerka.get_set_elements(context, c) - self.sheerka.log.info(f"elements : {items}") + print(f"elements : {items}") first = False def dump_sdp(self): snapshot = self.sheerka.sdp.get_snapshot(SheerkaDataProvider.HeadFile) state = self.sheerka.sdp.load_state(snapshot) - self.sheerka.log.info(get_pp().pformat(state.data)) + print(get_pp().pformat(state.data)) diff --git a/src/core/sheerka/services/SheerkaEvaluateConcept.py b/src/core/sheerka/services/SheerkaEvaluateConcept.py index 5285bd6..71d9655 100644 --- a/src/core/sheerka/services/SheerkaEvaluateConcept.py +++ b/src/core/sheerka/services/SheerkaEvaluateConcept.py @@ -1,8 +1,9 @@ from dataclasses import dataclass from core.builtin_concepts import BuiltinConcepts -from core.builtin_helpers import expect_one, only_successful, parse_unrecognized, evaluate -from core.concept import Concept, DoNotResolve, ConceptParts, InfiniteRecursionResolved, NotInit, ensure_concept +from core.builtin_helpers import expect_one, only_successful, parse_unrecognized, evaluate, ensure_concept +from core.concept import Concept, DoNotResolve, ConceptParts, InfiniteRecursionResolved, NotInit, AllConceptParts, \ + concept_part_value from core.sheerka.services.SheerkaExecute import ParserInput from core.sheerka.services.sheerka_service import BaseService from core.tokenizer import Tokenizer @@ -47,8 +48,10 @@ class SheerkaEvaluateConcept(BaseService): parent = context.get_parent() while parent is not None: - if parent.who == context.who and parent.action == BuiltinConcepts.EVALUATING_CONCEPT and \ - parent.obj == concept and parent.obj.compiled == concept.compiled: + if (parent.who == context.who and + parent.action == BuiltinConcepts.EVALUATING_CONCEPT and + parent.obj == concept and + parent.obj.get_compiled() == concept.get_compiled()): return True parent = parent.get_parent() @@ -90,15 +93,15 @@ class SheerkaEvaluateConcept(BaseService): vars_needed = False body_needed = False - if concept_part in concept.compiled and concept.compiled[concept_part] is not None: - concept_part_source = getattr(concept.metadata, concept_part.value) + if concept_part in concept.get_compiled() and concept.get_compiled()[concept_part] is not None: + concept_part_source = getattr(concept.get_metadata(), concept_part_value(concept_part)) assert concept_part_source is not None tokens = [t.str_value for t in Tokenizer(concept_part_source)] if check_vars: - for var_name in (v[0] for v in concept.metadata.variables): + for var_name in (v[0] for v in concept.get_metadata().variables): if var_name in tokens: vars_needed = True ret.append("variables") @@ -106,9 +109,9 @@ class SheerkaEvaluateConcept(BaseService): if check_body and "self" in tokens: body_needed = True - ret.append("body") + ret.append(ConceptParts.BODY) - ret.append(concept_part.value) + ret.append(concept_part) return ret, vars_needed, body_needed @@ -121,16 +124,16 @@ class SheerkaEvaluateConcept(BaseService): :param var_name: :return: """ - if concept.metadata.where is None or concept.metadata.where.strip() == "": + if concept.get_metadata().where is None or concept.get_metadata().where.strip() == "": return None - ret = ExpressionParser().parse(context, ParserInput(concept.metadata.where)) + ret = ExpressionParser().parse(context, ParserInput(concept.get_metadata().where)) if not ret.status: # TODO: manage invalid where clause return None expr = ret.body.body - to_trueify = [v[0] for v in concept.metadata.variables if v[0] != var_name] + to_trueify = [v[0] for v in concept.get_metadata().variables if v[0] != var_name] trueified_where = str(TrueifyVisitor(to_trueify, [var_name]).visit(expr)) tokens = [t.str_value for t in Tokenizer(trueified_where)] @@ -140,7 +143,7 @@ class SheerkaEvaluateConcept(BaseService): compiled = compile(trueified_where, "", "eval") except Exception: pass - return WhereClauseDef(concept, concept.metadata.where, trueified_where, var_name, compiled) + return WhereClauseDef(concept, concept.get_metadata().where, trueified_where, var_name, compiled) else: return None @@ -153,7 +156,8 @@ class SheerkaEvaluateConcept(BaseService): :return: """ ret = [] - for r in [r for r in return_values if r.status]: + valid_return_values = [r for r in return_values if r.status] + for r in valid_return_values: if where_clause_def.compiled: try: if eval(where_clause_def.compiled, {where_clause_def.prop: self.sheerka.objvalue(r)}): @@ -177,10 +181,13 @@ class SheerkaEvaluateConcept(BaseService): if len(ret) > 0: return ret + reason = [r.body for r in return_values] if len(valid_return_values) == 0 else None + return self.sheerka.new(BuiltinConcepts.CONDITION_FAILED, body=where_clause_def.clause, concept=where_clause_def.concept, - prop=where_clause_def.prop) + prop=where_clause_def.prop, + reason=reason) def manage_infinite_recursion(self, context): """ @@ -194,7 +201,7 @@ class SheerkaEvaluateConcept(BaseService): concepts_found = set() while parent and parent.obj: if parent.who == context.who and parent.action == BuiltinConcepts.EVALUATING_CONCEPT: - body = parent.obj.metadata.body + body = parent.obj.get_metadata().body try: return self.sheerka.ret(self.NAME, True, InfiniteRecursionResolved(eval(body))) except Exception: @@ -226,11 +233,11 @@ class SheerkaEvaluateConcept(BaseService): return self.sheerka.resolve(identifier) return None - for part_key in ConceptParts: - if part_key in concept.compiled: + for part_key in AllConceptParts: + if part_key in concept.get_compiled(): continue - source = getattr(concept.metadata, part_key.value) + source = getattr(concept.get_metadata(), concept_part_value(part_key)) if source is None: # or not isinstance(source, str): continue @@ -238,22 +245,22 @@ class SheerkaEvaluateConcept(BaseService): raise Exception("Invalid concept init. metadata must be a string") if source.strip() == "": - concept.compiled[part_key] = DoNotResolve(source) + concept.get_compiled()[part_key] = DoNotResolve(source) else: # first case, when the metadata references another concept via c:xxx: keyword if concept_found := parse_token_concept(source): context.log(f"Recognized concept '{concept_found}'", self.NAME) - concept.compiled[part_key] = concept_found + concept.get_compiled()[part_key] = concept_found else: res = parse_unrecognized(context, source, parsers="all", prop=part_key, filter_func=only_successful) - concept.compiled[part_key] = res.body.body if is_only_successful(res) else res + concept.get_compiled()[part_key] = res.body.body if is_only_successful(res) else res - for var_name, default_value in concept.metadata.variables: - if var_name in concept.compiled: + for var_name, default_value in concept.get_metadata().variables: + if var_name in concept.get_compiled(): continue if default_value is None: @@ -263,23 +270,23 @@ class SheerkaEvaluateConcept(BaseService): raise Exception("Invalid concept init. variable metadata must be a string") if default_value.strip() == "": - concept.compiled[var_name] = DoNotResolve(default_value) + concept.get_compiled()[var_name] = DoNotResolve(default_value) else: # first case, when the metadata references another concept via c:xxx: keyword if concept_found := parse_token_concept(default_value): context.log(f"Recognized concept '{concept_found}'", self.NAME) - concept.compiled[var_name] = concept_found + concept.get_compiled()[var_name] = concept_found else: res = parse_unrecognized(context, default_value, parsers="all", prop=var_name, filter_func=only_successful) - concept.compiled[var_name] = res.body.body if is_only_successful(res) else res + concept.get_compiled()[var_name] = res.body.body if is_only_successful(res) else res # Updates the cache of concepts when possible if self.sheerka.has_id(concept.id): - self.sheerka.get_by_id(concept.id).compiled = concept.compiled + self.sheerka.get_by_id(concept.id).set_compiled(concept.get_compiled()) def resolve(self, context, @@ -325,8 +332,8 @@ class SheerkaEvaluateConcept(BaseService): with context.push(BuiltinConcepts.EVALUATING_ATTRIBUTE, current_prop, desc=desc, - obj=current_concept, - path=path) as sub_context: + obj=current_concept) as sub_context: + sub_context.add_inputs(path=path) if force_evaluation: sub_context.protected_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED) @@ -352,7 +359,7 @@ class SheerkaEvaluateConcept(BaseService): else: # update short term memory with current concept variables if current_concept: - for var in current_concept.metadata.variables: + for var in current_concept.get_metadata().variables: value = current_concept.get_value(var[0]) if value != NotInit: sub_context.add_to_short_term_memory(var[0], current_concept.get_value(var[0])) @@ -433,23 +440,23 @@ class SheerkaEvaluateConcept(BaseService): :return: value of the evaluation or error """ - if concept.metadata.is_evaluated: + if concept.get_metadata().is_evaluated: return concept # I cannot use cache because of concept like 'number'. # They don't have variables, but their values change every time they are instantiated # TODO: Need to find a way to cache despite of them # need_body = eval_body or context.in_context(BuiltinConcepts.EVAL_BODY_REQUESTED) - # if need_body and len(concept.metadata.variables) == 0 and context.sheerka.has_id(concept.id): + # if need_body and len(concept.get_metadata().variables) == 0 and context.sheerka.has_id(concept.id): # from_cache = context.sheerka.get_by_id(concept.id) - # if from_cache.metadata.is_evaluated: + # if from_cache.get_metadata().is_evaluated: # concept.set_value(ConceptParts.BODY, from_cache.body) - # concept.metadata.is_evaluated = True + # concept.get_metadata().is_evaluated = True # return concept desc = f"Evaluating concept {concept}" - with context.push(BuiltinConcepts.EVALUATING_CONCEPT, concept, desc=desc, eval_body=eval_body) as sub_context: - + with context.push(BuiltinConcepts.EVALUATING_CONCEPT, concept, desc=desc) as sub_context: + sub_context.add_inputs(eval_body=eval_body) if eval_body: # ask for body evaluation sub_context.protected_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED) @@ -466,8 +473,8 @@ class SheerkaEvaluateConcept(BaseService): for metadata_to_eval in all_metadata_to_eval: if metadata_to_eval == "variables": - for var_name in (v for v in concept.variables() if v in concept.compiled): - prop_ast = concept.compiled[var_name] + for var_name in (v for v in concept.variables() if v in concept.get_compiled()): + prop_ast = concept.get_compiled()[var_name] w_clause = self.get_where_clause_def(context, concept, var_name) # TODO, manage when the where clause cannot be parsed @@ -485,17 +492,17 @@ class SheerkaEvaluateConcept(BaseService): else: concept.set_value(var_name, resolved) else: - part_key = ConceptParts(metadata_to_eval) + part_key = metadata_to_eval # do not evaluate where when the body is a set # Indeed, the way that the where clause is expressed is not a valid python or concept code if part_key == ConceptParts.WHERE and self.sheerka.isaset(sub_context, concept.body): continue - if part_key not in concept.compiled or concept.compiled[part_key] is None: + if part_key not in concept.get_compiled() or concept.get_compiled()[part_key] is None: continue - metadata_ast = concept.compiled[part_key] + metadata_ast = concept.get_compiled()[part_key] # if part_key is PRE, POST or WHERE, the concept need to be evaluated # if we want the predicates to be resolved => so force_eval = True @@ -514,7 +521,7 @@ class SheerkaEvaluateConcept(BaseService): # validate PRE and WHERE condition if part_key in (ConceptParts.PRE, ConceptParts.WHERE) and not self.sheerka.objvalue(resolved): return self.sheerka.new(BuiltinConcepts.CONDITION_FAILED, - body=getattr(concept.metadata, metadata_to_eval), + body=getattr(concept.get_metadata(), concept_part_value(metadata_to_eval)), concept=concept, prop=part_key) @@ -524,19 +531,19 @@ class SheerkaEvaluateConcept(BaseService): concept.init_key() # Necessary for old unit tests. To remove someday - if "body" in all_metadata_to_eval: - concept.metadata.is_evaluated = True + if ConceptParts.BODY in all_metadata_to_eval: + concept.get_metadata().is_evaluated = True # # update the cache for concepts with no variables # Cannot use cache. See the comment at the beginning of this method - # if len(concept.metadata.variables) == 0: + # if len(concept.get_metadata().variables) == 0: # self.sheerka.cache_manager.put(self.sheerka.CONCEPTS_BY_ID_ENTRY, concept.id, concept) - if not concept.metadata.is_builtin: + if not concept.get_metadata().is_builtin: self.sheerka.register_object(sub_context, concept.name, concept) # manage RET metadata - if sub_context.in_context(BuiltinConcepts.EVAL_BODY_REQUESTED) and ConceptParts.RET in concept.values: + if sub_context.in_context(BuiltinConcepts.EVAL_BODY_REQUESTED) and ConceptParts.RET in concept.values(): return concept.get_value(ConceptParts.RET) else: return concept @@ -547,7 +554,7 @@ class SheerkaEvaluateConcept(BaseService): needed, variables, body = self.get_needed_metadata(concept, ConceptParts.PRE, True, True) to_eval.extend(needed) - if context.in_context(BuiltinConcepts.EVAL_WHERE_REQUESTED) or concept.metadata.need_validation: + if context.in_context(BuiltinConcepts.EVAL_WHERE_REQUESTED) or concept.get_metadata().need_validation: # What are the cases where we do not need a validation ? # see test_sheerka_non_reg::test_i_can_evaluate_bnf_concept_with_where_clause() # res = sheerka.evaluate_user_input("foobar") @@ -572,7 +579,7 @@ class SheerkaEvaluateConcept(BaseService): to_eval.append('variables') if not body: - to_eval.append("body") + to_eval.append(ConceptParts.BODY) return to_eval diff --git a/src/core/sheerka/services/SheerkaEvaluateRules.py b/src/core/sheerka/services/SheerkaEvaluateRules.py new file mode 100644 index 0000000..d1ec1cb --- /dev/null +++ b/src/core/sheerka/services/SheerkaEvaluateRules.py @@ -0,0 +1,114 @@ +from core.builtin_concepts import BuiltinConcepts +from core.builtin_helpers import expect_one +from core.sheerka.services.sheerka_service import BaseService +from evaluators.ConceptEvaluator import ConceptEvaluator + +DISABLED_RULES = "#disabled#" +LOW_PRIORITY_RULES = "#low_priority#" + + +class SheerkaEvaluateRules(BaseService): + NAME = "EvaluateRules" + + def __init__(self, sheerka): + super().__init__(sheerka) + self.evaluators_by_name = None + + def initialize(self): + self.sheerka.bind_service_method(self.evaluate_format_rules, False) + self.reset_evaluators() + + def reset_evaluators(self): + # instantiate evaluators, once for all, only keep when it's enabled + evaluators = [e_class() for e_class in self.sheerka.evaluators] + evaluators = [e for e in evaluators if e.enabled] + self.evaluators_by_name = {e.short_name: e for e in evaluators} + + def evaluate_format_rules(self, context, bag, disabled): + return self.evaluate_rules(context, self.sheerka.get_format_rules(), bag, disabled) + + def evaluate_rules(self, context, rules, bag, disabled): + """ + evaluate the format rules, in the context of 'bag' + CAUTION : the rules MUST be sorted by priority + :param context: + :param rules: + :param bag: + :param disabled: disabled rules (because they have already been fired or whatever) + :return: { True : list of success, False :list of failed, '#disabled"': list of disabled...} + """ + with context.push(BuiltinConcepts.EVALUATING_RULES, bag, desc="Evaluating rules...") as sub_context: + sub_context.protected_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED) + sub_context.protected_hints.add(BuiltinConcepts.EVAL_WHERE_REQUESTED) + sub_context.protected_hints.add(BuiltinConcepts.EVAL_UNTIL_SUCCESS_REQUESTED) + sub_context.protected_hints.add(BuiltinConcepts.EVAL_QUESTION_REQUESTED) + sub_context.add_inputs(bag=bag) + + debugger = sub_context.get_debugger(SheerkaEvaluateRules.NAME, "evaluate_rules") + debugger.debug_entering(bag=bag) + + results = {} + + sub_context.sheerka.add_many_to_short_term_memory(sub_context, bag) + success_priority = None + for rule in rules: + if not rule.metadata.is_enabled or rule.id in disabled: + results.setdefault(DISABLED_RULES, []).append(rule) + continue + + if success_priority and rule.priority != success_priority: + results.setdefault(LOW_PRIORITY_RULES, []).append(rule) + continue + + res = self.evaluate_rule(sub_context, rule, bag) + ok = res.status and self.sheerka.is_success(self.sheerka.objvalue(res)) + results.setdefault(ok, []).append(rule) + if ok and success_priority is None: + success_priority = rule.priority + + debugger.debug_var("results", self.get_debug_format(results)) + + sub_context.add_values(rules_result=results) + return results + + def evaluate_rule(self, context, rule, bag): + """ + Evaluate all the predicate + :param context: + :param rule: + :param bag: + :return: + """ + + results = [] + for rule_predicate in rule.compiled_predicate: + + if rule_predicate.source in bag: + # simple case where the rule is an item of the bag. No need of complicate evaluation + results.append(context.sheerka.ret(self.NAME, True, bag[rule_predicate.source])) + + else: + + # do not forget to reset the 'is_evaluated' in the case of a concept + if rule_predicate.evaluator == ConceptEvaluator.NAME: + rule_predicate.concept.get_metadata().is_evaluated = False + + evaluator = self.evaluators_by_name[rule_predicate.evaluator] + results.append(evaluator.eval(context, rule_predicate.predicate)) + + debugger = context.get_debugger(SheerkaEvaluateRules.NAME, "evaluate_rule") + debugger.debug_rule(rule, results) + # if context.sheerka.debug_rule_activated(rule_id, context.id): + # context.debug(SheerkaEvaluateRules.NAME, "evaluate_rules", f"result(#{rule_id})", results) + + return expect_one(context, results) + + @staticmethod + def get_debug_format(result): + """ + Return the same dictionary, the with the short formatting of the rules + eg without the action clause + :param result: + :return: + """ + return {key: [str(r) if key == True else r.short_str() for r in rules] for key, rules in result.items()} diff --git a/src/core/sheerka/services/SheerkaEventManager.py b/src/core/sheerka/services/SheerkaEventManager.py new file mode 100644 index 0000000..71e43da --- /dev/null +++ b/src/core/sheerka/services/SheerkaEventManager.py @@ -0,0 +1,60 @@ +from threading import RLock + +from core.sheerka.services.sheerka_service import BaseService + + +class SheerkaEventManager(BaseService): + """ + This class implement a very basic publish and subscribe mechanism + It supposes that the subscriber has a little knowledge of how the publisher works + """ + NAME = "EventManager" + + def __init__(self, sheerka): + super().__init__(sheerka) + self._lock = RLock() + self.subscribers = {} + + def initialize(self): + self.sheerka.bind_service_method(self.subscribe, True, visible=False) + self.sheerka.bind_service_method(self.publish, True, visible=False) + + def subscribe(self, topic, callback): + """ + To subscribe to a topic, just give the callback to call + Note that the callback must be a function whose first argument is a context + :param topic: + :param callback: + :return: + """ + with self._lock: + self.subscribers.setdefault(topic, []).append(callback) + + def publish(self, context, topic, data=None): + """ + Publish on a topic + The data is not mandatory + :param context: + :param topic: + :param data: + :return: + """ + with self._lock: + try: + subscribers = self.subscribers[topic] + if data: + for callback in subscribers: + callback(context, data) + else: + for callback in subscribers: + callback(context) + except KeyError: + pass + + def reset_topic(self, topic): + """ + Remove all subsccribers from a given topic + :param topic: + :return: + """ + self.subscribers[topic].clear() diff --git a/src/core/sheerka/services/SheerkaExecute.py b/src/core/sheerka/services/SheerkaExecute.py index 59a76cb..ee4de60 100644 --- a/src/core/sheerka/services/SheerkaExecute.py +++ b/src/core/sheerka/services/SheerkaExecute.py @@ -5,6 +5,16 @@ from core.sheerka.services.sheerka_service import BaseService from core.tokenizer import Tokenizer, TokenKind, Token NO_MATCH = "** No Match **" +EVALUATOR_STEPS = [ + BuiltinConcepts.BEFORE_PARSING, + BuiltinConcepts.AFTER_PARSING, + BuiltinConcepts.BEFORE_EVALUATION, + BuiltinConcepts.EVALUATION, + BuiltinConcepts.AFTER_EVALUATION, + BuiltinConcepts.BEFORE_RENDERING, + BuiltinConcepts.RENDERING, + BuiltinConcepts.AFTER_RENDERING, +] class ParserInput: @@ -74,10 +84,10 @@ class ParserInput: if self.start == 0 and self.end == self.length: self.sub_text = self.text return self.sub_text - self.sub_text = self.get_text_from_tokens(self.tokens[self.start:self.end]) + self.sub_text = core.utils.get_text_from_tokens(self.tokens[self.start:self.end]) return self.sub_text else: - return self.get_text_from_tokens(self.as_tokens(), custom_switcher, tracker) + return core.utils.get_text_from_tokens(self.as_tokens(), custom_switcher, tracker) def as_tokens(self): if self.sub_tokens: @@ -145,36 +155,6 @@ class ParserInput: return True return False - @staticmethod - def get_text_from_tokens(tokens, custom_switcher=None, tracker=None): - """ - Create the source code, from the list of token - :param tokens: list of tokens - :param custom_switcher: to override the behaviour (the return value) of some token - :param tracker: keep track of the original token value when custom switched - :return: - """ - if tokens is None: - return "" - res = "" - - if not hasattr(tokens, "__iter__"): - tokens = [tokens] - - switcher = { - TokenKind.CONCEPT: lambda t: core.utils.str_concept(t.value), - } - - if custom_switcher: - switcher.update(custom_switcher) - - for token in tokens: - value = switcher.get(token.type, lambda t: t.value)(token) - res += value - if tracker is not None and token.type in custom_switcher: - tracker[value] = token.value - return res - class SheerkaExecute(BaseService): """ @@ -187,18 +167,156 @@ class SheerkaExecute(BaseService): def __init__(self, sheerka): super().__init__(sheerka) self.pi_cache = Cache(default=lambda key: ParserInput(key), max_size=20) + self.instantiated_evaluators = None + self.evaluators_by_name = None + self.grouped_evaluators_cache = {} # key=step, value=tuple(evaluators for this step, sorted priorities) + self.old_values = [] def initialize(self): self.sheerka.bind_service_method(self.execute, True) self.sheerka.cache_manager.register_cache(self.PARSERS_INPUTS_ENTRY, self.pi_cache, False) + self.reset_evaluators() + + def reset_evaluators(self): + # instantiate evaluators, once for all, only keep when it's enabled + self.instantiated_evaluators = [e_class() for e_class in self.sheerka.evaluators] + self.instantiated_evaluators = [e for e in self.instantiated_evaluators if e.enabled] + self.evaluators_by_name = {e.short_name: e for e in self.instantiated_evaluators} + + # get default evaluators by process step + for process_step in EVALUATOR_STEPS: + self.grouped_evaluators_cache[f"{process_step}|__default"] = self.get_grouped_evaluators( + [e for e in self.instantiated_evaluators if process_step in e.steps]) + + # @staticmethod + # def get_grouped_evaluators(instantiated_evaluators, process_step): + # """ + # For a given list of evaluators and a given process step + # Computes + # * the evaluators eligible for this step + # * the list of sorted priorities for theses evaluators + # :param instantiated_evaluators: + # :param process_step: + # :return: + # """ + # grouped = {} + # for evaluator in [e for e in instantiated_evaluators if e.enabled and process_step in e.steps]: + # grouped.setdefault(evaluator.priority, []).append(evaluator) + # + # sorted_groups = sorted(grouped.keys(), reverse=True) + # return grouped, sorted_groups + + @staticmethod + def get_grouped_evaluators(evaluators): + """ + For a given list of evaluators, + group them by priorities + sort the priorities + :param evaluators: + :return: tuple({priority: List of evaluators with this priority}, list of sorted priorities) + """ + grouped = {} + for evaluator in evaluators: + grouped.setdefault(evaluator.priority, []).append(evaluator) + + sorted_groups = sorted(grouped.keys(), reverse=True) + return grouped, sorted_groups + + def preprocess(self, items, preprocess_definitions): + for preprocess in preprocess_definitions: + for item in items: + if self.matches(item.name, preprocess.get_value("preprocess_name")): + for var_name, value in preprocess.values().items(): + if var_name == "preprocess_name": + continue + if hasattr(item, var_name): + self.old_values.append((item, var_name, getattr(item, var_name))) + setattr(item, var_name, value) + + def preprocess_old(self, context, parsers_or_evaluators, mode): + if mode == "parsers": + if not context.preprocess and not context.preprocess_parsers: + return parsers_or_evaluators + items = context.preprocess_parsers + elif mode == "evaluators": + if not context.preprocess and not context.preprocess_evaluators: + return parsers_or_evaluators + items = context.preprocess_evaluators + else: + raise ValueError(mode) + + if not hasattr(parsers_or_evaluators, "__iter__"): + single_one = True + parsers_or_evaluators = [parsers_or_evaluators] + else: + single_one = False + + if items: + res = [] + for item in items: + for e in parsers_or_evaluators: + if item == e.name: + res.append(e) + break + else: + raise ValueError(f"{item} not found.") + parsers_or_evaluators = res + + if context.preprocess: + for preprocess in context.preprocess: + for e in parsers_or_evaluators: + if self.matches(e.name, preprocess.get_value("name")): + for var_name in preprocess.values: + if var_name == "name": + continue + if hasattr(e, var_name): + self.old_values.append((e, var_name, getattr(e, var_name))) + setattr(e, var_name, preprocess.get_value(var_name)) + + return parsers_or_evaluators[0] if single_one else parsers_or_evaluators + + def get_evaluators(self, context, process_step): + """ + Returns the list of evaluators to use for a specific test + :param context: + :param process_step: + :return: + """ + # Normal case, the evaluators are the default one + if not context.preprocess_evaluators and not context.preprocess: + return self.grouped_evaluators_cache[f"{process_step}|__default"] + + # First case, only use a subset of evaluators + if context.preprocess_evaluators and not context.preprocess: + key = str(process_step) + "|" + "|".join(context.preprocess_evaluators) + try: + return self.grouped_evaluators_cache[key] + except KeyError: + evaluators = [self.evaluators_by_name[e] for e in context.preprocess_evaluators] + grouped = self.get_grouped_evaluators(evaluators) + self.grouped_evaluators_cache[key] = grouped + return grouped + + # final case, evaluators attributes are modified by the context + # So first, get the modified evaluators + evaluators = [self.evaluators_by_name[e] for e in + context.preprocess_evaluators] if context.preprocess_evaluators else self.instantiated_evaluators + self.preprocess(evaluators, context.preprocess) + evaluators = [e for e in evaluators if e.enabled] # make sure they are still enabled + key = str(process_step) + "|" + "|".join([e.name for e in evaluators if e.enabled]) + try: + return self.grouped_evaluators_cache[key] + except KeyError: + grouped = self.get_grouped_evaluators(evaluators) + self.grouped_evaluators_cache[key] = grouped + return grouped def get_parser_input(self, text, tokens=None): """ Returns new or existing parser input :param text: :param tokens: - :param length: :return: """ @@ -212,7 +330,7 @@ class SheerkaExecute(BaseService): self.pi_cache.put(text, pi) return pi - key = text or ParserInput.get_text_from_tokens(tokens) + key = text or core.utils.get_text_from_tokens(tokens) pi = ParserInput(key, tokens) self.pi_cache.put(key, pi) return pi @@ -251,7 +369,7 @@ class SheerkaExecute(BaseService): # group the parsers by priorities instantiated_parsers = [parser(sheerka=self.sheerka) for parser in self.sheerka.parsers.values()] - instantiated_parsers = self.preprocess(context, instantiated_parsers) + instantiated_parsers = self.preprocess_old(context, instantiated_parsers, "parsers") grouped_parsers = {} for parser in [p for p in instantiated_parsers if p.enabled]: @@ -272,13 +390,12 @@ class SheerkaExecute(BaseService): # if self.sheerka.log.isEnabledFor(logging.DEBUG): # debug_text = "'" + to_parse + "'" if isinstance(to_parse, str) \ - # else "'" + BaseParser.get_text_from_tokens(to_parse) + "' as tokens" + # else "'" + core.utils.get_text_from_tokens(to_parse) + "' as tokens" # context.log(f"Parsing {debug_text}") with context.push(BuiltinConcepts.PARSING, {"parser": parser.name}, - desc=f"Parsing using {parser.name}", - logger=parser.verbose_log) as sub_context: + desc=f"Parsing using {parser.name}") as sub_context: sub_context.add_inputs(to_parse=to_parse) res = parser.parse(sub_context, to_parse) if res is not None: @@ -318,27 +435,13 @@ class SheerkaExecute(BaseService): if not isinstance(return_values, list): return_values = [return_values] - # group the evaluators by priority and sort them - # The first one to be applied will be the one with the highest priority - grouped_evaluators = {} - instantiated_evaluators = [e_class() for e_class in self.sheerka.evaluators] + grouped_evaluators, sorted_priorities = self.get_evaluators(context, process_step) - # pre-process evaluators if needed - instantiated_evaluators = self.preprocess(context, instantiated_evaluators) - - for evaluator in [e for e in instantiated_evaluators if e.enabled and process_step in e.steps]: - grouped_evaluators.setdefault(evaluator.priority, []).append(evaluator) - - # order the groups by priority, the higher first - sorted_priorities = sorted(grouped_evaluators.keys(), reverse=True) - - # process iteration = 0 while True: with context.push(process_step, - {"iteration": iteration}, - desc=f"iteration #{iteration}", - iteration=iteration) as iteration_context: + {"step": process_step, "iteration": iteration}, + desc=f"iteration #{iteration}") as iteration_context: simple_digest = return_values[:] iteration_context.add_inputs(return_values=simple_digest) @@ -348,13 +451,14 @@ class SheerkaExecute(BaseService): evaluated_items = [] to_delete = [] for evaluator in grouped_evaluators[priority]: - evaluator = self.preprocess(context, evaluator.__class__()) # fresh copy + evaluator.reset() sub_context_desc = f"Evaluating using {evaluator.name} ({priority=})" with iteration_context.push(process_step, - {"iteration": iteration, "evaluator": evaluator.name}, - desc=sub_context_desc, - logger=evaluator.verbose_log) as sub_context: + {"step": process_step, + "iteration": iteration, + "evaluator": evaluator.name}, + desc=sub_context_desc) as sub_context: sub_context.add_inputs(return_values=original_items) # process evaluators that work on one simple return value at the time @@ -365,6 +469,8 @@ class SheerkaExecute(BaseService): if evaluator.matches(sub_context, item): # init the evaluator is possible + # KSI. 20201102 : Evaluators are now instantiated at startup, + # Can we move this section into reset_evaluators() if hasattr(evaluator, "init_evaluator") and not evaluator.is_initialized: evaluator.init_evaluator(sub_context, original_items) @@ -401,6 +507,7 @@ class SheerkaExecute(BaseService): # process evaluators that work on all return values else: if evaluator.matches(sub_context, original_items): + results = evaluator.eval(sub_context, original_items) if results is None: continue @@ -427,6 +534,8 @@ class SheerkaExecute(BaseService): # inc the iteration and continue iteration += 1 + self.undo_preprocess() + return return_values def execute(self, context, return_values, execution_steps): @@ -441,40 +550,30 @@ class SheerkaExecute(BaseService): for step in execution_steps: copy = return_values[:] if hasattr(return_values, "__iter__") else [return_values] with context.push(BuiltinConcepts.PROCESSING, - {"step": step}, - step=step, iteration=0, desc=f"{step=}") as sub_context: + {"step": step, "iteration": 0}, + desc=f"{step=}") as sub_context: + + sub_context.add_inputs(return_values=copy) if step == BuiltinConcepts.PARSING: return_values = self.call_parsers(sub_context, return_values) else: return_values = self.call_evaluators(sub_context, return_values, step) - if copy != return_values: + has_changed = copy != return_values + if has_changed: sub_context.log_result(return_values) sub_context.add_values(return_values=return_values) + sub_context.add_values(has_changed=has_changed) return return_values - def preprocess(self, context, parsers_or_evaluators): - if not context.preprocess: - return parsers_or_evaluators + def undo_preprocess(self): + for item, var_name, value in self.old_values: + setattr(item, var_name, value) - if not hasattr(parsers_or_evaluators, "__iter__"): - single_one = True - parsers_or_evaluators = [parsers_or_evaluators] - else: - single_one = False - - for preprocess in context.preprocess: - for e in parsers_or_evaluators: - if self.matches(e.name, preprocess.get_value("name")): - for var_name in preprocess.values: - if var_name == "name": - continue - if hasattr(e, var_name): - setattr(e, var_name, preprocess.get_value(var_name)) - return parsers_or_evaluators[0] if single_one else parsers_or_evaluators + self.old_values.clear() @staticmethod def matches(parser_or_evaluator_name, preprocessor_name): diff --git a/src/core/sheerka/services/SheerkaHasAManager.py b/src/core/sheerka/services/SheerkaHasAManager.py index cf2aca6..67dbb64 100644 --- a/src/core/sheerka/services/SheerkaHasAManager.py +++ b/src/core/sheerka/services/SheerkaHasAManager.py @@ -1,5 +1,5 @@ from core.builtin_concepts import BuiltinConcepts -from core.concept import ensure_concept +from core.builtin_helpers import ensure_concept from core.sheerka.services.sheerka_service import BaseService @@ -25,8 +25,8 @@ class SheerkaHasAManager(BaseService): context.log(f"Setting concept {concept_a} has a {concept_b}", who=self.NAME) ensure_concept(concept_a, concept_b) - if (BuiltinConcepts.HASA in concept_a.metadata.props and - concept_b in concept_a.metadata.props[BuiltinConcepts.HASA]): + if (BuiltinConcepts.HASA in concept_a.get_metadata().props and + concept_b in concept_a.get_metadata().props[BuiltinConcepts.HASA]): return self.sheerka.ret( self.NAME, False, @@ -49,5 +49,5 @@ class SheerkaHasAManager(BaseService): """ ensure_concept(concept_a, concept_b) - return (BuiltinConcepts.HASA in concept_a.metadata.props and - concept_b in concept_a.metadata.props[BuiltinConcepts.HASA]) + return (BuiltinConcepts.HASA in concept_a.get_metadata().props and + concept_b in concept_a.get_metadata().props[BuiltinConcepts.HASA]) diff --git a/src/core/sheerka/services/SheerkaMemory.py b/src/core/sheerka/services/SheerkaMemory.py index 1363dcc..05fa6d8 100644 --- a/src/core/sheerka/services/SheerkaMemory.py +++ b/src/core/sheerka/services/SheerkaMemory.py @@ -1,8 +1,10 @@ from dataclasses import dataclass +from cache.FastCache import FastCache from cache.ListIfNeededCache import ListIfNeededCache from core.builtin_concepts import BuiltinConcepts from core.concept import Concept +from core.global_symbols import CONTEXT_DISPOSED from core.sheerka.services.sheerka_service import BaseService, ServiceObj @@ -13,48 +15,72 @@ class MemoryObject(ServiceObj): class SheerkaMemory(BaseService): NAME = "Memory" + GLOBAL = "global" SHORT_TERM_OBJECTS_ENTRY = "Memory:ShortTermMemoryObjects" OBJECTS_ENTRY = "Memory:Objects" def __init__(self, sheerka): super().__init__(sheerka) - self.short_term_objects = ListIfNeededCache() + self.short_term_objects = FastCache() self.objects = ListIfNeededCache(default=lambda k: self.sheerka.sdp.get(self.OBJECTS_ENTRY, k)) self.registration = {} def initialize(self): self.sheerka.bind_service_method(self.get_from_short_term_memory, False, visible=False) self.sheerka.bind_service_method(self.add_to_short_term_memory, True, visible=False) + self.sheerka.bind_service_method(self.remove_context, True, as_name="clear_short_term_memory", visible=False) self.sheerka.bind_service_method(self.add_to_memory, True, visible=False) + self.sheerka.bind_service_method(self.add_many_to_short_term_memory, True, visible=False) self.sheerka.bind_service_method(self.get_from_memory, False) self.sheerka.bind_service_method(self.register_object, True, visible=False) self.sheerka.bind_service_method(self.unregister_object, True, visible=False) self.sheerka.bind_service_method(self.add_registered_objects, True, visible=False) self.sheerka.bind_service_method(self.memory, False) - self.sheerka.cache_manager.register_cache(self.SHORT_TERM_OBJECTS_ENTRY, self.short_term_objects, persist=False) self.sheerka.cache_manager.register_cache(self.OBJECTS_ENTRY, self.objects, persist=True, use_ref=True) + def reset(self): + self.short_term_objects.clear() + + def initialize_deferred(self, context, is_first_time): + self.sheerka.subscribe(CONTEXT_DISPOSED, self.remove_context) + def get_from_short_term_memory(self, context, key): while True: - key_to_use = (str(context.id) if context else "") + ":" + key - if (obj := self.sheerka.cache_manager.get(self.SHORT_TERM_OBJECTS_ENTRY, key_to_use)) is not None: - return obj + try: + id_to_use = context.id if context else self.GLOBAL + return self.short_term_objects.cache[id_to_use][key] + except KeyError: + if context is None: + return None - if context is None: - return None + context = context.get_parent() - context = context.get_parent() + def get_all_short_term_memory(self, context): + return self.short_term_objects.get(context.id) - def add_to_short_term_memory(self, context, key, concept): + def add_to_short_term_memory(self, context, key, value): if context: context.stm = True - key_to_use = (str(context.id) if context else "") + ":" + key - return self.sheerka.cache_manager.put(self.SHORT_TERM_OBJECTS_ENTRY, key_to_use, concept) + id_to_use = context.id + else: + id_to_use = SheerkaMemory.GLOBAL + + if id_to_use in self.short_term_objects.cache: + self.short_term_objects.cache[id_to_use][key] = value + else: + self.short_term_objects.put(id_to_use, {key: value}) + + def add_many_to_short_term_memory(self, context, bag): + context.stm = True + self.short_term_objects.put(context.id if context else self.GLOBAL, bag) def remove_context(self, context): - self.short_term_objects.evict_by_key(lambda k: k.startswith(str(context.id) + ":")) + try: + del self.short_term_objects.cache[context.id] + except KeyError: + pass def add_to_memory(self, context, key, concept): """ @@ -74,6 +100,11 @@ class SheerkaMemory(BaseService): def register_object(self, context, key, concept): """ Before adding objects to memory, they first need to be registered + More: + We don't want to add all evaluated concept into memory + (because some of them may be ref to concept already in memory) + So we first register them, and add the end of sheerka.evaluate_user_input() + all remaining registered concepts will be added to memory :param context: :param key: :param concept: diff --git a/src/core/sheerka/services/SheerkaModifyConcept.py b/src/core/sheerka/services/SheerkaModifyConcept.py index 0f4589d..e34cd25 100644 --- a/src/core/sheerka/services/SheerkaModifyConcept.py +++ b/src/core/sheerka/services/SheerkaModifyConcept.py @@ -1,7 +1,8 @@ from core.builtin_concepts import BuiltinConcepts -from core.concept import ensure_concept +from core.builtin_helpers import ensure_concept +from core.concept import NotInit, freeze_concept_attrs, Concept from core.sheerka.services.sheerka_service import BaseService -from parsers.BnfParser import BnfParser +from parsers.BnfDefinitionParser import BnfDefinitionParser class SheerkaModifyConcept(BaseService): @@ -47,6 +48,9 @@ class SheerkaModifyConcept(BaseService): BuiltinConcepts.CONCEPT_ALREADY_DEFINED, body=concept)) + # update attributes + freeze_concept_attrs(concept) + self.sheerka.cache_manager.update_concept(old_version, concept) # TODO : update concept by first keyword : have a look at update_references() below @@ -72,8 +76,8 @@ class SheerkaModifyConcept(BaseService): for concept_id in refs: concept = self.sheerka.get_by_id(concept_id) - if concept.bnf is not None: - BnfParser.update_recurse_id(context, concept_id, concept.bnf) + if concept.get_bnf() is not None: + BnfDefinitionParser.update_recurse_id(context, concept_id, concept.get_bnf()) # remove the grammar entry so that it can be recreated self.sheerka.cache_manager.delete(self.sheerka.CONCEPTS_GRAMMARS_ENTRY, concept_id) @@ -88,7 +92,9 @@ class SheerkaModifyConcept(BaseService): :return: """ ensure_concept(concept) - concept.set_value(attribute, value) + + attr = attribute.str_id if isinstance(attribute, Concept) else attribute + concept.set_value(attr, value) return self.sheerka.ret(self.NAME, True, self.sheerka.new(BuiltinConcepts.SUCCESS)) def get_attr(self, concept, attribute): @@ -103,6 +109,8 @@ class SheerkaModifyConcept(BaseService): if not self.sheerka.is_success(concept): return concept - if (value := concept.get_value(attribute)) == BuiltinConcepts.NOT_INITIALIZED: + attr = attribute.str_id if isinstance(attribute, Concept) else attribute + + if (value := concept.get_value(attr)) == NotInit: return self.sheerka.new(BuiltinConcepts.NOT_FOUND, body={"#concept": concept, "#attr": attribute}) return value diff --git a/src/core/sheerka/services/SheerkaOut.py b/src/core/sheerka/services/SheerkaOut.py new file mode 100644 index 0000000..0e7f0c7 --- /dev/null +++ b/src/core/sheerka/services/SheerkaOut.py @@ -0,0 +1,100 @@ +from core.builtin_concepts import BuiltinConcepts +from core.sheerka.services.sheerka_service import BaseService +from core.utils import as_bag +from out.ConsoleVisistor import ConsoleVisitor +from out.DeveloperVisitor import DeveloperVisitor + + +class SheerkaOut(BaseService): + NAME = "Out" + + def __init__(self, sheerka): + super().__init__(sheerka) + self.out_visitors = [ConsoleVisitor()] + + def initialize(self): + self.sheerka.bind_service_method(self.process_return_values, False) + + def create_out_tree(self, context, obj): + return self.create_out_tree_recursive(context, {'__obj': obj}, DeveloperVisitor(self, set(), 0)) + + def create_out_tree_recursive(self, context, bag, visitor): + debugger = context.get_debugger(SheerkaOut.NAME, "create_out_tree") + debugger.debug_entering(bag=bag) + + current_obj = bag["__obj"] + bag = self.update_bag(bag, visitor.list_recursion_depth) + + valid_rules = self.sheerka.evaluate_format_rules(context, bag, visitor.already_seen).get(True, None) + res = None + if valid_rules: + if len(valid_rules) > 1: + # TODO manage when too many rules + pass + + rule = valid_rules[0] + if rule.id in visitor.already_seen: + debugger.debug_log(f"Rule #{rule.id} already fired.") + else: + debugger.debug_log(f"Applying rule {rule}.") + visitor.already_seen.add(rule.id) + + bag.update(as_bag(current_obj)) # update with the current obj attributes + visitor.visit(context, rule.compiled_action, bag) + res = visitor.get_result() + + if res is None: + debugger.debug_log(f"No matching rule.") + res = current_obj + + debugger.debug_var("out_tree", res) + + return res + + def process_return_values(self, context, ret): + with context.push(BuiltinConcepts.BEFORE_RENDERING, + None, + desc=f"step='{BuiltinConcepts.BEFORE_RENDERING}'") as sub_context: + sub_context.protected_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED) + sub_context.protected_hints.add(BuiltinConcepts.EVAL_WHERE_REQUESTED) + sub_context.protected_hints.add(BuiltinConcepts.EVAL_UNTIL_SUCCESS_REQUESTED) + sub_context.protected_hints.add(BuiltinConcepts.EVAL_QUESTION_REQUESTED) + # sub_context.deactivate_push() + + out_tree = self.create_out_tree(sub_context, ret) + + # sub_context.activate_push() + + if out_tree: + for visitor in self.out_visitors: + visitor.visit(context, out_tree, None) + if hasattr(visitor, "finalize"): + visitor.finalize() + + return self.sheerka.ret(self.NAME, True, self.sheerka.new(BuiltinConcepts.SUCCESS)) + return self.sheerka.ret(self.NAME, False, self.sheerka.new(BuiltinConcepts.NO_RESULT)) + + def update_bag(self, bag, depth): + obj = bag["__obj"] + bag["__tab"] = " " * depth + + if self.sheerka.isinstance(obj, BuiltinConcepts.RETURN_VALUE): + bag["__ret"] = obj + if self.sheerka.is_container(obj.body): + bag["__ret_container"] = obj.body + bag["__ret_value"] = self.simplify_list(obj.body.body) + bag["__ret_val"] = bag["__ret_value"] + else: + bag["__ret_value"] = self.simplify_list(obj.body) + bag["__ret_val"] = bag["__ret_value"] + elif isinstance(obj, list) and len(obj) > 0 and self.sheerka.isinstance(obj[0], BuiltinConcepts.RETURN_VALUE): + bag["__rets"] = obj + + return bag + + @staticmethod + def simplify_list(item): + try: + return item[0] if hasattr(item, "__len__") and len(item) == 1 else item + except KeyError: + return item # Caution. it's a dict, not a list ! diff --git a/src/core/sheerka/services/SheerkaResultManager.py b/src/core/sheerka/services/SheerkaResultManager.py index 9945cc4..0a473b6 100644 --- a/src/core/sheerka/services/SheerkaResultManager.py +++ b/src/core/sheerka/services/SheerkaResultManager.py @@ -1,5 +1,8 @@ +import ast + from core.builtin_concepts import BuiltinConcepts from core.sheerka.services.sheerka_service import BaseService +from core.utils import as_bag class SheerkaResultConcept(BaseService): @@ -14,10 +17,31 @@ class SheerkaResultConcept(BaseService): self.sheerka.bind_service_method(self.get_results_by_command, True) # digest is recorded self.sheerka.bind_service_method(self.get_last_results, True) # digest is recorded self.sheerka.bind_service_method(self.get_results, False) + self.sheerka.bind_service_method(self.get_execution_item, False) - def get_results_by_digest(self, context, digest, record_digest=True): + @staticmethod + def get_predicate(**kwargs): + if len(kwargs) == 0: + return None + res = [] + if "filter" in kwargs: + res.append(kwargs["filter"]) + kwargs.pop("filter") + + for k, v in kwargs.items(): + if k in ("depth", "recursion_depth"): + continue + + if isinstance(v, str): + v = '"' + v.translate(str.maketrans({'"': r'\"'})) + '"' + res.append(f"{k} == {v}") + predicate = " and ".join(res) + return compile(ast.parse(predicate, mode="eval"), "", mode="eval") + + def get_results_by_digest(self, context, digest, filter=None, record_digest=True, **kwargs): """ Gets the entire execution tree for the given event digest + :param filter: :param context: :param digest: :param record_digest: @@ -26,27 +50,39 @@ class SheerkaResultConcept(BaseService): if digest is None: return None + if filter is not None: + kwargs["filter"] = filter + try: result = self.sheerka.sdp.load_result(digest) event = self.sheerka.sdp.load_event(digest) if record_digest: context.log(f"Recording digest '{digest}'") - self.sheerka.record(context, self.NAME, "digest", digest) + self.sheerka.record_var(context, self.NAME, "digest", digest) + + explanation = self.sheerka.new(BuiltinConcepts.EXPLANATION, + digest=event.get_digest(), + command=event.message, + body=self.as_list(result, self.get_predicate(**kwargs))) + + # add format instructions if applicable + if (depth := kwargs.get("depth", None)) is not None or \ + (depth := kwargs.get("recursion_depth", None)) is not None: + explanation.set_format_instr(recursion_depth=depth, recurse_on="_children") + + return explanation - return self.sheerka.new(BuiltinConcepts.EXPLANATION, - digest=event.get_digest(), - command=event.message, - body=self.as_list(result)) except FileNotFoundError as ex: context.log_error(f"Digest {digest} is not found.", self.NAME, ex) return self.sheerka.new(BuiltinConcepts.NOT_FOUND, body={"digest": digest}) - def get_results_by_command(self, context, command, record_digest=True): + def get_results_by_command(self, context, command, filter=None, record_digest=True, **kwargs): """ Get the result of the command that starts with command :param context: :param command: + :param filter: :param record_digest: :return: """ @@ -59,7 +95,7 @@ class SheerkaResultConcept(BaseService): for event in self.sheerka.sdp.load_events(self.page_size, start): consumed += 1 if event.message.startswith(command): - return self.get_results_by_digest(context, event.get_digest(), record_digest) + return self.get_results_by_digest(context, event.get_digest(), filter, record_digest, **kwargs) if consumed < self.page_size: break @@ -69,10 +105,11 @@ class SheerkaResultConcept(BaseService): return self.sheerka.new(BuiltinConcepts.NOT_FOUND, body={"command": command}) - def get_last_results(self, context, record_digest=True): + def get_last_results(self, context, filter=None, record_digest=True, **kwargs): """ Gets the results of the last command :param context: + :param filter: :param record_digest: :return: """ @@ -84,7 +121,7 @@ class SheerkaResultConcept(BaseService): for event in self.sheerka.sdp.load_events(page_size, start): consumed += 1 if self.sheerka.sdp.has_result(event.get_digest()): - return self.get_results_by_digest(context, event.get_digest(), record_digest) + return self.get_results_by_digest(context, event.get_digest(), filter, record_digest, **kwargs) if consumed < page_size: break @@ -97,27 +134,45 @@ class SheerkaResultConcept(BaseService): return self.sheerka.new(BuiltinConcepts.NOT_FOUND, body={"query": "last"}) - def get_results(self, context): + def get_results(self, context, filter=None, **kwargs): """ Use the last digest saved to get the execution results :param context: + :param filter: :return: """ - digest = self.sheerka.load(self.NAME, "digest") + digest = self.sheerka.load_var(self.NAME, "digest") if digest is None: context.log("No recorded digest found.") return None - return self.get_results_by_digest(context, digest, False) + return self.get_results_by_digest(context, digest, filter, False, **kwargs) + + def get_execution_item(self, context, item_id): + digest = self.sheerka.load_var(self.NAME, "digest") + if digest is None: + return self.sheerka.new(BuiltinConcepts.NOT_FOUND, body="no digest") + + try: + result = self.sheerka.sdp.load_result(digest) + items = list(self.as_list(result, self.get_predicate(id=item_id))) + + if len(items) == 0: + return self.sheerka.new(BuiltinConcepts.NOT_FOUND, body={"id": item_id}) + + return items[0] + + except FileNotFoundError as ex: + context.log_error(f"Digest {digest} is not found.", self.NAME, ex) + return self.sheerka.new(BuiltinConcepts.NOT_FOUND, body={"digest": digest}) @staticmethod - def as_list(execution_context): - + def as_list(execution_context, predicate): def _yield_result(lst): - for e in lst: - yield e + if predicate is None or eval(predicate, as_bag(e)): + yield e if e._children: yield from _yield_result(e._children) diff --git a/src/core/sheerka/services/SheerkaRuleManager.py b/src/core/sheerka/services/SheerkaRuleManager.py new file mode 100644 index 0000000..6beed59 --- /dev/null +++ b/src/core/sheerka/services/SheerkaRuleManager.py @@ -0,0 +1,642 @@ +import operator +import re +from dataclasses import dataclass +from typing import Union + +from cache.Cache import Cache +from core.builtin_concepts import BuiltinConcepts, ReturnValueConcept +from core.builtin_helpers import parse_unrecognized, only_successful, ensure_rule +from core.concept import Concept +from core.global_symbols import RULE_PRECEDENCE_MODIFIED, RULE_COMPARISON_CONTEXT +from core.rule import Rule +from core.sheerka.services.sheerka_service import BaseService +from core.tokenizer import Keywords, TokenKind, Token, IterParser +from core.utils import index_tokens, COLORS, get_text_from_tokens +from evaluators.ConceptEvaluator import ConceptEvaluator +from evaluators.PythonEvaluator import PythonEvaluator +from parsers.BaseNodeParser import SourceCodeWithConceptNode, ConceptNode, SourceCodeNode +from parsers.PythonParser import PythonNode + +identifier_regex = re.compile(r"[\w _.]+") + + +@dataclass +class FormatRuleError: + pass + + +@dataclass +class BraceMismatch(FormatRuleError): + lbrace: Token + + +@dataclass +class UnexpectedEof(FormatRuleError): + message: str + token: Token = None + + def __eq__(self, other): + if id(self) == id(other): + return True + + if not isinstance(other, UnexpectedEof): + return False + + return self.message == other.message and (other.token is None or other.token == self.token) + + def __hash__(self): + return hash(self.message, self.token) + + +@dataclass +class FormatRuleSyntaxError(FormatRuleError): + message: str + token: Token + + +@dataclass +class FormatAstNode: + @staticmethod + def repr_value(items): + if items is None: + return "" + + return ", ".join(repr(item) for item in items) + + +@dataclass +class FormatAstRawText(FormatAstNode): + text: str + + +@dataclass +class FormatAstVariable(FormatAstNode): + name: str + format: Union[str, None] = None + value: object = None + index: object = None + + +@dataclass +class FormatAstVariableNotFound(FormatAstNode): + name: str + + +@dataclass +class FormatAstGrid(FormatAstNode): + pass + + +@dataclass +class FormatAstList(FormatAstNode): + variable: str + items_prop: str = None # where to search the list if variable does not resolve to an iterable + recurse_on: str = None + recursion_depth: int = 0 + items: object = None + + +@dataclass +class FormatAstColor(FormatAstNode): + def __init__(self, color, format_ast): + self.color = color + self.format_ast = format_ast + + def __repr__(self): + return f"{self.color}({self.format_ast})" + + def __eq__(self, other): + if id(self) == id(other): + return True + + if not isinstance(other, FormatAstColor): + return False + + return self.color == other.color and self.format_ast == other.format_ast + + def __hash__(self): + return hash((self.color, self.format_ast)) + + +@dataclass +class FormatAstFunction(FormatAstNode): + name: str + args: list = None + kwargs: dict = None + + +class FormatAstSequence(FormatAstNode): + def __init__(self, items): + self.items = items + + def __repr__(self): + return "FormatAstSequence(" + self.repr_value(self.items) + ")" + + def __eq__(self, other): + if id(self) == id(other): + return True + + if not isinstance(other, FormatAstSequence): + return False + + return self.items == other.items + + +class FormatRuleParser(IterParser): + + @staticmethod + def to_text(list_or_dict_of_tokens): + """ + Works on list of list of tokens + or dict of list of tokens + :param list_or_dict_of_tokens: + :return: + """ + get_text = get_text_from_tokens + if isinstance(list_or_dict_of_tokens, list): + return [get_text(i) for i in list_or_dict_of_tokens] + if isinstance(list_or_dict_of_tokens, dict): + return {k: get_text(v) for k, v in list_or_dict_of_tokens.items()} + raise NotImplementedError("") + + def to_value(self, tokens): + """ + Works on list of tokens + return string or numeric value of the tokens + :return: + """ + + value = get_text_from_tokens(tokens) + if value[0] in ("'", '"'): + return value[1:-1] + + try: + return int(value) + except ValueError: + pass + + try: + return float(value) + except ValueError: + self.error_sink = FormatRuleSyntaxError(f"'{value}' is not numeric", None) + + def parse(self): + """ + Parses a format rule + format ::= {variable'} | function(...) | rawtext + :return: + """ + + if self.source == "": + return FormatAstRawText("") + + buffer = [] + result = [] + res = None + escaped = False + + def _flush_buffer(): + if len(buffer) > 0: + result.append(FormatAstRawText(get_text_from_tokens(buffer))) + buffer.clear() + + while self.next_token(skip_whitespace=False): + if not escaped: + if self.token.type == TokenKind.IDENTIFIER and self.the_token_after().type == TokenKind.LPAR: + _flush_buffer() + res = self.parse_function(self.token) + elif self.token.type == TokenKind.LBRACE: + _flush_buffer() + res = self.parse_variable(self.token) + elif self.token.type == TokenKind.BACK_SLASH: + escaped = True + else: + buffer.append(self.token) + else: + escaped = False + buffer.append(self.token) + + if self.error_sink: + break + + if res: + result.append(res) + res = None + + _flush_buffer() + + return [] if len(result) == 0 else result[0] if len(result) == 1 else FormatAstSequence(result) + + def parse_function(self, func_name): + self.next_token() + self.next_token() + + if self.token.type == TokenKind.EOF: + self.error_sink = UnexpectedEof("while parsing function", func_name) + return None + + param_buffer = [] + args = [] + kwargs = {} + get_text = get_text_from_tokens + + def _process_parameters(): + if len(param_buffer) == 0: + self.error_sink = FormatRuleSyntaxError("no parameter found", self.token) + return None + if (index := index_tokens(param_buffer, "=")) > 0: + kwargs[get_text(param_buffer[:index])] = param_buffer[index + 1:] + else: + args.append(param_buffer.copy()) + param_buffer.clear() + + while True: + if self.token.type == TokenKind.RPAR: + if len(param_buffer) > 0: + _process_parameters() + break + + elif self.token.type == TokenKind.COMMA: + _process_parameters() + if self.error_sink: + break + + else: + param_buffer.append(self.token) + + if not self.next_token(): + break + + if self.error_sink: + return None + + if self.token.type != TokenKind.RPAR: + self.error_sink = UnexpectedEof("while parsing function", func_name) + return None + + if func_name.value in COLORS: + return self.return_color(func_name.value, args, kwargs) + elif func_name.value == "list": + return self.return_list(args, kwargs) + + return FormatAstFunction(func_name.value, self.to_text(args), self.to_text(kwargs)) + + def parse_variable(self, lbrace): + self.next_token() + + if self.token.type == TokenKind.EOF: + self.error_sink = UnexpectedEof("while parsing variable", lbrace) + return None + + buffer = [] + while True: + if self.token.type == TokenKind.RBRACE: + break + buffer.append(self.token) + + if not self.next_token(): + break + + # if self.error_sink: + # return None + + if self.token.type != TokenKind.RBRACE: + self.error_sink = UnexpectedEof("while parsing variable", lbrace) + return None + + if len(buffer) == 0: + self.error_sink = FormatRuleSyntaxError("variable name not found", None) + return None + + variable = get_text_from_tokens(buffer) + try: + index = variable.index(":") + return FormatAstVariable(variable[:index], variable[index + 1:]) + except ValueError: + return FormatAstVariable(variable) + + def return_color(self, color, args, kwargs): + if len(kwargs) > 0: + self.error_sink = FormatRuleSyntaxError("keyword arguments are not supported", None) + return None + + if len(args) == 0: + return FormatAstColor(color, FormatAstRawText("")) + + if len(args) > 1: + self.error_sink = FormatRuleSyntaxError("only one parameter supported", args[1][0]) + return None + + source = get_text_from_tokens(args[0]) + if len(source) > 1 and source[0] in ("'", '"') and source[-1] in ("'", '"'): + source = source[1:-1] + parser = FormatRuleParser(source) + res = parser.parse() + self.error_sink = parser.error_sink + return FormatAstColor(color, res) + else: + try: + index = source.index(":") + variable, vformat = source[:index], source[index + 1:] + except ValueError: + variable, vformat = source, None + + if not identifier_regex.fullmatch(variable): + self.error_sink = FormatRuleSyntaxError("Invalid identifier", None) + return None + return FormatAstColor(color, FormatAstVariable(variable, vformat)) + + def return_list(self, args, kwargs): + len_args = len(args) + if len_args < 1: + self.error_sink = FormatRuleSyntaxError("variable name not found", None) + return None + + if len_args > 3: + self.error_sink = FormatRuleSyntaxError("too many positional arguments", args[3][0]) + return None + + variable_name = get_text_from_tokens(args[0]) + recurse_on, recursion_depth, items_prop = None, 0, None + + if len_args == 2: + recursion_depth = self.to_value(args[1]) + elif len_args == 3: + recursion_depth = self.to_value(args[1]) + recurse_on = self.to_value(args[2]) + + if "recurse_on" in kwargs: + recurse_on = self.to_value(kwargs["recurse_on"]) + + if "recursion_depth" in kwargs: + recursion_depth = self.to_value(kwargs["recursion_depth"]) + + if "items_prop" in kwargs: + items_prop = self.to_value(kwargs["items_prop"]) + + if self.error_sink: + return None + + if not isinstance(recursion_depth, int): + self.error_sink = FormatRuleSyntaxError("'recursion_depth' must be an integer", None) + return None + + return FormatAstList(variable_name, items_prop, recurse_on, recursion_depth) + + +@dataclass() +class RulePredicate: + source: str + evaluator: str + predicate: ReturnValueConcept + concept: Union[Concept, None] + + +class SheerkaRuleManager(BaseService): + NAME = "RuleManager" + RULE_IDS = "Rules_Ids" + FORMAT_RULE_ENTRY = "RuleManager:FormatRules" + EXEC_RULE_ENTRY = "RuleManager:ExecRules" + + def __init__(self, sheerka): + super().__init__(sheerka) + self.format_rule_cache = Cache(default=lambda k: self.sheerka.sdp.get(self.FORMAT_RULE_ENTRY, k)) + self.exec_rule_cache = Cache(default=lambda k: self.sheerka.sdp.get(self.EXEC_RULE_ENTRY, k)) + + self._format_rules = None # sorted by priority + + def initialize(self): + self.sheerka.bind_service_method(self.create_new_rule, True, visible=False) + self.sheerka.bind_service_method(self.get_rule_by_id, False) + self.sheerka.bind_service_method(self.dump_desc_rule, False, as_name="desc_rule") + self.sheerka.bind_service_method(self.get_format_rules, False, visible=False) + + self.sheerka.cache_manager.register_cache(self.FORMAT_RULE_ENTRY, self.format_rule_cache, True, True) + self.sheerka.cache_manager.register_cache(self.EXEC_RULE_ENTRY, self.exec_rule_cache, True, True) + + def initialize_deferred(self, context, is_first_time): + + if is_first_time: + # add builtin rules if it's the first initialization of Sheerka + self.init_builtin_rules(context) + + # adds the other rules (when it's not the first time) + self.format_rule_cache.populate(lambda: self.sheerka.sdp.list(self.FORMAT_RULE_ENTRY), lambda rule: rule.id) + self.exec_rule_cache.populate(lambda: self.sheerka.sdp.list(self.EXEC_RULE_ENTRY), lambda rule: rule.id) + + # compile all the rules + for rule_id in self.format_rule_cache: + rule = self.init_rule(context, self.format_rule_cache.get(rule_id)) + + # update rules priorities + self.update_rules_priorities(context) + + self.sheerka.subscribe(RULE_PRECEDENCE_MODIFIED, self.update_rules_priorities) + + def update_rules_priorities(self, context): + """ + Ask the SheerkaComparisonManager for the priorities + :return: + """ + # get the priorities + rules_weights = self.sheerka.get_concepts_weights(BuiltinConcepts.PRECEDENCE, RULE_COMPARISON_CONTEXT) + + # compile all the rules + for rule_id in self.format_rule_cache: + rule = self.format_rule_cache.get(rule_id) + if rule.str_id in rules_weights: + rule.priority = rules_weights[rule.str_id] + + self._format_rules = None + + def init_rule(self, context, rule: Rule): + if rule.metadata.is_compiled: + return + + if rule.compiled_predicate is None: + res = self.compile_when(context, self.NAME, rule.metadata.predicate) + if not isinstance(res, list): + rule.error_sink = [res.body] + return + rule.compiled_predicate = res + + if rule.compiled_action is None: + res = self.compile_print(context, rule.metadata.action) + if not res.status: + rule.error_sink = [res.body] + return + rule.compiled_action = res.body + + # rule.variables = self.get_variables() + + rule.metadata.is_compiled = True + rule.metadata.is_enabled = True + return rule + + def compile_when(self, context, name, source): + # parser_input = self.sheerka.services[SheerkaExecute.NAME].get_parser_input(source) + parsed = parse_unrecognized(context, + source, + parsers="all", + who=name, + prop=Keywords.WHEN, + filter_func=only_successful) + + if not parsed.status: + return parsed + + if self.sheerka.isinstance(parsed.body, BuiltinConcepts.ONLY_SUCCESSFUL): + parsed = parsed.body.body + + return self.add_evaluators(source, parsed if hasattr(parsed, "__iter__") else [parsed]) + + def compile_print(self, context, source): + parser = FormatRuleParser(source) + parsed = parser.parse() + if parser.error_sink: + return self.sheerka.ret(self.NAME, + False, + self.sheerka.new(BuiltinConcepts.ERROR, body=[parser.error_sink])) + else: + return self.sheerka.ret(self.NAME, True, parsed) + + def set_id_if_needed(self, rule: Rule): + """ + Set the id for the concept if needed + :param rule: + :return: + """ + if rule.metadata.id is not None: + return + + rule.metadata.id = str(self.sheerka.cache_manager.get(self.sheerka.CONCEPTS_KEYS_ENTRY, self.RULE_IDS)) + + def create_new_rule(self, context, rule): + """ + Saves the new rule in DB + :param context: + :param rule: + :return: + """ + sheerka = self.sheerka + + # set id before saving in db + self.set_id_if_needed(rule) + if rule.compiled_predicate and rule.compiled_action: + rule.metadata.is_compiled = True + rule.metadata.is_enabled = True + + # save it + if rule.metadata.action_type == "print": + self.sheerka.cache_manager.put(self.FORMAT_RULE_ENTRY, rule.metadata.id, rule) + self._format_rules = None + else: + self.sheerka.cache_manager.put(self.EXEC_RULE_ENTRY, rule.metadata.id, rule) + + # process the return if needed + ret = sheerka.ret(self.NAME, True, sheerka.new(BuiltinConcepts.NEW_RULE, body=rule)) + return ret + + def init_builtin_rules(self, context): + # self.sheerka.init_log.debug("Initializing default rules") + rules = [ + Rule("print", "Print return values", "__rets", "list(__rets)"), + Rule("print", "Print ReturnValue", + "__ret", + "\\ReturnValue(who={__ret.who}, status={__ret.status}, value={__ret.value})"), + Rule("print", "Failed ReturnValue in red", + "__ret and not __ret.status", + "red(__ret)"), + Rule("print", "List explanations", + "isinstance(__ret_container, BuiltinConcepts.EXPLANATION)", + "blue(__ret_container.digest) : {__ret_container.command}\nlist(__ret_container)"), + Rule("print", "Print ExecutionContext", + "isinstance(__obj, ExecutionContext)", + "[{id:3}] {__tab}{desc} ({status})"), + Rule("print", "Display formatted list", + "isinstance(__ret_container, BuiltinConcepts.TO_LIST)", + "list(__ret_container)"), + ] + + for r in rules: + self.create_new_rule(context, r) + + self.sheerka.set_is_less_than(context, BuiltinConcepts.PRECEDENCE, rules[1], rules[2], RULE_COMPARISON_CONTEXT) + self.sheerka.set_is_less_than(context, BuiltinConcepts.PRECEDENCE, rules[1], rules[3], RULE_COMPARISON_CONTEXT) + self.sheerka.set_is_less_than(context, BuiltinConcepts.PRECEDENCE, rules[1], rules[5], RULE_COMPARISON_CONTEXT) + self.sheerka.set_is_greatest(context, BuiltinConcepts.PRECEDENCE, rules[0], RULE_COMPARISON_CONTEXT) + + def get_rule_by_id(self, rule_id): + """ + Looks in the caches for a specific rule id + :param rule_id: + :return: + """ + if rule_id is None: + return None + + rule = self.format_rule_cache.get(rule_id) + if rule: + return rule + + rule = self.exec_rule_cache.get(rule_id) + if rule: + return rule + + metadata = [("id", rule_id)] + return self.sheerka.new(BuiltinConcepts.UNKNOWN_RULE, body=metadata) + + def dump_desc_rule(self, rules): + """ + dumps the definition of a rule + :param rules: + :return: + """ + ensure_rule(rules) + + if not hasattr(rules, "__iter__"): + rules = [rules] + + first = True + for rule in rules: + if not first: + self.sheerka.log.info("") + self.sheerka.log.info(f"id : {rule.id}") + self.sheerka.log.info(f"name : {rule.metadata.name}") + self.sheerka.log.info(f"type : {rule.metadata.action_type}") + self.sheerka.log.info(f"predicate : {rule.metadata.predicate}") + self.sheerka.log.info(f"action : {rule.metadata.action}") + self.sheerka.log.info(f"compiled : {rule.metadata.is_compiled}") + self.sheerka.log.info(f"enabled : {rule.metadata.is_enabled}") + + def get_format_rules(self): + if self._format_rules: + return self._format_rules + + self._format_rules = sorted(self.format_rule_cache.get_all(), key=operator.attrgetter('priority'), reverse=True) + return self._format_rules + + def add_evaluators(self, source, ret_vals): + """ + Browse the ReturnValueConcepts to determine the evaluator to use + Returns a list of tuple (evaluator_name, return_value) + :param source: + :param ret_vals: + :return: + """ + res = [] + for r in ret_vals: + underlying = self.sheerka.objvalue(r) + if isinstance(underlying, PythonNode): + res.append(RulePredicate(source, PythonEvaluator.NAME, r, None)) + elif isinstance(underlying, SourceCodeWithConceptNode): + res.append(RulePredicate(source, PythonEvaluator.NAME, r, None)) + elif isinstance(underlying, SourceCodeNode): + res.append(RulePredicate(source, PythonEvaluator.NAME, r, None)) + elif isinstance(underlying, Concept): + res.append(RulePredicate(source, ConceptEvaluator.NAME, r, underlying)) + elif hasattr(underlying, "__iter__") and len(underlying) == 1 and isinstance(underlying[0], ConceptNode): + res.append(RulePredicate(source, ConceptEvaluator.NAME, r, underlying[0].concept)) + else: + raise NotImplementedError(r) + return res diff --git a/src/core/sheerka/services/SheerkaSetsManager.py b/src/core/sheerka/services/SheerkaSetsManager.py index a460ecd..5717b8c 100644 --- a/src/core/sheerka/services/SheerkaSetsManager.py +++ b/src/core/sheerka/services/SheerkaSetsManager.py @@ -1,9 +1,9 @@ import core.builtin_helpers from cache.Cache import Cache from cache.SetCache import SetCache -from core.ast.nodes import python_to_concept +from core.ast_helpers import UnreferencedVariablesVisitor from core.builtin_concepts import BuiltinConcepts -from core.concept import Concept, ConceptParts, ensure_concept, DEFINITION_TYPE_BNF +from core.concept import Concept, ConceptParts, DEFINITION_TYPE_BNF from core.sheerka.services.SheerkaModifyConcept import SheerkaModifyConcept from core.sheerka.services.sheerka_service import BaseService @@ -41,9 +41,10 @@ class SheerkaSetsManager(BaseService): """ context.log(f"Setting concept {concept} is a {concept_set}", who=self.NAME) - ensure_concept(concept, concept_set) + core.builtin_helpers.ensure_concept(concept, concept_set) - if BuiltinConcepts.ISA in concept.metadata.props and concept_set in concept.metadata.props[BuiltinConcepts.ISA]: + if BuiltinConcepts.ISA in concept.get_metadata().props and concept_set in concept.get_metadata().props[ + BuiltinConcepts.ISA]: return self.sheerka.ret( self.NAME, False, @@ -71,7 +72,7 @@ class SheerkaSetsManager(BaseService): """ context.log(f"Adding concept {concept} to set {concept_set}", who=self.NAME) - ensure_concept(concept, concept_set) + core.builtin_helpers.ensure_concept(concept, concept_set) set_elements = self.sets.get(concept_set.id) if set_elements and concept.id in set_elements: @@ -98,7 +99,7 @@ class SheerkaSetsManager(BaseService): """ context.log(f"Adding concepts {concepts} to set {concept_set}", who=self.NAME) - ensure_concept(concept_set) + core.builtin_helpers.ensure_concept(concept_set) already_in_set = [] for concept in concepts: res = self.add_concept_to_set(context, concept, concept_set) @@ -124,7 +125,7 @@ class SheerkaSetsManager(BaseService): :return: """ - ensure_concept(concept) + core.builtin_helpers.ensure_concept(concept) def _get_set_elements(sub_concept): if not self.isaset(context, sub_concept): @@ -146,8 +147,8 @@ class SheerkaSetsManager(BaseService): concepts.extend(other_concepts) # apply the where clause if any - if sub_concept.metadata.where: - new_condition = self._validate_where_clause(sub_concept) + if sub_concept.get_metadata().where: + new_condition = self._validate_where_clause(context, sub_concept) if not new_condition: return self.sheerka.new(BuiltinConcepts.CONDITION_FAILED, body=sub_concept) @@ -179,7 +180,7 @@ class SheerkaSetsManager(BaseService): :return: """ - ensure_concept(a, b) + core.builtin_helpers.ensure_concept(a, b) # TODO, first check the 'isa' property of a if not (a.id and b.id): @@ -190,11 +191,11 @@ class SheerkaSetsManager(BaseService): def isa(self, a, b): - ensure_concept(a, b) - if BuiltinConcepts.ISA not in a.metadata.props: + core.builtin_helpers.ensure_concept(a, b) + if BuiltinConcepts.ISA not in a.get_metadata().props: return False - for c in a.metadata.props[BuiltinConcepts.ISA]: + for c in a.get_metadata().props[BuiltinConcepts.ISA]: if c == b: return True if self.isa(self.sheerka.get_by_id(c.id), b): @@ -216,7 +217,7 @@ class SheerkaSetsManager(BaseService): # KSI 20200629 # To resolve infinite recursion between group concepts and BNF concepts - if concept.metadata.definition_type == DEFINITION_TYPE_BNF: + if concept.get_metadata().definition_type == DEFINITION_TYPE_BNF: return False # check if it has a group @@ -231,18 +232,18 @@ class SheerkaSetsManager(BaseService): return self.isaset(context, concept.body) - def _validate_where_clause(self, concept): - python_parser_result = [r for r in concept.compiled[ConceptParts.WHERE] if r.who == "parsers.Python"] + def _validate_where_clause(self, context, concept): + python_parser_result = [r for r in concept.get_compiled()[ConceptParts.WHERE] if r.who == "parsers.Python"] if not python_parser_result or not python_parser_result[0].status: return None ast_ = python_parser_result[0].body.body.ast_ - ast_as_concepts = python_to_concept(ast_) - names = core.builtin_helpers.get_names(self.sheerka, ast_as_concepts) - if len(names) != 1 or names[0] != concept.metadata.body: + visitor = UnreferencedVariablesVisitor(context) + names = list(visitor.get_names(ast_)) + if len(names) != 1 or names[0] != concept.get_metadata().body: return None - condition = concept.metadata.where.replace(concept.metadata.body, "sheerka.objvalue(x)") + condition = concept.get_metadata().where.replace(concept.get_metadata().body, "sheerka.objvalue(x)") expression = f""" result=[] for x in xx__concepts__xx: @@ -277,7 +278,7 @@ for x in xx__concepts__xx: errors = [] for element_id in ids: concept = self.sheerka.get_by_id(element_id) - if len(concept.metadata.variables) == 0: + if len(concept.get_metadata().variables) == 0: # The concepts are directly taken from Sheerka.get_by_id, so variable cannot be filled # It's the reason why we only evaluate concept with no variable evaluated = self.sheerka.evaluate_concept(sub_context, concept) diff --git a/src/core/sheerka/services/SheerkaVariableManager.py b/src/core/sheerka/services/SheerkaVariableManager.py index 4feb7c1..48a5d6c 100644 --- a/src/core/sheerka/services/SheerkaVariableManager.py +++ b/src/core/sheerka/services/SheerkaVariableManager.py @@ -19,6 +19,9 @@ class Variable(ServiceObj): def get_key(self): return f"{self.who}|{self.key}" + def __str__(self): + return f"({self.who}){self.key}={self.value}" + class SheerkaVariableManager(BaseService): NAME = "VariableManager" @@ -26,18 +29,28 @@ class SheerkaVariableManager(BaseService): def __init__(self, sheerka): super().__init__(sheerka) + self.bound = { + "sheerka.enable_process_return_values": "enable_process_return_values", + "sheerka.save_execution_context": "save_execution_context" + } def initialize(self): - self.sheerka.bind_service_method(self.record, True) - self.sheerka.bind_service_method(self.load, False) - self.sheerka.bind_service_method(self.delete, True) - self.sheerka.bind_service_method(self.set, True) - self.sheerka.bind_service_method(self.get, False) + self.sheerka.bind_service_method(self.record_var, True, visible=False) + self.sheerka.bind_service_method(self.load_var, False, visible=False) + self.sheerka.bind_service_method(self.delete_var, True, visible=False) + self.sheerka.bind_service_method(self.set_var, True) + self.sheerka.bind_service_method(self.get_var, False) + self.sheerka.bind_service_method(self.list_vars, False) - cache = Cache(default=lambda k: self.sheerka.sdp.get(self.VARIABLES_ENTRY, k)) + cache = Cache() + cache.populate(lambda: self.sheerka.sdp.list(self.VARIABLES_ENTRY), lambda var: var.get_key()) self.sheerka.cache_manager.register_cache(self.VARIABLES_ENTRY, cache, True, True) - def record(self, context, who, key, value): + for variable in cache.get_all(): + if variable.key in self.bound: + setattr(self.sheerka, self.bound[variable.key], variable.value) + + def record_var(self, context, who, key, value): """ :param context: @@ -49,20 +62,33 @@ class SheerkaVariableManager(BaseService): variable = Variable(context.event.get_digest(), who, key, value, None) self.sheerka.cache_manager.put(self.VARIABLES_ENTRY, variable.get_key(), variable) + + # TODO: manage credentials + if key in self.bound: + setattr(self.sheerka, self.bound[key], value) + return self.sheerka.ret(self.NAME, True, self.sheerka.new(BuiltinConcepts.SUCCESS)) - def load(self, who, key): + def load_var(self, who, key): variable = self.sheerka.cache_manager.get(self.VARIABLES_ENTRY, who + "|" + key) if variable is None: return None return variable.value - def delete(self, context, who, key): + def delete_var(self, context, who, key): self.sheerka.cache_manager.delete(self.VARIABLES_ENTRY, who + "|" + key) - def set(self, context, key, value): - return self.record(context, context.event.user_id, key, value) + def set_var(self, context, key, value): + return self.record_var(context, context.event.user_id, key, value) - def get(self, context, key): - return self.load(context.event.user_id, key) + def get_var(self, context, key): + return self.load_var(context.event.user_id, key) + + def list_vars(self, context, all_vars=False): + if all_vars: + res = [str(v) for v in self.sheerka.cache_manager.copy(self.VARIABLES_ENTRY).values()] + else: + res = [str(v) for v in self.sheerka.cache_manager.copy(self.VARIABLES_ENTRY).values() if + v.who == context.event.user_id] + return res diff --git a/src/core/sheerka/services/sheerka_service.py b/src/core/sheerka/services/sheerka_service.py index d55e703..89910d9 100644 --- a/src/core/sheerka/services/sheerka_service.py +++ b/src/core/sheerka/services/sheerka_service.py @@ -10,6 +10,7 @@ class BaseService: """ Base class for services """ + def __init__(self, sheerka): self.sheerka = sheerka @@ -19,3 +20,13 @@ class BaseService: :return: """ pass + + def restore_values(self, *args): + """ + Use Variable Manager to restore the state of a service + :param args: + :return: + """ + for prop_name in args: + if (value := self.sheerka.load_var(self.NAME, prop_name)) is not None: + setattr(self, prop_name, value) diff --git a/src/core/tokenizer.py b/src/core/tokenizer.py index d471ca7..1550e9d 100644 --- a/src/core/tokenizer.py +++ b/src/core/tokenizer.py @@ -9,13 +9,14 @@ class TokenKind(Enum): KEYWORD = "keyword" IDENTIFIER = "identifier" CONCEPT = "concept" + RULE = "rule" STRING = "string" NUMBER = "number" TRUE = "true" FALSE = "false" LPAR = "lpar" RPAR = "rpar" - LBRACKET = "lbrace" + LBRACKET = "lbracket" RBRACKET = "rbracket" LBRACE = "lbrace" RBRACE = "rbrace" @@ -49,7 +50,7 @@ class TokenKind(Enum): WORD = "word" EQUALSEQUALS = "==" VAR_DEF = "__var__" - REGEX = "r'xxx' or r\"xxx\" or r:xxx: or r|xxx| or r/xxx/" + REGEX = "r'xxx' or r\"xxx\" or r|xxx| or r/xxx/" @dataclass() @@ -65,18 +66,7 @@ class Token: _repr_value: str = field(default=None, repr=False, compare=False, hash=None) def __repr__(self): - if self.type == TokenKind.IDENTIFIER: - value = str(self.value) - elif self.type == TokenKind.WHITESPACE: - value = "" if self.value == "" else "" if self.value[0] == "\t" else "" - elif self.type == TokenKind.NEWLINE: - value = "" - elif self.type == TokenKind.EOF: - value = "" - else: - value = self.value - - return f"Token({value})" + return f"Token({self.repr_value})" @property def strip_quote(self): @@ -102,9 +92,15 @@ class Token: if self.type == TokenKind.EOF: self._repr_value = "" elif self.type == TokenKind.WHITESPACE: - self._repr_value = "" + self._repr_value = "" if self.value == "" else "" if self.value[0] == "\t" else "" elif self.type == TokenKind.NEWLINE: self._repr_value = "" + elif self.type == TokenKind.CONCEPT: + from core.utils import str_concept + self._repr_value = str_concept(self.value) + elif self.type == TokenKind.RULE: + from core.utils import str_concept + self._repr_value = str_concept(self.value, prefix="r:") else: self._repr_value = self.str_value return self._repr_value @@ -121,6 +117,9 @@ class Token: elif self.type == TokenKind.CONCEPT: from core.utils import str_concept return str_concept(self.value) + elif self.type == TokenKind.RULE: + from core.utils import str_concept + return str_concept(self.value, prefix="r:") else: return str(self.value) @@ -192,18 +191,18 @@ class Tokenizer: self.column += 1 elif c == "_": from core.concept import VARIABLE_PREFIX - if self.i + 1 < self.text_len and self.text[self.i + 1].isalpha(): - identifier = self.eat_identifier(self.i) - yield Token(TokenKind.IDENTIFIER, identifier, self.i, self.line, self.column) - self.i += len(identifier) - self.column += len(identifier) - elif self.i + 7 < self.text_len and \ + if self.i + 7 < self.text_len and \ self.text[self.i: self.i + 7] == VARIABLE_PREFIX and \ self.text[self.i + 7].isdigit(): number = self.eat_number(self.i + 7) yield Token(TokenKind.VAR_DEF, VARIABLE_PREFIX + number, self.i, self.line, self.column) self.i += 7 + len(number) self.column += 7 + len(number) + elif self.i + 1 < self.text_len and (self.text[self.i + 1].isalpha() or self.text[self.i + 1] == "_"): + identifier = self.eat_identifier(self.i) + yield Token(TokenKind.IDENTIFIER, identifier, self.i, self.line, self.column) + self.i += len(identifier) + self.column += len(identifier) else: yield Token(TokenKind.UNDERSCORE, "_", self.i, self.line, self.column) self.i += 1 @@ -341,7 +340,12 @@ class Tokenizer: yield Token(TokenKind.CONCEPT, (name, id), self.i, self.line, self.column) self.i += length + 2 self.column += length + 2 - elif c == "r" and self.i + 1 < self.text_len and self.text[self.i + 1] in "'\":|/": + elif c == "r" and self.i + 1 < self.text_len and self.text[self.i + 1] == ":": + name, id, length = self.eat_concept(self.i + 2, self.line, self.column + 2) + yield Token(TokenKind.RULE, (name, id), self.i, self.line, self.column) + self.i += length + 2 + self.column += length + 2 + elif c == "r" and self.i + 1 < self.text_len and self.text[self.i + 1] in "'\"|/": string, newlines, column_index = self.eat_string(self.i + 1, self.line, self.column) yield Token(TokenKind.REGEX, string, self.i, self.line, self.column) # quotes are kept self.i += len(string) + 1 @@ -368,10 +372,10 @@ class Tokenizer: self.i += len(string) self.column = column_index # 1 if newlines > 0 else self.column + len(string) self.line += newlines - elif c == "_": - yield Token(TokenKind.UNDERSCORE, "_", self.i, self.line, self.column) - self.i += 1 - self.column += 1 + # elif c == "_": + # yield Token(TokenKind.UNDERSCORE, "_", self.i, self.line, self.column) + # self.i += 1 + # self.column += 1 else: raise LexerError(f"Unknown token '{c}'", self.text, self.i, self.line, self.column) @@ -518,3 +522,71 @@ class Tokenizer: break return result + + +class IterParser: + def __init__(self, source): + self.source = source + self.iterator = iter(Tokenizer(source)) + self.tokens_after = [] + self.token = None + self.error_sink = None + + def next_token(self, skip_whitespace=True): + try: + if len(self.tokens_after) > 0: + self.token = self.tokens_after.pop(0) + else: + self.token = next(self.iterator) + if skip_whitespace: + while self.token.type in (TokenKind.WHITESPACE, TokenKind.NEWLINE): + self.token = next(self.iterator) + return self.token.type != TokenKind.EOF + except StopIteration: + return False + + def the_token_after(self, skip_whitespace=True): + try: + token_after = next(self.iterator) + self.tokens_after.append(token_after) + if skip_whitespace: + while token_after.type in (TokenKind.WHITESPACE, TokenKind.NEWLINE): + token_after = next(self.iterator) + self.tokens_after.append(token_after) + + return token_after + except StopIteration: + return Token(TokenKind.EOF, -1, -1, -1, -1) + + +# @dataclass +# class PropDef: +# prop: str +# index: int +# +# +# class SimpleExpressionParser(IterParser): +# def __init__(self, source): +# super().__init__(source) +# self.properties = [] +# +# def parse(self): +# +# prop, index, key = None, None, None +# while self.next_token(): +# if self.token.type == TokenKind.DOT: +# self.properties.append(PropDef(prop, index, key)) +# prop, index, key = None, None, None +# continue +# +# if self.token.type == TokenKind.LBRACKET: +# index = self.parse_index() +# elif self.token.type == TokenKind.LBRACE: +# key = self.parse_key() +# else: +# prop = self.token.value +# +# if prop is not None: +# self.properties.append(PropDef(prop, index, key)) +# +# def parse_i diff --git a/src/core/utils.py b/src/core/utils.py index 4db3e7a..60b5f0b 100644 --- a/src/core/utils.py +++ b/src/core/utils.py @@ -1,15 +1,54 @@ +import ast import importlib import inspect import pkgutil import re -from core.tokenizer import TokenKind +from cache.Cache import Cache +from core.ast_helpers import ast_to_props +from core.tokenizer import TokenKind, Tokenizer default_debug_name = "*default*" debug_activated = set() +COLORS = { + "black", + "red", + "green", + "yellow", + "blue", + "magenta", + "cyan", + "white", +} + +CONSOLE_COLORS_MAP = { + "reset": "\u001b[0m", + "black": "\u001b[30m", + "red": "\u001b[31m", + "green": "\u001b[32m", + "yellow": "\u001b[33m", + "blue": "\u001b[34m", + "magenta": "\u001b[35m", + "cyan": "\u001b[36m", + "white": "\u001b[37m", +} + +PRIMITIVES_TYPES = (str, bool, type(None), int, float, list, dict, set, bytes, tuple, type) + +expressions_cache = Cache() + def my_debug(*args, check_started=None): + """ + Write one line per arg in 'debug.txt' + :param args: + :param check_started: + True : first check if start_debug() was called + : first check if start_debug(name) was called + list of : first check if start_debug() is called for all names + :return: + """ if check_started and default_debug_name not in debug_activated: return @@ -30,14 +69,14 @@ def my_debug(*args, check_started=None): f.write(f"{arg}\n") -def start_debug(msg=None, debug_name=default_debug_name): +def start_debug(debug_name=default_debug_name, msg=None): debug_activated.add(debug_name) if msg: with open("debug.txt", "a") as f: f.write(f"{msg}\n") -def stop_debug(msg=None, debug_name=default_debug_name): +def stop_debug(debug_name=default_debug_name, msg=None): if msg: with open("debug.txt", "a") as f: f.write(f"{msg}\n") @@ -357,6 +396,26 @@ def strip_tokens(tokens, strip_eof=False): return tokens[start: end + 1] +def index_tokens(tokens, value): + """ + Returns the index of the token whose value equals 'value' + >>> assert index_tokens(Tokenizer("xxx=yyy"), "=") == 1 + >>> assert index_tokens(Tokenizer("xxx = yyy"), "=") == 2 + >>> assert index_tokens(Tokenizer("yyy"), "=") == -1 + >>> assert index_tokens(Tokenizer("xxx = yyy"), " = ") == -1 # " = " is not valid token + :param tokens: + :param value: + :return: + """ + if not tokens: + return -1 + + for i, t in enumerate(tokens): + if t.value == value: + return i + return -1 + + def escape_char(text, to_escape): res = "" @@ -392,7 +451,7 @@ def decode_enum(enum_repr: str): return None -def str_concept(t, drop_name=None): +def str_concept(t, drop_name=None, prefix="c:"): """ The key,id identifiers of a concept are stored in a tuple we want to return the key and the id, separated by a pipe @@ -404,25 +463,29 @@ def str_concept(t, drop_name=None): >>> assert str_concept((None, None)) == "" >>> assert str_concept(Concept(name="foo", id="bar")) == "c:foo|bar:" >>> assert str_concept(Concept(name="foo", id="bar"), drop_name=True) == "c:|bar:" + >>> assert str_concept(("key", "id"), prefix='r:') == "r:key|id:" :param t: :param drop_name: True if we only want the id (and not the key) + :param prefix: :return: """ if isinstance(t, tuple): name, id_ = t[0], t[1] + elif prefix == "r:": + name, id_ = t.metadata.name, t.id else: name, id_ = t.key, t.id if name is None and id_ is None: return "" - result = 'c:' if (name is None or drop_name) else "c:" + name + result = prefix if (name is None or drop_name) else prefix + name if id_: result += "|" + id_ return result + ":" -def unstr_concept(concept_repr): +def unstr_concept(concept_repr, prefix='c:'): """ if concept_repr is like :c:key:id: return the key and the id @@ -430,6 +493,7 @@ def unstr_concept(concept_repr): >>> assert unstr_concept("c:key|id:") == ("key", "id") >>> assert unstr_concept("c:|id:") == ("None", "id") >>> assert unstr_concept("c:key|:") == ("key", "None") + >>> assert unstr_concept("r:key|id:", prefix='r:') == ("key", "id") >>> # Otherwise, return (None,None) :param concept_repr: @@ -437,7 +501,7 @@ def unstr_concept(concept_repr): """ if not (concept_repr and isinstance(concept_repr, str) and - concept_repr.startswith("c:") and + concept_repr.startswith(prefix) and concept_repr.endswith(":")): return None, None @@ -470,7 +534,7 @@ def unstr_concept(concept_repr): return key if key != "" else None, id if id != "" else None -def encode_concept(t): +def encode_concept(t, wrapper="C"): """ Given a tuple of concept id, concept id Create a valid Python identifier that can be parsed back @@ -480,24 +544,27 @@ def encode_concept(t): >>> assert encode_concept(("key", None)) == "__C__KEY_key__ID_00None00__C__" :param t: + :param wrapper: :return: """ key, id_ = (t[0], t[1]) if isinstance(t, tuple) else (t.key, t.id) - prefix = "__C" sanitized_key = "".join(c if c.isalnum() else "0" for c in key) if key else "00None00" - return prefix + f"__KEY_{sanitized_key}__ID_{id_ or '00None00'}__C__" + return f"__{wrapper}__KEY_{sanitized_key}__ID_{id_ or '00None00'}__{wrapper}__" -decode_regex = re.compile(r"__KEY_(\w+)__ID_(\w+)__C__") +concept_decode_regex = re.compile(r"__KEY_(\w+)__ID_(\w+)__C__") # it is compiled only once +rule_decode_regex = re.compile(r"__KEY_(\w+)__ID_(\w+)__R__") # it is compiled only once -def decode_concept(text): +def decode_concept(text, wrapper="C"): """ Decode what was encoded by encode_concept_key_id :param text: + :param wrapper: :return: """ + decode_regex = concept_decode_regex if wrapper == "C" else rule_decode_regex m = decode_regex.search(text) lookup = {"00None00": None} if m: @@ -539,7 +606,114 @@ def as_bag(obj): if hasattr(obj, "as_bag"): bag = obj.as_bag() else: - bag = {prop: getattr(obj, prop) for prop in dir(obj) if not prop.startswith("_")} + bag = {} if type(obj) in PRIMITIVES_TYPES else {prop: getattr(obj, prop) + for prop in dir(obj) if not prop.startswith("_")} bag["self"] = obj return bag + + +def flatten_all_children(item, get_children): + """ + Return a list containing the current item and all its children, recursively + :param item: + :param get_children: lambda to get the children + :return: + """ + + def inner_get_all_children(inner_item): + yield inner_item + for child in get_children(inner_item): + yield from inner_get_all_children(child) + + return inner_get_all_children(item) + + +def evaluate_expression(expr, bag): + """ + Try to evaluate expr in context of bag + :param expr: + :param bag: + :return: + """ + + if expr is None or expr.strip() == "": + return None + + if expr in bag: + return bag[expr] + + props_definitions = expressions_cache.get(expr) + if props_definitions is None: + _ast = ast.parse(expr, mode="eval") + props_definitions = [] + ast_to_props(props_definitions, _ast.body, None) + props_definitions.reverse() + expressions_cache.put(expr, props_definitions) + + return evaluate_object(bag, props_definitions) + + +def evaluate_object(bag, properties): + """ + Evaluate the properties of an object + Works with evaluate_expression + :param bag: + :param properties: List of ast_helpers.PropDef + :return: + """ + for prop in properties: + try: + obj = bag[prop.prop] + except KeyError: + try: + obj = bag["self"][prop.prop] + except Exception: + raise NameError(prop.prop) + + if obj is None: + return None + + if prop.index is not None: + obj = obj[prop.index] + + bag = as_bag(obj) + + return obj + + +def get_text_from_tokens(tokens, custom_switcher=None, tracker=None): + """ + Create the source code, from the list of token + :param tokens: list of tokens + :param custom_switcher: to override the behaviour (the return value) of some token + :param tracker: keep track of the original token value when custom switched + :return: + """ + if tokens is None: + return "" + res = "" + + if not hasattr(tokens, "__iter__"): + tokens = [tokens] + + switcher = { + # TokenKind.CONCEPT: lambda t: core.utils.str_concept(t.value), + } + + if custom_switcher: + switcher.update(custom_switcher) + + for token in tokens: + value = switcher.get(token.type, lambda t: t.str_value)(token) + res += value + if tracker is not None and token.type in custom_switcher: + tracker[value] = token + return res + + +def dump_ast(node): + dump = ast.dump(node) + for to_remove in [", ctx=Load()", ", kind=None", ", type_ignores=[]"]: + dump = dump.replace(to_remove, "") + return dump diff --git a/src/evaluators/BaseEvaluator.py b/src/evaluators/BaseEvaluator.py index 870ac7f..792f483 100644 --- a/src/evaluators/BaseEvaluator.py +++ b/src/evaluators/BaseEvaluator.py @@ -10,11 +10,12 @@ class BaseEvaluator: PREFIX = "evaluators." def __init__(self, name, steps, priority: int, enabled=True): - self.log = get_logger(self.PREFIX + self.__class__.__name__) - self.init_log = get_logger("init." + self.PREFIX + self.__class__.__name__) - self.verbose_log = get_logger("verbose." + self.PREFIX + self.__class__.__name__) + # self.log = get_logger(self.PREFIX + self.__class__.__name__) + # self.init_log = get_logger("init." + self.PREFIX + self.__class__.__name__) + # self.verbose_log = get_logger("verbose." + self.PREFIX + self.__class__.__name__) self.name = self.PREFIX + name + self.short_name = name self.steps = steps self.priority = priority self.enabled = enabled @@ -22,6 +23,9 @@ class BaseEvaluator: def __repr__(self): return f"{self.name} ({self.priority})" + def reset(self): + pass + class OneReturnValueEvaluator(BaseEvaluator): """ @@ -50,3 +54,5 @@ class AllReturnValuesEvaluator(BaseEvaluator): def eval(self, context: ExecutionContext, return_values): pass + def reset(self): + self.eaten.clear() diff --git a/src/evaluators/ConceptEvaluator.py b/src/evaluators/ConceptEvaluator.py index fee149b..95ec990 100644 --- a/src/evaluators/ConceptEvaluator.py +++ b/src/evaluators/ConceptEvaluator.py @@ -45,7 +45,7 @@ class ConceptEvaluator(OneReturnValueEvaluator): # Why ? # If we evaluate Concept("foo", body="a").set_prop("a", "'property_a'") # The body should be 'property_a', and not a concept called 'a' - if context.obj and concept.name in context.obj.values: + if context.obj and concept.name in context.obj.values(): value = context.obj.get_value(concept.name) context.log(f"{concept.name} is a property. Returning value '{value}'.", self.name) @@ -63,7 +63,7 @@ class ConceptEvaluator(OneReturnValueEvaluator): evaluated, parents=[return_value]) - if self.return_body and ConceptParts.BODY in evaluated.compiled: + if self.return_body and ConceptParts.BODY in evaluated.get_compiled(): return sheerka.ret(self.name, True, evaluated.body, parents=[return_value]) else: return sheerka.ret(self.name, True, evaluated, parents=[return_value]) diff --git a/src/evaluators/AddConceptEvaluator.py b/src/evaluators/DefConceptEvaluator.py similarity index 85% rename from src/evaluators/AddConceptEvaluator.py rename to src/evaluators/DefConceptEvaluator.py index c7091d1..967311f 100644 --- a/src/evaluators/AddConceptEvaluator.py +++ b/src/evaluators/DefConceptEvaluator.py @@ -1,7 +1,6 @@ import core.utils -from core.ast.nodes import python_to_concept +from core.ast_helpers import UnreferencedVariablesVisitor from core.builtin_concepts import ParserResultConcept, ReturnValueConcept, BuiltinConcepts -from core.builtin_helpers import get_names from core.concept import Concept, DEFINITION_TYPE_BNF, DEFINITION_TYPE_DEF from core.sheerka.services.SheerkaExecute import ParserInput from core.tokenizer import TokenKind, Tokenizer @@ -35,16 +34,18 @@ class ConceptOrRuleNameVisitor(ParsingExpressionVisitor): self.names.add(node.rule_name) -class AddConceptEvaluator(OneReturnValueEvaluator): +class DefConceptEvaluator(OneReturnValueEvaluator): """ Used to add a new concept """ - NAME = "AddNewConcept" + NAME = "DefConcept" def __init__(self): super().__init__(self.NAME, [BuiltinConcepts.EVALUATION], 50) def matches(self, context, return_value): + debugger = context.get_debugger(self.NAME, "matches") + debugger.debug_entering(return_value=return_value) return return_value.status and \ isinstance(return_value.value, ParserResultConcept) and \ isinstance(return_value.value.value, DefConceptNode) @@ -54,11 +55,14 @@ class AddConceptEvaluator(OneReturnValueEvaluator): def_concept_node = return_value.value.value sheerka = context.sheerka + debugger = context.get_debugger(self.NAME, "eval") + debugger.debug_entering(def_concept=def_concept_node) + # validate the node variables_found = set() - concept = Concept(def_concept_node.name) - concept.metadata.definition_type = def_concept_node.definition_type + concept = Concept(str(def_concept_node.name)) + concept.get_metadata().definition_type = def_concept_node.definition_type name_to_use = self.get_name_to_use(def_concept_node) for prop in ("definition", "where", "pre", "post", "body", "ret"): @@ -75,14 +79,14 @@ class AddConceptEvaluator(OneReturnValueEvaluator): ParserInput) else part_ret_val.value.source else: raise Exception("Unexpected") - setattr(concept.metadata, prop, source) + setattr(concept.get_metadata(), prop, source) # Do not try to resolve variables from itself - if prop == "definition" and concept.metadata.definition_type == DEFINITION_TYPE_DEF: + if prop == "definition" and concept.get_metadata().definition_type == DEFINITION_TYPE_DEF: continue # try to find what can be a property - for p in self.get_variables(sheerka, part_ret_val, name_to_use): + for p in self.get_variables(context, part_ret_val, name_to_use): variables_found.add(p) # add variables by order of appearance when possible @@ -93,7 +97,7 @@ class AddConceptEvaluator(OneReturnValueEvaluator): # add the remaining properties # They mainly come from BNF definition for p in variables_found: - if p not in concept.values: + if p not in concept.values(): concept.def_var(p, None) # initialize the key @@ -105,7 +109,7 @@ class AddConceptEvaluator(OneReturnValueEvaluator): # update the bnf definition if needed if not isinstance(def_concept_node.definition, NotInitializedNode) and \ def_concept_node.definition_type == DEFINITION_TYPE_BNF: - concept.bnf = def_concept_node.definition.value.value + concept.set_bnf(def_concept_node.definition.value.value) ret = sheerka.create_new_concept(context, concept) if not ret.status: @@ -119,7 +123,7 @@ class AddConceptEvaluator(OneReturnValueEvaluator): return [part.str_value for part in core.utils.strip_tokens(source.tokens, True)] @staticmethod - def get_variables(sheerka, ret_value, concept_name): + def get_variables(context, ret_value, concept_name): """ Try to find out the variables This function can only be a draft, as there may be tons of different situations @@ -149,8 +153,8 @@ class AddConceptEvaluator(OneReturnValueEvaluator): if isinstance(ret_value.value, ParserResultConcept) and isinstance(ret_value.value.value, PythonNode): if len(concept_name) > 1: python_node = ret_value.value.value - as_concept_node = python_to_concept(python_node.ast_) - names = get_names(sheerka, as_concept_node) + visitor = UnreferencedVariablesVisitor(context) + names = visitor.get_names(python_node.ast_) variables = filter(lambda x: x in concept_name, names) return set(variables) diff --git a/src/evaluators/FormatRuleEvaluator.py b/src/evaluators/FormatRuleEvaluator.py new file mode 100644 index 0000000..abca289 --- /dev/null +++ b/src/evaluators/FormatRuleEvaluator.py @@ -0,0 +1,46 @@ +import core.utils +from core.builtin_concepts import BuiltinConcepts, ParserResultConcept +from core.rule import Rule +from core.tokenizer import Keywords +from evaluators.BaseEvaluator import OneReturnValueEvaluator +from parsers.BaseParser import BaseParser +from parsers.FormatRuleParser import FormatRuleNode + + +class FormatRuleEvaluator(OneReturnValueEvaluator): + """ + Used to store a new format rule + """ + NAME = "FormatRule" + + def __init__(self): + super().__init__(self.NAME, [BuiltinConcepts.EVALUATION], 50) + + def matches(self, context, return_value): + return return_value.status and \ + isinstance(return_value.value, ParserResultConcept) and \ + isinstance(return_value.value.value, FormatRuleNode) + + def eval(self, context, return_value): + """ + Creates a Rule out of a FormatRuleNode and saves it in db + :param context: + :param return_value: + :return: + """ + + context.log("Adding a new format rule", self.name) + format_rule_node = return_value.value.value + sheerka = context.sheerka + + predicate = core.utils.get_text_from_tokens(format_rule_node.tokens[Keywords.WHEN][1:]) + action = core.utils.get_text_from_tokens(format_rule_node.tokens[Keywords.PRINT][1:]) + rule = Rule("print", None, predicate, action) + rule.compiled_predicate = format_rule_node.rule + rule.compiled_action = format_rule_node.format_ast + + ret = sheerka.create_new_rule(context, rule) + if not ret.status: + error_cause = sheerka.objvalue(ret.body) + context.log(f"Failed to add new rule '{rule}'. Reason: {error_cause}", self.name) + return sheerka.ret(self.name, ret.status, ret.value, parents=[return_value]) diff --git a/src/evaluators/LexerNodeEvaluator.py b/src/evaluators/LexerNodeEvaluator.py index 7c3e6cc..77708ed 100644 --- a/src/evaluators/LexerNodeEvaluator.py +++ b/src/evaluators/LexerNodeEvaluator.py @@ -16,6 +16,10 @@ class LexerNodeEvaluator(OneReturnValueEvaluator): self.identifiers = {} # cache for already created identifier (the key is id(concept)) self.identifiers_key = {} # number of identifiers with the same root (prefix) + def reset(self): + self.identifiers.clear() + self.identifiers_key.clear() + def matches(self, context, return_value): if not return_value.status: return False diff --git a/src/evaluators/MultipleErrorsEvaluator.py b/src/evaluators/MultipleErrorsEvaluator.py index fb4ac66..6c6b30a 100644 --- a/src/evaluators/MultipleErrorsEvaluator.py +++ b/src/evaluators/MultipleErrorsEvaluator.py @@ -15,6 +15,10 @@ class MultipleErrorsEvaluator(AllReturnValuesEvaluator): super().__init__(self.NAME, [BuiltinConcepts.AFTER_EVALUATION], 30) self.return_values_in_error = [] + def reset(self): + super().reset() + self.return_values_in_error.clear() + def matches(self, context, return_values): nb_evaluators_in_error = 0 to_process = False @@ -47,5 +51,5 @@ class MultipleErrorsEvaluator(AllReturnValuesEvaluator): return sheerka.ret( self.name, False, - sheerka.new(BuiltinConcepts.MULTIPLE_ERRORS, body=self.return_values_in_error), + sheerka.new(BuiltinConcepts.MULTIPLE_ERRORS, body=self.return_values_in_error.copy()), parents=self.eaten) diff --git a/src/evaluators/MutipleSameSuccessEvaluator.py b/src/evaluators/MutipleSameSuccessEvaluator.py index 0afd0e5..baf5e9f 100644 --- a/src/evaluators/MutipleSameSuccessEvaluator.py +++ b/src/evaluators/MutipleSameSuccessEvaluator.py @@ -21,6 +21,10 @@ class MultipleSameSuccessEvaluator(AllReturnValuesEvaluator): super().__init__(self.NAME, [BuiltinConcepts.AFTER_EVALUATION], 50) self.success = [] + def reset(self): + super().reset() + self.success.clear() + def matches(self, context, return_values): nb_successful_evaluators = 0 only_parsers_in_error = True diff --git a/src/evaluators/OneErrorEvaluator.py b/src/evaluators/OneErrorEvaluator.py index a4121e3..089c2e8 100644 --- a/src/evaluators/OneErrorEvaluator.py +++ b/src/evaluators/OneErrorEvaluator.py @@ -15,6 +15,10 @@ class OneErrorEvaluator(AllReturnValuesEvaluator): super().__init__(self.NAME, [BuiltinConcepts.AFTER_EVALUATION], 30) self.return_value_in_error = None + def reset(self): + super().reset() + self.return_value_in_error = None + def matches(self, context, return_values): nb_evaluators_in_error = 0 to_process = False diff --git a/src/evaluators/OneSuccessEvaluator.py b/src/evaluators/OneSuccessEvaluator.py index 53f33bb..68ecc12 100644 --- a/src/evaluators/OneSuccessEvaluator.py +++ b/src/evaluators/OneSuccessEvaluator.py @@ -16,6 +16,12 @@ class OneSuccessEvaluator(AllReturnValuesEvaluator): def __init__(self): super().__init__(self.NAME, [BuiltinConcepts.AFTER_EVALUATION], 60) # before MultipleSameSuccess self.successful_return_value = None + self.value_to_return = None + + def reset(self): + super().reset() + self.successful_return_value = None + self.value_to_return = None def matches(self, context, return_values): nb_successful_evaluators = 0 @@ -30,6 +36,7 @@ class OneSuccessEvaluator(AllReturnValuesEvaluator): elif ret.status and ret.who.startswith(self.PREFIX): nb_successful_evaluators += 1 self.successful_return_value = ret + self.value_to_return = ret if context.sheerka.is_container(ret) else ret.body self.eaten.append(ret) elif not ret.status: self.eaten.append(ret) @@ -41,4 +48,4 @@ class OneSuccessEvaluator(AllReturnValuesEvaluator): context.log(f"{self.successful_return_value}", who=self) sheerka = context.sheerka - return sheerka.ret(self.name, True, self.successful_return_value.value, parents=self.eaten) + return sheerka.ret(self.name, True, self.value_to_return, parents=self.eaten) diff --git a/src/evaluators/PostExecutionEvaluator.py b/src/evaluators/PostExecutionEvaluator.py index 08c25d4..c90cf60 100644 --- a/src/evaluators/PostExecutionEvaluator.py +++ b/src/evaluators/PostExecutionEvaluator.py @@ -1,5 +1,5 @@ from core.builtin_concepts import BuiltinConcepts -from core.concept import Concept +from core.concept import Concept, NotInit from evaluators.BaseEvaluator import OneReturnValueEvaluator @@ -25,11 +25,31 @@ class PostExecutionEvaluator(OneReturnValueEvaluator): return isinstance(value, Concept) and context.sheerka.isa(value, context.sheerka.new(BuiltinConcepts.AUTO_EVAL)) def eval(self, context, return_value): - # only support the rule for the COMMANDS - #body = return_value.body.body - body = context.sheerka.objvalue(return_value) + # only support the rule for the COMMANDS ?? return context.sheerka.ret( self.name, True, - body if body != BuiltinConcepts.NOT_INITIALIZED else return_value.body, + self.custom_obj_value(context, return_value.body), parents=[return_value]) + + @staticmethod + def custom_obj_value(context, obj): + """ + get concept inner value. + You cannot use sheerka.objvalue() as it does not handle container + And... + Do not try to make it manage it, it won't work ;-) + :param context: + :param obj: + :return: + """ + if context.sheerka.is_container(obj): + return obj + + if isinstance(obj, Concept): + if isinstance(obj.body, Concept): + return PostExecutionEvaluator.custom_obj_value(context, obj.body) + else: + return obj.body if obj.body != NotInit else obj + + return obj diff --git a/src/evaluators/PrepareEvalBodyEvaluator.py b/src/evaluators/PrepareEvalBodyEvaluator.py index 6a6db76..a0764c9 100644 --- a/src/evaluators/PrepareEvalBodyEvaluator.py +++ b/src/evaluators/PrepareEvalBodyEvaluator.py @@ -13,6 +13,9 @@ class PrepareEvalBodyEvaluator(OneReturnValueEvaluator): super().__init__(self.NAME, [BuiltinConcepts.BEFORE_PARSING], 90) self.text = None + def reset(self): + self.text = None + def matches(self, context, return_value): if not (return_value.status and context.sheerka.isinstance(return_value.body, BuiltinConcepts.USER_INPUT) and diff --git a/src/evaluators/PrepareEvalQuestionEvaluator.py b/src/evaluators/PrepareEvalQuestionEvaluator.py index d5509e7..cb101bd 100644 --- a/src/evaluators/PrepareEvalQuestionEvaluator.py +++ b/src/evaluators/PrepareEvalQuestionEvaluator.py @@ -13,6 +13,9 @@ class PrepareEvalQuestionEvaluator(OneReturnValueEvaluator): super().__init__(self.NAME, [BuiltinConcepts.BEFORE_PARSING], 90) self.question = None + def reset(self): + self.question = None + def matches(self, context, return_value): if not (return_value.status and context.sheerka.isinstance(return_value.body, BuiltinConcepts.USER_INPUT) and diff --git a/src/evaluators/PythonEvaluator.py b/src/evaluators/PythonEvaluator.py index f1876a1..ce7c048 100644 --- a/src/evaluators/PythonEvaluator.py +++ b/src/evaluators/PythonEvaluator.py @@ -3,12 +3,14 @@ import copy import traceback from dataclasses import dataclass, field -import core.ast.nodes +import core.builtin_helpers import core.utils -from core.ast.visitors import UnreferencedNamesVisitor +from core.ast_helpers import UnreferencedNamesVisitor, NamesWithAttributesVisitor from core.builtin_concepts import BuiltinConcepts, ParserResultConcept from core.concept import ConceptParts, Concept, NotInit -from core.sheerka.services.SheerkaFilter import Pipe +from core.rule import Rule +from core.sheerka.ExecutionContext import ExecutionContext +from core.tokenizer import Token, TokenKind from evaluators.BaseEvaluator import OneReturnValueEvaluator from parsers.PythonParser import PythonNode @@ -47,6 +49,20 @@ class PythonEvalError: traceback: str = field(repr=False) concepts: dict = field(repr=False) + def __eq__(self, other): + if id(self) == id(other): + return True + + if not isinstance(other, PythonEvalError): + return False + + return isinstance(self.error, type(other.error)) and \ + self.traceback == other.traceback and \ + self.concepts == other.concepts + + def __hash__(self): + return hash(self.error) + class PythonEvaluator(OneReturnValueEvaluator): NAME = "Python" @@ -54,37 +70,31 @@ class PythonEvaluator(OneReturnValueEvaluator): """ Evaluate a Python node, ie, evaluate some Python code """ + isinstance = None def __init__(self): super().__init__(self.NAME, [BuiltinConcepts.EVALUATION], 50) - self.globals = {} + + @staticmethod + def initialize(sheerka): + from core.sheerka.services.SheerkaAdmin import SheerkaAdmin + PythonEvaluator.isinstance = sheerka.services[SheerkaAdmin.NAME].extended_isinstance def matches(self, context, return_value): if not return_value.status or not isinstance(return_value.value, ParserResultConcept): return False body = return_value.value.value - return isinstance(body, PythonNode) or ( - hasattr(body, "python_node") and isinstance(body.python_node, PythonNode)) - # return return_value.status and \ - # isinstance(return_value.value, ParserResultConcept) and \ - # isinstance(return_value.value.value, PythonNode) + return isinstance(body, PythonNode) or hasattr(body, "python_node") def eval(self, context, return_value): sheerka = context.sheerka node = return_value.value.value if isinstance(return_value.value.value, PythonNode) else \ return_value.value.value.python_node + debugger = context.get_debugger(PythonEvaluator.NAME, "eval") + debugger.debug_entering(node=node) context.log(f"Evaluating python node {node}.", self.name) - # Do not evaluate if the ast refers to a concept (leave it to ConceptEvaluator) - # TODO: Remove this section when this check will be implemented in the AFTER_PARSING step - if isinstance(node.ast_, ast.Expression) and isinstance(node.ast_.body, ast.Name): - c = context.sheerka.resolve(node.ast_.body.id) - if c is not None: - context.log("It's a simple concept. Not for me.", self.name) - not_for_me = context.sheerka.new(BuiltinConcepts.NOT_FOR_ME, body=node) - return sheerka.ret(self.name, False, not_for_me, parents=[return_value]) - # If we evaluate a Concept metadata which is NOT the body ex (pre, post, where...) # We need to disable the function that may alter the state # It's a poor way to have source code security check @@ -104,11 +114,12 @@ class PythonEvaluator(OneReturnValueEvaluator): # get globals my_globals = self.get_globals(context, node, expression_only) - context.log(f"globals={my_globals}", self.name) + + debugger.debug_var("globals", my_globals) all_possible_globals = self.get_all_possible_globals(context, my_globals) - concepts_entries = None - evaluated = BuiltinConcepts.NOT_INITIALIZED + concepts_entries = None # entries in globals_ that refers to Concept objects + evaluated = NotInit errors = [] expect_success = context.in_context(BuiltinConcepts.EVAL_UNTIL_SUCCESS_REQUESTED) for globals_ in all_possible_globals: @@ -116,8 +127,7 @@ class PythonEvaluator(OneReturnValueEvaluator): # eval if isinstance(node.ast_, ast.Expression): context.log("Evaluating using 'eval'.", self.name) - compiled = compile(node.ast_, "", "eval") - evaluated = eval(compiled, globals_, sheerka.locals) + evaluated = eval(node.get_compiled(), globals_, sheerka.locals) else: context.log("Evaluating using 'exec'.", self.name) evaluated = self.exec_with_return(node.ast_, globals_, sheerka.locals) @@ -128,143 +138,125 @@ class PythonEvaluator(OneReturnValueEvaluator): if concepts_entries is None: concepts_entries = self.get_concepts_entries_from_globals(my_globals) errors.append(PythonEvalError(ex, - traceback.format_exc(), + traceback.format_exc() if context.debug_enabled else None, self.get_concepts_values_from_globals(globals_, concepts_entries))) - if evaluated == BuiltinConcepts.NOT_INITIALIZED: + if evaluated == NotInit: if len(errors) == 1: context.log_error(errors[0].error, who=self.name, exc=errors[0].traceback) - one_error = sheerka.new(BuiltinConcepts.ERROR, body=errors[0]) - return sheerka.ret(self.name, False, one_error, parents=[return_value]) + return sheerka.ret(self.name, False, sheerka.err(errors[0]), parents=[return_value]) if len(errors) > 1: for eval_error in errors: context.log_error(eval_error.error, who=self.name, exc=eval_error.traceback) - too_many_errors = sheerka.new(BuiltinConcepts.TOO_MANY_ERRORS, body=errors) - return sheerka.ret(self.name, False, too_many_errors, parents=[return_value]) + return sheerka.ret(self.name, False, sheerka.err(errors), parents=[return_value]) context.log(f"{evaluated=}", self.name) + debugger.debug_var("ret", evaluated) return sheerka.ret(self.name, True, evaluated, parents=[return_value]) def get_globals(self, context, node, expression_only): """ - Creates the global variables for python source code evaluation + Creates the globals variables :param context: :param node: - :param expression_only: most of the commands are refused + :param expression_only: :return: """ + unreferenced_names_visitor = UnreferencedNamesVisitor(context) + names = unreferenced_names_visitor.get_names(node.ast_) + if context.debug_enabled: + context.debug(self.NAME, "eval", "names", names) + return self.get_globals_by_names(context, names, node, expression_only) + + def get_sheerka_method(self, context, name, expression_only): + try: + method = context.sheerka.sheerka_methods[name] + context.log(f"Resolving '{name}'. It's a sheerka method.", self.name) + if expression_only and method.has_side_effect: + context.log(f"...but with side effect when {expression_only=}. Discarding.", self.name) + return None + else: + return inject_context(context)(method.method) if name in context.sheerka.methods_with_context \ + else method.method + except KeyError: + return None + + def get_globals_by_names(self, context, names, node, expression_only): my_globals = { "Concept": core.concept.Concept, "BuiltinConcepts": core.builtin_concepts.BuiltinConcepts, + "ExecutionContext": ExecutionContext, "in_context": context.in_context, } - if expression_only: - # disable some builtin - for statement in TO_DISABLED: - my_globals[statement] = None - - # has to be the first, to allow override - method_from_sheerka = self.update_globals_with_sheerka_methods(my_globals, context, expression_only) - - self.update_globals_with_context(my_globals, context) - already_know = set(my_globals.keys()) - self.update_globals_with_node(my_globals, context, node, already_know) - - if self.globals: # when extra values are given. Add them - my_globals.update(self.globals) - - my_globals["sheerka"] = Expando(method_from_sheerka) # it's the last, so I cannot be overridden - return my_globals - - @staticmethod - def update_globals_with_sheerka_methods(my_locals, context, expression_only): - methods_from_sheerka = {} - - # Add all the methods as a direct access - for method_name, method in context.sheerka.sheerka_methods.items(): - if expression_only and method.has_side_effect: + for name in names: + if name in my_globals: continue - if method_name in context.sheerka.methods_with_context: - my_locals[method_name] = inject_context(context)(method.method) - else: - my_locals[method_name] = method.method - methods_from_sheerka[method_name] = my_locals[method_name] - - # Add pipeable functions - for func_name, function in context.sheerka.sheerka_pipeables.items(): - if expression_only and function.has_side_effect: + if expression_only and name in TO_DISABLED: + my_globals[name] = None continue - my_locals[func_name] = Pipe(function.method, context) - - return methods_from_sheerka # to allow access using prefix "sheerka." - - def update_globals_with_context(self, my_globals, context): - """ - Update globals with the current object being evaluated (and its variables) - :param my_globals: - :param context: - :return: - """ - if context.obj: - context.log(f"Concept '{context.obj}' is in context. Adding it and its properties to locals.", self.name) - - for prop_name in context.obj.variables(): - value = context.obj.get_value(prop_name) - if value != NotInit: - my_globals[prop_name] = value - my_globals["self"] = context.obj - - def update_globals_with_node(self, my_globals, context, node, already_known): - """ - Try to find concepts using the names that appear in the AST of the node. - :param my_globals: dictionary to update - :param context: - :param node: - :param already_known: if the name is in this list, do no try to instantiate it again - :return: - """ - node_concept = core.ast.nodes.python_to_concept(node.ast_) - unreferenced_names_visitor = UnreferencedNamesVisitor(context.sheerka) - unreferenced_names_visitor.visit(node_concept) - - for name in unreferenced_names_visitor.names: - context.log(f"Resolving '{name}'.", self.name) - - # get the concept - if name in node.concepts: - # use it, even if it already in already_known - # This concept take precedence other the outer world - context.log(f"Using value from node.", self.name) - concept = self.resolve_concept(context, node.concepts[name]) - elif name in already_known: - context.log(f"Already known. Skipping.", self.name) - continue - elif (concept := context.get_from_short_term_memory(name)) is not None: - context.log(f"Using from STM known.", self.name) - else: - context.log(f"Instantiating new concept with {name}.", self.name) - concept = self.resolve_concept(context, name) - - if concept is None: - context.log(f"Concept '{name}' is not found or cannot be instantiated. Skipping.", self.name) + # need to add it manually to avoid conflict with sheerka.isinstance + if name == "isinstance": + my_globals["isinstance"] = PythonEvaluator.isinstance continue - # evaluate it if needed - if concept.metadata.is_evaluated: - context.log(f"Concept {name} is already evaluated.", self.name) - else: - context.log(f"Evaluating '{concept}'", self.name) - evaluated = context.sheerka.evaluate_concept(context, concept, eval_body=True) - if not context.sheerka.is_success(evaluated) and evaluated.key != concept.key: - context.log(f"Error while evaluating '{name}'. Skipping.", self.name) + # support reference to sheerka + if name == "sheerka": + bag = {} + visitor = NamesWithAttributesVisitor() + for sequence in visitor.get_sequences(node.ast_, "sheerka"): + if (len(sequence) > 1 and + (method := self.get_sheerka_method(context, sequence[1], expression_only)) is not None): + bag[sequence[1]] = method + my_globals["sheerka"] = Expando(bag) + continue + + # search in short term memory + if (obj := context.get_from_short_term_memory(name)) is not None: + context.log(f"Resolving '{name}'. Using value found in STM.", self.name) + my_globals[name] = obj + continue + + # search in sheerka methods + if (method := self.get_sheerka_method(context, name, expression_only)) is not None: + my_globals[name] = method + continue + + # search in context.obj (to replace by short time memory ?) + if context.obj: + if name == "self": + my_globals["self"] = context.obj continue - concept = evaluated - my_globals[name] = concept + try: + attribute = context.obj.variables()[name] + if attribute != NotInit: + my_globals[name] = attribute + continue + context.log(f"Resolving '{name}'. It's obj attribute (obj={context.obj}).", self.name) + except KeyError: + pass + + # search in current node (if the name was found during the parsing) + if name in node.objects: + context.log(f"Resolving '{name}'. Using value from node.", self.name) + obj = self.resolve_object(context, node.objects[name]) + + # at last, try to instantiate a new concept + else: + context.log(f"Resolving '{name}'. Instantiating new concept.", self.name) + obj = self.resolve_object(context, name) + + if obj is None: + context.log(f"...'{name}' is not found or cannot be instantiated. Skipping.", self.name) + continue + + my_globals[name] = obj + + return my_globals @staticmethod def get_all_possible_globals(context, my_globals): @@ -309,31 +301,37 @@ class PythonEvaluator(OneReturnValueEvaluator): return {name: my_globals[name] for name in names} @staticmethod - def resolve_concept(context, concept_hint): + def resolve_object(context, name): """ Try to find a concept by its name, id or the pattern c:key|id: :param context: - :param concept_hint: + :param name: :return: """ - if isinstance(concept_hint, Concept): - return concept_hint + if isinstance(name, Rule): + return name - concept = context.sheerka.resolve(concept_hint) + if isinstance(name, Concept): + name = core.builtin_helpers.ensure_evaluated(context, name) + return name + + if isinstance(name, Token) and name.type == TokenKind.RULE: + rule = context.sheerka.get_rule_by_id(name.value[1]) # TODO: need a resolve function for the rules + return rule if isinstance(rule, Rule) else None + + if isinstance(name, tuple): + raise Exception() + + # try to resolve by name + concept = context.sheerka.fast_resolve(name) if concept is None: return None - new_instance = context.sheerka.new_from_template(concept, concept.key) - if isinstance(concept_hint, tuple): - # It's means that it was requested by PythonParser which have found a concept token (c:xxx:) - # So a concept was explicitly required, not its value - # We mark the concept as already evaluated, so it's body will not be evaluated - new_instance.metadata.is_evaluated = True - if len(concept.metadata.variables) > 0: - # In this situation, it means that we are dealing with the concept and not its instantiation - # So do not try to evaluate it - new_instance.metadata.is_evaluated = True - return new_instance + if hasattr(concept, "__iter__"): + raise NotImplementedError("Too many concepts") + + concept = core.builtin_helpers.ensure_evaluated(context, concept) + return concept @staticmethod def expr_to_expression(expr): @@ -343,7 +341,8 @@ class PythonEvaluator(OneReturnValueEvaluator): return result - def exec_with_return(self, code_ast, my_globals, my_locals): + @staticmethod + def exec_with_return(code_ast, my_globals, my_locals): init_ast = copy.deepcopy(code_ast) init_ast.body = code_ast.body[:-1] @@ -353,6 +352,8 @@ class PythonEvaluator(OneReturnValueEvaluator): exec(compile(init_ast, "", "exec"), my_globals, my_locals) if type(last_ast.body[0]) == ast.Expr: - return eval(compile(self.expr_to_expression(last_ast.body[0]), "", "eval"), my_globals, my_locals) + return eval(compile(PythonEvaluator.expr_to_expression(last_ast.body[0]), "", "eval"), + my_globals, + my_locals) else: exec(compile(last_ast, "", "exec"), my_globals, my_locals) diff --git a/src/evaluators/ResolveAmbiguityEvaluator.py b/src/evaluators/ResolveAmbiguityEvaluator.py index 69c4efd..ed20a54 100644 --- a/src/evaluators/ResolveAmbiguityEvaluator.py +++ b/src/evaluators/ResolveAmbiguityEvaluator.py @@ -15,6 +15,10 @@ class ResolveAmbiguityEvaluator(AllReturnValuesEvaluator): super().__init__(self.NAME, [BuiltinConcepts.AFTER_PARSING], 50) self.sources = None + def reset(self): + super().reset() + self.sources = None + def matches(self, context, return_values): # first, arrange return_values by sources. # If they share the same source, that means that there are multiple results for one ParserInput diff --git a/src/evaluators/RetEvaluator.py b/src/evaluators/RetEvaluator.py index d58ded0..7984282 100644 --- a/src/evaluators/RetEvaluator.py +++ b/src/evaluators/RetEvaluator.py @@ -18,14 +18,14 @@ # def matches(self, context, return_value): # return return_value.status and \ # isinstance(return_value.value, Concept) and \ -# return_value.value.metadata.ret is not None +# return_value.value.get_metadata().ret is not None # # def eval(self, context, return_value): # sheerka = context.sheerka # concept = return_value.value # context.log(f"Processing ret value for concept {concept}.", self.name) # -# if not concept.metadata.is_evaluated: +# if not concept.get_metadata().is_evaluated: # evaluated = ensure_evaluated(context, concept) # if evaluated.key != concept.key: # context.log(f"Failed to evaluate concept '{concept}'") diff --git a/src/evaluators/ReturnBodyEvaluator.py b/src/evaluators/ReturnBodyEvaluator.py index 87a6a24..a0da08e 100644 --- a/src/evaluators/ReturnBodyEvaluator.py +++ b/src/evaluators/ReturnBodyEvaluator.py @@ -1,5 +1,5 @@ from core.builtin_concepts import BuiltinConcepts -from core.concept import Concept +from core.concept import Concept, NotInit from evaluators.BaseEvaluator import AllReturnValuesEvaluator @@ -23,7 +23,10 @@ class ReturnBodyEvaluator(AllReturnValuesEvaluator): result = [] for ret_val in return_values: - if ret_val.status and isinstance(ret_val.body, Concept) and ret_val.body.body != BuiltinConcepts.NOT_INITIALIZED: + if ret_val.status and \ + isinstance(ret_val.body, Concept) and \ + not sheerka.is_container(ret_val.body) and \ + ret_val.body.body != NotInit: context.log(f"Evaluating {ret_val.body}", who=self) result.append(sheerka.ret(self.name, True, ret_val.body.body, parents=[ret_val])) elif ret_val.status and sheerka.isaset(context, ret_val.body): diff --git a/src/evaluators/RuleEvaluator.py b/src/evaluators/RuleEvaluator.py new file mode 100644 index 0000000..ed661c9 --- /dev/null +++ b/src/evaluators/RuleEvaluator.py @@ -0,0 +1,47 @@ +from core.builtin_concepts import BuiltinConcepts, ParserResultConcept +from core.rule import Rule, ACTION_TYPE_DEFERRED +from evaluators.BaseEvaluator import OneReturnValueEvaluator + + +class RuleEvaluator(OneReturnValueEvaluator): + """ + This first version will only returns the rule found by the parser + """ + NAME = "Rule" + + def __init__(self): + """ + + """ + super().__init__(self.NAME, [BuiltinConcepts.EVALUATION], 50) + + def matches(self, context, return_value): + if not return_value.status: + return False + + if not isinstance(return_value.value, ParserResultConcept): + return False + + value = return_value.value.value + return isinstance(value, Rule) or isinstance(value, list) and len(value) > 0 and isinstance(value[0], Rule) + + def eval(self, context, return_value): + sheerka = context.sheerka + rules = return_value.value.value + + resolved = [] + success = True + for r in rules: + # Browse the rules to find possible deferred rules + if r.metadata.action_type == ACTION_TYPE_DEFERRED: + rule_id = sheerka.get_from_short_term_memory(context, r.id) + rule = sheerka.get_rule_by_id(str(rule_id or r.id)) + resolved.append(rule) + success &= isinstance(rule, Rule) + else: + resolved.append(r) + + return context.sheerka.ret(self.name, + success, + resolved if len(resolved) > 1 else resolved[0], + parents=[return_value]) diff --git a/src/evaluators/TooManySuccessEvaluator.py b/src/evaluators/TooManySuccessEvaluator.py index 06b53f5..ea20228 100644 --- a/src/evaluators/TooManySuccessEvaluator.py +++ b/src/evaluators/TooManySuccessEvaluator.py @@ -20,6 +20,10 @@ class TooManySuccessEvaluator(AllReturnValuesEvaluator): super().__init__(self.NAME, [BuiltinConcepts.AFTER_EVALUATION], 60) self.success = [] + def reset(self): + super().reset() + self.success.clear() + def matches(self, context, return_values): to_process = False @@ -39,10 +43,10 @@ class TooManySuccessEvaluator(AllReturnValuesEvaluator): def eval(self, context, return_values): sheerka = context.sheerka - if self.verbose_log.isEnabledFor(logging.DEBUG): - for s in self.success: - context.log(s, self.name) - context.log(f"value={sheerka.value(s.value)}", self.name) + # if self.verbose_log.isEnabledFor(logging.DEBUG): + # for s in self.success: + # context.log(s, self.name) + # context.log(f"value={sheerka.value(s.value)}", self.name) same_success = core.builtin_helpers.is_same_success(context, self.success) if same_success is None: @@ -50,7 +54,7 @@ class TooManySuccessEvaluator(AllReturnValuesEvaluator): if not same_success: context.log(f"Values are different. Raising {BuiltinConcepts.TOO_MANY_SUCCESS}.", self.name) - too_many_success = sheerka.new(BuiltinConcepts.TOO_MANY_SUCCESS, body=self.success) + too_many_success = sheerka.new(BuiltinConcepts.TOO_MANY_SUCCESS, body=self.success.copy()) return sheerka.ret(self.name, False, too_many_success, parents=self.eaten) context.log(f"Values are the same. Nothing to do.", self.name) diff --git a/src/evaluators/UpdateFunctionsParametersEvaluator.py b/src/evaluators/UpdateFunctionsParametersEvaluator.py index 6b4b2e9..afa8ae8 100644 --- a/src/evaluators/UpdateFunctionsParametersEvaluator.py +++ b/src/evaluators/UpdateFunctionsParametersEvaluator.py @@ -28,6 +28,9 @@ class UpdateFunctionsParametersEvaluator(OneReturnValueEvaluator): super().__init__(self.NAME, [BuiltinConcepts.AFTER_EVALUATION], 79) self.enabled = False + def reset(self): + self.enabled = False + def matches(self, context, return_value): """ True if the return value is the successful result of PythonEvaluator @@ -88,14 +91,14 @@ class UpdateFunctionsParametersEvaluator(OneReturnValueEvaluator): @staticmethod def get_function(node): if len(node.children) == 2 and \ - isinstance(node.children[0], Name) and \ - isinstance(node.children[1], PythonNode) and \ - node.children[1].type == "trailer" and \ - len(node.children[1].children) >= 2 and \ - isinstance(node.children[1].children[0], Operator) and \ - node.children[1].children[0].value == "(" and \ - isinstance(node.children[1].children[-1], Operator) and \ - node.children[1].children[-1].value == ")": + isinstance(node.children[0], Name) and \ + isinstance(node.children[1], PythonNode) and \ + node.children[1].type == "trailer" and \ + len(node.children[1].children) >= 2 and \ + isinstance(node.children[1].children[0], Operator) and \ + node.children[1].children[0].value == "(" and \ + isinstance(node.children[1].children[-1], Operator) and \ + node.children[1].children[-1].value == ")": name = node.children[0].value if len(node.children[1].children) == 2: params = [] diff --git a/src/out/ConsoleVisistor.py b/src/out/ConsoleVisistor.py new file mode 100644 index 0000000..630d692 --- /dev/null +++ b/src/out/ConsoleVisistor.py @@ -0,0 +1,55 @@ +from out.OutVisitor import OutVisitor +from core.sheerka.services.SheerkaRuleManager import FormatAstNode + + +class ConsoleVisitor(OutVisitor): + """ + Prints to the console + """ + COLORS = { + "reset": "\u001b[0m", + "black": "\u001b[30m", + "red": "\u001b[31m", + "green": "\u001b[32m", + "yellow": "\u001b[33m", + "blue": "\u001b[34m", + "magenta": "\u001b[35m", + "cyan": "\u001b[36m", + "white": "\u001b[37m", + } + + def __init__(self): + self.out = print + + def finalize(self): + self.out("") + + def visit_FormatAstRawText(self, context, format_ast, bag): + self.out(format_ast.text, end="") + + def visit_FormatAstVariable(self, context, format_ast, bag): + if isinstance(format_ast.value, FormatAstNode): + self.visit(context, format_ast.value, bag) + return + self.out(format_ast.value, end="") + + def visit_FormatAstVariableNotFound(self, context, format_ast, bag): + self.out(self.COLORS["red"] + format_ast.name + self.COLORS["reset"], end="") + + def visit_FormatAstSequence(self, context, format_ast, bag): + for item in format_ast.items: + self.visit(context, item, bag) + + def visit_FormatAstList(self, context, format_ast, bag): + first = True + for item in format_ast.items: + if not first: + self.out("") # print new line + self.visit(context, item, bag) + first = False + + def visit_FormatAstColor(self, context, format_ast, bag): + self.out(self.COLORS[format_ast.color], end="") + self.visit(context, format_ast.format_ast, bag) + self.out(self.COLORS["reset"], end="") + diff --git a/src/out/DeveloperVisitor.py b/src/out/DeveloperVisitor.py new file mode 100644 index 0000000..7c9d631 --- /dev/null +++ b/src/out/DeveloperVisitor.py @@ -0,0 +1,140 @@ +from core.sheerka.services.SheerkaRuleManager import FormatAstVariable, FormatAstVariableNotFound, FormatAstSequence, \ + FormatAstColor, FormatAstList, FormatAstRawText +from core.utils import evaluate_expression, as_bag +from out.OutVisitor import OutVisitor + +fstring = compile('f"{value:{format}}"', "DeveloperVisitor.fstring", mode="eval") + + +class DeveloperVisitor(OutVisitor): + """ + This visitor is used to resolve all the variables as well as all the lists + Once completed, it will be passed to the ConsoleVisitor for console print + """ + + def __init__(self, sheerka_out, already_seen, list_recursion_depth): + self._result = None + self.sheerka_out = sheerka_out + self.already_seen = already_seen + self.list_recursion_depth = list_recursion_depth + + def visit_FormatAstRawText(self, context, format_ast, bag): + if context.debug_enabled: + context.debug_entering("DeveloperVisitor", "visit_FormatAstRawText", format_ast=format_ast, bag=bag) + return self.set_result(format_ast) + + def visit_FormatAstVariable(self, context, format_ast, bag): + if context.debug_enabled: + context.debug_entering("DeveloperVisitor", "visit_FormatAstVariable", format_ast=format_ast, bag=bag) + + try: + value = evaluate_expression(format_ast.name, bag) + sub_bag = { + "__obj": value, + format_ast.name: value + } + try: + index = format_ast.name.rindex(".") + sub_bag[format_ast.name[index + 1:]] = value + except ValueError: + pass + res = self.sheerka_out.create_out_tree_recursive(context, sub_bag, self) + + if format_ast.format: + res = eval(fstring, {"value": res, "format": format_ast.format}) + + return self.set_result(FormatAstVariable(format_ast.name, + format_ast.format, + res, + format_ast.index)) + + except NameError as error: + context.debug("DeveloperVisitor", "visit_FormatAstList", "evaluate_expression", error, is_error=True) + return self.set_result(FormatAstVariableNotFound(format_ast.name)) + + def visit_FormatAstSequence(self, context, format_ast, bag): + if context.debug_enabled: + context.debug_entering("DeveloperVisitor", "visit_FormatAstSequence", format_ast=format_ast, bag=bag) + return self.set_result(FormatAstSequence([self.visit(context, item, bag) for item in format_ast.items])) + + def visit_FormatAstColor(self, context, format_ast, bag): + if context.debug_enabled: + context.debug_entering("DeveloperVisitor", "visit_FormatAstColor", format_ast=format_ast, bag=bag) + return self.set_result(FormatAstColor(format_ast.color, self.visit(context, format_ast.format_ast, bag))) + + def visit_FormatAstList(self, context, format_ast, bag): + if context.debug_enabled: + context.debug_entering("DeveloperVisitor", "visit_FormatAstList", format_ast=format_ast, bag=bag) + try: + value = evaluate_expression(format_ast.variable, bag) + if value is None: + return self.set_result(FormatAstVariable(format_ast.variable, format_ast.format, None)) + + if hasattr(value, "__iter__"): + items = value + else: + # the variable does not resolve to an iterable, let's look at one of its attribute + items_props = format_ast.items_prop or "body" + items = evaluate_expression(f"self.{items_props}", {"self": value}) + if not hasattr(items, "__iter__"): + # Definition error ? No list found + return self.set_result(FormatAstVariable(format_ast.variable, None, value)) + + recursion_depth, recurse_on = self.get_recurse_info(value, format_ast.recursion_depth, format_ast.recurse_on) + + result = [] # TODO change into generator + for i, item in enumerate(items): + bag["__item"] = item + sub_visitor = DeveloperVisitor(self.sheerka_out, set(), self.list_recursion_depth) + result.append(sub_visitor.visit(context, FormatAstVariable("__item", None, item, i), bag)) + + # recursion management + recursion_depth, recurse_on = self.get_recurse_info(item, recursion_depth, recurse_on) + if recursion_depth > 0: + sub_items = evaluate_expression(recurse_on, as_bag(item)) + if sub_items and hasattr(sub_items, "__iter__"): + sub_visitor = DeveloperVisitor(self.sheerka_out, set(), self.list_recursion_depth + 1) + bag[f"__{recurse_on}"] = sub_items + + sub_items = sub_visitor.visit(context, FormatAstList(f"__{recurse_on}", + None, + recurse_on, + recursion_depth - 1), bag) + result.append(sub_items) + + return self.set_result(FormatAstList(variable=format_ast.variable, + items_prop=format_ast.items_prop, + recurse_on=recurse_on, + recursion_depth=recursion_depth, + items=result)) + except NameError as error: + context.debug("DeveloperVisitor", "visit_FormatAstList", "evaluate_expression", error, is_error=True) + var_name = format_ast.variable if error.args[0] == format_ast.variable else \ + format_ast.variable + "." + error.args[0] + return self.set_result(FormatAstVariableNotFound(var_name)) + + def visit_FormatAstFunction(self, context, format_ast, bag): + if context.debug_enabled: + context.debug_entering("DeveloperVisitor", "visit_FormatAstFunction", format_ast=format_ast, bag=bag) + unknown = FormatAstColor("red", FormatAstRawText(f"function '{format_ast.name}' is unknown")) + return self.set_result(unknown) + + def set_result(self, result): + self._result = result + return result + + def get_result(self): + return self._result + + @staticmethod + def get_recurse_info(obj, recursion_depth, recurse_on): + depth, on = 0, None + + if hasattr(obj, "get_format_instr"): + depth = obj.get_format_instr("recursion_depth") + on = obj.get_format_instr("recurse_on") + + depth = depth or recursion_depth + on = on or recurse_on + + return depth or 0, on diff --git a/src/out/OutVisitor.py b/src/out/OutVisitor.py new file mode 100644 index 0000000..1b5b23f --- /dev/null +++ b/src/out/OutVisitor.py @@ -0,0 +1,10 @@ +class OutVisitor: + def visit(self, context, format_ast, bag): + name = format_ast.__class__.__name__ + + method = 'visit_' + name + visit_method = getattr(self, method, self.generic_visit) + return visit_method(context, format_ast, bag) + + def generic_visit(self, context, format_ast, bag): + pass diff --git a/src/core/ast/__init__.py b/src/out/__init__.py similarity index 100% rename from src/core/ast/__init__.py rename to src/out/__init__.py diff --git a/src/parsers/BaseCustomGrammarParser.py b/src/parsers/BaseCustomGrammarParser.py index 779d9d6..37e9791 100644 --- a/src/parsers/BaseCustomGrammarParser.py +++ b/src/parsers/BaseCustomGrammarParser.py @@ -128,7 +128,7 @@ class BaseCustomGrammarParser(BaseParser): return tokens[pos:] - def get_parts(self, keywords, expected_first_token=None): + def get_parts(self, keywords, expected_first_token=None, strip_tokens=False): """ Reads Parser Input and groups the tokens by keywords ex: @@ -148,6 +148,7 @@ class BaseCustomGrammarParser(BaseParser): :param keywords: :param expected_first_token: it must be a KeyW + :param strip_tokens: if True, the returned tokens will be trimmed :return: dictionary """ @@ -246,5 +247,7 @@ class BaseCustomGrammarParser(BaseParser): # replace double quoted strings by their content elif len(stripped) == 1 and stripped[0].type == TokenKind.STRING and stripped[0].value[0] == '"': res[k] = v[0:1] + list(Tokenizer(stripped[0].strip_quote, yield_eof=False)) + elif strip_tokens: + res[k] = core.utils.strip_tokens(v) return res diff --git a/src/parsers/BaseNodeParser.py b/src/parsers/BaseNodeParser.py index b25f0fc..331a262 100644 --- a/src/parsers/BaseNodeParser.py +++ b/src/parsers/BaseNodeParser.py @@ -6,6 +6,7 @@ from typing import Set import core.utils from core.builtin_concepts import BuiltinConcepts from core.concept import VARIABLE_PREFIX, Concept, DEFINITION_TYPE_BNF, ConceptParts +from core.rule import Rule from core.tokenizer import TokenKind, Token from parsers.BaseParser import Node, BaseParser, ErrorNode @@ -26,7 +27,7 @@ class LexerNode(Node): def __post_init__(self): if self.source is None: - self.source = BaseParser.get_text_from_tokens(self.tokens) + self.source = core.utils.get_text_from_tokens(self.tokens) def __eq__(self, other): if not isinstance(other, LexerNode): @@ -39,7 +40,7 @@ class LexerNode(Node): def fix_source(self, force=True): if force or self.source is None: - self.source = BaseParser.get_text_from_tokens(self.tokens) + self.source = core.utils.get_text_from_tokens(self.tokens) return self def clone(self): @@ -151,6 +152,40 @@ class UnrecognizedTokensNode(LexerNode): return f"UTN('{self.source}')" +class RuleNode(LexerNode): + def __init__(self, rule, start, end, tokens=None, source=None): + super().__init__(start, end, tokens, source) + self.rule = rule + self.fix_source(False) + + def __eq__(self, other): + if id(self) == id(other): + return True + + if isinstance(other, RN): + return other == self + + if not isinstance(other, RuleNode): + return False + + return self.rule == other.rule and \ + self.start == other.start and \ + self.end == other.end and \ + self.source == other.source + + def __hash__(self): + return hash((self.rule, self.start, self.end, self.source)) + + def __repr__(self): + return f"RuleNode(rule='{self.rule}', source='{self.source}', start={self.start}, end={self.end})" + + def clone(self): + return RuleNode(self.rule, self.start, self.end, self.tokens, self.source) + + def to_short_str(self): + return f'RN({self.rule})' + + class ConceptNode(LexerNode): """ Returned by the BnfNodeParser @@ -194,7 +229,7 @@ class ConceptNode(LexerNode): def __repr__(self): text = f"ConceptNode(concept='{self.concept}', source='{self.source}', start={self.start}, end={self.end}" if DEBUG_COMPILED: - for k, v in self.concept.compiled.items(): + for k, v in self.concept.get_compiled().items(): text += f", {k}='{v}'" return text + ")" @@ -213,7 +248,7 @@ class ConceptNode(LexerNode): bag[k] = v # if isinstance(self.concept, Concept): - # bag["compiled"] = self.concept.compiled + # bag["compiled"] = self.concept.get_compiled() return bag def to_short_str(self): @@ -607,7 +642,7 @@ class CNC(CN): It matches with ConceptNode But focuses on the 'compiled' property of the concept - CNC == ConceptNode if CNC.compiled == ConceptNode.concept.compiled + CNC == ConceptNode if CNC.get_compiled() == ConceptNode.concept.get_compiled() """ def __init__(self, concept_key, start=None, end=None, source=None, exclude_body=False, **kwargs): @@ -634,9 +669,9 @@ class CNC(CN): if self.source is not None and self.source != other.source: return False if self.exclude_body: - to_compare = {k: v for k, v in other.concept.compiled.items() if k != ConceptParts.BODY} + to_compare = {k: v for k, v in other.concept.get_compiled().items() if k != ConceptParts.BODY} else: - to_compare = other.concept.compiled + to_compare = other.concept.get_compiled() if self.compiled == to_compare: # expanded form to ease the debug return True else: @@ -675,10 +710,9 @@ class UTN(HelperWithPos): def __init__(self, source, start=None, end=None): """ - :param concept: Concept or concept_key (only the key is used anyway) + :param source: :param start: :param end: - :param source: """ super().__init__(start, end) self.source = source @@ -711,6 +745,65 @@ class UTN(HelperWithPos): return txt + ")" +class RN(HelperWithPos): + """ + Helper class to test RuleNode + """ + + def __init__(self, rule, start=None, end=None, source=None): + """ + + :param concept: Concept or concept_key (only the key is used anyway) + :param start: + :param end: + :param source: + """ + super().__init__(start, end) + self.rule_id = rule.id if isinstance(rule, Rule) else rule + self.source = source or core.utils.str_concept((None, self.rule_id), prefix="r:") + self.rule = rule if isinstance(rule, Rule) else None + + def __eq__(self, other): + if id(self) == id(other): + return True + + if isinstance(other, RuleNode): + if other.rule is None: + return False + if other.rule.id != self.rule_id: + return False + if self.start is not None and self.start != other.start: + return False + if self.end is not None and self.end != other.end: + return False + if self.source is not None and self.source != other.source: + return False + return True + + if not isinstance(other, RN): + return False + + return self.rule_id == other.rule_id and \ + self.start == other.start and \ + self.end == other.end and \ + self.source == other.source + + def __hash__(self): + return hash((self.rule_id, self.start, self.end, self.source)) + + def __repr__(self): + if self.rule: + txt = f"RN(rule='{self.rule}'" + else: + txt = f"RN(rule_id='{self.rule_id}'" + txt += f", source='{self.source}'" + if self.start is not None: + txt += f", start={self.start}" + if self.end is not None: + txt += f", end={self.end}" + return txt + ")" + + class BaseNodeParser(BaseParser): """ Parser that return LexerNode @@ -938,10 +1031,10 @@ class BaseNodeParser(BaseParser): :param concept: :return: """ - if concept.bnf: + if concept.get_bnf(): from parsers.BnfNodeParser import BnfNodeFirstTokenVisitor bnf_visitor = BnfNodeFirstTokenVisitor(sheerka) - bnf_visitor.visit(concept.bnf) + bnf_visitor.visit(concept.get_bnf()) return bnf_visitor.first_tokens else: keywords = concept.key.split() @@ -955,22 +1048,22 @@ class BaseNodeParser(BaseParser): @staticmethod def ensure_bnf(context, concept, parser_name="BaseNodeParser"): - if concept.metadata.definition_type == DEFINITION_TYPE_BNF and not concept.bnf: - from parsers.BnfParser import BnfParser - regex_parser = BnfParser() - desc = f"Resolving BNF '{concept.metadata.definition}'" + if concept.get_metadata().definition_type == DEFINITION_TYPE_BNF and not concept.get_bnf(): + from parsers.BnfDefinitionParser import BnfDefinitionParser + regex_parser = BnfDefinitionParser() + desc = f"Resolving BNF '{concept.get_metadata().definition}'" with context.push(BuiltinConcepts.INIT_BNF, concept, who=parser_name, obj=concept, desc=desc) as sub_context: - sub_context.add_inputs(parser_input=concept.metadata.definition) - bnf_parsing_ret_val = regex_parser.parse(sub_context, concept.metadata.definition) + sub_context.add_inputs(parser_input=concept.get_metadata().definition) + bnf_parsing_ret_val = regex_parser.parse(sub_context, concept.get_metadata().definition) sub_context.add_values(return_values=bnf_parsing_ret_val) if not bnf_parsing_ret_val.status: raise Exception(bnf_parsing_ret_val.value) - concept.bnf = bnf_parsing_ret_val.body.body + concept.set_bnf(bnf_parsing_ret_val.body.body) if concept.id: - context.sheerka.get_by_id(concept.id).bnf = concept.bnf # update bnf in cache + context.sheerka.get_by_id(concept.id).set_bnf(concept.get_bnf()) # update bnf in cache diff --git a/src/parsers/BaseParser.py b/src/parsers/BaseParser.py index 357e9c0..b672cb6 100644 --- a/src/parsers/BaseParser.py +++ b/src/parsers/BaseParser.py @@ -4,6 +4,7 @@ from typing import Union from core.builtin_concepts import BuiltinConcepts, ParserResultConcept from core.concept import Concept +from core.error import ErrorObj from core.sheerka.ExecutionContext import ExecutionContext from core.sheerka.services.SheerkaExecute import ParserInput from core.sheerka_logger import get_logger @@ -50,7 +51,7 @@ class NotInitializedNode(Node): @dataclass() -class ErrorNode(Node): +class ErrorNode(Node, ErrorObj): pass @@ -89,9 +90,9 @@ class BaseParser: PREFIX = "parsers." def __init__(self, name, priority: int, enabled=True, yield_eof=False): - self.log = get_logger("parsers." + self.__class__.__name__) - self.init_log = get_logger("init." + self.PREFIX + self.__class__.__name__) - self.verbose_log = get_logger("verbose." + self.PREFIX + self.__class__.__name__) + # self.log = get_logger("parsers." + self.__class__.__name__) + # self.init_log = get_logger("init." + self.PREFIX + self.__class__.__name__) + # self.verbose_log = get_logger("verbose." + self.PREFIX + self.__class__.__name__) self.name = self.PREFIX + name self.priority = priority @@ -141,23 +142,25 @@ class BaseParser: return len(self.error_sink) > 0 def log_result(self, context, source, ret): - if not self.log.isEnabledFor(logging.DEBUG): - return - - if ret.status: - value = context.return_value_to_str(ret) - context.log(f"Recognized '{source}' as {value}", self.name) - else: - context.log(f"Failed to recognize '{source}'", self.name) + pass + # if not self.log.isEnabledFor(logging.DEBUG): + # return + # + # if ret.status: + # value = context.return_value_to_str(ret) + # context.log(f"Recognized '{source}' as {value}", self.name) + # else: + # context.log(f"Failed to recognize '{source}'", self.name) def log_multiple_results(self, context, source, list_of_ret): - if not self.log.isEnabledFor(logging.DEBUG): - return - - context.log(f"Recognized '{source}' as multiple concepts", self.name) - for r in list_of_ret: - value = context.return_value_to_str(r) - context.log(f" Recognized '{value}'", self.name) + pass + # if not self.log.isEnabledFor(logging.DEBUG): + # return + # + # context.log(f"Recognized '{source}' as multiple concepts", self.name) + # for r in list_of_ret: + # value = context.return_value_to_str(r) + # context.log(f" Recognized '{value}'", self.name) def get_return_value_body(self, sheerka, source, parsed, try_parse): """ @@ -221,35 +224,35 @@ class BaseParser: lst.append(Token(TokenKind.EOF, "", -1, -1, -1)) return lst - @staticmethod - def get_text_from_tokens(tokens, custom_switcher=None, tracker=None): - """ - Create the source code, from the list of token - :param tokens: list of tokens - :param custom_switcher: to override the behaviour (the return value) of some token - :param tracker: keep track of the original token value when custom switched - :return: - """ - if tokens is None: - return "" - res = "" - - if not hasattr(tokens, "__iter__"): - tokens = [tokens] - - switcher = { - # TokenKind.CONCEPT: lambda t: core.utils.str_concept(t.value), - } - - if custom_switcher: - switcher.update(custom_switcher) - - for token in tokens: - value = switcher.get(token.type, lambda t: t.str_value)(token) - res += value - if tracker is not None and token.type in custom_switcher: - tracker[value] = token.value - return res + # @staticmethod + # def get_text_from_tokens(tokens, custom_switcher=None, tracker=None): + # """ + # Create the source code, from the list of token + # :param tokens: list of tokens + # :param custom_switcher: to override the behaviour (the return value) of some token + # :param tracker: keep track of the original token value when custom switched + # :return: + # """ + # if tokens is None: + # return "" + # res = "" + # + # if not hasattr(tokens, "__iter__"): + # tokens = [tokens] + # + # switcher = { + # # TokenKind.CONCEPT: lambda t: core.utils.str_concept(t.value), + # } + # + # if custom_switcher: + # switcher.update(custom_switcher) + # + # for token in tokens: + # value = switcher.get(token.type, lambda t: t.str_value)(token) + # res += value + # if tracker is not None and token.type in custom_switcher: + # tracker[value] = token.value + # return res @staticmethod def get_tokens_boundaries(tokens): diff --git a/src/parsers/BnfParser.py b/src/parsers/BnfDefinitionParser.py similarity index 96% rename from src/parsers/BnfParser.py rename to src/parsers/BnfDefinitionParser.py index eb7e499..99096a2 100644 --- a/src/parsers/BnfParser.py +++ b/src/parsers/BnfDefinitionParser.py @@ -14,7 +14,7 @@ class UnexpectedEndOfFileError(ErrorNode): pass -class BnfParser(BaseParser): +class BnfDefinitionParser(BaseParser): """ Parser used to transform literal into ParsingExpression example : @@ -28,10 +28,10 @@ class BnfParser(BaseParser): """ + NAME = "BnfDefinition" + def __init__(self, **kwargs): - super().__init__("Bnf", 50, False) - # self.error_sink = [] - # self.name = BaseParser.PREFIX + "Bnf" + super().__init__(BnfDefinitionParser.NAME, 50, False) self.lexer_iter = None self._current = None @@ -42,7 +42,7 @@ class BnfParser(BaseParser): self.sheerka = None def __eq__(self, other): - if not isinstance(other, BnfParser): + if not isinstance(other, BnfDefinitionParser): return False return True @@ -294,7 +294,7 @@ class BnfParser(BaseParser): expression.rule_name = token.value self.next_token() - if BnfParser.is_expression_a_set(self.context, expression): + if BnfDefinitionParser.is_expression_a_set(self.context, expression): root_concept = self.context.search(start_with_self=True, predicate=lambda ec: ec.action == BuiltinConcepts.INIT_BNF, get_obj=lambda ec: ec.action_context, @@ -313,8 +313,8 @@ class BnfParser(BaseParser): @staticmethod def update_recurse_id(context, concept_id, expression): - if BnfParser.is_expression_a_set(context, expression): + if BnfDefinitionParser.is_expression_a_set(context, expression): expression.recurse_id = expression.get_recurse_id(concept_id, expression.concept.id, expression.rule_name) for element in expression.elements: - BnfParser.update_recurse_id(context, concept_id, element) + BnfDefinitionParser.update_recurse_id(context, concept_id, element) diff --git a/src/parsers/BnfNodeParser.py b/src/parsers/BnfNodeParser.py index 6f2ad97..9fc163b 100644 --- a/src/parsers/BnfNodeParser.py +++ b/src/parsers/BnfNodeParser.py @@ -11,6 +11,7 @@ from dataclasses import dataclass from operator import attrgetter import core.builtin_helpers +import core.utils from cache.Cache import Cache from core.builtin_concepts import BuiltinConcepts from core.concept import DEFINITION_TYPE_BNF, DoNotResolve, ConceptParts, Concept @@ -19,7 +20,7 @@ from core.tokenizer import Tokenizer, TokenKind, Token from parsers.BaseNodeParser import BaseNodeParser, GrammarErrorNode, UnrecognizedTokensNode, ConceptNode, LexerNode from parsers.BaseParser import BaseParser -PARSERS = ["AtomNode", "SyaNode", "Python"] +PARSERS = ["Sequence", "Sya", "Python"] @dataclass @@ -41,7 +42,7 @@ class ParsingContext: :return: """ self.node.tokens = parser_helper.parser.parser_input.tokens[self.node.start: self.node.end + 1] - self.node.source = BaseParser.get_text_from_tokens(self.node.tokens) + self.node.source = core.utils.get_text_from_tokens(self.node.tokens) def __mul__(self, other): res = [self] @@ -1044,17 +1045,17 @@ class BnfConceptParserHelper: Adds a new entry, makes a list if the property already exists """ - if prop_name not in _concept.compiled or _concept.compiled[prop_name] is None: + if prop_name not in _concept.get_compiled() or _concept.get_compiled()[prop_name] is None: # new entry - _concept.compiled[prop_name] = value + _concept.get_compiled()[prop_name] = value else: # make a list if there was a value - previous_value = _concept.compiled[prop_name] + previous_value = _concept.get_compiled()[prop_name] if isinstance(previous_value, list): previous_value.append(value) else: new_value = [previous_value, value] - _concept.compiled[prop_name] = new_value + _concept.get_compiled()[prop_name] = new_value def _look_for_concept_match(_underlying): """ @@ -1094,18 +1095,18 @@ class BnfConceptParserHelper: if _underlying.parsing_expression.rule_name: value = _get_underlying_value(_underlying) _add_prop(_concept, _underlying.parsing_expression.rule_name, value) - _concept.metadata.need_validation = True + _concept.get_metadata().need_validation = True elif isinstance(_underlying, NonTerminalNode): for child in _underlying.children: _process_rule_name(_concept, child) - if init_empty_body and concept.metadata.body is None: + if init_empty_body and concept.get_metadata().body is None: value = _get_underlying_value(underlying) - concept.compiled[ConceptParts.BODY] = value + concept.get_compiled()[ConceptParts.BODY] = value if underlying.parsing_expression.rule_name: _add_prop(concept, underlying.parsing_expression.rule_name, value) - # KSI : Why don't we set concept.metadata.need_validation to True ? + # KSI : Why don't we set concept.get_metadata().need_validation to True ? if isinstance(underlying, NonTerminalNode) and not isinstance(underlying.parsing_expression, ConceptExpression): for node in underlying.children: @@ -1147,8 +1148,11 @@ class ToUpdate: class BnfNodeParser(BaseNodeParser): + + NAME = "Bnf" + def __init__(self, **kwargs): - super().__init__("BnfNode", 50, **kwargs) + super().__init__(BnfNodeParser.NAME, 50, **kwargs) if 'sheerka' in kwargs: sheerka = kwargs.get("sheerka") @@ -1162,11 +1166,11 @@ class BnfNodeParser(BaseNodeParser): @staticmethod def _is_eligible(concept): """ - Predicate that select concepts that must handled by AtomNodeParser + Predicate that select concepts that must handled by BnfNodeParser :param concept: :return: """ - return concept.metadata.definition_type == DEFINITION_TYPE_BNF + return concept.get_metadata().definition_type == DEFINITION_TYPE_BNF @staticmethod def get_valid(parsers_helpers): @@ -1422,7 +1426,6 @@ class BnfNodeParser(BaseNodeParser): with context.push(BuiltinConcepts.INIT_BNF, concept, who=self.name, obj=concept, - root_concept=concept, desc=desc) as sub_context: # get the parsing expression to_skip = {concept.id} @@ -1500,13 +1503,13 @@ class BnfNodeParser(BaseNodeParser): desc = f"Resolve concept parsing expression for '{concept}'. {key_to_use=}" with context.push(BuiltinConcepts.INIT_BNF, concept, who=self.name, obj=concept, desc=desc) as sub_context: - if not concept.bnf: # 'if' is done outside to save a function call. Not sure it worth it. + if not concept.get_bnf(): # 'if' is done outside to save a function call. Not sure it worth it. BaseNodeParser.ensure_bnf(sub_context, concept, self.name) grammar[key_to_use] = UnderConstruction(concept.id) - if concept.metadata.definition_type == DEFINITION_TYPE_BNF: - expression = concept.bnf + if concept.get_metadata().definition_type == DEFINITION_TYPE_BNF: + expression = concept.get_bnf() desc = f"Bnf concept detected. Resolving parsing expression '{expression}'" with sub_context.push(BuiltinConcepts.INIT_BNF, concept, who=self.name, obj=concept, desc=desc) as ssc: ssc.add_inputs(expression=expression) @@ -1630,7 +1633,7 @@ class BnfNodeParser(BaseNodeParser): if isinstance(concept, Concept): return concept - if concept in context.concepts: + if context.concepts and concept in context.concepts: return context.concepts[concept] return self.sheerka.get_by_key(concept) diff --git a/src/parsers/DefConceptParser.py b/src/parsers/DefConceptParser.py index 21077bf..10d93f6 100644 --- a/src/parsers/DefConceptParser.py +++ b/src/parsers/DefConceptParser.py @@ -8,7 +8,7 @@ from core.sheerka.services.SheerkaExecute import ParserInput, SheerkaExecute from core.tokenizer import TokenKind, Keywords from parsers.BaseCustomGrammarParser import BaseCustomGrammarParser, SyntaxErrorNode from parsers.BaseParser import Node, ErrorNode, NotInitializedNode, UnexpectedTokenErrorNode -from parsers.BnfParser import BnfParser +from parsers.BnfDefinitionParser import BnfDefinitionParser class ParsingException(Exception): @@ -221,7 +221,7 @@ class DefConceptParser(BaseCustomGrammarParser): if tokens[0].type == TokenKind.COLON: tokens = self.get_body(tokens[1:]) - bnf_regex_parser = BnfParser() + bnf_regex_parser = BnfDefinitionParser() desc = f"Resolving BNF {current_concept_def.definition}" with self.context.push(BuiltinConcepts.INIT_BNF, current_concept_def, diff --git a/src/parsers/ExactConceptParser.py b/src/parsers/ExactConceptParser.py index 025be7a..36df6da 100644 --- a/src/parsers/ExactConceptParser.py +++ b/src/parsers/ExactConceptParser.py @@ -75,12 +75,12 @@ class ExactConceptParser(BaseParser): index = int(token[len(VARIABLE_PREFIX):]) value = words[i] concept.def_var_by_index(index, str_concept(value) if isinstance(value, tuple) else value) - concept.metadata.need_validation = True - if self.verbose_log.isEnabledFor(logging.DEBUG): - prop_name = concept.metadata.variables[index][0] - context.log( - f"Added variable {index}: {prop_name}='{words[i]}'.", - self.name) + concept.get_metadata().need_validation = True + # if self.verbose_log.isEnabledFor(logging.DEBUG): + # prop_name = concept.get_metadata().variables[index][0] + # context.log( + # f"Added variable {index}: {prop_name}='{words[i]}'.", + # self.name) already_recognized.append(concept) diff --git a/src/parsers/FormatRuleParser.py b/src/parsers/FormatRuleParser.py index bdf17db..7321daf 100644 --- a/src/parsers/FormatRuleParser.py +++ b/src/parsers/FormatRuleParser.py @@ -1,24 +1,14 @@ from dataclasses import dataclass +import core.utils from core.builtin_concepts import BuiltinConcepts, ReturnValueConcept -from core.builtin_helpers import parse_unrecognized, expect_one -from core.sheerka.services.SheerkaExecute import ParserInput, SheerkaExecute +from core.sheerka.services.SheerkaExecute import ParserInput +from core.sheerka.services.SheerkaRuleManager import SheerkaRuleManager, FormatAstNode from core.tokenizer import Keywords -from core.utils import strip_tokens from parsers.BaseCustomGrammarParser import BaseCustomGrammarParser, KeywordNotFound from parsers.BaseParser import BaseParser, Node -@dataclass -class FormatAstNode: - pass - - -@dataclass -class FormatAstRawText(FormatAstNode): - text: str - - @dataclass class FormatRuleNode(Node): tokens: dict @@ -29,7 +19,7 @@ class FormatRuleNode(Node): class FormatRuleParser(BaseCustomGrammarParser): """ Class that will parse formatting rules definitions - when xxx print yyy + eg: when xxx print yyy where xxx will be evaluated in the context of BuiltinConcepts.EVAL_QUESTION_REQUESTED and yyy is a internal way to describe a format (yet another one) """ @@ -81,7 +71,7 @@ class FormatRuleParser(BaseCustomGrammarParser): return ret def parse_rule(self): - parts = self.get_parts(self.KEYWORDS_VALUES) + parts = self.get_parts(self.KEYWORDS_VALUES, strip_tokens=True) if parts is None: return None @@ -108,19 +98,14 @@ class FormatRuleParser(BaseCustomGrammarParser): :param tokens: :return: """ - source = self.sheerka.services[SheerkaExecute.NAME].get_parser_input(None, strip_tokens(tokens[1:])) - parsed = parse_unrecognized(self.context, - source, - parsers="all", - who=self.name, - prop=Keywords.WHEN, - filter_func=expect_one) + source = core.utils.get_text_from_tokens(core.utils.strip_tokens(tokens[1:])) + res = self.sheerka.services[SheerkaRuleManager.NAME].compile_when(self.context, self.name, source) - if not parsed.status: - self.add_error(parsed.value) + if not isinstance(res, list): + self.add_error(res.value) return None - return parsed + return res def get_print(self, tokens): """ @@ -128,5 +113,10 @@ class FormatRuleParser(BaseCustomGrammarParser): :param tokens: :return: """ - source = BaseParser.get_text_from_tokens(strip_tokens(tokens[1:])) - return FormatAstRawText(source) + source = core.utils.get_text_from_tokens(core.utils.strip_tokens(tokens[1:])) + res = self.sheerka.services[SheerkaRuleManager.NAME].compile_print(self.context, source) + if not res.status: + self.add_error(res.value) + return None + + return res.body diff --git a/src/parsers/FunctionParser.py b/src/parsers/FunctionParser.py index 70e6ba9..0ad1430 100644 --- a/src/parsers/FunctionParser.py +++ b/src/parsers/FunctionParser.py @@ -3,16 +3,24 @@ from typing import List from core.builtin_concepts import BuiltinConcepts from core.builtin_helpers import get_lexer_nodes_from_unrecognized, update_compiled +from core.concept import Concept from core.sheerka.services.SheerkaExecute import ParserInput from core.tokenizer import TokenKind, Token from core.utils import get_n_clones +from parsers.SequenceNodeParser import SequenceNodeParser from parsers.BaseNodeParser import SourceCodeNode, SourceCodeWithConceptNode, UnrecognizedTokensNode from parsers.BaseParser import BaseParser, UnexpectedTokenErrorNode, UnexpectedEofNode, Node +from parsers.BnfNodeParser import BnfNodeParser from parsers.PythonWithConceptsParser import PythonWithConceptsParser +from parsers.RuleParser import RuleParser +from parsers.SyaNodeParser import SyaNodeParser # No need to check for Python code as the source code node will resolve to python code anyway # I only look for concepts, so -PARSERS = ["BnfNode", "SyaNode", "AtomNode"] +PARSERS = [RuleParser.NAME, + SequenceNodeParser.NAME, + BnfNodeParser.NAME, + SyaNodeParser.NAME] @dataclass @@ -334,8 +342,6 @@ class FunctionParser(BaseParser): res = [SourceCodeWithConceptNode(function_node.first.to_unrecognized(), function_node.last.to_unrecognized())] - function_name = function_node.first.str_value() - for param in function_node.parameters: if isinstance(param.value, NamesNode): # try to recognize concepts @@ -383,7 +389,7 @@ class FunctionParser(BaseParser): # make sure that concepts found can be evaluated errors = [] - for c in source_code_node.python_node.concepts.values(): + for c in [c for c in source_code_node.python_node.objects.values() if isinstance(c, Concept)]: update_compiled(self.context, c, errors) return res diff --git a/src/parsers/PythonParser.py b/src/parsers/PythonParser.py index 1c2728f..07c57bd 100644 --- a/src/parsers/PythonParser.py +++ b/src/parsers/PythonParser.py @@ -20,12 +20,23 @@ class PythonErrorNode(ErrorNode): # self.log.debug("-> PythonErrorNode: " + str(self.exception)) +@dataclass() +class ConceptDetected(ErrorNode): + name: str + + class PythonNode(Node): - def __init__(self, source, ast_=None, concepts=None): + def __init__(self, source, ast_=None, objects=None): self.source = source self.ast_ = ast_ if ast_ else ast.parse(source, mode="eval") if source else None - self.concepts = concepts or {} # when concepts are recognized in the expression + self.objects = objects or {} # when objects (mainly concepts or rules) are recognized in the expression + self.compiled = None + + def get_compiled(self): + if self.compiled is None: + self.compiled = compile(self.ast_, "", "eval") + return self.compiled # def __repr__(self): # return "PythonNode(parser_input='" + self.parser_input + "', ast=" + self.get_dump(self.ast_) + ")" @@ -80,12 +91,16 @@ class PythonParser(BaseParser): self.source = kwargs.get("source", "") def parse(self, context, parser_input: ParserInput): + if not isinstance(parser_input, ParserInput): + return None + sheerka = context.sheerka tree = None tracker = {} # to keep track of concept tokens (c:xxx:) python_switcher = { - TokenKind.CONCEPT: lambda t: core.utils.encode_concept(t.value) + TokenKind.CONCEPT: lambda t: core.utils.encode_concept(t.value), + TokenKind.RULE: lambda t: core.utils.encode_concept(t.value, "R") } try: @@ -106,6 +121,12 @@ class PythonParser(BaseParser): except LexerError as e: self.error_sink.append(e) + # Python parser will refuse input that directly refers to a concept + if isinstance(tree, ast.Expression) and isinstance(tree.body, ast.Name): + if tree.body.id in tracker or context.sheerka.fast_resolve(tree.body.id, return_new=False) is not None: + context.log("It's a simple concept. Not for me.", self.name) + self.error_sink.append(ConceptDetected(tree.body.id)) + if self.has_error: ret = sheerka.ret( self.name, diff --git a/src/parsers/PythonWithConceptsParser.py b/src/parsers/PythonWithConceptsParser.py index 6551d41..c16b7fb 100644 --- a/src/parsers/PythonWithConceptsParser.py +++ b/src/parsers/PythonWithConceptsParser.py @@ -1,6 +1,6 @@ from core.builtin_concepts import BuiltinConcepts from core.sheerka.services.SheerkaExecute import SheerkaExecute -from parsers.BaseNodeParser import ConceptNode +from parsers.BaseNodeParser import ConceptNode, RuleNode from parsers.BaseNodeParser import SourceCodeWithConceptNode from parsers.BaseParser import BaseParser from parsers.PythonParser import PythonParser @@ -15,6 +15,9 @@ class PythonWithConceptsParser(BaseParser): @staticmethod def sanitize(identifier): + if identifier is None: + return "" + res = "" for c in identifier: res += c if c.isalnum() else "0" @@ -46,7 +49,7 @@ class PythonWithConceptsParser(BaseParser): identifiers_key = {} python_ids_mappings = {} - def _get_identifier(c): + def _get_identifier(c, wrapper): """ Get an identifier for a concept. Make sure to return the same identifier if the same concept @@ -61,7 +64,7 @@ class PythonWithConceptsParser(BaseParser): if id(c) in identifiers: return identifiers[id(c)] - identifier = "__C__" + self.sanitize(c.key or c.name) + identifier = wrapper + self.sanitize(c.key or c.name) if c.id: identifier += "__" + c.id @@ -71,7 +74,7 @@ class PythonWithConceptsParser(BaseParser): else: identifiers_key[identifier] = 0 - identifier += "__C__" + identifier += wrapper identifiers[id(c)] = identifier return identifier @@ -82,10 +85,19 @@ class PythonWithConceptsParser(BaseParser): if to_parse: to_parse += " " concept = node.concept - python_id = _get_identifier(concept) + python_id = _get_identifier(concept, "__C__") to_parse += python_id python_ids_mappings[python_id] = concept + elif isinstance(node, RuleNode): + source += node.source + if to_parse: + to_parse += " " + rule = node.rule + python_id = _get_identifier(rule, "__R__") + to_parse += python_id + python_ids_mappings[python_id] = rule + else: source += node.source to_parse += node.source @@ -100,7 +112,7 @@ class PythonWithConceptsParser(BaseParser): if result.status: python_node = result.body.body python_node.source = source - python_node.concepts = python_ids_mappings + python_node.objects = python_ids_mappings return sheerka.ret( self.name, diff --git a/src/parsers/RuleParser.py b/src/parsers/RuleParser.py new file mode 100644 index 0000000..b0e5567 --- /dev/null +++ b/src/parsers/RuleParser.py @@ -0,0 +1,88 @@ +from core.builtin_concepts import BuiltinConcepts +from core.rule import Rule, ACTION_TYPE_DEFERRED +from core.sheerka.services.SheerkaExecute import ParserInput +from core.tokenizer import LexerError, TokenKind +from parsers.BaseParser import BaseParser, ErrorNode, UnexpectedTokenErrorNode + + +class RuleNotFound(ErrorNode): + def __init__(self, id_as_tuple): + self.key = id_as_tuple[0] + self.id = id_as_tuple[1] + + def __repr__(self): + return f"RuleNotFound(id={self.id}, key={self.key}" + + +class RuleParser(BaseParser): + """ + Tries to recognize rules + """ + + NAME = "Rule" + + def __init__(self, **kwargs): + BaseParser.__init__(self, RuleParser.NAME, 80) + + def parse(self, context, parser_input: ParserInput): + """ + text can be string, but text can also be an list of tokens + :param context: + :param parser_input: + :return: + """ + + context.log(f"Parsing '{parser_input}'", self.name) + sheerka = context.sheerka + + if parser_input.is_empty(): + return sheerka.ret(self.name, + False, + sheerka.new(BuiltinConcepts.IS_EMPTY)) + + try: + parser_input.reset() + + parser_input.next_token() + if parser_input.token.type != TokenKind.RULE: + return sheerka.ret(self.name, + False, + sheerka.new(BuiltinConcepts.NOT_FOR_ME, body=parser_input.as_text())) + + token = parser_input.token + + if parser_input.next_token(): + reason = UnexpectedTokenErrorNode("Only one rule supported", + parser_input.token, + [TokenKind.EOF]) + return sheerka.ret(self.name, + False, + sheerka.new(BuiltinConcepts.NOT_FOR_ME, body=parser_input.as_text(), reason=reason)) + + if token.value[1] is None: + return sheerka.ret(self.name, + False, + sheerka.new(BuiltinConcepts.NOT_IMPLEMENTED)) + + if token.value[1].isdigit(): + rule = sheerka.get_rule_by_id(token.value[1]) + else: + rule = Rule().set_id(token.value[1]) + rule.metadata.action_type = ACTION_TYPE_DEFERRED + + if sheerka.isinstance(rule, BuiltinConcepts.UNKNOWN_RULE): + return sheerka.ret(self.name, + False, + sheerka.new(BuiltinConcepts.ERROR, + body=[RuleNotFound(token.value)])) + body = sheerka.new(BuiltinConcepts.PARSER_RESULT, + parser=self, + source=parser_input.as_text(), + body=[rule], + try_parsed=[rule]) + + return sheerka.ret(self.name, True, body) + + except LexerError as e: + context.log(f"Error found in tokenizer {e}", self.name) + return sheerka.ret(self.name, False, sheerka.new(BuiltinConcepts.ERROR, body=e)) diff --git a/src/parsers/AtomNodeParser.py b/src/parsers/SequenceNodeParser.py similarity index 95% rename from src/parsers/AtomNodeParser.py rename to src/parsers/SequenceNodeParser.py index 0186dfb..3ccfbc3 100644 --- a/src/parsers/AtomNodeParser.py +++ b/src/parsers/SequenceNodeParser.py @@ -8,8 +8,10 @@ from core.tokenizer import Tokenizer, TokenKind from core.utils import strip_tokens, make_unique from parsers.BaseNodeParser import BaseNodeParser, ConceptNode, UnrecognizedTokensNode, SourceCodeNode from parsers.BaseParser import UnexpectedTokenErrorNode, ErrorNode +from parsers.BnfNodeParser import BnfNodeParser +from parsers.SyaNodeParser import SyaNodeParser -PARSERS = ["BnfNode", "SyaNode", "Python"] +PARSERS = [BnfNodeParser.NAME, SyaNodeParser.NAME, "Python"] @dataclass() @@ -199,7 +201,7 @@ class AtomConceptParserHelper: return clone -class AtomNodeParser(BaseNodeParser): +class SequenceNodeParser(BaseNodeParser): """ Parser used to recognize atoms concepts or sequence of atoms concepts An atom concept is concept that does not have any property thought it may have a body @@ -216,17 +218,19 @@ class AtomNodeParser(BaseNodeParser): Note 'one plus two' will be recognized by the SyaParser """ + NAME = "Sequence" + def __init__(self, **kwargs): - super().__init__("AtomNode", 50, **kwargs) + super().__init__(SequenceNodeParser.NAME, 50, **kwargs) @staticmethod def _is_eligible(concept): """ - Predicate that select concepts that must handled by AtomNodeParser + Predicate that select concepts that must handled by SequenceNodeParser :param concept: :return: """ - return len(concept.metadata.variables) == 0 and concept.metadata.definition_type != DEFINITION_TYPE_BNF + return len(concept.get_metadata().variables) == 0 and concept.get_metadata().definition_type != DEFINITION_TYPE_BNF def get_concepts(self, token, to_keep, custom=None, to_map=None, strip_quotes=False): @@ -245,7 +249,7 @@ class AtomNodeParser(BaseNodeParser): return a if isinstance(a, list) else [a] - concepts_by_name = as_list(self.sheerka.resolve(token.value)) + concepts_by_name = as_list(self.sheerka.resolve(token)) concepts_by_first_keyword = new_instances(super().get_concepts(token, self._is_eligible)) if concepts_by_name is None: @@ -368,8 +372,8 @@ class AtomNodeParser(BaseNodeParser): for node in parser_helper.sequence: # if isinstance(node, ConceptNode): - # if len(node.concept.metadata.variables) > 0: - # node.concept.metadata.is_evaluated = True # Do not try to evaluate those concepts + # if len(node.concept.get_metadata().variables) > 0: + # node.concept.get_metadata().is_evaluated = True # Do not try to evaluate those concepts node.tokens = self.parser_input.tokens[node.start:node.end + 1] node.fix_source() diff --git a/src/parsers/SyaNodeParser.py b/src/parsers/SyaNodeParser.py index 5cd272c..597065b 100644 --- a/src/parsers/SyaNodeParser.py +++ b/src/parsers/SyaNodeParser.py @@ -7,6 +7,7 @@ from core import builtin_helpers from core.builtin_concepts import BuiltinConcepts from core.builtin_helpers import parse_function from core.concept import Concept, DEFINITION_TYPE_BNF +from core.global_symbols import CONCEPT_COMPARISON_CONTEXT from core.sheerka.services.SheerkaComparisonManager import SheerkaComparisonManager from core.sheerka.services.SheerkaExecute import ParserInput from core.tokenizer import Token, TokenKind, Tokenizer @@ -15,7 +16,7 @@ from parsers.BaseNodeParser import UnrecognizedTokensNode, ConceptNode, SourceCo SourceCodeWithConceptNode, BaseNodeParser from parsers.BaseParser import ErrorNode -PARSERS = ["BnfNode", "AtomNode", "Python"] +PARSERS = ["Sequence", "Bnf", "Python"] function_parser_res = namedtuple("FunctionParserRes", 'to_out function') @@ -159,7 +160,7 @@ class SyaConceptParserHelper: return len(self.expected) == 0 def is_atom(self): - return len(self.concept.concept.metadata.variables) == 0 and len(self.expected) == 0 + return len(self.concept.concept.get_metadata().variables) == 0 and len(self.expected) == 0 def is_next(self, token): """ @@ -1056,9 +1057,10 @@ class PostFixToItem: class SyaNodeParser(BaseNodeParser): + NAME = "Sya" def __init__(self, **kwargs): - super().__init__("SyaNode", 50, **kwargs) + super().__init__(SyaNodeParser.NAME, 50, **kwargs) if 'sheerka' in kwargs: sheerka = kwargs.get("sheerka") self.sya_definitions = sheerka.resolved_sya_def @@ -1085,16 +1087,15 @@ class SyaNodeParser(BaseNodeParser): @staticmethod def _is_eligible(concept): """ - Predicate that select concepts that must handled by AtomNodeParser + Predicate that select concepts that must handled by SyaNodeParser :param concept: :return: """ # We only concepts that has parameter (refuse atoms) # Bnf definitions are not supposed to be managed by this parser either - return len(concept.metadata.variables) > 0 and concept.metadata.definition_type != DEFINITION_TYPE_BNF + return len(concept.get_metadata().variables) > 0 and concept.get_metadata().definition_type != DEFINITION_TYPE_BNF - @staticmethod - def _get_sya_concept_def(parser, concept): + def _get_sya_concept_def(self, parser, concept): sya_concept_def = SyaConceptDef(concept) if concept.id in parser.sya_definitions: # Manage when precedence and associativity are given in the unit tests @@ -1105,9 +1106,9 @@ class SyaNodeParser(BaseNodeParser): sya_concept_def.associativity = sya_def[1] if parser.sheerka: - concept_weight = parser.sheerka.get_concepts_weights(BuiltinConcepts.PRECEDENCE) - if concept.id in concept_weight: - sya_concept_def.precedence = concept_weight[concept.id] + concept_weight = parser.sheerka.get_concepts_weights(BuiltinConcepts.PRECEDENCE, CONCEPT_COMPARISON_CONTEXT) + if concept.str_id in concept_weight: + sya_concept_def.precedence = concept_weight[concept.str_id] if associativity := concept.get_prop(BuiltinConcepts.ASSOCIATIVITY): sya_concept_def.associativity = SyaAssociativity(associativity) @@ -1137,7 +1138,7 @@ class SyaNodeParser(BaseNodeParser): res.extend(forked) forked.clear() - res = [InFixToPostFix(context, context.in_context(BuiltinConcepts.DEBUG))] + res = [InFixToPostFix(context, context.debug_enabled)] while self.parser_input.next_token(False): for infix_to_postfix in res: infix_to_postfix.reset() @@ -1184,13 +1185,13 @@ class SyaNodeParser(BaseNodeParser): infix_to_postfix.finalize(self.parser_input.pos) _add_forked_to_res() - if context.in_context(BuiltinConcepts.DEBUG): - context.debug(f"Parsing {parser_input}") - context.debug(f"{len(res)} InfixToPostFix(s) found") + if context.debug_enabled: + context.debug(self.name, "infix_to_postfix", None, f"Parsing {parser_input}") + context.debug(self.name, "infix_to_postfix", "nb_found", f"{len(res)} InfixToPostFix(s) found") for i, r in enumerate(res): - context.debug(f"#{i}") + context.debug(self.name, "infix_to_postfix", "infix_to_postfix", f"#{i}") for line in r.debug: - context.debug(line) + context.debug(self.name, "infix_to_postfix", "infix_to_postfix", line) return res @@ -1221,21 +1222,21 @@ class SyaNodeParser(BaseNodeParser): end = item.end has_unrecognized = False concept = sheerka.new_from_template(item.concept, item.concept.key) - for param_index in reversed(range(len(concept.metadata.variables))): + for param_index in reversed(range(len(concept.get_metadata().variables))): inner_item = self.postfix_to_item(sheerka, postfixed) if inner_item.start < start: start = inner_item.start if inner_item.end > end: end = inner_item.end has_unrecognized |= isinstance(inner_item, (UnrecognizedTokensNode, SourceCodeWithConceptNode)) or \ - hasattr(inner_item, "has_unrecognized") and inner_item.has_unrecognized + hasattr(inner_item, "has_unrecognized") and inner_item.has_unrecognized - param_name = concept.metadata.variables[param_index][0] + param_name = concept.get_metadata().variables[param_index][0] param_value = inner_item.concept if hasattr(inner_item, "concept") else \ [inner_item.return_value] if isinstance(inner_item, SourceCodeNode) else \ inner_item - concept.compiled[param_name] = param_value + concept.get_compiled()[param_name] = param_value return PostFixToItem(concept, start, end, has_unrecognized) diff --git a/src/parsers/UnrecognizedNodeParser.py b/src/parsers/UnrecognizedNodeParser.py index 4577122..27f4043 100644 --- a/src/parsers/UnrecognizedNodeParser.py +++ b/src/parsers/UnrecognizedNodeParser.py @@ -3,11 +3,18 @@ from dataclasses import dataclass import core.utils from core.builtin_concepts import BuiltinConcepts from core.builtin_helpers import only_successful, parse_unrecognized, get_lexer_nodes, update_compiled -from core.concept import Concept +from parsers.SequenceNodeParser import SequenceNodeParser from parsers.BaseNodeParser import ConceptNode, UnrecognizedTokensNode, SourceCodeNode, SourceCodeWithConceptNode from parsers.BaseParser import BaseParser, ErrorNode +from parsers.BnfNodeParser import BnfNodeParser +from parsers.SyaNodeParser import SyaNodeParser -PARSERS = ["EmptyString", "ShortTermMemory", "AtomNode", "BnfNode", "SyaNode", "Python"] +PARSERS = ["EmptyString", + "ShortTermMemory", + SequenceNodeParser.NAME, + BnfNodeParser.NAME, + SyaNodeParser.NAME, + "Python"] @dataclass() @@ -22,7 +29,7 @@ class UnrecognizedNodeParser(BaseParser): """ def __init__(self, **kwargs): - super().__init__("UnrecognizedNode", 45) # lower than AtomNode, BnfNode and SyaNode + super().__init__("UnrecognizedNode", 45) # lower than SequenceNode, BnfNode and SyaNode def add_error(self, error): if hasattr(error, "__iter__"): diff --git a/src/printer/SheerkaPrinter.py b/src/printer/SheerkaPrinter.py index ac87258..2858e97 100644 --- a/src/printer/SheerkaPrinter.py +++ b/src/printer/SheerkaPrinter.py @@ -1,7 +1,7 @@ import types from core.builtin_concepts import BuiltinConcepts -from core.concept import Concept +from core.concept import Concept, NotInit from printer.FormatInstructions import FormatInstructions, FormatDetailType from printer.Formatter import Formatter @@ -36,6 +36,7 @@ class SheerkaPrinter: self.custom_concepts_printers = { str(BuiltinConcepts.EXPLANATION): self.print_explanation, str(BuiltinConcepts.RETURN_VALUE): self.print_return_value, + str(BuiltinConcepts.TO_LIST): self.print_to_list, } self.formatter.reset_formats() self.formatter.register_format_l(EXECUTION_CONTEXT_CLASS, "[{id:3}] %tab%{desc} ({status})") @@ -110,7 +111,7 @@ class SheerkaPrinter: if instructions.recursive_props: for k, v in instructions.recursive_props.items(): - if hasattr(item, k) and v > 0 and (value := getattr(item, k)) != BuiltinConcepts.NOT_INITIALIZED: + if hasattr(item, k) and v > 0 and (value := getattr(item, k)) != NotInit: self.fp(instructions.recurse(k), value) @staticmethod @@ -137,11 +138,19 @@ class SheerkaPrinter: printer.fp(explanation_instructions, f"%blue%{item.digest}%reset% : {item.command}") printer.fp(explanation_instructions, item.body) + @staticmethod + def print_to_list(printer, instructions, item): + explanation_instructions = instructions.clone().merge(item.get_format_instructions()) + printer.fp(explanation_instructions, item.body) + @staticmethod def print_return_value(printer, instructions, item): if printer.sheerka.isinstance(item.body, BuiltinConcepts.EXPLANATION): return printer.fp(instructions, item.body) + if printer.sheerka.isinstance(item.body, BuiltinConcepts.TO_LIST): + return printer.fp(instructions, item.body) + if isinstance(item.body, (list, tuple, types.GeneratorType)): return printer.fp(instructions, item.body) diff --git a/src/repl/SheerkaPrompt.py b/src/repl/SheerkaPrompt.py index ac8ec3f..e5ff8a9 100644 --- a/src/repl/SheerkaPrompt.py +++ b/src/repl/SheerkaPrompt.py @@ -1,6 +1,7 @@ from os import path import click +import prompt_toolkit from core.sheerka.Sheerka import EXIT_COMMANDS from prompt_toolkit import prompt from prompt_toolkit.auto_suggest import AutoSuggestFromHistory @@ -22,15 +23,22 @@ class SheerkaPrompt: completer=SheerkaPromptCompleter(self.sheerka), # lexer=PygmentsLexer(PythonLexer) ) + _in = _in.strip() + if _in in EXIT_COMMANDS: print("Take care.") break + if _in == "clear": + prompt_toolkit.shortcuts.clear() + continue + if _in == '__': _in = click.edit() result = self.sheerka.evaluate_user_input(_in) - self.sheerka.print(result) + if not self.sheerka.enable_process_return_values: + self.sheerka.print(result) except KeyboardInterrupt: continue except EOFError: diff --git a/src/sdp/readme.md b/src/sdp/readme.md index 9a1d789..5a9469c 100644 --- a/src/sdp/readme.md +++ b/src/sdp/readme.md @@ -15,6 +15,7 @@ - R : executionContext ('R' stands for Result or ReturnValue, no history management) - O : ServiceObj (from pickle) - M : MemoryObject (using SheerkaPickle) +- X : Rule (from sheerkaPickle, 'X' stands for nothing, I am running out of meaningful letters) ## How concepts are serialized ? - get the id of the concept diff --git a/src/sdp/sheerkaDataProvider.py b/src/sdp/sheerkaDataProvider.py index 51629df..487048e 100644 --- a/src/sdp/sheerkaDataProvider.py +++ b/src/sdp/sheerkaDataProvider.py @@ -121,7 +121,7 @@ class SheerkaDataProviderTransaction: # save the event if needed self.event_digest = self.sdp.save_event(self.event) if isinstance(self.event, Event) else self.event - # load state + # load state. I need a fresh copy, not the one from sdp self.snapshot = self.sdp.get_snapshot(SheerkaDataProvider.HeadFile) self.state = self.sdp.load_state(self.snapshot) return self @@ -158,6 +158,18 @@ class SheerkaDataProviderTransaction: except KeyError: pass + def clear(self, entry): + """ + Clear an entire entry + :param entry: + :return: + """ + with self.sdp.lock: + try: + self.state.data[entry].clear() + except KeyError: + pass + def __exit__(self, exc_type, exc_val, exc_tb): self.state.parents = [] if self.snapshot is None else [self.snapshot] self.state.events = [self.event_digest] @@ -165,6 +177,7 @@ class SheerkaDataProviderTransaction: self.snapshot = self.sdp.save_state(self.state) self.sdp.set_snapshot(SheerkaDataProvider.HeadFile, self.snapshot) + self.sdp.update_state(self.state) # make sure to keep sync return False # let's escalate the exceptions @@ -193,6 +206,9 @@ class SheerkaDataProvider: self.serializer = Serializer() self.lock = RLock() + snapshot = self.get_snapshot(SheerkaDataProvider.HeadFile) + self.state = self.load_state(snapshot) + @staticmethod def get_stream_digest(stream): sha256_hash = hashlib.sha256() @@ -214,22 +230,19 @@ class SheerkaDataProvider: :param load_origin: adds the parent object digest to the object :return: """ + with self.lock: - snapshot = self.get_snapshot(SheerkaDataProvider.HeadFile) - state = self.load_state(snapshot) - - if entry not in state.data: + if entry not in self.state.data: return default - if key is not None and key not in state.data[entry]: + if key is not None and key not in self.state.data[entry]: return default - item = state.data[entry] if key is None else state.data[entry][key] - if isinstance(item, list): - return [self.load_ref_if_needed(i, load_origin) for i in item] - elif isinstance(item, set): - return {self.load_ref_if_needed(i, load_origin) for i in item} - return self.load_ref_if_needed(item, load_origin) + item = self.state.data[entry] if key is None else self.state.data[entry][key] + if isinstance(item, dict): + return item.copy() if key else {k: self.load_ref_if_needed_ex(v, load_origin) for k, v in item.items()} + else: + return self.load_ref_if_needed_ex(item, load_origin) def list(self, entry, filter=None): """ @@ -238,12 +251,10 @@ class SheerkaDataProvider: :param filter: filter to use :return: list of elements """ - snapshot = self.get_snapshot(SheerkaDataProvider.HeadFile) - state = self.load_state(snapshot) - if entry not in state.data: + if entry not in self.state.data: return [] - elements = state.data[entry] + elements = self.state.data[entry] if isinstance(elements, dict): # manage when elements have a key @@ -272,14 +283,11 @@ class SheerkaDataProvider: :return: """ with self.lock: - snapshot = self.get_snapshot(SheerkaDataProvider.HeadFile) - state = self.load_state(snapshot) - - exist = entry in state.data + exist = entry in self.state.data if not exist or key is None: return exist - return key in state.data[entry] + return key in self.state.data[entry] def reset(self): """ @@ -289,6 +297,8 @@ class SheerkaDataProvider: self.first_time = self.io.first_time if hasattr(self.io, "reset"): self.io.reset() + snapshot = self.get_snapshot(SheerkaDataProvider.HeadFile) + self.state = self.load_state(snapshot) def save_event(self, event: Event): """ @@ -384,6 +394,10 @@ class SheerkaDataProvider: self.io.write_binary(target_path, self.serializer.serialize(state, context).read()) return digest + def update_state(self, state: State): + with self.lock: + self.state = state + def get_result_file_path(self, digest, is_admin): ext = "_admin_result" if is_admin else "_result" return self.io.get_obj_path(SheerkaDataProvider.EventFolder, digest) + ext @@ -448,6 +462,22 @@ class SheerkaDataProvider: resolved = self.load_obj(obj[len(SheerkaDataProvider.REF_PREFIX):], load_origin) return resolved or obj + def load_ref_if_needed_ex(self, item, load_origin): + """ + Make sure we return the real object, even inside a collection + :param item: + :param load_origin: + :return: + """ + if isinstance(item, list): + return [self.load_ref_if_needed(i, load_origin) for i in item] + elif isinstance(item, set): + return {self.load_ref_if_needed(i, load_origin) for i in item} + elif isinstance(item, dict): + return item.copy() + else: + return self.load_ref_if_needed(item, load_origin) + def save_obj(self, obj): self.log.debug(f"Saving '{obj}' as reference...") context = SerializerContext(user_name="kodjo", sheerka=self.sheerka) diff --git a/src/sdp/sheerkaSerializer.py b/src/sdp/sheerkaSerializer.py index 6067bab..4de20a0 100644 --- a/src/sdp/sheerkaSerializer.py +++ b/src/sdp/sheerkaSerializer.py @@ -8,6 +8,7 @@ from enum import Enum import sheerkapickle from core.concept import Concept +from core.rule import Rule from core.sheerka_logger import get_logger from core.utils import get_full_qualified_name, get_class @@ -60,8 +61,9 @@ class Serializer: self.register(ConceptSerializer()) self.register(DictionarySerializer()) self.register(ExecutionContextSerializer()) - self.register(MemoryObjectSerializer()) # before ServiceObjSerializer + self.register(MemoryObjectSerializer()) # before ServiceObjSerializer self.register(ServiceObjSerializer()) + self.register(RuleSerializer()) def register(self, serializer): """ @@ -297,3 +299,8 @@ class MemoryObjectSerializer(SheerkaPickleSerializer): def __init__(self): super().__init__(lambda obj: get_full_qualified_name(obj) == self.CLASS_NAME, "R", 1) + + +class RuleSerializer(SheerkaPickleSerializer): + def __init__(self): + super().__init__(lambda obj: isinstance(obj, Rule), "X", 1) diff --git a/src/sheerkapickle/SheerkaPickler.py b/src/sheerkapickle/SheerkaPickler.py index ff081ec..5d16a8c 100644 --- a/src/sheerkapickle/SheerkaPickler.py +++ b/src/sheerkapickle/SheerkaPickler.py @@ -8,9 +8,7 @@ from sheerkapickle import utils, tags, handlers def encode(sheerka, obj): - pickler = SheerkaPickler(sheerka) - data = pickler.flatten(obj) - return json.dumps(data) + return json.dumps(SheerkaPickler(sheerka).flatten(obj)) class ToReduce: @@ -41,6 +39,9 @@ class SheerkaPickler: self.to_reduce.append(ToReduce(lambda o: isinstance(o, NotInitialized), lambda o: None)) def flatten(self, obj): + if utils.is_to_discard(obj): + return str(obj) + if utils.is_primitive(obj): return obj @@ -125,9 +126,8 @@ class SheerkaPickler: if hasattr(obj, "__dict__"): for k, v in obj.__dict__.items(): data[k] = self.flatten(v) - return data - return None + return data def exist(self, obj): try: diff --git a/src/sheerkapickle/SheerkaUnpickler.py b/src/sheerkapickle/SheerkaUnpickler.py index c504f78..ac3661e 100644 --- a/src/sheerkapickle/SheerkaUnpickler.py +++ b/src/sheerkapickle/SheerkaUnpickler.py @@ -5,8 +5,7 @@ from sheerkapickle import tags, utils, handlers def decode(sheerka, obj): - decoded = SheerkaUnpickler(sheerka).restore(json.loads(obj)) - return decoded + return SheerkaUnpickler(sheerka).restore(json.loads(obj)) class SheerkaUnpickler: @@ -74,6 +73,11 @@ class SheerkaUnpickler: self.objs.append(instance) instance = handler.restore(obj, instance) else: + # KSI 202011: Hack because Property is removed + # To suppress asap + if obj[tags.OBJECT] == "core.concept.Property": + return self.restore(obj["value"]) + cls = core.utils.get_class(obj[tags.OBJECT]) instance = cls.__new__(cls) self.objs.append(instance) diff --git a/src/sheerkapickle/sheerka_handlers.py b/src/sheerkapickle/sheerka_handlers.py index fd7c5d9..3468d7e 100644 --- a/src/sheerkapickle/sheerka_handlers.py +++ b/src/sheerkapickle/sheerka_handlers.py @@ -1,14 +1,20 @@ +import core.utils from core.builtin_concepts import UserInputConcept, ReturnValueConcept, BuiltinConcepts +from core.concept import Concept, PROPERTIES_TO_SERIALIZE as CONCEPT_PROPERTIES_TO_SERIALIZE, ConceptParts, NotInit, \ + get_concept_attrs +from core.rule import Rule +from core.sheerka.ExecutionContext import ExecutionContext, PROPERTIES_TO_SERIALIZE as CONTEXT_PROPERTIES_TO_SERIALIZE from core.sheerka.Sheerka import Sheerka from core.sheerka.services.SheerkaExecute import ParserInput from evaluators.BaseEvaluator import BaseEvaluator from parsers.BaseParser import BaseParser +from parsers.PythonParser import PythonNode from sheerkapickle.handlers import BaseHandler, registry -from core.concept import Concept, PROPERTIES_TO_SERIALIZE as CONCEPT_PROPERTIES_TO_SERIALIZE, ConceptParts, NotInit -from core.sheerka.ExecutionContext import ExecutionContext, PROPERTIES_TO_SERIALIZE as CONTEXT_PROPERTIES_TO_SERIALIZE default_concept = Concept() +default_concept_values = default_concept.values() CONCEPT_ID = "concept/id" +RULE_ID = "rule/id" class ConceptHandler(BaseHandler): @@ -17,28 +23,28 @@ class ConceptHandler(BaseHandler): pickler = self.context sheerka = self.sheerka - if obj.metadata.full_serialization: + if obj.get_metadata().full_serialization: ref = default_concept + ref_values = default_concept_values else: ref = sheerka.get_by_id(obj.id) + ref_values = ref.values() data[CONCEPT_ID] = (obj.key, obj.id) # transform metadata for name in CONCEPT_PROPERTIES_TO_SERIALIZE: - value = getattr(obj.metadata, name) - ref_value = getattr(ref.metadata, name) + value = getattr(obj.get_metadata(), name) + ref_value = getattr(ref.get_metadata(), name) if value != ref_value: value_to_use = [list(t) for t in value] if name == "variables" else value data["meta." + name] = pickler.flatten(value_to_use) # # transform values - for name in obj.values: - value = obj.get_value(name) - if name not in ref.values or value != ref.get_value(name): + for name, value in obj.values().items(): + if name not in ref_values or value != ref_values[name]: if "values" not in data: data["values"] = [] - key_to_use = "cParts." + name.value if isinstance(name, ConceptParts) else name - data["values"].append((pickler.flatten(key_to_use), pickler.flatten(value))) + data["values"].append((pickler.flatten(name), pickler.flatten(value))) return data @@ -62,16 +68,15 @@ class ConceptHandler(BaseHandler): for prop_name, prop_value in resolved_value: instance.def_var(prop_name, prop_value) else: - setattr(instance.metadata, resolved_prop, resolved_value) + setattr(instance.get_metadata(), resolved_prop, resolved_value) elif key == "values": # get properties for prop_name, prop_value in resolved_value: - key_to_use = ConceptParts(prop_name[7:]) if isinstance(prop_name, str) and prop_name.startswith("cParts.") else prop_name - instance.set_value(key_to_use, NotInit if prop_value is None else prop_value) + instance.set_value(prop_name, NotInit if prop_value is None else prop_value) else: raise Exception("Sanity check as it's not possible yet") - instance.freeze_definition_hash() + # instance.freeze_definition_hash() return instance @@ -80,9 +85,9 @@ class UserInputHandler(ConceptHandler): def flatten(self, obj: UserInputConcept, data): data[CONCEPT_ID] = (obj.key, obj.id) data["user_name"] = obj.user_name - data["text"] = BaseParser.get_text_from_tokens(obj.text) if isinstance(obj.text, list) else \ + data["text"] = core.utils.get_text_from_tokens(obj.text) if isinstance(obj.text, list) else \ obj.text.as_text() if isinstance(obj.text, ParserInput) else \ - obj.text + obj.text return data def new(self, data): @@ -90,8 +95,8 @@ class UserInputHandler(ConceptHandler): def restore(self, data, instance): instance.__init__(data["text"], data["user_name"]) - instance.metadata.key = data[CONCEPT_ID][0] - instance.metadata.id = data[CONCEPT_ID][1] + instance.get_metadata().key = data[CONCEPT_ID][0] + instance.get_metadata().id = data[CONCEPT_ID][1] instance.freeze_definition_hash() return instance @@ -121,8 +126,8 @@ class ReturnValueHandler(BaseHandler): if "parents" in data: instance.parents = pickler.restore(data["parents"]) - instance.metadata.key = data[CONCEPT_ID][0] - instance.metadata.id = data[CONCEPT_ID][1] + instance.get_metadata().key = data[CONCEPT_ID][0] + instance.get_metadata().id = data[CONCEPT_ID][1] instance.freeze_definition_hash() return instance @@ -156,7 +161,7 @@ class ExecutionContextHandler(BaseHandler): return data def new(self, data): - return ExecutionContext(data["who"], None, None, BuiltinConcepts.NOP, None) + return ExecutionContext(data["who"], None, self.sheerka, BuiltinConcepts.NOP, None) def restore(self, data, instance): pickler = self.context @@ -169,9 +174,49 @@ class ExecutionContextHandler(BaseHandler): return instance +class RuleContextHandler(BaseHandler): + + def flatten(self, obj, data): + data[RULE_ID] = obj.id + data["name"] = obj.metadata.name + data["predicate"] = obj.metadata.predicate + data["action_type"] = obj.metadata.action_type + data["action"] = obj.metadata.action + return data + + def new(self, data): + return Rule(data["action_type"], data["name"], data["predicate"], data["action"]) + + def restore(self, data, instance): + instance.metadata.id = data[RULE_ID] + return instance + + +class PythonNodeHandler(BaseHandler): + + def flatten(self, obj, data): + pickler = self.context + + data["source"] = obj.source + data["objects"] = pickler.flatten(obj.objects) + return data + + def new(self, data): + return PythonNode.__new__(PythonNode) + + def restore(self, data, instance): + pickler = self.context + + instance.__init__(data["source"], objects=pickler.restore(data["objects"])) + return instance + + def initialize_pickle_handlers(): registry.register(Concept, ConceptHandler, True) registry.register(UserInputConcept, UserInputHandler, True) registry.register(ReturnValueConcept, ReturnValueHandler, True) registry.register(Sheerka, SheerkaHandler, True) registry.register(ExecutionContext, ExecutionContextHandler, True) + registry.register(Rule, RuleContextHandler, True) + registry.register(PythonNode, PythonNodeHandler, True) + diff --git a/src/sheerkapickle/utils.py b/src/sheerkapickle/utils.py index 0f1c26b..6e989bf 100644 --- a/src/sheerkapickle/utils.py +++ b/src/sheerkapickle/utils.py @@ -21,36 +21,42 @@ def is_object(obj): """Returns True is obj is a reference to an object instance.""" return (isinstance(obj, object) and - not isinstance(obj, (type, types.FunctionType, - types.BuiltinFunctionType))) + not isinstance(obj, (type, + types.FunctionType, + types.BuiltinFunctionType, + types.GeneratorType))) + + +def is_to_discard(obj): + return isinstance(obj, (types.GeneratorType, types.CodeType)) def is_primitive(obj): - return type(obj) in PRIMITIVES + return isinstance(obj, PRIMITIVES) def is_dictionary(obj): - return type(obj) is dict + return isinstance(obj, dict) def is_list(obj): - return type(obj) is list + return isinstance(obj, list) def is_set(obj): - return type(obj) is set + return isinstance(obj, set) def is_bytes(obj): - return type(obj) is bytes + return isinstance(obj, bytes) def is_tuple(obj): - return type(obj) is tuple + return isinstance(obj, tuple) def is_class(obj): - return type(obj) is type + return isinstance(obj, type) def b64encode(data): diff --git a/tests/BaseTest.py b/tests/BaseTest.py index 71bc07a..86083f3 100644 --- a/tests/BaseTest.py +++ b/tests/BaseTest.py @@ -2,9 +2,11 @@ import ast from core.builtin_concepts import ReturnValueConcept, ParserResultConcept, BuiltinConcepts from core.concept import Concept, DEFINITION_TYPE_BNF, DEFINITION_TYPE_DEF +from core.rule import Rule from core.sheerka.ExecutionContext import ExecutionContext +from core.sheerka.services.SheerkaRuleManager import SheerkaRuleManager +from parsers.BnfDefinitionParser import BnfDefinitionParser from parsers.BnfNodeParser import StrMatch -from parsers.BnfParser import BnfParser from sdp.sheerkaDataProvider import Event @@ -40,6 +42,10 @@ class BaseTest: dump = dump.replace(to_remove, "") return dump + @staticmethod + def dump_tokens(tokens): + return [t.repr_value for t in tokens] + def init_concepts(self, *concepts, **kwargs): sheerka = self.get_sheerka(**kwargs) @@ -52,20 +58,21 @@ class BaseTest: if isinstance(c, str): c = Concept(c) - if c.metadata.definition and c.metadata.definition_type != DEFINITION_TYPE_DEF: - desc = f"Resolving BNF {c.metadata.definition}" + if c.get_metadata().definition and c.get_metadata().definition_type != DEFINITION_TYPE_DEF: + desc = f"Resolving BNF {c.get_metadata().definition}" with context.push(BuiltinConcepts.INIT_BNF, c, obj=c, desc=desc) as sub_context: - bnf_parser = BnfParser() - res = bnf_parser.parse(sub_context, c.metadata.definition) + bnf_parser = BnfDefinitionParser() + res = bnf_parser.parse(sub_context, c.get_metadata().definition) if res.status: - c.bnf = res.value.value - c.metadata.definition_type = DEFINITION_TYPE_BNF + c.set_bnf(res.value.value) + c.get_metadata().definition_type = DEFINITION_TYPE_BNF else: - raise Exception(f"Error in bnf definition '{c.metadata.definition}'", sheerka.get_error(res)) + raise Exception(f"Error in bnf definition '{c.get_metadata().definition}'", + sheerka.get_error(res)) if create_new: sheerka.create_new_concept(context, c) @@ -78,6 +85,40 @@ class BaseTest: return sheerka, context, *result + def init_format_rules(self, *rules, create_new=True, compile_rule=True, concepts=None, **kwargs): + if concepts: + sheerka, context, *concepts = self.init_concepts(*concepts, **kwargs) + else: + sheerka, context, *concepts = self.init_concepts(**kwargs) + + if create_new: + sheerka.cache_manager.caches[SheerkaRuleManager.FORMAT_RULE_ENTRY].cache.clear() + sheerka.cache_manager.delete(sheerka.CONCEPTS_KEYS_ENTRY, SheerkaRuleManager.RULE_IDS) + with sheerka.sdp.get_transaction(context.event.get_digest()) as transaction: + transaction.clear(SheerkaRuleManager.FORMAT_RULE_ENTRY) + + initialized = [] + for rule_blue_print in rules: + if isinstance(rule_blue_print, tuple): + rule = Rule("print", None, rule_blue_print[0], rule_blue_print[1]) + if not compile_rule: + rule.metadata.is_compiled = True + else: + rule = rule_blue_print + + is_enabled = rule.metadata.is_enabled + sheerka.services[SheerkaRuleManager.NAME].init_rule(context, rule) + if create_new: + res = sheerka.create_new_rule(context, rule) + initialized.append(res.body.body) + else: + initialized.append(rule) + + if is_enabled is not None: + rule.metadata.is_enabled = is_enabled + + return sheerka, context, *initialized + @staticmethod def get_concept_instance(sheerka, concept, **kwargs): """ @@ -88,10 +129,10 @@ class BaseTest: :return: """ instance = sheerka.new(concept.key if isinstance(concept, Concept) else concept) - for i, var in enumerate(instance.metadata.variables): + for i, var in enumerate(instance.get_metadata().variables): if var[0] in kwargs: assert isinstance(kwargs[var[0]], str), "variables definitions must be string" - instance.metadata.variables[i] = (var[0], kwargs[var[0]]) + instance.get_metadata().variables[i] = (var[0], kwargs[var[0]]) return instance @@ -110,10 +151,10 @@ class BaseTest: return sheerka.ret(who, True, obj) @staticmethod - def pretval(concept, source=None, parser="parsers.name", who="some_name", status=True): + def pretval(concept, source=None, parser="parsers.name", who=None, status=True): """ParserResult ret_val (p stands for ParserResult)""" return ReturnValueConcept( - who, + who or parser, status, ParserResultConcept(parser=parser, source=source or concept.name, @@ -135,8 +176,8 @@ class BaseTest: for v in variables: concept.def_var(v) if bnf: - concept.bnf = bnf - concept.metadata.definition_type = DEFINITION_TYPE_BNF + concept.set_bnf(bnf) + concept.get_metadata().definition_type = DEFINITION_TYPE_BNF concept.init_key() sheerka.set_id_if_needed(concept, False) sheerka.add_in_cache(concept) @@ -150,8 +191,8 @@ class BaseTest: name = concept concept = Concept(concept) - concept.bnf = expression or StrMatch(name) - concept.metadata.definition_type = DEFINITION_TYPE_BNF + concept.set_bnf(expression or StrMatch(name)) + concept.get_metadata().definition_type = DEFINITION_TYPE_BNF return concept @staticmethod @@ -164,9 +205,9 @@ class BaseTest: if kwargs: for k, v in kwargs.items(): if k in ("body", "pre", "post", "where"): - setattr(concept.metadata, k, v) + setattr(concept.get_metadata(), k, v) else: - concept.metadata.variables[k] = v + concept.get_metadata().variables[k] = v return concept def init_scenario(self, init_expressions): diff --git a/tests/TestUsingFileBasedSheerka.py b/tests/TestUsingFileBasedSheerka.py index 94069d1..8058d86 100644 --- a/tests/TestUsingFileBasedSheerka.py +++ b/tests/TestUsingFileBasedSheerka.py @@ -3,6 +3,7 @@ import shutil from os import path import pytest +from core.concept import ALL_ATTRIBUTES from core.sheerka.Sheerka import Sheerka from tests.BaseTest import BaseTest @@ -27,14 +28,16 @@ class TestUsingFileBasedSheerka(BaseTest): os.chdir(current_pwd) def get_sheerka(self, **kwargs): + reset_attrs = kwargs.get("reset_attrs", True) + if reset_attrs: + ALL_ATTRIBUTES.clear() + use_dict = kwargs.get("use_dict", False) # use dictionary based io instead of file # If you do so, information between two different instances of sheerka # won't be shared - use_dict = kwargs.get("use_dict", False) - root = "mem://" if use_dict else self.root_folder sheerka = Sheerka() - sheerka.initialize(root, save_execution_context=False) + sheerka.initialize(root, save_execution_context=False, enable_process_return_values=False) return sheerka diff --git a/tests/TestUsingMemoryBasedSheerka.py b/tests/TestUsingMemoryBasedSheerka.py index eaa0785..5c42c86 100644 --- a/tests/TestUsingMemoryBasedSheerka.py +++ b/tests/TestUsingMemoryBasedSheerka.py @@ -1,3 +1,4 @@ +from core.concept import ALL_ATTRIBUTES from core.sheerka.Sheerka import Sheerka from tests.BaseTest import BaseTest @@ -9,23 +10,28 @@ class TestUsingMemoryBasedSheerka(BaseTest): @staticmethod def _inner_get_sheerka(cache_only): + ALL_ATTRIBUTES.clear() sheerka = Sheerka(cache_only=cache_only) - sheerka.initialize("mem://", save_execution_context=False) + sheerka.initialize("mem://", save_execution_context=False, enable_process_return_values=False) return sheerka def get_sheerka(self, **kwargs): cache_only = kwargs.get("cache_only", True) use_singleton = kwargs.get("singleton", True) + reset_attrs = kwargs.get("reset_attrs", True) sheerka = kwargs.get("sheerka", None) if sheerka: return sheerka + if reset_attrs: + ALL_ATTRIBUTES.clear() + if use_singleton: singleton_instance = TestUsingMemoryBasedSheerka.singleton_instance if singleton_instance: singleton_instance.reset(cache_only) - singleton_instance.cache_manager.init_from(TestUsingMemoryBasedSheerka.dump) + singleton_instance.cache_manager.init_from_dump(TestUsingMemoryBasedSheerka.dump) return singleton_instance else: new_instance = self._inner_get_sheerka(cache_only) diff --git a/tests/cache/test_FastCache.py b/tests/cache/test_FastCache.py new file mode 100644 index 0000000..49ba177 --- /dev/null +++ b/tests/cache/test_FastCache.py @@ -0,0 +1,67 @@ +from cache.FastCache import FastCache + + +def test_i_can_put_an_retrieve_values(): + cache = FastCache() + cache.put("key", "value") + + assert cache.get("key") == "value" + assert cache.cache == {"key": "value"} + assert cache.lru == ["key"] + + +def test_i_can_put_and_retrieve_multiple_items(): + cache = FastCache() + cache.put("key1", "value1") + cache.put("key2", "value2") + cache.put("key3", "value3") + + assert cache.cache == {"key1": "value1", "key2": "value2", "key3": "value3"} + assert cache.lru == ["key1", "key2", "key3"] + + +def test_i_the_least_used_is_remove_first(): + cache = FastCache(3) + cache.put("key1", "value1") + cache.put("key2", "value2") + cache.put("key3", "value3") + + cache.put("key4", "value4") + assert cache.cache == {"key4": "value4", "key2": "value2", "key3": "value3"} + assert cache.lru == ["key2", "key3", "key4"] + + cache.put("key5", "value5") + assert cache.cache == {"key4": "value4", "key5": "value5", "key3": "value3"} + assert cache.lru == ["key3", "key4", "key5"] + + +def test_i_can_put_the_same_key_several_times(): + cache = FastCache() + cache.put("key1", "value1") + cache.put("key2", "value2") + cache.put("key1", "value3") + + assert cache.cache == {"key1": "value3", "key2": "value2"} + assert cache.lru == ["key2", "key1"] + + +def test_none_is_returned_when_not_found(): + cache = FastCache() + assert cache.get("foo") is None + + +def test_i_can_evict_by_key(): + cache = FastCache() + cache.put("key1", "value1") + cache.put("to_keep1", "to_keep_value1") + cache.put("key2", "value2") + cache.put("to_keep2", "to_keep_value2") + cache.put("key3", "value3") + cache.put("to_keep3", "to_keep_value3") + + cache.evict_by_key(lambda k: k.startswith("key")) + assert cache.cache == {"to_keep1": "to_keep_value1", + "to_keep2": "to_keep_value2", + "to_keep3": "to_keep_value3"} + + assert cache.lru == ["to_keep1", "to_keep2", "to_keep3"] diff --git a/tests/cache/test_cache.py b/tests/cache/test_cache.py index ffdf266..b224e5e 100644 --- a/tests/cache/test_cache.py +++ b/tests/cache/test_cache.py @@ -565,3 +565,35 @@ class TestCache(TestUsingMemoryBasedSheerka): cache.get(str(MAX_INITIALIZED_KEY + 1)) assert len(cache._initialized_keys) == 1 + + def test_i_can_populate(self): + items = [("1", "1"), ("2", "2"), ("3", "3")] + cache = Cache() + + cache.populate(lambda: items, lambda item: item[0]) + + assert len(cache) == 3 + assert cache.get("1") == ("1", "1") + assert cache.get("2") == ("2", "2") + assert cache.get("3") == ("3", "3") + + def test_max_size_is_respected_when_populate(self): + items = [("1", "1"), ("2", "2"), ("3", "3"), ("4", "4"), ("5", "5")] + cache = Cache(max_size=3) + + cache.populate(lambda: items, lambda item: item[0]) + + assert len(cache) == 3 + assert cache.get("3") == ("3", "3") + assert cache.get("4") == ("4", "4") + assert cache.get("5") == ("5", "5") + + def test_i_can_get_all(self): + items = [("1", "1"), ("2", "2"), ("3", "3")] + cache = Cache() + + cache.populate(lambda: items, lambda item: item[0]) + + res = cache.get_all() + assert len(res) == 3 + assert list(res) == [('1', '1'), ('2', '2'), ('3', '3')] diff --git a/tests/core/test_ExecutionContext.py b/tests/core/test_ExecutionContext.py index 4ac5f84..dd8bfc5 100644 --- a/tests/core/test_ExecutionContext.py +++ b/tests/core/test_ExecutionContext.py @@ -1,140 +1,196 @@ -import pytest - from core.builtin_concepts import BuiltinConcepts from core.concept import Concept from core.sheerka.ExecutionContext import ExecutionContext +from core.sheerka.services.SheerkaMemory import SheerkaMemory from sdp.sheerkaDataProvider import Event - -def test_id_is_incremented_by_event_digest(): - a = ExecutionContext("foo", Event("event_1"), None, BuiltinConcepts.NOP, None) - b = ExecutionContext("foo", Event("event_1"), None, BuiltinConcepts.NOP, None) - c = ExecutionContext("foo", Event("event_2"), None, BuiltinConcepts.NOP, None) - d = b.push(BuiltinConcepts.NOP, None) - e = c.push(BuiltinConcepts.NOP, None) - - assert a.id == 0 - assert b.id == 1 - assert c.id == 0 - assert d.id == 2 - assert e.id == 1 +from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka -def test_i_can_use_with_statement(): - with ExecutionContext("who_", Event("event"), "fake_sheerka", BuiltinConcepts.NOP, None) as e: - pass - assert e.elapsed > 0 +class TestExecutionContext(TestUsingMemoryBasedSheerka): + def test_id_is_incremented_by_event_digest(self): + sheerka = self.get_sheerka() -def test_i_can_push(): - a = ExecutionContext("foo", Event("event_1"), "fake_sheerka", BuiltinConcepts.NOP, None, - desc="some description", - obj=Concept("foo"), - step=BuiltinConcepts.EVALUATION, - iteration=15, - concepts={"bar": Concept("bar")}) - a.preprocess = set() - a.preprocess.add("preprocess") - a.protected_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED) - a.global_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED) + a = ExecutionContext("foo", Event("event_1"), sheerka, BuiltinConcepts.NOP, None) + b = ExecutionContext("foo", Event("event_1"), sheerka, BuiltinConcepts.NOP, None) + c = ExecutionContext("foo", Event("event_2"), sheerka, BuiltinConcepts.NOP, None) + d = b.push(BuiltinConcepts.NOP, None) + e = c.push(BuiltinConcepts.NOP, None) - b = a.push(BuiltinConcepts.EVALUATION, "sub action context", desc="sub description") + assert a.id == 0 + assert b.id == 1 + assert c.id == 0 + assert d.id == 2 + assert e.id == 1 - assert b._parent == a - assert b.who == a.who - assert b.event == a.event - assert b.sheerka == a.sheerka - assert b.action == BuiltinConcepts.EVALUATION - assert b.action_context == "sub action context" - assert b.desc == "sub description" - assert b.obj == a.obj - assert b.step == a.step - assert b.iteration == a.iteration - assert b.concepts == a.concepts - assert b.id == a.id + 1 - assert b._tab == a._tab + " " - assert b.preprocess == a.preprocess - assert b.protected_hints == {BuiltinConcepts.EVAL_BODY_REQUESTED} - assert b.global_hints == a.global_hints + def test_i_can_use_with_statement(self): + sheerka = self.get_sheerka() + with ExecutionContext("who_", Event("event"), sheerka, BuiltinConcepts.NOP, None) as e: + pass + assert e.elapsed > 0 -def test_children_i_created_when_i_push(): - e = ExecutionContext("who_", Event("event"), "fake_sheerka", BuiltinConcepts.NOP, None) - e.push(BuiltinConcepts.NOP, None, who="a", desc="I do something") - e.push(BuiltinConcepts.NOP, None, who="b", desc="oups! I did a again") - e.push(BuiltinConcepts.NOP, None, who="c", desc="I do something else") + def test_i_can_push(self): + sheerka = self.get_sheerka() - assert len(e._children) == 3 - assert e._children[0].who, e._children[0].who == ("a", "I do something") - assert e._children[1].who, e._children[1].who == ("b", "oups! I did a again") - assert e._children[2].who, e._children[2].who == ("c", "I do something else") + a = ExecutionContext("foo", Event("event_1"), sheerka, BuiltinConcepts.NOP, None, + desc="some description", + obj=Concept("foo"), + concepts={"bar": Concept("bar")}) + a.preprocess = set() + a.preprocess.add("preprocess") + a.preprocess_parsers = ["list of parsers"] + a.preprocess_evaluators = ["list of evaluators"] + a.protected_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED) + a.global_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED) + b = a.push(BuiltinConcepts.EVALUATION, "sub action context", desc="sub description") -def test_i_can_add_variable_when_i_push(): - e = ExecutionContext("who_", Event("event"), "fake_sheerka", BuiltinConcepts.NOP, None) - sub_e = e.push(BuiltinConcepts.NOP, None, who="a", my_new_var="new var value") + assert b._parent == a + assert b.who == a.who + assert b.event == a.event + assert b.sheerka == a.sheerka + assert b.action == BuiltinConcepts.EVALUATION + assert b.action_context == "sub action context" + assert b.desc == "sub description" + assert b.obj == a.obj + assert b.concepts == a.concepts + assert b.id == a.id + 1 + assert b.preprocess == a.preprocess + assert b.preprocess_parsers == a.preprocess_parsers + assert b.preprocess_evaluators == a.preprocess_evaluators + assert b.protected_hints == {BuiltinConcepts.EVAL_BODY_REQUESTED} + assert b.global_hints == a.global_hints - assert sub_e.my_new_var == "new var value" - with pytest.raises(AttributeError): - assert e.my_new_var == "" # my_new_var does not exist in parent + def test_children_i_created_when_i_push(self): + sheerka = self.get_sheerka() + e = ExecutionContext("who_", Event("event"), sheerka, BuiltinConcepts.NOP, None) + e.push(BuiltinConcepts.NOP, None, who="a", desc="I do something") + e.push(BuiltinConcepts.NOP, None, who="b", desc="oups! I did a again") + e.push(BuiltinConcepts.NOP, None, who="c", desc="I do something else") -def test_local_hints_are_local_and_global_hints_are_global(): - a = ExecutionContext("foo", Event("event_1"), "fake_sheerka", BuiltinConcepts.NOP, None) - a.protected_hints.add("local hint 1") - a.global_hints.add("global hint 1") + assert len(e._children) == 3 + assert e._children[0].who, e._children[0].who == ("a", "I do something") + assert e._children[1].who, e._children[1].who == ("b", "oups! I did a again") + assert e._children[2].who, e._children[2].who == ("c", "I do something else") - b = a.push(BuiltinConcepts.NOP, None) - b.protected_hints.add("local hint 2") - b.global_hints.add("global hint 2") + # def test_i_can_add_variable_when_i_push(self): + # sheerka = self.get_sheerka() + # + # e = ExecutionContext("who_", Event("event"), sheerka, BuiltinConcepts.NOP, None) + # sub_e = e.push(BuiltinConcepts.NOP, None, who="a", my_new_var="new var value") + # + # assert sub_e.my_new_var == "new var value" + # with pytest.raises(AttributeError): + # assert e.my_new_var == "" # my_new_var does not exist in parent - assert a.protected_hints == {"local hint 1"} - assert a.global_hints == {"global hint 1", "global hint 2"} + def test_local_hints_are_local_and_global_hints_are_global(self): + sheerka = self.get_sheerka() - assert b.protected_hints == {"local hint 1", "local hint 2"} - assert b.global_hints == {"global hint 1", "global hint 2"} + a = ExecutionContext("foo", Event("event_1"), sheerka, BuiltinConcepts.NOP, None) + a.protected_hints.add("local hint 1") + a.global_hints.add("global hint 1") + b = a.push(BuiltinConcepts.NOP, None) + b.protected_hints.add("local hint 2") + b.global_hints.add("global hint 2") -def test_global_hits_are_global_even_when_empty(): - a = ExecutionContext("foo", Event("event_1"), "fake_sheerka", BuiltinConcepts.NOP, None) + assert a.protected_hints == {"local hint 1"} + assert a.global_hints == {"global hint 1", "global hint 2"} - b = a.push(BuiltinConcepts.NOP, None) - b.global_hints.add("global hint 2") + assert b.protected_hints == {"local hint 1", "local hint 2"} + assert b.global_hints == {"global hint 1", "global hint 2"} - assert a.global_hints == {"global hint 2"} - assert b.global_hints == {"global hint 2"} + def test_global_hits_are_global_even_when_empty(self): + sheerka = self.get_sheerka() + a = ExecutionContext("foo", Event("event_1"), sheerka, BuiltinConcepts.NOP, None) -def test_i_can_search(): - a = ExecutionContext("foo", Event("event_1"), "fake_sheerka", BuiltinConcepts.TESTING, "a") - ab = a.push(BuiltinConcepts.TESTING, "ab", obj="obj_ab") - ac = a.push(BuiltinConcepts.TESTING, "ac", obj="obj_ac") - abb = ab.push(BuiltinConcepts.TESTING, "abb", obj="skip") - abbb = abb.push(BuiltinConcepts.TESTING, "abbb", obj="obj_abbb") + b = a.push(BuiltinConcepts.NOP, None) + b.global_hints.add("global hint 2") - assert list(abbb.search()) == [abb, ab, a] - assert list(abbb.search(start_with_self=True)) == [abbb, abb, ab, a] - assert list(abbb.search(lambda ec: ec.obj != "skip")) == [ab, a] - assert list(abbb.search(lambda ec: ec.obj != "skip", lambda ec: ec.action_context)) == ["ab", "a"] - assert list(abbb.search(stop=lambda ec: ec.obj == "skip")) == [abb] - assert list(abbb.search( - stop=lambda ec: ec.obj == "skip", - start_with_self=True, - get_obj=lambda ec: ec.obj)) == ["obj_abbb", "skip"] + assert a.global_hints == {"global hint 2"} + assert b.global_hints == {"global hint 2"} + def test_i_can_search(self): + sheerka = self.get_sheerka() -def test_variables_are_passed_to_children_but_not_to_parents(): - a = ExecutionContext("foo", Event("event_1"), "fake_sheerka", BuiltinConcepts.NOP, None) - assert not hasattr(a, "var") + a = ExecutionContext("foo", Event("event_1"), sheerka, BuiltinConcepts.TESTING, "a") + ab = a.push(BuiltinConcepts.TESTING, "ab", obj="obj_ab") + ac = a.push(BuiltinConcepts.TESTING, "ac", obj="obj_ac") + abb = ab.push(BuiltinConcepts.TESTING, "abb", obj="skip") + abbb = abb.push(BuiltinConcepts.TESTING, "abbb", obj="obj_abbb") - b = a.push(BuiltinConcepts.NOP, None, var="foo") - assert b.var == "foo" - assert not hasattr(a, "var") + assert list(abbb.search()) == [abb, ab, a] + assert list(abbb.search(start_with_self=True)) == [abbb, abb, ab, a] + assert list(abbb.search(lambda ec: ec.obj != "skip")) == [ab, a] + assert list(abbb.search(lambda ec: ec.obj != "skip", lambda ec: ec.action_context)) == ["ab", "a"] + assert list(abbb.search(stop=lambda ec: ec.obj == "skip")) == [abb] + assert list(abbb.search( + stop=lambda ec: ec.obj == "skip", + start_with_self=True, + get_obj=lambda ec: ec.obj)) == ["obj_abbb", "skip"] - c = b.push(BuiltinConcepts.NOP, None) - assert c.var == "foo" + def test_i_can_deactivate_push(self): + sheerka = self.get_sheerka() - c.var = "bar" - assert c.var == "bar" - assert b.var != "bar" - assert not hasattr(a, "var") + context = ExecutionContext("foo", Event("event_1"), sheerka, BuiltinConcepts.NOP, None) + context.deactivate_push() + sub_context1 = context.push(BuiltinConcepts.NOP, None) + sub_context2 = context.push(BuiltinConcepts.NOP, None) + sub_context3 = sub_context1.push(BuiltinConcepts.NOP, None) + + assert id(sub_context1) == id(sub_context2) + assert id(sub_context1) == id(sub_context3) + + def test_stm_are_copied_when_deactivate_push(self): + sheerka = self.get_sheerka() + + context = ExecutionContext("foo", Event("event_1"), sheerka, BuiltinConcepts.NOP, None) + context.add_to_short_term_memory("key1", "value1") + context.add_to_short_term_memory("key2", "value2") + + context.deactivate_push() + + stm_cache = sheerka.services[SheerkaMemory.NAME].short_term_objects.cache + assert stm_cache == { + context.id: {'key1': 'value1', 'key2': 'value2'}, + context.id + 1: {'key1': 'value1', 'key2': 'value2'} + } + + def test_has_parent(self): + sheerka = self.get_sheerka() + root = ExecutionContext("foo", Event("event_1"), sheerka, BuiltinConcepts.NOP, None) + + sub1 = root.push(BuiltinConcepts.TESTING, None) + sub2 = root.push(BuiltinConcepts.TESTING, None) + + sub11 = sub1.push(BuiltinConcepts.TESTING, None) + sub111 = sub11.push(BuiltinConcepts.TESTING, None) + + assert not root.has_parent(sub1.id) + assert sub1.has_parent(root.id) + assert sub11.has_parent(root.id) + assert sub111.has_parent(root.id) + assert sub2.has_parent(root.id) + assert not sub1.has_parent(sub2.id) + assert not sub2.has_parent(sub1.id) + # def test_variables_are_passed_to_children_but_not_to_parents(self): + # sheerka = self.get_sheerka() + # + # a = ExecutionContext("foo", Event("event_1"), sheerka, BuiltinConcepts.NOP, None) + # assert not hasattr(a, "var") + # + # b = a.push(BuiltinConcepts.NOP, None, var="foo") + # assert b.var == "foo" + # assert not hasattr(a, "var") + # + # c = b.push(BuiltinConcepts.NOP, None) + # assert c.var == "foo" + # + # c.var = "bar" + # assert c.var == "bar" + # assert b.var != "bar" + # assert not hasattr(a, "var") diff --git a/tests/core/test_SheerkaAdmin.py b/tests/core/test_SheerkaAdmin.py new file mode 100644 index 0000000..8820110 --- /dev/null +++ b/tests/core/test_SheerkaAdmin.py @@ -0,0 +1,65 @@ +from core.builtin_concepts import BuiltinConcepts + +from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka + + +class TestSheerkaAdmin(TestUsingMemoryBasedSheerka): + def test_i_can_get_last_ret(self): + pass + + def test_i_can_get_last_error_ret(self): + sheerka, context = self.init_concepts() + + # nothing in history + last_error_ret = sheerka.last_error_ret(context) + assert sheerka.isinstance(last_error_ret, BuiltinConcepts.RETURN_VALUE) + assert not last_error_ret.status + assert sheerka.isinstance(last_error_ret.body, BuiltinConcepts.NOT_FOUND) + + # only success in history + r_success = sheerka.ret("Test", True, "success") + sheerka.last_return_values.append([r_success]) + last_error_ret = sheerka.last_error_ret(context) + assert sheerka.isinstance(last_error_ret, BuiltinConcepts.RETURN_VALUE) + assert not last_error_ret.status + assert sheerka.isinstance(last_error_ret.body, BuiltinConcepts.NOT_FOUND) + + # at least on error + r_failure = sheerka.ret("Test", False, "failure") + sheerka.last_return_values.append([r_failure]) + assert sheerka.last_error_ret(context) == r_failure + + # add another success and make sure we get the same error + sheerka.last_return_values.append([r_success]) + assert sheerka.last_error_ret(context) == r_failure + + # and I only get the last failure + r_failure2 = sheerka.ret("Test", False, "another failure") + sheerka.last_return_values.append([r_failure2]) + assert sheerka.last_error_ret(context) == r_failure2 + + # but I still can get the previous error + assert sheerka.last_error_ret(context, -2) == r_failure + + def test_i_can_get_last_error_ret_when_only_one_ret_has_failed(self): + sheerka, context = self.init_concepts() + + r_success = sheerka.ret("Test", True, "success") + r_failure = sheerka.ret("Test", False, "False1") + + sheerka.last_return_values.append([r_success, r_failure]) + assert sheerka.last_error_ret(context) == r_failure + + def test_i_cannot_get_last_error_ret_when_too_many_errors(self): + sheerka, context = self.init_concepts() + + r1 = sheerka.ret("Test", True, "success") + r2 = sheerka.ret("Test", False, "False1") + r3 = sheerka.ret("Test", False, "False2") + + sheerka.last_return_values.append([r1, r2, r3]) + last_error_ret = sheerka.last_error_ret(context) + assert sheerka.isinstance(last_error_ret, BuiltinConcepts.RETURN_VALUE) + assert not last_error_ret.status + assert sheerka.isinstance(last_error_ret.body, BuiltinConcepts.TOO_MANY_ERRORS) + assert last_error_ret.body.body == [r2, r3] diff --git a/tests/core/test_SheerkaComparisonManager.py b/tests/core/test_SheerkaComparisonManager.py index eb21070..0e29a5a 100644 --- a/tests/core/test_SheerkaComparisonManager.py +++ b/tests/core/test_SheerkaComparisonManager.py @@ -1,5 +1,8 @@ import pytest from core.builtin_concepts import BuiltinConcepts +from core.concept import Concept +from core.global_symbols import CONCEPT_PRECEDENCE_MODIFIED, CONCEPT_COMPARISON_CONTEXT, RULE_PRECEDENCE_MODIFIED, \ + RULE_COMPARISON_CONTEXT from core.sheerka.services.SheerkaComparisonManager import SheerkaComparisonManager, ComparisonObj from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka @@ -31,15 +34,36 @@ class TestSheerkaGreaterThanManager(TestUsingMemoryBasedSheerka): assert sheerka.isinstance(res.body, BuiltinConcepts.SUCCESS) in_cache = sheerka.cache_manager.get(SheerkaComparisonManager.COMPARISON_ENTRY, "prop_name|#") - assert in_cache == [ComparisonObj(context.event.get_digest(), "prop_name", two.id, one.id, ">", "#")] + assert in_cache == [ComparisonObj(context.event.get_digest(), "prop_name", two.str_id, one.str_id, ">", "#")] weighted = sheerka.cache_manager.get(SheerkaComparisonManager.RESOLVED_COMPARISON_ENTRY, "prop_name|#") - assert weighted == {"1001": 1, "1002": 2} + assert weighted == {"c:one|1001:": 1, "c:two|1002:": 2} # I can commit sheerka.cache_manager.commit(context) in_db = sheerka.sdp.get(SheerkaComparisonManager.COMPARISON_ENTRY, "prop_name|#") - assert in_db == [ComparisonObj(context.event.get_digest(), "prop_name", two.id, one.id, ">", "#")] + assert in_db == [ComparisonObj(context.event.get_digest(), "prop_name", two.str_id, one.str_id, ">", "#")] + + def test_i_can_add_is_greater_than_for_rules(self): + sheerka, context, r1, r2 = self.init_format_rules(("True", "true"), ("False", "false"), + cache_only=False, + compile_rule=False) + service = sheerka.services[SheerkaComparisonManager.NAME] + + res = service.set_is_greater_than(context, "prop_name", r2, r1) + assert res.status + assert sheerka.isinstance(res.body, BuiltinConcepts.SUCCESS) + + in_cache = sheerka.cache_manager.get(SheerkaComparisonManager.COMPARISON_ENTRY, "prop_name|#") + assert in_cache == [ComparisonObj(context.event.get_digest(), "prop_name", r2.str_id, r1.str_id, ">", "#")] + + weighted = sheerka.cache_manager.get(SheerkaComparisonManager.RESOLVED_COMPARISON_ENTRY, "prop_name|#") + assert weighted == {"r:|1:": 1, "r:|2:": 2} + + # I can commit + sheerka.cache_manager.commit(context) + in_db = sheerka.sdp.get(SheerkaComparisonManager.COMPARISON_ENTRY, "prop_name|#") + assert in_db == [ComparisonObj(context.event.get_digest(), "prop_name", r2.str_id, r1.str_id, ">", "#")] def test_i_can_add_a_is_less_than(self): sheerka, context, one, two = self.init_concepts("one", "two", cache_only=False) @@ -50,15 +74,36 @@ class TestSheerkaGreaterThanManager(TestUsingMemoryBasedSheerka): assert sheerka.isinstance(res.body, BuiltinConcepts.SUCCESS) in_cache = sheerka.cache_manager.get(SheerkaComparisonManager.COMPARISON_ENTRY, "prop_name|#") - assert in_cache == [ComparisonObj(context.event.get_digest(), "prop_name", one.id, two.id, "<", "#")] + assert in_cache == [ComparisonObj(context.event.get_digest(), "prop_name", one.str_id, two.str_id, "<", "#")] weighted = sheerka.cache_manager.get(SheerkaComparisonManager.RESOLVED_COMPARISON_ENTRY, "prop_name|#") - assert weighted == {"1001": 1, "1002": 2} + assert weighted == {"c:one|1001:": 1, "c:two|1002:": 2} # I can commit sheerka.cache_manager.commit(context) in_db = sheerka.sdp.get(SheerkaComparisonManager.COMPARISON_ENTRY, "prop_name|#") - assert in_db == [ComparisonObj(context.event.get_digest(), "prop_name", one.id, two.id, "<", "#")] + assert in_db == [ComparisonObj(context.event.get_digest(), "prop_name", one.str_id, two.str_id, "<", "#")] + + def test_i_can_add_is_less_than_for_rules(self): + sheerka, context, r1, r2 = self.init_format_rules(("True", "true"), ("False", "false"), + cache_only=False, + compile_rule=False) + service = sheerka.services[SheerkaComparisonManager.NAME] + + res = service.set_is_less_than(context, "prop_name", r1, r2) + assert res.status + assert sheerka.isinstance(res.body, BuiltinConcepts.SUCCESS) + + in_cache = sheerka.cache_manager.get(SheerkaComparisonManager.COMPARISON_ENTRY, "prop_name|#") + assert in_cache == [ComparisonObj(context.event.get_digest(), "prop_name", r1.str_id, r2.str_id, "<", "#")] + + weighted = sheerka.cache_manager.get(SheerkaComparisonManager.RESOLVED_COMPARISON_ENTRY, "prop_name|#") + assert weighted == {"r:|1:": 1, "r:|2:": 2} + + # I can commit + sheerka.cache_manager.commit(context) + in_db = sheerka.sdp.get(SheerkaComparisonManager.COMPARISON_ENTRY, "prop_name|#") + assert in_db == [ComparisonObj(context.event.get_digest(), "prop_name", r1.str_id, r2.str_id, "<", "#")] def test_i_can_add_multiples_constraints(self): sheerka, context, one, two, three, four = self.init_concepts("one", "two", "three", "four", cache_only=False) @@ -69,16 +114,16 @@ class TestSheerkaGreaterThanManager(TestUsingMemoryBasedSheerka): in_cache = sheerka.cache_manager.get(SheerkaComparisonManager.COMPARISON_ENTRY, "prop_name|#") assert in_cache == [ - ComparisonObj(context.event.get_digest(), "prop_name", two.id, one.id, ">", "#"), - ComparisonObj(context.event.get_digest(), "prop_name", three.id, two.id, ">", "#") + ComparisonObj(context.event.get_digest(), "prop_name", two.str_id, one.str_id, ">", "#"), + ComparisonObj(context.event.get_digest(), "prop_name", three.str_id, two.str_id, ">", "#") ] # I can commit sheerka.cache_manager.commit(context) in_db = sheerka.sdp.get(SheerkaComparisonManager.COMPARISON_ENTRY, "prop_name|#") assert in_db == [ - ComparisonObj(context.event.get_digest(), "prop_name", two.id, one.id, ">", "#"), - ComparisonObj(context.event.get_digest(), "prop_name", three.id, two.id, ">", "#") + ComparisonObj(context.event.get_digest(), "prop_name", two.str_id, one.str_id, ">", "#"), + ComparisonObj(context.event.get_digest(), "prop_name", three.str_id, two.str_id, ">", "#") ] sheerka.cache_manager.clear(SheerkaComparisonManager.COMPARISON_ENTRY) # reset the cache @@ -86,21 +131,45 @@ class TestSheerkaGreaterThanManager(TestUsingMemoryBasedSheerka): service.set_is_greater_than(context, "prop_name", four, three) in_cache = sheerka.cache_manager.get(SheerkaComparisonManager.COMPARISON_ENTRY, "prop_name|#") assert in_cache == [ - ComparisonObj(context.event.get_digest(), "prop_name", two.id, one.id, ">", "#"), - ComparisonObj(context.event.get_digest(), "prop_name", three.id, two.id, ">", "#"), - ComparisonObj(context.event.get_digest(), "prop_name", four.id, three.id, ">", "#"), + ComparisonObj(context.event.get_digest(), "prop_name", two.str_id, one.str_id, ">", "#"), + ComparisonObj(context.event.get_digest(), "prop_name", three.str_id, two.str_id, ">", "#"), + ComparisonObj(context.event.get_digest(), "prop_name", four.str_id, three.str_id, ">", "#"), ] weighted = sheerka.cache_manager.get(SheerkaComparisonManager.RESOLVED_COMPARISON_ENTRY, "prop_name|#") - assert weighted == {"1001": 1, "1002": 2, "1003": 3, "1004": 4} + assert weighted == {"c:one|1001:": 1, "c:two|1002:": 2, "c:three|1003:": 3, "c:four|1004:": 4} + + def test_i_can_add_multiple_constraints_2(self): + sheerka, context, one, two, three = self.init_concepts("one", "two", "three") + service = sheerka.services[SheerkaComparisonManager.NAME] + + service.set_is_greater_than(context, "prop_name", two, one) + service.set_is_greater_than(context, "prop_name", three, two) + + weighted = sheerka.cache_manager.get(SheerkaComparisonManager.RESOLVED_COMPARISON_ENTRY, "prop_name|#") + assert weighted == {"c:one|1001:": 1, "c:two|1002:": 2, "c:three|1003:": 3} + + service.set_is_greater_than(context, "prop_name", three, one) # should not change order + weighted = sheerka.cache_manager.get(SheerkaComparisonManager.RESOLVED_COMPARISON_ENTRY, "prop_name|#") + assert weighted == {"c:one|1001:": 1, "c:two|1002:": 2, "c:three|1003:": 3} + + def test_i_lesser_than_and_opposite_greater_than(self): + sheerka, context, one, two = self.init_concepts("one", "two") + service = sheerka.services[SheerkaComparisonManager.NAME] + + service.set_is_greater_than(context, "prop_name", two, one) + service.set_is_less_than(context, "prop_name", one, two) + + weighted = sheerka.cache_manager.get(SheerkaComparisonManager.RESOLVED_COMPARISON_ENTRY, "prop_name|#") + assert weighted == {"c:one|1001:": 1, "c:two|1002:": 2} @pytest.mark.parametrize("entries, expected", [ - (["two > one"], {'1001': 1, '1002': 2}), - (["one < two"], {'1001': 1, '1002': 2}), - (["three > two", "one < two"], {'1001': 1, '1002': 2, '1003': 3}), - (["three > one", "one < two"], {'1001': 1, '1002': 2, '1003': 2}), - (["three >>"], {'1003': 2}), - (["one <<"], {'1001': 0}), + (["two > one"], {'c:one|1001:': 1, 'c:two|1002:': 2}), + (["one < two"], {'c:one|1001:': 1, 'c:two|1002:': 2}), + (["three > two", "one < two"], {'c:one|1001:': 1, 'c:two|1002:': 2, 'c:three|1003:': 3}), + (["three > one", "one < two"], {'c:one|1001:': 1, 'c:two|1002:': 2, 'c:three|1003:': 2}), + (["three >>"], {'c:three|1003:': 2}), + (["one <<"], {'c:one|1001:': 0}), ]) def test_i_can_get_concept_weight(self, entries, expected): sheerka, context, *concepts = self.init_concepts("one", "two", "three") @@ -125,7 +194,7 @@ class TestSheerkaGreaterThanManager(TestUsingMemoryBasedSheerka): service.set_is_lesser(context, "prop_name", one) sheerka.cache_manager.clear(service.RESOLVED_COMPARISON_ENTRY) - assert service.get_concepts_weights("prop_name") == {"1001": 0} + assert service.get_concepts_weights("prop_name") == {"c:one|1001:": 0} def test_i_can_get_partition(self): sheerka, context, one, two, three = self.init_concepts("one", "two", "three") @@ -137,9 +206,9 @@ class TestSheerkaGreaterThanManager(TestUsingMemoryBasedSheerka): res = service.get_partition("prop_name") assert res == { - 1: ["1001"], - 2: ["1002"], - 3: ["1003"], + 1: ["c:one|1001:"], + 2: ["c:two|1002:"], + 3: ["c:three|1003:"], } def test_i_can_detect_chicken_and_egg(self): @@ -176,7 +245,7 @@ class TestSheerkaGreaterThanManager(TestUsingMemoryBasedSheerka): service.set_is_less_than(context, "prop_name", one, two) weighted = sheerka.cache_manager.get(SheerkaComparisonManager.RESOLVED_COMPARISON_ENTRY, "prop_name|#") - assert weighted == {"1001": 1, "1002": 2} + assert weighted == {"c:one|1001:": 1, "c:two|1002:": 2} def test_methods_are_correctly_bound(self): sheerka, context, one, two = self.init_concepts("one", "two") @@ -188,17 +257,17 @@ class TestSheerkaGreaterThanManager(TestUsingMemoryBasedSheerka): service = sheerka.services[SheerkaComparisonManager.NAME] service.set_is_lesser(context, "prop_name", one) - assert service.get_concepts_weights("prop_name") == {"1001": 0} # DEFAULT_COMPARISON_VALUE - 1 + assert service.get_concepts_weights("prop_name") == {"c:one|1001:": 0} # DEFAULT_COMPARISON_VALUE - 1 sheerka.set_is_greater_than(context, "prop_name", three, two) - assert service.get_concepts_weights("prop_name") == {"1001": 0, "1002": 1, "1003": 2} + assert service.get_concepts_weights("prop_name") == {"c:one|1001:": 0, "c:two|1002:": 1, "c:three|1003:": 2} # I can commit sheerka.cache_manager.commit(context) in_db = sheerka.sdp.get(SheerkaComparisonManager.COMPARISON_ENTRY, "prop_name|#") assert in_db == [ - ComparisonObj(context.event.get_digest(), "prop_name", one.id, None, "<<", "#"), - ComparisonObj(context.event.get_digest(), "prop_name", three.id, two.id, ">", "#") + ComparisonObj(context.event.get_digest(), "prop_name", one.str_id, None, "<<", "#"), + ComparisonObj(context.event.get_digest(), "prop_name", three.str_id, two.str_id, ">", "#") ] def test_i_can_define_an_order_for_lesser_concepts(self): @@ -211,7 +280,9 @@ class TestSheerkaGreaterThanManager(TestUsingMemoryBasedSheerka): sheerka.set_is_less_than(context, "prop_name", less_l, lesser) sheerka.set_is_greater_than(context, "prop_name", less_l, even_more_l) - assert service.get_concepts_weights("prop_name") == {"1001": 0, "1002": -1, "1003": -2} + assert service.get_concepts_weights("prop_name") == {"c:lesser|1001:": 0, + "c:less_l|1002:": -1, + "c:even_less_l|1003:": -2} def test_i_cannot_define_less_than_a_lesser_if_not_a_lesser_itself(self): """ @@ -237,10 +308,10 @@ class TestSheerkaGreaterThanManager(TestUsingMemoryBasedSheerka): service = sheerka.services[SheerkaComparisonManager.NAME] service.set_is_greatest(context, "prop_name", three) - assert service.get_concepts_weights("prop_name") == {"1003": 2} # DEFAULT_COMPARISON_VALUE + 1 + assert service.get_concepts_weights("prop_name") == {"c:three|1003:": 2} # DEFAULT_COMPARISON_VALUE + 1 sheerka.set_is_greater_than(context, "prop_name", two, one) - assert service.get_concepts_weights("prop_name") == {"1001": 1, "1002": 2, "1003": 3} + assert service.get_concepts_weights("prop_name") == {"c:one|1001:": 1, "c:two|1002:": 2, "c:three|1003:": 3} def test_i_cannot_define_greater_than_a_greatest_if_not_a_greater_itself(self): sheerka, context, greatest, foo = self.init_concepts("greatest", "foo") @@ -280,7 +351,9 @@ class TestSheerkaGreaterThanManager(TestUsingMemoryBasedSheerka): sheerka.set_is_less_than(context, "prop_name", greatest, more_g) sheerka.set_is_greater_than(context, "prop_name", even_more_g, more_g) - assert service.get_concepts_weights("prop_name") == {"1001": 2, "1002": 3, "1003": 4} + assert service.get_concepts_weights("prop_name") == {"c:greatest|1001:": 2, + "c:more_g|1002:": 3, + "c:even_more_g|1003:": 4} def test_i_can_mix_all_comparisons(self): sheerka, context, one, two, three, four, five, six, three_and_half = self.init_concepts( @@ -298,25 +371,54 @@ class TestSheerkaGreaterThanManager(TestUsingMemoryBasedSheerka): sheerka.set_is_less_than(context, "prop_name", three, four) assert service.get_concepts_weights("prop_name") == { - "1001": -1, - "1002": 0, - "1003": 1, - "1004": 2, - "1005": 3, - "1006": 4 + "c:one|1001:": -1, + "c:two|1002:": 0, + "c:three|1003:": 1, + "c:four|1004:": 2, + "c:five|1005:": 3, + "c:six|1006:": 4 } sheerka.set_is_less_than(context, "prop_name", three_and_half, four) sheerka.set_is_greater_than(context, "prop_name", three_and_half, three) assert service.get_concepts_weights("prop_name") == { - "1001": -1, - "1002": 0, - "1003": 1, - "1007": 2, - "1004": 3, - "1005": 4, - "1006": 5 + "c:one|1001:": -1, + "c:two|1002:": 0, + "c:three|1003:": 1, + "c:two_and_half|1007:": 2, + "c:four|1004:": 3, + "c:five|1005:": 4, + "c:six|1006:": 5 + } + + def test_i_can_mix_all_comparisons_2(self): + sheerka, context, zero, one, two, three, four, five, thousand = self.init_concepts( + "zero", "one", "two", "three", "four", "five", "thousand") + service = sheerka.services[SheerkaComparisonManager.NAME] + + service.set_is_greater_than(context, "prop_name", four, one) + service.set_is_less_than(context, "prop_name", one, five) + service.set_is_less_than(context, "prop_name", one, three) + service.set_is_less_than(context, "prop_name", two, three) + service.set_is_greater_than(context, "prop_name", two, one) + service.set_is_greater_than(context, "prop_name", five, four) + service.set_is_less_than(context, "prop_name", three, four) + service.set_is_greatest(context, "prop_name", thousand) + service.set_is_lesser(context, "prop_name", zero) + service.set_is_less_than(context, "prop_name", two, five) + service.set_is_less_than(context, "prop_name", two, four) + service.set_is_greater_than(context, "prop_name", five, one) + service.set_is_greater_than(context, "prop_name", five, three) + + assert service.get_concepts_weights("prop_name") == { + "c:zero|1001:": 0, + "c:one|1002:": 1, + "c:two|1003:": 2, + "c:three|1004:": 3, + "c:four|1005:": 4, + "c:five|1006:": 5, + "c:thousand|1007:": 6 } @pytest.mark.parametrize("definition", [ @@ -335,3 +437,47 @@ class TestSheerkaGreaterThanManager(TestUsingMemoryBasedSheerka): assert not res.status assert sheerka.isinstance(res.body, BuiltinConcepts.CONCEPT_ALREADY_DEFINED) + + def test_an_event_is_fired_when_modifying_concepts_precedence(self): + sheerka, context, one, two, foo = self.init_concepts("one", "two", "foo") + event_received = False + + def receive_event(c): + nonlocal event_received + event_received = True + + sheerka.subscribe(CONCEPT_PRECEDENCE_MODIFIED, receive_event) + + sheerka.set_is_greater_than(context, foo, one, two) + assert not event_received + + sheerka.set_is_greater_than(context, BuiltinConcepts.PRECEDENCE, foo, one, two) + assert not event_received + + sheerka.set_is_greater_than(context, BuiltinConcepts.PRECEDENCE, one, two, CONCEPT_COMPARISON_CONTEXT) + assert event_received + + def test_an_event_is_fired_when_modifying_rule_precedence(self): + sheerka, context, r1, r2 = self.init_format_rules( + ("True", "True"), + ("False", "False"), + compile_rule=False, + ) + foo = Concept("foo") + event_received = False + sheerka.cache_manager.clear(SheerkaComparisonManager.COMPARISON_ENTRY) + + def receive_event(c): + nonlocal event_received + event_received = True + + sheerka.subscribe(RULE_PRECEDENCE_MODIFIED, receive_event) + + sheerka.set_is_greater_than(context, foo, r1, r2) + assert not event_received + + sheerka.set_is_greater_than(context, BuiltinConcepts.PRECEDENCE, foo, r1, r2) + assert not event_received + + sheerka.set_is_greater_than(context, BuiltinConcepts.PRECEDENCE, r1, r2, RULE_COMPARISON_CONTEXT) + assert event_received diff --git a/tests/core/test_SheerkaConceptAlgebra.py b/tests/core/test_SheerkaConceptAlgebra.py index 8d11d5a..5386777 100644 --- a/tests/core/test_SheerkaConceptAlgebra.py +++ b/tests/core/test_SheerkaConceptAlgebra.py @@ -19,7 +19,7 @@ class TestSheerkaConceptsAlgebra(TestUsingMemoryBasedSheerka): res = sheerka.cadd(context, sheerka.new("man"), sheerka.new("driver")) assert isinstance(res, Concept) - assert res.metadata.props == {BuiltinConcepts.ISA: {male, human}, + assert res.get_metadata().props == {BuiltinConcepts.ISA: {male, human}, BuiltinConcepts.HASA: {car, licence}, } def test_can_add_concepts_when_property_already_exist(self): @@ -32,7 +32,7 @@ class TestSheerkaConceptsAlgebra(TestUsingMemoryBasedSheerka): res = sheerka.cadd(context, sheerka.new("man"), sheerka.new("king")) assert isinstance(res, Concept) - assert res.metadata.props == {BuiltinConcepts.ISA: {male, human}} + assert res.get_metadata().props == {BuiltinConcepts.ISA: {male, human}} def test_i_can_subtract_concepts(self): sheerka, context, foo, bar, isa1, isa2, hasa1, hasa2 = self.init_concepts( @@ -53,14 +53,14 @@ class TestSheerkaConceptsAlgebra(TestUsingMemoryBasedSheerka): res = sheerka.csub(context, new_foo) assert isinstance(res, Concept) - assert res.metadata.props == res.metadata.props + assert res.get_metadata().props == res.get_metadata().props res = sheerka.csub(context, new_foo, new_bar) assert isinstance(res, Concept) - assert res.metadata.props == {BuiltinConcepts.ISA: {isa2}, + assert res.get_metadata().props == {BuiltinConcepts.ISA: {isa2}, BuiltinConcepts.HASA: {hasa2}, } - def test_i_can_recognize_myself_when_cache_only_is_not_set(self): + def test_i_can_recognize_myself_when_using_sdp_repository(self): sheerka, context, foo, isa1, hasa1, = self.init_concepts("foo", "isa1", "has1", cache_only=False, create_new=True) @@ -73,7 +73,7 @@ class TestSheerkaConceptsAlgebra(TestUsingMemoryBasedSheerka): assert sheerka.recognize(new_foo, all_scores=True) == [ConceptScore(1, new_foo, new_foo)] - def test_i_can_recognize_myself_when_cache_only_is_set(self): + def test_i_can_recognize_myself_when_not_using_sdp_repository(self): sheerka, context, foo, isa1, hasa1, = self.init_concepts("foo", "isa1", "has1") new_foo = sheerka.new("foo") diff --git a/tests/core/test_SheerkaCreateNewConcept.py b/tests/core/test_SheerkaCreateNewConcept.py index 0d09e79..ac02dcb 100644 --- a/tests/core/test_SheerkaCreateNewConcept.py +++ b/tests/core/test_SheerkaCreateNewConcept.py @@ -1,6 +1,6 @@ import pytest from core.builtin_concepts import BuiltinConcepts -from core.concept import PROPERTIES_TO_SERIALIZE, Concept, DEFINITION_TYPE_DEF +from core.concept import PROPERTIES_TO_SERIALIZE, Concept, DEFINITION_TYPE_DEF, get_concept_attrs from core.sheerka.Sheerka import Sheerka from tests.TestUsingFileBasedSheerka import TestUsingFileBasedSheerka @@ -22,10 +22,11 @@ class TestSheerkaCreateNewConcept(TestUsingMemoryBasedSheerka): concept_found = res.value.body for prop in PROPERTIES_TO_SERIALIZE: - assert getattr(concept_found.metadata, prop) == getattr(concept.metadata, prop) + assert getattr(concept_found.get_metadata(), prop) == getattr(concept.get_metadata(), prop) assert concept_found.key == "__var__0 + __var__1" assert concept_found.id == "1001" + assert get_concept_attrs(concept) == ['a', 'b'] # saved in cache assert sheerka.has_id(concept.id) @@ -63,7 +64,7 @@ class TestSheerkaCreateNewConcept(TestUsingMemoryBasedSheerka): concept_found = res.value.body for prop in PROPERTIES_TO_SERIALIZE: - assert getattr(concept_found.metadata, prop) == getattr(concept.metadata, prop) + assert getattr(concept_found.get_metadata(), prop) == getattr(concept.get_metadata(), prop) assert concept_found.key == "hello __var__0" assert concept_found.id == "1001" @@ -120,7 +121,7 @@ class TestSheerkaCreateNewConcept(TestUsingMemoryBasedSheerka): sheerka = self.get_sheerka() concept1 = self.get_default_concept() concept2 = self.get_default_concept() - concept2.metadata.body = "a+b" + concept2.get_metadata().body = "a+b" res1 = sheerka.create_new_concept(self.get_context(sheerka), concept1) res2 = sheerka.create_new_concept(self.get_context(sheerka), concept2) @@ -149,7 +150,7 @@ class TestSheerkaCreateNewConcept(TestUsingMemoryBasedSheerka): assert res.status assert sheerka.get_by_name(concept.name) == concept - assert sheerka.get_by_name(concept.metadata.definition) == concept + assert sheerka.get_by_name(concept.get_metadata().definition) == concept concept = Concept(name="foo", definition="foo", definition_type=DEFINITION_TYPE_DEF) res = sheerka.create_new_concept(context, concept) @@ -273,6 +274,7 @@ class TestSheerkaCreateNewConceptFileBased(TestUsingFileBasedSheerka): assert len(sheerka.sdp.get(Sheerka.CONCEPTS_BY_KEY_ENTRY, "foo")) == 2 sheerka = self.get_sheerka() # new instance + context = self.get_context(sheerka) sheerka.create_new_concept(context, Concept("foo", body="3")) sheerka.cache_manager.commit(context) diff --git a/tests/core/test_SheerkaDebugManager.py b/tests/core/test_SheerkaDebugManager.py new file mode 100644 index 0000000..b3b589b --- /dev/null +++ b/tests/core/test_SheerkaDebugManager.py @@ -0,0 +1,425 @@ +import pytest +from core.builtin_concepts import BuiltinConcepts +from core.sheerka.ExecutionContext import ExecutionContext +from core.sheerka.services.SheerkaDebugManager import SheerkaDebugManager, DebugVarSetting, DebugRuleSetting +from sdp.sheerkaDataProvider import Event + +from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka + + +class TestSheerkaDebugManager(TestUsingMemoryBasedSheerka): + + def test_i_can_activate_debug(self): + sheerka, context = self.init_concepts() + + sheerka.set_debug(context, True) + assert sheerka.debug_activated() + + sheerka.set_debug(context, False) + assert not sheerka.debug_activated() + + def test_when_debug_mode_is_activated_context_are_in_debug_mode(self): + sheerka = self.get_sheerka() + ExecutionContext.ids.clear() + root_context = self.get_context(sheerka) + + sheerka.set_debug(root_context, True) + + context = root_context.push(BuiltinConcepts.NOP, None) # sub_context.id = 1 + sub_context = context.push(BuiltinConcepts.NOP, None) # sub_sub_context.parent = 1 + sub_context2 = context.push(BuiltinConcepts.NOP, None) # sub_sub_context2.parent = 1 + sub_sub_context = sub_context.push(BuiltinConcepts.NOP, None) # is a child + + assert context.id == 1 + assert context.debug_enabled + assert sub_context.debug_enabled + assert sub_context2.debug_enabled + assert sub_sub_context.debug_enabled + assert not root_context.debug_enabled + + def test_i_can_activate_debug_for_new_context(self): + sheerka = self.get_sheerka() + ExecutionContext.ids.clear() + root_context = self.get_context(sheerka) + + sheerka.set_debug(root_context, True) + sheerka.set_explicit(root_context, True) + sheerka.activate_debug_for(root_context, 1) + + context = ExecutionContext("test", Event(), sheerka, BuiltinConcepts.TESTING, None) # context.id = 1 + assert context.debug_enabled + + def test_i_can_activate_debug_for_new_context_2(self): + """ + This time children is also requested + :return: + """ + sheerka = self.get_sheerka() + ExecutionContext.ids.clear() + root_context = self.get_context(sheerka) + + sheerka.set_debug(root_context, True) + sheerka.set_explicit(root_context, True) + sheerka.activate_debug_for(root_context, 1, children=True) + + context = ExecutionContext("test", Event(), sheerka, BuiltinConcepts.TESTING, None) # context.id = 1 + assert context.debug_enabled + + def test_i_can_activate_debug_for_a_context_using_push(self): + sheerka = self.get_sheerka() + ExecutionContext.ids.clear() + root_context = self.get_context(sheerka) + service = sheerka.services[SheerkaDebugManager.NAME] + + sheerka.set_debug(root_context, True) + sheerka.set_explicit(root_context, True) + sheerka.activate_debug_for(root_context, 1) + + context = root_context.push(BuiltinConcepts.NOP, None) # sub_context.id = 1 + sub_context = context.push(BuiltinConcepts.NOP, None) # sub_sub_context.parent = 1 + + assert context.id == 1 + assert 1 in service.context_cache + assert context.debug_enabled + assert not sub_context.debug_enabled + assert not root_context.debug_enabled + + def test_global_debug_must_be_activated_to_activate_context_debug(self): + sheerka = self.get_sheerka() + ExecutionContext.ids.clear() + root_context = self.get_context(sheerka) + service = sheerka.services[SheerkaDebugManager.NAME] + + sheerka.set_debug(root_context, False) + sheerka.activate_debug_for(root_context, 1) + + context = root_context.push(BuiltinConcepts.NOP, None) # sub_context.id = 1 + + assert context.id == 1 + assert 1 in service.context_cache + assert not context.debug_enabled + + def test_i_can_activate_debug_for_sub_children(self): + sheerka = self.get_sheerka() + ExecutionContext.ids.clear() + root_context = self.get_context(sheerka) + service = sheerka.services[SheerkaDebugManager.NAME] + + sheerka.set_debug(root_context, True) + sheerka.set_explicit(root_context, True) + sheerka.activate_debug_for(root_context, 1, children=True) + + context = root_context.push(BuiltinConcepts.NOP, None) # sub_context.id = 1 + sub_context = context.push(BuiltinConcepts.NOP, None) # sub_sub_context.parent = 1 + sub_context2 = context.push(BuiltinConcepts.NOP, None) # sub_sub_context2.parent = 1 + sub_sub_context = sub_context.push(BuiltinConcepts.NOP, None) # is a child + + assert context.id == 1 + assert 1 in service.context_cache + assert "1+" in service.context_cache + assert context.debug_enabled + assert sub_context.debug_enabled + assert sub_context2.debug_enabled + assert sub_sub_context.debug_enabled + assert not root_context.debug_enabled + + def test_i_can_deactivate_debug_for_a_context(self): + sheerka = self.get_sheerka() + ExecutionContext.ids.clear() + root_context = self.get_context(sheerka) + service = sheerka.services[SheerkaDebugManager.NAME] + + sheerka.set_debug(root_context, True) + sheerka.set_explicit(root_context, True) + sheerka.activate_debug_for(root_context, 1) + sheerka.deactivate_debug_for(root_context, 1) + + context = root_context.push(BuiltinConcepts.NOP, None) + + assert context.id == 1 + assert 1 not in service.context_cache + assert not context.debug_enabled + + def test_i_can_deactivate_context_but_not_children(self): + sheerka = self.get_sheerka() + ExecutionContext.ids.clear() + root_context = self.get_context(sheerka) + service = sheerka.services[SheerkaDebugManager.NAME] + + sheerka.set_debug(root_context, True) + sheerka.set_explicit(root_context, True) + sheerka.activate_debug_for(root_context, 1, True) + sheerka.deactivate_debug_for(root_context, 1) + + context = root_context.push(BuiltinConcepts.NOP, None) # sub_context.id = 1 + sub_context = context.push(BuiltinConcepts.NOP, None) # sub_sub_context.parent = 1 + sub_context2 = context.push(BuiltinConcepts.NOP, None) # sub_sub_context2.parent = 1 + sub_sub_context = sub_context.push(BuiltinConcepts.NOP, None) # is a child + + assert context.id == 1 + assert 1 not in service.context_cache + assert "1+" in service.context_cache + assert not context.debug_enabled + assert sub_context.debug_enabled + assert sub_context2.debug_enabled + assert sub_sub_context.debug_enabled + + def test_i_can_deactivate_context_and_children(self): + sheerka = self.get_sheerka() + ExecutionContext.ids.clear() + root_context = self.get_context(sheerka) + service = sheerka.services[SheerkaDebugManager.NAME] + + sheerka.set_debug(root_context, True) + sheerka.set_explicit(root_context, True) + sheerka.activate_debug_for(root_context, 1) + sheerka.deactivate_debug_for(root_context, 1, children=True) + + context = root_context.push(BuiltinConcepts.NOP, None) # sub_context.id = 1 + sub_context = context.push(BuiltinConcepts.NOP, None) # sub_sub_context.parent = 1 + sub_context2 = context.push(BuiltinConcepts.NOP, None) # sub_sub_context2.parent = 1 + sub_sub_context = sub_context.push(BuiltinConcepts.NOP, None) # is a child + + assert 1 not in service.context_cache + assert "1+" not in service.context_cache + assert not context.debug_enabled + assert not sub_context.debug_enabled + assert not sub_context2.debug_enabled + assert not sub_sub_context.debug_enabled + + def test_i_can_activate_debug_for_a_variable(self): + sheerka, context = self.init_concepts() + service = sheerka.services[SheerkaDebugManager.NAME] + sheerka.set_debug(context) + + sheerka.activate_debug_for(context, "Out") + assert "Out" in service.variable_cache + assert sheerka.debug_activated_for("Out") + + sheerka.deactivate_debug_for(context, "Out") + assert "Out" not in service.variable_cache + assert not sheerka.debug_activated_for("Out") + + def test_i_can_activate_debug_for_sub_children_using_the_simplified_form(self): + sheerka = self.get_sheerka() + ExecutionContext.ids.clear() + root_context = self.get_context(sheerka) + service = sheerka.services[SheerkaDebugManager.NAME] + + sheerka.set_debug(root_context, True) + sheerka.set_explicit(root_context, True) + sheerka.activate_debug_for(root_context, "1+") + + context = root_context.push(BuiltinConcepts.NOP, None) # sub_context.id = 1 + sub_context = context.push(BuiltinConcepts.NOP, None) # sub_sub_context.parent = 1 + sub_context2 = context.push(BuiltinConcepts.NOP, None) # sub_sub_context2.parent = 1 + sub_sub_context = sub_context.push(BuiltinConcepts.NOP, None) # is a child + + assert context.id == 1 + assert 1 in service.context_cache + assert "1+" in service.context_cache + assert context.debug_enabled + assert sub_context.debug_enabled + assert sub_context2.debug_enabled + assert sub_sub_context.debug_enabled + assert not root_context.debug_enabled + + @pytest.mark.parametrize("settings, expected", [ + ({"service": "my_service"}, True), + ({"service": "other_service"}, False), + ({"method": "my_method"}, True), + ({"method": "other_method"}, False), + ]) + def test_i_can_compute_debug(self, settings, expected): + sheerka, context = self.init_concepts() + service = sheerka.services[SheerkaDebugManager.NAME] + service.set_debug(context, True) + + service.debug_var(context, **settings) + assert service.compute_debug("my_service", "my_method", context) == expected + + def test_i_can_compute_debug_for_context(self): + sheerka, root_context = self.init_concepts() + service = sheerka.services[SheerkaDebugManager.NAME] + service.set_debug(root_context, True) + + context1 = root_context.push(BuiltinConcepts.TESTING, None) + context2 = root_context.push(BuiltinConcepts.TESTING, None) + service.debug_var(root_context, context_id=context1.id) + + assert service.compute_debug("my_service", "my_method", context1) + assert not service.compute_debug("my_service", "my_method", context2) + + def test_i_can_disable_debug_setting(self): + sheerka, context = self.init_concepts() + service = sheerka.services[SheerkaDebugManager.NAME] + + service.debug_var(context, service="my_service", enabled=False) + assert not service.compute_debug("my_service", "my_method", context) + + def test_i_can_compute_combination_of_debug_settings(self): + sheerka, context = self.init_concepts() + service = sheerka.services[SheerkaDebugManager.NAME] + service.set_debug(context, True) + + service.debug_var(context, service="my_service", enabled=True) + service.debug_var(context, method="my_method", enabled=False) + service.debug_var(context, variable="xxx", enabled=False) # is not used + service.debug_var(context, debug_id=1, enabled=False) # is not used + + assert not service.compute_debug("my_service", "my_method", context) # False > True + assert service.compute_debug("my_service", "another_method", context) + assert not service.compute_debug("another_service", "my_method", context) + + def test_variable_debug_is_deactivated_by_default(self): + sheerka, context = self.init_concepts() + service = sheerka.services[SheerkaDebugManager.NAME] + service.set_debug(context, True) + + assert not service.compute_var_debug("my_service", "my_method", 0, "my_variable", 0) + + def test_i_can_activate_variable_debug_for_all_variables(self): + sheerka, context = self.init_concepts() + service = sheerka.services[SheerkaDebugManager.NAME] + service.set_debug(context, True) + + service.debug_var(context, variable="*") + + assert service.compute_var_debug("my_service", "my_method", 0, "my_variable", 0) == True + + def test_i_can_activate_variable_display_for_all_variables(self): + sheerka, context = self.init_concepts() + service = sheerka.services[SheerkaDebugManager.NAME] + service.set_debug(context, True) + + service.debug_var(context, variable="*", enabled='list') + + assert service.compute_var_debug("my_service", "my_method", 0, "my_variable", 0) == 'list' + + def test_i_can_disable_variable_debug(self): + sheerka, context = self.init_concepts() + service = sheerka.services[SheerkaDebugManager.NAME] + service.set_debug(context, True) + + service.debug_var(context, variable="my_variable", enabled=False) + assert not service.compute_var_debug("my_service", "my_method", 0, "my_variable", 0) + + @pytest.mark.parametrize("enabled_1, enabled_2, expected", [ + (False, False, False), + (False, True, False), + (False, 'list', False), + (True, False, False), + (True, True, True), + (True, 'list', 'list'), + ('list', False, False), + ('list', True, 'list'), + ('list', 'list', 'list'), + ]) + def test_i_can_compute_combination_of_debug_var_settings(self, enabled_1, enabled_2, expected): + sheerka, context = self.init_concepts() + service = sheerka.services[SheerkaDebugManager.NAME] + service.set_debug(context, True) + + service.debug_var(context, variable="my_variable", enabled=enabled_1) + service.debug_var(context, variable="*", enabled=enabled_2) + + assert service.compute_var_debug("my_service", "my_method", 0, "my_variable", 0) == expected + + @pytest.mark.parametrize("settings, expected", [ + ({"service": "my_service"}, True), + ({"service": "other_service"}, False), + ({"method": "my_method"}, True), + ({"method": "other_method"}, False), + ({"context_id": 0}, True), + ({"context_id": 1}, False), + ]) + def test_service_and_method_and_context_id_are_tested_before_disabling_a_variable(self, settings, expected): + sheerka, context = self.init_concepts() + service = sheerka.services[SheerkaDebugManager.NAME] + service.set_debug(context, True) + + settings["variable"] = "my_variable" + + service.debug_var(context, **settings, enabled=True) + assert service.compute_var_debug("my_service", "my_method", 0, "my_variable", 0) == expected + + @pytest.mark.parametrize("context_children, expected", [ + (True, True), + (False, False), + ]) + def test_i_can_compute_debug_var_for_context_children(self, context_children, expected): + sheerka, context = self.init_concepts() + service = sheerka.services[SheerkaDebugManager.NAME] + service.set_debug(context, True) + + sub_context = context.push(BuiltinConcepts.TESTING, None) + sub_sub_context = sub_context.push(BuiltinConcepts.TESTING, None) + + service.debug_var(context, context_id=sub_context.id, context_children=context_children) + assert service.compute_debug("my_service", "my_method", sub_sub_context) == expected + + def test_i_can_update_debug_setting(self): + sheerka, context = self.init_concepts() + service = sheerka.services[SheerkaDebugManager.NAME] + + service.debug_var(context, service="my_service", enabled=True) + service.debug_var(context, service="my_service", enabled=False) + + assert len(service.debug_vars_settings) == 1 + assert not service.debug_vars_settings[0].enabled + + service.debug_var(context, service="my_service", enabled=True) + assert len(service.debug_vars_settings) == 1 + assert service.debug_vars_settings[0].enabled + + def test_i_can_reset_debug_settings(self): + sheerka, context = self.init_concepts() + service = sheerka.services[SheerkaDebugManager.NAME] + service.set_debug(context, True) + + service.debug_var(context, service="my_service") + assert service.compute_debug("my_service", "my_method", context) + + service.reset_debug(context) + assert not service.compute_debug("my_service", "my_method", context) + + @pytest.mark.parametrize("settings, expected", [ + ({"rule": "1"}, True), + ({"rule": "2"}, False), + ({"context_id": 0}, True), + ({"context_id": 1}, False), + ({"debug_id": 0}, True), + ({"debug_id": 1}, False), + ]) + def test_i_can_compute_rules_debug_settings(self, settings, expected): + sheerka, context = self.init_concepts() + service = sheerka.services[SheerkaDebugManager.NAME] + service.set_debug(context, True) + + service.debug_rule(context, **settings) + assert service.compute_debug_rule("1", 0, 0) == expected + + def test_state_is_saved_and_restored(self): + sheerka = self.get_sheerka() + ExecutionContext.ids.clear() + root_context = self.get_context(sheerka) + + sheerka.set_debug(root_context, True) + sheerka.set_explicit(root_context, False) + sheerka.activate_debug_for(root_context, 1, children=True) + sheerka.activate_debug_for(root_context, "SomeVar") + sheerka.debug_rule(root_context, "1", 10, 15) + sheerka.debug_var(root_context, service="service_name") + + another_service = SheerkaDebugManager(sheerka) + another_service.initialize_deferred(root_context, True) + + assert another_service.activated + assert not another_service.explicit + assert another_service.context_cache == {1, "1+"} + assert another_service.variable_cache == {"SomeVar"} + assert another_service.debug_rules_settings == [ + DebugRuleSetting("1", 10, 15, True) + ] + assert another_service.debug_vars_settings == [ + DebugVarSetting('service_name', None, None, None, False, None, False, True)] diff --git a/tests/core/test_SheerkaEvaluateConcept.py b/tests/core/test_SheerkaEvaluateConcept.py index 5742c0d..16fe67a 100644 --- a/tests/core/test_SheerkaEvaluateConcept.py +++ b/tests/core/test_SheerkaEvaluateConcept.py @@ -1,6 +1,7 @@ import pytest from core.builtin_concepts import BuiltinConcepts, ReturnValueConcept, ParserResultConcept -from core.concept import Concept, DoNotResolve, ConceptParts, Property, InfiniteRecursionResolved, CB, NotInit +from core.concept import Concept, DoNotResolve, ConceptParts, InfiniteRecursionResolved, CB, NotInit, \ + concept_part_value from core.sheerka.services.SheerkaEvaluateConcept import SheerkaEvaluateConcept from core.sheerka.services.SheerkaMemory import SheerkaMemory from parsers.PythonParser import PythonNode @@ -11,7 +12,7 @@ from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka): @pytest.mark.parametrize("body, expected", [ - (None, BuiltinConcepts.NOT_INITIALIZED), + (None, NotInit), ("", ""), ("1", 1), ("1+1", 2), @@ -27,13 +28,13 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka): assert evaluated.key == concept.key assert evaluated.body == expected - assert evaluated.metadata.body == body - assert evaluated.metadata.pre is None - assert evaluated.metadata.post is None - assert evaluated.metadata.where is None + assert evaluated.get_metadata().body == body + assert evaluated.get_metadata().pre is None + assert evaluated.get_metadata().post is None + assert evaluated.get_metadata().where is None assert evaluated.variables() == {} - assert evaluated.metadata.is_evaluated - assert len(evaluated.values) == 0 if body is None else 1 + assert evaluated.get_metadata().is_evaluated + assert len(evaluated.values()) == 0 if body is None else 1 assert "foo" in sheerka.services[SheerkaMemory.NAME].registration @@ -59,13 +60,13 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka): evaluated = sheerka.evaluate_concept(context, concept) assert evaluated.key == concept.key - assert evaluated.metadata.body is None - assert evaluated.metadata.post == expr - assert evaluated.metadata.pre is None - assert evaluated.metadata.where is None + assert evaluated.get_metadata().body is None + assert evaluated.get_metadata().post == expr + assert evaluated.get_metadata().pre is None + assert evaluated.get_metadata().where is None assert evaluated.get_value(ConceptParts.POST) == expected assert evaluated.variables() == {} - assert not evaluated.metadata.is_evaluated + assert not evaluated.get_metadata().is_evaluated assert len(evaluated.values) == 0 if expr is None else 1 @pytest.mark.parametrize("expr, expected", [ @@ -84,12 +85,12 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka): evaluated = sheerka.evaluate_concept(context, concept) assert evaluated.key == concept.key - assert evaluated.metadata.pre is None - assert evaluated.metadata.pre is None - assert evaluated.metadata.post is None - assert evaluated.metadata.where is None - assert evaluated.variables() == {"a": Property("a", expected)} - assert evaluated.metadata.is_evaluated + assert evaluated.get_metadata().pre is None + assert evaluated.get_metadata().pre is None + assert evaluated.get_metadata().post is None + assert evaluated.get_metadata().where is None + assert evaluated.variables() == {"a": expected} + assert evaluated.get_metadata().is_evaluated def test_i_can_evaluate_when_the_body_is_the_name_of_the_concept(self): # to prove that I can distinguish from a string @@ -102,33 +103,33 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka): def test_i_can_evaluate_metadata_using_do_not_resolve(self): sheerka, context, concept = self.init_concepts(Concept("foo"), eval_body=True) - concept.compiled[ConceptParts.BODY] = DoNotResolve("do not resolve") + concept.get_compiled()[ConceptParts.BODY] = DoNotResolve("do not resolve") evaluated = sheerka.evaluate_concept(context, concept) assert evaluated.body == "do not resolve" - assert evaluated.metadata.is_evaluated + assert evaluated.get_metadata().is_evaluated def test_i_can_evaluate_variable_using_do_not_resolve(self): sheerka, context, concept = self.init_concepts(Concept("foo").def_var("a"), eval_body=True) - concept.compiled["a"] = DoNotResolve("do not resolve") + concept.get_compiled()["a"] = DoNotResolve("do not resolve") evaluated = sheerka.evaluate_concept(context, concept) assert evaluated.get_value("a") == "do not resolve" - assert evaluated.metadata.is_evaluated + assert evaluated.get_metadata().is_evaluated def test_original_value_is_overridden_when_using_do_no_resolve(self): concept = Concept("foo", body="original value").def_var("a", "original value") sheerka, context, concept = self.init_concepts(concept, eval_body=True) - concept.compiled["a"] = DoNotResolve("do not resolve") - concept.compiled[ConceptParts.BODY] = DoNotResolve("do not resolve") + concept.get_compiled()["a"] = DoNotResolve("do not resolve") + concept.get_compiled()[ConceptParts.BODY] = DoNotResolve("do not resolve") evaluated = sheerka.evaluate_concept(context, concept) assert evaluated.body == "do not resolve" assert evaluated.get_value("a") == "do not resolve" - assert evaluated.metadata.is_evaluated + assert evaluated.get_metadata().is_evaluated def test_variables_are_evaluated_before_body(self): sheerka, context, concept = self.init_concepts(Concept("foo", body="a+1").def_var("a", "10"), eval_body=True) @@ -145,9 +146,9 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka): evaluated = sheerka.evaluate_concept(context, concept) - assert evaluated == CB("foo", CB("a", BuiltinConcepts.NOT_INITIALIZED)) - assert evaluated.metadata.is_evaluated - assert evaluated.body.metadata.is_evaluated + assert evaluated == CB("foo", CB("a", NotInit)) + assert evaluated.get_metadata().is_evaluated + assert evaluated.body.get_metadata().is_evaluated def test_i_can_evaluate_when_the_referenced_concept_has_a_body(self): sheerka, context, concept_a, concept = self.init_concepts( @@ -159,8 +160,8 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka): assert evaluated.key == concept.key assert evaluated.body == CB("a", 1) - assert not concept_a.metadata.is_evaluated - assert evaluated.metadata.is_evaluated + assert not concept_a.get_metadata().is_evaluated + assert evaluated.get_metadata().is_evaluated def test_i_can_evaluate_concept_of_concept_when_the_leaf_has_a_body(self): sheerka, context, concept_a, concept_b, concept_c, concept_d = self.init_concepts( @@ -176,7 +177,7 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka): expected = CB("c", CB("b", CB("a", "a"))) assert evaluated.body == expected assert sheerka.objvalue(evaluated) == 'a' - assert evaluated.metadata.is_evaluated + assert evaluated.get_metadata().is_evaluated def test_i_can_evaluate_concept_of_concept_does_not_have_a_body(self): sheerka, context, concept_a, concept_b, concept_c, concept_d = self.init_concepts( @@ -189,10 +190,10 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka): evaluated = sheerka.evaluate_concept(context, concept_d) assert evaluated.key == concept_d.key - expected = CB("c", CB("b", CB("a", BuiltinConcepts.NOT_INITIALIZED))) + expected = CB("c", CB("b", CB("a", NotInit))) assert evaluated.body == expected - assert sheerka.objvalue(evaluated) == CB("a", BuiltinConcepts.NOT_INITIALIZED) - assert evaluated.metadata.is_evaluated + assert sheerka.objvalue(evaluated) == CB("a", NotInit) + assert evaluated.get_metadata().is_evaluated def test_i_can_evaluate_concept_when_variables_reference_others_concepts_1(self): """ @@ -273,7 +274,7 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka): sheerka, context, concept_a = self.init_concepts(Concept(name="a", body="'a'"), eval_body=True) concept = Concept("foo").def_var("a") - concept.compiled["a"] = concept_a + concept.get_compiled()["a"] = concept_a evaluated = sheerka.evaluate_concept(context, concept) assert evaluated.key == concept.key @@ -306,7 +307,7 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka): foo = Concept("foo", body="1") concept = Concept("to_eval").def_var("prop") - concept.compiled["prop"] = [foo, DoNotResolve("1")] + concept.get_compiled()["prop"] = [foo, DoNotResolve("1")] evaluated = sheerka.evaluate_concept(self.get_context(sheerka, True), concept) variables = evaluated.get_value("prop") @@ -321,13 +322,13 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka): parser_result = ParserResultConcept(parser="who", value=python_node) concept = Concept("to_eval").def_var("prop") - concept.compiled["prop"] = [ReturnValueConcept("who", True, parser_result)] + concept.get_compiled()["prop"] = [ReturnValueConcept("who", True, parser_result)] evaluated = sheerka.evaluate_concept(self.get_context(sheerka, True), concept) assert evaluated.get_value("prop") == 2 # also works when only one return value concept = Concept("to_eval").def_var("prop") - concept.compiled["prop"] = ReturnValueConcept("who", True, parser_result) + concept.get_compiled()["prop"] = ReturnValueConcept("who", True, parser_result) evaluated = sheerka.evaluate_concept(self.get_context(sheerka, True), concept) assert evaluated.get_value("prop") == 2 @@ -342,7 +343,7 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka): evaluated = sheerka.evaluate_concept(context, add_instance) assert evaluated.key == add_instance.key - assert evaluated.metadata.is_evaluated + assert evaluated.get_metadata().is_evaluated assert sheerka.objvalue(evaluated) == 3 def test_i_can_evaluate_when_body_is_a_concept_with_its_own_variables_and_different_names(self): @@ -356,7 +357,7 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka): evaluated = sheerka.evaluate_concept(context, add_instance) assert evaluated.key == add_instance.key - assert evaluated.metadata.is_evaluated + assert evaluated.get_metadata().is_evaluated assert sheerka.objvalue(evaluated) == 3 def test_i_can_evaluate_when_body_is_a_concept_with_its_own_variables_multiple_levels(self): @@ -371,7 +372,7 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka): evaluated = sheerka.evaluate_concept(context, inc_instance) assert evaluated.key == inc_instance.key - assert evaluated.metadata.is_evaluated + assert evaluated.get_metadata().is_evaluated assert sheerka.objvalue(evaluated) == 2 def test_i_can_reference_sheerka(self): @@ -445,12 +446,12 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka): assert evaluated.key == concept.init_key().key @pytest.mark.parametrize("where_clause, expected, expected_prop, expected_body", [ - ("True", True, None, BuiltinConcepts.NOT_INITIALIZED), - ("False", False, ConceptParts.WHERE, BuiltinConcepts.NOT_INITIALIZED), + ("True", True, None, NotInit), + ("False", False, ConceptParts.WHERE, NotInit), ("self < 10", False, ConceptParts.WHERE, 10), ("self < 11", True, None, 10), - ("a < 20", False, "a", BuiltinConcepts.NOT_INITIALIZED), - ("a > 19", True, None, BuiltinConcepts.NOT_INITIALIZED), + ("a < 20", False, "a", NotInit), + ("a > 19", True, None, NotInit), ("a + self > 20", True, None, 10), ("a + self < 20", False, ConceptParts.WHERE, 10), ]) @@ -602,7 +603,7 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka): ) evaluated = sheerka.evaluate_concept(context, plus) - assert evaluated.key == BuiltinConcepts.CONDITION_FAILED + assert evaluated.key == str(BuiltinConcepts.CONDITION_FAILED) assert evaluated.body == "a is an int" def test_i_can_enable_disable_where_clause_evaluation(self): @@ -728,18 +729,18 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka): one = sheerka.new("one") number2 = sheerka.new("number") - number2.compiled["one"] = one - number2.compiled[ConceptParts.BODY] = one + number2.get_compiled()["one"] = one + number2.get_compiled()[ConceptParts.BODY] = one forties = sheerka.new("forties") - forties.compiled["forty"] = sheerka.new("forty") - forties.compiled["number"] = number2 + forties.get_compiled()["forty"] = sheerka.new("forty") + forties.get_compiled()["number"] = number2 number1 = sheerka.new("number") - number1.compiled["forties"] = forties - number1.compiled[ConceptParts.BODY] = forties + number1.get_compiled()["forties"] = forties + number1.get_compiled()[ConceptParts.BODY] = forties forty_one_thousand = sheerka.new("thousand") - forty_one_thousand.compiled["number"] = number1 + forty_one_thousand.get_compiled()["number"] = number1 evaluated = sheerka.evaluate_concept(context, forty_one_thousand) @@ -758,15 +759,15 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka): assert "a" in sheerka.locals @pytest.mark.parametrize("metadata", [ - "where", - "pre", - "post", - "ret" + ConceptParts.WHERE, + ConceptParts.PRE, + ConceptParts.POST, + ConceptParts.RET ]) def test_i_cannot_evaluate_python_statement_in_where_pre_post_ret(self, metadata, capsys): sheerka, context, foo = self.init_concepts("foo") - setattr(foo.metadata, metadata, "a=10; print('10')") - foo.metadata.need_validation = True + setattr(foo.get_metadata(), concept_part_value(metadata), "a=10; print('10')") + foo.get_metadata().need_validation = True evaluated = sheerka.evaluate_concept(context, foo, eval_body=True) captured = capsys.readouterr() @@ -774,7 +775,7 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka): assert sheerka.isinstance(evaluated, BuiltinConcepts.CONCEPT_EVAL_ERROR) error = evaluated.body assert sheerka.isinstance(error, BuiltinConcepts.PYTHON_SECURITY_ERROR) - assert error.prop.value == metadata + assert error.prop == metadata assert error.body == "a=10; print('10')" assert captured.out == "" @@ -813,13 +814,14 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka): @pytest.mark.parametrize("concept, eval_body, expected", [ (Concept("foo"), False, []), - (Concept("foo", pre="pre", post="post", ret="ret", where="where"), False, ["pre", "post"]), - (Concept("foo", pre="pr", post="p", ret="r", where="w"), True, ["pre", "ret", "post", "variables", "body"]), - (Concept("foo", pre="a").def_var("a"), False, ["variables", "pre"]), - (Concept("foo", pre="self"), False, ["body", "pre"]), - (Concept("foo", pre="self + a").def_var("a"), False, ["variables", "body", "pre"]), - (Concept("foo", pre="self + a", ret="ret").def_var("a"), False, ["variables", "body", "pre"]), - (Concept("foo", pre="self + a", ret="ret").def_var("a"), True, ["variables", "body", "pre", "ret"]), + (Concept("foo", pre="pre", post="post", ret="ret", where="where"), False, ["#pre#", "#post#"]), + (Concept("foo", pre="pr", post="p", ret="r", where="w"), True, + ["#pre#", "#ret#", "#post#", "variables", "#body#"]), + (Concept("foo", pre="a").def_var("a"), False, ["variables", "#pre#"]), + (Concept("foo", pre="self"), False, ["#body#", "#pre#"]), + (Concept("foo", pre="self + a").def_var("a"), False, ["variables", "#body#", "#pre#"]), + (Concept("foo", pre="self + a", ret="ret").def_var("a"), False, ["variables", "#body#", "#pre#"]), + (Concept("foo", pre="self + a", ret="ret").def_var("a"), True, ["variables", "#body#", "#pre#", "#ret#"]), (Concept("foo", body="body"), False, []) ]) def test_i_can_compute_metadata_to_eval(self, concept, eval_body, expected): @@ -839,27 +841,27 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka): context = self.get_context(sheerka, eval_where=True) concept = Concept("foo", where="where") service.initialize_concept_asts(context, concept) - assert service.compute_metadata_to_eval(context, concept) == ["where"] + assert service.compute_metadata_to_eval(context, concept) == ["#where#"] concept = Concept("foo", where="where a").def_var("a") service.initialize_concept_asts(context, concept) - assert service.compute_metadata_to_eval(context, concept) == ["variables", "where"] + assert service.compute_metadata_to_eval(context, concept) == ["variables", "#where#"] concept = Concept("foo", where="where self") service.initialize_concept_asts(context, concept) - assert service.compute_metadata_to_eval(context, concept) == ["body", "where"] + assert service.compute_metadata_to_eval(context, concept) == ["#body#", "#where#"] context = self.get_context(sheerka, eval_body=True) concept = Concept("foo") - assert service.compute_metadata_to_eval(context, concept) == ["variables", "body"] + assert service.compute_metadata_to_eval(context, concept) == ["variables", "#body#"] context = self.get_context(sheerka, eval_body=True) concept = Concept("foo").def_var("a") - assert service.compute_metadata_to_eval(context, concept) == ["variables", "body"] + assert service.compute_metadata_to_eval(context, concept) == ["variables", "#body#"] context = self.get_context(sheerka, eval_body=True) concept = Concept("foo", body="body").def_var("a") - assert service.compute_metadata_to_eval(context, concept) == ["variables", "body"] + assert service.compute_metadata_to_eval(context, concept) == ["variables", "#body#"] @pytest.mark.parametrize("concept, expected", [ (Concept("foo"), True), @@ -870,7 +872,7 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka): evaluated = sheerka.evaluate_concept(context, concept, eval_body=True) assert evaluated.key == concept.key - assert concept.metadata.is_evaluated == expected + assert concept.get_metadata().is_evaluated == expected def test_i_only_compute_the_requested_metadata(self): sheerka, context, concept = self.init_concepts( @@ -879,8 +881,8 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka): context.protected_hints.add(BuiltinConcepts.EVAL_WHERE_REQUESTED) # to prove that we do not care context.protected_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED) # to prove that we do not care - evaluated = sheerka.evaluate_concept(context, concept, metadata=['pre']) - assert evaluated.values == {"a": Property("a", NotInit), ConceptParts.PRE: Property(ConceptParts.PRE, 'pre')} + evaluated = sheerka.evaluate_concept(context, concept, metadata=[ConceptParts.PRE]) + assert evaluated.values() == {"a": NotInit, ConceptParts.PRE: 'pre'} def test_i_can_manage_ret(self): sheerka, context, foo, bar = self.init_concepts("foo", Concept("bar", ret="foo")) @@ -897,14 +899,20 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka): res = sheerka.evaluate_concept(context, bar, eval_body=False) assert res.id == bar.id + def test_i_can_eval_concept_with_rules(self): + sheerka, context, foo = self.init_concepts(Concept("foo a", body="a.name").def_var("a", "r:|1:")) + + res = sheerka.evaluate_concept(context, foo, eval_body=True) + assert res.body == "Print return values" + # I cannot implement value cache for now # def test_values_when_no_variables_are_computed_only_once(self): # sheerka, context, foo = self.init_concepts(Concept("foo", body="10")) # # evaluated = sheerka.evaluate_concept(context, sheerka.new("foo"), eval_body=True) # assert evaluated.body == 10 - # assert len(evaluated.compiled) > 0 + # assert len(evaluated.get_compiled()) > 0 # # evaluated_2 = sheerka.evaluate_concept(context, sheerka.new("foo"), eval_body=True) # assert evaluated_2.body == 10 - # assert len(evaluated_2.compiled) == 0 + # assert len(evaluated_2.get_compiled()) == 0 diff --git a/tests/core/test_SheerkaEvaluateRules.py b/tests/core/test_SheerkaEvaluateRules.py new file mode 100644 index 0000000..4af82de --- /dev/null +++ b/tests/core/test_SheerkaEvaluateRules.py @@ -0,0 +1,74 @@ +import operator + +from core.concept import Concept +from core.rule import Rule +from core.sheerka.services.SheerkaEvaluateRules import SheerkaEvaluateRules, LOW_PRIORITY_RULES, DISABLED_RULES + +from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka + + +class TestSheerkaEvaluateRules(TestUsingMemoryBasedSheerka): + def test_i_can_evaluate_python_rules(self): + sheerka, context, r1, r2, r3, r4, r5, r6, r7, r8, r9 = self.init_format_rules( + Rule(predicate="a == 1", action="", priority=1), # r1 + Rule(predicate="a == 2", action="", priority=1), # r2 + Rule(predicate="a == 3", action="", priority=0), # r3 + Rule(predicate="a == 4", action="", priority=0), # r4 + Rule(predicate="a == 5", action="", priority=0), # r5 + Rule(predicate="a == 1", action="", priority=1), # r6 + Rule(predicate="a == 7", action="", priority=1, is_enabled=False), # r7 + Rule(predicate="a == 8", action="", priority=1), # r8 + Rule(predicate="a == 9", action="", priority=2), # r9 + ) + service = sheerka.services[SheerkaEvaluateRules.NAME] + rules = sorted([r1, r2, r3, r4, r5, r6, r7, r8, r9], key=operator.attrgetter('priority'), reverse=True) + + res = service.evaluate_rules(context, rules, {"a": 1}, set()) + + assert res == { + True: [r1, r6], + False: [r9, r2, r8], + LOW_PRIORITY_RULES: [r3, r4, r5], + DISABLED_RULES: [r7] + } + + def test_i_can_evaluate_concept_rules(self): + sheerka, context, r1, r2, r3, r4, r5, r6, r7, r8, r9 = self.init_format_rules( + Rule(predicate="a equals 1", action="", priority=1), # r1 + Rule(predicate="a equals 2", action="", priority=1), # r2 + Rule(predicate="a equals 3", action="", priority=0), # r3 + Rule(predicate="a equals 4", action="", priority=0), # r4 + Rule(predicate="a equals 5", action="", priority=0), # r5 + Rule(predicate="a equals 1", action="", priority=1), # r6 + Rule(predicate="a equals 7", action="", priority=1, is_enabled=False), # r7 + Rule(predicate="a equals 8", action="", priority=1), # r8 + Rule(predicate="a equals 9", action="", priority=2), # r9 + concepts=[Concept("x equals y", body="x == y").def_var("x").def_var("y")], + ) + + service = sheerka.services[SheerkaEvaluateRules.NAME] + rules = sorted([r1, r2, r3, r4, r5, r6, r7, r8, r9], key=operator.attrgetter('priority'), reverse=True) + + res = service.evaluate_rules(context, rules, {"a": 1}, set()) + + assert res == { + True: [r1, r6], + False: [r9, r2, r8], + LOW_PRIORITY_RULES: [r3, r4, r5], + DISABLED_RULES: [r7] + } + + def test_i_can_disable_rules_at_runtime(self): + sheerka, context, r1, r2, = self.init_format_rules( + Rule(predicate="a == 1", action="", priority=2), # r1 + Rule(predicate="a == 1", action="", priority=1), # r2 + ) + service = sheerka.services[SheerkaEvaluateRules.NAME] + rules = sorted([r1, r2], key=operator.attrgetter('priority'), reverse=True) + + res = service.evaluate_rules(context, rules, {"a": 1}, {r1.id}) + + assert res == { + True: [r2], + DISABLED_RULES: [r1] + } diff --git a/tests/core/test_SheerkaEventManager.py b/tests/core/test_SheerkaEventManager.py new file mode 100644 index 0000000..ca670f0 --- /dev/null +++ b/tests/core/test_SheerkaEventManager.py @@ -0,0 +1,66 @@ +from core.sheerka.services.SheerkaEventManager import SheerkaEventManager + +from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka + + +def example_of_function(context): + print(f"example_of_class_method. event={context.event.get_digest()}") + + +def example_of_function_with_data(context, data): + print(f"example_of_class_method. event={context.event.get_digest()}, {data=}") + + +class TestSheerkaEventManager(TestUsingMemoryBasedSheerka): + + def example_of_class_method(self, context): + print(f"example_of_class_method. event={context.event.get_digest()}") + + @staticmethod + def example_of_static_method(context): + print(f"example_of_static_method. event={context.event.get_digest()}") + + def example_of_class_method_with_data(self, context, data): + print(f"example_of_class_method. event={context.event.get_digest()}, {data=}") + + @staticmethod + def example_of_static_method_with_data(context, data): + print(f"example_of_static_method. event={context.event.get_digest()}, {data=}") + + def test_i_can_subscribe_and_publish(self, capsys): + sheerka, context = self.init_concepts() + topic = "my topic" + + sheerka.subscribe(topic, self.example_of_class_method) + sheerka.subscribe(topic, self.example_of_static_method) + sheerka.subscribe(topic, example_of_function) + + sheerka.publish(context, topic) + + captured = capsys.readouterr() + assert captured.out == """example_of_class_method. event=xxx +example_of_static_method. event=xxx +example_of_class_method. event=xxx +""" + + service = sheerka.services[SheerkaEventManager.NAME] + service.reset_topic(topic) + + def test_i_can_subscribe_and_publish_with_data(self, capsys): + sheerka, context = self.init_concepts() + topic = "my topic" + + sheerka.subscribe(topic, self.example_of_class_method_with_data) + sheerka.subscribe(topic, self.example_of_static_method_with_data) + sheerka.subscribe(topic, example_of_function_with_data) + + sheerka.publish(context, topic, "42") + + captured = capsys.readouterr() + assert captured.out == """example_of_class_method. event=xxx, data='42' +example_of_static_method. event=xxx, data='42' +example_of_class_method. event=xxx, data='42' +""" + + service = sheerka.services[SheerkaEventManager.NAME] + service.reset_topic(topic) diff --git a/tests/core/test_SheerkaHistoryManager.py b/tests/core/test_SheerkaHistoryManager.py index 90e8878..feea883 100644 --- a/tests/core/test_SheerkaHistoryManager.py +++ b/tests/core/test_SheerkaHistoryManager.py @@ -32,7 +32,8 @@ class TestSheerkaHistoryManager(TestUsingMemoryBasedSheerka): hist("xxx", False), hist("one", True), hist("def concept one as 1", True), - hist("Initializing Sheerka.", None)] + hist("Initializing Sheerka.", None) + ] h = list(sheerka.history(2)) assert h == [ diff --git a/tests/core/test_SheerkaMemory.py b/tests/core/test_SheerkaMemory.py index 7d08b38..b8695c9 100644 --- a/tests/core/test_SheerkaMemory.py +++ b/tests/core/test_SheerkaMemory.py @@ -15,7 +15,7 @@ class TestSheerkaMemory(TestUsingMemoryBasedSheerka): foo = Concept("foo") sheerka.add_to_short_term_memory(None, "a", foo) - assert service.short_term_objects.copy() == {":a": foo} + assert service.short_term_objects.copy() == {'global': {'a': foo}} assert id(sheerka.get_from_short_term_memory(None, "a")) == id(foo) def test_i_can_add_context_short_term_memory(self): @@ -26,23 +26,36 @@ class TestSheerkaMemory(TestUsingMemoryBasedSheerka): sheerka.add_to_short_term_memory(context, "a", foo) context_id = ExecutionContext.ids[context.event.get_digest()] - assert service.short_term_objects.copy() == {f"{context_id}:a": foo} + assert service.short_term_objects.copy() == {context_id: {'a': foo}} assert id(sheerka.get_from_short_term_memory(context, "a")) == id(foo) assert sheerka.get_from_short_term_memory(None, "a") is None + def test_i_can_add_many(self): + sheerka, context = self.init_concepts() + bag = {"a": "foo", "b": "bar", } + context_id = ExecutionContext.ids[context.event.get_digest()] + service = sheerka.services[SheerkaMemory.NAME] + + sheerka.add_many_to_short_term_memory(context, bag) + assert service.short_term_objects.copy() == {context_id: bag} + def test_i_can_get_obj_from_parents(self): sheerka, context = self.init_concepts() - service = sheerka.services[SheerkaMemory.NAME] foo = Concept("foo") + sheerka.add_to_short_term_memory(None, "a", foo) + sheerka.add_to_short_term_memory(context, "b", foo) with context.push(BuiltinConcepts.TESTING, None) as sub_context: - assert service.short_term_objects.copy() == {":a": foo} assert id(sheerka.get_from_short_term_memory(sub_context, "a")) == id(foo) assert id(sheerka.get_from_short_term_memory(context, "a")) == id(foo) assert id(sheerka.get_from_short_term_memory(None, "a")) == id(foo) - def test_entry_are_removed_on_context_exit(self): + assert id(sheerka.get_from_short_term_memory(sub_context, "b")) == id(foo) + assert id(sheerka.get_from_short_term_memory(context, "b")) == id(foo) + assert sheerka.get_from_short_term_memory(None, "b") is None + + def test_short_term_memory_entries_are_removed_on_context_exit(self): sheerka, context = self.init_concepts() with context.push(BuiltinConcepts.TESTING, None) as sub_context: @@ -52,6 +65,17 @@ class TestSheerkaMemory(TestUsingMemoryBasedSheerka): assert sheerka.get_from_short_term_memory(sub_context, "a") is None + def test_short_term_memory_entries_are_removed_on_context_exit_2(self): + # this time we test the bulk insert + sheerka, context = self.init_concepts() + + with context.push(BuiltinConcepts.TESTING, None) as sub_context: + foo = Concept("foo") + sheerka.add_many_to_short_term_memory(sub_context, {"a": foo}) + assert id(sheerka.get_from_short_term_memory(sub_context, "a")) == id(foo) + + assert sheerka.get_from_short_term_memory(sub_context, "a") is None + def test_i_can_add_and_retrieve_from_memory(self): sheerka, context = self.init_concepts() service = sheerka.services[SheerkaMemory.NAME] diff --git a/tests/core/test_SheerkaModifyConcept.py b/tests/core/test_SheerkaModifyConcept.py index 403bbb4..67062e4 100644 --- a/tests/core/test_SheerkaModifyConcept.py +++ b/tests/core/test_SheerkaModifyConcept.py @@ -1,5 +1,5 @@ from core.builtin_concepts import BuiltinConcepts -from core.concept import Concept, ConceptParts +from core.concept import Concept, ConceptParts, get_concept_attrs from tests.TestUsingFileBasedSheerka import TestUsingFileBasedSheerka from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka @@ -10,8 +10,10 @@ class TestSheerkaModifyConcept(TestUsingMemoryBasedSheerka): def test_i_can_modify_a_concept(self): sheerka, context, foo, bar = self.init_concepts("foo", "bar", create_new=True, cache_only=False) + assert get_concept_attrs(foo) == [] + foo_instance = sheerka.new("foo") - foo_instance.metadata.body = "metadata value" # modify metadata + foo_instance.get_metadata().body = "metadata value" # modify metadata foo_instance.def_var("var_name", "default value") # modify definition of variables foo_instance.add_prop(BuiltinConcepts.ISA, bar) # modify property foo_instance.set_value(ConceptParts.BODY, "body value") # modify value @@ -20,30 +22,31 @@ class TestSheerkaModifyConcept(TestUsingMemoryBasedSheerka): assert res.status assert sheerka.isinstance(res.body, BuiltinConcepts.NEW_CONCEPT) - assert res.body.body.metadata.body == "metadata value" - assert res.body.body.metadata.variables == [("var_name", "default value")] + assert res.body.body.get_metadata().body == "metadata value" + assert res.body.body.get_metadata().variables == [("var_name", "default value")] assert res.body.body.get_prop(BuiltinConcepts.ISA) == {bar} assert res.body.body.body == "body value" assert res.body.body.get_value("var_name") == "var value" + assert get_concept_attrs(foo) == ["var_name"] # test that object foo_from_sheerka = sheerka.get_by_key("foo") - assert foo_from_sheerka.metadata.body == "metadata value" - assert foo_from_sheerka.metadata.variables == [("var_name", "default value")] + assert foo_from_sheerka.get_metadata().body == "metadata value" + assert foo_from_sheerka.get_metadata().variables == [("var_name", "default value")] assert foo_from_sheerka.get_prop(BuiltinConcepts.ISA) == {bar} assert foo_from_sheerka.body == "body value" assert foo_from_sheerka.get_value("var_name") == "var value" # other caches are also updated - assert sheerka.get_by_id(foo.id).metadata.body == "metadata value" - assert sheerka.get_by_name(foo.name).metadata.body == "metadata value" - assert sheerka.get_by_hash(foo_instance.get_definition_hash()).metadata.body == "metadata value" + assert sheerka.get_by_id(foo.id).get_metadata().body == "metadata value" + assert sheerka.get_by_name(foo.name).get_metadata().body == "metadata value" + assert sheerka.get_by_hash(foo_instance.get_definition_hash()).get_metadata().body == "metadata value" # sdp can be updated sheerka.cache_manager.commit(context) from_sdp = sheerka.sdp.get(sheerka.CONCEPTS_BY_ID_ENTRY, foo.id) - assert from_sdp.metadata.body == "metadata value" - assert from_sdp.metadata.variables == [("var_name", "default value")] + assert from_sdp.get_metadata().body == "metadata value" + assert from_sdp.get_metadata().variables == [("var_name", "default value")] assert from_sdp.get_prop(BuiltinConcepts.ISA) == {bar} assert from_sdp.body == "body value" assert from_sdp.get_value("var_name") == "var value" @@ -81,16 +84,16 @@ class TestSheerkaModifyConcept(TestUsingMemoryBasedSheerka): Concept("foo", body="2"), create_new=True) foo2_instance = sheerka.new("foo")[1] - foo2_instance.metadata.body = "value" + foo2_instance.get_metadata().body = "value" res = sheerka.modify_concept(context, foo2_instance) assert res.status assert sheerka.isinstance(res.body, BuiltinConcepts.NEW_CONCEPT) - assert res.body.body.metadata.body == "value" + assert res.body.body.get_metadata().body == "value" foo_from_sheerka = sheerka.new("foo") - assert foo_from_sheerka[0].metadata.body == "1" - assert foo_from_sheerka[1].metadata.body == "value" + assert foo_from_sheerka[0].get_metadata().body == "1" + assert foo_from_sheerka[1].get_metadata().body == "value" def test_i_can_get_and_set_attribute(self): sheerka, context = self.init_concepts() diff --git a/tests/core/test_SheerkaRuleManager.py b/tests/core/test_SheerkaRuleManager.py new file mode 100644 index 0000000..152df45 --- /dev/null +++ b/tests/core/test_SheerkaRuleManager.py @@ -0,0 +1,355 @@ +import ast + +import pytest +from core.builtin_concepts import BuiltinConcepts +from core.concept import Concept +from core.global_symbols import RULE_COMPARISON_CONTEXT +from core.rule import Rule +from core.sheerka.services.SheerkaRuleManager import SheerkaRuleManager, FormatRuleParser, \ + FormatAstRawText, FormatAstVariable, FormatAstSequence, FormatAstFunction, \ + FormatRuleSyntaxError, FormatAstList, UnexpectedEof, FormatAstColor, RulePredicate +from core.tokenizer import Token, TokenKind +from parsers.BaseNodeParser import SourceCodeWithConceptNode, SourceCodeNode +from parsers.PythonParser import PythonNode + +from tests.TestUsingFileBasedSheerka import TestUsingFileBasedSheerka +from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka + +seq = FormatAstSequence +raw = FormatAstRawText +var = FormatAstVariable +func = FormatAstFunction +lst = FormatAstList + +PYTHON_EVALUATOR_NAME = "Python" +CONCEPT_EVALUATOR_NAME = "Concept" + + +class TestSheerkaRuleManager(TestUsingMemoryBasedSheerka): + + @pytest.mark.parametrize("action_type, cache_entry", [ + ("print", SheerkaRuleManager.FORMAT_RULE_ENTRY), + ("exec", SheerkaRuleManager.EXEC_RULE_ENTRY), + ]) + def test_i_can_create_a_new_rule(self, action_type, cache_entry): + sheerka, context = self.init_concepts(cache_only=False) + previous_rules_number = sheerka.cache_manager.caches[sheerka.CONCEPTS_KEYS_ENTRY].cache.copy()[ + SheerkaRuleManager.RULE_IDS] + + rule = Rule(action_type, "name", "True", "Hello world") + + res = sheerka.create_new_rule(context, rule) + expected_id = str(previous_rules_number + 1) + + assert res.status + assert sheerka.isinstance(res.body, BuiltinConcepts.NEW_RULE) + + created_rule = res.body.body + assert created_rule.metadata.id == expected_id + assert created_rule.metadata.name == "name" + assert created_rule.metadata.predicate == "True" + assert created_rule.metadata.action_type == action_type + assert created_rule.metadata.action == "Hello world" + + # saved in cache + assert len(sheerka.cache_manager.caches[cache_entry].cache) > 0 + from_cache = sheerka.cache_manager.get(cache_entry, expected_id) + assert from_cache.metadata.id == expected_id + assert from_cache.metadata.name == "name" + assert from_cache.metadata.predicate == "True" + assert from_cache.metadata.action_type == action_type + assert from_cache.metadata.action == "Hello world" + + sheerka.cache_manager.commit(context) + + # saved in sdp + from_sdp = sheerka.sdp.get(cache_entry, expected_id) + assert from_sdp.metadata.id == expected_id + assert from_sdp.metadata.name == "name" + assert from_sdp.metadata.predicate == "True" + assert from_sdp.metadata.action_type == action_type + assert from_sdp.metadata.action == "Hello world" + + def test_i_can_create_multiple_rules(self): + sheerka, context = self.init_concepts(cache_only=False) + previous_rules_number = len(sheerka.cache_manager.caches[SheerkaRuleManager.FORMAT_RULE_ENTRY].cache) + + sheerka.create_new_rule(context, Rule("print", "name1", "True", "Hello world")) + sheerka.create_new_rule(context, Rule("print", "name2", "value() is __EXPLANATION", "list(value())")) + + assert len( + sheerka.cache_manager.caches[SheerkaRuleManager.FORMAT_RULE_ENTRY].cache) == 2 + previous_rules_number + + @pytest.mark.parametrize("text, expected", [ + ("", FormatAstRawText("")), + (" ", FormatAstRawText(" ")), + (" raw text ", FormatAstRawText(" raw text ")), + ("{variable}", FormatAstVariable("variable")), + ("{ variable }", FormatAstVariable("variable")), + (" xy {v} z", seq([raw(" xy "), var("v"), raw(" z")])), + (r"\{variable}", FormatAstRawText("{variable}")), + (r"\\{variable}", seq([raw("\\"), var("variable")])), + (r"\\\{variable}", FormatAstRawText(r"\{variable}")), + (r"{var1}{var2}", seq([var("var1"), var("var2")])), + ("func()", FormatAstFunction("func", [], {})), + ("func(a, 'string value', c)", FormatAstFunction("func", ["a", "'string value'", "c"], {})), + ("func(a=10, b='string value')", FormatAstFunction("func", [], {"a": "10", "b": "'string value'"})), + ("func('string value'='another string value')", func("func", [], {"'string value'": "'another string value'"})), + ("red(' xy {v}')", FormatAstColor("red", seq([raw(" xy "), var("v")]))), + ('blue(" xy {v}")', FormatAstColor("blue", seq([raw(" xy "), var("v")]))), + ('green( xy )', FormatAstColor("green", var("xy"))), + ('green()', FormatAstColor("green", raw(""))), + ('green("")', FormatAstColor("green", raw(""))), + ("list(var_name, 2, 'children')", FormatAstList("var_name", recurse_on="children", recursion_depth=2)), + ("list(var_name, recursion_depth=2, recurse_on='children')", FormatAstList("var_name", + recurse_on="children", + recursion_depth=2)), + ("list(var_name, recursion_depth=2, 'children')", FormatAstList("var_name", recursion_depth=2)), + ("list(var_name, 'children', recursion_depth=2)", FormatAstList("var_name", recursion_depth=2)), + ("list(var_name)", FormatAstList("var_name")), + ("{obj.prop1.prop2[0].prop3['value']}", FormatAstVariable("obj.prop1.prop2[0].prop3['value']")), + ("[{id}]", seq([raw("["), var("id"), raw("]")])), + ("{variable:format}", FormatAstVariable("variable", "format")), + ("{variable:3}", FormatAstVariable("variable", "3")), + (r"\not_a_function(a={var})", seq([raw("not_a_function(a="), var("var"), raw(")")])), + ]) + def test_i_can_parse_format_rule(self, text, expected): + assert FormatRuleParser(text).parse() == expected + + @pytest.mark.parametrize("text, expected_error", [ + ("{", UnexpectedEof("while parsing variable", Token(TokenKind.LBRACE, "{", 0, 1, 1))), + ("{var_name", UnexpectedEof("while parsing variable", Token(TokenKind.LBRACE, "{", 0, 1, 1))), + ("{}", FormatRuleSyntaxError("variable name not found", None)), + ("func(", UnexpectedEof("while parsing function", Token(TokenKind.IDENTIFIER, "func", 0, 1, 1))), + ("func(a,b,c", UnexpectedEof("while parsing function", Token(TokenKind.IDENTIFIER, "func", 0, 1, 1))), + ("func(a,,c", FormatRuleSyntaxError("no parameter found", Token(TokenKind.COMMA, ",", 7, 1, 8))), + ("func(a,,c)", FormatRuleSyntaxError("no parameter found", Token(TokenKind.COMMA, ",", 7, 1, 8))), + ("red(a,b)", FormatRuleSyntaxError("only one parameter supported", Token(TokenKind.IDENTIFIER, "b", 6, 1, 7))), + ("red(a=b)", FormatRuleSyntaxError("keyword arguments are not supported", None)), + ("red(xy {v})", FormatRuleSyntaxError("Invalid identifier", None)), + ("list()", FormatRuleSyntaxError("variable name not found", None)), + ("list(recursion_depth=2)", FormatRuleSyntaxError("variable name not found", None)), + ("list(a,b,c,d)", FormatRuleSyntaxError("too many positional arguments", + Token(TokenKind.IDENTIFIER, "d", 11, 1, 12))), + ("list(a, recursion_depth=hello)", FormatRuleSyntaxError("'hello' is not numeric", None)), + ("list(a, recursion_depth='hello')", FormatRuleSyntaxError("'recursion_depth' must be an integer", None)), + ]) + def test_i_cannot_parse_invalid_format(self, text, expected_error): + parser = FormatRuleParser(text) + parser.parse() + + assert parser.error_sink == expected_error + + @pytest.mark.parametrize("text", [ + "a == 5", + "foo == 5", + "func() == 5", + ]) + def test_i_can_compile_predicate_when_pure_python(self, text): + sheerka, context, *concepts = self.init_concepts("foo") + service = sheerka.services[SheerkaRuleManager.NAME] + ast_ = ast.parse(text, "", 'eval') + expected_python_node = PythonNode(text, ast_) + + res = service.compile_when(context, "test", text) + + assert len(res) == 1 + assert isinstance(res[0], RulePredicate) + assert res[0].evaluator == PYTHON_EVALUATOR_NAME + assert sheerka.isinstance(res[0].predicate, BuiltinConcepts.RETURN_VALUE) + assert sheerka.objvalue(res[0].predicate) == expected_python_node + assert res[0].concept is None + + @pytest.mark.parametrize("text, expected_type", [ + ("isinstance(a, int)", SourceCodeWithConceptNode), + ("func()", SourceCodeNode), + ]) + def test_i_can_compile_predicates_that_resolve_to_python(self, text, expected_type): + sheerka, context, *concepts = self.init_concepts() + service = sheerka.services[SheerkaRuleManager.NAME] + ast_ = ast.parse(text, "", 'eval') + expected_python_node = PythonNode(text, ast_) + + res = service.compile_when(context, "test", text) + + assert len(res) == 1 + assert isinstance(res[0], RulePredicate) + assert res[0].evaluator == PYTHON_EVALUATOR_NAME + assert sheerka.isinstance(res[0].predicate, BuiltinConcepts.RETURN_VALUE) + assert isinstance(sheerka.objvalue(res[0].predicate), expected_type) + assert sheerka.objvalue(res[0].predicate).python_node == expected_python_node + assert res[0].concept is None + + def test_i_can_compile_predicate_when_python_and_concept(self): + sheerka, context, *concepts = self.init_concepts(Concept("foo bar"), create_new=True) + service = sheerka.services[SheerkaRuleManager.NAME] + text = "foo bar == 5" + ast_ = ast.parse("__C__foo0bar__1001__C__ == 5", "", 'eval') + resolved_expected = PythonNode(text, ast_) + + res = service.compile_when(context, "test", text) + + assert len(res) == 1 + assert isinstance(res[0], RulePredicate) + assert res[0].evaluator == PYTHON_EVALUATOR_NAME + assert sheerka.isinstance(res[0].predicate, BuiltinConcepts.RETURN_VALUE) + assert sheerka.objvalue(res[0].predicate) == resolved_expected + assert res[0].concept is None + + @pytest.mark.parametrize("text, expected_variables", [ + ("cat is an animal", ["cat", "animal"]), + ("a is an animal", ["a", "animal"]), + ("cat is an b", ["cat", "b"]), + ]) + def test_i_can_compile_predicate_when_exact_concept(self, text, expected_variables): + sheerka, context, *concepts = self.init_concepts( + Concept("x is an y", pre="is_question()", body="isinstance(x, y)").def_var("x").def_var("y"), + Concept("cat"), + Concept("animal"), + create_new=True + ) + service = sheerka.services[SheerkaRuleManager.NAME] + + expected = concepts[0] + expected.get_metadata().variables = [('x', expected_variables[0]), ('y', expected_variables[1])] + + res = service.compile_when(context, "test", text) + + assert len(res) == 1 + assert isinstance(res[0], RulePredicate) + assert res[0].evaluator == CONCEPT_EVALUATOR_NAME + assert sheerka.isinstance(res[0].predicate, BuiltinConcepts.RETURN_VALUE) + assert sheerka.objvalue(res[0].predicate) == expected + assert res[0].concept == expected + + @pytest.mark.parametrize("text, expected_variables", [ + ("a cat is an animal", ["cat", "animal"]), + ("a cat is an b", ["a", "animal"]), + ]) + def test_i_can_compile_predicate_when_sya_node_parser(self, text, expected_variables): + sheerka, context, *concepts = self.init_concepts( + Concept("x is an y", pre="is_question()", body="isinstance(x, y)").def_var("x").def_var("y"), + Concept("a cat"), + Concept("animal"), + create_new=True + ) + service = sheerka.services[SheerkaRuleManager.NAME] + expected = concepts[0] + + res = service.compile_when(context, "test", text) + + assert len(res) == 1 + assert isinstance(res[0], RulePredicate) + assert res[0].evaluator == CONCEPT_EVALUATOR_NAME + assert sheerka.isinstance(res[0].predicate, BuiltinConcepts.RETURN_VALUE) + assert sheerka.objvalue(res[0].predicate)[0].concept == expected + assert res[0].concept == expected + + def test_i_can_compile_predicate_when_bnf_node_parser(self): + sheerka, context, *concepts = self.init_concepts( + Concept("animal"), + Concept("x is an y", pre="is_question()", definition="('cat'|'bird')=x 'is an' animal").def_var("x"), + create_new=True + ) + service = sheerka.services[SheerkaRuleManager.NAME] + expected = concepts[1] + + res = service.compile_when(context, "test", "cat is an animal") + + assert len(res) == 1 + assert isinstance(res[0], RulePredicate) + assert res[0].evaluator == CONCEPT_EVALUATOR_NAME + assert sheerka.isinstance(res[0].predicate, BuiltinConcepts.RETURN_VALUE) + assert sheerka.objvalue(res[0].predicate)[0].concept == expected + assert res[0].concept == expected + + def test_i_can_compile_predicate_when_multiple_choices(self): + sheerka, context, *concepts = self.init_concepts( + Concept("x is a y", pre="is_question()", body="isinstance(x, y)").def_var("x").def_var("y"), + Concept("x is a y", pre="is_question()", body="isa(x, y)").def_var("x").def_var("y"), + create_new=True + ) + service = sheerka.services[SheerkaRuleManager.NAME] + + res = service.compile_when(context, "test", "a is a b") + + assert len(res) == 2 + assert isinstance(res[0], RulePredicate) + assert res[0].evaluator == CONCEPT_EVALUATOR_NAME + assert sheerka.isinstance(res[0].predicate, BuiltinConcepts.RETURN_VALUE) + assert sheerka.objvalue(res[0].predicate)[0].concept == concepts[0] + assert res[0].concept == concepts[0] + + assert isinstance(res[1], RulePredicate) + assert res[1].evaluator == CONCEPT_EVALUATOR_NAME + assert sheerka.isinstance(res[1].predicate, BuiltinConcepts.RETURN_VALUE) + assert sheerka.objvalue(res[1].predicate)[0].concept == concepts[1] + assert res[1].concept == concepts[1] + + # @pytest.mark.skip + # @pytest.mark.parametrize("text, expected", [ + # ("cat is an animal", set()), + # ("a is an animal", {"a"}), + # ("a is an b", {"a", "b"}), + # ("cat is an b", {"b"}), + # ("a cat is an b", {"b"}), + # + # ("cat is a animal", set()), + # ("a is a animal", {"a"}), + # ("a is a b", {"a", "b"}), + # ("cat is a b", {"b"}), + # ("a cat is an b", {"b"}), + # + # ("a == 5", {"a"}), + # ("isinstance(a, int)", {"a"}), + # ("a cat == b", {"b"}) + # ]) + # def test_i_can_get_rules_variables(self, text, expected): + # sheerka, context, *concepts = self.init_concepts( + # Concept("x is a y", pre="is_question()", body="isinstance(x, y)").def_var("x").def_var("y"), + # Concept("x is a y", pre="is_question()", body="isa(x, y)").def_var("x").def_var("y"), + # Concept("x is an y", pre="is_question()", body="isinstance(x, y)").def_var("x").def_var("y"), + # Concept("cat"), + # Concept("animal"), + # Concept("a cat"), + # create_new=True + # ) + # service = sheerka.services[SheerkaRuleManager.NAME] + # + # compiled = service.compile_when(context, "test", "a is a b") + # + # assert service.get_unknown_variables(compiled) == expected + + +class TestSheerkaRuleManagerUsingFileBasedSheerka(TestUsingFileBasedSheerka): + def test_rules_are_initialized_at_startup(self): + sheerka, context, *rules = self.init_format_rules( + Rule("print", "name1", "True", "Hello world"), + Rule("print", "name2", "value() is __EXPLANATION", "list(value())") + ) + sheerka.set_is_greater_than(context, BuiltinConcepts.PRECEDENCE, + rules[0], + rules[1], + RULE_COMPARISON_CONTEXT) + + sheerka.cache_manager.commit(context) + assert len(sheerka.cache_manager.copy(SheerkaRuleManager.FORMAT_RULE_ENTRY)) == len(rules) + + sheerka = self.get_sheerka() # new instance + assert len(sheerka.cache_manager.copy(SheerkaRuleManager.FORMAT_RULE_ENTRY)) == len(rules) + + # manually update the rules (I need their new priorities) + service = sheerka.services[SheerkaRuleManager.NAME] + rules = [service.format_rule_cache.get(rule_id) for rule_id in service.format_rule_cache] + + # check if the rules are correctly initialized + rules_as_map = {rule.id: rule for rule in rules} + for rule_id in service.format_rule_cache: + actual = service.format_rule_cache.get(rule_id) + expected = rules_as_map[rule_id] + assert actual.metadata.is_compiled == expected.metadata.is_compiled + assert actual.metadata.is_enabled == expected.metadata.is_enabled + assert actual.compiled_action == expected.compiled_action + assert actual.compiled_predicate == expected.compiled_predicate + assert actual.priority is not None + assert actual.priority == expected.priority diff --git a/tests/core/test_SheerkaSetsManager.py b/tests/core/test_SheerkaSetsManager.py index 2d0a2e7..91e7480 100644 --- a/tests/core/test_SheerkaSetsManager.py +++ b/tests/core/test_SheerkaSetsManager.py @@ -268,12 +268,12 @@ class TestSheerkaSetsManager(TestUsingMemoryBasedSheerka): create_new=True ) - assert twenties.bnf.elements[1].recurse_id is None + assert twenties.get_bnf().elements[1].recurse_id is None # update number sheerka.set_isa(context, sheerka.new("one"), number) - assert twenties.bnf.elements[1].recurse_id == "1003#1002(number)" + assert twenties.get_bnf().elements[1].recurse_id == "1003#1002(number)" def test_concepts_in_group_cache_is_updated(self): sheerka, context, one, two, number = self.init_concepts("one", "two", "number") @@ -306,7 +306,8 @@ class TestSheerkaSetsManagerUsingFileBasedSheerka(TestUsingFileBasedSheerka): assert sheerka.add_concept_to_set(context, foo, group).status sheerka.cache_manager.commit(context) - sheerka = self.get_sheerka() # another session + sheerka = self.get_sheerka(reset_attrs=False) # another session + context = self.get_context(sheerka) assert sheerka.add_concept_to_set(context, bar, group).status # I can get the elements @@ -319,7 +320,8 @@ class TestSheerkaSetsManagerUsingFileBasedSheerka(TestUsingFileBasedSheerka): } # I can also add a group another elements - sheerka = self.get_sheerka() + sheerka = self.get_sheerka(reset_attrs=False) + context = self.get_context(sheerka) foo3 = Concept("foo3") foo4 = Concept("foo4") for c in [foo3, foo4]: @@ -348,19 +350,19 @@ class TestSheerkaSetsManagerUsingFileBasedSheerka(TestUsingFileBasedSheerka): # nothing was previously in ISA foo = sheerka.new(foo.key) - assert BuiltinConcepts.ISA not in foo.metadata.props + assert BuiltinConcepts.ISA not in foo.get_metadata().props res = sheerka.set_isa(context, foo, group) assert res.status sheerka.cache_manager.commit(context) - sheerka = self.get_sheerka() + sheerka = self.get_sheerka(reset_attrs=False) assert foo.get_prop(BuiltinConcepts.ISA) == {group} assert sheerka.isa(foo, group) assert sheerka.isinset(foo, group) assert sheerka.isaset(context, group) # I can do the same for bar - sheerka = self.get_sheerka() + sheerka = self.get_sheerka(reset_attrs=False) res = sheerka.set_isa(context, bar, group) assert res.status assert bar.get_prop(BuiltinConcepts.ISA) == {group} @@ -371,7 +373,7 @@ class TestSheerkaSetsManagerUsingFileBasedSheerka(TestUsingFileBasedSheerka): sheerka.cache_manager.commit(context) # they are both in the same group - sheerka = self.get_sheerka() + sheerka = self.get_sheerka(reset_attrs=False) all_entries = sheerka.sdp.get(SheerkaSetsManager.CONCEPTS_GROUPS_ENTRY) assert all_entries == { group.id: {foo.id, bar.id} diff --git a/tests/core/test_SheerkaVariableManager.py b/tests/core/test_SheerkaVariableManager.py index b5fa1e6..d92b3ae 100644 --- a/tests/core/test_SheerkaVariableManager.py +++ b/tests/core/test_SheerkaVariableManager.py @@ -1,6 +1,7 @@ from core.concept import Concept, ConceptParts from core.sheerka.services.SheerkaVariableManager import SheerkaVariableManager +from tests.TestUsingFileBasedSheerka import TestUsingFileBasedSheerka from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka @@ -9,8 +10,8 @@ class TestSheerkaVariable(TestUsingMemoryBasedSheerka): sheerka = self.get_sheerka(cache_only=False) context = self.get_context(sheerka) - sheerka.record(context, "TestSheerkaVariable", "my_variable", 1) - res = sheerka.load("TestSheerkaVariable", "my_variable") + sheerka.record_var(context, "TestSheerkaVariable", "my_variable", 1) + res = sheerka.load_var("TestSheerkaVariable", "my_variable") assert res == 1 # I can persist in db @@ -30,8 +31,8 @@ class TestSheerkaVariable(TestUsingMemoryBasedSheerka): concept = Concept("foo").set_value("a", "alpha").set_value(ConceptParts.BODY, 3.14) - sheerka.record(context, "TestSheerkaVariable", "my_variable", concept) - res = sheerka.load("TestSheerkaVariable", "my_variable") + sheerka.record_var(context, "TestSheerkaVariable", "my_variable", concept) + res = sheerka.load_var("TestSheerkaVariable", "my_variable") assert res == concept assert res.body == concept.body @@ -43,19 +44,19 @@ class TestSheerkaVariable(TestUsingMemoryBasedSheerka): concept = Concept("foo") - sheerka.record(context, "TestSheerkaVariable", "my_variable", concept) - assert sheerka.load("TestSheerkaVariable", "my_variable") is not None + sheerka.record_var(context, "TestSheerkaVariable", "my_variable", concept) + assert sheerka.load_var("TestSheerkaVariable", "my_variable") is not None - sheerka.delete(context, "TestSheerkaVariable", "my_variable") - assert sheerka.load("TestSheerkaVariable", "my_variable") is None + sheerka.delete_var(context, "TestSheerkaVariable", "my_variable") + assert sheerka.load_var("TestSheerkaVariable", "my_variable") is None def test_i_can_set_and_get_a_value(self): sheerka = self.get_sheerka(cache_only=False) context = self.get_context(sheerka) context.event.user_id = "Test_user" - sheerka.set(context, "my_variable", "my value") - res = sheerka.get(context, "my_variable") + sheerka.set_var(context, "my_variable", "my value") + res = sheerka.get_var(context, "my_variable") assert res == "my value" # I can persist in db @@ -69,13 +70,33 @@ class TestSheerkaVariable(TestUsingMemoryBasedSheerka): assert loaded.who == "Test_user" assert loaded.parents is None + +class TestSheerkaVariableUsingFileBasedSdp(TestUsingFileBasedSheerka): + + def test_i_can_bound_variables_to_sheerka(self): + sheerka, context = self.init_concepts() + + old_value = sheerka.enable_process_return_values + new_value = not old_value + sheerka.record_var(context, "TestSheerkaVariable", "sheerka.enable_process_return_values", new_value) + sheerka.cache_manager.commit(context) + + assert sheerka.enable_process_return_values == new_value + + # the modification is persisted upon new Sheerka creation + sheerka = self.get_sheerka() + assert sheerka.enable_process_return_values == new_value + + # reset old value + sheerka.enable_process_return_values = old_value + # def test_i_can_get_the_parent_when_modified(self): # sheerka = self.get_sheerka() # context = self.get_context(sheerka) # - # sheerka.record(context, "TestSheerkaVariable", "my_variable", 1) - # sheerka.record(context, "TestSheerkaVariable", "my_variable", 2) - # res = sheerka.load("TestSheerkaVariable", "my_variable") + # sheerka.record_var(context, "TestSheerkaVariable", "my_variable", 1) + # sheerka.record_var(context, "TestSheerkaVariable", "my_variable", 2) + # res = sheerka.load_var("TestSheerkaVariable", "my_variable") # assert res == 2 # # loaded = sheerka.sdp.get(SheerkaVariableManager.VARIABLES_ENTRY, "TestSheerkaVariable.my_variable") @@ -96,8 +117,8 @@ class TestSheerkaVariable(TestUsingMemoryBasedSheerka): # sheerka = self.get_sheerka(singleton=True) # context = self.get_context(sheerka) # - # sheerka.record(context, "TestSheerkaVariable", "my_variable", 1) - # sheerka.record(context, "TestSheerkaVariable", "my_variable", 1) + # sheerka.record_var(context, "TestSheerkaVariable", "my_variable", 1) + # sheerka.record_var(context, "TestSheerkaVariable", "my_variable", 1) # # loaded = sheerka.sdp.get(SheerkaVariableManager.VARIABLES_ENTRY, "TestSheerkaVariable.my_variable") # assert loaded.event_id == context.event.get_digest() diff --git a/tests/core/test_ast.py b/tests/core/test_ast.py deleted file mode 100644 index c3f29ac..0000000 --- a/tests/core/test_ast.py +++ /dev/null @@ -1,174 +0,0 @@ -import ast - -import core.ast.nodes -import pytest -from core.ast.nodes import NodeParent, GenericNodeConcept -from core.ast.visitors import ConceptNodeVisitor, UnreferencedNamesVisitor -from core.builtin_concepts import BuiltinConcepts -from core.sheerka.Sheerka import Sheerka - - -def get_sheerka(): - sheerka = Sheerka(cache_only=True) - sheerka.initialize("mem://") - - return sheerka - - -class TestNameVisitor(ConceptNodeVisitor): - """ - Test class for a basic Visitor test - """ - - def __init__(self): - self.names = [] - - def visit_Name(self, node): - self.names.append(node) - - -def test_i_can_transform_simple_ast_using_generic_node(): - source = """ -def my_function(a,b): - for i in range(b): - a = a+b - return a -""" - tree = ast.parse(source) - tree_as_concept = core.ast.nodes.python_to_concept(tree) - sheerka = get_sheerka() - - assert tree_as_concept.node_type == "Module" - assert sheerka.isinstance(tree_as_concept.get_value("body"), BuiltinConcepts.LIST) - - def_func = tree_as_concept.get_value("body").body[0] - assert sheerka.isinstance(def_func, BuiltinConcepts.GENERIC_NODE) - assert def_func.node_type == "FunctionDef" - assert def_func.parent == NodeParent(tree_as_concept, "body") - assert def_func.get_value("name") == "my_function" - - def_func_args = def_func.get_value("args") - assert sheerka.isinstance(def_func_args, BuiltinConcepts.GENERIC_NODE) - assert def_func_args.node_type == "arguments" - - def_func_args_real_args = def_func_args.get_value("args") - assert sheerka.isinstance(def_func_args_real_args, BuiltinConcepts.LIST) - assert len(def_func_args_real_args.body) == 2 - - assert sheerka.isinstance(def_func_args_real_args.body[0], BuiltinConcepts.GENERIC_NODE) - assert def_func_args_real_args.body[0].node_type == "arg" - assert def_func_args_real_args.body[0].parent == NodeParent(def_func_args, "args") - assert def_func_args_real_args.body[0].get_value("arg") == "a" - assert sheerka.isinstance(def_func_args_real_args.body[1], BuiltinConcepts.GENERIC_NODE) - assert def_func_args_real_args.body[1].node_type == "arg" - assert def_func_args_real_args.body[1].parent == NodeParent(def_func_args, "args") - assert def_func_args_real_args.body[1].get_value("arg") == "b" - - def_fun_body = def_func.get_value("body") - assert sheerka.isinstance(def_fun_body, BuiltinConcepts.LIST) - assert len(def_fun_body.body) == 2 - - def_fun_body_for = def_fun_body.body[0] - assert sheerka.isinstance(def_fun_body_for, BuiltinConcepts.GENERIC_NODE) - assert def_fun_body_for.node_type == "For" - assert def_fun_body_for.parent == NodeParent(def_func, "body") - - def_fun_body_return = def_fun_body.body[1] - assert sheerka.isinstance(def_fun_body_return, BuiltinConcepts.GENERIC_NODE) - assert def_fun_body_return.node_type == "Return" - assert def_fun_body_return.parent == NodeParent(def_func, "body") - - -def test_i_can_visit_concept_node(): - source = """ -def my_function(a,b): - for i in range(b): - a = a+b - return a - """ - - node = ast.parse(source) - concept_node = core.ast.nodes.python_to_concept(node) - - visitor = TestNameVisitor() - visitor.visit(concept_node) - - sheerka = get_sheerka() - assert sheerka.objvalue(visitor.names[0]) == "i" - assert sheerka.objvalue(visitor.names[1]) == "range" - assert sheerka.objvalue(visitor.names[2]) == "b" - assert sheerka.objvalue(visitor.names[3]) == "a" - assert sheerka.objvalue(visitor.names[4]) == "a" - assert sheerka.objvalue(visitor.names[5]) == "b" - assert sheerka.objvalue(visitor.names[6]) == "a" - - -def test_i_can_get_unreferenced_variables(): - source = """ -def my_function(a,b): - for i in range(b): - a = a+b - return a - -my_function(x,y) -""" - - sheerka = get_sheerka() - - node = ast.parse(source) - concept_node = core.ast.nodes.python_to_concept(node) - - visitor = UnreferencedNamesVisitor(sheerka) - visitor.visit(concept_node) - values = visitor.names - - assert len(visitor.names) == 2 - assert "x" in values - assert "y" in values - - -@pytest.mark.parametrize("source, expected", [ - ("a,b", ["a", "b"]), - ("isinstance(a, int)", ["a", "int"]) - -]) -def test_i_can_get_unreferenced_variables_from_simple_expressions(source, expected): - sheerka = get_sheerka() - - node = ast.parse(source) - concept_node = core.ast.nodes.python_to_concept(node) - - visitor = UnreferencedNamesVisitor(sheerka) - visitor.visit(concept_node) - - assert sorted(list(visitor.names)) == expected - - -def test_i_can_compare_NodeParent_with_tuple(): - node_parent = NodeParent(GenericNodeConcept("For", None), "target") - assert node_parent == ("For", "target") - - -def test_i_can_transform_back(): - source = """ -def my_function(a,b): - for i in range(b): - a = a + b - return a - - -my_function(x, y) - """ - - node = ast.parse(source) - concept_node = core.ast.nodes.python_to_concept(node) - - transformed_back = core.ast.nodes.concept_to_python(concept_node) - assert dump_ast(transformed_back) == dump_ast(node) - - -def dump_ast(node): - dump = ast.dump(node) - for to_remove in [", ctx=Load()", ", kind=None", ", type_ignores=[]"]: - dump = dump.replace(to_remove, "") - return dump diff --git a/tests/core/test_ast_helper.py b/tests/core/test_ast_helper.py new file mode 100644 index 0000000..5c75244 --- /dev/null +++ b/tests/core/test_ast_helper.py @@ -0,0 +1,44 @@ +import ast + +import pytest +from core.ast_helpers import UnreferencedNamesVisitor, UnreferencedVariablesVisitor + +from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka + + +class TestAstHelper(TestUsingMemoryBasedSheerka): + @pytest.mark.parametrize("source, expected", [ + ("a,b", {"a", "b"}), + ("isinstance(a, int)", {"isinstance", "a", "int"}), + ("date.today()", {"date"}), + ("test()", {"test"}), + ("sheerka.test()", {"sheerka"}), + ("for i in range(10): pass", set()) + + ]) + def test_i_can_get_unreferenced_names_from_simple_expressions(self, source, expected): + context = self.get_context() + + ast_ = ast.parse(source) + visitor = UnreferencedNamesVisitor(context) + visitor.visit(ast_) + + assert visitor.names == expected + + @pytest.mark.parametrize("source, expected", [ + ("a,b", {"a", "b"}), + ("isinstance(a, int)", {"a", "int"}), + ("date.today()", set()), + ("test()", set()), + ("sheerka.test()", set()), + ("for i in range(10): pass", set()) + + ]) + def test_i_can_get_unreferenced_variables_from_simple_expressions(self, source, expected): + context = self.get_context() + + ast_ = ast.parse(source) + visitor = UnreferencedVariablesVisitor(context) + visitor.visit(ast_) + + assert visitor.names == expected diff --git a/tests/core/test_builtin_helpers.py b/tests/core/test_builtin_helpers.py index 1e11a31..39aa035 100644 --- a/tests/core/test_builtin_helpers.py +++ b/tests/core/test_builtin_helpers.py @@ -105,44 +105,44 @@ class TestBuiltinHelpers(TestUsingMemoryBasedSheerka): assert not res.status assert res == item - @pytest.mark.parametrize("expression, vars_to_include, vars_to_exclude, expected_expr", [ - ("a == 1", [], [], []), - ("a == 1", ["a"], [], ["a == 1"]), - ("a == 1", [], ["a"], []), - ("predicate(a)", [], [], []), - ("predicate(a)", ["a"], [], ["predicate(a)"]), - ("predicate(a, b)", ["a"], [], ["predicate(a, b)"]), - ("predicate(a, b)", ["b"], [], ["predicate(a, b)"]), - ("predicate(a, b)", ["a", "b"], [], ["predicate(a, b)"]), - ("predicate(a, b)", ["a"], ["b"], []), - ("a + b == 1", [], [], []), - ("a + b == 1", ["a"], [], ["a + b == 1"]), - ("a + b == 1", ["a"], ["b"], []), - ("a + b == 1", ["b"], [], ["a + b == 1"]), - ("a + b == 1", ["a", "b"], [], ["a + b == 1"]), - ("a == 1 and b == 2", [], [], []), - ("a == 1 and b == 2", ["a"], [], ["a == 1"]), - ("a == 1 and b == 2", ["b"], [], ["b == 2"]), - ("a == 1 and b == 2", ["a"], ["b"], ["a == 1"]), - ("a == 1 and b == 2", ["a", "b"], [], ["a == 1 and b == 2"]), - ("predicate(a,c) and predicate(b,c)", ["a", "b"], [], ["predicate(a,c) and predicate(b,c)"]), - ("not(a == 1)", [], [], []), - ("not(a == 1)", ["a"], [], ["not(a==1)"]), - ("a == 1 or b == 2", [], [], []), - ("a == 1 or b == 2", ["a"], [], ["a == 1"]), - ("a == 1 or b == 2", ["b"], [], ["b == 2"]), - ("a == 1 or b == 2", ["a", "b"], [], ["a == 1 or b == 2"]), - ("predicate(a,c) or predicate(b,c)", ["a", "b"], [], ["predicate(a,c) or predicate(b,c)"]), - ("a < 1 and a > b", ["a"], [], ["a < 1 and a > b"]), - ]) - def test_i_can_extract_predicates(self, expression, vars_to_include, vars_to_exclude, expected_expr): - sheerka = self.get_sheerka() - expected = [ast.parse(expr, mode="eval") for expr in expected_expr] - - actual = core.builtin_helpers.extract_predicates(sheerka, expression, vars_to_include, vars_to_exclude) - assert len(actual) == len(expected) - for i in range(len(actual)): - assert self.dump_ast(actual[i]) == self.dump_ast(expected[i]) + # @pytest.mark.parametrize("expression, vars_to_include, vars_to_exclude, expected_expr", [ + # ("a == 1", [], [], []), + # ("a == 1", ["a"], [], ["a == 1"]), + # ("a == 1", [], ["a"], []), + # ("predicate(a)", [], [], []), + # ("predicate(a)", ["a"], [], ["predicate(a)"]), + # ("predicate(a, b)", ["a"], [], ["predicate(a, b)"]), + # ("predicate(a, b)", ["b"], [], ["predicate(a, b)"]), + # ("predicate(a, b)", ["a", "b"], [], ["predicate(a, b)"]), + # ("predicate(a, b)", ["a"], ["b"], []), + # ("a + b == 1", [], [], []), + # ("a + b == 1", ["a"], [], ["a + b == 1"]), + # ("a + b == 1", ["a"], ["b"], []), + # ("a + b == 1", ["b"], [], ["a + b == 1"]), + # ("a + b == 1", ["a", "b"], [], ["a + b == 1"]), + # ("a == 1 and b == 2", [], [], []), + # ("a == 1 and b == 2", ["a"], [], ["a == 1"]), + # ("a == 1 and b == 2", ["b"], [], ["b == 2"]), + # ("a == 1 and b == 2", ["a"], ["b"], ["a == 1"]), + # ("a == 1 and b == 2", ["a", "b"], [], ["a == 1 and b == 2"]), + # ("predicate(a,c) and predicate(b,c)", ["a", "b"], [], ["predicate(a,c) and predicate(b,c)"]), + # ("not(a == 1)", [], [], []), + # ("not(a == 1)", ["a"], [], ["not(a==1)"]), + # ("a == 1 or b == 2", [], [], []), + # ("a == 1 or b == 2", ["a"], [], ["a == 1"]), + # ("a == 1 or b == 2", ["b"], [], ["b == 2"]), + # ("a == 1 or b == 2", ["a", "b"], [], ["a == 1 or b == 2"]), + # ("predicate(a,c) or predicate(b,c)", ["a", "b"], [], ["predicate(a,c) or predicate(b,c)"]), + # ("a < 1 and a > b", ["a"], [], ["a < 1 and a > b"]), + # ]) + # def test_i_can_extract_predicates(self, expression, vars_to_include, vars_to_exclude, expected_expr): + # sheerka = self.get_sheerka() + # expected = [ast.parse(expr, mode="eval") for expr in expected_expr] + # + # actual = core.builtin_helpers.extract_predicates(sheerka, expression, vars_to_include, vars_to_exclude) + # assert len(actual) == len(expected) + # for i in range(len(actual)): + # assert self.dump_ast(actual[i]) == self.dump_ast(expected[i]) @pytest.mark.parametrize("concepts, expected", [ ([], []), diff --git a/tests/core/test_concept.py b/tests/core/test_concept.py index 4ca6c77..b8334ff 100644 --- a/tests/core/test_concept.py +++ b/tests/core/test_concept.py @@ -1,6 +1,7 @@ import pytest +from core.builtin_concepts import BuiltinConcepts -from core.concept import Concept, ConceptParts, DEFINITION_TYPE_DEF +from core.concept import Concept, ConceptParts, DEFINITION_TYPE_DEF, ALL_ATTRIBUTES, get_concept_attrs @pytest.mark.parametrize("name, variables, expected", [ @@ -24,7 +25,7 @@ from core.concept import Concept, ConceptParts, DEFINITION_TYPE_DEF def test_i_can_compute_the_key(name, variables, expected): concept = Concept(name) for var_name in variables: - concept.metadata.variables.append((var_name, None)) + concept.get_metadata().variables.append((var_name, None)) concept.init_key() assert concept.key == expected @@ -33,17 +34,17 @@ def test_i_can_compute_the_key(name, variables, expected): def test_i_can_compute_the_key_when_from_definition(): # if definition is not defined, use the name concept = Concept() - concept.metadata.name = "hello a" - concept.metadata.variables = [("a", None)] + concept.get_metadata().name = "hello a" + concept.get_metadata().variables = [("a", None)] concept.init_key() assert concept.key == "hello __var__0" # if definition is defined, use it concept = Concept() - concept.metadata.name = "greetings" - concept.metadata.definition = "hello a" - concept.metadata.definition_type = DEFINITION_TYPE_DEF - concept.metadata.variables = [("a", None)] + concept.get_metadata().name = "greetings" + concept.get_metadata().definition = "hello a" + concept.get_metadata().definition_type = DEFINITION_TYPE_DEF + concept.get_metadata().variables = [("a", None)] concept.init_key() assert concept.key == "hello __var__0" @@ -52,7 +53,7 @@ def test_key_does_not_use_variable_when_definition_is_set(): concept = Concept("plus").def_var('plus') concept.init_key() - assert concept.metadata.key == "plus" + assert concept.get_metadata().key == "plus" def test_i_can_serialize(): @@ -181,6 +182,8 @@ def test_i_can_compare_concepts(): def test_i_can_detect_concept_differences(): + ALL_ATTRIBUTES.clear() + assert Concept(name="concept_name") != Concept() assert Concept(is_builtin=True) != Concept() assert Concept(is_unique=True) != Concept() @@ -197,14 +200,18 @@ def test_i_can_detect_concept_differences(): assert Concept().add_prop("a", "b") != Concept() assert Concept().set_value("a", "b") != Concept() + +def test_compiled_is_not_used_when_comparing_concepts(): + ALL_ATTRIBUTES.clear() + concept = Concept() - concept.compiled["foo"] = "value" + concept.get_compiled()["foo"] = "value" assert concept == Concept() # compiled is not used in the comparison def test_i_can_compare_concept_with_circular_reference(): foo = Concept("foo") - foo.metadata.body = foo + foo.get_metadata().body = foo assert foo == foo @@ -213,7 +220,7 @@ def test_i_can_compare_concept_with_sophisticated_circular_reference(): foo = Concept("foo") bar = Concept("foo", body=foo) baz = Concept("foo", body=bar) - foo.metadata.body = baz + foo.get_metadata().body = baz assert foo != bar @@ -222,12 +229,14 @@ def test_i_can_compare_concept_with_sophisticated_circular_reference_in_other_me foo = Concept("foo") bar = Concept("foo", pre=foo) baz = Concept("foo", pre=bar) - foo.metadata.pre = baz + foo.get_metadata().pre = baz assert foo != bar def test_i_can_update_from(): + ALL_ATTRIBUTES.clear() + template = Concept( name="concept_name", is_builtin=True, @@ -242,6 +251,9 @@ def test_i_can_update_from(): desc="this this the desc", id="123456" ).def_var("a", "10").def_var("b", None) + template.add_prop(BuiltinConcepts.ISA, Concept("foo").def_var("x", "value_x")) + template.add_prop(BuiltinConcepts.HASA, Concept("bar").def_var("x", "value_x")) + template.set_prop(BuiltinConcepts.AUTO_EVAL, True) # make sure origin is preserved setattr(template, "##origin##", "digest") @@ -257,3 +269,28 @@ def test_i_can_update_from(): assert concept == template assert getattr(concept, "##origin##") == "digest" + + +def test_i_can_manage_concepts_attributes(): + ALL_ATTRIBUTES.clear() + concept = Concept("foo") + assert get_concept_attrs(concept) == [] + assert concept.values() == {} + + concept.set_value(ConceptParts.BODY, "I have a body!") + assert concept.values() == {"#body#": "I have a body!"} + + +def test_attributes_are_generated_once_for_all(): + ALL_ATTRIBUTES.clear() + concept = Concept("foo") + concept.get_metadata().id = "id" + + concept.set_value("key1", "value1") + concept.set_value("key2", "value2") + assert get_concept_attrs(concept) == ["key1", "key2"] + assert concept.values() == {"key1": "value1", "key2": "value2"} + + concept.set_value("key3", "value3") # too late for it ! + assert get_concept_attrs(concept) == ["key1", "key2"] + assert concept.values() == {"key1": "value1", "key2": "value2"} diff --git a/tests/core/test_sheerka.py b/tests/core/test_sheerka.py index 43c8e84..ca43438 100644 --- a/tests/core/test_sheerka.py +++ b/tests/core/test_sheerka.py @@ -1,8 +1,8 @@ import os import pytest -from core.builtin_concepts import BuiltinConcepts, ReturnValueConcept, UserInputConcept -from core.concept import Concept, PROPERTIES_TO_SERIALIZE, ConceptParts +from core.builtin_concepts import BuiltinConcepts, ReturnValueConcept, UserInputConcept, AllBuiltinConcepts +from core.concept import Concept, PROPERTIES_TO_SERIALIZE, ConceptParts, NotInit from core.sheerka.Sheerka import Sheerka, BASE_NODE_PARSER_CLASS from core.tokenizer import Token, TokenKind @@ -31,7 +31,7 @@ class TestSheerkaUsingMemoryBasedSheerka(TestUsingMemoryBasedSheerka): assert "parsers.DefConceptParser.DefConceptParser" in sheerka.parsers assert "parsers.BnfNodeParser.BnfNodeParser" in sheerka.parsers assert "parsers.SyaNodeParser.SyaNodeParser" in sheerka.parsers - assert "parsers.AtomNodeParser.AtomNodeParser" in sheerka.parsers + assert "parsers.SequenceNodeParser.SequenceNodeParser" in sheerka.parsers # make sure BaseNodeParser is properly initialized assert BASE_NODE_PARSER_CLASS not in sheerka.parsers @@ -70,7 +70,7 @@ class TestSheerkaUsingMemoryBasedSheerka(TestUsingMemoryBasedSheerka): assert loaded is not None assert sheerka.isinstance(loaded, BuiltinConcepts.UNKNOWN_CONCEPT) assert loaded.body == ("key", "key_that_does_not_exist") - assert loaded.metadata.is_evaluated + assert loaded.get_metadata().is_evaluated def test_i_cannot_get_when_id_is_not_found(self): sheerka = self.get_sheerka() @@ -80,7 +80,7 @@ class TestSheerkaUsingMemoryBasedSheerka(TestUsingMemoryBasedSheerka): assert loaded is not None assert sheerka.isinstance(loaded, BuiltinConcepts.UNKNOWN_CONCEPT) assert loaded.body == ("id", "id_that_does_not_exist") - assert loaded.metadata.is_evaluated + assert loaded.get_metadata().is_evaluated def test_i_can_instantiate_a_builtin_concept_when_it_has_its_own_class(self): sheerka = self.get_sheerka() @@ -112,7 +112,7 @@ class TestSheerkaUsingMemoryBasedSheerka(TestUsingMemoryBasedSheerka): assert sheerka.isinstance(new, concept) for prop in PROPERTIES_TO_SERIALIZE: - assert getattr(new.metadata, prop) == getattr(concept.metadata, prop) + assert getattr(new.get_metadata(), prop) == getattr(concept.get_metadata(), prop) assert new.get_value("a") == 10 assert new.get_value("b") == "value" @@ -127,17 +127,17 @@ class TestSheerkaUsingMemoryBasedSheerka(TestUsingMemoryBasedSheerka): concepts = sheerka.new("foo") assert len(concepts) == 2 assert concepts[0].id == "1001" - assert concepts[0].metadata.body == "foo1" + assert concepts[0].get_metadata().body == "foo1" assert concepts[1].id == "1002" - assert concepts[1].metadata.body == "foo2" + assert concepts[1].get_metadata().body == "foo2" # only one instance if the id is given foo1 = sheerka.new(("foo", "1001")) - assert foo1.metadata.body == "foo1" + assert foo1.get_metadata().body == "foo1" # only one instance if the id is given foo2 = sheerka.new(("foo", "1002")) - assert foo2.metadata.body == "foo2" + assert foo2.get_metadata().body == "foo2" def test_instances_are_different_when_asking_for_new(self): sheerka, context, concept = self.init_concepts(self.get_default_concept(), create_new=True) @@ -152,18 +152,18 @@ class TestSheerkaUsingMemoryBasedSheerka(TestUsingMemoryBasedSheerka): sheerka, context, foo, bar = self.init_concepts("foo", "bar", create_new=True) new_foo = sheerka.new("foo") - new_foo.metadata.body = "metadata value" # modify metadata + new_foo.get_metadata().body = "metadata value" # modify metadata new_foo.def_var("var_name", "default value") # modify definition of variables new_foo.add_prop(BuiltinConcepts.ISA, bar) # modify property - new_foo.compiled["var_name"] = "'var value'" + new_foo.get_compiled()["var_name"] = "'var value'" new_foo.set_value(ConceptParts.BODY, "body value") # modify value new_foo.set_value("var_name", "var value") # modify value - assert new_foo.metadata.body != foo.metadata.body - assert new_foo.metadata.variables != foo.metadata.variables - assert new_foo.metadata.props != foo.metadata.props + assert new_foo.get_metadata().body != foo.get_metadata().body + assert new_foo.get_metadata().variables != foo.get_metadata().variables + assert new_foo.get_metadata().props != foo.get_metadata().props assert new_foo.values != foo.values - assert new_foo.compiled != foo.compiled + assert new_foo.get_compiled() != foo.get_compiled() def test_i_get_the_same_instance_when_is_unique_is_true(self): sheerka, context, concept = self.init_concepts(Concept(name="unique", is_unique=True), create_new=True) @@ -181,16 +181,16 @@ class TestSheerkaUsingMemoryBasedSheerka(TestUsingMemoryBasedSheerka): eval_body=True) sheerka.evaluate_concept(context, sheerka.get_by_id(template.id)) - assert template.metadata.is_evaluated + assert template.get_metadata().is_evaluated assert template.body == "foo body" new = sheerka.new(template.key) - assert not new.metadata.is_evaluated - assert new.body == BuiltinConcepts.NOT_INITIALIZED + assert not new.get_metadata().is_evaluated + assert new.body == NotInit new = sheerka.new((None, template.id)) - assert not new.metadata.is_evaluated - assert new.body == BuiltinConcepts.NOT_INITIALIZED + assert not new.get_metadata().is_evaluated + assert new.body == NotInit def test_i_cannot_instantiate_an_unknown_concept(self): sheerka = self.get_sheerka() @@ -230,7 +230,7 @@ class TestSheerkaUsingMemoryBasedSheerka(TestUsingMemoryBasedSheerka): new = sheerka.new(("foo", "invalid_id")) assert sheerka.isinstance(new, "foo") - assert new.metadata.body == "foo1" + assert new.get_metadata().body == "foo1" def test_i_cannot_instantiate_when_properties_are_not_recognized(self): sheerka, context, concept = self.init_concepts(self.get_default_concept(), create_new=True) @@ -242,16 +242,16 @@ class TestSheerkaUsingMemoryBasedSheerka(TestUsingMemoryBasedSheerka): assert sheerka.isinstance(new.concept, concept) @pytest.mark.parametrize("concept, reduce_simple_list, expected", [ - # (None, False, None), - # (3.14, False, 3.14), - # ("foo", False, "foo"), - # (True, False, True), - # (Concept("name", body="foo"), False, "foo"), - # (Concept("name"), False, Concept("name")), - # (ConceptWithGetObjValue("name").set_value("my_prop", "my_value"), False, "my_value"), - # (ReturnValueConcept(value="return_value"), False, "return_value"), - # (ReturnValueConcept(value=Concept(key=BuiltinConcepts.USER_INPUT, body="text"), status=True), False, "text"), - # (ReturnValueConcept(value=UserInputConcept("text"), status=True), False, "text"), + (None, False, None), + (3.14, False, 3.14), + ("foo", False, "foo"), + (True, False, True), + (Concept("name", body="foo"), False, "foo"), + (Concept("name"), False, Concept("name")), + (ConceptWithGetObjValue("name").set_value("my_prop", "my_value"), False, "my_value"), + (ReturnValueConcept(value="return_value"), False, "return_value"), + (ReturnValueConcept(value=Concept(key=BuiltinConcepts.USER_INPUT, body="text"), status=True), False, "text"), + (ReturnValueConcept(value=UserInputConcept("text"), status=True), False, "text"), (ReturnValueConcept(value=Concept("foo", body=False).auto_init(), status=True), False, False), (Concept("name", body=["foo", "bar"]), False, ["foo", "bar"]), (Concept("name", body=["foo"]), True, "foo"), @@ -342,7 +342,7 @@ class TestSheerkaUsingFileBasedSheerka(TestUsingFileBasedSheerka): def test_builtin_concepts_are_initialized(self): sheerka = self.get_sheerka() - for concept_name in BuiltinConcepts: + for concept_name in AllBuiltinConcepts: assert sheerka.has_key(str(concept_name)) assert sheerka.sdp.get(sheerka.CONCEPTS_BY_KEY_ENTRY, str(concept_name)) is not None @@ -360,15 +360,15 @@ class TestSheerkaUsingFileBasedSheerka(TestUsingFileBasedSheerka): def test_builtin_concepts_can_be_updated(self): sheerka = self.get_sheerka() before_parsing = sheerka.get_by_key(BuiltinConcepts.BEFORE_PARSING) - before_parsing.metadata.desc = "I have a description" - before_parsing.metadata.full_serialization = True + before_parsing.get_metadata().desc = "I have a description" + before_parsing.get_metadata().full_serialization = True with sheerka.sdp.get_transaction("Test") as transac: transac.add(sheerka.CONCEPTS_BY_KEY_ENTRY, before_parsing.key, before_parsing, use_ref=True) sheerka = self.get_sheerka() # another fresh new instance before_parsing = sheerka.get_by_key(BuiltinConcepts.BEFORE_PARSING) - assert before_parsing.metadata.desc == "I have a description" + assert before_parsing.get_metadata().desc == "I have a description" def test_i_first_look_in_local_cache(self): sheerka, context, concept = self.init_concepts("foo", create_new=True) diff --git a/tests/core/test_sheerkaResultManager.py b/tests/core/test_sheerkaResultManager.py index 0cf48c6..d3f8886 100644 --- a/tests/core/test_sheerkaResultManager.py +++ b/tests/core/test_sheerkaResultManager.py @@ -1,6 +1,8 @@ import types +import pytest from core.builtin_concepts import BuiltinConcepts +from core.sheerka.ExecutionContext import ExecutionContext from core.sheerka.services.SheerkaResultManager import SheerkaResultConcept from sdp.sheerkaDataProvider import Event @@ -8,21 +10,28 @@ from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka class TestSheerkaResultManager(TestUsingMemoryBasedSheerka): + io_cache = None @classmethod def setup_class(cls): sheerka = cls().get_sheerka() sheerka.save_execution_context = True + sheerka.evaluate_user_input("def concept one as 1") + cls.io_cache = sheerka.sdp.io.cache.copy() @classmethod def teardown_class(cls): sheerka = cls().get_sheerka() sheerka.save_execution_context = False - def test_i_can_get_the_result_by_digest(self): + def init_test(self): sheerka, context = self.init_concepts() + sheerka.sdp.io.cache = self.io_cache.copy() + return sheerka, context + + def test_i_can_get_the_result_by_digest(self): + sheerka, context = self.init_test() - sheerka.evaluate_user_input("def concept one as 1") digest = sheerka.get_last_execution().event.get_digest() res = sheerka.get_results_by_digest(context, digest) @@ -32,7 +41,7 @@ class TestSheerkaResultManager(TestUsingMemoryBasedSheerka): assert res.digest == digest assert isinstance(res.body, types.GeneratorType) - assert sheerka.load(SheerkaResultConcept.NAME, "digest") == digest + assert sheerka.load_var(SheerkaResultConcept.NAME, "digest") == digest previous_results = list(res.body) @@ -58,11 +67,9 @@ class TestSheerkaResultManager(TestUsingMemoryBasedSheerka): assert sheerka.get_results(context) is None def test_i_can_get_the_result_by_command_name(self): - sheerka, context = self.init_concepts() + sheerka, context = self.init_test() - sheerka.evaluate_user_input("def concept one as 1") digest = sheerka.get_last_execution().event.get_digest() - sheerka.evaluate_user_input("one") # another command res = sheerka.get_results_by_command(context, "def concept") @@ -72,9 +79,8 @@ class TestSheerkaResultManager(TestUsingMemoryBasedSheerka): assert isinstance(res.body, types.GeneratorType) def test_i_can_get_the_result_by_command_when_not_in_the_same_page_size(self): - sheerka, context = self.init_concepts() + sheerka, context = self.init_test() - sheerka.evaluate_user_input("def concept one as 1") sheerka.evaluate_user_input("one") sheerka.evaluate_user_input("one") sheerka.evaluate_user_input("one") @@ -92,9 +98,8 @@ class TestSheerkaResultManager(TestUsingMemoryBasedSheerka): assert res.body == {'command': 'def concept'} def test_i_cannot_get_result_from_command_if_the_command_does_not_exists_multiple_pages(self): - sheerka, context = self.init_concepts() + sheerka, context = self.init_test() - sheerka.evaluate_user_input("def concept one as 1") sheerka.evaluate_user_input("one") sheerka.evaluate_user_input("one") sheerka.evaluate_user_input("one") @@ -105,9 +110,8 @@ class TestSheerkaResultManager(TestUsingMemoryBasedSheerka): assert res.body == {'command': 'fake command'} def test_i_can_get_last_results(self): - sheerka, context = self.init_concepts() + sheerka, context = self.init_test() - sheerka.evaluate_user_input("def concept one as 1") sheerka.evaluate_user_input("one") res = sheerka.get_last_results(context) @@ -144,4 +148,106 @@ class TestSheerkaResultManager(TestUsingMemoryBasedSheerka): assert sheerka.isinstance(res, BuiltinConcepts.NOT_FOUND) assert res.body == {'query': 'last'} + @pytest.mark.parametrize('kwargs', [ + {"desc": "Evaluating 'def concept one as 1'"}, + {"id": 0}, + {"desc": "Evaluating 'def concept one as 1'", "id": 0} + ]) + def test_i_can_get_last_results_using_kwarg(self, kwargs): + sheerka, context = self.init_test() + ExecutionContext.ids.clear() + res = sheerka.get_last_results(context, **kwargs) + items = list(res.body) + + assert len(items) == 1 + for k, v in kwargs.items(): + assert getattr(items[0], k) == v + + @pytest.mark.parametrize('predicate, expected', [ + ('desc.startswith("Evaluating \'def concept one as 1\'")', {"desc": "Evaluating 'def concept one as 1'"}), + ("id == 0", {"id": 0}), + ("'def concept one as 1' in desc and id == 0", {"desc": "Evaluating 'def concept one as 1'", "id": 0}) + ]) + def test_i_can_get_last_results_using_filter(self, predicate, expected): + sheerka, context = self.init_test() + ExecutionContext.ids.clear() + + res = sheerka.get_last_results(context, filter=predicate) + items = list(res.body) + + assert len(items) == 1 + for k, v in expected.items(): + assert getattr(items[0], k) == v + + @pytest.mark.parametrize('predicate, expected', [ + ('desc.startswith("Evaluating \'def concept one as 1\'")', {"desc": "Evaluating 'def concept one as 1'"}), + ("id == 0", {"id": 0}), + ("'def concept one as 1' in desc and id == 0", {"desc": "Evaluating 'def concept one as 1'", "id": 0}) + ]) + def test_i_can_get_last_results_using_the_first_argument_to_filter(self, predicate, expected): + sheerka, context = self.init_test() + ExecutionContext.ids.clear() + + res = sheerka.get_last_results(context, predicate) + items = list(res.body) + + assert len(items) == 1 + for k, v in expected.items(): + assert getattr(items[0], k) == v + + @pytest.mark.parametrize('kwargs', [ + {"desc": "Evaluating 'def concept one as 1'"}, + {"id": 0}, + {"desc": "Evaluating 'def concept one as 1'", "id": 0} + ]) + def test_i_can_get_results_using_kwarg(self, kwargs): + sheerka, context = self.init_test() + ExecutionContext.ids.clear() + sheerka.get_last_results(context) + + res = sheerka.get_results(context, **kwargs) + items = list(res.body) + + assert len(items) == 1 + for k, v in kwargs.items(): + assert getattr(items[0], k) == v + + @pytest.mark.parametrize('predicate, expected', [ + ('desc.startswith("Evaluating \'def concept one as 1\'")', {"desc": "Evaluating 'def concept one as 1'"}), + ("id == 0", {"id": 0}), + ("'def concept one as 1' in desc and id == 0", {"desc": "Evaluating 'def concept one as 1'", "id": 0}) + ]) + def test_i_can_get_results_using_filter(self, predicate, expected): + sheerka, context = self.init_test() + ExecutionContext.ids.clear() + sheerka.get_last_results(context) + + res = sheerka.get_results(context, filter=predicate) + items = list(res.body) + + assert len(items) == 1 + for k, v in expected.items(): + assert getattr(items[0], k) == v + + @pytest.mark.parametrize('predicate, expected', [ + ('desc.startswith("Evaluating \'def concept one as 1\'")', {"desc": "Evaluating 'def concept one as 1'"}), + ("id == 0", {"id": 0}), + ("'def concept one as 1' in desc and id == 0", {"desc": "Evaluating 'def concept one as 1'", "id": 0}) + ]) + def test_i_can_get_results_using_the_first_argument_to_filter(self, predicate, expected): + sheerka, context = self.init_test() + ExecutionContext.ids.clear() + sheerka.get_last_results(context) + + res = sheerka.get_results(context, predicate) + items = list(res.body) + + assert len(items) == 1 + for k, v in expected.items(): + assert getattr(items[0], k) == v + + def test_i_can_manage_invalid_predicates(self): + predicate = {"filter": "a b c"} + with pytest.raises(SyntaxError): + SheerkaResultConcept.get_predicate(**predicate) diff --git a/tests/core/test_sheerka_call_evaluators.py b/tests/core/test_sheerka_call_evaluators.py index b2bd45a..b2a2b3e 100644 --- a/tests/core/test_sheerka_call_evaluators.py +++ b/tests/core/test_sheerka_call_evaluators.py @@ -1,5 +1,6 @@ from core.builtin_concepts import BuiltinConcepts from core.concept import Concept +from core.sheerka.services.SheerkaExecute import SheerkaExecute from evaluators.BaseEvaluator import OneReturnValueEvaluator, BaseEvaluator, AllReturnValuesEvaluator from tests.BaseTest import BaseTest @@ -15,16 +16,18 @@ class Out: target = [str(r.body.key) for r in return_value] else: target = str(return_value.body.key) - step = str(context.step) - text = f"{step} [{context.iteration}] " + step = str(context.action_context["step"]) + iteration = context.action_context["iteration"] + text = f"{step} [{iteration}] " text += f"{name} - {method} - target={target}" self.debug_out.append(text) def out_all(self, method, name, context, return_values): name = name[len(BaseEvaluator.PREFIX):] target = [str(r.body.key) for r in return_values] - step = str(context.step) - text = f"{step} [{context.iteration}] " + step = str(context.action_context["step"]) + iteration = context.action_context["iteration"] + text = f"{step} [{iteration}] " text += f"{name} - {method} - target={target}" self.debug_out.append(text) @@ -295,6 +298,8 @@ class TestSheerkaExecuteEvaluators(TestUsingMemoryBasedSheerka): EvaluatorOneWithPriority10, EvaluatorOneWithPriority15, EvaluatorAllWithPriority20, ] + service = sheerka.services[SheerkaExecute.NAME] + service.reset_evaluators() entries = [self.tretval(sheerka, Concept("foo"))] Out.debug_out = [] @@ -318,6 +323,8 @@ class TestSheerkaExecuteEvaluators(TestUsingMemoryBasedSheerka): def test_that_predicate_is_checked_before_evaluation_for_one_return(self): sheerka = self.get_sheerka() sheerka.evaluators = [EvaluatorOneModifyFoo] + service = sheerka.services[SheerkaExecute.NAME] + service.reset_evaluators() entries = [self.tretval(sheerka, Concept("foo")), self.tretval(sheerka, Concept("baz"))] Out.debug_out = [] @@ -334,6 +341,8 @@ class TestSheerkaExecuteEvaluators(TestUsingMemoryBasedSheerka): def test_that_predicate_is_checked_before_evaluation_for_all_return(self): sheerka = self.get_sheerka() sheerka.evaluators = [EvaluatorAllReduceFooBar] + service = sheerka.services[SheerkaExecute.NAME] + service.reset_evaluators() entries = [self.tretval(sheerka, Concept("foo")), self.tretval(sheerka, Concept("bar"))] Out.debug_out = [] @@ -354,6 +363,8 @@ class TestSheerkaExecuteEvaluators(TestUsingMemoryBasedSheerka): def test_evaluation_continue_until_no_more_modification(self): sheerka = self.get_sheerka() sheerka.evaluators = [EvaluatorOneModifyFoo, EvaluatorOneModifyBar] + service = sheerka.services[SheerkaExecute.NAME] + service.reset_evaluators() entries = [self.tretval(sheerka, Concept("foo")), self.tretval(sheerka, Concept("baz"))] Out.debug_out = [] @@ -379,6 +390,8 @@ class TestSheerkaExecuteEvaluators(TestUsingMemoryBasedSheerka): def test_evaluation_steps_are_respected(self): sheerka = self.get_sheerka() sheerka.evaluators = [EvaluatorOneWithPriority10, EvaluatorOnePreEvaluation] + service = sheerka.services[SheerkaExecute.NAME] + service.reset_evaluators() entries = [self.tretval(sheerka, Concept("foo"))] Out.debug_out = [] @@ -392,6 +405,8 @@ class TestSheerkaExecuteEvaluators(TestUsingMemoryBasedSheerka): def test_evaluation_multi_steps_are_respected(self): sheerka = self.get_sheerka() sheerka.evaluators = [EvaluatorOneMultiSteps] + service = sheerka.services[SheerkaExecute.NAME] + service.reset_evaluators() entries = [self.tretval(sheerka, Concept("foo"))] Out.debug_out = [] @@ -405,9 +420,88 @@ class TestSheerkaExecuteEvaluators(TestUsingMemoryBasedSheerka): '__EVALUATION [0] multiStep - eval - target=foo', ] + def test_evaluators_can_be_selected_in_the_context(self): + sheerka, context = self.init_concepts() + sheerka.evaluators = [EvaluatorOneWithPriority20, + EvaluatorAllWithPriority15, + EvaluatorAllWithPriority10, ] + service = sheerka.services[SheerkaExecute.NAME] + service.reset_evaluators() + context.preprocess_evaluators = [EvaluatorAllWithPriority10().short_name] # it will be the only one to be evaluated + + entries = [self.tretval(sheerka, Concept("foo"))] + + Out.debug_out = [] + sheerka.execute(context, entries, [BuiltinConcepts.EVALUATION]) + assert Out.debug_out == [ + "__EVALUATION [0] all_priority10 - matches - target=['foo']", + "__EVALUATION [0] all_priority10 - eval - target=['foo']" + ] + + # grouped evaluator is in cache + service = sheerka.services[SheerkaExecute.NAME] + assert "__EVALUATION|all_priority10" in service.grouped_evaluators_cache + + def test_evaluators_priorities_can_be_tweaked_by_the_context(self): + sheerka, context = self.init_concepts() + sheerka.evaluators = [EvaluatorOneWithPriority20, + EvaluatorOneWithPriority15, + EvaluatorOneWithPriority10, ] + service = sheerka.services[SheerkaExecute.NAME] + service.reset_evaluators() + + # invert the priorities + context.add_preprocess(EvaluatorOneWithPriority20().name, priority=1) + context.add_preprocess(EvaluatorOneWithPriority15().name, priority=2) + context.add_preprocess(EvaluatorOneWithPriority10().name, priority=3) + + entries = [self.tretval(sheerka, Concept("foo"))] + + Out.debug_out = [] + sheerka.execute(context, entries, [BuiltinConcepts.EVALUATION]) + assert Out.debug_out == [ + "__EVALUATION [0] priority10 - matches - target=foo", + "__EVALUATION [0] priority10 - eval - target=foo", + "__EVALUATION [0] priority15 - matches - target=foo", + "__EVALUATION [0] priority15 - eval - target=foo", + '__EVALUATION [0] priority20 - matches - target=foo', + '__EVALUATION [0] priority20 - eval - target=foo', + ] + + # grouped evaluator is in cache + service = sheerka.services[SheerkaExecute.NAME] + assert "__EVALUATION|evaluators.priority20|evaluators.priority15|evaluators.priority10" in service.grouped_evaluators_cache + + def test_evaluators_enabled_can_be_tweaked_by_the_context(self): + sheerka, context = self.init_concepts() + sheerka.evaluators = [EvaluatorOneWithPriority20, + EvaluatorOneWithPriority15, + EvaluatorOneWithPriority10, ] + service = sheerka.services[SheerkaExecute.NAME] + service.reset_evaluators() + + # invert the priorities + context.add_preprocess(EvaluatorOneWithPriority20().name, enabled=False) + context.add_preprocess(EvaluatorOneWithPriority15().name, enabled=False) + + entries = [self.tretval(sheerka, Concept("foo"))] + + Out.debug_out = [] + sheerka.execute(context, entries, [BuiltinConcepts.EVALUATION]) + assert Out.debug_out == [ + "__EVALUATION [0] priority10 - matches - target=foo", + "__EVALUATION [0] priority10 - eval - target=foo", + ] + + # grouped evaluator is in cache + service = sheerka.services[SheerkaExecute.NAME] + assert "__EVALUATION|evaluators.priority10" in service.grouped_evaluators_cache + def test_evaluators_can_be_pre_processed(self): sheerka = self.get_sheerka() sheerka.evaluators = [EvaluatorOneModifyFoo] + service = sheerka.services[SheerkaExecute.NAME] + service.reset_evaluators() entries = [self.tretval(sheerka, Concept("foo"))] @@ -418,7 +512,7 @@ class TestSheerkaExecuteEvaluators(TestUsingMemoryBasedSheerka): sheerka.execute(context, entries, [BuiltinConcepts.EVALUATION]) assert Out.debug_out == [] - # other contextes are not impacted + # other evaluations are not impacted Out.debug_out = [] sheerka.execute(self.get_context(sheerka), entries, [BuiltinConcepts.EVALUATION]) assert Out.debug_out == [ @@ -430,6 +524,8 @@ class TestSheerkaExecuteEvaluators(TestUsingMemoryBasedSheerka): def test_i_can_initialize_evaluator_if_initialize_evaluator_is_defined(self): sheerka, context, foo, bar, baz = self.init_concepts("foo", "bar", "baz") sheerka.evaluators = [EvaluatorOneInitializationOnce] + service = sheerka.services[SheerkaExecute.NAME] + service.reset_evaluators() entries = [ self.tretval(sheerka, foo), @@ -451,6 +547,8 @@ class TestSheerkaExecuteEvaluators(TestUsingMemoryBasedSheerka): def test_i_can_initialize_evaluators_multiple_times(self): sheerka, context, foo, bar, baz = self.init_concepts("foo", "bar", "baz") sheerka.evaluators = [EvaluatorOneInitializationMultiple] + service = sheerka.services[SheerkaExecute.NAME] + service.reset_evaluators() entries = [ self.tretval(sheerka, foo), @@ -479,6 +577,8 @@ class TestSheerkaExecuteEvaluators(TestUsingMemoryBasedSheerka): """ sheerka = self.get_sheerka() sheerka.evaluators = [EvaluatorOneDoNotModifyExecutionFlow] + service = sheerka.services[SheerkaExecute.NAME] + service.reset_evaluators() entries = [self.tretval(sheerka, Concept("foo"))] Out.debug_out = [] @@ -499,6 +599,8 @@ class TestSheerkaExecuteEvaluators(TestUsingMemoryBasedSheerka): """ sheerka = self.get_sheerka() sheerka.evaluators = [EvaluatorOneModifyFoo] + service = sheerka.services[SheerkaExecute.NAME] + service.reset_evaluators() entries = [self.tretval(sheerka, Concept("foo"))] Out.debug_out = [] @@ -516,6 +618,8 @@ class TestSheerkaExecuteEvaluators(TestUsingMemoryBasedSheerka): def test_i_can_remove_return_values_from_the_execution_workflow(self): sheerka = self.get_sheerka() sheerka.evaluators = [EvaluatorAllSuppressEntries] + service = sheerka.services[SheerkaExecute.NAME] + service.reset_evaluators() entries = [self.tretval(sheerka, Concept("foo")), self.tretval(sheerka, Concept("bar")), diff --git a/tests/core/test_sheerka_printer.py b/tests/core/test_sheerka_printer.py index 1900b0a..5ec4700 100644 --- a/tests/core/test_sheerka_printer.py +++ b/tests/core/test_sheerka_printer.py @@ -353,25 +353,21 @@ second : 'value d' sheerka.print(lst) captured = capsys.readouterr() assert captured.out == """(1001)foo a b -a : 'value a' -var.a: *name 'var' is not defined* -b : 'value b' -var.b: *name 'var' is not defined* -id : '1001' -name : 'foo a b' -key : 'foo __var__0 __var__1' -body : __NOT_INITIALIZED -self : (1001)foo a b +a : 'value a' +b : 'value b' +id : '1001' +name: 'foo a b' +key : 'foo __var__0 __var__1' +body: **NotInit** +self: (1001)foo a b (1001)foo a b -a : 'value c' -var.a: *name 'var' is not defined* -b : 'value d' -var.b: *name 'var' is not defined* -id : '1001' -name : 'foo a b' -key : 'foo __var__0 __var__1' -body : __NOT_INITIALIZED -self : (1001)foo a b +a : 'value c' +b : 'value d' +id : '1001' +name: 'foo a b' +key : 'foo __var__0 __var__1' +body: **NotInit** +self: (1001)foo a b """ def test_i_can_format_d_when_dictionary(self, capsys): @@ -391,34 +387,32 @@ self : (1001)foo a b sheerka.print(lst) captured = capsys.readouterr() assert captured.out == """(1001)foo a b -a : 'value a' -var.a: *name 'var' is not defined* -b : {'a' : 'value a' - 'beta' : {'b1': 10 - 'b2': Obj(a='10', b=15) - 'b3': ['items', - 'in', - 'a', - 'list']} - 'gamma' : {'list' : ["'quoted string'", - ""double" 'single'", - 'c3'] - 'empty': []} - 'epsilon': ['a', - 'b', - 'c'] - 'g' : {'tuple': ('tuple-a', - 'tuple-b', - 'tuple-b') - 'empty': ()} - 'h' : {'set' : {'set-a'} - 'empty': {}}} -var.b: *name 'var' is not defined* -id : '1001' -name : 'foo a b' -key : 'foo __var__0 __var__1' -body : __NOT_INITIALIZED -self : (1001)foo a b +a : 'value a' +b : {'a' : 'value a' + 'beta' : {'b1': 10 + 'b2': Obj(a='10', b=15) + 'b3': ['items', + 'in', + 'a', + 'list']} + 'gamma' : {'list' : ["'quoted string'", + ""double" 'single'", + 'c3'] + 'empty': []} + 'epsilon': ['a', + 'b', + 'c'] + 'g' : {'tuple': ('tuple-a', + 'tuple-b', + 'tuple-b') + 'empty': ()} + 'h' : {'set' : {'set-a'} + 'empty': {}}} +id : '1001' +name: 'foo a b' +key : 'foo __var__0 __var__1' +body: **NotInit** +self: (1001)foo a b """ def test_i_can_manage_when_property_does_not_exist(self, capsys): diff --git a/tests/core/test_tokenizer.py b/tests/core/test_tokenizer.py index f53fd4e..1bd707c 100644 --- a/tests/core/test_tokenizer.py +++ b/tests/core/test_tokenizer.py @@ -4,7 +4,7 @@ from core.tokenizer import Tokenizer, Token, TokenKind, LexerError def test_i_can_tokenize(): source = "+*-/{}[]() ,;:.?\n\n\r\r\r\nidentifier_0\t \t10.15 10 'string\n' \"another string\"=|&<>c:name:" - source += "$£€!_identifier°~_^\\`==#__var__10r/regex\nregex/" + source += "$£€!_identifier°~_^\\`==#__var__10r/regex\nregex/r:xxx|1:" tokens = list(Tokenizer(source)) assert tokens[0] == Token(TokenKind.PLUS, "+", 0, 1, 1) assert tokens[1] == Token(TokenKind.STAR, "*", 1, 1, 2) @@ -56,12 +56,15 @@ def test_i_can_tokenize(): assert tokens[47] == Token(TokenKind.HASH, '#', 111, 6, 54) assert tokens[48] == Token(TokenKind.VAR_DEF, '__var__10', 112, 6, 55) assert tokens[49] == Token(TokenKind.REGEX, '/regex\nregex/', 121, 6, 64) + assert tokens[50] == Token(TokenKind.RULE, ("xxx", "1"), 135, 7, 7) - assert tokens[50] == Token(TokenKind.EOF, '', 135, 7, 7) + assert tokens[51] == Token(TokenKind.EOF, '', 143, 7, 15) @pytest.mark.parametrize("text, expected", [ ("_ident", True), + ("__ident", True), + ("___ident", True), ("ident", True), ("ident123", True), ("ident_123", True), @@ -168,3 +171,29 @@ def test_i_can_parse_concept_token(text, expected): assert tokens[0].type == TokenKind.CONCEPT assert tokens[0].value == expected + +@pytest.mark.parametrize("text, expected", [ + ("r:key:", ("key", None)), + ("r:key|id:", ("key", "id")), + ("r:key|:", ("key", None)), + ("r:|id:", (None, "id")), + ("r:125:", ("125", None)), +]) +def test_i_can_parse_concept_token(text, expected): + tokens = list(Tokenizer(text)) + + assert tokens[0].type == TokenKind.RULE + assert tokens[0].value == expected + + +@pytest.mark.parametrize("text, expected", [ + ("r|regex|", "|regex|"), + ("r/regex/", "/regex/"), + ("r'regex'", "'regex'"), + ('r"regex"', '"regex"'), +]) +def test_i_can_parse_regex_token(text, expected): + tokens = list(Tokenizer(text)) + + assert tokens[0].type == TokenKind.REGEX + assert tokens[0].value == expected diff --git a/tests/core/test_utils.py b/tests/core/test_utils.py index 9c253d0..911e46b 100644 --- a/tests/core/test_utils.py +++ b/tests/core/test_utils.py @@ -2,8 +2,8 @@ from dataclasses import dataclass import core.utils import pytest -from core.concept import ConceptParts, Concept -from core.tokenizer import Token, TokenKind +from core.concept import Concept +from core.tokenizer import Token, TokenKind, Tokenizer, Keywords @dataclass @@ -15,6 +15,12 @@ class Obj: return hash((self.prop1, self.prop1)) +@dataclass +class Obj2: + prop1: object + prop2: object + + def get_tokens(lst): res = [] for e in lst: @@ -134,11 +140,25 @@ def test_i_can_product(a, b, expected): ]) def test_i_can_strip(input_as_list, expected_as_list): - actual = core.utils.strip_tokens(get_tokens(input_as_list)) + actual = core.utils.strip_tokens(get_tokens(input_as_list)) # KSI 20201007 Why not use Tokenizer ?!! For perf ? expected = get_tokens(expected_as_list) assert actual == expected +@pytest.mark.parametrize("text, value, expected", [ + ("xxx=yyy", "=", 1), + ("xxx", "=", -1), + ("xxx = yyy", "=", 2), + ("xxx = yyy", " = ", -1), +]) +def test_i_can_index(text, value, expected): + assert core.utils.index_tokens(Tokenizer(text), value) == expected + + +def test_i_can_manage_non_in_index_tokens(): + assert core.utils.index_tokens(None, "=") == -1 + + def test_by_default_eof_is_not_stripped(): actual = core.utils.strip_tokens(get_tokens(["one", "two", " ", "\n", ""])) expected = get_tokens(["one", "two", " ", "\n", ""]) @@ -176,6 +196,16 @@ def test_i_can_unstr_concept(text, expected_key, expected_id): assert i == expected_id +@pytest.mark.parametrize("text, expected_key, expected_id", [ + ("r:key:", "key", None), + ("r:key|id:", "key", "id"), +]) +def test_i_can_unstr_concept_rules(text, expected_key, expected_id): + k, i = core.utils.unstr_concept(text, prefix="r:") + assert k == expected_key + assert i == expected_id + + def test_i_can_str_concept(): assert core.utils.str_concept(("key", "id")) == "c:key|id:" assert core.utils.str_concept((None, "id")) == "c:|id:" @@ -186,10 +216,12 @@ def test_i_can_str_concept(): concept = Concept("foo").init_key() assert core.utils.str_concept(concept) == "c:foo:" - concept.metadata.id = "1001" + concept.get_metadata().id = "1001" assert core.utils.str_concept(concept) == "c:foo|1001:" assert core.utils.str_concept(concept, drop_name=True) == "c:|1001:" + assert core.utils.str_concept(("key", "id"), prefix='r:') == "r:key|id:" + @pytest.mark.parametrize("text, expected", [ (None, None), @@ -198,7 +230,7 @@ def test_i_can_str_concept(): ("xxx", None), ("xxx.", None), ("xxx.yyy", None), - ("core.concept.ConceptParts.BODY", ConceptParts.BODY), + ("core.tokenizer.Keywords.CONCEPT", Keywords.CONCEPT), ]) def test_i_can_decode_enum(text, expected): actual = core.utils.decode_enum(text) @@ -214,15 +246,24 @@ def test_encode_concept_key_id(): concept = Concept("foo").init_key() assert core.utils.encode_concept(concept) == "__C__KEY_foo__ID_00None00__C__" - concept.metadata.id = "1001" + concept.get_metadata().id = "1001" assert core.utils.encode_concept(concept) == "__C__KEY_foo__ID_1001__C__" + assert core.utils.encode_concept(("key", "id"), "R") == "__R__KEY_key__ID_id__R__" + assert core.utils.encode_concept((None, "id"), "R") == "__R__KEY_00None00__ID_id__R__" + assert core.utils.encode_concept(("key", None), "R") == "__R__KEY_key__ID_00None00__R__" + assert core.utils.encode_concept(("k + y", "id"), "R") == "__R__KEY_k000y__ID_id__R__" + def test_decode_concept_key_id(): assert core.utils.decode_concept("__C__KEY_key__ID_id__C__") == ("key", "id") assert core.utils.decode_concept("__C__KEY_00None00__ID_id__C__") == (None, "id") assert core.utils.decode_concept("__C__KEY_key__ID_00None00__C__") == ("key", None) + assert core.utils.decode_concept("__R__KEY_key__ID_id__R__", "R") == ("key", "id") + assert core.utils.decode_concept("__R__KEY_00None00__ID_id__R__", "R") == (None, "id") + assert core.utils.decode_concept("__R__KEY_key__ID_00None00__R__", "R") == ("key", None) + @pytest.mark.parametrize("a,b,expected", [ ([], [], []), @@ -246,3 +287,54 @@ def test_i_can_make_unique(): assert core.utils.make_unique(["a", "a", "b", "c", "c"]) == ["a", "b", "c"] assert core.utils.make_unique([Obj("a", "b"), Obj("a", "c"), Obj("a", "b")]) == [Obj("a", "b"), Obj("a", "c")] assert core.utils.make_unique([Obj("a", "b"), Obj("a", "c")], lambda o: o.prop1) == [Obj("a", "b")] + + +@pytest.mark.parametrize("expression, bag, expected", [ + ("", {}, None), + (None, {}, None), + ("a", {"a": 1}, 1), + ("a.x", {"a.x": 1}, 1), + ("a.prop1", {"a": Obj("prop1", "prop2")}, "prop1"), + ("a.prop1.prop2.prop1", {"a": Obj2(Obj2("prop11", Obj2("prop121", "prop122")), "2")}, "prop121"), + ("a.prop1.prop2.prop1", {"a": Obj2(Obj2("prop11", None), "2")}, None), + ("a[1]", {"a": ['lst-first', 'lst-second']}, 'lst-second'), + ("a['bar']", {"a": {'foo': 'dict-first', 'bar': 'dict-second'}}, 'dict-second'), + ("a.prop1[0]", {"a": Obj2(['lst-first', 'lst-second'], None)}, 'lst-first'), + ("a.prop2['bar']", {"a": Obj2(None, {'foo': 'dict-first', 'bar': 'dict-second'})}, 'dict-second'), + ("a.bar", {"a": {'foo': 'dict-first', 'bar': 'dict-second'}}, 'dict-second'), + ("a.prop2.bar", {"a": Obj2(None, {'foo': 'dict-first', 'bar': 'dict-second'})}, 'dict-second'), +]) +def test_i_can_evaluate_expression(expression, bag, expected): + assert core.utils.evaluate_expression(expression, bag) == expected + + +@pytest.mark.parametrize("expression, bag, expected_error, prop_name", [ + ("a", {}, NameError, "a"), + ("a.prop3", {"a": Obj("prop1", "prop2")}, NameError, "prop3"), +]) +def test_i_cannot_evaluate_expression(expression, bag, expected_error, prop_name): + with pytest.raises(expected_error) as e: + core.utils.evaluate_expression(expression, bag) + + assert e.value.args == (prop_name,) + + +@pytest.mark.parametrize("text, expected_text", [ + ("hello world", "hello world"), + ("'hello' 'world'", "'hello' 'world'"), + ("def concept a from", "def concept a from"), + ("()[]{}1=1.5+-/*><&é", "()[]{}1=1.5+-/*><&é"), + ("execute(c:concept_name:)", "execute(c:concept_name:)") + +]) +def test_i_can_get_text_from_tokens(text, expected_text): + tokens = list(Tokenizer(text)) + assert core.utils.get_text_from_tokens(tokens) == expected_text + + +@pytest.mark.parametrize("text, custom, expected_text", [ + ("execute(c:concept_name:)", {TokenKind.CONCEPT: lambda t: f"__C__{t.value[0]}"}, "execute(__C__concept_name)") +]) +def test_i_can_get_text_from_tokens_with_custom_switcher(text, custom, expected_text): + tokens = list(Tokenizer(text)) + assert core.utils.get_text_from_tokens(tokens, custom) == expected_text diff --git a/tests/evaluators/test_AddConceptEvaluator.py b/tests/evaluators/test_DefConceptEvaluator.py similarity index 70% rename from tests/evaluators/test_AddConceptEvaluator.py rename to tests/evaluators/test_DefConceptEvaluator.py index 6df27a0..df12641 100644 --- a/tests/evaluators/test_AddConceptEvaluator.py +++ b/tests/evaluators/test_DefConceptEvaluator.py @@ -4,17 +4,17 @@ import pytest from core.builtin_concepts import ReturnValueConcept, ParserResultConcept, BuiltinConcepts from core.concept import VARIABLE_PREFIX, Concept, DEFINITION_TYPE_BNF, DEFINITION_TYPE_DEF from core.tokenizer import Tokenizer -from evaluators.AddConceptEvaluator import AddConceptEvaluator +from evaluators.DefConceptEvaluator import DefConceptEvaluator from parsers.BaseParser import BaseParser from parsers.BnfNodeParser import Sequence, StrMatch, ZeroOrMore, ConceptExpression -from parsers.BnfParser import BnfParser +from parsers.BnfDefinitionParser import BnfDefinitionParser from parsers.DefConceptParser import DefConceptNode, NameNode from parsers.PythonParser import PythonNode, PythonParser from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka -class TestAddConceptEvaluator(TestUsingMemoryBasedSheerka): +class TestDefConceptEvaluator(TestUsingMemoryBasedSheerka): @staticmethod def get_concept_part(part): @@ -47,7 +47,7 @@ class TestAddConceptEvaluator(TestUsingMemoryBasedSheerka): status=True, value=ParserResultConcept( source=source, - parser=BnfParser(), + parser=BnfDefinitionParser(), value=parsing_expression ) ) @@ -86,7 +86,7 @@ class TestAddConceptEvaluator(TestUsingMemoryBasedSheerka): ]) def test_i_can_match(self, ret_val, expected): context = self.get_context() - assert AddConceptEvaluator().matches(context, ret_val) == expected + assert DefConceptEvaluator().matches(context, ret_val) == expected def test_that_the_source_is_correctly_set_for_bnf_concept(self): context = self.get_context() @@ -98,21 +98,21 @@ class TestAddConceptEvaluator(TestUsingMemoryBasedSheerka): body="print('hello' + a)", ret="a") - evaluated = AddConceptEvaluator().eval(context, def_concept_return_value) + evaluated = DefConceptEvaluator().eval(context, def_concept_return_value) assert evaluated.status assert context.sheerka.isinstance(evaluated.body, BuiltinConcepts.NEW_CONCEPT) created_concept = evaluated.body.body - assert created_concept.metadata.name == "hello a" - assert created_concept.metadata.key == "hello __var__0" - assert created_concept.metadata.where == "isinstance(a, str )" - assert created_concept.metadata.pre == "a is not None" - assert created_concept.metadata.post is None # test that NotInitialized is mapped into None - assert created_concept.metadata.body == "print('hello' + a)" - assert created_concept.metadata.ret == "a" - assert created_concept.metadata.definition == "hello a" - assert created_concept.metadata.definition_type == "bnf" + assert created_concept.get_metadata().name == "hello a" + assert created_concept.get_metadata().key == "hello __var__0" + assert created_concept.get_metadata().where == "isinstance(a, str )" + assert created_concept.get_metadata().pre == "a is not None" + assert created_concept.get_metadata().post is None # test that NotInitialized is mapped into None + assert created_concept.get_metadata().body == "print('hello' + a)" + assert created_concept.get_metadata().ret == "a" + assert created_concept.get_metadata().definition == "hello a" + assert created_concept.get_metadata().definition_type == "bnf" def test_i_can_add_concept_with_the_correct_variables_when_referencing_other_concepts(self): context = self.get_context() @@ -121,13 +121,13 @@ class TestAddConceptEvaluator(TestUsingMemoryBasedSheerka): where=self.pretval(Concept("u is a v").def_var("u").def_var("v"), source="x is a number"), body=self.pretval(Concept("add a b").def_var("a").def_var("b"), source="add x y"), ) - evaluated = AddConceptEvaluator().eval(context, def_concept_return_value) + evaluated = DefConceptEvaluator().eval(context, def_concept_return_value) assert evaluated.status assert context.sheerka.isinstance(evaluated.body, BuiltinConcepts.NEW_CONCEPT) created_concept = evaluated.body.body - assert created_concept.metadata.variables == [("x", None), ("y", None)] + assert created_concept.get_metadata().variables == [("x", None), ("y", None)] def test_that_the_source_is_correctly_set_for_concept_with_simple_definition(self): context = self.get_context() @@ -138,20 +138,20 @@ class TestAddConceptEvaluator(TestUsingMemoryBasedSheerka): pre="a is not None", body="print('hello' + a)") - evaluated = AddConceptEvaluator().eval(context, def_concept_return_value) + evaluated = DefConceptEvaluator().eval(context, def_concept_return_value) assert evaluated.status assert context.sheerka.isinstance(evaluated.body, BuiltinConcepts.NEW_CONCEPT) created_concept = evaluated.body.body - assert created_concept.metadata.name == "greetings" - assert created_concept.metadata.key == "hello __var__0" - assert created_concept.metadata.where == "isinstance(a, str )" - assert created_concept.metadata.pre == "a is not None" - assert created_concept.metadata.post is None - assert created_concept.metadata.body == "print('hello' + a)" - assert created_concept.metadata.definition == "hello a" - assert created_concept.metadata.definition_type == "def" + assert created_concept.get_metadata().name == "greetings" + assert created_concept.get_metadata().key == "hello __var__0" + assert created_concept.get_metadata().where == "isinstance(a, str )" + assert created_concept.get_metadata().pre == "a is not None" + assert created_concept.get_metadata().post is None + assert created_concept.get_metadata().body == "print('hello' + a)" + assert created_concept.get_metadata().definition == "hello a" + assert created_concept.get_metadata().definition_type == "def" def test_that_the_new_concept_is_correctly_saved_in_db(self): context = self.get_context() @@ -166,23 +166,23 @@ class TestAddConceptEvaluator(TestUsingMemoryBasedSheerka): from_db = context.sheerka.get_by_key("hello " + VARIABLE_PREFIX + "0") assert context.sheerka.isinstance(from_db, BuiltinConcepts.UNKNOWN_CONCEPT) - AddConceptEvaluator().eval(context, def_concept_return_value) + DefConceptEvaluator().eval(context, def_concept_return_value) context.sheerka.concepts_cache = {} # reset cache from_db = context.sheerka.get_by_key("hello " + VARIABLE_PREFIX + "0") - assert from_db.metadata.key == f"hello {VARIABLE_PREFIX}0" - assert from_db.metadata.name == "hello a" - assert from_db.metadata.where == "isinstance(a, str )" - assert from_db.metadata.pre == "a is not None" - assert from_db.metadata.post is None - assert from_db.metadata.body == "print('hello' + a)" - assert from_db.metadata.definition == "hello a" - assert from_db.metadata.definition_type == "bnf" - assert len(from_db.metadata.variables) == 1 - assert from_db.metadata.variables[0] == ("a", None) - assert "a" in from_db.values + assert from_db.get_metadata().key == f"hello {VARIABLE_PREFIX}0" + assert from_db.get_metadata().name == "hello a" + assert from_db.get_metadata().where == "isinstance(a, str )" + assert from_db.get_metadata().pre == "a is not None" + assert from_db.get_metadata().post is None + assert from_db.get_metadata().body == "print('hello' + a)" + assert from_db.get_metadata().definition == "hello a" + assert from_db.get_metadata().definition_type == "bnf" + assert len(from_db.get_metadata().variables) == 1 + assert from_db.get_metadata().variables[0] == ("a", None) + assert "a" in from_db.values() - assert from_db.compiled == {} # ast is not saved in db + assert from_db.get_compiled() == {} # ast is not saved in db @pytest.mark.parametrize("expression, name, expected", [ ("isinstance(a, str)", "a b", {"a"}), @@ -195,39 +195,39 @@ class TestAddConceptEvaluator(TestUsingMemoryBasedSheerka): ret_val = self.get_concept_part(expression) context = self.get_context() - assert AddConceptEvaluator.get_variables(context.sheerka, ret_val, name.split()) == expected + assert DefConceptEvaluator.get_variables(context.sheerka, ret_val, name.split()) == expected def test_i_can_get_variables_when_keywords(self): sheerka, context = self.init_concepts() def_concept = self.get_def_concept("condition pre").value.value - name_to_use = AddConceptEvaluator.get_name_to_use(def_concept) + name_to_use = DefConceptEvaluator.get_name_to_use(def_concept) concept_part = self.get_concept_part("pre") - assert AddConceptEvaluator.get_variables(context.sheerka, concept_part, name_to_use) == {"pre"} + assert DefConceptEvaluator.get_variables(context.sheerka, concept_part, name_to_use) == {"pre"} def test_i_cannot_get_variables_from_python_node_when_name_has_only_one_token(self): ret_val = self.get_concept_part("isinstance(a, str)") context = self.get_context() - assert AddConceptEvaluator.get_variables(context.sheerka, ret_val, ["a"]) == [] + assert DefConceptEvaluator.get_variables(context.sheerka, ret_val, ["a"]) == [] def test_i_can_get_variables_from_definition(self): parsing_expression = Sequence(ConceptExpression('mult'), ZeroOrMore(Sequence(StrMatch("+"), ConceptExpression("add")))) ret_val = self.get_return_value("mult (('+'|'-') add)?", parsing_expression) - assert AddConceptEvaluator.get_variables(self.get_sheerka(), ret_val, []) == {"add", "mult"} + assert DefConceptEvaluator.get_variables(self.get_sheerka(), ret_val, []) == {"add", "mult"} def test_concept_that_references_itself_is_correctly_created(self): context = self.get_context() def_concept_as_return_value = self.get_def_concept("foo", body="foo") - ret_val = AddConceptEvaluator().eval(context, def_concept_as_return_value) + ret_val = DefConceptEvaluator().eval(context, def_concept_as_return_value) assert ret_val.status new_concept = ret_val.body.body assert new_concept.name == 'foo' - assert new_concept.metadata.body == 'foo' - assert new_concept.values == {} - assert new_concept.metadata.variables == [] + assert new_concept.get_metadata().body == 'foo' + assert new_concept.values() == {} + assert new_concept.get_metadata().variables == [] diff --git a/tests/evaluators/test_FormatRuleEvaluator.py b/tests/evaluators/test_FormatRuleEvaluator.py new file mode 100644 index 0000000..5c6b5ac --- /dev/null +++ b/tests/evaluators/test_FormatRuleEvaluator.py @@ -0,0 +1,51 @@ +import pytest +from core.builtin_concepts import ReturnValueConcept, ParserResultConcept, BuiltinConcepts +from core.rule import Rule, RuleMetadata +from core.sheerka.services.SheerkaExecute import ParserInput +from core.tokenizer import Tokenizer +from evaluators.FormatRuleEvaluator import FormatRuleEvaluator +from parsers.FormatRuleParser import FormatRuleNode, FormatRuleParser + +from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka + + +class TestFormatRuleEvaluator(TestUsingMemoryBasedSheerka): + + @staticmethod + def get_ret_val_from_rule(rule_def): + tokens = {k: list(Tokenizer(k.value + " " + v, yield_eof=False)) for k, v in rule_def.items()} + for v in tokens.values(): + del v[1] + + node = FormatRuleNode(tokens) + return ReturnValueConcept("parsers.FormatRule", True, ParserResultConcept(value=node)) + + @pytest.mark.parametrize("ret_val, expected", [ + (ReturnValueConcept("name", True, ParserResultConcept(value=FormatRuleNode({}))), True), + (ReturnValueConcept("name", True, ParserResultConcept(value="other object")), False), + (ReturnValueConcept("name", False, ParserResultConcept(value=FormatRuleNode({}))), False), + (ReturnValueConcept("name", False, FormatRuleNode({})), False), + ]) + def test_i_can_match(self, ret_val, expected): + context = self.get_context() + assert FormatRuleEvaluator().matches(context, ret_val) == expected + + def test_i_can_eval(self): + sheerka, context = self.init_concepts() + text = "when isinstance(value, __EXPLANATION) print list(value)" + ret_val = FormatRuleParser().parse(context, ParserInput(text)) + + res = FormatRuleEvaluator().eval(context, ret_val) + + assert res.status + assert sheerka.isinstance(res.body, BuiltinConcepts.NEW_RULE) + assert isinstance(res.body.body, Rule) + assert res.body.body.metadata == RuleMetadata("print", + None, + "isinstance(value, __EXPLANATION)", + "list(value)", + id=res.body.body.metadata.id, # no need to compare the id + is_compiled=True, + is_enabled=True) + assert res.body.body.compiled_predicate is not None + assert res.body.body.compiled_action is not None diff --git a/tests/evaluators/test_LexerNodeEvaluator.py b/tests/evaluators/test_LexerNodeEvaluator.py index bfeefbc..a7d9187 100644 --- a/tests/evaluators/test_LexerNodeEvaluator.py +++ b/tests/evaluators/test_LexerNodeEvaluator.py @@ -69,7 +69,7 @@ class TestLexerNodeEvaluator(TestUsingMemoryBasedSheerka): assert wrapper.parser == evaluator assert wrapper.source == "foo" assert return_value == foo - assert return_value.compiled[ConceptParts.BODY] == DoNotResolve("foo") + assert return_value.get_compiled()[ConceptParts.BODY] == DoNotResolve("foo") assert result.parents == [ret_val] def test_concept_python_node_is_returned_when_source_code(self): @@ -88,5 +88,5 @@ class TestLexerNodeEvaluator(TestUsingMemoryBasedSheerka): assert wrapper.source == "foo + 1" assert return_value == PythonNode('foo + 1', ast.parse("__C__foo__C__ + 1", mode="eval")) - assert return_value.concepts == {"__C__foo__C__": foo} + assert return_value.objects == {"__C__foo__C__": foo} assert result.parents == [ret_val] diff --git a/tests/evaluators/test_OneSuccessEvaluator.py b/tests/evaluators/test_OneSuccessEvaluator.py index f303c0a..04e8d36 100644 --- a/tests/evaluators/test_OneSuccessEvaluator.py +++ b/tests/evaluators/test_OneSuccessEvaluator.py @@ -1,13 +1,16 @@ import pytest - -from core.builtin_concepts import ReturnValueConcept, BuiltinConcepts +from core.builtin_concepts import ReturnValueConcept, BuiltinConcepts, ParserResultConcept from core.concept import Concept +from core.rule import Rule from evaluators.OneSuccessEvaluator import OneSuccessEvaluator + from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka -def r(value, status=True): - return ReturnValueConcept(value, status, value) +def r(name, status=True, value=None): + if isinstance(value, Rule): + value = ParserResultConcept(name, value=value, try_parsed=value) + return ReturnValueConcept(name, status, value or name) reduce_requested = ReturnValueConcept("some_name", True, Concept(key=BuiltinConcepts.REDUCE_REQUESTED)) @@ -25,6 +28,7 @@ class TestOneSuccessEvaluator(TestUsingMemoryBasedSheerka): ([r("evaluators.success"), r("evaluators.another success"), reduce_requested], False), ([r("evaluators.one success"), r("other success"), r("failed", False), reduce_requested], True), ([r("evaluators.one success"), r("parsers.success"), reduce_requested], False), + ([r("evaluators.one success"), r("parsers.success"), reduce_requested], False), ]) def test_i_can_match(self, return_values, expected): context = self.get_context() diff --git a/tests/evaluators/test_PythonEvaluator.py b/tests/evaluators/test_PythonEvaluator.py index e00c2b1..6aa5337 100644 --- a/tests/evaluators/test_PythonEvaluator.py +++ b/tests/evaluators/test_PythonEvaluator.py @@ -2,19 +2,20 @@ import ast import pytest from core.builtin_concepts import ReturnValueConcept, ParserResultConcept, BuiltinConcepts -from core.concept import Concept, CB, NotInit +from core.concept import Concept, CB from core.sheerka.services.SheerkaExecute import ParserInput from core.tokenizer import Tokenizer -from evaluators.PythonEvaluator import PythonEvaluator, PythonEvalError +from evaluators.PythonEvaluator import PythonEvaluator, PythonEvalError, NamesWithAttributesVisitor from parsers.BaseNodeParser import SourceCodeNode, SourceCodeWithConceptNode +from parsers.FunctionParser import FunctionParser from parsers.PythonParser import PythonNode, PythonParser from parsers.PythonWithConceptsParser import PythonWithConceptsParser from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka -def get_concept_name(concept): - return concept.name +def get_obj_name(obj): + return obj.name def get_source_code_node(source_code, concepts=None): @@ -36,7 +37,7 @@ def get_source_code_node(source_code, concepts=None): tokens = list(Tokenizer(source_code, yield_eof=False)) return SourceCodeNode(0, len(tokens), tokens, python_node=python_node) else: - python_node.concepts = concepts + python_node.objects = concepts scwcn = SourceCodeWithConceptNode(None, None) scwcn.python_node = python_node return scwcn @@ -62,10 +63,13 @@ class TestPythonEvaluator(TestUsingMemoryBasedSheerka): assert PythonEvaluator().matches(context, ret_val) == expected @pytest.mark.parametrize("text, expected", [ - ("1 + 1", 2), - ("test()", "I have access to Sheerka !"), + # ("1 + 1", 2), + # ("test()", "I have access to Sheerka !"), ("sheerka.test()", "I have access to Sheerka !"), ("a=10\na", 10), + ("Concept('foo')", Concept('foo')), + ("BuiltinConcepts.NOP", BuiltinConcepts.NOP), + ("in_context(BuiltinConcepts.EVAL_QUESTION_REQUESTED)", False), ]) def test_i_can_eval(self, text, expected): context = self.get_context() @@ -76,6 +80,29 @@ class TestPythonEvaluator(TestUsingMemoryBasedSheerka): assert evaluated.status assert evaluated.value == expected + def test_i_can_eval_isinstance(self): + sheerka, context, foo = self.init_concepts("foo") + parsed = PythonParser().parse(context, ParserInput("isinstance('foo', str)")) + evaluated = PythonEvaluator().eval(context, parsed) + assert evaluated.status + assert evaluated.value + + parsed = PythonParser().parse(context, ParserInput("isinstance(foo, 'foo')")) + evaluated = PythonEvaluator().eval(context, parsed) + assert evaluated.status + assert evaluated.value + + def test_i_can_eval_context_obj_properties(self): + sheerka, context, foo = self.init_concepts(Concept("foo").def_var("a", "hello world!").auto_init()) + context = self.get_context() + parsed = PythonParser().parse(context, ParserInput("a")) + + context.obj = foo + evaluated = PythonEvaluator().eval(context, parsed) + + assert evaluated.status + assert evaluated.value == "hello world!" + @pytest.mark.parametrize("source_code_node, expected", [ (get_source_code_node("1 + 1"), 2), (get_source_code_node("one + one", {"one": Concept("one", body="1")}), 2) @@ -89,14 +116,14 @@ class TestPythonEvaluator(TestUsingMemoryBasedSheerka): assert evaluated.status assert evaluated.value == expected - def test_i_can_eval_using_context(self): + def test_i_can_eval_a_method_that_requires_the_context(self): context = self.get_context() - parsed = PythonParser().parse(context, ParserInput("test_using_context('value for param1', 10)")) + parsed = PythonParser().parse(context, ParserInput("test_using_context('value for param')")) evaluated = PythonEvaluator().eval(context, parsed) assert evaluated.status - assert evaluated.value.startswith("I have access to Sheerka ! param1='value for param1', param2=10, event=") + assert evaluated.value == "I have access to Sheerka ! param='value for param', event='xxx'." def test_i_can_eval_using_context_when_self_is_not_sheerka(self): sheerka, context = self.init_concepts() @@ -107,21 +134,6 @@ class TestPythonEvaluator(TestUsingMemoryBasedSheerka): assert evaluated.status assert sheerka.has_key("foo") - @pytest.mark.parametrize("concept", [ - Concept("foo"), - Concept("foo", body="2"), - Concept("foo").def_var("prop", "'a'"), - Concept("foo", body="bar") - ]) - def test_simple_concepts_are_not_for_me(self, concept): - context = self.get_context() - context.sheerka.add_in_cache(Concept("foo")) - - parsed = PythonParser().parse(context, ParserInput("foo")) - evaluated = PythonEvaluator().eval(context, parsed) - - assert not evaluated.status - assert context.sheerka.isinstance(evaluated.value, BuiltinConcepts.NOT_FOR_ME) def test_i_can_eval_ast_expression_that_references_concepts(self): """ @@ -167,10 +179,10 @@ class TestPythonEvaluator(TestUsingMemoryBasedSheerka): def test_i_can_eval_concept_token(self): context = self.get_context() context.sheerka.add_in_cache(Concept("foo", body="2")) + context.add_to_short_term_memory("get_obj_name", get_obj_name) - parsed = PythonParser().parse(context, ParserInput("get_concept_name(c:foo:)")) + parsed = PythonParser().parse(context, ParserInput("get_obj_name(c:foo:)")) python_evaluator = PythonEvaluator() - python_evaluator.globals["get_concept_name"] = get_concept_name evaluated = python_evaluator.eval(context, parsed) assert evaluated.status @@ -210,7 +222,8 @@ class TestPythonEvaluator(TestUsingMemoryBasedSheerka): evaluated = python_evaluator.eval(context, parsed) assert evaluated.status - assert sheerka.get_concepts_weights(BuiltinConcepts.PRECEDENCE) == {'1001': 1, '1002': 2} + assert sheerka.get_concepts_weights(BuiltinConcepts.PRECEDENCE) == {'c:__var__0 plus __var__1|1001:': 1, + 'c:__var__0 mult __var__1|1002:': 2} def test_i_can_define_variables(self): sheerka, context = self.init_concepts() @@ -261,7 +274,7 @@ class TestPythonEvaluator(TestUsingMemoryBasedSheerka): evaluated = PythonEvaluator().eval(context, parsed) assert not evaluated.status - assert context.sheerka.isinstance(evaluated.value, BuiltinConcepts.TOO_MANY_ERRORS) + assert context.sheerka.isinstance(evaluated.value, BuiltinConcepts.ERROR) error0 = evaluated.body.body[0] assert isinstance(error0, PythonEvalError) @@ -275,21 +288,6 @@ class TestPythonEvaluator(TestUsingMemoryBasedSheerka): assert error1.error.args[0] == 'can only concatenate str (not "int") to str' assert error1.concepts == {'foo': 'string'} - def test_i_do_not_include_not_initialized_variables_when_evaluating(self): - sheerka, context, foo = self.init_concepts( - Concept("foo a", pre="a == 'True'").def_var("a", "'True'").def_var("b")) - - foo.set_value("b", "'Initialized!'") - context.obj = foo - - assert foo.get_value("a") == NotInit - assert foo.get_value("b") == "'Initialized!'" - - my_globals = {} - PythonEvaluator().update_globals_with_context(my_globals, context) - - assert my_globals == {"self": foo, "b": "'Initialized!'"} - def test_i_can_use_sheerka_locals(self): sheerka, context = self.init_concepts() @@ -307,11 +305,47 @@ class TestPythonEvaluator(TestUsingMemoryBasedSheerka): def test_i_can_eval_concept_with_ret(self): sheerka, context, one, the = self.init_concepts("one", Concept("the a", ret="a").def_var("a")) - ret_val = get_ret_val_from_source_code(context, "test_using_context(one, the one)", { - "the one": self.get_concept_instance(sheerka, the, a="one"), - "one": self.get_concept_instance(sheerka, "one") + ret_val = get_ret_val_from_source_code(context, "test_using_context(the one)", { + "the one": self.get_concept_instance(sheerka, the, a="one") }) evaluated = PythonEvaluator().eval(context, ret_val) assert evaluated.status - assert evaluated.value.startswith("I have access to Sheerka ! param1=(1001)one, param2=(1001)one, event=") + assert evaluated.value == "I have access to Sheerka ! param=(1001)one, event='xxx'." + + def test_i_can_eval_rules_from_python_parser(self): + sheerka, context = self.init_concepts() + parsed_ret_val = PythonParser().parse(context, ParserInput("r:|1:.id")) + + assert parsed_ret_val.status + + evaluated = PythonEvaluator().eval(context, parsed_ret_val) + assert evaluated.status + assert evaluated.value == "1" + + def test_i_can_eval_rules_from_function_parser(self): + context = self.get_context() + context.add_to_short_term_memory("get_obj_name", get_obj_name) + + parsed = FunctionParser().parse(context, ParserInput("get_obj_name(r:|1:)")) + python_evaluator = PythonEvaluator() + evaluated = python_evaluator.eval(context, parsed) + + assert evaluated.status + assert evaluated.value == "Print return values" + + @pytest.mark.parametrize("text, expected", [ + ("foo.bar.baz", [["foo", "bar", "baz"]]), + ("foo.bar.baz; one.two.three", [["foo", "bar", "baz"]]), + ("foo.bar.baz; one.two.three; foo.bar", [["foo", "bar", "baz"], ["foo", "bar"]]), + ("one.two.three", []), + ("foo", [["foo"]]), + ("foo.bar[1].baz", [["foo", "bar", "baz"]]), + ("foo[1].bar.baz", [["foo", "bar", "baz"]]), + ]) + def test_names_with_attributes_visitor(self, text, expected): + ast_ = ast.parse(text, "", mode="exec") + visitor = NamesWithAttributesVisitor() + + sequence = visitor.get_sequences(ast_, "foo") + assert sequence == expected diff --git a/tests/evaluators/test_ReturnBodyEvaluator.py b/tests/evaluators/test_ReturnBodyEvaluator.py new file mode 100644 index 0000000..b067ca3 --- /dev/null +++ b/tests/evaluators/test_ReturnBodyEvaluator.py @@ -0,0 +1,50 @@ +from core.builtin_concepts import BuiltinConcepts +from core.concept import Concept +from evaluators.ReturnBodyEvaluator import ReturnBodyEvaluator + +from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka + + +class TestReturnBodyEvaluator(TestUsingMemoryBasedSheerka): + def test_i_can_match(self): + sheerka, root_context = self.init_concepts() + root_context.add_to_global_hints(BuiltinConcepts.RETURN_BODY_REQUESTED) + + context = root_context.push(BuiltinConcepts.PROCESSING, None) + sub_context = context.push(BuiltinConcepts.PROCESSING, None) + sub_sub_context = sub_context.push(BuiltinConcepts.PROCESSING, None) + + evaluator = ReturnBodyEvaluator() + + assert evaluator.matches(root_context, []) # no parent, real root + assert evaluator.matches(context, []) # no BuiltinConcepts.PROCESSING, is root for unit test + assert evaluator.matches(sub_context, []) # only one BuiltinConcepts.PROCESSING, is root in prod + assert not evaluator.matches(sub_sub_context, []) # not a root + + def test_i_can_eval_concept(self): + sheerka, root_context = self.init_concepts() + root_context.add_to_global_hints(BuiltinConcepts.RETURN_BODY_REQUESTED) + context = root_context.push(BuiltinConcepts.PROCESSING, None) + evaluator = ReturnBodyEvaluator() + + return_values = [ + sheerka.ret("Who", True, Concept("foo", body="status -> will be reduce").auto_init()), + sheerka.ret("Who", False, Concept("foo", body="status -> won't be reduced").auto_init()), + sheerka.ret("Who", False, Concept("foo", body="body not init -> won't be reduced")), + ] + + res = evaluator.eval(context, return_values) + + assert res == [sheerka.ret(evaluator.name, True, "status -> will be reduce")] + assert res[0].parents == [return_values[0]] + + def test_i_do_not_reduce_containers_concepts(self): + sheerka, root_context = self.init_concepts() + root_context.add_to_global_hints(BuiltinConcepts.RETURN_BODY_REQUESTED) + context = root_context.push(BuiltinConcepts.PROCESSING, None) + evaluator = ReturnBodyEvaluator() + + container = sheerka.new(BuiltinConcepts.EXPLANATION, body=[]) # body is initialized + return_values = [sheerka.ret("who", True, container)] + + assert evaluator.eval(context, return_values) is None diff --git a/tests/evaluators/test_RuleEvaluator.py b/tests/evaluators/test_RuleEvaluator.py new file mode 100644 index 0000000..c1bf326 --- /dev/null +++ b/tests/evaluators/test_RuleEvaluator.py @@ -0,0 +1,70 @@ +import pytest +from core.builtin_concepts import ReturnValueConcept, ParserResultConcept, BuiltinConcepts +from core.rule import Rule, ACTION_TYPE_DEFERRED +from evaluators.RuleEvaluator import RuleEvaluator + +from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka + + +class TestRuleEvaluator(TestUsingMemoryBasedSheerka): + + @pytest.mark.parametrize("ret_val, expected", [ + (ReturnValueConcept("some_name", True, ParserResultConcept(value=[Rule()])), True), + (ReturnValueConcept("some_name", True, ParserResultConcept(value="Not a concept")), False), + (ReturnValueConcept("some_name", True, ParserResultConcept(value=[])), False), + (ReturnValueConcept("some_name", True, Rule), False), + (ReturnValueConcept("some_name", False, ParserResultConcept(value=Rule())), False), + (ReturnValueConcept("some_name", False, ParserResultConcept(value=[Rule()])), False), + ]) + def test_i_can_match(self, ret_val, expected): + context = self.get_context() + assert RuleEvaluator().matches(context, ret_val) == expected + + @pytest.mark.parametrize("body, expected", [ + ([Rule("exec", "rule1")], Rule("exec", "rule1")), + ([Rule("exec", "rule1"), Rule("exec", "rule2")], [Rule("exec", "rule1"), Rule("exec", "rule2")]), + + ]) + def test_i_can_evaluate(self, body, expected): + context = self.get_context() + ret_val = ReturnValueConcept("some_name", True, ParserResultConcept(value=body), True) + evaluator = RuleEvaluator() + + result = evaluator.eval(context, ret_val) + + assert result.who == evaluator.name + assert result.status + assert result.value == expected + assert result.parents == [ret_val] + + def test_i_can_evaluate_deferred_rules(self): + context = self.get_context() + + rule = Rule().set_id("i") + rule.metadata.action_type = ACTION_TYPE_DEFERRED + ret_val = ReturnValueConcept("some_name", True, ParserResultConcept(value=[rule]), True) + context.add_to_short_term_memory("i", 1) + evaluator = RuleEvaluator() + + result = evaluator.eval(context, ret_val) + + assert result.who == evaluator.name + assert result.status + assert result.value == context.sheerka.get_rule_by_id("1") + assert result.parents == [ret_val] + + def test_i_cannot_evaluate_deferred_rule_if_the_rule_is_unknown(self): + sheerka, context = self.init_concepts() + + rule = Rule().set_id("unknown variable") + rule.metadata.action_type = ACTION_TYPE_DEFERRED + ret_val = ReturnValueConcept("some_name", True, ParserResultConcept(value=[rule]), True) + evaluator = RuleEvaluator() + + result = evaluator.eval(context, ret_val) + + assert result.who == evaluator.name + assert not result.status + assert sheerka.isinstance(result.value, BuiltinConcepts.UNKNOWN_RULE) + assert result.value.body == [('id', 'unknown variable')] + assert result.parents == [ret_val] diff --git a/tests/non_reg/test_sheerka_non_reg.py b/tests/non_reg/test_sheerka_non_reg.py index 068d866..2699317 100644 --- a/tests/non_reg/test_sheerka_non_reg.py +++ b/tests/non_reg/test_sheerka_non_reg.py @@ -56,7 +56,7 @@ class TestSheerkaNonRegMemory(TestUsingMemoryBasedSheerka): # sanity check evaluated = sheerka.evaluate_concept(self.get_context(eval_body=True), return_value) - assert evaluated == simplec("un", simplec("one", BuiltinConcepts.NOT_INITIALIZED)) + assert evaluated == simplec("un", simplec("one", NotInit)) def test_i_can_recognize_concept_with_no_body(self): sheerka = self.get_sheerka() @@ -95,9 +95,9 @@ as: """ expected = self.get_default_concept() - expected.metadata.id = "1001" - expected.metadata.desc = None - expected.metadata.variables = [("a", None), ("b", None)] + expected.get_metadata().id = "1001" + expected.get_metadata().desc = None + expected.get_metadata().variables = [("a", None), ("b", None)] expected.init_key() sheerka = self.get_sheerka(cache_only=False) @@ -110,7 +110,7 @@ as: concept_saved = res[0].value.body for prop in PROPERTIES_TO_SERIALIZE: - assert getattr(concept_saved.metadata, prop) == getattr(expected.metadata, prop) + assert getattr(concept_saved.get_metadata(), prop) == getattr(expected.get_metadata(), prop) # cache is up to date assert sheerka.has_key(concept_saved.key) @@ -136,7 +136,7 @@ as: res = sheerka.evaluate_user_input("def concept a xx b as a plus b") expected = Concept(name="a xx b", body="a plus b").def_var("a").def_var("b").init_key() - expected.metadata.id = "1001" + expected.get_metadata().id = "1001" assert len(res) == 1 assert res[0].status @@ -145,7 +145,7 @@ as: concept_saved = res[0].value.body for prop in PROPERTIES_TO_SERIALIZE: - assert getattr(concept_saved.metadata, prop) == getattr(expected.metadata, prop) + assert getattr(concept_saved.get_metadata(), prop) == getattr(expected.get_metadata(), prop) assert sheerka.has_key(concept_saved.key) @@ -212,9 +212,9 @@ as: # sanity check evaluated = sheerka.evaluate_concept(self.get_context(eval_body=True), res[0].value) assert evaluated.body == "hello foo" - assert evaluated.metadata.is_evaluated + assert evaluated.get_metadata().is_evaluated assert evaluated.get_value("a") == simplec("foo", "foo") - assert evaluated.get_value("a").metadata.is_evaluated + assert evaluated.get_value("a").get_metadata().is_evaluated def test_i_can_recognize_duplicate_concepts_with_same_value(self): # when multiple result, choose the one that is the more specific (that has the less variables) @@ -227,7 +227,7 @@ as: assert len(res) == 1 assert res[0].status assert sheerka.isinstance(res[0].body, "hello foo") - assert res[0].value.body == BuiltinConcepts.NOT_INITIALIZED + assert res[0].value.body == NotInit assert res[0].who == sheerka.get_evaluator_name(OneSuccessEvaluator.NAME) def test_i_cannot_manage_duplicate_concepts_when_the_values_are_different(self): @@ -297,9 +297,9 @@ as: # sanity check evaluated = sheerka.evaluate_concept(self.get_context(sheerka=sheerka, eval_body=True), return_value) assert evaluated.body == "one three" - assert evaluated.metadata.is_evaluated + assert evaluated.get_metadata().is_evaluated assert evaluated.get_value("a") == sheerka.new(concept_a.key, body="one").init_key() - assert evaluated.get_value("a").metadata.is_evaluated + assert evaluated.get_value("a").get_metadata().is_evaluated @pytest.mark.parametrize("user_input", [ "def concept greetings from def hello a where a", @@ -315,7 +315,7 @@ as: concept_found = res[0].value assert sheerka.isinstance(concept_found, greetings) assert concept_found.get_value("a") == "foo" - assert concept_found.metadata.need_validation + assert concept_found.get_metadata().need_validation res = sheerka.evaluate_user_input("greetings") assert len(res) == 1 @@ -323,7 +323,7 @@ as: concept_found = res[0].value assert sheerka.isinstance(concept_found, greetings) assert concept_found.get_value("a") == NotInit - assert not concept_found.metadata.need_validation + assert not concept_found.get_metadata().need_validation @pytest.mark.parametrize("desc, definitions", [ ("Simple form", [ @@ -718,7 +718,7 @@ as: assert len(res) == 1 assert res[0].status assert sheerka.isinstance(res[0].body, "foobar") - assert res[0].body.body == BuiltinConcepts.NOT_INITIALIZED + assert res[0].body.body == NotInit res = sheerka.evaluate_user_input("eval foo bar") assert len(res) == 1 @@ -871,14 +871,17 @@ as: res = sheerka.evaluate_user_input("get_concepts_weights('some_prop')") assert res[0].status - assert res[0].body == {'1001': 1, '1002': 2, '1003': 3} + assert res[0].body == {'c:one|1001:': 1, 'c:two|1002:': 2, 'c:three|1003:': 3} # test i use a concept to define relation sheerka.evaluate_user_input("def concept a > b as set_is_greater_than('some_prop', a, b)") res = sheerka.evaluate_user_input("eval four > three") assert res[0].status - assert sheerka.get_concepts_weights("some_prop") == {'1001': 1, '1002': 2, '1003': 3, '1004': 4} + assert sheerka.get_concepts_weights("some_prop") == {'c:one|1001:': 1, + 'c:two|1002:': 2, + 'c:three|1003:': 3, + 'c:four|1004:': 4} def test_i_can_evaluate_expression_when_using_token_concept(self): sheerka, context, one, two, three, is_less_than = self.init_concepts( @@ -899,14 +902,14 @@ as: res = sheerka.evaluate_user_input(expression) assert res[0].status assert sheerka.isinstance(res[0].body, BuiltinConcepts.SUCCESS) - assert sheerka.get_concepts_weights("some_prop") == {'1001': 1, '1002': 2} + assert sheerka.get_concepts_weights("some_prop") == {'c:one|1001:': 1, 'c:two|1002:': 2} # it now also works using the concepts names expression = "eval two < three" res = sheerka.evaluate_user_input(expression) assert res[0].status assert sheerka.isinstance(res[0].body, BuiltinConcepts.SUCCESS) - assert sheerka.get_concepts_weights("some_prop") == {'1001': 1, '1002': 2, '1003': 3} + assert sheerka.get_concepts_weights("some_prop") == {'c:one|1001:': 1, 'c:two|1002:': 2, 'c:three|1003:': 3} def test_i_can_detect_multiple_errors_when_evaluating_a_concept(self): sheerka, context, foo, plus_one = self.init_concepts( @@ -918,7 +921,7 @@ as: assert not res[0].status assert context.sheerka.isinstance(res[0].body, BuiltinConcepts.CONCEPT_EVAL_ERROR) - assert context.sheerka.isinstance(res[0].body.body, BuiltinConcepts.TOO_MANY_ERRORS) + assert context.sheerka.isinstance(res[0].body.body, BuiltinConcepts.ERROR) error0 = res[0].body.body.body[0] assert isinstance(error0, PythonEvalError) @@ -951,7 +954,7 @@ as: # simulate that sheerka was stopped and restarted sheerka.cache_manager.clear(sheerka.CONCEPTS_GRAMMARS_ENTRY) - sheerka.cache_manager.get(sheerka.CONCEPTS_BY_KEY_ENTRY, "twenties").compiled = {} + sheerka.cache_manager.get(sheerka.CONCEPTS_BY_KEY_ENTRY, "twenties").set_compiled({}) res = sheerka.evaluate_user_input("eval twenty one") assert res[0].status @@ -985,8 +988,8 @@ as: plus = res[0].body assert isinstance(plus, Concept) assert plus.name == "plus" - assert plus.compiled["a"] == sheerka.new("one") - assert plus.compiled["b"] == CC(the, a=sheerka.new("one")) + assert plus.get_compiled()["a"] == sheerka.new("one") + assert plus.get_compiled()["b"] == CC(the, a=sheerka.new("one")) res = sheerka.evaluate_user_input("eval one plus the one") assert res[0].status @@ -1063,7 +1066,7 @@ as: "def concept q from q ? as question(q)", "set_auto_eval(q)", "def concept is_a from x is a y as isa(x,y) pre in_context(BuiltinConcepts.EVAL_QUESTION_REQUESTED)", - "set_is_greater_than(BuiltinConcepts.PRECEDENCE, c:is_a:, c:q:)", + "set_is_greater_than(BuiltinConcepts.PRECEDENCE, c:is_a:, c:q:, 'Sya')", ] sheerka = self.init_scenario(init) @@ -1185,20 +1188,20 @@ class TestSheerkaNonRegFile(TestUsingFileBasedSheerka): saved_concept = sheerka.sdp.get(sheerka.CONCEPTS_BY_KEY_ENTRY, "plus") assert saved_concept.key == "plus" - assert saved_concept.metadata.definition == "a ('plus' plus)?" - assert "a" in saved_concept.values - assert "plus" in saved_concept.values + assert saved_concept.get_metadata().definition == "a ('plus' plus)?" + assert "a" in saved_concept.values() + assert "plus" in saved_concept.values() expected_bnf = Sequence( ConceptExpression(concept_a, rule_name="a"), Optional(Sequence(StrMatch("plus"), ConceptExpression("plus")))) new_concept = res[0].value.body - assert new_concept.metadata.name == "plus" - assert new_concept.metadata.definition == "a ('plus' plus)?" - assert new_concept.bnf == expected_bnf - assert "a" in new_concept.values - assert "plus" in new_concept.values + assert new_concept.get_metadata().name == "plus" + assert new_concept.get_metadata().definition == "a ('plus' plus)?" + assert new_concept.get_bnf() == expected_bnf + assert "a" in new_concept.values() + assert "plus" in new_concept.values() def test_i_can_recognize_bnf_definitions_from_separate_instances(self): sheerka = self.get_sheerka() diff --git a/tests/out/__init__.py b/tests/out/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/out/test_SheerkaOut.py b/tests/out/test_SheerkaOut.py new file mode 100644 index 0000000..b2a08c4 --- /dev/null +++ b/tests/out/test_SheerkaOut.py @@ -0,0 +1,381 @@ +import pytest +from core.builtin_concepts import ReturnValueConcept, BuiltinConcepts +from core.concept import Concept +from core.rule import Rule +from core.sheerka.services.SheerkaOut import SheerkaOut +from core.sheerka.services.SheerkaRuleManager import FormatAstRawText, FormatAstVariable, FormatAstSequence, \ + FormatAstColor, FormatAstVariableNotFound, FormatAstList +from core.utils import flatten_all_children + +from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka + +seq = FormatAstSequence +raw = FormatAstRawText +var = FormatAstVariable + + +class TestSheerkaOut(TestUsingMemoryBasedSheerka): + + def init_service_with_rules(self, *rules, **kwargs): + sheerka, context, *rules = self.init_format_rules(*rules, **kwargs) + service = sheerka.services[SheerkaOut.NAME] + + return sheerka, context, service, *rules + + @pytest.mark.parametrize("rule, expected", [ + (("__ret", "hello world"), FormatAstRawText("hello world")), + (("__ret", "{__ret_value}"), FormatAstVariable("__ret_value", None, 1)), + (("__ret", "{status}"), FormatAstVariable("status", None, True)), + (("__ret", "{foo}"), FormatAstVariableNotFound("foo")), + (("__ret", "hello world {__ret_value} !"), seq([FormatAstRawText("hello world "), + FormatAstVariable("__ret_value", None, 1), + FormatAstRawText(" !")])), + (("__ret", "red(__ret_value)"), FormatAstColor("red", FormatAstVariable("__ret_value", None, 1))), + (("__ret", "blue('hello world {__ret_value} !')"), FormatAstColor("blue", + seq([FormatAstRawText("hello world "), + FormatAstVariable("__ret_value", None, + 1), + FormatAstRawText(" !")]))), + (("__ret", "list(foo)"), FormatAstVariableNotFound("foo")), + (("__ret", "{__ret_value:3}"), FormatAstVariable("__ret_value", "3", " 1")) + ]) + def test_i_can_develop_when_simple_rules(self, rule, expected): + sheerka, context, service, rule = self.init_service_with_rules(rule) + ret = sheerka.ret("Test", True, 1) + + res = service.create_out_tree(context, ret) + + assert res == expected + + def test_i_can_develop_rules_of_rules_using_obj_type(self): + sheerka, context, service, *rules = self.init_service_with_rules( + ("__ret", "status: {status}, value: {__ret_value}"), # ret_value is a 'foo' + ("isinstance(__obj, 'foo')", "{id}-{name}:{body}") + ) + sheerka.create_new_concept(context, Concept("foo")) + foo = sheerka.new("foo", body="hello world") + ret = sheerka.ret("Test", True, foo) + + res = service.create_out_tree(context, ret) + + assert res == seq([FormatAstRawText("status: "), + FormatAstVariable("status", None, True), + FormatAstRawText(", value: "), + FormatAstVariable("__ret_value", None, seq([FormatAstVariable("id", None, "1001"), + FormatAstRawText("-"), + FormatAstVariable("name", None, "foo"), + FormatAstRawText(":"), + FormatAstVariable("body", None, + "hello world")]))]) + + @pytest.mark.parametrize('second_rule', [ + ("__ret.body", "{id}-{name}:{body}"), + ("body", "{id}-{name}:{body}"), + ]) + def test_i_can_develop_rules_of_rules_using_obj_names(self, second_rule): + sheerka, context, service, *rules = self.init_service_with_rules( + ("__ret", "status: {status}, value: {__ret.body}"), + second_rule + ) + sheerka.create_new_concept(context, Concept("foo")) + foo = sheerka.new("foo", body="hello world") + ret = sheerka.ret("Test", True, foo) + + res = service.create_out_tree(context, ret) + + assert res == seq([FormatAstRawText("status: "), + FormatAstVariable("status", None, True), + FormatAstRawText(", value: "), + FormatAstVariable("__ret.body", None, seq([FormatAstVariable("id", None, "1001"), + FormatAstRawText("-"), + FormatAstVariable("name", None, "foo"), + FormatAstRawText(":"), + FormatAstVariable("body", None, + "hello world")]))]) + + def test_i_can_develop_list(self): + sheerka, context, service, *rules = self.init_service_with_rules(("isinstance(__obj, list)", "list(__obj)")) + lst = list(flatten_all_children(context, lambda item: item.achildren))[:3] + + res = service.create_out_tree(context, lst) + assert res == FormatAstList(variable="__obj", items=[FormatAstVariable("__item", None, lst[0], 0), + FormatAstVariable("__item", None, lst[1], 1), + FormatAstVariable("__item", None, lst[2], 2)]) + + def test_i_can_develop_list_using_the_default_items_prop(self): + sheerka, context, service, *rules = self.init_service_with_rules(("isinstance(__obj, 'foo')", "list(__obj)")) + lst = list(flatten_all_children(context, lambda item: item.achildren))[:3] + foo = Concept("foo", body=lst, key="foo").auto_init() + + res = service.create_out_tree(context, foo) + assert res == FormatAstList(variable="__obj", items=[FormatAstVariable("__item", None, lst[0], 0), + FormatAstVariable("__item", None, lst[1], 1), + FormatAstVariable("__item", None, lst[2], 2)]) + + def test_i_can_develop_list_using_the_custom_items_prop(self): + sheerka, context, service, *rules = self.init_service_with_rules( + ("isinstance(__obj, 'foo')", "list(__obj, items_prop='custom')")) + lst = list(flatten_all_children(context, lambda item: item.achildren))[:3] + foo = Concept("foo", key="foo").def_var("custom").auto_init() + foo.set_value("custom", lst) + + res = service.create_out_tree(context, foo) + assert res == FormatAstList(variable="__obj", items_prop='custom', + items=[FormatAstVariable("__item", None, lst[0], 0), + FormatAstVariable("__item", None, lst[1], 1), + FormatAstVariable("__item", None, lst[2], 2)]) + + def test_not_a_list_are_resolved_into_variable_ast(self): + sheerka, context, service, *rules = self.init_service_with_rules(("isinstance(__obj, 'foo')", "list(__obj)")) + foo = Concept("foo", key="foo") + + res = service.create_out_tree(context, foo) + assert res == FormatAstVariable("__obj", None, foo) + + def test_i_can_develop_list_of_return_values(self): + sheerka, context, service, *rules = self.init_service_with_rules(("__rets", "list(__rets)")) + lst = [ReturnValueConcept("foo", True, "value for True"), + ReturnValueConcept("bar", True, Concept("bar", body="value for bar").auto_init()), + ReturnValueConcept("baz", False, sheerka.new(BuiltinConcepts.ERROR, body="baz in error")), + ] + + res = service.create_out_tree(context, lst) + assert res == FormatAstList(variable="__rets", items=[FormatAstVariable("__item", None, lst[0], 0), + FormatAstVariable("__item", None, lst[1], 1), + FormatAstVariable("__item", None, lst[2], 2)]) + + def test_rules_are_correctly_reset_when_list(self): + sheerka, context, service, *rules = self.init_service_with_rules( + ("__rets", "list(__rets)"), + ("__ret and not __ret.status", "red(__ret)") + ) + lst = [ + ReturnValueConcept("who", False, "value1"), + ReturnValueConcept("who", False, "value2"), + ] + + res = service.create_out_tree(context, lst) + assert res == FormatAstList(variable="__rets", items=[ + FormatAstVariable("__item", None, FormatAstColor("red", FormatAstVariable("__ret", None, lst[0])), 0), + FormatAstVariable("__item", None, FormatAstColor("red", FormatAstVariable("__ret", None, lst[1])), 1), + ]) + + def test_i_can_develop_list_with_recurse(self): + sheerka, context, service, *rules = self.init_service_with_rules( + ("__rets", "list(__rets, recurse_on='parents', recursion_depth=2)")) + r1111 = ReturnValueConcept("Test", True, "r1111") + r111 = ReturnValueConcept("Test", True, "r111", parents=[r1111]) + r11 = ReturnValueConcept("Test", True, "r11", parents=[r111]) + r1 = ReturnValueConcept("Test", True, "r1", parents=[r11]) + r22 = ReturnValueConcept("Test", True, "r22") + r2 = ReturnValueConcept("Test", True, "r2", parents=[r22]) + + lst = [r1, r2] + + res = service.create_out_tree(context, lst) + assert res == FormatAstList(variable="__rets", recurse_on='parents', recursion_depth=2, items=[ + FormatAstVariable("__item", None, r1, 0), + FormatAstList(variable="__parents", recurse_on='parents', recursion_depth=1, items=[ + FormatAstVariable("__item", None, r11, 0), + FormatAstList(variable="__parents", recurse_on='parents', recursion_depth=0, items=[ + FormatAstVariable("__item", None, r111, 0)]) + ]), + FormatAstVariable("__item", None, r2, 1), + FormatAstList(variable="__parents", recurse_on='parents', recursion_depth=1, items=[ + FormatAstVariable("__item", None, r22, 0)]), + ]) + + def test_i_can_develop_list_with_recurse_using_container_format_instr(self): + sheerka, context, service, *rules = self.init_service_with_rules( + ("isinstance(__obj, 'foo')", "list(__obj)")) + r1111 = ReturnValueConcept("Test", True, "r1111") + r111 = ReturnValueConcept("Test", True, "r111", parents=[r1111]) + r11 = ReturnValueConcept("Test", True, "r11", parents=[r111]) + r1 = ReturnValueConcept("Test", True, "r1", parents=[r11]) + r22 = ReturnValueConcept("Test", True, "r22") + r2 = ReturnValueConcept("Test", True, "r2", parents=[r22]) + + foo = Concept("foo", key="foo", body=[r1, r2]).auto_init() + foo.set_format_instr(recurse_on='parents', recursion_depth=2) + + res = service.create_out_tree(context, foo) + assert res == FormatAstList(variable="__obj", recurse_on='parents', recursion_depth=2, items=[ + FormatAstVariable("__item", None, r1, 0), + FormatAstList(variable="__parents", recurse_on='parents', recursion_depth=1, items=[ + FormatAstVariable("__item", None, r11, 0), + FormatAstList(variable="__parents", recurse_on='parents', recursion_depth=0, items=[ + FormatAstVariable("__item", None, r111, 0)]) + ]), + FormatAstVariable("__item", None, r2, 1), + FormatAstList(variable="__parents", recurse_on='parents', recursion_depth=1, items=[ + FormatAstVariable("__item", None, r22, 0)]), + ]) + + def test_i_can_develop_using_variable_properties(self): + sheerka, context, service, *rules = self.init_service_with_rules( + ("isinstance(__obj, Concept)", "{__obj.body}"), + ) + lst = Concept("bar", body="value for bar").auto_init() + + res = service.create_out_tree(context, lst) + assert res == FormatAstVariable("__obj.body", None, "value for bar") + + @pytest.mark.parametrize("rule, expected", [ + (("__ret", "red('hello world')"), FormatAstColor("red", FormatAstRawText("hello world"))), + (("__ret", "blue(__ret_value)"), FormatAstColor("blue", FormatAstVariable("__ret_value", None, 1))), + ]) + def test_i_can_develop_color(self, rule, expected): + sheerka, context, service, rule = self.init_service_with_rules(rule) + ret = sheerka.ret("Test", True, 1) + + res = service.create_out_tree(context, ret) + + assert res == expected + + @pytest.mark.parametrize("rule, expected", [ + (("__ret", "{__ret}"), FormatAstVariable("__ret", None, + ReturnValueConcept(who="Test", status=True, value=1))), + (("__ret", "magenta(__ret)"), + FormatAstColor("magenta", + FormatAstVariable("__ret", None, + ReturnValueConcept(who="Test", status=True, value=1)))), + ]) + def test_i_can_manage_infinite_recursion(self, rule, expected): + sheerka, context, service, rule = self.init_service_with_rules(rule) + ret = sheerka.ret("Test", True, 1) + + res = service.create_out_tree(context, ret) + + assert res == expected + + def test_i_can_manage_infinite_recursion_when_multiple_rules(self): + sheerka, context, service, *rules = self.init_service_with_rules( + Rule("print", None, "__ret", "{__ret.value}", 1), + Rule("print", None, "__ret", "white(__ret)", 2), + ) + ret = sheerka.ret("Test", True, "hello world!") + + res = service.create_out_tree(context, ret) + + assert res == FormatAstColor("white", + FormatAstVariable("__ret", None, + FormatAstVariable("__ret.value", None, "hello world!"))) + + def test_i_can_print_out_the_result(self, capsys): + sheerka, context, service, *rules = self.init_service_with_rules( + ("__rets", "list(__rets)"), + ("__ret", "status: {status}, value: {__ret_value}"), + ("isinstance(__obj, 'foo')", "{id}-{name}:{body}") + ) + sheerka.create_new_concept(context, Concept("foo")) + foo = sheerka.new("foo", body="hello world") + ret = sheerka.ret("Test", True, foo) + + service.process_return_values(context, [ret]) + captured = capsys.readouterr() + assert captured.out == "status: True, value: 1001-foo:hello world\n" + + def test_i_can_print_out_a_list_of_results(self, capsys): + sheerka, context, service, *rules = self.init_service_with_rules( + ("__rets", "list(__rets)"), + ("isinstance(__ret_value, 'foo')", "foo: {body}"), + ("isinstance(__ret_value, 'bar')", "bar: {body}"), + ) + sheerka.create_new_concept(context, Concept("foo")) + sheerka.create_new_concept(context, Concept("bar")) + sheerka.create_new_concept(context, Concept("baz")) + + rets = [sheerka.ret("Test", True, sheerka.new("foo", body="foo value")), + sheerka.ret("Test", True, sheerka.new("bar", body="bar value")), + sheerka.ret("Test", True, sheerka.new("baz", body="baz value")), ] + + service.process_return_values(context, rets) + captured = capsys.readouterr() + assert captured.out == """foo: (1001)foo +bar: (1002)bar +ReturnValue(who=Test, status=True, value=(1003)baz, message=None) +""" + + def test_i_can_print_out_a_list_with_recurse(self, capsys): + sheerka, context, service, *rules = self.init_service_with_rules( + ("__rets", "list(__rets, recurse_on='parents', recursion_depth=2)"), + ("__ret", "{__tab}{__ret}"), + ) + r1111 = ReturnValueConcept("Test", True, "r1111") + r111 = ReturnValueConcept("Test", True, "r111", parents=[r1111]) + r11 = ReturnValueConcept("Test", True, "r11", parents=[r111]) + r1 = ReturnValueConcept("Test", True, "r1", parents=[r11]) + r22 = ReturnValueConcept("Test", True, "r22") + r2 = ReturnValueConcept("Test", True, "r2", parents=[r22]) + + rets = [r1, r2] + + service.process_return_values(context, rets) + captured = capsys.readouterr() + assert captured.out == """ReturnValue(who=Test, status=True, value=r1, message=None) + ReturnValue(who=Test, status=True, value=r11, message=None) + ReturnValue(who=Test, status=True, value=r111, message=None) +ReturnValue(who=Test, status=True, value=r2, message=None) + ReturnValue(who=Test, status=True, value=r22, message=None) +""" + + def test_i_can_print_out_a_list_with_recurse_using_format_instr(self, capsys): + sheerka, context, service, *rules = self.init_service_with_rules( + ("__rets", "list(__rets)"), + ("__ret", "{__tab}{__ret}"), + ) + r1111 = ReturnValueConcept("Test", True, "r1111") + r111 = ReturnValueConcept("Test", True, "r111", parents=[r1111]) + r11 = ReturnValueConcept("Test", True, "r11", parents=[r111]) + r1 = ReturnValueConcept("Test", True, "r1", parents=[r11]) + r1.set_format_instr(recursion_depth=2, recurse_on="parents") + r22 = ReturnValueConcept("Test", True, "r22") + r2 = ReturnValueConcept("Test", True, "r2", parents=[r22]) + r2.set_format_instr(recursion_depth=2, recurse_on="parents") + + rets = [r1, r2] + + service.process_return_values(context, rets) + captured = capsys.readouterr() + assert captured.out == """ReturnValue(who=Test, status=True, value=r1, message=None) + ReturnValue(who=Test, status=True, value=r11, message=None) + ReturnValue(who=Test, status=True, value=r111, message=None) +ReturnValue(who=Test, status=True, value=r2, message=None) + ReturnValue(who=Test, status=True, value=r22, message=None) +""" + + def test_i_can_print_out_a_list_with_recurse_using_container_format_instr(self, capsys): + sheerka, context, service, *rules = self.init_service_with_rules( + ("isinstance(__obj, 'foo')", "list(__obj)"), + ("__ret", "{__tab}{__ret}"), + ) + r1111 = ReturnValueConcept("Test", True, "r1111") + r111 = ReturnValueConcept("Test", True, "r111", parents=[r1111]) + r11 = ReturnValueConcept("Test", True, "r11", parents=[r111]) + r1 = ReturnValueConcept("Test", True, "r1", parents=[r11]) + r22 = ReturnValueConcept("Test", True, "r22") + r2 = ReturnValueConcept("Test", True, "r2", parents=[r22]) + + foo = Concept("foo", key="foo", body=[r1, r2]).auto_init() + foo.set_format_instr(recurse_on='parents', recursion_depth=2) + + service.process_return_values(context, foo) + captured = capsys.readouterr() + assert captured.out == """ReturnValue(who=Test, status=True, value=r1, message=None) + ReturnValue(who=Test, status=True, value=r11, message=None) + ReturnValue(who=Test, status=True, value=r111, message=None) +ReturnValue(who=Test, status=True, value=r2, message=None) + ReturnValue(who=Test, status=True, value=r22, message=None) +""" + + def test_i_can_print_out_color(self, capsys): + sheerka, context, service, *rules = self.init_service_with_rules( + ("__ret", "status: green(status), blue('value: {__ret_value}')"), + ) + sheerka.create_new_concept(context, Concept("foo")) + foo = sheerka.new("foo", body="hello world") + ret = sheerka.ret("Test", True, foo) + + service.process_return_values(context, ret) + captured = capsys.readouterr() + assert captured.out == "status: \x1b[32mTrue\x1b[0m, \x1b[34mvalue: (1001)foo\x1b[0m\n" diff --git a/tests/parsers/parsers_utils.py b/tests/parsers/parsers_utils.py index 187b146..fbcbf9a 100644 --- a/tests/parsers/parsers_utils.py +++ b/tests/parsers/parsers_utils.py @@ -1,7 +1,7 @@ from core.concept import CC, Concept, ConceptParts, DoNotResolve, CIO from core.tokenizer import Tokenizer, TokenKind, Token from parsers.BaseNodeParser import scnode, utnode, cnode, SCWC, CNC, short_cnode, SourceCodeWithConceptNode, CN, UTN, \ - SCN + SCN, RN from parsers.SyaNodeParser import SyaConceptParserHelper @@ -108,6 +108,12 @@ def get_node( sub_expr.fix_pos(node) return sub_expr + if isinstance(sub_expr, RN): + start, length = _index(expression_as_tokens, sub_expr.source, skip) + sub_expr.start = start + sub_expr.end = start + length - 1 + return sub_expr + if isinstance(sub_expr, (CNC, CC, CN)): concept_node = get_node( concepts_map, @@ -164,7 +170,7 @@ def get_node( concept_found = concepts_map.get(concept_key, None) if concept_found: concept_found = Concept().update_from(concept_found) # make a copy when massively used in tests - if sya and len(concept_found.metadata.variables) > 0 and not is_bnf: + if sya and len(concept_found.get_metadata().variables) > 0 and not is_bnf: return SyaConceptParserHelper(concept_found, start, start + length - 1) elif init_empty_body: node = CNC(concept_found, start, start + length - 1, source=sub_expr, exclude_body=exclude_body) @@ -183,7 +189,7 @@ def init_body(item, concept, value): del (item.compiled["body"]) return - if not concept or concept.metadata.body or ConceptParts.BODY in item.compiled: + if not concept or concept.get_metadata().body or ConceptParts.BODY in item.compiled: return item.compiled[ConceptParts.BODY] = DoNotResolve(value) diff --git a/tests/parsers/test_AtomsParser.py b/tests/parsers/test_AtomsParser.py index 56a6a4f..900fd90 100644 --- a/tests/parsers/test_AtomsParser.py +++ b/tests/parsers/test_AtomsParser.py @@ -2,7 +2,7 @@ import pytest from core.builtin_concepts import BuiltinConcepts from core.concept import Concept, DEFINITION_TYPE_DEF from core.sheerka.services.SheerkaExecute import ParserInput -from parsers.AtomNodeParser import AtomNodeParser +from parsers.SequenceNodeParser import SequenceNodeParser from parsers.BaseNodeParser import cnode, utnode, CNC, SCN, CN from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka @@ -17,9 +17,9 @@ class TestAtomsParser(TestUsingMemoryBasedSheerka): singleton=singleton) if use_sheerka: - parser = AtomNodeParser(sheerka=sheerka) + parser = SequenceNodeParser(sheerka=sheerka) else: - parser = AtomNodeParser() + parser = SequenceNodeParser() parser.init_from_concepts(context, updated_concepts) return sheerka, context, parser @@ -351,7 +351,7 @@ class TestAtomsParser(TestUsingMemoryBasedSheerka): lexer_nodes = res.body.body assert res.status - assert lexer_nodes[0].concept.metadata.is_evaluated == expected_is_evaluated + assert lexer_nodes[0].concept.get_metadata().is_evaluated == expected_is_evaluated def test_the_parser_always_return_a_new_instance_of_the_concept(self): concepts_map = { @@ -376,3 +376,9 @@ class TestAtomsParser(TestUsingMemoryBasedSheerka): assert not res.status assert sheerka.isinstance(res.body, BuiltinConcepts.NOT_FOR_ME) + def test_i_do_not_parse_rules_instead_of_concepts(self): + sheerka, context, parser = self.init_parser({}) + res = parser.parse(context, ParserInput("r:|20:")) + + assert not res.status + assert sheerka.isinstance(res.body, BuiltinConcepts.NOT_FOR_ME) diff --git a/tests/parsers/test_BaseCustomGrammarParser.py b/tests/parsers/test_BaseCustomGrammarParser.py index bbabb6e..4634af7 100644 --- a/tests/parsers/test_BaseCustomGrammarParser.py +++ b/tests/parsers/test_BaseCustomGrammarParser.py @@ -38,22 +38,26 @@ class TestBaseCustomGrammarParser(TestUsingMemoryBasedSheerka): return sheerka, context, parser - @pytest.mark.parametrize("text, expected", [ - ("when xxx yyy", {Keywords.WHEN: "when xxx yyy"}), - ("when uuu vvv print xxx yyy", {Keywords.WHEN: "when uuu vvv ", Keywords.PRINT: "print xxx yyy"}), - ("print xxx yyy when uuu vvv", {Keywords.WHEN: "when uuu vvv", Keywords.PRINT: "print xxx yyy "}), - (" when xxx", {Keywords.WHEN: "when xxx"}), + @pytest.mark.parametrize("text, strip_tokens, expected", [ + ("when xxx yyy", False, {Keywords.WHEN: "when xxx yyy"}), + ("when uuu vvv print xxx yyy", False, {Keywords.WHEN: "when uuu vvv ", Keywords.PRINT: "print xxx yyy"}), + ("print xxx yyy when uuu vvv", False, {Keywords.WHEN: "when uuu vvv", Keywords.PRINT: "print xxx yyy "}), + (" when xxx", False, {Keywords.WHEN: "when xxx"}), + ("when xxx yyy", True, {Keywords.WHEN: "when xxx yyy"}), + ("when uuu vvv print xxx yyy", True, {Keywords.WHEN: "when uuu vvv", Keywords.PRINT: "print xxx yyy"}), + ("print xxx yyy when uuu vvv", True, {Keywords.WHEN: "when uuu vvv", Keywords.PRINT: "print xxx yyy"}), + (" when xxx", True, {Keywords.WHEN: "when xxx"}), ]) - def test_i_can_get_parts(self, text, expected): + def test_i_can_get_parts(self, text, strip_tokens, expected): sheerka, context, parser = self.init_parser(text) - res = parser.get_parts(["when", "print"]) + res = parser.get_parts(["when", "print"], strip_tokens=strip_tokens) self.compare_results(res, expected) def test_i_can_get_parts_when_multilines(self): text = """when def func(x): - return x+1 +\treturn x+1 func(a) """ expected = {Keywords.WHEN: "when def func(x):\n\treturn x+1\nfunc(a)\n"} diff --git a/tests/parsers/test_BaseNodeParser.py b/tests/parsers/test_BaseNodeParser.py index bf3e7be..0e3609d 100644 --- a/tests/parsers/test_BaseNodeParser.py +++ b/tests/parsers/test_BaseNodeParser.py @@ -51,7 +51,7 @@ class TestBaseNodeParser(TestUsingMemoryBasedSheerka): sheerka.add_in_cache(bar) concept = Concept("foo").init_key() - concept.bnf = bnf + concept.set_bnf(bnf) sheerka.set_id_if_needed(concept, False) res = BaseNodeParser.get_concepts_by_first_token(context, [concept]) @@ -72,7 +72,7 @@ class TestBaseNodeParser(TestUsingMemoryBasedSheerka): sheerka.add_in_cache(baz) foo = Concept("foo").init_key() - foo.bnf = OrderedChoice(ConceptExpression("bar"), ConceptExpression("baz"), StrMatch("qux")) + foo.set_bnf(OrderedChoice(ConceptExpression("bar"), ConceptExpression("baz"), StrMatch("qux"))) sheerka.set_id_if_needed(foo, False) res = BaseNodeParser.get_concepts_by_first_token(context, [bar, baz, foo]) @@ -99,7 +99,7 @@ class TestBaseNodeParser(TestUsingMemoryBasedSheerka): sheerka.add_in_cache(bar) foo = Concept("foo").init_key() - foo.bnf = OrderedChoice(ConceptExpression("one"), ConceptExpression("bar"), StrMatch("qux")) + foo.set_bnf(OrderedChoice(ConceptExpression("one"), ConceptExpression("bar"), StrMatch("qux"))) sheerka.set_id_if_needed(foo, False) res = BaseNodeParser.get_concepts_by_first_token(context, [bar, foo], use_sheerka=True) diff --git a/tests/parsers/test_BaseParser.py b/tests/parsers/test_BaseParser.py index e50106e..26d0fab 100644 --- a/tests/parsers/test_BaseParser.py +++ b/tests/parsers/test_BaseParser.py @@ -4,26 +4,6 @@ from core.tokenizer import Tokenizer, TokenKind, Token from parsers.BaseParser import BaseParser, BaseSplitIterParser -@pytest.mark.parametrize("text, expected_text", [ - ("hello world", "hello world"), - ("'hello' 'world'", "'hello' 'world'"), - ("def concept a from", "def concept a from"), - ("()[]{}1=1.5+-/*><&é", "()[]{}1=1.5+-/*><&é"), - ("execute(c:concept_name:)", "execute(c:concept_name:)") - -]) -def test_i_can_get_text_from_tokens(text, expected_text): - tokens = list(Tokenizer(text)) - assert BaseParser.get_text_from_tokens(tokens) == expected_text - - -@pytest.mark.parametrize("text, custom, expected_text", [ - ("execute(c:concept_name:)", {TokenKind.CONCEPT: lambda t: f"__C__{t.value[0]}"}, "execute(__C__concept_name)") -]) -def test_i_can_get_text_from_tokens_with_custom_switcher(text, custom, expected_text): - tokens = list(Tokenizer(text)) - assert BaseParser.get_text_from_tokens(tokens, custom) == expected_text - @pytest.mark.parametrize("text, expected", [ ("", [""]), diff --git a/tests/parsers/test_BnfNodeParser.py b/tests/parsers/test_BnfNodeParser.py index 6f42d3d..80cd003 100644 --- a/tests/parsers/test_BnfNodeParser.py +++ b/tests/parsers/test_BnfNodeParser.py @@ -1,11 +1,11 @@ import pytest from core.builtin_concepts import BuiltinConcepts -from core.concept import Concept, ConceptParts, DoNotResolve, CC, DEFINITION_TYPE_BNF +from core.concept import Concept, ConceptParts, DoNotResolve, CC, DEFINITION_TYPE_BNF, NotInit from core.sheerka.services.SheerkaExecute import ParserInput from parsers.BaseNodeParser import CNC, UTN, CN from parsers.BnfNodeParser import StrMatch, TerminalNode, NonTerminalNode, Sequence, OrderedChoice, \ Optional, ZeroOrMore, OneOrMore, ConceptExpression, UnOrderedChoice, BnfNodeParser -from parsers.BnfParser import BnfParser +from parsers.BnfDefinitionParser import BnfDefinitionParser import tests.parsers.parsers_utils from tests.BaseTest import BaseTest @@ -137,11 +137,11 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka): @staticmethod def update_bnf(context, concept): - bnf_parser = BnfParser() - res = bnf_parser.parse(context, concept.metadata.definition) + bnf_parser = BnfDefinitionParser() + res = bnf_parser.parse(context, concept.get_metadata().definition) if res.status: - concept.bnf = res.value.value - concept.metadata.definition_type = DEFINITION_TYPE_BNF + concept.set_bnf(res.value.value) + concept.get_metadata().definition_type = DEFINITION_TYPE_BNF else: raise Exception(res) return concept @@ -572,16 +572,16 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka): # explicit validations of the compiled concept_foo = sequences[0][0].concept - assert concept_foo.body == BuiltinConcepts.NOT_INITIALIZED - assert concept_foo.compiled == {ConceptParts.BODY: DoNotResolve("one two")} + assert concept_foo.body == NotInit + assert concept_foo.get_compiled() == {ConceptParts.BODY: DoNotResolve("one two")} concept_bar = sequences[1][0].concept - assert concept_bar.body == BuiltinConcepts.NOT_INITIALIZED - assert concept_bar.compiled == { + assert concept_bar.body == NotInit + assert concept_bar.get_compiled() == { ConceptParts.BODY: concept_foo, "foo": concept_foo } - assert id(concept_bar.compiled[ConceptParts.BODY]) == id(concept_bar.compiled["foo"]) + assert id(concept_bar.get_compiled()[ConceptParts.BODY]) == id(concept_bar.get_compiled()["foo"]) def test_i_can_refer_to_other_concepts_with_body(self): my_map = { @@ -598,12 +598,12 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka): # explicit validations of the compiled concept_foo = sequences[0][0].concept - assert concept_foo.body == BuiltinConcepts.NOT_INITIALIZED - assert len(concept_foo.compiled) == 0 # because there is a body defined in the metadata + assert concept_foo.body == NotInit + assert len(concept_foo.get_compiled()) == 0 # because there is a body defined in the metadata concept_bar = sequences[1][0].concept - assert concept_bar.body == BuiltinConcepts.NOT_INITIALIZED - assert concept_bar.compiled == { + assert concept_bar.body == NotInit + assert concept_bar.get_compiled() == { ConceptParts.BODY: concept_foo, "foo": concept_foo } @@ -625,19 +625,19 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka): # explicit validations of the compiled concept_foo = sequences[0][0].concept - assert concept_foo.body == BuiltinConcepts.NOT_INITIALIZED - assert concept_foo.compiled == {ConceptParts.BODY: DoNotResolve("one two")} + assert concept_foo.body == NotInit + assert concept_foo.get_compiled() == {ConceptParts.BODY: DoNotResolve("one two")} concept_bar = sequences[1][0].concept - assert concept_bar.body == BuiltinConcepts.NOT_INITIALIZED - assert concept_bar.compiled == { + assert concept_bar.body == NotInit + assert concept_bar.get_compiled() == { ConceptParts.BODY: concept_foo, "foo": concept_foo } concept_baz = sequences[2][0].concept - assert concept_baz.body == BuiltinConcepts.NOT_INITIALIZED - assert concept_baz.compiled == { + assert concept_baz.body == NotInit + assert concept_baz.get_compiled() == { ConceptParts.BODY: concept_bar, "bar": concept_bar } @@ -655,21 +655,21 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka): expected = [CN("bar", source="twenty two")] sequences = self.validate_get_concepts_sequences(my_map, text, expected) concept_bar = sequences[0].concept - assert concept_bar.compiled == { + assert concept_bar.get_compiled() == { ConceptParts.BODY: DoNotResolve("twenty two"), "foo": my_map["foo"], } - assert concept_bar.compiled["foo"].compiled == {ConceptParts.BODY: DoNotResolve("twenty")} + assert concept_bar.get_compiled()["foo"].get_compiled() == {ConceptParts.BODY: DoNotResolve("twenty")} text = "thirty one" expected = [CN("bar", source="thirty one")] sequences = self.validate_get_concepts_sequences(my_map, text, expected) concept_bar = sequences[0].concept - assert concept_bar.compiled == { + assert concept_bar.get_compiled() == { ConceptParts.BODY: DoNotResolve("thirty one"), "foo": my_map["foo"], } - assert concept_bar.compiled["foo"].compiled == {ConceptParts.BODY: DoNotResolve("thirty")} + assert concept_bar.get_compiled()["foo"].get_compiled() == {ConceptParts.BODY: DoNotResolve("thirty")} text = "thirty three" expected = [[CN("foo", source="thirty"), CN("three")], []] @@ -712,21 +712,21 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka): sheerka, context, sequences = self.exec_get_concepts_sequences(my_map, text, expected) concept_bar = sequences[0].concept - assert concept_bar.compiled == { + assert concept_bar.get_compiled() == { ConceptParts.BODY: DoNotResolve("twenty two"), "foo": sheerka.new("foo"), } - assert concept_bar.compiled["foo"].compiled == {} # as foo as a body + assert concept_bar.get_compiled()["foo"].get_compiled() == {} # as foo as a body text = "thirty one" expected = [CN("bar", source="thirty one")] sequences = self.validate_get_concepts_sequences(my_map, text, expected) concept_bar = sequences[0].concept - assert concept_bar.compiled == { + assert concept_bar.get_compiled() == { ConceptParts.BODY: DoNotResolve("thirty one"), "foo": sheerka.new("foo"), } - assert concept_bar.compiled["foo"].compiled == {} + assert concept_bar.get_compiled()["foo"].get_compiled() == {} def test_i_can_mix_zero_and_more_and_reference_to_other_concepts(self): my_map = { @@ -738,13 +738,13 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka): expected = [CN("bar", source="one two three")] sequences = self.validate_get_concepts_sequences(my_map, text, expected) concept_bar = sequences[0].concept - assert concept_bar.compiled == { + assert concept_bar.get_compiled() == { ConceptParts.BODY: DoNotResolve("one two three"), "foo": [my_map["foo"], my_map["foo"], my_map["foo"]] } - assert concept_bar.compiled["foo"][0].compiled == {ConceptParts.BODY: DoNotResolve("one")} - assert concept_bar.compiled["foo"][1].compiled == {ConceptParts.BODY: DoNotResolve("two")} - assert concept_bar.compiled["foo"][2].compiled == {ConceptParts.BODY: DoNotResolve("three")} + assert concept_bar.get_compiled()["foo"][0].get_compiled() == {ConceptParts.BODY: DoNotResolve("one")} + assert concept_bar.get_compiled()["foo"][1].get_compiled() == {ConceptParts.BODY: DoNotResolve("two")} + assert concept_bar.get_compiled()["foo"][2].get_compiled() == {ConceptParts.BODY: DoNotResolve("three")} def test_i_can_parse_concept_reference_that_is_not_in_grammar(self): my_map = { @@ -763,7 +763,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka): expected = [CN("foo", source="twenty one")] sequences = self.validate_get_concepts_sequences(my_map, text, expected) concept_foo = sequences[0].concept - assert concept_foo.compiled == { + assert concept_foo.get_compiled() == { ConceptParts.BODY: DoNotResolve("twenty one"), "unit": my_map["one"], } @@ -786,8 +786,8 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka): # explicit validations of the compiled concept_foo = sequences[0].concept - assert concept_foo.body == BuiltinConcepts.NOT_INITIALIZED - assert concept_foo.compiled == {'number': CC(my_map["number"], body=my_map["two"], two=my_map["two"]), + assert concept_foo.body == NotInit + assert concept_foo.get_compiled() == {'number': CC(my_map["number"], body=my_map["two"], two=my_map["two"]), ConceptParts.BODY: DoNotResolve(value='twenty two')} text = "twenty one" @@ -796,8 +796,8 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka): # explicit validations of the compiled concept_foo = sequences[0].concept - assert concept_foo.body == BuiltinConcepts.NOT_INITIALIZED - assert concept_foo.compiled == {'number': CC(my_map["number"], body=my_map["one"], one=my_map["one"]), + assert concept_foo.body == NotInit + assert concept_foo.get_compiled() == {'number': CC(my_map["number"], body=my_map["one"], one=my_map["one"]), ConceptParts.BODY: DoNotResolve(value='twenty one')} @pytest.mark.parametrize("bar_expr, expected", [ @@ -1241,7 +1241,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka): sheerka, context, parser = self.init_parser(init_from_sheerka=True) sheerka.concepts_grammars.clear() # simulate restart for c in cmap.values(): - sheerka.get_by_id(c.id).bnf = None + sheerka.get_by_id(c.id).set_bnf(None) text = "thirty three" expected = CNC("thirties", diff --git a/tests/parsers/test_BnfParser.py b/tests/parsers/test_BnfParser.py index c632cc8..1db659f 100644 --- a/tests/parsers/test_BnfParser.py +++ b/tests/parsers/test_BnfParser.py @@ -8,7 +8,7 @@ from parsers.BaseParser import UnexpectedTokenErrorNode from parsers.BnfNodeParser import StrMatch, Optional, ZeroOrMore, OrderedChoice, Sequence, \ OneOrMore, ConceptExpression from parsers.BnfNodeParser import BnfNodeParser -from parsers.BnfParser import BnfParser, UnexpectedEndOfFileError +from parsers.BnfDefinitionParser import BnfDefinitionParser, UnexpectedEndOfFileError from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka @@ -27,7 +27,7 @@ def update_concepts_ids(sheerka, parsing_expression): if isinstance(parsing_expression, ConceptExpression): if not parsing_expression.concept.id: concept = sheerka.get_by_key(parsing_expression.concept.key) - parsing_expression.concept.metadata.id = concept.id + parsing_expression.concept.get_metadata().id = concept.id for pe in parsing_expression.elements: update_concepts_ids(sheerka, pe) @@ -40,7 +40,7 @@ class TestBnfParser(TestUsingMemoryBasedSheerka): def init_parser(self, *concepts): sheerka, context, *updated = self.init_concepts(*concepts, singleton=True) - parser = BnfParser() + parser = BnfDefinitionParser() return sheerka, context, parser, *updated @@ -177,10 +177,10 @@ class TestBnfParser(TestUsingMemoryBasedSheerka): sheerka, context, regex_parser, foo, bar = self.init_parser("foo", "bar") for concept in [foo, bar]: - concept.metadata.definition_type = DEFINITION_TYPE_BNF + concept.get_metadata().definition_type = DEFINITION_TYPE_BNF - foo.bnf = regex_parser.parse(context, "'twenty' | 'thirty'").value.value - bar.bnf = regex_parser.parse(context, "foo ('one' | 'two')").value.value + foo.set_bnf(regex_parser.parse(context, "'twenty' | 'thirty'").value.value) + bar.set_bnf(regex_parser.parse(context, "foo ('one' | 'two')").value.value) bnf_parser = BnfNodeParser() bnf_parser.init_from_concepts(context, [foo, bar]) diff --git a/tests/parsers/test_DefConceptParser.py b/tests/parsers/test_DefConceptParser.py index 6c14edd..f83da27 100644 --- a/tests/parsers/test_DefConceptParser.py +++ b/tests/parsers/test_DefConceptParser.py @@ -9,7 +9,7 @@ from core.tokenizer import Keywords, Tokenizer, LexerError from parsers.BaseNodeParser import SCWC from parsers.BaseParser import NotInitializedNode, UnexpectedEofNode from parsers.BnfNodeParser import OrderedChoice, ConceptExpression, StrMatch, Sequence -from parsers.BnfParser import BnfParser +from parsers.BnfDefinitionParser import BnfDefinitionParser from parsers.DefConceptParser import DefConceptParser, NameNode, SyntaxErrorNode from parsers.DefConceptParser import UnexpectedTokenErrorNode, DefConceptNode from parsers.FunctionParser import FunctionParser @@ -34,7 +34,7 @@ def get_def_concept(name, where=None, pre=None, post=None, body=None, definition def_concept.ret = get_concept_part(ret) if bnf_def: def_concept.definition = ReturnValueConcept( - "parsers.Bnf", + "parsers.BnfDefinition", True, bnf_def) def_concept.definition_type = DEFINITION_TYPE_BNF @@ -346,7 +346,7 @@ def concept add one to a as: node = res.value.value definition = OrderedChoice(ConceptExpression(a_concept, rule_name="a_concept"), StrMatch("a_string")) - parser_result = ParserResultConcept(BnfParser(), "a_concept | 'a_string'", None, definition, definition) + parser_result = ParserResultConcept(BnfDefinitionParser(), "a_concept | 'a_string'", None, definition, definition) expected = get_def_concept(name="name", body="__definition[0]", bnf_def=parser_result) assert res.status @@ -515,7 +515,7 @@ from give me the date ! assert res.status assert concept_defined.name.tokens == list(Tokenizer("def concept x", yield_eof=False)) - def test_i_can_parse_when_ambiguity_in_where_pre_clause(self): + def test_i_can_parse_when_ambiguity_in_where_or_pre_clause(self): sheerka, context, parser, *concepts = self.init_parser( Concept("x is a y", pre="in_context(BuiltinConcepts.EVAL_QUESTION_REQUESTED)"), Concept("x is a y") diff --git a/tests/parsers/test_ExactConceptParser.py b/tests/parsers/test_ExactConceptParser.py index fe784c2..4c9636d 100644 --- a/tests/parsers/test_ExactConceptParser.py +++ b/tests/parsers/test_ExactConceptParser.py @@ -7,7 +7,7 @@ from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka def variable_def(concept, prop_name): - for name, value in concept.metadata.variables: + for name, value in concept.get_metadata().variables: if name == prop_name: return value @@ -61,8 +61,8 @@ class TestExactConceptParser(TestUsingMemoryBasedSheerka): assert len(results) == 1 assert results[0].status assert concept_found == concept - assert not concept_found.metadata.need_validation - assert not concept_found.metadata.is_evaluated + assert not concept_found.get_metadata().need_validation + assert not concept_found.get_metadata().is_evaluated def test_i_can_parse_concepts_defined_several_times(self): sheerka = self.get_sheerka(singleton=True) @@ -79,11 +79,11 @@ class TestExactConceptParser(TestUsingMemoryBasedSheerka): assert results[0].status assert results[0].value.value.name == "hello a" assert variable_def(results[0].value.value, "a") == "world" - assert results[0].value.value.metadata.need_validation + assert results[0].value.value.get_metadata().need_validation assert results[1].status assert results[1].value.value.name == "hello world" - assert not results[1].value.value.metadata.need_validation + assert not results[1].value.value.get_metadata().need_validation def test_i_can_parse_a_concept_with_variables(self): sheerka = self.get_sheerka(singleton=True) @@ -98,8 +98,8 @@ class TestExactConceptParser(TestUsingMemoryBasedSheerka): concept_found = results[0].value.value assert concept_found == CMV(concept, a="10", b="5") - assert concept_found.metadata.need_validation - assert not concept_found.metadata.is_evaluated + assert concept_found.get_metadata().need_validation + assert not concept_found.get_metadata().is_evaluated def test_i_can_parse_a_concept_with_duplicate_variables(self): sheerka = self.get_sheerka(singleton=True) @@ -114,7 +114,7 @@ class TestExactConceptParser(TestUsingMemoryBasedSheerka): concept_found = results[0].value.value assert concept_found == CMV(concept, a="10", b="5") - assert concept_found.metadata.need_validation + assert concept_found.get_metadata().need_validation def test_i_can_parse_concept_when_defined_using_from_def(self): sheerka, context, plus = self.init_concepts( @@ -128,8 +128,8 @@ class TestExactConceptParser(TestUsingMemoryBasedSheerka): assert len(results) == 1 assert results[0].status assert concept_found == CMV(plus, a="10", b="5") - assert concept_found.metadata.need_validation - assert not concept_found.metadata.is_evaluated + assert concept_found.get_metadata().need_validation + assert not concept_found.get_metadata().is_evaluated def test_i_can_parse_concept_token(self): sheerka, context, foo = self.init_concepts("foo") @@ -141,8 +141,8 @@ class TestExactConceptParser(TestUsingMemoryBasedSheerka): assert len(results) == 1 assert results[0].status assert concept_found == foo - assert not concept_found.metadata.need_validation - assert concept_found.metadata.is_evaluated + assert not concept_found.get_metadata().need_validation + assert concept_found.get_metadata().is_evaluated def test_i_can_parse_concept_with_concept_tokens(self): sheerka, context, one, two, plus = self.init_concepts( @@ -158,8 +158,8 @@ class TestExactConceptParser(TestUsingMemoryBasedSheerka): assert len(results) == 1 assert results[0].status assert concept_found == CMV(plus, a="c:one:", b="c:two:") - assert concept_found.metadata.need_validation - assert not concept_found.metadata.is_evaluated + assert concept_found.get_metadata().need_validation + assert not concept_found.get_metadata().is_evaluated def test_i_can_parse_when_expression_contains_keyword(self): sheerka, context, isa, def_concept = self.init_concepts( @@ -174,8 +174,8 @@ class TestExactConceptParser(TestUsingMemoryBasedSheerka): assert len(results) == 1 assert results[0].status assert concept_found == CMV(isa, c="z") - assert concept_found.metadata.need_validation - assert not concept_found.metadata.is_evaluated + assert concept_found.get_metadata().need_validation + assert not concept_found.get_metadata().is_evaluated source = "def concept z" results = ExactConceptParser().parse(context, ParserInput(source)) @@ -184,8 +184,8 @@ class TestExactConceptParser(TestUsingMemoryBasedSheerka): assert len(results) == 1 assert results[0].status assert concept_found == CMV(def_concept, a="z") - assert concept_found.metadata.need_validation - assert not concept_found.metadata.is_evaluated + assert concept_found.get_metadata().need_validation + assert not concept_found.get_metadata().is_evaluated def test_i_can_manage_unknown_concept(self): context = self.get_context(self.get_sheerka(singleton=True)) @@ -218,4 +218,4 @@ class TestExactConceptParser(TestUsingMemoryBasedSheerka): # assert len(results) == 1 # assert results[0].status # assert results[0].value.value == concept - # assert not results[0].value.value.metadata.need_validation + # assert not results[0].value.value.get_metadata().need_validation diff --git a/tests/parsers/test_ExpressionParser.py b/tests/parsers/test_ExpressionParser.py index bba65f8..69f1467 100644 --- a/tests/parsers/test_ExpressionParser.py +++ b/tests/parsers/test_ExpressionParser.py @@ -33,14 +33,14 @@ class TestExpressionParser(TestUsingMemoryBasedSheerka): @pytest.mark.parametrize("expression, expected", [ ("one complicated expression", n("one complicated expression")), - # ("function_call(a,b,c)", n("function_call(a,b,c)")), - # ("one expression or another expression", OrNode(n("one expression"), n("another expression"))), - # ("one expression and another expression", AndNode(n("one expression"), n("another expression"))), - # ("one or two or three", OrNode(n("one"), n("two"), n("three"))), - # ("one and two and three", AndNode(n("one"), n("two"), n("three"))), - # ("one or two and three", OrNode(n("one"), AndNode(n("two"), n("three")))), - # ("one and two or three", OrNode(AndNode(n("one"), n("two")), n("three"))), - # ("one and (two or three)", AndNode(n("one"), OrNode(n("two"), n("three")))), + ("function_call(a,b,c)", n("function_call(a,b,c)")), + ("one expression or another expression", OrNode(n("one expression"), n("another expression"))), + ("one expression and another expression", AndNode(n("one expression"), n("another expression"))), + ("one or two or three", OrNode(n("one"), n("two"), n("three"))), + ("one and two and three", AndNode(n("one"), n("two"), n("three"))), + ("one or two and three", OrNode(n("one"), AndNode(n("two"), n("three")))), + ("one and two or three", OrNode(AndNode(n("one"), n("two")), n("three"))), + ("one and (two or three)", AndNode(n("one"), OrNode(n("two"), n("three")))), ]) def test_i_can_parse_expression(self, expression, expected): sheerka, context, parser = self.init_parser() diff --git a/tests/parsers/test_FormatRuleParser.py b/tests/parsers/test_FormatRuleParser.py index 1158515..0bd5ce9 100644 --- a/tests/parsers/test_FormatRuleParser.py +++ b/tests/parsers/test_FormatRuleParser.py @@ -1,12 +1,20 @@ import pytest from core.builtin_concepts import BuiltinConcepts +from core.concept import Concept from core.sheerka.services.SheerkaExecute import ParserInput +from core.sheerka.services.SheerkaRuleManager import FormatAstRawText, RulePredicate, FormatAstVariable +from core.tokenizer import Keywords, Tokenizer from parsers.BaseCustomGrammarParser import KeywordNotFound -from parsers.FormatRuleParser import FormatRuleParser, FormatAstRawText, FormatRuleNode +from parsers.FormatRuleParser import FormatRuleParser, FormatRuleNode from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka -cmap = {} +cmap = { + "is a": Concept("x is a y").def_var("x").def_var("y"), + "is a question": Concept("x is a y", pre="is_question()").def_var("x").def_var("y"), + "a is good": Concept("a is good").def_var("a"), + "b is good": Concept("b is good").def_var("b"), +} class TestFormatRuleParser(TestUsingMemoryBasedSheerka): @@ -45,16 +53,47 @@ class TestFormatRuleParser(TestUsingMemoryBasedSheerka): res = parser.parse(context, ParserInput(text)) parser_result = res.body format_rule = res.body.body - rule = format_rule.rule + rules = format_rule.rule format_ast = format_rule.format_ast assert res.status assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT) assert isinstance(format_rule, FormatRuleNode) + assert self.dump_tokens(format_rule.tokens[Keywords.WHEN][1:]) == self.dump_tokens( + Tokenizer("isinstance(last_value(), Concept)", False)) + assert self.dump_tokens(format_rule.tokens[Keywords.PRINT][1:]) == self.dump_tokens( + Tokenizer("hello world!", False)) - assert sheerka.isinstance(rule, BuiltinConcepts.RETURN_VALUE) + assert len(rules) == 1 + assert isinstance(rules[0], RulePredicate) assert format_ast == FormatAstRawText("hello world!") + def test_when_is_parsed_in_the_context_of_a_question(self): + sheerka, context, parser = self.init_parser() + + text = "when foo is a bar print hello world" + res = parser.parse(context, ParserInput(text)) + format_rule = res.body.body + rules = format_rule.rule + + assert res.status + assert len(rules) == 1 + assert isinstance(rules[0], RulePredicate) + assert rules[0].predicate.body.body.get_metadata().pre == "is_question()" + + def test_when_can_support_multiple_possibilities(self): + sheerka, context, parser = self.init_parser() + + text = "when foo is good print hello world" + res = parser.parse(context, ParserInput(text)) + format_rule = res.body.body + rules = format_rule.rule + + assert res.status + assert len(rules) == 2 + assert rules[0].predicate.body.body.get_metadata().name == "a is good" + assert rules[1].predicate.body.body.get_metadata().name == "b is good" + @pytest.mark.parametrize("text, error", [ ("hello world", [KeywordNotFound(None, keywords=['when', 'print'])]), ("when True", [KeywordNotFound([], keywords=['print'])]), @@ -69,3 +108,30 @@ class TestFormatRuleParser(TestUsingMemoryBasedSheerka): assert not res.status assert sheerka.isinstance(not_for_me, BuiltinConcepts.NOT_FOR_ME) assert not_for_me.reason == error + + @pytest.mark.parametrize("text, expected_error", [ + ("when a b print hello world!", BuiltinConcepts.TOO_MANY_ERRORS), + ]) + def test_i_cannot_parse_invalid_predicates(self, text, expected_error): + sheerka, context, parser = self.init_parser() + + res = parser.parse(context, ParserInput(text)) + + assert not res.status + assert sheerka.isinstance(res.body, expected_error) + + @pytest.mark.parametrize("expr, expected", [ + ("hello world", FormatAstRawText("hello world")), + ("{id}", FormatAstVariable("id")), + ]) + def test_i_can_parse_valid_print_expression(self, expr, expected): + sheerka, context, parser = self.init_parser() + + text = "when True print " + expr + res = parser.parse(context, ParserInput(text)) + format_rule = res.body.body + format_ast = format_rule.format_ast + + assert res.status + assert format_ast == expected + diff --git a/tests/parsers/test_FunctionParser.py b/tests/parsers/test_FunctionParser.py index e79085d..d0bd00c 100644 --- a/tests/parsers/test_FunctionParser.py +++ b/tests/parsers/test_FunctionParser.py @@ -1,8 +1,9 @@ import pytest from core.builtin_concepts import BuiltinConcepts from core.concept import Concept +from core.rule import Rule from core.sheerka.services.SheerkaExecute import ParserInput -from parsers.BaseNodeParser import SCN, SCWC, CN, UTN, CNC +from parsers.BaseNodeParser import SCN, SCWC, CN, UTN, CNC, RN from parsers.FunctionParser import FunctionParser, FN from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka @@ -76,6 +77,16 @@ class TestFunctionParser(TestUsingMemoryBasedSheerka): assert res == expected + def test_i_can_parse_function_when_rule(self): + sheerka, context, parser = self.init_parser() + + parser.reset_parser(context, ParserInput("func(r:|1:)")) + parser.parser_input.next_token() + + res = parser.parse_function() + + assert res == FN("func(", ")", ["r:|1:"]) + @pytest.mark.parametrize("text, expected", [ ("func()", SCN("func()")), (" func()", SCN("func()")), @@ -118,6 +129,63 @@ class TestFunctionParser(TestUsingMemoryBasedSheerka): assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT) assert expressions == resolved_expected + def test_i_can_parse_when_the_parameter_is_not_a_concept(self): + """ + It's not a concept, but it can be a valid short term memory object + :return: + """ + sheerka, context, parser = self.init_parser() + text = "func(unknown_concept)" + + res = parser.parse(context, ParserInput(text)) + + assert res.status + + def test_i_cannot_parse_when_the_concept_is_not_found(self): + """ + We do not check yet if it's a valid concept + If you find a cheap way to do so, simply remove this test + :return: + """ + sheerka, context, parser = self.init_parser() + text = "func(c:|xxx:)" + + res = parser.parse(context, ParserInput(text)) + + assert res.status + + def test_i_can_parse_when_rules(self): + sheerka, context, parser = self.init_parser() + text = "func(r:|1:)" + expected = SCWC("func(", ")", RN("1")) + resolved_expected = compute_expected_array(cmap, text, [expected])[0] + + res = parser.parse(context, ParserInput(text)) + parser_result = res.body + expression = res.body.body + + assert res.status + assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT) + assert expression == resolved_expected + assert expression.python_node is not None + assert expression.return_value is not None + + # def test_i_cannot_parse_when_rule_not_found(self): + # sheerka, context, parser = self.init_parser() + # text = "func(r:|fake:)" + # expected = SCWC("func(", ")", RN("fake")) + # resolved_expected = compute_expected_array(cmap, text, [expected])[0] + # + # res = parser.parse(context, ParserInput(text)) + # parser_result = res.body + # expression = res.body.body + # + # assert not res.status + # assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT) + # assert expression == resolved_expected + # assert expression.python_node is not None + # assert expression.return_value is not None + @pytest.mark.parametrize("text, expected_error_type", [ ("one", BuiltinConcepts.NOT_FOR_ME), ("$*!", BuiltinConcepts.NOT_FOR_ME), @@ -138,7 +206,7 @@ class TestFunctionParser(TestUsingMemoryBasedSheerka): @pytest.mark.parametrize("text, expected", [ ("func(one two)", SCWC("func(", ")", "one", "two")), ]) - def test_i_can_detect_non_function(self, text, expected): + def test_i_can_detect_none_function(self, text, expected): sheerka, context, parser = self.init_parser() resolved_expected = compute_expected_array(cmap, text, [expected])[0] @@ -169,11 +237,11 @@ class TestFunctionParser(TestUsingMemoryBasedSheerka): concept = res.body.body.nodes[0].concept assert res.status - assert isinstance(concept.compiled["a"], Concept) + assert isinstance(concept.get_compiled()["a"], Concept) # three is not recognized, # so it will be transformed into list of ReturnValueConcept that indicate how to recognized it - assert isinstance(concept.compiled["b"], list) - for item in concept.compiled["b"]: + assert isinstance(concept.get_compiled()["b"], list) + for item in concept.get_compiled()["b"]: assert sheerka.isinstance(item, BuiltinConcepts.RETURN_VALUE) diff --git a/tests/parsers/test_PythonParser.py b/tests/parsers/test_PythonParser.py index ba17227..ee35ecd 100644 --- a/tests/parsers/test_PythonParser.py +++ b/tests/parsers/test_PythonParser.py @@ -2,10 +2,10 @@ import ast import core.utils import pytest -from core.builtin_concepts import ParserResultConcept, NotForMeConcept +from core.builtin_concepts import ParserResultConcept, NotForMeConcept, BuiltinConcepts from core.sheerka.services.SheerkaExecute import ParserInput -from core.tokenizer import LexerError -from parsers.PythonParser import PythonNode, PythonParser, PythonErrorNode +from core.tokenizer import LexerError, TokenKind +from parsers.PythonParser import PythonNode, PythonParser, PythonErrorNode, ConceptDetected from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka @@ -70,6 +70,52 @@ class TestPythonParser(TestUsingMemoryBasedSheerka): assert res.value.value == PythonNode( "__C__KEY_name__ID_id__C__ + 1", ast.parse(encoded + "+1", mode="eval")) - assert res.value.value.concepts == { - encoded: ("name", "id") - } + assert len(res.value.value.objects) == 1 + assert res.value.value.objects[encoded].type == TokenKind.CONCEPT + assert res.value.value.objects[encoded].value == ("name", "id") + + def test_i_can_parse_a_rule(self): + text = "r:name|id: + 1" + + parser = PythonParser() + res = parser.parse(self.get_context(), ParserInput(text)) + encoded = core.utils.encode_concept(("name", "id"), "R") + + assert res + assert res.value.source == "r:name|id: + 1" + assert res.value.value == PythonNode( + "__R__KEY_name__ID_id__R__ + 1", + ast.parse(encoded + "+1", mode="eval")) + assert len(res.value.value.objects) == 1 + assert res.value.value.objects[encoded].type == TokenKind.RULE + assert res.value.value.objects[encoded].value == ("name", "id") + + def test_i_can_parse_a_rule_2(self): + text = "r:name|id:.name" + + parser = PythonParser() + res = parser.parse(self.get_context(), ParserInput(text)) + encoded = core.utils.encode_concept(("name", "id"), "R") + + assert res + assert res.value.source == "r:name|id:.name" + assert res.value.value == PythonNode( + "__R__KEY_name__ID_id__R__.name", + ast.parse(encoded + ".name", mode="eval")) + assert len(res.value.value.objects) == 1 + assert res.value.value.objects[encoded].type == TokenKind.RULE + assert res.value.value.objects[encoded].value == ("name", "id") + + @pytest.mark.parametrize("text, expected_id", [ + ("foo", "foo"), + ("c:foo:", "__C__KEY_foo__ID_00None00__C__") + ]) + def test_i_cannot_parse_simple_concepts(self, text, expected_id): + sheerka, context, foo = self.init_concepts("foo") + + parser = PythonParser() + res = parser.parse(context, ParserInput(text)) + + assert not res.status + assert sheerka.isinstance(res.value, BuiltinConcepts.NOT_FOR_ME) + assert res.value.reason == [ConceptDetected(expected_id)] diff --git a/tests/parsers/test_PythonWithConceptsParser.py b/tests/parsers/test_PythonWithConceptsParser.py index 9e741db..855d2fe 100644 --- a/tests/parsers/test_PythonWithConceptsParser.py +++ b/tests/parsers/test_PythonWithConceptsParser.py @@ -1,11 +1,13 @@ import ast import pytest +import core.utils from core.builtin_concepts import ParserResultConcept, BuiltinConcepts, ReturnValueConcept from core.concept import Concept +from core.rule import Rule from core.sheerka.services.SheerkaExecute import ParserInput from core.tokenizer import Token, TokenKind, Tokenizer -from parsers.BaseNodeParser import ConceptNode, UnrecognizedTokensNode +from parsers.BaseNodeParser import ConceptNode, UnrecognizedTokensNode, RuleNode from parsers.PythonParser import PythonNode from parsers.PythonWithConceptsParser import PythonWithConceptsParser from parsers.UnrecognizedNodeParser import UnrecognizedNodeParser @@ -23,6 +25,10 @@ def ret_val(*args): tokens = [Token(TokenKind.IDENTIFIER, item.name, 0, 0, 0)] result.append(ConceptNode(item, index, index, tokens, item.name)) index += 1 + elif isinstance(item, Rule): + tokens = [Token(TokenKind.RULE, (None, item.id), 0, 0, 0)] + result.append(RuleNode(item, index, index, tokens, f"r:|{item.id}:")) + index += 1 else: tokens = list(Tokenizer(item)) result.append(UnrecognizedTokensNode(index, index + len(tokens) - 1, tokens)) @@ -69,7 +75,7 @@ class TestPythonWithConceptsParser(TestUsingMemoryBasedSheerka): assert isinstance(return_value, PythonNode) assert return_value.source == "foo + 1" assert return_value.get_dump(return_value.ast_) == to_str_ast("__C__foo__C__ + 1") - assert return_value.concepts["__C__foo__C__"] == foo + assert return_value.objects["__C__foo__C__"] == foo def test_i_can_parse_concepts_and_python_when_concept_is_known(self): context = self.get_context() @@ -89,7 +95,7 @@ class TestPythonWithConceptsParser(TestUsingMemoryBasedSheerka): assert isinstance(return_value, PythonNode) assert return_value.source == "foo + 1" assert return_value.get_dump(return_value.ast_) == to_str_ast("__C__foo__1001__C__ + 1") - assert return_value.concepts["__C__foo__1001__C__"] == foo + assert return_value.objects["__C__foo__1001__C__"] == foo def test_i_can_parse_when_concept_name_has_invalid_characters(self): context = self.get_context() @@ -102,7 +108,7 @@ class TestPythonWithConceptsParser(TestUsingMemoryBasedSheerka): return_value = result.value.value assert result.status - assert return_value.concepts["__C__foo0et000000__1001__C__"] == foo + assert return_value.objects["__C__foo0et000000__1001__C__"] == foo def test_i_can_parse_when_multiple_concepts(self): sheerka, context, foo, bar = self.init_concepts("foo", "bar") @@ -120,8 +126,42 @@ class TestPythonWithConceptsParser(TestUsingMemoryBasedSheerka): assert isinstance(return_value, PythonNode) assert return_value.source == "func(foo, bar)" assert return_value.get_dump(return_value.ast_) == to_str_ast("func(__C__foo__1001__C__, __C__bar__1002__C__)") - assert return_value.concepts["__C__foo__1001__C__"] == foo - assert return_value.concepts["__C__bar__1002__C__"] == bar + assert return_value.objects["__C__foo__1001__C__"] == foo + assert return_value.objects["__C__bar__1002__C__"] == bar + + def test_i_can_parse_when_python_and_rule(self): + context = self.get_context() + rule = Rule().set_id("rule_id") + input_return_value = ret_val(rule, " + 1") + + parser = PythonWithConceptsParser() + result = parser.parse(context, input_return_value.body) + wrapper = result.value + return_value = result.value.value + + assert result.status + assert result.who == parser.name + assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT) + assert wrapper.source == "r:|rule_id: + 1" + assert isinstance(return_value, PythonNode) + assert return_value.source == "r:|rule_id: + 1" + assert return_value.get_dump(return_value.ast_) == to_str_ast("__R____rule_id__R__ + 1") + assert return_value.objects["__R____rule_id__R__"] == rule + + def test_python_ids_mappings_are_correct_when_rules_with_the_same_id(self): + context = self.get_context() + rule1 = Rule().set_id("rule_id") + rule2 = Rule().set_id("rule_id") + + input_return_value = ret_val(rule1, "+", rule2) + + parser = PythonWithConceptsParser() + result = parser.parse(context, input_return_value.body) + return_value = result.value.value + + assert result.status + assert return_value.objects["__R____rule_id__R__"] == rule1 + assert return_value.objects["__R____rule_id_1__R__"] == rule2 def test_python_ids_mappings_are_correct_when_concepts_with_the_same_name(self): context = self.get_context() @@ -137,10 +177,10 @@ class TestPythonWithConceptsParser(TestUsingMemoryBasedSheerka): return_value = result.value.value assert result.status - assert return_value.concepts["__C__foo__C__"] == foo1 - assert return_value.concepts["__C__foo_1__C__"] == foo2 - assert return_value.concepts["__C__foo__1001__C__"] == foo3 - assert return_value.concepts["__C__foo__1002__C__"] == foo4 + assert return_value.objects["__C__foo__C__"] == foo1 + assert return_value.objects["__C__foo_1__C__"] == foo2 + assert return_value.objects["__C__foo__1001__C__"] == foo3 + assert return_value.objects["__C__foo__1002__C__"] == foo4 def test_i_cannot_parse_if_syntax_error(self): context = self.get_context() diff --git a/tests/parsers/test_RuleParser.py b/tests/parsers/test_RuleParser.py new file mode 100644 index 0000000..8c693a7 --- /dev/null +++ b/tests/parsers/test_RuleParser.py @@ -0,0 +1,101 @@ +import pytest +from core.builtin_concepts import BuiltinConcepts +from core.sheerka.services.SheerkaExecute import ParserInput +from parsers.RuleParser import RuleParser, RuleNotFound + +from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka + +my_rules = {("__rets", "list(__rets)")} + + +class TestRuleParser(TestUsingMemoryBasedSheerka): + sheerka = None + + @classmethod + def setup_class(cls): + t = cls() + cls.sheerka, context, _ = t.init_parser(my_rules) + + def init_parser(self, rules=None): + if rules is not None: + sheerka, context, *concepts = self.init_format_rules(*rules) + else: + sheerka = TestRuleParser.sheerka + context = self.get_context(sheerka) + + parser = RuleParser() + return sheerka, context, parser + + def test_i_cannot_parse_is_empty(self): + sheerka, context, parser = self.init_parser() + + res = parser.parse(context, ParserInput("")) + + assert not res.status + assert sheerka.isinstance(res.body, BuiltinConcepts.IS_EMPTY) + + def test_i_cannot_parse_when_not_for_me(self): + sheerka, context, parser = self.init_parser() + + res = parser.parse(context, ParserInput("hello world!")) + + assert not res.status + assert sheerka.isinstance(res.body, BuiltinConcepts.NOT_FOR_ME) + + def test_i_cannot_parse_a_rule_by_name(self): + sheerka, context, parser = self.init_parser() + + res = parser.parse(context, ParserInput("r:1:")) + + assert not res.status + assert sheerka.isinstance(res.body, BuiltinConcepts.NOT_IMPLEMENTED) + + def test_i_cannot_parse_an_unknown_rule(self): + sheerka, context, parser = self.init_parser() + + res = parser.parse(context, ParserInput("r:|999999:")) + error = res.body + errors_causes = res.body.body + + assert not res.status + assert sheerka.isinstance(error, BuiltinConcepts.ERROR) + assert errors_causes == [RuleNotFound("999999")] + + def test_i_can_parse_rule(self): + sheerka, context, parser = self.init_parser() + + res = parser.parse(context, ParserInput("r:|1:")) + parser_result = res.body + rules = res.body.body + + assert res.status + assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT) + assert len(rules) == 1 + assert rules[0].id == "1" + + def test_rule_recognition_is_deferred_when_id_is_a_string(self): + sheerka, context, parser = self.init_parser() + + res = parser.parse(context, ParserInput("r:|xxx:")) + parser_result = res.body + rules = res.body.body + + assert res.status + assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT) + assert len(rules) == 1 + assert rules[0].id == "xxx" + assert rules[0].metadata.action_type == "deferred" + + @pytest.mark.parametrize("text", [ + "r:|1:xxx", + "xxxr:|1:", + "r:|1: + 1" + ]) + def test_i_can_only_parse_simple_rule(self, text): + sheerka, context, parser = self.init_parser() + + res = parser.parse(context, ParserInput(text)) + not_for_me = res.body + + assert not res.status + assert sheerka.isinstance(not_for_me, BuiltinConcepts.NOT_FOR_ME) diff --git a/tests/parsers/test_SyaNodeParser.py b/tests/parsers/test_SyaNodeParser.py index 91f96b4..86e90e5 100644 --- a/tests/parsers/test_SyaNodeParser.py +++ b/tests/parsers/test_SyaNodeParser.py @@ -1,7 +1,7 @@ import pytest from core.builtin_concepts import BuiltinConcepts -from core.concept import Concept, CIO -from core.sheerka.services.SheerkaComparisonManager import SheerkaComparisonManager +from core.concept import Concept, CIO, ALL_ATTRIBUTES +from core.global_symbols import CONCEPT_COMPARISON_CONTEXT from core.sheerka.services.SheerkaExecute import ParserInput from core.tokenizer import Tokenizer from parsers.BaseNodeParser import utnode, ConceptNode, cnode, short_cnode, UnrecognizedTokensNode, \ @@ -54,14 +54,16 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka): cmap["plus"].set_prop(BuiltinConcepts.ASSOCIATIVITY, "right") cmap["mult"].set_prop(BuiltinConcepts.ASSOCIATIVITY, "right") cmap["minus"].set_prop(BuiltinConcepts.ASSOCIATIVITY, "right") - TestSyaNodeParser.sheerka.services[SheerkaComparisonManager.NAME].set_is_greater_than(context, - BuiltinConcepts.PRECEDENCE, - cmap["mult"], - cmap["plus"]) - TestSyaNodeParser.sheerka.services[SheerkaComparisonManager.NAME].set_is_greater_than(context, - BuiltinConcepts.PRECEDENCE, - cmap["mult"], - cmap["minus"]) + TestSyaNodeParser.sheerka.set_is_greater_than(context, + BuiltinConcepts.PRECEDENCE, + cmap["mult"], + cmap["plus"], + CONCEPT_COMPARISON_CONTEXT) + TestSyaNodeParser.sheerka.set_is_greater_than(context, + BuiltinConcepts.PRECEDENCE, + cmap["mult"], + cmap["minus"], + CONCEPT_COMPARISON_CONTEXT) # TestSyaNodeParser.sheerka.force_sya_def(context, [ # (cmap["plus"].id, 5, SyaAssociativity.Right), @@ -91,6 +93,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka): sheerka = TestSyaNodeParser.sheerka context = self.get_context(sheerka) concepts = cmap.values() + ALL_ATTRIBUTES.clear() if post_init_concepts: post_init_concepts(sheerka, context) @@ -967,7 +970,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka): ' 7: => PUSH_UNREC', ' 8:: => ??', " _: => RECOG [[UTN('twenty '), CN((1001)one)], [CN((1016)twenties)]]", - " _: => POP ConceptNode(concept='(1016)twenties', source='twenty one', start=4, end=6, ConceptParts.BODY='DoNotResolve(value='twenty one')', unit='(1001)one')", + " _: => POP ConceptNode(concept='(1016)twenties', source='twenty one', start=4, end=6, #body#='DoNotResolve(value='twenty one')', unit='(1001)one')", ' 9: => EAT', ' 10:three => PUSH_UNREC', ' 11: => ??', @@ -978,7 +981,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka): ]) def test_i_can_debug(self, expression, expected_debugs): sheerka, context, parser = self.init_parser() - context.add_to_private_hints(BuiltinConcepts.DEBUG) + context.debug_enabled = True res = parser.infix_to_postfix(context, ParserInput(expression)) assert len(res) == len(expected_debugs) @@ -1000,10 +1003,10 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka): # check the compiled expected_concept = lexer_nodes[0].concept - assert expected_concept.compiled["a"] == cmap["one"] - assert expected_concept.compiled["b"] == cmap["mult"] - assert expected_concept.compiled["b"].compiled["a"] == cmap["two"] - assert expected_concept.compiled["b"].compiled["b"] == cmap["three"] + assert expected_concept.get_compiled()["a"] == cmap["one"] + assert expected_concept.get_compiled()["b"] == cmap["mult"] + assert expected_concept.get_compiled()["b"].get_compiled()["a"] == cmap["two"] + assert expected_concept.get_compiled()["b"].get_compiled()["b"] == cmap["three"] def test_i_can_parse_when_python_code(self): sheerka, context, parser = self.init_parser() @@ -1019,9 +1022,9 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka): # check the compiled expected_concept = lexer_nodes[0].concept - assert len(expected_concept.compiled["a"]) == 1 + assert len(expected_concept.get_compiled()["a"]) == 1 - return_value_a = expected_concept.compiled["a"][0] + return_value_a = expected_concept.get_compiled()["a"][0] assert sheerka.isinstance(return_value_a, BuiltinConcepts.RETURN_VALUE) assert return_value_a.status assert sheerka.isinstance(return_value_a.body, BuiltinConcepts.PARSER_RESULT) @@ -1044,8 +1047,8 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka): # check the compiled expected_concept = lexer_nodes[0].concept - assert sheerka.isinstance(expected_concept.compiled["a"], "twenties") - assert expected_concept.compiled["a"].compiled["unit"] == cmap["one"] + assert sheerka.isinstance(expected_concept.get_compiled()["a"], "twenties") + assert expected_concept.get_compiled()["a"].get_compiled()["unit"] == cmap["one"] def test_i_can_parse_sequences(self): sheerka, context, parser = self.init_parser() @@ -1062,9 +1065,9 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka): ConceptNode(cmap["suffixed"], 10, 12, source="suffixed two")] # check the compiled - concept_plus_a = lexer_nodes[0].concept.compiled["a"] - concept_plus_b = lexer_nodes[0].concept.compiled["b"] - concept_suffixed_a = lexer_nodes[1].concept.compiled["a"] + concept_plus_a = lexer_nodes[0].concept.get_compiled()["a"] + concept_plus_b = lexer_nodes[0].concept.get_compiled()["b"] + concept_suffixed_a = lexer_nodes[1].concept.get_compiled()["a"] assert concept_plus_a == cmap["one"] assert len(concept_plus_b) == 1 @@ -1199,7 +1202,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka): concept_found = lexer_nodes[0].concept for unrecognized in expected_unrecognized: - assert isinstance(concept_found.compiled[unrecognized], UnrecognizedTokensNode) + assert isinstance(concept_found.get_compiled()[unrecognized], UnrecognizedTokensNode) @pytest.mark.parametrize("text, expected", [ ("x$!# suffixed one", [utnode(0, 4, "x$!# "), cnode("suffixed __var__0", 5, 7, "suffixed one")]), diff --git a/tests/parsers/test_UnrecognizedNodeParser.py b/tests/parsers/test_UnrecognizedNodeParser.py index 72597a2..0e57ef7 100644 --- a/tests/parsers/test_UnrecognizedNodeParser.py +++ b/tests/parsers/test_UnrecognizedNodeParser.py @@ -1,8 +1,11 @@ from core.builtin_concepts import ParserResultConcept, BuiltinConcepts from core.concept import Concept, CC from core.tokenizer import Tokenizer, TokenKind +from parsers.SequenceNodeParser import SequenceNodeParser from parsers.BaseNodeParser import ConceptNode, UnrecognizedTokensNode, scnode, cnode, \ utnode, SyaAssociativity, CN, CNC, UTN, SourceCodeWithConceptNode, SCWC, SourceCodeNode +from parsers.BnfNodeParser import BnfNodeParser +from parsers.SyaNodeParser import SyaNodeParser from parsers.UnrecognizedNodeParser import UnrecognizedNodeParser from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka @@ -14,7 +17,7 @@ def get_input_nodes_from(my_concepts_map, full_expr, *args): if isinstance(n, CC): concept = n.concept or Concept.update_from(my_concepts_map[n.concept_key]) for k, v in n.compiled.items(): - concept.compiled[k] = _get_real_node(v) + concept.get_compiled()[k] = _get_real_node(v) return concept if isinstance(n, (utnode, UTN)): @@ -26,7 +29,7 @@ def get_input_nodes_from(my_concepts_map, full_expr, *args): tokens = full_expr_as_tokens[n.start: n.end + 1] if hasattr(n, "compiled"): for k, v in n.compiled.items(): - concept.compiled[k] = _get_real_node(v) + concept.get_compiled()[k] = _get_real_node(v) return ConceptNode(concept, n.start, n.end, tokens) if isinstance(n, SCWC): @@ -118,45 +121,45 @@ class TestUnrecognizedNodeParser(TestUsingMemoryBasedSheerka): concept = res.body.concept assert concept == concepts_map["5params"] - assert len(concept.compiled["a"]) == 1 - assert sheerka.isinstance(concept.compiled["a"][0], BuiltinConcepts.RETURN_VALUE) - assert concept.compiled["a"][0].status - assert concept.compiled["a"][0].who == "parsers.AtomNode" - assert concept.compiled["a"][0].body.body == [cnode("one", 1, 1, "one")] + assert len(concept.get_compiled()["a"]) == 1 + assert sheerka.isinstance(concept.get_compiled()["a"][0], BuiltinConcepts.RETURN_VALUE) + assert concept.get_compiled()["a"][0].status + assert concept.get_compiled()["a"][0].who == "parsers." + SequenceNodeParser.NAME + assert concept.get_compiled()["a"][0].body.body == [cnode("one", 1, 1, "one")] - assert len(concept.compiled["b"]) == 1 - assert sheerka.isinstance(concept.compiled["b"][0], BuiltinConcepts.RETURN_VALUE) - assert concept.compiled["b"][0].status - assert concept.compiled["b"][0].who == "parsers.AtomNode" - assert concept.compiled["b"][0].body.body == [cnode("two", 1, 1, "two"), cnode("three", 3, 3, "three")] + assert len(concept.get_compiled()["b"]) == 1 + assert sheerka.isinstance(concept.get_compiled()["b"][0], BuiltinConcepts.RETURN_VALUE) + assert concept.get_compiled()["b"][0].status + assert concept.get_compiled()["b"][0].who == "parsers." + SequenceNodeParser.NAME + assert concept.get_compiled()["b"][0].body.body == [cnode("two", 1, 1, "two"), cnode("three", 3, 3, "three")] - assert len(concept.compiled["c"]) == 1 - assert sheerka.isinstance(concept.compiled["c"][0], BuiltinConcepts.RETURN_VALUE) - assert concept.compiled["c"][0].status - assert concept.compiled["c"][0].who == "parsers.BnfNode" + assert len(concept.get_compiled()["c"]) == 1 + assert sheerka.isinstance(concept.get_compiled()["c"][0], BuiltinConcepts.RETURN_VALUE) + assert concept.get_compiled()["c"][0].status + assert concept.get_compiled()["c"][0].who == "parsers." + BnfNodeParser.NAME expected_nodes = compute_expected_array( concepts_map, " twenty one ", [CNC("twenties", source="twenty one", unit="one")]) - assert concept.compiled["c"][0].body.body == expected_nodes + assert concept.get_compiled()["c"][0].body.body == expected_nodes - assert len(concept.compiled["d"]) == 1 - assert sheerka.isinstance(concept.compiled["d"][0], BuiltinConcepts.RETURN_VALUE) - assert concept.compiled["d"][0].status - assert concept.compiled["d"][0].who == "parsers.Python" - assert concept.compiled["d"][0].body.source == " 1 + 2 " + assert len(concept.get_compiled()["d"]) == 1 + assert sheerka.isinstance(concept.get_compiled()["d"][0], BuiltinConcepts.RETURN_VALUE) + assert concept.get_compiled()["d"][0].status + assert concept.get_compiled()["d"][0].who == "parsers.Python" + assert concept.get_compiled()["d"][0].body.source == " 1 + 2 " - assert len(concept.compiled["e"]) == 1 - assert sheerka.isinstance(concept.compiled["e"][0], BuiltinConcepts.RETURN_VALUE) - assert concept.compiled["e"][0].status - assert concept.compiled["e"][0].who == "parsers.SyaNode" + assert len(concept.get_compiled()["e"]) == 1 + assert sheerka.isinstance(concept.get_compiled()["e"][0], BuiltinConcepts.RETURN_VALUE) + assert concept.get_compiled()["e"][0].status + assert concept.get_compiled()["e"][0].who == "parsers." + SyaNodeParser.NAME expected_nodes = compute_expected_array( concepts_map, " one plus two mult three ", [CNC("plus", a="one", b=CC("mult", a="two", b="three"))], exclude_body=True) - assert concept.compiled["e"][0].body.body == expected_nodes + assert concept.get_compiled()["e"][0].body.body == expected_nodes # # sanity check, I can evaluate the concept # evaluated = sheerka.evaluate_concept(self.get_context(sheerka, eval_body=True), concept) @@ -177,25 +180,25 @@ class TestUnrecognizedNodeParser(TestUsingMemoryBasedSheerka): assert res.status assert res.body.concept == concepts_map["plus"] - assert len(res.body.concept.compiled["a"]) == 1 - assert res.body.concept.compiled["a"][0].status - assert res.body.concept.compiled["a"][0].who == "parsers.Python" - assert res.body.concept.compiled["a"][0].body.source == "1 " + assert len(res.body.concept.get_compiled()["a"]) == 1 + assert res.body.concept.get_compiled()["a"][0].status + assert res.body.concept.get_compiled()["a"][0].who == "parsers.Python" + assert res.body.concept.get_compiled()["a"][0].body.source == "1 " - assert res.body.concept.compiled["b"] == concepts_map["mult"] - assert sheerka.isinstance(res.body.concept.compiled["b"].compiled["a"][0], BuiltinConcepts.RETURN_VALUE) - assert res.body.concept.compiled["b"].compiled["a"][0].status - assert res.body.concept.compiled["b"].compiled["a"][0].who == "parsers.Python" - assert res.body.concept.compiled["b"].compiled["a"][0].body.source == " 2 " + assert res.body.concept.get_compiled()["b"] == concepts_map["mult"] + assert sheerka.isinstance(res.body.concept.get_compiled()["b"].get_compiled()["a"][0], BuiltinConcepts.RETURN_VALUE) + assert res.body.concept.get_compiled()["b"].get_compiled()["a"][0].status + assert res.body.concept.get_compiled()["b"].get_compiled()["a"][0].who == "parsers.Python" + assert res.body.concept.get_compiled()["b"].get_compiled()["a"][0].body.source == " 2 " - assert sheerka.isinstance(res.body.concept.compiled["b"].compiled["b"][0], BuiltinConcepts.RETURN_VALUE) - assert res.body.concept.compiled["b"].compiled["b"][0].status - assert res.body.concept.compiled["b"].compiled["b"][0].who == "parsers.BnfNode" + assert sheerka.isinstance(res.body.concept.get_compiled()["b"].get_compiled()["b"][0], BuiltinConcepts.RETURN_VALUE) + assert res.body.concept.get_compiled()["b"].get_compiled()["b"][0].status + assert res.body.concept.get_compiled()["b"].get_compiled()["b"][0].who == "parsers.Bnf" expected_nodes = compute_expected_array( concepts_map, " twenty two", [CNC("twenties", source="twenty two", unit="two")]) - assert res.body.concept.compiled["b"].compiled["b"][0].body.body == expected_nodes + assert res.body.concept.get_compiled()["b"].get_compiled()["b"][0].body.body == expected_nodes # def test_i_can_validate_and_evaluate_a_concept_node_with_python(self): # sheerka, context, parser = self.init_parser() @@ -211,12 +214,12 @@ class TestUnrecognizedNodeParser(TestUsingMemoryBasedSheerka): # # assert res.status # assert res.body.concept == concepts_map["plus"] - # assert res.body.concept.compiled["a"] == concepts_map["one"] - # assert len(res.body.concept.compiled["b"]) == 1 - # assert sheerka.isinstance(res.body.concept.compiled["b"][0], BuiltinConcepts.RETURN_VALUE) - # assert res.body.concept.compiled["b"][0].status - # assert res.body.concept.compiled["b"][0].who == "parsers.Python" - # assert res.body.concept.compiled["b"][0].body.source == "1 + 1" + # assert res.body.concept.get_compiled()["a"] == concepts_map["one"] + # assert len(res.body.concept.get_compiled()["b"]) == 1 + # assert sheerka.isinstance(res.body.concept.get_compiled()["b"][0], BuiltinConcepts.RETURN_VALUE) + # assert res.body.concept.get_compiled()["b"][0].status + # assert res.body.concept.get_compiled()["b"][0].who == "parsers.Python" + # assert res.body.concept.get_compiled()["b"][0].body.source == "1 + 1" # # # # evaluate # # context = self.get_context(sheerka, eval_body=True) @@ -231,10 +234,10 @@ class TestUnrecognizedNodeParser(TestUsingMemoryBasedSheerka): # # assert res.status # assert res.body.concept == concepts_map["plus"] - # assert res.body.concept.compiled["a"] == concepts_map["one"] - # assert len(res.body.concept.compiled["b"]) == 1 - # assert res.body.concept.compiled["b"][0].status - # assert res.body.concept.compiled["b"][0].who == "parsers.BnfNode" + # assert res.body.concept.get_compiled()["a"] == concepts_map["one"] + # assert len(res.body.concept.get_compiled()["b"]) == 1 + # assert res.body.concept.get_compiled()["b"][0].status + # assert res.body.concept.get_compiled()["b"][0].who == "parsers.BnfNode" # # # evaluate # context = self.get_context(sheerka, eval_body=True) @@ -336,7 +339,7 @@ class TestUnrecognizedNodeParser(TestUsingMemoryBasedSheerka): assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT) assert parser_result.source == expression assert len(actual_nodes) == 1 - assert actual_nodes[0].nodes[0].concept.metadata.is_evaluated # 'a plus b' is recognized as concept definition + assert actual_nodes[0].nodes[0].concept.get_metadata().is_evaluated # 'a plus b' is recognized as concept definition def test_i_can_parse_unrecognized_source_code_with_concept_node_when_var_in_short_term_memory(self): sheerka, context, parser = self.init_parser() @@ -355,7 +358,7 @@ class TestUnrecognizedNodeParser(TestUsingMemoryBasedSheerka): assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT) assert parser_result.source == expression assert len(actual_nodes) == 1 - assert not actual_nodes[0].nodes[0].concept.metadata.is_evaluated # 'a plus b' need to be evaluated + assert not actual_nodes[0].nodes[0].concept.get_metadata().is_evaluated # 'a plus b' need to be evaluated def test_i_can_parse_unrecognized_sya_concept_that_references_source_code(self): sheerka, context, parser = self.init_parser() @@ -381,8 +384,8 @@ class TestUnrecognizedNodeParser(TestUsingMemoryBasedSheerka): expression, [CN("hello_sya", source="hello get_user_name(twenty one)")], exclude_body=True) assert actual_nodes == expected_array - assert isinstance(actual_nodes[0].concept.compiled["a"], list) - assert sheerka.isinstance(actual_nodes[0].concept.compiled["a"][0], BuiltinConcepts.RETURN_VALUE) + assert isinstance(actual_nodes[0].concept.get_compiled()["a"], list) + assert sheerka.isinstance(actual_nodes[0].concept.get_compiled()["a"][0], BuiltinConcepts.RETURN_VALUE) def test_i_can_parse_sequences(self): sheerka, context, parser = self.init_parser() diff --git a/tests/repl/test_SheerkaPromptCompleter.py b/tests/repl/test_SheerkaPromptCompleter.py index 8c760f1..325992a 100644 --- a/tests/repl/test_SheerkaPromptCompleter.py +++ b/tests/repl/test_SheerkaPromptCompleter.py @@ -19,10 +19,13 @@ class TestSheerkaPromptCompleter(TestUsingMemoryBasedSheerka): assert as_dict["get_partition"].display_text == "get_partition" assert as_dict["get_partition"].display_meta_text == "builtin" - assert "get_results" in as_dict - assert as_dict["get_results"].text == "get_results()" - assert as_dict["get_results"].display_text == "get_results" - assert as_dict["get_results"].display_meta_text == "builtin" + completions = SheerkaPromptCompleter(sheerka).get_completions(Document("cache"), CompleteEvent()) + as_dict = {c.display_text: c for c in completions} + + assert "caches_names" in as_dict + assert as_dict["caches_names"].text == "caches_names()" + assert as_dict["caches_names"].display_text == "caches_names" + assert as_dict["caches_names"].display_meta_text == "builtin" def test_i_can_complete_with_commands(self): sheerka = self.get_sheerka() diff --git a/tests/sdp/test_sheerkaDataProvider.py b/tests/sdp/test_sheerkaDataProvider.py index 8e2da64..4fc218e 100644 --- a/tests/sdp/test_sheerkaDataProvider.py +++ b/tests/sdp/test_sheerkaDataProvider.py @@ -242,8 +242,10 @@ def test_i_can_load_an_entry(root): transaction.add("entry", "key2", "bar") transaction.add("entry", "key3", "baz") - load_entry = sdp.get("entry") + item = sdp.get("entry", "key1") + assert item == "foo" + load_entry = sdp.get("entry") assert load_entry == { "key1": "foo", "key2": "bar", @@ -369,6 +371,11 @@ def test_i_can_add_an_object_and_save_it_as_a_reference(root): assert sdp.get("entry", "key2") == [ObjNoKey("a", "b"), ObjNoKey("c", "d")] assert sdp.get("entry", "key3") == {ObjNoKey("a", "b"), ObjNoKey("c", "d")} + # I can ask for the whole entry + assert sdp.get("entry") == {"key1": ObjNoKey("a", "b"), + "key2": [ObjNoKey("a", "b"), ObjNoKey("c", "d")], + "key3": {ObjNoKey("a", "b"), ObjNoKey("c", "d")}} + state = sdp.load_state(sdp.get_snapshot(SheerkaDataProvider.HeadFile)) assert state.data == { "entry": {'key1': '##REF##:8fac7e801d08361c3449c594b4261ab9c45ef47f1a08df68eb717db2b6919774', diff --git a/tests/sheerkapickle/test_SheerkaPickler.py b/tests/sheerkapickle/test_SheerkaPickler.py index 66ed573..0540c68 100644 --- a/tests/sheerkapickle/test_SheerkaPickler.py +++ b/tests/sheerkapickle/test_SheerkaPickler.py @@ -1,12 +1,13 @@ import logging import pytest -from core.concept import Concept, ConceptParts +from core.concept import Concept +from core.tokenizer import Keywords from sheerkapickle import tags from sheerkapickle.SheerkaPickler import SheerkaPickler from sheerkapickle.SheerkaUnpickler import SheerkaUnpickler -from tests.TestUsingFileBasedSheerka import TestUsingFileBasedSheerka +from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka class Obj: @@ -28,7 +29,7 @@ class Obj: return hash((self.a, self.b, self.c)) -class TestSheerkaPickler(TestUsingFileBasedSheerka): +class TestSheerkaPickler(TestUsingMemoryBasedSheerka): @pytest.mark.parametrize("obj, expected", [ (1, 1), @@ -44,7 +45,7 @@ class TestSheerkaPickler(TestUsingFileBasedSheerka): ([1, [3.14, "a string"]], [1, [3.14, "a string"]]), ([1, (3.14, "a string")], [1, {tags.TUPLE: [3.14, "a string"]}]), ([], []), - (ConceptParts.BODY, {tags.ENUM: "core.concept.ConceptParts.BODY"}), + (Keywords.DEF, {tags.ENUM: 'core.tokenizer.Keywords.DEF'}), ]) def test_i_can_flatten_and_restore_primitives(self, obj, expected): sheerka = self.get_sheerka() @@ -105,9 +106,22 @@ class TestSheerkaPickler(TestUsingFileBasedSheerka): decoded = SheerkaUnpickler(sheerka).restore(flatten) assert decoded == obj + def test_i_cannot_correctly_flatten_compiled_and_generator(self): + sheerka = self.get_sheerka() + + obj = Obj((i for i in range(3)), compile("a + b", "", mode="eval"), None) + + flatten = SheerkaPickler(sheerka).flatten(obj) + + assert isinstance(flatten["a"], str) + assert flatten["a"].startswith(" /dev/null + + if [ "$available" = "" ]; then + echo "Error. No available environment !" >&2 + else + echo "Available environments are:" + echo "$available" + fi + +} + +if [ "$#" -eq 0 ]; then + echo "Usage: $0 " + list_available + exit 0 +fi + +env_file="$BASEDIR"/../_concepts_"$1".txt +env_folder="$HOME/.sheerka_$1" + +if ! [ -e "$env_file" ]; then + echo "$env_file not found" >&2 + list_available + exit 1 +fi + +echo "Rebuilding $1..." + +if [ -e ~/.sheerka ]; then + rm -rf ~/.sheerka.bak + mv ~/.sheerka ~/.sheerka.bak +fi + +python $BASEDIR/../main.py "sheerka.restore('$1')" +rm -rf "$env_folder" +cp -R ~/.sheerka "$env_folder" diff --git a/utils/sheerka.reset.sh b/utils/sheerka.reset.sh new file mode 100755 index 0000000..9c24059 --- /dev/null +++ b/utils/sheerka.reset.sh @@ -0,0 +1,19 @@ +#!/bin/sh + +if [ "$#" -eq 0 ]; then + echo "Resetting Sheerka environment." + rm -rf ~/.sheerka + exit 0 +fi + +if ! [ -e "$HOME/.sheerka_$1" ]; then + echo "$HOME/.sheerka_$1 not found" >&2 + available=$(ls -d $HOME/.sheerka_* | awk -F_ '{ print " "$2}') + echo "Available environments are:" + echo "$available" + exit 1 +fi + +echo "Resetting Sheerka environment to '$1'." +rm -rf ~/.sheerka +cp -r "$HOME/.sheerka_$1" ~/.sheerka