From dd520c1680110316570bc662504a84ed9b9bcc29 Mon Sep 17 00:00:00 2001 From: Kodjo Sossouvi Date: Mon, 21 Sep 2020 21:30:38 +0200 Subject: [PATCH] Fixed memory() and RET usage --- _concepts.txt | 90 ++++++------ _concepts_lite.txt | 10 +- src/core/builtin_concepts.py | 8 +- src/core/builtin_helpers.py | 17 +-- src/core/sheerka/ExecutionContext.py | 4 +- src/core/sheerka/Sheerka.py | 20 ++- src/core/sheerka/services/SheerkaAdmin.py | 26 +++- .../services/SheerkaEvaluateConcept.py | 58 +++++--- src/core/sheerka/services/SheerkaMemory.py | 132 ++++++++++++++++++ .../sheerka/services/SheerkaModifyConcept.py | 50 +++++-- .../sheerka/services/SheerkaSetsManager.py | 16 ++- .../services/SheerkaShortTermMemory.py | 37 ----- src/evaluators/AddToMemoryEvaluator.py | 27 ++++ src/evaluators/ConceptEvaluator.py | 3 +- src/evaluators/PostExecutionEvaluator.py | 2 +- src/evaluators/PythonEvaluator.py | 5 +- src/evaluators/RetEvaluator.py | 81 +++++------ src/evaluators/TooManySuccessEvaluator.py | 10 +- src/parsers/BaseNodeParser.py | 2 - src/parsers/ExactConceptParser.py | 5 +- src/parsers/FunctionParser.py | 13 +- src/parsers/SyaNodeParser.py | 3 +- src/sdp/readme.md | 1 + src/sdp/sheerkaSerializer.py | 44 +++--- tests/BaseTest.py | 1 + tests/core/test_SheerkaCreateNewConcept.py | 19 ++- tests/core/test_SheerkaEvaluateConcept.py | 82 +++++++++-- tests/core/test_SheerkaMemory.py | 123 ++++++++++++++++ tests/core/test_SheerkaModifyConcept.py | 12 ++ tests/core/test_SheerkaShortTermMemory.py | 52 ------- tests/evaluators/test_ConceptEvaluator.py | 2 - tests/evaluators/test_PythonEvaluator.py | 26 ++++ tests/evaluators/test_RetEvaluator.py | 94 ++++++------- .../evaluators/test_TooManySucessEvaluator.py | 22 ++- tests/non_reg/test_sheerka_non_reg.py | 47 ++++--- tests/parsers/test_FunctionParser.py | 1 + tests/parsers/test_SyaNodeParser.py | 24 ++++ 37 files changed, 816 insertions(+), 353 deletions(-) create mode 100644 src/core/sheerka/services/SheerkaMemory.py delete mode 100644 src/core/sheerka/services/SheerkaShortTermMemory.py create mode 100644 src/evaluators/AddToMemoryEvaluator.py create mode 100644 tests/core/test_SheerkaMemory.py delete mode 100644 tests/core/test_SheerkaShortTermMemory.py diff --git a/_concepts.txt b/_concepts.txt index 396d006..503ad05 100644 --- a/_concepts.txt +++ b/_concepts.txt @@ -19,64 +19,64 @@ def concept eighteen as 18 def concept nineteen as 19 def concept twenty as 20 def concept number -one isa number -two isa number -three isa number -four isa number -five isa number -six isa number -seven isa number -eight isa number -nine isa number -ten isa number -eleven isa number -twelve isa number -thirteen isa number -fourteen isa number -fifteen isa number -sixteen isa number -seventeen isa number -eighteen isa number -nineteen isa number -twenty isa number +set_isa(one, number) +set_isa(two, number) +set_isa(three, number) +set_isa(four, number) +set_isa(five, number) +set_isa(six, number) +set_isa(seven, number) +set_isa(eight, number) +set_isa(nine, number) +set_isa(ten, number) +set_isa(eleven, number) +set_isa(twelve, number) +set_isa(thirteen, number) +set_isa(fourteen, number) +set_isa(fifteen, number) +set_isa(sixteen, number) +set_isa(seventeen, number) +set_isa(eighteen, number) +set_isa(nineteen, number) +set_isa(twenty, number) def concept twenties from bnf twenty number where number < 10 as twenty + number -twenties isa number +set_isa(twenties, number) def concept thirty as 30 -thirty isa number +set_isa(thirty, number) def concept thirties from bnf thirty number where number < 10 as thirty + number -thirties isa number +set_isa(thirties, number) def concept forty as 40 -forty isa number +set_isa(forty, number) def concept forties from bnf forty number where number < 10 as forty + number -forties isa number +set_isa(forties, number) def concept fifty as 50 -fifty isa number +set_isa(fifty, number) def concept fifties from bnf fifty number where number < 10 as fifty + number -fifties isa number +set_isa(fifties, number) def concept sixty as 60 -sixty isa number +set_isa(sixty, number) def concept sixties from bnf sixty number where number < 10 as sixty + number -sixties isa number +set_isa(sixties, number) def concept seventy as 70 -seventy isa number +set_isa(seventy, number) def concept seventies from bnf seventy number where number < 10 as seventy + number -seventies isa number +set_isa(seventies, number) def concept eighty as 80 -eighty isa number +set_isa(eighty, number) def concept eighties from bnf eighty number where number < 10 as eighty + number -eighties isa number +set_isa(eighties, number) def concept ninety as 90 -ninety isa number +set_isa(ninety, number) def concept nineties from bnf ninety number where number < 10 as ninety + number -nineties isa number +set_isa(nineties, number) def concept one hundred as 100 -one hundred isa number +set_isa(one hundred, number) def concept hundreds from bnf number=n1 'hundred' 'and' number=n2 where n1 < 10 and n2 < 100 as n1 * 100 + n2 -hundreds isa number +set_isa(hundreds, number) def concept hundreds from bnf number 'hundred' where number < 10 as number * 100 last_created_concept() is number def concept thousands from bnf number 'thousand' where number < 1000 as number * 1000 -thousands isa number +set_isa(thousands, number) def concept thousands from bnf number=n1 'thousand' 'and' number=n2 as n1 * 1000 + n2 where n1 < 1000 and n2 < 1000 last_created_concept() is number def concept history as history() @@ -92,16 +92,16 @@ def concept explain as get_results() | filter("id == 0") | recurse(2) def concept explain last as get_last_results() | filter("id == 0") | recurse(2) def concept explain x as get_results() | filter(f"id == {x}") | recurse(3) where x def concept explain x '--recurse' y as get_results() | filter(f"id == {x}") | recurse(y) where x,y -set_isa(c:explain:, __COMMAND) -set_isa(c:explain last:, __COMMAND) -set_isa(c:explain x:, __COMMAND) +set_isa(c:explain:, __AUTO_EVAL) +set_isa(c:explain last:, __AUTO_EVAL) +set_isa(c:explain x:, __AUTO_EVAL) def concept precedence a > precedence b as set_is_greater_than(__PRECEDENCE, a, b) -set_isa(c:precedence a > precedence b:, __COMMAND) -def concept x is a command as set_isa(x, __COMMAND) -set_isa(c:x is a command:, __COMMAND) +set_isa(c:precedence a > precedence b:, __AUTO_EVAL) +def concept x is a command as set_isa(x, __AUTO_EVAL) +set_isa(c:x is a command:, __AUTO_EVAL) def concept q from q ? as question(q) pre is_question() set_is_lesser(__PRECEDENCE, q) def concept x is a 'concept' as isinstance(x, Concept) pre is_question() def concept x is a y as isa(x,y) pre is_question() def concept explain x values where x as get_results() | filter(f"id=={x}") | format_d -set_isa(c:explain x values:, __COMMAND) \ No newline at end of file +set_isa(c:explain x values:, __AUTO_EVAL) \ No newline at end of file diff --git a/_concepts_lite.txt b/_concepts_lite.txt index 2f9310b..cd17adf 100644 --- a/_concepts_lite.txt +++ b/_concepts_lite.txt @@ -1,8 +1,12 @@ def concept one as 1 def concept two as 2 def concept explain as get_results() | filter("id == 0") | recurse(2) -set_isa(c:explain:, __COMMAND) +set_isa(c:explain:, __AUTO_EVAL) def concept explain last as get_last_results() | filter("id == 0") | recurse(2) -set_isa(c:explain last:, __COMMAND) +set_isa(c:explain last:, __AUTO_EVAL) def concept explain x as get_results() | filter(f"id == {x}") | recurse(3) where x -set_isa(c:explain x:, __COMMAND) \ No newline at end of file +set_isa(c:explain x:, __AUTO_EVAL) +def concept apple +def concept table +def concept location +def concept x is on y as set_attr(x, location, y) diff --git a/src/core/builtin_concepts.py b/src/core/builtin_concepts.py index 7fb2436..5516b40 100644 --- a/src/core/builtin_concepts.py +++ b/src/core/builtin_concepts.py @@ -15,9 +15,9 @@ class BuiltinConcepts(Enum): """ SHEERKA = "sheerka" - # processing instructions during sheerka.execute() + # processing instructions during sheerka.execute() or sheerka.evaluate_concept() # The instruction may alter how the actions work - DEBUG = "debug" # activate all debug information + DEBUG = "debug" # activate all debug information EVAL_BODY_REQUESTED = "eval body" # to evaluate the body EVAL_WHERE_REQUESTED = "eval where" # to evaluate the where clause RETURN_BODY_REQUESTED = "return body" # returns the body of the concept instead of the concept itself @@ -53,7 +53,7 @@ class BuiltinConcepts(Enum): # builtin attributes ISA = "is a" # when a concept is an instance of another one - COMMAND = "command" # when the concept must be auto evaluated + AUTO_EVAL = "auto eval" # when the concept must be auto evaluated # object USER_INPUT = "user input concept" # represent an input from an user @@ -155,7 +155,7 @@ BuiltinUnique = [ BuiltinConcepts.TESTING, BuiltinConcepts.ISA, - BuiltinConcepts.COMMAND, + BuiltinConcepts.AUTO_EVAL, BuiltinConcepts.INVALID_LESSER_OPERATION, BuiltinConcepts.INVALID_GREATEST_OPERATION, diff --git a/src/core/builtin_helpers.py b/src/core/builtin_helpers.py index 8b08617..0a4cf7a 100644 --- a/src/core/builtin_helpers.py +++ b/src/core/builtin_helpers.py @@ -23,7 +23,7 @@ def is_same_success(context, return_values): Returns True if all returns values are successful and have the same value :param context: :param return_values: - :return: + :return: True False or None (None if the concept is not evaluated) """ assert isinstance(return_values, list) @@ -31,17 +31,10 @@ def is_same_success(context, return_values): if not ret_val.status: raise Exception("Status is false") - if isinstance(ret_val.body, Concept): - if not ret_val.body.metadata.is_evaluated: - evaluated = context.sheerka.evaluate_concept(context, ret_val.body, eval_body=True) - if not context.sheerka.is_success(evaluated): - raise Exception("Failed to evaluate evaluate") + if isinstance(ret_val.body, Concept) and not ret_val.body.metadata.is_evaluated: + raise Exception("Concept is not evaluated") - return context.sheerka.objvalue(evaluated) - else: - return context.sheerka.objvalue(ret_val.body) - else: - return context.sheerka.objvalue(ret_val) + return context.sheerka.objvalue(ret_val) try: reference = _get_value(return_values[0]) @@ -53,7 +46,7 @@ def is_same_success(context, return_values): except Exception as ex: context.log_error(ex) - return False + return None return True diff --git a/src/core/sheerka/ExecutionContext.py b/src/core/sheerka/ExecutionContext.py index b573529..0973a56 100644 --- a/src/core/sheerka/ExecutionContext.py +++ b/src/core/sheerka/ExecutionContext.py @@ -4,7 +4,7 @@ import time from core.builtin_concepts import BuiltinConcepts, ParserResultConcept from core.concept import Concept from core.sheerka.services.SheerkaExecute import NO_MATCH -from core.sheerka.services.SheerkaShortTermMemory import SheerkaShortTermMemory +from core.sheerka.services.SheerkaMemory import SheerkaMemory from core.sheerka_logger import get_logger from sdp.sheerkaDataProvider import Event @@ -130,7 +130,7 @@ class ExecutionContext: def __exit__(self, exc_type, exc_val, exc_tb): if self.stm: - self.sheerka.services[SheerkaShortTermMemory.NAME].remove_context(self) + self.sheerka.services[SheerkaMemory.NAME].remove_context(self) self._stop = time.time_ns() if self._show_stats: diff --git a/src/core/sheerka/Sheerka.py b/src/core/sheerka/Sheerka.py index aaba5d5..fb87ef6 100644 --- a/src/core/sheerka/Sheerka.py +++ b/src/core/sheerka/Sheerka.py @@ -64,6 +64,7 @@ class Sheerka(Concept): USER_CONCEPTS_KEYS = "User_Concepts" # sequential key for user defined concepts MAX_EXECUTION_HISTORY = 100 + MAX_RETURN_VALUES_HISTORY = 100 def __init__(self, cache_only=False, debug=False, loggers=None): self.init_logging(debug, loggers) @@ -113,6 +114,7 @@ class Sheerka(Concept): self.locals = {} self.last_executions = [] + self.last_return_values = [] @property def resolved_concepts_by_first_keyword(self): @@ -138,21 +140,23 @@ class Sheerka(Concept): def chicken_and_eggs(self): return self.cache_manager.caches[self.CHICKEN_AND_EGG_CONCEPTS_ENTRY].cache - def bind_service_method(self, bound_method, has_side_effect, as_name=None): + def bind_service_method(self, bound_method, has_side_effect, as_name=None, visible=True): """ Bind service method to sheerka instance for ease of use ? :param bound_method: :param has_side_effect: False if the method is safe - :param as_name: + :param as_name: give another name to the method + :param visible: make the method visible to Sheerka :return: """ if as_name is None: as_name = bound_method.__name__ - signature = inspect.signature(bound_method) - if len(signature.parameters) > 0 and list(signature.parameters.keys())[0] == "context": - self.methods_with_context.add(as_name) - self.sheerka_methods[as_name] = SheerkaMethod(bound_method, has_side_effect) + if visible: + signature = inspect.signature(bound_method) + if len(signature.parameters) > 0 and list(signature.parameters.keys())[0] == "context": + self.methods_with_context.add(as_name) + self.sheerka_methods[as_name] = SheerkaMethod(bound_method, has_side_effect) setattr(self, as_name, bound_method) @@ -437,6 +441,10 @@ class Sheerka(Concept): del self.last_executions[0] self.last_executions.append(execution_context) + if len(self.last_return_values) == self.MAX_RETURN_VALUES_HISTORY: + del self.last_return_values[0] + self.last_return_values.append(ret) + return ret def print(self, result, instructions=None): diff --git a/src/core/sheerka/services/SheerkaAdmin.py b/src/core/sheerka/services/SheerkaAdmin.py index 47f576e..3cace1e 100644 --- a/src/core/sheerka/services/SheerkaAdmin.py +++ b/src/core/sheerka/services/SheerkaAdmin.py @@ -3,9 +3,10 @@ import time from core.builtin_concepts import BuiltinConcepts from core.sheerka.services.sheerka_service import BaseService -CONCEPTS_FILE = "_concepts_lite.txt" +CONCEPTS_FILE_LITE = "_concepts_lite.txt" CONCEPTS_FILE_ALL_CONCEPTS = "_concepts.txt" -CONCEPTS_FILE_TO_USE = CONCEPTS_FILE +CONCEPTS_FILE_TO_USE = CONCEPTS_FILE_ALL_CONCEPTS + class SheerkaAdmin(BaseService): NAME = "Admin" @@ -19,6 +20,7 @@ class SheerkaAdmin(BaseService): self.sheerka.bind_service_method(self.restore, True) self.sheerka.bind_service_method(self.concepts, False) self.sheerka.bind_service_method(self.last_created_concept, False) + self.sheerka.bind_service_method(self.last_ret, False) def caches_names(self): """ @@ -27,16 +29,20 @@ class SheerkaAdmin(BaseService): """ return list(self.sheerka.cache_manager.caches.keys()) - def cache(self, name): + def cache(self, name, *keys): """ Returns the content of a cache :param name: + :param keys: look for a specific key. May ask to sdp if the key is not in cache :return: """ if name not in self.sheerka.cache_manager.caches: return self.sheerka.new(BuiltinConcepts.NOT_FOUND, body={"cache": name}) - return self.sheerka.cache_manager.caches[name].cache.copy() + if not keys: + return self.sheerka.cache_manager.caches[name].cache.copy() + + return {key: self.sheerka.cache_manager.get(name, key) for key in keys} def restore(self, concept_file=CONCEPTS_FILE_TO_USE): """ @@ -53,6 +59,7 @@ class SheerkaAdmin(BaseService): try: start = time.time_ns() nb_lines = 0 + nb_lines_in_error = 0 self.sheerka.during_restore = True with open(concept_file, "r") as f: for line in f.readlines(): @@ -63,14 +70,19 @@ class SheerkaAdmin(BaseService): self.sheerka.log.info(line) res = self.sheerka.evaluate_user_input(line) if len(res) > 1 or not res[0].status: - self.sheerka.log.error("Error detected !") + nb_lines_in_error += 1 + self.sheerka.log.error("\u001b[31mError detected !\u001b[0m") self.sheerka.during_restore = False stop = time.time_ns() nano_sec = stop - start dt = nano_sec / 1e6 elapsed = f"{dt} ms" if dt < 1000 else f"{dt / 1000} s" - print(f"Imported {nb_lines} line(s) in {elapsed}.") + self.sheerka.log.info(f"Imported {nb_lines} line(s) in {elapsed}.") + if nb_lines_in_error > 0: + self.sheerka.log.info(f"\u001b[31m{nb_lines_in_error} errors(s) found.\u001b[0m") + else: + self.sheerka.log.info(f"No error.") except IOError: pass @@ -89,3 +101,5 @@ class SheerkaAdmin(BaseService): return self.sheerka.new(BuiltinConcepts.NOT_FOUND) + def last_ret(self, context, index=-1): + return self.sheerka.last_return_values[index] diff --git a/src/core/sheerka/services/SheerkaEvaluateConcept.py b/src/core/sheerka/services/SheerkaEvaluateConcept.py index cac635f..066b129 100644 --- a/src/core/sheerka/services/SheerkaEvaluateConcept.py +++ b/src/core/sheerka/services/SheerkaEvaluateConcept.py @@ -2,7 +2,7 @@ from dataclasses import dataclass from core.builtin_concepts import BuiltinConcepts from core.builtin_helpers import expect_one, only_successful, parse_unrecognized, evaluate -from core.concept import Concept, DoNotResolve, ConceptParts, InfiniteRecursionResolved, NotInit +from core.concept import Concept, DoNotResolve, ConceptParts, InfiniteRecursionResolved, NotInit, ensure_concept from core.sheerka.services.SheerkaExecute import ParserInput from core.sheerka.services.sheerka_service import BaseService from core.tokenizer import Tokenizer @@ -32,6 +32,7 @@ class SheerkaEvaluateConcept(BaseService): def initialize(self): self.sheerka.bind_service_method(self.evaluate_concept, True) + self.sheerka.bind_service_method(self.set_auto_eval, True) @staticmethod def infinite_recursion_detected(context, concept): @@ -229,9 +230,12 @@ class SheerkaEvaluateConcept(BaseService): continue source = getattr(concept.metadata, part_key.value) - if source is None or not isinstance(source, str): + if source is None: # or not isinstance(source, str): continue + if not isinstance(source, str): + raise Exception("Invalid concept init. metadata must be a string") + if source.strip() == "": concept.compiled[part_key] = DoNotResolve(source) else: @@ -251,9 +255,12 @@ class SheerkaEvaluateConcept(BaseService): if var_name in concept.compiled: continue - if default_value is None or not isinstance(default_value, str): + if default_value is None: continue + if not isinstance(default_value, str): + raise Exception("Invalid concept init. variable metadata must be a string") + if default_value.strip() == "": concept.compiled[var_name] = DoNotResolve(default_value) else: @@ -332,12 +339,13 @@ class SheerkaEvaluateConcept(BaseService): # when it's a concept, evaluate it if isinstance(to_resolve, Concept) and \ not context.sheerka.isinstance(to_resolve, BuiltinConcepts.RETURN_VALUE): + evaluated = self.evaluate_concept(sub_context, to_resolve) sub_context.add_values(return_values=evaluated) - if evaluated.key == to_resolve.key: # quicker (and dirtier) than sheerka.is_success() - return self.apply_ret(evaluated) - else: + if not context.sheerka.is_success(evaluated) and evaluated.key != to_resolve.key: error = evaluated + else: + return evaluated # otherwise, execute all return values to find out what is the value else: @@ -449,7 +457,7 @@ class SheerkaEvaluateConcept(BaseService): sub_context.protected_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED) # auto evaluate commands - if context.sheerka.isa(concept, context.sheerka.new(BuiltinConcepts.COMMAND)): + if context.sheerka.isa(concept, context.sheerka.new(BuiltinConcepts.AUTO_EVAL)): sub_context.protected_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED) self.initialize_concept_asts(sub_context, concept) @@ -531,12 +539,19 @@ class SheerkaEvaluateConcept(BaseService): if "body" in all_metadata_to_eval: concept.metadata.is_evaluated = True - # # update the cache for concepts with no variables - # Cannot use cache. See the comment at the beginning of this method - # if len(concept.metadata.variables) == 0: - # self.sheerka.cache_manager.put(self.sheerka.CONCEPTS_BY_ID_ENTRY, concept.id, concept) + # # update the cache for concepts with no variables + # Cannot use cache. See the comment at the beginning of this method + # if len(concept.metadata.variables) == 0: + # self.sheerka.cache_manager.put(self.sheerka.CONCEPTS_BY_ID_ENTRY, concept.id, concept) - return concept + if not concept.metadata.is_builtin: + self.sheerka.register_object(sub_context, concept.name, concept) + + # manage RET metadata + if sub_context.in_context(BuiltinConcepts.EVAL_BODY_REQUESTED) and ConceptParts.RET in concept.values: + return concept.get_value(ConceptParts.RET) + else: + return concept def compute_metadata_to_eval(self, context, concept): to_eval = [] @@ -553,10 +568,11 @@ class SheerkaEvaluateConcept(BaseService): body |= b to_eval.extend(needed) - needed, v, b = self.get_needed_metadata(concept, ConceptParts.RET, not variables, not body) - variables |= v - body |= b - to_eval.extend(needed) + if context.in_context(BuiltinConcepts.EVAL_BODY_REQUESTED): + needed, v, b = self.get_needed_metadata(concept, ConceptParts.RET, not variables, not body) + variables |= v + body |= b + to_eval.extend(needed) needed, v, b = self.get_needed_metadata(concept, ConceptParts.POST, not variables, not body) variables |= v @@ -571,3 +587,13 @@ class SheerkaEvaluateConcept(BaseService): to_eval.append("body") return to_eval + + def set_auto_eval(self, context, concept): + """ + add AUTO_EVAL to ISA + :param context: + :param concept: + :return: + """ + ensure_concept(concept) + return self.sheerka.set_isa(context, concept, self.sheerka.new(BuiltinConcepts.AUTO_EVAL)) diff --git a/src/core/sheerka/services/SheerkaMemory.py b/src/core/sheerka/services/SheerkaMemory.py new file mode 100644 index 0000000..1363dcc --- /dev/null +++ b/src/core/sheerka/services/SheerkaMemory.py @@ -0,0 +1,132 @@ +from dataclasses import dataclass + +from cache.ListIfNeededCache import ListIfNeededCache +from core.builtin_concepts import BuiltinConcepts +from core.concept import Concept +from core.sheerka.services.sheerka_service import BaseService, ServiceObj + + +@dataclass +class MemoryObject(ServiceObj): + obj: object + + +class SheerkaMemory(BaseService): + NAME = "Memory" + + SHORT_TERM_OBJECTS_ENTRY = "Memory:ShortTermMemoryObjects" + OBJECTS_ENTRY = "Memory:Objects" + + def __init__(self, sheerka): + super().__init__(sheerka) + self.short_term_objects = ListIfNeededCache() + self.objects = ListIfNeededCache(default=lambda k: self.sheerka.sdp.get(self.OBJECTS_ENTRY, k)) + self.registration = {} + + def initialize(self): + self.sheerka.bind_service_method(self.get_from_short_term_memory, False, visible=False) + self.sheerka.bind_service_method(self.add_to_short_term_memory, True, visible=False) + self.sheerka.bind_service_method(self.add_to_memory, True, visible=False) + self.sheerka.bind_service_method(self.get_from_memory, False) + self.sheerka.bind_service_method(self.register_object, True, visible=False) + self.sheerka.bind_service_method(self.unregister_object, True, visible=False) + self.sheerka.bind_service_method(self.add_registered_objects, True, visible=False) + self.sheerka.bind_service_method(self.memory, False) + + self.sheerka.cache_manager.register_cache(self.SHORT_TERM_OBJECTS_ENTRY, self.short_term_objects, persist=False) + self.sheerka.cache_manager.register_cache(self.OBJECTS_ENTRY, self.objects, persist=True, use_ref=True) + + def get_from_short_term_memory(self, context, key): + while True: + key_to_use = (str(context.id) if context else "") + ":" + key + if (obj := self.sheerka.cache_manager.get(self.SHORT_TERM_OBJECTS_ENTRY, key_to_use)) is not None: + return obj + + if context is None: + return None + + context = context.get_parent() + + def add_to_short_term_memory(self, context, key, concept): + if context: + context.stm = True + key_to_use = (str(context.id) if context else "") + ":" + key + return self.sheerka.cache_manager.put(self.SHORT_TERM_OBJECTS_ENTRY, key_to_use, concept) + + def remove_context(self, context): + self.short_term_objects.evict_by_key(lambda k: k.startswith(str(context.id) + ":")) + + def add_to_memory(self, context, key, concept): + """ + Adds an object to memory + :param context: + :param key: + :param concept: + :return: + """ + self.objects.put(key, MemoryObject(context.event.get_digest(), concept)) + + def get_from_memory(self, context, key): + """" + """ + return self.objects.get(key) + + def register_object(self, context, key, concept): + """ + Before adding objects to memory, they first need to be registered + :param context: + :param key: + :param concept: + :return: + """ + self.registration[key] = concept + + def unregister_object(self, context, key): + """ + To indicate that key is no longer to be remembered + :param context: + :param key: + :return: + """ + try: + del self.registration[key] + except KeyError: + pass + + def add_registered_objects(self, context): + """ + Adds all registered objects + :param context: + :return: + """ + for k, v in self.registration.items(): + self.add_to_memory(context, k, v) + self.registration.clear() + + def memory(self, context, name=None): + """ + Get the list of all objects in memory + :param context: + :param name: + :return: + """ + if name: + name_to_use = name.name if isinstance(name, Concept) else name + self.unregister_object(context, name_to_use) + obj = self.get_from_memory(context, name_to_use) + if obj is None: + return self.sheerka.new(BuiltinConcepts.NOT_FOUND, body={"#name": name}) + + if isinstance(obj, list): + obj = obj[-1] + + return obj.obj + + res = {} + for k in self.objects: + obj = self.objects.get(k) + if isinstance(obj, list): + obj = obj[-1] + res[k] = obj.obj + + return res diff --git a/src/core/sheerka/services/SheerkaModifyConcept.py b/src/core/sheerka/services/SheerkaModifyConcept.py index d81bb32..f466de1 100644 --- a/src/core/sheerka/services/SheerkaModifyConcept.py +++ b/src/core/sheerka/services/SheerkaModifyConcept.py @@ -1,18 +1,27 @@ from core.builtin_concepts import BuiltinConcepts +from core.concept import ensure_concept from core.sheerka.services.sheerka_service import BaseService from parsers.BnfParser import BnfParser class SheerkaModifyConcept(BaseService): NAME = "ModifyConcept" - + def __init__(self, sheerka): super().__init__(sheerka) def initialize(self): self.sheerka.bind_service_method(self.modify_concept, True) + self.sheerka.bind_service_method(self.set_attr, True) + self.sheerka.bind_service_method(self.get_attr, False) def modify_concept(self, context, concept): + """ + Modify the definition of a concept + :param context: + :param concept: + :return: + """ old_version = self.sheerka.get_by_id(concept.id) if old_version is None: @@ -38,16 +47,12 @@ class SheerkaModifyConcept(BaseService): BuiltinConcepts.ALREADY_DEFINED, body=concept)) - old_references = self.sheerka.cache_manager.get(self.sheerka.CONCEPTS_REFERENCES_ENTRY, concept.id) - if old_references: - old_references = old_references.copy() - self.sheerka.cache_manager.update_concept(old_version, concept) - # TODO : update concept by first keyword - # TODO : update resolved by first keyword - # TODO : update concepts grammars - # TODO : update when definition_type = DEFINITION_TYPE_DEF + # TODO : update concept by first keyword : have a look at update_references() below + # TODO : update resolved by first keyword : have a look at update_references() below + # TODO : update when definition_type = DEFINITION_TYPE_DEF : have a look at update_references() below + # TODO : Update concepts grammars : have a look at update_references() below ret = self.sheerka.ret(self.NAME, True, self.sheerka.new(BuiltinConcepts.NEW_CONCEPT, body=concept)) return ret @@ -69,3 +74,30 @@ class SheerkaModifyConcept(BaseService): if concept.bnf is not None: BnfParser.update_recurse_id(context, concept_id, concept.bnf) + + # remove the grammar entry so that it can be recreated + self.sheerka.cache_manager.delete(self.sheerka.CONCEPTS_GRAMMARS_ENTRY, concept_id) + + def set_attr(self, concept, attribute, value): + """ + Modifies an attribute of a concept (concept.values) + :param context: + :param concept: + :param attribute: + :param value: + :return: + """ + ensure_concept(concept) + concept.set_value(attribute, value) + return self.sheerka.ret(self.NAME, True, self.sheerka.new(BuiltinConcepts.SUCCESS)) + + def get_attr(self, concept, attribute): + """ + Returns the attribute of a concept + :param context: + :param concept: + :param attribute: + :return: + """ + ensure_concept() + return concept.get_value(attribute) diff --git a/src/core/sheerka/services/SheerkaSetsManager.py b/src/core/sheerka/services/SheerkaSetsManager.py index 5f8e9d9..eaa93bb 100644 --- a/src/core/sheerka/services/SheerkaSetsManager.py +++ b/src/core/sheerka/services/SheerkaSetsManager.py @@ -59,10 +59,6 @@ class SheerkaSetsManager(BaseService): res = self.add_concept_to_set(context, concept, concept_set) - # update concept_set references - self.sheerka.services[SheerkaModifyConcept.NAME].update_references(context, concept_set) - self.concepts_in_set.delete(concept_set.id) - return res def add_concept_to_set(self, context, concept, concept_set): @@ -85,11 +81,21 @@ class SheerkaSetsManager(BaseService): self.sheerka.new(BuiltinConcepts.CONCEPT_ALREADY_IN_SET, body=concept, concept_set=concept_set)) self.sets.put(concept_set.id, concept.id) + + # invalidate the cache of what contains concept_set self.concepts_in_set.delete(concept_set.id) + + # update concept_set references + self.sheerka.services[SheerkaModifyConcept.NAME].update_references(context, concept_set) + return self.sheerka.ret(self.NAME, True, self.sheerka.new(BuiltinConcepts.SUCCESS)) def add_concepts_to_set(self, context, concepts, concept_set): - """Adding multiple concepts at the same time""" + """ + Adding multiple concepts at the same time + ******** THIS METHOD IS FOR TEST ONLY ************* + As it is not optimized. It needs to be rewritten in case of production usage + """ context.log(f"Adding concepts {concepts} to set {concept_set}", who=self.NAME) ensure_concept(concept_set) diff --git a/src/core/sheerka/services/SheerkaShortTermMemory.py b/src/core/sheerka/services/SheerkaShortTermMemory.py deleted file mode 100644 index bce7c83..0000000 --- a/src/core/sheerka/services/SheerkaShortTermMemory.py +++ /dev/null @@ -1,37 +0,0 @@ -from cache.ListIfNeededCache import ListIfNeededCache -from core.sheerka.services.sheerka_service import BaseService - - -class SheerkaShortTermMemory(BaseService): - NAME = "ShortTermMemory" - - SHORT_TERM_MEMORY_ENTRY = "ShortTermMemory:Objects" - - def __init__(self, sheerka): - super().__init__(sheerka) - self.objects = ListIfNeededCache() - - def initialize(self): - self.sheerka.bind_service_method(self.get_from_short_term_memory, False) - self.sheerka.bind_service_method(self.add_to_short_term_memory, True) - self.sheerka.cache_manager.register_cache(self.SHORT_TERM_MEMORY_ENTRY, self.objects, persist=False) - - def get_from_short_term_memory(self, context, key): - while True: - key_to_use = (str(context.id) if context else "") + ":" + key - if (obj := self.sheerka.cache_manager.get(self.SHORT_TERM_MEMORY_ENTRY, key_to_use)) is not None: - return obj - - if context is None: - return None - - context = context.get_parent() - - def add_to_short_term_memory(self, context, key, concept): - if context: - context.stm = True - key_to_use = (str(context.id) if context else "") + ":" + key - return self.sheerka.cache_manager.put(self.SHORT_TERM_MEMORY_ENTRY, key_to_use, concept) - - def remove_context(self, context): - self.objects.evict_by_key(lambda k: k.startswith(str(context.id) + ":")) diff --git a/src/evaluators/AddToMemoryEvaluator.py b/src/evaluators/AddToMemoryEvaluator.py new file mode 100644 index 0000000..263d4e9 --- /dev/null +++ b/src/evaluators/AddToMemoryEvaluator.py @@ -0,0 +1,27 @@ +from core.builtin_concepts import BuiltinConcepts +from evaluators.BaseEvaluator import OneReturnValueEvaluator + + +class AddToMemoryEvaluator(OneReturnValueEvaluator): + """ + Last chance to alter the return_value + This evaluator is supposed to be a generic evaluator for all rules that must be executed just before + the aggregations + """ + + NAME = "AddToMemory" + + def __init__(self): + super().__init__(self.NAME, [BuiltinConcepts.AFTER_EVALUATION], 10) + + def matches(self, context, return_value): + evaluation_parents = context.get_parents(lambda c: c.action == BuiltinConcepts.PROCESSING) + if len(evaluation_parents) > 1: + return False # It must be executed only when the top level context + + from core.sheerka.services.SheerkaMemory import SheerkaMemory + return len(context.sheerka.services[SheerkaMemory.NAME].registration) > 0 + + def eval(self, context, return_value): + context.sheerka.add_registered_objects(context) + return None # no need to have a second pass diff --git a/src/evaluators/ConceptEvaluator.py b/src/evaluators/ConceptEvaluator.py index 3a23cd1..fee149b 100644 --- a/src/evaluators/ConceptEvaluator.py +++ b/src/evaluators/ConceptEvaluator.py @@ -53,8 +53,9 @@ class ConceptEvaluator(OneReturnValueEvaluator): evaluated = sheerka.evaluate_concept(context, concept) - if evaluated.key != concept.key: + if not sheerka.is_success(evaluated) and evaluated.key != concept.key: # evaluated.key != concept.key means that we have transformed the concept + # not sheerka.is_success(evaluated) means that it was transformed into an error # When you successfully evaluate an error, the status should not be false return sheerka.ret( self.name, diff --git a/src/evaluators/PostExecutionEvaluator.py b/src/evaluators/PostExecutionEvaluator.py index e12a7f7..b69a51b 100644 --- a/src/evaluators/PostExecutionEvaluator.py +++ b/src/evaluators/PostExecutionEvaluator.py @@ -22,7 +22,7 @@ class PostExecutionEvaluator(OneReturnValueEvaluator): # only support the rule for the COMMANDS value = return_value.body - return isinstance(value, Concept) and context.sheerka.isa(value, context.sheerka.new(BuiltinConcepts.COMMAND)) + return isinstance(value, Concept) and context.sheerka.isa(value, context.sheerka.new(BuiltinConcepts.AUTO_EVAL)) def eval(self, context, return_value): # only support the rule for the COMMANDS diff --git a/src/evaluators/PythonEvaluator.py b/src/evaluators/PythonEvaluator.py index bdd0237..f1876a1 100644 --- a/src/evaluators/PythonEvaluator.py +++ b/src/evaluators/PythonEvaluator.py @@ -85,6 +85,9 @@ class PythonEvaluator(OneReturnValueEvaluator): not_for_me = context.sheerka.new(BuiltinConcepts.NOT_FOR_ME, body=node) return sheerka.ret(self.name, False, not_for_me, parents=[return_value]) + # If we evaluate a Concept metadata which is NOT the body ex (pre, post, where...) + # We need to disable the function that may alter the state + # It's a poor way to have source code security check attr_under_eval = context.get_parents(lambda ec: ec.action == BuiltinConcepts.EVALUATING_ATTRIBUTE) if attr_under_eval: attr_under_eval = attr_under_eval[0] @@ -256,7 +259,7 @@ class PythonEvaluator(OneReturnValueEvaluator): else: context.log(f"Evaluating '{concept}'", self.name) evaluated = context.sheerka.evaluate_concept(context, concept, eval_body=True) - if evaluated.key != concept.key: + if not context.sheerka.is_success(evaluated) and evaluated.key != concept.key: context.log(f"Error while evaluating '{name}'. Skipping.", self.name) continue concept = evaluated diff --git a/src/evaluators/RetEvaluator.py b/src/evaluators/RetEvaluator.py index 79ca476..d58ded0 100644 --- a/src/evaluators/RetEvaluator.py +++ b/src/evaluators/RetEvaluator.py @@ -1,40 +1,41 @@ -from core.builtin_concepts import BuiltinConcepts -from core.builtin_helpers import ensure_evaluated -from core.concept import Concept, ConceptParts -from evaluators.BaseEvaluator import OneReturnValueEvaluator - - -class RetEvaluator(OneReturnValueEvaluator): - """ - The evaluator transforms the concept, using the RET metadata value - """ - - NAME = "Ret" - - def __init__(self): - super().__init__(self.NAME, [BuiltinConcepts.EVALUATION], 10) - - def matches(self, context, return_value): - return return_value.status and \ - isinstance(return_value.value, Concept) and \ - return_value.value.metadata.ret is not None - - def eval(self, context, return_value): - sheerka = context.sheerka - concept = return_value.value - context.log(f"Processing ret value for concept {concept}.", self.name) - - if not concept.metadata.is_evaluated: - evaluated = ensure_evaluated(context, concept) - if evaluated.key != concept.key: - context.log(f"Failed to evaluate concept '{concept}'") - return None - ret = evaluated.get_value(ConceptParts.RET) - else: - ret = concept.get_value(ConceptParts.RET) - - if isinstance(ret, Concept) and sheerka.is_known(ret): - return sheerka.ret(self.name, True, ret, parents=[return_value]) - - context.log(f"ret '{ret}' is not a concept!") - return None +# from core.builtin_concepts import BuiltinConcepts +# from core.builtin_helpers import ensure_evaluated +# from core.concept import Concept, ConceptParts +# from evaluators.BaseEvaluator import OneReturnValueEvaluator +# +# +# class RetEvaluator(OneReturnValueEvaluator): +# """ +# The evaluator transforms the concept, using the RET metadata value +# """ +# +# NAME = "Ret" +# +# def __init__(self): +# super().__init__(self.NAME, [BuiltinConcepts.EVALUATION], 10) +# self.enabled = False +# +# def matches(self, context, return_value): +# return return_value.status and \ +# isinstance(return_value.value, Concept) and \ +# return_value.value.metadata.ret is not None +# +# def eval(self, context, return_value): +# sheerka = context.sheerka +# concept = return_value.value +# context.log(f"Processing ret value for concept {concept}.", self.name) +# +# if not concept.metadata.is_evaluated: +# evaluated = ensure_evaluated(context, concept) +# if evaluated.key != concept.key: +# context.log(f"Failed to evaluate concept '{concept}'") +# return None +# ret = evaluated.get_value(ConceptParts.RET) +# else: +# ret = concept.get_value(ConceptParts.RET) +# +# if isinstance(ret, Concept) and sheerka.is_known(ret): +# return sheerka.ret(self.name, True, ret, parents=[return_value]) +# +# context.log(f"ret '{ret}' is not a concept!") +# return None diff --git a/src/evaluators/TooManySuccessEvaluator.py b/src/evaluators/TooManySuccessEvaluator.py index 5cd4414..06b53f5 100644 --- a/src/evaluators/TooManySuccessEvaluator.py +++ b/src/evaluators/TooManySuccessEvaluator.py @@ -1,8 +1,8 @@ import logging -from core.builtin_concepts import BuiltinConcepts import core.builtin_helpers -from evaluators.BaseEvaluator import AllReturnValuesEvaluator, BaseEvaluator +from core.builtin_concepts import BuiltinConcepts +from evaluators.BaseEvaluator import AllReturnValuesEvaluator from parsers.BaseParser import BaseParser @@ -44,7 +44,11 @@ class TooManySuccessEvaluator(AllReturnValuesEvaluator): context.log(s, self.name) context.log(f"value={sheerka.value(s.value)}", self.name) - if not core.builtin_helpers.is_same_success(context, self.success): + same_success = core.builtin_helpers.is_same_success(context, self.success) + if same_success is None: + return None + + if not same_success: context.log(f"Values are different. Raising {BuiltinConcepts.TOO_MANY_SUCCESS}.", self.name) too_many_success = sheerka.new(BuiltinConcepts.TOO_MANY_SUCCESS, body=self.success) return sheerka.ret(self.name, False, too_many_success, parents=self.eaten) diff --git a/src/parsers/BaseNodeParser.py b/src/parsers/BaseNodeParser.py index f3bf9c8..4269f95 100644 --- a/src/parsers/BaseNodeParser.py +++ b/src/parsers/BaseNodeParser.py @@ -772,8 +772,6 @@ class BaseNodeParser(BaseParser): if token.type == TokenKind.STRING: name = token.value[1:-1] if strip_quotes else token.value - elif token.type == TokenKind.KEYWORD: - name = token.value.value else: name = token.value diff --git a/src/parsers/ExactConceptParser.py b/src/parsers/ExactConceptParser.py index 70868a1..025be7a 100644 --- a/src/parsers/ExactConceptParser.py +++ b/src/parsers/ExactConceptParser.py @@ -48,7 +48,7 @@ class ExactConceptParser(BaseParser): for combination in self.combinations(words): concept_key = " ".join(combination) - result = sheerka.new(concept_key) # use new(), not get() because we need a new instance + result = sheerka.get_by_key(concept_key) # use new(), not get() because we need a new instance if sheerka.isinstance(result, BuiltinConcepts.UNKNOWN_CONCEPT): continue @@ -66,6 +66,9 @@ class ExactConceptParser(BaseParser): continue context.log(f"Recognized concept {concept}.", self.name) + # We can ask for a new instance + concept = sheerka.new_from_template(concept, concept.key) + # update the properties if needed for i, token in enumerate(combination): if token.startswith(VARIABLE_PREFIX): diff --git a/src/parsers/FunctionParser.py b/src/parsers/FunctionParser.py index ec15036..a8ab86f 100644 --- a/src/parsers/FunctionParser.py +++ b/src/parsers/FunctionParser.py @@ -38,6 +38,14 @@ class NamesNode(FunctionParserNode): def to_unrecognized(self): return UnrecognizedTokensNode(self.start, self.end, self.tokens).fix_source() + def to_str_unrecognized(self): + token = Token(TokenKind.STRING, + "'" + self.str_value() + "'", + self.tokens[0].index, + self.tokens[0].line, + self.tokens[0].column) + return UnrecognizedTokensNode(self.start, self.end, [token]).fix_source() + @dataclass() class FunctionParameter: @@ -322,10 +330,13 @@ class FunctionParser(BaseParser): scn.add_node(sep.to_unrecognized()) res = [SourceCodeWithConceptNode(function_node.first.to_unrecognized(), function_node.last.to_unrecognized())] + + function_name = function_node.first.str_value() + for param in function_node.parameters: if isinstance(param.value, NamesNode): - unrecognized = param.value.to_unrecognized() # try to recognize concepts + unrecognized = param.value.to_unrecognized() nodes_sequences = get_lexer_nodes_from_unrecognized(self.context, unrecognized, PARSERS) diff --git a/src/parsers/SyaNodeParser.py b/src/parsers/SyaNodeParser.py index d65e63c..7d887c7 100644 --- a/src/parsers/SyaNodeParser.py +++ b/src/parsers/SyaNodeParser.py @@ -1219,7 +1219,8 @@ class SyaNodeParser(BaseNodeParser): start = inner_item.start if inner_item.end > end: end = inner_item.end - has_unrecognized |= isinstance(inner_item, (UnrecognizedTokensNode, SourceCodeWithConceptNode)) + has_unrecognized |= isinstance(inner_item, (UnrecognizedTokensNode, SourceCodeWithConceptNode)) or \ + hasattr(inner_item, "has_unrecognized") and inner_item.has_unrecognized param_name = concept.metadata.variables[param_index][0] param_value = inner_item.concept if hasattr(inner_item, "concept") else \ diff --git a/src/sdp/readme.md b/src/sdp/readme.md index 43895e8..9a1d789 100644 --- a/src/sdp/readme.md +++ b/src/sdp/readme.md @@ -14,6 +14,7 @@ - D : concept definitions (no history management) - R : executionContext ('R' stands for Result or ReturnValue, no history management) - O : ServiceObj (from pickle) +- M : MemoryObject (using SheerkaPickle) ## How concepts are serialized ? - get the id of the concept diff --git a/src/sdp/sheerkaSerializer.py b/src/sdp/sheerkaSerializer.py index d8457b7..6067bab 100644 --- a/src/sdp/sheerkaSerializer.py +++ b/src/sdp/sheerkaSerializer.py @@ -60,6 +60,7 @@ class Serializer: self.register(ConceptSerializer()) self.register(DictionarySerializer()) self.register(ExecutionContextSerializer()) + self.register(MemoryObjectSerializer()) # before ServiceObjSerializer self.register(ServiceObjSerializer()) def register(self, serializer): @@ -229,13 +230,14 @@ class StateSerializer(PickleSerializer): 1) -class ConceptSerializer(BaseSerializer): +class SheerkaPickleSerializer(BaseSerializer): - def __init__(self): - BaseSerializer.__init__(self, "C", 1) + def __init__(self, predicate, name, version): + BaseSerializer.__init__(self, name, version) + self.predicate = predicate def matches(self, obj): - return isinstance(obj, Concept) + return self.predicate(obj) def dump(self, stream, obj, context): stream.write(sheerkapickle.encode(context.sheerka, obj).encode("utf-8")) @@ -248,6 +250,12 @@ class ConceptSerializer(BaseSerializer): return obj +class ConceptSerializer(SheerkaPickleSerializer): + + def __init__(self): + super().__init__(lambda obj: isinstance(obj, Concept), "C", 1) + + class DictionarySerializer(BaseSerializer): def __init__(self): super().__init__("D", 1) @@ -267,25 +275,11 @@ class DictionarySerializer(BaseSerializer): return obj -class ExecutionContextSerializer(BaseSerializer): +class ExecutionContextSerializer(SheerkaPickleSerializer): CLASS_NAME = "core.sheerka.ExecutionContext.ExecutionContext" def __init__(self): - BaseSerializer.__init__(self, "R", 1) - - def matches(self, obj): - return get_full_qualified_name(obj) == self.CLASS_NAME - - def dump(self, stream, obj, context): - stream.write(sheerkapickle.encode(context.sheerka, obj).encode("utf-8")) - stream.seek(0) - return stream - - def load(self, stream, context): - json_stream = stream.read().decode("utf-8") - obj = sheerkapickle.decode(context.sheerka, json_stream) - # json_message = json.loads(json_stream) - return obj + super().__init__(lambda obj: get_full_qualified_name(obj) == self.CLASS_NAME, "R", 1) class ServiceObjSerializer(PickleSerializer): @@ -297,7 +291,9 @@ class ServiceObjSerializer(PickleSerializer): "O", 1) -# -# class SheerkaSerializer(ObjectSerializer): -# def __init__(self): -# ObjectSerializer.__init__(self, "core.sheerka.Sheerka", "C", 1) + +class MemoryObjectSerializer(SheerkaPickleSerializer): + CLASS_NAME = "core.sheerka.services.SheerkaMemory.MemoryObject" + + def __init__(self): + super().__init__(lambda obj: get_full_qualified_name(obj) == self.CLASS_NAME, "R", 1) diff --git a/tests/BaseTest.py b/tests/BaseTest.py index f8591e3..bf112ee 100644 --- a/tests/BaseTest.py +++ b/tests/BaseTest.py @@ -90,6 +90,7 @@ class BaseTest: instance = sheerka.new(concept.key if isinstance(concept, Concept) else concept) for i, var in enumerate(instance.metadata.variables): if var[0] in kwargs: + assert isinstance(kwargs[var[0]], str), "variables definitions must be string" instance.metadata.variables[i] = (var[0], kwargs[var[0]]) return instance diff --git a/tests/core/test_SheerkaCreateNewConcept.py b/tests/core/test_SheerkaCreateNewConcept.py index c829a52..ee69812 100644 --- a/tests/core/test_SheerkaCreateNewConcept.py +++ b/tests/core/test_SheerkaCreateNewConcept.py @@ -182,7 +182,7 @@ class TestSheerkaCreateNewConcept(TestUsingMemoryBasedSheerka): assert sheerka.cache_manager.get(sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, "-") == [bnf_concept.id] assert sheerka.cache_manager.get(sheerka.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY, "-") == [bnf_concept.id] - def test_concept_references_are_updated(self): + def test_concept_references_are_updated_1(self): sheerka, context, one, two, number, twenty, twenties = self.init_concepts( "one", "two", @@ -198,6 +198,23 @@ class TestSheerkaCreateNewConcept(TestUsingMemoryBasedSheerka): assert sheerka.cache_manager.get(sheerka.CONCEPTS_REFERENCES_ENTRY, twenty.id) == {twenties.id} assert sheerka.cache_manager.get(sheerka.CONCEPTS_REFERENCES_ENTRY, twenties.id) is None + def test_concept_references_are_updated_2(self): + sheerka, context, one, two, number, twenty, twenties = self.init_concepts( + "one", + "two", + "number", + "twenty", + Concept("twenties", definition="twenty number"), + create_new=True + ) + + assert sheerka.cache_manager.get(sheerka.CONCEPTS_REFERENCES_ENTRY, one.id) is None + assert sheerka.cache_manager.get(sheerka.CONCEPTS_REFERENCES_ENTRY, two.id) is None + assert sheerka.cache_manager.get(sheerka.CONCEPTS_REFERENCES_ENTRY, number.id) == {twenties.id} + assert sheerka.cache_manager.get(sheerka.CONCEPTS_REFERENCES_ENTRY, twenty.id) == {twenties.id} + assert sheerka.cache_manager.get(sheerka.CONCEPTS_REFERENCES_ENTRY, twenties.id) is None + + class TestSheerkaCreateNewConceptFileBased(TestUsingFileBasedSheerka): def test_i_can_add_several_concepts(self): diff --git a/tests/core/test_SheerkaEvaluateConcept.py b/tests/core/test_SheerkaEvaluateConcept.py index bc80a3b..5742c0d 100644 --- a/tests/core/test_SheerkaEvaluateConcept.py +++ b/tests/core/test_SheerkaEvaluateConcept.py @@ -2,6 +2,7 @@ import pytest from core.builtin_concepts import BuiltinConcepts, ReturnValueConcept, ParserResultConcept from core.concept import Concept, DoNotResolve, ConceptParts, Property, InfiniteRecursionResolved, CB, NotInit from core.sheerka.services.SheerkaEvaluateConcept import SheerkaEvaluateConcept +from core.sheerka.services.SheerkaMemory import SheerkaMemory from parsers.PythonParser import PythonNode from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka @@ -34,6 +35,8 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka): assert evaluated.metadata.is_evaluated assert len(evaluated.values) == 0 if body is None else 1 + assert "foo" in sheerka.services[SheerkaMemory.NAME].registration + @pytest.mark.parametrize("expr, expected", [ ("", ""), ("1", 1), @@ -191,7 +194,13 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka): assert sheerka.objvalue(evaluated) == CB("a", BuiltinConcepts.NOT_INITIALIZED) assert evaluated.metadata.is_evaluated - def test_i_can_evaluate_concept_when_variables_reference_others_concepts(self): + def test_i_can_evaluate_concept_when_variables_reference_others_concepts_1(self): + """ + The body references a variable. + The variable reference a concept + The variable name is also the name of a concept + :return: + """ sheerka, context, concept_a, concept = self.init_concepts( Concept("a"), Concept("foo", body="a").def_var("a", "a"), @@ -218,6 +227,35 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka): assert evaluated.key == concept.key assert evaluated.body == concept_a + def test_i_can_evaluate_concept_when_variables_reference_others_concepts_3(self): + """ + The body references a variable. + The variable reference a concept + The name of the variable is also the name of a concept, but the variable points to something else + :return: + """ + sheerka, context, concept_a, concept_b = self.init_concepts("a", "b", eval_body=True) + + concept = Concept("foo", body="a").def_var("a", "b") + evaluated = sheerka.evaluate_concept(context, concept) + + assert evaluated.key == concept.key + assert evaluated.body == concept_b + + def test_i_can_evaluate_concept_when_variables_reference_others_concepts_4(self): + """ + The body references a variable. + The variable reference a concept + :return: + """ + sheerka, context, concept_b = self.init_concepts("b", eval_body=True) + + concept = Concept("foo", body="a").def_var("a", "b") + evaluated = sheerka.evaluate_concept(context, concept) + + assert evaluated.key == concept.key + assert evaluated.body == concept_b + def test_i_can_evaluate_concept_when_variables_reference_others_concepts_with_body(self): sheerka, context, *concepts = self.init_concepts( Concept(name="a", body="1"), @@ -714,7 +752,7 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka): assert evaluated.key == command.key assert "a" not in sheerka.locals - sheerka.set_isa(context, command, sheerka.new(BuiltinConcepts.COMMAND)) + sheerka.set_isa(context, command, sheerka.new(BuiltinConcepts.AUTO_EVAL)) evaluated = sheerka.evaluate_concept(context, sheerka.new("command")) assert evaluated.key == command.key assert "a" in sheerka.locals @@ -730,7 +768,7 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka): setattr(foo.metadata, metadata, "a=10; print('10')") foo.metadata.need_validation = True - evaluated = sheerka.evaluate_concept(context, foo) + evaluated = sheerka.evaluate_concept(context, foo, eval_body=True) captured = capsys.readouterr() assert sheerka.isinstance(evaluated, BuiltinConcepts.CONCEPT_EVAL_ERROR) @@ -773,19 +811,24 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka): captured = capsys.readouterr() assert captured.out == "" - @pytest.mark.parametrize("concept, expected", [ - (Concept("foo"), []), - (Concept("foo", pre="pre", post="post", ret="ret", where="where"), ["pre", "ret", "post"]), - (Concept("foo", pre="a").def_var("a"), ["variables", "pre"]), - (Concept("foo", pre="self"), ["body", "pre"]), - (Concept("foo", pre="self + a").def_var("a"), ["variables", "body", "pre"]), - (Concept("foo", pre="self + a", ret="ret").def_var("a"), ["variables", "body", "pre", "ret"]), - (Concept("foo", body="body"), []) # only if eval_body_is_set + @pytest.mark.parametrize("concept, eval_body, expected", [ + (Concept("foo"), False, []), + (Concept("foo", pre="pre", post="post", ret="ret", where="where"), False, ["pre", "post"]), + (Concept("foo", pre="pr", post="p", ret="r", where="w"), True, ["pre", "ret", "post", "variables", "body"]), + (Concept("foo", pre="a").def_var("a"), False, ["variables", "pre"]), + (Concept("foo", pre="self"), False, ["body", "pre"]), + (Concept("foo", pre="self + a").def_var("a"), False, ["variables", "body", "pre"]), + (Concept("foo", pre="self + a", ret="ret").def_var("a"), False, ["variables", "body", "pre"]), + (Concept("foo", pre="self + a", ret="ret").def_var("a"), True, ["variables", "body", "pre", "ret"]), + (Concept("foo", body="body"), False, []) ]) - def test_i_can_compute_metadata_to_eval(self, concept, expected): + def test_i_can_compute_metadata_to_eval(self, concept, eval_body, expected): sheerka, context, concept = self.init_concepts(concept) service = sheerka.services[SheerkaEvaluateConcept.NAME] + if eval_body: + context.add_to_protected_hints(BuiltinConcepts.EVAL_BODY_REQUESTED) + service.initialize_concept_asts(context, concept) assert service.compute_metadata_to_eval(context, concept) == expected @@ -839,6 +882,21 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka): evaluated = sheerka.evaluate_concept(context, concept, metadata=['pre']) assert evaluated.values == {"a": Property("a", NotInit), ConceptParts.PRE: Property(ConceptParts.PRE, 'pre')} + def test_i_can_manage_ret(self): + sheerka, context, foo, bar = self.init_concepts("foo", Concept("bar", ret="foo")) + + res = sheerka.evaluate_concept(context, bar) + assert res.id == bar.id + + res = sheerka.evaluate_concept(context, bar, eval_body=True) + assert res.id == foo.id + + def test_ret_is_evaluated_only_is_body_is_requested(self): + sheerka, context, foo, bar = self.init_concepts("foo", Concept("bar", ret="__NOT_FOUND")) + + res = sheerka.evaluate_concept(context, bar, eval_body=False) + assert res.id == bar.id + # I cannot implement value cache for now # def test_values_when_no_variables_are_computed_only_once(self): # sheerka, context, foo = self.init_concepts(Concept("foo", body="10")) diff --git a/tests/core/test_SheerkaMemory.py b/tests/core/test_SheerkaMemory.py new file mode 100644 index 0000000..7d08b38 --- /dev/null +++ b/tests/core/test_SheerkaMemory.py @@ -0,0 +1,123 @@ +from core.builtin_concepts import BuiltinConcepts +from core.concept import Concept +from core.sheerka.ExecutionContext import ExecutionContext +from core.sheerka.services.SheerkaMemory import SheerkaMemory, MemoryObject + +from tests.TestUsingFileBasedSheerka import TestUsingFileBasedSheerka +from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka + + +class TestSheerkaMemory(TestUsingMemoryBasedSheerka): + def test_i_can_add_to_global_short_term_memory(self): + sheerka = self.get_sheerka() + service = sheerka.services[SheerkaMemory.NAME] + + foo = Concept("foo") + sheerka.add_to_short_term_memory(None, "a", foo) + + assert service.short_term_objects.copy() == {":a": foo} + assert id(sheerka.get_from_short_term_memory(None, "a")) == id(foo) + + def test_i_can_add_context_short_term_memory(self): + sheerka, context = self.init_concepts() + service = sheerka.services[SheerkaMemory.NAME] + + foo = Concept("foo") + sheerka.add_to_short_term_memory(context, "a", foo) + + context_id = ExecutionContext.ids[context.event.get_digest()] + assert service.short_term_objects.copy() == {f"{context_id}:a": foo} + assert id(sheerka.get_from_short_term_memory(context, "a")) == id(foo) + assert sheerka.get_from_short_term_memory(None, "a") is None + + def test_i_can_get_obj_from_parents(self): + sheerka, context = self.init_concepts() + service = sheerka.services[SheerkaMemory.NAME] + foo = Concept("foo") + sheerka.add_to_short_term_memory(None, "a", foo) + + with context.push(BuiltinConcepts.TESTING, None) as sub_context: + assert service.short_term_objects.copy() == {":a": foo} + assert id(sheerka.get_from_short_term_memory(sub_context, "a")) == id(foo) + assert id(sheerka.get_from_short_term_memory(context, "a")) == id(foo) + assert id(sheerka.get_from_short_term_memory(None, "a")) == id(foo) + + def test_entry_are_removed_on_context_exit(self): + sheerka, context = self.init_concepts() + + with context.push(BuiltinConcepts.TESTING, None) as sub_context: + foo = Concept("foo") + sheerka.add_to_short_term_memory(sub_context, "a", foo) + assert id(sheerka.get_from_short_term_memory(sub_context, "a")) == id(foo) + + assert sheerka.get_from_short_term_memory(sub_context, "a") is None + + def test_i_can_add_and_retrieve_from_memory(self): + sheerka, context = self.init_concepts() + service = sheerka.services[SheerkaMemory.NAME] + + assert sheerka.get_from_memory(context, "a") is None + + foo = Concept("foo") + sheerka.add_to_memory(context, "a", foo) + + assert service.objects.copy() == {"a": MemoryObject(context.event.get_digest(), foo)} + assert id(sheerka.get_from_memory(context, "a").obj) == id(foo) + + def test_i_can_use_memory_to_get_the_list_of_all_objects(self): + sheerka, context = self.init_concepts() + foo = Concept("foo") + bar = Concept("bar") + + sheerka.add_to_memory(context, "foo", 'value that will not appear') + sheerka.add_to_memory(context, "foo", foo) + sheerka.add_to_memory(context, "bar", bar) + + assert sheerka.memory(context) == {"foo": foo, "bar": bar} + + def test_i_can_use_memory_with_a_string(self): + sheerka, context = self.init_concepts() + + foo = Concept("foo") + sheerka.add_to_memory(context, "foo", foo) + + assert sheerka.memory(context, "foo") == foo + + def test_i_can_use_memory_with_a_concept(self): + sheerka, context = self.init_concepts() + + foo = Concept("foo") + sheerka.add_to_memory(context, "foo", foo) + + assert sheerka.memory(context, Concept("foo")) == foo + + def test_concept_not_found_is_return_when_not_found(self): + sheerka, context = self.init_concepts() + + res = sheerka.memory(context, "foo") + + assert sheerka.isinstance(res, BuiltinConcepts.NOT_FOUND) + assert res.body == {"#name": "foo"} + + def test_memory_only_returns_the_last_object(self): + sheerka, context = self.init_concepts() + + foo = Concept("foo") + bar = Concept("bar") + + sheerka.add_to_memory(context, "item", foo) + sheerka.add_to_memory(context, "item", bar) + + assert sheerka.memory(context, "item") == bar + + +class TestSheerkaMemoryUsingFileBase(TestUsingFileBasedSheerka): + def test_i_can_record_memory_objects(self): + sheerka, context = self.init_concepts() + + sheerka.add_to_memory(context, "item", Concept("foo")) + sheerka.cache_manager.commit(context) + + sheerka = self.get_sheerka() + context = self.get_context(sheerka) + assert sheerka.get_from_memory(context, "item").obj == Concept("foo") diff --git a/tests/core/test_SheerkaModifyConcept.py b/tests/core/test_SheerkaModifyConcept.py index 2411c29..acdd3bc 100644 --- a/tests/core/test_SheerkaModifyConcept.py +++ b/tests/core/test_SheerkaModifyConcept.py @@ -92,6 +92,18 @@ class TestSheerkaModifyConcept(TestUsingMemoryBasedSheerka): assert foo_from_sheerka[0].metadata.body == "1" assert foo_from_sheerka[1].metadata.body == "value" + def test_i_can_get_and_set_attribute(self): + sheerka, context = self.init_concepts() + foo = Concept("foo") + prop = Concept("property") + bar = Concept("bar") + + res = sheerka.set_attr(foo, prop, bar) + assert res.status + assert sheerka.isinstance(res.body, BuiltinConcepts.SUCCESS) + + assert sheerka.get_attr(foo, prop) == bar + class TestSheerkaModifyConceptUsingFile(TestUsingFileBasedSheerka): diff --git a/tests/core/test_SheerkaShortTermMemory.py b/tests/core/test_SheerkaShortTermMemory.py deleted file mode 100644 index 2cb395c..0000000 --- a/tests/core/test_SheerkaShortTermMemory.py +++ /dev/null @@ -1,52 +0,0 @@ -from core.builtin_concepts import BuiltinConcepts -from core.concept import Concept -from core.sheerka.ExecutionContext import ExecutionContext -from core.sheerka.services.SheerkaShortTermMemory import SheerkaShortTermMemory - -from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka - - -class TestSheerkaShortTermMemory(TestUsingMemoryBasedSheerka): - def test_i_can_add_to_global_short_term_memory(self): - sheerka = self.get_sheerka() - service = sheerka.services[SheerkaShortTermMemory.NAME] - - foo = Concept("foo") - sheerka.add_to_short_term_memory(None, "a", foo) - - assert service.objects.copy() == {":a": foo} - assert id(sheerka.get_from_short_term_memory(None, "a")) == id(foo) - - def test_i_can_add_context_short_term_memory(self): - sheerka, context = self.init_concepts() - service = sheerka.services[SheerkaShortTermMemory.NAME] - - foo = Concept("foo") - sheerka.add_to_short_term_memory(context, "a", foo) - - context_id = ExecutionContext.ids[context.event.get_digest()] - assert service.objects.copy() == {f"{context_id}:a": foo} - assert id(sheerka.get_from_short_term_memory(context, "a")) == id(foo) - assert sheerka.get_from_short_term_memory(None, "a") is None - - def test_i_can_get_obj_from_parents(self): - sheerka, context = self.init_concepts() - service = sheerka.services[SheerkaShortTermMemory.NAME] - foo = Concept("foo") - sheerka.add_to_short_term_memory(None, "a", foo) - - with context.push(BuiltinConcepts.TESTING, None) as sub_context: - assert service.objects.copy() == {":a": foo} - assert id(sheerka.get_from_short_term_memory(sub_context, "a")) == id(foo) - assert id(sheerka.get_from_short_term_memory(context, "a")) == id(foo) - assert id(sheerka.get_from_short_term_memory(None, "a")) == id(foo) - - def test_entry_are_removed_on_context_exit(self): - sheerka, context = self.init_concepts() - - with context.push(BuiltinConcepts.TESTING, None) as sub_context: - foo = Concept("foo") - sheerka.add_to_short_term_memory(sub_context, "a", foo) - assert id(sheerka.get_from_short_term_memory(sub_context, "a")) == id(foo) - - assert sheerka.get_from_short_term_memory(sub_context, "a") is None diff --git a/tests/evaluators/test_ConceptEvaluator.py b/tests/evaluators/test_ConceptEvaluator.py index 1f6794f..b96cd43 100644 --- a/tests/evaluators/test_ConceptEvaluator.py +++ b/tests/evaluators/test_ConceptEvaluator.py @@ -24,7 +24,6 @@ class TestAddConceptEvaluator(TestUsingMemoryBasedSheerka): concept = Concept(name="foo", where="True", pre="2 > 1", - ret="3", post="4").def_var("a", "5").def_var("b", "6") evaluator = ConceptEvaluator() @@ -36,7 +35,6 @@ class TestAddConceptEvaluator(TestUsingMemoryBasedSheerka): assert result.value.name == "foo" assert result.value.get_value(ConceptParts.WHERE) == True assert result.value.get_value(ConceptParts.PRE) == True - assert result.value.get_value(ConceptParts.RET) == 3 assert result.value.get_value(ConceptParts.POST) == 4 assert result.value.get_value("a") == 5 assert result.value.get_value("b") == 6 diff --git a/tests/evaluators/test_PythonEvaluator.py b/tests/evaluators/test_PythonEvaluator.py index 028b06d..e00c2b1 100644 --- a/tests/evaluators/test_PythonEvaluator.py +++ b/tests/evaluators/test_PythonEvaluator.py @@ -8,6 +8,7 @@ from core.tokenizer import Tokenizer from evaluators.PythonEvaluator import PythonEvaluator, PythonEvalError from parsers.BaseNodeParser import SourceCodeNode, SourceCodeWithConceptNode from parsers.PythonParser import PythonNode, PythonParser +from parsers.PythonWithConceptsParser import PythonWithConceptsParser from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka @@ -17,6 +18,15 @@ def get_concept_name(concept): def get_source_code_node(source_code, concepts=None): + if concepts: + for concept_name, concept in sorted(concepts.items(), key=lambda kv: len(kv[0]), reverse=True): + identifier = "__C__" + PythonWithConceptsParser.sanitize(concept.name) + if concept.id: + identifier += "__" + concept.id + identifier += "__C__" + source_code = source_code.replace(concept_name, identifier) + concepts[identifier] = concept + if source_code: python_node = PythonNode(source_code, ast.parse(source_code, f"", 'eval')) else: @@ -32,6 +42,11 @@ def get_source_code_node(source_code, concepts=None): return scwcn +def get_ret_val_from_source_code(context, source_code, concepts): + parsed = get_source_code_node(source_code, concepts) + return context.sheerka.ret("parsers.??", True, ParserResultConcept(value=parsed)) + + class TestPythonEvaluator(TestUsingMemoryBasedSheerka): @pytest.mark.parametrize("ret_val, expected", [ @@ -289,3 +304,14 @@ class TestPythonEvaluator(TestUsingMemoryBasedSheerka): assert evaluated.status assert evaluated.value == 11 + + def test_i_can_eval_concept_with_ret(self): + sheerka, context, one, the = self.init_concepts("one", Concept("the a", ret="a").def_var("a")) + ret_val = get_ret_val_from_source_code(context, "test_using_context(one, the one)", { + "the one": self.get_concept_instance(sheerka, the, a="one"), + "one": self.get_concept_instance(sheerka, "one") + }) + + evaluated = PythonEvaluator().eval(context, ret_val) + assert evaluated.status + assert evaluated.value.startswith("I have access to Sheerka ! param1=(1001)one, param2=(1001)one, event=") diff --git a/tests/evaluators/test_RetEvaluator.py b/tests/evaluators/test_RetEvaluator.py index 2862c08..24f5919 100644 --- a/tests/evaluators/test_RetEvaluator.py +++ b/tests/evaluators/test_RetEvaluator.py @@ -1,47 +1,47 @@ -import pytest -from core.builtin_concepts import ReturnValueConcept -from core.concept import Concept -from evaluators.RetEvaluator import RetEvaluator - -from tests.BaseTest import BaseTest -from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka - - -class TestRetEvaluator(TestUsingMemoryBasedSheerka): - - @pytest.mark.parametrize("ret_val, expected", [ - (ReturnValueConcept("who", True, Concept("foo", ret="bar")), True), - (ReturnValueConcept("who", False, Concept("foo", ret="bar")), False), - (ReturnValueConcept("who", True, Concept("foo")), False), - (BaseTest.pretval(Concept("foo", ret="bar")), False), - (ReturnValueConcept("who", True, "not even a concept"), False), - ]) - def test_i_can_match(self, ret_val, expected): - context = self.get_context() - assert RetEvaluator().matches(context, ret_val) == expected - - def test_i_can_evaluate_fully_initialized(self): - sheerka, context, foo, the_concept = self.init_concepts("foo", Concept("the concept", ret="a").def_var("a")) - - instance = sheerka.new("the concept") - instance.set_value("a", sheerka.new("foo")) - ret_value = self.tretval(sheerka, instance) - - res = RetEvaluator().eval(context, ret_value) - assert res.status - assert sheerka.isinstance(res.body, "foo") - - @pytest.mark.parametrize("instance, expected", [ - (Concept("with ret", ret="a").def_var("a", "foo"), "foo"), - (Concept("with ret", ret="a", body="10").def_var("a", "foo"), "foo"), - (Concept("with ret", ret="a").def_var("a", "bar"), "bar"), - ]) - def test_i_can_evaluate(self, instance, expected): - sheerka, context, foo, bar = self.init_concepts("foo", Concept("bar", body="10")) - - ret_value = self.tretval(sheerka, instance) - - res = RetEvaluator().eval(context, ret_value) - assert res.status - assert sheerka.isinstance(res.body, expected) - +# import pytest +# from core.builtin_concepts import ReturnValueConcept +# from core.concept import Concept +# from evaluators.RetEvaluator import RetEvaluator +# +# from tests.BaseTest import BaseTest +# from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka +# +# +# class TestRetEvaluator(TestUsingMemoryBasedSheerka): +# +# @pytest.mark.parametrize("ret_val, expected", [ +# (ReturnValueConcept("who", True, Concept("foo", ret="bar")), True), +# (ReturnValueConcept("who", False, Concept("foo", ret="bar")), False), +# (ReturnValueConcept("who", True, Concept("foo")), False), +# (BaseTest.pretval(Concept("foo", ret="bar")), False), +# (ReturnValueConcept("who", True, "not even a concept"), False), +# ]) +# def test_i_can_match(self, ret_val, expected): +# context = self.get_context() +# assert RetEvaluator().matches(context, ret_val) == expected +# +# def test_i_can_evaluate_fully_initialized(self): +# sheerka, context, foo, the_concept = self.init_concepts("foo", Concept("the concept", ret="a").def_var("a")) +# +# instance = sheerka.new("the concept") +# instance.set_value("a", sheerka.new("foo")) +# ret_value = self.tretval(sheerka, instance) +# +# res = RetEvaluator().eval(context, ret_value) +# assert res.status +# assert sheerka.isinstance(res.body, "foo") +# +# @pytest.mark.parametrize("instance, expected", [ +# (Concept("with ret", ret="a").def_var("a", "foo"), "foo"), +# (Concept("with ret", ret="a", body="10").def_var("a", "foo"), "foo"), +# (Concept("with ret", ret="a").def_var("a", "bar"), "bar"), +# ]) +# def test_i_can_evaluate(self, instance, expected): +# sheerka, context, foo, bar = self.init_concepts("foo", Concept("bar", body="10")) +# +# ret_value = self.tretval(sheerka, instance) +# +# res = RetEvaluator().eval(context, ret_value) +# assert res.status +# assert sheerka.isinstance(res.body, expected) +# diff --git a/tests/evaluators/test_TooManySucessEvaluator.py b/tests/evaluators/test_TooManySucessEvaluator.py index 98f78d3..87e3ec5 100644 --- a/tests/evaluators/test_TooManySucessEvaluator.py +++ b/tests/evaluators/test_TooManySucessEvaluator.py @@ -1,5 +1,4 @@ import pytest - from core.builtin_concepts import ReturnValueConcept, BuiltinConcepts from core.concept import Concept from evaluators.TooManySuccessEvaluator import TooManySuccessEvaluator @@ -44,8 +43,8 @@ class TestTooManySuccessEvaluator(TestUsingMemoryBasedSheerka): def test_i_can_eval(self): context = self.get_context() - value1 = r("evaluators.a", value=Concept("c1", body="1")) - value2 = r("evaluators.a", value=Concept("c2", body="2")) + value1 = r("evaluators.a", value=Concept("c1", body="1").auto_init()) + value2 = r("evaluators.a", value=Concept("c2", body="2").auto_init()) return_values = [ value1, value2, @@ -80,6 +79,23 @@ class TestTooManySuccessEvaluator(TestUsingMemoryBasedSheerka): assert matches assert res is None + def test_i_do_not_eval_when_the_concepts_are_not_evaluated(self): + context = self.get_context() + + return_values = [ + r("evaluators.a", value=Concept("c1", body="1")), + r("evaluators.a", value=Concept("c2", body="2")), + r("other", False), + reduce_requested + ] + + evaluator = TooManySuccessEvaluator() + matches = evaluator.matches(context, return_values) + res = evaluator.eval(context, return_values) + + assert matches + assert res is None + def test_other_success_are_not_reduced(self): context = self.get_context() diff --git a/tests/non_reg/test_sheerka_non_reg.py b/tests/non_reg/test_sheerka_non_reg.py index aaa469c..e95f103 100644 --- a/tests/non_reg/test_sheerka_non_reg.py +++ b/tests/non_reg/test_sheerka_non_reg.py @@ -1,6 +1,6 @@ import pytest from core.builtin_concepts import BuiltinConcepts -from core.concept import Concept, PROPERTIES_TO_SERIALIZE, simplec, CMV, NotInit +from core.concept import Concept, PROPERTIES_TO_SERIALIZE, simplec, CMV, NotInit, CC from evaluators.MutipleSameSuccessEvaluator import MultipleSameSuccessEvaluator from evaluators.OneSuccessEvaluator import OneSuccessEvaluator from evaluators.PythonEvaluator import PythonEvalError @@ -236,16 +236,16 @@ as: self.create_and_add_in_cache_concept(sheerka, Concept(name="hello b", body="'hello you ' + b").def_var("b")) self.create_and_add_in_cache_concept(sheerka, Concept(name="foo", body="'another value'")) - res = sheerka.evaluate_user_input("hello foo") + res = sheerka.evaluate_user_input("eval hello foo") assert len(res) == 1 assert not res[0].status assert sheerka.isinstance(res[0].value, BuiltinConcepts.TOO_MANY_SUCCESS) concepts = res[0].value.body assert len(concepts) == 2 - sorted_values = sorted(concepts, key=lambda x: x.value.body) - assert sorted_values[0].value.body == "hello another value" - assert sorted_values[1].value.body == "hello you another value" + sorted_values = sorted(concepts, key=lambda x: x.value) + assert sorted_values[0].value == "hello another value" + assert sorted_values[1].value == "hello you another value" def test_i_can_manage_concepts_with_the_same_key_when_values_are_the_same(self): sheerka = self.get_sheerka() @@ -254,10 +254,10 @@ as: sheerka.create_new_concept(context, Concept(name="hello a", body="'hello ' + a").def_var("a")) sheerka.create_new_concept(context, Concept(name="hello b", body="'hello ' + b").def_var("b")) - res = sheerka.evaluate_user_input("hello 'foo'") + res = sheerka.evaluate_user_input("eval hello 'foo'") assert len(res) == 1 assert res[0].status - assert res[0].value.body == "hello foo" # I don't know yet the one to choose + assert res[0].value == "hello foo" # I don't know yet the one to choose assert res[0].who == sheerka.get_evaluator_name(MultipleSameSuccessEvaluator.NAME) def test_i_can_create_concepts_with_python_code_as_body(self): @@ -574,7 +574,6 @@ as: assert res[0].status assert res[0].body == 3 - def test_eval_does_not_break_valid_result(self): sheerka = self.get_sheerka() sheerka.evaluate_user_input("def concept one as 1") @@ -981,13 +980,13 @@ as: sheerka = self.init_scenario(init) the = sheerka.get_by_name("the a") - # res = sheerka.evaluate_user_input("one plus the one") - # assert res[0].status - # plus = res[0].body - # assert isinstance(plus, Concept) - # assert plus.name == "plus" - # assert plus.compiled["a"] == sheerka.new("one") - # assert plus.compiled["b"] == CC(the, a=sheerka.new("one")) + res = sheerka.evaluate_user_input("one plus the one") + assert res[0].status + plus = res[0].body + assert isinstance(plus, Concept) + assert plus.name == "plus" + assert plus.compiled["a"] == sheerka.new("one") + assert plus.compiled["b"] == CC(the, a=sheerka.new("one")) res = sheerka.evaluate_user_input("eval one plus the one") assert res[0].status @@ -996,7 +995,7 @@ as: def test_i_can_evaluate_command(self): init = [ "def concept command as 'Executed !'", - "set_isa(c:command:, __COMMAND)", + "set_isa(c:command:, __AUTO_EVAL)", ] # Since command is a __COMMAND, the body is auto evaluated @@ -1144,6 +1143,22 @@ as: assert res[0].status assert sheerka.isinstance(res[0].body, BuiltinConcepts.NEW_CONCEPT) + def test_bnf_node_parsers_are_updated_when_concepts_are_modified(self): + init = [ + "def concept number", + "def concept one", + "def concept twenties from bnf 'twenty' number as 20 + number", + ] + sheerka = self.init_scenario(init) + + res = sheerka.evaluate_user_input("twenty one") + assert len(res) > 1 + + sheerka.evaluate_user_input("set_isa(one, number)") + res = sheerka.evaluate_user_input("twenty one") + assert len(res) == 1 + assert res[0].status + class TestSheerkaNonRegFile(TestUsingFileBasedSheerka): def test_i_can_def_several_concepts(self): diff --git a/tests/parsers/test_FunctionParser.py b/tests/parsers/test_FunctionParser.py index 35c3428..76fb95e 100644 --- a/tests/parsers/test_FunctionParser.py +++ b/tests/parsers/test_FunctionParser.py @@ -174,3 +174,4 @@ class TestFunctionParser(TestUsingMemoryBasedSheerka): assert isinstance(concept.compiled["b"], list) for item in concept.compiled["b"]: assert sheerka.isinstance(item, BuiltinConcepts.RETURN_VALUE) + diff --git a/tests/parsers/test_SyaNodeParser.py b/tests/parsers/test_SyaNodeParser.py index e63e41d..91f96b4 100644 --- a/tests/parsers/test_SyaNodeParser.py +++ b/tests/parsers/test_SyaNodeParser.py @@ -1154,6 +1154,30 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka): assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT) assert lexer_nodes == expected_array + @pytest.mark.parametrize("text, expected_result", [ + ("a plus b", [CN("plus", source="a plus b")]), + ("suffixed a plus b", [CN("suffixed", source="suffixed a plus b")]), + ]) + def test_i_can_almost_parse_concept_definition(self, text, expected_result): + """ + In these examples, 'a' and 'b' are not defined. + So the status of the return value cannot be True + :param text: + :param expected_result: + :return: + """ + sheerka, context, parser = self.init_parser() + + res = parser.parse(context, ParserInput(text)) + + wrapper = res.body + lexer_nodes = res.body.body + + expected_array = compute_expected_array(cmap, text, expected_result) + assert not res.status + assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT) + assert lexer_nodes == expected_array + @pytest.mark.parametrize("text, expected_concept, expected_unrecognized", [ ("x$!# prefixed", "prefixed", ["a"]), ("suffixed x$!#", "suffixed", ["a"]),