From 7310bc5522a4547b1d72ec894662e4773ad07acc Mon Sep 17 00:00:00 2001 From: Kodjo Sossouvi Date: Tue, 23 Jun 2020 15:22:27 +0200 Subject: [PATCH] Fixed infinite recursion when parsing complex BNF node --- .gitignore | 1 + _concepts.txt | 12 +- src/cache/CacheManager.py | 1 + src/core/builtin_concepts.py | 1 + src/core/concept.py | 12 +- src/core/sheerka/ExecutionContext.py | 33 +- src/core/sheerka/Sheerka.py | 2 +- src/core/sheerka/services/SheerkaAdmin.py | 9 + .../services/SheerkaComparisonManager.py | 8 +- .../services/SheerkaCreateNewConcept.py | 2 +- .../services/SheerkaEvaluateConcept.py | 5 + src/core/sheerka/services/SheerkaExecute.py | 11 + .../sheerka/services/SheerkaSetsManager.py | 42 +- .../services/SheerkaVariableManager.py | 8 + src/core/utils.py | 27 +- src/parsers/BaseNodeParser.py | 94 ++- src/parsers/BnfNodeParser.py | 347 ++++++++--- tests/BaseTest.py | 12 +- tests/core/test_ExecutionContext.py | 18 + tests/core/test_SheerkaComparisonManager.py | 40 +- tests/core/test_SheerkaVariableManager.py | 21 +- tests/evaluators/test_PythonEvaluator.py | 2 +- tests/non_reg/test_sheerka_non_reg.py | 67 ++- tests/parsers/parsers_utils.py | 2 +- tests/parsers/test_BaseNodeParser.py | 22 +- tests/parsers/test_BnfNodeParser.py | 539 ++++++++++++++++-- tests/parsers/test_SyaNodeParser.py | 12 +- tests/sheerkapickle/test_sheerka_handlers.py | 8 +- 28 files changed, 1082 insertions(+), 276 deletions(-) diff --git a/.gitignore b/.gitignore index 6a68cec..7ef544b 100644 --- a/.gitignore +++ b/.gitignore @@ -6,5 +6,6 @@ __pycache__ build _build prof +log.txt tests/_concepts.txt tests/**/*result_test \ No newline at end of file diff --git a/_concepts.txt b/_concepts.txt index 8ef973f..8a5e8d0 100644 --- a/_concepts.txt +++ b/_concepts.txt @@ -69,10 +69,10 @@ def concept ninety as 90 ninety isa number def concept nineties from bnf ninety number where number < 10 as ninety + number nineties isa number -def concept hundreds1 from number hundred where number1 < 10 as number1 * 100 -def concept hundreds2 from number1 hundred and number2 where number1 < 10 and number2 < 100 as number1 * 100 + number2 -def concept one hundred as 100 -one hundred isa number -hundreds1 isa number -hundreds2 isa number +# def concept hundreds1 from number 'hundred' where number < 10 as number * 100 +# def concept hundreds2 from number=number1 'hundred' 'and' number=number2 where number1 < 10 and number2 < 100 as number1 * 100 + number2 +# def concept one hundred as 100 +# c:one hundred: isa number +# hundreds1 isa number +# hundreds2 isa number def concept history as history() diff --git a/src/cache/CacheManager.py b/src/cache/CacheManager.py index 783b4e9..7e9cab9 100644 --- a/src/cache/CacheManager.py +++ b/src/cache/CacheManager.py @@ -262,6 +262,7 @@ class CacheManager: def reset(self, cache_only): """For unit test speed enhancement""" + self.clear() self.cache_only = cache_only self.caches.clear() self.concept_caches.clear() diff --git a/src/core/builtin_concepts.py b/src/core/builtin_concepts.py index 20d7c93..d509a44 100644 --- a/src/core/builtin_concepts.py +++ b/src/core/builtin_concepts.py @@ -37,6 +37,7 @@ class BuiltinConcepts(Enum): MANAGE_INFINITE_RECURSION = "manage infinite recursion" PARSE_CODE = "execute source code" EXEC_CODE = "execute source code" + TESTING = "testing" USER_INPUT = "user input" # represent an input from an user SUCCESS = "success" diff --git a/src/core/concept.py b/src/core/concept.py index 5272911..4c39beb 100644 --- a/src/core/concept.py +++ b/src/core/concept.py @@ -332,6 +332,9 @@ class Concept: for k in other.values: self.set_value(k, other.get_value(k)) + # update bnf definition + self.bnf = other.bnf + # origin from sdp.sheerkaSerializer import Serializer if hasattr(other, Serializer.ORIGIN): @@ -533,6 +536,10 @@ class CC: self.end = None # for debug purpose, indicate where the concept ends self.exclude_body = exclude_body + if "body" in self.compiled: + self.compiled[ConceptParts.BODY] = self.compiled["body"] + del self.compiled["body"] + def __eq__(self, other): if id(self) == id(other): return True @@ -544,7 +551,10 @@ class CC: to_compare = {k: v for k, v in other.compiled.items() if k != ConceptParts.BODY} else: to_compare = other.compiled - return self.compiled == to_compare + if self.compiled == to_compare: + return True + else: + return False if not isinstance(other, CC): return False diff --git a/src/core/sheerka/ExecutionContext.py b/src/core/sheerka/ExecutionContext.py index 69257ef..de78f6c 100644 --- a/src/core/sheerka/ExecutionContext.py +++ b/src/core/sheerka/ExecutionContext.py @@ -334,7 +334,10 @@ class ExecutionContext: for k, v in self._bag.items(): bag[k] = v bag["bag." + k] = v - for prop in ("id", "who", "desc", "obj", "inputs", "values", "concepts"): + for prop in ("id", "who", "action", "desc", "obj", "inputs", "values", "concepts"): + bag[prop] = getattr(self, prop) + bag["action"] = self.action_context + for prop in ("desc", "obj", "inputs", "values", "concepts"): bag[prop] = getattr(self, prop) bag["status"] = self.get_status() bag["elapsed"] = self.elapsed @@ -362,15 +365,23 @@ class ExecutionContext: :param predicate: :return: """ - res = [] - current = self - while True: - parent = current._parent - if parent: - if predicate is None or predicate(parent): - res.append(parent) - current = parent - else: + return list(self.search(predicate, None, False)) + + def search(self, predicate=None, get_obj=None, start_with_self=False, stop=None): + """ + Iter thru execution context parent and return the list of obj + :param predicate: what execution context to keep + :param get_obj: lambda to compute what to return + :param start_with_self: include the current execution context in the search + :param stop: stop the search if matched + :return: + """ + current = self if start_with_self else self._parent + while current: + if stop and stop(current): break - return res + if predicate is None or predicate(current): + yield current if get_obj is None else get_obj(current) + + current = current._parent diff --git a/src/core/sheerka/Sheerka.py b/src/core/sheerka/Sheerka.py index 9de89c1..c605170 100644 --- a/src/core/sheerka/Sheerka.py +++ b/src/core/sheerka/Sheerka.py @@ -328,7 +328,7 @@ class Sheerka(Concept): self.evaluators.append(evaluator) def initialize_concept_node_parsing(self, context): - self.init_log.debug("Initializing concept node parsing.") + self.init_log.debug("siInitializing concepts by first keyword.") concepts_by_first_keyword = self.cache_manager.copy(self.CONCEPTS_BY_FIRST_KEYWORD_ENTRY) res = self.bnp.resolve_concepts_by_first_keyword(context, concepts_by_first_keyword) diff --git a/src/core/sheerka/services/SheerkaAdmin.py b/src/core/sheerka/services/SheerkaAdmin.py index 3de1a87..9a3241d 100644 --- a/src/core/sheerka/services/SheerkaAdmin.py +++ b/src/core/sheerka/services/SheerkaAdmin.py @@ -1,3 +1,5 @@ +import time + from core.builtin_concepts import BuiltinConcepts from core.sheerka.services.sheerka_service import BaseService @@ -39,6 +41,7 @@ class SheerkaAdmin(BaseService): :return: """ try: + start = time.time_ns() self.sheerka.during_restore = True with open(CONCEPTS_FILE, "r") as f: for line in f.readlines(): @@ -50,5 +53,11 @@ class SheerkaAdmin(BaseService): if len(res) > 1 or not res[0].status: self.sheerka.log.error("Error detected !") self.sheerka.during_restore = False + stop = time.time_ns() + + nano_sec = stop - start + dt = nano_sec / 1e6 + elapsed = f"{dt} ms" if dt < 1000 else f"{dt / 1000} s" + print(f"Execution time: {elapsed}") except IOError: pass diff --git a/src/core/sheerka/services/SheerkaComparisonManager.py b/src/core/sheerka/services/SheerkaComparisonManager.py index c87bcbc..767d45b 100644 --- a/src/core/sheerka/services/SheerkaComparisonManager.py +++ b/src/core/sheerka/services/SheerkaComparisonManager.py @@ -90,12 +90,12 @@ class SheerkaComparisonManager(BaseService): cache = Cache() self.sheerka.cache_manager.register_cache(self.RESOLVED_COMPARISON_ENTRY, cache, persist=False) - self.sheerka.bind_service_method(self.is_greater_than) - self.sheerka.bind_service_method(self.is_less_than) + self.sheerka.bind_service_method(self.set_is_greater_than) + self.sheerka.bind_service_method(self.set_is_less_than) self.sheerka.bind_service_method(self.get_partition) self.sheerka.bind_service_method(self.get_concepts_weights) - def is_greater_than(self, context, prop_name, concept_a, concept_b, comparison_context="#"): + def set_is_greater_than(self, context, prop_name, concept_a, concept_b, comparison_context="#"): """ Records that the property of concept a is greater than concept b's one :param context: @@ -112,7 +112,7 @@ class SheerkaComparisonManager(BaseService): comparison_obj = ComparisonObj(event_digest, prop_name, concept_a.id, concept_b.id, ">", comparison_context) return self._inner_add_comparison(comparison_obj) - def is_less_than(self, context, prop_name, concept_a, concept_b, comparison_context="#"): + def set_is_less_than(self, context, prop_name, concept_a, concept_b, comparison_context="#"): """ Records that the property of concept a is lesser than concept b's one :param context: diff --git a/src/core/sheerka/services/SheerkaCreateNewConcept.py b/src/core/sheerka/services/SheerkaCreateNewConcept.py index 26988dd..fad3a3b 100644 --- a/src/core/sheerka/services/SheerkaCreateNewConcept.py +++ b/src/core/sheerka/services/SheerkaCreateNewConcept.py @@ -52,7 +52,7 @@ class SheerkaCreateNewConcept(BaseService): sheerka.set_id_if_needed(concept, False) # compute new concepts_by_first_keyword - init_ret_value = self.bnp.get_concepts_by_first_keyword(context, [concept], True) + init_ret_value = self.bnp.get_concepts_by_first_token(context, [concept], True) if not init_ret_value.status: return sheerka.ret(self.NAME, False, ErrorConcept(init_ret_value.value)) concepts_by_first_keyword = init_ret_value.body diff --git a/src/core/sheerka/services/SheerkaEvaluateConcept.py b/src/core/sheerka/services/SheerkaEvaluateConcept.py index 95fb2bb..2fa3df8 100644 --- a/src/core/sheerka/services/SheerkaEvaluateConcept.py +++ b/src/core/sheerka/services/SheerkaEvaluateConcept.py @@ -296,6 +296,11 @@ class SheerkaEvaluateConcept(BaseService): concept.init_key() # only does it if needed concept.metadata.is_evaluated = "body" in all_metadata_to_eval + + # # update the cache for concepts with no variable + # if len(concept.metadata.variables) == 0: + # self.sheerka.cache_manager.put(self.sheerka.CONCEPTS_BY_ID_ENTRY, concept.id, concept) + return concept def choose_metadata_to_eval(self, context, concept): diff --git a/src/core/sheerka/services/SheerkaExecute.py b/src/core/sheerka/services/SheerkaExecute.py index e56fb2b..aa122db 100644 --- a/src/core/sheerka/services/SheerkaExecute.py +++ b/src/core/sheerka/services/SheerkaExecute.py @@ -173,6 +173,17 @@ class SheerkaExecute(BaseService): return pi def call_parsers(self, context, return_values): + """ + Call all the parsers, ordered by priority + Possible return value for a parser: + None : indicate that you do no need to care about the result + ParserResult with status False : Success + ParserResult with status False : failed to parse, but the result will be reused by other parsers + NotForMe (status is False) : Failed to parse. Do no reuse the result + :param context: + :param return_values: + :return: + """ # return_values must be a list if not isinstance(return_values, list): diff --git a/src/core/sheerka/services/SheerkaSetsManager.py b/src/core/sheerka/services/SheerkaSetsManager.py index 165a3f4..fac9fce 100644 --- a/src/core/sheerka/services/SheerkaSetsManager.py +++ b/src/core/sheerka/services/SheerkaSetsManager.py @@ -1,4 +1,5 @@ import core.builtin_helpers +from cache.Cache import Cache from cache.SetCache import SetCache from core.ast.nodes import python_to_concept from core.builtin_concepts import BuiltinConcepts @@ -11,9 +12,12 @@ GROUP_PREFIX = 'All_' class SheerkaSetsManager(BaseService): NAME = "SetsManager" CONCEPTS_GROUPS_ENTRY = "SetsManager:Concepts_Groups" + CONCEPTS_IN_GROUPS_ENTRY = "SetsManager:Concepts_In_Groups" # cache for get_set_elements() def __init__(self, sheerka): super().__init__(sheerka) + self.sets = SetCache(default=lambda k: self.sheerka.sdp.get(self.CONCEPTS_GROUPS_ENTRY, k)) + self.concepts_in_set = Cache() def initialize(self): self.sheerka.bind_service_method(self.set_isa) @@ -23,8 +27,8 @@ class SheerkaSetsManager(BaseService): self.sheerka.bind_service_method(self.isa) self.sheerka.bind_service_method(self.isaset) - cache = SetCache(default=lambda k: self.sheerka.sdp.get(self.CONCEPTS_GROUPS_ENTRY, k)) - self.sheerka.cache_manager.register_cache(self.CONCEPTS_GROUPS_ENTRY, cache) + self.sheerka.cache_manager.register_cache(self.CONCEPTS_GROUPS_ENTRY, self.sets) + self.sheerka.cache_manager.register_cache(self.CONCEPTS_IN_GROUPS_ENTRY, self.concepts_in_set, persist=False) def set_isa(self, context, concept, concept_set): """ @@ -64,14 +68,14 @@ class SheerkaSetsManager(BaseService): context.log(f"Adding concept {concept} to set {concept_set}", who=self.NAME) ensure_concept(concept, concept_set) - set_elements = self.sheerka.cache_manager.get(self.CONCEPTS_GROUPS_ENTRY, concept_set.id) + set_elements = self.sets.get(concept_set.id) if set_elements and concept.id in set_elements: return self.sheerka.ret( self.NAME, False, self.sheerka.new(BuiltinConcepts.CONCEPT_ALREADY_IN_SET, body=concept, concept_set=concept_set)) - self.sheerka.cache_manager.put(self.CONCEPTS_GROUPS_ENTRY, concept_set.id, concept.id) + self.sets.put(concept_set.id, concept.id) return self.sheerka.ret(self.NAME, True, self.sheerka.new(BuiltinConcepts.SUCCESS)) def add_concepts_to_set(self, context, concepts, concept_set): @@ -109,8 +113,8 @@ class SheerkaSetsManager(BaseService): if not self.isaset(context, sub_concept): return self.sheerka.new(BuiltinConcepts.NOT_A_SET, body=concept) - # first, try to see if sub_context has it's own group entry - ids = self.sheerka.cache_manager.get(self.CONCEPTS_GROUPS_ENTRY, sub_concept.id) + # first, try to see if sub_concept has it's own group entry + ids = self.sets.get(sub_concept.id) concepts = self._get_concepts(context, ids, True) # aggregate with en entries from its body @@ -139,7 +143,12 @@ class SheerkaSetsManager(BaseService): return concepts - return _get_set_elements(concept) + if res := self.concepts_in_set.get(concept.id): + return res + + res = _get_set_elements(concept) + self.concepts_in_set.put(concept.id, res) + return res def isinset(self, a, b): """ @@ -156,7 +165,7 @@ class SheerkaSetsManager(BaseService): if not (a.id and b.id): return False - group_elements = self.sheerka.cache_manager.get(self.CONCEPTS_GROUPS_ENTRY, b.id) + group_elements = self.sets.get(b.id) return group_elements and a.id in group_elements def isa(self, a, b): @@ -187,7 +196,7 @@ class SheerkaSetsManager(BaseService): # check if it has a group # TODO: use cache instead of directly requesting sdp - if self.sheerka.cache_manager.get(self.CONCEPTS_GROUPS_ENTRY, concept.id): + if self.sets.get(concept.id): return True # it may be a concept that references a set @@ -240,9 +249,18 @@ for x in xx__concepts__xx: desc=f"Evaluating concepts of a set") as sub_context: sub_context.add_inputs(ids=ids) sub_context.local_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED) + errors = [] for element_id in ids: concept = self.sheerka.get_by_id(element_id) - evaluated = self.sheerka.evaluate_concept(sub_context, concept) - result.append(evaluated) - sub_context.add_inputs(return_value=result) + if len(concept.metadata.variables) == 0: + # only evaluate + evaluated = self.sheerka.evaluate_concept(sub_context, concept) + if context.sheerka.is_success(evaluated): + result.append(evaluated) + else: + errors.append(evaluated) + else: + result.append(concept) + sub_context.add_values(return_value=result) + sub_context.add_values(errors=errors) return result diff --git a/src/core/sheerka/services/SheerkaVariableManager.py b/src/core/sheerka/services/SheerkaVariableManager.py index 200cf07..288b16e 100644 --- a/src/core/sheerka/services/SheerkaVariableManager.py +++ b/src/core/sheerka/services/SheerkaVariableManager.py @@ -30,6 +30,8 @@ class SheerkaVariableManager(BaseService): self.sheerka.bind_service_method(self.record) self.sheerka.bind_service_method(self.load) self.sheerka.bind_service_method(self.delete) + self.sheerka.bind_service_method(self.set) + self.sheerka.bind_service_method(self.get) cache = Cache(default=lambda k: self.sheerka.sdp.get(self.VARIABLES_ENTRY, k)) self.sheerka.cache_manager.register_cache(self.VARIABLES_ENTRY, cache, True, True) @@ -56,3 +58,9 @@ class SheerkaVariableManager(BaseService): def delete(self, context, who, key): self.sheerka.cache_manager.delete(self.VARIABLES_ENTRY, who + "|" + key) + + def set(self, context, key, value): + return self.record(context, context.event.user_id, key, value) + + def get(self, context, key): + return self.load(context.event.user_id, key) diff --git a/src/core/utils.py b/src/core/utils.py index 37aca86..c993d48 100644 --- a/src/core/utils.py +++ b/src/core/utils.py @@ -187,6 +187,32 @@ def remove_list_from_list(lst, to_remove): return lst +def make_unique(lst, get_id=None): + """ + All items in the list are now uniq and the order is kept + >>> assert make_unique(["a", "a", "b", "c", "c"]) == ["a", "b", "c"] + :param lst: + :param get_id: define your own way to recognize the items + :return: + """ + + def _make_unique(seq, get_id=None): + seen = set() + if get_id is None: + for x in seq: + if x not in seen: + seen.add(x) + yield x + else: + for x in seq: + x = get_id(x) + if x not in seen: + seen.add(x) + yield x + + return list(_make_unique(lst, get_id)) + + def product(a, b): """ Kind of cartesian product between lists a and b @@ -241,7 +267,6 @@ def dict_product(a, b): return res - def strip_quotes(text): if not isinstance(text, str): return text diff --git a/src/parsers/BaseNodeParser.py b/src/parsers/BaseNodeParser.py index 90e220a..f0b92b8 100644 --- a/src/parsers/BaseNodeParser.py +++ b/src/parsers/BaseNodeParser.py @@ -1,6 +1,7 @@ from collections import namedtuple from dataclasses import dataclass from enum import Enum +from typing import Set import core.utils from core.builtin_concepts import BuiltinConcepts @@ -12,6 +13,11 @@ from parsers.BaseParser import Node, BaseParser, ErrorNode DEBUG_COMPILED = True +@dataclass +class ChickenAndEggError(Exception): + concepts: Set[str] + + @dataclass() class LexerNode(Node): start: int # starting index in the tokens list @@ -422,7 +428,7 @@ class CN(HelperWithPos): ConceptNode tester class It matches with ConceptNode but with less constraints - CNC == ConceptNode if concept key, start, end and source are the same + CN == ConceptNode if concept key, start, end and source are the same """ def __init__(self, concept, start=None, end=None, source=None): @@ -496,6 +502,9 @@ class CNC(CN): super().__init__(concept_key, start, end, source) self.compiled = kwargs self.exclude_body = exclude_body + if "body" in self.compiled: + self.compiled[ConceptParts.BODY] = self.compiled["body"] + del self.compiled["body"] def __eq__(self, other): if id(self) == id(other): @@ -516,7 +525,10 @@ class CNC(CN): to_compare = {k: v for k, v in other.concept.compiled.items() if k != ConceptParts.BODY} else: to_compare = other.concept.compiled - return self.compiled == to_compare + if self.compiled == to_compare: + return True + else: + return False if not isinstance(other, CNC): return False @@ -613,7 +625,7 @@ class BaseNodeParser(BaseParser): :param concepts :return: """ - concepts_by_first_keyword = self.get_concepts_by_first_keyword(context, concepts).body + concepts_by_first_keyword = self.get_concepts_by_first_token(context, concepts).body self.concepts_by_first_keyword = self.resolve_concepts_by_first_keyword(context, concepts_by_first_keyword).body def reset_parser(self, context, parser_input: ParserInput): @@ -626,38 +638,6 @@ class BaseNodeParser(BaseParser): self.add_error(self.sheerka.new(BuiltinConcepts.ERROR, body=e), False) return False return True - # self.text = text - # - # try: - # self.tokens = list(self.get_input_as_tokens(text)) - # - # - # self.token = None - # self.pos = -1 - # return True - - # def add_error(self, error, next_token=True): - # self.error_sink.append(error) - # if next_token: - # self.parser_input.next_token() - # return error - - # def get_token(self) -> Token: - # return self.token - # - # def next_token(self, skip_whitespace=True): - # if self.token and self.token.type == TokenKind.EOF: - # return False - # - # self.pos += 1 - # self.token = self.tokens[self.pos] - # - # if skip_whitespace: - # while self.token.type == TokenKind.WHITESPACE or self.token.type == TokenKind.NEWLINE: - # self.pos += 1 - # self.token = self.tokens[self.pos] - # - # return self.token.type != TokenKind.EOF def get_concepts(self, token, to_keep, custom=None, to_map=None, strip_quotes=False): """ @@ -698,7 +678,7 @@ class BaseNodeParser(BaseParser): return custom_concepts if custom else None @staticmethod - def get_concepts_by_first_keyword(context, concepts, use_sheerka=False): + def get_concepts_by_first_token(context, concepts, use_sheerka=False): """ Create the map describing the first token expected by a concept :param context: @@ -718,22 +698,26 @@ class BaseNodeParser(BaseParser): for keyword in keywords: res.setdefault(keyword, []).append(concept.id) + # 'uniquify' the lists + for k, v in res.items(): + res[k] = core.utils.make_unique(v) + return sheerka.ret("BaseNodeParser", True, res) @staticmethod def resolve_concepts_by_first_keyword(context, concepts_by_first_keyword): sheerka = context.sheerka - def _make_unique(elements): - keys = {} - for e in elements: - keys[e] = 1 - return list(keys.keys()) - - def _resolve_concepts(concept_str): - resolved = [] - to_resolve = [] + def resolve_concepts(concept_str): + resolved = set() + to_resolve = set() concept = sheerka.get_by_id(core.utils.unstr_concept(concept_str)[1]) + + if concept.id in already_seen: + raise ChickenAndEggError(already_seen) + else: + already_seen.add(concept.id) + if sheerka.isaset(context, concept): concepts = sheerka.get_set_elements(context, concept) else: @@ -743,25 +727,31 @@ class BaseNodeParser(BaseParser): BaseNodeParser.ensure_bnf(context, concept) # need to make sure that it cannot fail keywords = BaseNodeParser.get_first_tokens(sheerka, concept) for keyword in keywords: - (to_resolve if keyword.startswith("c:|") else resolved).append(keyword) + (to_resolve if keyword.startswith("c:|") else resolved).add(keyword) for concept_to_resolve_str in to_resolve: - resolved += _resolve_concepts(concept_to_resolve_str) + resolved |= resolve_concepts(concept_to_resolve_str) return resolved res = {} for k, v in concepts_by_first_keyword.items(): if k.startswith("c:|"): - resolved_keywords = _resolve_concepts(k) - for resolved in resolved_keywords: - res.setdefault(resolved, []).extend(v) + try: + already_seen = set() + resolved_keywords = resolve_concepts(k) + for resolved in resolved_keywords: + res.setdefault(resolved, []).extend(v) + except ChickenAndEggError as ex: + context.log(f"Chicken and egg detected for {k}, concepts={ex.concepts}") + # res[k] = sheerka.new(BuiltinConcepts.CHICKEN_AND_EGG, + # body=[sheerka.get_by_id(c) for c in ex.concepts]) else: res.setdefault(k, []).extend(v) # 'uniquify' the lists for k, v in res.items(): - res[k] = _make_unique(v) + res[k] = core.utils.make_unique(v) return sheerka.ret("BaseNodeParser", True, res) @@ -797,7 +787,7 @@ class BaseNodeParser(BaseParser): if concept.metadata.definition_type == DEFINITION_TYPE_BNF and not concept.bnf: from parsers.BnfParser import BnfParser regex_parser = BnfParser() - desc = f"Resolving BNF {concept.metadata.definition}" + desc = f"Resolving BNF '{concept.metadata.definition}'" with context.push(BuiltinConcepts.INIT_BNF, concept, who=parser_name, diff --git a/src/parsers/BnfNodeParser.py b/src/parsers/BnfNodeParser.py index 5fa9e54..26eeb69 100644 --- a/src/parsers/BnfNodeParser.py +++ b/src/parsers/BnfNodeParser.py @@ -154,18 +154,6 @@ class ConceptExpression(ParsingExpression): [node]) -# class ConceptGroupExpression(ConceptExpression): -# def _parse(self, parser_helper): -# node = self.nodes[0].parse(parser_helper) -# if node is None: -# return None -# return NonTerminalNode(self, -# node.start, -# node.end, -# node.tokens, # node is an OrderedChoice -# [node]) - - class Sequence(ParsingExpression): """ Will match sequence of parser expressions in exact order they are defined. @@ -422,6 +410,69 @@ class StrMatch(Match): return None +# class RegExMatch(Match): +# ''' +# This Match class will perform input matching based on Regular Expressions. +# +# Args: +# to_match (regex string): A regular expression string to match. +# It will be used to create regular expression using re.compile. +# ignore_case(bool): If case insensitive match is needed. +# Default is None to support propagation from global parser setting. +# multiline(bool): allow regex to works on multiple lines +# (re.DOTALL flag). Default is None to support propagation from +# global parser setting. +# str_repr(str): A string that is used to represent this regex. +# re_flags: flags parameter for re.compile if neither ignore_case +# or multiple are set. +# +# ''' +# def __init__(self, to_match, rule_name='', root=False, ignore_case=None, +# multiline=None, str_repr=None, re_flags=re.MULTILINE): +# super(RegExMatch, self).__init__(rule_name, root) +# self.to_match_regex = to_match +# self.ignore_case = ignore_case +# self.multiline = multiline +# self.explicit_flags = re_flags +# +# self.to_match = str_repr if str_repr is not None else to_match +# +# def compile(self): +# flags = self.explicit_flags +# if self.multiline is True: +# flags |= re.DOTALL +# if self.multiline is False and flags & re.DOTALL: +# flags -= re.DOTALL +# if self.ignore_case is True: +# flags |= re.IGNORECASE +# if self.ignore_case is False and flags & re.IGNORECASE: +# flags -= re.IGNORECASE +# self.regex = re.compile(self.to_match_regex, flags) +# +# def __str__(self): +# return self.to_match +# +# def __unicode__(self): +# return self.__str__() +# +# def _parse(self, parser): +# c_pos = parser.position +# m = self.regex.match(parser.input, c_pos) +# if m: +# matched = m.group() +# if parser.debug: +# parser.dprint( +# "++ Match '%s' at %d => '%s'" % +# (matched, c_pos, parser.context(len(matched)))) +# parser.position += len(matched) +# if matched: +# return Terminal(self, c_pos, matched, extra_info=m) +# else: +# if parser.debug: +# parser.dprint("-- NoMatch at {}".format(c_pos)) +# parser._nm_raise(self, c_pos, parser) + + class ParsingExpressionVisitor: """ visit ParsingExpression @@ -550,7 +601,7 @@ class BnfConceptParserHelper: forked.eat_concept(concept, token) # init - parsing_expression = self.parser.get_parsing_expression(concept) + parsing_expression = self.parser.get_parsing_expression(self.parser.context, concept) if not isinstance(parsing_expression, ParsingExpression): self.debug.append(concept) error_msg = f"Failed to parse concept '{concept}'" @@ -733,6 +784,11 @@ class BnfConceptParserHelper: return concept +@dataclass +class UnderConstruction: + concept_id: str + + class BnfNodeParser(BaseNodeParser): def __init__(self, **kwargs): super().__init__("BnfNode", 50, **kwargs) @@ -769,6 +825,11 @@ class BnfNodeParser(BaseNodeParser): return valid_parser_helpers def get_concepts_sequences(self): + """ + Main method that parses the tokens and extract the concepts + :return: + """ + def _add_forked_to_concept_parser_helpers(): # check that if some new InfixToPostfix are created for parser in concept_parser_helpers: @@ -836,110 +897,214 @@ class BnfNodeParser(BaseNodeParser): return concept_parser_helpers - def get_parsing_expression(self, concept, already_seen=None): + def check_for_infinite_recursion(self, parsing_expression, already_found, only_first=False): + + if isinstance(parsing_expression, ConceptExpression): + if parsing_expression.concept in already_found: + return True + already_found.add(parsing_expression.concept) + return self.check_for_infinite_recursion(parsing_expression.nodes[0], already_found, False) + + if isinstance(parsing_expression, Sequence): + # for sequence, we need to check all nodes + if only_first: + nodes = [] if len(parsing_expression.nodes) == 0 else [parsing_expression.nodes[0]] + else: + nodes = parsing_expression.nodes + for node in nodes: + already_found_for_current_node = already_found.copy() + if self.check_for_infinite_recursion(node, already_found_for_current_node, False): + already_found.update(already_found_for_current_node) + return True + return False + + if isinstance(parsing_expression, OrderedChoice): + # for ordered choice, if there is at least one node that does not resolved to a recursion + # we are safe + for node in parsing_expression.nodes: + already_found_for_current_node = already_found.copy() + if self.check_for_infinite_recursion(node, already_found, True): + already_found.update(already_found_for_current_node) + return True + else: + return False + return False + + return False + + def get_parsing_expression(self, context, concept): if concept.id in self.concepts_grammars: return self.concepts_grammars.get(concept.id) - if not concept.bnf: - BaseNodeParser.ensure_bnf(self.context, concept, self.name) + grammar = self.concepts_grammars.copy() + to_resolve = {} # the key is the instance id of the parsing expression + isa_concepts = set() + self.resolve_concept_parsing_expression(context, concept, grammar, to_resolve, isa_concepts) - expression = concept.bnf - desc = f"Resolving parsing expression {expression}" - with self.context.push(BuiltinConcepts.INIT_BNF, concept, who=self.name, obj=concept, desc=desc) as sub_context: - sub_context.add_inputs(expression=expression) - resolved = self.resolve_parsing_expression(expression, already_seen or set()) - sub_context.add_values(return_values=resolved) + for _id, pe in to_resolve.items(): + for i, node in enumerate(pe.nodes): + if isinstance(node, UnderConstruction): + pe.nodes[i] = grammar.get(node.concept_id) - self.concepts_grammars.put(concept.id, resolved) + concepts_in_recursion = set() + if self.check_for_infinite_recursion(pe, concepts_in_recursion): + cycle = context.sheerka.new(BuiltinConcepts.CHICKEN_AND_EGG, body={c.id for c in concepts_in_recursion}) + for concept in concepts_in_recursion: + grammar[concept.id] = cycle - if self.has_error: - return None + # Make sure you do not put isa concepts in cache + # why : + # twenties = 'twenty' number where number < 10 + # hundreds = number 'hundred' where number < 99 + # the concept of number depends on its utilisation + for concept_id in [c for c in grammar if c not in isa_concepts]: + self.concepts_grammars.put(concept_id, grammar[concept_id]) return self.concepts_grammars.get(concept.id) - def resolve_parsing_expression(self, parsing_expression, already_seen): + def resolve_concept_parsing_expression(self, context, concept, grammar, to_resolve, isa_concepts): + if concept.id in grammar: + return grammar.get(concept.id) - def inner_resolve(expression, inner_already_seen): - # if isinstance(expression, Concept): - # if self.sheerka.isaset(self.context, expression): - # ret = ConceptGroupExpression(expression, rule_name=expression.name) - # else: - # ret = ConceptExpression(expression, rule_name=expression.name) - # possible_recursion.add(expression) - if isinstance(expression, str): - ret = StrMatch(expression, ignore_case=self.ignore_case) + desc = f"Get parsing expression for '{concept}'" + with context.push(BuiltinConcepts.INIT_BNF, concept, who=self.name, obj=concept, desc=desc) as sub_context: + if not concept.bnf: # to save a function call. Not sure it worth it. + BaseNodeParser.ensure_bnf(sub_context, concept, self.name) - elif not isinstance(expression, ParsingExpression): - return expression # escalate the error + grammar[concept.id] = UnderConstruction(concept.id) + sheerka = context.sheerka - elif isinstance(expression, ConceptExpression): - concept = self.get_concept(expression.concept) - if concept in inner_already_seen: - return self.sheerka.new(BuiltinConcepts.CHICKEN_AND_EGG, body=concept) - expression.concept = concept - inner_already_seen.add(concept) + if concept.metadata.definition_type == DEFINITION_TYPE_BNF: + expression = concept.bnf + desc = f"Bnf concept detected. Resolving parsing expression '{expression}'" + with sub_context.push(BuiltinConcepts.INIT_BNF, concept, who=self.name, obj=concept, desc=desc) as ssc: + ssc.add_inputs(expression=expression) + resolved = self.resolve_parsing_expression(ssc, expression, grammar, to_resolve, isa_concepts) + ssc.add_values(return_values=resolved) - if not self.sheerka.is_known(concept): - unknown_concept = self.sheerka.new(BuiltinConcepts.UNKNOWN_CONCEPT, body=concept) - return self.add_error(unknown_concept) + elif sheerka.isaset(context, concept): + desc = f"Concept is a group. Resolving parsing expression using 'isa'" + with sub_context.push(BuiltinConcepts.INIT_BNF, concept, who=self.name, obj=concept, desc=desc) as ssc: + ssc.add_inputs(concept=concept) + isa_concepts.add(concept.id) + concepts_in_group = self.sheerka.get_set_elements(ssc, concept) - # bnf concept - elif concept.metadata.definition_type == DEFINITION_TYPE_BNF: - pe = self.get_parsing_expression(concept, inner_already_seen) + # concepts_in_group comes from a set, so the order of its elements is not guaranteed + # to avoid random failure (ie random CHICKEN_AND_EGG), we need to rearrange + # We also remove the root concept (the one from get_parsing_expression()) - elif self.sheerka.isaset(self.context, concept): - concepts_in_group = self.sheerka.get_set_elements(self.context, concept) - nodes = [ConceptExpression(c, rule_name=c.name) for c in concepts_in_group] - pe = inner_resolve(OrderedChoice(*nodes), inner_already_seen) + root_concept_as_set = set(context.search( + predicate=lambda ec: ec.action == BuiltinConcepts.INIT_BNF, + get_obj=lambda ec: ec.obj, + stop=lambda ec: ec.action != BuiltinConcepts.INIT_BNF)) # there only one item in the set + root_concept = list(root_concept_as_set)[0] + reordered = [] + for c in concepts_in_group: + if c.id == root_concept.id: + continue - else: - # regular concepts - tokens = Tokenizer(concept.name) - nodes = [StrMatch(token.strip_quote) for token in list(tokens)[:-1]] - pe = inner_resolve(nodes[0] if len(nodes) == 1 else Sequence(nodes), inner_already_seen) + # I do not guaranty the same order every time, but I minimize the ChickenAndEgg random issue + if c.metadata.definition_type == DEFINITION_TYPE_BNF or sheerka.isaset(ssc, c): + reordered.append(c) + else: + reordered.insert(0, c) - if not isinstance(pe, ParsingExpression): - return pe - expression.nodes = [pe] - expression.rule_name = expression.rule_name or concept.name - ret = expression - - elif isinstance(expression, StrMatch): - ret = expression - if ret.ignore_case is None: - ret.ignore_case = self.ignore_case - - elif isinstance(expression, Sequence) or \ - isinstance(expression, OrderedChoice) or \ - isinstance(expression, ZeroOrMore) or \ - isinstance(expression, OneOrMore) or \ - isinstance(expression, Optional): - ret = expression - ret.nodes = [] - for e in ret.elements: - pe = inner_resolve(e, already_seen.copy()) - if not isinstance(pe, ParsingExpression): - return pe - ret.nodes.append(pe) + nodes = [ConceptExpression(c, rule_name=c.name) for c in reordered] + resolved = self.resolve_parsing_expression(ssc, + OrderedChoice(*nodes), + grammar, + to_resolve, + isa_concepts) + ssc.add_values(concepts_in_group=concepts_in_group) + ssc.add_values(return_values=resolved) else: - ret = self.add_error(GrammarErrorNode(f"Unrecognized grammar element '{expression}'."), False) + desc = f"Concept is a simple concept." + with sub_context.push(BuiltinConcepts.INIT_BNF, concept, who=self.name, obj=concept, desc=desc) as ssc: + tokens = Tokenizer(concept.name, yield_eof=False) + nodes = [StrMatch(token.strip_quote) for token in tokens] + expression = nodes[0] if len(nodes) == 1 else Sequence(nodes) + resolved = self.resolve_parsing_expression(ssc, expression, grammar, to_resolve, isa_concepts) - # Translate separator expression. - if isinstance(expression, Repetition) and expression.sep: - expression.sep = inner_resolve(expression.sep, already_seen) + grammar[concept.id] = resolved - return ret + if self.has_error: + sub_context.add_values(errors=self.error_sink) + return None - parsing_expression = inner_resolve(parsing_expression, already_seen) - return parsing_expression + sub_context.add_values(return_values=resolved) + return resolved - def get_concept(self, concept): + def resolve_parsing_expression(self, context, expression, grammar, to_resolve, isa_concepts): + + if isinstance(expression, str): + ret = StrMatch(expression, ignore_case=self.ignore_case) + + elif not isinstance(expression, ParsingExpression): + return expression # escalate the error + + elif isinstance(expression, ConceptExpression): + concept = self.get_concept(context, expression.concept) + expression.concept = concept + + if not self.sheerka.is_known(concept): + unknown_concept = self.sheerka.new(BuiltinConcepts.UNKNOWN_CONCEPT, body=concept) + return self.add_error(unknown_concept) + + pe = self.resolve_concept_parsing_expression(context, concept, grammar, to_resolve, isa_concepts) + + if not isinstance(pe, (ParsingExpression, UnderConstruction)): + return pe # an error is detected, escalate it + # + # if isinstance(pe, UnderConstruction) and expression.concept.id == pe.concept_id: + # return pe # we are looking for ourself, just return it + + if isinstance(pe, UnderConstruction): + to_resolve[id(expression)] = expression + + expression.nodes = [pe] + expression.rule_name = expression.rule_name or concept.name + ret = expression + + elif isinstance(expression, StrMatch): + ret = expression + if ret.ignore_case is None: + ret.ignore_case = self.ignore_case + + elif isinstance(expression, Sequence) or \ + isinstance(expression, OrderedChoice) or \ + isinstance(expression, ZeroOrMore) or \ + isinstance(expression, OneOrMore) or \ + isinstance(expression, Optional): + ret = expression + ret.nodes = [] + for e in ret.elements: + pe = self.resolve_parsing_expression(context, e, grammar, to_resolve, isa_concepts) + if not isinstance(pe, (ParsingExpression, UnderConstruction)): + return pe # an error is detected, escalate it + if isinstance(pe, UnderConstruction): + to_resolve[id(ret)] = ret # remember that there is an unresolved parsing expression + ret.nodes.append(pe) + + else: + ret = self.add_error(GrammarErrorNode(f"Unrecognized grammar element '{expression}'."), False) + + # Translate separator expression. + if isinstance(ret, Repetition) and expression.sep: + expression.sep = self.resolve_parsing_expression(context, + expression.sep, + grammar, + to_resolve, + isa_concepts) + + return ret + + def get_concept(self, context, concept): if isinstance(concept, Concept): return concept - if concept in self.context.concepts: - return self.context.concepts[concept] + if concept in context.concepts: + return context.concepts[concept] return self.sheerka.get_by_key(concept) def parse(self, context, parser_input: ParserInput): diff --git a/tests/BaseTest.py b/tests/BaseTest.py index 36b27b6..3b1969c 100644 --- a/tests/BaseTest.py +++ b/tests/BaseTest.py @@ -13,7 +13,7 @@ class BaseTest: pass def get_context(self, sheerka=None, eval_body=False, eval_where=False): - context = ExecutionContext("test", Event(), sheerka or self.get_sheerka(), BuiltinConcepts.NOP, None) + context = ExecutionContext("test", Event(), sheerka or self.get_sheerka(), BuiltinConcepts.TESTING, None) if eval_body: context.local_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED) if eval_where: @@ -145,3 +145,13 @@ class BaseTest: else: concept.metadata.variables[k] = v return concept + + def init_scenario(self, init_expressions): + sheerka = self.get_sheerka() + + for expression in init_expressions: + res = sheerka.evaluate_user_input(expression) + assert len(res) == 1, f"Failed to execute '{expression}'" + assert res[0].status, f"Error while executing '{expression}'" + + return sheerka diff --git a/tests/core/test_ExecutionContext.py b/tests/core/test_ExecutionContext.py index a70bd06..f917e53 100644 --- a/tests/core/test_ExecutionContext.py +++ b/tests/core/test_ExecutionContext.py @@ -103,3 +103,21 @@ def test_global_hits_are_global_even_when_empty(): assert a.global_hints == {"global hint 2"} assert b.global_hints == {"global hint 2"} + + +def test_i_can_search(): + a = ExecutionContext("foo", Event("event_1"), "fake_sheerka", BuiltinConcepts.TESTING, "a") + ab = a.push(BuiltinConcepts.TESTING, "ab", obj="obj_ab") + ac = a.push(BuiltinConcepts.TESTING, "ac", obj="obj_ac") + abb = ab.push(BuiltinConcepts.TESTING, "abb", obj="skip") + abbb = abb.push(BuiltinConcepts.TESTING, "abbb", obj="obj_abbb") + + assert list(abbb.search()) == [abb, ab, a] + assert list(abbb.search(start_with_self=True)) == [abbb, abb, ab, a] + assert list(abbb.search(lambda ec: ec.obj != "skip")) == [ab, a] + assert list(abbb.search(lambda ec: ec.obj != "skip", lambda ec: ec.action_context)) == ["ab", "a"] + assert list(abbb.search(stop=lambda ec: ec.obj == "skip")) == [] + assert list(abbb.search( + stop=lambda ec: ec.obj == "skip", + start_with_self=True, + get_obj=lambda ec: ec.obj)) == ["obj_abbb"] diff --git a/tests/core/test_SheerkaComparisonManager.py b/tests/core/test_SheerkaComparisonManager.py index f76183c..fbc38cd 100644 --- a/tests/core/test_SheerkaComparisonManager.py +++ b/tests/core/test_SheerkaComparisonManager.py @@ -10,7 +10,7 @@ class TestSheerkaGreaterThanManager(TestUsingMemoryBasedSheerka): sheerka, context, one, two = self.init_concepts("one", "two", cache_only=False) service = sheerka.services[SheerkaComparisonManager.NAME] - res = service.is_greater_than(context, "prop_name", two, one) + res = service.set_is_greater_than(context, "prop_name", two, one) assert res.status assert sheerka.isinstance(res.body, BuiltinConcepts.SUCCESS) @@ -29,7 +29,7 @@ class TestSheerkaGreaterThanManager(TestUsingMemoryBasedSheerka): sheerka, context, one, two = self.init_concepts("one", "two", cache_only=False) service = sheerka.services[SheerkaComparisonManager.NAME] - res = service.is_less_than(context, "prop_name", one, two) + res = service.set_is_less_than(context, "prop_name", one, two) assert res.status assert sheerka.isinstance(res.body, BuiltinConcepts.SUCCESS) @@ -48,8 +48,8 @@ class TestSheerkaGreaterThanManager(TestUsingMemoryBasedSheerka): sheerka, context, one, two, three, four = self.init_concepts("one", "two", "three", "four", cache_only=False) service = sheerka.services[SheerkaComparisonManager.NAME] - service.is_greater_than(context, "prop_name", two, one) - service.is_greater_than(context, "prop_name", three, two) + service.set_is_greater_than(context, "prop_name", two, one) + service.set_is_greater_than(context, "prop_name", three, two) in_cache = sheerka.cache_manager.get(SheerkaComparisonManager.COMPARISON_ENTRY, "prop_name|#") assert in_cache == [ @@ -67,7 +67,7 @@ class TestSheerkaGreaterThanManager(TestUsingMemoryBasedSheerka): sheerka.cache_manager.clear(SheerkaComparisonManager.COMPARISON_ENTRY) # reset the cache - service.is_greater_than(context, "prop_name", four, three) + service.set_is_greater_than(context, "prop_name", four, three) in_cache = sheerka.cache_manager.get(SheerkaComparisonManager.COMPARISON_ENTRY, "prop_name|#") assert in_cache == [ ComparisonObj(context.event.get_digest(), "prop_name", two.id, one.id, ">", "#"), @@ -92,10 +92,10 @@ class TestSheerkaGreaterThanManager(TestUsingMemoryBasedSheerka): for entry in entries: if ">" in entry: a, b = [concepts_map[e.strip()] for e in entry.split(">")] - service.is_greater_than(context, "prop_name", a, b) + service.set_is_greater_than(context, "prop_name", a, b) else: a, b = [concepts_map[e.strip()] for e in entry.split("<")] - service.is_less_than(context, "prop_name", a, b) + service.set_is_less_than(context, "prop_name", a, b) assert service.get_concepts_weights("prop_name") == expected @@ -103,8 +103,8 @@ class TestSheerkaGreaterThanManager(TestUsingMemoryBasedSheerka): sheerka, context, one, two, three = self.init_concepts("one", "two", "three") service = sheerka.services[SheerkaComparisonManager.NAME] - service.is_greater_than(context, "prop_name", two, one) - service.is_less_than(context, "prop_name", two, three) + service.set_is_greater_than(context, "prop_name", two, one) + service.set_is_less_than(context, "prop_name", two, three) res = service.get_partition("prop_name") @@ -118,8 +118,8 @@ class TestSheerkaGreaterThanManager(TestUsingMemoryBasedSheerka): sheerka, context, one, two = self.init_concepts("one", "two") service = sheerka.services[SheerkaComparisonManager.NAME] - service.is_greater_than(context, "prop_name", two, one) - res = service.is_greater_than(context, "prop_name", one, two) + service.set_is_greater_than(context, "prop_name", two, one) + res = service.set_is_greater_than(context, "prop_name", one, two) assert not res.status assert sheerka.isinstance(res.body, BuiltinConcepts.CHICKEN_AND_EGG) @@ -129,15 +129,15 @@ class TestSheerkaGreaterThanManager(TestUsingMemoryBasedSheerka): sheerka, context, one, two, three, four, five = self.init_concepts("one", "two", "three", "four", "five") service = sheerka.services[SheerkaComparisonManager.NAME] - service.is_greater_than(context, "prop_name", two, one) - service.is_greater_than(context, "prop_name", five, four) - service.is_greater_than(context, "prop_name", four, three) - service.is_greater_than(context, "prop_name", five, two) + service.set_is_greater_than(context, "prop_name", two, one) + service.set_is_greater_than(context, "prop_name", five, four) + service.set_is_greater_than(context, "prop_name", four, three) + service.set_is_greater_than(context, "prop_name", five, two) - res = service.is_greater_than(context, "prop_name", two, one) + res = service.set_is_greater_than(context, "prop_name", two, one) assert res.status - res = service.is_greater_than(context, "prop_name", one, five) + res = service.set_is_greater_than(context, "prop_name", one, five) assert not res.status assert sheerka.isinstance(res.body, BuiltinConcepts.CHICKEN_AND_EGG) @@ -147,13 +147,13 @@ class TestSheerkaGreaterThanManager(TestUsingMemoryBasedSheerka): sheerka, context, one, two = self.init_concepts("one", "two") service = sheerka.services[SheerkaComparisonManager.NAME] - service.is_greater_than(context, "prop_name", two, one) - service.is_less_than(context, "prop_name", one, two) + service.set_is_greater_than(context, "prop_name", two, one) + service.set_is_less_than(context, "prop_name", one, two) weighted = sheerka.cache_manager.get(SheerkaComparisonManager.RESOLVED_COMPARISON_ENTRY, "prop_name|#") assert weighted == {"1001": 1, "1002": 2} def test_methods_are_correctly_bound(self): sheerka, context, one, two = self.init_concepts("one", "two") - res = sheerka.is_greater_than(context, "prop_name", two, one) + res = sheerka.set_is_greater_than(context, "prop_name", two, one) assert res.status diff --git a/tests/core/test_SheerkaVariableManager.py b/tests/core/test_SheerkaVariableManager.py index f210413..b5fa1e6 100644 --- a/tests/core/test_SheerkaVariableManager.py +++ b/tests/core/test_SheerkaVariableManager.py @@ -1,5 +1,4 @@ from core.concept import Concept, ConceptParts -from core.sheerka.Sheerka import Sheerka from core.sheerka.services.SheerkaVariableManager import SheerkaVariableManager from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka @@ -50,6 +49,26 @@ class TestSheerkaVariable(TestUsingMemoryBasedSheerka): sheerka.delete(context, "TestSheerkaVariable", "my_variable") assert sheerka.load("TestSheerkaVariable", "my_variable") is None + def test_i_can_set_and_get_a_value(self): + sheerka = self.get_sheerka(cache_only=False) + context = self.get_context(sheerka) + context.event.user_id = "Test_user" + + sheerka.set(context, "my_variable", "my value") + res = sheerka.get(context, "my_variable") + assert res == "my value" + + # I can persist in db + sheerka.cache_manager.commit(context) + + assert sheerka.sdp.exists(SheerkaVariableManager.VARIABLES_ENTRY, "Test_user|my_variable") + loaded = sheerka.sdp.get(SheerkaVariableManager.VARIABLES_ENTRY, "Test_user|my_variable") + assert loaded.event_id == context.event.get_digest() + assert loaded.key == "my_variable" + assert loaded.value == "my value" + assert loaded.who == "Test_user" + assert loaded.parents is None + # def test_i_can_get_the_parent_when_modified(self): # sheerka = self.get_sheerka() # context = self.get_context(sheerka) diff --git a/tests/evaluators/test_PythonEvaluator.py b/tests/evaluators/test_PythonEvaluator.py index 23e8aa0..4e9ef3f 100644 --- a/tests/evaluators/test_PythonEvaluator.py +++ b/tests/evaluators/test_PythonEvaluator.py @@ -132,7 +132,7 @@ class TestPythonEvaluator(TestUsingMemoryBasedSheerka): self.from_def_concept("mult", "a mult b", ["a", "b"]), ) - parsed = PythonParser().parse(context, ParserInput("is_greater_than(BuiltinConcepts.PRECEDENCE, mult, plus)")) + parsed = PythonParser().parse(context, ParserInput("set_is_greater_than(BuiltinConcepts.PRECEDENCE, mult, plus)")) python_evaluator = PythonEvaluator() evaluated = python_evaluator.eval(context, parsed) diff --git a/tests/non_reg/test_sheerka_non_reg.py b/tests/non_reg/test_sheerka_non_reg.py index fece58a..5cc7140 100644 --- a/tests/non_reg/test_sheerka_non_reg.py +++ b/tests/non_reg/test_sheerka_non_reg.py @@ -1,6 +1,6 @@ import pytest from core.builtin_concepts import BuiltinConcepts -from core.concept import Concept, PROPERTIES_TO_SERIALIZE, simplec, CMV, CB, CC, CV +from core.concept import Concept, PROPERTIES_TO_SERIALIZE, simplec, CMV from evaluators.MutipleSameSuccessEvaluator import MultipleSameSuccessEvaluator from evaluators.PythonEvaluator import PythonEvalError from parsers.BaseNodeParser import SyaAssociativity @@ -12,16 +12,6 @@ from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka class TestSheerkaNonRegMemory(TestUsingMemoryBasedSheerka): - def init_scenario(self, init_expressions): - sheerka = self.get_sheerka() - - for expression in init_expressions: - res = sheerka.evaluate_user_input(expression) - assert len(res) == 1, f"Failed to execute '{expression}'" - assert res[0].status, f"Error while executing '{expression}'" - - return sheerka - @pytest.mark.parametrize("text, expected", [ ("1 + 1", 2), ("sheerka.test()", 'I have access to Sheerka !') @@ -880,10 +870,10 @@ as: sheerka = self.init_scenario(definitions) - res = sheerka.evaluate_user_input("is_greater_than('some_prop', two, one)") + res = sheerka.evaluate_user_input("set_is_greater_than('some_prop', two, one)") assert res[0].status - res = sheerka.evaluate_user_input("is_less_than('some_prop', two, three)") + res = sheerka.evaluate_user_input("set_is_less_than('some_prop', two, three)") assert res[0].status res = sheerka.evaluate_user_input("get_concepts_weights('some_prop')") @@ -891,7 +881,7 @@ as: assert res[0].body == {'1001': 1, '1002': 2, '1003': 3} # test i use a concept to define relation - sheerka.evaluate_user_input("def concept a > b as is_greater_than('some_prop', a, b)") + sheerka.evaluate_user_input("def concept a > b as set_is_greater_than('some_prop', a, b)") res = sheerka.evaluate_user_input("eval four > three") assert res[0].status @@ -901,7 +891,7 @@ as: sheerka, context, one, two, plus = self.init_concepts( Concept("one", body="1"), Concept("two", body="2"), - self.from_def_concept("<", "a < b", ["a", "b"], body="is_less_than('some_prop', a, b)") + self.from_def_concept("<", "a < b", ["a", "b"], body="set_is_less_than('some_prop', a, b)") ) expression = "c:one: < c:two:" @@ -946,6 +936,33 @@ as: assert isinstance(error1.error, TypeError) assert error1.error.args[0] == "unsupported operand type(s) for +: 'Concept' and 'int'" + def test_i_can_evaluate_bnf_concept_defined_with_group_after_restart(self): + """ + BNF Concepts defined with group and being themselves part a s group get messed up after restart + :return: + """ + init = [ + "def concept one as 1", + "def concept two as 2", + "def concept twenty as 20", + "def concept number", + "one isa number", + "two isa number", + "twenty isa number", + "def concept twenties from bnf twenty number where number < 10 as twenty + number", + "twenties isa number", + ] + + sheerka = self.init_scenario(init) + + # simulate that sheerka was stopped and restarted + sheerka.cache_manager.clear(sheerka.CONCEPTS_GRAMMARS_ENTRY) + sheerka.cache_manager.get(sheerka.CONCEPTS_BY_KEY_ENTRY, "twenties").compiled = {} + + res = sheerka.evaluate_user_input("eval twenty one") + assert res[0].status + assert res[0].body == 21 + class TestSheerkaNonRegFile(TestUsingFileBasedSheerka): def test_i_can_def_several_concepts(self): @@ -1013,3 +1030,23 @@ class TestSheerkaNonRegFile(TestUsingFileBasedSheerka): evaluated = sheerka.evaluate_concept(self.get_context(eval_body=True), res[0].value) assert evaluated.body == "one two three" assert evaluated.get_value("a") == sheerka.new(concept_a.key, body="one two").init_key() + + def test_i_can_eval_sophisticated_bnf_concepts_after_restart(self): + self.init_scenario([ + "def concept one as 1", + "def concept number", + "one isa number", + "def concept twenty as 20", + "twenty isa number", + "def concept twenties from bnf twenty number where number < 10 as twenty + number", + "twenties isa number", + "def concept thirty as 30", + "thirty isa number", + "def concept thirties from bnf thirty number where number < 10 as thirty + number", + "thirties isa number", + ]) + + sheerka = self.get_sheerka() # another instance + + assert sheerka.evaluate_user_input("eval twenty one")[0].body == 21 + assert sheerka.evaluate_user_input("eval thirty one")[0].body == 31 diff --git a/tests/parsers/parsers_utils.py b/tests/parsers/parsers_utils.py index 6211cb9..971926e 100644 --- a/tests/parsers/parsers_utils.py +++ b/tests/parsers/parsers_utils.py @@ -71,7 +71,7 @@ def get_node( if sub_expr == "')'": return ")" - if isinstance(sub_expr, (scnode, utnode)): + if isinstance(sub_expr, (scnode, utnode, DoNotResolve)): return sub_expr if isinstance(sub_expr, cnode): diff --git a/tests/parsers/test_BaseNodeParser.py b/tests/parsers/test_BaseNodeParser.py index 6496917..8929263 100644 --- a/tests/parsers/test_BaseNodeParser.py +++ b/tests/parsers/test_BaseNodeParser.py @@ -24,7 +24,7 @@ class TestBaseNodeParser(TestUsingMemoryBasedSheerka): sheerka, context, *updated = self.init_concepts(concept) - res = BaseNodeParser.get_concepts_by_first_keyword(context, updated) + res = BaseNodeParser.get_concepts_by_first_token(context, updated) assert res.status assert res.body == expected @@ -54,7 +54,7 @@ class TestBaseNodeParser(TestUsingMemoryBasedSheerka): concept.bnf = bnf sheerka.set_id_if_needed(concept, False) - res = BaseNodeParser.get_concepts_by_first_keyword(context, [concept]) + res = BaseNodeParser.get_concepts_by_first_token(context, [concept]) assert res.status assert res.body == expected @@ -75,7 +75,7 @@ class TestBaseNodeParser(TestUsingMemoryBasedSheerka): foo.bnf = OrderedChoice(ConceptExpression("bar"), ConceptExpression("baz"), StrMatch("qux")) sheerka.set_id_if_needed(foo, False) - res = BaseNodeParser.get_concepts_by_first_keyword(context, [bar, baz, foo]) + res = BaseNodeParser.get_concepts_by_first_token(context, [bar, baz, foo]) assert res.status assert res.body == { @@ -102,7 +102,7 @@ class TestBaseNodeParser(TestUsingMemoryBasedSheerka): foo.bnf = OrderedChoice(ConceptExpression("one"), ConceptExpression("bar"), StrMatch("qux")) sheerka.set_id_if_needed(foo, False) - res = BaseNodeParser.get_concepts_by_first_keyword(context, [bar, foo], use_sheerka=True) + res = BaseNodeParser.get_concepts_by_first_token(context, [bar, foo], use_sheerka=True) assert res.status assert res.body == { @@ -149,7 +149,7 @@ class TestBaseNodeParser(TestUsingMemoryBasedSheerka): sheerka.set_isa(context, sheerka.new("one"), number) sheerka.set_isa(context, sheerka.new("two"), number) - cbfk = BaseNodeParser.get_concepts_by_first_keyword(context, [one, two, three, number, foo]).body + cbfk = BaseNodeParser.get_concepts_by_first_token(context, [one, two, three, number, foo]).body resolved_ret_val = BaseNodeParser.resolve_concepts_by_first_keyword(context, cbfk) @@ -171,7 +171,7 @@ class TestBaseNodeParser(TestUsingMemoryBasedSheerka): ConceptExpression("foo"), ConceptExpression("bar"))) - concepts_by_first_keywords = BaseNodeParser.get_concepts_by_first_keyword( + concepts_by_first_keywords = BaseNodeParser.get_concepts_by_first_token( context, [good, foo, bar, baz]).body resolved_ret_val = BaseNodeParser.resolve_concepts_by_first_keyword(context, concepts_by_first_keywords) @@ -187,7 +187,7 @@ class TestBaseNodeParser(TestUsingMemoryBasedSheerka): a = self.create_and_add_in_cache_concept(sheerka, "a", bnf=Sequence("one", "two")) b = self.create_and_add_in_cache_concept(sheerka, "b", bnf=Sequence(ConceptExpression("a"), "two")) - concepts_by_first_keywords = BaseNodeParser.get_concepts_by_first_keyword( + concepts_by_first_keywords = BaseNodeParser.get_concepts_by_first_token( context, [a, b]).body resolved_ret_val = BaseNodeParser.resolve_concepts_by_first_keyword(context, concepts_by_first_keywords) @@ -202,7 +202,7 @@ class TestBaseNodeParser(TestUsingMemoryBasedSheerka): # foo = self.get_concept(sheerka, "foo", ConceptExpression("bar")) # bar = self.get_concept(sheerka, "bar", ConceptExpression("foo")) # - # concepts_by_first_keywords = BaseNodeParser.get_concepts_by_first_keyword(sheerka, [good, foo, bar]).body + # concepts_by_first_keywords = BaseNodeParser.get_concepts_by_first_token(sheerka, [good, foo, bar]).body # # resolved_ret_val = BaseNodeParser.resolve_concepts_by_first_keyword(sheerka, concepts_by_first_keywords) # assert resolved_ret_val.status @@ -218,7 +218,7 @@ class TestBaseNodeParser(TestUsingMemoryBasedSheerka): # two = self.get_concept(sheerka, "two", ConceptExpression("three")) # three = self.get_concept(sheerka, "three", ConceptExpression("two")) # - # concepts_by_first_keywords = BaseNodeParser.get_concepts_by_first_keyword(sheerka, [good, one, two, three]).body + # concepts_by_first_keywords = BaseNodeParser.get_concepts_by_first_token(sheerka, [good, one, two, three]).body # # resolved_ret_val = BaseNodeParser.resolve_concepts_by_first_keyword(sheerka, concepts_by_first_keywords) # assert resolved_ret_val.status @@ -233,7 +233,7 @@ class TestBaseNodeParser(TestUsingMemoryBasedSheerka): # one = self.get_concept(sheerka, "one", ConceptExpression("two")) # two = self.get_concept(sheerka, "two", OrderedChoice(ConceptExpression("one"), ConceptExpression("two"))) # - # concepts_by_first_keywords = BaseNodeParser.get_concepts_by_first_keyword(sheerka, [good, one, two]).body + # concepts_by_first_keywords = BaseNodeParser.get_concepts_by_first_token(sheerka, [good, one, two]).body # # resolved_ret_val = BaseNodeParser.resolve_concepts_by_first_keyword(sheerka, concepts_by_first_keywords) # assert resolved_ret_val.status @@ -248,7 +248,7 @@ class TestBaseNodeParser(TestUsingMemoryBasedSheerka): # one = self.get_concept(sheerka, "one", ConceptExpression("two")) # two = self.get_concept(sheerka, "two", Sequence(StrMatch("yes"), ConceptExpression("one"))) # - # concepts_by_first_keywords = BaseNodeParser.get_concepts_by_first_keyword(sheerka, [good, one, two]).body + # concepts_by_first_keywords = BaseNodeParser.get_concepts_by_first_token(sheerka, [good, one, two]).body # # resolved_ret_val = BaseNodeParser.resolve_concepts_by_first_keyword(sheerka, concepts_by_first_keywords) # assert resolved_ret_val.status diff --git a/tests/parsers/test_BnfNodeParser.py b/tests/parsers/test_BnfNodeParser.py index 402a59b..fe72fa0 100644 --- a/tests/parsers/test_BnfNodeParser.py +++ b/tests/parsers/test_BnfNodeParser.py @@ -1,25 +1,40 @@ import pytest from core.builtin_concepts import BuiltinConcepts -from core.concept import Concept, ConceptParts, DoNotResolve +from core.concept import Concept, ConceptParts, DoNotResolve, CC, DEFINITION_TYPE_BNF from core.sheerka.services.SheerkaExecute import ParserInput from parsers.BaseNodeParser import CNC, UTN, CN from parsers.BnfNodeParser import BnfNodeParser, StrMatch, TerminalNode, NonTerminalNode, Sequence, OrderedChoice, \ Optional, ZeroOrMore, OneOrMore, ConceptExpression +from parsers.BnfParser import BnfParser import tests.parsers.parsers_utils +from tests.BaseTest import BaseTest from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka cmap = { "one": Concept("one"), "two": Concept("two"), "three": Concept("three"), - "plus": Concept(name="a plus b").def_var("a").def_var("b"), - "bnf one": Concept("bnf_one", definition="'one'"), - 'one and two': Concept("one and two", definition="one two"), - 'one or more three': Concept("one or more three", definition="three+"), - 'two or four': Concept("two or four", definition="two | 'four'"), - "twenties": Concept("twenties", definition="'twenty' c:two or four:=unit"), - "one or more plus": Concept("one or more plus", definition="c:a plus b:+"), # TODO + "four": Concept("four"), + "thirty": Concept("thirty", body=30), + "forty": Concept("forty", body=40), + "fifty": Concept("fifty", body=50), + "number": Concept("number"), + "foo": Concept("foo"), + "bar": Concept("bar"), + "baz": Concept("baz"), + + "bnf baz": Concept("bnf baz", definition="'baz'"), # this one should be chosen + + "plus": Concept("plus", definition="one 'plus' two").def_var("a").def_var("b"), + + 'foo then bar': Concept("foo then bar", definition="foo bar").def_var("foo").def_var("bar"), + 'foo or bar': Concept("foo or bar", definition="foo | bar").def_var("foo").def_var("bar"), + 'one or more foo': Concept("one or more foo", definition="foo+").def_var("foo"), + + "t1": Concept("t1", definition="'twenty' (one|two)=unit").def_var("unit").def_var("one").def_var("two"), + "three_four": Concept("three_four", definition="three | four").def_var("three").def_var("four"), + "t2": Concept("t2", definition="'twenty' three_four=unit").def_var("unit").def_var("three").def_var("four"), # testing keywords "def_only": Concept("def"), @@ -65,15 +80,57 @@ def compute_expected_array(my_concepts_map, expression, expected, exclude_body=F class TestBnfNodeParser(TestUsingMemoryBasedSheerka): sheerka = None + @staticmethod + def update_bnf(context, concept): + bnf_parser = BnfParser() + res = bnf_parser.parse(context, concept.metadata.definition) + if res.status: + concept.bnf = res.value.value + concept.metadata.definition_type = DEFINITION_TYPE_BNF + else: + raise Exception(res) + return concept + @classmethod def setup_class(cls): - t = TestBnfNodeParser() + t = cls() TestBnfNodeParser.sheerka, context, _ = t.init_parser( cmap, singleton=False, create_new=True, init_from_sheerka=True) + # end of initialisation + sheerka = TestBnfNodeParser.sheerka + sheerka.set_isa(context, sheerka.new("one"), sheerka.new("number")) + sheerka.set_isa(context, sheerka.new("two"), sheerka.new("number")) + sheerka.set_isa(context, sheerka.new("three"), sheerka.new("number")) + sheerka.set_isa(context, sheerka.new("four"), sheerka.new("number")) + sheerka.set_isa(context, sheerka.new("thirty"), sheerka.new("number")) + sheerka.set_isa(context, sheerka.new("forty"), sheerka.new("number")) + sheerka.set_isa(context, sheerka.new("fifty"), sheerka.new("number")) + + thirties = cls.update_bnf(context, Concept("thirties", + definition="thirty number", + where="number < 10", + body="thirty + number").def_var("thirty").def_var("number")) + cmap["thirties"] = sheerka.create_new_concept(context, thirties).body.body + sheerka.set_isa(context, sheerka.new("thirties"), sheerka.new("number")) + + forties = cls.update_bnf(context, Concept("forties", + definition="forty number", + where="number < 10", + body="forty + number").def_var("forty").def_var("number")) + cmap["forties"] = sheerka.create_new_concept(context, forties).body.body + sheerka.set_isa(context, sheerka.new("forties"), sheerka.new("number")) + + fifties = cls.update_bnf(context, Concept("fifties", + definition="fifty number", + where="number < 10", + body="fifty + number").def_var("fifty").def_var("number")) + cmap["fifties"] = sheerka.create_new_concept(context, fifties).body.body + sheerka.set_isa(context, sheerka.new("fifties"), sheerka.new("number")) + def init_parser(self, my_concepts_map=None, init_from_sheerka=False, **kwargs): if my_concepts_map is not None: sheerka, context, *updated = self.init_concepts(*my_concepts_map.values(), **kwargs) @@ -174,6 +231,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka): self.validate_get_concepts_sequences(my_map, text, expected) def test_i_can_use_skip_whitespace_when_mixing_sequence_and_strmatch(self): + # to match '--filter' in one word my_map = { "filter": self.bnf_concept("filter", Sequence(StrMatch("-", skip_whitespace=False), @@ -236,20 +294,50 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka): self.validate_get_concepts_sequences(my_map, text, expected) + @pytest.mark.parametrize("concept_three, expected", [ + (Concept("three"), []), + (BaseTest.bnf_concept("three", StrMatch("three")), [UTN('twenty '), "three"]) + ]) + def test_i_can_manage_sequence_with_wrong_order_choice(self, concept_three, expected): + my_map = { + "foo": self.bnf_concept("foo", + Sequence( + StrMatch("twenty"), + OrderedChoice(StrMatch("one"), StrMatch("two")))), + "three": concept_three} + + text = "twenty three" + self.validate_get_concepts_sequences(my_map, text, expected) + @pytest.mark.parametrize("text, expected", [ - ("thirty one ok", [CNC("foo", source="thirty one ok")]), - ("twenty one ok", [CNC("foo", source="twenty one ok")]), + ("ok thirty one", [CNC("foo", source="ok thirty one")]), + ("ok twenty one", [CNC("foo", source="ok twenty one")]), + ("ok one", []), ]) def test_i_can_mix_sequence_and_ordered(self, text, expected): my_map = { "foo": self.bnf_concept("foo", Sequence( + StrMatch("ok"), OrderedChoice(StrMatch("twenty"), StrMatch("thirty")), - StrMatch("one"), - StrMatch("ok")) + StrMatch("one")) )} self.validate_get_concepts_sequences(my_map, text, expected) + @pytest.mark.parametrize("text, expected", [ + # ("twenty one", [CNC("foo", source="twenty one")]), + # ("twenty three", []), # three does not exist + ("twenty four", []), # four exists but should not be seen + ]) + def test_i_can_mix_sequence_and_ordered_2(self, text, expected): + my_map = { + "foo": self.bnf_concept("foo", + Sequence( + StrMatch("twenty"), + OrderedChoice(StrMatch("one"), StrMatch("two")))), + "four": Concept("four")} + self.validate_get_concepts_sequences(my_map, text, expected) + @pytest.mark.parametrize("text, expected", [ ("twenty thirty", [CNC("foo", source="twenty thirty")]), ("one", [CNC("foo", source="one")]), @@ -531,6 +619,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka): "bar": self.bnf_concept("bar", Sequence( ConceptExpression("foo"), OrderedChoice(StrMatch("one"), StrMatch("two")))), + "three": Concept("three") } text = "twenty two" @@ -553,6 +642,33 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka): } assert concept_bar.compiled["foo"].compiled == {ConceptParts.BODY: DoNotResolve("thirty")} + text = "thirty three" + expected = [[CN("foo", source="thirty"), CN("three")], []] + self.validate_get_concepts_sequences(my_map, text, expected, multiple_result=True) + + def test_i_can_mix_reference_to_other_concepts_2(self): + # this time, we use concept expression + my_map = { + "twenty": self.bnf_concept("twenty", StrMatch("twenty")), + "number": self.bnf_concept("number", OrderedChoice(StrMatch("one"), StrMatch("two"))), + "twenties": self.bnf_concept("twenties", + Sequence(ConceptExpression("twenty"), ConceptExpression("number"))), + "three": Concept("three") + } + + text = "twenty two" + + expected = [CNC("twenties", + source="twenty two", + twenty=CC("twenty", body=DoNotResolve("twenty")), + number=CC("number", source="two", body=DoNotResolve("two")) + )] + self.validate_get_concepts_sequences(my_map, text, expected) + + text = "twenty three" + expected = [[CN("twenty"), CN("three")], []] + self.validate_get_concepts_sequences(my_map, text, expected, multiple_result=True) + def test_i_can_mix_reference_to_other_concepts_when_body(self): my_map = { "foo": self.bnf_concept(Concept("foo", body="'foo'"), @@ -654,12 +770,12 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka): 'one': my_map["one"], ConceptParts.BODY: DoNotResolve(value='twenty one')} - @pytest.mark.parametrize("bar_expr", [ - ConceptExpression("foo"), - OrderedChoice(ConceptExpression("foo"), StrMatch("one")), - Sequence(StrMatch("one"), ConceptExpression("foo"), StrMatch("two")) + @pytest.mark.parametrize("bar_expr, expected", [ + (ConceptExpression("foo"), {}), + (OrderedChoice(ConceptExpression("foo"), StrMatch("one")), {'one': ['1002']}), + (Sequence(StrMatch("one"), ConceptExpression("foo"), StrMatch("two")), {'one': ['1001', '1002']}) ]) - def test_i_can_detect_infinite_recursion(self, bar_expr): + def test_i_can_detect_infinite_recursion(self, bar_expr, expected): my_map = { "foo": self.bnf_concept("foo", ConceptExpression("bar")), "bar": self.bnf_concept("bar", bar_expr), @@ -669,14 +785,64 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka): parser.context = context parser.sheerka = sheerka - parsing_expression = parser.get_parsing_expression(my_map["foo"]) + # every obvious cyclic recursion are removed from concept_by_first_keyword dict + parser.init_from_concepts(context, my_map.values()) + assert parser.concepts_by_first_keyword == expected + + # get_parsing_expression() also returns CHICKEN_AND_EGG + parsing_expression = parser.get_parsing_expression(context, my_map["foo"]) assert sheerka.isinstance(parsing_expression, BuiltinConcepts.CHICKEN_AND_EGG) assert sheerka.isinstance(parser.concepts_grammars.get(my_map["foo"].id), BuiltinConcepts.CHICKEN_AND_EGG) - parsing_expression = parser.get_parsing_expression(my_map["bar"]) + parsing_expression = parser.get_parsing_expression(context, my_map["bar"]) assert sheerka.isinstance(parsing_expression, BuiltinConcepts.CHICKEN_AND_EGG) assert sheerka.isinstance(parser.concepts_grammars.get(my_map["bar"].id), BuiltinConcepts.CHICKEN_AND_EGG) + def test_i_can_detect_longer_infinite_recursion(self): + my_map = { + "foo": self.bnf_concept("foo", ConceptExpression("bar")), + "bar": self.bnf_concept("bar", ConceptExpression("baz")), + "baz": self.bnf_concept("baz", ConceptExpression("qux")), + "qux": self.bnf_concept("qux", ConceptExpression("foo")), + } + + sheerka, context, parser = self.init_parser(my_map, singleton=True) + parser.context = context + parser.sheerka = sheerka + + # every obvious cyclic recursion are removed from concept_by_first_keyword dict + parser.init_from_concepts(context, my_map.values()) + assert parser.concepts_by_first_keyword == {} + + parsing_expression = parser.get_parsing_expression(context, my_map["foo"]) + assert sheerka.isinstance(parsing_expression, BuiltinConcepts.CHICKEN_AND_EGG) + assert sheerka.isinstance(parser.concepts_grammars.get(my_map["foo"].id), BuiltinConcepts.CHICKEN_AND_EGG) + assert parser.concepts_grammars.get(my_map["foo"].id).body == {"1001", "1002", "1003", "1004"} + + assert sheerka.isinstance(parser.concepts_grammars.get(my_map["bar"].id), BuiltinConcepts.CHICKEN_AND_EGG) + assert sheerka.isinstance(parser.concepts_grammars.get(my_map["baz"].id), BuiltinConcepts.CHICKEN_AND_EGG) + assert sheerka.isinstance(parser.concepts_grammars.get(my_map["qux"].id), BuiltinConcepts.CHICKEN_AND_EGG) + + @pytest.mark.parametrize("expr, expected", [ + (OrderedChoice(StrMatch("bar"), ConceptExpression("foo")), False), + (OrderedChoice(ConceptExpression("foo"), StrMatch("bar")), True), + (OrderedChoice(Sequence(StrMatch("bar"), ConceptExpression("foo")), StrMatch("baz")), False), + (OrderedChoice(Sequence(ConceptExpression("foo"), StrMatch("bar")), StrMatch("baz")), True) + ]) + def test_i_can_detect_ordered_choice_infinite_recursion(self, expr, expected): + my_map = { + "foo": self.bnf_concept("foo", expr), + } + + sheerka, context, parser = self.init_parser(my_map, singleton=True) + parser.init_from_concepts(context, my_map.values()) + parser.context = context + parser.sheerka = sheerka + + res = parser.get_parsing_expression(context, my_map["foo"]) + assert sheerka.isinstance(res, BuiltinConcepts.CHICKEN_AND_EGG) == expected + + def test_i_can_get_parsing_expression_when_concept_isa(self): my_map = { "one": Concept("one"), @@ -690,15 +856,87 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka): sheerka.set_isa(context, sheerka.new("one"), my_map["number"]) sheerka.set_isa(context, sheerka.new("twenty"), my_map["number"]) - parsing_expression = parser.get_parsing_expression(my_map["twenties"]) + parser.concepts_grammars.clear() # make sure parsing expression is created from scratch + + parsing_expression = parser.get_parsing_expression(context, my_map["twenties"]) assert parsing_expression == Sequence( ConceptExpression(my_map["twenty"], rule_name="twenty"), ConceptExpression(my_map["number"], rule_name="number")) - assert parsing_expression.nodes[0].nodes == [StrMatch("twenty")] - assert isinstance(parsing_expression.nodes[1].nodes[0], OrderedChoice) - assert ConceptExpression(my_map["one"], rule_name="one") in parsing_expression.nodes[1].nodes[0].elements - assert ConceptExpression(my_map["twenty"], rule_name="twenty") in parsing_expression.nodes[1].nodes[0].elements + assert len(parsing_expression.nodes) == len(parsing_expression.elements) + twenty_nodes = parsing_expression.nodes[0].nodes + assert twenty_nodes == [StrMatch("twenty")] + + number_nodes = parsing_expression.nodes[1].nodes + assert len(number_nodes) == 1 + assert isinstance(number_nodes[0], OrderedChoice) + assert len(number_nodes[0].nodes) == len(number_nodes[0].elements) + assert ConceptExpression(my_map["one"], rule_name="one") in number_nodes[0].nodes + assert ConceptExpression(my_map["twenty"], rule_name="twenty") in number_nodes[0].nodes + + assert my_map["number"].id not in parser.concepts_grammars + + # + # def test_i_cannot_get_parsing_expression_when_concept_is_part_of_a_group(self): + # """ + # In this test, twenties isa number + # # So 'number' in Sequence(thirty, number) will spawn 'twenties' which, because there is no other indication, + # # will create an infinite loop + # :return: + # """ + # my_map = { + # "one": Concept("one"), + # "twenty": Concept("twenty"), + # "number": Concept("number"), + # "twenties": self.bnf_concept("twenties", Sequence(ConceptExpression("twenty"), ConceptExpression("number"))) + # } + # sheerka, context, parser = self.init_parser(my_map, singleton=True) + # parser.context = context + # parser.sheerka = sheerka + # sheerka.set_isa(context, sheerka.new("one"), my_map["number"]) + # sheerka.set_isa(context, sheerka.new("twenty"), my_map["number"]) + # sheerka.set_isa(context, sheerka.new("twenties"), my_map["number"]) # <- twenties is also a number + # + # parser.concepts_grammars.clear() # make sure parsing expression is created from scratch + # + # parsing_expression = parser.get_parsing_expression(context, my_map["twenties"]) + # assert sheerka.isinstance(parsing_expression, BuiltinConcepts.CHICKEN_AND_EGG) + # assert parsing_expression.body == {my_map["twenties"].id, my_map["number"].id} + # + # assert isinstance(parser.concepts_grammars.get(my_map["one"].id), ParsingExpression) + # assert isinstance(parser.concepts_grammars.get(my_map["twenty"].id), ParsingExpression) + + def test_i_can_get_parsing_expression_when_concept_is_part_of_a_group(self): + my_map = { + "one": Concept("one"), + "twenty": Concept("twenty"), + "number": Concept("number"), + "twenties": self.bnf_concept("twenties", Sequence(ConceptExpression("twenty"), ConceptExpression("number"))) + } + sheerka, context, parser = self.init_parser(my_map, singleton=True) + parser.context = context + parser.sheerka = sheerka + sheerka.set_isa(context, sheerka.new("one"), my_map["number"]) + sheerka.set_isa(context, sheerka.new("twenty"), my_map["number"]) + sheerka.set_isa(context, sheerka.new("twenties"), my_map["number"]) # <- twenties is also a number + + parser.concepts_grammars.clear() # make sure parsing expression is created from scratch + + parsing_expression = parser.get_parsing_expression(context, my_map["twenties"]) + assert parsing_expression == Sequence( + ConceptExpression(my_map["twenty"], rule_name="twenty"), + ConceptExpression(my_map["number"], rule_name="number")) + + assert len(parsing_expression.nodes) == len(parsing_expression.elements) + twenty_nodes = parsing_expression.nodes[0].nodes + assert twenty_nodes == [StrMatch("twenty")] + + number_nodes = parsing_expression.nodes[1].nodes + assert len(number_nodes) == 1 + assert isinstance(number_nodes[0], OrderedChoice) + assert len(number_nodes[0].nodes) == len(number_nodes[0].elements) + assert ConceptExpression(my_map["one"], rule_name="one") in number_nodes[0].nodes + assert ConceptExpression(my_map["twenty"], rule_name="twenty") in number_nodes[0].nodes def test_i_can_get_parsing_expression_when_sequence_of_concept(self): my_map = { @@ -709,7 +947,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka): parser.context = context parser.sheerka = sheerka - parsing_expression = parser.get_parsing_expression(my_map["two_ones"]) + parsing_expression = parser.get_parsing_expression(context, my_map["two_ones"]) assert parsing_expression == Sequence( ConceptExpression(my_map["one"], rule_name="one"), ConceptExpression(my_map["one"], rule_name="one")) @@ -726,7 +964,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka): self.validate_get_concepts_sequences(my_map, text, expected) - def test_i_can_recognize_unknown_then_they_look_like_known(self): + def test_i_can_recognize_unknown_when_they_look_like_known(self): my_map = { "one two": self.bnf_concept("one two", Sequence("one", "two")), "three": self.bnf_concept("three") @@ -752,15 +990,13 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka): assert len(sequence) == 1 @pytest.mark.parametrize("parser_input, expected_status, expected", [ - ("one", True, [CNC("bnf one", source="one")]), # the bnf one is chosen - ("one two", True, [CN("one and two", source="one two")]), - ("three three three", True, [CN("one or more three", source="three three three")]), - ("twenty two", True, [CN("twenties", source="twenty two")]), - ("twenty four", True, [CN("twenties", source="twenty four")]), - ("twenty one", False, [UTN("twenty "), CN("bnf one", source="one")]), - ("twenty two + 1", True, [CN("twenties", source="twenty two"), " + 1"]), + ("baz", True, [CNC("bnf baz", source="baz")]), # the bnf one is chosen + ("foo bar", True, [CNC("foo then bar", source="foo bar", foo="foo", bar="bar")]), + ("bar", True, [CNC("foo or bar", source="bar", bar="bar", body="bar")]), + ("one plus two", True, [CNC("plus", source="one plus two", one="one", two="two")]), + ("twenty one", True, [CNC("t1", source="twenty one", unit="one", one="one")]), ]) - def test_i_can_parse(self, parser_input, expected_status, expected): + def test_i_can_parse_simple_expressions(self, parser_input, expected_status, expected): sheerka, context, parser = self.init_parser(init_from_sheerka=True) res = parser.parse(context, ParserInput(parser_input)) @@ -772,6 +1008,167 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka): assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT) assert concepts_nodes == expected_array + def test_i_can_when_multiple_times_the_same_variable(self): + sheerka, context, parser = self.init_parser(init_from_sheerka=True) + + text = "foo foo foo" + expected_array = compute_expected_array(cmap, text, [CNC("one or more foo", source=text)]) + expected_array[0].compiled["foo"] = [cmap["foo"], cmap["foo"], cmap["foo"]] + + res = parser.parse(context, ParserInput(text)) + parser_result = res.value + concepts_nodes = res.value.value + + assert res.status + assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT) + assert concepts_nodes == expected_array + + def test_i_can_test_when_expression_references_other_expressions(self): + sheerka, context, parser = self.init_parser(init_from_sheerka=True) + + text = "twenty four" + expected = CNC("t2", + source=text, + unit=CC("three_four", + source="four", + four=CC("four", body=DoNotResolve("four")), + body=CC("four", body=DoNotResolve("four"))), + four="four") + expected_array = compute_expected_array(cmap, text, [expected]) + + res = parser.parse(context, ParserInput(text)) + parser_result = res.value + concepts_nodes = res.value.value + + assert res.status + assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT) + assert concepts_nodes == expected_array + + # def test_i_cannot_parse_bnf_concept_mixed_with_isa_concepts(self): + # sheerka, context, parser = self.init_parser(init_from_sheerka=True) + # + # # thirties = cls.update_bnf(context, Concept("thirties", + # # definition="thirty number", + # # where="number < 10", + # # body="thirty + number").def_var("thirty").def_var("number")) + # # with thirties isa number + # # So number in 'thirty number' will spawn 'thirties' which, because there is no other indication, will + # # create an infinite loop + # + # text = "thirty one" + # expected = CNC("thirties", + # source=text, + # number=CC("number", + # source="one", + # one=CC("one", body=DoNotResolve("one")), + # body=CC("one", body=DoNotResolve("one"))), + # one=CC("one", body=DoNotResolve("one")), + # thirty="thirty") + # expected_array = compute_expected_array(cmap, text, [expected]) + # + # res = parser.parse(context, ParserInput(text)) + # not_for_me = res.value + # reason = res.value.body + # + # assert not res.status + # assert sheerka.isinstance(not_for_me, BuiltinConcepts.NOT_FOR_ME) + # assert sheerka.isinstance(reason, BuiltinConcepts.CHICKEN_AND_EGG) + # assert reason.body == {cmap["thirties"].id, cmap["number"].id} + + def test_i_can_parse_bnf_concept_mixed_with_isa_concepts(self): + sheerka, context, parser = self.init_parser(init_from_sheerka=True) + + # thirties = cls.update_bnf(context, Concept("thirties", + # definition="thirty number", + # where="number < 10", + # body="thirty + number").def_var("thirty").def_var("number")) + + text = "thirty one" + expected = CNC("thirties", + source=text, + number=CC("number", + source="one", + one=CC("one", body=DoNotResolve("one")), + body=CC("one", body=DoNotResolve("one"))), + one=CC("one", body=DoNotResolve("one")), + thirty="thirty") + expected_array = compute_expected_array(cmap, text, [expected]) + + res = parser.parse(context, ParserInput(text)) + parser_result = res.value + concepts_nodes = res.value.value + + assert res.status + assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT) + assert concepts_nodes == expected_array + + def test_i_can_parse_bnf_concept_mixed_with_isa_concepts_2(self): + # this time, three is a number, and also part of three_four, even if it is not relevant in t3 + sheerka, context, parser = self.init_parser(init_from_sheerka=True) + + text = "thirty three" + expected = CNC("thirties", + source=text, + number=CC("number", + source="three", + three=CC("three", body=DoNotResolve("three")), + body=CC("three", body=DoNotResolve("three"))), + three=CC("three", body=DoNotResolve("three")), + thirty="thirty") + expected_array = compute_expected_array(cmap, text, [expected]) + + res = parser.parse(context, ParserInput(text)) + parser_result = res.value + concepts_nodes = res.value.value + + assert res.status + assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT) + assert concepts_nodes == expected_array + + def test_i_can_parse_bnf_concept_mixed_with_isa_after_restart(self): + sheerka, context, parser = self.init_parser(init_from_sheerka=True) + sheerka.concepts_grammars.clear() # simulate restart + for c in cmap.values(): + sheerka.get_by_id(c.id).bnf = None + + text = "thirty three" + expected = CNC("thirties", + source=text, + number=CC("number", + source="three", + three=CC("three", body=DoNotResolve("three")), + body=CC("three", body=DoNotResolve("three"))), + three=CC("three", body=DoNotResolve("three")), + thirty="thirty") + expected_array = compute_expected_array(cmap, text, [expected]) + + res = parser.parse(context, ParserInput(text)) + parser_result = res.value + concepts_nodes = res.value.value + + assert res.status + assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT) + assert concepts_nodes == expected_array + + text = "forty one" + expected = CNC("forties", + source=text, + number=CC("number", + source="one", + one=CC("one", body=DoNotResolve("one")), + body=CC("one", body=DoNotResolve("one"))), + one=CC("one", body=DoNotResolve("one")), + forty="forty") + expected_array = compute_expected_array(cmap, text, [expected]) + + res = parser.parse(context, ParserInput(text)) + parser_result = res.value + concepts_nodes = res.value.value + + assert res.status + assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT) + assert concepts_nodes == expected_array + def test_i_can_parse_when_keyword(self): sheerka, context, parser = self.init_parser(init_from_sheerka=True) @@ -800,10 +1197,80 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka): parser_result = res.value concepts_nodes = res.value.value - assert res.status == True + assert res.status assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT) assert concepts_nodes == expected_array + def test_i_can_parse_descent_grammar(self): + my_map = { + "factor": Concept("factor", definition="1 | 2 | 3"), + "term": Concept("term", definition="factor ('*' factor)*"), + "expr": Concept("expr", definition="term ('+' term)*"), + } + + sheerka, context, parser = self.init_parser(my_map, singleton=True) + parser.init_from_concepts(context, my_map.values()) + + text = "1 + 2 * 3" + + res = parser.parse(context, ParserInput(text)) + parser_result = res.value + concepts_nodes = res.value.value + + factor = my_map["factor"] + term = my_map["term"] + expr = my_map["expr"] + + assert res.status + assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT) + assert concepts_nodes == [CNC(expr, + term=[CC(term, + body=CC(factor, body=DoNotResolve("1")), + factor=CC(factor, body=DoNotResolve("1"))), + CC(term, + body=DoNotResolve("2 * 3"), + factor=[ + CC(factor, body=DoNotResolve("2")), + CC(factor, body=DoNotResolve("3")), + ])], + factor=[ + CC(factor, body=DoNotResolve("1")), + CC(factor, body=DoNotResolve("2")), + CC(factor, body=DoNotResolve("3"))], + body=DoNotResolve("1 + 2 * 3"))] + + def test_i_can_parse_recursive_descent_grammar(self): + my_map = { + "factor": Concept("factor", definition="1 | 2 | 3"), + "term": self.bnf_concept("term", OrderedChoice( + Sequence(ConceptExpression("factor"), StrMatch("*"), ConceptExpression("term")), + ConceptExpression("factor"))), + "expr": self.bnf_concept("expr", OrderedChoice( + Sequence(ConceptExpression("term"), StrMatch("+"), ConceptExpression("expr")), + ConceptExpression("term"))), + } + sheerka, context, parser = self.init_parser(my_map, singleton=True) + parser.init_from_concepts(context, my_map.values()) + + text = "1 + 2 * 3" + + res = parser.parse(context, ParserInput(text)) + # concepts_nodes = res.value.value is too complicated to be validated + assert res.status + + def test_i_can_parse_simple_recursive_grammar(self): + my_map = { + "foo": self.bnf_concept("foo", Sequence(StrMatch("foo"), + OrderedChoice(StrMatch("bar"), ConceptExpression("foo")))), + } + + sheerka, context, parser = self.init_parser(my_map, singleton=True) + parser.init_from_concepts(context, my_map.values()) + + assert parser.parse(context, ParserInput("foo bar")).status + assert parser.parse(context, ParserInput("foo foo foo bar")).status + assert not parser.parse(context, ParserInput("foo baz")).status + # @pytest.mark.parametrize("parser_input, expected", [ # ("one", [ # (True, [CNC("bnf_one", source="one", one="one", body="one")]), diff --git a/tests/parsers/test_SyaNodeParser.py b/tests/parsers/test_SyaNodeParser.py index dcd8541..91d4713 100644 --- a/tests/parsers/test_SyaNodeParser.py +++ b/tests/parsers/test_SyaNodeParser.py @@ -54,12 +54,12 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka): cmap["plus"].set_prop(BuiltinConcepts.ASSOCIATIVITY, "right") cmap["mult"].set_prop(BuiltinConcepts.ASSOCIATIVITY, "right") cmap["minus"].set_prop(BuiltinConcepts.ASSOCIATIVITY, "right") - TestSyaNodeParser.sheerka.services[SheerkaComparisonManager.NAME].is_greater_than(context, - BuiltinConcepts.PRECEDENCE, - cmap["mult"], cmap["plus"]) - TestSyaNodeParser.sheerka.services[SheerkaComparisonManager.NAME].is_greater_than(context, - BuiltinConcepts.PRECEDENCE, - cmap["mult"], cmap["minus"]) + TestSyaNodeParser.sheerka.services[SheerkaComparisonManager.NAME].set_is_greater_than(context, + BuiltinConcepts.PRECEDENCE, + cmap["mult"], cmap["plus"]) + TestSyaNodeParser.sheerka.services[SheerkaComparisonManager.NAME].set_is_greater_than(context, + BuiltinConcepts.PRECEDENCE, + cmap["mult"], cmap["minus"]) # TestSyaNodeParser.sheerka.force_sya_def(context, [ # (cmap["plus"].id, 5, SyaAssociativity.Right), diff --git a/tests/sheerkapickle/test_sheerka_handlers.py b/tests/sheerkapickle/test_sheerka_handlers.py index 091ea79..f9e3a65 100644 --- a/tests/sheerkapickle/test_sheerka_handlers.py +++ b/tests/sheerkapickle/test_sheerka_handlers.py @@ -201,7 +201,7 @@ class TestSheerkaPickleHandler(TestUsingMemoryBasedSheerka): to_string = sheerkapickle.encode(sheerka, user_input) decoded = sheerkapickle.decode(sheerka, to_string) assert decoded == user_input - assert to_string == '{"_sheerka/obj": "core.builtin_concepts.UserInputConcept", "concept/id": ["__USER_INPUT", "22"], "user_name": "my_user_name", "text": "my_text"}' + assert to_string == '{"_sheerka/obj": "core.builtin_concepts.UserInputConcept", "concept/id": ["__USER_INPUT", "23"], "user_name": "my_user_name", "text": "my_text"}' def test_i_can_encode_decode_user_input_when_tokens(self): sheerka = self.get_sheerka() @@ -213,7 +213,7 @@ class TestSheerkaPickleHandler(TestUsingMemoryBasedSheerka): to_string = sheerkapickle.encode(sheerka, user_input) decoded = sheerkapickle.decode(sheerka, to_string) assert decoded == sheerka.new(BuiltinConcepts.USER_INPUT, body=text, user_name="my_user_name") - assert to_string == '{' + f'"_sheerka/obj": "core.builtin_concepts.UserInputConcept", "concept/id": ["__USER_INPUT", "22"], "user_name": "my_user_name", "text": "{text}"' + '}' + assert to_string == '{' + f'"_sheerka/obj": "core.builtin_concepts.UserInputConcept", "concept/id": ["__USER_INPUT", "23"], "user_name": "my_user_name", "text": "{text}"' + '}' def test_i_can_encode_decode_return_value(self): sheerka = self.get_sheerka() @@ -223,7 +223,7 @@ class TestSheerkaPickleHandler(TestUsingMemoryBasedSheerka): to_string = sheerkapickle.encode(sheerka, ret_val) decoded = sheerkapickle.decode(sheerka, to_string) assert decoded == ret_val - assert to_string == '{"_sheerka/obj": "core.builtin_concepts.ReturnValueConcept", "concept/id": ["__RETURN_VALUE", "27"], "who": "who", "status": true, "value": 10}' + assert to_string == '{"_sheerka/obj": "core.builtin_concepts.ReturnValueConcept", "concept/id": ["__RETURN_VALUE", "28"], "who": "who", "status": true, "value": 10}' def test_i_can_encode_decode_return_value_with_parent(self): sheerka = self.get_sheerka() @@ -236,7 +236,7 @@ class TestSheerkaPickleHandler(TestUsingMemoryBasedSheerka): decoded = sheerkapickle.decode(sheerka, to_string) assert decoded == ret_val assert decoded.parents == ret_val.parents - id_str = ', "concept/id": ["__RETURN_VALUE", "27"]' + id_str = ', "concept/id": ["__RETURN_VALUE", "28"]' parents_str = '[{"_sheerka/obj": "core.builtin_concepts.ReturnValueConcept"' + id_str + ', "who": "parent_who", "status": true, "value": "10"}, {"_sheerka/id": 1}]' assert to_string == '{"_sheerka/obj": "core.builtin_concepts.ReturnValueConcept"' + id_str + ', "who": "who", "status": true, "value": 10, "parents": ' + parents_str + '}'