diff --git a/_concepts_numbers.txt b/_concepts_numbers.txt index 3e9b64a..f34af85 100644 --- a/_concepts_numbers.txt +++ b/_concepts_numbers.txt @@ -89,4 +89,8 @@ def concept divided from a divided by b as a * b set_is_greater_than(__PRECEDENCE, multiplied, plus, 'Sya') set_is_greater_than(__PRECEDENCE, divided, plus, 'Sya') set_is_greater_than(__PRECEDENCE, multiplied, minus, 'Sya') -set_is_greater_than(__PRECEDENCE, divided, minus, 'Sya') \ No newline at end of file +set_is_greater_than(__PRECEDENCE, divided, minus, 'Sya') + +def concept quantity +def concept quantify x from bnf number x as set_attr(x, quantity, number) ret x +def concept how many x pre is_question() as get_attr(memory(x), quantity) \ No newline at end of file diff --git a/src/core/builtin_helpers.py b/src/core/builtin_helpers.py index 46e0352..f77a82a 100644 --- a/src/core/builtin_helpers.py +++ b/src/core/builtin_helpers.py @@ -5,11 +5,11 @@ from cache.Cache import Cache from core.ast_helpers import ast_to_props from core.builtin_concepts import BuiltinConcepts from core.concept import Concept, ConceptParts, DEFINITION_TYPE_BNF, concept_part_value -from core.global_symbols import NotInit, NotFound +from core.global_symbols import NotInit, NotFound, CURRENT_OBJ from core.rule import Rule from core.utils import as_bag from parsers.BaseNodeParser import SourceCodeNode, ConceptNode, UnrecognizedTokensNode, SourceCodeWithConceptNode, \ - RuleNode + RuleNode, VariableNode from parsers.BaseParser import ParsingError PARSE_STEPS = [BuiltinConcepts.BEFORE_PARSING, BuiltinConcepts.PARSING, BuiltinConcepts.AFTER_PARSING] @@ -498,6 +498,18 @@ def get_lexer_nodes_from_unrecognized(context, unrecognized_tokens_node, parsers :return: """ + # first look into short term memory to see if the unrecognized is not a variable of the current object + if (current_obj := context.sheerka.get_from_short_term_memory(context, CURRENT_OBJ)) is not NotFound: + if isinstance(current_obj, Concept): + source = unrecognized_tokens_node.source + if source in current_obj.get_compiled() or source in current_obj.variables(): + return [[VariableNode(current_obj, + source, + unrecognized_tokens_node.start, + unrecognized_tokens_node.end, + unrecognized_tokens_node.tokens, + unrecognized_tokens_node.source)]] + res = context.sheerka.parse_unrecognized(context, unrecognized_tokens_node.source, parsers) res = only_parsers_results(context, res) diff --git a/src/core/concept.py b/src/core/concept.py index c593994..139fbe5 100644 --- a/src/core/concept.py +++ b/src/core/concept.py @@ -1,9 +1,7 @@ import hashlib -from collections import namedtuple from copy import deepcopy from dataclasses import dataclass from threading import RLock -from typing import Union import core.utils from core.builtin_concepts_ids import BuiltinDynamicAttrs @@ -68,14 +66,21 @@ all_attributes_lock = RLock() def get_concept_attrs(concept): + # look for instance attributes + if concept.get_all_attributes() is not None: + return concept.get_all_attributes() + + # look for class attributes defined within the concept (for real classes that inherit from Concept) if concept.ALL_ATTRIBUTES is not None: return concept.ALL_ATTRIBUTES try: + # class attributes defined globally return ALL_ATTRIBUTES[concept.id] except KeyError: pass + # create a class attribute with all_attributes_lock: all_attributes = [k for k in concept.__dict__ if k[0] != "_" and k[0] != "#"] if concept.id and concept.key not in BuiltinDynamicAttrs: @@ -97,7 +102,8 @@ def copy_concepts_attrs(): def load_concepts_attrs(attrs): global ALL_ATTRIBUTES with all_attributes_lock: - ALL_ATTRIBUTES = attrs + ALL_ATTRIBUTES.clear() + ALL_ATTRIBUTES.update(attrs) class Concept: @@ -107,7 +113,7 @@ class Concept: Everything is a concept """ - ALL_ATTRIBUTES = None + ALL_ATTRIBUTES = None # class attributes (only use when for Concept subclasses where it id is not yet defined) def __init__(self, name=None, is_builtin=False, @@ -151,22 +157,16 @@ class Concept: self._original_definition_hash = None # concept hash before any alteration of the metadata self._format = None # how to print the concept self._hints = {} # extra processing information to help processing + self._all_attributes = None # instance attributes def __repr__(self): text = f"({self._metadata.id}){self._metadata.name}" return text + " (" + self._metadata.pre + ")" if self._metadata.pre else text def __eq__(self, other): - if id(self) == id(other): return True - if isinstance(other, simplec): - return self.name == other.name and self.body == other.body - - if isinstance(other, (CC, CB, CV, CMV, CIO)): - return other == self - if not isinstance(other, Concept): return False @@ -208,14 +208,8 @@ class Concept: def __hash__(self): return hash(self._metadata.name) - # def __getattr__(self, item): - # # I have this complicated implementation because of the usage of Pickle - # - # if 'values' in vars(self) and item in self.values: - # return self.get_value(item) - # - # name = self.name if 'metadata' in vars(self) else 'Concept' - # raise AttributeError(f"'{name}' concept has no attribute '{item}'") + def get_all_attributes(self): + return self._all_attributes def def_var(self, var_name, default_value=None): """ @@ -466,6 +460,15 @@ class Concept: setattr(self, ConceptParts.BODY, value) elif self._bound_body and name == ConceptParts.BODY: setattr(self, self._bound_body, value) + + # KSI 2021-03-04 + # I am not sure how cost efficient it is to check for new attribute everytime + # Need to find a better way + if name not in get_concept_attrs(self): + if self._all_attributes is None: + self._all_attributes = get_concept_attrs(self).copy() + self._all_attributes.append(name) + except AttributeError: print(f"Cannot set {name}") return self @@ -607,275 +610,3 @@ class InfiniteRecursionResolved: def get_obj_value(self): return self.value - - -# ################################ -# -# Class created for tests purpose -# -# ################################ - - -class CC: - """ - Concept class for test purpose - CC means concept for compiled (or concept with compiled) - It matches a concept if the compiles are equals - """ - - # The only properties that are testes are concept_key and compiled - # The other properties (concept, source, start and end) - # are used in tests/parsers/parsers_utils.py to help creating helper objects - - def __init__(self, concept, source=None, exclude_body=False, **kwargs): - self.concept_key = concept.key if isinstance(concept, Concept) else concept - self.compiled = kwargs - self.concept = concept if isinstance(concept, Concept) else None - self.source = source # to use when the key is different from the sub str to search when filling start and stop - self.start = None # for debug purpose, indicate where the concept starts - self.end = None # for debug purpose, indicate where the concept ends - self.exclude_body = exclude_body - - if "body" in self.compiled: - self.compiled[ConceptParts.BODY] = self.compiled["body"] - del self.compiled["body"] - - def __eq__(self, other): - if id(self) == id(other): - return True - - if isinstance(other, Concept): - if other.key != self.concept_key: - return False - if self.exclude_body: - to_compare = {k: v for k, v in other.get_compiled().items() if k != ConceptParts.BODY} - else: - to_compare = other.get_compiled() - if self.compiled == to_compare: - return True - else: - return False - - if not isinstance(other, CC): - return False - - if self.concept_key != other.concept_key: - return False - - return self.compiled == other.compiled - - def __hash__(self): - if self.concept: - return hash(self.concept) - return hash(self.concept_key) - - def __repr__(self): - if self.concept: - txt = f"CC(concept='{self.concept}'" - else: - txt = f"CC(concept_key='{self.concept_key}'" - - for k, v in self.compiled.items(): - txt += f", {k}='{v}'" - return txt + ")" - - def fix_pos(self, node): - start = node.start if hasattr(node, "start") else \ - node[0] if isinstance(node, tuple) else None - end = node.end if hasattr(node, "end") else \ - node[1] if isinstance(node, tuple) else None - - if start is not None: - if self.start is None or start < self.start: - self.start = start - - if end is not None: - if self.end is None or end > self.end: - self.end = end - return self - - def to_compare(self, other, to_compare_delegate): - """ - Transform other into CNC, to ease the comparison - :param other: - :param to_compare_delegate: - :return: - """ - - if isinstance(other, CC): - return other - - if isinstance(other, Concept): - if self.exclude_body: - compiled = {k: v for k, v in other.get_compiled().items() if k != ConceptParts.BODY} - else: - compiled = other.get_compiled() - - self_compile_to_use = self.compiled or compiled - - compiled = to_compare_delegate(self_compile_to_use, compiled, to_compare_delegate) - return CC(other, - self.source, - self.exclude_body, - **compiled) - - raise NotImplementedError(f"CC, {other=}") - - -@dataclass() -class CB: - """ - Concept with body only - Test class that tests only the body of the concept - """ - concept: Union[str, Concept] - body: object - - def __eq__(self, other): - if isinstance(other, Concept): - key = self.concept if isinstance(self.concept, str) else self.concept.key - return key == other.key and self.body == other.body - - if not isinstance(other, CB): - return False - - return self.concept == other.concept and self.body == other.body - - def __hash__(self): - return hash((self.concept, self.body)) - - def __repr__(self): - return f"CB({self.body})" - - -class CV: - """ - Concept with all values - Test class that tests all the values (not the metadata, so not the properties) of a concept - """ - - def __init__(self, concept, **kwargs): - self.concept_key = concept.key if isinstance(concept, Concept) else concept - self.concept = concept if isinstance(concept, Concept) else None - self.values = {} - for k, v in kwargs.items(): - if f"#{k}#" in AllConceptParts: - self.values[f"#{k}#"] = v - else: - self.values[k] = v - - def __eq__(self, other): - if isinstance(other, Concept): - if self.concept_key != other.key: - return False - for k, v in self.values.items(): - if self.values[k] != other.get_value(k): - return False - return True - - if not isinstance(other, CV): - return False - - return self.concept_key == other.concept_key and self.values == other.values - - def __hash__(self): - return hash((self.concept_key, self.values)) - - def __repr__(self): - return f"CV(key={self.concept_key}, values={self.values})" - - -class CMV: - """ - Concept with metadata variables - CMV stands for Concept Metadata Variables - Test class that only compare the key and the metadata variables - """ - - def __init__(self, concept, **kwargs): - self.concept_key = concept.key if isinstance(concept, Concept) else concept - self.concept = concept if isinstance(concept, Concept) else None - self.variables = kwargs - - def __eq__(self, other): - if id(self) == id(other): - return True - - if isinstance(other, Concept): - if other.key != self.concept_key: - return False - - if len(other._metadata.variables) != len(self.variables): - return False - - for name, value in other._metadata.variables: - if self.variables[name] != value: - return False - return True - - if not isinstance(other, CMV): - return False - - if self.concept_key != other.concept_key: - return False - - return self.variables == other.variables - - def __hash__(self): - if self.concept: - return hash(self.concept) - return hash(self.concept_key) - - def __repr__(self): - if self.concept: - txt = f"CMV(concept='{self.concept}'" - else: - txt = f"CMV(concept_key='{self.concept_key}'" - - for k, v in self.variables.items(): - txt += f", {k}='{v}'" - return txt + ")" - - -class CIO: - """ - Concept id only - only test the id - """ - - def __init__(self, concept, source=None): - if isinstance(concept, str): - self.concept_name = concept - self.concept_id = None - self.concept = None - elif isinstance(concept, Concept): - self.concept_id = concept.id - self.concept = concept - self.source = source - self.start = None - self.end = None - - def set_concept(self, concept): - self.concept = concept - self.concept_id = concept.id - - def __eq__(self, other): - if id(self) == id(other): - return True - - if isinstance(other, Concept): - return self.concept_id == other.id - - if not isinstance(other, CIO): - return False - - return self.concept_id == other.concept_id - - def __hash__(self): - return hash(self.concept_id) - - def __repr__(self): - return f"CIO(concept='{self.concept}')" if self.concept else f"CIO(name='{self.concept_name}')" - - -simplec = namedtuple("concept", "name body") # for simple concept (tests purposes only) diff --git a/src/core/global_symbols.py b/src/core/global_symbols.py index 164b05e..5485ca4 100644 --- a/src/core/global_symbols.py +++ b/src/core/global_symbols.py @@ -31,6 +31,9 @@ class CustomType: def __eq__(self, other): return isinstance(other, CustomType) and self.value == other.value + def __hash__(self): + return hash(self.value) + class NotInitType(CustomType): def __init__(self): @@ -63,3 +66,6 @@ class ErrorObj: To indicate that somehow, the underlying object is (or has) an error """ pass + + +CURRENT_OBJ = "__obj" diff --git a/src/core/sheerka/services/SheerkaAdmin.py b/src/core/sheerka/services/SheerkaAdmin.py index f421d9e..80284d7 100644 --- a/src/core/sheerka/services/SheerkaAdmin.py +++ b/src/core/sheerka/services/SheerkaAdmin.py @@ -35,13 +35,15 @@ class SheerkaAdmin(BaseService): self.sheerka.bind_service_method(self.ontologies, False) self.sheerka.bind_service_method(self.in_memory, False) self.sheerka.bind_service_method(self.admin_history, False, as_name="history") + self.sheerka.bind_service_method(self.admin_history, False, as_name="history") + self.sheerka.bind_service_method(self.sdp, False) def caches_names(self): """ Returns the name of all the caches :return: """ - return list(self.sheerka.om.current_cache_manager().caches.keys()) + return self.sheerka.new(BuiltinConcepts.TO_LIST, body=self.sheerka.om.current_cache_manager().caches.keys()) def cache(self, name, *keys): """ @@ -58,6 +60,16 @@ class SheerkaAdmin(BaseService): return {key: self.sheerka.om.get(name, key) for key in keys} + def sdp(self, name=None): + if name: + for ontology in self.sheerka.om.ontologies: + if ontology.name == name: + return ontology.cache_manager.sdp + + return self.sheerka.err(self.sheerka.new(BuiltinConcepts.NOT_FOUND, {"sdp_name", name})) + + return self.sheerka.om.current_sdp() + def restore(self, concept_file=CONCEPTS_FILE_TO_USE): """ Restore the state with all previous valid concept definitions diff --git a/src/core/sheerka/services/SheerkaConceptManager.py b/src/core/sheerka/services/SheerkaConceptManager.py index fa73f2c..01bc118 100644 --- a/src/core/sheerka/services/SheerkaConceptManager.py +++ b/src/core/sheerka/services/SheerkaConceptManager.py @@ -237,36 +237,22 @@ class SheerkaConceptManager(BaseService): except Exception as ex: return sheerka.ret(self.NAME, False, ex.args[0]) - # compute first token and/or first regex - init_ret_value = self.compute_concepts_by_first_item(context, [concept], True) - if not init_ret_value.status: - return sheerka.ret(self.NAME, False, ErrorConcept(init_ret_value.value)) - by_first_keyword, by_first_regex = init_ret_value.body - - # computes resolved concepts_by_first_keyword - init_ret_value = self.resolve_concepts_by_first_keyword(context, by_first_keyword) - if not init_ret_value.status: - return sheerka.ret(self.NAME, False, ErrorConcept(init_ret_value.value)) - resolved_concepts_by_first_keyword = init_ret_value.body - - # compile regex - compile_ret = self.compile_concepts_by_first_regex(context, by_first_regex) - if not compile_ret.status: - return sheerka.ret(self.NAME, False, ErrorConcept(compile_ret.value)) - compiled_concepts_by_first_regex = compile_ret.body + # recompute concepts by first tokens and concept by first regex + update_items_res = self.recompute_first_items(context, None, [concept]) + if not update_items_res.status: + return update_items_res + by_first_keyword, by_first_regex, resolved_by_first_keyword, compiled_by_first_regex = update_items_res.body # if everything is fine freeze_concept_attrs(concept) concept.freeze_definition_hash() om.add_concept(concept) - om.put(self.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, False, by_first_keyword) - om.put(self.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY, False, resolved_concepts_by_first_keyword) - om.put(self.CONCEPTS_BY_REGEX_ENTRY, False, {k.serialize(): v for k, v in by_first_regex.items()}) - - # update the compiled regex - self.compiled_concepts_by_regex.clear() - self.compiled_concepts_by_regex.extend(compiled_concepts_by_first_regex) + self.update_first_items_caches(context, + by_first_keyword, + by_first_regex, + resolved_by_first_keyword, + compiled_by_first_regex) if concept.get_metadata().definition_type == DEFINITION_TYPE_DEF and concept.get_metadata().definition != concept.name: # allow search by definition when definition relevant @@ -284,9 +270,7 @@ class SheerkaConceptManager(BaseService): # publish the new concept sheerka.publish(context, EVENT_CONCEPT_CREATED, concept) - # process the return if needed - ret = sheerka.ret(self.NAME, True, sheerka.new(BuiltinConcepts.NEW_CONCEPT, body=concept)) - return ret + return sheerka.ret(self.NAME, True, sheerka.new(BuiltinConcepts.NEW_CONCEPT, body=concept)) def modify_concept(self, context, concept, to_add=None, to_remove=None, modify_source=False): """ @@ -324,40 +308,17 @@ class SheerkaConceptManager(BaseService): # modify the metadata. Almost all ConceptMetadata attributes except variables and props new_concept = sheerka.new_from_template(concept, concept.key) # reload from cache or database ? - res = self._update_concept(context, new_concept, to_add, to_remove) if res is not None: return res - # To update concept by first keyword and first regex - # first remove old first token and first regex entries - concepts_by_first_keyword, concepts_by_regex = self._remove_concept_first_token_and_first_regex(concept) + # recompute concepts by first tokens and concept by first regex + update_items_res = self.recompute_first_items(context, concept, [new_concept]) + if not update_items_res.status: + return update_items_res + by_first_keyword, by_first_regex, resolved_by_first_keyword, compiled_by_first_regex = update_items_res.body - # and then update - init_ret_value = self.compute_concepts_by_first_item(context, - [new_concept], - False, - concepts_by_first_keyword, - concepts_by_regex) - if not init_ret_value.status: - return sheerka.ret(self.NAME, False, ErrorConcept(init_ret_value.value)) - concepts_by_first_keyword, concepts_by_regex = init_ret_value.body - - # computes resolved concepts_by_first_keyword - init_ret_value = self.resolve_concepts_by_first_keyword(context, - concepts_by_first_keyword, - {new_concept.id: new_concept}) - if not init_ret_value.status: - return sheerka.ret(self.NAME, False, ErrorConcept(init_ret_value.value)) - resolved_concepts_by_first_keyword = init_ret_value.body - - # compile new regex - compile_ret = self.compile_concepts_by_first_regex(context, concepts_by_regex) - if not compile_ret.status: - return sheerka.ret(self.NAME, False, ErrorConcept(compile_ret.value)) - compiled_concepts_by_first_regex = compile_ret.body - - # update concept that referenced the old concept and clear old references + # update concepts that referenced the old concept and clear old references self.update_references(context, concept, new_concept, to_add) for ref in self.compute_references(concept): om.delete(self.CONCEPTS_REFERENCES_ENTRY, ref, concept.id) @@ -368,13 +329,11 @@ class SheerkaConceptManager(BaseService): # everything is ok, update the caches om.update_concept(concept, new_concept) - om.put(self.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, False, concepts_by_first_keyword) - om.put(self.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY, False, resolved_concepts_by_first_keyword) - om.put(self.CONCEPTS_BY_REGEX_ENTRY, False, {k.serialize(): v for k, v in concepts_by_regex.items()}) - - # update the compiled regex - self.compiled_concepts_by_regex.clear() - self.compiled_concepts_by_regex.extend(compiled_concepts_by_first_regex) + self.update_first_items_caches(context, + by_first_keyword, + by_first_regex, + resolved_by_first_keyword, + compiled_by_first_regex) # everything seems to be fine. Update the list of attributes # Caution. Must be done AFTER update_concept() @@ -386,8 +345,7 @@ class SheerkaConceptManager(BaseService): self._update_concept(context, concept, to_add, to_remove) # KSI 2021-02-16 publish the modification of the concept only when someone needs it - ret = sheerka.ret(self.NAME, True, sheerka.new(BuiltinConcepts.NEW_CONCEPT, body=new_concept)) - return ret + return sheerka.ret(self.NAME, True, sheerka.new(BuiltinConcepts.NEW_CONCEPT, body=new_concept)) def remove_concept(self, context, concept): """ @@ -410,30 +368,19 @@ class SheerkaConceptManager(BaseService): refs_instances = [sheerka.new_from_template(c, c.key) for c in [self.get_by_id(ref) for ref in refs]] return sheerka.ret(self.NAME, False, sheerka.err(ConceptIsReferenced(refs_instances))) - concepts_by_first_keyword, concepts_by_regex = self._remove_concept_first_token_and_first_regex(concept) - - # computes resolved concepts_by_first_keyword - init_ret_value = self.resolve_concepts_by_first_keyword(context, concepts_by_first_keyword) - if not init_ret_value.status: - return sheerka.ret(self.NAME, False, ErrorConcept(init_ret_value.value)) - resolved_concepts_by_first_keyword = init_ret_value.body - - # compile new regex - compile_ret = self.compile_concepts_by_first_regex(context, concepts_by_regex) - if not compile_ret.status: - return sheerka.ret(self.NAME, False, ErrorConcept(compile_ret.value)) - compiled_concepts_by_first_regex = compile_ret.body + # recompute concepts by first tokens and concept by first regex + update_items_res = self.recompute_first_items(context, concept, None) + if not update_items_res.status: + return update_items_res + by_first_keyword, by_first_regex, resolved_by_first_keyword, compiled_by_first_regex = update_items_res.body # everything seems fine. I can commit the modification and remove om.remove_concept(concept) - - om.put(self.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, False, concepts_by_first_keyword) - om.put(self.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY, False, resolved_concepts_by_first_keyword) - om.put(self.CONCEPTS_BY_REGEX_ENTRY, False, {k.serialize(): v for k, v in concepts_by_regex.items()}) - - # update the compiled regex - self.compiled_concepts_by_regex.clear() - self.compiled_concepts_by_regex.extend(compiled_concepts_by_first_regex) + self.update_first_items_caches(context, + by_first_keyword, + by_first_regex, + resolved_by_first_keyword, + compiled_by_first_regex) sheerka.publish(context, EVENT_CONCEPT_DELETED, concept) return sheerka.ret(self.NAME, True, sheerka.new(BuiltinConcepts.SUCCESS)) @@ -568,9 +515,9 @@ class SheerkaConceptManager(BaseService): """ Updates all the concepts that reference concept :param context: - :param concept: - :param modified_concept: - :param modifications: + :param concept: Old version of the concept + :param modified_concept: new version of the concept + :param modifications: what are the modification :return: """ @@ -582,11 +529,12 @@ class SheerkaConceptManager(BaseService): # remove the grammar entry so that it can be recreated self.sheerka.om.delete(self.CONCEPTS_BNF_DEFINITIONS_ENTRY, concept_id) + to_update = self.get_by_id(concept_id) + metadata = to_update.get_metadata() + # reset the bnf definition if needed if modified_concept: - if self.has_id(concept_id): - to_update = self.get_by_id(concept_id) - metadata = to_update.get_metadata() + if self.has_id(concept_id): # reset only it the bnf definition is in cache if metadata.definition_type == DEFINITION_TYPE_BNF and self._name_has_changed(modifications): tokens = list(Tokenizer(metadata.definition)) modified = False @@ -601,6 +549,19 @@ class SheerkaConceptManager(BaseService): to_update.get_metadata().definition = core.utils.get_text_from_tokens(tokens) to_update.set_bnf(None) + # update concept_by_first_token + if metadata.definition_type == DEFINITION_TYPE_BNF: + # recompute concepts by first tokens and concept by first regex + res = self.recompute_first_items(context, concept, [concept]) + if not res.status: + return res + by_first_keyword, by_first_regex, resolved_by_first_keyword, compiled_by_first_regex = res.body + self.update_first_items_caches(context, + by_first_keyword, + by_first_regex, + resolved_by_first_keyword, + compiled_by_first_regex) + def compute_references(self, concept): """ We need to keep a track of all concepts used by the current concept @@ -723,6 +684,8 @@ class SheerkaConceptManager(BaseService): else: return sheerka.ret(self.NAME, False, sheerka.err(UnknownAttribute(k))) core.utils.remove_list_from_list(concept.get_metadata().variables, variables_to_remove) + if concept.get_all_attributes(): + core.utils.remove_list_from_list(concept.get_all_attributes(), [v[0] for v in variables_to_remove]) concept.get_metadata().key = None if self._definition_has_changed(to_add) and concept.get_metadata().definition_type == DEFINITION_TYPE_BNF: @@ -1046,3 +1009,78 @@ class SheerkaConceptManager(BaseService): def get_concepts_bnf_definitions(self): return self.sheerka.om.current_cache_manager().caches[self.CONCEPTS_BNF_DEFINITIONS_ENTRY].cache + + def recompute_first_items(self, context, old_concept, new_concepts): + """ + Recompute + concepts fy first items + resolved concept by first items + concepts by first regex + compiled concepts by first regex + + Do not update anything + :param context: + :param old_concept: + :param new_concepts: + :return: + """ + sheerka = context.sheerka + + if old_concept and not new_concepts: + # remove + modified_concepts = None + by_first_keyword, by_first_regex = self._remove_concept_first_token_and_first_regex(old_concept) + + elif old_concept and new_concepts: + # remove + by_first_keyword, by_first_regex = self._remove_concept_first_token_and_first_regex(old_concept) + + # and then update + init_ret_value = self.compute_concepts_by_first_item(context, + new_concepts, + False, + by_first_keyword, + by_first_regex) + if not init_ret_value.status: + return sheerka.ret(self.NAME, False, ErrorConcept(init_ret_value.value)) + by_first_keyword, by_first_regex = init_ret_value.body + modified_concepts = {new_concept.id: new_concept for new_concept in new_concepts} + + elif not old_concept and new_concepts: + # only update + modified_concepts = None + init_ret_value = self.compute_concepts_by_first_item(context, new_concepts, True) + if not init_ret_value.status: + return sheerka.ret(self.NAME, False, ErrorConcept(init_ret_value.value)) + by_first_keyword, by_first_regex = init_ret_value.body + + # computes resolved concepts_by_first_keyword + init_ret_value = self.resolve_concepts_by_first_keyword(context, by_first_keyword, modified_concepts) + if not init_ret_value.status: + return sheerka.ret(self.NAME, False, ErrorConcept(init_ret_value.value)) + resolved_by_first_keyword = init_ret_value.body + + # compile new regex + compile_ret = self.compile_concepts_by_first_regex(context, by_first_regex) + if not compile_ret.status: + return sheerka.ret(self.NAME, False, ErrorConcept(compile_ret.value)) + compiled_by_first_regex = compile_ret.body + + return sheerka.ret(self.NAME, + True, + (by_first_keyword, by_first_regex, resolved_by_first_keyword, compiled_by_first_regex)) + + def update_first_items_caches(self, + context, + by_first_keyword, + by_first_regex, + resolved_by_first_keyword, + compiled_by_first_regex): + om = context.sheerka.om + om.put(self.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, False, by_first_keyword) + om.put(self.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY, False, resolved_by_first_keyword) + om.put(self.CONCEPTS_BY_REGEX_ENTRY, False, {k.serialize(): v for k, v in by_first_regex.items()}) + + # update the compiled regex + self.compiled_concepts_by_regex.clear() + self.compiled_concepts_by_regex.extend(compiled_by_first_regex) diff --git a/src/core/sheerka/services/SheerkaEvaluateConcept.py b/src/core/sheerka/services/SheerkaEvaluateConcept.py index 2c9a684..f1cfde5 100644 --- a/src/core/sheerka/services/SheerkaEvaluateConcept.py +++ b/src/core/sheerka/services/SheerkaEvaluateConcept.py @@ -4,7 +4,7 @@ from core.builtin_concepts import BuiltinConcepts from core.builtin_helpers import expect_one, only_successful, evaluate, ensure_concept from core.concept import Concept, DoNotResolve, ConceptParts, InfiniteRecursionResolved, AllConceptParts, \ concept_part_value -from core.global_symbols import NotInit +from core.global_symbols import NotInit, CURRENT_OBJ from core.rule import Rule from core.sheerka.services.SheerkaConceptManager import SheerkaConceptManager from core.sheerka.services.SheerkaExecute import ParserInput @@ -535,6 +535,8 @@ class SheerkaEvaluateConcept(BaseService): if context.sheerka.isa(concept, context.sheerka.new(BuiltinConcepts.AUTO_EVAL)): sub_context.protected_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED) + sub_context.add_to_short_term_memory(CURRENT_OBJ, concept) + try: self.initialize_concept_asts(sub_context, concept) except ChickenAndEggException as ex: diff --git a/src/core/sheerka/services/SheerkaIsAManager.py b/src/core/sheerka/services/SheerkaIsAManager.py index 6cef2a7..8bbd0cd 100644 --- a/src/core/sheerka/services/SheerkaIsAManager.py +++ b/src/core/sheerka/services/SheerkaIsAManager.py @@ -280,17 +280,16 @@ for x in xx__concepts__xx: sub_context.protected_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED) errors = [] for element_id in ids: - concept = self.sheerka.get_by_id(element_id) - if len(concept.get_metadata().variables) == 0: - # The concepts are directly taken from Sheerka.get_by_id, so variable cannot be filled - # It's the reason why we only evaluate concept with no variable - evaluated = self.sheerka.evaluate_concept(sub_context, concept) - if context.sheerka.is_success(evaluated): - result.append(evaluated) + concept = self.sheerka.fast_resolve((None, element_id)) + if concept: + if len(concept.get_metadata().variables) == 0: + evaluated = self.sheerka.evaluate_concept(sub_context, concept) + if context.sheerka.is_success(evaluated): + result.append(evaluated) + else: + errors.append(evaluated) else: - errors.append(evaluated) - else: - result.append(concept) + result.append(concept) sub_context.add_values(return_value=result) sub_context.add_values(errors=errors) return result diff --git a/src/core/sheerka/services/SheerkaMemory.py b/src/core/sheerka/services/SheerkaMemory.py index 5d017b0..7e51627 100644 --- a/src/core/sheerka/services/SheerkaMemory.py +++ b/src/core/sheerka/services/SheerkaMemory.py @@ -12,6 +12,15 @@ from core.sheerka.services.sheerka_service import BaseService, ServiceObj class MemoryObject(ServiceObj): obj: object + def __eq__(self, other): + if not isinstance(other, MemoryObject): + return False + + return self.obj == other.obj and self.event_id == other.event_id + + def __hash__(self): + return hash((self.event_id, self.obj)) + class SheerkaMemory(BaseService): NAME = "Memory" @@ -26,6 +35,7 @@ class SheerkaMemory(BaseService): def initialize(self): self.sheerka.bind_service_method(self.get_from_short_term_memory, False, visible=False) + self.sheerka.bind_service_method(self.get_all_short_term_memory, False, visible=False) self.sheerka.bind_service_method(self.add_to_short_term_memory, True, visible=False) self.sheerka.bind_service_method(self.remove_context, True, as_name="clear_short_term_memory", visible=False) self.sheerka.bind_service_method(self.add_to_memory, True, visible=False) @@ -62,8 +72,26 @@ class SheerkaMemory(BaseService): context = context.get_parent() - def get_all_short_term_memory(self, context): - return self.short_term_objects.get(context.id) + def get_all_short_term_memory(self, context, recursive=False): + id_to_use = context.id if context else self.GLOBAL + + if not recursive: + return self.short_term_objects.get(id_to_use) + + all_vars = {} + while True: + try: + all_vars.update(self.short_term_objects.cache[id_to_use]) + except KeyError: + pass + + if id_to_use == self.GLOBAL: + break + else: + context = context.get_parent() + id_to_use = context.id if context else self.GLOBAL + + return all_vars def add_to_short_term_memory(self, context, key, value): if context: @@ -99,12 +127,12 @@ class SheerkaMemory(BaseService): if last is NotFound: self.sheerka.om.put(SheerkaMemory.OBJECTS_ENTRY, key, MemoryObject(context.event.get_digest(), concept)) return - + if not isinstance(last, list) and last.obj == concept: self.sheerka.om.delete(SheerkaMemory.OBJECTS_ENTRY, key, last) self.sheerka.om.put(SheerkaMemory.OBJECTS_ENTRY, key, MemoryObject(context.event.get_digest(), concept)) return - + if isinstance(last, list) and last[-1].obj == concept: self.sheerka.om.delete(SheerkaMemory.OBJECTS_ENTRY, key, last[-1]) self.sheerka.om.put(SheerkaMemory.OBJECTS_ENTRY, key, MemoryObject(context.event.get_digest(), concept)) diff --git a/src/core/utils.py b/src/core/utils.py index 65fdf16..a770d09 100644 --- a/src/core/utils.py +++ b/src/core/utils.py @@ -444,7 +444,7 @@ def str_concept(t, drop_name=None, prefix="c:"): elif prefix == "r:": name, id_ = t.metadata.name, t.id else: - name, id_ = t.key, t.id + name, id_ = t.key or t.name, t.id if name is None and id_ is None: return "" diff --git a/src/core/var_ref.py b/src/core/var_ref.py new file mode 100644 index 0000000..10be32b --- /dev/null +++ b/src/core/var_ref.py @@ -0,0 +1,32 @@ +from core.concept import Concept +from core.rule import Rule + + +class VariableRef: + """ + Represents the reference to the property/attribute of an object + """ + + def __init__(self, obj, prop): + self.obj = obj + self.prop = prop + + def __eq__(self, other): + if not isinstance(other, VariableRef): + return False + + return self.obj == other.obj and self.prop == other.prop + + def __hash__(self): + return hash((self.obj, self.prop)) + + @property + def key(self): + return self.obj.key or self.obj.name if isinstance(self.obj, (Concept, Rule)) else type(self.obj).__name__ + + @property + def id(self): + if self.prop is None: + return "" + + return self.prop diff --git a/src/evaluators/PythonEvaluator.py b/src/evaluators/PythonEvaluator.py index 05a1d40..6515caf 100644 --- a/src/evaluators/PythonEvaluator.py +++ b/src/evaluators/PythonEvaluator.py @@ -13,6 +13,7 @@ from core.rule import Rule from core.sheerka.ExecutionContext import ExecutionContext from core.sheerka.services.SheerkaMemory import SheerkaMemory from core.tokenizer import Token, TokenKind +from core.var_ref import VariableRef from evaluators.BaseEvaluator import OneReturnValueEvaluator from parsers.PythonParser import PythonNode @@ -332,6 +333,10 @@ class PythonEvaluator(OneReturnValueEvaluator): :param name: :return: """ + + if isinstance(name, VariableRef): + return getattr(name.obj, name.prop) + if isinstance(name, Rule): return context.sheerka.resolve_rule(context, name) diff --git a/src/parsers/BaseNodeParser.py b/src/parsers/BaseNodeParser.py index ee8ddda..a943779 100644 --- a/src/parsers/BaseNodeParser.py +++ b/src/parsers/BaseNodeParser.py @@ -1,11 +1,9 @@ -from collections import namedtuple from dataclasses import dataclass from enum import Enum import core.utils -from core.concept import Concept, ConceptParts -from core.rule import Rule from core.tokenizer import TokenKind, Token +from core.var_ref import VariableRef from parsers.BaseParser import Node, BaseParser, ParsingError DEBUG_COMPILED = True @@ -117,14 +115,6 @@ class UnrecognizedTokensNode(LexerNode): return self.tokens[-1].type def __eq__(self, other): - if isinstance(other, utnode): - return self.start == other.start and \ - self.end == other.end and \ - self.source == other.source - - if isinstance(other, UTN): - return other == self - if not isinstance(other, UnrecognizedTokensNode): return False @@ -158,9 +148,6 @@ class RuleNode(LexerNode): if id(self) == id(other): return True - if isinstance(other, RN): - return other == self - if not isinstance(other, RuleNode): return False @@ -198,18 +185,6 @@ class ConceptNode(LexerNode): if id(self) == id(other): return True - if isinstance(other, (CN, CNC)): - return other == self - - if isinstance(other, cnode): - return self.concept.key == other.concept_key and \ - self.start == other.start and \ - self.end == other.end and \ - self.source == other.source - - if isinstance(other, short_cnode): - return self.concept.key == other.concept_key and self.source == other.source - if not isinstance(other, ConceptNode): return False @@ -255,7 +230,8 @@ class SourceCodeNode(LexerNode): Returned when some source code (like Python source code is recognized) """ - def __init__(self, start, end, tokens=None, source=None, python_node=None, return_value=None): + def __init__(self, start, end, tokens=None, source=None, + python_node=None, return_value=None, error_when_parsing=None): """ :param start: start position (index of the first token) @@ -269,18 +245,12 @@ class SourceCodeNode(LexerNode): You should have return_value.body.body == node """ super().__init__(start, end, tokens, source) + self.python_node = python_node # The PythonNode (or whatever language node) that is found self.return_value = return_value # original result of the parsing + self.error_when_parsing = error_when_parsing # if python_node is still None after parsing, it explains why def __eq__(self, other): - if isinstance(other, scnode): - return self.start == other.start and \ - self.end == other.end and \ - self.source == other.source - - if isinstance(other, SCN): - return other == self - if not isinstance(other, SourceCodeNode): return False @@ -336,6 +306,7 @@ class SourceCodeWithConceptNode(LexerNode): self.python_node = None # if the source code node is validated against a python parse, here is the PythonNode self.return_value = None # return_value that produced the PythonNode + self.error_when_parsing = None # if python_node is still None after parsing, it explains why def add_node(self, node): self.nodes.append(node) @@ -348,9 +319,6 @@ class SourceCodeWithConceptNode(LexerNode): if id(self) == id(other): return True - if isinstance(other, SCWC): - return other == self - if not isinstance(other, SourceCodeWithConceptNode): return False @@ -436,6 +404,44 @@ class SourceCodeWithConceptNode(LexerNode): return self.python_node.source +class VariableNode(LexerNode): + """ + When trying to parser source code, a reference to a variable is recognized + Not sure yet if it has to be a lexer node + """ + + def __init__(self, obj, prop, start, end, tokens=None, source=None): + super().__init__(start, end, tokens, source) + self.var_ref = VariableRef(obj, prop) + + def __eq__(self, other): + if id(self) == id(other): + return True + + if not isinstance(other, VariableNode): + return False + + return self.var_ref == other.var_ref and \ + self.start == other.start and \ + self.end == other.end and \ + self.source == other.source + + def __hash__(self): + return hash((self.var_ref.obj, self.var_ref.prop, self.start, self.end, self.source)) + + def __repr__(self): + ret = f"VariableNode(obj={self.var_ref.obj}, prop={self.var_ref.prop}, " + ret += f"start={self.start}, end={self.end}, source='{self.source}')" + return ret + + def to_short_str(self): + return f"VN({self.var_ref.obj})" if self.var_ref.prop is None else f"VN({self.var_ref.obj}.{self.var_ref.prop})" + + def clone(self): + clone = VariableNode(self.var_ref.obj, self.var_ref.prop, self.start, self.end, self.tokens, self.source) + return clone + + @dataclass() class GrammarErrorNode(ParsingError): message: str @@ -455,426 +461,6 @@ class SyaAssociativity(Enum): return self.value -cnode = namedtuple("ConceptNode", "concept_key start end source") -short_cnode = namedtuple("ConceptNode", "concept_key source") -utnode = namedtuple("utnode", "start end source") -scnode = namedtuple("scnode", "start end source") - - -class HelperWithPos: - def __init__(self, start=None, end=None): - self.start = start - self.end = end - - self.start_is_fixed = start is not None - self.end_is_fixed = end is not None - - def fix_pos(self, node): - if not self.start_is_fixed: - start = node.start if hasattr(node, "start") else \ - node[0] if isinstance(node, tuple) else None - - if start is not None and (self.start is None or start < self.start): - self.start = start - - if not self.end_is_fixed: - end = node.end if hasattr(node, "end") else \ - node[1] if isinstance(node, tuple) else None - - if end is not None and (self.end is None or end > self.end): - self.end = end - return self - - -class SCN(HelperWithPos): - """ - SourceCodeNode tester class - It matches with SourceCodeNode but with less constraints - - SCN == SourceCodeNode if source, start, end (start and end are not validated when None) - """ - - def __init__(self, source, start=None, end=None): - super().__init__(start, end) - self.source = source - - def __eq__(self, other): - if id(self) == id(other): - return True - - if isinstance(other, SourceCodeNode): - if self.source != other.source: - return False - if self.start is not None and self.start != other.start: - return False - if self.end is not None and self.end != other.end: - return False - - return True - - if not isinstance(other, CN): - return False - - return self.source == other.source and \ - self.start == other.start and \ - self.end == other.end - - def __hash__(self): - return hash((self.source, self.start, self.end)) - - def __repr__(self): - txt = f"SCN(source='{self.source}'" - if self.start is not None: - txt += f", start={self.start}" - if self.end is not None: - txt += f", end={self.end}" - return txt + ")" - - -class SCWC(HelperWithPos): - """ - SourceNodeWithConcept tester class - It matches with a SourceNodeWithConcept - but it's easier to instantiate during the tests - """ - - def __init__(self, first, last, *args): - super().__init__(None, None) - self.first = first - self.last = last - self.content = args - - def __eq__(self, other): - if id(self) == id(other): - return True - - if isinstance(other, SourceCodeWithConceptNode): - if self.first != other.first: - return False - - if self.last != other.last: - return False - - if len(self.content) != len(other.nodes): - return False - - for self_node, other_node in zip(self.content, other.nodes): - if self_node != other_node: - return False - - # at last - return True - - def __repr__(self): - txt = "SCWC(" - if self.start is not None: - txt += f"start={self.start}" - if self.end is not None: - txt += f", end={self.end}" - txt += f", source='{self.source}'" - return txt + ")" - - @property - def source(self): - """ - this code is a copy and paste from SourceCodeWithConceptNode.pseudo_fix_source - TODO: create a common function or whatever... - :return: - """ - source = self.first.source if hasattr(self.first, "source") else self.first - for n in self.content: - source += " " - if hasattr(n, "source"): - source += n.source - elif hasattr(n, "concept"): - source += str(n.concept) - else: - source += " unknown" - source += self.last.source if hasattr(self.last, "source") else self.last - return source - - -class CN(HelperWithPos): - """ - ConceptNode tester class - It matches with ConceptNode but with less constraints - - CN == ConceptNode if concept key, start, end and source are the same - """ - - def __init__(self, concept, start=None, end=None, source=None): - """ - - :param concept: Concept or concept_key (only the key is used anyway) - :param start: - :param end: - :param source: - """ - super().__init__(start, end) - self.concept_key = concept.key if isinstance(concept, Concept) else concept - self.source = source - self.concept = concept if isinstance(concept, Concept) else None - - def fix_source(self, str_tokens): - self.source = "".join(str_tokens) - return self - - def __eq__(self, other): - if id(self) == id(other): - return True - - if isinstance(other, ConceptNode): - if other.concept is None: - return False - if other.concept.key != self.concept_key: - return False - if self.start is not None and self.start != other.start: - return False - if self.end is not None and self.end != other.end: - return False - if self.source is not None and self.source != other.source: - return False - return True - - if not isinstance(other, CN): - return False - - return self.concept_key == other.concept_key and \ - self.start == other.start and \ - self.end == other.end and \ - self.source == other.source - - def __hash__(self): - return hash((self.concept_key, self.start, self.end, self.source)) - - def __repr__(self): - if self.concept: - txt = f"CN(concept='{self.concept}'" - else: - txt = f"CN(concept_key='{self.concept_key}'" - txt += f", source='{self.source}'" - if self.start is not None: - txt += f", start={self.start}" - if self.end is not None: - txt += f", end={self.end}" - return txt + ")" - - -class CNC(CN): - """ - ConceptNode for Compiled tester class - It matches with ConceptNode - But focuses on the 'compiled' property of the concept - - CNC == ConceptNode if CNC.get_compiled() == ConceptNode.concept.get_compiled() - """ - - def __init__(self, concept_key, start=None, end=None, source=None, exclude_body=False, **kwargs): - super().__init__(concept_key, start, end, source) - self.compiled = kwargs - self.exclude_body = exclude_body - if "body" in self.compiled: - self.compiled[ConceptParts.BODY] = self.compiled["body"] - del self.compiled["body"] - - def __eq__(self, other): - if id(self) == id(other): - return True - - if isinstance(other, ConceptNode): - if other.concept is None: - return False - if other.concept.key != self.concept_key: - return False - if self.start is not None and self.start != other.start: - return False - if self.end is not None and self.end != other.end: - return False - if self.source is not None and self.source != other.source: - return False - if self.exclude_body: - to_compare = {k: v for k, v in other.concept.get_compiled().items() if k != ConceptParts.BODY} - else: - to_compare = other.concept.get_compiled() - if self.compiled == to_compare: # expanded form to ease the debug - return True - else: - return False - - if not isinstance(other, CNC): - return False - - return self.concept_key == other.concept_key and \ - self.start == other.start and \ - self.end == other.end and \ - self.source == other.source and \ - self.compiled == other.compiled - - def __repr__(self): - if self.concept: - txt = f"CNC(concept='{self.concept}'" - else: - txt = f"CNC(concept_key='{self.concept_key}'" - txt += f", source='{self.source}'" - if self.start is not None: - txt += f", start={self.start}" - if self.end is not None: - txt += f", end={self.end}" - - for k, v in self.compiled.items(): - txt += f", {k}='{v}'" - return txt + ")" - - def to_compare(self, other, to_compare_delegate): - """ - Transform other into CNC, to ease the comparison - :param other: - :param to_compare_delegate: - :return: - """ - - if isinstance(other, CNC): - return other - - if isinstance(other, ConceptNode): - if self.exclude_body: - compiled = {k: v for k, v in other.concept.get_compiled().items() if k != ConceptParts.BODY} - else: - compiled = other.concept.get_compiled() - - self_compile_to_use = self.compiled or compiled - - compiled = to_compare_delegate(self_compile_to_use, compiled, to_compare_delegate) - return CNC(other.concept, - other.start if self.start is not None else None, - other.end if self.end is not None else None, - other.source if self.source is not None else None, - self.exclude_body, - **compiled) - - raise NotImplementedError("CNC") - - -class UTN(HelperWithPos): - """ - Tester class for UnrecognizedTokenNode - compare the source, and start, end if defined - """ - - def __init__(self, source, start=None, end=None): - """ - :param source: - :param start: - :param end: - """ - super().__init__(start, end) - self.source = source - - def __eq__(self, other): - if id(self) == id(other): - return True - - if isinstance(other, UnrecognizedTokensNode): - return self.start == other.start and \ - self.end == other.end and \ - self.source == other.source - - if not isinstance(other, UTN): - return False - - return self.start == other.start and \ - self.end == other.end and \ - self.source == other.source - - def __hash__(self): - return hash((self.source, self.start, self.end)) - - def __repr__(self): - txt = f"UTN(source='{self.source}'" - if self.start is not None: - txt += f", start={self.start}" - if self.end is not None: - txt += f", end={self.end}" - return txt + ")" - - def to_compare(self, other, to_compare_delegate): - """ - Transform other into CNC, to ease the comparison - :param other: - :param to_compare_delegate: - :return: - """ - - if isinstance(other, UTN): - return other - - if isinstance(other, UnrecognizedTokensNode): - return UTN(other.source, - other.start, - other.end) - - raise NotImplementedError("UTN") - - -class RN(HelperWithPos): - """ - Helper class to test RuleNode - """ - - def __init__(self, rule, start=None, end=None, source=None): - """ - - :param concept: Concept or concept_key (only the key is used anyway) - :param start: - :param end: - :param source: - """ - super().__init__(start, end) - self.rule_id = rule.id if isinstance(rule, Rule) else rule - self.source = source or core.utils.str_concept((None, self.rule_id), prefix="r:") - self.rule = rule if isinstance(rule, Rule) else None - - def __eq__(self, other): - if id(self) == id(other): - return True - - if isinstance(other, RuleNode): - if other.rule is None: - return False - if other.rule.id != self.rule_id: - return False - if self.start is not None and self.start != other.start: - return False - if self.end is not None and self.end != other.end: - return False - if self.source is not None and self.source != other.source: - return False - return True - - if not isinstance(other, RN): - return False - - return self.rule_id == other.rule_id and \ - self.start == other.start and \ - self.end == other.end and \ - self.source == other.source - - def __hash__(self): - return hash((self.rule_id, self.start, self.end, self.source)) - - def __repr__(self): - if self.rule: - txt = f"RN(rule='{self.rule}'" - else: - txt = f"RN(rule_id='{self.rule_id}'" - txt += f", source='{self.source}'" - if self.start is not None: - txt += f", start={self.start}" - if self.end is not None: - txt += f", end={self.end}" - return txt + ")" - - class BaseNodeParser(BaseParser): """ Parser that return LexerNode diff --git a/src/parsers/BaseParser.py b/src/parsers/BaseParser.py index 70e5abd..b429f6f 100644 --- a/src/parsers/BaseParser.py +++ b/src/parsers/BaseParser.py @@ -168,15 +168,22 @@ class BaseParser: if expected_parser and parser_input.parser != expected_parser: return None - if len(parser_input.value) == 0: - return None - - for node in parser_input.value: - from parsers.BaseNodeParser import LexerNode - if not isinstance(node, LexerNode): + from parsers.BaseNodeParser import LexerNode + if isinstance(parser_input.value, list): + if len(parser_input.value) == 0: return None - return parser_input.value + for node in parser_input.value: + if not isinstance(node, LexerNode): + return None + + return parser_input.value + + else: + if not isinstance(parser_input.value, LexerNode): + return None + + return [parser_input.value] @staticmethod def get_tokens_boundaries(tokens): diff --git a/src/parsers/FunctionParser.py b/src/parsers/FunctionParser.py index 2408633..9de73ee 100644 --- a/src/parsers/FunctionParser.py +++ b/src/parsers/FunctionParser.py @@ -53,59 +53,6 @@ class FunctionNode(FunctionParserNode): parameters: list -class FN(FunctionNode): - """ - Test class only - It matches with FunctionNode but with less constraints - - Thereby, - FN("first", "last", ["param1," ...]) can be compared to - FunctionNode(NameExprNode("first"), NameExprNode("second"), [FunctionParameter(NamesNodes("param1"), NamesNodes(", ")]) - - Note that FunctionParameter can easily be defined with a single string - * "param" -> FunctionParameter(NameExprNode("param"), None) - * "param, " -> FunctionParameter(NameExprNode("param"), NameExprNode(", ")) - For more complicated situations, you can use a tuple (value, sep) to define the value part and the separator part - """ - - def __init__(self, first, last, parameters): - self.first = first - self.last = last - self.parameters = [] - for param in parameters: - if isinstance(param, tuple): - self.parameters.append(param) - elif isinstance(param, str) and (pos := param.find(",")) != -1: - self.parameters.append((param[:pos], param[pos:])) - else: - self.parameters.append((param, None)) - - def __eq__(self, other): - if id(self) == id(other): - return True - - if isinstance(other, FN): - return self.first == other.first and self.last == other.last and self.parameters == other.parameters - - if isinstance(other, FunctionNode): - if self.first != other.first.value or self.last != other.last.value: - return False - if len(self.parameters) != len(other.parameters): - return False - for self_parameter, other_parameter in zip(self.parameters, other.parameters): - value = other_parameter.value.value if isinstance(self_parameter[0], str) else other_parameter.value - sep = other_parameter.separator.value if other_parameter.separator else None - if self_parameter[0] != value or self_parameter[1] != sep: - return False - - return True - - return False - - def __hash__(self): - return hash((self.first, self.last, self.parameters)) - - class FunctionParser(BaseParser): """ The parser will be used to parse func(x, y, z) @@ -126,6 +73,17 @@ class FunctionParser(BaseParser): self.longest_concepts_only = longest_concepts_only self.record_errors = True + def function_parser_get_return_value_body(self, source_code_node): + if source_code_node.error_when_parsing: + return self.sheerka.new(BuiltinConcepts.ERROR, + body=source_code_node.error_when_parsing) + + return self.sheerka.new(BuiltinConcepts.PARSER_RESULT, + parser=self, + source=self.parser_input.as_text(), + body=source_code_node, + try_parsed=source_code_node) + def add_error(self, error, next_token=True): if not self.record_errors: return @@ -166,6 +124,9 @@ class FunctionParser(BaseParser): [TokenKind.EOF])) if self.has_error: + if len(self.error_sink) == 1 and isinstance(self.error_sink[0], Concept): + return self.error_sink[0] + if node is None: body = context.sheerka.new(BuiltinConcepts.NOT_FOR_ME, body=parser_input.as_text(), @@ -178,12 +139,8 @@ class FunctionParser(BaseParser): res = [] for source_code_node in source_code_nodes: - value = self.get_return_value_body(context.sheerka, - self.parser_input.as_text(), - source_code_node, - source_code_node) - - res.append(self.sheerka.ret(self.name, source_code_node.python_node is not None, value)) + body = self.function_parser_get_return_value_body(source_code_node) + res.append(self.sheerka.ret(self.name, source_code_node.python_node is not None, body)) return res[0] if len(res) == 1 else res @@ -288,6 +245,25 @@ class FunctionParser(BaseParser): def to_source_code_node(self, function_node: FunctionNode): python_parser = PythonWithConceptsParser() + def update_source_code_node(scn, nodes, sep): + if hasattr(nodes, "__iter__"): + for n in nodes: + scn.add_node(n) + else: + scn.add_node(nodes) + + if sep: + scn.add_node(sep.to_unrecognized()) + + def get_errors_from_python_parsing(parsing_res): + if parsing_res.status: + return None + + if self.sheerka.isinstance(parsing_res.body, BuiltinConcepts.NOT_FOR_ME): + return parsing_res.body.reason + else: + return parsing_res.body.body + if len(function_node.parameters) == 0: # validate the source nodes_to_parse = [function_node.first.to_unrecognized(), function_node.last.to_unrecognized()] @@ -298,17 +274,8 @@ class FunctionParser(BaseParser): end=function_node.last.end, tokens=function_node.first.tokens + function_node.last.tokens, python_node=python_node, - return_value=python_parsing_res)] - - def update_source_code_node(scn, nodes, sep): - if hasattr(nodes, "__iter__"): - for n in nodes: - scn.add_node(n) - else: - scn.add_node(nodes) - - if sep: - scn.add_node(sep.to_unrecognized()) + return_value=python_parsing_res, + error_when_parsing=get_errors_from_python_parsing(python_parsing_res))] res = [SourceCodeWithConceptNode(function_node.first.to_unrecognized(), function_node.last.to_unrecognized())] @@ -363,6 +330,12 @@ class FunctionParser(BaseParser): for c in [c for c in source_code_node.python_node.objects.values() if isinstance(c, Concept)]: update_compiled(self.context, c, errors) + if errors: + source_code_node.error_when_parsing = errors + + else: + source_code_node.error_when_parsing = get_errors_from_python_parsing(python_parsing_res) + return res @staticmethod diff --git a/src/parsers/PythonWithConceptsParser.py b/src/parsers/PythonWithConceptsParser.py index 1dd86c7..222b269 100644 --- a/src/parsers/PythonWithConceptsParser.py +++ b/src/parsers/PythonWithConceptsParser.py @@ -1,6 +1,6 @@ from core.builtin_concepts import BuiltinConcepts from core.builtin_helpers import CreateObjectIdentifiers -from parsers.BaseNodeParser import ConceptNode, RuleNode +from parsers.BaseNodeParser import ConceptNode, RuleNode, VariableNode from parsers.BaseNodeParser import SourceCodeWithConceptNode from parsers.BaseParser import BaseParser from parsers.UnrecognizedNodeParser import UnrecognizedNodeParser @@ -64,6 +64,19 @@ class PythonWithConceptsParser(BaseParser): python_ids_mappings[python_id] = rule last_token_index = node.end + elif isinstance(node, VariableNode): + if node.start != last_token_index + 1 and source: # put back missing whitespace + source += " " + to_parse += " " + + source += node.source + var_ref = node.var_ref + python_id = ids_manager.get_identifier(var_ref, "__V__") + to_parse += python_id + python_ids_mappings[python_id] = var_ref + last_token_index = node.end + + else: source += node.source to_parse += node.get_source_to_parse() diff --git a/src/parsers/SyaNodeParser.py b/src/parsers/SyaNodeParser.py index 7f7aabe..d29d150 100644 --- a/src/parsers/SyaNodeParser.py +++ b/src/parsers/SyaNodeParser.py @@ -12,7 +12,7 @@ from core.sheerka.services.SheerkaExecute import ParserInput from core.tokenizer import Token, TokenKind, Tokenizer from core.utils import get_n_clones, get_text_from_tokens, NextIdManager from parsers.BaseNodeParser import UnrecognizedTokensNode, ConceptNode, SourceCodeNode, SyaAssociativity, \ - SourceCodeWithConceptNode, BaseNodeParser + SourceCodeWithConceptNode, BaseNodeParser, VariableNode from parsers.BaseParser import ParsingError PARSERS = ["Sequence", "Bnf", "Python"] @@ -1262,7 +1262,7 @@ class SyaNodeParser(BaseNodeParser): def postfix_to_item(self, sheerka, postfixed): item = postfixed.pop() - if isinstance(item, (UnrecognizedTokensNode, SourceCodeNode, ConceptNode)): + if isinstance(item, (UnrecognizedTokensNode, SourceCodeNode, ConceptNode, VariableNode)): return item if isinstance(item, SourceCodeWithConceptNode): diff --git a/src/sdp/sheerkaDataProvider.py b/src/sdp/sheerkaDataProvider.py index 8b93ddc..4bffa6c 100644 --- a/src/sdp/sheerkaDataProvider.py +++ b/src/sdp/sheerkaDataProvider.py @@ -235,6 +235,16 @@ class SheerkaDataProvider: def get_transaction(self, event) -> SheerkaDataProviderTransaction: return SheerkaDataProviderTransaction(self, event) + def get_ref(self, entry, key): + self.log.debug(f"getting object ref_id {entry=}, {key=}") + if entry not in self.state.data: + return NotFound + + if key not in self.state.data[entry]: + return NotFound + + return self.state.data[entry][key] + def get(self, entry, key=None, default=NotFound, load_origin=True): """ Get an element diff --git a/tests/core/test_SheerkaConceptManager.py b/tests/core/test_SheerkaConceptManager.py index 2e6faa2..d3da83e 100644 --- a/tests/core/test_SheerkaConceptManager.py +++ b/tests/core/test_SheerkaConceptManager.py @@ -487,6 +487,9 @@ class TestSheerkaConceptManager(TestUsingMemoryBasedSheerka): assert get_concept_attrs(foo) == ["b", "c"] assert get_concept_attrs(new_concept) == ["b", "c"] + new_foo = sheerka.new(foo) + assert get_concept_attrs(new_foo) == ["b", "c"] + def test_key_is_modified_when_modifying_name_or_variables(self): sheerka, context, foo = self.init_concepts(Concept("foo a b").def_var("a").def_var("b")) diff --git a/tests/core/test_SheerkaEvaluateConcept.py b/tests/core/test_SheerkaEvaluateConcept.py index 31071d3..c4b6b3e 100644 --- a/tests/core/test_SheerkaEvaluateConcept.py +++ b/tests/core/test_SheerkaEvaluateConcept.py @@ -1,7 +1,7 @@ import pytest from core.builtin_concepts import BuiltinConcepts, ReturnValueConcept, ParserResultConcept -from core.concept import Concept, DoNotResolve, ConceptParts, InfiniteRecursionResolved, CB, \ +from core.concept import Concept, DoNotResolve, ConceptParts, InfiniteRecursionResolved, \ concept_part_value, DEFINITION_TYPE_DEF from core.global_symbols import NotInit, NotFound from core.sheerka.services.SheerkaEvaluateConcept import SheerkaEvaluateConcept @@ -10,6 +10,7 @@ from parsers.BaseParser import BaseParser from parsers.PythonParser import PythonNode, PythonParser from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka from tests.evaluators.EvaluatorTestsUtils import pr_ret_val, python_ret_val +from tests.parsers.parsers_utils import CB, compare_with_test_object class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka): @@ -82,7 +83,7 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka): ("True", True), ("1 > 2", False), ]) - def test_i_can_evaluate_a_concept_with_prop(self, expr, expected): + def test_i_can_evaluate_a_concept_with_variable(self, expr, expected): sheerka, context, concept = self.init_concepts(Concept("foo").def_var("a", expr), eval_body=True) evaluated = sheerka.evaluate_concept(context, concept) @@ -141,7 +142,7 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka): evaluated = sheerka.evaluate_concept(context, concept) - assert evaluated == CB(concept, 11) + compare_with_test_object(evaluated, CB(concept, 11)) def test_i_can_evaluate_when_another_concept_is_referenced(self): sheerka, context, concept_a, concept = self.init_concepts( @@ -151,7 +152,7 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka): evaluated = sheerka.evaluate_concept(context, concept) - assert evaluated == CB("foo", CB("a", NotInit)) + compare_with_test_object(evaluated, CB("foo", CB("a", NotInit))) assert evaluated.get_metadata().is_evaluated assert evaluated.body.get_metadata().is_evaluated @@ -164,7 +165,7 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka): evaluated = sheerka.evaluate_concept(context, concept) assert evaluated.key == concept.key - assert evaluated.body == CB("a", 1) + compare_with_test_object(evaluated.body, CB("a", 1)) assert not concept_a.get_metadata().is_evaluated assert evaluated.get_metadata().is_evaluated @@ -180,7 +181,7 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka): assert evaluated.key == concept_d.key expected = CB("c", CB("b", CB("a", "a"))) - assert evaluated.body == expected + compare_with_test_object(evaluated.body, expected) assert sheerka.objvalue(evaluated) == 'a' assert evaluated.get_metadata().is_evaluated @@ -196,8 +197,8 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka): assert evaluated.key == concept_d.key expected = CB("c", CB("b", CB("a", NotInit))) - assert evaluated.body == expected - assert sheerka.objvalue(evaluated) == CB("a", NotInit) + compare_with_test_object(evaluated.body, expected) + compare_with_test_object(sheerka.objvalue(evaluated), CB("a", NotInit)) assert evaluated.get_metadata().is_evaluated def test_i_can_evaluate_concept_when_variables_reference_others_concepts_1(self): @@ -283,7 +284,7 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka): evaluated = sheerka.evaluate_concept(context, concept) assert evaluated.key == concept.key - assert evaluated.get_value("a") == CB("a", "a") + compare_with_test_object(evaluated.get_value("a"), CB("a", "a")) def test_i_can_evaluate_concept_when_variable_is_a_concept_token(self): sheerka, context, concept_a, concept_b = self.init_concepts( @@ -317,7 +318,7 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka): variables = evaluated.get_value("prop") assert len(variables) == 2 - assert variables[0] == CB("foo", 1) + compare_with_test_object(variables[0], CB("foo", 1)) assert variables[1] == "1" def test_i_can_evaluate_when_compiled_is_set_up_with_return_value(self): @@ -419,7 +420,7 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka): concept = Concept("foo", body="a") evaluated = sheerka.evaluate_concept(context, concept) assert evaluated.key == concept.key - assert evaluated.body == CB("a", "concept_a") # this test was already done + compare_with_test_object(evaluated.body, CB("a", "concept_a")) # this test was already done # so check this one. concept = Concept("foo", body="a").def_var("a", "'property_a'") @@ -431,7 +432,7 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka): concept = Concept("foo", body="a").def_var("a", "b") evaluated = sheerka.evaluate_concept(context, concept) assert evaluated.key == concept.key - assert evaluated.body == CB("b", "concept_b") + compare_with_test_object(evaluated.body, CB("b", "concept_b")) def test_variables_values_takes_precedence(self): sheerka, context, concept_a, concept_b = self.init_concepts( @@ -453,7 +454,7 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka): concept = Concept("foo", body="a.subProp").def_var("a", "concept_a") evaluated = sheerka.evaluate_concept(context, concept) - assert evaluated == CB(concept.key, "sub_a") + compare_with_test_object(evaluated, CB(concept.key, "sub_a")) def test_i_cannot_evaluate_concept_if_variable_is_in_error(self): sheerka = self.get_sheerka() diff --git a/tests/core/test_SheerkaIsAManager.py b/tests/core/test_SheerkaIsAManager.py index 679e727..0b4da52 100644 --- a/tests/core/test_SheerkaIsAManager.py +++ b/tests/core/test_SheerkaIsAManager.py @@ -120,7 +120,10 @@ class TestSheerkaIsAManager(TestUsingMemoryBasedSheerka): service.add_concepts_to_set(context, [one, two, three, four, five], number) assert sheerka.isaset(context, sub_number) - assert set(sheerka.get_set_elements(context, sub_number)) == {one, two, three} + # compare ids, as concepts are evaluated in get_set_elements + actual_ids = set(c.id for c in sheerka.get_set_elements(context, sub_number)) + expected_ids = set(c.id for c in [one, two, three]) + assert actual_ids == expected_ids def test_i_can_define_subset_of_subset(self): sheerka, context, one, two, three, four, five, number, sub_number, sub_sub_number = self.init_concepts( @@ -137,7 +140,11 @@ class TestSheerkaIsAManager(TestUsingMemoryBasedSheerka): service.add_concepts_to_set(context, [one, two, three, four, five], number) assert sheerka.isaset(context, sub_sub_number) - assert set(sheerka.get_set_elements(context, sub_sub_number)) == {three} + + # compare ids, as concepts are evaluated in get_set_elements + actual_ids = set(c.id for c in sheerka.get_set_elements(context, sub_sub_number)) + expected_ids = set(c.id for c in [three]) + assert actual_ids == expected_ids def test_i_can_define_subset_of_another_set_when_some_concept_do_not_have_a_defined_body(self): """ @@ -161,7 +168,10 @@ class TestSheerkaIsAManager(TestUsingMemoryBasedSheerka): service.add_concepts_to_set(context, [one, two, three, four, five], number) assert sheerka.isaset(context, sub_number) - assert set(sheerka.get_set_elements(context, sub_number)) == {one, three} + # compare ids, as concepts are evaluated in get_set_elements + actual_ids = set(c.id for c in sheerka.get_set_elements(context, sub_number)) + expected_ids = set(c.id for c in [one, three]) + assert actual_ids == expected_ids def test_i_can_define_subset_of_another_set_when_some_concept_are_bnf(self): """ @@ -183,7 +193,10 @@ class TestSheerkaIsAManager(TestUsingMemoryBasedSheerka): service.add_concepts_to_set(context, [one, two, twenty, twenties], number) assert sheerka.isaset(context, sub_number) - assert sheerka.get_set_elements(context, sub_number) == [twenty] # what is expected ? + # compare ids, as concepts are evaluated in get_set_elements + actual_ids = set(c.id for c in sheerka.get_set_elements(context, sub_number)) + expected_ids = set(c.id for c in [twenty]) + assert actual_ids == expected_ids def test_bnf_elements_can_be_part_of_a_set(self): """ diff --git a/tests/core/test_SheerkaMemory.py b/tests/core/test_SheerkaMemory.py index d386552..dcd4273 100644 --- a/tests/core/test_SheerkaMemory.py +++ b/tests/core/test_SheerkaMemory.py @@ -1,4 +1,5 @@ from core.builtin_concepts import BuiltinConcepts +from core.builtin_helpers import ensure_evaluated from core.concept import Concept from core.global_symbols import NotFound from core.sheerka.ExecutionContext import ExecutionContext @@ -6,6 +7,7 @@ from core.sheerka.services.SheerkaMemory import SheerkaMemory, MemoryObject from tests.TestUsingFileBasedSheerka import TestUsingFileBasedSheerka from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka +from tests.parsers.parsers_utils import CV, compare_with_test_object, CB class TestSheerkaMemory(TestUsingMemoryBasedSheerka): @@ -31,7 +33,7 @@ class TestSheerkaMemory(TestUsingMemoryBasedSheerka): assert id(sheerka.get_from_short_term_memory(context, "a")) == id(foo) assert sheerka.get_from_short_term_memory(None, "a") is NotFound - def test_i_can_add_many(self): + def test_i_can_add_many_to_short_term_memory(self): sheerka, context = self.init_test().unpack() bag = {"a": "foo", "b": "bar", } context_id = ExecutionContext.ids[context.event.get_digest()] @@ -191,6 +193,27 @@ class TestSheerkaMemory(TestUsingMemoryBasedSheerka): assert from_memory[-1].obj == from_memory_again[-1].obj assert from_memory[-1].event_id != from_memory_again[-1].event_id + def test_object_values_are_recorded(self): + sheerka, context, foo, prop, value, inner_value = self.init_test(cache_only=False).with_concepts( + Concept("foo").def_var("x"), + Concept("prop"), + Concept("value x").def_var("x"), + Concept("one", body="1") + ).unpack() + + instantiated_foo = sheerka.new(foo, x="value1") + instantiated_one = ensure_evaluated(context, sheerka.new(inner_value)) + instantiated_value = sheerka.new(value, x=instantiated_one) + sheerka.set_attr(instantiated_foo, prop, instantiated_value) + + sheerka.add_to_memory(context, "foo", instantiated_foo) + sheerka.om.commit(context) + + from_db = sheerka.om.current_sdp().get(SheerkaMemory.OBJECTS_ENTRY, "foo") + assert isinstance(from_db, MemoryObject) + assert sheerka.isinstance(from_db.obj, foo) + compare_with_test_object(sheerka.get_attr(from_db.obj, prop), CV(value, x=CB(inner_value, 1))) + class TestSheerkaMemoryUsingFileBase(TestUsingFileBasedSheerka): def test_i_can_record_memory_objects(self): diff --git a/tests/core/test_SheerkaRuleManager.py b/tests/core/test_SheerkaRuleManager.py index 62fd68e..5c508b4 100644 --- a/tests/core/test_SheerkaRuleManager.py +++ b/tests/core/test_SheerkaRuleManager.py @@ -3,7 +3,7 @@ import ast import pytest from core.builtin_concepts import BuiltinConcepts -from core.concept import Concept, CMV, DEFINITION_TYPE_DEF, CC, DoNotResolve +from core.concept import Concept, DEFINITION_TYPE_DEF, DoNotResolve from core.global_symbols import RULE_COMPARISON_CONTEXT, NotFound, EVENT_RULE_DELETED from core.rule import Rule, ACTION_TYPE_PRINT, ACTION_TYPE_EXEC from core.sheerka.Sheerka import RECOGNIZED_BY_ID, RECOGNIZED_BY_NAME @@ -20,6 +20,7 @@ from sheerkarete.common import V from sheerkarete.conditions import Condition, AndConditions from tests.TestUsingFileBasedSheerka import TestUsingFileBasedSheerka from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka +from tests.parsers.parsers_utils import CMV, CC, compare_with_test_object, get_test_obj seq = FormatAstSequence raw = FormatAstRawText @@ -494,8 +495,8 @@ class TestSheerkaRuleManager(TestUsingMemoryBasedSheerka): assert isinstance(res[0], RuleCompiledPredicate) assert res[0].evaluator == CONCEPT_EVALUATOR_NAME assert sheerka.isinstance(res[0].predicate, BuiltinConcepts.RETURN_VALUE) - assert sheerka.objvalue(res[0].predicate)[0].concept == expected - assert res[0].concept == expected + compare_with_test_object(sheerka.objvalue(res[0].predicate)[0].concept, expected) + compare_with_test_object(res[0].concept, expected) def test_i_can_compile_predicate_when_bnf_node_parser(self): sheerka, context, *concepts = self.init_test().with_concepts( @@ -540,11 +541,13 @@ class TestSheerkaRuleManager(TestUsingMemoryBasedSheerka): assert isinstance(res[0], RuleCompiledPredicate) python_node = res[0].predicate.body.body assert python_node == expected_python_node - assert python_node.objects == { + expected = { "__C__00var0000is0a000var001__1005__C__": CC(is_a, x=cat, y=pet), "__C__00var0000is0an0y__1006__C__": CC(is_an, exclude_body=True, x=DoNotResolve("bird"), animal=animal), "__C__00var0000is0a000var001__1005_1__C__": CMV(is_a, x="dog", y="pet"), } + transformed = get_test_obj(python_node.objects, expected) + assert transformed == expected @pytest.mark.parametrize("text", [ "a and not b", @@ -595,11 +598,11 @@ class TestSheerkaRuleManager(TestUsingMemoryBasedSheerka): assert isinstance(res[0], RuleCompiledPredicate) python_node = res[0].predicate.body.body assert python_node == expected_python_node - assert python_node.objects == { + compare_with_test_object(python_node.objects, { "__C__00var0000is0a000var001__1005__C__": CC(is_a, x=cat, y=pet), "__C__00var0000is0an0y__1006__C__": CC(is_an, exclude_body=True, x=DoNotResolve("bird"), animal=animal), "__C__00var0000is0a000var001__1005_1__C__": CMV(is_a, x="dog", y="pet"), - } + }) def test_i_can_compile_predicate_when_multiple_choices(self): sheerka, context, *concepts = self.init_test().with_concepts( @@ -616,14 +619,14 @@ class TestSheerkaRuleManager(TestUsingMemoryBasedSheerka): assert isinstance(res[0], RuleCompiledPredicate) assert res[0].evaluator == CONCEPT_EVALUATOR_NAME assert sheerka.isinstance(res[0].predicate, BuiltinConcepts.RETURN_VALUE) - assert sheerka.objvalue(res[0].predicate)[0].concept == CMV(concepts[0], x="a", y="b") - assert res[0].concept == CMV(concepts[0], x="a", y="b") + compare_with_test_object(sheerka.objvalue(res[0].predicate)[0].concept, CMV(concepts[0], x="a", y="b")) + compare_with_test_object(res[0].concept, CMV(concepts[0], x="a", y="b")) assert isinstance(res[1], RuleCompiledPredicate) assert res[1].evaluator == CONCEPT_EVALUATOR_NAME assert sheerka.isinstance(res[1].predicate, BuiltinConcepts.RETURN_VALUE) - assert sheerka.objvalue(res[1].predicate)[0].concept == CMV(concepts[1], x="a", y="b") - assert res[1].concept == CMV(concepts[1], x="a", y="b") + compare_with_test_object(sheerka.objvalue(res[1].predicate)[0].concept, CMV(concepts[1], x="a", y="b")) + compare_with_test_object(res[1].concept, CMV(concepts[1], x="a", y="b")) def test_i_can_compile_predicate_when_mix_and_multiple_choices(self): sheerka, context, *concepts = self.init_test().with_concepts( diff --git a/tests/core/test_concept.py b/tests/core/test_concept.py index b8334ff..7655221 100644 --- a/tests/core/test_concept.py +++ b/tests/core/test_concept.py @@ -1,7 +1,9 @@ import pytest -from core.builtin_concepts import BuiltinConcepts -from core.concept import Concept, ConceptParts, DEFINITION_TYPE_DEF, ALL_ATTRIBUTES, get_concept_attrs +from core.builtin_concepts import BuiltinConcepts +from core.concept import Concept, ConceptParts, DEFINITION_TYPE_DEF, ALL_ATTRIBUTES, get_concept_attrs, \ + freeze_concept_attrs +from core.global_symbols import NotInit @pytest.mark.parametrize("name, variables, expected", [ @@ -121,6 +123,7 @@ def test_i_can_deserialize(): } concept = Concept().from_dict(from_dict) + freeze_concept_attrs(concept) assert concept == Concept( name="concept_name", @@ -281,16 +284,21 @@ def test_i_can_manage_concepts_attributes(): assert concept.values() == {"#body#": "I have a body!"} -def test_attributes_are_generated_once_for_all(): +def test_i_can_manage_instance_attributes(): ALL_ATTRIBUTES.clear() - concept = Concept("foo") - concept.get_metadata().id = "id" - concept.set_value("key1", "value1") - concept.set_value("key2", "value2") - assert get_concept_attrs(concept) == ["key1", "key2"] - assert concept.values() == {"key1": "value1", "key2": "value2"} + foo = Concept("foo", id="foo_id").def_var("x") - concept.set_value("key3", "value3") # too late for it ! - assert get_concept_attrs(concept) == ["key1", "key2"] - assert concept.values() == {"key1": "value1", "key2": "value2"} + assert foo.values() == {"x": NotInit} + assert foo.get_all_attributes() is None + assert ALL_ATTRIBUTES == {"foo_id": ["x"]} + + foo.set_value("x", "value for x") + assert foo.values() == {"x": "value for x"} + assert foo.get_all_attributes() is None + assert ALL_ATTRIBUTES == {"foo_id": ["x"]} + + foo.set_value("y", "value for y") + assert foo.values() == {"x": "value for x", "y": "value for y"} + assert foo.get_all_attributes() == ["x", "y"] + assert ALL_ATTRIBUTES == {"foo_id": ["x"]} diff --git a/tests/evaluators/test_PythonEvaluator.py b/tests/evaluators/test_PythonEvaluator.py index 498aa82..2bebeaa 100644 --- a/tests/evaluators/test_PythonEvaluator.py +++ b/tests/evaluators/test_PythonEvaluator.py @@ -4,7 +4,7 @@ import pytest from core.builtin_concepts import ReturnValueConcept, ParserResultConcept, BuiltinConcepts from core.builtin_helpers import CreateObjectIdentifiers -from core.concept import Concept, CB +from core.concept import Concept from core.sheerka.services.SheerkaConceptManager import SheerkaConceptManager from core.sheerka.services.SheerkaExecute import ParserInput from core.tokenizer import Tokenizer @@ -13,6 +13,7 @@ from parsers.BaseNodeParser import SourceCodeNode, SourceCodeWithConceptNode from parsers.FunctionParser import FunctionParser from parsers.PythonParser import PythonNode, PythonParser from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka +from tests.parsers.parsers_utils import CB, compare_with_test_object def get_obj_name(obj): @@ -178,7 +179,7 @@ class TestPythonEvaluator(TestUsingMemoryBasedSheerka): evaluated = PythonEvaluator().eval(context, parsed) assert evaluated.status - assert evaluated.value == CB("foo", 2) + compare_with_test_object(evaluated.value, CB("foo", 2)) def test_i_can_eval_concept_token(self): context = self.get_context() @@ -253,7 +254,7 @@ class TestPythonEvaluator(TestUsingMemoryBasedSheerka): all_globals = python_evaluator.get_all_possible_globals(context, my_globals) assert len(all_globals) == 2 - assert all_globals[0]["foo"] == CB(foo, "foo") + compare_with_test_object(all_globals[0]["foo"], CB(foo, "foo")) assert all_globals[1]["foo"] == 'foo' # body is evaluated def test_i_can_detect_one_error(self): @@ -284,7 +285,7 @@ class TestPythonEvaluator(TestUsingMemoryBasedSheerka): assert isinstance(error0, PythonEvalError) assert isinstance(error0.error, TypeError) assert error0.error.args[0] == "unsupported operand type(s) for +: 'Concept' and 'int'" - assert error0.concepts == {'foo': CB(foo, 'string')} + compare_with_test_object(error0.concepts, {'foo': CB(foo, 'string')}) error1 = evaluated.body.body[1] assert isinstance(error1, PythonEvalError) diff --git a/tests/non_reg/test_sheerka_non_reg.py b/tests/non_reg/test_sheerka_non_reg.py index bced11b..bcdbb30 100644 --- a/tests/non_reg/test_sheerka_non_reg.py +++ b/tests/non_reg/test_sheerka_non_reg.py @@ -1,7 +1,7 @@ import pytest from core.builtin_concepts import BuiltinConcepts -from core.concept import Concept, PROPERTIES_TO_SERIALIZE, simplec, CMV, CC +from core.concept import Concept, PROPERTIES_TO_SERIALIZE from core.global_symbols import NotInit from core.sheerka.services.SheerkaConceptManager import SheerkaConceptManager from evaluators.MutipleSameSuccessEvaluator import MultipleSameSuccessEvaluator @@ -10,6 +10,7 @@ from evaluators.PythonEvaluator import PythonEvalError from parsers.BnfNodeParser import Sequence, StrMatch, OrderedChoice, Optional, ConceptExpression from tests.TestUsingFileBasedSheerka import TestUsingFileBasedSheerka from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka +from tests.parsers.parsers_utils import CMV, CC, compare_with_test_object, CB class TestSheerkaNonRegMemory(TestUsingMemoryBasedSheerka): @@ -40,7 +41,7 @@ class TestSheerkaNonRegMemory(TestUsingMemoryBasedSheerka): # sanity check evaluated = sheerka.evaluate_concept(self.get_context(eval_body=True), res[0].value) - assert evaluated == simplec("one", 1) + compare_with_test_object(evaluated, CB("one", 1)) def test_i_can_recognize_concept_with_concept_body(self): sheerka, context, concept_one, concept_un = self.init_concepts("one", Concept(name="un", body="one")) @@ -53,7 +54,7 @@ class TestSheerkaNonRegMemory(TestUsingMemoryBasedSheerka): # sanity check evaluated = sheerka.evaluate_concept(self.get_context(eval_body=True), return_value) - assert evaluated == simplec("un", simplec("one", NotInit)) + compare_with_test_object(evaluated, CB("un", CB("one", NotInit))) def test_i_can_recognize_concept_with_no_body(self): sheerka = self.get_sheerka() @@ -212,7 +213,7 @@ as: evaluated = sheerka.evaluate_concept(self.get_context(eval_body=True), res[0].value) assert evaluated.body == "hello foo" assert evaluated.get_metadata().is_evaluated - assert evaluated.get_value("a") == simplec("foo", "foo") + compare_with_test_object(evaluated.get_value("a"), CB("foo", "foo")) assert evaluated.get_value("a").get_metadata().is_evaluated def test_i_can_recognize_duplicate_concepts_with_same_value(self): @@ -889,7 +890,7 @@ as: expression = "c:one: < c:two:" res = sheerka.evaluate_user_input(expression) assert res[0].status - assert res[0].body == CMV(is_less_than, a="c:one:", b="c:two:") + compare_with_test_object(res[0].body, CMV(is_less_than, a="c:one:", b="c:two:")) assert res[0].body.a == NotInit # concept is not evaluated assert res[0].body.b == NotInit # concept is not evaluated @@ -985,7 +986,7 @@ as: assert isinstance(plus, Concept) assert plus.name == "plus" assert plus.get_compiled()["a"] == sheerka.new("one") - assert plus.get_compiled()["b"] == CC(the, a=sheerka.new("one")) + compare_with_test_object(plus.get_compiled()["b"], CC(the, a=sheerka.new("one"))) res = sheerka.evaluate_user_input("eval one plus the one") assert res[0].status @@ -1249,6 +1250,55 @@ as: assert res[0].status assert sheerka.isinstance(res[0].body, "binary") + def test_parsing_expression_are_dynamic(self): + init = [ + "def concept two", + "def concept number", + "def concept nb times from bnf number 'times'", + "set_isa(two, number)", + ] + sheerka = self.init_scenario(init) + + res = sheerka.evaluate_user_input("two times") + + assert len(res) == 1 + assert res[0].status + assert sheerka.isinstance(res[0].body, "nb times") + + def test_i_can_evaluate_when_body_is_a_function_that_references_inner_variables(self): + init = [ + "def concept two as 2", + "def concept number", + "set_isa(two, number)", + "def concept cars", + "def concept quantify x from bnf number x as set_attr(x, 'qty', number) ret x" + ] + sheerka = self.init_scenario(init) + + res = sheerka.evaluate_user_input("eval two cars") + + assert len(res) == 1 + assert res[0].status + + assert sheerka.objvalue(res[0].body.get_value("qty")) == 2 + + def test_i_can_evaluate_when_body_is_a_function_that_references_inner_variables_using_alias(self): + init = [ + "def concept two as 2", + "def concept number", + "set_isa(two, number)", + "def concept cars", + "def concept quantify x from bnf number=n1 x as set_attr(x, 'qty', n1) ret x" + ] + sheerka = self.init_scenario(init) + + res = sheerka.evaluate_user_input("eval two cars") + + assert len(res) == 1 + assert res[0].status + + assert sheerka.objvalue(res[0].body.get_value("qty")) == 2 + class TestSheerkaNonRegFile(TestUsingFileBasedSheerka): def test_i_can_def_several_concepts(self): diff --git a/tests/parsers/parsers_utils.py b/tests/parsers/parsers_utils.py index 5790d0b..f441bae 100644 --- a/tests/parsers/parsers_utils.py +++ b/tests/parsers/parsers_utils.py @@ -1,13 +1,16 @@ import ast from dataclasses import dataclass +from typing import Union from core.builtin_concepts import ReturnValueConcept from core.builtin_helpers import CreateObjectIdentifiers -from core.concept import CC, Concept, ConceptParts, DoNotResolve, CIO, CMV +from core.concept import Concept, ConceptParts, DoNotResolve, AllConceptParts +from core.rule import Rule from core.tokenizer import Tokenizer, TokenKind, Token -from core.utils import get_text_from_tokens, tokens_index -from parsers.BaseNodeParser import scnode, utnode, cnode, SCWC, CNC, short_cnode, CN, UTN, \ - SCN, RN, UnrecognizedTokensNode, SourceCodeNode +from core.utils import get_text_from_tokens, tokens_index, str_concept +from parsers.BaseNodeParser import UnrecognizedTokensNode, SourceCodeNode, RuleNode, ConceptNode, \ + SourceCodeWithConceptNode +from parsers.FunctionParser import FunctionNode from parsers.PythonParser import PythonNode from parsers.SyaNodeParser import SyaConceptParserHelper from parsers.expressions import NameExprNode, AndNode, OrNode, NotNode, VariableNode, ComparisonNode, ComparisonType @@ -115,6 +118,825 @@ class NIN: # for NOT INT source = None +class CC: + """ + Concept class for test purpose + CC means concept for compiled (or concept with compiled) + It matches a concept if the compiles are equals + """ + + # The only properties that are testes are concept_key and compiled + # The other properties (concept, source, start and end) + # are used in tests/parsers/parsers_utils.py to help creating helper objects + + def __init__(self, concept, source=None, exclude_body=False, **kwargs): + self.concept_key = concept.key if isinstance(concept, Concept) else concept + self.compiled = kwargs + self.concept = concept if isinstance(concept, Concept) else None + self.source = source # to use when the key is different from the sub str to search when filling start and stop + self.start = None # for debug purpose, indicate where the concept starts + self.end = None # for debug purpose, indicate where the concept ends + self.exclude_body = exclude_body + + if "body" in self.compiled: + self.compiled[ConceptParts.BODY] = self.compiled["body"] + del self.compiled["body"] + + def __eq__(self, other): + if id(self) == id(other): + return True + + if isinstance(other, Concept): + if other.key != self.concept_key: + return False + if self.exclude_body: + to_compare = {k: v for k, v in other.get_compiled().items() if k != ConceptParts.BODY} + else: + to_compare = other.get_compiled() + if self.compiled == to_compare: + return True + else: + return False + + if not isinstance(other, CC): + return False + + if self.concept_key != other.concept_key: + return False + + return self.compiled == other.compiled + + def __hash__(self): + if self.concept: + return hash(self.concept) + return hash(self.concept_key) + + def __repr__(self): + if self.concept: + txt = f"CC(concept='{self.concept}'" + else: + txt = f"CC(concept_key='{self.concept_key}'" + + for k, v in self.compiled.items(): + txt += f", {k}='{v}'" + return txt + ")" + + def fix_pos(self, node): + start = node.start if hasattr(node, "start") else \ + node[0] if isinstance(node, tuple) else None + end = node.end if hasattr(node, "end") else \ + node[1] if isinstance(node, tuple) else None + + if start is not None: + if self.start is None or start < self.start: + self.start = start + + if end is not None: + if self.end is None or end > self.end: + self.end = end + return self + + def transform_real_obj(self, other, to_compare_delegate): + """ + Transform other into CNC, to ease the comparison + :param other: + :param to_compare_delegate: + :return: + """ + + if isinstance(other, CC): + return other + + if isinstance(other, Concept): + if self.exclude_body: + compiled = {k: v for k, v in other.get_compiled().items() if k != ConceptParts.BODY} + else: + compiled = other.get_compiled() + + self_compile_to_use = self.compiled or compiled + + compiled = to_compare_delegate(self_compile_to_use, compiled, to_compare_delegate) + return CC(other, + self.source, + self.exclude_body, + **compiled) + + raise NotImplementedError(f"CC, {other=}") + + +class CB: + """ + Concept with body only + Test class that tests only the body of the concept + """ + + def __init__(self, concept: Union[str, Concept], body: object): + self.concept_key = concept.key if isinstance(concept, Concept) else concept + self.concept = concept if isinstance(concept, Concept) else None + self.body = body + + def __eq__(self, other): + if not isinstance(other, CB): + return False + + return self.concept_key == other.concept_key and self.body == other.body + + def __hash__(self): + return hash((self.concept, self.body)) + + def __repr__(self): + concept_debug = f"concept={self.concept}" if self.concept else f"concept_key={self.concept_key}" + return f"CB({concept_debug}, body='{self.body}')" + + def transform_real_obj(self, other, get_test_obj_delegate): + if isinstance(other, CB): + return other + + if isinstance(other, Concept): + concept = other.key if not self.concept else other + if isinstance(other.body, Concept): + body = get_test_obj_delegate(other.body, self.body, get_test_obj_delegate) + else: + body = other.body + return CB(concept, body) + + raise NotImplementedError(f"CB, {other=}") + + +class CV: + """ + Concept with all values + Test class that tests all the values (not the metadata, so not the properties) of a concept + """ + + def __init__(self, concept, **kwargs): + self.concept_key = concept.key if isinstance(concept, Concept) else concept + self.concept = concept if isinstance(concept, Concept) else None + self.values = {} + for k, v in kwargs.items(): + if f"#{k}#" in AllConceptParts: + self.values[f"#{k}#"] = v + else: + self.values[k] = v + + def __eq__(self, other): + if not isinstance(other, CV): + return False + + return self.concept_key == other.concept_key and self.values == other.values + + def __hash__(self): + return hash((self.concept_key, self.values)) + + def __repr__(self): + concept_debug = f"concept={self.concept}" if self.concept else f"concept_key={self.concept_key}" + return f"CV({concept_debug}, values={self.values})" + + def transform_real_obj(self, other, get_test_obj_delegate): + if isinstance(other, CV): + return other + + if isinstance(other, Concept): + concept = other.key if not self.concept else other + values = get_test_obj_delegate(other.values(), self.values, get_test_obj_delegate) + return CV(concept, **values) + + raise NotImplementedError(f"CV, {other=}") + + +class CMV: + """ + Concept with metadata variables + CMV stands for Concept Metadata Variables + Test class that only compare the key and the metadata variables + """ + + def __init__(self, concept, **kwargs): + self.concept_key = concept.key if isinstance(concept, Concept) else concept + self.concept = concept if isinstance(concept, Concept) else None + self.variables = kwargs + + def __eq__(self, other): + if id(self) == id(other): + return True + + if not isinstance(other, CMV): + return False + + if self.concept_key != other.concept_key: + return False + + return self.variables == other.variables + + def __hash__(self): + if self.concept: + return hash(self.concept) + return hash(self.concept_key) + + def __repr__(self): + if self.concept: + txt = f"CMV(concept='{self.concept}'" + else: + txt = f"CMV(concept_key='{self.concept_key}'" + + for k, v in self.variables.items(): + txt += f", {k}='{v}'" + return txt + ")" + + def transform_real_obj(self, other, get_test_obj_delegate): + if isinstance(other, CMV): + return other + + if isinstance(other, Concept): + concept = other.key if not self.concept else other + variables = {name: value for name, value in other.get_metadata().variables} + return CMV(concept, **variables) + + raise NotImplementedError(f"CMV, {other=}") + + +class CIO: + """ + Concept id only + only test the id + """ + + def __init__(self, concept, source=None): + if isinstance(concept, str): + self.concept_name = concept + self.concept_id = None + self.concept = None + elif isinstance(concept, Concept): + self.concept_id = concept.id + self.concept = concept + self.source = source + self.start = None + self.end = None + + def set_concept(self, concept): + self.concept = concept + self.concept_id = concept.id + + def __eq__(self, other): + if id(self) == id(other): + return True + + if not isinstance(other, CIO): + return False + + return self.concept_id == other.concept_id + + def __hash__(self): + return hash(self.concept_id) + + def __repr__(self): + return f"CIO(concept='{self.concept}')" if self.concept else f"CIO(name='{self.concept_name}')" + + def transform_real_obj(self, other, get_test_obj_delegate): + if isinstance(other, CIO): + return other + + if isinstance(other, Concept): + return CIO(other) + + raise NotImplementedError(f"CIO, {other=}") + + +class HelperWithPos: + def __init__(self, start=None, end=None): + self.start = start + self.end = end + + self.start_is_fixed = start is not None + self.end_is_fixed = end is not None + + def fix_pos(self, node): + if not self.start_is_fixed: + start = node.start if hasattr(node, "start") else \ + node[0] if isinstance(node, tuple) else None + + if start is not None and (self.start is None or start < self.start): + self.start = start + + if not self.end_is_fixed: + end = node.end if hasattr(node, "end") else \ + node[1] if isinstance(node, tuple) else None + + if end is not None and (self.end is None or end > self.end): + self.end = end + return self + + +class SCN(HelperWithPos): + """ + SourceCodeNode tester class + It matches with SourceCodeNode but with less constraints + + SCN == SourceCodeNode if source, start, end (start and end are not validated when None) + """ + + def __init__(self, source, start=None, end=None): + super().__init__(start, end) + self.source = source + + def __eq__(self, other): + if id(self) == id(other): + return True + + if isinstance(other, SourceCodeNode): + if self.source != other.source: + return False + if self.start is not None and self.start != other.start: + return False + if self.end is not None and self.end != other.end: + return False + + return True + + if not isinstance(other, SCN): + return False + + return self.source == other.source and \ + self.start == other.start and \ + self.end == other.end + + def __hash__(self): + return hash((self.source, self.start, self.end)) + + def __repr__(self): + txt = f"SCN(source='{self.source}'" + if self.start is not None: + txt += f", start={self.start}" + if self.end is not None: + txt += f", end={self.end}" + return txt + ")" + + def transform_real_obj(self, other, to_compare_delegate): + """ + Transform other into CNC, to ease the comparison + :param other: + :param to_compare_delegate: + :return: + """ + + if isinstance(other, SCN): + return other + + if isinstance(other, SourceCodeNode): + return SCN(other.source, + other.start if self.start is not None else None, + other.end if self.end is not None else None) + + raise NotImplementedError(f"SCN, {other=}") + + +class SCWC(HelperWithPos): + """ + SourceNodeWithConcept tester class + It matches with a SourceNodeWithConcept + but it's easier to instantiate during the tests + """ + + def __init__(self, first, last, *args): + super().__init__(None, None) + self.first = first + self.last = last + self.content = list(args) + + def __eq__(self, other): + if id(self) == id(other): + return True + + if isinstance(other, SourceCodeWithConceptNode): + if self.first != other.first: + return False + + if self.last != other.last: + return False + + if len(self.content) != len(other.nodes): + return False + + for self_node, other_node in zip(self.content, other.nodes): + if self_node != other_node: + return False + + # at last + return True + + if not isinstance(other, SCWC): + return False + + return (self.start == other.start and + self.end == other.end and + self.first == other.first and + self.last == other.last and + self.content == other.content) + + def __repr__(self): + txt = "SCWC(" + if self.start is not None: + txt += f"start={self.start}" + if self.end is not None: + txt += f", end={self.end}" + for item in [self.first, self.last, *self.content]: + txt += f", {item}" + return txt + ")" + + def transform_real_obj(self, other, get_test_obj_delegate): + """ + Transform other into CNC, to ease the comparison + :param other: + :param get_test_obj_delegate: + :return: + """ + + if isinstance(other, SCWC): + return other + + if isinstance(other, SourceCodeWithConceptNode): + first = get_test_obj_delegate(other.first, self.first) + last = get_test_obj_delegate(other.last, self.last) + content = [get_test_obj_delegate(r, t) for r, t in zip(other.nodes, self.content)] + res = SCWC(first, last, *content) + res.start = other.start + res.end = other.end + return res + + raise NotImplementedError(f"SCWC, {other=}") + + @property + def source(self): + """ + this code is a copy and paste from SourceCodeWithConceptNode.pseudo_fix_source + TODO: create a common function or whatever... + :return: + """ + source = self.first.source if hasattr(self.first, "source") else self.first + for n in self.content: + source += " " + if hasattr(n, "source"): + source += n.source + elif hasattr(n, "concept"): + source += str(n.concept) + else: + source += " unknown" + source += self.last.source if hasattr(self.last, "source") else self.last + return source + + +class CN(HelperWithPos): + """ + ConceptNode tester class + It matches with ConceptNode but with less constraints + + CN == ConceptNode if concept key, start, end and source are the same + """ + + def __init__(self, concept, source=None, start=None, end=None): + """ + + :param concept: Concept or concept_key (only the key is used anyway) + :param start: + :param end: + :param source: + """ + super().__init__(start, end) + self.concept_key = concept.key if isinstance(concept, Concept) else concept + self.source = source + self.concept = concept if isinstance(concept, Concept) else None + + def fix_source(self, str_tokens): + self.source = "".join(str_tokens) + return self + + def __eq__(self, other): + if id(self) == id(other): + return True + + if not isinstance(other, CN): + return False + + return self.concept_key == other.concept_key and \ + self.start == other.start and \ + self.end == other.end and \ + self.source == other.source + + def __hash__(self): + return hash((self.concept_key, self.start, self.end, self.source)) + + def __repr__(self): + if self.concept: + txt = f"CN(concept='{self.concept}'" + else: + txt = f"CN(concept_key='{self.concept_key}'" + txt += f", source='{self.source}'" + if self.start is not None: + txt += f", start={self.start}" + if self.end is not None: + txt += f", end={self.end}" + return txt + ")" + + def transform_real_obj(self, other, get_test_obj_delegate): + """ + Transform other into CNC, to ease the comparison + :param other: + :param get_test_obj_delegate: + :return: + """ + + if isinstance(other, CN): + return other + + if isinstance(other, ConceptNode): + return CN(other.concept, + other.source if self.source is not None else None, + other.start if self.start is not None else None, + other.end if self.end is not None else None) + + raise NotImplementedError(f"CN, {other=}") + + +class CNC(CN): + """ + ConceptNode for Compiled tester class + It matches with ConceptNode + But focuses on the 'compiled' property of the concept + + CNC == ConceptNode if CNC.get_compiled() == ConceptNode.concept.get_compiled() + """ + + def __init__(self, concept_key, source=None, start=None, end=None, exclude_body=False, **kwargs): + super().__init__(concept_key, source, start, end) + self.compiled = kwargs + self.exclude_body = exclude_body + if "body" in self.compiled: + self.compiled[ConceptParts.BODY] = self.compiled["body"] + del self.compiled["body"] + + def __eq__(self, other): + if id(self) == id(other): + return True + + if not isinstance(other, CNC): + return False + + return self.concept_key == other.concept_key and \ + self.start == other.start and \ + self.end == other.end and \ + self.source == other.source and \ + self.compiled == other.compiled + + def __repr__(self): + if self.concept: + txt = f"CNC(concept='{self.concept}'" + else: + txt = f"CNC(concept_key='{self.concept_key}'" + txt += f", source='{self.source}'" + if self.start is not None: + txt += f", start={self.start}" + if self.end is not None: + txt += f", end={self.end}" + + for k, v in self.compiled.items(): + txt += f", {k}='{v}'" + return txt + ")" + + def transform_real_obj(self, other, get_test_obj_delegate): + """ + Transform other into CNC, to ease the comparison + :param other: + :param get_test_obj_delegate: + :return: + """ + + if isinstance(other, CNC): + return other + + if isinstance(other, ConceptNode): + if self.exclude_body: + compiled = {k: v for k, v in other.concept.get_compiled().items() if k != ConceptParts.BODY} + else: + compiled = other.concept.get_compiled() + + self_compile_to_use = self.compiled or compiled + + compiled = get_test_obj_delegate(self_compile_to_use, compiled, get_test_obj_delegate) + return CNC(other.concept, + other.source if self.source is not None else None, + other.start if self.start is not None else None, + other.end if self.end is not None else None, + self.exclude_body, + **compiled) + + raise NotImplementedError(f"CNC, {other=}") + + +class UTN(HelperWithPos): + """ + Tester class for UnrecognizedTokenNode + compare the source, and start, end if defined + """ + + def __init__(self, source, start=None, end=None): + """ + :param source: + :param start: + :param end: + """ + super().__init__(start, end) + self.source = source + + def __eq__(self, other): + if id(self) == id(other): + return True + + if isinstance(other, UnrecognizedTokensNode): + return self.start == other.start and \ + self.end == other.end and \ + self.source == other.source + + if not isinstance(other, UTN): + return False + + return self.start == other.start and \ + self.end == other.end and \ + self.source == other.source + + def __hash__(self): + return hash((self.source, self.start, self.end)) + + def __repr__(self): + txt = f"UTN(source='{self.source}'" + if self.start is not None: + txt += f", start={self.start}" + if self.end is not None: + txt += f", end={self.end}" + return txt + ")" + + def transform_real_obj(self, other, get_test_obj_delegate): + """ + Transform other into CNC, to ease the comparison + :param other: + :param get_test_obj_delegate: + :return: + """ + + if isinstance(other, UTN): + return other + + if isinstance(other, UnrecognizedTokensNode): + return UTN(other.source, + other.start, + other.end) + + raise NotImplementedError(f"UTN, {other=}") + + +class RN(HelperWithPos): + """ + Helper class to test RuleNode + """ + + def __init__(self, rule, source=None, start=None, end=None): + """ + + :param source: + :param start: + :param end: + """ + super().__init__(start, end) + self.rule_id = rule.id if isinstance(rule, Rule) else rule + self.source = source or str_concept((None, self.rule_id), prefix="r:") if self.rule_id else None + self.rule = rule if isinstance(rule, Rule) else None + + def __eq__(self, other): + if id(self) == id(other): + return True + + if not isinstance(other, RN): + return False + + return self.rule_id == other.rule_id and \ + self.start == other.start and \ + self.end == other.end and \ + self.source == other.source + + def __hash__(self): + return hash((self.rule_id, self.start, self.end, self.source)) + + def __repr__(self): + if self.rule: + txt = f"RN(rule='{self.rule}'" + else: + txt = f"RN(rule_id='{self.rule_id}'" + txt += f", source='{self.source}'" + if self.start is not None: + txt += f", start={self.start}" + if self.end is not None: + txt += f", end={self.end}" + return txt + ")" + + def transform_real_obj(self, other, get_test_obj_delegate): + """ + Transform other into CNC, to ease the comparison + :param other: + :param get_test_obj_delegate: + :return: + """ + + if isinstance(other, RN): + return other + + if isinstance(other, RuleNode): + return RN(other.rule, + other.source if self.source is not None else None, + other.start if self.start is not None else None, + other.end if self.end is not None else None) + + raise NotImplementedError(f"RN, {other=}") + + +class FN: + """ + Test class only + It matches with FunctionNode but with less constraints + + Thereby, + FN("first", "last", ["param1," ...]) can be compared to + FunctionNode(NameExprNode("first"), NameExprNode("second"), [FunctionParameter(NamesNodes("param1"), NamesNodes(", ")]) + + Note that FunctionParameter can easily be defined with a single string + * "param" -> FunctionParameter(NameExprNode("param"), None) + * "param, " -> FunctionParameter(NameExprNode("param"), NameExprNode(", ")) + For more complicated situations, you can use a tuple (value, sep) to define the value part and the separator part + """ + + def __init__(self, first, last, parameters): + self.first = first + self.last = last + self.parameters = [] + for param in parameters: + if isinstance(param, tuple): + self.parameters.append(param) + elif isinstance(param, str) and (pos := param.find(",")) != -1: + self.parameters.append((param[:pos], param[pos:])) + else: + self.parameters.append((param, None)) + + def __repr__(self): + res = self.first + for param in self.parameters: + if param[1]: + res += f"{param[0]}{param[1]} " + else: + res += f"{param[0]}" + return res + self.last + + def __eq__(self, other): + if id(self) == id(other): + return True + + if isinstance(other, FN): + return self.first == other.first and self.last == other.last and self.parameters == other.parameters + + # if isinstance(other, FunctionNode): + # if self.first != other.first.value or self.last != other.last.value: + # return False + # if len(self.parameters) != len(other.parameters): + # return False + # for self_parameter, other_parameter in zip(self.parameters, other.parameters): + # value = other_parameter.value.value if isinstance(self_parameter[0], str) else other_parameter.value + # sep = other_parameter.separator.value if other_parameter.separator else None + # if self_parameter[0] != value or self_parameter[1] != sep: + # return False + # + # return True + + return False + + def __hash__(self): + return hash((self.first, self.last, self.parameters)) + + def transform_real_obj(self, other, get_test_obj_delegate): + if isinstance(other, FN): + return other + + if isinstance(other, FunctionNode): + params = [] + for self_parameter, other_parameter in zip(self.parameters, other.parameters): + if isinstance(self_parameter[0], str): + value = other_parameter.value.value + else: + value = get_test_obj_delegate(other_parameter.value, self_parameter[0]) + sep = other_parameter.separator.value if other_parameter.separator else None + params.append((value, sep)) + + return FN(other.first.value, other.last.value, params) + + raise NotImplementedError(f"FN, {other=}") + + comparison_type_mapping = { "EQ": ComparisonType.EQUALS, "NEQ": ComparisonType.NOT_EQUAlS, @@ -260,7 +1082,7 @@ def get_node( if isinstance(sub_expr, ReturnValueConcept): return sub_expr - if isinstance(sub_expr, (scnode, utnode, DoNotResolve)): + if isinstance(sub_expr, DoNotResolve): return sub_expr if isinstance(sub_expr, CIO): @@ -272,18 +1094,6 @@ def get_node( sub_expr.end = node.end return sub_expr - if isinstance(sub_expr, cnode): - # for cnode, map the concept key to the one from concepts_maps if needed - if sub_expr.concept_key.startswith("#"): - return cnode( - concepts_map[sub_expr.concept_key[1:]].key, - sub_expr.start, - sub_expr.end, - sub_expr.source - ) - else: - return sub_expr - if isinstance(sub_expr, SCWC): sub_expr.first = get_node(concepts_map, expression_as_tokens, sub_expr.first, sya=sya) sub_expr.last = get_node(concepts_map, expression_as_tokens, sub_expr.last, sya=sya) @@ -342,10 +1152,6 @@ def get_node( sub_expr.fix_pos(node) return sub_expr - if isinstance(sub_expr, short_cnode): - return get_node(concepts_map, expression_as_tokens, sub_expr.source, - concept_key=sub_expr.concept_key, skip=skip, is_bnf=True, sya=sya) - if isinstance(sub_expr, tuple): return get_node(concepts_map, expression_as_tokens, sub_expr[0], concept_key=concept_key, skip=sub_expr[1], is_bnf=is_bnf, sya=sya) @@ -364,11 +1170,11 @@ def get_node( if sya and len(concept_found.get_metadata().variables) > 0 and not is_bnf: return SyaConceptParserHelper(concept_found, start, start + length - 1) elif init_empty_body: - node = CNC(concept_found, start, start + length - 1, source=sub_expr, exclude_body=exclude_body) + node = CNC(concept_found, sub_expr, start, start + length - 1, exclude_body=exclude_body) init_body(node, concept_found, sub_expr) return node else: - return CN(concept_found, start, start + length - 1, source=sub_expr) + return CN(concept_found, sub_expr, start, start + length - 1) else: # else an UnrecognizedTokensNode return UTN(sub_expr, start, start + length - 1) @@ -489,27 +1295,32 @@ def get_rete_conditions(*conditions_as_string): return AndConditions(res) -def get_test_obj(test_obj, real_obj, to_compare_delegate=None): +def get_test_obj(real_obj, test_obj, get_test_obj_delegate=None): """ From a production object (Concept, ConceptNode, ....) Create a test object (CNC, CC ...) that can be used to validate the unit tests :param test_obj: :param real_obj: - :param to_compare_delegate: + :param get_test_obj_delegate: :return: """ if isinstance(test_obj, list): if len(test_obj) != len(real_obj): - raise Exception(f"Not the same size ! {test_obj=}, {real_obj=}") - return [get_test_obj(t, r) for t, r in zip(test_obj, real_obj)] + raise Exception(f"Not the same size ! {real_obj=}, {test_obj=}") + return [get_test_obj(r, t) for r, t in zip(real_obj, test_obj)] if isinstance(test_obj, dict): if len(test_obj) != len(real_obj): - raise Exception(f"Not the same size ! {test_obj=}, {real_obj=}") + raise Exception(f"Not the same size ! {real_obj=}, {test_obj=}") - return {k: get_test_obj(v, real_obj[k]) for k, v in test_obj.items()} + return {k: get_test_obj(real_obj[k], v) for k, v in test_obj.items()} - if not hasattr(test_obj, "to_compare"): + if not hasattr(test_obj, "transform_real_obj"): return real_obj - return test_obj.to_compare(real_obj, get_test_obj) + return test_obj.transform_real_obj(real_obj, get_test_obj) + + +def compare_with_test_object(actual, expected): + to_compare = get_test_obj(actual, expected) + assert to_compare == expected diff --git a/tests/parsers/test_BnfNodeParser.py b/tests/parsers/test_BnfNodeParser.py index 9338b15..3fff468 100644 --- a/tests/parsers/test_BnfNodeParser.py +++ b/tests/parsers/test_BnfNodeParser.py @@ -4,11 +4,11 @@ import pytest import tests.parsers.parsers_utils from core.builtin_concepts import BuiltinConcepts -from core.concept import Concept, ConceptParts, DoNotResolve, CC, DEFINITION_TYPE_BNF +from core.concept import Concept, ConceptParts, DoNotResolve, DEFINITION_TYPE_BNF from core.global_symbols import NotInit from core.sheerka.services.SheerkaConceptManager import SheerkaConceptManager from core.sheerka.services.SheerkaExecute import ParserInput -from parsers.BaseNodeParser import CNC, UTN, CN, NoMatchingTokenError, SCN +from parsers.BaseNodeParser import NoMatchingTokenError from parsers.BnfDefinitionParser import BnfDefinitionParser from parsers.BnfNodeParser import StrMatch, TerminalNode, NonTerminalNode, Sequence, OrderedChoice, \ Optional, ZeroOrMore, OneOrMore, ConceptExpression, UnOrderedChoice, BnfNodeParser, RegExMatch, \ @@ -16,6 +16,7 @@ from parsers.BnfNodeParser import StrMatch, TerminalNode, NonTerminalNode, Seque from tests.BaseTest import BaseTest from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka from tests.evaluators.EvaluatorTestsUtils import python_ret_val +from tests.parsers.parsers_utils import CNC, CN, UTN, CC, SCN, get_test_obj, compare_with_test_object cmap = { "one": Concept("one"), @@ -210,7 +211,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka): assert len(bnf_parsers_helpers) == len(expected_array) for parser_helper, expected_sequence in zip(bnf_parsers_helpers, expected_array): - to_compare = tests.parsers.parsers_utils.get_test_obj(expected_sequence, parser_helper.sequence) + to_compare = tests.parsers.parsers_utils.get_test_obj(parser_helper.sequence, expected_sequence) # assert parser_helper.sequence == expected_sequence assert to_compare == expected_sequence @@ -305,7 +306,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka): self.validate_get_concepts_sequences(my_map, text, expected) @pytest.mark.parametrize("text, expected", [ - ("one two three", [CNC("foo", source="one two three")]), + ("one two three", [CNC("foo", "one two three")]), ("one two", []), ("one two four", []), ]) @@ -345,7 +346,8 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka): parser.reset_parser(context, ParserInput(text)) bnf_parsers_helpers = parser.get_concepts_sequences(context) - assert bnf_parsers_helpers[0].sequence == expected_array + transformed = get_test_obj(bnf_parsers_helpers[0].sequence, expected_array) + assert transformed == expected_array assert not bnf_parsers_helpers[0].has_unrecognized # but I cannot parse @@ -362,8 +364,8 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka): text = "one two three one two" expected = [ - CNC("foo", source="one two three"), - CNC("bar", source="one two", start=6, end=8)] + CNC("foo", "one two three"), + CNC("bar", "one two", 6, 8)] self.validate_get_concepts_sequences(my_map, text, expected) @@ -406,8 +408,8 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka): self.validate_get_concepts_sequences(my_map, text, expected) @pytest.mark.parametrize("text, expected", [ - ("ok thirty one", [CNC("foo", source="ok thirty one")]), - ("ok twenty one", [CNC("foo", source="ok twenty one")]), + ("ok thirty one", [CNC("foo", "ok thirty one")]), + ("ok twenty one", [CNC("foo", "ok twenty one")]), ("ok one", []), ]) def test_i_can_mix_sequence_and_ordered(self, text, expected): @@ -421,7 +423,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka): self.validate_get_concepts_sequences(my_map, text, expected) @pytest.mark.parametrize("text, expected", [ - ("twenty one", [CNC("foo", source="twenty one")]), + ("twenty one", [CNC("foo", "twenty one")]), ("twenty three", []), # three does not exist ("twenty four", []), # four exists but should not be seen ]) @@ -435,8 +437,8 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka): self.validate_get_concepts_sequences(my_map, text, expected) @pytest.mark.parametrize("text, expected", [ - ("twenty thirty", [CNC("foo", source="twenty thirty")]), - ("one", [CNC("foo", source="one")]), + ("twenty thirty", [CNC("foo", "twenty thirty")]), + ("one", [CNC("foo", "one")]), ]) def test_i_can_mix_ordered_choices_and_sequences(self, text, expected): my_map = { @@ -448,8 +450,8 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka): self.validate_get_concepts_sequences(my_map, text, expected) @pytest.mark.parametrize("text, expected", [ - ("one", [CNC("foo", source="one")]), - ("one two", [CNC("foo", source="one two")]), + ("one", [CNC("foo", "one")]), + ("one two", [CNC("foo", "one two")]), ("three", []), ]) @@ -463,7 +465,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka): self.validate_get_concepts_sequences(my_map, text, expected) @pytest.mark.parametrize("text, expected", [ - ("one", [CNC("foo", source="one")]), + ("one", [CNC("foo", "one")]), ("", []), ("two", []), ]) @@ -475,8 +477,8 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka): self.validate_get_concepts_sequences(my_map, text, expected) @pytest.mark.parametrize("text, expected", [ - ("twenty one", [CNC("foo", source="twenty one")]), - ("one", [CNC("foo", source="one")]), + ("twenty one", [CNC("foo", "twenty one")]), + ("one", [CNC("foo", "one")]), ]) def test_i_can_match_sequence_starting_with_optional(self, text, expected): my_map = { @@ -489,8 +491,8 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka): self.validate_get_concepts_sequences(my_map, text, expected) @pytest.mark.parametrize("text, expected", [ - ("one two three", [CNC("foo", source="one two three")]), - ("one two", [CNC("foo", source="one two")]), + ("one two three", [CNC("foo", "one two three")]), + ("one two", [CNC("foo", "one two")]), ]) def test_i_can_match_sequence_ending_with_optional(self, text, expected): my_map = { @@ -504,8 +506,8 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka): self.validate_get_concepts_sequences(my_map, text, expected) @pytest.mark.parametrize("text, expected", [ - ("one two three", [CNC("foo", source="one two three")]), - ("one three", [CNC("foo", source="one three")]), + ("one two three", [CNC("foo", "one two three")]), + ("one three", [CNC("foo", "one three")]), ]) def test_i_can_match_sequence_with_optional_in_between(self, text, expected): my_map = { @@ -521,8 +523,8 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka): @pytest.mark.parametrize("text, expected", [ ("", []), ("two", []), - ("one", [CNC("foo", source="one")]), - ("one one", [CNC("foo", source="one one")]), + ("one", [CNC("foo", "one")]), + ("one one", [CNC("foo", "one one")]), ]) def test_i_can_match_zero_or_more(self, text, expected): my_map = { @@ -532,9 +534,9 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka): self.validate_get_concepts_sequences(my_map, text, expected) @pytest.mark.parametrize("text, expected", [ - ("two", [CNC("foo", source="two")]), - ("one two", [CNC("foo", source="one two")]), - ("one one two", [CNC("foo", source="one one two")]), + ("two", [CNC("foo", "two")]), + ("one two", [CNC("foo", "one two")]), + ("one one two", [CNC("foo", "one one two")]), ]) def test_i_can_match_sequence_and_zero_or_more(self, text, expected): my_map = { @@ -548,7 +550,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka): self.validate_get_concepts_sequences(my_map, text, expected) @pytest.mark.parametrize("text, expected", [ - ("one, one , one", [CNC("foo", source="one, one , one")]), + ("one, one , one", [CNC("foo", "one, one , one")]), ]) def test_i_can_match_zero_or_more_with_separator(self, text, expected): my_map = { @@ -564,14 +566,14 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka): } text = "one one one" - expected = [CNC("foo", source=text)] + expected = [CNC("foo", text)] self.validate_get_concepts_sequences(my_map, text, expected) @pytest.mark.parametrize("text, expected", [ ("", []), ("two", []), - ("one", [CNC("foo", source="one")]), - ("one one one", [CNC("foo", source="one one one")]), + ("one", [CNC("foo", "one")]), + ("one one one", [CNC("foo", "one one one")]), ]) def test_i_can_match_one_or_more(self, text, expected): my_map = { @@ -582,8 +584,8 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka): @pytest.mark.parametrize("text, expected", [ ("two", []), - ("one two", [CNC("foo", source="one two")]), - ("one one two", [CNC("foo", source="one one two")]), + ("one two", [CNC("foo", "one two")]), + ("one one two", [CNC("foo", "one one two")]), ]) def test_i_can_match_sequence_one_and_or_more(self, text, expected): my_map = { @@ -597,7 +599,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka): self.validate_get_concepts_sequences(my_map, text, expected) @pytest.mark.parametrize("text, expected", [ - ("one, one , one", [CNC("foo", source="one, one , one")]), + ("one, one , one", [CNC("foo", "one, one , one")]), ]) def test_i_can_match_one_or_more_with_separator(self, text, expected): my_map = { @@ -613,18 +615,18 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka): } text = "one one one" - expected = [CNC("foo", source=text)] + expected = [CNC("foo", text)] self.validate_get_concepts_sequences(my_map, text, expected) @pytest.mark.parametrize("text, expected", [ ("one two", [ - [CNC("foo", source="one two")], - [CNC("bar", source="one two")]]), + [CNC("foo", "one two")], + [CNC("bar", "one two")]]), ("one two one two", [ - [CNC("bar", source="one two"), CNC("bar", source="one two")], - [CNC("foo", source="one two"), CNC("bar", source="one two")], - [CNC("bar", source="one two"), CNC("foo", source="one two")], - [CNC("foo", source="one two"), CNC("foo", source="one two")]]), + [CNC("bar", "one two"), CNC("bar", "one two")], + [CNC("foo", "one two"), CNC("bar", "one two")], + [CNC("bar", "one two"), CNC("foo", "one two")], + [CNC("foo", "one two"), CNC("foo", "one two")]]), ]) def test_i_can_have_multiple_results(self, text, expected): my_map = { @@ -635,7 +637,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka): } text = "one two" - expected = [[CNC("foo", source=text)], [CNC("bar", source=text)]] + expected = [[CNC("foo", text)], [CNC("bar", text)]] self.validate_get_concepts_sequences(my_map, text, expected, multiple_result=True) def test_i_can_refer_to_other_concepts(self): @@ -646,8 +648,8 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka): text = "one two" expected = [ - [CNC("foo", source=text)], - [CN("bar", source=text)] # Do not check the compiled part + [CNC("foo", text)], + [CN("bar", text)] # Do not check the compiled part ] sequences = self.validate_get_concepts_sequences(my_map, text, expected, multiple_result=True) @@ -672,8 +674,8 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka): text = "one two" expected = [ - [CNC("foo", source=text)], - [CN("bar", source=text)] # Do not check the compiled part + [CNC("foo", text)], + [CN("bar", text)] # Do not check the compiled part ] sequences = self.validate_get_concepts_sequences(my_map, text, expected, multiple_result=True) @@ -698,9 +700,9 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka): text = "one two" expected = [ - [CNC("foo", source=text)], - [CN("bar", source=text)], # Do not check the compiled part - [CN("baz", source=text)], # Do not check the compiled part + [CNC("foo", text)], + [CN("bar", text)], # Do not check the compiled part + [CN("baz", text)], # Do not check the compiled part ] sequences = self.validate_get_concepts_sequences(my_map, text, expected, multiple_result=True) @@ -769,7 +771,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka): text = "twenty two" expected = [CNC("twenties", - source="twenty two", + "twenty two", twenty=CC("twenty", body=DoNotResolve("twenty")), number=CC("number", source="two", body=DoNotResolve("two")) )] @@ -842,7 +844,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka): } text = "twenty one" - expected = [CN("foo", source="twenty one")] + expected = [CN("foo", "twenty one")] sequences = self.validate_get_concepts_sequences(my_map, text, expected) concept_foo = sequences[0].concept assert concept_foo.get_compiled() == { @@ -869,8 +871,9 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka): # explicit validations of the compiled concept_foo = sequences[0].concept assert concept_foo.body == NotInit - assert concept_foo.get_compiled() == {'number': CC(my_map["number"], body=my_map["two"], two=my_map["two"]), - ConceptParts.BODY: DoNotResolve(value='twenty two')} + compare_with_test_object(concept_foo.get_compiled(), { + 'number': CC(my_map["number"], body=my_map["two"], two=my_map["two"]), + ConceptParts.BODY: DoNotResolve(value='twenty two')}) text = "twenty one" expected = [CN("foo", source="twenty one")] @@ -879,14 +882,15 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka): # explicit validations of the compiled concept_foo = sequences[0].concept assert concept_foo.body == NotInit - assert concept_foo.get_compiled() == {'number': CC(my_map["number"], body=my_map["one"], one=my_map["one"]), - ConceptParts.BODY: DoNotResolve(value='twenty one')} + compare_with_test_object(concept_foo.get_compiled(), { + 'number': CC(my_map["number"], body=my_map["one"], one=my_map["one"]), + ConceptParts.BODY: DoNotResolve(value='twenty one')}) @pytest.mark.parametrize("expr, expected", [ - ("one 'car'", [CNC("foo", source="one 'car'", x=python_ret_val("'car'"))]), # python - ("one bar", [CNC("foo", source="one bar", x=CC("bar"))]), # simple concept - ("one super car", [CNC("foo", source="one super car", x=CC("super car"))]), # long concept - ("one shoe", [CNC("foo", source="one shoe", x=CC("thing", source="shoe", body=DoNotResolve("shoe")))]), # bnf + ("one 'car'", [CNC("foo", "one 'car'", x=python_ret_val("'car'"))]), # python + ("one bar", [CNC("foo", "one bar", x=CC("bar"))]), # simple concept + ("one super car", [CNC("foo", "one super car", x=CC("super car"))]), # long concept + ("one shoe", [CNC("foo", "one shoe", x=CC("thing", source="shoe", body=DoNotResolve("shoe")))]), # bnf ]) def test_i_can_match_variable_when_ending_with_one_variable(self, expr, expected): my_map = { @@ -910,8 +914,8 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka): expr = "one bar plus baz" expected = [ - [CNC("foo", source="one bar", x=CC("bar")), UTN(" plus "), CN("baz")], - [CNC("foo", source="one bar plus baz", x=CC("plus", source="bar plus baz", x="bar", y="baz"))], + [CNC("foo", "one bar", x=CC("bar")), UTN(" plus "), CN("baz")], + [CNC("foo", "one bar plus baz", x=CC("plus", source="bar plus baz", x="bar", y="baz"))], ] self.validate_get_concepts_sequences(my_map, expr, expected, multiple_result=True) @@ -925,8 +929,8 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka): expr = "one pretty big" expected = [ - [CNC("foo", source="one pretty big", x=CC("pretty big"))], - [CNC("foo", source="one pretty big", x=CC("pbig", source="pretty big"))] + [CNC("foo", "one pretty big", x=CC("pretty big"))], + [CNC("foo", "one pretty big", x=CC("pbig", source="pretty big"))] ] self.validate_get_concepts_sequences(my_map, expr, expected, multiple_result=True) @@ -940,16 +944,16 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka): expr = "one pretty big" expected = [ - [CNC("foo", source="one pretty big", x=CC("pretty"), y=CC("big"))], - [CNC("foo", source="one pretty big", x=CC("pretty2", source="pretty"), y=CC("big"))] + [CNC("foo", "one pretty big", x=CC("pretty"), y=CC("big"))], + [CNC("foo", "one pretty big", x=CC("pretty2", source="pretty"), y=CC("big"))] ] self.validate_get_concepts_sequences(my_map, expr, expected, multiple_result=True) @pytest.mark.parametrize("expr, expected", [ - ("'my' shoe", [CNC("foo", source="'my' shoe", x=python_ret_val("'my' "))]), # python - ("one shoe", [CNC("foo", source="one shoe", x=CC("one"))]), # concept - ("my little shoe", [CNC("foo", source="my little shoe", x=CC("my little"))]), # long concept - ("black shoe", [CNC("foo", source="black shoe", x=CC("color", source="black", body=DoNotResolve('black')))]), + ("'my' shoe", [CNC("foo", "'my' shoe", x=python_ret_val("'my' "))]), # python + ("one shoe", [CNC("foo", "one shoe", x=CC("one"))]), # concept + ("my little shoe", [CNC("foo", "my little shoe", x=CC("my little"))]), # long concept + ("black shoe", [CNC("foo", "black shoe", x=CC("color", source="black", body=DoNotResolve('black')))]), ]) def test_i_can_match_variable_when_starting_with_one_variable(self, expr, expected): my_map = { @@ -972,9 +976,9 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka): expr = "tiny but beautiful shoe" expected_res = [ CNC("foo", - source="tiny but beautiful shoe", + "tiny but beautiful shoe", x=CC("but", source="tiny but beautiful", x="tiny", y="beautiful"))] - unwanted_res = [CN("tiny"), UTN(" but "), CNC("foo", source="beautiful shoe", x=CC("beautiful"))] + unwanted_res = [CN("tiny"), UTN(" but "), CNC("foo", "beautiful shoe", x=CC("beautiful"))] self.validate_get_concepts_sequences(my_map, expr, [unwanted_res, expected_res], multiple_result=True) def test_i_can_match_variable_when_starting_with_multiple_variables(self): @@ -992,7 +996,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka): unwanted_res = [CN("one"), SCN(" 'one' "), ("one", 1), UTN(" plus "), CN("two")] expected_res = [CNC("foo", - source="one 'one' one plus two shoe", + "one 'one' one plus two shoe", x=CC("one"), y=python_ret_val(" 'one' "), z=CC("plus", source="one plus two", x="one", y="two"))] @@ -1009,18 +1013,18 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka): "one": Concept("one") } text = "one foo bar baz" - expected = [CNC("foo", source="one foo bar baz", x=CC("one"))] + expected = [CNC("foo", "one foo bar baz", x=CC("one"))] self.validate_get_concepts_sequences(my_map, text, expected) @pytest.mark.parametrize("expr, expected", [ - ("one 'pretty' shoe", [CNC("foo", source="one 'pretty' shoe", x=python_ret_val("'pretty' "))]), # python - ("one little shoe", [CNC("foo", source="one little shoe", x=CC("little"))]), # concept - ("one very big shoe", [CNC("foo", source="one very big shoe", x=CC("very big"))]), # long concept + ("one 'pretty' shoe", [CNC("foo", "one 'pretty' shoe", x=python_ret_val("'pretty' "))]), # python + ("one little shoe", [CNC("foo", "one little shoe", x=CC("little"))]), # concept + ("one very big shoe", [CNC("foo", "one very big shoe", x=CC("very big"))]), # long concept ("one black shoe", - [CNC("foo", source="one black shoe", x=CC("color", source="black", body=DoNotResolve('black')))]), + [CNC("foo", "one black shoe", x=CC("color", source="black", body=DoNotResolve('black')))]), ("one tiny but beautiful shoe", [CNC("foo", - source="one tiny but beautiful shoe", + "one tiny but beautiful shoe", x=CC("but", source="tiny but beautiful", x="tiny", y="beautiful "))]), ]) def test_i_can_match_variable_in_between(self, expr, expected): @@ -1043,8 +1047,8 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka): expr = "one pretty big shoe" expected = [ - [CNC("foo", source="one pretty big shoe", x=CC("pretty big"))], - [CNC("foo", source="one pretty big shoe", x=CC("pbig", source="pretty big"))] + [CNC("foo", "one pretty big shoe", x=CC("pretty big"))], + [CNC("foo", "one pretty big shoe", x=CC("pbig", source="pretty big"))] ] self.validate_get_concepts_sequences(my_map, expr, expected, multiple_result=True) @@ -1055,7 +1059,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka): "shoe": Concept("shoe") } text = "onyx shoe" - expected = [CNC("foo", source="onyx shoe", x=CC("shoe"))] + expected = [CNC("foo", "onyx shoe", x=CC("shoe"))] self.validate_get_concepts_sequences(my_map, text, expected) def test_i_can_match_variable_and_regex(self): @@ -1065,7 +1069,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka): "one": Concept("one") } text = "one onyx" - expected = [CNC("foo", source="one onyx", x=CC("one"))] + expected = [CNC("foo", "one onyx", x=CC("one"))] self.validate_get_concepts_sequences(my_map, text, expected) def test_i_can_reuse_the_same_variable(self): @@ -1083,12 +1087,12 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka): # same variable appears only once in the compiled variables text = "one equals one" - expected = [CNC("foo", source="one equals one", x=CC("one"))] + expected = [CNC("foo", "one equals one", x=CC("one"))] expected_sequence = compute_expected_array(my_map, text, expected) parser.reset_parser(context, ParserInput(text)) bnf_parsers_helpers = parser.get_concepts_sequences(context) - to_compare = tests.parsers.parsers_utils.get_test_obj(expected_sequence, bnf_parsers_helpers[0].sequence) + to_compare = tests.parsers.parsers_utils.get_test_obj(bnf_parsers_helpers[0].sequence, expected_sequence) assert to_compare == expected def test_i_cannot_match_variable_when_variables_discrepancy(self): @@ -1314,9 +1318,9 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka): ConceptExpression(my_map["one"], rule_name="one")) @pytest.mark.parametrize("expr, text, expected", [ - (ZeroOrMore(StrMatch("one"), sep=","), "one,", [CNC("foo", source="one"), UTN(",")]), - (StrMatch("one"), "one two", [CNC("foo", source="one"), UTN(" two")]), - (StrMatch("one"), "two one", [UTN("two "), CNC("foo", source="one")]), + (ZeroOrMore(StrMatch("one"), sep=","), "one,", [CNC("foo", "one"), UTN(",")]), + (StrMatch("one"), "one two", [CNC("foo", "one"), UTN(" two")]), + (StrMatch("one"), "two one", [UTN("two "), CNC("foo", "one")]), ]) def test_i_can_recognize_unknown_concepts(self, expr, text, expected): my_map = { @@ -1332,7 +1336,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka): } text = "one three" - expected = [UTN("one "), CNC("three", source="three")] + expected = [UTN("one "), CNC("three", "three")] self.validate_get_concepts_sequences(my_map, text, expected) def test_i_can_remove_duplicates(self): @@ -1350,12 +1354,12 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka): assert len(sequence) == 1 @pytest.mark.parametrize("parser_input, expected_status, expected", [ - ("baz", True, [CNC("bnf baz", source="baz")]), # the bnf one is chosen - ("foo bar", True, [CNC("foo then bar", source="foo bar", foo="foo", bar="bar")]), - ("bar", True, [CNC("foo or bar", source="bar", bar="bar", body="bar")]), - ("one plus two", True, [CNC("plus", source="one plus two", one="one", two="two")]), - ("twenty one", True, [CNC("t1", source="twenty one", unit="one")]), - ("one 'car'", True, [CNC("one thing", source="one 'car'", x=python_ret_val("'car'"), one="one")]) + ("baz", True, [CNC("bnf baz", "baz")]), # the bnf one is chosen + ("foo bar", True, [CNC("foo then bar", "foo bar", foo="foo", bar="bar")]), + ("bar", True, [CNC("foo or bar", "bar", bar="bar", body="bar")]), + ("one plus two", True, [CNC("plus", "one plus two", one="one", two="two")]), + ("twenty one", True, [CNC("t1", "twenty one", unit="one")]), + ("one 'car'", True, [CNC("one thing", "one 'car'", x=python_ret_val("'car'"), one="one")]) ]) def test_i_can_parse_simple_expressions(self, parser_input, expected_status, expected): sheerka, context, parser = self.init_parser(init_from_sheerka=True) @@ -1367,7 +1371,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka): assert res.status == expected_status assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT) - assert concepts_nodes == expected_array + compare_with_test_object(concepts_nodes, expected_array) def test_i_can_parse_when_multiple_times_the_same_variable(self): sheerka, context, parser = self.init_parser(init_from_sheerka=True) @@ -1382,7 +1386,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka): assert res.status assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT) - assert concepts_nodes == expected_array + compare_with_test_object(concepts_nodes, expected_array) def test_i_can_test_when_expression_references_other_expressions(self): sheerka, context, parser = self.init_parser(init_from_sheerka=True) @@ -1402,7 +1406,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka): assert res.status assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT) - assert concepts_nodes == expected_array + compare_with_test_object(concepts_nodes, expected_array) def test_i_can_parse_bnf_concept_mixed_with_isa_concepts(self): sheerka, context, parser = self.init_parser(init_from_sheerka=True) @@ -1428,7 +1432,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka): assert res.status assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT) - assert concepts_nodes == expected_array + compare_with_test_object(concepts_nodes, expected_array) def test_i_can_parse_bnf_concept_mixed_with_isa_concepts_2(self): # this time, three is a number, and also part of three_four, even if it is not relevant in t3 @@ -1450,7 +1454,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka): assert res.status assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT) - assert concepts_nodes == expected_array + compare_with_test_object(concepts_nodes, expected_array) def test_i_can_parse_when_starting_by_isa_concept(self): """ @@ -1476,7 +1480,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka): assert res.status assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT) - assert concepts_nodes == expected_array + compare_with_test_object(concepts_nodes, expected_array) def test_i_can_parse_fifty_one_thousand(self): """ @@ -1515,10 +1519,10 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka): res = parser.parse(context, ParserInput(text)) assert res[0].status - assert res[0].value.value == expected_thousands + compare_with_test_object(res[0].value.value, expected_thousands) assert res[1].status - assert res[1].value.value == expected_fifties + compare_with_test_object(res[1].value.value, expected_fifties) def test_i_can_parse_one_hundred_thousand(self): sheerka, context, parser = self.init_parser(init_from_sheerka=True) @@ -1565,7 +1569,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka): assert res.status assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT) - assert concepts_nodes == expected_array + compare_with_test_object(concepts_nodes, expected_array) def test_i_can_parse_bnf_concept_mixed_with_isa_after_restart(self): sheerka, context, parser = self.init_parser(init_from_sheerka=True) @@ -1589,7 +1593,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka): assert res.status assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT) - assert concepts_nodes == expected_array + compare_with_test_object(concepts_nodes, expected_array) text = "forty one" expected = CNC("forties", @@ -1607,13 +1611,13 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka): assert res.status assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT) - assert concepts_nodes == expected_array + compare_with_test_object(concepts_nodes, expected_array) def test_i_can_parse_when_keyword(self): sheerka, context, parser = self.init_parser(init_from_sheerka=True) parser_input = "def one" - expected = [CNC("def number", source="def one", number="one")] + expected = [CNC("def number", "def one", number="one")] res = parser.parse(context, ParserInput(parser_input)) expected_array = compute_expected_array(cmap, parser_input, expected) @@ -1624,7 +1628,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka): assert res.status assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT) - assert concepts_nodes == expected_array + compare_with_test_object(concepts_nodes, expected_array) def test_i_can_parse_filter(self): sheerka, context, parser = self.init_parser(init_from_sheerka=True) @@ -1639,7 +1643,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka): assert res.status assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT) - assert concepts_nodes == expected_array + compare_with_test_object(concepts_nodes, expected_array) def test_i_can_parse_descent_grammar(self): my_map = { @@ -1662,17 +1666,17 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka): assert res.status assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT) - assert concepts_nodes == [CNC(expr, - term=[CC(term, - body=CC(factor, body=DoNotResolve("1")), - factor=CC(factor, body=DoNotResolve("1"))), - CC(term, - body=DoNotResolve("2 * 3"), - factor=[ - CC(factor, body=DoNotResolve("2")), - CC(factor, body=DoNotResolve("3")), - ])], - body=DoNotResolve("1 + 2 * 3"))] + compare_with_test_object(concepts_nodes, [CNC(expr, + term=[CC(term, + body=CC(factor, body=DoNotResolve("1")), + factor=CC(factor, body=DoNotResolve("1"))), + CC(term, + body=DoNotResolve("2 * 3"), + factor=[ + CC(factor, body=DoNotResolve("2")), + CC(factor, body=DoNotResolve("3")), + ])], + body=DoNotResolve("1 + 2 * 3"))]) def test_i_can_parse_recursive_descent_grammar(self): my_map = { @@ -1698,25 +1702,27 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka): # concepts_nodes = res.value.value is too complicated to be validated assert res.status assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT) - assert concepts_nodes == [CNC(expr, - term=CC(term, - body=CC(factor, body=DoNotResolve("1")), - factor=CC(factor, body=DoNotResolve("1"))), - expr=CC(expr, - body=CC(term, - body=DoNotResolve("2 * 3"), - factor=CC(factor, body=DoNotResolve("2")), + compare_with_test_object(concepts_nodes, [CNC(expr, term=CC(term, - body=CC(factor, body=DoNotResolve("3")), - factor=CC(factor, body=DoNotResolve("3")))), - term=CC(term, - body=DoNotResolve("2 * 3"), - factor=CC(factor, body=DoNotResolve("2")), - term=CC(term, - body=CC(factor, body=DoNotResolve("3")), - factor=CC(factor, body=DoNotResolve("3"))))), + body=CC(factor, body=DoNotResolve("1")), + factor=CC(factor, body=DoNotResolve("1"))), + expr=CC(expr, + body=CC(term, + body=DoNotResolve("2 * 3"), + factor=CC(factor, body=DoNotResolve("2")), + term=CC(term, + body=CC(factor, body=DoNotResolve("3")), + factor=CC(factor, + body=DoNotResolve("3")))), + term=CC(term, + body=DoNotResolve("2 * 3"), + factor=CC(factor, body=DoNotResolve("2")), + term=CC(term, + body=CC(factor, body=DoNotResolve("3")), + factor=CC(factor, + body=DoNotResolve("3"))))), - body=DoNotResolve("1 + 2 * 3"))] + body=DoNotResolve("1 + 2 * 3"))]) def test_i_can_parse_simple_recursive_grammar(self): my_map = { @@ -1752,14 +1758,14 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka): text = "thirty one" res = parser.parse(context, ParserInput(text)) assert res.status - assert res.value.value == compute_expected_array(cmap, text, [CN("thirties", source=text)]) + compare_with_test_object(res.value.value, compute_expected_array(cmap, text, [CN("thirties", text)])) # add a layer, I still can parse the text sheerka.push_ontology(context, "new layer") parser = BnfNodeParser(sheerka=sheerka) res = parser.parse(context, ParserInput(text)) assert res.status - assert res.value.value == compute_expected_array(cmap, text, [CN("thirties", source=text)]) + compare_with_test_object(res.value.value, compute_expected_array(cmap, text, [CN("thirties", text)])) def test_i_do_not_eat_unwanted_tokens_at_the_beginning_when_concept_with_variable(self): my_map = { @@ -1772,9 +1778,9 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka): text = "two one shoe" res = parser.parse(context, ParserInput(text)) assert res.status - assert res.value.value == compute_expected_array(my_map, text, [ + compare_with_test_object(res.value.value, compute_expected_array(my_map, text, [ CN("two"), - CNC("foo", source="one shoe", x=CC("one"))]) + CNC("foo", "one shoe", x=CC("one"))])) def test_i_do_not_eat_unwanted_tokens_at_the_end_when_concept_with_variable(self): my_map = { @@ -1787,9 +1793,9 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka): text = "one bar baz" res = parser.parse(context, ParserInput(text)) assert res.status - assert res.value.value == compute_expected_array(my_map, text, [ - CNC("foo", source="one bar", x=CC("bar")), - CN("baz")]) + compare_with_test_object(res.value.value, compute_expected_array(my_map, text, [ + CNC("foo", "one bar", x=CC("bar")), + CN("baz")])) @pytest.mark.parametrize("parsing_expression, expected", [ (RegExMatch("a"), [RegExDef("a")]), diff --git a/tests/parsers/test_BnfParser.py b/tests/parsers/test_BnfParser.py index 184a21c..86866c3 100644 --- a/tests/parsers/test_BnfParser.py +++ b/tests/parsers/test_BnfParser.py @@ -4,13 +4,13 @@ from core.builtin_concepts import BuiltinConcepts from core.concept import Concept, DEFINITION_TYPE_BNF from core.sheerka.services.SheerkaExecute import ParserInput from core.tokenizer import Tokenizer, TokenKind, LexerError -from parsers.BaseNodeParser import cnode from parsers.BaseParser import UnexpectedTokenParsingError, UnexpectedEofParsingError from parsers.BnfDefinitionParser import BnfDefinitionParser from parsers.BnfNodeParser import BnfNodeParser, RegExMatch, VariableExpression from parsers.BnfNodeParser import StrMatch, Optional, ZeroOrMore, OrderedChoice, Sequence, \ OneOrMore, ConceptExpression from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka +from tests.parsers.parsers_utils import CN, compare_with_test_object class ClassWithName: @@ -226,15 +226,15 @@ class TestBnfParser(TestUsingMemoryBasedSheerka): res = bnf_parser.parse(context, ParserInput("twenty two")) assert res.status - assert res.value.body == [cnode("bar", 0, 2, "twenty two")] + compare_with_test_object(res.value.body, [CN("bar", "twenty two", 0, 2)]) res = bnf_parser.parse(context, ParserInput("thirty one")) assert res.status - assert res.value.body == [cnode("bar", 0, 2, "thirty one")] + compare_with_test_object(res.value.body, [CN("bar", "thirty one", 0, 2)]) res = bnf_parser.parse(context, ParserInput("twenty")) assert res.status - assert res.value.body == [cnode("foo", 0, 0, "twenty")] + compare_with_test_object(res.value.body, [CN("foo", "twenty", 0, 0)]) def test_i_cannot_parse_when_too_many_concepts(self): sheerka, context, regex_parser, foo1, foo2 = self.init_parser( diff --git a/tests/parsers/test_DefConceptParser.py b/tests/parsers/test_DefConceptParser.py index 78d2942..a094894 100644 --- a/tests/parsers/test_DefConceptParser.py +++ b/tests/parsers/test_DefConceptParser.py @@ -4,11 +4,10 @@ from dataclasses import dataclass import pytest from core.builtin_concepts import ParserResultConcept, BuiltinConcepts, ReturnValueConcept -from core.concept import DEFINITION_TYPE_BNF, DEFINITION_TYPE_DEF, Concept, CV +from core.concept import DEFINITION_TYPE_BNF, DEFINITION_TYPE_DEF, Concept from core.global_symbols import NotInit from core.sheerka.services.SheerkaExecute import ParserInput from core.tokenizer import Keywords, Tokenizer, LexerError -from parsers.BaseNodeParser import SCWC from parsers.BaseParser import UnexpectedEofParsingError from parsers.BnfDefinitionParser import BnfDefinitionParser from parsers.BnfNodeParser import OrderedChoice, ConceptExpression, StrMatch, Sequence, RegExMatch, OneOrMore, \ @@ -18,7 +17,7 @@ from parsers.DefConceptParser import UnexpectedTokenParsingError, DefConceptNode from parsers.FunctionParser import FunctionParser from parsers.PythonParser import PythonParser, PythonNode from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka -from tests.parsers.parsers_utils import compute_expected_array +from tests.parsers.parsers_utils import compute_expected_array, SCWC, CV, compare_with_test_object def get_def_concept(name, where=None, pre=None, post=None, body=None, definition=None, bnf_def=None, ret=None): @@ -265,7 +264,6 @@ class TestDefConceptParser(TestUsingMemoryBasedSheerka): text = """def concept a mult b where a,b pre isinstance(a, int) and isinstance(b, int) -post isinstance(res, a) as res = a * b ret a if isinstance(a, Concept) else self """ @@ -276,7 +274,6 @@ ret a if isinstance(a, Concept) else self name="a mult b", where="a,b\n", pre="isinstance(a, int) and isinstance(b, int)\n", - post=FN("isinstance(res, a)\n", "isinstance(", ")", ["res", ", ", "a"]), body=PN("res = a * b\n", "exec"), ret="a if isinstance(a, Concept) else self\n" ) @@ -542,29 +539,27 @@ from give me the date ! text = "def concept foo x y where x is a y" res = parser.parse(context, ParserInput(text)) - expected_body = self.pretval(CV(concepts[0], pre=True), source="x is a y", who="parsers.DefConcept", - parser="parsers.ExactConcept") - expected = get_def_concept("foo x y", where=expected_body) node = res.value.value assert res.status assert res.who == parser.name assert res.value.source == text assert isinstance(res.value, ParserResultConcept) - assert node == expected + assert isinstance(node, DefConceptNode) + assert sheerka.isinstance(node.where, BuiltinConcepts.RETURN_VALUE) + compare_with_test_object(node.where.body.body, CV(concepts[0], pre=True)) text = "def concept foo x y pre x is a y" res = parser.parse(context, ParserInput(text)) - expected_body = self.pretval(CV(concepts[0], pre=True), source="x is a y", who="parsers.DefConcept", - parser="parsers.ExactConcept") - expected = get_def_concept("foo x y", pre=expected_body) node = res.value.value assert res.status assert res.who == parser.name assert res.value.source == text assert isinstance(res.value, ParserResultConcept) - assert node == expected + assert isinstance(node, DefConceptNode) + assert sheerka.isinstance(node.pre, BuiltinConcepts.RETURN_VALUE) + compare_with_test_object(node.pre.body.body, CV(concepts[0], pre=True)) def test_i_can_parse_bnf_concept_with_regex(self): sheerka, context, parser, number = self.init_parser("number") diff --git a/tests/parsers/test_ExactConceptParser.py b/tests/parsers/test_ExactConceptParser.py index 53bb043..845ca43 100644 --- a/tests/parsers/test_ExactConceptParser.py +++ b/tests/parsers/test_ExactConceptParser.py @@ -1,9 +1,10 @@ from core.builtin_concepts import BuiltinConcepts -from core.concept import Concept, CMV +from core.concept import Concept from core.sheerka.services.SheerkaExecute import ParserInput from parsers.ExactConceptParser import ExactConceptParser from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka +from tests.parsers.parsers_utils import CMV, compare_with_test_object def variable_def(concept, prop_name): @@ -97,7 +98,7 @@ class TestExactConceptParser(TestUsingMemoryBasedSheerka): assert results[0].status concept_found = results[0].value.value - assert concept_found == CMV(concept, a="10", b="5") + compare_with_test_object(concept_found, CMV(concept, a="10", b="5")) assert concept_found.get_metadata().need_validation assert not concept_found.get_metadata().is_evaluated @@ -113,7 +114,7 @@ class TestExactConceptParser(TestUsingMemoryBasedSheerka): assert results[0].status concept_found = results[0].value.value - assert concept_found == CMV(concept, a="10", b="5") + compare_with_test_object(concept_found, CMV(concept, a="10", b="5")) assert concept_found.get_metadata().need_validation def test_i_can_parse_concept_when_defined_using_from_def(self): @@ -127,7 +128,7 @@ class TestExactConceptParser(TestUsingMemoryBasedSheerka): assert len(results) == 1 assert results[0].status - assert concept_found == CMV(plus, a="10", b="5") + compare_with_test_object(concept_found, CMV(plus, a="10", b="5")) assert concept_found.get_metadata().need_validation assert not concept_found.get_metadata().is_evaluated @@ -157,7 +158,7 @@ class TestExactConceptParser(TestUsingMemoryBasedSheerka): assert len(results) == 1 assert results[0].status - assert concept_found == CMV(plus, a="c:one:", b="c:two:") + compare_with_test_object(concept_found, CMV(plus, a="c:one:", b="c:two:")) assert concept_found.get_metadata().need_validation assert not concept_found.get_metadata().is_evaluated @@ -173,7 +174,7 @@ class TestExactConceptParser(TestUsingMemoryBasedSheerka): assert len(results) == 1 assert results[0].status - assert concept_found == CMV(isa, c="z") + compare_with_test_object(concept_found, CMV(isa, c="z")) assert concept_found.get_metadata().need_validation assert not concept_found.get_metadata().is_evaluated @@ -183,7 +184,7 @@ class TestExactConceptParser(TestUsingMemoryBasedSheerka): assert len(results) == 1 assert results[0].status - assert concept_found == CMV(def_concept, a="z") + compare_with_test_object(concept_found, CMV(def_concept, a="z")) assert concept_found.get_metadata().need_validation assert not concept_found.get_metadata().is_evaluated diff --git a/tests/parsers/test_ExpressionParser.py b/tests/parsers/test_ExpressionParser.py index c64e7b3..5fd0729 100644 --- a/tests/parsers/test_ExpressionParser.py +++ b/tests/parsers/test_ExpressionParser.py @@ -3,11 +3,10 @@ import ast import pytest from core.builtin_concepts import BuiltinConcepts, ReturnValueConcept -from core.concept import Concept, CMV, DoNotResolve, CC +from core.concept import Concept, DoNotResolve from core.rule import Rule from core.sheerka.services.SheerkaExecute import ParserInput from core.tokenizer import TokenKind -from parsers.BaseNodeParser import CNC from parsers.BaseParser import UnexpectedEofParsingError, UnexpectedTokenParsingError from parsers.ExpressionParser import ExpressionParser, LeftPartNotFoundError, ParenthesisMismatchError from parsers.PythonParser import PythonNode @@ -15,7 +14,7 @@ from parsers.expressions import TrueifyVisitor, IsAQuestionVisitor, AndNode from sheerkarete.network import ReteNetwork from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka from tests.parsers.parsers_utils import compute_expected_array, resolve_test_concept, EXPR, OR, AND, NOT, \ - get_expr_node_from_test_node, get_rete_conditions + get_expr_node_from_test_node, get_rete_conditions, CMV, CNC, CC, compare_with_test_object class TestExpressionParser(TestUsingMemoryBasedSheerka): @@ -215,9 +214,9 @@ class TestExpressionParser(TestUsingMemoryBasedSheerka): ("foo", "foo"), ("one two", "one two"), ("foo is a bar", CMV("is a", x='foo', y='bar')), - ("one two is a bar", [CNC("is a", source="one two is a bar", x="one two", y="bar")]), + ("one two is a bar", [CNC("is a", "one two is a bar", x="one two", y="bar")]), ("foo is an foo bar", - [CNC("is an", source="foo is an foo bar", x=DoNotResolve(value='foo'), exclude_body=True)]), + [CNC("is an", "foo is an foo bar", x=DoNotResolve(value='foo'), exclude_body=True)]), ]) def test_i_can_get_compiled_expr_from_simple_concepts_expressions(self, expression, expected): concepts_map = { @@ -238,10 +237,10 @@ class TestExpressionParser(TestUsingMemoryBasedSheerka): if isinstance(expected, list): expected_nodes = compute_expected_array(concepts_map, expression, expected) - assert ret.body.body == expected_nodes + compare_with_test_object(ret.body.body, expected_nodes) else: expected_concept = resolve_test_concept(concepts_map, expected) - assert ret.body.body == expected_concept + compare_with_test_object(ret.body.body, expected_concept) @pytest.mark.parametrize("expression", [ "a == 5", @@ -338,11 +337,11 @@ class TestExpressionParser(TestUsingMemoryBasedSheerka): ret = return_values[0] python_node = ret.body.body assert python_node == expected_python_node - assert python_node.objects == { + compare_with_test_object(python_node.objects, { "__C__00var0000is0a000var001__1005__C__": CC(is_a, x=cat, y=pet), "__C__00var0000is0an0y__1006__C__": CC(is_an, exclude_body=True, x=DoNotResolve("bird"), animal=animal), "__C__00var0000is0a000var001__1005_1__C__": CMV(is_a, x="dog", y="pet"), - } + }) def test_i_can_get_compiled_expr_from_mix(self): sheerka, context, animal, cat, dog, pet, is_a, is_an = self.init_test().with_concepts( @@ -369,11 +368,11 @@ class TestExpressionParser(TestUsingMemoryBasedSheerka): python_node = ret.body.body assert python_node == expected_python_node - assert python_node.objects == { + compare_with_test_object(python_node.objects, { "__C__00var0000is0a000var001__1005__C__": CC(is_a, x=cat, y=pet), "__C__00var0000is0an0y__1006__C__": CC(is_an, exclude_body=True, x=DoNotResolve("bird"), animal=animal), "__C__00var0000is0a000var001__1005_1__C__": CMV(is_a, x="dog", y="pet"), - } + }) def test_i_can_get_compiled_expr_when_multiple_choices(self): sheerka, context, *concepts = self.init_test().with_concepts( @@ -390,10 +389,10 @@ class TestExpressionParser(TestUsingMemoryBasedSheerka): assert len(return_values) == 2 ret = return_values[0] - assert sheerka.objvalue(ret)[0].concept == CMV(concepts[0], x="a", y="b") + compare_with_test_object(sheerka.objvalue(ret)[0].concept, CMV(concepts[0], x="a", y="b")) ret = return_values[1] - assert sheerka.objvalue(ret)[0].concept == CMV(concepts[1], x="a", y="b") + compare_with_test_object(sheerka.objvalue(ret)[0].concept, CMV(concepts[1], x="a", y="b")) def test_i_can_get_compiled_expr_from_mix_when_multiple_choices(self): sheerka, context, *concepts = self.init_test().with_concepts( diff --git a/tests/parsers/test_FunctionParser.py b/tests/parsers/test_FunctionParser.py index 1859f43..070d3cc 100644 --- a/tests/parsers/test_FunctionParser.py +++ b/tests/parsers/test_FunctionParser.py @@ -3,10 +3,10 @@ import pytest from core.builtin_concepts import BuiltinConcepts from core.concept import Concept from core.sheerka.services.SheerkaExecute import ParserInput -from parsers.BaseNodeParser import SCN, SCWC, CN, UTN, CNC, RN -from parsers.FunctionParser import FunctionParser, FN +from parsers.FunctionParser import FunctionParser +from parsers.PythonParser import PythonErrorNode from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka -from tests.parsers.parsers_utils import compute_expected_array +from tests.parsers.parsers_utils import compute_expected_array, SCN, SCWC, CN, UTN, CNC, RN, FN, get_test_obj cmap = { "one": Concept("one"), @@ -80,60 +80,73 @@ class TestFunctionParser(TestUsingMemoryBasedSheerka): parser.parser_input.next_token() res = parser.parse_function() - - assert res == expected + transformed_res = get_test_obj(res, expected) + assert transformed_res == expected def test_i_can_parse_function_when_rule(self): sheerka, context, parser = self.init_parser() + expected = FN("func(", ")", ["r:|1:"]) parser.reset_parser(context, ParserInput("func(r:|1:)")) parser.parser_input.next_token() res = parser.parse_function() - assert res == FN("func(", ")", ["r:|1:"]) + transformed_res = get_test_obj(res, expected) + assert transformed_res == expected @pytest.mark.parametrize("text, expected", [ ("func()", SCN("func()")), (" func()", SCN("func()")), ("func(one)", SCWC("func(", ")", CN("one"))), ("func(one, unknown, two)", SCWC("func(", ")", CN("one"), ", ", UTN("unknown"), (", ", 1), CN("two"))), - ("func(one, twenty two)", SCWC("func(", ")", "one", ", ", CN("twenties", source="twenty two"))), + ("func(one, twenty two)", SCWC("func(", ")", "one", ", ", CN("twenties", "twenty two"))), ("func(one plus two, three)", SCWC("func(", ")", CNC("plus", a="one", b="two"), ", ", UTN("three"))), ("func(func1(one), two)", SCWC("func(", (")", 1), SCWC("func1(", ")", "one"), ", ", "two")) ]) def test_i_can_parse(self, text, expected): sheerka, context, parser = self.init_parser() resolved_expected = compute_expected_array(cmap, text, [expected])[0] - res = parser.parse(context, ParserInput(text)) parser_result = res.body expression = res.body.body assert res.status assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT) - assert expression == resolved_expected + transformed_expression = get_test_obj(expression, resolved_expected) + assert transformed_expression == resolved_expected assert expression.python_node is not None assert expression.return_value is not None def test_i_can_parse_when_multiple_results_when_requested(self): + # the previous output was + # [ + # SCWC("func(", ")", "one", ", ", "twenty ", "two"), + # SCWC("func(", ")", "one", ", ", CN("twenties", "twenty two")) + # ] + # But the first one is now filtered out, as it's not a valid python function call sheerka, context, parser = self.init_parser() parser.longest_concepts_only = False text = "func(one, twenty two)" - expected = [SCWC("func(", ")", "one", ", ", "twenty ", "two"), - SCWC("func(", ")", "one", ", ", CN("twenties", source="twenty two"))] - all_resolved_expected = compute_expected_array(cmap, text, expected) + expected = [SCWC("func(", ")", "one", ", ", CN("twenties", "twenty two"))] + resolved_expected = compute_expected_array(cmap, text, expected) results = parser.parse(context, ParserInput(text)) assert len(results) == 2 - for res, resolved_expected in zip(results, all_resolved_expected): - parser_result = res.body - expressions = res.body.body + res = results[0] + assert not res.status + assert sheerka.isinstance(res.body, BuiltinConcepts.ERROR) + assert len(res.body.body) == 1 + assert (res.body.body[0], PythonErrorNode) - assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT) - assert expressions == resolved_expected + res = results[1] + parser_result = res.body + expressions = res.body.body + assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT) + transformed_expressions = get_test_obj(expressions, resolved_expected[0]) + assert transformed_expressions == resolved_expected[0] def test_i_can_parse_when_the_parameter_is_not_a_concept(self): """ @@ -144,10 +157,15 @@ class TestFunctionParser(TestUsingMemoryBasedSheerka): text = "func(unknown_concept)" res = parser.parse(context, ParserInput(text)) + expected = [SCWC("func(", ")", "unknown_concept")] + resolved_expected = compute_expected_array(cmap, text, expected) assert res.status + parsed = res.body.body + transformed_parsed = get_test_obj([parsed], resolved_expected) + assert transformed_parsed == resolved_expected - def test_i_cannot_parse_when_the_concept_is_not_found(self): + def test_i_can_parse_when_the_concept_is_not_found(self): """ We do not check yet if it's a valid concept If you find a cheap way to do so, simply remove this test @@ -169,37 +187,24 @@ class TestFunctionParser(TestUsingMemoryBasedSheerka): res = parser.parse(context, ParserInput(text)) parser_result = res.body expression = res.body.body + transformed_expression = get_test_obj(expression, resolved_expected) assert res.status assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT) - assert expression == resolved_expected + assert transformed_expression == resolved_expected assert expression.python_node is not None assert expression.return_value is not None - # def test_i_cannot_parse_when_rule_not_found(self): - # sheerka, context, parser = self.init_parser() - # text = "func(r:|fake:)" - # expected = SCWC("func(", ")", RN("fake")) - # resolved_expected = compute_expected_array(cmap, text, [expected])[0] - # - # res = parser.parse(context, ParserInput(text)) - # parser_result = res.body - # expression = res.body.body - # - # assert not res.status - # assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT) - # assert expression == resolved_expected - # assert expression.python_node is not None - # assert expression.return_value is not None - @pytest.mark.parametrize("text, expected_error_type", [ - ("one", BuiltinConcepts.NOT_FOR_ME), - ("$*!", BuiltinConcepts.NOT_FOR_ME), - ("func(", BuiltinConcepts.ERROR), - ("func(one", BuiltinConcepts.ERROR), - ("func(one, two, ", BuiltinConcepts.ERROR), - ("func(one) and func(two)", BuiltinConcepts.ERROR), - ("one func(one)", BuiltinConcepts.NOT_FOR_ME), + ("one", BuiltinConcepts.NOT_FOR_ME), # no function found + ("$*!", BuiltinConcepts.NOT_FOR_ME), # no function found + ("func(", BuiltinConcepts.ERROR), # function found, but incomplete + ("func(one", BuiltinConcepts.ERROR), # function found, but incomplete + ("func(one, two, ", BuiltinConcepts.ERROR), # function found, but incomplete + ("func(one) and func(two)", BuiltinConcepts.ERROR), # to many function + ("one func(one)", BuiltinConcepts.NOT_FOR_ME), # function not found ! (as it is not the first) + ("func(a=b, c)", BuiltinConcepts.ERROR), # function found, but cannot be parsed + ("func(one two)", BuiltinConcepts.ERROR), # function found, but cannot be parsed ]) def test_i_cannot_parse(self, text, expected_error_type): sheerka, context, parser = self.init_parser() @@ -209,23 +214,6 @@ class TestFunctionParser(TestUsingMemoryBasedSheerka): assert not res.status assert sheerka.isinstance(res.body, expected_error_type) - @pytest.mark.parametrize("text, expected", [ - ("func(one two)", SCWC("func(", ")", "one", "two")), - ]) - def test_i_can_detect_none_function(self, text, expected): - sheerka, context, parser = self.init_parser() - resolved_expected = compute_expected_array(cmap, text, [expected])[0] - - res = parser.parse(context, ParserInput(text)) - parser_result = res.body - expression = res.body.body - - assert not res.status - assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT) - assert expression == resolved_expected - assert expression.python_node is None - assert expression.return_value is None - @pytest.mark.parametrize("sequence, expected", [ (None, None), ([["a"]], [["a"]]), diff --git a/tests/parsers/test_PythonWithConceptsParser.py b/tests/parsers/test_PythonWithConceptsParser.py index c5af4b9..ee7fa25 100644 --- a/tests/parsers/test_PythonWithConceptsParser.py +++ b/tests/parsers/test_PythonWithConceptsParser.py @@ -7,7 +7,8 @@ from core.concept import Concept from core.rule import Rule from core.sheerka.services.SheerkaExecute import ParserInput from core.tokenizer import Token, TokenKind, Tokenizer -from parsers.BaseNodeParser import ConceptNode, UnrecognizedTokensNode, RuleNode, SourceCodeNode +from core.var_ref import VariableRef +from parsers.BaseNodeParser import ConceptNode, UnrecognizedTokensNode, RuleNode, VariableNode from parsers.PythonParser import PythonNode from parsers.PythonWithConceptsParser import PythonWithConceptsParser from parsers.UnrecognizedNodeParser import UnrecognizedNodeParser @@ -29,8 +30,12 @@ def ret_val(*args): tokens = [Token(TokenKind.RULE, (None, item.id), 0, 0, 0)] result.append(RuleNode(item, index, index, tokens, f"r:|{item.id}:")) index += 1 + elif isinstance(item, VariableRef): + tokens = list(Tokenizer(item.prop, yield_eof=False)) + result.append(VariableNode(item.obj, item.prop, index, index + len(tokens) - 1, tokens, f"{item.prop}")) + index += len(tokens) else: - tokens = list(Tokenizer(item)) + tokens = list(Tokenizer(item, yield_eof=False)) result.append(UnrecognizedTokensNode(index, index + len(tokens) - 1, tokens)) index += len(tokens) @@ -58,10 +63,10 @@ class TestPythonWithConceptsParser(TestUsingMemoryBasedSheerka): else: assert res is None - def test_i_can_parse_concepts_and_python(self): + def test_i_can_parse_concepts_python_and_variable_ref(self): context = self.get_context() foo = Concept("foo") - input_return_value = ret_val(foo, " + 1") + input_return_value = ret_val(foo, " + 1 + ", VariableRef(foo, "var_name")) parser = PythonWithConceptsParser() result = parser.parse(context, input_return_value.body) @@ -71,12 +76,13 @@ class TestPythonWithConceptsParser(TestUsingMemoryBasedSheerka): assert result.status assert result.who == parser.name assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT) - assert wrapper.source == "foo + 1" + assert wrapper.source == "foo + 1 + var_name" assert isinstance(return_value, PythonNode) - assert return_value.source == "__C__foo__C__ + 1" - assert return_value.original_source == "foo + 1" - assert return_value.get_dump(return_value.ast_) == to_str_ast("__C__foo__C__ + 1") - assert return_value.objects["__C__foo__C__"] == foo + assert return_value.source == "__C__foo__C__ + 1 + __V__foo__var_name__V__" + assert return_value.original_source == "foo + 1 + var_name" + assert return_value.get_dump(return_value.ast_) == to_str_ast("__C__foo__C__ + 1 + __V__foo__var_name__V__") + assert return_value.objects == {"__C__foo__C__": foo, + "__V__foo__var_name__V__": VariableRef(foo, "var_name")} def test_i_can_parse_concepts_and_python_when_concept_is_known(self): context = self.get_context() diff --git a/tests/parsers/test_SequenceNodeParser.py b/tests/parsers/test_SequenceNodeParser.py index d0b96ce..74ad842 100644 --- a/tests/parsers/test_SequenceNodeParser.py +++ b/tests/parsers/test_SequenceNodeParser.py @@ -1,12 +1,12 @@ import pytest + from core.builtin_concepts import BuiltinConcepts from core.concept import Concept, DEFINITION_TYPE_DEF from core.sheerka.services.SheerkaExecute import ParserInput from parsers.SequenceNodeParser import SequenceNodeParser -from parsers.BaseNodeParser import cnode, utnode, CNC, SCN, CN - from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka -from tests.parsers.parsers_utils import compute_expected_array +from tests.parsers.parsers_utils import compute_expected_array, CN, CNC, SCN, get_test_obj, compare_with_test_object, \ + UTN class TestAtomsParser(TestUsingMemoryBasedSheerka): @@ -40,8 +40,8 @@ class TestAtomsParser(TestUsingMemoryBasedSheerka): (" foo ", ["foo"]), ("foo bar", ["foo", "bar"]), ("foo bar twenties", ["foo", "bar", "twenties"]), - ("a plus b", [CN("plus", 0, 4)]), - ("mult", [CN("mult", 0, 0, "mult")]), + ("a plus b", [CN("plus", None, 0, 4)]), + ("mult", [CN("mult", "mult", 0, 0)]), ]) def test_i_can_parse_simple_sequences(self, text, expected): concepts_map = { @@ -62,7 +62,7 @@ class TestAtomsParser(TestUsingMemoryBasedSheerka): expected_array = compute_expected_array(concepts_map, text, expected) assert sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT) - assert lexer_nodes == expected_array + compare_with_test_object(lexer_nodes, expected_array) @pytest.mark.parametrize("text, expected", [ ("foo bar", ["foo bar"]), @@ -85,7 +85,7 @@ class TestAtomsParser(TestUsingMemoryBasedSheerka): expected_array = compute_expected_array(concepts_map, text, expected) assert sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT) - assert lexer_nodes == expected_array + compare_with_test_object(lexer_nodes, expected_array) @pytest.mark.parametrize("text, expected_status, expected", [ ("foo bar suffixed one", False, ["foo bar", " suffixed ", "one"]), @@ -123,12 +123,12 @@ class TestAtomsParser(TestUsingMemoryBasedSheerka): expected_array = compute_expected_array(concepts_map, text, expected) assert sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT) - assert lexer_nodes == expected_array + compare_with_test_object(lexer_nodes, expected_array) @pytest.mark.parametrize("text, expected_status, expected", [ - (" one two ", True, [cnode("one", 1, 1, "one"), cnode("two", 3, 3, "two")]), - (" one x$!# ", False, [cnode("one", 1, 1, "one"), utnode(2, 7, " x$!# ")]), - (" foo bar x$!# ", False, [cnode("foo bar", 1, 3, "foo bar"), utnode(4, 9, " x$!# ")]), + (" one two ", True, [CN("one", "one", 1, 1), CN("two", "two", 3, 3)]), + (" one x$!# ", False, [CN("one", "one", 1, 1), UTN(" x$!# ", 2, 7)]), + (" foo bar x$!# ", False, [CN("foo bar", "foo bar", 1, 3), UTN(" x$!# ", 4, 9)]), ]) def test_i_can_parse_when_surrounded_by_spaces(self, text, expected_status, expected): concepts_map = { @@ -150,7 +150,7 @@ class TestAtomsParser(TestUsingMemoryBasedSheerka): expected_array = compute_expected_array(concepts_map, text, expected) assert sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT) - assert lexer_nodes == expected_array + compare_with_test_object(lexer_nodes, expected_array) @pytest.mark.parametrize("text, expected", [ ("one two", [["one", "two"], ["one two"]]) @@ -173,7 +173,7 @@ class TestAtomsParser(TestUsingMemoryBasedSheerka): assert res.status expected_array = compute_expected_array(concepts_map, text, expected[i]) assert sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT) - assert lexer_nodes == expected_array + compare_with_test_object(lexer_nodes, expected_array) def test_i_can_parse_multiple_concepts_when_long_names_and_unrecognized(self): concepts_map = { @@ -204,7 +204,7 @@ class TestAtomsParser(TestUsingMemoryBasedSheerka): assert res.status == expected[0] expected_array = compute_expected_array(concepts_map, text, expected[1]) assert sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT) - assert lexer_nodes == expected_array + compare_with_test_object(lexer_nodes, expected_array) def test_i_can_parse_concepts_with_isa(self): concepts_map = { @@ -218,7 +218,7 @@ class TestAtomsParser(TestUsingMemoryBasedSheerka): res = parser.parse(context, ParserInput("one")) lexer_nodes = res.body.body expected_array = compute_expected_array(concepts_map, "one", ["one"]) - assert lexer_nodes == expected_array + compare_with_test_object(lexer_nodes, expected_array) def test_i_can_parse_concepts_with_keyword(self): concepts_map = { @@ -231,12 +231,12 @@ class TestAtomsParser(TestUsingMemoryBasedSheerka): res = parser.parse(context, ParserInput("a special concept")) lexer_nodes = res.body.body expected_array = compute_expected_array(concepts_map, "a special concept", ["a special concept"]) - assert lexer_nodes == expected_array + compare_with_test_object(lexer_nodes, expected_array) res = parser.parse(context, ParserInput("isa")) lexer_nodes = res.body.body expected_array = compute_expected_array(concepts_map, "isa", ["isa"]) - assert lexer_nodes == expected_array + compare_with_test_object(lexer_nodes, expected_array) def test_i_can_parse_concepts_when_sub_tokens(self): concepts_map = { @@ -256,7 +256,7 @@ class TestAtomsParser(TestUsingMemoryBasedSheerka): expected_array = compute_expected_array(concepts_map, text, expected) assert sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT) - assert lexer_nodes == expected_array + compare_with_test_object(lexer_nodes, expected_array) @pytest.mark.parametrize("text", [ "foo", @@ -283,8 +283,8 @@ class TestAtomsParser(TestUsingMemoryBasedSheerka): @pytest.mark.parametrize("text, expected", [ ("hello foo bar", [ - (True, [CNC("hello1", source="hello foo ", a="foo "), "bar"]), - (True, [CNC("hello2", source="hello foo ", b="foo "), "bar"]), + (True, [CNC("hello1", "hello foo ", a="foo "), "bar"]), + (True, [CNC("hello2", "hello foo ", b="foo "), "bar"]), ]), ]) def test_i_can_parse_when_unrecognized_yield_multiple_values(self, text, expected): @@ -304,9 +304,10 @@ class TestAtomsParser(TestUsingMemoryBasedSheerka): lexer_nodes = res.body.body assert res.status == expected[0] - expected_array = compute_expected_array(concepts_map, text, expected[1]) assert sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT) - assert lexer_nodes == expected_array + expected_array = compute_expected_array(concepts_map, text, expected[1]) + transformed_nodes = get_test_obj(lexer_nodes, expected_array) + assert transformed_nodes == expected_array @pytest.mark.parametrize("text, expected", [ ("1 + twenty one", [SCN("1 + twenty "), "one"]), @@ -326,7 +327,8 @@ class TestAtomsParser(TestUsingMemoryBasedSheerka): expected_array = compute_expected_array(concepts_map, text, expected) assert sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT) - assert lexer_nodes == expected_array + transformed_nodes = get_test_obj(lexer_nodes, expected_array) + assert transformed_nodes == expected_array @pytest.mark.parametrize("text, expected_is_evaluated", [ ("foo", False), diff --git a/tests/parsers/test_SyaNodeParser.py b/tests/parsers/test_SyaNodeParser.py index 46ae7d7..fc555c0 100644 --- a/tests/parsers/test_SyaNodeParser.py +++ b/tests/parsers/test_SyaNodeParser.py @@ -2,18 +2,19 @@ import pytest import tests.parsers.parsers_utils from core.builtin_concepts import BuiltinConcepts -from core.concept import Concept, CIO, CMV +from core.concept import Concept from core.global_symbols import CONCEPT_COMPARISON_CONTEXT from core.sheerka.services.SheerkaExecute import ParserInput from core.tokenizer import Tokenizer from core.utils import NextIdManager -from parsers.BaseNodeParser import utnode, cnode, short_cnode, UnrecognizedTokensNode, \ - SCWC, CNC, UTN, SCN, CN +from parsers.BaseNodeParser import UnrecognizedTokensNode from parsers.PythonParser import PythonNode from parsers.SyaNodeParser import SyaNodeParser, SyaConceptParserHelper, SyaAssociativity, \ NoneAssociativeSequenceError, TooManyParametersFoundError, InFixToPostFix, ParenthesisMismatchError from tests.TestUsingFileBasedSheerka import TestUsingFileBasedSheerka from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka +from tests.parsers.parsers_utils import UTN, SCWC, CNC, SCN, CIO, CN, compute_debug_array, CMV, get_test_obj, \ + compare_with_test_object def compute_expected_array(concepts_map, expression, expected): @@ -102,6 +103,16 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka): parser.init_from_concepts(context, concepts, sya=sya_def_to_use) return sheerka, context, parser + @staticmethod + def compare_results(res, expected_sequences, concept_map, expression, validate_errors=True): + assert len(res) == len(expected_sequences) + for res_i, expected in zip(res, expected_sequences): + if validate_errors: + assert len(res_i.errors) == 0 + expected_array = compute_expected_array(concept_map, expression, expected) + res_i_as_test_obj = get_test_obj(res_i.out, expected_array) + assert res_i_as_test_obj == expected_array + @pytest.mark.parametrize("expression, expected_sequences", [ ("one plus two", [["one", "two", "plus"]]), ("1 + 1 plus two", [["1 + 1", "two", "plus"]]), @@ -110,7 +121,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka): ["one + two", "three", "plus"]]), ("twenty one plus two", [ ["twenty ", "one", "two", "plus"], - [short_cnode("twenties", "twenty one"), "two", "plus"] + [CN("twenties", "twenty one"), "two", "plus"] ]), ("x$!# plus two", [["x$!#", "two", "plus"]]), @@ -122,7 +133,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka): ]), ("twenty one plus 1 + 1", [ ["twenty ", "one", "1 + 1", "plus"], - [cnode("twenties", 0, 2, "twenty one"), "1 + 1", "plus"] + [CN("twenties", "twenty one", 0, 2), "1 + 1", "plus"] ]), ("x$!# plus 1 + 1", [["x$!#", "1 + 1", "plus"]]), @@ -142,9 +153,9 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka): ]), ("twenty one plus two + three", [ ["twenty ", "one", "two", "plus", " + ", "three"], - [cnode("twenties", 0, 2, "twenty one"), "two", "plus", " + ", "three"], + [CN("twenties", "twenty one", 0, 2), "two", "plus", " + ", "three"], ["twenty ", "one", "two + three", "plus"], - [cnode("twenties", 0, 2, "twenty one"), "two + three", "plus"], + [CN("twenties", "twenty one", 0, 2), "two + three", "plus"], ]), ("x$!# plus two + three", [ ["x$!#", "two", "plus", " + ", "three"], @@ -153,28 +164,28 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka): ("one plus twenty two", [ ["one", "twenty ", "plus", "two"], - ["one", cnode("twenties", 4, 6, "twenty two"), "plus"], + ["one", CN("twenties", "twenty two", 4, 6), "plus"], ]), ("1 + 1 plus twenty one", [ ["1 + 1", "twenty ", "plus", "one"], - ["1 + 1", cnode("twenties", 8, 10, "twenty one"), "plus"], + ["1 + 1", CN("twenties", "twenty one", 8, 10), "plus"], ]), ("one + two plus twenty one", [ ["one", " + ", "two", "twenty ", "plus", ("one", 1)], ["one + two", "twenty ", "plus", ("one", 1)], - ["one", " + ", "two", cnode("twenties", 8, 10, "twenty one"), "plus"], - ["one + two", cnode("twenties", 8, 10, "twenty one"), "plus"], + ["one", " + ", "two", CN("twenties", "twenty one", 8, 10), "plus"], + ["one + two", CN("twenties", "twenty one", 8, 10), "plus"], ]), ("twenty one plus twenty two", [ ["twenty ", "one", ("twenty ", 1), "plus", "two"], - [cnode("twenties", 0, 2, "twenty one"), ("twenty ", 1), "plus", "two"], - ["twenty ", "one", cnode("twenties", 6, 8, "twenty two"), "plus"], - [cnode("twenties", 0, 2, "twenty one"), cnode("twenties", 6, 8, "twenty two"), "plus"], + [CN("twenties", "twenty one", 0, 2), ("twenty ", 1), "plus", "two"], + ["twenty ", "one", CN("twenties", "twenty two", 6, 8), "plus"], + [CN("twenties", "twenty one", 0, 2), CN("twenties", "twenty two", 6, 8), "plus"], ]), ("x$!# plus twenty two", [ ["x$!#", "twenty ", "plus", "two"], - ["x$!#", cnode("twenties", 7, 9, "twenty two"), "plus"] + ["x$!#", CN("twenties", "twenty two", 7, 9), "plus"] ]), ("one plus z$!#", [["one", "z$!#", "plus"]]), @@ -185,7 +196,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka): ]), ("twenty one plus z$!#", [ ["twenty ", "one", "z$!#", "plus"], - [cnode("twenties", 0, 2, "twenty one"), "z$!#", "plus"], + [CN("twenties", "twenty one", 0, 2), "z$!#", "plus"], ]), ("x$!# plus z$!#", [["x$!#", "z$!#", "plus"]]), ]) @@ -194,17 +205,13 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka): res = parser.infix_to_postfix(context, ParserInput(expression)) - assert len(res) == len(expected_sequences) - for res_i, expected in zip(res, expected_sequences): - assert len(res_i.errors) == 0 - expected_array = compute_expected_array(cmap, expression, expected) - assert res_i.out == expected_array + self.compare_results(res, expected_sequences, cmap, expression) @pytest.mark.parametrize("expression, expected_sequences", [ ("one plus plus plus 1 + 1", [["one", "1 + 1", "plus plus plus"]]), ("x$!# another long name infix twenty two", [ ["x$!#", "twenty ", "another long name infix", "two"], - ["x$!#", cnode("twenties", 13, 15, "twenty two"), "another long name infix"], + ["x$!#", CN("twenties", "twenty two", 13, 15), "another long name infix"], ]), ]) def test_i_can_post_fix_infix_concepts_with_long_name(self, expression, expected_sequences): @@ -220,11 +227,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka): res = parser.infix_to_postfix(context, ParserInput(expression)) - assert len(res) == len(expected_sequences) - for res_i, expected in zip(res, expected_sequences): - assert len(res_i.errors) == 0 - expected_array = compute_expected_array(concepts_map, expression, expected) - assert res_i.out == expected_array + self.compare_results(res, expected_sequences, concepts_map, expression) @pytest.mark.parametrize("expression, expected_sequences", [ ("one prefixed", [["one", "prefixed"]]), @@ -235,7 +238,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka): ]), ("twenty one prefixed", [ ["twenty ", "one", "prefixed"], - [cnode("twenties", 0, 2, "twenty one"), "prefixed"], + [CN("twenties", "twenty one", 0, 2), "prefixed"], ]), ("x$!# prefixed", [["x$!#", "prefixed"]]), ]) @@ -244,11 +247,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka): res = parser.infix_to_postfix(context, ParserInput(expression)) - assert len(res) == len(expected_sequences) - for res_i, expected in zip(res, expected_sequences): - assert len(res_i.errors) == 0 - expected_array = compute_expected_array(cmap, expression, expected) - assert res_i.out == expected_array + self.compare_results(res, expected_sequences, cmap, expression) @pytest.mark.parametrize("expression, expected_sequences", [ ("one prefixed prefixed", [["one", "prefixed prefixed"]]), @@ -259,7 +258,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka): ]), ("twenty one prefixed prefixed", [ ["twenty ", "one", "prefixed prefixed"], - [cnode("twenties", 0, 2, "twenty one"), "prefixed prefixed"], + [CN("twenties", "twenty one", 0, 2), "prefixed prefixed"], ]), ("x$!# prefixed prefixed", [["x$!#", "prefixed prefixed"]]), @@ -271,7 +270,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka): ]), ("twenty one long name prefixed", [ ["twenty ", "one", "long name prefixed"], - [cnode("twenties", 0, 2, "twenty one"), "long name prefixed"], + [CN("twenties", "twenty one", 0, 2), "long name prefixed"], ]), ("x$!# long name prefixed", [["x$!#", "long name prefixed"]]), ]) @@ -287,11 +286,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka): res = parser.infix_to_postfix(context, ParserInput(expression)) - assert len(res) == len(expected_sequences) - for res_i, expected in zip(res, expected_sequences): - assert len(res_i.errors) == 0 - expected_array = compute_expected_array(concepts_map, expression, expected) - assert res_i.out == expected_array + self.compare_results(res, expected_sequences, concepts_map, expression) @pytest.mark.parametrize("expression, expected_sequences", [ ("suffixed one", [["one", "suffixed"]]), @@ -302,7 +297,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka): ]), ("suffixed twenty one", [ ["twenty ", "suffixed", "one"], - [cnode("twenties", 2, 4, "twenty one"), "suffixed"], + [CN("twenties", "twenty one", 2, 4), "suffixed"], ]), ("suffixed x$!#", [["x$!#", "suffixed"]]), ]) @@ -311,11 +306,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka): res = parser.infix_to_postfix(context, ParserInput(expression)) - assert len(res) == len(expected_sequences) - for res_i, expected in zip(res, expected_sequences): - assert len(res_i.errors) == 0 - expected_array = compute_expected_array(cmap, expression, expected) - assert res_i.out == expected_array + self.compare_results(res, expected_sequences, cmap, expression) @pytest.mark.parametrize("expression, expected", [ ("suffixed suffixed one", ["one", "suffixed suffixed"]), @@ -333,8 +324,10 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka): res = parser.infix_to_postfix(context, ParserInput(expression)) expected_array = compute_expected_array(concepts_map, expression, expected) + assert len(res) == 1 - assert res[0].out == expected_array + transformed_out = get_test_obj(res[0].out, expected_array) + assert transformed_out == expected_array @pytest.mark.parametrize("expression, expected_sequences", [ ("one ? two : three", [["one", "two", "three", "?"]]), @@ -342,14 +335,14 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka): ("1+1 ? one + two : twenty one", [ ["1+1", "one", " + ", "two"], # error is detected so the parsing has stopped ["1+1", "one + two", "twenty ", "?", ("one", 1)], - ["1+1", "one + two", short_cnode("twenties", "twenty one"), "?"], + ["1+1", "one + two", CN("twenties", "twenty one"), "?"], ]), ("x$!# ? y$!# : z$!#", [["x$!#", "y$!#", "z$!#", "?"]]), ("if one then two else three end", [["one", "two", "three", "if"]]), ("if 1+1 then x$!# else twenty one end", [ ["1+1", "x$!#", "twenty ", "one"], # an error is detected - ["1+1", "x$!#", short_cnode("twenties", "twenty one"), "if"], + ["1+1", "x$!#", CN("twenties", "twenty one"), "if"], ]), ("if x$!# then one + two else z$!# end", [ ["x$!#", "one", " + ", "two"], # error is detected so the parsing has stopped @@ -373,24 +366,20 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka): res = parser.infix_to_postfix(context, ParserInput(expression)) - assert len(res) == len(expected_sequences) - for res_i, expected in zip(res, expected_sequences): - # assert len(res_i.errors) == 0 # Do not validate errors - expected_array = compute_expected_array(cmap, expression, expected) - assert res_i.out == expected_array + self.compare_results(res, expected_sequences, cmap, expression, validate_errors=False) @pytest.mark.parametrize("expression, expected_sequences", [ ("one ? ? two : : three", [["one", "two", "three", "? ?"]]), ("1+1 ? ? one + two : : twenty one", [ ["1+1", "one", " + ", "two"], # error ["1+1", "one + two", "twenty ", "? ?", ("one", 1)], - ["1+1", "one + two", short_cnode("twenties", "twenty one"), "? ?"], + ["1+1", "one + two", CN("twenties", "twenty one"), "? ?"], ]), ("if if one then then two else else three end end ", [["one", "two", "three", "if if"]]), ("if if 1+1 then then x$!# else else twenty one end end ", [ ["1+1", "x$!#", "twenty ", "one"], # error - ["1+1", "x$!#", short_cnode("twenties", "twenty one"), "if if"]]), + ["1+1", "x$!#", CN("twenties", "twenty one"), "if if"]]), ]) def test_i_can_post_fix_ternary_concept_with_long_names(self, expression, expected_sequences): concepts_map = { @@ -405,11 +394,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka): res = parser.infix_to_postfix(context, ParserInput(expression)) - assert len(res) == len(expected_sequences) - for res_i, expected in zip(res, expected_sequences): - # assert len(res_i.errors) == 0 # Do not validate errors - expected_array = compute_expected_array(concepts_map, expression, expected) - assert res_i.out == expected_array + self.compare_results(res, expected_sequences, concepts_map, expression, validate_errors=False) @pytest.mark.parametrize("expression, expected", [ ("foo bar baz", ["baz", "bar", "foo"]), @@ -428,7 +413,8 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka): expected_array = compute_expected_array(concepts_map, expression, expected) assert len(res) == 1 - assert res[0].out == expected_array + transformed_out = get_test_obj(res[0].out, expected_array) + assert transformed_out == expected_array @pytest.mark.parametrize("expression, expected", [ ("baz bar foo", ["baz", "bar", "foo"]), @@ -451,7 +437,8 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka): expected_array = compute_expected_array(concepts_map, expression, expected) assert len(res) == 1 - assert res[0].out == expected_array + transformed_out = get_test_obj(res[0].out, expected_array) + assert transformed_out == expected_array @pytest.mark.parametrize("expression, expected", [ ("one plus two mult three", ["one", "two", "three", "mult", "plus"]), @@ -466,7 +453,8 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka): expected_array = compute_expected_array(cmap, expression, expected) assert len(res) == 1 - assert res[0].out == expected_array + transformed_out = get_test_obj(res[0].out, expected_array) + assert transformed_out == expected_array def test_i_can_post_fix_unary_with_precedence(self): concepts_map = { @@ -487,7 +475,8 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka): expected_array = compute_expected_array(concepts_map, expression, expected) assert len(res) == 1 - assert res[0].out == expected_array + transformed_out = get_test_obj(res[0].out, expected_array) + assert transformed_out == expected_array # change the precedence sya_def = { @@ -502,7 +491,8 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka): expected_array = compute_expected_array(concepts_map, expression, expected) assert len(res) == 1 - assert res[0].out == expected_array + transformed_out = get_test_obj(res[0].out, expected_array) + assert transformed_out == expected_array def test_i_can_post_fix_right_associated_binary(self): concepts_map = { @@ -524,7 +514,8 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka): expected_array = compute_expected_array(concepts_map, expression, expected) assert len(res) == 1 - assert res[0].out == expected_array + transformed_out = get_test_obj(res[0].out, expected_array) + assert transformed_out == expected_array def test_i_can_post_fix_left_associated_binary(self): concepts_map = { @@ -546,7 +537,8 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka): expected_array = compute_expected_array(concepts_map, expression, expected) assert len(res) == 1 - assert res[0].out == expected_array + transformed_out = get_test_obj(res[0].out, expected_array) + assert transformed_out == expected_array @pytest.mark.parametrize("expression, expected", [ ("x$!# ? y$!# : z$!# ? two : three", ["x$!#", "y$!#", "z$!#", "two", "three", ("?", 1), "?"]), @@ -572,8 +564,10 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka): res = parser.infix_to_postfix(context, ParserInput(expression)) expected_array = compute_expected_array(concepts_map, expression, expected) + assert len(res) == 1 - assert res[0].out == expected_array + transformed_out = get_test_obj(res[0].out, expected_array) + assert transformed_out == expected_array @pytest.mark.parametrize("expression, expected", [ ("x$!# ? y$!# : z$!# ? two : three", ["x$!#", "y$!#", "z$!#", "?", "two", "three", ("?", 1)]), @@ -599,8 +593,10 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka): res = parser.infix_to_postfix(context, ParserInput(expression)) expected_array = compute_expected_array(concepts_map, expression, expected) + assert len(res) == 1 - assert res[0].out == expected_array + transformed_out = get_test_obj(res[0].out, expected_array) + assert transformed_out == expected_array def test_i_can_post_fix_when_multiple_concepts_are_found(self): concepts_map = { @@ -617,11 +613,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka): ["baz", "foo bar"] ] - assert len(res) == len(expected_sequences) - for res_i, expected in zip(res, expected_sequences): - assert len(res_i.errors) == 0 - expected_array = compute_expected_array(concepts_map, expression, expected) - assert res_i.out == expected_array + self.compare_results(res, expected_sequences, concepts_map, expression) @pytest.mark.parametrize("expression, expected", [ # ("function(one plus three) minus two", @@ -667,7 +659,8 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka): expected_array = compute_expected_array(cmap, expression, expected) assert len(res) == 1 - assert res[0].out == expected_array + transformed_out = get_test_obj(res[0].out, expected_array) + assert transformed_out == expected_array @pytest.mark.parametrize("expression, expected_sequences", [ # composition @@ -681,20 +674,18 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka): [[SCWC("function(", ")", CNC("prefixed", a=CIO("twenties", source="twenty two")))]]), ("function(if one then twenty two else three end)", [[SCWC("function(", ")", CNC("if", a="one", b=CIO("twenties", source="twenty two"), c="three", end=16))]]), - ("func1(func2(one two) three)", - [[SCWC("func1(", (")", 1), SCWC("func2(", ")", "one", "two"), "three")]]), ("twenty two(suffixed one)", [ ["twenty ", SCWC("two(", ")", CNC("suffixed", a="one"))], - [CN("twenties", source="twenty two"), "one", "suffixed"], + [CN("twenties", "twenty two"), "one", "suffixed"], ]), ("twenty two(one prefixed)", [ ["twenty ", SCWC("two(", ")", CNC("prefixed", a="one"))], - [CN("twenties", source="twenty two"), "one", "prefixed"], + [CN("twenties", "twenty two"), "one", "prefixed"], ]), - ("f1(one plus two mult three) plus f2(suffixed x$!# prefixed)", [ - [SCWC("f1(", ")", CN("plus", source="one plus two mult three")), - SCWC("f2(", (")", 1), CN("suffixed", source="suffixed x$!# prefixed")), + ("f1(one plus two mult three) plus f2(suffixed xxx prefixed)", [ + [SCWC("f1(", ")", CN("plus", "one plus two mult three")), + SCWC("f2(", (")", 1), CN("suffixed", "suffixed xxx prefixed")), ("plus", 1)] ]), @@ -706,8 +697,8 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka): [SCWC("f1(", ")", "one"), SCWC("f2(", (")", 1), "two"), SCWC("f3(", (")", 2), "three"), "if"]]), # Sequence - ("if one then two else three end function(x$!#)", [ - ["one", "two", "three", "if", UTN(" ", start=13, end=13), SCWC("function(", ")", "x$!#")]]), + ("if one then two else three end function(xxx)", [ + ["one", "two", "three", "if", UTN(" ", start=13, end=13), SCWC("function(", ")", "xxx")]]), ("one prefixed function(two)", [["one", "prefixed", UTN(" ", start=3, end=3), SCWC("function(", ")", "two")]]), ("suffixed one function(two)", [["one", "suffixed", UTN(" ", start=3, end=3), SCWC("function(", ")", "two")]]), ("func(one, two, three)", [[SCWC("func(", ")", "one", ", ", "two", (", ", 1), "three")]]), @@ -717,10 +708,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka): res = parser.infix_to_postfix(context, ParserInput(expression)) - assert len(res) == len(expected_sequences) - for res_i, expected in zip(res, expected_sequences): - expected_array = compute_expected_array(cmap, expression, expected) - assert res_i.out == expected_array + self.compare_results(res, expected_sequences, cmap, expression) @pytest.mark.parametrize("expression, expected", [ ("(", ("(", 0)), @@ -800,7 +788,8 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka): expected_array = compute_expected_array(cmap, expression, expected) assert len(res) == 1 - assert res[0].out == expected_array + transformed_out = get_test_obj(res[0].out, expected_array) + assert transformed_out == expected_array @pytest.mark.parametrize("expression", [ "one ? two : three", @@ -818,7 +807,8 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka): expected_array = compute_expected_array(cmap, expression, expected) assert len(res) == 1 - assert res[0].out == expected_array + transformed_out = get_test_obj(res[0].out, expected_array) + assert transformed_out == expected_array def test_the_more_concepts_the_more_results(self): concepts_map = { @@ -837,7 +827,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka): expression = "a plus plus equals b" res = parser.infix_to_postfix(context, ParserInput(expression)) - expected_array = tests.parsers.parsers_utils.compute_debug_array(res) + expected_array = compute_debug_array(res) assert len(expected_array) == len([ ["T(a)", "C(a plus b)", "C(a plus b)", "T(equals)", "T(b)"], ["T(a)", "C(a plus b)", "C(a plus plus)", "T(equals)", "T(b)"], @@ -861,14 +851,17 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka): sheerka, context, parser = self.init_parser(concepts_map, None) res = parser.infix_to_postfix(context, ParserInput("one ? ? two '::' three")) - assert len(res) == 1 - assert res[0].out == [ - cnode("one", start=0, end=0, source="one"), - cnode("two", start=6, end=6, source="two"), - cnode("three", start=10, end=10, source="three"), + expected_array = [ + CN("one", start=0, end=0, source="one"), + CN("two", start=6, end=6, source="two"), + CN("three", start=10, end=10, source="three"), SyaConceptParserHelper(concepts_map["ternary"], 2), ] + assert len(res) == 1 + transformed_out = get_test_obj(res[0].out, expected_array) + assert transformed_out == expected_array + def test_i_cannot_chain_non_associative(self): concepts_map = { "less than": Concept("a less than b").def_var("a").def_var("b"), @@ -897,10 +890,12 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka): expression = "suffixed twenties" res = parser.infix_to_postfix(context, ParserInput(expression)) - expected = [cnode("twenties", 2, 2, "twenties"), "suffixed"] + expected = [CN("twenties", "twenties", 2, 2), "suffixed"] expected_array = compute_expected_array(cmap, expression, expected) + assert len(res) == 1 - assert res[0].out == expected_array + transformed_out = get_test_obj(res[0].out, expected_array) + assert transformed_out == expected_array @pytest.mark.parametrize("expression, expected_debugs", [ ("one", [[" 0:one => PUSH_UNREC"]]), @@ -999,12 +994,12 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka): assert res.status assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT) - assert lexer_nodes == [CN(cmap["plus"], 0, 8, source=text)] + compare_with_test_object(lexer_nodes, [CN(cmap["plus"], text, 0, 8)]) # check the compiled expected_concept = lexer_nodes[0].concept assert expected_concept.get_compiled()["a"] == cmap["one"] - assert expected_concept.get_compiled()["b"] == CMV(cmap["mult"], a="two", b="three") + compare_with_test_object(expected_concept.get_compiled()["b"], CMV(cmap["mult"], a="two", b="three")) assert expected_concept.get_compiled()["b"].get_compiled()["a"] == cmap["two"] assert expected_concept.get_compiled()["b"].get_compiled()["b"] == cmap["three"] @@ -1023,7 +1018,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka): assert res.status assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT) - assert lexer_nodes == [CN(cmap["suffixed"], 0, 6, source=text)] + compare_with_test_object(lexer_nodes, [CN(cmap["suffixed"], text, 0, 6)]) # check the compiled expected_concept = lexer_nodes[0].concept @@ -1051,7 +1046,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka): lexer_nodes = res[1].body.body assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT) - assert lexer_nodes == [CN(cmap["suffixed"], 0, 4, source=text)] + compare_with_test_object(lexer_nodes, [CN(cmap["suffixed"], text, 0, 4)]) # check the compiled expected_concept = lexer_nodes[0].concept @@ -1071,9 +1066,9 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka): assert res.status assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT) - assert lexer_nodes == [ - CN(cmap["plus"], 0, 9, source="one plus 1 + 1 "), - CN(cmap["suffixed"], 10, 12, source="suffixed two")] + compare_with_test_object(lexer_nodes, [ + CN(cmap["plus"], "one plus 1 + 1 ", 0, 9), + CN(cmap["suffixed"], "suffixed two", 10, 12)]) # check the compiled concept_plus_a = lexer_nodes[0].concept.get_compiled()["a"] @@ -1105,7 +1100,9 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka): expected_array = compute_expected_array(cmap, text, expected_result) assert res.status == expected_status assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT) - assert lexer_nodes == expected_array + + transformed_nodes = get_test_obj(lexer_nodes, expected_array) + assert transformed_nodes == expected_array @pytest.mark.parametrize("text", [ "function(suffixed one)", @@ -1166,11 +1163,15 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka): expected_array = compute_expected_array(cmap, text, expected_result) assert not res.status assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT) - assert lexer_nodes == expected_array + + transformed_nodes = get_test_obj(lexer_nodes, expected_array) + assert transformed_nodes == expected_array + + # assert lexer_nodes == expected_array @pytest.mark.parametrize("text, expected_result", [ - ("a plus b", [CN("plus", source="a plus b")]), - ("suffixed a plus b", [CN("suffixed", source="suffixed a plus b")]), + ("a plus b", [CN("plus", "a plus b")]), + ("suffixed a plus b", [CN("suffixed", "suffixed a plus b")]), ]) def test_i_can_almost_parse_concept_definition(self, text, expected_result): """ @@ -1190,7 +1191,9 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka): expected_array = compute_expected_array(cmap, text, expected_result) assert not res.status assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT) - assert lexer_nodes == expected_array + transformed_nodes = get_test_obj(lexer_nodes, expected_array) + assert transformed_nodes == expected_array + # assert lexer_nodes == expected_array @pytest.mark.parametrize("text, expected_concept, expected_unrecognized", [ ("x$!# prefixed", "prefixed", ["a"]), @@ -1209,15 +1212,18 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka): assert not res.status assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT) - assert lexer_nodes == [CN(cmap[expected_concept], 0, expected_end, source=text)] + expected_array = [CN(cmap[expected_concept], text, 0, expected_end)] + transformed_nodes = get_test_obj(lexer_nodes, expected_array) + assert transformed_nodes == expected_array + # assert lexer_nodes == [CN(cmap[expected_concept], text, 0, expected_end)] concept_found = lexer_nodes[0].concept for unrecognized in expected_unrecognized: assert isinstance(concept_found.get_compiled()[unrecognized], UnrecognizedTokensNode) @pytest.mark.parametrize("text, expected", [ - ("x$!# suffixed one", [utnode(0, 4, "x$!# "), cnode("suffixed __var__0", 5, 7, "suffixed one")]), - ("one prefixed x$!#", [cnode("__var__0 prefixed", 0, 2, "one prefixed"), utnode(3, 7, " x$!#")]), + ("x$!# suffixed one", [UTN("x$!# ", 0, 4), CN("suffixed __var__0", "suffixed one", 5, 7)]), + ("one prefixed x$!#", [CN("__var__0 prefixed", "one prefixed", 0, 2), UTN(" x$!#", 3, 7)]), ]) def test_i_cannot_parse_when_part_of_the_sequence_is_not_recognized(self, text, expected): sheerka, context, parser = self.init_parser() @@ -1228,7 +1234,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka): assert not res.status assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT) - assert lexer_nodes == expected + compare_with_test_object(lexer_nodes, expected) def test_i_cannot_parse_function_using_short_name(self): concepts_map = { @@ -1301,15 +1307,15 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka): assert len(actual) == 1 - assert actual[0].to_out == resolved_to_out + compare_with_test_object(actual[0].to_out, resolved_to_out) actual[0].function.fix_source() - assert actual[0].function == resolved_function_name[0] + compare_with_test_object(actual[0].function, resolved_function_name[0]) @pytest.mark.parametrize("expression, expected_list", [ ("twenty two function(", [(["twenty ", "two", UTN(" ", 3, 3)], "function("), - ([CN("twenties", source="twenty two"), UTN(" ", 3, 3)], "function(")]), + ([CN("twenties", "twenty two"), UTN(" ", 3, 3)], "function(")]), ("twenty two(", [(["twenty "], "two("), - ([CN("twenties", source="twenty two")], None)]), + ([CN("twenties", "twenty two")], None)]), ]) def test_i_can_get_functions_names_from_unrecognized_when_multiple_results(self, expression, expected_list): sheerka, context, parser = self.init_parser() @@ -1326,11 +1332,11 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka): for actual, expected in zip(actual_list, expected_list): resolved_to_out = compute_expected_array(cmap, expression, expected[0]) - assert actual.to_out == resolved_to_out + compare_with_test_object(actual.to_out, resolved_to_out) if actual.function: actual.function.fix_source() resolved_function_name = compute_expected_array(cmap, expression, [expected[1]]) - assert actual.function == resolved_function_name[0] + compare_with_test_object(actual.function, resolved_function_name[0]) else: assert actual.function is None @@ -1344,7 +1350,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka): assert res.status assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT) - assert lexer_nodes == [CN(cmap["suffixed"], 0, 6, source=text)] + compare_with_test_object(lexer_nodes, [CN(cmap["suffixed"], text, 0, 6)]) # add an ontology layer and make sure will still can parse sheerka.push_ontology(context, "new ontology") @@ -1356,7 +1362,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka): assert res.status assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT) - assert lexer_nodes == [CN(cmap["suffixed"], 0, 6, source=text)] + compare_with_test_object(lexer_nodes, [CN(cmap["suffixed"], text, 0, 6)]) class TestFileBaseSyaNodeParser(TestUsingFileBasedSheerka): diff --git a/tests/parsers/test_UnrecognizedNodeParser.py b/tests/parsers/test_UnrecognizedNodeParser.py index 7d04c62..6ba20a0 100644 --- a/tests/parsers/test_UnrecognizedNodeParser.py +++ b/tests/parsers/test_UnrecognizedNodeParser.py @@ -1,14 +1,14 @@ from core.builtin_concepts import ParserResultConcept, BuiltinConcepts -from core.concept import Concept, CC +from core.concept import Concept from core.tokenizer import Tokenizer, TokenKind -from parsers.BaseNodeParser import ConceptNode, UnrecognizedTokensNode, scnode, cnode, \ - utnode, CN, CNC, UTN, SourceCodeWithConceptNode, SCWC, SourceCodeNode +from parsers.BaseNodeParser import ConceptNode, UnrecognizedTokensNode, SourceCodeWithConceptNode, SourceCodeNode from parsers.BnfNodeParser import BnfNodeParser from parsers.SequenceNodeParser import SequenceNodeParser from parsers.SyaNodeParser import SyaNodeParser from parsers.UnrecognizedNodeParser import UnrecognizedNodeParser from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka -from tests.parsers.parsers_utils import compute_expected_array, get_node +from tests.parsers.parsers_utils import compute_expected_array, get_node, CC, UTN, CNC, CN, SCWC, \ + compare_with_test_object, SCN def get_input_nodes_from(my_concepts_map, full_expr, *args): @@ -19,10 +19,10 @@ def get_input_nodes_from(my_concepts_map, full_expr, *args): concept.get_compiled()[k] = _get_real_node(v) return concept - if isinstance(n, (utnode, UTN)): + if isinstance(n, UTN): return UnrecognizedTokensNode(n.start, n.end, full_expr_as_tokens[n.start: n.end + 1]) - if isinstance(n, (CNC, CN, cnode)): + if isinstance(n, (CNC, CN)): concept = n.concept if hasattr(n, "concept") and n.concept else \ Concept().update_from(my_concepts_map[n.concept_key]) tokens = full_expr_as_tokens[n.start: n.end + 1] @@ -116,7 +116,7 @@ class TestUnrecognizedNodeParser(TestUsingMemoryBasedSheerka): a=" one ", b=" two three ", c=" twenty one ", - d=utnode(12, 18, " 1 + 2 "), + d=UTN(" 1 + 2 ", 12, 18), e=" one plus two mult three"))[0] res = UnrecognizedNodeParser().validate_concept_node(context, node) @@ -130,13 +130,14 @@ class TestUnrecognizedNodeParser(TestUsingMemoryBasedSheerka): assert sheerka.isinstance(concept.get_compiled()["a"][0], BuiltinConcepts.RETURN_VALUE) assert concept.get_compiled()["a"][0].status assert concept.get_compiled()["a"][0].who == "parsers." + SequenceNodeParser.NAME - assert concept.get_compiled()["a"][0].body.body == [cnode("one", 1, 1, "one")] + compare_with_test_object(concept.get_compiled()["a"][0].body.body, [CN("one", "one", 1, 1)]) assert len(concept.get_compiled()["b"]) == 1 assert sheerka.isinstance(concept.get_compiled()["b"][0], BuiltinConcepts.RETURN_VALUE) assert concept.get_compiled()["b"][0].status assert concept.get_compiled()["b"][0].who == "parsers." + SequenceNodeParser.NAME - assert concept.get_compiled()["b"][0].body.body == [cnode("two", 1, 1, "two"), cnode("three", 3, 3, "three")] + compare_with_test_object(concept.get_compiled()["b"][0].body.body, + [CN("two", "two", 1, 1), CN("three", "three", 3, 3)]) assert len(concept.get_compiled()["c"]) == 1 assert sheerka.isinstance(concept.get_compiled()["c"][0], BuiltinConcepts.RETURN_VALUE) @@ -145,8 +146,8 @@ class TestUnrecognizedNodeParser(TestUsingMemoryBasedSheerka): expected_nodes = compute_expected_array( concepts_map, " twenty one ", - [CNC("twenties", source="twenty one", unit="one")]) - assert concept.get_compiled()["c"][0].body.body == expected_nodes + [CNC("twenties", "twenty one", unit="one")]) + compare_with_test_object(concept.get_compiled()["c"][0].body.body, expected_nodes) assert len(concept.get_compiled()["d"]) == 1 assert sheerka.isinstance(concept.get_compiled()["d"][0], BuiltinConcepts.RETURN_VALUE) @@ -164,7 +165,7 @@ class TestUnrecognizedNodeParser(TestUsingMemoryBasedSheerka): [CNC("plus", a="one", b=CC("mult", a="two", b="three"))], exclude_body=True) - assert concept.get_compiled()["e"][0].body.body == expected_nodes + compare_with_test_object(concept.get_compiled()["e"][0].body.body, expected_nodes) # # sanity check, I can evaluate the concept # evaluated = sheerka.evaluate_concept(self.get_context(sheerka, eval_body=True), concept) @@ -204,8 +205,8 @@ class TestUnrecognizedNodeParser(TestUsingMemoryBasedSheerka): expected_nodes = compute_expected_array( concepts_map, " twenty two", - [CNC("twenties", source="twenty two", unit="two")]) - assert res.body.concept.get_compiled()["b"].get_compiled()["b"][0].body.body == expected_nodes + [CNC("twenties", "twenty two", unit="two")]) + compare_with_test_object(res.body.concept.get_compiled()["b"].get_compiled()["b"][0].body.body, expected_nodes) def test_i_can_validate_and_evaluate_a_concept_node_with_python(self): sheerka, context, parser = self.init_parser() @@ -281,7 +282,7 @@ class TestUnrecognizedNodeParser(TestUsingMemoryBasedSheerka): assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT) assert parser_result.source == expression assert len(actual_nodes) == 1 - assert actual_nodes[0] == scnode(0, 4, expression) + compare_with_test_object(actual_nodes[0], SCN(expression, 0, 4)) def test_i_cannot_parse_unrecognized_python_that_looks_like_concept(self): sheerka, context, parser = self.init_parser() @@ -319,7 +320,7 @@ class TestUnrecognizedNodeParser(TestUsingMemoryBasedSheerka): expected_array = compute_expected_array( concepts_map, expression, [CNC("twenties", source=expression, unit="one")]) - assert actual_nodes == expected_array + compare_with_test_object(actual_nodes, expected_array) def test_i_can_parse_unrecognized_sya_concept_node(self): sheerka, context, parser = self.init_parser() @@ -343,7 +344,7 @@ class TestUnrecognizedNodeParser(TestUsingMemoryBasedSheerka): a="one", b=CC("mult", source="two mult three", a="two", b="three"))], exclude_body=True) - assert actual_nodes == expected_array + compare_with_test_object(actual_nodes, expected_array) def test_i_can_parse_unrecognized_source_code_with_concept_node(self): sheerka, context, parser = self.init_parser() @@ -388,8 +389,8 @@ class TestUnrecognizedNodeParser(TestUsingMemoryBasedSheerka): expression = "hello get_user_name(twenty one)" tmp_node = CNC("hello_sya", - source="hello get_user_name(twenty one)", - a=SCWC("get_user_name(", ")", CNC("twenties", source="twenty one", unit="one"))) + "hello get_user_name(twenty one)", + a=SCWC("get_user_name(", ")", CNC("twenties", "twenty one", unit="one"))) nodes = get_input_nodes_from(concepts_map, expression, tmp_node) parser_input = ParserResultConcept("parsers.xxx", source=expression, value=nodes) @@ -404,9 +405,9 @@ class TestUnrecognizedNodeParser(TestUsingMemoryBasedSheerka): expected_array = compute_expected_array( concepts_map, - expression, [CN("hello_sya", source="hello get_user_name(twenty one)")], + expression, [CN("hello_sya", "hello get_user_name(twenty one)")], exclude_body=True) - assert actual_nodes == expected_array + compare_with_test_object(actual_nodes, expected_array) assert isinstance(actual_nodes[0].concept.get_compiled()["a"], list) assert sheerka.isinstance(actual_nodes[0].concept.get_compiled()["a"][0], BuiltinConcepts.RETURN_VALUE) @@ -416,7 +417,7 @@ class TestUnrecognizedNodeParser(TestUsingMemoryBasedSheerka): expression = "one plus two three" sequence = get_input_nodes_from(concepts_map, expression, CNC("plus", a="one", b="two"), - utnode(5, 6, " three")) + UTN(" three", 5, 6)) parser_input = ParserResultConcept("parsers.xxx", source="one plus two three", value=sequence) res = parser.parse(context, parser_input) @@ -429,7 +430,7 @@ class TestUnrecognizedNodeParser(TestUsingMemoryBasedSheerka): expression, [ CNC("plus", a="one", b="two"), CN("three", start=6, end=6)]) - assert actual_nodes == expected_array + compare_with_test_object(actual_nodes, expected_array) def test_i_can_parse_when_multiple_atom_and_sya(self): sheerka, context, parser = self.init_parser() @@ -445,19 +446,18 @@ class TestUnrecognizedNodeParser(TestUsingMemoryBasedSheerka): actual_nodes0 = res[0].body.body expected_0 = compute_expected_array(concepts_map, expression, [ - CN("two", 0, 0), + CN("two", start=0, end=0), CN("hello_atom", source="hello one", start=2, end=4), - CN("three", 6, 6)]) - assert actual_nodes0 == expected_0 + CN("three", start=6, end=6)]) + compare_with_test_object(actual_nodes0, expected_0) actual_nodes1 = res[1].body.body expected_1 = compute_expected_array(concepts_map, expression, [ - CN("two", 0, 0), - CNC("hello_sya", source="hello one", start=2, end=4, a="one"), - CN("three", 6, 6)], + CN("two", start=0, end=0), + CNC("hello_sya", "hello one", start=2, end=4, a="one"), + CN("three", start=6, end=6)], exclude_body=True) - - assert actual_nodes1 == expected_1 + compare_with_test_object(actual_nodes1, expected_1) def test_i_can_parse_when_multiple_sya_concepts(self): sheerka, context, parser = self.init_parser() @@ -474,12 +474,12 @@ class TestUnrecognizedNodeParser(TestUsingMemoryBasedSheerka): actual_nodes0 = res[0].body.body expected_0 = compute_expected_array(concepts_map, expression, [ CNC("greetings_a", source="greetings two", start=0, end=2, a="two")], exclude_body=True) - assert actual_nodes0 == expected_0 + compare_with_test_object(actual_nodes0, expected_0) actual_nodes1 = res[1].body.body expected_1 = compute_expected_array(concepts_map, expression, [ CNC("greetings_b", source="greetings two", start=0, end=2, b="two")], exclude_body=True) - assert actual_nodes1 == expected_1 + compare_with_test_object(actual_nodes1, expected_1) def test_i_cannot_parse_when_some_unrecognized_remain(self): sheerka, context, parser = self.init_parser() diff --git a/tests/parsers/test_parsers_utils.py b/tests/parsers/test_parsers_utils.py index 37d5ea1..75a77b4 100644 --- a/tests/parsers/test_parsers_utils.py +++ b/tests/parsers/test_parsers_utils.py @@ -1,10 +1,13 @@ -from core.concept import Concept, ConceptParts, CC +from core.concept import Concept, ConceptParts +from core.global_symbols import NotInit +from core.rule import Rule, ACTION_TYPE_EXEC from core.sheerka.services.SheerkaExecute import ParserInput -from parsers.BaseNodeParser import CNC +from parsers.BaseNodeParser import RuleNode from parsers.BnfNodeParser import BnfNodeParser +from parsers.FunctionParser import FunctionParser from parsers.SyaNodeParser import SyaNodeParser from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka -from tests.parsers.parsers_utils import get_test_obj +from tests.parsers.parsers_utils import get_test_obj, CNC, CC, CN, SCN, SCWC, UTN, RN, CB class TestParsersUtils(TestUsingMemoryBasedSheerka): @@ -20,12 +23,12 @@ class TestParsersUtils(TestUsingMemoryBasedSheerka): cnode = parser.parse(context, ParserInput("one plus two")).body.body[0] # compare all attributes - cnc_res = get_test_obj(CNC(concept_key="key", start=0, end=1, source="", exclude_body=False), cnode) + cnc_res = get_test_obj(cnode, CNC(concept_key="key", start=0, end=1, source="", exclude_body=False)) assert isinstance(cnc_res, CNC) - assert cnc_res == CNC("__var__0 plus __var__1", 0, 4, "one plus two", False, **cnode.concept.get_compiled()) + assert cnc_res == CNC("__var__0 plus __var__1", "one plus two", 0, 4, False, **cnode.concept.get_compiled()) # I can discard start, end and source - cnc_res = get_test_obj(CNC(concept_key="key"), cnode) + cnc_res = get_test_obj(cnode, CNC(concept_key="key")) assert isinstance(cnc_res, CNC) assert cnc_res == CNC("__var__0 plus __var__1", None, None, None, False, **cnode.concept.get_compiled()) @@ -40,12 +43,12 @@ class TestParsersUtils(TestUsingMemoryBasedSheerka): cnode = parser.parse(context, ParserInput("twenty one")).body.body[0] # compare all attributes - cnc_res = get_test_obj(CNC(concept_key="key", start=0, end=1, source="", exclude_body=False), cnode) + cnc_res = get_test_obj(cnode, CNC(concept_key="key", start=0, end=1, source="", exclude_body=False)) assert isinstance(cnc_res, CNC) - assert cnc_res == CNC("twenties", 0, 2, "twenty one", False, **cnode.concept.get_compiled()) + assert cnc_res == CNC("twenties", "twenty one", 0, 2, False, **cnode.concept.get_compiled()) # I can exclude body - cnc_res = get_test_obj(CNC(concept_key="key", exclude_body=True), cnode) + cnc_res = get_test_obj(cnode, CNC(concept_key="key", exclude_body=True)) expected_compiled = {k: v for k, v in cnode.concept.get_compiled().items()} del expected_compiled[ConceptParts.BODY] assert isinstance(cnc_res, CNC) @@ -61,13 +64,13 @@ class TestParsersUtils(TestUsingMemoryBasedSheerka): parser = SyaNodeParser().init_from_concepts(context, [one, two, plus]) cnode = parser.parse(context, ParserInput("one plus two")).body.body[0] - res = get_test_obj([CNC("key1"), CNC("key", 0, 1, "")], [cnode, cnode]) + res = get_test_obj([cnode, cnode], [CNC("key1"), CNC("key", 0, 1, "")]) assert len(res) == 2 assert isinstance(res[0], CNC) assert res[0] == CNC("__var__0 plus __var__1", None, None, None, False, **cnode.concept.get_compiled()) assert isinstance(res[1], CNC) - assert res[1] == CNC("__var__0 plus __var__1", 0, 4, "one plus two", False, **cnode.concept.get_compiled()) + assert res[1] == CNC("__var__0 plus __var__1", "one plus two", 0, 4, False, **cnode.concept.get_compiled()) def test_i_can_get_test_obj_when_dict(self): sheerka, context, one, two, plus = self.init_concepts( @@ -79,12 +82,12 @@ class TestParsersUtils(TestUsingMemoryBasedSheerka): parser = SyaNodeParser().init_from_concepts(context, [one, two, plus]) cnode = parser.parse(context, ParserInput("one plus two")).body.body[0] - res = get_test_obj({"key1": CNC("key1"), "key2": CNC("key", 0, 1, "")}, {"key1": cnode, "key2": cnode}) + res = get_test_obj({"key1": cnode, "key2": cnode}, {"key1": CNC("key1"), "key2": CNC("key", 0, 1, "")}) assert len(res) == 2 assert isinstance(res["key1"], CNC) assert res["key1"] == CNC("__var__0 plus __var__1", None, None, None, False, **cnode.concept.get_compiled()) assert isinstance(res["key2"], CNC) - assert res["key2"] == CNC("__var__0 plus __var__1", 0, 4, "one plus two", False, **cnode.concept.get_compiled()) + assert res["key2"] == CNC("__var__0 plus __var__1", "one plus two", 0, 4, False, **cnode.concept.get_compiled()) def test_i_can_get_test_obj_when_CC(self): sheerka, context, one, two, plus = self.init_concepts( @@ -97,13 +100,115 @@ class TestParsersUtils(TestUsingMemoryBasedSheerka): cc = parser.parse(context, ParserInput("twenty one")).body.body[0].concept # compare all attributes - cc_res = get_test_obj(CC(concept="key", source="", exclude_body=False), cc) + cc_res = get_test_obj(cc, CC(concept="key", source="", exclude_body=False)) assert isinstance(cc_res, CC) assert cc_res == CC("twenties", "twenty one", False, **cc.get_compiled()) # I can exclude body - cnc_res = get_test_obj(CC(concept="key", exclude_body=True), cc) + cnc_res = get_test_obj(cc, CC(concept="key", exclude_body=True)) expected_compiled = {k: v for k, v in cc.get_compiled().items()} del expected_compiled[ConceptParts.BODY] assert isinstance(cnc_res, CC) assert cnc_res == CC("twenties", "twenty one", True, **expected_compiled) + + def test_i_can_get_test_obj_when_CN(self): + sheerka, context, one, two, plus = self.init_concepts( + "one", + "two", + Concept("a plus b").def_var("a").def_var("b") + ) + + parser = SyaNodeParser().init_from_concepts(context, [one, two, plus]) + cnode = parser.parse(context, ParserInput("one plus two")).body.body[0] + + cn_res = get_test_obj(cnode, CN(concept="key", start=0, end=1, source="")) + assert isinstance(cn_res, CN) + assert cn_res == CN(plus, "one plus two", 0, 4) + + # I can discard start, end and source + cnc_res = get_test_obj(cnode, CN(concept="key")) + assert isinstance(cnc_res, CN) + assert cnc_res == CN(plus, None, None, None) + + def test_i_can_get_test_obj_when_SCN(self): + sheerka, context = self.init_test().unpack() + + parser = FunctionParser() + scn = parser.parse(context, ParserInput("test()")).body.body + + scn_res = get_test_obj(scn, SCN("", start=0, end=1)) + assert isinstance(scn_res, SCN) + assert scn_res == SCN("test()", 0, 2) + + # I can discard start and end + scn_res = get_test_obj(scn, SCN("")) + assert isinstance(scn_res, SCN) + assert scn_res == SCN("test()", None, None) + + def test_i_can_get_test_obj_when_SCWC(self): + sheerka, context = self.init_test().unpack() + + parser = FunctionParser() + scwc = parser.parse(context, ParserInput("test(param1, test2())")).body.body + + scwc_res = get_test_obj(scwc, SCWC(UTN(""), UTN(""), UTN(""), UTN(""), SCN("", 0, 0))) + assert isinstance(scwc_res, SCWC) + expected = SCWC(UTN("test(", 0, 1), + UTN(")", 8, 8), + UTN("param1", 2, 2), + UTN(", ", 3, 4), + SCN("test2()", 5, 7)) + expected.start = 0 + expected.end = 8 + assert scwc_res == expected + + assert isinstance(scwc_res.first, UTN) + assert isinstance(scwc_res.last, UTN) + assert isinstance(scwc_res.content[0], UTN) + assert isinstance(scwc_res.content[1], UTN) + assert isinstance(scwc_res.content[2], SCN) + + def test_i_can_get_test_obj_when_RN(self): + rule = Rule(ACTION_TYPE_EXEC, "test_rule", "True", "True") + rn = RuleNode(rule, 1, 1, source="r:|xxx:") + + rn_res = get_test_obj(rn, RN("", "", 0, 1)) + + assert isinstance(rn_res, RN) + assert rn_res == RN(rule, rn.source, rn.start, rn.end) + + # I can discard start and end + rn_res = get_test_obj(rn, RN("", None, None, None)) + assert isinstance(rn_res, RN) + assert rn_res == RN(rule, None, None, None) + + def test_i_can_get_test_obj_when_CB(self): + a = Concept("a", key="a", body="10").auto_init() + b = Concept("b", key="b", body=a).auto_init() + c = Concept("c", key="c") + other_c = Concept("c", key="c", body="i don't care") + + # i can test when no body + res = get_test_obj(c, CB("", "")) + assert isinstance(res, CB) + assert res == CB("c", NotInit) + + # i can test with a concept instead of a key + res = get_test_obj(c, CB(Concept(), "")) + assert isinstance(res, CB) + assert res == CB(other_c, NotInit) # don't care if it's not the real 'c', only test key and body + + # # i can test when the body is a concept + res = get_test_obj(b, CB("", "")) + assert isinstance(res, CB) + assert res == CB("b", a) + + # i can go into recursion when body is a concept + res = get_test_obj(b, CB("", CB("", ""))) + assert isinstance(res, CB) + assert res == CB("b", CB("a", "10")) + + # I can try to go into recursion when there nothing to found + res = get_test_obj(c, CB("", CB("", ""))) + assert isinstance(res, CB) + assert res == CB("c", NotInit)