kodjo/Sheerka-Old
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fixed #32 : concept groups are not correctly updated

Browse Source 
Fixed #35 : Refactor test helper class (CNC, CC, CIO)
Fixed #36 : Concept values are not used when declared with variable expression
Fixed #37 : Objects in memory lose their values are restart
Fixed #38 : func(a=b, c) (which is not allowed) raise an exception
This commit is contained in:
Kodjo Sossouvi
2021-03-05 11:16:19 +01:00
parent 646c428edb
commit 05577012f3
38 changed files with 1942 additions and 1463 deletions
Download Patch File Download Diff File Expand all files Collapse all files
_concepts_numbers.txt
+5 -1
View File
@@ -89,4 +89,8 @@ def concept divided from a divided by b as a * b
set_is_greater_than(__PRECEDENCE, multiplied, plus, 'Sya')
set_is_greater_than(__PRECEDENCE, divided, plus, 'Sya')
set_is_greater_than(__PRECEDENCE, multiplied, minus, 'Sya')
set_is_greater_than(__PRECEDENCE, divided, minus, 'Sya')
set_is_greater_than(__PRECEDENCE, divided, minus, 'Sya')
def concept quantity
def concept quantify x from bnf number x as set_attr(x, quantity, number) ret x
def concept how many x pre is_question() as get_attr(memory(x), quantity)
src/core/builtin_helpers.py
+14 -2
View File
@@ -5,11 +5,11 @@ from cache.Cache import Cache
from core.ast_helpers import ast_to_props
from core.builtin_concepts import BuiltinConcepts
from core.concept import Concept, ConceptParts, DEFINITION_TYPE_BNF, concept_part_value
from core.global_symbols import NotInit, NotFound
from core.global_symbols import NotInit, NotFound, CURRENT_OBJ
from core.rule import Rule
from core.utils import as_bag
from parsers.BaseNodeParser import SourceCodeNode, ConceptNode, UnrecognizedTokensNode, SourceCodeWithConceptNode, \
RuleNode
RuleNode, VariableNode
from parsers.BaseParser import ParsingError
PARSE_STEPS = [BuiltinConcepts.BEFORE_PARSING, BuiltinConcepts.PARSING, BuiltinConcepts.AFTER_PARSING]
@@ -498,6 +498,18 @@ def get_lexer_nodes_from_unrecognized(context, unrecognized_tokens_node, parsers
:return:
"""
# first look into short term memory to see if the unrecognized is not a variable of the current object
if (current_obj := context.sheerka.get_from_short_term_memory(context, CURRENT_OBJ)) is not NotFound:
if isinstance(current_obj, Concept):
source = unrecognized_tokens_node.source
if source in current_obj.get_compiled() or source in current_obj.variables():
return [[VariableNode(current_obj,
source,
unrecognized_tokens_node.start,
unrecognized_tokens_node.end,
unrecognized_tokens_node.tokens,
unrecognized_tokens_node.source)]]
res = context.sheerka.parse_unrecognized(context, unrecognized_tokens_node.source, parsers)
res = only_parsers_results(context, res)
src/core/concept.py
+22 -291
View File
@@ -1,9 +1,7 @@
import hashlib
from collections import namedtuple
from copy import deepcopy
from dataclasses import dataclass
from threading import RLock
from typing import Union
import core.utils
from core.builtin_concepts_ids import BuiltinDynamicAttrs
@@ -68,14 +66,21 @@ all_attributes_lock = RLock()
def get_concept_attrs(concept):
# look for instance attributes
if concept.get_all_attributes() is not None:
return concept.get_all_attributes()
# look for class attributes defined within the concept (for real classes that inherit from Concept)
if concept.ALL_ATTRIBUTES is not None:
return concept.ALL_ATTRIBUTES
try:
# class attributes defined globally
return ALL_ATTRIBUTES[concept.id]
except KeyError:
pass
# create a class attribute
with all_attributes_lock:
all_attributes = [k for k in concept.__dict__ if k[0] != "_" and k[0] != "#"]
if concept.id and concept.key not in BuiltinDynamicAttrs:
@@ -97,7 +102,8 @@ def copy_concepts_attrs():
def load_concepts_attrs(attrs):
global ALL_ATTRIBUTES
with all_attributes_lock:
ALL_ATTRIBUTES = attrs
ALL_ATTRIBUTES.clear()
ALL_ATTRIBUTES.update(attrs)
class Concept:
@@ -107,7 +113,7 @@ class Concept:
Everything is a concept
"""
ALL_ATTRIBUTES = None
ALL_ATTRIBUTES = None # class attributes (only use when for Concept subclasses where it id is not yet defined)
def __init__(self, name=None,
is_builtin=False,
@@ -151,22 +157,16 @@ class Concept:
self._original_definition_hash = None # concept hash before any alteration of the metadata
self._format = None # how to print the concept
self._hints = {} # extra processing information to help processing
self._all_attributes = None # instance attributes
def __repr__(self):
text = f"({self._metadata.id}){self._metadata.name}"
return text + " (" + self._metadata.pre + ")" if self._metadata.pre else text
def __eq__(self, other):
if id(self) == id(other):
return True
if isinstance(other, simplec):
return self.name == other.name and self.body == other.body
if isinstance(other, (CC, CB, CV, CMV, CIO)):
return other == self
if not isinstance(other, Concept):
return False
@@ -208,14 +208,8 @@ class Concept:
def __hash__(self):
return hash(self._metadata.name)
# def __getattr__(self, item):
# # I have this complicated implementation because of the usage of Pickle
#
# if 'values' in vars(self) and item in self.values:
# return self.get_value(item)
#
# name = self.name if 'metadata' in vars(self) else 'Concept'
# raise AttributeError(f"'{name}' concept has no attribute '{item}'")
def get_all_attributes(self):
return self._all_attributes
def def_var(self, var_name, default_value=None):
"""
@@ -466,6 +460,15 @@ class Concept:
setattr(self, ConceptParts.BODY, value)
elif self._bound_body and name == ConceptParts.BODY:
setattr(self, self._bound_body, value)
# KSI 2021-03-04
# I am not sure how cost efficient it is to check for new attribute everytime
# Need to find a better way
if name not in get_concept_attrs(self):
if self._all_attributes is None:
self._all_attributes = get_concept_attrs(self).copy()
self._all_attributes.append(name)
except AttributeError:
print(f"Cannot set {name}")
return self
@@ -607,275 +610,3 @@ class InfiniteRecursionResolved:
def get_obj_value(self):
return self.value
# ################################
#
# Class created for tests purpose
#
# ################################
class CC:
"""
Concept class for test purpose
CC means concept for compiled (or concept with compiled)
It matches a concept if the compiles are equals
"""
# The only properties that are testes are concept_key and compiled
# The other properties (concept, source, start and end)
# are used in tests/parsers/parsers_utils.py to help creating helper objects
def __init__(self, concept, source=None, exclude_body=False, **kwargs):
self.concept_key = concept.key if isinstance(concept, Concept) else concept
self.compiled = kwargs
self.concept = concept if isinstance(concept, Concept) else None
self.source = source # to use when the key is different from the sub str to search when filling start and stop
self.start = None # for debug purpose, indicate where the concept starts
self.end = None # for debug purpose, indicate where the concept ends
self.exclude_body = exclude_body
if "body" in self.compiled:
self.compiled[ConceptParts.BODY] = self.compiled["body"]
del self.compiled["body"]
def __eq__(self, other):
if id(self) == id(other):
return True
if isinstance(other, Concept):
if other.key != self.concept_key:
return False
if self.exclude_body:
to_compare = {k: v for k, v in other.get_compiled().items() if k != ConceptParts.BODY}
else:
to_compare = other.get_compiled()
if self.compiled == to_compare:
return True
else:
return False
if not isinstance(other, CC):
return False
if self.concept_key != other.concept_key:
return False
return self.compiled == other.compiled
def __hash__(self):
if self.concept:
return hash(self.concept)
return hash(self.concept_key)
def __repr__(self):
if self.concept:
txt = f"CC(concept='{self.concept}'"
else:
txt = f"CC(concept_key='{self.concept_key}'"
for k, v in self.compiled.items():
txt += f", {k}='{v}'"
return txt + ")"
def fix_pos(self, node):
start = node.start if hasattr(node, "start") else \
node[0] if isinstance(node, tuple) else None
end = node.end if hasattr(node, "end") else \
node[1] if isinstance(node, tuple) else None
if start is not None:
if self.start is None or start < self.start:
self.start = start
if end is not None:
if self.end is None or end > self.end:
self.end = end
return self
def to_compare(self, other, to_compare_delegate):
"""
Transform other into CNC, to ease the comparison
:param other:
:param to_compare_delegate:
:return:
"""
if isinstance(other, CC):
return other
if isinstance(other, Concept):
if self.exclude_body:
compiled = {k: v for k, v in other.get_compiled().items() if k != ConceptParts.BODY}
else:
compiled = other.get_compiled()
self_compile_to_use = self.compiled or compiled
compiled = to_compare_delegate(self_compile_to_use, compiled, to_compare_delegate)
return CC(other,
self.source,
self.exclude_body,
**compiled)
raise NotImplementedError(f"CC, {other=}")
@dataclass()
class CB:
"""
Concept with body only
Test class that tests only the body of the concept
"""
concept: Union[str, Concept]
body: object
def __eq__(self, other):
if isinstance(other, Concept):
key = self.concept if isinstance(self.concept, str) else self.concept.key
return key == other.key and self.body == other.body
if not isinstance(other, CB):
return False
return self.concept == other.concept and self.body == other.body
def __hash__(self):
return hash((self.concept, self.body))
def __repr__(self):
return f"CB({self.body})"
class CV:
"""
Concept with all values
Test class that tests all the values (not the metadata, so not the properties) of a concept
"""
def __init__(self, concept, **kwargs):
self.concept_key = concept.key if isinstance(concept, Concept) else concept
self.concept = concept if isinstance(concept, Concept) else None
self.values = {}
for k, v in kwargs.items():
if f"#{k}#" in AllConceptParts:
self.values[f"#{k}#"] = v
else:
self.values[k] = v
def __eq__(self, other):
if isinstance(other, Concept):
if self.concept_key != other.key:
return False
for k, v in self.values.items():
if self.values[k] != other.get_value(k):
return False
return True
if not isinstance(other, CV):
return False
return self.concept_key == other.concept_key and self.values == other.values
def __hash__(self):
return hash((self.concept_key, self.values))
def __repr__(self):
return f"CV(key={self.concept_key}, values={self.values})"
class CMV:
"""
Concept with metadata variables
CMV stands for Concept Metadata Variables
Test class that only compare the key and the metadata variables
"""
def __init__(self, concept, **kwargs):
self.concept_key = concept.key if isinstance(concept, Concept) else concept
self.concept = concept if isinstance(concept, Concept) else None
self.variables = kwargs
def __eq__(self, other):
if id(self) == id(other):
return True
if isinstance(other, Concept):
if other.key != self.concept_key:
return False
if len(other._metadata.variables) != len(self.variables):
return False
for name, value in other._metadata.variables:
if self.variables[name] != value:
return False
return True
if not isinstance(other, CMV):
return False
if self.concept_key != other.concept_key:
return False
return self.variables == other.variables
def __hash__(self):
if self.concept:
return hash(self.concept)
return hash(self.concept_key)
def __repr__(self):
if self.concept:
txt = f"CMV(concept='{self.concept}'"
else:
txt = f"CMV(concept_key='{self.concept_key}'"
for k, v in self.variables.items():
txt += f", {k}='{v}'"
return txt + ")"
class CIO:
"""
Concept id only
only test the id
"""
def __init__(self, concept, source=None):
if isinstance(concept, str):
self.concept_name = concept
self.concept_id = None
self.concept = None
elif isinstance(concept, Concept):
self.concept_id = concept.id
self.concept = concept
self.source = source
self.start = None
self.end = None
def set_concept(self, concept):
self.concept = concept
self.concept_id = concept.id
def __eq__(self, other):
if id(self) == id(other):
return True
if isinstance(other, Concept):
return self.concept_id == other.id
if not isinstance(other, CIO):
return False
return self.concept_id == other.concept_id
def __hash__(self):
return hash(self.concept_id)
def __repr__(self):
return f"CIO(concept='{self.concept}')" if self.concept else f"CIO(name='{self.concept_name}')"
simplec = namedtuple("concept", "name body") # for simple concept (tests purposes only)
src/core/global_symbols.py
+6
View File
@@ -31,6 +31,9 @@ class CustomType:
def __eq__(self, other):
return isinstance(other, CustomType) and self.value == other.value
def __hash__(self):
return hash(self.value)
class NotInitType(CustomType):
def __init__(self):
@@ -63,3 +66,6 @@ class ErrorObj:
To indicate that somehow, the underlying object is (or has) an error
"""
pass
CURRENT_OBJ = "__obj"
src/core/sheerka/services/SheerkaAdmin.py
+13 -1
View File
@@ -35,13 +35,15 @@ class SheerkaAdmin(BaseService):
self.sheerka.bind_service_method(self.ontologies, False)
self.sheerka.bind_service_method(self.in_memory, False)
self.sheerka.bind_service_method(self.admin_history, False, as_name="history")
self.sheerka.bind_service_method(self.admin_history, False, as_name="history")
self.sheerka.bind_service_method(self.sdp, False)
def caches_names(self):
"""
Returns the name of all the caches
:return:
"""
return list(self.sheerka.om.current_cache_manager().caches.keys())
return self.sheerka.new(BuiltinConcepts.TO_LIST, body=self.sheerka.om.current_cache_manager().caches.keys())
def cache(self, name, *keys):
"""
@@ -58,6 +60,16 @@ class SheerkaAdmin(BaseService):
return {key: self.sheerka.om.get(name, key) for key in keys}
def sdp(self, name=None):
if name:
for ontology in self.sheerka.om.ontologies:
if ontology.name == name:
return ontology.cache_manager.sdp
return self.sheerka.err(self.sheerka.new(BuiltinConcepts.NOT_FOUND, {"sdp_name", name}))
return self.sheerka.om.current_sdp()
def restore(self, concept_file=CONCEPTS_FILE_TO_USE):
"""
Restore the state with all previous valid concept definitions
src/core/sheerka/services/SheerkaConceptManager.py
+130 -92
View File
@@ -237,36 +237,22 @@ class SheerkaConceptManager(BaseService):
except Exception as ex:
return sheerka.ret(self.NAME, False, ex.args[0])
# compute first token and/or first regex
init_ret_value = self.compute_concepts_by_first_item(context, [concept], True)
if not init_ret_value.status:
return sheerka.ret(self.NAME, False, ErrorConcept(init_ret_value.value))
by_first_keyword, by_first_regex = init_ret_value.body
# computes resolved concepts_by_first_keyword
init_ret_value = self.resolve_concepts_by_first_keyword(context, by_first_keyword)
if not init_ret_value.status:
return sheerka.ret(self.NAME, False, ErrorConcept(init_ret_value.value))
resolved_concepts_by_first_keyword = init_ret_value.body
# compile regex
compile_ret = self.compile_concepts_by_first_regex(context, by_first_regex)
if not compile_ret.status:
return sheerka.ret(self.NAME, False, ErrorConcept(compile_ret.value))
compiled_concepts_by_first_regex = compile_ret.body
# recompute concepts by first tokens and concept by first regex
update_items_res = self.recompute_first_items(context, None, [concept])
if not update_items_res.status:
return update_items_res
by_first_keyword, by_first_regex, resolved_by_first_keyword, compiled_by_first_regex = update_items_res.body
# if everything is fine
freeze_concept_attrs(concept)
concept.freeze_definition_hash()
om.add_concept(concept)
om.put(self.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, False, by_first_keyword)
om.put(self.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY, False, resolved_concepts_by_first_keyword)
om.put(self.CONCEPTS_BY_REGEX_ENTRY, False, {k.serialize(): v for k, v in by_first_regex.items()})
# update the compiled regex
self.compiled_concepts_by_regex.clear()
self.compiled_concepts_by_regex.extend(compiled_concepts_by_first_regex)
self.update_first_items_caches(context,
by_first_keyword,
by_first_regex,
resolved_by_first_keyword,
compiled_by_first_regex)
if concept.get_metadata().definition_type == DEFINITION_TYPE_DEF and concept.get_metadata().definition != concept.name:
# allow search by definition when definition relevant
@@ -284,9 +270,7 @@ class SheerkaConceptManager(BaseService):
# publish the new concept
sheerka.publish(context, EVENT_CONCEPT_CREATED, concept)
# process the return if needed
ret = sheerka.ret(self.NAME, True, sheerka.new(BuiltinConcepts.NEW_CONCEPT, body=concept))
return ret
return sheerka.ret(self.NAME, True, sheerka.new(BuiltinConcepts.NEW_CONCEPT, body=concept))
def modify_concept(self, context, concept, to_add=None, to_remove=None, modify_source=False):
"""
@@ -324,40 +308,17 @@ class SheerkaConceptManager(BaseService):
# modify the metadata. Almost all ConceptMetadata attributes except variables and props
new_concept = sheerka.new_from_template(concept, concept.key) # reload from cache or database ?
res = self._update_concept(context, new_concept, to_add, to_remove)
if res is not None:
return res
# To update concept by first keyword and first regex
# first remove old first token and first regex entries
concepts_by_first_keyword, concepts_by_regex = self._remove_concept_first_token_and_first_regex(concept)
# recompute concepts by first tokens and concept by first regex
update_items_res = self.recompute_first_items(context, concept, [new_concept])
if not update_items_res.status:
return update_items_res
by_first_keyword, by_first_regex, resolved_by_first_keyword, compiled_by_first_regex = update_items_res.body
# and then update
init_ret_value = self.compute_concepts_by_first_item(context,
[new_concept],
False,
concepts_by_first_keyword,
concepts_by_regex)
if not init_ret_value.status:
return sheerka.ret(self.NAME, False, ErrorConcept(init_ret_value.value))
concepts_by_first_keyword, concepts_by_regex = init_ret_value.body
# computes resolved concepts_by_first_keyword
init_ret_value = self.resolve_concepts_by_first_keyword(context,
concepts_by_first_keyword,
{new_concept.id: new_concept})
if not init_ret_value.status:
return sheerka.ret(self.NAME, False, ErrorConcept(init_ret_value.value))
resolved_concepts_by_first_keyword = init_ret_value.body
# compile new regex
compile_ret = self.compile_concepts_by_first_regex(context, concepts_by_regex)
if not compile_ret.status:
return sheerka.ret(self.NAME, False, ErrorConcept(compile_ret.value))
compiled_concepts_by_first_regex = compile_ret.body
# update concept that referenced the old concept and clear old references
# update concepts that referenced the old concept and clear old references
self.update_references(context, concept, new_concept, to_add)
for ref in self.compute_references(concept):
om.delete(self.CONCEPTS_REFERENCES_ENTRY, ref, concept.id)
@@ -368,13 +329,11 @@ class SheerkaConceptManager(BaseService):
# everything is ok, update the caches
om.update_concept(concept, new_concept)
om.put(self.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, False, concepts_by_first_keyword)
om.put(self.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY, False, resolved_concepts_by_first_keyword)
om.put(self.CONCEPTS_BY_REGEX_ENTRY, False, {k.serialize(): v for k, v in concepts_by_regex.items()})
# update the compiled regex
self.compiled_concepts_by_regex.clear()
self.compiled_concepts_by_regex.extend(compiled_concepts_by_first_regex)
self.update_first_items_caches(context,
by_first_keyword,
by_first_regex,
resolved_by_first_keyword,
compiled_by_first_regex)
# everything seems to be fine. Update the list of attributes
# Caution. Must be done AFTER update_concept()
@@ -386,8 +345,7 @@ class SheerkaConceptManager(BaseService):
self._update_concept(context, concept, to_add, to_remove)
# KSI 2021-02-16 publish the modification of the concept only when someone needs it
ret = sheerka.ret(self.NAME, True, sheerka.new(BuiltinConcepts.NEW_CONCEPT, body=new_concept))
return ret
return sheerka.ret(self.NAME, True, sheerka.new(BuiltinConcepts.NEW_CONCEPT, body=new_concept))
def remove_concept(self, context, concept):
"""
@@ -410,30 +368,19 @@ class SheerkaConceptManager(BaseService):
refs_instances = [sheerka.new_from_template(c, c.key) for c in [self.get_by_id(ref) for ref in refs]]
return sheerka.ret(self.NAME, False, sheerka.err(ConceptIsReferenced(refs_instances)))
concepts_by_first_keyword, concepts_by_regex = self._remove_concept_first_token_and_first_regex(concept)
# computes resolved concepts_by_first_keyword
init_ret_value = self.resolve_concepts_by_first_keyword(context, concepts_by_first_keyword)
if not init_ret_value.status:
return sheerka.ret(self.NAME, False, ErrorConcept(init_ret_value.value))
resolved_concepts_by_first_keyword = init_ret_value.body
# compile new regex
compile_ret = self.compile_concepts_by_first_regex(context, concepts_by_regex)
if not compile_ret.status:
return sheerka.ret(self.NAME, False, ErrorConcept(compile_ret.value))
compiled_concepts_by_first_regex = compile_ret.body
# recompute concepts by first tokens and concept by first regex
update_items_res = self.recompute_first_items(context, concept, None)
if not update_items_res.status:
return update_items_res
by_first_keyword, by_first_regex, resolved_by_first_keyword, compiled_by_first_regex = update_items_res.body
# everything seems fine. I can commit the modification and remove
om.remove_concept(concept)
om.put(self.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, False, concepts_by_first_keyword)
om.put(self.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY, False, resolved_concepts_by_first_keyword)
om.put(self.CONCEPTS_BY_REGEX_ENTRY, False, {k.serialize(): v for k, v in concepts_by_regex.items()})
# update the compiled regex
self.compiled_concepts_by_regex.clear()
self.compiled_concepts_by_regex.extend(compiled_concepts_by_first_regex)
self.update_first_items_caches(context,
by_first_keyword,
by_first_regex,
resolved_by_first_keyword,
compiled_by_first_regex)
sheerka.publish(context, EVENT_CONCEPT_DELETED, concept)
return sheerka.ret(self.NAME, True, sheerka.new(BuiltinConcepts.SUCCESS))
@@ -568,9 +515,9 @@ class SheerkaConceptManager(BaseService):
"""
Updates all the concepts that reference concept
:param context:
:param concept:
:param modified_concept:
:param modifications:
:param concept: Old version of the concept
:param modified_concept: new version of the concept
:param modifications: what are the modification
:return:
"""
@@ -582,11 +529,12 @@ class SheerkaConceptManager(BaseService):
# remove the grammar entry so that it can be recreated
self.sheerka.om.delete(self.CONCEPTS_BNF_DEFINITIONS_ENTRY, concept_id)
to_update = self.get_by_id(concept_id)
metadata = to_update.get_metadata()
# reset the bnf definition if needed
if modified_concept:
if self.has_id(concept_id):
to_update = self.get_by_id(concept_id)
metadata = to_update.get_metadata()
if self.has_id(concept_id): # reset only it the bnf definition is in cache
if metadata.definition_type == DEFINITION_TYPE_BNF and self._name_has_changed(modifications):
tokens = list(Tokenizer(metadata.definition))
modified = False
@@ -601,6 +549,19 @@ class SheerkaConceptManager(BaseService):
to_update.get_metadata().definition = core.utils.get_text_from_tokens(tokens)
to_update.set_bnf(None)
# update concept_by_first_token
if metadata.definition_type == DEFINITION_TYPE_BNF:
# recompute concepts by first tokens and concept by first regex
res = self.recompute_first_items(context, concept, [concept])
if not res.status:
return res
by_first_keyword, by_first_regex, resolved_by_first_keyword, compiled_by_first_regex = res.body
self.update_first_items_caches(context,
by_first_keyword,
by_first_regex,
resolved_by_first_keyword,
compiled_by_first_regex)
def compute_references(self, concept):
"""
We need to keep a track of all concepts used by the current concept
@@ -723,6 +684,8 @@ class SheerkaConceptManager(BaseService):
else:
return sheerka.ret(self.NAME, False, sheerka.err(UnknownAttribute(k)))
core.utils.remove_list_from_list(concept.get_metadata().variables, variables_to_remove)
if concept.get_all_attributes():
core.utils.remove_list_from_list(concept.get_all_attributes(), [v[0] for v in variables_to_remove])
concept.get_metadata().key = None
if self._definition_has_changed(to_add) and concept.get_metadata().definition_type == DEFINITION_TYPE_BNF:
@@ -1046,3 +1009,78 @@ class SheerkaConceptManager(BaseService):
def get_concepts_bnf_definitions(self):
return self.sheerka.om.current_cache_manager().caches[self.CONCEPTS_BNF_DEFINITIONS_ENTRY].cache
def recompute_first_items(self, context, old_concept, new_concepts):
"""
Recompute
concepts fy first items
resolved concept by first items
concepts by first regex
compiled concepts by first regex
Do not update anything
:param context:
:param old_concept:
:param new_concepts:
:return:
"""
sheerka = context.sheerka
if old_concept and not new_concepts:
# remove
modified_concepts = None
by_first_keyword, by_first_regex = self._remove_concept_first_token_and_first_regex(old_concept)
elif old_concept and new_concepts:
# remove
by_first_keyword, by_first_regex = self._remove_concept_first_token_and_first_regex(old_concept)
# and then update
init_ret_value = self.compute_concepts_by_first_item(context,
new_concepts,
False,
by_first_keyword,
by_first_regex)
if not init_ret_value.status:
return sheerka.ret(self.NAME, False, ErrorConcept(init_ret_value.value))
by_first_keyword, by_first_regex = init_ret_value.body
modified_concepts = {new_concept.id: new_concept for new_concept in new_concepts}
elif not old_concept and new_concepts:
# only update
modified_concepts = None
init_ret_value = self.compute_concepts_by_first_item(context, new_concepts, True)
if not init_ret_value.status:
return sheerka.ret(self.NAME, False, ErrorConcept(init_ret_value.value))
by_first_keyword, by_first_regex = init_ret_value.body
# computes resolved concepts_by_first_keyword
init_ret_value = self.resolve_concepts_by_first_keyword(context, by_first_keyword, modified_concepts)
if not init_ret_value.status:
return sheerka.ret(self.NAME, False, ErrorConcept(init_ret_value.value))
resolved_by_first_keyword = init_ret_value.body
# compile new regex
compile_ret = self.compile_concepts_by_first_regex(context, by_first_regex)
if not compile_ret.status:
return sheerka.ret(self.NAME, False, ErrorConcept(compile_ret.value))
compiled_by_first_regex = compile_ret.body
return sheerka.ret(self.NAME,
True,
(by_first_keyword, by_first_regex, resolved_by_first_keyword, compiled_by_first_regex))
def update_first_items_caches(self,
context,
by_first_keyword,
by_first_regex,
resolved_by_first_keyword,
compiled_by_first_regex):
om = context.sheerka.om
om.put(self.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, False, by_first_keyword)
om.put(self.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY, False, resolved_by_first_keyword)
om.put(self.CONCEPTS_BY_REGEX_ENTRY, False, {k.serialize(): v for k, v in by_first_regex.items()})
# update the compiled regex
self.compiled_concepts_by_regex.clear()
self.compiled_concepts_by_regex.extend(compiled_by_first_regex)
src/core/sheerka/services/SheerkaEvaluateConcept.py
+3 -1
View File
@@ -4,7 +4,7 @@ from core.builtin_concepts import BuiltinConcepts
from core.builtin_helpers import expect_one, only_successful, evaluate, ensure_concept
from core.concept import Concept, DoNotResolve, ConceptParts, InfiniteRecursionResolved, AllConceptParts, \
concept_part_value
from core.global_symbols import NotInit
from core.global_symbols import NotInit, CURRENT_OBJ
from core.rule import Rule
from core.sheerka.services.SheerkaConceptManager import SheerkaConceptManager
from core.sheerka.services.SheerkaExecute import ParserInput
@@ -535,6 +535,8 @@ class SheerkaEvaluateConcept(BaseService):
if context.sheerka.isa(concept, context.sheerka.new(BuiltinConcepts.AUTO_EVAL)):
sub_context.protected_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
sub_context.add_to_short_term_memory(CURRENT_OBJ, concept)
try:
self.initialize_concept_asts(sub_context, concept)
except ChickenAndEggException as ex:
src/core/sheerka/services/SheerkaIsAManager.py
+9 -10
View File
@@ -280,17 +280,16 @@ for x in xx__concepts__xx:
sub_context.protected_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
errors = []
for element_id in ids:
concept = self.sheerka.get_by_id(element_id)
if len(concept.get_metadata().variables) == 0:
# The concepts are directly taken from Sheerka.get_by_id, so variable cannot be filled
# It's the reason why we only evaluate concept with no variable
evaluated = self.sheerka.evaluate_concept(sub_context, concept)
if context.sheerka.is_success(evaluated):
result.append(evaluated)
concept = self.sheerka.fast_resolve((None, element_id))
if concept:
if len(concept.get_metadata().variables) == 0:
evaluated = self.sheerka.evaluate_concept(sub_context, concept)
if context.sheerka.is_success(evaluated):
result.append(evaluated)
else:
errors.append(evaluated)
else:
errors.append(evaluated)
else:
result.append(concept)
result.append(concept)
sub_context.add_values(return_value=result)
sub_context.add_values(errors=errors)
return result
src/core/sheerka/services/SheerkaMemory.py
+32 -4
View File
@@ -12,6 +12,15 @@ from core.sheerka.services.sheerka_service import BaseService, ServiceObj
class MemoryObject(ServiceObj):
obj: object
def __eq__(self, other):
if not isinstance(other, MemoryObject):
return False
return self.obj == other.obj and self.event_id == other.event_id
def __hash__(self):
return hash((self.event_id, self.obj))
class SheerkaMemory(BaseService):
NAME = "Memory"
@@ -26,6 +35,7 @@ class SheerkaMemory(BaseService):
def initialize(self):
self.sheerka.bind_service_method(self.get_from_short_term_memory, False, visible=False)
self.sheerka.bind_service_method(self.get_all_short_term_memory, False, visible=False)
self.sheerka.bind_service_method(self.add_to_short_term_memory, True, visible=False)
self.sheerka.bind_service_method(self.remove_context, True, as_name="clear_short_term_memory", visible=False)
self.sheerka.bind_service_method(self.add_to_memory, True, visible=False)
@@ -62,8 +72,26 @@ class SheerkaMemory(BaseService):
context = context.get_parent()
def get_all_short_term_memory(self, context):
return self.short_term_objects.get(context.id)
def get_all_short_term_memory(self, context, recursive=False):
id_to_use = context.id if context else self.GLOBAL
if not recursive:
return self.short_term_objects.get(id_to_use)
all_vars = {}
while True:
try:
all_vars.update(self.short_term_objects.cache[id_to_use])
except KeyError:
pass
if id_to_use == self.GLOBAL:
break
else:
context = context.get_parent()
id_to_use = context.id if context else self.GLOBAL
return all_vars
def add_to_short_term_memory(self, context, key, value):
if context:
@@ -99,12 +127,12 @@ class SheerkaMemory(BaseService):
if last is NotFound:
self.sheerka.om.put(SheerkaMemory.OBJECTS_ENTRY, key, MemoryObject(context.event.get_digest(), concept))
return
if not isinstance(last, list) and last.obj == concept:
self.sheerka.om.delete(SheerkaMemory.OBJECTS_ENTRY, key, last)
self.sheerka.om.put(SheerkaMemory.OBJECTS_ENTRY, key, MemoryObject(context.event.get_digest(), concept))
return
if isinstance(last, list) and last[-1].obj == concept:
self.sheerka.om.delete(SheerkaMemory.OBJECTS_ENTRY, key, last[-1])
self.sheerka.om.put(SheerkaMemory.OBJECTS_ENTRY, key, MemoryObject(context.event.get_digest(), concept))
src/core/utils.py
+1 -1
View File
@@ -444,7 +444,7 @@ def str_concept(t, drop_name=None, prefix="c:"):
elif prefix == "r:":
name, id_ = t.metadata.name, t.id
else:
name, id_ = t.key, t.id
name, id_ = t.key or t.name, t.id
if name is None and id_ is None:
return ""
src/core/var_ref.py
+32
View File
@@ -0,0 +1,32 @@
from core.concept import Concept
from core.rule import Rule
class VariableRef:
"""
Represents the reference to the property/attribute of an object
"""
def __init__(self, obj, prop):
self.obj = obj
self.prop = prop
def __eq__(self, other):
if not isinstance(other, VariableRef):
return False
return self.obj == other.obj and self.prop == other.prop
def __hash__(self):
return hash((self.obj, self.prop))
@property
def key(self):
return self.obj.key or self.obj.name if isinstance(self.obj, (Concept, Rule)) else type(self.obj).__name__
@property
def id(self):
if self.prop is None:
return ""
return self.prop
src/evaluators/PythonEvaluator.py
+5
View File
@@ -13,6 +13,7 @@ from core.rule import Rule
from core.sheerka.ExecutionContext import ExecutionContext
from core.sheerka.services.SheerkaMemory import SheerkaMemory
from core.tokenizer import Token, TokenKind
from core.var_ref import VariableRef
from evaluators.BaseEvaluator import OneReturnValueEvaluator
from parsers.PythonParser import PythonNode
@@ -332,6 +333,10 @@ class PythonEvaluator(OneReturnValueEvaluator):
:param name:
:return:
"""
if isinstance(name, VariableRef):
return getattr(name.obj, name.prop)
if isinstance(name, Rule):
return context.sheerka.resolve_rule(context, name)
src/parsers/BaseNodeParser.py
+44 -458
View File
@@ -1,11 +1,9 @@
from collections import namedtuple
from dataclasses import dataclass
from enum import Enum
import core.utils
from core.concept import Concept, ConceptParts
from core.rule import Rule
from core.tokenizer import TokenKind, Token
from core.var_ref import VariableRef
from parsers.BaseParser import Node, BaseParser, ParsingError
DEBUG_COMPILED = True
@@ -117,14 +115,6 @@ class UnrecognizedTokensNode(LexerNode):
return self.tokens[-1].type
def __eq__(self, other):
if isinstance(other, utnode):
return self.start == other.start and \
self.end == other.end and \
self.source == other.source
if isinstance(other, UTN):
return other == self
if not isinstance(other, UnrecognizedTokensNode):
return False
@@ -158,9 +148,6 @@ class RuleNode(LexerNode):
if id(self) == id(other):
return True
if isinstance(other, RN):
return other == self
if not isinstance(other, RuleNode):
return False
@@ -198,18 +185,6 @@ class ConceptNode(LexerNode):
if id(self) == id(other):
return True
if isinstance(other, (CN, CNC)):
return other == self
if isinstance(other, cnode):
return self.concept.key == other.concept_key and \
self.start == other.start and \
self.end == other.end and \
self.source == other.source
if isinstance(other, short_cnode):
return self.concept.key == other.concept_key and self.source == other.source
if not isinstance(other, ConceptNode):
return False
@@ -255,7 +230,8 @@ class SourceCodeNode(LexerNode):
Returned when some source code (like Python source code is recognized)
"""
def __init__(self, start, end, tokens=None, source=None, python_node=None, return_value=None):
def __init__(self, start, end, tokens=None, source=None,
python_node=None, return_value=None, error_when_parsing=None):
"""
:param start: start position (index of the first token)
@@ -269,18 +245,12 @@ class SourceCodeNode(LexerNode):
You should have return_value.body.body == node
"""
super().__init__(start, end, tokens, source)
self.python_node = python_node # The PythonNode (or whatever language node) that is found
self.return_value = return_value # original result of the parsing
self.error_when_parsing = error_when_parsing # if python_node is still None after parsing, it explains why
def __eq__(self, other):
if isinstance(other, scnode):
return self.start == other.start and \
self.end == other.end and \
self.source == other.source
if isinstance(other, SCN):
return other == self
if not isinstance(other, SourceCodeNode):
return False
@@ -336,6 +306,7 @@ class SourceCodeWithConceptNode(LexerNode):
self.python_node = None # if the source code node is validated against a python parse, here is the PythonNode
self.return_value = None # return_value that produced the PythonNode
self.error_when_parsing = None # if python_node is still None after parsing, it explains why
def add_node(self, node):
self.nodes.append(node)
@@ -348,9 +319,6 @@ class SourceCodeWithConceptNode(LexerNode):
if id(self) == id(other):
return True
if isinstance(other, SCWC):
return other == self
if not isinstance(other, SourceCodeWithConceptNode):
return False
@@ -436,6 +404,44 @@ class SourceCodeWithConceptNode(LexerNode):
return self.python_node.source
class VariableNode(LexerNode):
"""
When trying to parser source code, a reference to a variable is recognized
Not sure yet if it has to be a lexer node
"""
def __init__(self, obj, prop, start, end, tokens=None, source=None):
super().__init__(start, end, tokens, source)
self.var_ref = VariableRef(obj, prop)
def __eq__(self, other):
if id(self) == id(other):
return True
if not isinstance(other, VariableNode):
return False
return self.var_ref == other.var_ref and \
self.start == other.start and \
self.end == other.end and \
self.source == other.source
def __hash__(self):
return hash((self.var_ref.obj, self.var_ref.prop, self.start, self.end, self.source))
def __repr__(self):
ret = f"VariableNode(obj={self.var_ref.obj}, prop={self.var_ref.prop}, "
ret += f"start={self.start}, end={self.end}, source='{self.source}')"
return ret
def to_short_str(self):
return f"VN({self.var_ref.obj})" if self.var_ref.prop is None else f"VN({self.var_ref.obj}.{self.var_ref.prop})"
def clone(self):
clone = VariableNode(self.var_ref.obj, self.var_ref.prop, self.start, self.end, self.tokens, self.source)
return clone
@dataclass()
class GrammarErrorNode(ParsingError):
message: str
@@ -455,426 +461,6 @@ class SyaAssociativity(Enum):
return self.value
cnode = namedtuple("ConceptNode", "concept_key start end source")
short_cnode = namedtuple("ConceptNode", "concept_key source")
utnode = namedtuple("utnode", "start end source")
scnode = namedtuple("scnode", "start end source")
class HelperWithPos:
def __init__(self, start=None, end=None):
self.start = start
self.end = end
self.start_is_fixed = start is not None
self.end_is_fixed = end is not None
def fix_pos(self, node):
if not self.start_is_fixed:
start = node.start if hasattr(node, "start") else \
node[0] if isinstance(node, tuple) else None
if start is not None and (self.start is None or start < self.start):
self.start = start
if not self.end_is_fixed:
end = node.end if hasattr(node, "end") else \
node[1] if isinstance(node, tuple) else None
if end is not None and (self.end is None or end > self.end):
self.end = end
return self
class SCN(HelperWithPos):
"""
SourceCodeNode tester class
It matches with SourceCodeNode but with less constraints
SCN == SourceCodeNode if source, start, end (start and end are not validated when None)
"""
def __init__(self, source, start=None, end=None):
super().__init__(start, end)
self.source = source
def __eq__(self, other):
if id(self) == id(other):
return True
if isinstance(other, SourceCodeNode):
if self.source != other.source:
return False
if self.start is not None and self.start != other.start:
return False
if self.end is not None and self.end != other.end:
return False
return True
if not isinstance(other, CN):
return False
return self.source == other.source and \
self.start == other.start and \
self.end == other.end
def __hash__(self):
return hash((self.source, self.start, self.end))
def __repr__(self):
txt = f"SCN(source='{self.source}'"
if self.start is not None:
txt += f", start={self.start}"
if self.end is not None:
txt += f", end={self.end}"
return txt + ")"
class SCWC(HelperWithPos):
"""
SourceNodeWithConcept tester class
It matches with a SourceNodeWithConcept
but it's easier to instantiate during the tests
"""
def __init__(self, first, last, *args):
super().__init__(None, None)
self.first = first
self.last = last
self.content = args
def __eq__(self, other):
if id(self) == id(other):
return True
if isinstance(other, SourceCodeWithConceptNode):
if self.first != other.first:
return False
if self.last != other.last:
return False
if len(self.content) != len(other.nodes):
return False
for self_node, other_node in zip(self.content, other.nodes):
if self_node != other_node:
return False
# at last
return True
def __repr__(self):
txt = "SCWC("
if self.start is not None:
txt += f"start={self.start}"
if self.end is not None:
txt += f", end={self.end}"
txt += f", source='{self.source}'"
return txt + ")"
@property
def source(self):
"""
this code is a copy and paste from SourceCodeWithConceptNode.pseudo_fix_source
TODO: create a common function or whatever...
:return:
"""
source = self.first.source if hasattr(self.first, "source") else self.first
for n in self.content:
source += " "
if hasattr(n, "source"):
source += n.source
elif hasattr(n, "concept"):
source += str(n.concept)
else:
source += " unknown"
source += self.last.source if hasattr(self.last, "source") else self.last
return source
class CN(HelperWithPos):
"""
ConceptNode tester class
It matches with ConceptNode but with less constraints
CN == ConceptNode if concept key, start, end and source are the same
"""
def __init__(self, concept, start=None, end=None, source=None):
"""
:param concept: Concept or concept_key (only the key is used anyway)
:param start:
:param end:
:param source:
"""
super().__init__(start, end)
self.concept_key = concept.key if isinstance(concept, Concept) else concept
self.source = source
self.concept = concept if isinstance(concept, Concept) else None
def fix_source(self, str_tokens):
self.source = "".join(str_tokens)
return self
def __eq__(self, other):
if id(self) == id(other):
return True
if isinstance(other, ConceptNode):
if other.concept is None:
return False
if other.concept.key != self.concept_key:
return False
if self.start is not None and self.start != other.start:
return False
if self.end is not None and self.end != other.end:
return False
if self.source is not None and self.source != other.source:
return False
return True
if not isinstance(other, CN):
return False
return self.concept_key == other.concept_key and \
self.start == other.start and \
self.end == other.end and \
self.source == other.source
def __hash__(self):
return hash((self.concept_key, self.start, self.end, self.source))
def __repr__(self):
if self.concept:
txt = f"CN(concept='{self.concept}'"
else:
txt = f"CN(concept_key='{self.concept_key}'"
txt += f", source='{self.source}'"
if self.start is not None:
txt += f", start={self.start}"
if self.end is not None:
txt += f", end={self.end}"
return txt + ")"
class CNC(CN):
"""
ConceptNode for Compiled tester class
It matches with ConceptNode
But focuses on the 'compiled' property of the concept
CNC == ConceptNode if CNC.get_compiled() == ConceptNode.concept.get_compiled()
"""
def __init__(self, concept_key, start=None, end=None, source=None, exclude_body=False, **kwargs):
super().__init__(concept_key, start, end, source)
self.compiled = kwargs
self.exclude_body = exclude_body
if "body" in self.compiled:
self.compiled[ConceptParts.BODY] = self.compiled["body"]
del self.compiled["body"]
def __eq__(self, other):
if id(self) == id(other):
return True
if isinstance(other, ConceptNode):
if other.concept is None:
return False
if other.concept.key != self.concept_key:
return False
if self.start is not None and self.start != other.start:
return False
if self.end is not None and self.end != other.end:
return False
if self.source is not None and self.source != other.source:
return False
if self.exclude_body:
to_compare = {k: v for k, v in other.concept.get_compiled().items() if k != ConceptParts.BODY}
else:
to_compare = other.concept.get_compiled()
if self.compiled == to_compare: # expanded form to ease the debug
return True
else:
return False
if not isinstance(other, CNC):
return False
return self.concept_key == other.concept_key and \
self.start == other.start and \
self.end == other.end and \
self.source == other.source and \
self.compiled == other.compiled
def __repr__(self):
if self.concept:
txt = f"CNC(concept='{self.concept}'"
else:
txt = f"CNC(concept_key='{self.concept_key}'"
txt += f", source='{self.source}'"
if self.start is not None:
txt += f", start={self.start}"
if self.end is not None:
txt += f", end={self.end}"
for k, v in self.compiled.items():
txt += f", {k}='{v}'"
return txt + ")"
def to_compare(self, other, to_compare_delegate):
"""
Transform other into CNC, to ease the comparison
:param other:
:param to_compare_delegate:
:return:
"""
if isinstance(other, CNC):
return other
if isinstance(other, ConceptNode):
if self.exclude_body:
compiled = {k: v for k, v in other.concept.get_compiled().items() if k != ConceptParts.BODY}
else:
compiled = other.concept.get_compiled()
self_compile_to_use = self.compiled or compiled
compiled = to_compare_delegate(self_compile_to_use, compiled, to_compare_delegate)
return CNC(other.concept,
other.start if self.start is not None else None,
other.end if self.end is not None else None,
other.source if self.source is not None else None,
self.exclude_body,
**compiled)
raise NotImplementedError("CNC")
class UTN(HelperWithPos):
"""
Tester class for UnrecognizedTokenNode
compare the source, and start, end if defined
"""
def __init__(self, source, start=None, end=None):
"""
:param source:
:param start:
:param end:
"""
super().__init__(start, end)
self.source = source
def __eq__(self, other):
if id(self) == id(other):
return True
if isinstance(other, UnrecognizedTokensNode):
return self.start == other.start and \
self.end == other.end and \
self.source == other.source
if not isinstance(other, UTN):
return False
return self.start == other.start and \
self.end == other.end and \
self.source == other.source
def __hash__(self):
return hash((self.source, self.start, self.end))
def __repr__(self):
txt = f"UTN(source='{self.source}'"
if self.start is not None:
txt += f", start={self.start}"
if self.end is not None:
txt += f", end={self.end}"
return txt + ")"
def to_compare(self, other, to_compare_delegate):
"""
Transform other into CNC, to ease the comparison
:param other:
:param to_compare_delegate:
:return:
"""
if isinstance(other, UTN):
return other
if isinstance(other, UnrecognizedTokensNode):
return UTN(other.source,
other.start,
other.end)
raise NotImplementedError("UTN")
class RN(HelperWithPos):
"""
Helper class to test RuleNode
"""
def __init__(self, rule, start=None, end=None, source=None):
"""
:param concept: Concept or concept_key (only the key is used anyway)
:param start:
:param end:
:param source:
"""
super().__init__(start, end)
self.rule_id = rule.id if isinstance(rule, Rule) else rule
self.source = source or core.utils.str_concept((None, self.rule_id), prefix="r:")
self.rule = rule if isinstance(rule, Rule) else None
def __eq__(self, other):
if id(self) == id(other):
return True
if isinstance(other, RuleNode):
if other.rule is None:
return False
if other.rule.id != self.rule_id:
return False
if self.start is not None and self.start != other.start:
return False
if self.end is not None and self.end != other.end:
return False
if self.source is not None and self.source != other.source:
return False
return True
if not isinstance(other, RN):
return False
return self.rule_id == other.rule_id and \
self.start == other.start and \
self.end == other.end and \
self.source == other.source
def __hash__(self):
return hash((self.rule_id, self.start, self.end, self.source))
def __repr__(self):
if self.rule:
txt = f"RN(rule='{self.rule}'"
else:
txt = f"RN(rule_id='{self.rule_id}'"
txt += f", source='{self.source}'"
if self.start is not None:
txt += f", start={self.start}"
if self.end is not None:
txt += f", end={self.end}"
return txt + ")"
class BaseNodeParser(BaseParser):
"""
Parser that return LexerNode
src/parsers/BaseParser.py
+14 -7
View File
@@ -168,15 +168,22 @@ class BaseParser:
if expected_parser and parser_input.parser != expected_parser:
return None
if len(parser_input.value) == 0:
return None
for node in parser_input.value:
from parsers.BaseNodeParser import LexerNode
if not isinstance(node, LexerNode):
from parsers.BaseNodeParser import LexerNode
if isinstance(parser_input.value, list):
if len(parser_input.value) == 0:
return None
return parser_input.value
for node in parser_input.value:
if not isinstance(node, LexerNode):
return None
return parser_input.value
else:
if not isinstance(parser_input.value, LexerNode):
return None
return [parser_input.value]
@staticmethod
def get_tokens_boundaries(tokens):
src/parsers/FunctionParser.py
+43 -70
View File
@@ -53,59 +53,6 @@ class FunctionNode(FunctionParserNode):
parameters: list
class FN(FunctionNode):
"""
Test class only
It matches with FunctionNode but with less constraints
Thereby,
FN("first", "last", ["param1," ...]) can be compared to
FunctionNode(NameExprNode("first"), NameExprNode("second"), [FunctionParameter(NamesNodes("param1"), NamesNodes(", ")])
Note that FunctionParameter can easily be defined with a single string
* "param" -> FunctionParameter(NameExprNode("param"), None)
* "param, " -> FunctionParameter(NameExprNode("param"), NameExprNode(", "))
For more complicated situations, you can use a tuple (value, sep) to define the value part and the separator part
"""
def __init__(self, first, last, parameters):
self.first = first
self.last = last
self.parameters = []
for param in parameters:
if isinstance(param, tuple):
self.parameters.append(param)
elif isinstance(param, str) and (pos := param.find(",")) != -1:
self.parameters.append((param[:pos], param[pos:]))
else:
self.parameters.append((param, None))
def __eq__(self, other):
if id(self) == id(other):
return True
if isinstance(other, FN):
return self.first == other.first and self.last == other.last and self.parameters == other.parameters
if isinstance(other, FunctionNode):
if self.first != other.first.value or self.last != other.last.value:
return False
if len(self.parameters) != len(other.parameters):
return False
for self_parameter, other_parameter in zip(self.parameters, other.parameters):
value = other_parameter.value.value if isinstance(self_parameter[0], str) else other_parameter.value
sep = other_parameter.separator.value if other_parameter.separator else None
if self_parameter[0] != value or self_parameter[1] != sep:
return False
return True
return False
def __hash__(self):
return hash((self.first, self.last, self.parameters))
class FunctionParser(BaseParser):
"""
The parser will be used to parse func(x, y, z)
@@ -126,6 +73,17 @@ class FunctionParser(BaseParser):
self.longest_concepts_only = longest_concepts_only
self.record_errors = True
def function_parser_get_return_value_body(self, source_code_node):
if source_code_node.error_when_parsing:
return self.sheerka.new(BuiltinConcepts.ERROR,
body=source_code_node.error_when_parsing)
return self.sheerka.new(BuiltinConcepts.PARSER_RESULT,
parser=self,
source=self.parser_input.as_text(),
body=source_code_node,
try_parsed=source_code_node)
def add_error(self, error, next_token=True):
if not self.record_errors:
return
@@ -166,6 +124,9 @@ class FunctionParser(BaseParser):
[TokenKind.EOF]))
if self.has_error:
if len(self.error_sink) == 1 and isinstance(self.error_sink[0], Concept):
return self.error_sink[0]
if node is None:
body = context.sheerka.new(BuiltinConcepts.NOT_FOR_ME,
body=parser_input.as_text(),
@@ -178,12 +139,8 @@ class FunctionParser(BaseParser):
res = []
for source_code_node in source_code_nodes:
value = self.get_return_value_body(context.sheerka,
self.parser_input.as_text(),
source_code_node,
source_code_node)
res.append(self.sheerka.ret(self.name, source_code_node.python_node is not None, value))
body = self.function_parser_get_return_value_body(source_code_node)
res.append(self.sheerka.ret(self.name, source_code_node.python_node is not None, body))
return res[0] if len(res) == 1 else res
@@ -288,6 +245,25 @@ class FunctionParser(BaseParser):
def to_source_code_node(self, function_node: FunctionNode):
python_parser = PythonWithConceptsParser()
def update_source_code_node(scn, nodes, sep):
if hasattr(nodes, "__iter__"):
for n in nodes:
scn.add_node(n)
else:
scn.add_node(nodes)
if sep:
scn.add_node(sep.to_unrecognized())
def get_errors_from_python_parsing(parsing_res):
if parsing_res.status:
return None
if self.sheerka.isinstance(parsing_res.body, BuiltinConcepts.NOT_FOR_ME):
return parsing_res.body.reason
else:
return parsing_res.body.body
if len(function_node.parameters) == 0:
# validate the source
nodes_to_parse = [function_node.first.to_unrecognized(), function_node.last.to_unrecognized()]
@@ -298,17 +274,8 @@ class FunctionParser(BaseParser):
end=function_node.last.end,
tokens=function_node.first.tokens + function_node.last.tokens,
python_node=python_node,
return_value=python_parsing_res)]
def update_source_code_node(scn, nodes, sep):
if hasattr(nodes, "__iter__"):
for n in nodes:
scn.add_node(n)
else:
scn.add_node(nodes)
if sep:
scn.add_node(sep.to_unrecognized())
return_value=python_parsing_res,
error_when_parsing=get_errors_from_python_parsing(python_parsing_res))]
res = [SourceCodeWithConceptNode(function_node.first.to_unrecognized(), function_node.last.to_unrecognized())]
@@ -363,6 +330,12 @@ class FunctionParser(BaseParser):
for c in [c for c in source_code_node.python_node.objects.values() if isinstance(c, Concept)]:
update_compiled(self.context, c, errors)
if errors:
source_code_node.error_when_parsing = errors
else:
source_code_node.error_when_parsing = get_errors_from_python_parsing(python_parsing_res)
return res
@staticmethod
src/parsers/PythonWithConceptsParser.py
+14 -1
View File
@@ -1,6 +1,6 @@
from core.builtin_concepts import BuiltinConcepts
from core.builtin_helpers import CreateObjectIdentifiers
from parsers.BaseNodeParser import ConceptNode, RuleNode
from parsers.BaseNodeParser import ConceptNode, RuleNode, VariableNode
from parsers.BaseNodeParser import SourceCodeWithConceptNode
from parsers.BaseParser import BaseParser
from parsers.UnrecognizedNodeParser import UnrecognizedNodeParser
@@ -64,6 +64,19 @@ class PythonWithConceptsParser(BaseParser):
python_ids_mappings[python_id] = rule
last_token_index = node.end
elif isinstance(node, VariableNode):
if node.start != last_token_index + 1 and source: # put back missing whitespace
source += " "
to_parse += " "
source += node.source
var_ref = node.var_ref
python_id = ids_manager.get_identifier(var_ref, "__V__")
to_parse += python_id
python_ids_mappings[python_id] = var_ref
last_token_index = node.end
else:
source += node.source
to_parse += node.get_source_to_parse()
src/parsers/SyaNodeParser.py
+2 -2
View File
@@ -12,7 +12,7 @@ from core.sheerka.services.SheerkaExecute import ParserInput
from core.tokenizer import Token, TokenKind, Tokenizer
from core.utils import get_n_clones, get_text_from_tokens, NextIdManager
from parsers.BaseNodeParser import UnrecognizedTokensNode, ConceptNode, SourceCodeNode, SyaAssociativity, \
SourceCodeWithConceptNode, BaseNodeParser
SourceCodeWithConceptNode, BaseNodeParser, VariableNode
from parsers.BaseParser import ParsingError
PARSERS = ["Sequence", "Bnf", "Python"]
@@ -1262,7 +1262,7 @@ class SyaNodeParser(BaseNodeParser):
def postfix_to_item(self, sheerka, postfixed):
item = postfixed.pop()
if isinstance(item, (UnrecognizedTokensNode, SourceCodeNode, ConceptNode)):
if isinstance(item, (UnrecognizedTokensNode, SourceCodeNode, ConceptNode, VariableNode)):
return item
if isinstance(item, SourceCodeWithConceptNode):
src/sdp/sheerkaDataProvider.py
+10
View File
@@ -235,6 +235,16 @@ class SheerkaDataProvider:
def get_transaction(self, event) -> SheerkaDataProviderTransaction:
return SheerkaDataProviderTransaction(self, event)
def get_ref(self, entry, key):
self.log.debug(f"getting object ref_id {entry=}, {key=}")
if entry not in self.state.data:
return NotFound
if key not in self.state.data[entry]:
return NotFound
return self.state.data[entry][key]
def get(self, entry, key=None, default=NotFound, load_origin=True):
"""
Get an element
tests/core/test_SheerkaConceptManager.py
+3
View File
@@ -487,6 +487,9 @@ class TestSheerkaConceptManager(TestUsingMemoryBasedSheerka):
assert get_concept_attrs(foo) == ["b", "c"]
assert get_concept_attrs(new_concept) == ["b", "c"]
new_foo = sheerka.new(foo)
assert get_concept_attrs(new_foo) == ["b", "c"]
def test_key_is_modified_when_modifying_name_or_variables(self):
sheerka, context, foo = self.init_concepts(Concept("foo a b").def_var("a").def_var("b"))
tests/core/test_SheerkaEvaluateConcept.py
+14 -13
View File
@@ -1,7 +1,7 @@
import pytest
from core.builtin_concepts import BuiltinConcepts, ReturnValueConcept, ParserResultConcept
from core.concept import Concept, DoNotResolve, ConceptParts, InfiniteRecursionResolved, CB, \
from core.concept import Concept, DoNotResolve, ConceptParts, InfiniteRecursionResolved, \
concept_part_value, DEFINITION_TYPE_DEF
from core.global_symbols import NotInit, NotFound
from core.sheerka.services.SheerkaEvaluateConcept import SheerkaEvaluateConcept
@@ -10,6 +10,7 @@ from parsers.BaseParser import BaseParser
from parsers.PythonParser import PythonNode, PythonParser
from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
from tests.evaluators.EvaluatorTestsUtils import pr_ret_val, python_ret_val
from tests.parsers.parsers_utils import CB, compare_with_test_object
class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
@@ -82,7 +83,7 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
("True", True),
("1 > 2", False),
])
def test_i_can_evaluate_a_concept_with_prop(self, expr, expected):
def test_i_can_evaluate_a_concept_with_variable(self, expr, expected):
sheerka, context, concept = self.init_concepts(Concept("foo").def_var("a", expr), eval_body=True)
evaluated = sheerka.evaluate_concept(context, concept)
@@ -141,7 +142,7 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
evaluated = sheerka.evaluate_concept(context, concept)
assert evaluated == CB(concept, 11)
compare_with_test_object(evaluated, CB(concept, 11))
def test_i_can_evaluate_when_another_concept_is_referenced(self):
sheerka, context, concept_a, concept = self.init_concepts(
@@ -151,7 +152,7 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
evaluated = sheerka.evaluate_concept(context, concept)
assert evaluated == CB("foo", CB("a", NotInit))
compare_with_test_object(evaluated, CB("foo", CB("a", NotInit)))
assert evaluated.get_metadata().is_evaluated
assert evaluated.body.get_metadata().is_evaluated
@@ -164,7 +165,7 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
evaluated = sheerka.evaluate_concept(context, concept)
assert evaluated.key == concept.key
assert evaluated.body == CB("a", 1)
compare_with_test_object(evaluated.body, CB("a", 1))
assert not concept_a.get_metadata().is_evaluated
assert evaluated.get_metadata().is_evaluated
@@ -180,7 +181,7 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
assert evaluated.key == concept_d.key
expected = CB("c", CB("b", CB("a", "a")))
assert evaluated.body == expected
compare_with_test_object(evaluated.body, expected)
assert sheerka.objvalue(evaluated) == 'a'
assert evaluated.get_metadata().is_evaluated
@@ -196,8 +197,8 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
assert evaluated.key == concept_d.key
expected = CB("c", CB("b", CB("a", NotInit)))
assert evaluated.body == expected
assert sheerka.objvalue(evaluated) == CB("a", NotInit)
compare_with_test_object(evaluated.body, expected)
compare_with_test_object(sheerka.objvalue(evaluated), CB("a", NotInit))
assert evaluated.get_metadata().is_evaluated
def test_i_can_evaluate_concept_when_variables_reference_others_concepts_1(self):
@@ -283,7 +284,7 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
evaluated = sheerka.evaluate_concept(context, concept)
assert evaluated.key == concept.key
assert evaluated.get_value("a") == CB("a", "a")
compare_with_test_object(evaluated.get_value("a"), CB("a", "a"))
def test_i_can_evaluate_concept_when_variable_is_a_concept_token(self):
sheerka, context, concept_a, concept_b = self.init_concepts(
@@ -317,7 +318,7 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
variables = evaluated.get_value("prop")
assert len(variables) == 2
assert variables[0] == CB("foo", 1)
compare_with_test_object(variables[0], CB("foo", 1))
assert variables[1] == "1"
def test_i_can_evaluate_when_compiled_is_set_up_with_return_value(self):
@@ -419,7 +420,7 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
concept = Concept("foo", body="a")
evaluated = sheerka.evaluate_concept(context, concept)
assert evaluated.key == concept.key
assert evaluated.body == CB("a", "concept_a") # this test was already done
compare_with_test_object(evaluated.body, CB("a", "concept_a")) # this test was already done
# so check this one.
concept = Concept("foo", body="a").def_var("a", "'property_a'")
@@ -431,7 +432,7 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
concept = Concept("foo", body="a").def_var("a", "b")
evaluated = sheerka.evaluate_concept(context, concept)
assert evaluated.key == concept.key
assert evaluated.body == CB("b", "concept_b")
compare_with_test_object(evaluated.body, CB("b", "concept_b"))
def test_variables_values_takes_precedence(self):
sheerka, context, concept_a, concept_b = self.init_concepts(
@@ -453,7 +454,7 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
concept = Concept("foo", body="a.subProp").def_var("a", "concept_a")
evaluated = sheerka.evaluate_concept(context, concept)
assert evaluated == CB(concept.key, "sub_a")
compare_with_test_object(evaluated, CB(concept.key, "sub_a"))
def test_i_cannot_evaluate_concept_if_variable_is_in_error(self):
sheerka = self.get_sheerka()
tests/core/test_SheerkaIsAManager.py
+17 -4
View File
@@ -120,7 +120,10 @@ class TestSheerkaIsAManager(TestUsingMemoryBasedSheerka):
service.add_concepts_to_set(context, [one, two, three, four, five], number)
assert sheerka.isaset(context, sub_number)
assert set(sheerka.get_set_elements(context, sub_number)) == {one, two, three}
# compare ids, as concepts are evaluated in get_set_elements
actual_ids = set(c.id for c in sheerka.get_set_elements(context, sub_number))
expected_ids = set(c.id for c in [one, two, three])
assert actual_ids == expected_ids
def test_i_can_define_subset_of_subset(self):
sheerka, context, one, two, three, four, five, number, sub_number, sub_sub_number = self.init_concepts(
@@ -137,7 +140,11 @@ class TestSheerkaIsAManager(TestUsingMemoryBasedSheerka):
service.add_concepts_to_set(context, [one, two, three, four, five], number)
assert sheerka.isaset(context, sub_sub_number)
assert set(sheerka.get_set_elements(context, sub_sub_number)) == {three}
# compare ids, as concepts are evaluated in get_set_elements
actual_ids = set(c.id for c in sheerka.get_set_elements(context, sub_sub_number))
expected_ids = set(c.id for c in [three])
assert actual_ids == expected_ids
def test_i_can_define_subset_of_another_set_when_some_concept_do_not_have_a_defined_body(self):
"""
@@ -161,7 +168,10 @@ class TestSheerkaIsAManager(TestUsingMemoryBasedSheerka):
service.add_concepts_to_set(context, [one, two, three, four, five], number)
assert sheerka.isaset(context, sub_number)
assert set(sheerka.get_set_elements(context, sub_number)) == {one, three}
# compare ids, as concepts are evaluated in get_set_elements
actual_ids = set(c.id for c in sheerka.get_set_elements(context, sub_number))
expected_ids = set(c.id for c in [one, three])
assert actual_ids == expected_ids
def test_i_can_define_subset_of_another_set_when_some_concept_are_bnf(self):
"""
@@ -183,7 +193,10 @@ class TestSheerkaIsAManager(TestUsingMemoryBasedSheerka):
service.add_concepts_to_set(context, [one, two, twenty, twenties], number)
assert sheerka.isaset(context, sub_number)
assert sheerka.get_set_elements(context, sub_number) == [twenty] # what is expected ?
# compare ids, as concepts are evaluated in get_set_elements
actual_ids = set(c.id for c in sheerka.get_set_elements(context, sub_number))
expected_ids = set(c.id for c in [twenty])
assert actual_ids == expected_ids
def test_bnf_elements_can_be_part_of_a_set(self):
"""
tests/core/test_SheerkaMemory.py
+24 -1
View File
@@ -1,4 +1,5 @@
from core.builtin_concepts import BuiltinConcepts
from core.builtin_helpers import ensure_evaluated
from core.concept import Concept
from core.global_symbols import NotFound
from core.sheerka.ExecutionContext import ExecutionContext
@@ -6,6 +7,7 @@ from core.sheerka.services.SheerkaMemory import SheerkaMemory, MemoryObject
from tests.TestUsingFileBasedSheerka import TestUsingFileBasedSheerka
from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
from tests.parsers.parsers_utils import CV, compare_with_test_object, CB
class TestSheerkaMemory(TestUsingMemoryBasedSheerka):
@@ -31,7 +33,7 @@ class TestSheerkaMemory(TestUsingMemoryBasedSheerka):
assert id(sheerka.get_from_short_term_memory(context, "a")) == id(foo)
assert sheerka.get_from_short_term_memory(None, "a") is NotFound
def test_i_can_add_many(self):
def test_i_can_add_many_to_short_term_memory(self):
sheerka, context = self.init_test().unpack()
bag = {"a": "foo", "b": "bar", }
context_id = ExecutionContext.ids[context.event.get_digest()]
@@ -191,6 +193,27 @@ class TestSheerkaMemory(TestUsingMemoryBasedSheerka):
assert from_memory[-1].obj == from_memory_again[-1].obj
assert from_memory[-1].event_id != from_memory_again[-1].event_id
def test_object_values_are_recorded(self):
sheerka, context, foo, prop, value, inner_value = self.init_test(cache_only=False).with_concepts(
Concept("foo").def_var("x"),
Concept("prop"),
Concept("value x").def_var("x"),
Concept("one", body="1")
).unpack()
instantiated_foo = sheerka.new(foo, x="value1")
instantiated_one = ensure_evaluated(context, sheerka.new(inner_value))
instantiated_value = sheerka.new(value, x=instantiated_one)
sheerka.set_attr(instantiated_foo, prop, instantiated_value)
sheerka.add_to_memory(context, "foo", instantiated_foo)
sheerka.om.commit(context)
from_db = sheerka.om.current_sdp().get(SheerkaMemory.OBJECTS_ENTRY, "foo")
assert isinstance(from_db, MemoryObject)
assert sheerka.isinstance(from_db.obj, foo)
compare_with_test_object(sheerka.get_attr(from_db.obj, prop), CV(value, x=CB(inner_value, 1)))
class TestSheerkaMemoryUsingFileBase(TestUsingFileBasedSheerka):
def test_i_can_record_memory_objects(self):
tests/core/test_SheerkaRuleManager.py
+13 -10
View File
@@ -3,7 +3,7 @@ import ast
import pytest
from core.builtin_concepts import BuiltinConcepts
from core.concept import Concept, CMV, DEFINITION_TYPE_DEF, CC, DoNotResolve
from core.concept import Concept, DEFINITION_TYPE_DEF, DoNotResolve
from core.global_symbols import RULE_COMPARISON_CONTEXT, NotFound, EVENT_RULE_DELETED
from core.rule import Rule, ACTION_TYPE_PRINT, ACTION_TYPE_EXEC
from core.sheerka.Sheerka import RECOGNIZED_BY_ID, RECOGNIZED_BY_NAME
@@ -20,6 +20,7 @@ from sheerkarete.common import V
from sheerkarete.conditions import Condition, AndConditions
from tests.TestUsingFileBasedSheerka import TestUsingFileBasedSheerka
from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
from tests.parsers.parsers_utils import CMV, CC, compare_with_test_object, get_test_obj
seq = FormatAstSequence
raw = FormatAstRawText
@@ -494,8 +495,8 @@ class TestSheerkaRuleManager(TestUsingMemoryBasedSheerka):
assert isinstance(res[0], RuleCompiledPredicate)
assert res[0].evaluator == CONCEPT_EVALUATOR_NAME
assert sheerka.isinstance(res[0].predicate, BuiltinConcepts.RETURN_VALUE)
assert sheerka.objvalue(res[0].predicate)[0].concept == expected
assert res[0].concept == expected
compare_with_test_object(sheerka.objvalue(res[0].predicate)[0].concept, expected)
compare_with_test_object(res[0].concept, expected)
def test_i_can_compile_predicate_when_bnf_node_parser(self):
sheerka, context, *concepts = self.init_test().with_concepts(
@@ -540,11 +541,13 @@ class TestSheerkaRuleManager(TestUsingMemoryBasedSheerka):
assert isinstance(res[0], RuleCompiledPredicate)
python_node = res[0].predicate.body.body
assert python_node == expected_python_node
assert python_node.objects == {
expected = {
"__C__00var0000is0a000var001__1005__C__": CC(is_a, x=cat, y=pet),
"__C__00var0000is0an0y__1006__C__": CC(is_an, exclude_body=True, x=DoNotResolve("bird"), animal=animal),
"__C__00var0000is0a000var001__1005_1__C__": CMV(is_a, x="dog", y="pet"),
}
transformed = get_test_obj(python_node.objects, expected)
assert transformed == expected
@pytest.mark.parametrize("text", [
"a and not b",
@@ -595,11 +598,11 @@ class TestSheerkaRuleManager(TestUsingMemoryBasedSheerka):
assert isinstance(res[0], RuleCompiledPredicate)
python_node = res[0].predicate.body.body
assert python_node == expected_python_node
assert python_node.objects == {
compare_with_test_object(python_node.objects, {
"__C__00var0000is0a000var001__1005__C__": CC(is_a, x=cat, y=pet),
"__C__00var0000is0an0y__1006__C__": CC(is_an, exclude_body=True, x=DoNotResolve("bird"), animal=animal),
"__C__00var0000is0a000var001__1005_1__C__": CMV(is_a, x="dog", y="pet"),
}
})
def test_i_can_compile_predicate_when_multiple_choices(self):
sheerka, context, *concepts = self.init_test().with_concepts(
@@ -616,14 +619,14 @@ class TestSheerkaRuleManager(TestUsingMemoryBasedSheerka):
assert isinstance(res[0], RuleCompiledPredicate)
assert res[0].evaluator == CONCEPT_EVALUATOR_NAME
assert sheerka.isinstance(res[0].predicate, BuiltinConcepts.RETURN_VALUE)
assert sheerka.objvalue(res[0].predicate)[0].concept == CMV(concepts[0], x="a", y="b")
assert res[0].concept == CMV(concepts[0], x="a", y="b")
compare_with_test_object(sheerka.objvalue(res[0].predicate)[0].concept, CMV(concepts[0], x="a", y="b"))
compare_with_test_object(res[0].concept, CMV(concepts[0], x="a", y="b"))
assert isinstance(res[1], RuleCompiledPredicate)
assert res[1].evaluator == CONCEPT_EVALUATOR_NAME
assert sheerka.isinstance(res[1].predicate, BuiltinConcepts.RETURN_VALUE)
assert sheerka.objvalue(res[1].predicate)[0].concept == CMV(concepts[1], x="a", y="b")
assert res[1].concept == CMV(concepts[1], x="a", y="b")
compare_with_test_object(sheerka.objvalue(res[1].predicate)[0].concept, CMV(concepts[1], x="a", y="b"))
compare_with_test_object(res[1].concept, CMV(concepts[1], x="a", y="b"))
def test_i_can_compile_predicate_when_mix_and_multiple_choices(self):
sheerka, context, *concepts = self.init_test().with_concepts(
tests/core/test_concept.py
+20 -12
View File
@@ -1,7 +1,9 @@
import pytest
from core.builtin_concepts import BuiltinConcepts
from core.concept import Concept, ConceptParts, DEFINITION_TYPE_DEF, ALL_ATTRIBUTES, get_concept_attrs
from core.builtin_concepts import BuiltinConcepts
from core.concept import Concept, ConceptParts, DEFINITION_TYPE_DEF, ALL_ATTRIBUTES, get_concept_attrs, \
freeze_concept_attrs
from core.global_symbols import NotInit
@pytest.mark.parametrize("name, variables, expected", [
@@ -121,6 +123,7 @@ def test_i_can_deserialize():
}
concept = Concept().from_dict(from_dict)
freeze_concept_attrs(concept)
assert concept == Concept(
name="concept_name",
@@ -281,16 +284,21 @@ def test_i_can_manage_concepts_attributes():
assert concept.values() == {"#body#": "I have a body!"}
def test_attributes_are_generated_once_for_all():
def test_i_can_manage_instance_attributes():
ALL_ATTRIBUTES.clear()
concept = Concept("foo")
concept.get_metadata().id = "id"
concept.set_value("key1", "value1")
concept.set_value("key2", "value2")
assert get_concept_attrs(concept) == ["key1", "key2"]
assert concept.values() == {"key1": "value1", "key2": "value2"}
foo = Concept("foo", id="foo_id").def_var("x")
concept.set_value("key3", "value3") # too late for it !
assert get_concept_attrs(concept) == ["key1", "key2"]
assert concept.values() == {"key1": "value1", "key2": "value2"}
assert foo.values() == {"x": NotInit}
assert foo.get_all_attributes() is None
assert ALL_ATTRIBUTES == {"foo_id": ["x"]}
foo.set_value("x", "value for x")
assert foo.values() == {"x": "value for x"}
assert foo.get_all_attributes() is None
assert ALL_ATTRIBUTES == {"foo_id": ["x"]}
foo.set_value("y", "value for y")
assert foo.values() == {"x": "value for x", "y": "value for y"}
assert foo.get_all_attributes() == ["x", "y"]
assert ALL_ATTRIBUTES == {"foo_id": ["x"]}
tests/evaluators/test_PythonEvaluator.py
+5 -4
View File
@@ -4,7 +4,7 @@ import pytest
from core.builtin_concepts import ReturnValueConcept, ParserResultConcept, BuiltinConcepts
from core.builtin_helpers import CreateObjectIdentifiers
from core.concept import Concept, CB
from core.concept import Concept
from core.sheerka.services.SheerkaConceptManager import SheerkaConceptManager
from core.sheerka.services.SheerkaExecute import ParserInput
from core.tokenizer import Tokenizer
@@ -13,6 +13,7 @@ from parsers.BaseNodeParser import SourceCodeNode, SourceCodeWithConceptNode
from parsers.FunctionParser import FunctionParser
from parsers.PythonParser import PythonNode, PythonParser
from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
from tests.parsers.parsers_utils import CB, compare_with_test_object
def get_obj_name(obj):
@@ -178,7 +179,7 @@ class TestPythonEvaluator(TestUsingMemoryBasedSheerka):
evaluated = PythonEvaluator().eval(context, parsed)
assert evaluated.status
assert evaluated.value == CB("foo", 2)
compare_with_test_object(evaluated.value, CB("foo", 2))
def test_i_can_eval_concept_token(self):
context = self.get_context()
@@ -253,7 +254,7 @@ class TestPythonEvaluator(TestUsingMemoryBasedSheerka):
all_globals = python_evaluator.get_all_possible_globals(context, my_globals)
assert len(all_globals) == 2
assert all_globals[0]["foo"] == CB(foo, "foo")
compare_with_test_object(all_globals[0]["foo"], CB(foo, "foo"))
assert all_globals[1]["foo"] == 'foo' # body is evaluated
def test_i_can_detect_one_error(self):
@@ -284,7 +285,7 @@ class TestPythonEvaluator(TestUsingMemoryBasedSheerka):
assert isinstance(error0, PythonEvalError)
assert isinstance(error0.error, TypeError)
assert error0.error.args[0] == "unsupported operand type(s) for +: 'Concept' and 'int'"
assert error0.concepts == {'foo': CB(foo, 'string')}
compare_with_test_object(error0.concepts, {'foo': CB(foo, 'string')})
error1 = evaluated.body.body[1]
assert isinstance(error1, PythonEvalError)
tests/non_reg/test_sheerka_non_reg.py
+56 -6
View File
@@ -1,7 +1,7 @@
import pytest
from core.builtin_concepts import BuiltinConcepts
from core.concept import Concept, PROPERTIES_TO_SERIALIZE, simplec, CMV, CC
from core.concept import Concept, PROPERTIES_TO_SERIALIZE
from core.global_symbols import NotInit
from core.sheerka.services.SheerkaConceptManager import SheerkaConceptManager
from evaluators.MutipleSameSuccessEvaluator import MultipleSameSuccessEvaluator
@@ -10,6 +10,7 @@ from evaluators.PythonEvaluator import PythonEvalError
from parsers.BnfNodeParser import Sequence, StrMatch, OrderedChoice, Optional, ConceptExpression
from tests.TestUsingFileBasedSheerka import TestUsingFileBasedSheerka
from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
from tests.parsers.parsers_utils import CMV, CC, compare_with_test_object, CB
class TestSheerkaNonRegMemory(TestUsingMemoryBasedSheerka):
@@ -40,7 +41,7 @@ class TestSheerkaNonRegMemory(TestUsingMemoryBasedSheerka):
# sanity check
evaluated = sheerka.evaluate_concept(self.get_context(eval_body=True), res[0].value)
assert evaluated == simplec("one", 1)
compare_with_test_object(evaluated, CB("one", 1))
def test_i_can_recognize_concept_with_concept_body(self):
sheerka, context, concept_one, concept_un = self.init_concepts("one", Concept(name="un", body="one"))
@@ -53,7 +54,7 @@ class TestSheerkaNonRegMemory(TestUsingMemoryBasedSheerka):
# sanity check
evaluated = sheerka.evaluate_concept(self.get_context(eval_body=True), return_value)
assert evaluated == simplec("un", simplec("one", NotInit))
compare_with_test_object(evaluated, CB("un", CB("one", NotInit)))
def test_i_can_recognize_concept_with_no_body(self):
sheerka = self.get_sheerka()
@@ -212,7 +213,7 @@ as:
evaluated = sheerka.evaluate_concept(self.get_context(eval_body=True), res[0].value)
assert evaluated.body == "hello foo"
assert evaluated.get_metadata().is_evaluated
assert evaluated.get_value("a") == simplec("foo", "foo")
compare_with_test_object(evaluated.get_value("a"), CB("foo", "foo"))
assert evaluated.get_value("a").get_metadata().is_evaluated
def test_i_can_recognize_duplicate_concepts_with_same_value(self):
@@ -889,7 +890,7 @@ as:
expression = "c:one: < c:two:"
res = sheerka.evaluate_user_input(expression)
assert res[0].status
assert res[0].body == CMV(is_less_than, a="c:one:", b="c:two:")
compare_with_test_object(res[0].body, CMV(is_less_than, a="c:one:", b="c:two:"))
assert res[0].body.a == NotInit # concept is not evaluated
assert res[0].body.b == NotInit # concept is not evaluated
@@ -985,7 +986,7 @@ as:
assert isinstance(plus, Concept)
assert plus.name == "plus"
assert plus.get_compiled()["a"] == sheerka.new("one")
assert plus.get_compiled()["b"] == CC(the, a=sheerka.new("one"))
compare_with_test_object(plus.get_compiled()["b"], CC(the, a=sheerka.new("one")))
res = sheerka.evaluate_user_input("eval one plus the one")
assert res[0].status
@@ -1249,6 +1250,55 @@ as:
assert res[0].status
assert sheerka.isinstance(res[0].body, "binary")
def test_parsing_expression_are_dynamic(self):
init = [
"def concept two",
"def concept number",
"def concept nb times from bnf number 'times'",
"set_isa(two, number)",
]
sheerka = self.init_scenario(init)
res = sheerka.evaluate_user_input("two times")
assert len(res) == 1
assert res[0].status
assert sheerka.isinstance(res[0].body, "nb times")
def test_i_can_evaluate_when_body_is_a_function_that_references_inner_variables(self):
init = [
"def concept two as 2",
"def concept number",
"set_isa(two, number)",
"def concept cars",
"def concept quantify x from bnf number x as set_attr(x, 'qty', number) ret x"
]
sheerka = self.init_scenario(init)
res = sheerka.evaluate_user_input("eval two cars")
assert len(res) == 1
assert res[0].status
assert sheerka.objvalue(res[0].body.get_value("qty")) == 2
def test_i_can_evaluate_when_body_is_a_function_that_references_inner_variables_using_alias(self):
init = [
"def concept two as 2",
"def concept number",
"set_isa(two, number)",
"def concept cars",
"def concept quantify x from bnf number=n1 x as set_attr(x, 'qty', n1) ret x"
]
sheerka = self.init_scenario(init)
res = sheerka.evaluate_user_input("eval two cars")
assert len(res) == 1
assert res[0].status
assert sheerka.objvalue(res[0].body.get_value("qty")) == 2
class TestSheerkaNonRegFile(TestUsingFileBasedSheerka):
def test_i_can_def_several_concepts(self):
tests/parsers/parsers_utils.py
+842 -31
View File
@@ -1,13 +1,16 @@
import ast
from dataclasses import dataclass
from typing import Union
from core.builtin_concepts import ReturnValueConcept
from core.builtin_helpers import CreateObjectIdentifiers
from core.concept import CC, Concept, ConceptParts, DoNotResolve, CIO, CMV
from core.concept import Concept, ConceptParts, DoNotResolve, AllConceptParts
from core.rule import Rule
from core.tokenizer import Tokenizer, TokenKind, Token
from core.utils import get_text_from_tokens, tokens_index
from parsers.BaseNodeParser import scnode, utnode, cnode, SCWC, CNC, short_cnode, CN, UTN, \
SCN, RN, UnrecognizedTokensNode, SourceCodeNode
from core.utils import get_text_from_tokens, tokens_index, str_concept
from parsers.BaseNodeParser import UnrecognizedTokensNode, SourceCodeNode, RuleNode, ConceptNode, \
SourceCodeWithConceptNode
from parsers.FunctionParser import FunctionNode
from parsers.PythonParser import PythonNode
from parsers.SyaNodeParser import SyaConceptParserHelper
from parsers.expressions import NameExprNode, AndNode, OrNode, NotNode, VariableNode, ComparisonNode, ComparisonType
@@ -115,6 +118,825 @@ class NIN: # for NOT INT
source = None
class CC:
"""
Concept class for test purpose
CC means concept for compiled (or concept with compiled)
It matches a concept if the compiles are equals
"""
# The only properties that are testes are concept_key and compiled
# The other properties (concept, source, start and end)
# are used in tests/parsers/parsers_utils.py to help creating helper objects
def __init__(self, concept, source=None, exclude_body=False, **kwargs):
self.concept_key = concept.key if isinstance(concept, Concept) else concept
self.compiled = kwargs
self.concept = concept if isinstance(concept, Concept) else None
self.source = source # to use when the key is different from the sub str to search when filling start and stop
self.start = None # for debug purpose, indicate where the concept starts
self.end = None # for debug purpose, indicate where the concept ends
self.exclude_body = exclude_body
if "body" in self.compiled:
self.compiled[ConceptParts.BODY] = self.compiled["body"]
del self.compiled["body"]
def __eq__(self, other):
if id(self) == id(other):
return True
if isinstance(other, Concept):
if other.key != self.concept_key:
return False
if self.exclude_body:
to_compare = {k: v for k, v in other.get_compiled().items() if k != ConceptParts.BODY}
else:
to_compare = other.get_compiled()
if self.compiled == to_compare:
return True
else:
return False
if not isinstance(other, CC):
return False
if self.concept_key != other.concept_key:
return False
return self.compiled == other.compiled
def __hash__(self):
if self.concept:
return hash(self.concept)
return hash(self.concept_key)
def __repr__(self):
if self.concept:
txt = f"CC(concept='{self.concept}'"
else:
txt = f"CC(concept_key='{self.concept_key}'"
for k, v in self.compiled.items():
txt += f", {k}='{v}'"
return txt + ")"
def fix_pos(self, node):
start = node.start if hasattr(node, "start") else \
node[0] if isinstance(node, tuple) else None
end = node.end if hasattr(node, "end") else \
node[1] if isinstance(node, tuple) else None
if start is not None:
if self.start is None or start < self.start:
self.start = start
if end is not None:
if self.end is None or end > self.end:
self.end = end
return self
def transform_real_obj(self, other, to_compare_delegate):
"""
Transform other into CNC, to ease the comparison
:param other:
:param to_compare_delegate:
:return:
"""
if isinstance(other, CC):
return other
if isinstance(other, Concept):
if self.exclude_body:
compiled = {k: v for k, v in other.get_compiled().items() if k != ConceptParts.BODY}
else:
compiled = other.get_compiled()
self_compile_to_use = self.compiled or compiled
compiled = to_compare_delegate(self_compile_to_use, compiled, to_compare_delegate)
return CC(other,
self.source,
self.exclude_body,
**compiled)
raise NotImplementedError(f"CC, {other=}")
class CB:
"""
Concept with body only
Test class that tests only the body of the concept
"""
def __init__(self, concept: Union[str, Concept], body: object):
self.concept_key = concept.key if isinstance(concept, Concept) else concept
self.concept = concept if isinstance(concept, Concept) else None
self.body = body
def __eq__(self, other):
if not isinstance(other, CB):
return False
return self.concept_key == other.concept_key and self.body == other.body
def __hash__(self):
return hash((self.concept, self.body))
def __repr__(self):
concept_debug = f"concept={self.concept}" if self.concept else f"concept_key={self.concept_key}"
return f"CB({concept_debug}, body='{self.body}')"
def transform_real_obj(self, other, get_test_obj_delegate):
if isinstance(other, CB):
return other
if isinstance(other, Concept):
concept = other.key if not self.concept else other
if isinstance(other.body, Concept):
body = get_test_obj_delegate(other.body, self.body, get_test_obj_delegate)
else:
body = other.body
return CB(concept, body)
raise NotImplementedError(f"CB, {other=}")
class CV:
"""
Concept with all values
Test class that tests all the values (not the metadata, so not the properties) of a concept
"""
def __init__(self, concept, **kwargs):
self.concept_key = concept.key if isinstance(concept, Concept) else concept
self.concept = concept if isinstance(concept, Concept) else None
self.values = {}
for k, v in kwargs.items():
if f"#{k}#" in AllConceptParts:
self.values[f"#{k}#"] = v
else:
self.values[k] = v
def __eq__(self, other):
if not isinstance(other, CV):
return False
return self.concept_key == other.concept_key and self.values == other.values
def __hash__(self):
return hash((self.concept_key, self.values))
def __repr__(self):
concept_debug = f"concept={self.concept}" if self.concept else f"concept_key={self.concept_key}"
return f"CV({concept_debug}, values={self.values})"
def transform_real_obj(self, other, get_test_obj_delegate):
if isinstance(other, CV):
return other
if isinstance(other, Concept):
concept = other.key if not self.concept else other
values = get_test_obj_delegate(other.values(), self.values, get_test_obj_delegate)
return CV(concept, **values)
raise NotImplementedError(f"CV, {other=}")
class CMV:
"""
Concept with metadata variables
CMV stands for Concept Metadata Variables
Test class that only compare the key and the metadata variables
"""
def __init__(self, concept, **kwargs):
self.concept_key = concept.key if isinstance(concept, Concept) else concept
self.concept = concept if isinstance(concept, Concept) else None
self.variables = kwargs
def __eq__(self, other):
if id(self) == id(other):
return True
if not isinstance(other, CMV):
return False
if self.concept_key != other.concept_key:
return False
return self.variables == other.variables
def __hash__(self):
if self.concept:
return hash(self.concept)
return hash(self.concept_key)
def __repr__(self):
if self.concept:
txt = f"CMV(concept='{self.concept}'"
else:
txt = f"CMV(concept_key='{self.concept_key}'"
for k, v in self.variables.items():
txt += f", {k}='{v}'"
return txt + ")"
def transform_real_obj(self, other, get_test_obj_delegate):
if isinstance(other, CMV):
return other
if isinstance(other, Concept):
concept = other.key if not self.concept else other
variables = {name: value for name, value in other.get_metadata().variables}
return CMV(concept, **variables)
raise NotImplementedError(f"CMV, {other=}")
class CIO:
"""
Concept id only
only test the id
"""
def __init__(self, concept, source=None):
if isinstance(concept, str):
self.concept_name = concept
self.concept_id = None
self.concept = None
elif isinstance(concept, Concept):
self.concept_id = concept.id
self.concept = concept
self.source = source
self.start = None
self.end = None
def set_concept(self, concept):
self.concept = concept
self.concept_id = concept.id
def __eq__(self, other):
if id(self) == id(other):
return True
if not isinstance(other, CIO):
return False
return self.concept_id == other.concept_id
def __hash__(self):
return hash(self.concept_id)
def __repr__(self):
return f"CIO(concept='{self.concept}')" if self.concept else f"CIO(name='{self.concept_name}')"
def transform_real_obj(self, other, get_test_obj_delegate):
if isinstance(other, CIO):
return other
if isinstance(other, Concept):
return CIO(other)
raise NotImplementedError(f"CIO, {other=}")
class HelperWithPos:
def __init__(self, start=None, end=None):
self.start = start
self.end = end
self.start_is_fixed = start is not None
self.end_is_fixed = end is not None
def fix_pos(self, node):
if not self.start_is_fixed:
start = node.start if hasattr(node, "start") else \
node[0] if isinstance(node, tuple) else None
if start is not None and (self.start is None or start < self.start):
self.start = start
if not self.end_is_fixed:
end = node.end if hasattr(node, "end") else \
node[1] if isinstance(node, tuple) else None
if end is not None and (self.end is None or end > self.end):
self.end = end
return self
class SCN(HelperWithPos):
"""
SourceCodeNode tester class
It matches with SourceCodeNode but with less constraints
SCN == SourceCodeNode if source, start, end (start and end are not validated when None)
"""
def __init__(self, source, start=None, end=None):
super().__init__(start, end)
self.source = source
def __eq__(self, other):
if id(self) == id(other):
return True
if isinstance(other, SourceCodeNode):
if self.source != other.source:
return False
if self.start is not None and self.start != other.start:
return False
if self.end is not None and self.end != other.end:
return False
return True
if not isinstance(other, SCN):
return False
return self.source == other.source and \
self.start == other.start and \
self.end == other.end
def __hash__(self):
return hash((self.source, self.start, self.end))
def __repr__(self):
txt = f"SCN(source='{self.source}'"
if self.start is not None:
txt += f", start={self.start}"
if self.end is not None:
txt += f", end={self.end}"
return txt + ")"
def transform_real_obj(self, other, to_compare_delegate):
"""
Transform other into CNC, to ease the comparison
:param other:
:param to_compare_delegate:
:return:
"""
if isinstance(other, SCN):
return other
if isinstance(other, SourceCodeNode):
return SCN(other.source,
other.start if self.start is not None else None,
other.end if self.end is not None else None)
raise NotImplementedError(f"SCN, {other=}")
class SCWC(HelperWithPos):
"""
SourceNodeWithConcept tester class
It matches with a SourceNodeWithConcept
but it's easier to instantiate during the tests
"""
def __init__(self, first, last, *args):
super().__init__(None, None)
self.first = first
self.last = last
self.content = list(args)
def __eq__(self, other):
if id(self) == id(other):
return True
if isinstance(other, SourceCodeWithConceptNode):
if self.first != other.first:
return False
if self.last != other.last:
return False
if len(self.content) != len(other.nodes):
return False
for self_node, other_node in zip(self.content, other.nodes):
if self_node != other_node:
return False
# at last
return True
if not isinstance(other, SCWC):
return False
return (self.start == other.start and
self.end == other.end and
self.first == other.first and
self.last == other.last and
self.content == other.content)
def __repr__(self):
txt = "SCWC("
if self.start is not None:
txt += f"start={self.start}"
if self.end is not None:
txt += f", end={self.end}"
for item in [self.first, self.last, *self.content]:
txt += f", {item}"
return txt + ")"
def transform_real_obj(self, other, get_test_obj_delegate):
"""
Transform other into CNC, to ease the comparison
:param other:
:param get_test_obj_delegate:
:return:
"""
if isinstance(other, SCWC):
return other
if isinstance(other, SourceCodeWithConceptNode):
first = get_test_obj_delegate(other.first, self.first)
last = get_test_obj_delegate(other.last, self.last)
content = [get_test_obj_delegate(r, t) for r, t in zip(other.nodes, self.content)]
res = SCWC(first, last, *content)
res.start = other.start
res.end = other.end
return res
raise NotImplementedError(f"SCWC, {other=}")
@property
def source(self):
"""
this code is a copy and paste from SourceCodeWithConceptNode.pseudo_fix_source
TODO: create a common function or whatever...
:return:
"""
source = self.first.source if hasattr(self.first, "source") else self.first
for n in self.content:
source += " "
if hasattr(n, "source"):
source += n.source
elif hasattr(n, "concept"):
source += str(n.concept)
else:
source += " unknown"
source += self.last.source if hasattr(self.last, "source") else self.last
return source
class CN(HelperWithPos):
"""
ConceptNode tester class
It matches with ConceptNode but with less constraints
CN == ConceptNode if concept key, start, end and source are the same
"""
def __init__(self, concept, source=None, start=None, end=None):
"""
:param concept: Concept or concept_key (only the key is used anyway)
:param start:
:param end:
:param source:
"""
super().__init__(start, end)
self.concept_key = concept.key if isinstance(concept, Concept) else concept
self.source = source
self.concept = concept if isinstance(concept, Concept) else None
def fix_source(self, str_tokens):
self.source = "".join(str_tokens)
return self
def __eq__(self, other):
if id(self) == id(other):
return True
if not isinstance(other, CN):
return False
return self.concept_key == other.concept_key and \
self.start == other.start and \
self.end == other.end and \
self.source == other.source
def __hash__(self):
return hash((self.concept_key, self.start, self.end, self.source))
def __repr__(self):
if self.concept:
txt = f"CN(concept='{self.concept}'"
else:
txt = f"CN(concept_key='{self.concept_key}'"
txt += f", source='{self.source}'"
if self.start is not None:
txt += f", start={self.start}"
if self.end is not None:
txt += f", end={self.end}"
return txt + ")"
def transform_real_obj(self, other, get_test_obj_delegate):
"""
Transform other into CNC, to ease the comparison
:param other:
:param get_test_obj_delegate:
:return:
"""
if isinstance(other, CN):
return other
if isinstance(other, ConceptNode):
return CN(other.concept,
other.source if self.source is not None else None,
other.start if self.start is not None else None,
other.end if self.end is not None else None)
raise NotImplementedError(f"CN, {other=}")
class CNC(CN):
"""
ConceptNode for Compiled tester class
It matches with ConceptNode
But focuses on the 'compiled' property of the concept
CNC == ConceptNode if CNC.get_compiled() == ConceptNode.concept.get_compiled()
"""
def __init__(self, concept_key, source=None, start=None, end=None, exclude_body=False, **kwargs):
super().__init__(concept_key, source, start, end)
self.compiled = kwargs
self.exclude_body = exclude_body
if "body" in self.compiled:
self.compiled[ConceptParts.BODY] = self.compiled["body"]
del self.compiled["body"]
def __eq__(self, other):
if id(self) == id(other):
return True
if not isinstance(other, CNC):
return False
return self.concept_key == other.concept_key and \
self.start == other.start and \
self.end == other.end and \
self.source == other.source and \
self.compiled == other.compiled
def __repr__(self):
if self.concept:
txt = f"CNC(concept='{self.concept}'"
else:
txt = f"CNC(concept_key='{self.concept_key}'"
txt += f", source='{self.source}'"
if self.start is not None:
txt += f", start={self.start}"
if self.end is not None:
txt += f", end={self.end}"
for k, v in self.compiled.items():
txt += f", {k}='{v}'"
return txt + ")"
def transform_real_obj(self, other, get_test_obj_delegate):
"""
Transform other into CNC, to ease the comparison
:param other:
:param get_test_obj_delegate:
:return:
"""
if isinstance(other, CNC):
return other
if isinstance(other, ConceptNode):
if self.exclude_body:
compiled = {k: v for k, v in other.concept.get_compiled().items() if k != ConceptParts.BODY}
else:
compiled = other.concept.get_compiled()
self_compile_to_use = self.compiled or compiled
compiled = get_test_obj_delegate(self_compile_to_use, compiled, get_test_obj_delegate)
return CNC(other.concept,
other.source if self.source is not None else None,
other.start if self.start is not None else None,
other.end if self.end is not None else None,
self.exclude_body,
**compiled)
raise NotImplementedError(f"CNC, {other=}")
class UTN(HelperWithPos):
"""
Tester class for UnrecognizedTokenNode
compare the source, and start, end if defined
"""
def __init__(self, source, start=None, end=None):
"""
:param source:
:param start:
:param end:
"""
super().__init__(start, end)
self.source = source
def __eq__(self, other):
if id(self) == id(other):
return True
if isinstance(other, UnrecognizedTokensNode):
return self.start == other.start and \
self.end == other.end and \
self.source == other.source
if not isinstance(other, UTN):
return False
return self.start == other.start and \
self.end == other.end and \
self.source == other.source
def __hash__(self):
return hash((self.source, self.start, self.end))
def __repr__(self):
txt = f"UTN(source='{self.source}'"
if self.start is not None:
txt += f", start={self.start}"
if self.end is not None:
txt += f", end={self.end}"
return txt + ")"
def transform_real_obj(self, other, get_test_obj_delegate):
"""
Transform other into CNC, to ease the comparison
:param other:
:param get_test_obj_delegate:
:return:
"""
if isinstance(other, UTN):
return other
if isinstance(other, UnrecognizedTokensNode):
return UTN(other.source,
other.start,
other.end)
raise NotImplementedError(f"UTN, {other=}")
class RN(HelperWithPos):
"""
Helper class to test RuleNode
"""
def __init__(self, rule, source=None, start=None, end=None):
"""
:param source:
:param start:
:param end:
"""
super().__init__(start, end)
self.rule_id = rule.id if isinstance(rule, Rule) else rule
self.source = source or str_concept((None, self.rule_id), prefix="r:") if self.rule_id else None
self.rule = rule if isinstance(rule, Rule) else None
def __eq__(self, other):
if id(self) == id(other):
return True
if not isinstance(other, RN):
return False
return self.rule_id == other.rule_id and \
self.start == other.start and \
self.end == other.end and \
self.source == other.source
def __hash__(self):
return hash((self.rule_id, self.start, self.end, self.source))
def __repr__(self):
if self.rule:
txt = f"RN(rule='{self.rule}'"
else:
txt = f"RN(rule_id='{self.rule_id}'"
txt += f", source='{self.source}'"
if self.start is not None:
txt += f", start={self.start}"
if self.end is not None:
txt += f", end={self.end}"
return txt + ")"
def transform_real_obj(self, other, get_test_obj_delegate):
"""
Transform other into CNC, to ease the comparison
:param other:
:param get_test_obj_delegate:
:return:
"""
if isinstance(other, RN):
return other
if isinstance(other, RuleNode):
return RN(other.rule,
other.source if self.source is not None else None,
other.start if self.start is not None else None,
other.end if self.end is not None else None)
raise NotImplementedError(f"RN, {other=}")
class FN:
"""
Test class only
It matches with FunctionNode but with less constraints
Thereby,
FN("first", "last", ["param1," ...]) can be compared to
FunctionNode(NameExprNode("first"), NameExprNode("second"), [FunctionParameter(NamesNodes("param1"), NamesNodes(", ")])
Note that FunctionParameter can easily be defined with a single string
* "param" -> FunctionParameter(NameExprNode("param"), None)
* "param, " -> FunctionParameter(NameExprNode("param"), NameExprNode(", "))
For more complicated situations, you can use a tuple (value, sep) to define the value part and the separator part
"""
def __init__(self, first, last, parameters):
self.first = first
self.last = last
self.parameters = []
for param in parameters:
if isinstance(param, tuple):
self.parameters.append(param)
elif isinstance(param, str) and (pos := param.find(",")) != -1:
self.parameters.append((param[:pos], param[pos:]))
else:
self.parameters.append((param, None))
def __repr__(self):
res = self.first
for param in self.parameters:
if param[1]:
res += f"{param[0]}{param[1]} "
else:
res += f"{param[0]}"
return res + self.last
def __eq__(self, other):
if id(self) == id(other):
return True
if isinstance(other, FN):
return self.first == other.first and self.last == other.last and self.parameters == other.parameters
# if isinstance(other, FunctionNode):
# if self.first != other.first.value or self.last != other.last.value:
# return False
# if len(self.parameters) != len(other.parameters):
# return False
# for self_parameter, other_parameter in zip(self.parameters, other.parameters):
# value = other_parameter.value.value if isinstance(self_parameter[0], str) else other_parameter.value
# sep = other_parameter.separator.value if other_parameter.separator else None
# if self_parameter[0] != value or self_parameter[1] != sep:
# return False
#
# return True
return False
def __hash__(self):
return hash((self.first, self.last, self.parameters))
def transform_real_obj(self, other, get_test_obj_delegate):
if isinstance(other, FN):
return other
if isinstance(other, FunctionNode):
params = []
for self_parameter, other_parameter in zip(self.parameters, other.parameters):
if isinstance(self_parameter[0], str):
value = other_parameter.value.value
else:
value = get_test_obj_delegate(other_parameter.value, self_parameter[0])
sep = other_parameter.separator.value if other_parameter.separator else None
params.append((value, sep))
return FN(other.first.value, other.last.value, params)
raise NotImplementedError(f"FN, {other=}")
comparison_type_mapping = {
"EQ": ComparisonType.EQUALS,
"NEQ": ComparisonType.NOT_EQUAlS,
@@ -260,7 +1082,7 @@ def get_node(
if isinstance(sub_expr, ReturnValueConcept):
return sub_expr
if isinstance(sub_expr, (scnode, utnode, DoNotResolve)):
if isinstance(sub_expr, DoNotResolve):
return sub_expr
if isinstance(sub_expr, CIO):
@@ -272,18 +1094,6 @@ def get_node(
sub_expr.end = node.end
return sub_expr
if isinstance(sub_expr, cnode):
# for cnode, map the concept key to the one from concepts_maps if needed
if sub_expr.concept_key.startswith("#"):
return cnode(
concepts_map[sub_expr.concept_key[1:]].key,
sub_expr.start,
sub_expr.end,
sub_expr.source
)
else:
return sub_expr
if isinstance(sub_expr, SCWC):
sub_expr.first = get_node(concepts_map, expression_as_tokens, sub_expr.first, sya=sya)
sub_expr.last = get_node(concepts_map, expression_as_tokens, sub_expr.last, sya=sya)
@@ -342,10 +1152,6 @@ def get_node(
sub_expr.fix_pos(node)
return sub_expr
if isinstance(sub_expr, short_cnode):
return get_node(concepts_map, expression_as_tokens, sub_expr.source,
concept_key=sub_expr.concept_key, skip=skip, is_bnf=True, sya=sya)
if isinstance(sub_expr, tuple):
return get_node(concepts_map, expression_as_tokens, sub_expr[0],
concept_key=concept_key, skip=sub_expr[1], is_bnf=is_bnf, sya=sya)
@@ -364,11 +1170,11 @@ def get_node(
if sya and len(concept_found.get_metadata().variables) > 0 and not is_bnf:
return SyaConceptParserHelper(concept_found, start, start + length - 1)
elif init_empty_body:
node = CNC(concept_found, start, start + length - 1, source=sub_expr, exclude_body=exclude_body)
node = CNC(concept_found, sub_expr, start, start + length - 1, exclude_body=exclude_body)
init_body(node, concept_found, sub_expr)
return node
else:
return CN(concept_found, start, start + length - 1, source=sub_expr)
return CN(concept_found, sub_expr, start, start + length - 1)
else:
# else an UnrecognizedTokensNode
return UTN(sub_expr, start, start + length - 1)
@@ -489,27 +1295,32 @@ def get_rete_conditions(*conditions_as_string):
return AndConditions(res)
def get_test_obj(test_obj, real_obj, to_compare_delegate=None):
def get_test_obj(real_obj, test_obj, get_test_obj_delegate=None):
"""
From a production object (Concept, ConceptNode, ....)
Create a test object (CNC, CC ...) that can be used to validate the unit tests
:param test_obj:
:param real_obj:
:param to_compare_delegate:
:param get_test_obj_delegate:
:return:
"""
if isinstance(test_obj, list):
if len(test_obj) != len(real_obj):
raise Exception(f"Not the same size ! {test_obj=}, {real_obj=}")
return [get_test_obj(t, r) for t, r in zip(test_obj, real_obj)]
raise Exception(f"Not the same size ! {real_obj=}, {test_obj=}")
return [get_test_obj(r, t) for r, t in zip(real_obj, test_obj)]
if isinstance(test_obj, dict):
if len(test_obj) != len(real_obj):
raise Exception(f"Not the same size ! {test_obj=}, {real_obj=}")
raise Exception(f"Not the same size ! {real_obj=}, {test_obj=}")
return {k: get_test_obj(v, real_obj[k]) for k, v in test_obj.items()}
return {k: get_test_obj(real_obj[k], v) for k, v in test_obj.items()}
if not hasattr(test_obj, "to_compare"):
if not hasattr(test_obj, "transform_real_obj"):
return real_obj
return test_obj.to_compare(real_obj, get_test_obj)
return test_obj.transform_real_obj(real_obj, get_test_obj)
def compare_with_test_object(actual, expected):
to_compare = get_test_obj(actual, expected)
assert to_compare == expected
tests/parsers/test_BnfNodeParser.py
+148 -142
View File
@@ -4,11 +4,11 @@ import pytest
import tests.parsers.parsers_utils
from core.builtin_concepts import BuiltinConcepts
from core.concept import Concept, ConceptParts, DoNotResolve, CC, DEFINITION_TYPE_BNF
from core.concept import Concept, ConceptParts, DoNotResolve, DEFINITION_TYPE_BNF
from core.global_symbols import NotInit
from core.sheerka.services.SheerkaConceptManager import SheerkaConceptManager
from core.sheerka.services.SheerkaExecute import ParserInput
from parsers.BaseNodeParser import CNC, UTN, CN, NoMatchingTokenError, SCN
from parsers.BaseNodeParser import NoMatchingTokenError
from parsers.BnfDefinitionParser import BnfDefinitionParser
from parsers.BnfNodeParser import StrMatch, TerminalNode, NonTerminalNode, Sequence, OrderedChoice, \
Optional, ZeroOrMore, OneOrMore, ConceptExpression, UnOrderedChoice, BnfNodeParser, RegExMatch, \
@@ -16,6 +16,7 @@ from parsers.BnfNodeParser import StrMatch, TerminalNode, NonTerminalNode, Seque
from tests.BaseTest import BaseTest
from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
from tests.evaluators.EvaluatorTestsUtils import python_ret_val
from tests.parsers.parsers_utils import CNC, CN, UTN, CC, SCN, get_test_obj, compare_with_test_object
cmap = {
"one": Concept("one"),
@@ -210,7 +211,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
assert len(bnf_parsers_helpers) == len(expected_array)
for parser_helper, expected_sequence in zip(bnf_parsers_helpers, expected_array):
to_compare = tests.parsers.parsers_utils.get_test_obj(expected_sequence, parser_helper.sequence)
to_compare = tests.parsers.parsers_utils.get_test_obj(parser_helper.sequence, expected_sequence)
# assert parser_helper.sequence == expected_sequence
assert to_compare == expected_sequence
@@ -305,7 +306,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
self.validate_get_concepts_sequences(my_map, text, expected)
@pytest.mark.parametrize("text, expected", [
("one two three", [CNC("foo", source="one two three")]),
("one two three", [CNC("foo", "one two three")]),
("one two", []),
("one two four", []),
])
@@ -345,7 +346,8 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
parser.reset_parser(context, ParserInput(text))
bnf_parsers_helpers = parser.get_concepts_sequences(context)
assert bnf_parsers_helpers[0].sequence == expected_array
transformed = get_test_obj(bnf_parsers_helpers[0].sequence, expected_array)
assert transformed == expected_array
assert not bnf_parsers_helpers[0].has_unrecognized
# but I cannot parse
@@ -362,8 +364,8 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
text = "one two three one two"
expected = [
CNC("foo", source="one two three"),
CNC("bar", source="one two", start=6, end=8)]
CNC("foo", "one two three"),
CNC("bar", "one two", 6, 8)]
self.validate_get_concepts_sequences(my_map, text, expected)
@@ -406,8 +408,8 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
self.validate_get_concepts_sequences(my_map, text, expected)
@pytest.mark.parametrize("text, expected", [
("ok thirty one", [CNC("foo", source="ok thirty one")]),
("ok twenty one", [CNC("foo", source="ok twenty one")]),
("ok thirty one", [CNC("foo", "ok thirty one")]),
("ok twenty one", [CNC("foo", "ok twenty one")]),
("ok one", []),
])
def test_i_can_mix_sequence_and_ordered(self, text, expected):
@@ -421,7 +423,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
self.validate_get_concepts_sequences(my_map, text, expected)
@pytest.mark.parametrize("text, expected", [
("twenty one", [CNC("foo", source="twenty one")]),
("twenty one", [CNC("foo", "twenty one")]),
("twenty three", []), # three does not exist
("twenty four", []), # four exists but should not be seen
])
@@ -435,8 +437,8 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
self.validate_get_concepts_sequences(my_map, text, expected)
@pytest.mark.parametrize("text, expected", [
("twenty thirty", [CNC("foo", source="twenty thirty")]),
("one", [CNC("foo", source="one")]),
("twenty thirty", [CNC("foo", "twenty thirty")]),
("one", [CNC("foo", "one")]),
])
def test_i_can_mix_ordered_choices_and_sequences(self, text, expected):
my_map = {
@@ -448,8 +450,8 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
self.validate_get_concepts_sequences(my_map, text, expected)
@pytest.mark.parametrize("text, expected", [
("one", [CNC("foo", source="one")]),
("one two", [CNC("foo", source="one two")]),
("one", [CNC("foo", "one")]),
("one two", [CNC("foo", "one two")]),
("three", []),
])
@@ -463,7 +465,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
self.validate_get_concepts_sequences(my_map, text, expected)
@pytest.mark.parametrize("text, expected", [
("one", [CNC("foo", source="one")]),
("one", [CNC("foo", "one")]),
("", []),
("two", []),
])
@@ -475,8 +477,8 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
self.validate_get_concepts_sequences(my_map, text, expected)
@pytest.mark.parametrize("text, expected", [
("twenty one", [CNC("foo", source="twenty one")]),
("one", [CNC("foo", source="one")]),
("twenty one", [CNC("foo", "twenty one")]),
("one", [CNC("foo", "one")]),
])
def test_i_can_match_sequence_starting_with_optional(self, text, expected):
my_map = {
@@ -489,8 +491,8 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
self.validate_get_concepts_sequences(my_map, text, expected)
@pytest.mark.parametrize("text, expected", [
("one two three", [CNC("foo", source="one two three")]),
("one two", [CNC("foo", source="one two")]),
("one two three", [CNC("foo", "one two three")]),
("one two", [CNC("foo", "one two")]),
])
def test_i_can_match_sequence_ending_with_optional(self, text, expected):
my_map = {
@@ -504,8 +506,8 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
self.validate_get_concepts_sequences(my_map, text, expected)
@pytest.mark.parametrize("text, expected", [
("one two three", [CNC("foo", source="one two three")]),
("one three", [CNC("foo", source="one three")]),
("one two three", [CNC("foo", "one two three")]),
("one three", [CNC("foo", "one three")]),
])
def test_i_can_match_sequence_with_optional_in_between(self, text, expected):
my_map = {
@@ -521,8 +523,8 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
@pytest.mark.parametrize("text, expected", [
("", []),
("two", []),
("one", [CNC("foo", source="one")]),
("one one", [CNC("foo", source="one one")]),
("one", [CNC("foo", "one")]),
("one one", [CNC("foo", "one one")]),
])
def test_i_can_match_zero_or_more(self, text, expected):
my_map = {
@@ -532,9 +534,9 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
self.validate_get_concepts_sequences(my_map, text, expected)
@pytest.mark.parametrize("text, expected", [
("two", [CNC("foo", source="two")]),
("one two", [CNC("foo", source="one two")]),
("one one two", [CNC("foo", source="one one two")]),
("two", [CNC("foo", "two")]),
("one two", [CNC("foo", "one two")]),
("one one two", [CNC("foo", "one one two")]),
])
def test_i_can_match_sequence_and_zero_or_more(self, text, expected):
my_map = {
@@ -548,7 +550,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
self.validate_get_concepts_sequences(my_map, text, expected)
@pytest.mark.parametrize("text, expected", [
("one, one , one", [CNC("foo", source="one, one , one")]),
("one, one , one", [CNC("foo", "one, one , one")]),
])
def test_i_can_match_zero_or_more_with_separator(self, text, expected):
my_map = {
@@ -564,14 +566,14 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
}
text = "one one one"
expected = [CNC("foo", source=text)]
expected = [CNC("foo", text)]
self.validate_get_concepts_sequences(my_map, text, expected)
@pytest.mark.parametrize("text, expected", [
("", []),
("two", []),
("one", [CNC("foo", source="one")]),
("one one one", [CNC("foo", source="one one one")]),
("one", [CNC("foo", "one")]),
("one one one", [CNC("foo", "one one one")]),
])
def test_i_can_match_one_or_more(self, text, expected):
my_map = {
@@ -582,8 +584,8 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
@pytest.mark.parametrize("text, expected", [
("two", []),
("one two", [CNC("foo", source="one two")]),
("one one two", [CNC("foo", source="one one two")]),
("one two", [CNC("foo", "one two")]),
("one one two", [CNC("foo", "one one two")]),
])
def test_i_can_match_sequence_one_and_or_more(self, text, expected):
my_map = {
@@ -597,7 +599,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
self.validate_get_concepts_sequences(my_map, text, expected)
@pytest.mark.parametrize("text, expected", [
("one, one , one", [CNC("foo", source="one, one , one")]),
("one, one , one", [CNC("foo", "one, one , one")]),
])
def test_i_can_match_one_or_more_with_separator(self, text, expected):
my_map = {
@@ -613,18 +615,18 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
}
text = "one one one"
expected = [CNC("foo", source=text)]
expected = [CNC("foo", text)]
self.validate_get_concepts_sequences(my_map, text, expected)
@pytest.mark.parametrize("text, expected", [
("one two", [
[CNC("foo", source="one two")],
[CNC("bar", source="one two")]]),
[CNC("foo", "one two")],
[CNC("bar", "one two")]]),
("one two one two", [
[CNC("bar", source="one two"), CNC("bar", source="one two")],
[CNC("foo", source="one two"), CNC("bar", source="one two")],
[CNC("bar", source="one two"), CNC("foo", source="one two")],
[CNC("foo", source="one two"), CNC("foo", source="one two")]]),
[CNC("bar", "one two"), CNC("bar", "one two")],
[CNC("foo", "one two"), CNC("bar", "one two")],
[CNC("bar", "one two"), CNC("foo", "one two")],
[CNC("foo", "one two"), CNC("foo", "one two")]]),
])
def test_i_can_have_multiple_results(self, text, expected):
my_map = {
@@ -635,7 +637,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
}
text = "one two"
expected = [[CNC("foo", source=text)], [CNC("bar", source=text)]]
expected = [[CNC("foo", text)], [CNC("bar", text)]]
self.validate_get_concepts_sequences(my_map, text, expected, multiple_result=True)
def test_i_can_refer_to_other_concepts(self):
@@ -646,8 +648,8 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
text = "one two"
expected = [
[CNC("foo", source=text)],
[CN("bar", source=text)] # Do not check the compiled part
[CNC("foo", text)],
[CN("bar", text)] # Do not check the compiled part
]
sequences = self.validate_get_concepts_sequences(my_map, text, expected, multiple_result=True)
@@ -672,8 +674,8 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
text = "one two"
expected = [
[CNC("foo", source=text)],
[CN("bar", source=text)] # Do not check the compiled part
[CNC("foo", text)],
[CN("bar", text)] # Do not check the compiled part
]
sequences = self.validate_get_concepts_sequences(my_map, text, expected, multiple_result=True)
@@ -698,9 +700,9 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
text = "one two"
expected = [
[CNC("foo", source=text)],
[CN("bar", source=text)], # Do not check the compiled part
[CN("baz", source=text)], # Do not check the compiled part
[CNC("foo", text)],
[CN("bar", text)], # Do not check the compiled part
[CN("baz", text)], # Do not check the compiled part
]
sequences = self.validate_get_concepts_sequences(my_map, text, expected, multiple_result=True)
@@ -769,7 +771,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
text = "twenty two"
expected = [CNC("twenties",
source="twenty two",
"twenty two",
twenty=CC("twenty", body=DoNotResolve("twenty")),
number=CC("number", source="two", body=DoNotResolve("two"))
)]
@@ -842,7 +844,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
}
text = "twenty one"
expected = [CN("foo", source="twenty one")]
expected = [CN("foo", "twenty one")]
sequences = self.validate_get_concepts_sequences(my_map, text, expected)
concept_foo = sequences[0].concept
assert concept_foo.get_compiled() == {
@@ -869,8 +871,9 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
# explicit validations of the compiled
concept_foo = sequences[0].concept
assert concept_foo.body == NotInit
assert concept_foo.get_compiled() == {'number': CC(my_map["number"], body=my_map["two"], two=my_map["two"]),
ConceptParts.BODY: DoNotResolve(value='twenty two')}
compare_with_test_object(concept_foo.get_compiled(), {
'number': CC(my_map["number"], body=my_map["two"], two=my_map["two"]),
ConceptParts.BODY: DoNotResolve(value='twenty two')})
text = "twenty one"
expected = [CN("foo", source="twenty one")]
@@ -879,14 +882,15 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
# explicit validations of the compiled
concept_foo = sequences[0].concept
assert concept_foo.body == NotInit
assert concept_foo.get_compiled() == {'number': CC(my_map["number"], body=my_map["one"], one=my_map["one"]),
ConceptParts.BODY: DoNotResolve(value='twenty one')}
compare_with_test_object(concept_foo.get_compiled(), {
'number': CC(my_map["number"], body=my_map["one"], one=my_map["one"]),
ConceptParts.BODY: DoNotResolve(value='twenty one')})
@pytest.mark.parametrize("expr, expected", [
("one 'car'", [CNC("foo", source="one 'car'", x=python_ret_val("'car'"))]), # python
("one bar", [CNC("foo", source="one bar", x=CC("bar"))]), # simple concept
("one super car", [CNC("foo", source="one super car", x=CC("super car"))]), # long concept
("one shoe", [CNC("foo", source="one shoe", x=CC("thing", source="shoe", body=DoNotResolve("shoe")))]), # bnf
("one 'car'", [CNC("foo", "one 'car'", x=python_ret_val("'car'"))]), # python
("one bar", [CNC("foo", "one bar", x=CC("bar"))]), # simple concept
("one super car", [CNC("foo", "one super car", x=CC("super car"))]), # long concept
("one shoe", [CNC("foo", "one shoe", x=CC("thing", source="shoe", body=DoNotResolve("shoe")))]), # bnf
])
def test_i_can_match_variable_when_ending_with_one_variable(self, expr, expected):
my_map = {
@@ -910,8 +914,8 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
expr = "one bar plus baz"
expected = [
[CNC("foo", source="one bar", x=CC("bar")), UTN(" plus "), CN("baz")],
[CNC("foo", source="one bar plus baz", x=CC("plus", source="bar plus baz", x="bar", y="baz"))],
[CNC("foo", "one bar", x=CC("bar")), UTN(" plus "), CN("baz")],
[CNC("foo", "one bar plus baz", x=CC("plus", source="bar plus baz", x="bar", y="baz"))],
]
self.validate_get_concepts_sequences(my_map, expr, expected, multiple_result=True)
@@ -925,8 +929,8 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
expr = "one pretty big"
expected = [
[CNC("foo", source="one pretty big", x=CC("pretty big"))],
[CNC("foo", source="one pretty big", x=CC("pbig", source="pretty big"))]
[CNC("foo", "one pretty big", x=CC("pretty big"))],
[CNC("foo", "one pretty big", x=CC("pbig", source="pretty big"))]
]
self.validate_get_concepts_sequences(my_map, expr, expected, multiple_result=True)
@@ -940,16 +944,16 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
expr = "one pretty big"
expected = [
[CNC("foo", source="one pretty big", x=CC("pretty"), y=CC("big"))],
[CNC("foo", source="one pretty big", x=CC("pretty2", source="pretty"), y=CC("big"))]
[CNC("foo", "one pretty big", x=CC("pretty"), y=CC("big"))],
[CNC("foo", "one pretty big", x=CC("pretty2", source="pretty"), y=CC("big"))]
]
self.validate_get_concepts_sequences(my_map, expr, expected, multiple_result=True)
@pytest.mark.parametrize("expr, expected", [
("'my' shoe", [CNC("foo", source="'my' shoe", x=python_ret_val("'my' "))]), # python
("one shoe", [CNC("foo", source="one shoe", x=CC("one"))]), # concept
("my little shoe", [CNC("foo", source="my little shoe", x=CC("my little"))]), # long concept
("black shoe", [CNC("foo", source="black shoe", x=CC("color", source="black", body=DoNotResolve('black')))]),
("'my' shoe", [CNC("foo", "'my' shoe", x=python_ret_val("'my' "))]), # python
("one shoe", [CNC("foo", "one shoe", x=CC("one"))]), # concept
("my little shoe", [CNC("foo", "my little shoe", x=CC("my little"))]), # long concept
("black shoe", [CNC("foo", "black shoe", x=CC("color", source="black", body=DoNotResolve('black')))]),
])
def test_i_can_match_variable_when_starting_with_one_variable(self, expr, expected):
my_map = {
@@ -972,9 +976,9 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
expr = "tiny but beautiful shoe"
expected_res = [
CNC("foo",
source="tiny but beautiful shoe",
"tiny but beautiful shoe",
x=CC("but", source="tiny but beautiful", x="tiny", y="beautiful"))]
unwanted_res = [CN("tiny"), UTN(" but "), CNC("foo", source="beautiful shoe", x=CC("beautiful"))]
unwanted_res = [CN("tiny"), UTN(" but "), CNC("foo", "beautiful shoe", x=CC("beautiful"))]
self.validate_get_concepts_sequences(my_map, expr, [unwanted_res, expected_res], multiple_result=True)
def test_i_can_match_variable_when_starting_with_multiple_variables(self):
@@ -992,7 +996,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
unwanted_res = [CN("one"), SCN(" 'one' "), ("one", 1), UTN(" plus "), CN("two")]
expected_res = [CNC("foo",
source="one 'one' one plus two shoe",
"one 'one' one plus two shoe",
x=CC("one"),
y=python_ret_val(" 'one' "),
z=CC("plus", source="one plus two", x="one", y="two"))]
@@ -1009,18 +1013,18 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
"one": Concept("one")
}
text = "one foo bar baz"
expected = [CNC("foo", source="one foo bar baz", x=CC("one"))]
expected = [CNC("foo", "one foo bar baz", x=CC("one"))]
self.validate_get_concepts_sequences(my_map, text, expected)
@pytest.mark.parametrize("expr, expected", [
("one 'pretty' shoe", [CNC("foo", source="one 'pretty' shoe", x=python_ret_val("'pretty' "))]), # python
("one little shoe", [CNC("foo", source="one little shoe", x=CC("little"))]), # concept
("one very big shoe", [CNC("foo", source="one very big shoe", x=CC("very big"))]), # long concept
("one 'pretty' shoe", [CNC("foo", "one 'pretty' shoe", x=python_ret_val("'pretty' "))]), # python
("one little shoe", [CNC("foo", "one little shoe", x=CC("little"))]), # concept
("one very big shoe", [CNC("foo", "one very big shoe", x=CC("very big"))]), # long concept
("one black shoe",
[CNC("foo", source="one black shoe", x=CC("color", source="black", body=DoNotResolve('black')))]),
[CNC("foo", "one black shoe", x=CC("color", source="black", body=DoNotResolve('black')))]),
("one tiny but beautiful shoe",
[CNC("foo",
source="one tiny but beautiful shoe",
"one tiny but beautiful shoe",
x=CC("but", source="tiny but beautiful", x="tiny", y="beautiful "))]),
])
def test_i_can_match_variable_in_between(self, expr, expected):
@@ -1043,8 +1047,8 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
expr = "one pretty big shoe"
expected = [
[CNC("foo", source="one pretty big shoe", x=CC("pretty big"))],
[CNC("foo", source="one pretty big shoe", x=CC("pbig", source="pretty big"))]
[CNC("foo", "one pretty big shoe", x=CC("pretty big"))],
[CNC("foo", "one pretty big shoe", x=CC("pbig", source="pretty big"))]
]
self.validate_get_concepts_sequences(my_map, expr, expected, multiple_result=True)
@@ -1055,7 +1059,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
"shoe": Concept("shoe")
}
text = "onyx shoe"
expected = [CNC("foo", source="onyx shoe", x=CC("shoe"))]
expected = [CNC("foo", "onyx shoe", x=CC("shoe"))]
self.validate_get_concepts_sequences(my_map, text, expected)
def test_i_can_match_variable_and_regex(self):
@@ -1065,7 +1069,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
"one": Concept("one")
}
text = "one onyx"
expected = [CNC("foo", source="one onyx", x=CC("one"))]
expected = [CNC("foo", "one onyx", x=CC("one"))]
self.validate_get_concepts_sequences(my_map, text, expected)
def test_i_can_reuse_the_same_variable(self):
@@ -1083,12 +1087,12 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
# same variable appears only once in the compiled variables
text = "one equals one"
expected = [CNC("foo", source="one equals one", x=CC("one"))]
expected = [CNC("foo", "one equals one", x=CC("one"))]
expected_sequence = compute_expected_array(my_map, text, expected)
parser.reset_parser(context, ParserInput(text))
bnf_parsers_helpers = parser.get_concepts_sequences(context)
to_compare = tests.parsers.parsers_utils.get_test_obj(expected_sequence, bnf_parsers_helpers[0].sequence)
to_compare = tests.parsers.parsers_utils.get_test_obj(bnf_parsers_helpers[0].sequence, expected_sequence)
assert to_compare == expected
def test_i_cannot_match_variable_when_variables_discrepancy(self):
@@ -1314,9 +1318,9 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
ConceptExpression(my_map["one"], rule_name="one"))
@pytest.mark.parametrize("expr, text, expected", [
(ZeroOrMore(StrMatch("one"), sep=","), "one,", [CNC("foo", source="one"), UTN(",")]),
(StrMatch("one"), "one two", [CNC("foo", source="one"), UTN(" two")]),
(StrMatch("one"), "two one", [UTN("two "), CNC("foo", source="one")]),
(ZeroOrMore(StrMatch("one"), sep=","), "one,", [CNC("foo", "one"), UTN(",")]),
(StrMatch("one"), "one two", [CNC("foo", "one"), UTN(" two")]),
(StrMatch("one"), "two one", [UTN("two "), CNC("foo", "one")]),
])
def test_i_can_recognize_unknown_concepts(self, expr, text, expected):
my_map = {
@@ -1332,7 +1336,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
}
text = "one three"
expected = [UTN("one "), CNC("three", source="three")]
expected = [UTN("one "), CNC("three", "three")]
self.validate_get_concepts_sequences(my_map, text, expected)
def test_i_can_remove_duplicates(self):
@@ -1350,12 +1354,12 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
assert len(sequence) == 1
@pytest.mark.parametrize("parser_input, expected_status, expected", [
("baz", True, [CNC("bnf baz", source="baz")]), # the bnf one is chosen
("foo bar", True, [CNC("foo then bar", source="foo bar", foo="foo", bar="bar")]),
("bar", True, [CNC("foo or bar", source="bar", bar="bar", body="bar")]),
("one plus two", True, [CNC("plus", source="one plus two", one="one", two="two")]),
("twenty one", True, [CNC("t1", source="twenty one", unit="one")]),
("one 'car'", True, [CNC("one thing", source="one 'car'", x=python_ret_val("'car'"), one="one")])
("baz", True, [CNC("bnf baz", "baz")]), # the bnf one is chosen
("foo bar", True, [CNC("foo then bar", "foo bar", foo="foo", bar="bar")]),
("bar", True, [CNC("foo or bar", "bar", bar="bar", body="bar")]),
("one plus two", True, [CNC("plus", "one plus two", one="one", two="two")]),
("twenty one", True, [CNC("t1", "twenty one", unit="one")]),
("one 'car'", True, [CNC("one thing", "one 'car'", x=python_ret_val("'car'"), one="one")])
])
def test_i_can_parse_simple_expressions(self, parser_input, expected_status, expected):
sheerka, context, parser = self.init_parser(init_from_sheerka=True)
@@ -1367,7 +1371,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
assert res.status == expected_status
assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT)
assert concepts_nodes == expected_array
compare_with_test_object(concepts_nodes, expected_array)
def test_i_can_parse_when_multiple_times_the_same_variable(self):
sheerka, context, parser = self.init_parser(init_from_sheerka=True)
@@ -1382,7 +1386,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
assert res.status
assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT)
assert concepts_nodes == expected_array
compare_with_test_object(concepts_nodes, expected_array)
def test_i_can_test_when_expression_references_other_expressions(self):
sheerka, context, parser = self.init_parser(init_from_sheerka=True)
@@ -1402,7 +1406,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
assert res.status
assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT)
assert concepts_nodes == expected_array
compare_with_test_object(concepts_nodes, expected_array)
def test_i_can_parse_bnf_concept_mixed_with_isa_concepts(self):
sheerka, context, parser = self.init_parser(init_from_sheerka=True)
@@ -1428,7 +1432,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
assert res.status
assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT)
assert concepts_nodes == expected_array
compare_with_test_object(concepts_nodes, expected_array)
def test_i_can_parse_bnf_concept_mixed_with_isa_concepts_2(self):
# this time, three is a number, and also part of three_four, even if it is not relevant in t3
@@ -1450,7 +1454,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
assert res.status
assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT)
assert concepts_nodes == expected_array
compare_with_test_object(concepts_nodes, expected_array)
def test_i_can_parse_when_starting_by_isa_concept(self):
"""
@@ -1476,7 +1480,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
assert res.status
assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT)
assert concepts_nodes == expected_array
compare_with_test_object(concepts_nodes, expected_array)
def test_i_can_parse_fifty_one_thousand(self):
"""
@@ -1515,10 +1519,10 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
res = parser.parse(context, ParserInput(text))
assert res[0].status
assert res[0].value.value == expected_thousands
compare_with_test_object(res[0].value.value, expected_thousands)
assert res[1].status
assert res[1].value.value == expected_fifties
compare_with_test_object(res[1].value.value, expected_fifties)
def test_i_can_parse_one_hundred_thousand(self):
sheerka, context, parser = self.init_parser(init_from_sheerka=True)
@@ -1565,7 +1569,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
assert res.status
assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT)
assert concepts_nodes == expected_array
compare_with_test_object(concepts_nodes, expected_array)
def test_i_can_parse_bnf_concept_mixed_with_isa_after_restart(self):
sheerka, context, parser = self.init_parser(init_from_sheerka=True)
@@ -1589,7 +1593,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
assert res.status
assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT)
assert concepts_nodes == expected_array
compare_with_test_object(concepts_nodes, expected_array)
text = "forty one"
expected = CNC("forties",
@@ -1607,13 +1611,13 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
assert res.status
assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT)
assert concepts_nodes == expected_array
compare_with_test_object(concepts_nodes, expected_array)
def test_i_can_parse_when_keyword(self):
sheerka, context, parser = self.init_parser(init_from_sheerka=True)
parser_input = "def one"
expected = [CNC("def number", source="def one", number="one")]
expected = [CNC("def number", "def one", number="one")]
res = parser.parse(context, ParserInput(parser_input))
expected_array = compute_expected_array(cmap, parser_input, expected)
@@ -1624,7 +1628,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
assert res.status
assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT)
assert concepts_nodes == expected_array
compare_with_test_object(concepts_nodes, expected_array)
def test_i_can_parse_filter(self):
sheerka, context, parser = self.init_parser(init_from_sheerka=True)
@@ -1639,7 +1643,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
assert res.status
assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT)
assert concepts_nodes == expected_array
compare_with_test_object(concepts_nodes, expected_array)
def test_i_can_parse_descent_grammar(self):
my_map = {
@@ -1662,17 +1666,17 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
assert res.status
assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT)
assert concepts_nodes == [CNC(expr,
term=[CC(term,
body=CC(factor, body=DoNotResolve("1")),
factor=CC(factor, body=DoNotResolve("1"))),
CC(term,
body=DoNotResolve("2 * 3"),
factor=[
CC(factor, body=DoNotResolve("2")),
CC(factor, body=DoNotResolve("3")),
])],
body=DoNotResolve("1 + 2 * 3"))]
compare_with_test_object(concepts_nodes, [CNC(expr,
term=[CC(term,
body=CC(factor, body=DoNotResolve("1")),
factor=CC(factor, body=DoNotResolve("1"))),
CC(term,
body=DoNotResolve("2 * 3"),
factor=[
CC(factor, body=DoNotResolve("2")),
CC(factor, body=DoNotResolve("3")),
])],
body=DoNotResolve("1 + 2 * 3"))])
def test_i_can_parse_recursive_descent_grammar(self):
my_map = {
@@ -1698,25 +1702,27 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
# concepts_nodes = res.value.value is too complicated to be validated
assert res.status
assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT)
assert concepts_nodes == [CNC(expr,
term=CC(term,
body=CC(factor, body=DoNotResolve("1")),
factor=CC(factor, body=DoNotResolve("1"))),
expr=CC(expr,
body=CC(term,
body=DoNotResolve("2 * 3"),
factor=CC(factor, body=DoNotResolve("2")),
compare_with_test_object(concepts_nodes, [CNC(expr,
term=CC(term,
body=CC(factor, body=DoNotResolve("3")),
factor=CC(factor, body=DoNotResolve("3")))),
term=CC(term,
body=DoNotResolve("2 * 3"),
factor=CC(factor, body=DoNotResolve("2")),
term=CC(term,
body=CC(factor, body=DoNotResolve("3")),
factor=CC(factor, body=DoNotResolve("3"))))),
body=CC(factor, body=DoNotResolve("1")),
factor=CC(factor, body=DoNotResolve("1"))),
expr=CC(expr,
body=CC(term,
body=DoNotResolve("2 * 3"),
factor=CC(factor, body=DoNotResolve("2")),
term=CC(term,
body=CC(factor, body=DoNotResolve("3")),
factor=CC(factor,
body=DoNotResolve("3")))),
term=CC(term,
body=DoNotResolve("2 * 3"),
factor=CC(factor, body=DoNotResolve("2")),
term=CC(term,
body=CC(factor, body=DoNotResolve("3")),
factor=CC(factor,
body=DoNotResolve("3"))))),
body=DoNotResolve("1 + 2 * 3"))]
body=DoNotResolve("1 + 2 * 3"))])
def test_i_can_parse_simple_recursive_grammar(self):
my_map = {
@@ -1752,14 +1758,14 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
text = "thirty one"
res = parser.parse(context, ParserInput(text))
assert res.status
assert res.value.value == compute_expected_array(cmap, text, [CN("thirties", source=text)])
compare_with_test_object(res.value.value, compute_expected_array(cmap, text, [CN("thirties", text)]))
# add a layer, I still can parse the text
sheerka.push_ontology(context, "new layer")
parser = BnfNodeParser(sheerka=sheerka)
res = parser.parse(context, ParserInput(text))
assert res.status
assert res.value.value == compute_expected_array(cmap, text, [CN("thirties", source=text)])
compare_with_test_object(res.value.value, compute_expected_array(cmap, text, [CN("thirties", text)]))
def test_i_do_not_eat_unwanted_tokens_at_the_beginning_when_concept_with_variable(self):
my_map = {
@@ -1772,9 +1778,9 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
text = "two one shoe"
res = parser.parse(context, ParserInput(text))
assert res.status
assert res.value.value == compute_expected_array(my_map, text, [
compare_with_test_object(res.value.value, compute_expected_array(my_map, text, [
CN("two"),
CNC("foo", source="one shoe", x=CC("one"))])
CNC("foo", "one shoe", x=CC("one"))]))
def test_i_do_not_eat_unwanted_tokens_at_the_end_when_concept_with_variable(self):
my_map = {
@@ -1787,9 +1793,9 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
text = "one bar baz"
res = parser.parse(context, ParserInput(text))
assert res.status
assert res.value.value == compute_expected_array(my_map, text, [
CNC("foo", source="one bar", x=CC("bar")),
CN("baz")])
compare_with_test_object(res.value.value, compute_expected_array(my_map, text, [
CNC("foo", "one bar", x=CC("bar")),
CN("baz")]))
@pytest.mark.parametrize("parsing_expression, expected", [
(RegExMatch("a"), [RegExDef("a")]),
tests/parsers/test_BnfParser.py
+4 -4
View File
@@ -4,13 +4,13 @@ from core.builtin_concepts import BuiltinConcepts
from core.concept import Concept, DEFINITION_TYPE_BNF
from core.sheerka.services.SheerkaExecute import ParserInput
from core.tokenizer import Tokenizer, TokenKind, LexerError
from parsers.BaseNodeParser import cnode
from parsers.BaseParser import UnexpectedTokenParsingError, UnexpectedEofParsingError
from parsers.BnfDefinitionParser import BnfDefinitionParser
from parsers.BnfNodeParser import BnfNodeParser, RegExMatch, VariableExpression
from parsers.BnfNodeParser import StrMatch, Optional, ZeroOrMore, OrderedChoice, Sequence, \
OneOrMore, ConceptExpression
from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
from tests.parsers.parsers_utils import CN, compare_with_test_object
class ClassWithName:
@@ -226,15 +226,15 @@ class TestBnfParser(TestUsingMemoryBasedSheerka):
res = bnf_parser.parse(context, ParserInput("twenty two"))
assert res.status
assert res.value.body == [cnode("bar", 0, 2, "twenty two")]
compare_with_test_object(res.value.body, [CN("bar", "twenty two", 0, 2)])
res = bnf_parser.parse(context, ParserInput("thirty one"))
assert res.status
assert res.value.body == [cnode("bar", 0, 2, "thirty one")]
compare_with_test_object(res.value.body, [CN("bar", "thirty one", 0, 2)])
res = bnf_parser.parse(context, ParserInput("twenty"))
assert res.status
assert res.value.body == [cnode("foo", 0, 0, "twenty")]
compare_with_test_object(res.value.body, [CN("foo", "twenty", 0, 0)])
def test_i_cannot_parse_when_too_many_concepts(self):
sheerka, context, regex_parser, foo1, foo2 = self.init_parser(
tests/parsers/test_DefConceptParser.py
+8 -13
View File
@@ -4,11 +4,10 @@ from dataclasses import dataclass
import pytest
from core.builtin_concepts import ParserResultConcept, BuiltinConcepts, ReturnValueConcept
from core.concept import DEFINITION_TYPE_BNF, DEFINITION_TYPE_DEF, Concept, CV
from core.concept import DEFINITION_TYPE_BNF, DEFINITION_TYPE_DEF, Concept
from core.global_symbols import NotInit
from core.sheerka.services.SheerkaExecute import ParserInput
from core.tokenizer import Keywords, Tokenizer, LexerError
from parsers.BaseNodeParser import SCWC
from parsers.BaseParser import UnexpectedEofParsingError
from parsers.BnfDefinitionParser import BnfDefinitionParser
from parsers.BnfNodeParser import OrderedChoice, ConceptExpression, StrMatch, Sequence, RegExMatch, OneOrMore, \
@@ -18,7 +17,7 @@ from parsers.DefConceptParser import UnexpectedTokenParsingError, DefConceptNode
from parsers.FunctionParser import FunctionParser
from parsers.PythonParser import PythonParser, PythonNode
from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
from tests.parsers.parsers_utils import compute_expected_array
from tests.parsers.parsers_utils import compute_expected_array, SCWC, CV, compare_with_test_object
def get_def_concept(name, where=None, pre=None, post=None, body=None, definition=None, bnf_def=None, ret=None):
@@ -265,7 +264,6 @@ class TestDefConceptParser(TestUsingMemoryBasedSheerka):
text = """def concept a mult b
where a,b
pre isinstance(a, int) and isinstance(b, int)
post isinstance(res, a)
as res = a * b
ret a if isinstance(a, Concept) else self
"""
@@ -276,7 +274,6 @@ ret a if isinstance(a, Concept) else self
name="a mult b",
where="a,b\n",
pre="isinstance(a, int) and isinstance(b, int)\n",
post=FN("isinstance(res, a)\n", "isinstance(", ")", ["res", ", ", "a"]),
body=PN("res = a * b\n", "exec"),
ret="a if isinstance(a, Concept) else self\n"
)
@@ -542,29 +539,27 @@ from give me the date !
text = "def concept foo x y where x is a y"
res = parser.parse(context, ParserInput(text))
expected_body = self.pretval(CV(concepts[0], pre=True), source="x is a y", who="parsers.DefConcept",
parser="parsers.ExactConcept")
expected = get_def_concept("foo x y", where=expected_body)
node = res.value.value
assert res.status
assert res.who == parser.name
assert res.value.source == text
assert isinstance(res.value, ParserResultConcept)
assert node == expected
assert isinstance(node, DefConceptNode)
assert sheerka.isinstance(node.where, BuiltinConcepts.RETURN_VALUE)
compare_with_test_object(node.where.body.body, CV(concepts[0], pre=True))
text = "def concept foo x y pre x is a y"
res = parser.parse(context, ParserInput(text))
expected_body = self.pretval(CV(concepts[0], pre=True), source="x is a y", who="parsers.DefConcept",
parser="parsers.ExactConcept")
expected = get_def_concept("foo x y", pre=expected_body)
node = res.value.value
assert res.status
assert res.who == parser.name
assert res.value.source == text
assert isinstance(res.value, ParserResultConcept)
assert node == expected
assert isinstance(node, DefConceptNode)
assert sheerka.isinstance(node.pre, BuiltinConcepts.RETURN_VALUE)
compare_with_test_object(node.pre.body.body, CV(concepts[0], pre=True))
def test_i_can_parse_bnf_concept_with_regex(self):
sheerka, context, parser, number = self.init_parser("number")
tests/parsers/test_ExactConceptParser.py
+8 -7
View File
@@ -1,9 +1,10 @@
from core.builtin_concepts import BuiltinConcepts
from core.concept import Concept, CMV
from core.concept import Concept
from core.sheerka.services.SheerkaExecute import ParserInput
from parsers.ExactConceptParser import ExactConceptParser
from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
from tests.parsers.parsers_utils import CMV, compare_with_test_object
def variable_def(concept, prop_name):
@@ -97,7 +98,7 @@ class TestExactConceptParser(TestUsingMemoryBasedSheerka):
assert results[0].status
concept_found = results[0].value.value
assert concept_found == CMV(concept, a="10", b="5")
compare_with_test_object(concept_found, CMV(concept, a="10", b="5"))
assert concept_found.get_metadata().need_validation
assert not concept_found.get_metadata().is_evaluated
@@ -113,7 +114,7 @@ class TestExactConceptParser(TestUsingMemoryBasedSheerka):
assert results[0].status
concept_found = results[0].value.value
assert concept_found == CMV(concept, a="10", b="5")
compare_with_test_object(concept_found, CMV(concept, a="10", b="5"))
assert concept_found.get_metadata().need_validation
def test_i_can_parse_concept_when_defined_using_from_def(self):
@@ -127,7 +128,7 @@ class TestExactConceptParser(TestUsingMemoryBasedSheerka):
assert len(results) == 1
assert results[0].status
assert concept_found == CMV(plus, a="10", b="5")
compare_with_test_object(concept_found, CMV(plus, a="10", b="5"))
assert concept_found.get_metadata().need_validation
assert not concept_found.get_metadata().is_evaluated
@@ -157,7 +158,7 @@ class TestExactConceptParser(TestUsingMemoryBasedSheerka):
assert len(results) == 1
assert results[0].status
assert concept_found == CMV(plus, a="c:one:", b="c:two:")
compare_with_test_object(concept_found, CMV(plus, a="c:one:", b="c:two:"))
assert concept_found.get_metadata().need_validation
assert not concept_found.get_metadata().is_evaluated
@@ -173,7 +174,7 @@ class TestExactConceptParser(TestUsingMemoryBasedSheerka):
assert len(results) == 1
assert results[0].status
assert concept_found == CMV(isa, c="z")
compare_with_test_object(concept_found, CMV(isa, c="z"))
assert concept_found.get_metadata().need_validation
assert not concept_found.get_metadata().is_evaluated
@@ -183,7 +184,7 @@ class TestExactConceptParser(TestUsingMemoryBasedSheerka):
assert len(results) == 1
assert results[0].status
assert concept_found == CMV(def_concept, a="z")
compare_with_test_object(concept_found, CMV(def_concept, a="z"))
assert concept_found.get_metadata().need_validation
assert not concept_found.get_metadata().is_evaluated
tests/parsers/test_ExpressionParser.py
+12 -13
View File
@@ -3,11 +3,10 @@ import ast
import pytest
from core.builtin_concepts import BuiltinConcepts, ReturnValueConcept
from core.concept import Concept, CMV, DoNotResolve, CC
from core.concept import Concept, DoNotResolve
from core.rule import Rule
from core.sheerka.services.SheerkaExecute import ParserInput
from core.tokenizer import TokenKind
from parsers.BaseNodeParser import CNC
from parsers.BaseParser import UnexpectedEofParsingError, UnexpectedTokenParsingError
from parsers.ExpressionParser import ExpressionParser, LeftPartNotFoundError, ParenthesisMismatchError
from parsers.PythonParser import PythonNode
@@ -15,7 +14,7 @@ from parsers.expressions import TrueifyVisitor, IsAQuestionVisitor, AndNode
from sheerkarete.network import ReteNetwork
from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
from tests.parsers.parsers_utils import compute_expected_array, resolve_test_concept, EXPR, OR, AND, NOT, \
get_expr_node_from_test_node, get_rete_conditions
get_expr_node_from_test_node, get_rete_conditions, CMV, CNC, CC, compare_with_test_object
class TestExpressionParser(TestUsingMemoryBasedSheerka):
@@ -215,9 +214,9 @@ class TestExpressionParser(TestUsingMemoryBasedSheerka):
("foo", "foo"),
("one two", "one two"),
("foo is a bar", CMV("is a", x='foo', y='bar')),
("one two is a bar", [CNC("is a", source="one two is a bar", x="one two", y="bar")]),
("one two is a bar", [CNC("is a", "one two is a bar", x="one two", y="bar")]),
("foo is an foo bar",
[CNC("is an", source="foo is an foo bar", x=DoNotResolve(value='foo'), exclude_body=True)]),
[CNC("is an", "foo is an foo bar", x=DoNotResolve(value='foo'), exclude_body=True)]),
])
def test_i_can_get_compiled_expr_from_simple_concepts_expressions(self, expression, expected):
concepts_map = {
@@ -238,10 +237,10 @@ class TestExpressionParser(TestUsingMemoryBasedSheerka):
if isinstance(expected, list):
expected_nodes = compute_expected_array(concepts_map, expression, expected)
assert ret.body.body == expected_nodes
compare_with_test_object(ret.body.body, expected_nodes)
else:
expected_concept = resolve_test_concept(concepts_map, expected)
assert ret.body.body == expected_concept
compare_with_test_object(ret.body.body, expected_concept)
@pytest.mark.parametrize("expression", [
"a == 5",
@@ -338,11 +337,11 @@ class TestExpressionParser(TestUsingMemoryBasedSheerka):
ret = return_values[0]
python_node = ret.body.body
assert python_node == expected_python_node
assert python_node.objects == {
compare_with_test_object(python_node.objects, {
"__C__00var0000is0a000var001__1005__C__": CC(is_a, x=cat, y=pet),
"__C__00var0000is0an0y__1006__C__": CC(is_an, exclude_body=True, x=DoNotResolve("bird"), animal=animal),
"__C__00var0000is0a000var001__1005_1__C__": CMV(is_a, x="dog", y="pet"),
}
})
def test_i_can_get_compiled_expr_from_mix(self):
sheerka, context, animal, cat, dog, pet, is_a, is_an = self.init_test().with_concepts(
@@ -369,11 +368,11 @@ class TestExpressionParser(TestUsingMemoryBasedSheerka):
python_node = ret.body.body
assert python_node == expected_python_node
assert python_node.objects == {
compare_with_test_object(python_node.objects, {
"__C__00var0000is0a000var001__1005__C__": CC(is_a, x=cat, y=pet),
"__C__00var0000is0an0y__1006__C__": CC(is_an, exclude_body=True, x=DoNotResolve("bird"), animal=animal),
"__C__00var0000is0a000var001__1005_1__C__": CMV(is_a, x="dog", y="pet"),
}
})
def test_i_can_get_compiled_expr_when_multiple_choices(self):
sheerka, context, *concepts = self.init_test().with_concepts(
@@ -390,10 +389,10 @@ class TestExpressionParser(TestUsingMemoryBasedSheerka):
assert len(return_values) == 2
ret = return_values[0]
assert sheerka.objvalue(ret)[0].concept == CMV(concepts[0], x="a", y="b")
compare_with_test_object(sheerka.objvalue(ret)[0].concept, CMV(concepts[0], x="a", y="b"))
ret = return_values[1]
assert sheerka.objvalue(ret)[0].concept == CMV(concepts[1], x="a", y="b")
compare_with_test_object(sheerka.objvalue(ret)[0].concept, CMV(concepts[1], x="a", y="b"))
def test_i_can_get_compiled_expr_from_mix_when_multiple_choices(self):
sheerka, context, *concepts = self.init_test().with_concepts(
tests/parsers/test_FunctionParser.py
+47 -59
View File
@@ -3,10 +3,10 @@ import pytest
from core.builtin_concepts import BuiltinConcepts
from core.concept import Concept
from core.sheerka.services.SheerkaExecute import ParserInput
from parsers.BaseNodeParser import SCN, SCWC, CN, UTN, CNC, RN
from parsers.FunctionParser import FunctionParser, FN
from parsers.FunctionParser import FunctionParser
from parsers.PythonParser import PythonErrorNode
from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
from tests.parsers.parsers_utils import compute_expected_array
from tests.parsers.parsers_utils import compute_expected_array, SCN, SCWC, CN, UTN, CNC, RN, FN, get_test_obj
cmap = {
"one": Concept("one"),
@@ -80,60 +80,73 @@ class TestFunctionParser(TestUsingMemoryBasedSheerka):
parser.parser_input.next_token()
res = parser.parse_function()
assert res == expected
transformed_res = get_test_obj(res, expected)
assert transformed_res == expected
def test_i_can_parse_function_when_rule(self):
sheerka, context, parser = self.init_parser()
expected = FN("func(", ")", ["r:|1:"])
parser.reset_parser(context, ParserInput("func(r:|1:)"))
parser.parser_input.next_token()
res = parser.parse_function()
assert res == FN("func(", ")", ["r:|1:"])
transformed_res = get_test_obj(res, expected)
assert transformed_res == expected
@pytest.mark.parametrize("text, expected", [
("func()", SCN("func()")),
(" func()", SCN("func()")),
("func(one)", SCWC("func(", ")", CN("one"))),
("func(one, unknown, two)", SCWC("func(", ")", CN("one"), ", ", UTN("unknown"), (", ", 1), CN("two"))),
("func(one, twenty two)", SCWC("func(", ")", "one", ", ", CN("twenties", source="twenty two"))),
("func(one, twenty two)", SCWC("func(", ")", "one", ", ", CN("twenties", "twenty two"))),
("func(one plus two, three)", SCWC("func(", ")", CNC("plus", a="one", b="two"), ", ", UTN("three"))),
("func(func1(one), two)", SCWC("func(", (")", 1), SCWC("func1(", ")", "one"), ", ", "two"))
])
def test_i_can_parse(self, text, expected):
sheerka, context, parser = self.init_parser()
resolved_expected = compute_expected_array(cmap, text, [expected])[0]
res = parser.parse(context, ParserInput(text))
parser_result = res.body
expression = res.body.body
assert res.status
assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT)
assert expression == resolved_expected
transformed_expression = get_test_obj(expression, resolved_expected)
assert transformed_expression == resolved_expected
assert expression.python_node is not None
assert expression.return_value is not None
def test_i_can_parse_when_multiple_results_when_requested(self):
# the previous output was
# [
# SCWC("func(", ")", "one", ", ", "twenty ", "two"),
# SCWC("func(", ")", "one", ", ", CN("twenties", "twenty two"))
# ]
# But the first one is now filtered out, as it's not a valid python function call
sheerka, context, parser = self.init_parser()
parser.longest_concepts_only = False
text = "func(one, twenty two)"
expected = [SCWC("func(", ")", "one", ", ", "twenty ", "two"),
SCWC("func(", ")", "one", ", ", CN("twenties", source="twenty two"))]
all_resolved_expected = compute_expected_array(cmap, text, expected)
expected = [SCWC("func(", ")", "one", ", ", CN("twenties", "twenty two"))]
resolved_expected = compute_expected_array(cmap, text, expected)
results = parser.parse(context, ParserInput(text))
assert len(results) == 2
for res, resolved_expected in zip(results, all_resolved_expected):
parser_result = res.body
expressions = res.body.body
res = results[0]
assert not res.status
assert sheerka.isinstance(res.body, BuiltinConcepts.ERROR)
assert len(res.body.body) == 1
assert (res.body.body[0], PythonErrorNode)
assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT)
assert expressions == resolved_expected
res = results[1]
parser_result = res.body
expressions = res.body.body
assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT)
transformed_expressions = get_test_obj(expressions, resolved_expected[0])
assert transformed_expressions == resolved_expected[0]
def test_i_can_parse_when_the_parameter_is_not_a_concept(self):
"""
@@ -144,10 +157,15 @@ class TestFunctionParser(TestUsingMemoryBasedSheerka):
text = "func(unknown_concept)"
res = parser.parse(context, ParserInput(text))
expected = [SCWC("func(", ")", "unknown_concept")]
resolved_expected = compute_expected_array(cmap, text, expected)
assert res.status
parsed = res.body.body
transformed_parsed = get_test_obj([parsed], resolved_expected)
assert transformed_parsed == resolved_expected
def test_i_cannot_parse_when_the_concept_is_not_found(self):
def test_i_can_parse_when_the_concept_is_not_found(self):
"""
We do not check yet if it's a valid concept
If you find a cheap way to do so, simply remove this test
@@ -169,37 +187,24 @@ class TestFunctionParser(TestUsingMemoryBasedSheerka):
res = parser.parse(context, ParserInput(text))
parser_result = res.body
expression = res.body.body
transformed_expression = get_test_obj(expression, resolved_expected)
assert res.status
assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT)
assert expression == resolved_expected
assert transformed_expression == resolved_expected
assert expression.python_node is not None
assert expression.return_value is not None
# def test_i_cannot_parse_when_rule_not_found(self):
# sheerka, context, parser = self.init_parser()
# text = "func(r:|fake:)"
# expected = SCWC("func(", ")", RN("fake"))
# resolved_expected = compute_expected_array(cmap, text, [expected])[0]
#
# res = parser.parse(context, ParserInput(text))
# parser_result = res.body
# expression = res.body.body
#
# assert not res.status
# assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT)
# assert expression == resolved_expected
# assert expression.python_node is not None
# assert expression.return_value is not None
@pytest.mark.parametrize("text, expected_error_type", [
("one", BuiltinConcepts.NOT_FOR_ME),
("$*!", BuiltinConcepts.NOT_FOR_ME),
("func(", BuiltinConcepts.ERROR),
("func(one", BuiltinConcepts.ERROR),
("func(one, two, ", BuiltinConcepts.ERROR),
("func(one) and func(two)", BuiltinConcepts.ERROR),
("one func(one)", BuiltinConcepts.NOT_FOR_ME),
("one", BuiltinConcepts.NOT_FOR_ME), # no function found
("$*!", BuiltinConcepts.NOT_FOR_ME), # no function found
("func(", BuiltinConcepts.ERROR), # function found, but incomplete
("func(one", BuiltinConcepts.ERROR), # function found, but incomplete
("func(one, two, ", BuiltinConcepts.ERROR), # function found, but incomplete
("func(one) and func(two)", BuiltinConcepts.ERROR), # to many function
("one func(one)", BuiltinConcepts.NOT_FOR_ME), # function not found ! (as it is not the first)
("func(a=b, c)", BuiltinConcepts.ERROR), # function found, but cannot be parsed
("func(one two)", BuiltinConcepts.ERROR), # function found, but cannot be parsed
])
def test_i_cannot_parse(self, text, expected_error_type):
sheerka, context, parser = self.init_parser()
@@ -209,23 +214,6 @@ class TestFunctionParser(TestUsingMemoryBasedSheerka):
assert not res.status
assert sheerka.isinstance(res.body, expected_error_type)
@pytest.mark.parametrize("text, expected", [
("func(one two)", SCWC("func(", ")", "one", "two")),
])
def test_i_can_detect_none_function(self, text, expected):
sheerka, context, parser = self.init_parser()
resolved_expected = compute_expected_array(cmap, text, [expected])[0]
res = parser.parse(context, ParserInput(text))
parser_result = res.body
expression = res.body.body
assert not res.status
assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT)
assert expression == resolved_expected
assert expression.python_node is None
assert expression.return_value is None
@pytest.mark.parametrize("sequence, expected", [
(None, None),
([["a"]], [["a"]]),
tests/parsers/test_PythonWithConceptsParser.py
+15 -9
View File
@@ -7,7 +7,8 @@ from core.concept import Concept
from core.rule import Rule
from core.sheerka.services.SheerkaExecute import ParserInput
from core.tokenizer import Token, TokenKind, Tokenizer
from parsers.BaseNodeParser import ConceptNode, UnrecognizedTokensNode, RuleNode, SourceCodeNode
from core.var_ref import VariableRef
from parsers.BaseNodeParser import ConceptNode, UnrecognizedTokensNode, RuleNode, VariableNode
from parsers.PythonParser import PythonNode
from parsers.PythonWithConceptsParser import PythonWithConceptsParser
from parsers.UnrecognizedNodeParser import UnrecognizedNodeParser
@@ -29,8 +30,12 @@ def ret_val(*args):
tokens = [Token(TokenKind.RULE, (None, item.id), 0, 0, 0)]
result.append(RuleNode(item, index, index, tokens, f"r:|{item.id}:"))
index += 1
elif isinstance(item, VariableRef):
tokens = list(Tokenizer(item.prop, yield_eof=False))
result.append(VariableNode(item.obj, item.prop, index, index + len(tokens) - 1, tokens, f"{item.prop}"))
index += len(tokens)
else:
tokens = list(Tokenizer(item))
tokens = list(Tokenizer(item, yield_eof=False))
result.append(UnrecognizedTokensNode(index, index + len(tokens) - 1, tokens))
index += len(tokens)
@@ -58,10 +63,10 @@ class TestPythonWithConceptsParser(TestUsingMemoryBasedSheerka):
else:
assert res is None
def test_i_can_parse_concepts_and_python(self):
def test_i_can_parse_concepts_python_and_variable_ref(self):
context = self.get_context()
foo = Concept("foo")
input_return_value = ret_val(foo, " + 1")
input_return_value = ret_val(foo, " + 1 + ", VariableRef(foo, "var_name"))
parser = PythonWithConceptsParser()
result = parser.parse(context, input_return_value.body)
@@ -71,12 +76,13 @@ class TestPythonWithConceptsParser(TestUsingMemoryBasedSheerka):
assert result.status
assert result.who == parser.name
assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
assert wrapper.source == "foo + 1"
assert wrapper.source == "foo + 1 + var_name"
assert isinstance(return_value, PythonNode)
assert return_value.source == "__C__foo__C__ + 1"
assert return_value.original_source == "foo + 1"
assert return_value.get_dump(return_value.ast_) == to_str_ast("__C__foo__C__ + 1")
assert return_value.objects["__C__foo__C__"] == foo
assert return_value.source == "__C__foo__C__ + 1 + __V__foo__var_name__V__"
assert return_value.original_source == "foo + 1 + var_name"
assert return_value.get_dump(return_value.ast_) == to_str_ast("__C__foo__C__ + 1 + __V__foo__var_name__V__")
assert return_value.objects == {"__C__foo__C__": foo,
"__V__foo__var_name__V__": VariableRef(foo, "var_name")}
def test_i_can_parse_concepts_and_python_when_concept_is_known(self):
context = self.get_context()
tests/parsers/test_SequenceNodeParser.py
+25 -23
View File
@@ -1,12 +1,12 @@
import pytest
from core.builtin_concepts import BuiltinConcepts
from core.concept import Concept, DEFINITION_TYPE_DEF
from core.sheerka.services.SheerkaExecute import ParserInput
from parsers.SequenceNodeParser import SequenceNodeParser
from parsers.BaseNodeParser import cnode, utnode, CNC, SCN, CN
from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
from tests.parsers.parsers_utils import compute_expected_array
from tests.parsers.parsers_utils import compute_expected_array, CN, CNC, SCN, get_test_obj, compare_with_test_object, \
UTN
class TestAtomsParser(TestUsingMemoryBasedSheerka):
@@ -40,8 +40,8 @@ class TestAtomsParser(TestUsingMemoryBasedSheerka):
(" foo ", ["foo"]),
("foo bar", ["foo", "bar"]),
("foo bar twenties", ["foo", "bar", "twenties"]),
("a plus b", [CN("plus", 0, 4)]),
("mult", [CN("mult", 0, 0, "mult")]),
("a plus b", [CN("plus", None, 0, 4)]),
("mult", [CN("mult", "mult", 0, 0)]),
])
def test_i_can_parse_simple_sequences(self, text, expected):
concepts_map = {
@@ -62,7 +62,7 @@ class TestAtomsParser(TestUsingMemoryBasedSheerka):
expected_array = compute_expected_array(concepts_map, text, expected)
assert sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
assert lexer_nodes == expected_array
compare_with_test_object(lexer_nodes, expected_array)
@pytest.mark.parametrize("text, expected", [
("foo bar", ["foo bar"]),
@@ -85,7 +85,7 @@ class TestAtomsParser(TestUsingMemoryBasedSheerka):
expected_array = compute_expected_array(concepts_map, text, expected)
assert sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
assert lexer_nodes == expected_array
compare_with_test_object(lexer_nodes, expected_array)
@pytest.mark.parametrize("text, expected_status, expected", [
("foo bar suffixed one", False, ["foo bar", " suffixed ", "one"]),
@@ -123,12 +123,12 @@ class TestAtomsParser(TestUsingMemoryBasedSheerka):
expected_array = compute_expected_array(concepts_map, text, expected)
assert sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
assert lexer_nodes == expected_array
compare_with_test_object(lexer_nodes, expected_array)
@pytest.mark.parametrize("text, expected_status, expected", [
(" one two ", True, [cnode("one", 1, 1, "one"), cnode("two", 3, 3, "two")]),
(" one x$!# ", False, [cnode("one", 1, 1, "one"), utnode(2, 7, " x$!# ")]),
(" foo bar x$!# ", False, [cnode("foo bar", 1, 3, "foo bar"), utnode(4, 9, " x$!# ")]),
(" one two ", True, [CN("one", "one", 1, 1), CN("two", "two", 3, 3)]),
(" one x$!# ", False, [CN("one", "one", 1, 1), UTN(" x$!# ", 2, 7)]),
(" foo bar x$!# ", False, [CN("foo bar", "foo bar", 1, 3), UTN(" x$!# ", 4, 9)]),
])
def test_i_can_parse_when_surrounded_by_spaces(self, text, expected_status, expected):
concepts_map = {
@@ -150,7 +150,7 @@ class TestAtomsParser(TestUsingMemoryBasedSheerka):
expected_array = compute_expected_array(concepts_map, text, expected)
assert sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
assert lexer_nodes == expected_array
compare_with_test_object(lexer_nodes, expected_array)
@pytest.mark.parametrize("text, expected", [
("one two", [["one", "two"], ["one two"]])
@@ -173,7 +173,7 @@ class TestAtomsParser(TestUsingMemoryBasedSheerka):
assert res.status
expected_array = compute_expected_array(concepts_map, text, expected[i])
assert sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
assert lexer_nodes == expected_array
compare_with_test_object(lexer_nodes, expected_array)
def test_i_can_parse_multiple_concepts_when_long_names_and_unrecognized(self):
concepts_map = {
@@ -204,7 +204,7 @@ class TestAtomsParser(TestUsingMemoryBasedSheerka):
assert res.status == expected[0]
expected_array = compute_expected_array(concepts_map, text, expected[1])
assert sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
assert lexer_nodes == expected_array
compare_with_test_object(lexer_nodes, expected_array)
def test_i_can_parse_concepts_with_isa(self):
concepts_map = {
@@ -218,7 +218,7 @@ class TestAtomsParser(TestUsingMemoryBasedSheerka):
res = parser.parse(context, ParserInput("one"))
lexer_nodes = res.body.body
expected_array = compute_expected_array(concepts_map, "one", ["one"])
assert lexer_nodes == expected_array
compare_with_test_object(lexer_nodes, expected_array)
def test_i_can_parse_concepts_with_keyword(self):
concepts_map = {
@@ -231,12 +231,12 @@ class TestAtomsParser(TestUsingMemoryBasedSheerka):
res = parser.parse(context, ParserInput("a special concept"))
lexer_nodes = res.body.body
expected_array = compute_expected_array(concepts_map, "a special concept", ["a special concept"])
assert lexer_nodes == expected_array
compare_with_test_object(lexer_nodes, expected_array)
res = parser.parse(context, ParserInput("isa"))
lexer_nodes = res.body.body
expected_array = compute_expected_array(concepts_map, "isa", ["isa"])
assert lexer_nodes == expected_array
compare_with_test_object(lexer_nodes, expected_array)
def test_i_can_parse_concepts_when_sub_tokens(self):
concepts_map = {
@@ -256,7 +256,7 @@ class TestAtomsParser(TestUsingMemoryBasedSheerka):
expected_array = compute_expected_array(concepts_map, text, expected)
assert sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
assert lexer_nodes == expected_array
compare_with_test_object(lexer_nodes, expected_array)
@pytest.mark.parametrize("text", [
"foo",
@@ -283,8 +283,8 @@ class TestAtomsParser(TestUsingMemoryBasedSheerka):
@pytest.mark.parametrize("text, expected", [
("hello foo bar",
[
(True, [CNC("hello1", source="hello foo ", a="foo "), "bar"]),
(True, [CNC("hello2", source="hello foo ", b="foo "), "bar"]),
(True, [CNC("hello1", "hello foo ", a="foo "), "bar"]),
(True, [CNC("hello2", "hello foo ", b="foo "), "bar"]),
]),
])
def test_i_can_parse_when_unrecognized_yield_multiple_values(self, text, expected):
@@ -304,9 +304,10 @@ class TestAtomsParser(TestUsingMemoryBasedSheerka):
lexer_nodes = res.body.body
assert res.status == expected[0]
expected_array = compute_expected_array(concepts_map, text, expected[1])
assert sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
assert lexer_nodes == expected_array
expected_array = compute_expected_array(concepts_map, text, expected[1])
transformed_nodes = get_test_obj(lexer_nodes, expected_array)
assert transformed_nodes == expected_array
@pytest.mark.parametrize("text, expected", [
("1 + twenty one", [SCN("1 + twenty "), "one"]),
@@ -326,7 +327,8 @@ class TestAtomsParser(TestUsingMemoryBasedSheerka):
expected_array = compute_expected_array(concepts_map, text, expected)
assert sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
assert lexer_nodes == expected_array
transformed_nodes = get_test_obj(lexer_nodes, expected_array)
assert transformed_nodes == expected_array
@pytest.mark.parametrize("text, expected_is_evaluated", [
("foo", False),
tests/parsers/test_SyaNodeParser.py
+129 -123
View File
@@ -2,18 +2,19 @@ import pytest
import tests.parsers.parsers_utils
from core.builtin_concepts import BuiltinConcepts
from core.concept import Concept, CIO, CMV
from core.concept import Concept
from core.global_symbols import CONCEPT_COMPARISON_CONTEXT
from core.sheerka.services.SheerkaExecute import ParserInput
from core.tokenizer import Tokenizer
from core.utils import NextIdManager
from parsers.BaseNodeParser import utnode, cnode, short_cnode, UnrecognizedTokensNode, \
SCWC, CNC, UTN, SCN, CN
from parsers.BaseNodeParser import UnrecognizedTokensNode
from parsers.PythonParser import PythonNode
from parsers.SyaNodeParser import SyaNodeParser, SyaConceptParserHelper, SyaAssociativity, \
NoneAssociativeSequenceError, TooManyParametersFoundError, InFixToPostFix, ParenthesisMismatchError
from tests.TestUsingFileBasedSheerka import TestUsingFileBasedSheerka
from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
from tests.parsers.parsers_utils import UTN, SCWC, CNC, SCN, CIO, CN, compute_debug_array, CMV, get_test_obj, \
compare_with_test_object
def compute_expected_array(concepts_map, expression, expected):
@@ -102,6 +103,16 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
parser.init_from_concepts(context, concepts, sya=sya_def_to_use)
return sheerka, context, parser
@staticmethod
def compare_results(res, expected_sequences, concept_map, expression, validate_errors=True):
assert len(res) == len(expected_sequences)
for res_i, expected in zip(res, expected_sequences):
if validate_errors:
assert len(res_i.errors) == 0
expected_array = compute_expected_array(concept_map, expression, expected)
res_i_as_test_obj = get_test_obj(res_i.out, expected_array)
assert res_i_as_test_obj == expected_array
@pytest.mark.parametrize("expression, expected_sequences", [
("one plus two", [["one", "two", "plus"]]),
("1 + 1 plus two", [["1 + 1", "two", "plus"]]),
@@ -110,7 +121,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
["one + two", "three", "plus"]]),
("twenty one plus two", [
["twenty ", "one", "two", "plus"],
[short_cnode("twenties", "twenty one"), "two", "plus"]
[CN("twenties", "twenty one"), "two", "plus"]
]),
("x$!# plus two", [["x$!#", "two", "plus"]]),
@@ -122,7 +133,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
]),
("twenty one plus 1 + 1", [
["twenty ", "one", "1 + 1", "plus"],
[cnode("twenties", 0, 2, "twenty one"), "1 + 1", "plus"]
[CN("twenties", "twenty one", 0, 2), "1 + 1", "plus"]
]),
("x$!# plus 1 + 1", [["x$!#", "1 + 1", "plus"]]),
@@ -142,9 +153,9 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
]),
("twenty one plus two + three", [
["twenty ", "one", "two", "plus", " + ", "three"],
[cnode("twenties", 0, 2, "twenty one"), "two", "plus", " + ", "three"],
[CN("twenties", "twenty one", 0, 2), "two", "plus", " + ", "three"],
["twenty ", "one", "two + three", "plus"],
[cnode("twenties", 0, 2, "twenty one"), "two + three", "plus"],
[CN("twenties", "twenty one", 0, 2), "two + three", "plus"],
]),
("x$!# plus two + three", [
["x$!#", "two", "plus", " + ", "three"],
@@ -153,28 +164,28 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
("one plus twenty two", [
["one", "twenty ", "plus", "two"],
["one", cnode("twenties", 4, 6, "twenty two"), "plus"],
["one", CN("twenties", "twenty two", 4, 6), "plus"],
]),
("1 + 1 plus twenty one", [
["1 + 1", "twenty ", "plus", "one"],
["1 + 1", cnode("twenties", 8, 10, "twenty one"), "plus"],
["1 + 1", CN("twenties", "twenty one", 8, 10), "plus"],
]),
("one + two plus twenty one", [
["one", " + ", "two", "twenty ", "plus", ("one", 1)],
["one + two", "twenty ", "plus", ("one", 1)],
["one", " + ", "two", cnode("twenties", 8, 10, "twenty one"), "plus"],
["one + two", cnode("twenties", 8, 10, "twenty one"), "plus"],
["one", " + ", "two", CN("twenties", "twenty one", 8, 10), "plus"],
["one + two", CN("twenties", "twenty one", 8, 10), "plus"],
]),
("twenty one plus twenty two",
[
["twenty ", "one", ("twenty ", 1), "plus", "two"],
[cnode("twenties", 0, 2, "twenty one"), ("twenty ", 1), "plus", "two"],
["twenty ", "one", cnode("twenties", 6, 8, "twenty two"), "plus"],
[cnode("twenties", 0, 2, "twenty one"), cnode("twenties", 6, 8, "twenty two"), "plus"],
[CN("twenties", "twenty one", 0, 2), ("twenty ", 1), "plus", "two"],
["twenty ", "one", CN("twenties", "twenty two", 6, 8), "plus"],
[CN("twenties", "twenty one", 0, 2), CN("twenties", "twenty two", 6, 8), "plus"],
]),
("x$!# plus twenty two", [
["x$!#", "twenty ", "plus", "two"],
["x$!#", cnode("twenties", 7, 9, "twenty two"), "plus"]
["x$!#", CN("twenties", "twenty two", 7, 9), "plus"]
]),
("one plus z$!#", [["one", "z$!#", "plus"]]),
@@ -185,7 +196,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
]),
("twenty one plus z$!#", [
["twenty ", "one", "z$!#", "plus"],
[cnode("twenties", 0, 2, "twenty one"), "z$!#", "plus"],
[CN("twenties", "twenty one", 0, 2), "z$!#", "plus"],
]),
("x$!# plus z$!#", [["x$!#", "z$!#", "plus"]]),
])
@@ -194,17 +205,13 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
res = parser.infix_to_postfix(context, ParserInput(expression))
assert len(res) == len(expected_sequences)
for res_i, expected in zip(res, expected_sequences):
assert len(res_i.errors) == 0
expected_array = compute_expected_array(cmap, expression, expected)
assert res_i.out == expected_array
self.compare_results(res, expected_sequences, cmap, expression)
@pytest.mark.parametrize("expression, expected_sequences", [
("one plus plus plus 1 + 1", [["one", "1 + 1", "plus plus plus"]]),
("x$!# another long name infix twenty two", [
["x$!#", "twenty ", "another long name infix", "two"],
["x$!#", cnode("twenties", 13, 15, "twenty two"), "another long name infix"],
["x$!#", CN("twenties", "twenty two", 13, 15), "another long name infix"],
]),
])
def test_i_can_post_fix_infix_concepts_with_long_name(self, expression, expected_sequences):
@@ -220,11 +227,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
res = parser.infix_to_postfix(context, ParserInput(expression))
assert len(res) == len(expected_sequences)
for res_i, expected in zip(res, expected_sequences):
assert len(res_i.errors) == 0
expected_array = compute_expected_array(concepts_map, expression, expected)
assert res_i.out == expected_array
self.compare_results(res, expected_sequences, concepts_map, expression)
@pytest.mark.parametrize("expression, expected_sequences", [
("one prefixed", [["one", "prefixed"]]),
@@ -235,7 +238,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
]),
("twenty one prefixed", [
["twenty ", "one", "prefixed"],
[cnode("twenties", 0, 2, "twenty one"), "prefixed"],
[CN("twenties", "twenty one", 0, 2), "prefixed"],
]),
("x$!# prefixed", [["x$!#", "prefixed"]]),
])
@@ -244,11 +247,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
res = parser.infix_to_postfix(context, ParserInput(expression))
assert len(res) == len(expected_sequences)
for res_i, expected in zip(res, expected_sequences):
assert len(res_i.errors) == 0
expected_array = compute_expected_array(cmap, expression, expected)
assert res_i.out == expected_array
self.compare_results(res, expected_sequences, cmap, expression)
@pytest.mark.parametrize("expression, expected_sequences", [
("one prefixed prefixed", [["one", "prefixed prefixed"]]),
@@ -259,7 +258,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
]),
("twenty one prefixed prefixed", [
["twenty ", "one", "prefixed prefixed"],
[cnode("twenties", 0, 2, "twenty one"), "prefixed prefixed"],
[CN("twenties", "twenty one", 0, 2), "prefixed prefixed"],
]),
("x$!# prefixed prefixed", [["x$!#", "prefixed prefixed"]]),
@@ -271,7 +270,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
]),
("twenty one long name prefixed", [
["twenty ", "one", "long name prefixed"],
[cnode("twenties", 0, 2, "twenty one"), "long name prefixed"],
[CN("twenties", "twenty one", 0, 2), "long name prefixed"],
]),
("x$!# long name prefixed", [["x$!#", "long name prefixed"]]),
])
@@ -287,11 +286,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
res = parser.infix_to_postfix(context, ParserInput(expression))
assert len(res) == len(expected_sequences)
for res_i, expected in zip(res, expected_sequences):
assert len(res_i.errors) == 0
expected_array = compute_expected_array(concepts_map, expression, expected)
assert res_i.out == expected_array
self.compare_results(res, expected_sequences, concepts_map, expression)
@pytest.mark.parametrize("expression, expected_sequences", [
("suffixed one", [["one", "suffixed"]]),
@@ -302,7 +297,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
]),
("suffixed twenty one", [
["twenty ", "suffixed", "one"],
[cnode("twenties", 2, 4, "twenty one"), "suffixed"],
[CN("twenties", "twenty one", 2, 4), "suffixed"],
]),
("suffixed x$!#", [["x$!#", "suffixed"]]),
])
@@ -311,11 +306,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
res = parser.infix_to_postfix(context, ParserInput(expression))
assert len(res) == len(expected_sequences)
for res_i, expected in zip(res, expected_sequences):
assert len(res_i.errors) == 0
expected_array = compute_expected_array(cmap, expression, expected)
assert res_i.out == expected_array
self.compare_results(res, expected_sequences, cmap, expression)
@pytest.mark.parametrize("expression, expected", [
("suffixed suffixed one", ["one", "suffixed suffixed"]),
@@ -333,8 +324,10 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
res = parser.infix_to_postfix(context, ParserInput(expression))
expected_array = compute_expected_array(concepts_map, expression, expected)
assert len(res) == 1
assert res[0].out == expected_array
transformed_out = get_test_obj(res[0].out, expected_array)
assert transformed_out == expected_array
@pytest.mark.parametrize("expression, expected_sequences", [
("one ? two : three", [["one", "two", "three", "?"]]),
@@ -342,14 +335,14 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
("1+1 ? one + two : twenty one", [
["1+1", "one", " + ", "two"], # error is detected so the parsing has stopped
["1+1", "one + two", "twenty ", "?", ("one", 1)],
["1+1", "one + two", short_cnode("twenties", "twenty one"), "?"],
["1+1", "one + two", CN("twenties", "twenty one"), "?"],
]),
("x$!# ? y$!# : z$!#", [["x$!#", "y$!#", "z$!#", "?"]]),
("if one then two else three end", [["one", "two", "three", "if"]]),
("if 1+1 then x$!# else twenty one end", [
["1+1", "x$!#", "twenty ", "one"], # an error is detected
["1+1", "x$!#", short_cnode("twenties", "twenty one"), "if"],
["1+1", "x$!#", CN("twenties", "twenty one"), "if"],
]),
("if x$!# then one + two else z$!# end", [
["x$!#", "one", " + ", "two"], # error is detected so the parsing has stopped
@@ -373,24 +366,20 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
res = parser.infix_to_postfix(context, ParserInput(expression))
assert len(res) == len(expected_sequences)
for res_i, expected in zip(res, expected_sequences):
# assert len(res_i.errors) == 0 # Do not validate errors
expected_array = compute_expected_array(cmap, expression, expected)
assert res_i.out == expected_array
self.compare_results(res, expected_sequences, cmap, expression, validate_errors=False)
@pytest.mark.parametrize("expression, expected_sequences", [
("one ? ? two : : three", [["one", "two", "three", "? ?"]]),
("1+1 ? ? one + two : : twenty one", [
["1+1", "one", " + ", "two"], # error
["1+1", "one + two", "twenty ", "? ?", ("one", 1)],
["1+1", "one + two", short_cnode("twenties", "twenty one"), "? ?"],
["1+1", "one + two", CN("twenties", "twenty one"), "? ?"],
]),
("if if one then then two else else three end end ", [["one", "two", "three", "if if"]]),
("if if 1+1 then then x$!# else else twenty one end end ", [
["1+1", "x$!#", "twenty ", "one"], # error
["1+1", "x$!#", short_cnode("twenties", "twenty one"), "if if"]]),
["1+1", "x$!#", CN("twenties", "twenty one"), "if if"]]),
])
def test_i_can_post_fix_ternary_concept_with_long_names(self, expression, expected_sequences):
concepts_map = {
@@ -405,11 +394,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
res = parser.infix_to_postfix(context, ParserInput(expression))
assert len(res) == len(expected_sequences)
for res_i, expected in zip(res, expected_sequences):
# assert len(res_i.errors) == 0 # Do not validate errors
expected_array = compute_expected_array(concepts_map, expression, expected)
assert res_i.out == expected_array
self.compare_results(res, expected_sequences, concepts_map, expression, validate_errors=False)
@pytest.mark.parametrize("expression, expected", [
("foo bar baz", ["baz", "bar", "foo"]),
@@ -428,7 +413,8 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
expected_array = compute_expected_array(concepts_map, expression, expected)
assert len(res) == 1
assert res[0].out == expected_array
transformed_out = get_test_obj(res[0].out, expected_array)
assert transformed_out == expected_array
@pytest.mark.parametrize("expression, expected", [
("baz bar foo", ["baz", "bar", "foo"]),
@@ -451,7 +437,8 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
expected_array = compute_expected_array(concepts_map, expression, expected)
assert len(res) == 1
assert res[0].out == expected_array
transformed_out = get_test_obj(res[0].out, expected_array)
assert transformed_out == expected_array
@pytest.mark.parametrize("expression, expected", [
("one plus two mult three", ["one", "two", "three", "mult", "plus"]),
@@ -466,7 +453,8 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
expected_array = compute_expected_array(cmap, expression, expected)
assert len(res) == 1
assert res[0].out == expected_array
transformed_out = get_test_obj(res[0].out, expected_array)
assert transformed_out == expected_array
def test_i_can_post_fix_unary_with_precedence(self):
concepts_map = {
@@ -487,7 +475,8 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
expected_array = compute_expected_array(concepts_map, expression, expected)
assert len(res) == 1
assert res[0].out == expected_array
transformed_out = get_test_obj(res[0].out, expected_array)
assert transformed_out == expected_array
# change the precedence
sya_def = {
@@ -502,7 +491,8 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
expected_array = compute_expected_array(concepts_map, expression, expected)
assert len(res) == 1
assert res[0].out == expected_array
transformed_out = get_test_obj(res[0].out, expected_array)
assert transformed_out == expected_array
def test_i_can_post_fix_right_associated_binary(self):
concepts_map = {
@@ -524,7 +514,8 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
expected_array = compute_expected_array(concepts_map, expression, expected)
assert len(res) == 1
assert res[0].out == expected_array
transformed_out = get_test_obj(res[0].out, expected_array)
assert transformed_out == expected_array
def test_i_can_post_fix_left_associated_binary(self):
concepts_map = {
@@ -546,7 +537,8 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
expected_array = compute_expected_array(concepts_map, expression, expected)
assert len(res) == 1
assert res[0].out == expected_array
transformed_out = get_test_obj(res[0].out, expected_array)
assert transformed_out == expected_array
@pytest.mark.parametrize("expression, expected", [
("x$!# ? y$!# : z$!# ? two : three", ["x$!#", "y$!#", "z$!#", "two", "three", ("?", 1), "?"]),
@@ -572,8 +564,10 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
res = parser.infix_to_postfix(context, ParserInput(expression))
expected_array = compute_expected_array(concepts_map, expression, expected)
assert len(res) == 1
assert res[0].out == expected_array
transformed_out = get_test_obj(res[0].out, expected_array)
assert transformed_out == expected_array
@pytest.mark.parametrize("expression, expected", [
("x$!# ? y$!# : z$!# ? two : three", ["x$!#", "y$!#", "z$!#", "?", "two", "three", ("?", 1)]),
@@ -599,8 +593,10 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
res = parser.infix_to_postfix(context, ParserInput(expression))
expected_array = compute_expected_array(concepts_map, expression, expected)
assert len(res) == 1
assert res[0].out == expected_array
transformed_out = get_test_obj(res[0].out, expected_array)
assert transformed_out == expected_array
def test_i_can_post_fix_when_multiple_concepts_are_found(self):
concepts_map = {
@@ -617,11 +613,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
["baz", "foo bar"]
]
assert len(res) == len(expected_sequences)
for res_i, expected in zip(res, expected_sequences):
assert len(res_i.errors) == 0
expected_array = compute_expected_array(concepts_map, expression, expected)
assert res_i.out == expected_array
self.compare_results(res, expected_sequences, concepts_map, expression)
@pytest.mark.parametrize("expression, expected", [
# ("function(one plus three) minus two",
@@ -667,7 +659,8 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
expected_array = compute_expected_array(cmap, expression, expected)
assert len(res) == 1
assert res[0].out == expected_array
transformed_out = get_test_obj(res[0].out, expected_array)
assert transformed_out == expected_array
@pytest.mark.parametrize("expression, expected_sequences", [
# composition
@@ -681,20 +674,18 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
[[SCWC("function(", ")", CNC("prefixed", a=CIO("twenties", source="twenty two")))]]),
("function(if one then twenty two else three end)",
[[SCWC("function(", ")", CNC("if", a="one", b=CIO("twenties", source="twenty two"), c="three", end=16))]]),
("func1(func2(one two) three)",
[[SCWC("func1(", (")", 1), SCWC("func2(", ")", "one", "two"), "three")]]),
("twenty two(suffixed one)", [
["twenty ", SCWC("two(", ")", CNC("suffixed", a="one"))],
[CN("twenties", source="twenty two"), "one", "suffixed"],
[CN("twenties", "twenty two"), "one", "suffixed"],
]),
("twenty two(one prefixed)", [
["twenty ", SCWC("two(", ")", CNC("prefixed", a="one"))],
[CN("twenties", source="twenty two"), "one", "prefixed"],
[CN("twenties", "twenty two"), "one", "prefixed"],
]),
("f1(one plus two mult three) plus f2(suffixed x$!# prefixed)", [
[SCWC("f1(", ")", CN("plus", source="one plus two mult three")),
SCWC("f2(", (")", 1), CN("suffixed", source="suffixed x$!# prefixed")),
("f1(one plus two mult three) plus f2(suffixed xxx prefixed)", [
[SCWC("f1(", ")", CN("plus", "one plus two mult three")),
SCWC("f2(", (")", 1), CN("suffixed", "suffixed xxx prefixed")),
("plus", 1)]
]),
@@ -706,8 +697,8 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
[SCWC("f1(", ")", "one"), SCWC("f2(", (")", 1), "two"), SCWC("f3(", (")", 2), "three"), "if"]]),
# Sequence
("if one then two else three end function(x$!#)", [
["one", "two", "three", "if", UTN(" ", start=13, end=13), SCWC("function(", ")", "x$!#")]]),
("if one then two else three end function(xxx)", [
["one", "two", "three", "if", UTN(" ", start=13, end=13), SCWC("function(", ")", "xxx")]]),
("one prefixed function(two)", [["one", "prefixed", UTN(" ", start=3, end=3), SCWC("function(", ")", "two")]]),
("suffixed one function(two)", [["one", "suffixed", UTN(" ", start=3, end=3), SCWC("function(", ")", "two")]]),
("func(one, two, three)", [[SCWC("func(", ")", "one", ", ", "two", (", ", 1), "three")]]),
@@ -717,10 +708,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
res = parser.infix_to_postfix(context, ParserInput(expression))
assert len(res) == len(expected_sequences)
for res_i, expected in zip(res, expected_sequences):
expected_array = compute_expected_array(cmap, expression, expected)
assert res_i.out == expected_array
self.compare_results(res, expected_sequences, cmap, expression)
@pytest.mark.parametrize("expression, expected", [
("(", ("(", 0)),
@@ -800,7 +788,8 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
expected_array = compute_expected_array(cmap, expression, expected)
assert len(res) == 1
assert res[0].out == expected_array
transformed_out = get_test_obj(res[0].out, expected_array)
assert transformed_out == expected_array
@pytest.mark.parametrize("expression", [
"one ? two : three",
@@ -818,7 +807,8 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
expected_array = compute_expected_array(cmap, expression, expected)
assert len(res) == 1
assert res[0].out == expected_array
transformed_out = get_test_obj(res[0].out, expected_array)
assert transformed_out == expected_array
def test_the_more_concepts_the_more_results(self):
concepts_map = {
@@ -837,7 +827,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
expression = "a plus plus equals b"
res = parser.infix_to_postfix(context, ParserInput(expression))
expected_array = tests.parsers.parsers_utils.compute_debug_array(res)
expected_array = compute_debug_array(res)
assert len(expected_array) == len([
["T(a)", "C(a plus b)", "C(a plus b)", "T(equals)", "T(b)"],
["T(a)", "C(a plus b)", "C(a plus plus)", "T(equals)", "T(b)"],
@@ -861,14 +851,17 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
sheerka, context, parser = self.init_parser(concepts_map, None)
res = parser.infix_to_postfix(context, ParserInput("one ? ? two '::' three"))
assert len(res) == 1
assert res[0].out == [
cnode("one", start=0, end=0, source="one"),
cnode("two", start=6, end=6, source="two"),
cnode("three", start=10, end=10, source="three"),
expected_array = [
CN("one", start=0, end=0, source="one"),
CN("two", start=6, end=6, source="two"),
CN("three", start=10, end=10, source="three"),
SyaConceptParserHelper(concepts_map["ternary"], 2),
]
assert len(res) == 1
transformed_out = get_test_obj(res[0].out, expected_array)
assert transformed_out == expected_array
def test_i_cannot_chain_non_associative(self):
concepts_map = {
"less than": Concept("a less than b").def_var("a").def_var("b"),
@@ -897,10 +890,12 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
expression = "suffixed twenties"
res = parser.infix_to_postfix(context, ParserInput(expression))
expected = [cnode("twenties", 2, 2, "twenties"), "suffixed"]
expected = [CN("twenties", "twenties", 2, 2), "suffixed"]
expected_array = compute_expected_array(cmap, expression, expected)
assert len(res) == 1
assert res[0].out == expected_array
transformed_out = get_test_obj(res[0].out, expected_array)
assert transformed_out == expected_array
@pytest.mark.parametrize("expression, expected_debugs", [
("one", [[" 0:one => PUSH_UNREC"]]),
@@ -999,12 +994,12 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
assert res.status
assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
assert lexer_nodes == [CN(cmap["plus"], 0, 8, source=text)]
compare_with_test_object(lexer_nodes, [CN(cmap["plus"], text, 0, 8)])
# check the compiled
expected_concept = lexer_nodes[0].concept
assert expected_concept.get_compiled()["a"] == cmap["one"]
assert expected_concept.get_compiled()["b"] == CMV(cmap["mult"], a="two", b="three")
compare_with_test_object(expected_concept.get_compiled()["b"], CMV(cmap["mult"], a="two", b="three"))
assert expected_concept.get_compiled()["b"].get_compiled()["a"] == cmap["two"]
assert expected_concept.get_compiled()["b"].get_compiled()["b"] == cmap["three"]
@@ -1023,7 +1018,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
assert res.status
assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
assert lexer_nodes == [CN(cmap["suffixed"], 0, 6, source=text)]
compare_with_test_object(lexer_nodes, [CN(cmap["suffixed"], text, 0, 6)])
# check the compiled
expected_concept = lexer_nodes[0].concept
@@ -1051,7 +1046,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
lexer_nodes = res[1].body.body
assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
assert lexer_nodes == [CN(cmap["suffixed"], 0, 4, source=text)]
compare_with_test_object(lexer_nodes, [CN(cmap["suffixed"], text, 0, 4)])
# check the compiled
expected_concept = lexer_nodes[0].concept
@@ -1071,9 +1066,9 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
assert res.status
assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
assert lexer_nodes == [
CN(cmap["plus"], 0, 9, source="one plus 1 + 1 "),
CN(cmap["suffixed"], 10, 12, source="suffixed two")]
compare_with_test_object(lexer_nodes, [
CN(cmap["plus"], "one plus 1 + 1 ", 0, 9),
CN(cmap["suffixed"], "suffixed two", 10, 12)])
# check the compiled
concept_plus_a = lexer_nodes[0].concept.get_compiled()["a"]
@@ -1105,7 +1100,9 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
expected_array = compute_expected_array(cmap, text, expected_result)
assert res.status == expected_status
assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
assert lexer_nodes == expected_array
transformed_nodes = get_test_obj(lexer_nodes, expected_array)
assert transformed_nodes == expected_array
@pytest.mark.parametrize("text", [
"function(suffixed one)",
@@ -1166,11 +1163,15 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
expected_array = compute_expected_array(cmap, text, expected_result)
assert not res.status
assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
assert lexer_nodes == expected_array
transformed_nodes = get_test_obj(lexer_nodes, expected_array)
assert transformed_nodes == expected_array
# assert lexer_nodes == expected_array
@pytest.mark.parametrize("text, expected_result", [
("a plus b", [CN("plus", source="a plus b")]),
("suffixed a plus b", [CN("suffixed", source="suffixed a plus b")]),
("a plus b", [CN("plus", "a plus b")]),
("suffixed a plus b", [CN("suffixed", "suffixed a plus b")]),
])
def test_i_can_almost_parse_concept_definition(self, text, expected_result):
"""
@@ -1190,7 +1191,9 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
expected_array = compute_expected_array(cmap, text, expected_result)
assert not res.status
assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
assert lexer_nodes == expected_array
transformed_nodes = get_test_obj(lexer_nodes, expected_array)
assert transformed_nodes == expected_array
# assert lexer_nodes == expected_array
@pytest.mark.parametrize("text, expected_concept, expected_unrecognized", [
("x$!# prefixed", "prefixed", ["a"]),
@@ -1209,15 +1212,18 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
assert not res.status
assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
assert lexer_nodes == [CN(cmap[expected_concept], 0, expected_end, source=text)]
expected_array = [CN(cmap[expected_concept], text, 0, expected_end)]
transformed_nodes = get_test_obj(lexer_nodes, expected_array)
assert transformed_nodes == expected_array
# assert lexer_nodes == [CN(cmap[expected_concept], text, 0, expected_end)]
concept_found = lexer_nodes[0].concept
for unrecognized in expected_unrecognized:
assert isinstance(concept_found.get_compiled()[unrecognized], UnrecognizedTokensNode)
@pytest.mark.parametrize("text, expected", [
("x$!# suffixed one", [utnode(0, 4, "x$!# "), cnode("suffixed __var__0", 5, 7, "suffixed one")]),
("one prefixed x$!#", [cnode("__var__0 prefixed", 0, 2, "one prefixed"), utnode(3, 7, " x$!#")]),
("x$!# suffixed one", [UTN("x$!# ", 0, 4), CN("suffixed __var__0", "suffixed one", 5, 7)]),
("one prefixed x$!#", [CN("__var__0 prefixed", "one prefixed", 0, 2), UTN(" x$!#", 3, 7)]),
])
def test_i_cannot_parse_when_part_of_the_sequence_is_not_recognized(self, text, expected):
sheerka, context, parser = self.init_parser()
@@ -1228,7 +1234,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
assert not res.status
assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
assert lexer_nodes == expected
compare_with_test_object(lexer_nodes, expected)
def test_i_cannot_parse_function_using_short_name(self):
concepts_map = {
@@ -1301,15 +1307,15 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
assert len(actual) == 1
assert actual[0].to_out == resolved_to_out
compare_with_test_object(actual[0].to_out, resolved_to_out)
actual[0].function.fix_source()
assert actual[0].function == resolved_function_name[0]
compare_with_test_object(actual[0].function, resolved_function_name[0])
@pytest.mark.parametrize("expression, expected_list", [
("twenty two function(", [(["twenty ", "two", UTN(" ", 3, 3)], "function("),
([CN("twenties", source="twenty two"), UTN(" ", 3, 3)], "function(")]),
([CN("twenties", "twenty two"), UTN(" ", 3, 3)], "function(")]),
("twenty two(", [(["twenty "], "two("),
([CN("twenties", source="twenty two")], None)]),
([CN("twenties", "twenty two")], None)]),
])
def test_i_can_get_functions_names_from_unrecognized_when_multiple_results(self, expression, expected_list):
sheerka, context, parser = self.init_parser()
@@ -1326,11 +1332,11 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
for actual, expected in zip(actual_list, expected_list):
resolved_to_out = compute_expected_array(cmap, expression, expected[0])
assert actual.to_out == resolved_to_out
compare_with_test_object(actual.to_out, resolved_to_out)
if actual.function:
actual.function.fix_source()
resolved_function_name = compute_expected_array(cmap, expression, [expected[1]])
assert actual.function == resolved_function_name[0]
compare_with_test_object(actual.function, resolved_function_name[0])
else:
assert actual.function is None
@@ -1344,7 +1350,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
assert res.status
assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
assert lexer_nodes == [CN(cmap["suffixed"], 0, 6, source=text)]
compare_with_test_object(lexer_nodes, [CN(cmap["suffixed"], text, 0, 6)])
# add an ontology layer and make sure will still can parse
sheerka.push_ontology(context, "new ontology")
@@ -1356,7 +1362,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
assert res.status
assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
assert lexer_nodes == [CN(cmap["suffixed"], 0, 6, source=text)]
compare_with_test_object(lexer_nodes, [CN(cmap["suffixed"], text, 0, 6)])
class TestFileBaseSyaNodeParser(TestUsingFileBasedSheerka):
tests/parsers/test_UnrecognizedNodeParser.py
+33 -33
View File
@@ -1,14 +1,14 @@
from core.builtin_concepts import ParserResultConcept, BuiltinConcepts
from core.concept import Concept, CC
from core.concept import Concept
from core.tokenizer import Tokenizer, TokenKind
from parsers.BaseNodeParser import ConceptNode, UnrecognizedTokensNode, scnode, cnode, \
utnode, CN, CNC, UTN, SourceCodeWithConceptNode, SCWC, SourceCodeNode
from parsers.BaseNodeParser import ConceptNode, UnrecognizedTokensNode, SourceCodeWithConceptNode, SourceCodeNode
from parsers.BnfNodeParser import BnfNodeParser
from parsers.SequenceNodeParser import SequenceNodeParser
from parsers.SyaNodeParser import SyaNodeParser
from parsers.UnrecognizedNodeParser import UnrecognizedNodeParser
from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
from tests.parsers.parsers_utils import compute_expected_array, get_node
from tests.parsers.parsers_utils import compute_expected_array, get_node, CC, UTN, CNC, CN, SCWC, \
compare_with_test_object, SCN
def get_input_nodes_from(my_concepts_map, full_expr, *args):
@@ -19,10 +19,10 @@ def get_input_nodes_from(my_concepts_map, full_expr, *args):
concept.get_compiled()[k] = _get_real_node(v)
return concept
if isinstance(n, (utnode, UTN)):
if isinstance(n, UTN):
return UnrecognizedTokensNode(n.start, n.end, full_expr_as_tokens[n.start: n.end + 1])
if isinstance(n, (CNC, CN, cnode)):
if isinstance(n, (CNC, CN)):
concept = n.concept if hasattr(n, "concept") and n.concept else \
Concept().update_from(my_concepts_map[n.concept_key])
tokens = full_expr_as_tokens[n.start: n.end + 1]
@@ -116,7 +116,7 @@ class TestUnrecognizedNodeParser(TestUsingMemoryBasedSheerka):
a=" one ",
b=" two three ",
c=" twenty one ",
d=utnode(12, 18, " 1 + 2 "),
d=UTN(" 1 + 2 ", 12, 18),
e=" one plus two mult three"))[0]
res = UnrecognizedNodeParser().validate_concept_node(context, node)
@@ -130,13 +130,14 @@ class TestUnrecognizedNodeParser(TestUsingMemoryBasedSheerka):
assert sheerka.isinstance(concept.get_compiled()["a"][0], BuiltinConcepts.RETURN_VALUE)
assert concept.get_compiled()["a"][0].status
assert concept.get_compiled()["a"][0].who == "parsers." + SequenceNodeParser.NAME
assert concept.get_compiled()["a"][0].body.body == [cnode("one", 1, 1, "one")]
compare_with_test_object(concept.get_compiled()["a"][0].body.body, [CN("one", "one", 1, 1)])
assert len(concept.get_compiled()["b"]) == 1
assert sheerka.isinstance(concept.get_compiled()["b"][0], BuiltinConcepts.RETURN_VALUE)
assert concept.get_compiled()["b"][0].status
assert concept.get_compiled()["b"][0].who == "parsers." + SequenceNodeParser.NAME
assert concept.get_compiled()["b"][0].body.body == [cnode("two", 1, 1, "two"), cnode("three", 3, 3, "three")]
compare_with_test_object(concept.get_compiled()["b"][0].body.body,
[CN("two", "two", 1, 1), CN("three", "three", 3, 3)])
assert len(concept.get_compiled()["c"]) == 1
assert sheerka.isinstance(concept.get_compiled()["c"][0], BuiltinConcepts.RETURN_VALUE)
@@ -145,8 +146,8 @@ class TestUnrecognizedNodeParser(TestUsingMemoryBasedSheerka):
expected_nodes = compute_expected_array(
concepts_map,
" twenty one ",
[CNC("twenties", source="twenty one", unit="one")])
assert concept.get_compiled()["c"][0].body.body == expected_nodes
[CNC("twenties", "twenty one", unit="one")])
compare_with_test_object(concept.get_compiled()["c"][0].body.body, expected_nodes)
assert len(concept.get_compiled()["d"]) == 1
assert sheerka.isinstance(concept.get_compiled()["d"][0], BuiltinConcepts.RETURN_VALUE)
@@ -164,7 +165,7 @@ class TestUnrecognizedNodeParser(TestUsingMemoryBasedSheerka):
[CNC("plus", a="one", b=CC("mult", a="two", b="three"))],
exclude_body=True)
assert concept.get_compiled()["e"][0].body.body == expected_nodes
compare_with_test_object(concept.get_compiled()["e"][0].body.body, expected_nodes)
# # sanity check, I can evaluate the concept
# evaluated = sheerka.evaluate_concept(self.get_context(sheerka, eval_body=True), concept)
@@ -204,8 +205,8 @@ class TestUnrecognizedNodeParser(TestUsingMemoryBasedSheerka):
expected_nodes = compute_expected_array(
concepts_map,
" twenty two",
[CNC("twenties", source="twenty two", unit="two")])
assert res.body.concept.get_compiled()["b"].get_compiled()["b"][0].body.body == expected_nodes
[CNC("twenties", "twenty two", unit="two")])
compare_with_test_object(res.body.concept.get_compiled()["b"].get_compiled()["b"][0].body.body, expected_nodes)
def test_i_can_validate_and_evaluate_a_concept_node_with_python(self):
sheerka, context, parser = self.init_parser()
@@ -281,7 +282,7 @@ class TestUnrecognizedNodeParser(TestUsingMemoryBasedSheerka):
assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT)
assert parser_result.source == expression
assert len(actual_nodes) == 1
assert actual_nodes[0] == scnode(0, 4, expression)
compare_with_test_object(actual_nodes[0], SCN(expression, 0, 4))
def test_i_cannot_parse_unrecognized_python_that_looks_like_concept(self):
sheerka, context, parser = self.init_parser()
@@ -319,7 +320,7 @@ class TestUnrecognizedNodeParser(TestUsingMemoryBasedSheerka):
expected_array = compute_expected_array(
concepts_map,
expression, [CNC("twenties", source=expression, unit="one")])
assert actual_nodes == expected_array
compare_with_test_object(actual_nodes, expected_array)
def test_i_can_parse_unrecognized_sya_concept_node(self):
sheerka, context, parser = self.init_parser()
@@ -343,7 +344,7 @@ class TestUnrecognizedNodeParser(TestUsingMemoryBasedSheerka):
a="one",
b=CC("mult", source="two mult three", a="two", b="three"))],
exclude_body=True)
assert actual_nodes == expected_array
compare_with_test_object(actual_nodes, expected_array)
def test_i_can_parse_unrecognized_source_code_with_concept_node(self):
sheerka, context, parser = self.init_parser()
@@ -388,8 +389,8 @@ class TestUnrecognizedNodeParser(TestUsingMemoryBasedSheerka):
expression = "hello get_user_name(twenty one)"
tmp_node = CNC("hello_sya",
source="hello get_user_name(twenty one)",
a=SCWC("get_user_name(", ")", CNC("twenties", source="twenty one", unit="one")))
"hello get_user_name(twenty one)",
a=SCWC("get_user_name(", ")", CNC("twenties", "twenty one", unit="one")))
nodes = get_input_nodes_from(concepts_map, expression, tmp_node)
parser_input = ParserResultConcept("parsers.xxx", source=expression, value=nodes)
@@ -404,9 +405,9 @@ class TestUnrecognizedNodeParser(TestUsingMemoryBasedSheerka):
expected_array = compute_expected_array(
concepts_map,
expression, [CN("hello_sya", source="hello get_user_name(twenty one)")],
expression, [CN("hello_sya", "hello get_user_name(twenty one)")],
exclude_body=True)
assert actual_nodes == expected_array
compare_with_test_object(actual_nodes, expected_array)
assert isinstance(actual_nodes[0].concept.get_compiled()["a"], list)
assert sheerka.isinstance(actual_nodes[0].concept.get_compiled()["a"][0], BuiltinConcepts.RETURN_VALUE)
@@ -416,7 +417,7 @@ class TestUnrecognizedNodeParser(TestUsingMemoryBasedSheerka):
expression = "one plus two three"
sequence = get_input_nodes_from(concepts_map, expression,
CNC("plus", a="one", b="two"),
utnode(5, 6, " three"))
UTN(" three", 5, 6))
parser_input = ParserResultConcept("parsers.xxx", source="one plus two three", value=sequence)
res = parser.parse(context, parser_input)
@@ -429,7 +430,7 @@ class TestUnrecognizedNodeParser(TestUsingMemoryBasedSheerka):
expression, [
CNC("plus", a="one", b="two"),
CN("three", start=6, end=6)])
assert actual_nodes == expected_array
compare_with_test_object(actual_nodes, expected_array)
def test_i_can_parse_when_multiple_atom_and_sya(self):
sheerka, context, parser = self.init_parser()
@@ -445,19 +446,18 @@ class TestUnrecognizedNodeParser(TestUsingMemoryBasedSheerka):
actual_nodes0 = res[0].body.body
expected_0 = compute_expected_array(concepts_map, expression, [
CN("two", 0, 0),
CN("two", start=0, end=0),
CN("hello_atom", source="hello one", start=2, end=4),
CN("three", 6, 6)])
assert actual_nodes0 == expected_0
CN("three", start=6, end=6)])
compare_with_test_object(actual_nodes0, expected_0)
actual_nodes1 = res[1].body.body
expected_1 = compute_expected_array(concepts_map, expression, [
CN("two", 0, 0),
CNC("hello_sya", source="hello one", start=2, end=4, a="one"),
CN("three", 6, 6)],
CN("two", start=0, end=0),
CNC("hello_sya", "hello one", start=2, end=4, a="one"),
CN("three", start=6, end=6)],
exclude_body=True)
assert actual_nodes1 == expected_1
compare_with_test_object(actual_nodes1, expected_1)
def test_i_can_parse_when_multiple_sya_concepts(self):
sheerka, context, parser = self.init_parser()
@@ -474,12 +474,12 @@ class TestUnrecognizedNodeParser(TestUsingMemoryBasedSheerka):
actual_nodes0 = res[0].body.body
expected_0 = compute_expected_array(concepts_map, expression, [
CNC("greetings_a", source="greetings two", start=0, end=2, a="two")], exclude_body=True)
assert actual_nodes0 == expected_0
compare_with_test_object(actual_nodes0, expected_0)
actual_nodes1 = res[1].body.body
expected_1 = compute_expected_array(concepts_map, expression, [
CNC("greetings_b", source="greetings two", start=0, end=2, b="two")], exclude_body=True)
assert actual_nodes1 == expected_1
compare_with_test_object(actual_nodes1, expected_1)
def test_i_cannot_parse_when_some_unrecognized_remain(self):
sheerka, context, parser = self.init_parser()
tests/parsers/test_parsers_utils.py
+120 -15
View File
@@ -1,10 +1,13 @@
from core.concept import Concept, ConceptParts, CC
from core.concept import Concept, ConceptParts
from core.global_symbols import NotInit
from core.rule import Rule, ACTION_TYPE_EXEC
from core.sheerka.services.SheerkaExecute import ParserInput
from parsers.BaseNodeParser import CNC
from parsers.BaseNodeParser import RuleNode
from parsers.BnfNodeParser import BnfNodeParser
from parsers.FunctionParser import FunctionParser
from parsers.SyaNodeParser import SyaNodeParser
from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
from tests.parsers.parsers_utils import get_test_obj
from tests.parsers.parsers_utils import get_test_obj, CNC, CC, CN, SCN, SCWC, UTN, RN, CB
class TestParsersUtils(TestUsingMemoryBasedSheerka):
@@ -20,12 +23,12 @@ class TestParsersUtils(TestUsingMemoryBasedSheerka):
cnode = parser.parse(context, ParserInput("one plus two")).body.body[0]
# compare all attributes
cnc_res = get_test_obj(CNC(concept_key="key", start=0, end=1, source="", exclude_body=False), cnode)
cnc_res = get_test_obj(cnode, CNC(concept_key="key", start=0, end=1, source="", exclude_body=False))
assert isinstance(cnc_res, CNC)
assert cnc_res == CNC("__var__0 plus __var__1", 0, 4, "one plus two", False, **cnode.concept.get_compiled())
assert cnc_res == CNC("__var__0 plus __var__1", "one plus two", 0, 4, False, **cnode.concept.get_compiled())
# I can discard start, end and source
cnc_res = get_test_obj(CNC(concept_key="key"), cnode)
cnc_res = get_test_obj(cnode, CNC(concept_key="key"))
assert isinstance(cnc_res, CNC)
assert cnc_res == CNC("__var__0 plus __var__1", None, None, None, False, **cnode.concept.get_compiled())
@@ -40,12 +43,12 @@ class TestParsersUtils(TestUsingMemoryBasedSheerka):
cnode = parser.parse(context, ParserInput("twenty one")).body.body[0]
# compare all attributes
cnc_res = get_test_obj(CNC(concept_key="key", start=0, end=1, source="", exclude_body=False), cnode)
cnc_res = get_test_obj(cnode, CNC(concept_key="key", start=0, end=1, source="", exclude_body=False))
assert isinstance(cnc_res, CNC)
assert cnc_res == CNC("twenties", 0, 2, "twenty one", False, **cnode.concept.get_compiled())
assert cnc_res == CNC("twenties", "twenty one", 0, 2, False, **cnode.concept.get_compiled())
# I can exclude body
cnc_res = get_test_obj(CNC(concept_key="key", exclude_body=True), cnode)
cnc_res = get_test_obj(cnode, CNC(concept_key="key", exclude_body=True))
expected_compiled = {k: v for k, v in cnode.concept.get_compiled().items()}
del expected_compiled[ConceptParts.BODY]
assert isinstance(cnc_res, CNC)
@@ -61,13 +64,13 @@ class TestParsersUtils(TestUsingMemoryBasedSheerka):
parser = SyaNodeParser().init_from_concepts(context, [one, two, plus])
cnode = parser.parse(context, ParserInput("one plus two")).body.body[0]
res = get_test_obj([CNC("key1"), CNC("key", 0, 1, "")], [cnode, cnode])
res = get_test_obj([cnode, cnode], [CNC("key1"), CNC("key", 0, 1, "")])
assert len(res) == 2
assert isinstance(res[0], CNC)
assert res[0] == CNC("__var__0 plus __var__1", None, None, None, False, **cnode.concept.get_compiled())
assert isinstance(res[1], CNC)
assert res[1] == CNC("__var__0 plus __var__1", 0, 4, "one plus two", False, **cnode.concept.get_compiled())
assert res[1] == CNC("__var__0 plus __var__1", "one plus two", 0, 4, False, **cnode.concept.get_compiled())
def test_i_can_get_test_obj_when_dict(self):
sheerka, context, one, two, plus = self.init_concepts(
@@ -79,12 +82,12 @@ class TestParsersUtils(TestUsingMemoryBasedSheerka):
parser = SyaNodeParser().init_from_concepts(context, [one, two, plus])
cnode = parser.parse(context, ParserInput("one plus two")).body.body[0]
res = get_test_obj({"key1": CNC("key1"), "key2": CNC("key", 0, 1, "")}, {"key1": cnode, "key2": cnode})
res = get_test_obj({"key1": cnode, "key2": cnode}, {"key1": CNC("key1"), "key2": CNC("key", 0, 1, "")})
assert len(res) == 2
assert isinstance(res["key1"], CNC)
assert res["key1"] == CNC("__var__0 plus __var__1", None, None, None, False, **cnode.concept.get_compiled())
assert isinstance(res["key2"], CNC)
assert res["key2"] == CNC("__var__0 plus __var__1", 0, 4, "one plus two", False, **cnode.concept.get_compiled())
assert res["key2"] == CNC("__var__0 plus __var__1", "one plus two", 0, 4, False, **cnode.concept.get_compiled())
def test_i_can_get_test_obj_when_CC(self):
sheerka, context, one, two, plus = self.init_concepts(
@@ -97,13 +100,115 @@ class TestParsersUtils(TestUsingMemoryBasedSheerka):
cc = parser.parse(context, ParserInput("twenty one")).body.body[0].concept
# compare all attributes
cc_res = get_test_obj(CC(concept="key", source="", exclude_body=False), cc)
cc_res = get_test_obj(cc, CC(concept="key", source="", exclude_body=False))
assert isinstance(cc_res, CC)
assert cc_res == CC("twenties", "twenty one", False, **cc.get_compiled())
# I can exclude body
cnc_res = get_test_obj(CC(concept="key", exclude_body=True), cc)
cnc_res = get_test_obj(cc, CC(concept="key", exclude_body=True))
expected_compiled = {k: v for k, v in cc.get_compiled().items()}
del expected_compiled[ConceptParts.BODY]
assert isinstance(cnc_res, CC)
assert cnc_res == CC("twenties", "twenty one", True, **expected_compiled)
def test_i_can_get_test_obj_when_CN(self):
sheerka, context, one, two, plus = self.init_concepts(
"one",
"two",
Concept("a plus b").def_var("a").def_var("b")
)
parser = SyaNodeParser().init_from_concepts(context, [one, two, plus])
cnode = parser.parse(context, ParserInput("one plus two")).body.body[0]
cn_res = get_test_obj(cnode, CN(concept="key", start=0, end=1, source=""))
assert isinstance(cn_res, CN)
assert cn_res == CN(plus, "one plus two", 0, 4)
# I can discard start, end and source
cnc_res = get_test_obj(cnode, CN(concept="key"))
assert isinstance(cnc_res, CN)
assert cnc_res == CN(plus, None, None, None)
def test_i_can_get_test_obj_when_SCN(self):
sheerka, context = self.init_test().unpack()
parser = FunctionParser()
scn = parser.parse(context, ParserInput("test()")).body.body
scn_res = get_test_obj(scn, SCN("", start=0, end=1))
assert isinstance(scn_res, SCN)
assert scn_res == SCN("test()", 0, 2)
# I can discard start and end
scn_res = get_test_obj(scn, SCN(""))
assert isinstance(scn_res, SCN)
assert scn_res == SCN("test()", None, None)
def test_i_can_get_test_obj_when_SCWC(self):
sheerka, context = self.init_test().unpack()
parser = FunctionParser()
scwc = parser.parse(context, ParserInput("test(param1, test2())")).body.body
scwc_res = get_test_obj(scwc, SCWC(UTN(""), UTN(""), UTN(""), UTN(""), SCN("", 0, 0)))
assert isinstance(scwc_res, SCWC)
expected = SCWC(UTN("test(", 0, 1),
UTN(")", 8, 8),
UTN("param1", 2, 2),
UTN(", ", 3, 4),
SCN("test2()", 5, 7))
expected.start = 0
expected.end = 8
assert scwc_res == expected
assert isinstance(scwc_res.first, UTN)
assert isinstance(scwc_res.last, UTN)
assert isinstance(scwc_res.content[0], UTN)
assert isinstance(scwc_res.content[1], UTN)
assert isinstance(scwc_res.content[2], SCN)
def test_i_can_get_test_obj_when_RN(self):
rule = Rule(ACTION_TYPE_EXEC, "test_rule", "True", "True")
rn = RuleNode(rule, 1, 1, source="r:|xxx:")
rn_res = get_test_obj(rn, RN("", "", 0, 1))
assert isinstance(rn_res, RN)
assert rn_res == RN(rule, rn.source, rn.start, rn.end)
# I can discard start and end
rn_res = get_test_obj(rn, RN("", None, None, None))
assert isinstance(rn_res, RN)
assert rn_res == RN(rule, None, None, None)
def test_i_can_get_test_obj_when_CB(self):
a = Concept("a", key="a", body="10").auto_init()
b = Concept("b", key="b", body=a).auto_init()
c = Concept("c", key="c")
other_c = Concept("c", key="c", body="i don't care")
# i can test when no body
res = get_test_obj(c, CB("", ""))
assert isinstance(res, CB)
assert res == CB("c", NotInit)
# i can test with a concept instead of a key
res = get_test_obj(c, CB(Concept(), ""))
assert isinstance(res, CB)
assert res == CB(other_c, NotInit) # don't care if it's not the real 'c', only test key and body
# # i can test when the body is a concept
res = get_test_obj(b, CB("", ""))
assert isinstance(res, CB)
assert res == CB("b", a)
# i can go into recursion when body is a concept
res = get_test_obj(b, CB("", CB("", "")))
assert isinstance(res, CB)
assert res == CB("b", CB("a", "10"))
# I can try to go into recursion when there nothing to found
res = get_test_obj(c, CB("", CB("", "")))
assert isinstance(res, CB)
assert res == CB("c", NotInit)