kodjo/Sheerka-Old
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fixed #109 : Mix python and concept. List comprehension

Browse Source 
Fixed #110 : SheerkaDebugManager: add list_debug_settings
Fixed #111 : SheerkaDebugManager: Implement ListDebugLogger
Fixed #112 : SyaNodeParser: rewrite this parser
Fixed #113 : Sheerka.: Add enable_parser_caching to disable parsers caching
Fixed #114 : SyaNodeParser : Implement fast cache to resolve unrecognized tokens requests
Fixed #115 : BnfNodeParser : Implement fast cache to resolve unrecognized tokens requests
Fixed #116 : SequenceNodeParser : Implement fast cache to resolve unrecognized tokens requests
Fixed #117 : ResolveMultiplePluralAmbiguityEvaluator: Resolve Multiple plural ambiguity
This commit is contained in:
Kodjo Sossouvi
2021-09-06 11:51:50 +02:00
parent 71d1b1d1ca
commit 54e5681c5a
57 changed files with 5179 additions and 3125 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/BaseTest.py
+10 -2
View File
@@ -1,12 +1,13 @@
import ast
from dataclasses import dataclass, field
from core.builtin_concepts import ReturnValueConcept, ParserResultConcept, BuiltinConcepts
from core.builtin_concepts import BuiltinConcepts, ParserResultConcept, ReturnValueConcept
from core.concept import Concept, DEFINITION_TYPE_BNF, DEFINITION_TYPE_DEF, freeze_concept_attrs
from core.rule import Rule, ACTION_TYPE_PRINT, ACTION_TYPE_EXEC
from core.rule import ACTION_TYPE_EXEC, ACTION_TYPE_PRINT, Rule
from core.sheerka.ExecutionContext import ExecutionContext
from core.sheerka.Sheerka import Sheerka
from core.sheerka.services.SheerkaConceptManager import SheerkaConceptManager
from core.sheerka.services.SheerkaDebugManager import ListDebugLogger
from core.sheerka.services.SheerkaRuleManager import SheerkaRuleManager
from parsers.BnfDefinitionParser import BnfDefinitionParser
from parsers.BnfNodeParser import StrMatch
@@ -290,3 +291,10 @@ class BaseTest:
@staticmethod
def successful_return_values(return_values):
return [ret_val for ret_val in return_values if ret_val.status]
@staticmethod
def activate_debug(context, pattern="Sya.*.*"):
sheerka = context.sheerka
sheerka.set_debug(context, True)
sheerka.set_debug_var(context, pattern)
sheerka.set_debug_logger_definition(ListDebugLogger)
tests/core/test_SheerkaDebugManager.py
+16 -12
View File
@@ -73,6 +73,7 @@ class TestSheerkaDebugManager(TestUsingMemoryBasedSheerka):
item_container = f"debug_{item_type}_settings"
assert getattr(service, item_container) == [DebugItem(
item_type,
item,
service_name,
method_name,
@@ -89,7 +90,7 @@ class TestSheerkaDebugManager(TestUsingMemoryBasedSheerka):
service.add_or_update_debug_item(context, "vars", item="item")
assert service.debug_vars_settings == [
DebugItem("item", None, None, None, False, None, False, True)
DebugItem("vars", "item", None, None, None, False, None, False, True)
]
def test_i_can_update_debug_item(self):
@@ -101,8 +102,8 @@ class TestSheerkaDebugManager(TestUsingMemoryBasedSheerka):
service.add_or_update_debug_item(context, "vars", "item", "service_name", "method_name", enabled=False)
assert service.debug_vars_settings == [
DebugItem("item", "service_name", "method_name", None, False, None, False, False),
DebugItem("item2", "service_name", "method_name", None, False, None, False, True),
DebugItem("vars", "item", "service_name", "method_name", None, False, None, False, False),
DebugItem("vars", "item2", "service_name", "method_name", None, False, None, False, True),
]
@pytest.mark.parametrize("settings, expected", [
@@ -407,7 +408,7 @@ class TestSheerkaDebugManager(TestUsingMemoryBasedSheerka):
sheerka.set_debug_var(context, "s.m.v", "1+", 10, variable="my_var")
assert service.debug_vars_settings == [
DebugItem("my_var", "s", "m", 1, True, 10, False, True)
DebugItem("vars", "my_var", "s", "m", 1, True, 10, False, True)
]
assert service.debug_concepts_settings == []
assert service.debug_rules_settings == []
@@ -418,7 +419,7 @@ class TestSheerkaDebugManager(TestUsingMemoryBasedSheerka):
sheerka.set_debug_rule(context, "s.m.v", "1+", 10, rule="my_rule")
assert service.debug_rules_settings == [
DebugItem("my_rule", "s", "m", 1, True, 10, False, True)
DebugItem("rules", "my_rule", "s", "m", 1, True, 10, False, True)
]
assert service.debug_concepts_settings == []
assert service.debug_vars_settings == []
@@ -429,7 +430,7 @@ class TestSheerkaDebugManager(TestUsingMemoryBasedSheerka):
sheerka.set_debug_concept(context, "s.m.v", "1+", 10, concept="my_concept")
assert service.debug_concepts_settings == [
DebugItem("my_concept", "s", "m", 1, True, 10, False, True)
DebugItem("concepts", "my_concept", "s", "m", 1, True, 10, False, True)
]
assert service.debug_rules_settings == []
assert service.debug_vars_settings == []
@@ -449,11 +450,11 @@ class TestSheerkaDebugManager(TestUsingMemoryBasedSheerka):
assert another_service.activated
assert another_service.debug_vars_settings == [
DebugItem('var', 'service_name', None, None, False, None, False, True)]
DebugItem("vars", 'var', 'service_name', None, None, False, None, False, True)]
assert another_service.debug_rules_settings == [
DebugItem('1', None, None, None, False, None, False, True)]
DebugItem("rules", '1', None, None, None, False, None, False, True)]
assert another_service.debug_concepts_settings == [
DebugItem('1001', None, None, None, False, None, False, True)]
DebugItem("concepts", '1001', None, None, None, False, None, False, True)]
def test_i_can_inspect_concept_all_attributes(self):
sheerka, context, foo = self.init_concepts("foo")
@@ -766,9 +767,12 @@ class TestSheerkaDebugManager(TestUsingMemoryBasedSheerka):
sheerka.pop_ontology(context)
assert service.activated
assert service.debug_vars_settings == [DebugItem("v_name", "v_service", "v_method", 1, True, 1, False, True)]
assert service.debug_rules_settings == [DebugItem("r_name", "r_service", "r_method", 2, True, 2, False, True)]
assert service.debug_concepts_settings == [DebugItem("c_name", "c_serv", "c_method", 3, True, 3, False, True)]
assert service.debug_vars_settings == [
DebugItem("vars", "v_name", "v_service", "v_method", 1, True, 1, False, True)]
assert service.debug_rules_settings == [
DebugItem("rules", "r_name", "r_service", "r_method", 2, True, 2, False, True)]
assert service.debug_concepts_settings == [
DebugItem("concepts", "c_name", "c_serv", "c_method", 3, True, 3, False, True)]
def test_i_can_register_debug_item(self):
sheerka, context = self.init_concepts()
tests/core/test_SheerkaEvaluateConcept.py
+29 -23
View File
@@ -4,7 +4,7 @@ from core.builtin_concepts import BuiltinConcepts, ReturnValueConcept, ParserRes
from core.concept import Concept, DoNotResolve, ConceptParts, InfiniteRecursionResolved, \
DEFINITION_TYPE_DEF
from core.global_symbols import NotInit, NotFound
from core.sheerka.services.SheerkaEvaluateConcept import SheerkaEvaluateConcept
from core.sheerka.services.SheerkaEvaluateConcept import EvaluationHints, SheerkaEvaluateConcept
from core.sheerka.services.SheerkaExecute import ParserInput
from core.sheerka.services.SheerkaMemory import SheerkaMemory
from parsers.BaseParser import BaseParser
@@ -13,7 +13,7 @@ from parsers.ExpressionParser import ExpressionParser
from parsers.PythonParser import PythonNode, PythonParser
from parsers.SyaNodeParser import SyaNodeParser
from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
from tests.evaluators.EvaluatorTestsUtils import pr_ret_val, python_ret_val
from tests.evaluators.EvaluatorTestsUtils import exact, pr_ret_val, python_ret_val
from tests.parsers.parsers_utils import CB, compare_with_test_object
@@ -487,7 +487,9 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
eval_where=True,
)
evaluated = sheerka.evaluate_concept(self.get_context(sheerka, False, True), concept, eval_body=False)
evaluated = sheerka.evaluate_concept(self.get_context(sheerka, False, True),
concept,
hints=EvaluationHints(eval_body=False))
if expected:
assert evaluated.key == concept.key
@@ -532,13 +534,13 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
)
foo_instance = sheerka.new("foo")
evaluated = sheerka.evaluate_concept(context, foo_instance, eval_body=False)
evaluated = sheerka.evaluate_concept(context, foo_instance, hints=EvaluationHints(eval_body=False))
assert ConceptParts.BODY in evaluated.get_compiled()
assert evaluated.body == NotInit
assert not evaluated.get_hints().is_evaluated
evaluated = sheerka.evaluate_concept(context, foo_instance, eval_body=True) # evaluate the body this time
evaluated = sheerka.evaluate_concept(context, foo_instance, hints=EvaluationHints(eval_body=True)) # evaluate the body this time
assert isinstance(evaluated.body, bool) and evaluated.body
def test_i_can_apply_intermediate_where_condition_using_python(self):
@@ -811,7 +813,7 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
Concept("bar", pre="print('10')"), # print won't be executed
)
evaluated = sheerka.evaluate_concept(context, foo, eval_body=True)
evaluated = sheerka.evaluate_concept(context, foo, hints=EvaluationHints(eval_body=True))
captured = capsys.readouterr()
assert evaluated.key == foo.key
assert captured.out == "10\n"
@@ -828,7 +830,7 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
Concept("foo", pre="in_context('foo')", body="print('10')"),
)
evaluated = sheerka.evaluate_concept(context, concept, eval_body=True)
evaluated = sheerka.evaluate_concept(context, concept, hints=EvaluationHints(eval_body=True))
assert sheerka.isinstance(evaluated, BuiltinConcepts.CONDITION_FAILED)
assert evaluated.body == "in_context('foo')"
assert evaluated.concept == concept
@@ -898,7 +900,7 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
def test_is_evaluated_is_correctly_set(self, concept, expected):
sheerka, context, concept = self.init_concepts(concept)
evaluated = sheerka.evaluate_concept(context, concept, eval_body=True)
evaluated = sheerka.evaluate_concept(context, concept, hints=EvaluationHints(eval_body=True))
assert evaluated.key == concept.key
assert concept.get_hints().is_evaluated == expected
@@ -919,24 +921,24 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
res = sheerka.evaluate_concept(context, bar)
assert sheerka.isinstance(res, "bar")
res = sheerka.evaluate_concept(context, bar, eval_body=True)
res = sheerka.evaluate_concept(context, bar, hints=EvaluationHints(eval_body=True))
assert sheerka.isinstance(res, "foo")
# And the result is still the same after a second call
assert bar.get_hints().is_evaluated
res = sheerka.evaluate_concept(context, bar, eval_body=True)
res = sheerka.evaluate_concept(context, bar, hints=EvaluationHints(eval_body=True))
assert sheerka.isinstance(res, "foo")
def test_ret_is_evaluated_only_is_body_is_requested(self):
sheerka, context, foo, bar = self.init_concepts("foo", Concept("bar", ret="__NOT_FOUND"))
res = sheerka.evaluate_concept(context, bar, eval_body=False)
res = sheerka.evaluate_concept(context, bar, hints=EvaluationHints(eval_body=False))
assert res.id == bar.id
def test_i_can_eval_concept_with_rules(self):
sheerka, context, foo = self.init_concepts(Concept("foo a", body="a.name").def_var("a", "r:|1:"))
res = sheerka.evaluate_concept(context, foo, eval_body=True)
res = sheerka.evaluate_concept(context, foo, hints=EvaluationHints(eval_body=True))
assert res.body == "Print return values"
def test_i_can_manage_python_concept_infinite_recursion_when_initializing_ast(self):
@@ -957,18 +959,18 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
evaluator = SheerkaEvaluateConcept(sheerka)
# 'def concept foo as foo'
return_values = [pr_ret_val(foo, parser="ExactConcept"), python_ret_val("foo")]
return_values = [pr_ret_val(foo, parser=exact), python_ret_val("foo")]
res = evaluator.get_recursive_definitions(context, foo, return_values)
assert list(res) == [BaseParser.get_name("ExactConcept")]
assert list(r.name for r in res) == [BaseParser.get_name("ExactConcept")]
def test_i_can_detect_when_no_recursive_definition(self):
sheerka, context, foo, bar = self.init_concepts("foo", "bar")
evaluator = SheerkaEvaluateConcept(sheerka)
# 'def concept foo as bar'
return_values = [pr_ret_val(bar, parser="ExactConcept"), python_ret_val("foo")]
return_values = [pr_ret_val(bar, parser=exact), python_ret_val("foo")]
res = evaluator.get_recursive_definitions(context, foo, return_values)
@@ -980,7 +982,7 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
evaluator = SheerkaEvaluateConcept(sheerka)
# i dunno how to construct the return value
return_values = [pr_ret_val(q, parser="ExactConcept")]
return_values = [pr_ret_val(q, parser=exact)]
res = evaluator.get_recursive_definitions(context, q, return_values)
@@ -1004,7 +1006,7 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
assert evaluated.get_compiled()["y"][0].body.body.get_hints().use_copy
# get the body
evaluated = evaluator.evaluate_concept(context, concept, eval_body=True)
evaluated = evaluator.evaluate_concept(context, concept, hints=EvaluationHints(eval_body=True))
assert evaluated.get_compiled()["x"][0].body.body.get_hints().use_copy
assert evaluated.get_compiled()["y"][0].body.body.get_hints().use_copy
assert not evaluated.get_value("x").get_hints().use_copy
@@ -1025,7 +1027,7 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
# get the body
context.add_to_protected_hints(BuiltinConcepts.EVAL_QUESTION_REQUESTED)
evaluated = evaluator.evaluate_concept(context, concept, eval_body=True)
evaluated = evaluator.evaluate_concept(context, concept, hints=EvaluationHints(eval_body=True))
assert evaluated.get_compiled()["x"][0].body.body.get_hints().use_copy
assert evaluated.get_compiled()["y"][0].body.body.get_hints().use_copy
assert not evaluated.get_value("x").get_hints().use_copy
@@ -1043,7 +1045,9 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
# Sanity check for normal behaviour
to_evaluate1 = parsed_ret_val.body.body[0].concept.copy()
evaluated1 = sheerka.evaluate_concept(context, to_evaluate1, eval_body=True, validation_only=False)
evaluated1 = sheerka.evaluate_concept(context,
to_evaluate1,
hints=EvaluationHints(eval_body=True, expression_only=False))
assert sheerka.isinstance(evaluated1, shirt)
assert evaluated1.get_value("body_ax_is_evaluated") == True
@@ -1053,7 +1057,9 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
# check validation_only behaviour
to_evaluate2 = parsed_ret_val.body.body[0].concept.copy()
evaluated2 = sheerka.evaluate_concept(context, to_evaluate2, eval_body=True, validation_only=True)
evaluated2 = sheerka.evaluate_concept(context,
to_evaluate2,
hints=EvaluationHints(eval_body=True, expression_only=True))
assert sheerka.isinstance(evaluated2, shirt)
assert evaluated2.get_value("body_ax_is_evaluated") == NotInit
@@ -1073,7 +1079,7 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
parsed_ret_val = SyaNodeParser().parse(context, ParserInput("a red shirt"))
to_evaluate = parsed_ret_val.body.body[0].concept
evaluated = sheerka.evaluate_concept(context, to_evaluate, eval_body=False)
evaluated = sheerka.evaluate_concept(context, to_evaluate, hints=EvaluationHints(eval_body=False))
assert sheerka.isinstance(evaluated, a_x)
assert "x" in evaluated.get_compiled()
@@ -1085,7 +1091,7 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
# sanity check
parsed_ret_val = SyaNodeParser().parse(context, ParserInput("a red shirt"))
to_evaluate = parsed_ret_val.body.body[0].concept
evaluated = sheerka.evaluate_concept(context, to_evaluate, eval_body=True)
evaluated = sheerka.evaluate_concept(context, to_evaluate, hints=EvaluationHints(eval_body=True))
assert sheerka.isinstance(evaluated, shirt)
assert evaluated.get_value("body_ax_is_evaluated") == True
@@ -1095,7 +1101,7 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
def test_concept_is_not_evaluated_when_method_access_error(self):
sheerka, context, foo = self.init_concepts(Concept("foo", body="set_attr(self, 'prop_name', 'prop_value')"))
evaluated = sheerka.evaluate_concept(context, foo, eval_body=True, validation_only=True)
evaluated = sheerka.evaluate_concept(context, foo, hints=EvaluationHints(eval_body=True, expression_only=True))
assert sheerka.isinstance(evaluated, foo)
assert not foo.get_hints().is_evaluated
tests/evaluators/EvaluatorTestsUtils.py
+39 -17
View File
@@ -1,14 +1,22 @@
import ast
from core.builtin_concepts import ReturnValueConcept, BuiltinConcepts, ParserResultConcept
from core.builtin_concepts import BuiltinConcepts, ParserResultConcept, ReturnValueConcept
from core.concept import Concept
from evaluators.BaseEvaluator import BaseEvaluator
from parsers.BaseParser import BaseParser
from parsers.PythonParser import PythonNode
from parsers.BaseNodeParser import ConceptNode
from parsers.ExactConceptParser import ExactConceptParser
from parsers.PythonParser import PythonNode, PythonParser
from parsers.SequenceNodeParser import SequenceNodeParser
from parsers.SyaNodeParser import SyaNodeParser
reduced_requested = ReturnValueConcept("Sheerka", True, Concept(name=BuiltinConcepts.REDUCE_REQUESTED,
key=BuiltinConcepts.REDUCE_REQUESTED))
sequence = SequenceNodeParser()
sya = SyaNodeParser()
exact = ExactConceptParser()
python = PythonParser()
def ret_val(value="value", who="who", status=True):
"""
@@ -21,7 +29,7 @@ def ret_val(value="value", who="who", status=True):
return ReturnValueConcept(who, status, value)
def p_ret_val(value="value", parser="parser", status=True):
def p_ret_val(value="value", parser=exact, status=True):
"""
ReturnValueConcept from parser
:param value:
@@ -29,7 +37,7 @@ def p_ret_val(value="value", parser="parser", status=True):
:param status:
:return:
"""
return ReturnValueConcept(BaseParser.get_name(parser), status, value)
return ReturnValueConcept(parser.name, status, value)
def e_ret_val(value="value", evaluator="evaluator", status=True):
@@ -43,7 +51,7 @@ def e_ret_val(value="value", evaluator="evaluator", status=True):
return ReturnValueConcept(BaseEvaluator.PREFIX + evaluator, status, value)
def p_ret_val_false(value="value", parser="parser"):
def p_ret_val_false(value="value", parser=exact):
"""
Failed ReturnValueConcept from parser
:param value:
@@ -53,7 +61,7 @@ def p_ret_val_false(value="value", parser="parser"):
return p_ret_val(value, parser, status=False)
def p_ret_val_true(value="value", parser="parser"):
def p_ret_val_true(value="value", parser=exact):
"""
Successful ReturnValueConcept from parser
:param value:
@@ -63,24 +71,24 @@ def p_ret_val_true(value="value", parser="parser"):
return p_ret_val(value, parser, status=True)
def e_ret_val_false(value="value", parser="parser"):
def e_ret_val_false(value="value", evaluator="evaluator"):
"""
Failed ReturnValueConcept from evaluator
:param value:
:param parser:
:param evaluator:
:return:
"""
return e_ret_val(value, parser, status=False)
return e_ret_val(value, evaluator, status=False)
def e_ret_val_true(value="value", parser="parser"):
def e_ret_val_true(value="value", evaluator="evaluator"):
"""
Successful ReturnValueConcept from evaluator
:param value:
:param parser:
:param evaluator:
:return:
"""
return e_ret_val(value, parser, status=True)
return e_ret_val(value, evaluator, status=True)
def e_ret_val_new(key, evaluator="evaluator", status=True, **kwargs):
@@ -96,7 +104,7 @@ def e_ret_val_new(key, evaluator="evaluator", status=True, **kwargs):
return e_ret_val(body, evaluator, status)
def pr_ret_val(value, parser="parser", source=None, status=True):
def pr_ret_val(value, parser=exact, source=None, status=True):
"""
ParserResult ReturnValue
eg: ReturnValue with a ParserResult
@@ -107,7 +115,7 @@ def pr_ret_val(value, parser="parser", source=None, status=True):
:return:
"""
source = source or (value.name if isinstance(value, Concept) else "source")
parser_result = ParserResultConcept(BaseParser.get_name(parser), source=source, value=value)
parser_result = ParserResultConcept(parser, source=source, value=value)
return p_ret_val(value=parser_result, parser=parser, status=status)
@@ -117,8 +125,14 @@ def python_ret_val(source):
:param source:
:return:
"""
python_node = PythonNode(source.strip(), ast.parse(source.strip(), f"<source>", 'eval'))
return pr_ret_val(python_node, parser="Python", source=source)
python_node = PythonNode(source.lstrip(), ast.parse(source.strip(), f"<source>", 'eval'))
return pr_ret_val(python_node, parser=python, source=source)
def cnode_ret_val(concept, source=None, parser=sya):
source = source or concept.name
cnode = ConceptNode(concept, 0, 0, source=source)
return pr_ret_val([cnode], parser=parser, source=source)
def new_concept(key, **kwargs):
@@ -129,3 +143,11 @@ def new_concept(key, **kwargs):
res.get_hints().is_evaluated = True
return res
def new_plural(name):
name_stripped_s = name.lstrip("s")
single = Concept(name_stripped_s)
concept = Concept(key=name, name=name, id=f"{name}:{BuiltinConcepts.PLURAL}", is_builtin=False, is_unique=False)
concept.set_prop(BuiltinConcepts.PLURAL, single)
return concept
tests/evaluators/test_DefConceptEvaluator.py
+12
View File
@@ -449,6 +449,18 @@ class TestDefConceptEvaluator(TestUsingMemoryBasedSheerka):
assert sheerka.get_property(created_concept, BuiltinConcepts.ISA) == {sheerka.new(BuiltinConcepts.AUTO_EVAL)}
def test_i_can_eval_when_variable_are_forced(self):
sheerka, context = self.init_test().unpack()
definition = "def concept foo from [z for x in y] def_var x def_var y def_var z"
def_ret_val = DefConceptParser().parse(context, ParserInput(definition))
evaluated = DefConceptEvaluator().eval(context, def_ret_val)
assert evaluated.status
assert context.sheerka.isinstance(evaluated.body, BuiltinConcepts.NEW_CONCEPT)
created_concept = evaluated.body.body
assert created_concept.get_metadata().parameters == ["z", "x", "y"]
def test_i_cannot_eval_bnf_concept_with_unknown_variable(self):
# testing MandatoryVariable
context = self.get_context()
tests/evaluators/test_ResolveMultiplePluralAmbiguityEvaluator.py
+60
View File
@@ -0,0 +1,60 @@
import pytest
from core.concept import Concept
from evaluators.ResolveMultiplePluralAmbiguityEvaluator import ResolveMultiplePluralAmbiguityEvaluator
from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
from tests.evaluators.EvaluatorTestsUtils import cnode_ret_val, exact, new_plural, pr_ret_val, python_ret_val, sequence, \
sya
class TestResolveMultiplePluralAmbiguityEvaluator(TestUsingMemoryBasedSheerka):
@pytest.mark.parametrize("return_values, expected", [
([python_ret_val("numbers"), cnode_ret_val(new_plural("numbers"), parser=sequence)], True),
([python_ret_val("numbers"), cnode_ret_val(new_plural("numbers"), parser=sequence), pr_ret_val("other")], True),
([python_ret_val("numbers"), cnode_ret_val(new_plural("numbers"), parser=sya)], False),
([python_ret_val("numbers"), cnode_ret_val(Concept("numbers"), parser=sequence)], False),
([python_ret_val("numbers"), pr_ret_val(new_plural("numbers"), parser=exact)], False),
])
def test_i_can_match(self, return_values, expected):
context = self.get_context()
assert ResolveMultiplePluralAmbiguityEvaluator().matches(context, return_values) == expected
def test_i_can_eval_when_nothing_in_memory(self):
sheerka, context = self.init_test().unpack()
return_values = [
python_ret_val("numbers"),
cnode_ret_val(new_plural("numbers"), source="source", parser=sequence)
]
evaluator = ResolveMultiplePluralAmbiguityEvaluator()
assert evaluator.matches(context, return_values)
rets = evaluator.eval(context, return_values)
assert len(rets) == 1
ret = rets[0]
assert ret.who == evaluator.name
assert ret.status == return_values[1].status
assert ret.value == return_values[1].value
assert ret.parents == return_values
def test_i_can_eval_when_plural_in_memory(self):
sheerka, context = self.init_test().unpack()
sheerka.add_to_memory(context, "numbers", "something")
return_values = [
python_ret_val("numbers"),
cnode_ret_val(new_plural("numbers"), source="source", parser=sequence)
]
evaluator = ResolveMultiplePluralAmbiguityEvaluator()
assert evaluator.matches(context, return_values)
rets = evaluator.eval(context, return_values)
assert len(rets) == 1
ret = rets[0]
assert ret.who == evaluator.name
assert ret.status == return_values[0].status
assert ret.value == return_values[0].value
assert ret.parents == return_values
tests/non_reg/test_sheerka_non_reg2.py
+31 -9
View File
@@ -45,13 +45,35 @@ class TestSheerkaNonRegMemory2(TestUsingMemoryBasedSheerka):
assert res[0].status
assert sheerka.isa(sheerka.new("one"), sheerka.new("number"))
# def test_i_can_define_plural(self):
# init = [
# "def concept man",
# "def concept men as set_plural(man) ret man auto_eval True",
# ]
# sheerka = self.init_scenario(init)
#
# res = sheerka.evaluate_user_input("men")
# assert res[0].status
def test_i_can_get_sequence_when_evaluation_plural(self):
init = [
"def concept one",
"def concept two",
"def concept number",
"global_truth(set_isa(one, number))",
"global_truth(set_isa(two, number))",
]
sheerka = self.init_scenario(init)
res = sheerka.evaluate_user_input("eval numbers")
assert res[0].status
assert set(res[0].body) == {sheerka.new("one"), sheerka.new("two")}
def test_i_can_use_list_comprehension(self):
init = [
"def concept rex",
"def concept rantanplan",
"def concept dog",
"def concept x is a y as set_isa(x, y)",
]
sheerka = self.init_scenario(init)
res = sheerka.evaluate_user_input("global_truth([ x is a dog for x in [rex, rantanplan]])")
assert len(res) == 1
assert res[0].status
rex = sheerka.new("rex")
dog = sheerka.new("dog")
assert sheerka.isa(rex, dog)
tests/non_reg/test_sheerka_non_reg_out.py
+21
View File
@@ -180,3 +180,24 @@ two: (1002)two
captured = capsys.readouterr()
assert " : test()" in captured.out
assert " : history()" in captured.out
def test_i_can_list_debug_settings(self, capsys):
init = [
"set_debug_var('Sya.parsers.*', 45)",
"set_debug_concept('c:|1015', '13+')",
"set_debug_rule('Out')",
]
sheerka = self.init_scenario(init)
capsys.readouterr()
sheerka.enable_process_return_values = True
res = sheerka.evaluate_user_input(f"list_debug_settings()")
assert len(res) == 1
assert res[0].status
captured = capsys.readouterr()
assert captured.out == """DebugItem(type=vars, setting=Sya.parsers.*, context_id=45, debug_id=None, context_children=False, debug_children=False (enabled=True))
DebugItem(type=concepts, setting=c:|1015.*.*, context_id=13, debug_id=None, context_children=True, debug_children=False (enabled=True))
DebugItem(type=rules, setting=Out.*.*, context_id=None, debug_id=None, context_children=False, debug_children=False (enabled=True))
"""
tests/parsers/parsers_utils.py
+292 -172
View File
@@ -1,24 +1,23 @@
import ast
from dataclasses import dataclass
from typing import Union, List
from typing import List, Union
from core.builtin_concepts import ReturnValueConcept
from core.builtin_helpers import CreateObjectIdentifiers
from core.concept import Concept, ConceptParts, DoNotResolve, AllConceptParts
from core.concept import AllConceptParts, Concept, ConceptParts, DoNotResolve
from core.rule import Rule
from core.tokenizer import Tokenizer, TokenKind, Token
from core.utils import get_text_from_tokens, tokens_index, str_concept
from parsers.BaseExpressionParser import NameExprNode, AndNode, OrNode, NotNode, VariableNode, ComparisonNode, \
ComparisonType, \
FunctionParameter
from parsers.BaseNodeParser import UnrecognizedTokensNode, SourceCodeNode, RuleNode, ConceptNode, \
SourceCodeWithConceptNode
from core.tokenizer import Token, TokenKind, Tokenizer
from core.utils import get_text_from_tokens, str_concept, tokens_index
from parsers.BaseExpressionParser import AndNode, ComparisonNode, ComparisonType, Comprehension, FunctionParameter, \
ListComprehensionNode, ListNode, NameExprNode, \
NotNode, OrNode, VariableNode, comma
from parsers.BaseNodeParser import ConceptNode, RuleNode, SourceCodeNode, SourceCodeWithConceptNode, \
UnrecognizedTokensNode
from parsers.FunctionParser import FunctionNode
from parsers.PythonParser import PythonNode
from parsers.SyaNodeParser import SyaConceptParserHelper
from sheerkapython.python_wrapper import sheerka_globals
from sheerkarete.common import V
from sheerkarete.conditions import Condition, AndConditions, NegatedCondition, NegatedConjunctiveConditions
from sheerkarete.conditions import AndConditions, Condition, NegatedCondition, NegatedConjunctiveConditions
@dataclass
@@ -29,104 +28,254 @@ class Obj:
parent: object = None
class AND:
class ExprTestObj:
@staticmethod
def get_pos(nodes):
start, end = None, None
for n in nodes:
if start is None or start > n.start:
start = n.start
if end is None or end < n.end:
end = n.end
return start, end
@staticmethod
def get_pos_from_source(source, full_text_as_tokens):
if isinstance(source, tuple):
source, to_skip = source[0], source[1]
else:
to_skip = 0
source_as_node = list(Tokenizer(source, yield_eof=False))
start = tokens_index(full_text_as_tokens, source_as_node, skip=to_skip)
end = start + len(source_as_node) - 1
return start, end
@staticmethod
def as_tokens(source):
if isinstance(source, tuple):
source, to_skip = source
else:
source, to_skip = source, 0
return list(Tokenizer(source, yield_eof=False)), to_skip
def get_expr_node(self, full_text_as_tokens=None):
raise NotImplementedError
@staticmethod
def safe_get_expr_node(obj, full_text_as_tokens):
if obj is None:
return None
obj = EXPR(obj) if isinstance(obj, (str, tuple)) else obj
return obj.get_expr_node(full_text_as_tokens)
class AND(ExprTestObj):
""" Test class for AndNode"""
def __init__(self, *parts, source=None):
self.parts = parts
self.source = source
def get_expr_node(self, full_text_as_tokens=None):
parts = [part.get_expr_node(full_text_as_tokens) for part in self.parts]
start, end = self.get_pos_from_source(self.source, full_text_as_tokens) if self.source else self.get_pos(parts)
return AndNode(start, end, full_text_as_tokens[start: end + 1], *parts)
class OR:
class OR(ExprTestObj):
""" Test class for OrNode"""
def __init__(self, *parts, source=None):
self.parts = parts
self.source = source
def get_expr_node(self, full_text_as_tokens=None):
parts = [part.get_expr_node(full_text_as_tokens) for part in self.parts]
start, end = self.get_pos_from_source(self.source, full_text_as_tokens) if self.source else self.get_pos(parts)
return OrNode(start, end, full_text_as_tokens[start: end + 1], *parts)
@dataclass
class NOT:
class NOT(ExprTestObj):
""" Test class for NotNode"""
expr: object
expr: ExprTestObj
source: str = None
def get_expr_node(self, full_text_as_tokens=None):
part = self.expr.get_expr_node(full_text_as_tokens)
start, end = self.get_pos_from_source(self.source, full_text_as_tokens) if self.source else (
part.start - 2, part.end)
return NotNode(start, end, full_text_as_tokens[start: end + 1], part)
@dataclass
class EXPR:
"""Test class for NameNode. E stands for Expression"""
class EXPR(ExprTestObj):
"""Test class for NameNode"""
source: str
def get_expr_node(self, full_text_as_tokens=None):
value_as_tokens, to_skip = self.as_tokens(self.source)
start = tokens_index(full_text_as_tokens, value_as_tokens, to_skip)
end = start + len(value_as_tokens) - 1
return NameExprNode(start, end, full_text_as_tokens[start: end + 1])
@dataclass
class VAR:
class VAR(ExprTestObj):
"""Test class for VarNode"""
full_name: str
source: str = None
def get_expr_node(self, full_text_as_tokens=None):
value_as_tokens = list(Tokenizer(self.source or self.full_name, yield_eof=False))
start = tokens_index(full_text_as_tokens, value_as_tokens, 0)
end = start + len(value_as_tokens) - 1
parts = self.full_name.split(".")
if len(parts) == 1:
return VariableNode(start, end, full_text_as_tokens[start: end + 1], parts[0])
else:
return VariableNode(start, end, full_text_as_tokens[start: end + 1], parts[0], *parts[1:])
@dataclass
class EQ:
left: object
right: object
class CompExprTestObj(ExprTestObj):
"""
Test object for comparison ==, <=, ...
"""
left: ExprTestObj
right: ExprTestObj
source: str = None
@dataclass
class NEQ:
left: object
right: object
source: str = None
def get_expr_node(self, full_text_as_tokens=None):
node_type = comparison_type_mapping[type(self).__name__]
left_node = self.left.get_expr_node(full_text_as_tokens)
right_node = self.right.get_expr_node(full_text_as_tokens)
start, end = self.get_pos_from_source(self.source, full_text_as_tokens) if self.source else \
self.get_pos([left_node, right_node])
return ComparisonNode(start, end, full_text_as_tokens[start: end + 1], node_type, left_node, right_node)
@dataclass
class GT:
left: object
right: object
source: str = None
class EQ(CompExprTestObj):
pass
@dataclass
class GTE:
left: object
right: object
source: str = None
class NEQ(CompExprTestObj):
pass
@dataclass
class LT:
left: object
right: object
source: str = None
class GT(CompExprTestObj):
pass
@dataclass
class LTE:
left: object
right: object
source: str = None
class GTE(CompExprTestObj):
pass
@dataclass
class IN:
left: object
right: object
source: str = None
class LT(CompExprTestObj):
pass
@dataclass
class NIN: # for NOT INT
left: object
right: object
source: str = None
class LTE(CompExprTestObj):
pass
@dataclass
class PAREN: # for parenthesis node
class IN(CompExprTestObj):
pass
@dataclass
class NIN(CompExprTestObj): # for NOT INT
pass
@dataclass
class PAREN(ExprTestObj): # for parenthesis node
node: object
source: str = None
class L_EXPR(ExprTestObj):
def __init__(self, first, last, *items, sep=None, source=None):
self.first = first
self.last = last
self.items = items
self.sep = sep or comma
self.source = source
def get_expr_node(self, full_text_as_tokens=None):
first = self.safe_get_expr_node(self.first, full_text_as_tokens)
last = self.safe_get_expr_node(self.last, full_text_as_tokens)
items = [self.safe_get_expr_node(item, full_text_as_tokens) for item in self.items]
if self.source is None:
source = self.first if self.first else ""
source += f"{self.sep.value} ".join(item.get_source() for item in items)
if self.last:
source += self.last
else:
source = self.source
start, end = self.get_pos_from_source(source, full_text_as_tokens)
return ListNode(start, end, full_text_as_tokens[start: end + 1], first, last, items, self.sep)
@dataclass
class LCC:
"""
List comprehension comprehension
"""
target: object
iterable: object
if_expr: object
@dataclass
class LC(ExprTestObj): # for List Comprehension node
element: object
generators: list
source: str = None
def get_expr_node(self, full_text_as_tokens=None):
# first transform str into NameExprTestObj (ie EXPR)
if isinstance(self.element, str):
self.element = EXPR(self.element)
comprehensions = []
nodes = []
for comp in self.generators:
target = EXPR(comp[0]) if isinstance(comp[0], (str, tuple)) else comp[0]
iterable = EXPR(comp[1]) if isinstance(comp[1], (str, tuple)) else comp[1]
if_expr = EXPR(comp[2]) if isinstance(comp[2], (str, tuple)) else comp[2]
comprehensions.append(LCC(target, iterable, if_expr))
self.generators = comprehensions
# then transform into ListComprehensionNode
element = self.element.get_expr_node(full_text_as_tokens)
nodes.append(element)
comprehensions = []
for comp in self.generators:
target = comp.target.get_expr_node(full_text_as_tokens)
iterable = comp.iterable.get_expr_node(full_text_as_tokens)
if_expr = comp.if_expr.get_expr_node(full_text_as_tokens) if comp.if_expr else None
comprehensions.append(Comprehension(target, iterable, if_expr))
nodes.extend([target, iterable, if_expr])
start, end = self.get_pos_from_source(self.source, full_text_as_tokens) if self.source else self.get_pos(nodes)
return ListComprehensionNode(start, end, full_text_as_tokens[start: end + 1], element, comprehensions)
class CC:
"""
Concept class for test purpose
@@ -320,10 +469,13 @@ class CMV:
Test class that only compare the key and the metadata variables
"""
def __init__(self, concept, **kwargs):
def __init__(self, concept, source=None, **kwargs):
self.concept_key = concept.key if isinstance(concept, Concept) else concept
self.concept = concept if isinstance(concept, Concept) else None
self.variables = kwargs
self.source = source # to use when the key is different from the sub str to search when filling start and stop
self.start = None # for debug purpose, indicate where the concept starts
self.end = None # for debug purpose, indicate where the concept ends
def __eq__(self, other):
if id(self) == id(other):
@@ -352,6 +504,21 @@ class CMV:
txt += f", {k}='{v}'"
return txt + ")"
def fix_pos(self, node):
start = node.start if hasattr(node, "start") else \
node[0] if isinstance(node, tuple) else None
end = node.end if hasattr(node, "end") else \
node[1] if isinstance(node, tuple) else None
if start is not None:
if self.start is None or start < self.start:
self.start = start
if end is not None:
if self.end is None or end > self.end:
self.end = end
return self
def transform_real_obj(self, other, get_test_obj_delegate):
if isinstance(other, CMV):
return other
@@ -730,7 +897,7 @@ class CNC(CN):
self_compile_to_use = self.compiled or compiled
compiled = get_test_obj_delegate(self_compile_to_use, compiled, get_test_obj_delegate)
compiled = get_test_obj_delegate(compiled, self_compile_to_use, get_test_obj_delegate)
return CNC(other.concept,
other.source if self.source is not None else None,
other.start if self.start is not None else None,
@@ -865,7 +1032,7 @@ class RN(HelperWithPos):
raise Exception(f"Expecting RuleNode but received {other=}")
class FN:
class FN(ExprTestObj):
"""
Test class only
It matches with FunctionNode but with less constraints
@@ -931,6 +1098,32 @@ class FN:
raise Exception(f"Expecting FunctionNode but received {other=}")
def get_expr_node(self, full_text_as_tokens=None):
start, end = self.get_pos_from_source(self.first, full_text_as_tokens)
first = NameExprNode(start, end, full_text_as_tokens[start: end + 1])
start, end = self.get_pos_from_source(self.last, full_text_as_tokens)
last = NameExprNode(start, end, full_text_as_tokens[start: end + 1])
parameters = []
for param_value, sep in self.parameters:
if isinstance(param_value, str):
start, end = self.get_pos_from_source(param_value, full_text_as_tokens)
param_as_expr_node = NameExprNode(start, end, full_text_as_tokens[start: end + 1])
else:
param_as_expr_node = param_value.get_expr_node(full_text_as_tokens)
if sep:
sep_tokens = Tokenizer(sep, yield_eof=False)
start = param_as_expr_node.end + 1
end = start + len(list(sep_tokens)) - 1
sep_as_expr_node = NameExprNode(start, end, full_text_as_tokens[start: end + 1])
else:
sep_as_expr_node = None
parameters.append(FunctionParameter(param_as_expr_node, sep_as_expr_node))
start, end = first.start, last.end
return FunctionNode(start, end, full_text_as_tokens[start: end + 1], first, last, parameters)
@dataclass()
class NEGCOND:
@@ -966,94 +1159,7 @@ def get_expr_node_from_test_node(full_text, test_node):
Returns EXPR, OR, NOT, AND object to ease the comparison with the real ExprNode
"""
full_text_as_tokens = list(Tokenizer(full_text, yield_eof=False))
def get_pos(nodes):
start, end = None, None
for n in nodes:
if start is None or start > n.start:
start = n.start
if end is None or end < n.end:
end = n.end
return start, end
def get_pos_from_source(source):
if isinstance(source, tuple):
source, to_skip = source[0], source[1]
else:
to_skip = 0
source_as_node = list(Tokenizer(source, yield_eof=False))
start = tokens_index(full_text_as_tokens, source_as_node, skip=to_skip)
end = start + len(source_as_node) - 1
return start, end
def get_expr_node(node):
if isinstance(node, EXPR):
value_as_tokens = list(Tokenizer(node.source, yield_eof=False))
start = tokens_index(full_text_as_tokens, value_as_tokens, 0)
end = start + len(value_as_tokens) - 1
return NameExprNode(start, end, full_text_as_tokens[start: end + 1])
if isinstance(node, AND):
parts = [get_expr_node(part) for part in node.parts]
start, end = get_pos_from_source(node.source) if node.source else get_pos(parts)
return AndNode(start, end, full_text_as_tokens[start: end + 1], *parts)
if isinstance(node, OR):
parts = [get_expr_node(part) for part in node.parts]
start, end = get_pos_from_source(node.source) if node.source else get_pos(parts)
return OrNode(start, end, full_text_as_tokens[start: end + 1], *parts)
if isinstance(node, NOT):
part = get_expr_node(node.expr)
start, end = get_pos_from_source(node.source) if node.source else (part.start - 2, part.end)
return NotNode(start, end, full_text_as_tokens[start: end + 1], part)
if isinstance(node, VAR):
value_as_tokens = list(Tokenizer(node.source or node.full_name, yield_eof=False))
start = tokens_index(full_text_as_tokens, value_as_tokens, 0)
end = start + len(value_as_tokens) - 1
parts = node.full_name.split(".")
if len(parts) == 1:
return VariableNode(start, end, full_text_as_tokens[start: end + 1], parts[0])
else:
return VariableNode(start, end, full_text_as_tokens[start: end + 1], parts[0], *parts[1:])
if isinstance(node, (EQ, NEQ, GT, GTE, LT, LTE, IN, NIN)):
node_type = comparison_type_mapping[type(node).__name__]
left_node, right_node = get_expr_node(node.left), get_expr_node(node.right)
start, end = get_pos_from_source(node.source) if node.source else get_pos([left_node, right_node])
return ComparisonNode(start, end, full_text_as_tokens[start: end + 1],
node_type, left_node, right_node)
if isinstance(node, FN):
start, end = get_pos_from_source(node.first)
first = NameExprNode(start, end, full_text_as_tokens[start: end + 1])
start, end = get_pos_from_source(node.last)
last = NameExprNode(start, end, full_text_as_tokens[start: end + 1])
parameters = []
for param_value, sep in node.parameters:
if isinstance(param_value, str):
start, end = get_pos_from_source(param_value)
param_as_expr_node = NameExprNode(start, end, full_text_as_tokens[start: end + 1])
else:
param_as_expr_node = get_expr_node(param_value)
if sep:
sep_tokens = Tokenizer(sep, yield_eof=False)
start = param_as_expr_node.end + 1
end = start + len(list(sep_tokens)) - 1
sep_as_expr_node = NameExprNode(start, end, full_text_as_tokens[start: end + 1])
else:
sep_as_expr_node = None
parameters.append(FunctionParameter(param_as_expr_node, sep_as_expr_node))
start, end = first.start, last.end
return FunctionNode(start, end, full_text_as_tokens[start: end + 1], first, last, parameters)
return get_expr_node(test_node)
return test_node.get_expr_node(full_text_as_tokens)
def _index(tokens, expr, index):
@@ -1096,15 +1202,15 @@ def compute_debug_array(res):
def get_node(
concepts_map,
expression_as_tokens,
sub_expr,
concept_key=None,
skip=0,
is_bnf=False,
sya=False,
init_empty_body=False,
exclude_body=False):
concepts_map,
expression_as_tokens,
sub_expr,
concept_key=None,
skip=0,
is_bnf=False,
sya=False,
init_empty_body=False,
exclude_body=False):
"""
Tries to find sub in expression
When found, transform it to its correct type
@@ -1157,18 +1263,20 @@ def get_node(
sub_expr.end = start + length - 1
return sub_expr
if isinstance(sub_expr, (CNC, CC, CN)):
concept_node = get_node(
concepts_map,
expression_as_tokens,
sub_expr.source or sub_expr.concept_key,
sub_expr.concept_key, sya=sya)
if not hasattr(concept_node, "concept"):
raise Exception(f"'{sub_expr.concept_key}' is not a concept. Check your map.")
concept_found = concept_node.concept
sub_expr.concept_key = concept_found.key
sub_expr.concept = concept_found
sub_expr.fix_pos((concept_node.start, concept_node.end if hasattr(concept_node, "end") else concept_node.start))
if isinstance(sub_expr, (CNC, CC, CN, CMV)):
if sub_expr.concept is None or sub_expr.start is None or sub_expr.end is None:
concept_node = get_node(
concepts_map,
expression_as_tokens,
sub_expr.source or sub_expr.concept_key,
sub_expr.concept_key, sya=sya)
if not hasattr(concept_node, "concept"):
raise Exception(f"'{sub_expr.concept_key}' is not a concept. Check your map.")
concept_found = concept_node.concept
sub_expr.concept_key = concept_found.key
sub_expr.concept = concept_found
sub_expr.fix_pos(
(concept_node.start, concept_node.end if hasattr(concept_node, "end") else concept_node.start))
if hasattr(sub_expr, "compiled"):
for k, v in sub_expr.compiled.items():
node = get_node(concepts_map, expression_as_tokens, v, sya=sya,
@@ -1210,9 +1318,9 @@ def get_node(
concept_found = concepts_map.get(concept_key, None)
if concept_found:
concept_found = Concept().update_from(concept_found) # make a copy when massively used in tests
if sya and len(concept_found.get_metadata().variables) > 0 and not is_bnf:
return SyaConceptParserHelper(concept_found, start, start + length - 1)
elif init_empty_body:
# if sya and len(concept_found.get_metadata().variables) > 0 and not is_bnf:
# return SyaConceptParserHelper(concept_found, start, start + length - 1)
if init_empty_body:
node = CNC(concept_found, sub_expr, start, start + length - 1, exclude_body=exclude_body)
init_body(node, concept_found, sub_expr)
return node
@@ -1354,8 +1462,8 @@ def get_test_obj(real_obj, test_obj, get_test_obj_delegate=None):
"""
From a production object (Concept, ConceptNode, ....)
Create a test object (CNC, CC ...) that can be used to validate the unit tests
:param test_obj:
:param real_obj:
:param test_obj: test object used as a template
:param get_test_obj_delegate:
:return:
"""
@@ -1367,13 +1475,25 @@ def get_test_obj(real_obj, test_obj, get_test_obj_delegate=None):
if isinstance(test_obj, dict):
if len(test_obj) != len(real_obj):
raise Exception(f"Not the same size ! {real_obj=}, {test_obj=}")
return {k: get_test_obj(real_obj[k], v) for k, v in test_obj.items()}
if not hasattr(test_obj, "transform_real_obj"):
return real_obj
if hasattr(test_obj, "transform_real_obj"):
return test_obj.transform_real_obj(real_obj, get_test_obj)
return test_obj.transform_real_obj(real_obj, get_test_obj)
return real_obj
def prepare_nodes_comparison(concepts_map, expression, real_obj, test_obj):
if isinstance(real_obj, list):
assert len(real_obj) == len(
test_obj), f"The two lists do not have the same size {len(real_obj)} != {len(test_obj)}"
resolved_test_obj = compute_expected_array(concepts_map, expression, test_obj)
real_obj_as_test = [get_test_obj(r, t) for r, t in zip(real_obj, resolved_test_obj)]
return real_obj_as_test, resolved_test_obj
else:
resolved_test_obj = compute_expected_array(concepts_map, expression, [test_obj])[0]
real_obj_as_test = get_test_obj(real_obj, resolved_test_obj)
return real_obj_as_test, resolved_test_obj
def compare_with_test_object(actual, expected):
tests/parsers/test_BaseCustomGrammarParser.py
+1 -1
View File
@@ -124,7 +124,7 @@ func(a)
assert parser.get_parts(["print", "when"]) is not None
assert len(parser.error_sink) == 1
assert isinstance(parser.error_sink[0], UnexpectedEofParsingError)
assert parser.error_sink[0].message == "While parsing keyword 'print'."
assert parser.error_sink[0].message == "while parsing keyword 'print'"
def test_i_can_double_quoted_strings_are_expanded(self):
"""
tests/parsers/test_BnfNodeParser.py
+13 -9
View File
@@ -4,19 +4,20 @@ import pytest
import tests.parsers.parsers_utils
from core.builtin_concepts import BuiltinConcepts
from core.concept import Concept, ConceptParts, DoNotResolve, DEFINITION_TYPE_BNF
from core.concept import Concept, ConceptParts, DEFINITION_TYPE_BNF, DoNotResolve
from core.global_symbols import NotInit
from core.sheerka.services.SheerkaConceptManager import SheerkaConceptManager
from core.sheerka.services.SheerkaExecute import ParserInput
from core.sheerka.services.SheerkaIsAManager import SheerkaIsAManager
from parsers.BaseNodeParser import NoMatchingTokenError
from parsers.BnfDefinitionParser import BnfDefinitionParser
from parsers.BnfNodeParser import StrMatch, TerminalNode, NonTerminalNode, Sequence, OrderedChoice, \
Optional, ZeroOrMore, OneOrMore, ConceptExpression, UnOrderedChoice, BnfNodeParser, RegExMatch, \
BnfNodeFirstTokenVisitor, Match, RegExDef, VariableExpression
from parsers.BnfNodeParser import BnfNodeFirstTokenVisitor, BnfNodeParser, ConceptExpression, Match, NonTerminalNode, \
OneOrMore, Optional, OrderedChoice, RegExDef, RegExMatch, Sequence, StrMatch, TerminalNode, UnOrderedChoice, \
VariableExpression, ZeroOrMore
from tests.BaseTest import BaseTest
from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
from tests.evaluators.EvaluatorTestsUtils import python_ret_val
from tests.parsers.parsers_utils import CNC, CN, UTN, CC, SCN, get_test_obj, compare_with_test_object
from tests.parsers.parsers_utils import CC, CMV, CN, CNC, SCN, UTN, compare_with_test_object, get_test_obj
cmap = {
"one": Concept("one"),
@@ -1027,7 +1028,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
("one tiny but beautiful shoe",
[CNC("foo",
"one tiny but beautiful shoe",
x=CC("but", source="tiny but beautiful", x="tiny", y="beautiful "))]),
x=CMV("but", source="tiny but beautiful", x="tiny ", y="beautiful "))]),
])
def test_i_can_match_variable_in_between(self, expr, expected):
my_map = {
@@ -1896,7 +1897,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
def test_i_can_simplify_unordered_choices_that_refer_to_the_same_isa(self):
my_map = {
"light_red": Concept("light red"),
"light_red": Concept("light red", key="light_red"),
"dark_red": Concept("dark red"),
"red colors": Concept("red colors"),
"color": Concept("color"),
@@ -1916,6 +1917,10 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
sheerka.set_isa(global_truth_context, my_map["red colors"], my_map["color"])
sheerka.set_isa(global_truth_context, my_map["red colors"], my_map["adjective"])
# hack to ease the tests
sheerka.get_by_id(my_map["light_red"].id).get_metadata().key = "light_red"
sheerka.om.clear(SheerkaIsAManager.CONCEPTS_IN_GROUPS_ENTRY)
text = "light red table"
expected = CNC("qualified table",
@@ -1940,7 +1945,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
"isafoo": Concept("x is an foo", body="False", pre="is_question()").def_var("x"),
"q": Concept("q ?", body="question(a)").def_var("q")
}
# I need the concept isafoo to fool SyaNodeParser when parsing the sub text 'is an hex ?'"
# The parser will try to recognize 'is an foo', will fail and will revert the result to UTN()
# It's this UTN that need to be properly handled
@@ -1953,7 +1958,6 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
res = parser.parse(context, ParserInput(text))
assert not res.status
# @pytest.mark.parametrize("parser_input, expected", [
# ("one", [
# (True, [CNC("bnf_one", source="one", one="one", body="one")]),
tests/parsers/test_DefConceptParser.py
+2 -2
View File
@@ -387,8 +387,8 @@ def concept add one to a as:
("def concept name from def", SyntaxErrorNode([], "Empty 'from' declaration.")),
("def concept name from def ", SyntaxErrorNode([], "Empty 'from' declaration.")),
("def concept name from as True", SyntaxErrorNode([], "Empty 'from' declaration.")),
("def concept name from", UnexpectedEofParsingError("While parsing keyword 'from'.")),
("def concept name from ", UnexpectedEofParsingError("While parsing keyword 'from'.")),
("def concept name from", UnexpectedEofParsingError("while parsing keyword 'from'")),
("def concept name from ", UnexpectedEofParsingError("while parsing keyword 'from'")),
])
def test_i_can_detect_empty_def_declaration(self, text, error):
sheerka, context, parser, *concepts = self.init_parser()
tests/parsers/test_DefRuleParser.py
+2 -2
View File
@@ -190,8 +190,8 @@ class TestDefRuleParser(TestUsingMemoryBasedSheerka):
assert sheerka.isinstance(res.body, expected_error)
@pytest.mark.parametrize("text, error_message", [
("def rule rule_name as", "While parsing 'when'."),
("def rule rule_name as ", "While parsing 'when'."),
("def rule rule_name as", "while parsing 'when'"),
("def rule rule_name as ", "while parsing 'when'"),
])
def test_i_cannot_parse_when_unexpected_eof(self, text, error_message):
sheerka, context, parser = self.init_parser()
tests/parsers/test_ListComprehensionParser.py
+180
View File
@@ -0,0 +1,180 @@
import pytest
from core.builtin_concepts_ids import BuiltinConcepts
from core.sheerka.services.SheerkaExecute import ParserInput
from core.tokenizer import Token, TokenKind
from parsers.BaseExpressionParser import ParenthesisMismatchError
from parsers.BaseParser import UnexpectedEofParsingError, UnexpectedTokenParsingError
from parsers.ListComprehensionParser import ElementNotFound, FailedToParse, ForNotFound, LeadingParenthesisNotFound, \
ListComprehensionParser
from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
from tests.parsers.parsers_utils import LC, L_EXPR, get_expr_node_from_test_node
class TestListComprehensionParser(TestUsingMemoryBasedSheerka):
def init_parser(self):
sheerka, context = self.init_concepts()
parser = ListComprehensionParser(auto_compile=False)
return sheerka, context, parser
@pytest.mark.parametrize("text, reason", [
("foo", LeadingParenthesisNotFound()),
("[]", ForNotFound()),
("[ x ]", ForNotFound()),
("[ x for]", FailedToParse("target", 5)),
("[ x for x]", UnexpectedEofParsingError("while parsing comprehension")),
("[ x for x in ]", UnexpectedEofParsingError("while parsing comprehension")),
("[ x for x in lst for]", FailedToParse("target", 13)),
("[", UnexpectedEofParsingError("when start parsing")),
("[]", ForNotFound()),
("[ for x in z ]", ElementNotFound()),
("[ x for in z ]", FailedToParse("target", 6)),
("[ x for x in ]", UnexpectedEofParsingError("while parsing comprehension")),
("[ x for x in z if ]", UnexpectedEofParsingError("while parsing comprehension")),
("[ x for x in z", ParenthesisMismatchError(Token(TokenKind.RBRACKET, "]", -1, -1, -1))),
("[ x for x in z if t", ParenthesisMismatchError(Token(TokenKind.RBRACKET, "]", -1, -1, -1))),
("zzz [ x for x in z if t ]", LeadingParenthesisNotFound()),
("[ x for x in z )", ParenthesisMismatchError(Token(TokenKind.RBRACKET, "]", -1, -1, -1))),
("[ x for x in z if t )", ParenthesisMismatchError(Token(TokenKind.RBRACKET, "]", -1, -1, -1))),
])
def test_i_cannot_parse_when_not_for_me(self, text, reason):
sheerka, context, parser = self.init_parser()
res = parser.parse(context, ParserInput(text))
assert not res.status
assert sheerka.isinstance(res.body, BuiltinConcepts.NOT_FOR_ME)
assert res.body.reason == [reason]
def test_i_cannot_parse_when_trailing_elements(self):
sheerka, context, parser = self.init_parser()
text = "[ x for x in z if t ] zzz"
res = parser.parse(context, ParserInput(text))
assert not res.status
assert sheerka.isinstance(res.body, BuiltinConcepts.ERROR)
assert len(res.body.body) == 1
error = res.body.body[0]
assert isinstance(error, UnexpectedTokenParsingError)
def test_i_can_parse_a_simple_expression(self):
sheerka, context, parser = self.init_parser()
expression = "[x for x in ['a', 'b'] if x == 'a']"
res = parser.parse(context, ParserInput(expression))
wrapper = res.body
lc_node = res.body.body
assert res.status
assert sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
expected = LC(L_EXPR(None, None, "x", source="x "), [(("x", 1), "['a', 'b']", "x == 'a'")], source=expression)
to_compare_to = get_expr_node_from_test_node(expression, expected)
assert lc_node == to_compare_to
def test_i_can_parse_when_no_if(self):
sheerka, context, parser = self.init_parser()
expression = "[x for x in ['a', 'b']]"
res = parser.parse(context, ParserInput(expression))
wrapper = res.body
lc_node = res.body.body
assert res.status
assert sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
expected = LC(L_EXPR(None, None, "x", source="x "), [(("x", 1), "['a', 'b']", None)], source=expression)
to_compare_to = get_expr_node_from_test_node(expression, expected)
assert lc_node == to_compare_to
def test_i_can_parse_when_element_is_a_tuple(self):
sheerka, context, parser = self.init_parser()
expression = "[(x + 1, x + 2) for x in [1, 2]]"
res = parser.parse(context, ParserInput(expression))
wrapper = res.body
lc_node = res.body.body
assert res.status
assert sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
elt = L_EXPR("(", ")", "x + 1", "x + 2")
expected = LC(elt, [(("x", 2), "[1, 2]", None)], source=expression)
to_compare_to = get_expr_node_from_test_node(expression, expected)
assert lc_node == to_compare_to
def test_i_can_parse_when_element_is_a_tuple_with_missing_parenthesis(self):
sheerka, context, parser = self.init_parser()
expression = "[x + 1, x + 2 for x in [1, 2]]"
res = parser.parse(context, ParserInput(expression))
wrapper = res.body
lc_node = res.body.body
assert res.status
assert sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
elt = L_EXPR(None, None, "x + 1", "x + 2", source="x + 1, x + 2 ")
expected = LC(elt, [(("x", 2), "[1, 2]", None)], source=expression)
to_compare_to = get_expr_node_from_test_node(expression, expected)
assert lc_node == to_compare_to
def test_i_can_parse_when_element_is_a_context_that_contains_for(self):
sheerka, context, parser = self.init_parser()
expression = "[handle x for me and for you for x in [1, 2]]"
res = parser.parse(context, ParserInput(expression))
wrapper = res.body
lc_node = res.body.body
assert res.status
assert sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
elt = L_EXPR(None, None, "handle x for me and for you", source="handle x for me and for you ")
expected = LC(elt, [(("x", 1), "[1, 2]", None)], source=expression)
to_compare_to = get_expr_node_from_test_node(expression, expected)
assert lc_node == to_compare_to
def test_i_can_parse_when_multiple_generators(self):
sheerka, context, parser = self.init_parser()
expression = "[(x, y) for x in ['a', 'b'] if x == 'a' for y in ['c', 'd'] if y == 'c']"
res = parser.parse(context, ParserInput(expression))
wrapper = res.body
lc_node = res.body.body
assert res.status
assert sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
elt = L_EXPR("(", ")", "x", "y")
expected = LC(elt,
[(("x", 1), "['a', 'b']", "x == 'a'"),
(("y", 1), "['c', 'd']", "y == 'c'")],
source=expression)
to_compare_to = get_expr_node_from_test_node(expression, expected)
assert lc_node == to_compare_to
def test_i_can_parse_when_multiple_generators_when_no_if(self):
sheerka, context, parser = self.init_parser()
expression = "[x, y for x in ['a', 'b'] for y in ['c', 'd'] if y == 'c']"
res = parser.parse(context, ParserInput(expression))
wrapper = res.body
lc_node = res.body.body
assert res.status
assert sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
elt = L_EXPR(None, None, "x", "y", source="x, y ")
expected = LC(elt,
[(("x", 1), "['a', 'b']", None),
(("y", 1), "['c', 'd']", "y == 'c'")],
source=expression)
to_compare_to = get_expr_node_from_test_node(expression, expected)
assert lc_node == to_compare_to
tests/parsers/test_ListParser.py
+69
View File
@@ -0,0 +1,69 @@
import pytest
from core.builtin_concepts_ids import BuiltinConcepts
from core.sheerka.services.SheerkaExecute import ParserInput
from core.tokenizer import Token, TokenKind
from parsers.BaseExpressionParser import ParenthesisMismatchError, end_parenthesis_mapping
from parsers.BaseParser import ErrorSink
from parsers.ListParser import ListParser
from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
from tests.parsers.parsers_utils import EXPR, L_EXPR, get_expr_node_from_test_node
semi_colon = Token(TokenKind.SEMICOLON, ";", -1, -1, -1)
or_token = Token(TokenKind.IDENTIFIER, "or", -1, -1, -1)
class TestListParser(TestUsingMemoryBasedSheerka):
def init_parser(self, sep=None):
sheerka, context = self.init_concepts()
parser = ListParser(sep)
return sheerka, context, parser
@pytest.mark.parametrize("expression, sep, expected", [
("()", None, L_EXPR("(", ")")),
("(x , foo y,z)", None, L_EXPR("(", ")", EXPR("x"), EXPR("foo y"), EXPR("z"), source="(x , foo y,z)")),
("x , foo y,z", None, L_EXPR(None, None, EXPR("x"), EXPR("foo y"), EXPR("z"), source="x , foo y,z")),
("x", None, L_EXPR(None, None, EXPR("x"))),
("[x, foo y, z]", None, L_EXPR("[", "]", EXPR("x"), EXPR("foo y"), EXPR("z"))),
("{x, foo y, z}", None, L_EXPR("{", "}", EXPR("x"), EXPR("foo y"), EXPR("z"))),
("(x; y; z)", semi_colon, L_EXPR("(", ")", EXPR("x"), EXPR("y"), EXPR("z"), sep=semi_colon, source="(x; y; z)")),
("x; y; z", semi_colon, L_EXPR(None, None, EXPR("x"), EXPR("y"), EXPR("z"), sep=semi_colon, source="x; y; z")),
("x or y or z", or_token, L_EXPR(None, None, EXPR("x"), EXPR("y"), EXPR("z"), sep=or_token, source="x or y or z")),
])
def test_i_can_parse_expression(self, expression, sep, expected):
sheerka, context, parser = self.init_parser(sep)
expected = get_expr_node_from_test_node(expression, expected)
res = parser.parse(context, ParserInput(expression))
wrapper = res.body
expressions = res.body.body
assert res.status
assert sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
assert expressions == expected
@pytest.mark.parametrize("expression, starting", [
("(", TokenKind.LPAR),
("(x, y", TokenKind.LPAR),
("{x, y", TokenKind.LBRACE),
("[x, y", TokenKind.LBRACKET),
])
def test_i_cannot_parse_when_missing_trailing_parenthesis(self, expression, starting):
sheerka, context, parser = self.init_parser()
res = parser.parse(context, ParserInput(expression))
assert not res.status
assert sheerka.isinstance(res.body, BuiltinConcepts.ERROR)
assert res.body.body == [ParenthesisMismatchError(end_parenthesis_mapping[starting])]
def test_none_is_return_when_empty_parser_input(self):
sheerka, context, parser = self.init_parser()
parser_input = ParserInput(" ").reset()
parser_input.next_token()
error_sink = ErrorSink()
parsed = parser.parse_input(context, parser_input, error_sink)
assert parsed is None
tests/parsers/test_LogicalOperatorParser.py
+6 -6
View File
@@ -100,12 +100,12 @@ class TestLogicalOperatorParser(TestUsingMemoryBasedSheerka):
assert expressions == expected
@pytest.mark.parametrize("expression, expected_errors", [
("one or", [UnexpectedEofParsingError("When parsing 'or'")]),
("one and", [UnexpectedEofParsingError("When parsing 'and'")]),
("and one", [LeftPartNotFoundError()]),
("or one", [LeftPartNotFoundError()]),
("or", [LeftPartNotFoundError(), UnexpectedEofParsingError("When parsing 'or'")]),
("and", [LeftPartNotFoundError(), UnexpectedEofParsingError("When parsing 'and'")]),
("one or", [UnexpectedEofParsingError("while parsing 'or'")]),
("one and", [UnexpectedEofParsingError("while parsing 'and'")]),
("and one", [LeftPartNotFoundError("and", 0)]),
("or one", [LeftPartNotFoundError("or", 0)]),
("or", [LeftPartNotFoundError("or", 0), UnexpectedEofParsingError("while parsing 'or'")]),
("and", [LeftPartNotFoundError("and", 0), UnexpectedEofParsingError("while parsing 'and'")]),
])
def test_i_can_detect_error(self, expression, expected_errors):
sheerka, context, parser = self.init_parser()
tests/parsers/test_SequenceNodeParser.py
+22 -2
View File
@@ -5,8 +5,7 @@ from core.concept import Concept, DEFINITION_TYPE_DEF
from core.sheerka.services.SheerkaExecute import ParserInput
from parsers.SequenceNodeParser import SequenceNodeParser
from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
from tests.parsers.parsers_utils import compute_expected_array, CN, SCN, get_test_obj, compare_with_test_object, \
UTN
from tests.parsers.parsers_utils import CN, SCN, UTN, compare_with_test_object, compute_expected_array, get_test_obj
class TestSequenceNodeParser(TestUsingMemoryBasedSheerka):
@@ -463,3 +462,24 @@ class TestSequenceNodeParser(TestUsingMemoryBasedSheerka):
assert concept_found.name == "boys"
assert concept_found.key == "boys"
assert concept_found.get_prop(BuiltinConcepts.PLURAL) == boy
def test_i_can_set_body_for_plurals_that_are_a_set(self):
concepts_map = {
"boy": Concept("boy"),
"girl": Concept("girl"),
"human": Concept("human"),
}
sheerka, context, parser = self.init_parser(concepts_map)
global_truth_concept = self.get_context(sheerka, global_truth=True)
sheerka.set_isa(global_truth_concept, concepts_map["boy"], concepts_map["human"])
sheerka.set_isa(global_truth_concept, concepts_map["girl"], concepts_map["human"])
res = parser.parse(context, ParserInput("humans"))
assert res.status
lexer_nodes = res.body.body
assert len(lexer_nodes) == 1
concept_found = lexer_nodes[0].concept
assert concept_found.get_metadata().body == "get_set_elements(c:|1003:)"
tests/parsers/test_SyaNodeParser.py
+1370 -1118
View File
File diff suppressed because it is too large Load Diff
tests/sheerkapython/test_ExprToPython.py
+422
View File
@@ -0,0 +1,422 @@
import pytest
from core.builtin_concepts_ids import BuiltinConcepts
from core.concept import Concept
from core.sheerka.services.SheerkaEvaluateConcept import EvaluationHints
from core.sheerka.services.SheerkaExecute import ParserInput
from evaluators.PythonEvaluator import PythonEvaluator
from parsers.BaseParser import ErrorSink
from parsers.ExpressionParser import ExpressionParser
from parsers.ListComprehensionParser import ListComprehensionParser
from parsers.PythonParser import PythonNode
from sheerkapython.ExprToPython import PythonExprVisitor
from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
class TestExprToPython(TestUsingMemoryBasedSheerka):
@staticmethod
def get_expr_node(context, expression, parser=None):
parser = parser or ExpressionParser()
error_sink = ErrorSink()
parser_input = ParserInput(expression)
parser.reset_parser_input(parser_input, error_sink)
parsed = parser.parse_input(context, parser_input, error_sink)
assert not error_sink.has_error
return parsed
@staticmethod
def eval(context, return_value, namespace=None):
evaluator = PythonEvaluator()
assert evaluator.matches(context, return_value)
if namespace:
for k, v in namespace.items():
context.add_to_short_term_memory(k, v)
res = evaluator.eval(context, return_value)
assert res.status
return res.body
@pytest.mark.parametrize("expression, source, objects", [
("foo w", "call_concept(__o_00__, x=w)", {"__o_00__": "foo"}),
("foo z + 2", "call_concept(__o_00__, x=z + 2)", {"__o_00__": "foo"}),
("foo a and bar b",
"call_concept(__o_00__, x=a) and call_concept(__o_01__, y=b)",
{"__o_00__": "foo", "__o_01__": "bar"}),
("foo a or bar b",
"call_concept(__o_00__, x=a) or call_concept(__o_01__, y=b)",
{"__o_00__": "foo", "__o_01__": "bar"}),
("not foo w", "not call_concept(__o_00__, x=w)", {"__o_00__": "foo"}),
("foo a >= bar b",
"call_concept(__o_00__, x=a) >= call_concept(__o_01__, y=b)",
{"__o_00__": "foo", "__o_01__": "bar"}),
("function(foo a, bar b)",
"function(call_concept(__o_00__, x=a), call_concept(__o_01__, y=b))",
{"__o_00__": "foo", "__o_01__": "bar"})
])
def test_i_can_compile_concept_when_is_question_is_false(self, expression, source, objects):
sheerka, context, foo, bar = self.init_test().with_concepts(
Concept("foo x", body="x").def_var("x"),
Concept("bar y", body="y").def_var("y"),
create_new=True
).unpack()
concepts = {
"foo": foo,
"bar": bar
}
node = self.get_expr_node(context, expression)
visitor = PythonExprVisitor(context)
ret = visitor.compile(node)
assert len(ret) == 1
python_node = ret[0].body.body
assert python_node.original_source == expression
assert python_node.source == source
for obj_name, obj_value in objects.items():
assert obj_name in python_node.objects
obj = python_node.objects[obj_name]
if isinstance(obj, Concept):
assert sheerka.isinstance(obj, concepts[obj_value])
assert obj.get_hints().use_copy
assert obj.get_hints().is_evaluated
else:
assert False
@pytest.mark.parametrize("expression, source, objects", [
("foo w", "evaluate_question(__o_00__, x=w)", {"__o_00__": "foo"}),
("foo z + 2", "evaluate_question(__o_00__, x=z + 2)", {"__o_00__": "foo"}),
])
def test_i_can_compile_concept_when_is_question_is_true(self, expression, source, objects):
sheerka, context, foo = self.init_test().with_concepts(
Concept("foo x", body="x", pre="is_question()").def_var("x"),
create_new=True
).unpack()
concepts = {
"foo": foo
}
node = self.get_expr_node(context, expression)
visitor = PythonExprVisitor(context)
ret = visitor.compile(node, EvaluationHints(eval_body=True, eval_question=True))
assert len(ret) == 1
python_node = ret[0].body.body
assert python_node.original_source == expression
assert python_node.source == source
for obj_name, obj_value in objects.items():
assert obj_name in python_node.objects
obj = python_node.objects[obj_name]
if isinstance(obj, Concept):
assert sheerka.isinstance(obj, concepts[obj_value])
assert obj.get_hints().use_copy
assert obj.get_hints().is_evaluated
else:
assert False
def test_i_can_compile_simple_list_comprehension(self):
sheerka, context = self.init_test().unpack()
expression = "[ x for x in ['a', 'b'] if x == 'a' ]"
node = self.get_expr_node(context, expression, parser=ListComprehensionParser())
visitor = PythonExprVisitor(context)
ret = visitor.compile(node)
assert len(ret) == 1
assert sheerka.isinstance(ret[0], BuiltinConcepts.RETURN_VALUE)
assert sheerka.isinstance(ret[0].body, BuiltinConcepts.PARSER_RESULT)
assert isinstance(ret[0].body.body, PythonNode)
python_node = ret[0].body.body
assert python_node.original_source == expression
assert python_node.source == expression
assert self.eval(context, ret[0]) == ["a"]
def test_i_can_compile_simple_list_comprehension_when_no_if(self):
sheerka, context = self.init_test().unpack()
expression = "[ x for x in ['a', 'b'] ]"
node = self.get_expr_node(context, expression, parser=ListComprehensionParser())
visitor = PythonExprVisitor(context)
ret = visitor.compile(node)
assert len(ret) == 1
assert sheerka.isinstance(ret[0], BuiltinConcepts.RETURN_VALUE)
assert sheerka.isinstance(ret[0].body, BuiltinConcepts.PARSER_RESULT)
assert isinstance(ret[0].body.body, PythonNode)
python_node = ret[0].body.body
assert python_node.original_source == expression
assert python_node.source == expression
assert self.eval(context, ret[0]) == ['a', 'b']
def test_i_can_compile_list_comprehension_when_element_is_a_concept(self):
sheerka, context, foo = self.init_test().with_concepts(
Concept("foo x", body="x").def_var("x")
).unpack()
expression = "[ foo w for w in ['a', 'b'] ]"
node = self.get_expr_node(context, expression, parser=ListComprehensionParser())
visitor = PythonExprVisitor(context)
ret = visitor.compile(node)
assert len(ret) == 1
assert sheerka.isinstance(ret[0], BuiltinConcepts.RETURN_VALUE)
assert sheerka.isinstance(ret[0].body, BuiltinConcepts.PARSER_RESULT)
assert isinstance(ret[0].body.body, PythonNode)
python_node = ret[0].body.body
assert python_node.original_source == expression
assert python_node.source == "[ call_concept(__o_00__, x=w) for w in ['a', 'b'] ]"
assert "__o_00__" in python_node.objects
concept = python_node.objects["__o_00__"]
assert sheerka.isinstance(concept, foo)
assert concept.get_hints().use_copy
assert concept.get_hints().is_evaluated
assert self.eval(context, ret[0]) == ["a", "b"]
def test_i_can_compile_list_comprehension_when_concept_with_complex_parameter(self):
sheerka, context, foo = self.init_test().with_concepts(
Concept("foo x", body="x").def_var("x"),
create_new=True
).unpack()
expression = "[ foo w + 1 for w in [1, 2] ]"
node = self.get_expr_node(context, expression, parser=ListComprehensionParser())
visitor = PythonExprVisitor(context)
ret = visitor.compile(node)
assert len(ret) == 1
assert sheerka.isinstance(ret[0], BuiltinConcepts.RETURN_VALUE)
assert sheerka.isinstance(ret[0].body, BuiltinConcepts.PARSER_RESULT)
assert isinstance(ret[0].body.body, PythonNode)
python_node = ret[0].body.body
assert python_node.original_source == expression
assert python_node.source == "[ call_concept(__o_00__, x=w + 1) for w in [1, 2] ]"
assert "__o_00__" in python_node.objects
assert self.eval(context, ret[0]) == [2, 3]
def test_i_can_compile_list_comprehension_when_iter_is_a_concept(self):
sheerka, context, red, blue, color, foo = self.init_test().with_concepts(
"red",
"blue",
"color",
Concept("foo x", body="x").def_var("x")
).unpack()
global_truth_context = self.get_context(sheerka, global_truth=True)
sheerka.set_isa(global_truth_context, red, color)
sheerka.set_isa(global_truth_context, blue, color)
expression = "[ foo x for x in colors ]"
node = self.get_expr_node(context, expression, parser=ListComprehensionParser())
visitor = PythonExprVisitor(context)
ret = visitor.compile(node)
assert len(ret) == 1
assert sheerka.isinstance(ret[0], BuiltinConcepts.RETURN_VALUE)
assert sheerka.isinstance(ret[0].body, BuiltinConcepts.PARSER_RESULT)
assert isinstance(ret[0].body.body, PythonNode)
python_node = ret[0].body.body
assert python_node.original_source == expression
assert python_node.source == "[ call_concept(__o_00__, x=x) for x in call_concept(__o_01__) ]"
assert "__o_00__" in python_node.objects
assert "__o_01__" in python_node.objects
concept0 = python_node.objects["__o_00__"]
assert sheerka.isinstance(concept0, foo)
assert concept0.get_hints().use_copy
assert concept0.get_hints().is_evaluated
concept1 = python_node.objects["__o_01__"]
assert sheerka.isinstance(concept1, "colors")
assert concept1.get_hints().use_copy
assert concept1.get_hints().is_evaluated
assert set(self.eval(context, ret[0])) == {red, blue}
def test_i_can_compile_list_comprehension_when_if_expression_is_a_concept(self):
sheerka, context, red, blue, color, foo, startswith = self.init_test().with_concepts(
"red",
"blue",
"color",
Concept("foo x", body="x").def_var("x"),
Concept("x starts with y", body="x.name.startswith(y)", pre="is_question()").def_var("x").def_var("y")
).unpack()
global_truth_context = self.get_context(sheerka, global_truth=True)
sheerka.set_isa(global_truth_context, red, color)
sheerka.set_isa(global_truth_context, blue, color)
expression = "[ foo x for x in colors if x starts with 'b' ]"
node = self.get_expr_node(context, expression, parser=ListComprehensionParser())
visitor = PythonExprVisitor(context)
ret = visitor.compile(node)
assert len(ret) == 1
assert sheerka.isinstance(ret[0], BuiltinConcepts.RETURN_VALUE)
assert sheerka.isinstance(ret[0].body, BuiltinConcepts.PARSER_RESULT)
assert isinstance(ret[0].body.body, PythonNode)
python_node = ret[0].body.body
assert python_node.original_source == expression
assert python_node.source == "[ call_concept(__o_00__, x=x) for x in call_concept(__o_01__) if evaluate_question(__o_02__, x=x, y='b') ]"
assert "__o_00__" in python_node.objects
assert "__o_01__" in python_node.objects
assert "__o_02__" in python_node.objects
assert visitor.obj_counter == 3
assert set(self.eval(context, ret[0])) == {blue}
def test_i_can_compile_list_comprehension_when_multiple_concepts(self):
sheerka, context, foo1, foo2, bar1, bar2, colors1, colors2, = self.init_test().with_concepts(
Concept("foo x").def_var("x"),
Concept("foo y", ).def_var("y"),
Concept("bar x", pre="is_question()").def_var("x"),
Concept("bar y", pre="is_question()").def_var("y"),
Concept("colors", body="[1]"),
Concept("colors", body="[2]"),
).unpack()
expression = "[ foo a for a in colors if bar a ]"
node = self.get_expr_node(context, expression, parser=ListComprehensionParser())
visitor = PythonExprVisitor(context)
ret = visitor.compile(node)
assert len(ret) == 8
python_node = ret[0].body.body
object_to_compare = {k: v.name for k, v in python_node.objects.items()}
assert python_node.source == '[ call_concept(__o_00__, x=a) for a in call_concept(__o_02__) if evaluate_question(__o_04__, x=a) ]'
assert object_to_compare == {"__o_00__": "foo x", "__o_02__": "colors", "__o_04__": "bar x"}
# ...
python_node = ret[7].body.body
object_to_compare = {k: v.name for k, v in python_node.objects.items()}
assert python_node.source == '[ call_concept(__o_01__, y=a) for a in call_concept(__o_03__) if evaluate_question(__o_05__, y=a) ]'
assert object_to_compare == {"__o_01__": "foo y", "__o_03__": "colors", "__o_05__": "bar y"}
def test_i_can_compile_list_comprehension_when_missing_concept_parameter(self):
sheerka, context, foo = self.init_test().with_concepts(
Concept("foo x y", body="x").def_var("x").def_var("y")
).unpack()
expression = "[ foo x k for x in ['a', 'b'] ]"
node = self.get_expr_node(context, expression, parser=ListComprehensionParser())
visitor = PythonExprVisitor(context)
ret = visitor.compile(node)
assert len(ret) == 1
python_node = ret[0].body.body
object_to_compare = {k: v.name for k, v in python_node.objects.items()}
assert python_node.source == "[ call_concept(__o_00__, x=x, y=k) for x in ['a', 'b'] ]"
assert object_to_compare == {"__o_00__": "foo x y"}
def test_i_can_compile_simple_list_comprehension_when_multiple_for(self):
sheerka, context = self.init_test().unpack()
expression = "[ (x, y) for x in ['a', 'b'] if x == 'a' for y in ['c', 'd'] if y == 'c' ]"
node = self.get_expr_node(context, expression, parser=ListComprehensionParser())
visitor = PythonExprVisitor(context)
ret = visitor.compile(node)
assert len(ret) == 1
assert sheerka.isinstance(ret[0], BuiltinConcepts.RETURN_VALUE)
assert sheerka.isinstance(ret[0].body, BuiltinConcepts.PARSER_RESULT)
assert isinstance(ret[0].body.body, PythonNode)
python_node = ret[0].body.body
assert python_node.original_source == expression
assert python_node.source == expression
assert self.eval(context, ret[0]) == [("a", "c")]
def test_i_can_compile_and_when_multiple_results(self):
sheerka, context, foo, foo2, bar, bar2 = self.init_test().with_concepts(
Concept("foo x", body="x").def_var("x"),
Concept("foo y", body="y").def_var("y"),
Concept("bar x", body="x").def_var("x"),
Concept("bar y", body="y").def_var("y"),
create_new=True
).unpack()
node = self.get_expr_node(context, "foo a and bar b")
visitor = PythonExprVisitor(context)
ret = visitor.compile(node)
assert len(ret) == 4
python_node = ret[0].body.body
object_to_compare = {k: v.name for k, v in python_node.objects.items()}
assert python_node.source == 'call_concept(__o_00__, x=a) and call_concept(__o_02__, x=b)'
assert object_to_compare == {"__o_00__": "foo x", "__o_02__": "bar x"}
python_node = ret[1].body.body
object_to_compare = {k: v.name for k, v in python_node.objects.items()}
assert python_node.source == 'call_concept(__o_00__, x=a) and call_concept(__o_03__, y=b)'
assert object_to_compare == {"__o_00__": "foo x", "__o_03__": "bar y"}
python_node = ret[2].body.body
object_to_compare = {k: v.name for k, v in python_node.objects.items()}
assert python_node.source == 'call_concept(__o_01__, y=a) and call_concept(__o_02__, x=b)'
assert object_to_compare == {"__o_01__": "foo y", "__o_02__": "bar x"}
python_node = ret[3].body.body
object_to_compare = {k: v.name for k, v in python_node.objects.items()}
assert python_node.source == 'call_concept(__o_01__, y=a) and call_concept(__o_03__, y=b)'
assert object_to_compare == {"__o_01__": "foo y", "__o_03__": "bar y"}
def test_i_can_compile_when_element_is_missing_its_parenthesis(self):
sheerka, context, foo = self.init_test().with_concepts(
Concept("foo x", body="x").def_var("x")
).unpack()
expression = "[ w, foo w for w in ['a', 'b'] ]"
node = self.get_expr_node(context, expression, parser=ListComprehensionParser())
visitor = PythonExprVisitor(context)
ret = visitor.compile(node)
assert len(ret) == 1
assert sheerka.isinstance(ret[0], BuiltinConcepts.RETURN_VALUE)
assert sheerka.isinstance(ret[0].body, BuiltinConcepts.PARSER_RESULT)
assert isinstance(ret[0].body.body, PythonNode)
python_node = ret[0].body.body
assert python_node.original_source == expression
assert python_node.source == "[ (w, call_concept(__o_00__, x=w)) for w in ['a', 'b'] ]"
assert "__o_00__" in python_node.objects
concept = python_node.objects["__o_00__"]
assert sheerka.isinstance(concept, foo)
assert concept.get_hints().use_copy
assert concept.get_hints().is_evaluated
assert self.eval(context, ret[0]) == [("a", "a"), ("b", "b")]