Fixed #29: Parsers: Implement parsing memoization

Fixed #77 : Parser: ShortTermMemoryParser should be called separately
Fixed #78 : Remove VariableNode usage
Fixed #79 : ConceptManager: Implement compile caching
Fixed #80 : SheerkaExecute : parsers_key is not correctly computed
Fixed #81 : ValidateConceptEvaluator : Validate concept's where and pre clauses right after the parsing
Fixed #82 : SheerkaIsAManager: isa() failed when the set as a body
Fixed #83 : ValidateConceptEvaluator : Support BNF and SYA Concepts
Fixed #84 : ExpressionParser: Implement the parser as a standard parser
Fixed #85 : Services: Give order to services
Fixed #86 : cannot manage smart_get_attr(the short, color)
This commit is contained in:
2021-06-07 21:14:03 +02:00
parent 1059ce25c5
commit 7dcaa9c111
92 changed files with 4263 additions and 1890 deletions
@@ -4,6 +4,7 @@ import pytest
from core.builtin_concepts import ReturnValueConcept
from core.builtin_concepts_ids import BuiltinConcepts
from core.builtin_helpers import ensure_evaluated
from core.concept import Concept, DEFINITION_TYPE_DEF
from core.rule import Rule
from core.sheerka.services.SheerkaEvaluateRules import SheerkaEvaluateRules
@@ -23,6 +24,19 @@ from tests.parsers.parsers_utils import get_rete_conditions, NEGCOND
class BaseTestSheerkaRuleManagerRulesCompilation(TestUsingMemoryBasedSheerka):
@staticmethod
def get_conditions(context, expression):
parser = ExpressionParser()
error_sink = ErrorSink()
parser_input = ParserInput(expression)
parser.reset_parser_input(parser_input, error_sink)
parsed = parser.parse_input(context, parser_input, error_sink)
visitor = PythonConditionExprVisitor(context)
conditions = visitor.get_conditions(parsed)
return conditions
@staticmethod
def check_against_rete(rule_expression, rule_conditions, objects):
"""
@@ -72,6 +86,9 @@ class BaseTestSheerkaRuleManagerRulesCompilation(TestUsingMemoryBasedSheerka):
sub_context.sheerka.add_many_to_short_term_memory(sub_context, objects)
evaluator = PythonEvaluator()
for c in condition.concepts_to_reset:
c.get_hints().is_evaluated = False
return evaluator.eval(sub_context, condition.return_value)
@classmethod
@@ -130,15 +147,7 @@ class BaseTestSheerkaRuleManagerRulesCompilation(TestUsingMemoryBasedSheerka):
expected_not_variables,
expected_objects):
sheerka = context.sheerka
parser = ExpressionParser()
error_sink = ErrorSink()
parser_input = ParserInput(expression)
parser.reset_parser_input(parser_input, error_sink)
parsed = parser.parse_input(context, parser_input, error_sink)
visitor = PythonConditionExprVisitor(context)
conditions = visitor.get_conditions(parsed)
conditions = BaseTestSheerkaRuleManagerRulesCompilation.get_conditions(context, expression)
assert len(conditions) == 1
assert isinstance(conditions[0], CompiledCondition)
@@ -312,6 +321,132 @@ class TestSheerkaRuleManagerRulesCompilationNotExists(BaseTestSheerkaRuleManager
self.check_against_python(context, expression, conditions, objects)
class TestSheerkaRuleManagerRulesCompilationSimplePython(BaseTestSheerkaRuleManagerRulesCompilation):
"""
Testing Python
True
False
10 + 5
'hello world'
a + self
a + 10
a + " world !"
a + foo
a + twenty one
a + my friend
"""
@pytest.mark.parametrize("expression, e_compiled, e_text, e_variables, e_objects, e_result", [
(
"True",
"True",
"True",
set(),
set(),
True
),
(
"False",
"False",
"False",
set(),
set(),
False
),
(
"10 + 5",
"__o_00__",
"10 + 5",
set(),
{("__o_00__", 15)},
15
),
(
"'hello world'",
"__o_00__",
"'hello world'",
set(),
{("__o_00__", 'hello world')},
'hello world'
),
(
"a + self",
"a + self",
"a + self",
{("a", 10), ("self", "foo")},
set(),
15
),
(
"a + 10",
"a + 10",
"a + 10",
{("a", 10)},
set(),
20
),
(
"a + 'world !'",
"a + 'world !'",
"a + 'world !'",
{("a", "hello ")},
set(),
"hello world !"
),
(
"a + foo",
"a + foo",
"a + foo",
{("a", 10), ("foo", "foo")},
set(),
15
),
(
"a + twenty one",
"a + __C__twenties__1004__C__",
"a + twenty one",
{("a", 10)},
{"__C__twenties__1004__C__"},
31
),
(
"a + my friend",
"a + __C__my0friend__1005__C__",
"a + my friend",
{("a", "hello ")},
{'__C__my0friend__1005__C__'},
"hello my friend"
),
])
def test_python(self, expression, e_compiled, e_text, e_variables, e_objects, e_result):
sheerka, context, foo, one, two, twenties, my_friend = self.init_concepts(
Concept("foo", body="5"),
Concept("one", body="1"),
Concept("two", body="2"),
Concept("twenties", definition="'twenty' (one|two)=n", body='20 + n').def_var("n"),
Concept("my friend", body="'my friend'"),
create_new=True
)
ensure_evaluated(context, foo, eval_body=True)
ensure_evaluated(context, my_friend, eval_body=True)
conditions = self.validate_python_test(context,
expression,
e_compiled,
e_text,
e_variables,
set(),
e_objects)
# check against SheerkaEvaluateRules
variables_mapping = {
"foo": foo,
}
namespace = self.get_testing_objects(context, e_variables, variables_mapping)
res = self.evaluate_condition(context, expression, conditions[0], namespace)
assert res.status
assert sheerka.objvalue(res) == e_result
class TestSheerkaRuleManagerRulesCompilationEquality(BaseTestSheerkaRuleManagerRulesCompilation):
"""
Testing simple equality:
@@ -321,6 +456,7 @@ class TestSheerkaRuleManagerRulesCompilationEquality(BaseTestSheerkaRuleManagerR
self == sheerka
self == BuiltinConcepts.TO_DICT
self == hello 'my friend'
a == b
"""
@pytest.mark.parametrize("expression, expected_as_list_of_str, expected_variables", [
@@ -339,7 +475,12 @@ class TestSheerkaRuleManagerRulesCompilationEquality(BaseTestSheerkaRuleManagerR
"#__x_00__|key|'hello __var__0'",
"#__x_00__|a|'my friend'"],
{("self", "hello_my_friend")}
)
),
# ("a == b",
# ["#__x_00__|__name__|'a'",
# "#__x_01__|__name__|'b'",
# "#__x_00__|__self__|#__x_01__"],
# {("a", 10), ("b", 10)}),
])
def test_rete(self, expression, expected_as_list_of_str, expected_variables):
sheerka, context, greetings = self.init_test().with_concepts(
@@ -365,23 +506,27 @@ class TestSheerkaRuleManagerRulesCompilationEquality(BaseTestSheerkaRuleManagerR
objects = self.get_testing_objects(context, expected_variables, objects_mappings)
self.check_against_rete(expression, conditions, objects)
# KSI: 2021-05-06 The last test done not produce any match because the WME (b, __self__, 10)
# is not added to memory.
@pytest.mark.parametrize("expression, expected_compiled, expected_variables, expected_objects", [
("a == 10", "a == __o_00__", {("a", 10)}, {("__o_00__", 10)}),
("__ret.status == True", "__ret.status == __o_00__", {"__ret"}, {("__o_00__", True)}),
("__ret.status == True", "__ret.status == True", {"__ret"}, set()),
("self == 'a'", "self == __o_00__", {("self", 'a')}, {("__o_00__", 'a')}),
("self == sheerka", "is_sheerka(self)", {("self", "sheerka")}, {}),
(
"self == BuiltinConcepts.TO_DICT",
"self == __o_00__",
"self == BuiltinConcepts.TO_DICT",
{("self", BuiltinConcepts.TO_DICT)},
{("__o_00__", BuiltinConcepts.TO_DICT)}
set()
),
(
"self == hello 'my friend'",
"""isinstance(self, Concept) and self.key == 'hello __var__0' and self.a == __o_01__""",
{("self", "hello_my_friend")},
{("__o_01__", "my friend")}
)
),
("a == b", "a == b", {("a", 10), ("b", 10)}, {}),
])
def test_python(self, expression, expected_compiled, expected_variables, expected_objects):
sheerka, context, greetings = self.init_test().with_concepts(
@@ -403,6 +548,176 @@ class TestSheerkaRuleManagerRulesCompilationEquality(BaseTestSheerkaRuleManagerR
self.check_against_python(context, expression, conditions, objects)
class TestSheerkaRuleManagerRulesCompilationOtherConditions(BaseTestSheerkaRuleManagerRulesCompilation):
"""
Testing other conditions than equality
a > 10
a >= 10
a < 10
a <= 10
a != 10
a > 10 and b <= 5
__ret.value > 10
10 > __ret.value
a + self > 10
a + 10 > 10
a + " world !" == "hello world !"
a + foo > 10
a + twenty one > 21
a + my friend == 'hello my friend'
10 < a + self
10 < a + 10
'hello world !' == a + ' world !'
10 < a + foo
10 > a + twenty one
'hello my friend' == a + my friend
"""
@pytest.mark.parametrize("expression, e_compiled, e_variables, e_objects, e_result", [
("a > 10", "a > __o_00__", {("a", 10)}, {("__o_00__", 10)}, False),
("a >= 10", "a >= __o_00__", {("a", 10)}, {("__o_00__", 10)}, True),
("a < 10", "a < __o_00__", {("a", 10)}, {("__o_00__", 10)}, False),
("a <= 10", "a <= __o_00__", {("a", 10)}, {("__o_00__", 10)}, True),
("a != 10", "a != __o_00__", {("a", 10)}, {("__o_00__", 10)}, False),
(
"a > 10 and b <= 5",
"a > __o_00__ and b <= __o_01__",
{("a", 11), ("b", 4)},
{("__o_00__", 10), ("__o_01__", 5)},
True
),
(
"__ret.value > 10",
"__ret.value > __o_00__",
{("__ret", 15)},
{("__o_00__", 10)},
True
),
(
"10 > __ret.value",
"__o_00__ > __ret.value",
{("__ret", 15)},
{("__o_00__", 10)},
False
),
(
"a + self > 10",
"a + self > __o_00__",
{("a", 6), ("self", "foo")},
{("__o_00__", 10)},
True
),
(
"a + 10 > 10",
"a + 10 > __o_00__",
{("a", 5)},
{("__o_00__", 10)},
True
),
(
"a + 'world !' == 'hello world !'",
"a + 'world !' == __o_00__",
{("a", "hello ")},
{("__o_00__", 'hello world !')},
True
),
(
"a + foo > 10",
"a + foo > __o_00__",
{("a", 6), ("foo", "foo")},
{("__o_00__", 10)},
True
),
(
"a + twenty one > 21",
"a + __C__twenties__1004__C__ > __o_00__",
{("a", 5)},
{"__C__twenties__1004__C__", ("__o_00__", 21)},
True
),
(
"a + my friend == 'hello my friend'",
"a + __C__my0friend__1005__C__ == __o_00__",
{("a", "hello ")},
{"__C__my0friend__1005__C__", ("__o_00__", 'hello my friend')},
True
),
(
"10 < a + self",
"__o_00__ < a + self",
{("a", 6), ("self", "foo")},
{("__o_00__", 10)},
True
),
(
"10 > a + 10",
"__o_00__ > a + 10",
{("a", 5)},
{("__o_00__", 10)},
False
),
(
"'hello world !' != a + 'world !'",
"__o_00__ != a + 'world !'",
{("a", "hello ")},
{("__o_00__", 'hello world !')},
False
),
(
"10 < a + foo",
"__o_00__ < a + foo",
{("a", 6), ("foo", "foo")},
{("__o_00__", 10)},
True
),
(
"21 > a + twenty one",
"__o_00__ > a + __C__twenties__1004__C__",
{("a", 5)},
{"__C__twenties__1004__C__", ("__o_00__", 21)},
False
),
(
"'hello my friend' == a + my friend",
"__o_00__ == a + __C__my0friend__1005__C__",
{("a", "hello ")},
{"__C__my0friend__1005__C__", ("__o_00__", 'hello my friend')},
True
),
])
def test_python(self, expression, e_compiled, e_variables, e_objects, e_result):
sheerka, context, foo, one, two, twenties, my_friend = self.init_concepts(
Concept("foo", body="5"),
Concept("one", body="1"),
Concept("two", body="2"),
Concept("twenties", definition="'twenty' (one|two)=n", body='20 + n').def_var("n"),
Concept("my friend", body="'my friend'"),
create_new=True
)
ensure_evaluated(context, foo, eval_body=True)
ensure_evaluated(context, my_friend, eval_body=True)
conditions = self.validate_python_test(context,
expression,
e_compiled,
expression,
e_variables,
set(),
e_objects)
# check against SheerkaEvaluateRules
variables_mapping = {
"foo": foo,
}
objects = self.get_testing_objects(context, e_variables, variables_mapping)
self.check_against_python(context, expression, conditions, objects, expected_result=e_result)
class TestSheerkaRuleManagerRulesCompilationFunctionsCall(BaseTestSheerkaRuleManagerRulesCompilation):
"""
Testing functions
@@ -795,6 +1110,8 @@ class TestSheerkaRuleManagerRulesCompilationEvalQuestionConcept(BaseTestSheerkaR
with long concept : the little boy is a human being
with long concept + variable : the little boy is a self
with long concept + variable : self is a human being
with a special symbol : self is a 'human'
with a special symbol : the little boy is a 'human'
"""
def test_rete(self):
@@ -826,6 +1143,16 @@ class TestSheerkaRuleManagerRulesCompilationEvalQuestionConcept(BaseTestSheerkaR
"evaluate_question(__o_00__)",
{"self"},
),
(
"self is a 'human'",
"evaluate_question(__o_00__)",
{"self"},
),
(
"the little boy is a 'human'",
"evaluate_question(__o_00__)",
set(),
),
])
def test_python(self, expression, expected_compiled, expected_variables):
sheerka, context, girl, human, little_boy, human_being, isa = self.init_test().with_concepts(
@@ -980,11 +1307,11 @@ class TestSheerkaRuleManagerRulesCompilationEvalConceptMixedWithOther(BaseTestSh
expected_objects)
# check against SheerkaEvaluateRules
variable_mapping = {
variables_mapping = {
"girl": girl,
"human being": human_being
}
testing_objects = self.get_testing_objects(context, expected_variables, variable_mapping)
testing_objects = self.get_testing_objects(context, expected_variables, variables_mapping)
self.check_against_python(context,
expression,
conditions,
@@ -1005,6 +1332,8 @@ class TestSheerkaRuleManagerRulesCompilationEvalNonQuestionConcept(BaseTestSheer
with function: func_identity(twenty two + twenty one)
with function: func_identity(twenty two plus one)
with function: func_identity(twenty two plus twenty one)
with special char : 'one' plus 'two'
with special char : twenty two plus 2
"""
def test_rete(self):
@@ -1081,6 +1410,20 @@ class TestSheerkaRuleManagerRulesCompilationEvalNonQuestionConcept(BaseTestSheer
{"__o_00__"},
43
),
(
"'one' plus 'two'",
"__o_00__",
"'one' plus 'two'",
{"__o_00__"},
'onetwo'
),
(
"twenty two plus 2",
"__o_00__",
"twenty two plus 2",
{"__o_00__"},
24
),
])
def test_python(self, expression, e_compiled, e_text, e_objects, e_result):
sheerka, context, one, two, twenties, plus = self.init_test().with_concepts(
@@ -1109,7 +1452,7 @@ class TestSheerkaRuleManagerRulesCompilationEvalNonQuestionConcept(BaseTestSheer
class TestSheerkaRuleManagerRulesCompilationMultipleSameConcept(BaseTestSheerkaRuleManagerRulesCompilation):
"""
Test when a concept returns multiple results
The compilation should fail : No need to execute a condition if we are not sure of the meaning ?
The compilation should fail : No need to execute a condition if we are not sure of the meaning
self is a bar
"""
@@ -1156,6 +1499,99 @@ class TestSheerkaRuleManagerRulesCompilationNot(BaseTestSheerkaRuleManagerRulesC
Testing not
not __ret.status == True
not recognize(__ret.body, hello sheerka)
not a cat is a pet and not bird is an animal # where x is a y is a concept
not a cat is a pet and not x > 5 # concept mixed with python
"""
pass
class TestCompiledCondition(BaseTestSheerkaRuleManagerRulesCompilation):
@pytest.mark.parametrize("expression, expected", [
("self is a 'foo'", {"x is a y"}),
("set self is a 'foo'", set()),
])
def test_i_can_get_concept_to_reset(self, expression, expected):
"""
When compiled conditions, sometimes there are concepts to reset between two usages
:param expression:
:param expected:
:return:
"""
sheerka, context, question, not_a_question = self.init_concepts(
Concept("x is a y", pre="is_question()").def_var("x").def_var("y"),
Concept("set x is a y").def_var("x").def_var("y"),
)
conditions = self.get_conditions(context, expression)
assert len(conditions) == 1
assert set(c.name for c in conditions[0].concepts_to_reset) == expected
def test_i_can_reset_concepts_when_multiple_levels(self):
"""
When compiled conditions, sometimes there are concepts to reset between two usages
:return:
"""
sheerka, context, is_instance, is_int, is_integer = self.init_concepts(
Concept("x is an instance of y", pre="is_question()", body="isinstance(x, y)").def_var("x").def_var("y"),
Concept("x is a int", pre="is_question()", body="x is an instance of int").def_var("x"),
Concept("x is an integer", pre="is_question()", body="x is a int").def_var("x"),
)
expression = "self is an integer"
conditions = conditions = self.get_conditions(context, expression)
assert len(conditions) == 1
assert set(c.name for c in conditions[0].concepts_to_reset) == {"x is an instance of y",
"x is a int",
"x is an integer"}
# So I can evaluate multiple times
res = self.evaluate_condition(context, expression, conditions[0], {'self': 10})
assert res.status
assert sheerka.objvalue(res.body)
res = self.evaluate_condition(context, expression, conditions[0], {'self': "string"})
assert res.status
assert not sheerka.objvalue(res.body)
def test_i_can_reset_concepts_when_multiple_levels_and_concept_node(self):
"""
When compiled conditions, sometimes there are concepts to reset between two usages
:return:
"""
# in this example, x + 2 is an int won't be parsed as an ExactNodeConcept, but as a ConceptNode
sheerka, context, is_int, is_integer = self.init_concepts(
Concept("x is a int", pre="is_question()", body="isinstance(x, int)").def_var("x"),
Concept("x is an integer", pre="is_question()", body="x + 2 is a int").def_var("x"),
create_new=True
)
expression = "self is an integer"
conditions = self.get_conditions(context, expression)
assert len(conditions) == 1
assert set(c.name for c in conditions[0].concepts_to_reset) == {"x is a int",
"x is an integer"}
# So I can evaluate multiple times
res = self.evaluate_condition(context, expression, conditions[0], {'self': 10})
assert res.status
assert sheerka.objvalue(res.body)
res = self.evaluate_condition(context, expression, conditions[0], {'self': "string"})
assert not res.status
def test_long_name_concept_set_are_not_considered_as_variables(self):
sheerka, context, one, number = self.init_concepts(
"one",
"all numbers",
)
sheerka.set_isa(context, one, number)
expression = "all numbers < 5"
conditions = self.get_conditions(context, expression)
assert len(conditions) == 1
assert conditions[0].return_value.body.body.source == '__o_00__ < __o_01__'