Fixed #29: Parsers: Implement parsing memoization
Fixed #77 : Parser: ShortTermMemoryParser should be called separately Fixed #78 : Remove VariableNode usage Fixed #79 : ConceptManager: Implement compile caching Fixed #80 : SheerkaExecute : parsers_key is not correctly computed Fixed #81 : ValidateConceptEvaluator : Validate concept's where and pre clauses right after the parsing Fixed #82 : SheerkaIsAManager: isa() failed when the set as a body Fixed #83 : ValidateConceptEvaluator : Support BNF and SYA Concepts Fixed #84 : ExpressionParser: Implement the parser as a standard parser Fixed #85 : Services: Give order to services Fixed #86 : cannot manage smart_get_attr(the short, color)
This commit is contained in:
@@ -11,53 +11,112 @@ reduced_requested = ReturnValueConcept("Sheerka", True, Concept(name=BuiltinConc
|
||||
|
||||
|
||||
def ret_val(value="value", who="who", status=True):
|
||||
"""
|
||||
ReturnValueConcept
|
||||
:param value:
|
||||
:param who:
|
||||
:param status:
|
||||
:return:
|
||||
"""
|
||||
return ReturnValueConcept(who, status, value)
|
||||
|
||||
|
||||
def p_ret_val(value="value", parser="parser", status=True):
|
||||
"""
|
||||
ReturnValueConcept from parser
|
||||
:param value:
|
||||
:param parser:
|
||||
:param status:
|
||||
:return:
|
||||
"""
|
||||
return ReturnValueConcept(BaseParser.get_name(parser), status, value)
|
||||
|
||||
|
||||
def e_ret_val(value="value", evaluator="evaluator", status=True):
|
||||
"""
|
||||
ReturnValueConcept from evaluator
|
||||
:param value:
|
||||
:param evaluator:
|
||||
:param status:
|
||||
:return:
|
||||
"""
|
||||
return ReturnValueConcept(BaseEvaluator.PREFIX + evaluator, status, value)
|
||||
|
||||
|
||||
def p_ret_val_false(value="value", parser="parser"):
|
||||
"""
|
||||
Failed ReturnValueConcept from parser
|
||||
:param value:
|
||||
:param parser:
|
||||
:return:
|
||||
"""
|
||||
return p_ret_val(value, parser, status=False)
|
||||
|
||||
|
||||
def p_ret_val_true(value="value", parser="parser"):
|
||||
"""
|
||||
Successful ReturnValueConcept from parser
|
||||
:param value:
|
||||
:param parser:
|
||||
:return:
|
||||
"""
|
||||
return p_ret_val(value, parser, status=True)
|
||||
|
||||
|
||||
def e_ret_val_false(value="value", parser="parser"):
|
||||
"""
|
||||
Failed ReturnValueConcept from evaluator
|
||||
:param value:
|
||||
:param parser:
|
||||
:return:
|
||||
"""
|
||||
return e_ret_val(value, parser, status=False)
|
||||
|
||||
|
||||
def e_ret_val_true(value="value", parser="parser"):
|
||||
"""
|
||||
Successful ReturnValueConcept from evaluator
|
||||
:param value:
|
||||
:param parser:
|
||||
:return:
|
||||
"""
|
||||
return e_ret_val(value, parser, status=True)
|
||||
|
||||
|
||||
def e_ret_val_new(key, evaluator="evaluator", status=True, **kwargs):
|
||||
"""
|
||||
Successful ReturnValueConcept from evaluator that returns a concept
|
||||
:param key:
|
||||
:param evaluator:
|
||||
:param status:
|
||||
:param kwargs:
|
||||
:return:
|
||||
"""
|
||||
body = new_concept(key, **kwargs)
|
||||
return e_ret_val(body, evaluator, status)
|
||||
|
||||
|
||||
def pr_ret_val(value, parser="parser", source=None):
|
||||
def pr_ret_val(value, parser="parser", source=None, status=True):
|
||||
"""
|
||||
ParserResult ReturnValue
|
||||
eg: ReturnValue with a ParserResult
|
||||
:param value:
|
||||
:param parser:
|
||||
:param source:
|
||||
:param status:
|
||||
:return:
|
||||
"""
|
||||
source = source or (value.name if isinstance(value, Concept) else "source")
|
||||
parser_result = ParserResultConcept(BaseParser.get_name(parser), source=source, value=value)
|
||||
return p_ret_val(parser_result, parser)
|
||||
return p_ret_val(value=parser_result, parser=parser, status=status)
|
||||
|
||||
|
||||
def python_ret_val(source):
|
||||
"""
|
||||
ReturnValueConcept with a PythonNode
|
||||
:param source:
|
||||
:return:
|
||||
"""
|
||||
python_node = PythonNode(source.strip(), ast.parse(source.strip(), f"<source>", 'eval'))
|
||||
return pr_ret_val(python_node, parser="Python", source=source)
|
||||
|
||||
@@ -68,5 +127,5 @@ def new_concept(key, **kwargs):
|
||||
to_use = "#" + k + "#" if k in ("body", "pre", "post", "ret") else k
|
||||
res.set_value(to_use, v)
|
||||
|
||||
res.get_metadata().is_evaluated = True
|
||||
res.get_hints().is_evaluated = True
|
||||
return res
|
||||
|
||||
@@ -47,7 +47,7 @@ class TestAddConceptEvaluator(TestUsingMemoryBasedSheerka):
|
||||
concept = Concept(name="foo",
|
||||
body="'I have a value'",
|
||||
where="True",
|
||||
pre="2",
|
||||
pre="True",
|
||||
post="3").set_value("a", "4").set_value("b", "5")
|
||||
|
||||
evaluator = ConceptEvaluator(return_body=True)
|
||||
@@ -64,7 +64,7 @@ class TestAddConceptEvaluator(TestUsingMemoryBasedSheerka):
|
||||
concept = Concept(name="foo",
|
||||
body="'I have a value'",
|
||||
where="True",
|
||||
pre="2",
|
||||
pre="True",
|
||||
post="3").set_value("a", "4").set_value("b", "5")
|
||||
|
||||
evaluator = ConceptEvaluator(return_body=False) # which is the default behaviour
|
||||
|
||||
@@ -204,7 +204,9 @@ class TestDefConceptEvaluator(TestUsingMemoryBasedSheerka):
|
||||
assert DefConceptEvaluator.get_variables(context, concept_definition, []) == expected
|
||||
|
||||
def test_i_can_recognize_variables_when_referencing_other_concepts(self):
|
||||
sheerka, context, isa_concept = self.init_concepts(Concept("x is an y").def_var("x").def_var("y"))
|
||||
sheerka, context, isa_concept = self.init_concepts(
|
||||
Concept("x is an y", pre="is_question()").def_var("x").def_var("y")
|
||||
)
|
||||
|
||||
text = "def concept what x is y pre is_question() where x is an adjective as get_attr(x, y)"
|
||||
def_ret_val = DefConceptParser().parse(context, ParserInput(text))
|
||||
|
||||
@@ -0,0 +1,83 @@
|
||||
import pytest
|
||||
|
||||
from core.builtin_concepts_ids import BuiltinConcepts
|
||||
from core.concept import Concept
|
||||
from core.sheerka.services.SheerkaExecute import ParserInput
|
||||
from evaluators.BaseEvaluator import BaseEvaluator
|
||||
from evaluators.ExpressionEvaluator import ExpressionEvaluator
|
||||
from parsers.BaseExpressionParser import AndNode
|
||||
from parsers.ExpressionParser import ExpressionParser
|
||||
from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
|
||||
from tests.evaluators.EvaluatorTestsUtils import pr_ret_val, e_ret_val_true
|
||||
|
||||
|
||||
class TestExpressionEvaluator(TestUsingMemoryBasedSheerka):
|
||||
|
||||
@pytest.mark.parametrize("return_value, expected", [
|
||||
(pr_ret_val(AndNode(0, 0, [])), True),
|
||||
(pr_ret_val(AndNode(0, 0, []), status=False), False),
|
||||
(pr_ret_val("not a ExprNode", status=True), False),
|
||||
(e_ret_val_true("no parser result"), False),
|
||||
])
|
||||
def test_i_can_match(self, return_value, expected):
|
||||
sheerka, context = self.init_concepts()
|
||||
evaluator = ExpressionEvaluator()
|
||||
|
||||
assert evaluator.matches(context, return_value) == expected
|
||||
|
||||
def test_i_can_eval(self):
|
||||
sheerka, context, one, number, isa = self.init_concepts(
|
||||
"one",
|
||||
"number",
|
||||
Concept("x is a y", body="isa(x,y)", pre="is_question()").def_var("x").def_var("y"))
|
||||
evaluator = ExpressionEvaluator()
|
||||
|
||||
context.add_to_protected_hints(BuiltinConcepts.EVAL_QUESTION_REQUESTED)
|
||||
|
||||
parsed_return_value = ExpressionParser().parse(context, ParserInput("one is a number"))
|
||||
|
||||
# first time, it returns False
|
||||
res = evaluator.eval(context, parsed_return_value)
|
||||
assert res.status
|
||||
assert res.who == BaseEvaluator.PREFIX + ExpressionEvaluator.NAME
|
||||
assert not res.body
|
||||
|
||||
# second time
|
||||
sheerka.set_isa(context, one, number)
|
||||
parsed_return_value = ExpressionParser().parse(context, ParserInput("one is a number"))
|
||||
res = evaluator.eval(context, parsed_return_value)
|
||||
assert res.status
|
||||
assert res.body
|
||||
|
||||
def test_i_do_not_mess_up_use_copy(self):
|
||||
sheerka, context, one, number, isa = self.init_concepts(
|
||||
"one",
|
||||
"number",
|
||||
Concept("x is a y", body="isa(x,y)", pre="is_question()").def_var("x").def_var("y"))
|
||||
evaluator = ExpressionEvaluator()
|
||||
|
||||
parsed_return_value = ExpressionParser().parse(context, ParserInput("one is a number"))
|
||||
concept = next(iter(parsed_return_value.body.body.compiled[0].return_value.body.body.objects.values()))
|
||||
assert concept.get_compiled()["x"][0].body.body.get_hints().use_copy
|
||||
assert concept.get_compiled()["y"][0].body.body.get_hints().use_copy
|
||||
|
||||
evaluator.eval(context, parsed_return_value)
|
||||
concept = next(iter(parsed_return_value.body.body.compiled[0].return_value.body.body.objects.values()))
|
||||
assert concept.get_compiled()["x"][0].body.body.get_hints().use_copy
|
||||
assert concept.get_compiled()["y"][0].body.body.get_hints().use_copy
|
||||
|
||||
def test_i_cannot_eval_when_variables_are_missing(self):
|
||||
sheerka, context, one, number, isa = self.init_concepts(
|
||||
"one",
|
||||
"number",
|
||||
Concept("x is a y", body="isa(x,y)", pre="is_question()").def_var("x").def_var("y"))
|
||||
evaluator = ExpressionEvaluator()
|
||||
|
||||
parsed_return_value = ExpressionParser().parse(context, ParserInput("self is a number"))
|
||||
res = evaluator.eval(context, parsed_return_value)
|
||||
|
||||
assert not res.status
|
||||
assert sheerka.isinstance(res.body, BuiltinConcepts.ERROR)
|
||||
assert len(res.body.body) == 1
|
||||
assert isinstance(res.body.body[0], NameError)
|
||||
assert res.body.body[0].args == ("self",)
|
||||
@@ -0,0 +1,198 @@
|
||||
import pytest
|
||||
|
||||
from core.builtin_concepts import ParserResultConcept
|
||||
from core.builtin_concepts_ids import BuiltinConcepts
|
||||
from core.concept import Concept
|
||||
from core.sheerka.services.SheerkaExecute import ParserInput
|
||||
from evaluators.BaseEvaluator import BaseEvaluator
|
||||
from evaluators.ValidateConceptEvaluator import ValidateConceptEvaluator
|
||||
from parsers.BaseNodeParser import ConceptNode
|
||||
from parsers.BaseParser import BaseParser
|
||||
from parsers.BnfNodeParser import BnfNodeParser
|
||||
from parsers.SyaNodeParser import SyaNodeParser
|
||||
from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
|
||||
from tests.evaluators.EvaluatorTestsUtils import p_ret_val, pr_ret_val, ret_val
|
||||
|
||||
|
||||
class TestValidateConceptEvaluator(TestUsingMemoryBasedSheerka):
|
||||
|
||||
@pytest.mark.parametrize("return_value, need_validation, expected", [
|
||||
(pr_ret_val(Concept("foo", where="something"), status=True), True, True),
|
||||
(pr_ret_val(Concept("foo", pre="something"), status=True), True, True),
|
||||
(pr_ret_val(Concept("foo", where="something"), status=True), False, False),
|
||||
(pr_ret_val(Concept("foo", pre="something"), status=True), False, False),
|
||||
(pr_ret_val(Concept("foo"), status=True), True, False),
|
||||
(pr_ret_val(Concept("foo"), status=True), True, False),
|
||||
(pr_ret_val(Concept("foo", where=None), status=True), True, False),
|
||||
(pr_ret_val(Concept("foo", pre=None), status=True), True, False),
|
||||
(pr_ret_val(Concept("foo", where=""), status=True), True, False),
|
||||
(pr_ret_val(Concept("foo", pre=""), status=True), True, False),
|
||||
(pr_ret_val(Concept("foo", where="something"), status=False), True, False),
|
||||
(pr_ret_val(Concept("foo", pre="something"), status=False), True, False),
|
||||
(pr_ret_val("Not a concept", status=True), True, False),
|
||||
(pr_ret_val("Not a concept", status=False), True, False),
|
||||
(p_ret_val("Not a parser result", status=True), True, False),
|
||||
(p_ret_val("Not a parser result", status=False), True, False),
|
||||
])
|
||||
def test_i_can_match(self, return_value, need_validation, expected):
|
||||
sheerka, context = self.init_concepts()
|
||||
evaluator = ValidateConceptEvaluator()
|
||||
if (sheerka.isinstance(return_value.body, BuiltinConcepts.PARSER_RESULT) and
|
||||
isinstance(return_value.body.body, Concept)):
|
||||
return_value.body.body.get_hints().need_validation = need_validation
|
||||
|
||||
assert evaluator.matches(context, return_value) == expected
|
||||
|
||||
def test_i_cannot_match_when_the_return_value_is_not_a_direct_parsing_result(self):
|
||||
# I match only if the return_value comes from a parser (not from an after_parsing evaluator)
|
||||
sheerka, context = self.init_concepts()
|
||||
|
||||
concept = Concept("foo", pre="something")
|
||||
parser_result = ParserResultConcept(BaseParser.get_name("parser"), source=concept.name, value=concept)
|
||||
return_value = ret_val(value=parser_result, who="evaluators.something", status=True)
|
||||
return_value.body.body.get_hints().need_validation = True
|
||||
|
||||
evaluator = ValidateConceptEvaluator()
|
||||
assert not evaluator.matches(context, return_value)
|
||||
|
||||
@pytest.mark.parametrize("concept", [
|
||||
Concept("foo", pre="False"),
|
||||
Concept("foo", where="False"),
|
||||
])
|
||||
def test_i_can_eval_when_constraint_is_false(self, concept):
|
||||
sheerka, context, foo = self.init_concepts(concept)
|
||||
|
||||
ret_val = pr_ret_val(foo, status=True)
|
||||
res = ValidateConceptEvaluator().eval(context, ret_val)
|
||||
|
||||
assert not res.status
|
||||
assert res.who == BaseEvaluator.PREFIX + ValidateConceptEvaluator.NAME
|
||||
assert sheerka.isinstance(res.body, BuiltinConcepts.FILTERED)
|
||||
assert res.body.body == foo
|
||||
|
||||
@pytest.mark.parametrize("concept", [
|
||||
Concept("foo"),
|
||||
Concept("foo", pre="True"),
|
||||
Concept("foo", where="True"),
|
||||
])
|
||||
def test_i_can_eval_when_constraint_is_true_or_when_no_constrain(self, concept):
|
||||
sheerka, context, foo = self.init_concepts(concept)
|
||||
|
||||
ret_val = pr_ret_val(foo, status=True)
|
||||
res = ValidateConceptEvaluator().eval(context, ret_val)
|
||||
|
||||
assert res is None
|
||||
|
||||
@pytest.mark.parametrize("concept", [
|
||||
Concept("foo"),
|
||||
Concept("foo", pre="True"),
|
||||
Concept("foo", where="True"),
|
||||
])
|
||||
def test_i_can_eval_when_constraint_is_true_or_when_no_constrain_use_copy(self, concept):
|
||||
sheerka, context, foo = self.init_concepts(concept)
|
||||
|
||||
foo.get_hints().use_copy = True
|
||||
ret_val = pr_ret_val(foo, status=True)
|
||||
res = ValidateConceptEvaluator().eval(context, ret_val)
|
||||
|
||||
assert res.status
|
||||
assert res.who == BaseEvaluator.PREFIX + ValidateConceptEvaluator.NAME
|
||||
assert res.body.body.id == foo.id
|
||||
assert not res.body.body.get_hints().use_copy
|
||||
|
||||
def test_i_can_eval_when_is_question(self):
|
||||
sheerka, context, foo = self.init_concepts(Concept("foo", pre="is_question()"))
|
||||
|
||||
ret_val = pr_ret_val(foo, status=True)
|
||||
|
||||
res = ValidateConceptEvaluator().eval(context, ret_val)
|
||||
assert not res.status
|
||||
|
||||
context.add_to_protected_hints(BuiltinConcepts.EVAL_QUESTION_REQUESTED)
|
||||
res = ValidateConceptEvaluator().eval(context, ret_val)
|
||||
assert res is None
|
||||
|
||||
def test_i_can_eval_when_unknown_variables_but_no_constraint(self):
|
||||
sheerka, context, foo = self.init_concepts(Concept("a plus b").def_var("a").def_var("b"))
|
||||
|
||||
ret_val = pr_ret_val(foo, status=True)
|
||||
|
||||
res = ValidateConceptEvaluator().eval(context, ret_val)
|
||||
assert res is None
|
||||
|
||||
def test_i_can_eval_when_unknown_variables(self):
|
||||
# ValidateConceptEvaluator must filter only if all the variables are known
|
||||
# In this example, 'b' is not set, so the return_value must not be filtered
|
||||
sheerka, context, foo = self.init_concepts(
|
||||
Concept("a plus b", where="isinstance(b, int)").def_var("a", "10").def_var("b"))
|
||||
|
||||
ret_val = pr_ret_val(foo, status=True)
|
||||
|
||||
res = ValidateConceptEvaluator().eval(context, ret_val)
|
||||
assert res is None
|
||||
|
||||
def test_i_can_eval_when_constraint_on_variable_fails(self):
|
||||
sheerka, context, foo = self.init_concepts(
|
||||
Concept("a plus b", where="isinstance(b, int)").def_var("a").def_var("b", "'a string'"))
|
||||
|
||||
ret_val = pr_ret_val(foo, status=True)
|
||||
|
||||
res = ValidateConceptEvaluator().eval(context, ret_val)
|
||||
assert not res.status
|
||||
|
||||
def test_i_can_eval_bnf_concepts(self):
|
||||
sheerka, context, quantify_x = self.init_concepts(
|
||||
Concept("quantify x", definition="('one'|'two') x", where="x != 'one'").def_var("x"),
|
||||
create_new=True)
|
||||
evaluator = ValidateConceptEvaluator()
|
||||
|
||||
# success
|
||||
ret_val = BnfNodeParser().parse(context, ParserInput("one 'two'"))
|
||||
assert evaluator.matches(context, ret_val)
|
||||
res = evaluator.eval(context, ret_val)
|
||||
assert isinstance(res.body.body, list)
|
||||
assert len(res.body.body) == 1
|
||||
assert isinstance(res.body.body[0], ConceptNode)
|
||||
assert res.body.body[0].concept.id == ret_val.body.body[0].concept.id
|
||||
|
||||
# failure
|
||||
ret_val = BnfNodeParser().parse(context, ParserInput("one 'one'"))
|
||||
assert evaluator.matches(context, ret_val)
|
||||
res = evaluator.eval(context, ret_val)
|
||||
assert not res.status
|
||||
assert sheerka.isinstance(res.body, BuiltinConcepts.FILTERED)
|
||||
assert res.body.body == ret_val.body.body
|
||||
|
||||
def test_i_can_eval_sya_concepts(self):
|
||||
sheerka, context, quantify_x = self.init_concepts(
|
||||
Concept("a plus b", where="a < 10").def_var("a").def_var("b"),
|
||||
create_new=True)
|
||||
evaluator = ValidateConceptEvaluator()
|
||||
|
||||
# success
|
||||
ret_val = SyaNodeParser().parse(context, ParserInput("5 plus 3"))
|
||||
assert evaluator.matches(context, ret_val)
|
||||
res = evaluator.eval(context, ret_val)
|
||||
assert isinstance(res.body.body, list)
|
||||
assert len(res.body.body) == 1
|
||||
assert isinstance(res.body.body[0], ConceptNode)
|
||||
assert res.body.body[0].concept.id == ret_val.body.body[0].concept.id
|
||||
|
||||
# failure
|
||||
ret_val = SyaNodeParser().parse(context, ParserInput("15 plus 3"))
|
||||
assert evaluator.matches(context, ret_val)
|
||||
res = evaluator.eval(context, ret_val)
|
||||
assert not res.status
|
||||
assert sheerka.isinstance(res.body, BuiltinConcepts.FILTERED)
|
||||
assert res.body.body == ret_val.body.body
|
||||
|
||||
def test_i_can_manage_infinite_recursion(self):
|
||||
sheerka, context, a_and_b = self.init_concepts(
|
||||
Concept("a and b", where="is_question()", body="a and b").def_var("a").def_var("b"),
|
||||
create_new=True)
|
||||
evaluator = ValidateConceptEvaluator()
|
||||
|
||||
ret_val = pr_ret_val(a_and_b)
|
||||
res = evaluator.eval(context, ret_val)
|
||||
|
||||
assert res is None # infinite recursion detected, res is None to drop the validator
|
||||
Reference in New Issue
Block a user