kodjo/Sheerka-Old
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added mandatory evaluator steps

Browse Source
This commit is contained in:
Kodjo Sossouvi
2019-12-22 19:51:17 +01:00
parent a683d4cd42
commit 5c90b07e1a
13 changed files with 210 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_sheerka_call_evaluators.py
+180 -35
View File
@@ -1,8 +1,8 @@
# Make sure that the evaluators works as expected
from core.builtin_concepts import BuiltinConcepts
from core.builtin_concepts import BuiltinConcepts, SuccessConcept
from core.concept import Concept
from core.sheerka import Sheerka, ExecutionContext
from evaluators.BaseEvaluator import OneReturnValueEvaluator, BaseEvaluator
from evaluators.BaseEvaluator import OneReturnValueEvaluator, BaseEvaluator, AllReturnValuesEvaluator
def get_sheerka():
@@ -22,44 +22,94 @@ def get_ret_val(sheerka, concept, who="who"):
return sheerka.ret(who, True, sheerka.new(concept.key))
class EvaluatorWithPriority(OneReturnValueEvaluator):
out = []
class Out:
debug_out = []
def __init__(self, name, priority):
super().__init__(name, priority)
def matches(self, context, return_value):
target = str(return_value.body.key)
def out(self, method, name, context, return_value):
name = name[len(BaseEvaluator.PREFIX):]
if isinstance(return_value, list):
target = [str(r.body.key) for r in return_value]
else:
target = str(return_value.body.key)
step = str(context.step)
text = f"{step} [{context.iteration}] "
text += f"{self.name[len(BaseEvaluator.PREFIX):]} - matches - target={target}"
self.out.append(text)
text += f"{name} - {method} - target={target}"
self.debug_out.append(text)
def out_all(self, method, name, context, return_values):
name = name[len(BaseEvaluator.PREFIX):]
target = [str(r.body.key) for r in return_values]
step = str(context.step)
text = f"{step} [{context.iteration}] "
text += f"{name} - {method} - target={target}"
self.debug_out.append(text)
class OneReturnValueEvaluatorForTestingPurpose(OneReturnValueEvaluator, Out):
def __init__(self, name, steps, priority):
super().__init__(name, steps, priority)
def matches(self, context, return_value):
self.out("matches", self.name, context, return_value)
return True
def eval(self, context, return_value):
target = str(return_value.body.key)
step = str(context.step)
text = f"{step} [{context.iteration}] "
text += f"{self.name[len(BaseEvaluator.PREFIX):]} - eval - target={target}"
self.out.append(text)
self.out("eval", self.name, context, return_value)
class EvaluatorWithPriority10(EvaluatorWithPriority):
class AllReturnValueEvaluatorForTestingPurpose(AllReturnValuesEvaluator, Out):
def __init__(self, name, steps, priority):
super().__init__(name, steps, priority)
def matches(self, context, return_values):
self.out("matches", self.name, context, return_values)
return True
def eval(self, context, return_values):
self.out("eval", self.name, context, return_values)
class EvaluatorOneWithPriority(OneReturnValueEvaluatorForTestingPurpose):
def __init__(self, name, priority):
super().__init__(name, [BuiltinConcepts.EVALUATION], priority)
class EvaluatorOneWithPriority10(EvaluatorOneWithPriority):
def __init__(self):
super().__init__("priority10", 10)
class EvaluatorWithPriority15(EvaluatorWithPriority):
class EvaluatorOneWithPriority15(EvaluatorOneWithPriority):
def __init__(self):
super().__init__("priority15", 15)
class EvaluatorWithPriority20(EvaluatorWithPriority):
class EvaluatorOneWithPriority20(EvaluatorOneWithPriority):
def __init__(self):
super().__init__("priority20", 20)
class EvaluatorModifyFoo(EvaluatorWithPriority):
class EvaluatorAllWithPriority(AllReturnValueEvaluatorForTestingPurpose):
def __init__(self, name, priority):
super().__init__(name, [BuiltinConcepts.EVALUATION], priority)
class EvaluatorAllWithPriority10(EvaluatorAllWithPriority):
def __init__(self):
super().__init__("all_priority10", 10)
class EvaluatorAllWithPriority15(EvaluatorAllWithPriority):
def __init__(self):
super().__init__("all_priority15", 15)
class EvaluatorAllWithPriority20(EvaluatorAllWithPriority):
def __init__(self):
super().__init__("all_priority20", 20)
class EvaluatorOneModifyFoo(EvaluatorOneWithPriority):
def __init__(self):
super().__init__("modifyFoo", 10)
@@ -72,7 +122,7 @@ class EvaluatorModifyFoo(EvaluatorWithPriority):
return get_ret_val(context.sheerka, Concept("bar"))
class EvaluatorModifyBar(EvaluatorWithPriority):
class EvaluatorOneModifyBar(EvaluatorOneWithPriority):
def __init__(self):
super().__init__("modifyBar", 10)
@@ -85,6 +135,32 @@ class EvaluatorModifyBar(EvaluatorWithPriority):
return get_ret_val(context.sheerka, Concept("baz"))
class EvaluatorOnePreEvaluation(OneReturnValueEvaluatorForTestingPurpose):
def __init__(self):
super().__init__("preEval", [BuiltinConcepts.BEFORE_EVALUATION], 10)
class EvaluatorOneMultiSteps(OneReturnValueEvaluatorForTestingPurpose):
def __init__(self):
super().__init__("multiStep", [BuiltinConcepts.BEFORE_EVALUATION, BuiltinConcepts.EVALUATION], 10)
class EvaluatorAllReduceFooBar(EvaluatorAllWithPriority):
def __init__(self):
super().__init__("all_reduce_foobar", 10)
def matches(self, context, return_values):
super().matches(context, return_values)
keys = [c.body.key for c in return_values]
return "foo" in keys and "bar" in keys
def eval(self, context, return_values):
super().eval(context, return_values)
ret = get_ret_val(context.sheerka, SuccessConcept())
ret.parents = return_values
return ret
def test_that_return_values_is_unchanged_when_no_evaluator():
sheerka = get_sheerka()
sheerka.evaluators = []
@@ -107,10 +183,10 @@ def test_i_can_use_a_list_as_input():
def test_step_concept_is_removed_after_processing_if_not_reduced():
"""
The entry is not modified by an evaluator
No evaluator
"""
sheerka = get_sheerka()
sheerka.evaluators = [EvaluatorWithPriority10]
sheerka.evaluators = []
entry = get_ret_val(sheerka, Concept("foo"))
return_values = sheerka.execute(get_context(sheerka), entry, [BuiltinConcepts.EVALUATION])
@@ -124,7 +200,7 @@ def test_step_concept_is_removed_after_processing_if_not_reduced_2():
nevertheless, step concept is removed
"""
sheerka = get_sheerka()
sheerka.evaluators = [EvaluatorModifyFoo]
sheerka.evaluators = [EvaluatorOneModifyFoo]
entry = get_ret_val(sheerka, Concept("foo"))
return_values = sheerka.execute(get_context(sheerka), entry, [BuiltinConcepts.EVALUATION])
@@ -134,36 +210,49 @@ def test_step_concept_is_removed_after_processing_if_not_reduced_2():
def test_that_higher_priority_evaluators_are_evaluated_first():
sheerka = get_sheerka()
sheerka.evaluators = [EvaluatorWithPriority20, EvaluatorWithPriority10, EvaluatorWithPriority15]
sheerka.evaluators = [
EvaluatorAllWithPriority10,
EvaluatorOneWithPriority20,
EvaluatorAllWithPriority15,
EvaluatorOneWithPriority10,
EvaluatorOneWithPriority15,
EvaluatorAllWithPriority20, ]
entries = [get_ret_val(sheerka, Concept("foo"))]
EvaluatorWithPriority.out = []
Out.debug_out = []
sheerka.execute(get_context(sheerka), entries, [BuiltinConcepts.EVALUATION])
assert EvaluatorWithPriority.out == [
assert Out.debug_out == [
'__EVALUATION [0] priority20 - matches - target=foo',
'__EVALUATION [0] priority20 - eval - target=foo',
'__EVALUATION [0] priority20 - matches - target=__EVALUATION',
'__EVALUATION [0] priority20 - eval - target=__EVALUATION',
"__EVALUATION [0] all_priority20 - matches - target=['foo', '__EVALUATION']",
"__EVALUATION [0] all_priority20 - eval - target=['foo', '__EVALUATION']",
"__EVALUATION [0] all_priority15 - matches - target=['foo', '__EVALUATION']",
"__EVALUATION [0] all_priority15 - eval - target=['foo', '__EVALUATION']",
'__EVALUATION [0] priority15 - matches - target=foo',
'__EVALUATION [0] priority15 - eval - target=foo',
'__EVALUATION [0] priority15 - matches - target=__EVALUATION',
'__EVALUATION [0] priority15 - eval - target=__EVALUATION',
"__EVALUATION [0] all_priority10 - matches - target=['foo', '__EVALUATION']",
"__EVALUATION [0] all_priority10 - eval - target=['foo', '__EVALUATION']",
'__EVALUATION [0] priority10 - matches - target=foo',
'__EVALUATION [0] priority10 - eval - target=foo',
'__EVALUATION [0] priority10 - matches - target=__EVALUATION',
'__EVALUATION [0] priority10 - eval - target=__EVALUATION']
'__EVALUATION [0] priority10 - eval - target=__EVALUATION'
]
def test_that_predicate_is_checked_before_evaluation():
def test_that_predicate_is_checked_before_evaluation_for_one_return():
sheerka = get_sheerka()
sheerka.evaluators = [EvaluatorModifyFoo]
sheerka.evaluators = [EvaluatorOneModifyFoo]
entries = [get_ret_val(sheerka, Concept("foo")), get_ret_val(sheerka, Concept("baz"))]
EvaluatorWithPriority.out = []
Out.debug_out = []
sheerka.execute(get_context(sheerka), entries, [BuiltinConcepts.EVALUATION])
assert EvaluatorWithPriority.out == [
assert Out.debug_out == [
'__EVALUATION [0] modifyFoo - matches - target=foo',
'__EVALUATION [0] modifyFoo - eval - target=foo',
'__EVALUATION [0] modifyFoo - matches - target=baz',
@@ -174,15 +263,36 @@ def test_that_predicate_is_checked_before_evaluation():
]
def test_that_predicate_is_checked_before_evaluation_for_all_return():
sheerka = get_sheerka()
sheerka.evaluators = [EvaluatorAllReduceFooBar]
entries = [get_ret_val(sheerka, Concept("foo")), get_ret_val(sheerka, Concept("bar"))]
Out.debug_out = []
sheerka.execute(get_context(sheerka), entries, [BuiltinConcepts.EVALUATION])
assert Out.debug_out == [
"__EVALUATION [0] all_reduce_foobar - matches - target=['foo', 'bar', '__EVALUATION']",
"__EVALUATION [0] all_reduce_foobar - eval - target=['foo', 'bar', '__EVALUATION']",
"__EVALUATION [1] all_reduce_foobar - matches - target=['__SUCCESS']"
]
entries = [get_ret_val(sheerka, Concept("foo"))]
Out.debug_out = []
sheerka.execute(get_context(sheerka), entries, [BuiltinConcepts.EVALUATION])
assert Out.debug_out == [
"__EVALUATION [0] all_reduce_foobar - matches - target=['foo', '__EVALUATION']"
]
def test_evaluation_continue_until_no_more_modification():
sheerka = get_sheerka()
sheerka.evaluators = [EvaluatorModifyFoo, EvaluatorModifyBar]
sheerka.evaluators = [EvaluatorOneModifyFoo, EvaluatorOneModifyBar]
entries = [get_ret_val(sheerka, Concept("foo")), get_ret_val(sheerka, Concept("baz"))]
EvaluatorWithPriority.out = []
Out.debug_out = []
sheerka.execute(get_context(sheerka), entries, [BuiltinConcepts.EVALUATION])
assert EvaluatorWithPriority.out == [
assert Out.debug_out == [
'__EVALUATION [0] modifyFoo - matches - target=foo',
'__EVALUATION [0] modifyFoo - eval - target=foo',
'__EVALUATION [0] modifyFoo - matches - target=baz',
@@ -204,3 +314,38 @@ def test_evaluation_continue_until_no_more_modification():
'__EVALUATION [2] modifyBar - matches - target=baz',
'__EVALUATION [2] modifyBar - matches - target=__EVALUATION'
]
def test_evaluation_steps_are_respected():
sheerka = get_sheerka()
sheerka.evaluators = [EvaluatorOneWithPriority10, EvaluatorOnePreEvaluation]
entries = [get_ret_val(sheerka, Concept("foo"))]
BaseEvaluator.debug_out = []
sheerka.execute(get_context(sheerka), entries, [BuiltinConcepts.BEFORE_EVALUATION])
assert BaseEvaluator.debug_out == [
'__BEFORE_EVALUATION [0] preEval - matches - target=foo',
'__BEFORE_EVALUATION [0] preEval - eval - target=foo',
'__BEFORE_EVALUATION [0] preEval - matches - target=__BEFORE_EVALUATION',
'__BEFORE_EVALUATION [0] preEval - eval - target=__BEFORE_EVALUATION']
def test_evaluation_multi_steps_are_respected():
sheerka = get_sheerka()
sheerka.evaluators = [EvaluatorOneMultiSteps]
entries = [get_ret_val(sheerka, Concept("foo"))]
BaseEvaluator.debug_out = []
sheerka.execute(get_context(sheerka), entries, [BuiltinConcepts.BEFORE_EVALUATION, BuiltinConcepts.EVALUATION])
assert BaseEvaluator.debug_out == [
'__BEFORE_EVALUATION [0] multiStep - matches - target=foo',
'__BEFORE_EVALUATION [0] multiStep - eval - target=foo',
'__BEFORE_EVALUATION [0] multiStep - matches - target=__BEFORE_EVALUATION',
'__BEFORE_EVALUATION [0] multiStep - eval - target=__BEFORE_EVALUATION',
'__EVALUATION [0] multiStep - matches - target=foo',
'__EVALUATION [0] multiStep - eval - target=foo',
'__EVALUATION [0] multiStep - matches - target=__EVALUATION',
'__EVALUATION [0] multiStep - eval - target=__EVALUATION'
]
tests/test_sheerka_non_reg.py
+15 -15
View File
@@ -207,21 +207,21 @@ as:
assert sheerka.isinstance(res[0].value, BuiltinConcepts.CONCEPT_ALREADY_DEFINED)
def test_i_can_disable_an_evaluator():
sheerka = get_sheerka()
concept = Concept(name="one", body="1")
sheerka.add_in_cache(concept)
text = "one"
p = next(e for e in sheerka.evaluators if e.__name__ == "PythonEvaluator")
p.enabled = False # not that you disable the class, not the instance
res = sheerka.evaluate_user_input(text)
assert len(res) == 1
assert res[0].status
assert sheerka.isinstance(res[0].value, BuiltinConcepts.PARSER_RESULT)
p.enabled = True # put back for the remaining unit tests
# def test_i_can_disable_an_evaluator():
# sheerka = get_sheerka()
# concept = Concept(name="one", body="1")
# sheerka.add_in_cache(concept)
#
# text = "one"
# p = next(e for e in sheerka.evaluators if e.__name__ == "PythonEvaluator")
# p.enabled = False # not that you disable the class, not the instance
#
# res = sheerka.evaluate_user_input(text)
# assert len(res) == 1
# assert res[0].status
# assert sheerka.isinstance(res[0].value, BuiltinConcepts.PARSER_RESULT)
#
# p.enabled = True # put back for the remaining unit tests
@pytest.mark.parametrize("text", [