352 lines
13 KiB
Python
352 lines
13 KiB
Python
# Make sure that the evaluators works as expected
|
|
from core.builtin_concepts import BuiltinConcepts, SuccessConcept
|
|
from core.concept import Concept
|
|
from core.sheerka import Sheerka, ExecutionContext
|
|
from evaluators.BaseEvaluator import OneReturnValueEvaluator, BaseEvaluator, AllReturnValuesEvaluator
|
|
|
|
|
|
def get_sheerka():
|
|
sheerka = Sheerka()
|
|
sheerka.initialize("mem://")
|
|
return sheerka
|
|
|
|
|
|
def get_context(sheerka):
|
|
return ExecutionContext("test", "xxx", sheerka)
|
|
|
|
|
|
def get_ret_val(sheerka, concept, who="who"):
|
|
concept.init_key()
|
|
if concept.key not in sheerka.concepts_cache:
|
|
sheerka.concepts_cache[concept.key] = concept
|
|
return sheerka.ret(who, True, sheerka.new(concept.key))
|
|
|
|
|
|
class Out:
|
|
debug_out = []
|
|
|
|
def out(self, method, name, context, return_value):
|
|
name = name[len(BaseEvaluator.PREFIX):]
|
|
if isinstance(return_value, list):
|
|
target = [str(r.body.key) for r in return_value]
|
|
else:
|
|
target = str(return_value.body.key)
|
|
step = str(context.step)
|
|
text = f"{step} [{context.iteration}] "
|
|
text += f"{name} - {method} - target={target}"
|
|
self.debug_out.append(text)
|
|
|
|
def out_all(self, method, name, context, return_values):
|
|
name = name[len(BaseEvaluator.PREFIX):]
|
|
target = [str(r.body.key) for r in return_values]
|
|
step = str(context.step)
|
|
text = f"{step} [{context.iteration}] "
|
|
text += f"{name} - {method} - target={target}"
|
|
self.debug_out.append(text)
|
|
|
|
|
|
class OneReturnValueEvaluatorForTestingPurpose(OneReturnValueEvaluator, Out):
|
|
def __init__(self, name, steps, priority):
|
|
super().__init__(name, steps, priority)
|
|
|
|
def matches(self, context, return_value):
|
|
self.out("matches", self.name, context, return_value)
|
|
return True
|
|
|
|
def eval(self, context, return_value):
|
|
self.out("eval", self.name, context, return_value)
|
|
|
|
|
|
class AllReturnValueEvaluatorForTestingPurpose(AllReturnValuesEvaluator, Out):
|
|
def __init__(self, name, steps, priority):
|
|
super().__init__(name, steps, priority)
|
|
|
|
def matches(self, context, return_values):
|
|
self.out("matches", self.name, context, return_values)
|
|
return True
|
|
|
|
def eval(self, context, return_values):
|
|
self.out("eval", self.name, context, return_values)
|
|
|
|
|
|
class EvaluatorOneWithPriority(OneReturnValueEvaluatorForTestingPurpose):
|
|
def __init__(self, name, priority):
|
|
super().__init__(name, [BuiltinConcepts.EVALUATION], priority)
|
|
|
|
|
|
class EvaluatorOneWithPriority10(EvaluatorOneWithPriority):
|
|
def __init__(self):
|
|
super().__init__("priority10", 10)
|
|
|
|
|
|
class EvaluatorOneWithPriority15(EvaluatorOneWithPriority):
|
|
def __init__(self):
|
|
super().__init__("priority15", 15)
|
|
|
|
|
|
class EvaluatorOneWithPriority20(EvaluatorOneWithPriority):
|
|
def __init__(self):
|
|
super().__init__("priority20", 20)
|
|
|
|
|
|
class EvaluatorAllWithPriority(AllReturnValueEvaluatorForTestingPurpose):
|
|
def __init__(self, name, priority):
|
|
super().__init__(name, [BuiltinConcepts.EVALUATION], priority)
|
|
|
|
|
|
class EvaluatorAllWithPriority10(EvaluatorAllWithPriority):
|
|
def __init__(self):
|
|
super().__init__("all_priority10", 10)
|
|
|
|
|
|
class EvaluatorAllWithPriority15(EvaluatorAllWithPriority):
|
|
def __init__(self):
|
|
super().__init__("all_priority15", 15)
|
|
|
|
|
|
class EvaluatorAllWithPriority20(EvaluatorAllWithPriority):
|
|
def __init__(self):
|
|
super().__init__("all_priority20", 20)
|
|
|
|
|
|
class EvaluatorOneModifyFoo(EvaluatorOneWithPriority):
|
|
def __init__(self):
|
|
super().__init__("modifyFoo", 10)
|
|
|
|
def matches(self, context, return_value):
|
|
super().matches(context, return_value)
|
|
return context.sheerka.isinstance(return_value.body, "foo")
|
|
|
|
def eval(self, context, return_value):
|
|
super().eval(context, return_value)
|
|
return get_ret_val(context.sheerka, Concept("bar"))
|
|
|
|
|
|
class EvaluatorOneModifyBar(EvaluatorOneWithPriority):
|
|
def __init__(self):
|
|
super().__init__("modifyBar", 10)
|
|
|
|
def matches(self, context, return_value):
|
|
super().matches(context, return_value)
|
|
return context.sheerka.isinstance(return_value.body, "bar")
|
|
|
|
def eval(self, context, return_value):
|
|
super().eval(context, return_value)
|
|
return get_ret_val(context.sheerka, Concept("baz"))
|
|
|
|
|
|
class EvaluatorOnePreEvaluation(OneReturnValueEvaluatorForTestingPurpose):
|
|
def __init__(self):
|
|
super().__init__("preEval", [BuiltinConcepts.BEFORE_EVALUATION], 10)
|
|
|
|
|
|
class EvaluatorOneMultiSteps(OneReturnValueEvaluatorForTestingPurpose):
|
|
def __init__(self):
|
|
super().__init__("multiStep", [BuiltinConcepts.BEFORE_EVALUATION, BuiltinConcepts.EVALUATION], 10)
|
|
|
|
|
|
class EvaluatorAllReduceFooBar(EvaluatorAllWithPriority):
|
|
def __init__(self):
|
|
super().__init__("all_reduce_foobar", 10)
|
|
|
|
def matches(self, context, return_values):
|
|
super().matches(context, return_values)
|
|
keys = [c.body.key for c in return_values]
|
|
return "foo" in keys and "bar" in keys
|
|
|
|
def eval(self, context, return_values):
|
|
super().eval(context, return_values)
|
|
ret = get_ret_val(context.sheerka, SuccessConcept())
|
|
ret.parents = return_values
|
|
return ret
|
|
|
|
|
|
def test_that_return_values_is_unchanged_when_no_evaluator():
|
|
sheerka = get_sheerka()
|
|
sheerka.evaluators = []
|
|
|
|
entries = get_ret_val(sheerka, Concept("foo"))
|
|
return_values = sheerka.execute(get_context(sheerka), entries, [BuiltinConcepts.EVALUATION])
|
|
|
|
assert return_values == [entries]
|
|
|
|
|
|
def test_i_can_use_a_list_as_input():
|
|
sheerka = get_sheerka()
|
|
sheerka.evaluators = []
|
|
|
|
entries = [get_ret_val(sheerka, Concept("foo"))]
|
|
return_values = sheerka.execute(get_context(sheerka), entries, [BuiltinConcepts.EVALUATION])
|
|
|
|
assert return_values == entries
|
|
|
|
|
|
def test_step_concept_is_removed_after_processing_if_not_reduced():
|
|
"""
|
|
No evaluator
|
|
"""
|
|
sheerka = get_sheerka()
|
|
sheerka.evaluators = []
|
|
|
|
entry = get_ret_val(sheerka, Concept("foo"))
|
|
return_values = sheerka.execute(get_context(sheerka), entry, [BuiltinConcepts.EVALUATION])
|
|
|
|
assert BuiltinConcepts.EVALUATION not in [r.body.key for r in return_values]
|
|
|
|
|
|
def test_step_concept_is_removed_after_processing_if_not_reduced_2():
|
|
"""
|
|
This time the entry is modified by an evaluator,
|
|
nevertheless, step concept is removed
|
|
"""
|
|
sheerka = get_sheerka()
|
|
sheerka.evaluators = [EvaluatorOneModifyFoo]
|
|
|
|
entry = get_ret_val(sheerka, Concept("foo"))
|
|
return_values = sheerka.execute(get_context(sheerka), entry, [BuiltinConcepts.EVALUATION])
|
|
|
|
assert BuiltinConcepts.EVALUATION not in [r.body.key for r in return_values]
|
|
|
|
|
|
def test_that_higher_priority_evaluators_are_evaluated_first():
|
|
sheerka = get_sheerka()
|
|
sheerka.evaluators = [
|
|
EvaluatorAllWithPriority10,
|
|
EvaluatorOneWithPriority20,
|
|
EvaluatorAllWithPriority15,
|
|
EvaluatorOneWithPriority10,
|
|
EvaluatorOneWithPriority15,
|
|
EvaluatorAllWithPriority20, ]
|
|
|
|
entries = [get_ret_val(sheerka, Concept("foo"))]
|
|
Out.debug_out = []
|
|
sheerka.execute(get_context(sheerka), entries, [BuiltinConcepts.EVALUATION])
|
|
|
|
assert Out.debug_out == [
|
|
'__EVALUATION [0] priority20 - matches - target=foo',
|
|
'__EVALUATION [0] priority20 - eval - target=foo',
|
|
'__EVALUATION [0] priority20 - matches - target=__EVALUATION',
|
|
'__EVALUATION [0] priority20 - eval - target=__EVALUATION',
|
|
"__EVALUATION [0] all_priority20 - matches - target=['foo', '__EVALUATION']",
|
|
"__EVALUATION [0] all_priority20 - eval - target=['foo', '__EVALUATION']",
|
|
"__EVALUATION [0] all_priority15 - matches - target=['foo', '__EVALUATION']",
|
|
"__EVALUATION [0] all_priority15 - eval - target=['foo', '__EVALUATION']",
|
|
'__EVALUATION [0] priority15 - matches - target=foo',
|
|
'__EVALUATION [0] priority15 - eval - target=foo',
|
|
'__EVALUATION [0] priority15 - matches - target=__EVALUATION',
|
|
'__EVALUATION [0] priority15 - eval - target=__EVALUATION',
|
|
"__EVALUATION [0] all_priority10 - matches - target=['foo', '__EVALUATION']",
|
|
"__EVALUATION [0] all_priority10 - eval - target=['foo', '__EVALUATION']",
|
|
'__EVALUATION [0] priority10 - matches - target=foo',
|
|
'__EVALUATION [0] priority10 - eval - target=foo',
|
|
'__EVALUATION [0] priority10 - matches - target=__EVALUATION',
|
|
'__EVALUATION [0] priority10 - eval - target=__EVALUATION'
|
|
]
|
|
|
|
|
|
def test_that_predicate_is_checked_before_evaluation_for_one_return():
|
|
sheerka = get_sheerka()
|
|
sheerka.evaluators = [EvaluatorOneModifyFoo]
|
|
|
|
entries = [get_ret_val(sheerka, Concept("foo")), get_ret_val(sheerka, Concept("baz"))]
|
|
Out.debug_out = []
|
|
sheerka.execute(get_context(sheerka), entries, [BuiltinConcepts.EVALUATION])
|
|
|
|
assert Out.debug_out == [
|
|
'__EVALUATION [0] modifyFoo - matches - target=foo',
|
|
'__EVALUATION [0] modifyFoo - eval - target=foo',
|
|
'__EVALUATION [0] modifyFoo - matches - target=baz',
|
|
'__EVALUATION [0] modifyFoo - matches - target=__EVALUATION',
|
|
'__EVALUATION [1] modifyFoo - matches - target=bar',
|
|
'__EVALUATION [1] modifyFoo - matches - target=baz',
|
|
'__EVALUATION [1] modifyFoo - matches - target=__EVALUATION'
|
|
]
|
|
|
|
|
|
def test_that_predicate_is_checked_before_evaluation_for_all_return():
|
|
sheerka = get_sheerka()
|
|
sheerka.evaluators = [EvaluatorAllReduceFooBar]
|
|
|
|
entries = [get_ret_val(sheerka, Concept("foo")), get_ret_val(sheerka, Concept("bar"))]
|
|
Out.debug_out = []
|
|
sheerka.execute(get_context(sheerka), entries, [BuiltinConcepts.EVALUATION])
|
|
assert Out.debug_out == [
|
|
"__EVALUATION [0] all_reduce_foobar - matches - target=['foo', 'bar', '__EVALUATION']",
|
|
"__EVALUATION [0] all_reduce_foobar - eval - target=['foo', 'bar', '__EVALUATION']",
|
|
"__EVALUATION [1] all_reduce_foobar - matches - target=['__SUCCESS']"
|
|
]
|
|
|
|
entries = [get_ret_val(sheerka, Concept("foo"))]
|
|
Out.debug_out = []
|
|
sheerka.execute(get_context(sheerka), entries, [BuiltinConcepts.EVALUATION])
|
|
assert Out.debug_out == [
|
|
"__EVALUATION [0] all_reduce_foobar - matches - target=['foo', '__EVALUATION']"
|
|
]
|
|
|
|
|
|
def test_evaluation_continue_until_no_more_modification():
|
|
sheerka = get_sheerka()
|
|
sheerka.evaluators = [EvaluatorOneModifyFoo, EvaluatorOneModifyBar]
|
|
|
|
entries = [get_ret_val(sheerka, Concept("foo")), get_ret_val(sheerka, Concept("baz"))]
|
|
Out.debug_out = []
|
|
sheerka.execute(get_context(sheerka), entries, [BuiltinConcepts.EVALUATION])
|
|
|
|
assert Out.debug_out == [
|
|
'__EVALUATION [0] modifyFoo - matches - target=foo',
|
|
'__EVALUATION [0] modifyFoo - eval - target=foo',
|
|
'__EVALUATION [0] modifyFoo - matches - target=baz',
|
|
'__EVALUATION [0] modifyFoo - matches - target=__EVALUATION',
|
|
'__EVALUATION [0] modifyBar - matches - target=foo',
|
|
'__EVALUATION [0] modifyBar - matches - target=baz',
|
|
'__EVALUATION [0] modifyBar - matches - target=__EVALUATION',
|
|
'__EVALUATION [1] modifyFoo - matches - target=bar',
|
|
'__EVALUATION [1] modifyFoo - matches - target=baz',
|
|
'__EVALUATION [1] modifyFoo - matches - target=__EVALUATION',
|
|
'__EVALUATION [1] modifyBar - matches - target=bar',
|
|
'__EVALUATION [1] modifyBar - eval - target=bar',
|
|
'__EVALUATION [1] modifyBar - matches - target=baz',
|
|
'__EVALUATION [1] modifyBar - matches - target=__EVALUATION',
|
|
'__EVALUATION [2] modifyFoo - matches - target=baz',
|
|
'__EVALUATION [2] modifyFoo - matches - target=baz',
|
|
'__EVALUATION [2] modifyFoo - matches - target=__EVALUATION',
|
|
'__EVALUATION [2] modifyBar - matches - target=baz',
|
|
'__EVALUATION [2] modifyBar - matches - target=baz',
|
|
'__EVALUATION [2] modifyBar - matches - target=__EVALUATION'
|
|
]
|
|
|
|
|
|
def test_evaluation_steps_are_respected():
|
|
sheerka = get_sheerka()
|
|
sheerka.evaluators = [EvaluatorOneWithPriority10, EvaluatorOnePreEvaluation]
|
|
|
|
entries = [get_ret_val(sheerka, Concept("foo"))]
|
|
BaseEvaluator.debug_out = []
|
|
sheerka.execute(get_context(sheerka), entries, [BuiltinConcepts.BEFORE_EVALUATION])
|
|
|
|
assert BaseEvaluator.debug_out == [
|
|
'__BEFORE_EVALUATION [0] preEval - matches - target=foo',
|
|
'__BEFORE_EVALUATION [0] preEval - eval - target=foo',
|
|
'__BEFORE_EVALUATION [0] preEval - matches - target=__BEFORE_EVALUATION',
|
|
'__BEFORE_EVALUATION [0] preEval - eval - target=__BEFORE_EVALUATION']
|
|
|
|
|
|
def test_evaluation_multi_steps_are_respected():
|
|
sheerka = get_sheerka()
|
|
sheerka.evaluators = [EvaluatorOneMultiSteps]
|
|
|
|
entries = [get_ret_val(sheerka, Concept("foo"))]
|
|
BaseEvaluator.debug_out = []
|
|
sheerka.execute(get_context(sheerka), entries, [BuiltinConcepts.BEFORE_EVALUATION, BuiltinConcepts.EVALUATION])
|
|
|
|
assert BaseEvaluator.debug_out == [
|
|
'__BEFORE_EVALUATION [0] multiStep - matches - target=foo',
|
|
'__BEFORE_EVALUATION [0] multiStep - eval - target=foo',
|
|
'__BEFORE_EVALUATION [0] multiStep - matches - target=__BEFORE_EVALUATION',
|
|
'__BEFORE_EVALUATION [0] multiStep - eval - target=__BEFORE_EVALUATION',
|
|
'__EVALUATION [0] multiStep - matches - target=foo',
|
|
'__EVALUATION [0] multiStep - eval - target=foo',
|
|
'__EVALUATION [0] multiStep - matches - target=__EVALUATION',
|
|
'__EVALUATION [0] multiStep - eval - target=__EVALUATION'
|
|
]
|