kodjo/Sheerka-Old
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fixed #51 : I can compile simple recognize

Browse Source
This commit is contained in:
Kodjo Sossouvi
2021-03-19 18:06:29 +01:00
parent 36e3ce0bcb
commit 88c96ee9a8
5 changed files with 376 additions and 76 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/core/sheerka/services/SheerkaEvaluateRules.py
+1 -1
View File
@@ -159,7 +159,7 @@ class SheerkaEvaluateRules(BaseService):
if compiled_condition.evaluator_type == ConceptEvaluator.NAME:
compiled_condition.concept.get_metadata().is_evaluated = False
evaluator = self.evaluators_by_name[compiled_condition.evaluator]
evaluator = self.evaluators_by_name[compiled_condition.evaluator_type]
res = evaluator.eval(context, compiled_condition.return_value)
if res.status and isinstance(res.body, bool) and res.body:
# one successful value found. No need to look any further
src/core/sheerka/services/SheerkaRuleManager.py
+153 -5
View File
@@ -1,6 +1,7 @@
import operator
import re
from dataclasses import dataclass
from itertools import product
from typing import Union, Set, List
from cache.Cache import Cache
@@ -14,7 +15,7 @@ from core.rule import Rule, ACTION_TYPE_PRINT
from core.sheerka.Sheerka import RECOGNIZED_BY_NAME, RECOGNIZED_BY_ID
from core.sheerka.services.sheerka_service import BaseService, FailedToCompileError
from core.tokenizer import Keywords, TokenKind, Token, IterParser
from core.utils import index_tokens, COLORS, get_text_from_tokens
from core.utils import index_tokens, COLORS, get_text_from_tokens, merge_dictionaries, merge_sets
from evaluators.ConceptEvaluator import ConceptEvaluator
from evaluators.PythonEvaluator import PythonEvaluator, Expando
from parsers.BaseExpressionParser import AndNode, ExpressionVisitor, VariableNode, ComparisonNode, FunctionNode
@@ -1309,19 +1310,166 @@ class ReteConditionExprVisitor(ExpressionVisitor):
conditions.append(Condition(left, attr, value))
@dataclass()
class PythonConditionExprVisitorObj:
text: Union[str, None] # human readable
source: Union[str, None] # python expression to compile
objects: dict
variables: set
@staticmethod
def combine_with_and(left, right):
left_as_list = left if isinstance(left, list) else [left]
right_as_list = right if isinstance(right, list) else [right]
def create_and(a, b):
if a is None and b is None:
return None
if a is None or a == "":
return b
if b is None or b == "":
return a
return a + " and " + b
left_right_product = list(product(left_as_list, right_as_list))
res = []
for left_obj, right_obj in left_right_product:
res.append(PythonConditionExprVisitorObj(create_and(left_obj.text, right_obj.text),
create_and(left_obj.source, right_obj.source),
merge_dictionaries(left_obj.objects, right_obj.objects),
merge_sets(left_obj.variables, right_obj.variables)))
return res[0] if len(res) == 1 else res
class PythonConditionExprVisitor(ExpressionVisitor):
def __init__(self, context):
self.context = context
self.var_counter = 0
self.variables = set()
self.variables = {}
def get_conditions(self, expr_node):
self.var_counter = 0
self.variables.clear()
condition = self.visit(expr_node)
return [condition]
visitor_obj = self.visit(expr_node)
if visitor_obj.source:
if self.variables:
variables_definitions = "\n".join([f"{v} = {k}" for k, v in self.variables.items()])
source = variables_definitions + "\n" + visitor_obj.source
text = variables_definitions + "\n" + visitor_obj.text
else:
source = visitor_obj.source
text = visitor_obj.text
ret = self.context.sheerka.parse_python(self.context, source)
if ret.status:
ret.body.body.original_source = text
ret.body.body.objects = visitor_obj.objects
return [CompiledCondition(PythonEvaluator.NAME, ret, visitor_obj.variables)]
else:
return [CompiledCondition(None, None, visitor_obj.variables)]
def add_variable(self, target):
var_name = f"__x_{self.var_counter:02}__"
self.var_counter += 1
self.variables[target] = var_name
return var_name
def init_or_get_variable_from_name(self, variable_path: List[str], obj_variables):
if len(variable_path) > 1:
left = variable_path[:-1]
right = [variable_path[-1]]
while left:
var_name = ".".join(left)
if var_name in self.variables:
return self.variables[var_name], ".".join(right)
right.insert(0, left.pop())
if variable_path[0] not in self.variables:
self.add_variable(variable_path[0])
obj_variables.add(variable_path[0])
return self.variables[variable_path[0]], ".".join(variable_path[1:])
def init_or_get_variable_from_path(self, variable_path: List[str], obj_variables):
path = ".".join(variable_path)
if path in self.variables:
return self.variables[path]
obj_variables.add(variable_path[0])
return self.add_variable(path)
def visit_VariableNode(self, expr_node: VariableNode):
# no evaluator to call, simply check that the variable is in the bag
return CompiledCondition(None, None, {expr_node.name})
if not expr_node.attributes and expr_node.name.startswith("__"):
return PythonConditionExprVisitorObj(None, None, {}, {expr_node.name})
source = expr_node.get_source() + " == True"
return PythonConditionExprVisitorObj(source, source, {}, {expr_node.name})
def visit_ComparisonNode(self, expr_node: ComparisonNode):
if isinstance(expr_node.left, VariableNode):
source = expr_node.get_source()
return PythonConditionExprVisitorObj(source, source, {}, {expr_node.left.name})
else:
raise FailedToCompileError([expr_node])
def visit_AndNode(self, expr_node: AndNode):
current_visitor_obj = self.visit(expr_node.parts[0])
for node in expr_node.parts[1:]:
visitor_obj = self.visit(node)
current_visitor_obj = PythonConditionExprVisitorObj.combine_with_and(current_visitor_obj, visitor_obj)
return current_visitor_obj
def visit_FunctionNode(self, expr_node: FunctionNode):
if expr_node.first.value == "recognize(":
if not isinstance(expr_node.parameters[0].value, VariableNode):
return FailedToCompileError([f"Cannot recognize '{expr_node.parameters[0].value}'"])
return self.recognize_concept(expr_node.parameters[0].value.unpack(),
expr_node.parameters[1].value,
{})
def recognize_concept(self, variable_path, concept_to_recognize, concept_variables: dict):
if not isinstance(concept_to_recognize, Concept):
concept_as_str = concept_to_recognize.get_source()
if not concept_as_str:
return FailedToCompileError([f"Missing concept in for {variable_path}"])
res = evaluate(self.context,
concept_as_str,
evaluators=CONDITIONS_VISITOR_EVALUATORS,
desc=None,
eval_body=True,
eval_where=False,
is_question=False,
expect_success=False,
stm=None)
res = expect_one(self.context, res)
if not res.status:
return FailedToCompileError([f"Unknown concept {concept_as_str}"])
concept = res.body
else:
concept = concept_to_recognize
obj_variables = set()
variable = self.init_or_get_variable_from_path(variable_path, obj_variables)
source = f"isinstance({variable}, Concept)"
if concept.get_hint(BuiltinConcepts.RECOGNIZED_BY) == RECOGNIZED_BY_NAME:
source += f" and {variable}.name == '{concept.name}'"
elif concept.get_hint(BuiltinConcepts.RECOGNIZED_BY) == RECOGNIZED_BY_ID:
source += f" and {variable}.id == '{concept.id}'"
else:
source += f" and {variable}.key == '{concept.key}'"
concept_variables.update({k: v for k, v in concept.variables().items() if v is not NotInit})
return PythonConditionExprVisitorObj(source, source, {}, obj_variables)
src/core/utils.py
+34
View File
@@ -305,6 +305,40 @@ def dict_product(a, b):
return res
def merge_dictionaries(a, b):
"""
Returns a new dictionary which is the merge
:param a:
:param b:
:return:
"""
if a is None and b is None:
return None
res = {}
if a:
res.update(a)
if b:
res.update(b)
return res
def merge_sets(a, b):
if a is None and b is None:
return None
res = set()
if a:
res.update(a)
if b:
res.update(b)
return res
def get_n_clones(obj, n):
objs = [obj]
for i in range(n - 1):
src/parsers/BaseExpressionParser.py
+40 -23
View File
@@ -1,5 +1,5 @@
from dataclasses import dataclass
from typing import List, Tuple, Union
from typing import List, Union
from core.builtin_concepts_ids import BuiltinConcepts
from core.sheerka.services.SheerkaExecute import ParserInput
@@ -33,15 +33,11 @@ class ParenthesisMismatchError(ParsingError):
token: Token
@dataclass(init=False)
class ExprNode(Node):
"""
Base ExprNode
eval() must be overridden
"""
start: int # index of the first token
end: int # index of the last token
tokens: List[Token]
def __init__(self, start: int, end: int, tokens: List[Token]):
self.start = start
@@ -118,10 +114,7 @@ class NameExprNode(ExprNode):
return UnrecognizedTokensNode(self.start, self.end, [token]).fix_source()
@dataclass(init=False)
class AndNode(ExprNode):
parts: Tuple[ExprNode]
def __init__(self, start, end, tokens, *parts: ExprNode):
super().__init__(start, end, tokens)
self.parts = parts
@@ -154,10 +147,7 @@ class AndNode(ExprNode):
return hash((self.start, self.end, self.parts))
@dataclass(init=False)
class OrNode(ExprNode):
parts: Tuple[ExprNode]
def __init__(self, start, end, tokens, *parts: ExprNode):
super().__init__(start, end, tokens)
self.parts = parts
@@ -190,9 +180,10 @@ class OrNode(ExprNode):
return hash((self.start, self.end, self.parts))
@dataclass()
class NotNode(ExprNode):
node: ExprNode
def __init__(self, start, end, tokens, node: ExprNode):
super().__init__(start, end, tokens)
self.node = node
def eval(self, obj):
return not self.node.eval(obj)
@@ -222,13 +213,15 @@ class NotNode(ExprNode):
return hash((self.start, self.end, self.node))
@dataclass()
class ParenthesisNode(ExprNode):
"""
Contains the boundaries of an expression inside parenthesis
Need it, just to keep track of the boundaries of the parenthesis
"""
node: ExprNode
def __init__(self, start, end, tokens, node: ExprNode):
super().__init__(start, end, tokens)
self.node = node
def __eq__(self, other):
if not isinstance(other, ParenthesisNode):
@@ -291,11 +284,12 @@ class VariableNode(ExprNode):
return [self.name] + self.attributes
@dataclass
class ComparisonNode(ExprNode):
comp: str
left: ExprNode
right: ExprNode
def __init__(self, start, end, tokens, comp: str, left: ExprNode, right: ExprNode):
super().__init__(start, end, tokens)
self.comp = comp
self.left = left
self.right = right
def __eq__(self, other):
if id(self) == id(other):
@@ -338,11 +332,34 @@ class FunctionParameter:
return UnrecognizedTokensNode(self.separator.start, self.separator.end, self.separator.tokens).fix_source()
@dataclass
class FunctionNode(ExprNode):
first: NameExprNode # beginning of the function (it should represent the name of the function)
last: NameExprNode # last part of the function (it should be the trailing parenthesis)
parameters: Union[None, List[FunctionParameter]]
def __init__(self, start, end, tokens,
first: NameExprNode, last: NameExprNode, parameters: Union[None, List[FunctionParameter]]):
super().__init__(start, end, tokens)
self.first = first
self.last = last
self.parameters = parameters
def __eq__(self, other):
if id(self) == id(other):
return True
if not isinstance(other, FunctionNode):
return False
return (self.first == other.first and
self.last == other.last and
self.parameters == other.parameters)
def __hash__(self):
return hash((self.first, self.last, self.parameters))
def __repr__(self):
return f"FunctionNode(start={self.start}, end={self.end}, {self.first!r} {self.last} {self.parameters!r})"
def __str__(self):
return f"{self.first} {self.parameters} {self.last}"
class BaseExpressionParser(BaseParser):
tests/core/test_SheerkaRuleManager.py
+118 -17
View File
@@ -1154,7 +1154,7 @@ isinstance(var, Concept) and var.key == 'hello __var__0'""" + \
),
])
def test_i_can_get_rete_using_recognized_function(self, test_name, expression, variable_name, expected_as_str):
def test_i_can_get_rete_using_recognize_function(self, test_name, expression, variable_name, expected_as_str):
sheerka, context, greetings, foo = self.init_test().with_concepts(
Concept("greetings", definition="hello a", definition_type=DEFINITION_TYPE_DEF).def_var("a"),
Concept("foo"),
@@ -1290,9 +1290,14 @@ isinstance(var, Concept) and var.key == 'hello __var__0'""" + \
matches = list(network.matches)
assert len(matches) == 1
def test_i_can_get_compiled_conditions_when_testing_data_existence(self):
@pytest.mark.parametrize("expression, expected_compiled", [
("__ret", None),
("__ret.status == True", "__ret.status == True"),
("__ret.status", "__ret.status == True"),
("__ret and __ret.status", "__ret.status == True")
])
def test_i_can_get_compiled_conditions(self, expression, expected_compiled):
sheerka, context = self.init_test().unpack()
expression = "__ret"
parser = ExpressionParser()
error_sink = ErrorSink()
@@ -1305,6 +1310,13 @@ isinstance(var, Concept) and var.key == 'hello __var__0'""" + \
assert len(conditions) == 1
assert isinstance(conditions[0], CompiledCondition)
if expected_compiled:
ast_ = ast.parse(expected_compiled, "<source>", 'eval')
expected_python_node = PythonNode(expected_compiled, ast_)
assert conditions[0].evaluator_type == PYTHON_EVALUATOR_NAME
assert sheerka.isinstance(conditions[0].return_value, BuiltinConcepts.RETURN_VALUE)
assert sheerka.objvalue(conditions[0].return_value) == expected_python_node
else:
assert conditions[0].evaluator_type is None
assert conditions[0].return_value is None
assert conditions[0].concept is None
@@ -1313,20 +1325,102 @@ isinstance(var, Concept) and var.key == 'hello __var__0'""" + \
# check against SheerkaEvaluateRules
evaluate_rules_service = sheerka.services[SheerkaEvaluateRules.NAME]
bag = {"__ret": ReturnValueConcept("Test", True, None)}
with context.push(BuiltinConcepts.RULES_EVALUATION, bag, desc="Evaluating rules...") as sub_context:
sub_context.sheerka.add_many_to_short_term_memory(sub_context, bag)
rule = Rule(name="test_i_can_get_compiled_conditions", predicate=expression)
rule.compiled_conditions = conditions
res = evaluate_rules_service.evaluate_rule(context, rule, bag)
res = evaluate_rules_service.evaluate_rule(sub_context, rule, bag)
assert res.status
assert self.sheerka.is_success(self.sheerka.objvalue(res))
@pytest.mark.parametrize("expression", [
"__ret",
"__ret.status == True",
"__ret.status",
"__ret and __ret.status",
@pytest.mark.parametrize("expression, variable_name, expected_compiled", [
(
"recognize(__ret.body, greetings)",
None,
"__x_00__ = __ret.body\nisinstance(__x_00__, Concept) and __x_00__.name == 'greetings'"
),
# (
# "recognize(__ret.body, c:|1001:)",
# None,
# ["#__x_00__|__name__|'__ret'",
# "#__x_00__|body|#__x_01__",
# "#__x_01__|__is_concept__|True",
# "#__x_01__|id|'1001'"]
# ),
# (
# "recognize(__ret.body, c:greetings:)",
# None,
# ["#__x_00__|__name__|'__ret'",
# "#__x_00__|body|#__x_01__",
# "#__x_01__|__is_concept__|True",
# "#__x_01__|name|'greetings'"]
# ),
# (
# "recognize(__ret.body, greetings) and __ret.body.a == 'my friend'",
# "my friend",
# ["#__x_00__|__name__|'__ret'",
# "#__x_00__|body|#__x_01__",
# "#__x_01__|__is_concept__|True",
# "#__x_01__|name|'greetings'",
# "#__x_01__|a|'my friend'"]
# ),
# (
# "recognize(__ret.body, greetings) and __ret.body.a == sheerka",
# "sheerka",
# ["#__x_00__|__name__|'__ret'",
# "#__x_00__|body|#__x_01__",
# "#__x_01__|__is_concept__|True",
# "#__x_01__|name|'greetings'",
# "#__x_01__|a|'__sheerka__'"]
# ),
# (
# "recognize(__ret.body, greetings) and __ret.body.a == foo",
# "foo",
# ["#__x_00__|__name__|'__ret'",
# "#__x_00__|body|#__x_01__",
# "#__x_01__|__is_concept__|True",
# "#__x_01__|name|'greetings'",
# "#__x_01__|a|#__x_02__",
# "#__x_02__|__is_concept__|True",
# "#__x_02__|key|'foo'"]
# ),
# (
# "recognize(__ret.body, hello sheerka)",
# "sheerka",
# ["#__x_00__|__name__|'__ret'",
# "#__x_00__|body|#__x_01__",
# "#__x_01__|__is_concept__|True",
# "#__x_01__|key|'hello __var__0'",
# "#__x_01__|a|'__sheerka__'"]
# ),
# (
# "recognize(__ret.body, hello 'my friend')",
# "my friend",
# ["#__x_00__|__name__|'__ret'",
# "#__x_00__|body|#__x_01__",
# "#__x_01__|__is_concept__|True",
# "#__x_01__|key|'hello __var__0'",
# "#__x_01__|a|'my friend'"]
# ),
# (
# "recognize(__ret.body, hello foo)",
# "foo",
# ["#__x_00__|__name__|'__ret'",
# "#__x_00__|body|#__x_01__",
# "#__x_01__|__is_concept__|True",
# "#__x_01__|key|'hello __var__0'",
# "#__x_01__|a|#__x_02__",
# "#__x_02__|__is_concept__|True",
# "#__x_02__|key|'foo'",
# ]
# ),
])
def test_i_can_get_compiled_conditions(self, expression):
sheerka, context = self.init_test().unpack()
def test_i_can_get_compiled_using_recognize_function(self, expression, variable_name, expected_compiled):
sheerka, context, greetings, foo = self.init_test().with_concepts(
Concept("greetings", definition="hello a", definition_type=DEFINITION_TYPE_DEF).def_var("a"),
Concept("foo"),
).unpack()
parser = ExpressionParser()
error_sink = ErrorSink()
@@ -1337,23 +1431,30 @@ isinstance(var, Concept) and var.key == 'hello __var__0'""" + \
visitor = PythonConditionExprVisitor(context)
conditions = visitor.get_conditions(parsed)
ast_ = ast.parse(expression, "<source>", 'eval')
expected_python_node = PythonNode(expression, ast_)
assert len(conditions) == 1
assert isinstance(conditions[0], CompiledCondition)
if expected_compiled:
ast_ = ast.parse(expected_compiled, "<source>", 'exec')
expected_python_node = PythonNode(expected_compiled, ast_, expected_compiled)
assert conditions[0].evaluator_type == PYTHON_EVALUATOR_NAME
assert sheerka.isinstance(conditions[0].return_value, BuiltinConcepts.RETURN_VALUE)
assert sheerka.objvalue(conditions[0].return_value) == expected_python_node
else:
assert conditions[0].evaluator_type is None
assert conditions[0].return_value is None
assert conditions[0].concept is None
assert conditions[0].variables == {"__ret"}
# check against SheerkaEvaluateRules
evaluate_rules_service = sheerka.services[SheerkaEvaluateRules.NAME]
bag = {"__ret": ReturnValueConcept("Test", True, None)}
rule = Rule(name="test_i_can_get_compiled_conditions", predicate=expression)
variable = foo if variable_name == "foo" else sheerka if variable_name == "sheerka" else variable_name
to_recognize = sheerka.new_from_template(greetings, greetings.key, a=variable)
bag = {"__ret": ReturnValueConcept("Test", True, to_recognize)}
with context.push(BuiltinConcepts.RULES_EVALUATION, bag, desc="Evaluating rules...") as sub_context:
sub_context.sheerka.add_many_to_short_term_memory(sub_context, bag)
rule = Rule(name="test_i_can_get_compiled_using_recognize_function", predicate=expression)
rule.compiled_conditions = conditions
res = evaluate_rules_service.evaluate_rule(context, rule, bag)
res = evaluate_rules_service.evaluate_rule(sub_context, rule, bag)
assert res.status
assert self.sheerka.is_success(self.sheerka.objvalue(res))