kodjo/Sheerka-Old
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fixed #49 : ExpressionParser: Implement ExpressionParser

Browse Source
This commit is contained in:
Kodjo Sossouvi
2021-03-15 19:41:06 +01:00
parent 42bc6abf97
commit 27bc6c4ba1
7 changed files with 245 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/core/sheerka/services/SheerkaEvaluateRules.py
-2
View File
@@ -29,8 +29,6 @@ class SheerkaEvaluateRules(BaseService):
self.sheerka.register_debug_vars(self.NAME, "evaluate_rules", "results")
self.sheerka.register_debug_rules(self.NAME, "evaluate_rule", "*")
def reset_evaluators(self):
# instantiate evaluators, once for all, only keep when it's enabled
evaluators = [e_class() for e_class in self.sheerka.evaluators]
src/core/sheerka/services/SheerkaRuleManager.py
+49 -14
View File
@@ -14,10 +14,10 @@ from core.rule import Rule, ACTION_TYPE_PRINT
from core.sheerka.Sheerka import RECOGNIZED_BY_NAME, RECOGNIZED_BY_ID
from core.sheerka.services.sheerka_service import BaseService, FailedToCompileError
from core.tokenizer import Keywords, TokenKind, Token, IterParser
from core.utils import index_tokens, COLORS, get_text_from_tokens
from core.utils import index_tokens, COLORS, get_text_from_tokens, unstr_concept
from evaluators.ConceptEvaluator import ConceptEvaluator
from evaluators.PythonEvaluator import PythonEvaluator, Expando
from parsers.BaseExpressionParser import AndNode, ExpressionVisitor, VariableNode, ComparisonNode
from parsers.BaseExpressionParser import AndNode, ExpressionVisitor, VariableNode, ComparisonNode, FunctionNode
from parsers.BaseNodeParser import SourceCodeWithConceptNode, ConceptNode, SourceCodeNode
from parsers.LogicalOperatorParser import LogicalOperatorParser
from parsers.PythonParser import PythonNode
@@ -1127,7 +1127,6 @@ class ReteConditionExprVisitor(ExpressionVisitor):
self.context = context
self.var_counter = 0
self.variables = {}
self.res = []
def add_variable(self, target):
var_name = f"__x_{self.var_counter:02}__"
@@ -1135,35 +1134,71 @@ class ReteConditionExprVisitor(ExpressionVisitor):
self.variables[target] = var_name
return var_name
def init_variable_if_needed(self, node):
def init_or_get_variable_from_name(self, node, conditions):
if node.name not in self.variables:
var_name = self.add_variable(node.name)
self.res.append(Condition(V(var_name), "__name__", node.name))
conditions.append(Condition(V(var_name), "__name__", node.name))
return V(self.variables[node.name])
def init_or_get_variable_from_attr(self, node, conditions):
path = f"{node.name}.{node.attributes_str}"
if path in self.variables:
return self.variables[path]
root = self.init_or_get_variable_from_name(node, conditions)
var_name = self.add_variable(path)
variable = V(var_name)
conditions.append(Condition(root, node.attributes_str, variable))
return variable
def get_conditions(self, expr_node):
self.res.clear()
self.var_counter = 0
self.variables.clear()
self.visit(expr_node)
return AndConditions(self.res)
conditions = self.visit(expr_node)
return [AndConditions(conditions)]
def visit_VariableNode(self, expr_node):
var_name = self.init_variable_if_needed(expr_node)
conditions = []
var_name = self.init_or_get_variable_from_name(expr_node, conditions)
if expr_node.attributes_str is not None:
self.res.append(Condition(var_name, expr_node.attributes_str, True))
conditions.append(Condition(var_name, expr_node.attributes_str, True))
return conditions
def visit_AndNode(self, expr_node: AndNode):
conditions = []
for node in expr_node.parts:
self.visit(node)
conditions.extend(self.visit(node))
return conditions
def visit_ComparisonNode(self, expr_node: ComparisonNode):
if isinstance(expr_node.left, VariableNode):
left = self.init_variable_if_needed(expr_node.left)
conditions = []
left = self.init_or_get_variable_from_name(expr_node.left, conditions)
attr = expr_node.left.attributes_str or "__self__"
right = eval(get_text_from_tokens(expr_node.right.tokens))
self.res.append(Condition(left, attr, right))
conditions.append(Condition(left, attr, right))
return conditions
else:
raise FailedToCompileError(expr_node)
raise FailedToCompileError([expr_node])
def visit_FunctionNode(self, expr_node: FunctionNode):
if expr_node.first.value == "recognize(":
return self.function_recognize(expr_node.parameters[0].value, expr_node.parameters[1].value)
def function_recognize(self, source, target):
conditions = []
body_var = self.init_or_get_variable_from_attr(source, conditions)
conditions.append(Condition(body_var, "__is_concept__", True))
if isinstance(target, VariableNode):
conditions.append(Condition(body_var, "name", target.name))
else:
concept_key, concept_id = unstr_concept(target.value)
if concept_id:
conditions.append(Condition(body_var, "id", concept_id))
elif concept_key:
conditions.append(Condition(body_var, "name", concept_key))
return conditions
src/sheerkarete/network.py
+22 -14
View File
@@ -230,10 +230,17 @@ class ReteNetwork:
for cond in conditions:
# update requested attributes for a fact
if isinstance(cond, Condition):
# Manage list of requested attributes
# Manage list of requested attributes when using __name__ indirection
if isinstance(cond.identifier, V) and cond.attribute == "__name__":
vars_ids_mappings[cond.identifier] = cond.value
# Manage list of requested attributes when bounding a new variable
if (cond.identifier in vars_ids_mappings and
isinstance(cond.attribute, str) and
isinstance(cond.value, V)):
vars_ids_mappings[cond.value] = f"{vars_ids_mappings[cond.identifier]}.{cond.attribute}"
identifier = vars_ids_mappings[cond.identifier] if cond.identifier in vars_ids_mappings else \
cond.identifier if not isinstance(cond.identifier, V) else \
None
@@ -365,22 +372,23 @@ class ReteNetwork:
for wme in to_remove:
self.remove_wme(wme)
def add_obj(self, name, obj, use_bag=False, root=True):
def add_obj(self, name, obj, fact_id=None, use_bag=False):
"""
Adds a new object to the working memory
"""
def inner_add_vme(ident, attr, val):
if val is NotInit:
def inner_add_vme(name_, fact_id_, attr_, value_):
if value_ is NotInit:
pass
elif attr != "self" and isinstance(val, Concept):
new_name = f"{ident}.{attr}"
self.add_wme(WME(ident, attr, new_name))
self.add_obj(new_name, val, use_bag=True, root=False)
elif attr_ != "self" and isinstance(value_, Concept):
new_name = f"{name_}.{attr_}"
new_fact_id = f"{fact_id_}.{attr_}"
self.add_wme(WME(fact_id_, attr_, new_fact_id))
self.add_obj(new_name, value_, new_fact_id)
else:
self.add_wme(WME(ident, attr, val))
self.add_wme(WME(fact_id_, attr_, value_))
if root:
if fact_id is None:
if hasattr(obj, FACT_ID):
raise ValueError("Object already has an id, cannot add")
@@ -388,8 +396,6 @@ class ReteNetwork:
setattr(obj, FACT_ID, fact_id)
self.facts[fact_id] = obj
self.fact_counter += 1
else:
fact_id = name
requested_attributes = "*" if use_bag else \
self.attributes_by_id[name] if name in self.attributes_by_id else \
@@ -399,13 +405,15 @@ class ReteNetwork:
if attribute == "*":
bag = as_bag(obj)
for k, v in bag.items():
inner_add_vme(fact_id, k, v)
inner_add_vme(name, fact_id, k, v)
elif attribute == "__name__":
self.add_wme(WME(fact_id, "__name__", name))
elif attribute == "__is_concept__":
self.add_wme(WME(fact_id, "__is_concept__", isinstance(obj, Concept)))
else:
try:
value = getattr(obj, attribute)
inner_add_vme(fact_id, attribute, value)
inner_add_vme(name, fact_id, attribute, value)
except AttributeError:
pass
tests/core/test_SheerkaRuleManager.py
+139 -3
View File
@@ -2,7 +2,7 @@ import ast
import pytest
from core.builtin_concepts import BuiltinConcepts
from core.builtin_concepts import BuiltinConcepts, ReturnValueConcept
from core.concept import Concept, DEFINITION_TYPE_DEF, DoNotResolve
from core.global_symbols import RULE_COMPARISON_CONTEXT, NotFound, EVENT_RULE_DELETED
from core.rule import Rule, ACTION_TYPE_PRINT, ACTION_TYPE_EXEC
@@ -21,6 +21,7 @@ from parsers.ExpressionParser import ExpressionParser
from parsers.PythonParser import PythonNode
from sheerkarete.common import V
from sheerkarete.conditions import Condition, AndConditions
from sheerkarete.network import ReteNetwork
from tests.TestUsingFileBasedSheerka import TestUsingFileBasedSheerka
from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
from tests.parsers.parsers_utils import CMV, CC, compare_with_test_object, get_test_obj, get_rete_conditions
@@ -1029,7 +1030,7 @@ isinstance(var, Concept) and var.key == 'hello __var__0'""" + \
),
])
def test_i_can_get_rete_conditions(self, expression, expected_as_str):
sheerka, context, = self.init_test().unpack()
sheerka, context = self.init_test().unpack()
parser = ExpressionParser()
expected = get_rete_conditions(*expected_as_str)
@@ -1041,7 +1042,142 @@ isinstance(var, Concept) and var.key == 'hello __var__0'""" + \
visitor = ReteConditionExprVisitor(context)
conditions = visitor.get_conditions(parsed)
assert conditions == expected
assert conditions == [expected]
# check against a Rete network
network = ReteNetwork()
rule = Rule("test", expression, None)
rule.metadata.id = 9999
rule.metadata.is_compiled = True
rule.metadata.is_enabled = True
rule.rete_disjunctions = conditions
network.add_rule(rule)
network.add_obj("__ret", ReturnValueConcept("Test", True, None))
matches = list(network.matches)
assert len(matches) == 1
@pytest.mark.parametrize("test_name, expression, variable_name, expected_as_str", [
(
"recognize by name",
"recognize(__ret.body, greetings)",
None,
["#__x_00__|__name__|'__ret'",
"#__x_00__|body|#__x_01__",
"#__x_01__|__is_concept__|True",
"#__x_01__|name|'greetings'"]
),
(
"recognize by id",
"recognize(__ret.body, c:|1001:)",
None,
["#__x_00__|__name__|'__ret'",
"#__x_00__|body|#__x_01__",
"#__x_01__|__is_concept__|True",
"#__x_01__|id|'1001'"]
),
(
"recognize by name using c_str",
"recognize(__ret.body, c:greetings:)",
None,
["#__x_00__|__name__|'__ret'",
"#__x_00__|body|#__x_01__",
"#__x_01__|__is_concept__|True",
"#__x_01__|name|'greetings'"]
),
(
"recognize by name and variable sheerka",
"recognize(__ret.body, greetings, a=sheerka)",
"sheerka",
["#__x_00__|__name__|'__ret'",
"#__x_00__|body|#__x_01__",
"#__x_01__|__is_concept__|True",
"#__x_01__|name|'greetings'",
"#__x_01__|a|'__sheerka__'"]
),
(
"recognize by name and str variable",
"recognize(__ret.body, greetings, a='my friend')",
"'my friend'",
["#__x_00__|__name__|'__ret'",
"#__x_00__|body|#__x_01__",
"#__x_01__|__is_concept__|True",
"#__x_01__|name|'greetings'",
"#__x_01__|a|'my friend'"]
),
(
"recognize by name and concept variable",
"recognize(__ret.body, greetings, a=foo)",
"foo",
["#__x_00__|__name__|'__ret'",
"#__x_00__|body|#__x_01__",
"#__x_01__|__is_concept__|True",
"#__x_01__|name|'greetings'",
"#__x_01__|a.name|'foo'"]
),
(
"recognize by name and add other conditions (str)",
"recognize(__ret.body, greetings) and __ret.body.a == 'my friend'",
"foo",
["#__x_00__|__name__|'__ret'",
"#__x_00__|body|#__x_01__",
"#__x_01__|__is_concept__|True",
"#__x_01__|name|'greetings'",
"#__x_01__|a|'my friend'"]
),
(
"recognize by name and add other conditions (sheerka)",
"recognize(__ret.body, greetings) and __ret.body.a == sheerka",
"foo",
["#__x_00__|__name__|'__ret'",
"#__x_00__|body|#__x_01__",
"#__x_01__|__is_concept__|True",
"#__x_01__|name|'greetings'",
"#__x_01__|a|'kodjo'"]
),
(
"recognize by name and add other conditions (concept)",
"recognize(__ret.body, greetings) and __ret.body.a == foo",
"foo",
["#__x_00__|__name__|'__ret'",
"#__x_00__|body|#__x_01__",
"#__x_01__|__is_concept__|True",
"#__x_01__|name|'greetings'",
"#__x_01__|a|'kodjo'"]
),
])
def test_i_can_get_rete_conditions_from_recognized(self, test_name, expression, variable_name, expected_as_str):
sheerka, context, greetings, foo = self.init_test().with_concepts(
Concept("greetings", definition="hello a", definition_type=DEFINITION_TYPE_DEF).def_var("a"),
Concept("foo"),
).unpack()
parser = ExpressionParser()
expected = get_rete_conditions(*expected_as_str)
error_sink = ErrorSink()
parser_input = ParserInput(expression)
parser.reset_parser_input(parser_input, error_sink)
parsed = parser.parse_input(context, parser_input, error_sink)
visitor = ReteConditionExprVisitor(context)
conditions = visitor.get_conditions(parsed)
assert conditions == [expected]
# check against a Rete network
network = ReteNetwork()
rule = Rule("test", expression, None)
rule.metadata.id = 9999
rule.metadata.is_compiled = True
rule.metadata.is_enabled = True
rule.rete_disjunctions = conditions
network.add_rule(rule)
variable = foo if variable_name == "foo" else variable_name
to_recognize = sheerka.new_from_template(greetings, greetings.key, a=variable)
network.add_obj("__ret", ReturnValueConcept("Test", True, to_recognize))
matches = list(network.matches)
assert len(matches) == 1
class TestSheerkaRuleManagerUsingFileBasedSheerka(TestUsingFileBasedSheerka):
tests/parsers/parsers_utils.py
+17 -11
View File
@@ -8,13 +8,14 @@ from core.concept import Concept, ConceptParts, DoNotResolve, AllConceptParts
from core.rule import Rule
from core.tokenizer import Tokenizer, TokenKind, Token
from core.utils import get_text_from_tokens, tokens_index, str_concept
from parsers.BaseExpressionParser import NameExprNode, AndNode, OrNode, NotNode, VariableNode, ComparisonNode, \
ComparisonType, \
FunctionParameter
from parsers.BaseNodeParser import UnrecognizedTokensNode, SourceCodeNode, RuleNode, ConceptNode, \
SourceCodeWithConceptNode
from parsers.FunctionParser import FunctionNode
from parsers.PythonParser import PythonNode
from parsers.SyaNodeParser import SyaConceptParserHelper
from parsers.BaseExpressionParser import NameExprNode, AndNode, OrNode, NotNode, VariableNode, ComparisonNode, ComparisonType, \
FunctionParameter
from sheerkarete.common import V
from sheerkarete.conditions import Condition, AndConditions
@@ -1304,17 +1305,22 @@ def get_rete_conditions(*conditions_as_string):
"identifier|__name__|'True'" -> Condition(identifier, '__name__', 'True') # the string 'True'
"identifier|__name__|True" -> Condition(identifier, '__name__', True) # the bool True
"""
def get_value(obj):
if obj.startswith("#"):
return V(obj[1:])
if obj.startswith("'"):
return obj[1:-1]
if obj in ("True", "False"):
return obj == "True"
return int(obj)
res = []
for as_string in conditions_as_string:
identifier, attribute, value = as_string.split("|")
if identifier.startswith("#"):
identifier = V(identifier[1:])
if value.startswith("'"):
value = value[1:-1]
elif value in ("True", "False"):
value = (value == "True")
else:
value = int(value)
parts = as_string.split("|")
identifier = get_value(parts[0])
attribute = parts[1]
value = get_value(parts[2])
res.append(Condition(identifier, attribute, value))
return AndConditions(res)
tests/parsers/test_ExpressionParser.py
+1
View File
@@ -56,6 +56,7 @@ class TestExpressionParser(TestUsingMemoryBasedSheerka):
("func(var1.attr1 > var2.attr2)", FN("func(", ")", [GT(VAR("var1.attr1"), VAR("var2.attr2"))])),
("func1(var1) and func2(var2)", AND(FN("func1(", ")", [VAR("var1")]), FN("func2(", (")", 1), [VAR("var2")]))),
("__ret", VAR("__ret")),
#("func1().func2()", [])
])
def test_i_can_parse_input(self, expression, expected):
sheerka, context, parser, parser_input, error_sink = self.init_parser_with_source(expression)
tests/sheerkarete/test_network.py
+17 -13
View File
@@ -65,16 +65,28 @@ class TestReteNetwork(TestUsingMemoryBasedSheerka):
assert len(network.pnodes) == 1
assert network.pnodes[0].rules == [rule1, rule2]
def test_i_can_update_conditions_attributes_by_id_when_constraint_on__name__(self):
def test_i_can_update_conditions_attributes_by_id_when_constraints(self):
network = ReteNetwork()
conditions = [Condition(V("x"), "__name__", "fact_name"),
Condition(V("x"), "attr1", "value1"),
Condition(V("x"), "attr2", "value1")]
Condition(V("x"), "body", V("y")),
Condition(V("y"), "__is_concept__", True),
Condition(V("y"), "name", "SubConcept"),
Condition(V("x"), "value", V("z")),
Condition(V("z"), "status", False),
Condition(V("z"), "body", V("zz")),
Condition(V("zz"), "sub_value", "sub_value"),
]
rule = RuleForTestingRete(AndConditions(conditions))
network.add_rule(rule)
assert network.attributes_by_id == {"fact_name": ["__name__", "attr1", "attr2"]}
assert network.attributes_by_id == {
"fact_name": ["__name__", "attr1", "body", "value"],
"fact_name.body": ["__is_concept__", "name"],
"fact_name.value": ["status", "body"],
"fact_name.value.body": ["sub_value"],
}
def test_adding_obj_when_no_rule_has_no_effect(self):
network = ReteNetwork()
@@ -193,20 +205,11 @@ class TestReteNetwork(TestUsingMemoryBasedSheerka):
WME("f-00000", "__name__", "__ret"),
WME("f-00000", "status", True),
WME("f-00000", "body", "f-00000.body"),
WME("f-00000.body", "id", "1003"),
WME("f-00000.body", "name", "greetings"),
WME("f-00000.body", "key", "hello __var__0"),
WME("f-00000.body", "a", "f-00000.body.a"),
WME("f-00000.body", "self", ret.body),
WME("f-00000.body.a", "id", "1002"),
WME("f-00000.body.a", "name", "the x"),
WME("f-00000.body.a", "key", "the __var__0"),
WME("f-00000.body.a", "x", "f-00000.body.a.x"),
WME("f-00000.body.a", "self", the_boy),
WME("f-00000.body.a.x", "id", "1001"),
WME("f-00000.body.a.x", "name", "boy"),
WME("f-00000.body.a.x", "key", "boy"),
WME("f-00000.body.a.x", "self", boy),
}
# sanity check that the WME produced match the condition
@@ -649,7 +652,8 @@ class TestReteNetwork(TestUsingMemoryBasedSheerka):
assert len(rule.rete_p_nodes) > 0
def test_format_rule_is_not_added_to_rete_network_when_it_is_created(self):
sheerka, context, rule = self.init_test().with_format_rules(("rule_name", "id.attr == 'value'", 'True')).unpack()
sheerka, context, rule = self.init_test().with_format_rules(
("rule_name", "id.attr == 'value'", 'True')).unpack()
evaluation_service = sheerka.services[SheerkaEvaluateRules.NAME]
rete_network = evaluation_service.network