Fixed #29: Parsers: Implement parsing memoization
Fixed #77 : Parser: ShortTermMemoryParser should be called separately Fixed #78 : Remove VariableNode usage Fixed #79 : ConceptManager: Implement compile caching Fixed #80 : SheerkaExecute : parsers_key is not correctly computed Fixed #81 : ValidateConceptEvaluator : Validate concept's where and pre clauses right after the parsing Fixed #82 : SheerkaIsAManager: isa() failed when the set as a body Fixed #83 : ValidateConceptEvaluator : Support BNF and SYA Concepts Fixed #84 : ExpressionParser: Implement the parser as a standard parser Fixed #85 : Services: Give order to services Fixed #86 : cannot manage smart_get_attr(the short, color)
This commit is contained in:
@@ -4,28 +4,18 @@ from core.builtin_concepts import BuiltinConcepts, ReturnValueConcept
|
||||
from core.concept import Concept, DEFINITION_TYPE_DEF
|
||||
from core.global_symbols import RULE_COMPARISON_CONTEXT, NotFound, EVENT_RULE_DELETED
|
||||
from core.rule import Rule, ACTION_TYPE_PRINT, ACTION_TYPE_EXEC
|
||||
from core.sheerka.Sheerka import RECOGNIZED_BY_ID, RECOGNIZED_BY_NAME
|
||||
from core.sheerka.services.SheerkaExecute import ParserInput
|
||||
from core.sheerka.services.SheerkaRuleManager import SheerkaRuleManager, FormatRuleActionParser, \
|
||||
FormatAstRawText, FormatAstVariable, FormatAstSequence, FormatAstFunction, \
|
||||
FormatRuleSyntaxError, FormatAstList, UnexpectedEof, FormatAstColor, FormatAstDict, \
|
||||
FormatAstMulti, \
|
||||
PythonCodeEmitter, FormatAstNode, ReteConditionExprVisitor
|
||||
from core.sheerka.services.SheerkaRuleManager import SheerkaRuleManager, ReteConditionExprVisitor
|
||||
from core.tokenizer import Token, TokenKind
|
||||
from parsers.BaseParser import ErrorSink
|
||||
from parsers.ExpressionParser import ExpressionParser
|
||||
from parsers.FormatRuleActionParser import FormatAstNode
|
||||
from sheerkarete.conditions import FilterCondition
|
||||
from sheerkarete.network import ReteNetwork
|
||||
from tests.TestUsingFileBasedSheerka import TestUsingFileBasedSheerka
|
||||
from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
|
||||
from tests.parsers.parsers_utils import get_rete_conditions, NEGCOND, NCCOND
|
||||
|
||||
seq = FormatAstSequence
|
||||
raw = FormatAstRawText
|
||||
var = FormatAstVariable
|
||||
func = FormatAstFunction
|
||||
lst = FormatAstList
|
||||
|
||||
PYTHON_EVALUATOR_NAME = "Python"
|
||||
CONCEPT_EVALUATOR_NAME = "Concept"
|
||||
|
||||
@@ -175,7 +165,7 @@ class TestSheerkaRuleManager(TestUsingMemoryBasedSheerka):
|
||||
sheerka, context = self.init_test(cache_only=False).unpack()
|
||||
service = sheerka.services[SheerkaRuleManager.NAME]
|
||||
|
||||
rule = Rule(ACTION_TYPE_EXEC, "name", "cannot build = False", "'Hello back at you !'")
|
||||
rule = Rule(action_type, "name", "cannot build = False", action)
|
||||
rule.metadata.is_enabled = True # it should be disabled
|
||||
rule = service.init_rule(context, rule)
|
||||
|
||||
@@ -209,71 +199,6 @@ class TestSheerkaRuleManager(TestUsingMemoryBasedSheerka):
|
||||
assert not rule.metadata.is_enabled
|
||||
assert rule.compiled_action is None
|
||||
|
||||
@pytest.mark.parametrize("text, expected", [
|
||||
("", FormatAstRawText("")),
|
||||
(" ", FormatAstRawText(" ")),
|
||||
(" raw text ", FormatAstRawText(" raw text ")),
|
||||
("{variable}", FormatAstVariable("variable")),
|
||||
("{ variable }", FormatAstVariable("variable")),
|
||||
(" xy {v} z", seq([raw(" xy "), var("v"), raw(" z")])),
|
||||
(r"\{variable}", FormatAstRawText("{variable}")),
|
||||
(r"\\{variable}", seq([raw("\\"), var("variable")])),
|
||||
(r"\\\{variable}", FormatAstRawText(r"\{variable}")),
|
||||
(r"{var1}{var2}", seq([var("var1"), var("var2")])),
|
||||
("func()", FormatAstFunction("func", [], {})),
|
||||
("func(a, 'string value', c)", FormatAstFunction("func", ["a", "'string value'", "c"], {})),
|
||||
("func(a=10, b='string value')", FormatAstFunction("func", [], {"a": "10", "b": "'string value'"})),
|
||||
("func('string value'='another string value')", func("func", [], {"'string value'": "'another string value'"})),
|
||||
("red(' xy {v}')", FormatAstColor("red", seq([raw(" xy "), var("v")]))),
|
||||
('blue(" xy {v}")', FormatAstColor("blue", seq([raw(" xy "), var("v")]))),
|
||||
('green( xy )', FormatAstColor("green", var("xy"))),
|
||||
('green()', FormatAstColor("green", raw(""))),
|
||||
('green("")', FormatAstColor("green", raw(""))),
|
||||
("list(var_name, 2, 'children')", FormatAstList("var_name", recurse_on="children", recursion_depth=2)),
|
||||
("list(var_name, recursion_depth=2, recurse_on='children')", FormatAstList("var_name",
|
||||
recurse_on="children",
|
||||
recursion_depth=2)),
|
||||
("list(var_name, recursion_depth=2, 'children')", FormatAstList("var_name", recursion_depth=2)),
|
||||
("list(var_name, 'children', recursion_depth=2)", FormatAstList("var_name", recursion_depth=2)),
|
||||
("list(var_name)", FormatAstList("var_name")),
|
||||
("{obj.prop1.prop2[0].prop3['value']}", FormatAstVariable("obj.prop1.prop2[0].prop3['value']")),
|
||||
("[{id}]", seq([raw("["), var("id"), raw("]")])),
|
||||
("{variable:format}", FormatAstVariable("variable", "format")),
|
||||
("{variable:3}", FormatAstVariable("variable", "3")),
|
||||
(r"\not_a_function(a={var})", seq([raw("not_a_function(a="), var("var"), raw(")")])),
|
||||
("dict(var_name)", FormatAstDict("var_name")),
|
||||
("dict(var_name, items_prop='props')", FormatAstDict("var_name", items_prop='props')),
|
||||
("dict(var_name, debug=True)", FormatAstDict("var_name", debug=True, prefix="{", suffix="}")),
|
||||
("multi(var_name)", FormatAstMulti("var_name")),
|
||||
])
|
||||
def test_i_can_parse_format_rule(self, text, expected):
|
||||
assert FormatRuleActionParser(text).parse() == expected
|
||||
|
||||
@pytest.mark.parametrize("text, expected_error", [
|
||||
("{", UnexpectedEof("while parsing variable", Token(TokenKind.LBRACE, "{", 0, 1, 1))),
|
||||
("{var_name", UnexpectedEof("while parsing variable", Token(TokenKind.LBRACE, "{", 0, 1, 1))),
|
||||
("{}", FormatRuleSyntaxError("variable name not found", None)),
|
||||
("func(", UnexpectedEof("while parsing function", Token(TokenKind.IDENTIFIER, "func", 0, 1, 1))),
|
||||
("func(a,b,c", UnexpectedEof("while parsing function", Token(TokenKind.IDENTIFIER, "func", 0, 1, 1))),
|
||||
("func(a,,c", FormatRuleSyntaxError("no parameter found", Token(TokenKind.COMMA, ",", 7, 1, 8))),
|
||||
("func(a,,c)", FormatRuleSyntaxError("no parameter found", Token(TokenKind.COMMA, ",", 7, 1, 8))),
|
||||
("red(a,b)", FormatRuleSyntaxError("only one parameter supported", Token(TokenKind.IDENTIFIER, "b", 6, 1, 7))),
|
||||
("red(a=b)", FormatRuleSyntaxError("keyword arguments are not supported", None)),
|
||||
("red(xy {v})", FormatRuleSyntaxError("Invalid identifier", None)),
|
||||
("list()", FormatRuleSyntaxError("variable name not found", None)),
|
||||
("list(recursion_depth=2)", FormatRuleSyntaxError("variable name not found", None)),
|
||||
("list(a,b,c,d,e)", FormatRuleSyntaxError("too many positional arguments",
|
||||
Token(TokenKind.IDENTIFIER, "e", 13, 1, 14))),
|
||||
("list(a, recursion_depth=hello)", FormatRuleSyntaxError("'hello' is not numeric", None)),
|
||||
("list(a, recursion_depth='hello')", FormatRuleSyntaxError("'recursion_depth' must be an integer", None)),
|
||||
("dict()", FormatRuleSyntaxError("variable name not found", None)),
|
||||
])
|
||||
def test_i_cannot_parse_invalid_format(self, text, expected_error):
|
||||
parser = FormatRuleActionParser(text)
|
||||
parser.parse()
|
||||
|
||||
assert parser.error_sink == expected_error
|
||||
|
||||
def test_i_can_get_rule_priorities(self):
|
||||
sheerka, context, rule_true, rule_false = self.init_test().with_format_rules(("True", "True"),
|
||||
("False", "False")).unpack()
|
||||
@@ -337,106 +262,6 @@ class TestSheerkaRuleManager(TestUsingMemoryBasedSheerka):
|
||||
unresolved.metadata.id_is_unresolved = True
|
||||
assert sheerka.resolve_rule(context, unresolved) == rule
|
||||
|
||||
@pytest.mark.parametrize("obj, expected", [
|
||||
("text value", "var == 'text value'"),
|
||||
("text 'value'", '''var == "text 'value'"'''),
|
||||
('text "value"', """var == 'text "value"'"""),
|
||||
(10, "var == 10"),
|
||||
(10.01, "var == 10.01"),
|
||||
])
|
||||
def test_i_can_test_python_code_emitter_for_basic_types(self, obj, expected):
|
||||
sheerka, context = self.init_test().unpack()
|
||||
|
||||
assert PythonCodeEmitter(context).recognize(obj, "var").get_text() == expected
|
||||
assert PythonCodeEmitter(context, "status").recognize(obj, "var").get_text() == "status and " + expected
|
||||
|
||||
@pytest.mark.parametrize("recognized_by, expected", [
|
||||
(RECOGNIZED_BY_ID, "isinstance(var, Concept) and var.id == '1001'"),
|
||||
(RECOGNIZED_BY_NAME, "isinstance(var, Concept) and var.name == 'greetings'"),
|
||||
(None, "isinstance(var, Concept) and var.key == 'hello'"),
|
||||
])
|
||||
def test_i_can_test_python_code_emitter_for_concepts(self, recognized_by, expected):
|
||||
sheerka, context, foo = self.init_concepts(
|
||||
Concept("greetings", definition="hello", definition_type=DEFINITION_TYPE_DEF))
|
||||
|
||||
instance = sheerka.new_from_template(foo, foo.key)
|
||||
if recognized_by:
|
||||
instance.set_hint(BuiltinConcepts.RECOGNIZED_BY, recognized_by)
|
||||
|
||||
assert PythonCodeEmitter(context).recognize(instance, "var").get_text() == expected
|
||||
assert PythonCodeEmitter(context, "status").recognize(instance, "var").get_text() == "status and " + expected
|
||||
|
||||
def test_i_can_test_python_code_emitter_for_concepts_with_variable(self):
|
||||
sheerka, context, greetings, little, foo, bar, and_concept = self.init_concepts(
|
||||
Concept("greetings", definition="hello a", definition_type=DEFINITION_TYPE_DEF).def_var("a"),
|
||||
Concept("little x").def_var("x"),
|
||||
"foo",
|
||||
"bar",
|
||||
Concept("a and b").def_var("a").def_var("b")
|
||||
)
|
||||
|
||||
# variable is a string
|
||||
greetings_instance = sheerka.new_from_template(greetings, greetings.key, a='sheerka')
|
||||
expected = "isinstance(var, Concept) and var.key == 'hello __var__0' and var.get_value('a') == 'sheerka'"
|
||||
text = PythonCodeEmitter(context).recognize(greetings_instance, "var").get_text()
|
||||
assert text == expected
|
||||
|
||||
# variable is a concept recognized by id
|
||||
foo_instance = sheerka.new_from_template(foo, foo.key)
|
||||
foo_instance.set_hint(BuiltinConcepts.RECOGNIZED_BY, RECOGNIZED_BY_ID)
|
||||
greetings_instance = sheerka.new_from_template(greetings, greetings.key, a=foo_instance)
|
||||
expected = """__x_00__ = var.get_value('a')
|
||||
isinstance(var, Concept) and var.key == 'hello __var__0' and isinstance(__x_00__, Concept) and __x_00__.id == '1003'"""
|
||||
text = PythonCodeEmitter(context).recognize(greetings_instance, "var").get_text()
|
||||
assert text == expected
|
||||
|
||||
# variable is a concept recognized by name
|
||||
foo_instance = sheerka.new_from_template(foo, foo.key)
|
||||
foo_instance.set_hint(BuiltinConcepts.RECOGNIZED_BY, RECOGNIZED_BY_NAME)
|
||||
greetings_instance = sheerka.new_from_template(greetings, greetings.key, a=foo_instance)
|
||||
expected = """__x_00__ = var.get_value('a')
|
||||
isinstance(var, Concept) and var.key == 'hello __var__0' and isinstance(__x_00__, Concept) and __x_00__.name == 'foo'"""
|
||||
text = PythonCodeEmitter(context).recognize(greetings_instance, "var").get_text()
|
||||
assert text == expected
|
||||
|
||||
# variable is a concept recognized by value
|
||||
foo_instance = sheerka.new_from_template(foo, foo.key)
|
||||
greetings_instance = sheerka.new_from_template(greetings, greetings.key, a=foo_instance)
|
||||
expected = """__x_00__ = var.get_value('a')
|
||||
isinstance(var, Concept) and var.key == 'hello __var__0' and isinstance(__x_00__, Concept) and __x_00__.key == 'foo'"""
|
||||
text = PythonCodeEmitter(context).recognize(greetings_instance, "var").get_text()
|
||||
assert text == expected
|
||||
|
||||
# variable is a concept witch has itself some variable
|
||||
foo_instance = sheerka.new_from_template(foo, foo.key)
|
||||
little_instance = sheerka.new_from_template(little, little.key, x=foo_instance)
|
||||
greetings_instance = sheerka.new_from_template(greetings, greetings.key, a=little_instance)
|
||||
expected = """__x_00__ = var.get_value('a')
|
||||
__x_01__ = __x_00__.get_value('x')
|
||||
isinstance(var, Concept) and var.key == 'hello __var__0'""" + \
|
||||
" and isinstance(__x_00__, Concept) and __x_00__.key == 'little __var__0'" + \
|
||||
" and isinstance(__x_01__, Concept) and __x_01__.key == 'foo'"""
|
||||
text = PythonCodeEmitter(context).recognize(greetings_instance, "var").get_text()
|
||||
assert text == expected
|
||||
|
||||
# concept with multiple variables (which are themselves concepts)
|
||||
foo_instance = sheerka.new_from_template(foo, foo.key)
|
||||
bar_instance = sheerka.new_from_template(bar, bar.key)
|
||||
little_instance = sheerka.new_from_template(little, little.key, x=foo_instance)
|
||||
and_instance = sheerka.new_from_template(and_concept, and_concept.key, a=bar_instance, b=little_instance)
|
||||
greetings_instance = sheerka.new_from_template(greetings, greetings.key, a=and_instance)
|
||||
expected = """__x_00__ = var.get_value('a')
|
||||
__x_01__ = __x_00__.get_value('a')
|
||||
__x_02__ = __x_00__.get_value('b')
|
||||
__x_03__ = __x_02__.get_value('x')
|
||||
isinstance(var, Concept) and var.key == 'hello __var__0'""" + \
|
||||
" and isinstance(__x_00__, Concept) and __x_00__.key == '__var__0 and __var__1'" + \
|
||||
" and isinstance(__x_01__, Concept) and __x_01__.key == 'bar'" + \
|
||||
" and isinstance(__x_02__, Concept) and __x_02__.key == 'little __var__0'" + \
|
||||
" and isinstance(__x_03__, Concept) and __x_03__.key == 'foo'"
|
||||
text = PythonCodeEmitter(context).recognize(greetings_instance, "var").get_text()
|
||||
assert text == expected
|
||||
|
||||
def test_i_can_get_format_rules(self):
|
||||
sheerka, context = self.init_test().unpack()
|
||||
service = sheerka.services[SheerkaRuleManager.NAME]
|
||||
@@ -674,3 +499,103 @@ class TestSheerkaRuleManagerUsingFileBasedSheerka(TestUsingFileBasedSheerka):
|
||||
assert rule.compiled_conditions == expected.compiled_conditions
|
||||
assert rule.priority is not None
|
||||
assert rule.priority == expected.priority
|
||||
|
||||
# @pytest.mark.parametrize("obj, expected", [
|
||||
# ("text value", "var == 'text value'"),
|
||||
# ("text 'value'", '''var == "text 'value'"'''),
|
||||
# ('text "value"', """var == 'text "value"'"""),
|
||||
# (10, "var == 10"),
|
||||
# (10.01, "var == 10.01"),
|
||||
# ])
|
||||
# def test_i_can_test_python_code_emitter_for_basic_types(self, obj, expected):
|
||||
# sheerka, context = self.init_test().unpack()
|
||||
#
|
||||
# assert PythonCodeEmitter(context).recognize(obj, "var").get_text() == expected
|
||||
# assert PythonCodeEmitter(context, "status").recognize(obj, "var").get_text() == "status and " + expected
|
||||
#
|
||||
# @pytest.mark.parametrize("recognized_by, expected", [
|
||||
# (RECOGNIZED_BY_ID, "isinstance(var, Concept) and var.id == '1001'"),
|
||||
# (RECOGNIZED_BY_NAME, "isinstance(var, Concept) and var.name == 'greetings'"),
|
||||
# (None, "isinstance(var, Concept) and var.key == 'hello'"),
|
||||
# ])
|
||||
# def test_i_can_test_python_code_emitter_for_concepts(self, recognized_by, expected):
|
||||
# sheerka, context, foo = self.init_concepts(
|
||||
# Concept("greetings", definition="hello", definition_type=DEFINITION_TYPE_DEF))
|
||||
#
|
||||
# instance = sheerka.new_from_template(foo, foo.key)
|
||||
# if recognized_by:
|
||||
# instance.get_hints().recognized_by = recognized_by
|
||||
#
|
||||
# assert PythonCodeEmitter(context).recognize(instance, "var").get_text() == expected
|
||||
# assert PythonCodeEmitter(context, "status").recognize(instance, "var").get_text() == "status and " + expected
|
||||
#
|
||||
# def test_i_can_test_python_code_emitter_for_concepts_with_variable(self):
|
||||
# sheerka, context, greetings, little, foo, bar, and_concept = self.init_concepts(
|
||||
# Concept("greetings", definition="hello a", definition_type=DEFINITION_TYPE_DEF).def_var("a"),
|
||||
# Concept("little x").def_var("x"),
|
||||
# "foo",
|
||||
# "bar",
|
||||
# Concept("a and b").def_var("a").def_var("b")
|
||||
# )
|
||||
#
|
||||
# # variable is a string
|
||||
# greetings_instance = sheerka.new_from_template(greetings, greetings.key, a='sheerka')
|
||||
# expected = "isinstance(var, Concept) and var.key == 'hello __var__0' and var.get_value('a') == 'sheerka'"
|
||||
# text = PythonCodeEmitter(context).recognize(greetings_instance, "var").get_text()
|
||||
# assert text == expected
|
||||
#
|
||||
# # variable is a concept recognized by id
|
||||
# foo_instance = sheerka.new_from_template(foo, foo.key)
|
||||
# foo_instance.get_hints().recognized_by = RECOGNIZED_BY_ID
|
||||
# greetings_instance = sheerka.new_from_template(greetings, greetings.key, a=foo_instance)
|
||||
# expected = """__x_00__ = var.get_value('a')
|
||||
# isinstance(var, Concept) and var.key == 'hello __var__0' and isinstance(__x_00__, Concept) and __x_00__.id == '1003'"""
|
||||
# text = PythonCodeEmitter(context).recognize(greetings_instance, "var").get_text()
|
||||
# assert text == expected
|
||||
#
|
||||
# # variable is a concept recognized by name
|
||||
# foo_instance = sheerka.new_from_template(foo, foo.key)
|
||||
# foo_instance.get_hints().recognized_by = RECOGNIZED_BY_NAME
|
||||
# greetings_instance = sheerka.new_from_template(greetings, greetings.key, a=foo_instance)
|
||||
# expected = """__x_00__ = var.get_value('a')
|
||||
# isinstance(var, Concept) and var.key == 'hello __var__0' and isinstance(__x_00__, Concept) and __x_00__.name == 'foo'"""
|
||||
# text = PythonCodeEmitter(context).recognize(greetings_instance, "var").get_text()
|
||||
# assert text == expected
|
||||
#
|
||||
# # variable is a concept recognized by value
|
||||
# foo_instance = sheerka.new_from_template(foo, foo.key)
|
||||
# greetings_instance = sheerka.new_from_template(greetings, greetings.key, a=foo_instance)
|
||||
# expected = """__x_00__ = var.get_value('a')
|
||||
# isinstance(var, Concept) and var.key == 'hello __var__0' and isinstance(__x_00__, Concept) and __x_00__.key == 'foo'"""
|
||||
# text = PythonCodeEmitter(context).recognize(greetings_instance, "var").get_text()
|
||||
# assert text == expected
|
||||
#
|
||||
# # variable is a concept witch has itself some variable
|
||||
# foo_instance = sheerka.new_from_template(foo, foo.key)
|
||||
# little_instance = sheerka.new_from_template(little, little.key, x=foo_instance)
|
||||
# greetings_instance = sheerka.new_from_template(greetings, greetings.key, a=little_instance)
|
||||
# expected = """__x_00__ = var.get_value('a')
|
||||
# __x_01__ = __x_00__.get_value('x')
|
||||
# isinstance(var, Concept) and var.key == 'hello __var__0'""" + \
|
||||
# " and isinstance(__x_00__, Concept) and __x_00__.key == 'little __var__0'" + \
|
||||
# " and isinstance(__x_01__, Concept) and __x_01__.key == 'foo'"""
|
||||
# text = PythonCodeEmitter(context).recognize(greetings_instance, "var").get_text()
|
||||
# assert text == expected
|
||||
#
|
||||
# # concept with multiple variables (which are themselves concepts)
|
||||
# foo_instance = sheerka.new_from_template(foo, foo.key)
|
||||
# bar_instance = sheerka.new_from_template(bar, bar.key)
|
||||
# little_instance = sheerka.new_from_template(little, little.key, x=foo_instance)
|
||||
# and_instance = sheerka.new_from_template(and_concept, and_concept.key, a=bar_instance, b=little_instance)
|
||||
# greetings_instance = sheerka.new_from_template(greetings, greetings.key, a=and_instance)
|
||||
# expected = """__x_00__ = var.get_value('a')
|
||||
# __x_01__ = __x_00__.get_value('a')
|
||||
# __x_02__ = __x_00__.get_value('b')
|
||||
# __x_03__ = __x_02__.get_value('x')
|
||||
# isinstance(var, Concept) and var.key == 'hello __var__0'""" + \
|
||||
# " and isinstance(__x_00__, Concept) and __x_00__.key == '__var__0 and __var__1'" + \
|
||||
# " and isinstance(__x_01__, Concept) and __x_01__.key == 'bar'" + \
|
||||
# " and isinstance(__x_02__, Concept) and __x_02__.key == 'little __var__0'" + \
|
||||
# " and isinstance(__x_03__, Concept) and __x_03__.key == 'foo'"
|
||||
# text = PythonCodeEmitter(context).recognize(greetings_instance, "var").get_text()
|
||||
# assert text == expected
|
||||
|
||||
Reference in New Issue
Block a user