kodjo/Sheerka-Old
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fixed #29: Parsers: Implement parsing memoization

Browse Source 
Fixed #77 : Parser: ShortTermMemoryParser should be called separately
Fixed #78 : Remove VariableNode usage
Fixed #79 : ConceptManager: Implement compile caching
Fixed #80 : SheerkaExecute : parsers_key is not correctly computed
Fixed #81 : ValidateConceptEvaluator : Validate concept's where and pre clauses right after the parsing
Fixed #82 : SheerkaIsAManager: isa() failed when the set as a body
Fixed #83 : ValidateConceptEvaluator : Support BNF and SYA Concepts
Fixed #84 : ExpressionParser: Implement the parser as a standard parser
Fixed #85 : Services: Give order to services
Fixed #86 : cannot manage smart_get_attr(the short, color)
This commit is contained in:
Kodjo Sossouvi
2021-06-07 21:14:03 +02:00
parent 1059ce25c5
commit 7dcaa9c111
92 changed files with 4263 additions and 1890 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/parsers/parsers_utils.py
+12 -13
View File
@@ -230,7 +230,7 @@ class CC:
self.exclude_body,
**compiled)
raise NotImplementedError(f"CC, {other=}")
raise Exception(f"Expecting Concept but received {other=}")
class CB:
@@ -269,7 +269,7 @@ class CB:
body = other.body
return CB(concept, body)
raise NotImplementedError(f"CB, {other=}")
raise Exception(f"Expecting Concept but received {other=}")
class CV:
@@ -310,7 +310,7 @@ class CV:
values = get_test_obj_delegate(other.values(), self.values, get_test_obj_delegate)
return CV(concept, **values)
raise NotImplementedError(f"CV, {other=}")
raise Exception(f"Expecting Concept but received {other=}")
class CMV:
@@ -361,7 +361,7 @@ class CMV:
variables = {name: value for name, value in other.get_metadata().variables}
return CMV(concept, **variables)
raise NotImplementedError(f"CMV, {other=}")
raise Exception(f"Expecting Concept but received {other=}")
class CIO:
@@ -408,7 +408,7 @@ class CIO:
if isinstance(other, Concept):
return CIO(other)
raise NotImplementedError(f"CIO, {other=}")
raise Exception(f"Expecting Concept but received {other=}")
class HelperWithPos:
@@ -496,7 +496,7 @@ class SCN(HelperWithPos):
other.start if self.start is not None else None,
other.end if self.end is not None else None)
raise NotImplementedError(f"SCN, {other=}")
raise Exception(f"Expecting SourceCodeNode but received {other=}")
class SCWC(HelperWithPos):
@@ -572,7 +572,7 @@ class SCWC(HelperWithPos):
res.end = other.end
return res
raise NotImplementedError(f"SCWC, {other=}")
raise Exception(f"Expecting SourceCodeWithConceptNode but received {other=}")
@property
def source(self):
@@ -663,7 +663,7 @@ class CN(HelperWithPos):
other.start if self.start is not None else None,
other.end if self.end is not None else None)
raise NotImplementedError(f"CN, {other=}")
raise Exception(f"Expecting ConceptNode but received {other=}")
class CNC(CN):
@@ -737,8 +737,7 @@ class CNC(CN):
other.end if self.end is not None else None,
self.exclude_body,
**compiled)
raise NotImplementedError(f"CNC, {other=}")
raise Exception(f"Expecting ConceptNode but received {other=}")
class UTN(HelperWithPos):
@@ -799,7 +798,7 @@ class UTN(HelperWithPos):
other.start,
other.end)
raise NotImplementedError(f"UTN, {other=}")
raise Exception(f"Expecting UnrecognizedTokensNode but received {other=}")
class RN(HelperWithPos):
@@ -863,7 +862,7 @@ class RN(HelperWithPos):
other.start if self.start is not None else None,
other.end if self.end is not None else None)
raise NotImplementedError(f"RN, {other=}")
raise Exception(f"Expecting RuleNode but received {other=}")
class FN:
@@ -930,7 +929,7 @@ class FN:
return FN(other.first.value, other.last.value, params)
raise NotImplementedError(f"FN, {other=}")
raise Exception(f"Expecting FunctionNode but received {other=}")
@dataclass()
tests/parsers/test_BnfNodeParser.py
+21
View File
@@ -1927,6 +1927,27 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
assert res.status
compare_with_test_object(res.value.value, expected_array)
def test_i_cannot_parse_regex_concept_mixed_with_unrecognized_sya(self):
my_map = {
"hex": self.bnf_concept("hex", RegExMatch("[a-f0-9]{8}")),
"isa": Concept("x is an y", body="isinstance(x, y)", pre="is_question()").def_var("x").def_var("y"),
"isafoo": Concept("x is an foo", body="False", pre="is_question()").def_var("x"),
"q": Concept("q ?", body="question(a)").def_var("q")
}
# I need the concept isafoo to fool SyaNodeParser when parsing the sub text 'is an hex ?'"
# The parser will try to recognize 'is an foo', will fail and will revert the result to UTN()
# It's this UTN that need to be properly handled
sheerka, context, parser = self.init_parser(my_map, init_from_sheerka=True, create_new=True)
sheerka.set_precedence(context, my_map["isa"], my_map["q"])
sheerka.set_precedence(context, my_map["isafoo"], my_map["q"])
text = "01234567 is an hexadecimal ?"
res = parser.parse(context, ParserInput(text))
assert not res.status
# @pytest.mark.parametrize("parser_input, expected", [
# ("one", [
# (True, [CNC("bnf_one", source="one", one="one", body="one")]),
tests/parsers/test_DefConceptParser.py
+32 -36
View File
@@ -8,16 +8,18 @@ from core.concept import DEFINITION_TYPE_BNF, DEFINITION_TYPE_DEF, Concept
from core.global_symbols import NotInit
from core.sheerka.services.SheerkaExecute import ParserInput
from core.tokenizer import Keywords, Tokenizer, LexerError
from parsers.BaseExpressionParser import VariableNode, ExprNode
from parsers.BaseParser import UnexpectedEofParsingError
from parsers.BnfDefinitionParser import BnfDefinitionParser
from parsers.BnfNodeParser import OrderedChoice, ConceptExpression, StrMatch, Sequence, RegExMatch, OneOrMore, \
VariableExpression
from parsers.DefConceptParser import DefConceptParser, NameNode, SyntaxErrorNode, CannotHandleParsingError
from parsers.DefConceptParser import UnexpectedTokenParsingError, DefConceptNode
from parsers.ExpressionParser import ExpressionParser
from parsers.FunctionParser import FunctionParser
from parsers.PythonParser import PythonParser, PythonNode
from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
from tests.parsers.parsers_utils import compute_expected_array, SCWC, CV, compare_with_test_object
from tests.parsers.parsers_utils import compute_expected_array, SCWC, compare_with_test_object, CIO
def get_def_concept(name, where=None, pre=None, post=None, body=None, definition=None, bnf_def=None, ret=None):
@@ -26,9 +28,9 @@ def get_def_concept(name, where=None, pre=None, post=None, body=None, definition
if body:
def_concept.body = get_concept_part(body)
if where:
def_concept.where = get_concept_part(where)
def_concept.where = get_concept_part(where, use_expression=True)
if pre:
def_concept.pre = get_concept_part(pre)
def_concept.pre = get_concept_part(pre, use_expression=True)
if post:
def_concept.post = get_concept_part(post)
if ret:
@@ -46,11 +48,21 @@ def get_def_concept(name, where=None, pre=None, post=None, body=None, definition
return def_concept
def get_concept_part(part):
if isinstance(part, str):
node = PythonNode(part.strip(), ast.parse(part.strip(), mode="eval"))
def get_concept_part(part, use_expression=False):
if use_expression:
node = VariableNode(0, 0, [], part)
return ReturnValueConcept(
who="parsers.DefConcept",
who="parsers.Expression",
status=True,
value=ParserResultConcept(
source=part,
parser=ExpressionParser(),
value=node))
if isinstance(part, str):
node = PythonNode(part.lstrip(), ast.parse(part.lstrip(), mode="eval"))
return ReturnValueConcept(
who="parsers.Python",
status=True,
value=ParserResultConcept(
source=part,
@@ -61,7 +73,7 @@ def get_concept_part(part):
# node = PythonNode(part.strip(), ast.parse(part.strip(), mode="eval"))
nodes = compute_expected_array({}, part.source, [SCWC(part.first, part.last, *part.content)])
return ReturnValueConcept(
who="parsers.DefConcept",
who="parsers.Python",
status=True,
value=ParserResultConcept(
source=part.source,
@@ -70,9 +82,9 @@ def get_concept_part(part):
try_parsed=nodes[0]))
if isinstance(part, PN):
node = PythonNode(part.source.strip(), ast.parse(part.source.strip(), mode=part.mode))
node = PythonNode(part.source.lstrip(), ast.parse(part.source.lstrip(), mode=part.mode))
return ReturnValueConcept(
who="parsers.DefConcept",
who="parsers.Python",
status=True,
value=ParserResultConcept(
source=part.source,
@@ -81,7 +93,7 @@ def get_concept_part(part):
if isinstance(part, PythonNode):
return ReturnValueConcept(
who="parsers.DefConcept",
who="parsers.Python",
status=True,
value=ParserResultConcept(
source=part.source,
@@ -248,7 +260,7 @@ class TestDefConceptParser(TestUsingMemoryBasedSheerka):
assert isinstance(res.value, ParserResultConcept)
part_mapping = "body" if part == "as" else part
args = {part_mapping: get_concept_part("True")}
args = {part_mapping: "True"}
expected = get_def_concept("foo", **args)
assert node == expected
@@ -260,28 +272,6 @@ class TestDefConceptParser(TestUsingMemoryBasedSheerka):
assert not res.status
assert sheerka.isinstance(return_value, BuiltinConcepts.TOO_MANY_ERRORS)
def test_i_can_parse_complex_def_concept_statement(self):
text = """def concept a mult b
where a,b
pre isinstance(a, int) and isinstance(b, int)
as res = a * b
ret a if isinstance(a, Concept) else self
"""
sheerka, context, parser, *concepts = self.init_parser()
res = parser.parse(context, ParserInput(text))
return_value = res.value
expected_concept = get_def_concept(
name="a mult b",
where="a,b\n",
pre="isinstance(a, int) and isinstance(b, int)\n",
body=PN("res = a * b\n", "exec"),
ret="a if isinstance(a, Concept) else self\n"
)
assert res.status
assert isinstance(return_value, ParserResultConcept)
assert return_value.value == expected_concept
def test_i_can_parse_mutilines_declarations(self):
text = """
def concept add one to a as
@@ -547,7 +537,10 @@ from give me the date !
assert isinstance(res.value, ParserResultConcept)
assert isinstance(node, DefConceptNode)
assert sheerka.isinstance(node.where, BuiltinConcepts.RETURN_VALUE)
compare_with_test_object(node.where.body.body, CV(concepts[0], pre=True))
where_condition = node.where.body.body
assert isinstance(where_condition, ExprNode)
concept_found = where_condition.compiled[0].objects["__o_00__"]
compare_with_test_object(concept_found, CIO(concepts[0]))
text = "def concept foo x y pre x is a y"
res = parser.parse(context, ParserInput(text))
@@ -559,7 +552,10 @@ from give me the date !
assert isinstance(res.value, ParserResultConcept)
assert isinstance(node, DefConceptNode)
assert sheerka.isinstance(node.pre, BuiltinConcepts.RETURN_VALUE)
compare_with_test_object(node.pre.body.body, CV(concepts[0], pre=True))
pre_condition = node.pre.body.body
assert isinstance(pre_condition, ExprNode)
concept_found = pre_condition.compiled[0].objects["__o_00__"]
compare_with_test_object(concept_found, CIO(concepts[0]))
def test_i_can_parse_bnf_concept_with_regex(self):
sheerka, context, parser, number = self.init_parser("number")
tests/parsers/test_DefRuleParser.py
+2 -1
View File
@@ -3,12 +3,13 @@ import pytest
from core.builtin_concepts_ids import BuiltinConcepts
from core.concept import Concept
from core.sheerka.services.SheerkaExecute import ParserInput
from core.sheerka.services.SheerkaRuleManager import FormatAstNode, CompiledCondition
from core.sheerka.services.SheerkaRuleManager import CompiledCondition
from core.tokenizer import Tokenizer, Keywords
from core.utils import tokens_are_matching
from parsers.BaseCustomGrammarParser import KeywordNotFound, NameNode, SyntaxErrorNode
from parsers.BaseParser import UnexpectedEofParsingError
from parsers.DefRuleParser import DefRuleParser, DefExecRuleNode, DefFormatRuleNode
from parsers.FormatRuleActionParser import FormatAstNode
from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
cmap = {
tests/parsers/test_ExactConceptParser.py
+18 -18
View File
@@ -62,8 +62,8 @@ class TestExactConceptParser(TestUsingMemoryBasedSheerka):
assert len(results) == 1
assert results[0].status
assert concept_found == concept
assert not concept_found.get_metadata().need_validation
assert not concept_found.get_metadata().is_evaluated
assert not concept_found.get_hints().need_validation
assert not concept_found.get_hints().is_evaluated
def test_i_can_parse_concepts_defined_several_times(self):
sheerka = self.get_sheerka(singleton=True)
@@ -80,11 +80,11 @@ class TestExactConceptParser(TestUsingMemoryBasedSheerka):
assert results[0].status
assert results[0].value.value.name == "hello a"
assert variable_def(results[0].value.value, "a") == "world"
assert results[0].value.value.get_metadata().need_validation
assert results[0].value.value.get_hints().need_validation
assert results[1].status
assert results[1].value.value.name == "hello world"
assert not results[1].value.value.get_metadata().need_validation
assert not results[1].value.value.get_hints().need_validation
def test_i_can_parse_a_concept_with_variables(self):
sheerka = self.get_sheerka(singleton=True)
@@ -99,8 +99,8 @@ class TestExactConceptParser(TestUsingMemoryBasedSheerka):
concept_found = results[0].value.value
compare_with_test_object(concept_found, CMV(concept, a="10", b="5"))
assert concept_found.get_metadata().need_validation
assert not concept_found.get_metadata().is_evaluated
assert concept_found.get_hints().need_validation
assert not concept_found.get_hints().is_evaluated
def test_i_can_parse_a_concept_with_duplicate_variables(self):
sheerka = self.get_sheerka(singleton=True)
@@ -115,7 +115,7 @@ class TestExactConceptParser(TestUsingMemoryBasedSheerka):
concept_found = results[0].value.value
compare_with_test_object(concept_found, CMV(concept, a="10", b="5"))
assert concept_found.get_metadata().need_validation
assert concept_found.get_hints().need_validation
def test_i_can_parse_concept_when_defined_using_from_def(self):
sheerka, context, plus = self.init_concepts(
@@ -129,8 +129,8 @@ class TestExactConceptParser(TestUsingMemoryBasedSheerka):
assert len(results) == 1
assert results[0].status
compare_with_test_object(concept_found, CMV(plus, a="10", b="5"))
assert concept_found.get_metadata().need_validation
assert not concept_found.get_metadata().is_evaluated
assert concept_found.get_hints().need_validation
assert not concept_found.get_hints().is_evaluated
def test_i_can_parse_concept_token(self):
sheerka, context, foo = self.init_concepts("foo")
@@ -142,8 +142,8 @@ class TestExactConceptParser(TestUsingMemoryBasedSheerka):
assert len(results) == 1
assert results[0].status
assert concept_found == foo
assert not concept_found.get_metadata().need_validation
assert concept_found.get_metadata().is_evaluated
assert not concept_found.get_hints().need_validation
assert concept_found.get_hints().is_evaluated
def test_i_can_parse_concept_with_concept_tokens(self):
sheerka, context, one, two, plus = self.init_concepts(
@@ -159,8 +159,8 @@ class TestExactConceptParser(TestUsingMemoryBasedSheerka):
assert len(results) == 1
assert results[0].status
compare_with_test_object(concept_found, CMV(plus, a="c:one:", b="c:two:"))
assert concept_found.get_metadata().need_validation
assert not concept_found.get_metadata().is_evaluated
assert concept_found.get_hints().need_validation
assert not concept_found.get_hints().is_evaluated
def test_i_can_parse_when_expression_contains_keyword(self):
sheerka, context, isa, def_concept = self.init_concepts(
@@ -175,8 +175,8 @@ class TestExactConceptParser(TestUsingMemoryBasedSheerka):
assert len(results) == 1
assert results[0].status
compare_with_test_object(concept_found, CMV(isa, c="z"))
assert concept_found.get_metadata().need_validation
assert not concept_found.get_metadata().is_evaluated
assert concept_found.get_hints().need_validation
assert not concept_found.get_hints().is_evaluated
source = "def concept z"
results = ExactConceptParser().parse(context, ParserInput(source))
@@ -185,8 +185,8 @@ class TestExactConceptParser(TestUsingMemoryBasedSheerka):
assert len(results) == 1
assert results[0].status
compare_with_test_object(concept_found, CMV(def_concept, a="z"))
assert concept_found.get_metadata().need_validation
assert not concept_found.get_metadata().is_evaluated
assert concept_found.get_hints().need_validation
assert not concept_found.get_hints().is_evaluated
def test_i_can_manage_unknown_concept(self):
context = self.get_context(self.get_sheerka(singleton=True))
@@ -219,4 +219,4 @@ class TestExactConceptParser(TestUsingMemoryBasedSheerka):
# assert len(results) == 1
# assert results[0].status
# assert results[0].value.value == concept
# assert not results[0].value.value.get_metadata().need_validation
# assert not results[0].value.value.get_hints().need_validation
tests/parsers/test_ExpressionParser.py
+24 -4
View File
@@ -3,8 +3,8 @@ import pytest
from core.builtin_concepts_ids import BuiltinConcepts
from core.sheerka.services.SheerkaExecute import ParserInput
from core.tokenizer import Tokenizer
from parsers.BaseExpressionParser import VariableNode, ComparisonNode
from parsers.BaseParser import ErrorSink
from parsers.BaseExpressionParser import VariableNode, ComparisonNode, ExprNode
from parsers.BaseParser import ErrorSink, BaseParser
from parsers.ExpressionParser import ExpressionParser
from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
from tests.parsers.parsers_utils import get_expr_node_from_test_node, VAR, EXPR, FN, AND, NOT, OR, GT, GTE, LT, LTE, EQ, \
@@ -71,7 +71,7 @@ class TestExpressionParser(TestUsingMemoryBasedSheerka):
"var.attr1.attr2",
"var . attr1 . attr2",
])
def test_i_can_parse_variable(self, expression):
def test_i_can_parse_input_variable(self, expression):
sheerka, context, parser, parser_input, error_sink = self.init_parser_with_source(expression)
parsed = parser.parse_input(context, parser_input, error_sink)
@@ -80,7 +80,7 @@ class TestExpressionParser(TestUsingMemoryBasedSheerka):
assert parsed.name == "var"
assert parsed.attributes == ["attr1", "attr2"]
def test_i_can_parse_sub_tokens(self):
def test_i_can_parse_input_sub_tokens(self):
sheerka, context, parser = self.init_parser()
expression = "do not care var1 + var2 do not care either"
@@ -105,3 +105,23 @@ class TestExpressionParser(TestUsingMemoryBasedSheerka):
new_source = ComparisonNode.rebuild_source("new_var", parsed.comp, parsed.right.get_source())
assert new_source == expected
def test_i_cannot_parse_empty_string(self):
sheerka, context, parser = self.init_parser()
res = parser.parse(context, ParserInput(""))
assert not res.status
assert sheerka.isinstance(res.body, BuiltinConcepts.IS_EMPTY)
def test_i_can_compile(self):
sheerka, context, parser = self.init_parser()
text = ParserInput("a > b and c < d")
res = parser.parse(context, text)
assert res.who == BaseParser.PREFIX + ExpressionParser.NAME
assert res.status
assert sheerka.isinstance(res.body, BuiltinConcepts.PARSER_RESULT)
assert isinstance(res.body.body, ExprNode)
assert res.body.body.compiled is not None
tests/parsers/test_FormatRuleActionParser.py
+81
View File
@@ -0,0 +1,81 @@
import pytest
from core.tokenizer import Token, TokenKind
from parsers.FormatRuleActionParser import FormatAstSequence, FormatAstRawText, FormatAstVariable, FormatAstFunction, \
FormatAstList, FormatAstColor, FormatAstDict, FormatAstMulti, FormatRuleActionParser, UnexpectedEof, \
FormatRuleSyntaxError
from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
seq = FormatAstSequence
raw = FormatAstRawText
var = FormatAstVariable
func = FormatAstFunction
lst = FormatAstList
class TestFormatRuleActionParser(TestUsingMemoryBasedSheerka):
@pytest.mark.parametrize("text, expected", [
("", FormatAstRawText("")),
(" ", FormatAstRawText(" ")),
(" raw text ", FormatAstRawText(" raw text ")),
("{variable}", FormatAstVariable("variable")),
("{ variable }", FormatAstVariable("variable")),
(" xy {v} z", seq([raw(" xy "), var("v"), raw(" z")])),
(r"\{variable}", FormatAstRawText("{variable}")),
(r"\\{variable}", seq([raw("\\"), var("variable")])),
(r"\\\{variable}", FormatAstRawText(r"\{variable}")),
(r"{var1}{var2}", seq([var("var1"), var("var2")])),
("func()", FormatAstFunction("func", [], {})),
("func(a, 'string value', c)", FormatAstFunction("func", ["a", "'string value'", "c"], {})),
("func(a=10, b='string value')", FormatAstFunction("func", [], {"a": "10", "b": "'string value'"})),
("func('string value'='another string value')", func("func", [], {"'string value'": "'another string value'"})),
("red(' xy {v}')", FormatAstColor("red", seq([raw(" xy "), var("v")]))),
('blue(" xy {v}")', FormatAstColor("blue", seq([raw(" xy "), var("v")]))),
('green( xy )', FormatAstColor("green", var("xy"))),
('green()', FormatAstColor("green", raw(""))),
('green("")', FormatAstColor("green", raw(""))),
("list(var_name, 2, 'children')", FormatAstList("var_name", recurse_on="children", recursion_depth=2)),
("list(var_name, recursion_depth=2, recurse_on='children')", FormatAstList("var_name",
recurse_on="children",
recursion_depth=2)),
("list(var_name, recursion_depth=2, 'children')", FormatAstList("var_name", recursion_depth=2)),
("list(var_name, 'children', recursion_depth=2)", FormatAstList("var_name", recursion_depth=2)),
("list(var_name)", FormatAstList("var_name")),
("{obj.prop1.prop2[0].prop3['value']}", FormatAstVariable("obj.prop1.prop2[0].prop3['value']")),
("[{id}]", seq([raw("["), var("id"), raw("]")])),
("{variable:format}", FormatAstVariable("variable", "format")),
("{variable:3}", FormatAstVariable("variable", "3")),
(r"\not_a_function(a={var})", seq([raw("not_a_function(a="), var("var"), raw(")")])),
("dict(var_name)", FormatAstDict("var_name")),
("dict(var_name, items_prop='props')", FormatAstDict("var_name", items_prop='props')),
("dict(var_name, debug=True)", FormatAstDict("var_name", debug=True, prefix="{", suffix="}")),
("multi(var_name)", FormatAstMulti("var_name")),
])
def test_i_can_parse_format_rule(self, text, expected):
assert FormatRuleActionParser(text).parse() == expected
@pytest.mark.parametrize("text, expected_error", [
("{", UnexpectedEof("while parsing variable", Token(TokenKind.LBRACE, "{", 0, 1, 1))),
("{var_name", UnexpectedEof("while parsing variable", Token(TokenKind.LBRACE, "{", 0, 1, 1))),
("{}", FormatRuleSyntaxError("variable name not found", None)),
("func(", UnexpectedEof("while parsing function", Token(TokenKind.IDENTIFIER, "func", 0, 1, 1))),
("func(a,b,c", UnexpectedEof("while parsing function", Token(TokenKind.IDENTIFIER, "func", 0, 1, 1))),
("func(a,,c", FormatRuleSyntaxError("no parameter found", Token(TokenKind.COMMA, ",", 7, 1, 8))),
("func(a,,c)", FormatRuleSyntaxError("no parameter found", Token(TokenKind.COMMA, ",", 7, 1, 8))),
("red(a,b)", FormatRuleSyntaxError("only one parameter supported", Token(TokenKind.IDENTIFIER, "b", 6, 1, 7))),
("red(a=b)", FormatRuleSyntaxError("keyword arguments are not supported", None)),
("red(xy {v})", FormatRuleSyntaxError("Invalid identifier", None)),
("list()", FormatRuleSyntaxError("variable name not found", None)),
("list(recursion_depth=2)", FormatRuleSyntaxError("variable name not found", None)),
("list(a,b,c,d,e)", FormatRuleSyntaxError("too many positional arguments",
Token(TokenKind.IDENTIFIER, "e", 13, 1, 14))),
("list(a, recursion_depth=hello)", FormatRuleSyntaxError("'hello' is not numeric", None)),
("list(a, recursion_depth='hello')", FormatRuleSyntaxError("'recursion_depth' must be an integer", None)),
("dict()", FormatRuleSyntaxError("variable name not found", None)),
])
def test_i_cannot_parse_invalid_format(self, text, expected_error):
parser = FormatRuleActionParser(text)
parser.parse()
assert parser.error_sink == expected_error
tests/parsers/test_LogicalOperatorParser.py
+281 -287
View File
@@ -1,21 +1,15 @@
import ast
import pytest
from core.builtin_concepts import BuiltinConcepts, ReturnValueConcept
from core.concept import Concept, DoNotResolve
from core.rule import Rule
from core.builtin_concepts import BuiltinConcepts
from core.sheerka.services.SheerkaExecute import ParserInput
from core.tokenizer import TokenKind
from parsers.BaseExpressionParser import TrueifyVisitor, IsAQuestionVisitor, AndNode, LeftPartNotFoundError, \
from parsers.BaseExpressionParser import TrueifyVisitor, IsAQuestionVisitor, LeftPartNotFoundError, \
ParenthesisMismatchError
from parsers.BaseParser import UnexpectedEofParsingError, UnexpectedTokenParsingError
from parsers.LogicalOperatorParser import LogicalOperatorParser
from parsers.PythonParser import PythonNode
from sheerkarete.network import ReteNetwork
from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
from tests.parsers.parsers_utils import compute_expected_array, resolve_test_concept, EXPR, OR, AND, NOT, \
get_expr_node_from_test_node, get_rete_conditions, CMV, CNC, CC, compare_with_test_object
from tests.parsers.parsers_utils import EXPR, OR, AND, NOT, \
get_expr_node_from_test_node
class TestLogicalOperatorParser(TestUsingMemoryBasedSheerka):
@@ -181,281 +175,281 @@ class TestLogicalOperatorParser(TestUsingMemoryBasedSheerka):
assert IsAQuestionVisitor().visit(expr_node) == expected
@pytest.mark.parametrize("expression, expected", [
("foo", "foo"),
("one two", "one two"),
("foo is a bar", CMV("is a", x='foo', y='bar')),
("one two is a bar", [CNC("is a", "one two is a bar", x="one two", y="bar")]),
("foo is an foo bar",
[CNC("is an", "foo is an foo bar", x=DoNotResolve(value='foo'), exclude_body=True)]),
])
def test_i_can_get_compiled_expr_from_simple_concepts_expressions(self, expression, expected):
concepts_map = {
"foo": Concept("foo"),
"bar": Concept("bar"),
"one two": Concept("one two"),
"is a": Concept("x is a y").def_var("x").def_var("y"),
"is an": Concept("x is an y", definition="('foo'|'bar')=x 'is an' 'foo bar'").def_var("x"),
}
sheerka, context, *concepts = self.init_test().with_concepts(*concepts_map.values(), create_new=True).unpack()
# @pytest.mark.parametrize("expression, expected", [
# ("foo", "foo"),
# ("one two", "one two"),
# ("foo is a bar", CMV("is a", x='foo', y='bar')),
# ("one two is a bar", [CNC("is a", "one two is a bar", x="one two", y="bar")]),
# ("foo is an foo bar",
# [CNC("is an", "foo is an foo bar", x=DoNotResolve(value='foo'), exclude_body=True)]),
# ])
# def test_i_can_get_compiled_expr_from_simple_concepts_expressions(self, expression, expected):
# concepts_map = {
# "foo": Concept("foo"),
# "bar": Concept("bar"),
# "one two": Concept("one two"),
# "is a": Concept("x is a y").def_var("x").def_var("y"),
# "is an": Concept("x is an y", definition="('foo'|'bar')=x 'is an' 'foo bar'").def_var("x"),
# }
# sheerka, context, *concepts = self.init_test().with_concepts(*concepts_map.values(), create_new=True).unpack()
#
# parser = LogicalOperatorParser()
# expr_node = parser.parse(context, ParserInput(expression)).body.body
# return_values, _ = parser.compile_conjunctions(context, [expr_node], "test")
#
# assert len(return_values) == 1
# ret = return_values[0]
#
# if isinstance(expected, list):
# expected_nodes = compute_expected_array(concepts_map, expression, expected)
# compare_with_test_object(ret.body.body, expected_nodes)
# else:
# expected_concept = resolve_test_concept(concepts_map, expected)
# compare_with_test_object(ret.body.body, expected_concept)
parser = LogicalOperatorParser()
expr_node = parser.parse(context, ParserInput(expression)).body.body
return_values, _ = parser.compile_conjunctions(context, [expr_node], "test")
assert len(return_values) == 1
ret = return_values[0]
if isinstance(expected, list):
expected_nodes = compute_expected_array(concepts_map, expression, expected)
compare_with_test_object(ret.body.body, expected_nodes)
else:
expected_concept = resolve_test_concept(concepts_map, expected)
compare_with_test_object(ret.body.body, expected_concept)
@pytest.mark.parametrize("expression", [
"a == 5",
"foo > 5",
"func() == 5",
"not a == 5",
"not foo > 5",
"not func() == 5",
"isinstance(a, int)",
"func()",
"not isinstance(a, int)",
"not func()"
])
def test_i_can_get_compiled_expr_from_simple_python_expressions(self, expression):
sheerka, context, = self.init_test().unpack()
parser = LogicalOperatorParser()
expr_node = parser.parse(context, ParserInput(expression)).body.body
return_values, _ = parser.compile_conjunctions(context, [expr_node], "test")
assert len(return_values) == 1
ret = return_values[0]
assert ret.status
python_node = ret.body.body.get_python_node()
_ast = ast.parse(expression, mode="eval")
expected_python_node = PythonNode(expression, _ast)
assert python_node == expected_python_node
@pytest.mark.parametrize("expression", [
"a and not b",
"not b and a",
"__ret and not __ret.status",
])
def test_i_can_compile_negative_conjunctions_when_pure_python(self, expression):
sheerka, context, *concepts = self.init_concepts("foo")
parser = LogicalOperatorParser()
expr_node = parser.parse(context, ParserInput(expression)).body.body
return_values, _ = parser.compile_conjunctions(context, expr_node.parts, "test")
ast_ = ast.parse(expression, "<source>", 'eval')
expected_python_node = PythonNode(expression, ast_)
assert len(return_values) == 1
ret = return_values[0]
assert sheerka.objvalue(ret) == expected_python_node
@pytest.mark.parametrize("expression, text_to_compile", [
("foo bar == 5", "__C__foo0bar__1001__C__ == 5"),
("not foo bar == 5", "not __C__foo0bar__1001__C__ == 5"),
])
def test_i_can_get_compiled_expr_from_python_and_concept(self, expression, text_to_compile):
sheerka, context, *concepts = self.init_test().with_concepts(Concept("foo bar"), create_new=True).unpack()
parser = LogicalOperatorParser()
expr_node = parser.parse(context, ParserInput(expression)).body.body
return_values, _ = parser.compile_conjunctions(context, [expr_node], "test")
assert len(return_values) == 1
ret = return_values[0]
assert ret.status
python_node = ret.body.body.get_python_node()
_ast = ast.parse(text_to_compile, mode="eval")
expected_python_node = PythonNode(text_to_compile, _ast, expression)
assert python_node == expected_python_node
def test_i_can_get_compiled_expr_from__mix_of_concepts_and_python(self):
sheerka, context, animal, cat, dog, pet, is_a, is_an = self.init_test().with_concepts(
Concept("animal"),
Concept("a cat"),
Concept("dog"),
Concept("pet"),
Concept("x is a y", pre="is_question()", body="isinstance(x, y)").def_var("x").def_var("y"),
Concept("x is an y", pre="is_question()", definition="('cat'|'bird')=x 'is an' animal").def_var("x"),
create_new=True
).unpack()
parser = LogicalOperatorParser()
expression = "not a cat is a pet and not bird is an animal and not x > 5 and not dog is a pet"
expr_node = parser.parse(context, ParserInput(expression)).body.body
return_values, _ = parser.compile_conjunctions(context, expr_node.parts, "test")
to_compile = 'not __C__00var0000is0a000var001__1005__C__'
to_compile += ' and not __C__00var0000is0an0y__1006__C__'
to_compile += ' and not x > 5'
to_compile += ' and not __C__00var0000is0a000var001__1005_1__C__'
ast_ = ast.parse(to_compile, "<source>", 'eval')
expected_python_node = PythonNode(to_compile, ast_, expression)
assert len(return_values) == 1
ret = return_values[0]
python_node = ret.body.body
assert python_node == expected_python_node
compare_with_test_object(python_node.objects, {
"__C__00var0000is0a000var001__1005__C__": CC(is_a, x=cat, y=pet),
"__C__00var0000is0an0y__1006__C__": CC(is_an, exclude_body=True, x=DoNotResolve("bird"), animal=animal),
"__C__00var0000is0a000var001__1005_1__C__": CMV(is_a, x="dog", y="pet"),
})
def test_i_can_get_compiled_expr_from_mix(self):
sheerka, context, animal, cat, dog, pet, is_a, is_an = self.init_test().with_concepts(
Concept("animal"),
Concept("a cat"),
Concept("dog"),
Concept("pet"),
Concept("x is a y", pre="is_question()", body="isinstance(x, y)").def_var("x").def_var("y"),
Concept("x is an y", pre="is_question()", definition="('cat'|'bird')=x 'is an' animal").def_var("x"),
create_new=True
).unpack()
expression = "a cat is a pet and bird is an animal and x > 5 and dog is a pet"
parser = LogicalOperatorParser()
expr_node = parser.parse(context, ParserInput(expression)).body.body
return_values, _ = parser.compile_conjunctions(context, expr_node.parts, "test")
assert len(return_values) == 1
ret = return_values[0]
to_compile = '__C__00var0000is0a000var001__1005__C__ and __C__00var0000is0an0y__1006__C__ and x > 5 and __C__00var0000is0a000var001__1005_1__C__'
ast_ = ast.parse(to_compile, "<source>", 'eval')
expected_python_node = PythonNode(to_compile, ast_, expression)
python_node = ret.body.body
assert python_node == expected_python_node
compare_with_test_object(python_node.objects, {
"__C__00var0000is0a000var001__1005__C__": CC(is_a, x=cat, y=pet),
"__C__00var0000is0an0y__1006__C__": CC(is_an, exclude_body=True, x=DoNotResolve("bird"), animal=animal),
"__C__00var0000is0a000var001__1005_1__C__": CMV(is_a, x="dog", y="pet"),
})
def test_i_can_get_compiled_expr_when_multiple_choices(self):
sheerka, context, *concepts = self.init_test().with_concepts(
Concept("x is a y", pre="is_question()", body="isinstance(x, y)").def_var("x").def_var("y"),
Concept("x is a y", pre="is_question()", body="isa(x, y)").def_var("x").def_var("y"),
create_new=True
).unpack()
parser = LogicalOperatorParser()
expression = "a is a b"
expr_node = parser.parse(context, ParserInput(expression)).body.body
return_values, _ = parser.compile_conjunctions(context, [expr_node], "test")
assert len(return_values) == 2
ret = return_values[0]
compare_with_test_object(sheerka.objvalue(ret)[0].concept, CMV(concepts[0], x="a", y="b"))
ret = return_values[1]
compare_with_test_object(sheerka.objvalue(ret)[0].concept, CMV(concepts[1], x="a", y="b"))
def test_i_can_get_compiled_expr_from_mix_when_multiple_choices(self):
sheerka, context, *concepts = self.init_test().with_concepts(
Concept("animal"),
Concept("a cat"),
Concept("dog"),
Concept("pet"),
Concept("x is a y", pre="is_question()", body="isinstance(x, y)").def_var("x").def_var("y"),
Concept("x is a y", pre="is_question()", body="isa(x, y)").def_var("x").def_var("y"),
Concept("x is an y", pre="is_question()", definition="('cat'|'bird')=x 'is an' animal").def_var("x"),
create_new=True
).unpack()
expression = "a cat is a pet and bird is an animal and x > 5 and dog is a pet"
parser = LogicalOperatorParser()
expr_node = parser.parse(context, ParserInput(expression)).body.body
return_values, _ = parser.compile_conjunctions(context, expr_node.parts, "test")
assert len(return_values) == 4
trimmed_source = "a cat is a pet and bird is an animal and x > 5 and dog is a pet"
current_ret = return_values[0]
python_source = "__C__00var0000is0a000var001__1005__C__ and __C__00var0000is0an0y__1007__C__ and x > 5 and __C__00var0000is0a000var001__1005_1__C__"
ast_ = ast.parse(python_source, "<source>", 'eval')
resolved_expected = PythonNode(python_source, ast_, trimmed_source)
assert sheerka.objvalue(current_ret) == resolved_expected
current_ret = return_values[1]
assert sheerka.isinstance(current_ret, BuiltinConcepts.RETURN_VALUE)
python_source = "__C__00var0000is0a000var001__1005__C__ and __C__00var0000is0an0y__1007__C__ and x > 5 and __C__00var0000is0a000var001__1006__C__"
ast_ = ast.parse(python_source, "<source>", 'eval')
resolved_expected = PythonNode(python_source, ast_, trimmed_source)
assert sheerka.objvalue(current_ret) == resolved_expected
current_ret = return_values[2]
assert sheerka.isinstance(current_ret, BuiltinConcepts.RETURN_VALUE)
python_source = "__C__00var0000is0a000var001__1006__C__ and __C__00var0000is0an0y__1007__C__ and x > 5 and __C__00var0000is0a000var001__1005__C__"
ast_ = ast.parse(python_source, "<source>", 'eval')
resolved_expected = PythonNode(python_source, ast_, trimmed_source)
assert sheerka.objvalue(current_ret) == resolved_expected
current_ret = return_values[3]
python_source = "__C__00var0000is0a000var001__1006__C__ and __C__00var0000is0an0y__1007__C__ and x > 5 and __C__00var0000is0a000var001__1006_1__C__"
ast_ = ast.parse(python_source, "<source>", 'eval')
resolved_expected = PythonNode(python_source, ast_, trimmed_source)
assert sheerka.objvalue(current_ret) == resolved_expected
@pytest.mark.skip
@pytest.mark.parametrize("expression, expected_conditions, test_obj", [
(
"__ret",
["#__x_00__|__name__|'__ret'"],
ReturnValueConcept("Test", True, None)
),
(
"__ret.status == True",
["#__x_00__|__name__|'__ret'", "#__x_00__|status|True"],
ReturnValueConcept("Test", True, None)
),
(
"__ret.status",
["#__x_00__|__name__|'__ret'", "#__x_00__|status|True"],
ReturnValueConcept("Test", True, None)
),
(
"__ret and __ret.status",
["#__x_00__|__name__|'__ret'", "#__x_00__|status|True"],
ReturnValueConcept("Test", True, None)
),
])
def test_i_can_get_rete_condition_from_python(self, expression, expected_conditions, test_obj):
sheerka, context, = self.init_test().unpack()
expected_full_condition = get_rete_conditions(*expected_conditions)
parser = LogicalOperatorParser()
expr_node = parser.parse(context, ParserInput(expression)).body.body
nodes = expr_node.parts if isinstance(expr_node, AndNode) else [expr_node]
_, rete_disjunctions = parser.compile_conjunctions(context, nodes, "test")
assert len(rete_disjunctions) == 1
assert rete_disjunctions == [expected_full_condition]
# check against a Rete network
network = ReteNetwork()
rule = Rule("test", expression, None)
rule.metadata.id = 9999
rule.metadata.is_compiled = True
rule.metadata.is_enabled = True
rule.rete_disjunctions = rete_disjunctions
network.add_rule(rule)
network.add_obj("__ret", test_obj)
matches = list(network.matches)
assert len(matches) > 0
# @pytest.mark.parametrize("expression", [
# "a == 5",
# "foo > 5",
# "func() == 5",
# "not a == 5",
# "not foo > 5",
# "not func() == 5",
# "isinstance(a, int)",
# "func()",
# "not isinstance(a, int)",
# "not func()"
# ])
# def test_i_can_get_compiled_expr_from_simple_python_expressions(self, expression):
# sheerka, context, = self.init_test().unpack()
#
# parser = LogicalOperatorParser()
# expr_node = parser.parse(context, ParserInput(expression)).body.body
# return_values, _ = parser.compile_conjunctions(context, [expr_node], "test")
#
# assert len(return_values) == 1
# ret = return_values[0]
#
# assert ret.status
# python_node = ret.body.body.get_python_node()
# _ast = ast.parse(expression, mode="eval")
# expected_python_node = PythonNode(expression, _ast)
# assert python_node == expected_python_node
#
# @pytest.mark.parametrize("expression", [
# "a and not b",
# "not b and a",
# "__ret and not __ret.status",
# ])
# def test_i_can_compile_negative_conjunctions_when_pure_python(self, expression):
# sheerka, context, *concepts = self.init_concepts("foo")
#
# parser = LogicalOperatorParser()
# expr_node = parser.parse(context, ParserInput(expression)).body.body
# return_values, _ = parser.compile_conjunctions(context, expr_node.parts, "test")
#
# ast_ = ast.parse(expression, "<source>", 'eval')
# expected_python_node = PythonNode(expression, ast_)
#
# assert len(return_values) == 1
# ret = return_values[0]
#
# assert sheerka.objvalue(ret) == expected_python_node
#
# @pytest.mark.parametrize("expression, text_to_compile", [
# ("foo bar == 5", "__C__foo0bar__1001__C__ == 5"),
# ("not foo bar == 5", "not __C__foo0bar__1001__C__ == 5"),
# ])
# def test_i_can_get_compiled_expr_from_python_and_concept(self, expression, text_to_compile):
# sheerka, context, *concepts = self.init_test().with_concepts(Concept("foo bar"), create_new=True).unpack()
#
# parser = LogicalOperatorParser()
# expr_node = parser.parse(context, ParserInput(expression)).body.body
# return_values, _ = parser.compile_conjunctions(context, [expr_node], "test")
#
# assert len(return_values) == 1
# ret = return_values[0]
#
# assert ret.status
# python_node = ret.body.body.get_python_node()
# _ast = ast.parse(text_to_compile, mode="eval")
# expected_python_node = PythonNode(text_to_compile, _ast, expression)
# assert python_node == expected_python_node
#
# def test_i_can_get_compiled_expr_from__mix_of_concepts_and_python(self):
# sheerka, context, animal, cat, dog, pet, is_a, is_an = self.init_test().with_concepts(
# Concept("animal"),
# Concept("a cat"),
# Concept("dog"),
# Concept("pet"),
# Concept("x is a y", pre="is_question()", body="isinstance(x, y)").def_var("x").def_var("y"),
# Concept("x is an y", pre="is_question()", definition="('cat'|'bird')=x 'is an' animal").def_var("x"),
# create_new=True
# ).unpack()
#
# parser = LogicalOperatorParser()
# expression = "not a cat is a pet and not bird is an animal and not x > 5 and not dog is a pet"
# expr_node = parser.parse(context, ParserInput(expression)).body.body
# return_values, _ = parser.compile_conjunctions(context, expr_node.parts, "test")
#
# to_compile = 'not __C__00var0000is0a000var001__1005__C__'
# to_compile += ' and not __C__00var0000is0an0y__1006__C__'
# to_compile += ' and not x > 5'
# to_compile += ' and not __C__00var0000is0a000var001__1005_1__C__'
# ast_ = ast.parse(to_compile, "<source>", 'eval')
# expected_python_node = PythonNode(to_compile, ast_, expression)
#
# assert len(return_values) == 1
# ret = return_values[0]
# python_node = ret.body.body
# assert python_node == expected_python_node
# compare_with_test_object(python_node.objects, {
# "__C__00var0000is0a000var001__1005__C__": CC(is_a, x=cat, y=pet),
# "__C__00var0000is0an0y__1006__C__": CC(is_an, exclude_body=True, x=DoNotResolve("bird"), animal=animal),
# "__C__00var0000is0a000var001__1005_1__C__": CMV(is_a, x="dog", y="pet"),
# })
#
# def test_i_can_get_compiled_expr_from_mix(self):
# sheerka, context, animal, cat, dog, pet, is_a, is_an = self.init_test().with_concepts(
# Concept("animal"),
# Concept("a cat"),
# Concept("dog"),
# Concept("pet"),
# Concept("x is a y", pre="is_question()", body="isinstance(x, y)").def_var("x").def_var("y"),
# Concept("x is an y", pre="is_question()", definition="('cat'|'bird')=x 'is an' animal").def_var("x"),
# create_new=True
# ).unpack()
#
# expression = "a cat is a pet and bird is an animal and x > 5 and dog is a pet"
# parser = LogicalOperatorParser()
# expr_node = parser.parse(context, ParserInput(expression)).body.body
# return_values, _ = parser.compile_conjunctions(context, expr_node.parts, "test")
#
# assert len(return_values) == 1
# ret = return_values[0]
#
# to_compile = '__C__00var0000is0a000var001__1005__C__ and __C__00var0000is0an0y__1006__C__ and x > 5 and __C__00var0000is0a000var001__1005_1__C__'
# ast_ = ast.parse(to_compile, "<source>", 'eval')
# expected_python_node = PythonNode(to_compile, ast_, expression)
#
# python_node = ret.body.body
# assert python_node == expected_python_node
# compare_with_test_object(python_node.objects, {
# "__C__00var0000is0a000var001__1005__C__": CC(is_a, x=cat, y=pet),
# "__C__00var0000is0an0y__1006__C__": CC(is_an, exclude_body=True, x=DoNotResolve("bird"), animal=animal),
# "__C__00var0000is0a000var001__1005_1__C__": CMV(is_a, x="dog", y="pet"),
# })
#
# def test_i_can_get_compiled_expr_when_multiple_choices(self):
# sheerka, context, *concepts = self.init_test().with_concepts(
# Concept("x is a y", pre="is_question()", body="isinstance(x, y)").def_var("x").def_var("y"),
# Concept("x is a y", pre="is_question()", body="isa(x, y)").def_var("x").def_var("y"),
# create_new=True
# ).unpack()
#
# parser = LogicalOperatorParser()
# expression = "a is a b"
# expr_node = parser.parse(context, ParserInput(expression)).body.body
# return_values, _ = parser.compile_conjunctions(context, [expr_node], "test")
#
# assert len(return_values) == 2
#
# ret = return_values[0]
# compare_with_test_object(sheerka.objvalue(ret)[0].concept, CMV(concepts[0], x="a", y="b"))
#
# ret = return_values[1]
# compare_with_test_object(sheerka.objvalue(ret)[0].concept, CMV(concepts[1], x="a", y="b"))
#
# def test_i_can_get_compiled_expr_from_mix_when_multiple_choices(self):
# sheerka, context, *concepts = self.init_test().with_concepts(
# Concept("animal"),
# Concept("a cat"),
# Concept("dog"),
# Concept("pet"),
# Concept("x is a y", pre="is_question()", body="isinstance(x, y)").def_var("x").def_var("y"),
# Concept("x is a y", pre="is_question()", body="isa(x, y)").def_var("x").def_var("y"),
# Concept("x is an y", pre="is_question()", definition="('cat'|'bird')=x 'is an' animal").def_var("x"),
# create_new=True
# ).unpack()
#
# expression = "a cat is a pet and bird is an animal and x > 5 and dog is a pet"
# parser = LogicalOperatorParser()
# expr_node = parser.parse(context, ParserInput(expression)).body.body
# return_values, _ = parser.compile_conjunctions(context, expr_node.parts, "test")
#
# assert len(return_values) == 4
# trimmed_source = "a cat is a pet and bird is an animal and x > 5 and dog is a pet"
#
# current_ret = return_values[0]
# python_source = "__C__00var0000is0a000var001__1005__C__ and __C__00var0000is0an0y__1007__C__ and x > 5 and __C__00var0000is0a000var001__1005_1__C__"
# ast_ = ast.parse(python_source, "<source>", 'eval')
# resolved_expected = PythonNode(python_source, ast_, trimmed_source)
# assert sheerka.objvalue(current_ret) == resolved_expected
#
# current_ret = return_values[1]
# assert sheerka.isinstance(current_ret, BuiltinConcepts.RETURN_VALUE)
# python_source = "__C__00var0000is0a000var001__1005__C__ and __C__00var0000is0an0y__1007__C__ and x > 5 and __C__00var0000is0a000var001__1006__C__"
# ast_ = ast.parse(python_source, "<source>", 'eval')
# resolved_expected = PythonNode(python_source, ast_, trimmed_source)
# assert sheerka.objvalue(current_ret) == resolved_expected
#
# current_ret = return_values[2]
# assert sheerka.isinstance(current_ret, BuiltinConcepts.RETURN_VALUE)
# python_source = "__C__00var0000is0a000var001__1006__C__ and __C__00var0000is0an0y__1007__C__ and x > 5 and __C__00var0000is0a000var001__1005__C__"
# ast_ = ast.parse(python_source, "<source>", 'eval')
# resolved_expected = PythonNode(python_source, ast_, trimmed_source)
# assert sheerka.objvalue(current_ret) == resolved_expected
#
# current_ret = return_values[3]
# python_source = "__C__00var0000is0a000var001__1006__C__ and __C__00var0000is0an0y__1007__C__ and x > 5 and __C__00var0000is0a000var001__1006_1__C__"
# ast_ = ast.parse(python_source, "<source>", 'eval')
# resolved_expected = PythonNode(python_source, ast_, trimmed_source)
# assert sheerka.objvalue(current_ret) == resolved_expected
#
# @pytest.mark.skip
# @pytest.mark.parametrize("expression, expected_conditions, test_obj", [
# (
# "__ret",
# ["#__x_00__|__name__|'__ret'"],
# ReturnValueConcept("Test", True, None)
# ),
# (
# "__ret.status == True",
# ["#__x_00__|__name__|'__ret'", "#__x_00__|status|True"],
# ReturnValueConcept("Test", True, None)
# ),
# (
# "__ret.status",
# ["#__x_00__|__name__|'__ret'", "#__x_00__|status|True"],
# ReturnValueConcept("Test", True, None)
# ),
# (
# "__ret and __ret.status",
# ["#__x_00__|__name__|'__ret'", "#__x_00__|status|True"],
# ReturnValueConcept("Test", True, None)
# ),
# ])
# def test_i_can_get_rete_condition_from_python(self, expression, expected_conditions, test_obj):
# sheerka, context, = self.init_test().unpack()
# expected_full_condition = get_rete_conditions(*expected_conditions)
#
# parser = LogicalOperatorParser()
# expr_node = parser.parse(context, ParserInput(expression)).body.body
#
# nodes = expr_node.parts if isinstance(expr_node, AndNode) else [expr_node]
# _, rete_disjunctions = parser.compile_conjunctions(context, nodes, "test")
#
# assert len(rete_disjunctions) == 1
# assert rete_disjunctions == [expected_full_condition]
#
# # check against a Rete network
# network = ReteNetwork()
# rule = Rule("test", expression, None)
# rule.metadata.id = 9999
# rule.metadata.is_compiled = True
# rule.metadata.is_enabled = True
# rule.rete_disjunctions = rete_disjunctions
# network.add_rule(rule)
#
# network.add_obj("__ret", test_obj)
# matches = list(network.matches)
# assert len(matches) > 0
tests/parsers/test_PythonWithConceptsParser.py
+38
View File
@@ -9,8 +9,10 @@ from core.sheerka.services.SheerkaExecute import ParserInput
from core.tokenizer import Token, TokenKind, Tokenizer
from core.var_ref import VariableRef
from parsers.BaseNodeParser import ConceptNode, UnrecognizedTokensNode, RuleNode, VariableNode
from parsers.BnfNodeParser import BnfNodeParser
from parsers.PythonParser import PythonNode
from parsers.PythonWithConceptsParser import PythonWithConceptsParser
from parsers.SequenceNodeParser import SequenceNodeParser
from parsers.UnrecognizedNodeParser import UnrecognizedNodeParser
from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
from tests.parsers.parsers_utils import get_source_code_node
@@ -226,3 +228,39 @@ class TestPythonWithConceptsParser(TestUsingMemoryBasedSheerka):
assert result_python_node.ast_str == PythonNode.get_dump(expected_ast)
assert result_python_node.original_source == "not foo == 1 and bar < 1"
assert result_python_node.objects == {"__C__foo__C__": foo, "__C__bar__C__": bar}
def test_can_parse_after_unrecognized_bnf(self):
sheerka, context, one, two, twenties = self.init_concepts(
Concept("one", body="1"),
Concept("two", body="2"),
Concept("twenties", definition="'twenty' (one|two)=n", body='20 + n').def_var("n"),
create_new=True
)
bnf_parser_ret_val = BnfNodeParser().parse(context, ParserInput("a + twenty one"))
unrec_node_ret_val = UnrecognizedNodeParser().parse(context, bnf_parser_ret_val.body)
parser = PythonWithConceptsParser()
result = parser.parse(context, unrec_node_ret_val.body)
assert result.status
python_node = result.body.body
assert isinstance(python_node, PythonNode)
assert python_node.source == 'a + __C__twenties__1003__C__'
assert "__C__twenties__1003__C__" in python_node.objects
def test_i_cannot_parse_unrecognized_sequence(self):
sheerka, context, one, two, twenties = self.init_concepts(
Concept("one", body="1"),
Concept("two", body="2"),
Concept("twenties", definition="'twenty' (one|two)=n", body='20 + n').def_var("n"),
create_new=True
)
sequence_parser_ret_val = SequenceNodeParser().parse(context, ParserInput("a + twenty one"))
unrec_node_ret_val = UnrecognizedNodeParser().parse(context, sequence_parser_ret_val.body)
parser = PythonWithConceptsParser()
result = parser.parse(context, unrec_node_ret_val.body)
assert not result.status
tests/parsers/test_SequenceNodeParser.py
+27 -5
View File
@@ -9,7 +9,7 @@ from tests.parsers.parsers_utils import compute_expected_array, CN, CNC, SCN, ge
UTN
class TestAtomsParser(TestUsingMemoryBasedSheerka):
class TestSequenceNodeParser(TestUsingMemoryBasedSheerka):
def init_parser(self, my_map, create_new=False, singleton=True, use_sheerka=False):
sheerka, context, *updated_concepts = self.init_test().with_concepts(
*my_map.values(),
@@ -283,8 +283,8 @@ class TestAtomsParser(TestUsingMemoryBasedSheerka):
@pytest.mark.parametrize("text, expected", [
("hello foo bar",
[
(True, [CNC("hello1", "hello foo ", a="foo "), "bar"]),
(True, [CNC("hello2", "hello foo ", b="foo "), "bar"]),
("a", [CN("hello1", "hello foo "), "bar"]),
("b", [CN("hello2", "hello foo "), "bar"]),
]),
])
def test_i_can_parse_when_unrecognized_yield_multiple_values(self, text, expected):
@@ -303,11 +303,12 @@ class TestAtomsParser(TestUsingMemoryBasedSheerka):
wrapper = res.body
lexer_nodes = res.body.body
assert res.status == expected[0]
assert res.status
assert sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
expected_array = compute_expected_array(concepts_map, text, expected[1])
transformed_nodes = get_test_obj(lexer_nodes, expected_array)
assert transformed_nodes == expected_array
assert lexer_nodes[0].concept.get_metadata().variables == [(expected[0], "foo ")]
@pytest.mark.parametrize("text, expected", [
("1 + twenty one", [SCN("1 + twenty "), "one"]),
@@ -352,7 +353,7 @@ class TestAtomsParser(TestUsingMemoryBasedSheerka):
lexer_nodes = res.body.body
assert res.status
assert lexer_nodes[0].concept.get_metadata().is_evaluated == expected_is_evaluated
assert lexer_nodes[0].concept.get_hints().is_evaluated == expected_is_evaluated
def test_the_parser_always_return_a_new_instance_of_the_concept(self):
concepts_map = {
@@ -383,3 +384,24 @@ class TestAtomsParser(TestUsingMemoryBasedSheerka):
assert not res.status
assert sheerka.isinstance(res.body, BuiltinConcepts.NOT_FOR_ME)
@pytest.mark.parametrize("text", [
"foo",
"foo bar",
"long concept",
"unrecognized foo",
"bar unrecognized",
])
def test_i_correctly_set_up_use_copy(self, text):
concepts_map = {
"foo": Concept("foo"),
"bar": Concept("bar"),
"long concept": Concept("long concept"),
}
sheerka, context, parser = self.init_parser(concepts_map)
res = parser.parse(context, ParserInput(text))
for node in res.body.body:
if hasattr(node, "concept"):
assert node.concept.get_hints().use_copy
tests/parsers/test_TestShortTermMemoryParser.py → tests/parsers/test_ShortTermMemoryParser.py
View File
tests/parsers/test_SyaNodeParser.py
+32 -44
View File
@@ -7,7 +7,6 @@ from core.global_symbols import CONCEPT_COMPARISON_CONTEXT
from core.sheerka.services.SheerkaExecute import ParserInput
from core.tokenizer import Tokenizer
from core.utils import NextIdManager
from parsers.BaseNodeParser import UnrecognizedTokensNode
from parsers.PythonParser import PythonNode
from parsers.SyaNodeParser import SyaNodeParser, SyaConceptParserHelper, SyaAssociativity, \
NoneAssociativeSequenceError, TooManyParametersFoundError, InFixToPostFix, ParenthesisMismatchError
@@ -1081,29 +1080,6 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
assert isinstance(concept_plus_b[0].body.body, PythonNode)
assert concept_suffixed_a == cmap["two"]
@pytest.mark.parametrize("text, expected_status, expected_result", [
("f1(one prefixed) plus f2(suffixed two)", False, [
CNC("plus",
a=SCWC("f1(", ")", CNC("prefixed", a="one")),
b=SCWC("f2(", (")", 1), CNC("suffixed", a="two")))
]),
("one is a concept", True, [CNC("is a concept", c="one")]),
("a is a concept", False, [CNC("is a concept", c=UTN("a"))]),
])
def test_i_can_parse_when_one_result(self, text, expected_status, expected_result):
sheerka, context, parser = self.init_parser()
res = parser.parse(context, ParserInput(text))
wrapper = res.body
lexer_nodes = res.body.body
expected_array = compute_expected_array(cmap, text, expected_result)
assert res.status == expected_status
assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
transformed_nodes = get_test_obj(lexer_nodes, expected_array)
assert transformed_nodes == expected_array
@pytest.mark.parametrize("text", [
"function(suffixed one)",
"function(one plus two mult three)",
@@ -1144,8 +1120,6 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
("one plus two a long other b", [CNC("plus", a="one", b="two"), UTN(" a long other b")]),
("one plus two a long infixed", [CNC("plus", a="one", b="two"), UTN(" a long infixed")]),
("one plus two a long", [CNC("plus", a="one", b="two"), UTN(" a long")]),
("one ? a long infixed : two", [CNC("?", a="one", b=UTN("a long infixed"), c="two")]),
("one ? a long infix : two", [CNC("?", a="one", b=UTN("a long infix"), c="two")]),
])
def test_i_can_almost_parse_when_one_part_is_recognized_but_not_the_rest(self, text, expected_result):
"""
@@ -1195,31 +1169,25 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
assert transformed_nodes == expected_array
# assert lexer_nodes == expected_array
@pytest.mark.parametrize("text, expected_concept, expected_unrecognized", [
("x$!# prefixed", "prefixed", ["a"]),
("suffixed x$!#", "suffixed", ["a"]),
("one infix x$!#", "infix", ["b"]),
("x$!# infix one", "infix", ["a"]),
("x$!# infix z$!#", "infix", ["a", "b"]),
@pytest.mark.parametrize("text, expected_error", [
("x$!# prefixed", "Cannot parse 'x$!#'"),
("suffixed x$!#", "Cannot parse 'x$!#'"),
("one infix x$!#", "Cannot parse 'x$!#'"),
("x$!# infix one", "Cannot parse 'x$!#'"),
("x$!# infix z$!#", ["Cannot parse 'z$!#'", "Cannot parse 'x$!#'"]),
("suffixed alpha beta", "Cannot parse 'alpha beta'"),
("alpha beta prefixed", "Cannot parse 'alpha beta'"),
("one plus alpha beta", "Cannot parse 'alpha beta'"),
])
def test_i_cannot_parse_when_unrecognized(self, text, expected_concept, expected_unrecognized):
def test_i_cannot_parse_when_unrecognized(self, text, expected_error):
sheerka, context, parser = self.init_parser()
res = parser.parse(context, ParserInput(text))
wrapper = res.body
lexer_nodes = res.body.body
expected_end = len(list(Tokenizer(text))) - 2
assert not res.status
assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
expected_array = [CN(cmap[expected_concept], text, 0, expected_end)]
transformed_nodes = get_test_obj(lexer_nodes, expected_array)
assert transformed_nodes == expected_array
# assert lexer_nodes == [CN(cmap[expected_concept], text, 0, expected_end)]
concept_found = lexer_nodes[0].concept
for unrecognized in expected_unrecognized:
assert isinstance(concept_found.get_compiled()[unrecognized], UnrecognizedTokensNode)
assert sheerka.isinstance(wrapper, BuiltinConcepts.ERROR)
assert wrapper.body == expected_error
@pytest.mark.parametrize("text, expected", [
("x$!# suffixed one", [UTN("x$!# ", 0, 4), CN("suffixed __var__0", "suffixed one", 5, 7)]),
@@ -1364,6 +1332,26 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
compare_with_test_object(lexer_nodes, [CN(cmap["suffixed"], text, 0, 6)])
def test_i_correctly_set_up_use_copy(self):
my_map = {
"shirt": Concept("shirt"),
"a x": Concept("a x", ret="x").def_var("x"),
"red x": Concept("red x", ret="x").def_var("x"),
}
sheerka, context, parser = self.init_parser(my_map)
res = parser.parse(context, ParserInput("a red shirt"))
concept_found = res.body.body[0].concept
assert concept_found.get_hints().use_copy
concept_found_x = concept_found.get_compiled()["x"]
assert concept_found_x.get_hints().use_copy
concept_found_x_x = concept_found_x.get_compiled()["x"]
assert concept_found_x_x.get_hints().use_copy
class TestFileBaseSyaNodeParser(TestUsingFileBasedSheerka):
def test_i_can_parse_after_restart(self):
tests/parsers/test_UnrecognizedNodeParser.py
+3 -3
View File
@@ -363,7 +363,7 @@ class TestUnrecognizedNodeParser(TestUsingMemoryBasedSheerka):
assert parser_result.source == expression
assert len(actual_nodes) == 1
assert actual_nodes[0].nodes[
0].concept.get_metadata().is_evaluated # 'a plus b' is recognized as concept definition
0].concept.get_hints().is_evaluated # 'a plus b' is recognized as concept definition
def test_i_can_parse_unrecognized_source_code_with_concept_node_when_var_in_short_term_memory(self):
sheerka, context, parser = self.init_parser()
@@ -373,7 +373,7 @@ class TestUnrecognizedNodeParser(TestUsingMemoryBasedSheerka):
nodes = get_input_nodes_from(concepts_map, expression, source_code_concepts)
parser_input = ParserResultConcept("parsers.xxx", source=expression, value=nodes)
context.add_to_short_term_memory("a", 1)
context.add_to_short_term_memory("a", 1) # -> a plus b is now an instance of the concept
res = parser.parse(context, parser_input)
parser_result = res.body
actual_nodes = res.body.body
@@ -382,7 +382,7 @@ class TestUnrecognizedNodeParser(TestUsingMemoryBasedSheerka):
assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT)
assert parser_result.source == expression
assert len(actual_nodes) == 1
assert not actual_nodes[0].nodes[0].concept.get_metadata().is_evaluated # 'a plus b' need to be evaluated
assert not actual_nodes[0].nodes[0].concept.get_hints().is_evaluated # 'a plus b' need to be evaluated
def test_i_can_parse_unrecognized_sya_concept_that_references_source_code(self):
sheerka, context, parser = self.init_parser()