Fixed #29: Parsers: Implement parsing memoization
Fixed #77 : Parser: ShortTermMemoryParser should be called separately Fixed #78 : Remove VariableNode usage Fixed #79 : ConceptManager: Implement compile caching Fixed #80 : SheerkaExecute : parsers_key is not correctly computed Fixed #81 : ValidateConceptEvaluator : Validate concept's where and pre clauses right after the parsing Fixed #82 : SheerkaIsAManager: isa() failed when the set as a body Fixed #83 : ValidateConceptEvaluator : Support BNF and SYA Concepts Fixed #84 : ExpressionParser: Implement the parser as a standard parser Fixed #85 : Services: Give order to services Fixed #86 : cannot manage smart_get_attr(the short, color)
This commit is contained in:
@@ -7,7 +7,6 @@ from core.global_symbols import CONCEPT_COMPARISON_CONTEXT
|
||||
from core.sheerka.services.SheerkaExecute import ParserInput
|
||||
from core.tokenizer import Tokenizer
|
||||
from core.utils import NextIdManager
|
||||
from parsers.BaseNodeParser import UnrecognizedTokensNode
|
||||
from parsers.PythonParser import PythonNode
|
||||
from parsers.SyaNodeParser import SyaNodeParser, SyaConceptParserHelper, SyaAssociativity, \
|
||||
NoneAssociativeSequenceError, TooManyParametersFoundError, InFixToPostFix, ParenthesisMismatchError
|
||||
@@ -1081,29 +1080,6 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
|
||||
assert isinstance(concept_plus_b[0].body.body, PythonNode)
|
||||
assert concept_suffixed_a == cmap["two"]
|
||||
|
||||
@pytest.mark.parametrize("text, expected_status, expected_result", [
|
||||
("f1(one prefixed) plus f2(suffixed two)", False, [
|
||||
CNC("plus",
|
||||
a=SCWC("f1(", ")", CNC("prefixed", a="one")),
|
||||
b=SCWC("f2(", (")", 1), CNC("suffixed", a="two")))
|
||||
]),
|
||||
("one is a concept", True, [CNC("is a concept", c="one")]),
|
||||
("a is a concept", False, [CNC("is a concept", c=UTN("a"))]),
|
||||
])
|
||||
def test_i_can_parse_when_one_result(self, text, expected_status, expected_result):
|
||||
sheerka, context, parser = self.init_parser()
|
||||
|
||||
res = parser.parse(context, ParserInput(text))
|
||||
wrapper = res.body
|
||||
lexer_nodes = res.body.body
|
||||
|
||||
expected_array = compute_expected_array(cmap, text, expected_result)
|
||||
assert res.status == expected_status
|
||||
assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
|
||||
|
||||
transformed_nodes = get_test_obj(lexer_nodes, expected_array)
|
||||
assert transformed_nodes == expected_array
|
||||
|
||||
@pytest.mark.parametrize("text", [
|
||||
"function(suffixed one)",
|
||||
"function(one plus two mult three)",
|
||||
@@ -1144,8 +1120,6 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
|
||||
("one plus two a long other b", [CNC("plus", a="one", b="two"), UTN(" a long other b")]),
|
||||
("one plus two a long infixed", [CNC("plus", a="one", b="two"), UTN(" a long infixed")]),
|
||||
("one plus two a long", [CNC("plus", a="one", b="two"), UTN(" a long")]),
|
||||
("one ? a long infixed : two", [CNC("?", a="one", b=UTN("a long infixed"), c="two")]),
|
||||
("one ? a long infix : two", [CNC("?", a="one", b=UTN("a long infix"), c="two")]),
|
||||
])
|
||||
def test_i_can_almost_parse_when_one_part_is_recognized_but_not_the_rest(self, text, expected_result):
|
||||
"""
|
||||
@@ -1195,31 +1169,25 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
|
||||
assert transformed_nodes == expected_array
|
||||
# assert lexer_nodes == expected_array
|
||||
|
||||
@pytest.mark.parametrize("text, expected_concept, expected_unrecognized", [
|
||||
("x$!# prefixed", "prefixed", ["a"]),
|
||||
("suffixed x$!#", "suffixed", ["a"]),
|
||||
("one infix x$!#", "infix", ["b"]),
|
||||
("x$!# infix one", "infix", ["a"]),
|
||||
("x$!# infix z$!#", "infix", ["a", "b"]),
|
||||
@pytest.mark.parametrize("text, expected_error", [
|
||||
("x$!# prefixed", "Cannot parse 'x$!#'"),
|
||||
("suffixed x$!#", "Cannot parse 'x$!#'"),
|
||||
("one infix x$!#", "Cannot parse 'x$!#'"),
|
||||
("x$!# infix one", "Cannot parse 'x$!#'"),
|
||||
("x$!# infix z$!#", ["Cannot parse 'z$!#'", "Cannot parse 'x$!#'"]),
|
||||
("suffixed alpha beta", "Cannot parse 'alpha beta'"),
|
||||
("alpha beta prefixed", "Cannot parse 'alpha beta'"),
|
||||
("one plus alpha beta", "Cannot parse 'alpha beta'"),
|
||||
])
|
||||
def test_i_cannot_parse_when_unrecognized(self, text, expected_concept, expected_unrecognized):
|
||||
def test_i_cannot_parse_when_unrecognized(self, text, expected_error):
|
||||
sheerka, context, parser = self.init_parser()
|
||||
|
||||
res = parser.parse(context, ParserInput(text))
|
||||
wrapper = res.body
|
||||
lexer_nodes = res.body.body
|
||||
expected_end = len(list(Tokenizer(text))) - 2
|
||||
|
||||
assert not res.status
|
||||
assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
|
||||
expected_array = [CN(cmap[expected_concept], text, 0, expected_end)]
|
||||
transformed_nodes = get_test_obj(lexer_nodes, expected_array)
|
||||
assert transformed_nodes == expected_array
|
||||
# assert lexer_nodes == [CN(cmap[expected_concept], text, 0, expected_end)]
|
||||
|
||||
concept_found = lexer_nodes[0].concept
|
||||
for unrecognized in expected_unrecognized:
|
||||
assert isinstance(concept_found.get_compiled()[unrecognized], UnrecognizedTokensNode)
|
||||
assert sheerka.isinstance(wrapper, BuiltinConcepts.ERROR)
|
||||
assert wrapper.body == expected_error
|
||||
|
||||
@pytest.mark.parametrize("text, expected", [
|
||||
("x$!# suffixed one", [UTN("x$!# ", 0, 4), CN("suffixed __var__0", "suffixed one", 5, 7)]),
|
||||
@@ -1364,6 +1332,26 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
|
||||
assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
|
||||
compare_with_test_object(lexer_nodes, [CN(cmap["suffixed"], text, 0, 6)])
|
||||
|
||||
def test_i_correctly_set_up_use_copy(self):
|
||||
my_map = {
|
||||
"shirt": Concept("shirt"),
|
||||
"a x": Concept("a x", ret="x").def_var("x"),
|
||||
"red x": Concept("red x", ret="x").def_var("x"),
|
||||
}
|
||||
|
||||
sheerka, context, parser = self.init_parser(my_map)
|
||||
|
||||
res = parser.parse(context, ParserInput("a red shirt"))
|
||||
concept_found = res.body.body[0].concept
|
||||
|
||||
assert concept_found.get_hints().use_copy
|
||||
|
||||
concept_found_x = concept_found.get_compiled()["x"]
|
||||
assert concept_found_x.get_hints().use_copy
|
||||
|
||||
concept_found_x_x = concept_found_x.get_compiled()["x"]
|
||||
assert concept_found_x_x.get_hints().use_copy
|
||||
|
||||
|
||||
class TestFileBaseSyaNodeParser(TestUsingFileBasedSheerka):
|
||||
def test_i_can_parse_after_restart(self):
|
||||
|
||||
Reference in New Issue
Block a user