kodjo/Sheerka-Old
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Implemented FunctionParser

Browse Source
This commit is contained in:
Kodjo Sossouvi
2020-09-17 14:11:09 +02:00
parent 8a866880bc
commit 177a6b1d5f
40 changed files with 1752 additions and 561 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/core/test_SheerkaEvaluateConcept.py
+9
View File
@@ -88,6 +88,15 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
assert evaluated.variables() == {"a": Property("a", expected)}
assert evaluated.metadata.is_evaluated
def test_i_can_evaluate_when_the_body_is_the_name_of_the_concept(self):
# to prove that I can distinguish from a string
sheerka, context, concept = self.init_concepts(Concept("foo", body="'foo'"), eval_body=True, create_new=True)
evaluated = sheerka.evaluate_concept(context, concept)
assert evaluated.key == concept.key
assert evaluated.body == "foo"
def test_i_can_evaluate_metadata_using_do_not_resolve(self):
sheerka, context, concept = self.init_concepts(Concept("foo"), eval_body=True)
concept.compiled[ConceptParts.BODY] = DoNotResolve("do not resolve")
tests/core/test_tokenizer.py
+1 -14
View File
@@ -1,5 +1,5 @@
import pytest
from core.tokenizer import Tokenizer, Token, TokenKind, LexerError, Keywords
from core.tokenizer import Tokenizer, Token, TokenKind, LexerError
def test_i_can_tokenize():
@@ -156,19 +156,6 @@ def test_i_can_parse_numbers(text):
assert tokens[0].value == text
@pytest.mark.parametrize("text, expected", [
("def", Keywords.DEF),
("concept", Keywords.CONCEPT),
("as", Keywords.AS),
("pre", Keywords.PRE),
("post", Keywords.POST)
])
def test_i_can_recognize_keywords(text, expected):
tokens = list(Tokenizer(text))
assert tokens[0].type == TokenKind.KEYWORD
assert tokens[0].value == expected
@pytest.mark.parametrize("text, expected", [
("c:key:", ("key", None)),
("c:key|id:", ("key", "id")),
tests/evaluators/test_LexerNodeEvaluator.py
+2 -3
View File
@@ -27,7 +27,7 @@ class TestLexerNodeEvaluator(TestUsingMemoryBasedSheerka):
for fragment in fragments:
if isinstance(fragment, str):
node = PythonNode(fragment, ast.parse(fragment.strip(), mode="eval"))
nodes.append(SourceCodeNode(node, 0, 0, [], fragment))
nodes.append(SourceCodeNode(0, 0, [], fragment, node))
else:
nodes.append(ConceptNode(fragment, 0, 0, [], fragment.name))
@@ -82,10 +82,9 @@ class TestLexerNodeEvaluator(TestUsingMemoryBasedSheerka):
wrapper = result.body
return_value = result.body.body
assert result.who == evaluator.name
assert result.who == "parsers.PythonWithConcepts"
assert result.status
assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
assert wrapper.parser == evaluator
assert wrapper.source == "foo + 1"
assert return_value == PythonNode('foo + 1', ast.parse("__C__foo__C__ + 1", mode="eval"))
tests/evaluators/test_PythonEvaluator.py
+50
View File
@@ -1,8 +1,12 @@
import ast
import pytest
from core.builtin_concepts import ReturnValueConcept, ParserResultConcept, BuiltinConcepts
from core.concept import Concept, CB, NotInit
from core.sheerka.services.SheerkaExecute import ParserInput
from core.tokenizer import Tokenizer
from evaluators.PythonEvaluator import PythonEvaluator, PythonEvalError
from parsers.BaseNodeParser import SourceCodeNode, SourceCodeWithConceptNode
from parsers.PythonParser import PythonNode, PythonParser
from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
@@ -12,10 +16,28 @@ def get_concept_name(concept):
return concept.name
def get_source_code_node(source_code, concepts=None):
if source_code:
python_node = PythonNode(source_code, ast.parse(source_code, f"<source>", 'eval'))
else:
python_node = PythonNode("", None)
if concepts is None:
tokens = list(Tokenizer(source_code, yield_eof=False))
return SourceCodeNode(0, len(tokens), tokens, python_node=python_node)
else:
python_node.concepts = concepts
scwcn = SourceCodeWithConceptNode(None, None)
scwcn.python_node = python_node
return scwcn
class TestPythonEvaluator(TestUsingMemoryBasedSheerka):
@pytest.mark.parametrize("ret_val, expected", [
(ReturnValueConcept("some_name", True, ParserResultConcept(value=PythonNode("", None))), True),
(ReturnValueConcept("some_name", True, ParserResultConcept(value=get_source_code_node(""))), True),
(ReturnValueConcept("some_name", True, ParserResultConcept(value=get_source_code_node("", {}))), True),
(ReturnValueConcept("some_name", True, ParserResultConcept(value="other thing")), False),
(ReturnValueConcept("some_name", False, "not relevant"), False),
(ReturnValueConcept("some_name", True, Concept()), False)
@@ -39,6 +61,19 @@ class TestPythonEvaluator(TestUsingMemoryBasedSheerka):
assert evaluated.status
assert evaluated.value == expected
@pytest.mark.parametrize("source_code_node, expected", [
(get_source_code_node("1 + 1"), 2),
(get_source_code_node("one + one", {"one": Concept("one", body="1")}), 2)
])
def test_i_can_eval_source_code_node(self, source_code_node, expected):
context = self.get_context()
return_value = context.sheerka.ret("parsers.??", True, ParserResultConcept(value=source_code_node))
evaluated = PythonEvaluator().eval(context, return_value)
assert evaluated.status
assert evaluated.value == expected
def test_i_can_eval_using_context(self):
context = self.get_context()
parsed = PythonParser().parse(context, ParserInput("test_using_context('value for param1', 10)"))
@@ -239,3 +274,18 @@ class TestPythonEvaluator(TestUsingMemoryBasedSheerka):
PythonEvaluator().update_globals_with_context(my_globals, context)
assert my_globals == {"self": foo, "b": "'Initialized!'"}
def test_i_can_use_sheerka_locals(self):
sheerka, context = self.init_concepts()
def func(i):
return i + 1
sheerka.locals["func"] = func
parsed = PythonParser().parse(context, ParserInput("func(10)"))
python_evaluator = PythonEvaluator()
evaluated = python_evaluator.eval(context, parsed)
assert evaluated.status
assert evaluated.value == 11
tests/non_reg/test_sheerka_non_reg.py
+55 -36
View File
@@ -348,8 +348,8 @@ as:
"def concept one as 1",
"def concept two as 2",
"def concept number",
"one isa number",
"two isa number",
"set_isa(one, number)",
"set_isa(two, number)",
"def concept twenties from bnf 'twenty' number as 20 + number"
]),
("When using isa and concept twenty", [
@@ -357,8 +357,8 @@ as:
"def concept two as 2",
"def concept twenty as 20",
"def concept number",
"one isa number",
"two isa number",
"set_isa(one, number)",
"set_isa(two, number)",
"def concept twenties from bnf twenty number as 20 + number"
]),
])
@@ -408,8 +408,8 @@ as:
sheerka.evaluate_user_input("def concept one as 1")
sheerka.evaluate_user_input("def concept two as 2")
sheerka.evaluate_user_input("def concept number")
sheerka.evaluate_user_input("one isa number")
sheerka.evaluate_user_input("two isa number")
sheerka.evaluate_user_input("set_isa(one, number)")
sheerka.evaluate_user_input("set_isa(two, number)")
sheerka.evaluate_user_input("def concept twenties from bnf 'twenty' number as 20 + number")
res = sheerka.evaluate_user_input("twenty one")
@@ -450,8 +450,8 @@ as:
"def concept one as 1",
"def concept twenty as 20",
"def concept number",
"one isa number",
"twenty isa number",
"set_isa(one, number)",
"set_isa(twenty, number)",
"def concept twenties from bnf twenty number as twenty + number"
]
@@ -563,7 +563,7 @@ as:
definitions = [
"def concept two as 2",
"def concept number",
"two isa number",
"set_isa(two, number)",
"def concept plus_one from bnf number=n1 'plus_one' as n1 + 1",
]
@@ -574,15 +574,6 @@ as:
assert res[0].status
assert res[0].body == 3
def test_i_can_say_that_a_concept_isa_another_concept(self):
sheerka = self.get_sheerka()
sheerka.evaluate_user_input("def concept foo")
sheerka.evaluate_user_input("def concept bar")
res = sheerka.evaluate_user_input("foo isa bar")
assert len(res) == 1
assert res[0].status
assert sheerka.isinstance(res[0].body, BuiltinConcepts.SUCCESS)
def test_eval_does_not_break_valid_result(self):
sheerka = self.get_sheerka()
@@ -662,9 +653,9 @@ as:
"def concept three as 3",
"def concept twenty as 20",
"def concept number",
"one isa number",
"two isa number",
"three isa number",
"set_isa(one, number)",
"set_isa(two, number)",
"set_isa(three, number)",
"def concept twenties from bnf twenty number where number <= 2 as twenty + number"
]
@@ -759,7 +750,7 @@ as:
definitions = [
"def concept one as 1",
"def concept number",
"one isa number",
"set_isa(one, number)",
"def concept hundreds from bnf number=n1 'hundred' ('and' number=n2)? where n1<10 and n2<100 as n1 * 100 + n2",
]
@@ -782,7 +773,7 @@ as:
sheerka.evaluate_user_input("def concept two as 2")
sheerka.evaluate_user_input("def concept twenties from bnf 'twenty' (one|two)=unit as 20 + unit")
res = sheerka.evaluate_user_input("twenties isa number")
res = sheerka.evaluate_user_input("set_isa(twenties, number)")
assert len(res) == 1
assert res[0].status
@@ -950,11 +941,11 @@ as:
"def concept two as 2",
"def concept twenty as 20",
"def concept number",
"one isa number",
"two isa number",
"twenty isa number",
"set_isa(one, number)",
"set_isa(two, number)",
"set_isa(twenty, number)",
"def concept twenties from bnf twenty number where number < 10 as twenty + number",
"twenties isa number",
"set_isa(twenties, number)",
]
sheerka = self.init_scenario(init)
@@ -975,7 +966,7 @@ as:
sheerka = self.init_scenario(init)
res = sheerka.evaluate_user_input("last_created_concept() isa number")
res = sheerka.evaluate_user_input("set_isa(last_created_concept(), number)")
assert res[0].status
assert sheerka.isa(sheerka.new("one"), sheerka.new("number"))
@@ -1021,7 +1012,7 @@ as:
"def concept one",
"def concept foo",
"def concept number",
"one isa number",
"set_isa(one, number)",
"def concept x is a y as isa(x,y) pre in_context(BuiltinConcepts.EVAL_QUESTION_REQUESTED)",
"def concept x is a y as set_isa(x,y)",
]
@@ -1041,7 +1032,7 @@ as:
init = [
"def concept one as 1",
"def concept number",
"one isa number",
"set_isa(one, number)",
"def concept one as 10", # to make sure that it won't be rejected because of the cast
"def concept x is a y as isa(x,y) pre in_context(BuiltinConcepts.EVAL_QUESTION_REQUESTED)",
"def concept x is a y as set_isa(x,y)",
@@ -1069,7 +1060,7 @@ as:
"def concept one",
"def concept foo",
"def concept number",
"one isa number",
"set_isa(one, number)",
"def concept q from q ? as question(q)",
"def concept is_a from x is a y as isa(x,y) pre in_context(BuiltinConcepts.EVAL_QUESTION_REQUESTED)",
"set_is_greater_than(BuiltinConcepts.PRECEDENCE, c:is_a:, c:q:)"
@@ -1125,6 +1116,34 @@ as:
assert len(res) == 1
assert res[0].status
def test_i_can_eval_concepts_fed_with_functions(self):
init = [
"def concept inc a as a + 1",
"def concept one as 1"
]
def times_five(i):
return i * 5
sheerka = self.init_scenario(init)
sheerka.locals["times_five"] = times_five
res = sheerka.evaluate_user_input("eval inc times_five(one)")
assert len(res) == 1
assert res[0].status
assert res[0].body == 6
def test_i_can_define_a_concept_when_where_clause_contains_the_name_of_the_variable(self):
init = [
"def concept x is a y as isa(x,y) pre is_question()",
]
sheerka = self.init_scenario(init)
res = sheerka.evaluate_user_input("def concept a x b where a is a number as a + b")
assert len(res) == 1
assert res[0].status
assert sheerka.isinstance(res[0].body, BuiltinConcepts.NEW_CONCEPT)
class TestSheerkaNonRegFile(TestUsingFileBasedSheerka):
def test_i_can_def_several_concepts(self):
@@ -1197,15 +1216,15 @@ class TestSheerkaNonRegFile(TestUsingFileBasedSheerka):
self.init_scenario([
"def concept one as 1",
"def concept number",
"one isa number",
"set_isa(one, number)",
"def concept twenty as 20",
"twenty isa number",
"set_isa(twenty, number)",
"def concept twenties from bnf twenty number where number < 10 as twenty + number",
"twenties isa number",
"set_isa(twenties, number)",
"def concept thirty as 30",
"thirty isa number",
"set_isa(thirty, number)",
"def concept thirties from bnf thirty number where number < 10 as thirty + number",
"thirties isa number",
"set_isa(thirties, number)",
])
sheerka = self.get_sheerka() # another instance
tests/parsers/parsers_utils.py
+11 -3
View File
@@ -1,4 +1,4 @@
from core.concept import CC, Concept, ConceptParts, DoNotResolve
from core.concept import CC, Concept, ConceptParts, DoNotResolve, CIO
from core.tokenizer import Tokenizer, TokenKind, Token
from parsers.BaseNodeParser import scnode, utnode, cnode, SCWC, CNC, short_cnode, SourceCodeWithConceptNode, CN, UTN, \
SCN
@@ -13,7 +13,7 @@ def _index(tokens, expr, index):
:param index:
:return:
"""
expected = [token.value for token in Tokenizer(expr) if token.type != TokenKind.EOF]
expected = [token.str_value for token in Tokenizer(expr) if token.type != TokenKind.EOF]
for i in range(0, len(tokens) - len(expected) + 1):
for j in range(len(expected)):
if tokens[i + j] != expected[j]:
@@ -74,6 +74,14 @@ def get_node(
if isinstance(sub_expr, (scnode, utnode, DoNotResolve)):
return sub_expr
if isinstance(sub_expr, CIO):
sub_expr.set_concept(concepts_map[sub_expr.concept_name])
if sub_expr.source:
node = get_node(concepts_map, expression_as_tokens, sub_expr.source, sya=sya)
sub_expr.start = node.start
sub_expr.end = node.end
return sub_expr
if isinstance(sub_expr, cnode):
# for cnode, map the concept key to the one from concepts_maps if needed
if sub_expr.concept_key.startswith("#"):
@@ -192,7 +200,7 @@ def compute_expected_array(concepts_map, expression, expected, sya=False, init_e
:param exclude_body: do not include ConceptParts.BODY in comparison
:return:
"""
expression_as_tokens = [token.value for token in Tokenizer(expression) if token.type != TokenKind.EOF]
expression_as_tokens = [token.str_value for token in Tokenizer(expression) if token.type != TokenKind.EOF]
return [get_node(
concepts_map,
expression_as_tokens,
tests/parsers/test_AtomsParser.py
+29
View File
@@ -34,6 +34,11 @@ class TestAtomsParser(TestUsingMemoryBasedSheerka):
@pytest.mark.parametrize("text, expected", [
("foo", ["foo"]),
("c:foo:", [CN("foo", source="c:foo:")]),
("c:|1001:", [CN("foo", source="c:|1001:")]),
(" foo", ["foo"]),
("foo ", ["foo"]),
(" foo ", ["foo"]),
("foo bar", ["foo", "bar"]),
("foo bar twenties", ["foo", "bar", "twenties"]),
("a plus b", [CN("plus", 0, 4)]),
@@ -347,3 +352,27 @@ class TestAtomsParser(TestUsingMemoryBasedSheerka):
assert res.status
assert lexer_nodes[0].concept.metadata.is_evaluated == expected_is_evaluated
def test_the_parser_always_return_a_new_instance_of_the_concept(self):
concepts_map = {
"foo": Concept("foo"),
}
sheerka, context, parser = self.init_parser(concepts_map, create_new=True, use_sheerka=True)
res = parser.parse(context, ParserInput("foo"))
assert res.status
assert id(res.body.body[0].concept) != id(sheerka.get_by_name("foo"))
def test_i_can_only_parse_when_the_name_is_an_identifier(self):
# to prove that I can distinguish string from actual concept name
concepts_map = {
"foo": Concept("foo"),
}
sheerka, context, parser = self.init_parser(concepts_map, create_new=True, use_sheerka=True)
res = parser.parse(context, ParserInput("'foo'"))
assert not res.status
assert sheerka.isinstance(res.body, BuiltinConcepts.NOT_FOR_ME)
tests/parsers/test_DefaultParser.py
+56 -20
View File
@@ -6,13 +6,16 @@ from core.builtin_concepts import ParserResultConcept, BuiltinConcepts, ReturnVa
from core.concept import DEFINITION_TYPE_BNF, DEFINITION_TYPE_DEF, Concept, CV
from core.sheerka.services.SheerkaExecute import ParserInput
from core.tokenizer import Keywords, Tokenizer, LexerError
from parsers.BaseNodeParser import SCN, SCWC
from parsers.BnfNodeParser import OrderedChoice, ConceptExpression, StrMatch
from parsers.BnfParser import BnfParser
from parsers.DefaultParser import DefaultParser, NameNode, SyntaxErrorNode, CannotHandleErrorNode, IsaConceptNode
from parsers.DefaultParser import DefaultParser, NameNode, SyntaxErrorNode, CannotHandleErrorNode
from parsers.DefaultParser import UnexpectedTokenErrorNode, DefConceptNode
from parsers.FunctionParser import FunctionParser
from parsers.PythonParser import PythonParser, PythonNode
from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
from tests.parsers.parsers_utils import get_node, compute_expected_array
def get_def_concept(name, where=None, pre=None, post=None, body=None, definition=None, bnf_def=None, ret=None):
@@ -52,6 +55,18 @@ def get_concept_part(part):
parser=PythonParser(),
value=node))
if isinstance(part, FN):
# node = PythonNode(part.strip(), ast.parse(part.strip(), mode="eval"))
nodes = compute_expected_array({}, part.source, [SCWC(part.first, part.last, *part.content)])
return ReturnValueConcept(
who="parsers.Default",
status=True,
value=ParserResultConcept(
source=part.source,
parser=FunctionParser(),
value=nodes[0],
try_parsed=nodes[0]))
if isinstance(part, PN):
node = PythonNode(part.source.strip(), ast.parse(part.source.strip(), mode=part.mode))
return ReturnValueConcept(
@@ -84,6 +99,17 @@ class PN:
mode: str # compilation mode
@dataclass
class FN:
"""
Function Node
"""
source: str
first: str
last: str
content: list
class TestDefaultParser(TestUsingMemoryBasedSheerka):
def init_parser(self, *concepts):
@@ -117,7 +143,7 @@ class TestDefaultParser(TestUsingMemoryBasedSheerka):
def test_i_can_parse_complex_def_concept_statement(self):
text = """def concept a mult b
where a,b
pre isinstance(b, int)
pre isinstance(a, int) and isinstance(b, int)
post isinstance(res, a)
as res = a * b
ret a if isinstance(a, Concept) else self
@@ -128,8 +154,8 @@ ret a if isinstance(a, Concept) else self
expected_concept = get_def_concept(
name="a mult b",
where="a,b\n",
pre="isinstance(b, int)\n",
post="isinstance(res, a)\n",
pre="isinstance(a, int) and isinstance(b, int)\n",
post=FN("isinstance(res, a)\n", "isinstance(", ")", ["res", ", ", "a"]),
body=PN("res = a * b\n", "exec"),
ret="a if isinstance(a, Concept) else self\n"
)
@@ -354,24 +380,21 @@ def concept add one to a as
assert context.sheerka.isinstance(res.value, BuiltinConcepts.NOT_FOR_ME)
assert isinstance(res.value.body[0], CannotHandleErrorNode)
def test_i_can_parse_is_a(self):
text = "the name of my 'concept' isa the name of the set"
sheerka, context, parser = self.init_parser()
res = parser.parse(context, ParserInput(text))
expected = IsaConceptNode([],
concept=NameNode(list(Tokenizer("the name of my 'concept'"))),
set=NameNode(list(Tokenizer("the name of the set"))))
assert res.status
assert res.who == parser.name
assert res.value.source == text
assert isinstance(res.value, ParserResultConcept)
assert res.value.value == expected
# def test_i_can_parse_is_a(self):
# text = "the name of my 'concept' isa the name of the set"
# sheerka, context, parser = self.init_parser()
# res = parser.parse(context, ParserInput(text))
# expected = IsaConceptNode([],
# concept=NameNode(list(Tokenizer("the name of my 'concept'"))),
# set=NameNode(list(Tokenizer("the name of the set"))))
#
# assert res.status
# assert res.who == parser.name
# assert res.value.source == text
# assert isinstance(res.value, ParserResultConcept)
# assert res.value.value == expected
@pytest.mark.parametrize("text", [
"concept",
"isa number",
"name isa",
"def",
"def concept_name"
])
@@ -383,6 +406,19 @@ def concept add one to a as
assert sheerka.isinstance(res.body, BuiltinConcepts.ERROR)
assert isinstance(res.body.body[0], UnexpectedTokenErrorNode)
@pytest.mark.parametrize("text", [
"concept",
"isa number",
"name isa",
])
def test_i_cannot_parse_not_for_me_entries(self, text):
sheerka, context, parser = self.init_parser()
res = parser.parse(context, ParserInput(text))
assert not res.status
assert sheerka.isinstance(res.body, BuiltinConcepts.NOT_FOR_ME)
assert isinstance(res.body.body[0], CannotHandleErrorNode)
@pytest.mark.parametrize("text, error_msg, error_text", [
("'name", "Missing Trailing quote", "'name"),
("foo isa 'name", "Missing Trailing quote", "'name"),
tests/parsers/test_FunctionParser.py
+176
View File
@@ -0,0 +1,176 @@
import pytest
from core.builtin_concepts import BuiltinConcepts
from core.concept import Concept
from core.sheerka.services.SheerkaExecute import ParserInput
from parsers.BaseNodeParser import SCN, SCWC, CN, UTN, CNC
from parsers.FunctionParser import FunctionParser, FN
from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
from tests.parsers.parsers_utils import compute_expected_array
cmap = {
"one": Concept("one"),
"two": Concept("two"),
"twenties": Concept("twenties", definition="'twenty' (one|two)=unit").def_var("unit"),
"plus": Concept("a plus b").def_var("a").def_var("b"),
}
class TestFunctionParser(TestUsingMemoryBasedSheerka):
sheerka = None
@classmethod
def setup_class(cls):
t = cls()
cls.sheerka, context, _ = t.init_parser(cmap)
def init_parser(self, concepts_map=None):
if concepts_map is not None:
sheerka, context, *concepts = self.init_concepts(*concepts_map.values(), create_new=True)
else:
sheerka = TestFunctionParser.sheerka
context = self.get_context(sheerka)
parser = FunctionParser()
return sheerka, context, parser
def test_i_can_detect_empty_expression(self):
sheerka, context, parser = self.init_parser()
res = parser.parse(context, ParserInput(""))
assert not res.status
assert sheerka.isinstance(res.body, BuiltinConcepts.IS_EMPTY)
def test_input_must_be_a_parser_input(self):
sheerka, context, parser = self.init_parser()
parser.parse(context, "not a parser input") is None
def test_i_cannot_parse_when_not_a_function(self):
sheerka, context, parser = self.init_parser()
res = parser.parse(context, ParserInput("not a function"))
assert not res.status
assert sheerka.isinstance(res.body, BuiltinConcepts.NOT_FOR_ME)
@pytest.mark.parametrize("expression, expected", [
("func()", FN("func(", ")", [])),
("concept(one)", FN("concept(", ")", ["one"])),
("func(one)", FN("func(", ")", ["one"])),
("func(a long two, 'three', ;:$*)", FN("func(", ")", ["a long two, ", "'three', ", ";:$*"])),
("func(func1(one), two, func2(func3(), func4(three)))", FN("func(", ")", [
(FN("func1(", ")", ["one"]), ", "),
"two, ",
(FN("func2(", ")", [
(FN("func3(", ")", []), ", "),
(FN("func4(", ")", ["three"]), None),
]), None)
])),
])
def test_i_can_parse_function(self, expression, expected):
sheerka, context, parser = self.init_parser()
parser.reset_parser(context, ParserInput(expression))
res = parser.parse_function()
assert res == expected
@pytest.mark.parametrize("text, expected", [
("func()", SCN("func()")),
(" func()", SCN("func()")),
("func(one)", SCWC("func(", ")", CN("one"))),
("func(one, unknown, two)", SCWC("func(", ")", CN("one"), ", ", UTN("unknown"), (", ", 1), CN("two"))),
("func(one, twenty two)", SCWC("func(", ")", "one", ", ", CN("twenties", source="twenty two"))),
("func(one plus two, three)", SCWC("func(", ")", CNC("plus", a="one", b="two"), ", ", UTN("three"))),
("func(func1(one), two)", SCWC("func(", (")", 1), SCWC("func1(", ")", "one"), ", ", "two"))
])
def test_i_can_parse(self, text, expected):
sheerka, context, parser = self.init_parser()
resolved_expected = compute_expected_array(cmap, text, [expected])[0]
res = parser.parse(context, ParserInput(text))
parser_result = res.body
expression = res.body.body
assert res.status
assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT)
assert expression == resolved_expected
assert expression.python_node is not None
assert expression.return_value is not None
def test_i_can_parse_when_multiple_results_when_requested(self):
sheerka, context, parser = self.init_parser()
parser.longest_concepts_only = False
text = "func(one, twenty two)"
expected = [SCWC("func(", ")", "one", ", ", "twenty ", "two"),
SCWC("func(", ")", "one", ", ", CN("twenties", source="twenty two"))]
all_resolved_expected = compute_expected_array(cmap, text, expected)
results = parser.parse(context, ParserInput(text))
assert len(results) == 2
for res, resolved_expected in zip(results, all_resolved_expected):
parser_result = res.body
expressions = res.body.body
assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT)
assert expressions == resolved_expected
@pytest.mark.parametrize("text, expected_error_type", [
("one", BuiltinConcepts.NOT_FOR_ME),
("$*!", BuiltinConcepts.NOT_FOR_ME),
("func(", BuiltinConcepts.ERROR),
("func(one", BuiltinConcepts.ERROR),
("func(one, two, ", BuiltinConcepts.ERROR),
("func(one) and func(two)", BuiltinConcepts.ERROR),
("one func(one)", BuiltinConcepts.NOT_FOR_ME),
])
def test_i_cannot_parse(self, text, expected_error_type):
sheerka, context, parser = self.init_parser()
res = parser.parse(context, ParserInput(text))
assert not res.status
assert sheerka.isinstance(res.body, expected_error_type)
@pytest.mark.parametrize("text, expected", [
("func(one two)", SCWC("func(", ")", "one", "two")),
])
def test_i_can_detect_non_function(self, text, expected):
sheerka, context, parser = self.init_parser()
resolved_expected = compute_expected_array(cmap, text, [expected])[0]
res = parser.parse(context, ParserInput(text))
parser_result = res.body
expression = res.body.body
assert not res.status
assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT)
assert expression == resolved_expected
assert expression.python_node is None
assert expression.return_value is None
@pytest.mark.parametrize("sequence, expected", [
(None, None),
([["a"]], [["a"]]),
([["a"], ["b", "c"]], [["a"]]),
([["b", "c"], ["a"]], [["a"]]),
([["b", "c"], ["a"], ["d", "e"], ["f"]], [["a"], ["f"]]),
])
def test_i_can_get_the_longest_concept_sequence(self, sequence, expected):
assert FunctionParser.get_longest_concepts(sequence) == expected
def test_concepts_found_are_fully_initialized(self):
sheerka, context, parser = self.init_parser()
res = parser.parse(context, ParserInput("func(one plus three)"))
concept = res.body.body.nodes[0].concept
assert res.status
assert isinstance(concept.compiled["a"], Concept)
# three is not recognized,
# so it will be transformed into list of ReturnValueConcept that indicate how to recognized it
assert isinstance(concept.compiled["b"], list)
for item in concept.compiled["b"]:
assert sheerka.isinstance(item, BuiltinConcepts.RETURN_VALUE)
tests/parsers/test_PythonWithConceptsParser.py
+19
View File
@@ -104,6 +104,25 @@ class TestPythonWithConceptsParser(TestUsingMemoryBasedSheerka):
assert result.status
assert return_value.concepts["__C__foo0et000000__1001__C__"] == foo
def test_i_can_parse_when_multiple_concepts(self):
sheerka, context, foo, bar = self.init_concepts("foo", "bar")
input_return_value = ret_val("func(", foo, ", ", bar, ")")
parser = PythonWithConceptsParser()
result = parser.parse(context, input_return_value.body)
parser_result = result.value
return_value = result.value.value
assert result.status
assert result.who == parser.name
assert context.sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT)
assert parser_result.source == "func(foo, bar)"
assert isinstance(return_value, PythonNode)
assert return_value.source == "func(foo, bar)"
assert return_value.get_dump(return_value.ast_) == to_str_ast("func(__C__foo__1001__C__, __C__bar__1002__C__)")
assert return_value.concepts["__C__foo__1001__C__"] == foo
assert return_value.concepts["__C__bar__1002__C__"] == bar
def test_python_ids_mappings_are_correct_when_concepts_with_the_same_name(self):
context = self.get_context()
foo1 = Concept("foo")
tests/parsers/test_SyaNodeParser.py
+201 -100
View File
@@ -1,14 +1,14 @@
import pytest
from core.builtin_concepts import BuiltinConcepts
from core.concept import Concept, CC
from core.concept import Concept, CIO
from core.sheerka.services.SheerkaComparisonManager import SheerkaComparisonManager
from core.sheerka.services.SheerkaExecute import ParserInput
from core.tokenizer import Tokenizer
from parsers.BaseNodeParser import utnode, ConceptNode, cnode, short_cnode, UnrecognizedTokensNode, \
SCWC, CNC, UTN, SourceCodeWithConceptNode
SCWC, CNC, UTN, SCN, CN
from parsers.PythonParser import PythonNode
from parsers.SyaNodeParser import SyaNodeParser, SyaConceptParserHelper, SyaAssociativity, \
NoneAssociativeSequenceErrorNode, TooManyParametersFound
NoneAssociativeSequenceErrorNode, TooManyParametersFound, InFixToPostFix, ParenthesisMismatchErrorNode
import tests.parsers.parsers_utils
from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
@@ -633,21 +633,25 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
assert res_i.out == expected_array
@pytest.mark.parametrize("expression, expected", [
# I can't manage source code functions :-(
# ("function(one plus three) minus two", []),
# ("function(one plus three) minus two",
# [SCWC("function(", ")", CNC("plus", a="one", b="three")), "two", "minus"]),
("two minus function(one plus three)",
["two", SCWC("function(", ")", CNC("plus", a="one", b="three")), "minus"]),
("func1() minus func2()", [SCN("func1()"), SCN("func2()"), "minus"]),
("func1() comes with func2()", [SCN("func1()"), UTN(" comes with "), SCN("func2()")]),
# ("(one plus two) ", ["one", "two", "plus"]),
# ("(one prefixed) ", ["one", "prefixed"]),
# ("(suffixed one) ", ["one", "suffixed"]),
# ("(one ? two : three)", ["one", "two", "three", "?"]),
# ("square(square(one))", ["one", ("square", 1), "square"]),
# ("square ( square ( one ) )", ["one", ("square", 1), "square"]),
#
# ("square(one plus three) minus two", ["one", "three", "plus", "square", "two", "minus"]),
# ("square( one plus three ) minus two", ["one", "three", "plus", "square", "two", "minus"]),
# ("one minus square( two plus three ) ", ["one", "two", "three", "plus", "square", "minus"]),
#
# ("((one prefixed) prefixed)", ["one", "prefixed", ("prefixed", 1)]),
("(one plus two) ", ["one", "two", "plus"]),
("(one prefixed) ", ["one", "prefixed"]),
("(suffixed one) ", ["one", "suffixed"]),
("(one ? two : three)", ["one", "two", "three", "?"]),
("square(square(one))", ["one", ("square", 1), "square"]),
("square ( square ( one ) )", ["one", ("square", 1), "square"]),
("square(one plus three) minus two", ["one", "three", "plus", "square", "two", "minus"]),
("square( one plus three ) minus two", ["one", "three", "plus", "square", "two", "minus"]),
("one minus square( two plus three ) ", ["one", "two", "three", "plus", "square", "minus"]),
("((one prefixed) prefixed)", ["one", "prefixed", ("prefixed", 1)]),
("( ( one prefixed ) prefixed)", ["one", "prefixed", ("prefixed", 1)]),
("( ( square( one ) prefixed ) prefixed)", ["one", "square", "prefixed", ("prefixed", 1)]),
@@ -666,6 +670,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
])
def test_i_can_pos_fix_when_parenthesis(self, expression, expected):
sheerka, context, parser = self.init_parser()
context.add_to_protected_hints(BuiltinConcepts.DEBUG)
res = parser.infix_to_postfix(context, ParserInput(expression))
expected_array = compute_expected_array(cmap, expression, expected)
@@ -675,34 +680,30 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
@pytest.mark.parametrize("expression, expected_sequences", [
# composition
("function(suffixed one)", [[SCWC("function(", ")", "one", "suffixed")]]),
("function(one prefixed)", [[SCWC("function(", ")", "one", "prefixed")]]),
("function(if one then two else three end)", [[SCWC("function(", ")", "one", "two", "three", "if")]]),
("function(suffixed twenty two)", [
[SCWC("function(", ")", "twenty ", "suffixed", "two")],
[SCWC("function(", ")", short_cnode("twenties", "twenty two"), "suffixed")]]),
("function(twenty two prefixed)", [
[SCWC("function(", ")", "twenty ", "two", "prefixed")],
[SCWC("function(", ")", short_cnode("twenties", "twenty two"), "prefixed")],
]),
("function(if one then twenty two else three end)", [
["')'", "one", "twenty ", "two"], # error
[SCWC("function(", ")", "one", short_cnode("twenties", "twenty two"), "three", "if")]
]),
("func1(func2(one two) three)", [
[SCWC("func1(", (")", 1), SCWC("func2(", ")", "one", "two"), "three")]]),
("function(suffixed one)", [[SCWC("function(", ")", CNC("suffixed", a="one"))]]),
("function(one prefixed)", [[SCWC("function(", ")", CNC("prefixed", a="one"))]]),
("function(if one then two else three end)",
[[SCWC("function(", ")", CNC("if", a="one", b="two", c="three", end=14))]]),
("function(suffixed twenty two)",
[[SCWC("function(", ")", CNC("suffixed", a=CIO("twenties", source="twenty two")))]]),
("function(twenty two prefixed)",
[[SCWC("function(", ")", CNC("prefixed", a=CIO("twenties", source="twenty two")))]]),
("function(if one then twenty two else three end)",
[[SCWC("function(", ")", CNC("if", a="one", b=CIO("twenties", source="twenty two"), c="three", end=16))]]),
("func1(func2(one two) three)",
[[SCWC("func1(", (")", 1), SCWC("func2(", ")", "one", "two"), "three")]]),
("twenty two(suffixed one)", [
["twenty ", SCWC("two(", ")", "one", "suffixed")],
[SCWC("twenty two(", ")", "one", "suffixed")],
["twenty ", SCWC("two(", ")", CNC("suffixed", a="one"))],
[CN("twenties", source="twenty two"), "one", "suffixed"],
]),
("twenty two(one prefixed)", [
["twenty ", SCWC("two(", ")", "one", "prefixed")],
[SCWC("twenty two(", ")", "one", "prefixed")],
["twenty ", SCWC("two(", ")", CNC("prefixed", a="one"))],
[CN("twenties", source="twenty two"), "one", "prefixed"],
]),
("f1(one plus two mult three) plus f2(suffixed x$!# prefixed)", [
[SCWC("f1(", ")", "one", "two", "three", "mult", "plus"),
SCWC("f2(", (")", 1), "x$!#", "prefixed", "suffixed"),
[SCWC("f1(", ")", CN("plus", source="one plus two mult three")),
SCWC("f2(", (")", 1), CN("suffixed", source="suffixed x$!# prefixed")),
("plus", 1)]
]),
@@ -715,12 +716,10 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
# Sequence
("if one then two else three end function(x$!#)", [
["one", "two", "three", "if", SCWC(" function(", ")", "x$!#")]]),
("one prefixed function(two)", [["one", "prefixed", SCWC(" function(", ")", "two")]]),
("suffixed one function(two)", [["one", "suffixed", SCWC(" function(", ")", "two")]]),
(
"func1(suffixed one func2(two))",
[[SCWC("func1(", (")", 1), "one", "suffixed", SCWC(" func2(", ")", "two"))]]),
["one", "two", "three", "if", UTN(" ", start=13, end=13), SCWC("function(", ")", "x$!#")]]),
("one prefixed function(two)", [["one", "prefixed", UTN(" ", start=3, end=3), SCWC("function(", ")", "two")]]),
("suffixed one function(two)", [["one", "suffixed", UTN(" ", start=3, end=3), SCWC("function(", ")", "two")]]),
("func(one, two, three)", [[SCWC("func(", ")", "one", ", ", "two", (", ", 1), "three")]]),
])
def test_i_can_post_fix_when_parenthesis_and_unknown(self, expression, expected_sequences):
sheerka, context, parser = self.init_parser()
@@ -737,6 +736,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
("one plus ( 1 + ", ("(", 4)),
("one( 1 + ", ("(", 1)),
("one ( 1 + ", ("(", 2)),
("function(", ("(", 1)),
("function( 1 + ", ("(", 1)),
("function ( 1 + ", ("(", 2)),
("one plus ) 1 + ", (")", 4)),
@@ -754,7 +754,16 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
res = parser.infix_to_postfix(context, ParserInput(expression))
assert len(res) == 1
assert res[0].errors == [expected]
assert res[0].errors == [ParenthesisMismatchErrorNode(expected)]
def test_i_can_detect_parenthesis_mismatch_error_special_case(self):
sheerka, context, parser = self.init_parser()
expression = "one ? function( : two"
expected = [ParenthesisMismatchErrorNode(("(", 5)), ParenthesisMismatchErrorNode(("(", 5))]
res = parser.infix_to_postfix(context, ParserInput(expression))
assert len(res) == 1
assert res[0].errors == expected
@pytest.mark.parametrize("expression, expected", [
("one ? one two : three", ("?", ":")),
@@ -802,29 +811,6 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
assert len(res) == 1
assert res[0].out == expected_array
def test_i_cannot_post_fix_using_concept_short_name(self):
concepts_map = {
"infixed": self.from_def_concept("infixed", "a infixed b", ["a", "b"]),
"suffixed": self.from_def_concept("suffixed", "suffixed a", ["a"]),
"prefixed": self.from_def_concept("prefixed", "a prefixed", ["a"]),
}
sheerka, context, parser = self.init_parser(concepts_map)
res = parser.infix_to_postfix(context, ParserInput("desc(infixed)"))
assert len(res) == 1
assert isinstance(res[0].out[0], SourceCodeWithConceptNode)
assert res[0].out[0].nodes[0].error == 'Not enough prefix parameters'
res = parser.infix_to_postfix(context, ParserInput("desc(suffixed)"))
assert len(res) == 1
assert isinstance(res[0].out[0], SourceCodeWithConceptNode)
assert res[0].out[0].nodes[0].error == 'Not enough suffix parameters'
res = parser.infix_to_postfix(context, ParserInput("desc(prefixed)"))
assert len(res) == 1
assert isinstance(res[0].out[0], SourceCodeWithConceptNode)
assert res[0].out[0].nodes[0].error == 'Not enough prefix parameters'
@pytest.mark.parametrize("expression", [
"one ? two : three",
"one?two:three",
@@ -861,7 +847,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
expression = "a plus plus equals b"
res = parser.infix_to_postfix(context, ParserInput(expression))
expected_array = tests.parsers.parsers_utils.compute_debug_array(res)
assert expected_array == [
assert len(expected_array) == len([
["T(a)", "C(a plus b)", "C(a plus b)", "T(equals)", "T(b)"],
["T(a)", "C(a plus b)", "C(a plus plus)", "T(equals)", "T(b)"],
["T(a)", "C(a plus b)", "C(a plus equals b)", "T(equals)", "T(b)"],
@@ -871,27 +857,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
["T(a)", "C(a plus equals b)", "C(a plus b)", "T(equals)", "T(b)"],
["T(a)", "C(a plus equals b)", "C(a plus plus)", "T(equals)", "T(b)"],
["T(a)", "C(a plus equals b)", "C(a plus equals b)", "T(equals)", "T(b)"],
]
def test_non_reg(self):
concepts_map = {
"plus": Concept("a plus b").def_var("a").def_var("b"),
"complex infix": Concept("a complex infix b ").def_var("a").def_var("b"),
}
sya_def = {
# concepts_map["plus"]: (1, SyaAssociativity.Right),
# concepts_map["plus plus"]: (1, SyaAssociativity.Right),
# concepts_map["plus equals"]: (1, SyaAssociativity.Right),
}
sheerka, context, parser = self.init_parser(concepts_map, sya_def)
expression = "a plus complex infix b"
res = parser.infix_to_postfix(context, ParserInput(expression))
res = parser.parse(context, ParserInput(expression))
pass
])
def test_i_can_use_string_instead_of_identifier(self):
concepts_map = {
@@ -945,6 +911,81 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
assert len(res) == 1
assert res[0].out == expected_array
@pytest.mark.parametrize("expression, expected_debugs", [
("one", [[" 0:one => PUSH_UNREC"]]),
("one plus two", [[
' 0:one => PUSH_UNREC',
' 1:<ws> => PUSH_UNREC',
' 2:plus(SyaConceptDef(concept=(1005)a plus b, precedence=1, associativity=right)) => ??',
" _: => RECOG [[CN((1001)one)]]",
" _: => POP ConceptNode(concept='(1001)one', source='one', start=0, end=0)",
' 2:plus(SyaConceptDef(concept=(1005)a plus b, precedence=1, associativity=right)) => PUSH',
' 3:<ws> => EAT',
' 4:two => PUSH_UNREC',
' 5:<EOF> => ??',
" _: => RECOG [[CN((1002)two)]]",
" _: => POP ConceptNode(concept='(1002)two', source='two', start=4, end=4)",
' _: => POP SyaConceptParserHelper(concept=(1005)a plus b, start=2, error=None)']]),
("suffixed one", [[
' 0:suffixed(SyaConceptDef(concept=(1009)suffixed a, precedence=1, associativity=right)) => PUSH',
' 1:<ws> => EAT',
' 2:one => PUSH_UNREC',
' 3:<EOF> => ??',
" _: => RECOG [[CN((1001)one)]]",
" _: => POP ConceptNode(concept='(1001)one', source='one', start=2, end=2)",
' _: => POP SyaConceptParserHelper(concept=(1009)suffixed a, start=0, error=None)'
]]),
("one ? twenty one : three", [[
' 0:one => PUSH_UNREC',
' 1:<ws> => PUSH_UNREC',
' 2:?(SyaConceptDef(concept=(1011)a ? b : c, precedence=1, associativity=right)) => ??',
" _: => RECOG [[CN((1001)one)]]",
" _: => POP ConceptNode(concept='(1001)one', source='one', start=0, end=0)",
' 2:?(SyaConceptDef(concept=(1011)a ? b : c, precedence=1, associativity=right)) => PUSH',
' 3:<ws> => EAT',
' 4:twenty => PUSH_UNREC',
' 5:<ws> => PUSH_UNREC',
' 6:one => PUSH_UNREC',
' 7:<ws> => PUSH_UNREC',
' 8:: => ??',
" _: => RECOG [[UTN('twenty '), CN((1001)one)], [CN((1016)twenties)]]",
" _: => POP UnrecognizedTokensNode(source='twenty ', start=4, end=5)",
" _: => POP ConceptNode(concept='(1001)one', source='one', start=6, end=6)",
" _: => => ERROR Too many parameters found for '(1011)a ? b : c' before token 'Token(:)'",
' 8:: => EAT',
], [
' 0:one => PUSH_UNREC',
' 1:<ws> => PUSH_UNREC',
' 2:?(SyaConceptDef(concept=(1011)a ? b : c, precedence=1, associativity=right)) => ??',
' _: => RECOG [[CN((1001)one)]]',
" _: => POP ConceptNode(concept='(1001)one', source='one', start=0, end=0)",
' 2:?(SyaConceptDef(concept=(1011)a ? b : c, precedence=1, associativity=right)) => PUSH',
' 3:<ws> => EAT',
' 4:twenty => PUSH_UNREC',
' 5:<ws> => PUSH_UNREC',
' 6:one => PUSH_UNREC',
' 7:<ws> => PUSH_UNREC',
' 8:: => ??',
" _: => RECOG [[UTN('twenty '), CN((1001)one)], [CN((1016)twenties)]]",
" _: => POP ConceptNode(concept='(1016)twenties', source='twenty one', start=4, end=6, ConceptParts.BODY='DoNotResolve(value='twenty one')', unit='(1001)one')",
' 9:<ws> => EAT',
' 10:three => PUSH_UNREC',
' 11:<EOF> => ??',
' _: => RECOG [[CN((1003)three)]]',
" _: => POP ConceptNode(concept='(1003)three', source='three', start=10, end=10)",
' _: => POP SyaConceptParserHelper(concept=(1011)a ? b : c, start=2, error=None)'
]]),
])
def test_i_can_debug(self, expression, expected_debugs):
sheerka, context, parser = self.init_parser()
context.add_to_private_hints(BuiltinConcepts.DEBUG)
res = parser.infix_to_postfix(context, ParserInput(expression))
assert len(res) == len(expected_debugs)
for res_i, expected_debug in zip(res, expected_debugs):
actual_debug = [str(di) for di in res_i.debug]
assert actual_debug == expected_debug
def test_i_can_parse_when_concept_atom_only(self):
sheerka, context, parser = self.init_parser()
@@ -1032,17 +1073,11 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
assert concept_suffixed_a == cmap["two"]
@pytest.mark.parametrize("text, expected_status, expected_result", [
("function(suffixed one)", True, [
SCWC("function(", ")", CNC("suffixed", 2, 4, a="one"))]),
("function(one plus two mult three)", True, [
SCWC("function(", ")", CNC("plus", 2, 10, a="one", b=CC("mult", a="two", b="three")))]),
("f1(one prefixed) plus f2(suffixed two)", True, [
("f1(one prefixed) plus f2(suffixed two)", False, [
CNC("plus",
a=SCWC("f1(", ")", CNC("prefixed", a="one")),
b=SCWC("f2(", (")", 1), CNC("suffixed", a="two")))
]),
("function(suffixed x$!#)", False, [
SCWC("function(", ")", CNC("suffixed", 2, 7, a="x$!#"))]),
("one is a concept", True, [CNC("is a concept", c="one")]),
("a is a concept", False, [CNC("is a concept", c=UTN("a"))]),
])
@@ -1058,6 +1093,19 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
assert lexer_nodes == expected_array
@pytest.mark.parametrize("text", [
"function(suffixed one)",
"function(one plus two mult three)",
"function(suffixed x$!#)"
])
def test_i_cannot_parse_when_function_only(self, text):
sheerka, context, parser = self.init_parser()
res = parser.parse(context, ParserInput(text))
assert not res.status
assert sheerka.isinstance(res.body, BuiltinConcepts.NOT_FOR_ME)
@pytest.mark.parametrize("text", [
"foo bar (one",
"foo bar one",
@@ -1082,14 +1130,13 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
("one plus two foo bar baz", [CNC("plus", a="one", b="two"), UTN(" foo bar baz")]),
("one plus two foo bar", [CNC("plus", a="one", b="two"), UTN(" foo bar")]),
("foo bar one plus two", [UTN("foo bar "), CNC("plus", a="one", b="two")]),
("foo bar (one plus two", [UTN("foo bar ("), CNC("plus", a="one", b="two")]),
("one plus two a long other b", [CNC("plus", a="one", b="two"), UTN(" a long other b")]),
("one plus two a long infixed", [CNC("plus", a="one", b="two"), UTN(" a long infixed")]),
("one plus two a long", [CNC("plus", a="one", b="two"), UTN(" a long")]),
("one ? a long infixed : two", [CNC("?", a="one", b=UTN("a long infixed"), c="two")]),
("one ? a long infix : two", [CNC("?", a="one", b=UTN("a long infix"), c="two")]),
])
def test_i_cannot_parse_when_one_part_is_recognized_but_not_the_rest(self, text, expected_result):
def test_i_can_almost_parse_when_one_part_is_recognized_but_not_the_rest(self, text, expected_result):
"""
We test that the parsed concept seems like a known one, but it was not.
The parser has to detected that the predication was incorrect
@@ -1194,3 +1241,57 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
assert not res.status
assert sheerka.isinstance(res.body, BuiltinConcepts.IS_EMPTY)
@pytest.mark.parametrize("expression, expected", [
("function(", ([], "function(")),
("before the function(", (["before the "], "function(")),
("one two function(", (["one", "two", UTN(" ", 3, 3)], "function(")),
("one(", ([], "one(")),
("one before the function(", (["one", " before the "], "function(")),
])
def test_i_can_get_functions_names_from_unrecognized(self, expression, expected):
sheerka, context, parser = self.init_parser()
infix_to_postfix = InFixToPostFix(context)
tokens = list(Tokenizer(expression, yield_eof=False))
for pos, token in enumerate(tokens[:-1]):
infix_to_postfix.eat_unrecognized(token, pos)
resolved_to_out = compute_expected_array(cmap, expression, expected[0])
resolved_function_name = compute_expected_array(cmap, expression, [expected[1]])
actual = infix_to_postfix.get_functions_names_from_unrecognized(tokens[-1], len(tokens) - 1)
assert len(actual) == 1
assert actual[0].to_out == resolved_to_out
actual[0].function.fix_source()
assert actual[0].function == resolved_function_name[0]
@pytest.mark.parametrize("expression, expected_list", [
("twenty two function(", [(["twenty ", "two", UTN(" ", 3, 3)], "function("),
([CN("twenties", source="twenty two"), UTN(" ", 3, 3)], "function(")]),
("twenty two(", [(["twenty "], "two("),
([CN("twenties", source="twenty two")], None)]),
])
def test_i_can_get_functions_names_from_unrecognized_when_multiple_results(self, expression, expected_list):
sheerka, context, parser = self.init_parser()
infix_to_postfix = InFixToPostFix(context)
tokens = list(Tokenizer(expression, yield_eof=False))
for pos, token in enumerate(tokens[:-1]):
infix_to_postfix.eat_unrecognized(token, pos)
actual_list = infix_to_postfix.get_functions_names_from_unrecognized(tokens[-1], len(tokens) - 1)
assert len(actual_list) == len(expected_list)
for actual, expected in zip(actual_list, expected_list):
resolved_to_out = compute_expected_array(cmap, expression, expected[0])
assert actual.to_out == resolved_to_out
if actual.function:
actual.function.fix_source()
resolved_function_name = compute_expected_array(cmap, expression, [expected[1]])
assert actual.function == resolved_function_name[0]
else:
assert actual.function is None
tests/parsers/test_UnrecognizedNodeParser.py
+36 -3
View File
@@ -31,9 +31,9 @@ def get_input_nodes_from(my_concepts_map, full_expr, *args):
if isinstance(n, SCWC):
n.first = _get_real_node(n.first)
n.last = _get_real_node(n.first)
n.last = _get_real_node(n.last)
n.content = tuple(_get_real_node(nn) for nn in n.content)
return SourceCodeWithConceptNode(n.first, n.last, list(n.content))
return SourceCodeWithConceptNode(n.first, n.last, list(n.content)).pseudo_fix_source()
if isinstance(n, (UnrecognizedTokensNode, ConceptNode, SourceCodeNode, SourceCodeWithConceptNode)):
return n
@@ -254,6 +254,7 @@ class TestUnrecognizedNodeParser(TestUsingMemoryBasedSheerka):
assert res.status
assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT)
assert parser_result.source == expression
assert len(actual_nodes) == 1
assert actual_nodes[0] == scnode(0, 4, expression)
@@ -270,6 +271,7 @@ class TestUnrecognizedNodeParser(TestUsingMemoryBasedSheerka):
assert not res.status
assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT)
assert parser_result.source == expression
assert len(actual_nodes) == 1
assert actual_nodes[0] == nodes[0]
@@ -287,6 +289,7 @@ class TestUnrecognizedNodeParser(TestUsingMemoryBasedSheerka):
assert res.status
assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT)
assert parser_result.source == expression
assert len(actual_nodes) == 1
expected_array = compute_expected_array(
concepts_map,
@@ -306,6 +309,7 @@ class TestUnrecognizedNodeParser(TestUsingMemoryBasedSheerka):
assert res.status
assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT)
assert parser_result.source == expression
assert len(actual_nodes) == 1
expected_array = compute_expected_array(
@@ -328,8 +332,9 @@ class TestUnrecognizedNodeParser(TestUsingMemoryBasedSheerka):
parser_result = res.body
actual_nodes = res.body.body
assert not res.status # status is False to let PythonWithConceptParser validate the code
assert not res.status # status is False to let PythonWithConceptParser validate the code
assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT)
assert parser_result.source == expression
assert len(actual_nodes) == 1
assert actual_nodes[0].nodes[0].concept.metadata.is_evaluated # 'a plus b' is recognized as concept definition
@@ -348,9 +353,37 @@ class TestUnrecognizedNodeParser(TestUsingMemoryBasedSheerka):
assert not res.status # status is False to let PythonWithConceptParser validate the code
assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT)
assert parser_result.source == expression
assert len(actual_nodes) == 1
assert not actual_nodes[0].nodes[0].concept.metadata.is_evaluated # 'a plus b' need to be evaluated
def test_i_can_parse_unrecognized_sya_concept_that_references_source_code(self):
sheerka, context, parser = self.init_parser()
expression = "hello get_user_name(twenty one)"
tmp_node = CNC("hello_sya",
source="hello get_user_name(twenty one)",
a=SCWC("get_user_name(", ")", CNC("twenties", source="twenty one", unit="one")))
nodes = get_input_nodes_from(concepts_map, expression, tmp_node)
parser_input = ParserResultConcept("parsers.xxx", source=expression, value=nodes)
res = parser.parse(context, parser_input)
parser_result = res.body
actual_nodes = res.body.body
assert res.status
assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT)
assert parser_result.source == expression
assert len(actual_nodes) == 1
expected_array = compute_expected_array(
concepts_map,
expression, [CN("hello_sya", source="hello get_user_name(twenty one)")],
exclude_body=True)
assert actual_nodes == expected_array
assert isinstance(actual_nodes[0].concept.compiled["a"], list)
assert sheerka.isinstance(actual_nodes[0].concept.compiled["a"][0], BuiltinConcepts.RETURN_VALUE)
def test_i_can_parse_sequences(self):
sheerka, context, parser = self.init_parser()