Introduced ParserInput

This commit is contained in:
2020-05-25 18:09:12 +02:00
parent c79403443f
commit 479461c0a4
35 changed files with 768 additions and 480 deletions
+35 -14
View File
@@ -1,6 +1,7 @@
import pytest
from core.builtin_concepts import BuiltinConcepts
from core.concept import Concept, DEFINITION_TYPE_DEF
from core.sheerka.services.SheerkaExecute import ParserInput
from parsers.AtomNodeParser import AtomNodeParser
from parsers.BaseNodeParser import cnode, utnode, CNC, SCN, CN
@@ -26,7 +27,7 @@ class TestAtomsParser(TestUsingMemoryBasedSheerka):
def test_i_cannot_parse_empty_string(self):
sheerka, context, parser = self.init_parser({})
res = parser.parse(context, "")
res = parser.parse(context, ParserInput(""))
assert not res.status
assert sheerka.isinstance(res.body, BuiltinConcepts.IS_EMPTY)
@@ -49,7 +50,7 @@ class TestAtomsParser(TestUsingMemoryBasedSheerka):
}
sheerka, context, parser = self.init_parser(concepts_map, create_new=True, use_sheerka=True)
res = parser.parse(context, text)
res = parser.parse(context, ParserInput(text))
wrapper = res.body
lexer_nodes = res.body.body
@@ -72,7 +73,7 @@ class TestAtomsParser(TestUsingMemoryBasedSheerka):
}
sheerka, context, parser = self.init_parser(concepts_map)
res = parser.parse(context, text)
res = parser.parse(context, ParserInput(text))
wrapper = res.body
lexer_nodes = res.body.body
@@ -110,7 +111,7 @@ class TestAtomsParser(TestUsingMemoryBasedSheerka):
}
sheerka, context, parser = self.init_parser(concepts_map)
res = parser.parse(context, text)
res = parser.parse(context, ParserInput(text))
wrapper = res.body
lexer_nodes = res.body.body
@@ -137,7 +138,7 @@ class TestAtomsParser(TestUsingMemoryBasedSheerka):
}
sheerka, context, parser = self.init_parser(concepts_map)
res = parser.parse(context, text)
res = parser.parse(context, ParserInput(text))
wrapper = res.body
lexer_nodes = res.body.body
@@ -158,7 +159,7 @@ class TestAtomsParser(TestUsingMemoryBasedSheerka):
}
sheerka, context, parser = self.init_parser(concepts_map)
list_of_res = parser.parse(context, text)
list_of_res = parser.parse(context, ParserInput(text))
assert len(list_of_res) == len(expected)
for i, res in enumerate(list_of_res):
@@ -180,7 +181,7 @@ class TestAtomsParser(TestUsingMemoryBasedSheerka):
text = "one two x$!# one two"
sheerka, context, parser = self.init_parser(concepts_map)
list_of_res = parser.parse(context, text)
list_of_res = parser.parse(context, ParserInput(text))
expected = [
(False, ["one", "two", " x$!# ", ("one", 1), ("two", 1)]),
@@ -210,7 +211,7 @@ class TestAtomsParser(TestUsingMemoryBasedSheerka):
sheerka, context, parser = self.init_parser(concepts_map)
sheerka.set_isa(context, sheerka.new("one"), sheerka.new("number"))
res = parser.parse(context, "one")
res = parser.parse(context, ParserInput("one"))
lexer_nodes = res.body.body
expected_array = compute_expected_array(concepts_map, "one", ["one"])
assert lexer_nodes == expected_array
@@ -223,16 +224,36 @@ class TestAtomsParser(TestUsingMemoryBasedSheerka):
sheerka, context, parser = self.init_parser(concepts_map)
res = parser.parse(context, "a special concept")
res = parser.parse(context, ParserInput("a special concept"))
lexer_nodes = res.body.body
expected_array = compute_expected_array(concepts_map, "a special concept", ["a special concept"])
assert lexer_nodes == expected_array
res = parser.parse(context, "isa")
res = parser.parse(context, ParserInput("isa"))
lexer_nodes = res.body.body
expected_array = compute_expected_array(concepts_map, "isa", ["isa"])
assert lexer_nodes == expected_array
def test_i_can_parse_concepts_when_sub_tokens(self):
concepts_map = {
"foo": Concept("foo"),
"bar": Concept("bar"),
}
sheerka, context, parser = self.init_parser(concepts_map, create_new=True, use_sheerka=True)
text = "not recognized foo bar not not recognized"
expected = ["foo", "bar"]
parser_input = ParserInput(text, start=3, end=7)
res = parser.parse(context, parser_input)
wrapper = res.body
lexer_nodes = res.body.body
assert res.status
expected_array = compute_expected_array(concepts_map, text, expected)
assert sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
assert lexer_nodes == expected_array
@pytest.mark.parametrize("text", [
"foo",
f"foo one",
@@ -249,7 +270,7 @@ class TestAtomsParser(TestUsingMemoryBasedSheerka):
}
sheerka, context, parser = self.init_parser(concepts_map)
res = parser.parse(context, text)
res = parser.parse(context, ParserInput(text))
assert not res.status
assert sheerka.isinstance(res.body, BuiltinConcepts.NOT_FOR_ME)
@@ -270,7 +291,7 @@ class TestAtomsParser(TestUsingMemoryBasedSheerka):
}
sheerka, context, parser = self.init_parser(concepts_map, create_new=True, singleton=False)
list_of_res = parser.parse(context, text)
list_of_res = parser.parse(context, ParserInput(text))
assert len(list_of_res) == len(expected)
@@ -293,7 +314,7 @@ class TestAtomsParser(TestUsingMemoryBasedSheerka):
}
sheerka, context, parser = self.init_parser(concepts_map)
res = parser.parse(context, text)
res = parser.parse(context, ParserInput(text))
wrapper = res.body
lexer_nodes = res.body.body
@@ -321,7 +342,7 @@ class TestAtomsParser(TestUsingMemoryBasedSheerka):
}
sheerka, context, parser = self.init_parser(concepts_map, create_new=True, use_sheerka=True)
res = parser.parse(context, text)
res = parser.parse(context, ParserInput(text))
lexer_nodes = res.body.body
assert res.status
+1
View File
@@ -81,3 +81,4 @@ def test_i_can_test_split_iter_parser_indexes():
])
def test_i_can_get_tokens_boundaries(tokens, expected):
assert BaseParser.get_tokens_boundaries(tokens) == expected
+6 -28
View File
@@ -1,6 +1,7 @@
import pytest
from core.builtin_concepts import BuiltinConcepts
from core.concept import Concept, ConceptParts, DoNotResolve, DEFINITION_TYPE_DEF, CC
from core.sheerka.services.SheerkaExecute import ParserInput
from parsers.BaseNodeParser import CNC, UTN, CN
from parsers.BnfNodeParser import BnfNodeParser, StrMatch, TerminalNode, NonTerminalNode, Sequence, OrderedChoice, \
Optional, ZeroOrMore, OneOrMore, ConceptExpression
@@ -95,7 +96,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
parser = BnfNodeParser()
parser.init_from_concepts(context, updated)
parser.reset_parser(context, text)
parser.reset_parser(context, ParserInput(text))
bnf_parsers_helpers = parser.get_concepts_sequences()
@@ -113,34 +114,11 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
my_map, text, expected, multiple_result, post_init_concepts
)
return sequences
# sheerka, context, *updated = self.init_concepts(*my_map.values(), create_new=False, singleton=True)
# if not multiple_result:
# expected_array = [compute_expected_array(my_map, text, expected)]
# else:
# expected_array = [compute_expected_array(my_map, text, e) for e in expected]
#
# if post_init_concepts:
# post_init_concepts(sheerka, context)
#
# parser = BnfNodeParser()
# parser.init_from_concepts(context, updated)
# parser.reset_parser(context, text)
#
# bnf_parsers_helpers = parser.get_concepts_sequences()
#
# assert len(bnf_parsers_helpers) == len(expected_array)
# for parser_helper, expected_sequence in zip(bnf_parsers_helpers, expected_array):
# assert parser_helper.sequence == expected_sequence
#
# if len(bnf_parsers_helpers) == 1:
# return bnf_parsers_helpers[0].sequence
# else:
# return [pe.sequence for pe in bnf_parsers_helpers]
def test_i_cannot_parse_empty_strings(self):
sheerka, context, parser = self.init_parser({}, singleton=True)
res = parser.parse(context, "")
res = parser.parse(context, ParserInput(""))
assert not res.status
assert sheerka.isinstance(res.body, BuiltinConcepts.NOT_FOR_ME)
@@ -738,7 +716,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
sheerka, context, parser = self.init_parser(my_map, singleton=True)
parser.init_from_concepts(context, my_map.values())
parser.reset_parser(context, "one three")
parser.reset_parser(context, ParserInput("one three"))
sequences = parser.get_concepts_sequences()
sequence = parser.get_valid(sequences)
@@ -756,7 +734,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
def test_i_can_parse(self, parser_input, expected_status, expected):
sheerka, context, parser = self.init_parser(init_from_sheerka=True)
res = parser.parse(context, parser_input)
res = parser.parse(context, ParserInput(parser_input))
expected_array = compute_expected_array(cmap, parser_input, expected)
parser_result = res.value
concepts_nodes = res.value.value
@@ -771,7 +749,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
parser_input = "def one"
expected = [CNC("def number", source="def one", number="one", one="one")]
res = parser.parse(context, parser_input)
res = parser.parse(context, ParserInput(parser_input))
expected_array = compute_expected_array(cmap, parser_input, expected)
expected_array[0].compiled["def"] = cmap["def_only"]
+15 -4
View File
@@ -1,6 +1,7 @@
import pytest
from core.builtin_concepts import BuiltinConcepts
from core.concept import Concept, DEFINITION_TYPE_BNF
from core.sheerka.services.SheerkaExecute import ParserInput
from core.tokenizer import Tokenizer, TokenKind, LexerError, Token
from parsers.BaseNodeParser import cnode
from parsers.BaseParser import UnexpectedTokenErrorNode
@@ -135,7 +136,6 @@ class TestBnfParser(TestUsingMemoryBasedSheerka):
("1|", UnexpectedEndOfFileError()),
("(1|)", UnexpectedTokenErrorNode("Unexpected token 'Token(<EOF>)'", eof_token, [TokenKind.RPAR])),
("1=", UnexpectedTokenErrorNode("Unexpected token 'Token(<EOF>)'", eof_token, [TokenKind.IDENTIFIER])),
("'name", LexerError("Missing Trailing quote", "'name", 5, 1, 6))
])
def test_i_can_detect_errors(self, expression, error):
sheerka, context, parser = self.init_parser()
@@ -146,6 +146,17 @@ class TestBnfParser(TestUsingMemoryBasedSheerka):
assert not res.status
assert ret_value[0] == error
def test_i_can_detect_lexer_error(self):
sheerka, context, parser = self.init_parser()
res = parser.parse(context, Tokenizer("'name"))
assert not res.status
assert sheerka.isinstance(res.body, BuiltinConcepts.ERROR)
assert isinstance(res.body.body[0], LexerError)
assert res.body.body[0].message == "Missing Trailing quote"
assert res.body.body[0].text == "'name"
def test_i_can_use_the_result_of_regex_parsing_to_parse_a_text(self):
sheerka, context, regex_parser, foo, bar = self.init_parser("foo", "bar")
@@ -158,15 +169,15 @@ class TestBnfParser(TestUsingMemoryBasedSheerka):
bnf_parser = BnfNodeParser()
bnf_parser.init_from_concepts(context, [foo, bar])
res = bnf_parser.parse(context, "twenty two")
res = bnf_parser.parse(context, ParserInput("twenty two"))
assert res.status
assert res.value.body == [cnode("bar", 0, 2, "twenty two")]
res = bnf_parser.parse(context, "thirty one")
res = bnf_parser.parse(context, ParserInput("thirty one"))
assert res.status
assert res.value.body == [cnode("bar", 0, 2, "thirty one")]
res = bnf_parser.parse(context, "twenty")
res = bnf_parser.parse(context, ParserInput("twenty"))
assert res.status
assert res.value.body == [cnode("foo", 0, 0, "twenty")]
+47 -32
View File
@@ -1,8 +1,10 @@
import ast
from dataclasses import dataclass
import pytest
from core.builtin_concepts import ParserResultConcept, BuiltinConcepts, ReturnValueConcept
from core.concept import DEFINITION_TYPE_BNF, DEFINITION_TYPE_DEF
from core.sheerka.services.SheerkaExecute import ParserInput
from core.tokenizer import Keywords, Tokenizer, LexerError
from parsers.BnfNodeParser import OrderedChoice, ConceptExpression, StrMatch
from parsers.BnfParser import BnfParser
@@ -39,7 +41,7 @@ def get_def_concept(name, where=None, pre=None, post=None, body=None, definition
def get_concept_part(part):
if isinstance(part, str):
node = PythonNode(part, ast.parse(part, mode="eval"))
node = PythonNode(part.strip(), ast.parse(part.strip(), mode="eval"))
return ReturnValueConcept(
who="parsers.Default",
status=True,
@@ -48,6 +50,16 @@ def get_concept_part(part):
parser=PythonParser(),
value=node))
if isinstance(part, PN):
node = PythonNode(part.source.strip(), ast.parse(part.source.strip(), mode=part.mode))
return ReturnValueConcept(
who="parsers.Default",
status=True,
value=ParserResultConcept(
source=part.source,
parser=PythonParser(),
value=node))
if isinstance(part, PythonNode):
return ReturnValueConcept(
who="parsers.Default",
@@ -61,6 +73,12 @@ def get_concept_part(part):
return part
@dataclass
class PN:
source: str # parser result source
mode: str # compilation mode
class TestDefaultParser(TestUsingMemoryBasedSheerka):
def init_parser(self, *concepts):
@@ -82,7 +100,7 @@ class TestDefaultParser(TestUsingMemoryBasedSheerka):
])
def test_i_can_parse_def_concept(self, text, expected):
sheerka, context, parser = self.init_parser()
res = parser.parse(context, text)
res = parser.parse(context, ParserInput(text))
node = res.value.value
assert res.status
@@ -99,14 +117,14 @@ post isinstance(res, int)
as res = a + b
"""
sheerka, context, parser = self.init_parser()
res = parser.parse(context, text)
res = parser.parse(context, ParserInput(text))
return_value = res.value
expected_concept = get_def_concept(
name="a plus b",
where="a,b",
pre="isinstance(a, int) and isinstance(b, float)",
post="isinstance(res, int)",
body=PythonNode("res = a + b", ast.parse("res = a + b", mode="exec"))
where="a,b\n",
pre="isinstance(a, int) and isinstance(b, float)\n",
post="isinstance(res, int)\n",
body=PN("res = a + b\n ", "exec")
)
assert res.status
@@ -119,17 +137,15 @@ def concept add one to a as
def func(x):
return x+1
func(a)
"""
"""
expected_concept = get_def_concept(
name="add one to a ",
body=PythonNode(
"def func(x):\n return x+1\nfunc(a)",
ast.parse("def func(x):\n return x+1\nfunc(a)", mode="exec"))
body=PN("def func(x):\n return x+1\nfunc(a)\n", "exec")
)
sheerka, context, parser = self.init_parser()
res = parser.parse(context, text)
res = parser.parse(context, ParserInput(text))
return_value = res.value
assert res.status
@@ -141,8 +157,7 @@ func(a)
def concept add one to a as:
def func(x):
return x+1
func(a)
"""
func(a)"""
expected_concept = get_def_concept(
name="add one to a ",
@@ -152,7 +167,7 @@ def concept add one to a as:
)
sheerka, context, parser = self.init_parser()
res = parser.parse(context, text)
res = parser.parse(context, ParserInput(text))
return_value = res.value
assert res.status
@@ -168,7 +183,7 @@ func(a)
"""
sheerka, context, parser = self.init_parser()
res = parser.parse(context, text)
res = parser.parse(context, ParserInput(text))
return_value = res.value
assert not res.status
@@ -184,7 +199,7 @@ def concept add one to a as
func(a)
"""
sheerka, context, parser = self.init_parser()
res = parser.parse(context, text)
res = parser.parse(context, ParserInput(text))
return_value = res.value
assert not res.status
@@ -194,7 +209,7 @@ def concept add one to a as
text = "def concept as 'hello'"
sheerka, context, parser = self.init_parser()
res = parser.parse(context, text)
res = parser.parse(context, ParserInput(text))
return_value = res.value
assert not res.status
@@ -205,9 +220,9 @@ def concept add one to a as
def test_concept_keyword_is_mandatory_but_the_concept_is_recognized(self):
text = "def hello as a where b pre c post d"
expected_concept = get_def_concept(name="hello", body="a", where="b", pre="c", post="d")
expected_concept = get_def_concept(name="hello", body="a ", where="b ", pre="c ", post="d")
sheerka, context, parser = self.init_parser()
res = parser.parse(context, text)
res = parser.parse(context, ParserInput(text))
return_value = res.value
assert not res.status
@@ -225,7 +240,7 @@ def concept add one to a as
])
def test_i_can_detect_error_in_declaration(self, text):
sheerka, context, parser = self.init_parser()
res = parser.parse(context, text)
res = parser.parse(context, ParserInput(text))
return_value = res.value
assert not res.status
@@ -235,7 +250,7 @@ def concept add one to a as
text = "def concept hello \n my friend as 'hello'"
sheerka, context, parser = self.init_parser()
res = parser.parse(context, text)
res = parser.parse(context, ParserInput(text))
return_value = res.value
assert not res.status
@@ -244,7 +259,7 @@ def concept add one to a as
def test_i_can_parse_def_concept_from_bnf(self):
text = "def concept name from bnf a_concept | 'a_string' as __definition[0]"
sheerka, context, parser, a_concept = self.init_parser("a_concept")
res = parser.parse(context, text)
res = parser.parse(context, ParserInput(text))
node = res.value.value
definition = OrderedChoice(ConceptExpression(a_concept, rule_name="a_concept"), StrMatch("a_string"))
@@ -260,7 +275,7 @@ def concept add one to a as
def test_i_can_parse_def_concept_where_bnf_references_itself(self):
text = "def concept name from bnf 'a' + name?"
sheerka, context, parser, a_concept = self.init_parser("a_concept")
parser.parse(context, text)
parser.parse(context, ParserInput(text))
assert not parser.has_error
@@ -271,7 +286,7 @@ def concept add one to a as
])
def test_i_can_detect_empty_bnf_declaration(self, text):
sheerka, context, parser = self.init_parser()
res = parser.parse(context, text)
res = parser.parse(context, ParserInput(text))
assert not res.status
assert res.value.value[0] == SyntaxErrorNode([], "Empty declaration")
@@ -281,7 +296,7 @@ def concept add one to a as
"def concept addition from def a plus b as a + b"])
def test_i_can_def_concept_from_definition(self, text):
sheerka, context, parser = self.init_parser()
res = parser.parse(context, text)
res = parser.parse(context, ParserInput(text))
expected = get_def_concept("addition", definition="a plus b", body="a + b")
node = res.value.value
@@ -294,7 +309,7 @@ def concept add one to a as
def test_i_can_detect_not_for_me(self):
text = "hello world"
sheerka, context, parser = self.init_parser()
res = parser.parse(context, text)
res = parser.parse(context, ParserInput(text))
assert not res.status
assert context.sheerka.isinstance(res.value, BuiltinConcepts.NOT_FOR_ME)
@@ -303,7 +318,7 @@ def concept add one to a as
def test_i_can_parse_is_a(self):
text = "the name of my 'concept' isa the name of the set"
sheerka, context, parser = self.init_parser()
res = parser.parse(context, text)
res = parser.parse(context, ParserInput(text))
expected = IsaConceptNode([],
concept=NameNode(list(Tokenizer("the name of my 'concept'"))),
set=NameNode(list(Tokenizer("the name of the set"))))
@@ -323,7 +338,7 @@ def concept add one to a as
])
def test_i_cannot_parse_invalid_entries(self, text):
sheerka, context, parser = self.init_parser()
res = parser.parse(context, text)
res = parser.parse(context, ParserInput(text))
assert not res.status
assert isinstance(res.body, ParserResultConcept)
@@ -339,10 +354,10 @@ def concept add one to a as
])
def test_i_cannot_parse_when_tokenizer_fails(self, text, error_msg, error_text):
sheerka, context, parser = self.init_parser()
res = parser.parse(context, text)
res = parser.parse(context, ParserInput(text))
assert not res.status
assert isinstance(res.body, ParserResultConcept)
assert sheerka.isinstance(res.body, BuiltinConcepts.ERROR)
assert isinstance(res.body.body[0], LexerError)
assert res.body.body[0].message == error_msg
assert res.body.body[0].text == error_text
@@ -350,7 +365,7 @@ def concept add one to a as
def test_i_cannot_parse_bnf_definition_referencing_unknown_concept(self):
text = "def concept name from bnf unknown"
sheerka, context, parser = self.init_parser()
res = parser.parse(context, text)
res = parser.parse(context, ParserInput(text))
assert not res.status
assert context.sheerka.isinstance(res.value, BuiltinConcepts.UNKNOWN_CONCEPT)
+24 -23
View File
@@ -1,5 +1,6 @@
from core.builtin_concepts import BuiltinConcepts
from core.concept import Concept, CMV
from core.sheerka.services.SheerkaExecute import ParserInput
from core.tokenizer import Tokenizer
from parsers.ExactConceptParser import ExactConceptParser
@@ -55,7 +56,7 @@ class TestExactConceptParser(TestUsingMemoryBasedSheerka):
concept = self.create_and_add_in_cache_concept(sheerka, "hello world")
source = "hello world"
results = ExactConceptParser().parse(context, source)
results = ExactConceptParser().parse(context, ParserInput(source))
concept_found = results[0].value.value
assert len(results) == 1
@@ -71,7 +72,7 @@ class TestExactConceptParser(TestUsingMemoryBasedSheerka):
self.create_and_add_in_cache_concept(sheerka, "hello a", variables=["a"])
source = "hello world"
results = ExactConceptParser().parse(context, source)
results = ExactConceptParser().parse(context, ParserInput(source))
assert len(results) == 2
results = sorted(results, key=lambda x: x.value.value.name) # because of the usage of sets
@@ -91,7 +92,7 @@ class TestExactConceptParser(TestUsingMemoryBasedSheerka):
concept = self.create_and_add_in_cache_concept(sheerka, "a + b", ["a", "b"])
source = "10 + 5"
results = ExactConceptParser().parse(context, source)
results = ExactConceptParser().parse(context, ParserInput(source))
assert len(results) == 1
assert results[0].status
@@ -107,7 +108,7 @@ class TestExactConceptParser(TestUsingMemoryBasedSheerka):
concept = self.create_and_add_in_cache_concept(sheerka, "a + b + a", ["a", "b"])
source = "10 + 5 + 10"
results = ExactConceptParser(max_word_size=5).parse(context, source)
results = ExactConceptParser(max_word_size=5).parse(context, ParserInput(source))
assert len(results) == 1
assert results[0].status
@@ -122,7 +123,7 @@ class TestExactConceptParser(TestUsingMemoryBasedSheerka):
)
source = "10 + 5"
results = ExactConceptParser().parse(context, source)
results = ExactConceptParser().parse(context, ParserInput(source))
concept_found = results[0].value.value
assert len(results) == 1
@@ -135,7 +136,7 @@ class TestExactConceptParser(TestUsingMemoryBasedSheerka):
sheerka, context, foo = self.init_concepts("foo")
source = "c:foo:"
results = ExactConceptParser().parse(context, source)
results = ExactConceptParser().parse(context, ParserInput(source))
concept_found = results[0].value.value
assert len(results) == 1
@@ -152,7 +153,7 @@ class TestExactConceptParser(TestUsingMemoryBasedSheerka):
)
source = "c:one: plus c:two:"
results = ExactConceptParser().parse(context, source)
results = ExactConceptParser().parse(context, ParserInput(source))
concept_found = results[0].value.value
assert len(results) == 1
@@ -168,7 +169,7 @@ class TestExactConceptParser(TestUsingMemoryBasedSheerka):
)
source = "z is a concept"
results = ExactConceptParser().parse(context, source)
results = ExactConceptParser().parse(context, ParserInput(source))
concept_found = results[0].value.value
assert len(results) == 1
@@ -178,7 +179,7 @@ class TestExactConceptParser(TestUsingMemoryBasedSheerka):
assert not concept_found.metadata.is_evaluated
source = "def concept z"
results = ExactConceptParser().parse(context, source)
results = ExactConceptParser().parse(context, ParserInput(source))
concept_found = results[0].value.value
assert len(results) == 1
@@ -190,7 +191,7 @@ class TestExactConceptParser(TestUsingMemoryBasedSheerka):
def test_i_can_manage_unknown_concept(self):
context = self.get_context(self.get_sheerka(singleton=True))
source = "def concept hello" # this is not a concept by itself
res = ExactConceptParser().parse(context, source)
res = ExactConceptParser().parse(context, ParserInput(source))
assert not res.status
assert context.sheerka.isinstance(res.value, BuiltinConcepts.UNKNOWN_CONCEPT)
@@ -199,7 +200,7 @@ class TestExactConceptParser(TestUsingMemoryBasedSheerka):
def test_i_can_detect_concepts_too_long(self):
context = self.get_context(self.get_sheerka(singleton=True))
source = "a very but finally too long concept"
res = ExactConceptParser().parse(context, source)
res = ExactConceptParser().parse(context, ParserInput(source))
assert not res.status
assert context.sheerka.isinstance(res.value, BuiltinConcepts.NOT_FOR_ME)
@@ -207,15 +208,15 @@ class TestExactConceptParser(TestUsingMemoryBasedSheerka):
assert res.value.reason.body == source
assert res.value.body == source
def test_i_can_detect_concept_from_tokens(self):
context = self.get_context(self.get_sheerka(singleton=True))
concept = get_concept("hello world", [])
context.sheerka.add_in_cache(concept)
source = "hello world"
results = ExactConceptParser().parse(context, list(Tokenizer(source)))
assert len(results) == 1
assert results[0].status
assert results[0].value.value == concept
assert not results[0].value.value.metadata.need_validation
# def test_i_can_detect_concept_from_tokens(self):
# context = self.get_context(self.get_sheerka(singleton=True))
# concept = get_concept("hello world", [])
# context.sheerka.add_in_cache(concept)
#
# source = "hello world"
# results = ExactConceptParser().parse(context, list(Tokenizer(source)))
#
# assert len(results) == 1
# assert results[0].status
# assert results[0].value.value == concept
# assert not results[0].value.value.metadata.need_validation
+10 -23
View File
@@ -3,7 +3,8 @@ import ast
import core.utils
import pytest
from core.builtin_concepts import ParserResultConcept, NotForMeConcept
from core.tokenizer import Tokenizer, LexerError
from core.sheerka.services.SheerkaExecute import ParserInput
from core.tokenizer import LexerError
from parsers.PythonParser import PythonNode, PythonParser, PythonErrorNode
from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
@@ -17,21 +18,7 @@ class TestPythonParser(TestUsingMemoryBasedSheerka):
])
def test_i_can_parse_a_simple_expression(self, text, expected):
parser = PythonParser()
res = parser.parse(self.get_context(), text)
assert res.status
assert res.who == parser.name
assert isinstance(res.value, ParserResultConcept)
assert res.value.value == expected
@pytest.mark.parametrize("text, expected", [
("1+1", PythonNode("1+1", ast.parse("1+1", mode="eval"))),
("a=10", PythonNode("a=10", ast.parse("a=10", mode="exec"))),
])
def test_i_can_parse_from_tokens(self, text, expected):
parser = PythonParser()
tokens = list(Tokenizer(text))
res = parser.parse(self.get_context(), tokens)
res = parser.parse(self.get_context(), ParserInput(text))
assert res.status
assert res.who == parser.name
@@ -40,12 +27,11 @@ class TestPythonParser(TestUsingMemoryBasedSheerka):
@pytest.mark.parametrize("text", [
"1+",
"'name",
"foo = 'name"
"&#é",
])
def test_i_can_detect_error(self, text):
def test_i_can_detect_python_error(self, text):
parser = PythonParser()
res = parser.parse(self.get_context(), text)
res = parser.parse(self.get_context(), ParserInput(text))
assert not res.status
assert res.who == parser.name
@@ -61,7 +47,7 @@ class TestPythonParser(TestUsingMemoryBasedSheerka):
])
def test_i_can_detect_lexer_errors(self, text, error_msg, error_text):
parser = PythonParser()
res = parser.parse(self.get_context(), text)
res = parser.parse(self.get_context(), ParserInput(text))
assert not res.status
assert isinstance(res.value, NotForMeConcept)
@@ -76,12 +62,13 @@ class TestPythonParser(TestUsingMemoryBasedSheerka):
text = "c:name|id: + 1"
parser = PythonParser()
res = parser.parse(self.get_context(), text)
res = parser.parse(self.get_context(), ParserInput(text))
encoded = core.utils.encode_concept(("name", "id"))
assert res
assert res.value.source == "c:name|id: + 1"
assert res.value.value == PythonNode(
"c:name|id: + 1",
"__C__KEY_name__ID_id__C__ + 1",
ast.parse(encoded + "+1", mode="eval"))
assert res.value.value.concepts == {
encoded: ("name", "id")
@@ -3,6 +3,7 @@ import ast
import pytest
from core.builtin_concepts import ParserResultConcept, BuiltinConcepts, ReturnValueConcept
from core.concept import Concept
from core.sheerka.services.SheerkaExecute import ParserInput
from core.tokenizer import Token, TokenKind, Tokenizer
from parsers.BaseNodeParser import ConceptNode, UnrecognizedTokensNode
from parsers.PythonParser import PythonNode
@@ -38,6 +39,7 @@ class TestPythonWithConceptsParser(TestUsingMemoryBasedSheerka):
@pytest.mark.parametrize("text, interested", [
("not parser result", False),
(ParserInput("not parser result"), False),
(ParserResultConcept(parser="not multiple_concepts_parser"), False),
(ParserResultConcept(parser=unrecognized_nodes_parser, value=[UnrecognizedTokensNode(0, 0, [])]), True),
])
+42 -41
View File
@@ -2,6 +2,7 @@ import pytest
from core.builtin_concepts import BuiltinConcepts
from core.concept import Concept, CC
from core.sheerka.services.SheerkaComparisonManager import SheerkaComparisonManager
from core.sheerka.services.SheerkaExecute import ParserInput
from core.tokenizer import Tokenizer
from parsers.BaseNodeParser import utnode, ConceptNode, cnode, short_cnode, UnrecognizedTokensNode, \
SCWC, CNC, UTN
@@ -198,7 +199,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
def test_i_can_post_fix_simple_infix_concepts(self, expression, expected_sequences):
sheerka, context, parser = self.init_parser()
res = parser.infix_to_postfix(context, expression)
res = parser.infix_to_postfix(context, ParserInput(expression))
assert len(res) == len(expected_sequences)
for res_i, expected in zip(res, expected_sequences):
@@ -224,7 +225,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
}
sheerka, context, parser = self.init_parser(concepts_map, create_new=True, init_from_sheerka=True)
res = parser.infix_to_postfix(context, expression)
res = parser.infix_to_postfix(context, ParserInput(expression))
assert len(res) == len(expected_sequences)
for res_i, expected in zip(res, expected_sequences):
@@ -248,7 +249,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
def test_i_can_post_fix_simple_prefixed_concepts(self, expression, expected_sequences):
sheerka, context, parser = self.init_parser()
res = parser.infix_to_postfix(context, expression)
res = parser.infix_to_postfix(context, ParserInput(expression))
assert len(res) == len(expected_sequences)
for res_i, expected in zip(res, expected_sequences):
@@ -291,7 +292,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
}
sheerka, context, parser = self.init_parser(concepts_map, create_new=True, init_from_sheerka=True)
res = parser.infix_to_postfix(context, expression)
res = parser.infix_to_postfix(context, ParserInput(expression))
assert len(res) == len(expected_sequences)
for res_i, expected in zip(res, expected_sequences):
@@ -315,7 +316,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
def test_i_can_post_fix_simple_suffixed_concepts(self, expression, expected_sequences):
sheerka, context, parser = self.init_parser()
res = parser.infix_to_postfix(context, expression)
res = parser.infix_to_postfix(context, ParserInput(expression))
assert len(res) == len(expected_sequences)
for res_i, expected in zip(res, expected_sequences):
@@ -337,7 +338,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
}
sheerka, context, parser = self.init_parser(concepts_map, None)
res = parser.infix_to_postfix(context, expression)
res = parser.infix_to_postfix(context, ParserInput(expression))
expected_array = compute_expected_array(concepts_map, expression, expected)
assert len(res) == 1
assert res[0].out == expected_array
@@ -377,7 +378,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
sheerka, context, parser = self.init_parser()
res = parser.infix_to_postfix(context, expression)
res = parser.infix_to_postfix(context, ParserInput(expression))
assert len(res) == len(expected_sequences)
for res_i, expected in zip(res, expected_sequences):
@@ -409,7 +410,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
}
sheerka, context, parser = self.init_parser(concepts_map, create_new=True, init_from_sheerka=True)
res = parser.infix_to_postfix(context, expression)
res = parser.infix_to_postfix(context, ParserInput(expression))
assert len(res) == len(expected_sequences)
for res_i, expected in zip(res, expected_sequences):
@@ -430,7 +431,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
}
sheerka, context, parser = self.init_parser(concepts_map, None)
res = parser.infix_to_postfix(context, expression)
res = parser.infix_to_postfix(context, ParserInput(expression))
expected_array = compute_expected_array(concepts_map, expression, expected)
assert len(res) == 1
@@ -453,7 +454,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
}
sheerka, context, parser = self.init_parser(concepts_map, sya_def)
res = parser.infix_to_postfix(context, expression)
res = parser.infix_to_postfix(context, ParserInput(expression))
expected_array = compute_expected_array(concepts_map, expression, expected)
assert len(res) == 1
@@ -468,7 +469,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
def test_i_can_post_fix_binary_with_precedence(self, expression, expected):
sheerka, context, parser = self.init_parser()
res = parser.infix_to_postfix(context, expression)
res = parser.infix_to_postfix(context, ParserInput(expression))
expected_array = compute_expected_array(cmap, expression, expected)
assert len(res) == 1
@@ -489,7 +490,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
expression = "suffixed a prefixed"
expected = ["a", "prefixed", "suffixed"]
res = parser.infix_to_postfix(context, expression)
res = parser.infix_to_postfix(context, ParserInput(expression))
expected_array = compute_expected_array(concepts_map, expression, expected)
assert len(res) == 1
@@ -504,7 +505,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
expression = "suffixed a prefixed"
expected = ["a", "suffixed", "prefixed"]
res = parser.infix_to_postfix(context, expression)
res = parser.infix_to_postfix(context, ParserInput(expression))
expected_array = compute_expected_array(concepts_map, expression, expected)
assert len(res) == 1
@@ -524,7 +525,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
sheerka, context, parser = self.init_parser(concepts_map, sya_def)
expression = "one equals two equals three"
res = parser.infix_to_postfix(context, expression)
res = parser.infix_to_postfix(context, ParserInput(expression))
expected = ["one", "two", "three", ("equals", 1), "equals"]
expected_array = compute_expected_array(concepts_map, expression, expected)
@@ -546,7 +547,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
sheerka, context, parser = self.init_parser(concepts_map, sya_def)
expression = "one plus two plus three"
res = parser.infix_to_postfix(context, expression)
res = parser.infix_to_postfix(context, ParserInput(expression))
expected = ["one", "two", "plus", "three", ("plus", 1)]
expected_array = compute_expected_array(concepts_map, expression, expected)
@@ -576,7 +577,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
}
sheerka, context, parser = self.init_parser(concepts_map, sya_def)
res = parser.infix_to_postfix(context, expression)
res = parser.infix_to_postfix(context, ParserInput(expression))
expected_array = compute_expected_array(concepts_map, expression, expected)
assert len(res) == 1
assert res[0].out == expected_array
@@ -603,7 +604,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
}
sheerka, context, parser = self.init_parser(concepts_map, sya_def)
res = parser.infix_to_postfix(context, expression)
res = parser.infix_to_postfix(context, ParserInput(expression))
expected_array = compute_expected_array(concepts_map, expression, expected)
assert len(res) == 1
assert res[0].out == expected_array
@@ -617,7 +618,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
sheerka, context, parser = self.init_parser(concepts_map, None)
expression = "foo bar baz"
res = parser.infix_to_postfix(context, expression)
res = parser.infix_to_postfix(context, ParserInput(expression))
expected_sequences = [
[UTN("bar "), "foo", "baz"],
["baz", "foo bar"]
@@ -664,7 +665,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
def test_i_can_pos_fix_when_parenthesis(self, expression, expected):
sheerka, context, parser = self.init_parser()
res = parser.infix_to_postfix(context, expression)
res = parser.infix_to_postfix(context, ParserInput(expression))
expected_array = compute_expected_array(cmap, expression, expected)
assert len(res) == 1
@@ -719,7 +720,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
def test_i_can_post_fix_when_parenthesis_and_unknown(self, expression, expected_sequences):
sheerka, context, parser = self.init_parser()
res = parser.infix_to_postfix(context, expression)
res = parser.infix_to_postfix(context, ParserInput(expression))
assert len(res) == len(expected_sequences)
for res_i, expected in zip(res, expected_sequences):
@@ -745,7 +746,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
def test_i_can_detect_parenthesis_mismatch_error_when_post_fixing(self, expression, expected):
sheerka, context, parser = self.init_parser()
res = parser.infix_to_postfix(context, expression)
res = parser.infix_to_postfix(context, ParserInput(expression))
assert len(res) == 1
assert res[0].errors == [expected]
@@ -756,7 +757,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
def test_i_can_detected_when_too_many_parameters(self, expression, expected):
sheerka, context, parser = self.init_parser(cmap, None)
res = parser.infix_to_postfix(context, expression)
res = parser.infix_to_postfix(context, ParserInput(expression))
assert len(res) == 1
assert len(res[0].errors) == 1
@@ -790,7 +791,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
def test_i_can_post_fix_sequences(self, expression, expected):
sheerka, context, parser = self.init_parser()
res = parser.infix_to_postfix(context, expression)
res = parser.infix_to_postfix(context, ParserInput(expression))
expected_array = compute_expected_array(cmap, expression, expected)
assert len(res) == 1
@@ -808,7 +809,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
sheerka, context, parser = self.init_parser()
expected = ["one", "two", "three", "?"]
res = parser.infix_to_postfix(context, expression)
res = parser.infix_to_postfix(context, ParserInput(expression))
expected_array = compute_expected_array(cmap, expression, expected)
assert len(res) == 1
@@ -830,7 +831,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
sheerka, context, parser = self.init_parser(concepts_map, sya_def)
expression = "a plus plus equals b"
res = parser.infix_to_postfix(context, expression)
res = parser.infix_to_postfix(context, ParserInput(expression))
expected_array = tests.parsers.parsers_utils.compute_debug_array(res)
assert expected_array == [
["T(a)", "C(a plus b)", "C(a plus b)", "T(equals)", "T(b)"],
@@ -859,9 +860,9 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
sheerka, context, parser = self.init_parser(concepts_map, sya_def)
expression = "a plus complex infix b"
res = parser.infix_to_postfix(context, expression)
res = parser.infix_to_postfix(context, ParserInput(expression))
res = parser.parse(context, expression)
res = parser.parse(context, ParserInput(expression))
pass
def test_i_can_use_string_instead_of_identifier(self):
@@ -874,7 +875,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
sheerka, context, parser = self.init_parser(concepts_map, None)
res = parser.infix_to_postfix(context, "one ? ? two '::' three")
res = parser.infix_to_postfix(context, ParserInput("one ? ? two '::' three"))
assert len(res) == 1
assert res[0].out == [
cnode("one", start=0, end=0, source="one"),
@@ -896,7 +897,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
sheerka, context, parser = self.init_parser(concepts_map, sya_def)
res = parser.infix_to_postfix(context, "one less than two less than three")
res = parser.infix_to_postfix(context, ParserInput("one less than two less than three"))
assert len(res) == 1
assert res[0].errors == [NoneAssociativeSequenceErrorNode(concepts_map["less than"], 2, 8)]
@@ -909,7 +910,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
sheerka, context, parser = self.init_parser()
expression = "suffixed twenties"
res = parser.infix_to_postfix(context, expression)
res = parser.infix_to_postfix(context, ParserInput(expression))
expected = [cnode("twenties", 2, 2, "twenties"), "suffixed"]
expected_array = compute_expected_array(cmap, expression, expected)
@@ -920,7 +921,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
sheerka, context, parser = self.init_parser()
text = "one plus two mult three"
res = parser.parse(context, text)
res = parser.parse(context, ParserInput(text))
wrapper = res.body
lexer_nodes = res.body.body
@@ -939,7 +940,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
sheerka, context, parser = self.init_parser()
text = "suffixed 1 + 1"
res = parser.parse(context, text)
res = parser.parse(context, ParserInput(text))
wrapper = res.body
lexer_nodes = res.body.body
@@ -962,7 +963,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
sheerka, context, parser = self.init_parser()
text = "suffixed twenty one"
res = parser.parse(context, text)
res = parser.parse(context, ParserInput(text))
assert len(res) == 2
assert res[1].status
@@ -981,7 +982,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
sheerka, context, parser = self.init_parser()
text = "one plus 1 + 1 suffixed two"
res = parser.parse(context, text)
res = parser.parse(context, ParserInput(text))
wrapper = res.body
lexer_nodes = res.body.body
@@ -1020,7 +1021,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
def test_i_can_parse_when_one_result(self, text, expected_status, expected_result):
sheerka, context, parser = self.init_parser()
res = parser.parse(context, text)
res = parser.parse(context, ParserInput(text))
wrapper = res.body
lexer_nodes = res.body.body
@@ -1043,7 +1044,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
"""
sheerka, context, parser = self.init_parser()
res = parser.parse(context, text)
res = parser.parse(context, ParserInput(text))
assert not res.status
assert sheerka.isinstance(res.body, BuiltinConcepts.NOT_FOR_ME)
@@ -1068,7 +1069,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
"""
sheerka, context, parser = self.init_parser()
res = parser.parse(context, text)
res = parser.parse(context, ParserInput(text))
wrapper = res.body
lexer_nodes = res.body.body
@@ -1088,7 +1089,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
def test_i_cannot_parse_when_unrecognized(self, text, expected_concept, expected_unrecognized):
sheerka, context, parser = self.init_parser()
res = parser.parse(context, text)
res = parser.parse(context, ParserInput(text))
wrapper = res.body
lexer_nodes = res.body.body
expected_end = len(list(Tokenizer(text))) - 2
@@ -1108,7 +1109,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
def test_i_cannot_parse_when_part_of_the_sequence_is_not_recognized(self, text, expected):
sheerka, context, parser = self.init_parser()
res = parser.parse(context, text)
res = parser.parse(context, ParserInput(text))
wrapper = res.body
lexer_nodes = res.body.body
@@ -1132,7 +1133,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
"""
sheerka, context, parser = self.init_parser()
res = parser.parse(context, text)
res = parser.parse(context, ParserInput(text))
assert not res.status
assert sheerka.isinstance(res.body, BuiltinConcepts.NOT_FOR_ME)
@@ -1141,7 +1142,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
def test_i_cannot_parse_empty_string(self):
sheerka, context, parser = self.init_parser({}, None)
res = parser.parse(context, "")
res = parser.parse(context, ParserInput(""))
assert not res.status
assert sheerka.isinstance(res.body, BuiltinConcepts.IS_EMPTY)
+3 -3
View File
@@ -135,7 +135,7 @@ class TestUnrecognizedNodeParser(TestUsingMemoryBasedSheerka):
assert sheerka.isinstance(concept.compiled["d"][0], BuiltinConcepts.RETURN_VALUE)
assert concept.compiled["d"][0].status
assert concept.compiled["d"][0].who == "parsers.Python"
assert concept.compiled["d"][0].body.source == "1 + 2"
assert concept.compiled["d"][0].body.source == " 1 + 2 "
assert len(concept.compiled["e"]) == 1
assert sheerka.isinstance(concept.compiled["e"][0], BuiltinConcepts.RETURN_VALUE)
@@ -171,13 +171,13 @@ class TestUnrecognizedNodeParser(TestUsingMemoryBasedSheerka):
assert len(res.body.concept.compiled["a"]) == 1
assert res.body.concept.compiled["a"][0].status
assert res.body.concept.compiled["a"][0].who == "parsers.Python"
assert res.body.concept.compiled["a"][0].body.source == "1"
assert res.body.concept.compiled["a"][0].body.source == "1 "
assert res.body.concept.compiled["b"] == concepts_map["mult"]
assert sheerka.isinstance(res.body.concept.compiled["b"].compiled["a"][0], BuiltinConcepts.RETURN_VALUE)
assert res.body.concept.compiled["b"].compiled["a"][0].status
assert res.body.concept.compiled["b"].compiled["a"][0].who == "parsers.Python"
assert res.body.concept.compiled["b"].compiled["a"][0].body.source == "2"
assert res.body.concept.compiled["b"].compiled["a"][0].body.source == " 2 "
assert sheerka.isinstance(res.body.concept.compiled["b"].compiled["b"][0], BuiltinConcepts.RETURN_VALUE)
assert res.body.concept.compiled["b"].compiled["b"][0].status