import ast from dataclasses import dataclass import pytest from core.builtin_concepts import ParserResultConcept, BuiltinConcepts, ReturnValueConcept from core.concept import DEFINITION_TYPE_BNF, DEFINITION_TYPE_DEF, Concept from core.global_symbols import NotInit from core.sheerka.services.SheerkaExecute import ParserInput from core.tokenizer import Keywords, Tokenizer, LexerError from parsers.BaseParser import UnexpectedEofParsingError from parsers.BnfDefinitionParser import BnfDefinitionParser from parsers.BnfNodeParser import OrderedChoice, ConceptExpression, StrMatch, Sequence, RegExMatch, OneOrMore, \ VariableExpression from parsers.DefConceptParser import DefConceptParser, NameNode, SyntaxErrorNode, CannotHandleParsingError from parsers.DefConceptParser import UnexpectedTokenParsingError, DefConceptNode from parsers.FunctionParser import FunctionParser from parsers.PythonParser import PythonParser, PythonNode from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka from tests.parsers.parsers_utils import compute_expected_array, SCWC, CV, compare_with_test_object def get_def_concept(name, where=None, pre=None, post=None, body=None, definition=None, bnf_def=None, ret=None): def_concept = DefConceptNode([], name=NameNode(list(Tokenizer(name)))) if body: def_concept.body = get_concept_part(body) if where: def_concept.where = get_concept_part(where) if pre: def_concept.pre = get_concept_part(pre) if post: def_concept.post = get_concept_part(post) if ret: def_concept.ret = get_concept_part(ret) if bnf_def: def_concept.definition = ReturnValueConcept( "parsers.BnfDefinition", True, bnf_def) def_concept.definition_type = DEFINITION_TYPE_BNF if definition: def_concept.definition = NameNode(list(Tokenizer(definition))) def_concept.definition_type = DEFINITION_TYPE_DEF return def_concept def get_concept_part(part): if isinstance(part, str): node = PythonNode(part.strip(), ast.parse(part.strip(), mode="eval")) return ReturnValueConcept( who="parsers.DefConcept", status=True, value=ParserResultConcept( source=part, parser=PythonParser(), value=node)) if isinstance(part, FN): # node = PythonNode(part.strip(), ast.parse(part.strip(), mode="eval")) nodes = compute_expected_array({}, part.source, [SCWC(part.first, part.last, *part.content)]) return ReturnValueConcept( who="parsers.DefConcept", status=True, value=ParserResultConcept( source=part.source, parser=FunctionParser(), value=nodes[0], try_parsed=nodes[0])) if isinstance(part, PN): node = PythonNode(part.source.strip(), ast.parse(part.source.strip(), mode=part.mode)) return ReturnValueConcept( who="parsers.DefConcept", status=True, value=ParserResultConcept( source=part.source, parser=PythonParser(), value=node)) if isinstance(part, PythonNode): return ReturnValueConcept( who="parsers.DefConcept", status=True, value=ParserResultConcept( source=part.source, parser=PythonParser(), value=part)) if isinstance(part, ReturnValueConcept): return part @dataclass class PN: """ Python Node """ source: str # parser result source mode: str # compilation mode @dataclass class FN: """ Function Node """ source: str first: str last: str content: list class TestDefConceptParser(TestUsingMemoryBasedSheerka): def init_parser(self, *concepts): sheerka, context, *updated = self.init_concepts(*concepts, singleton=True) parser = DefConceptParser() return sheerka, context, parser, *updated @pytest.mark.parametrize("text, error", [ ("concept", UnexpectedTokenParsingError("'def' keyword not found.", "concept", [Keywords.DEF])), ("hello word", UnexpectedTokenParsingError("'def' keyword not found.", "hello", [Keywords.DEF])), ("def hello", UnexpectedTokenParsingError("'concept' keyword not found.", "hello", [Keywords.CONCEPT])), ]) def test_i_can_detect_not_for_me(self, text, error): sheerka, context, parser, *concepts = self.init_parser() res = parser.parse(context, ParserInput(text)) assert not res.status assert context.sheerka.isinstance(res.value, BuiltinConcepts.NOT_FOR_ME) assert res.value.reason == [error] @pytest.mark.parametrize("text, expected", [ ("def concept hello", get_def_concept(name="hello")), ("def concept hello ", get_def_concept(name="hello")), ("def concept a + b", get_def_concept(name="a + b")), ("def concept a+b", get_def_concept(name="a + b")), ("def concept 'a+b'+c", get_def_concept(name="'a+b' + c")), ("def concept 'as if'", get_def_concept(name="'as if'")), ("def concept 'as' if", get_def_concept(name="'as' if")), ('def concept "as if"', get_def_concept(name="as if")), ]) def test_i_can_parse_def_concept_name(self, text, expected): sheerka, context, parser, *concepts = self.init_parser() res = parser.parse(context, ParserInput(text)) node = res.value.value assert res.status assert res.who == parser.name assert res.value.source == text assert isinstance(res.value, ParserResultConcept) assert node == expected def test_name_is_mandatory(self): text = "def concept as 'hello'" sheerka, context, parser, *concepts = self.init_parser() res = parser.parse(context, ParserInput(text)) return_value = res.value assert not res.status assert sheerka.isinstance(return_value, BuiltinConcepts.ERROR) assert isinstance(return_value.body[0], SyntaxErrorNode) assert return_value.body[0].message == "Name is mandatory" @pytest.mark.parametrize("text", [ "def concept hello\nmy friend", "def concept hello \nmy friend", "def concept hello\n my friend", "def concept hello \n my friend", "def concept hello from hello\nmy friend", "def concept hello from def hello\nmy friend", "def concept hello from bnf hello\nmy friend", "def concept hello from:\n\thello\nmy friend", "def concept hello from def:\n\thello\nmy friend", "def concept hello from bnf:\n\thello\nmy friend", ]) def test_new_line_is_not_allowed_in_the_name(self, text): text = "def concept hello \n my friend as 'hello'" sheerka, context, parser, *concepts = self.init_parser() res = parser.parse(context, ParserInput(text)) return_value = res.value assert not res.status assert sheerka.isinstance(return_value, BuiltinConcepts.ERROR) assert return_value.body == [SyntaxErrorNode(None, "Newline are not allowed in name.")] def test_concept_keyword_is_mandatory_but_the_concept_is_recognized(self): text = "def hello as a where b pre c post d" sheerka, context, parser, *concepts = self.init_parser() res = parser.parse(context, ParserInput(text)) return_value = res.value assert not res.status assert sheerka.isinstance(return_value, BuiltinConcepts.NOT_FOR_ME) assert isinstance(return_value.reason[0], UnexpectedTokenParsingError) assert return_value.reason[0].message == "'concept' keyword not found." assert return_value.reason[0].expected_tokens == [Keywords.CONCEPT] assert return_value.reason[0].token.value == "hello" def test_i_can_detect_empty_declaration(self): sheerka, context, parser, *concepts = self.init_parser() text = "def concept foo as where True" res = parser.parse(context, ParserInput(text)) error = res.body.body[0] assert not res.status assert sheerka.isinstance(res.value, BuiltinConcepts.ERROR) assert isinstance(error, SyntaxErrorNode) assert error.message == "Empty 'as' declaration." def test_empty_parts_are_not_initialized(self): sheerka, context, parser, *concepts = self.init_parser() text = "def concept foo" res = parser.parse(context, ParserInput(text)) parser_result = res.body node = res.body.body assert res.status assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT) assert isinstance(node, DefConceptNode) assert node.body is NotInit assert node.where is NotInit assert node.pre is NotInit assert node.post is NotInit assert node.ret is NotInit @pytest.mark.parametrize("part", [ "as", "pre", "post", "ret", "where" ]) def test_i_can_parse_def_concept_parts(self, part): sheerka, context, parser, *concepts = self.init_parser() text = "def concept foo " + part + " True" res = parser.parse(context, ParserInput(text)) node = res.value.value assert res.status assert res.who == parser.name assert res.value.source == text assert isinstance(res.value, ParserResultConcept) part_mapping = "body" if part == "as" else part args = {part_mapping: get_concept_part("True")} expected = get_def_concept("foo", **args) assert node == expected def test_i_can_detect_error_in_declaration(self): sheerka, context, parser, *concepts = self.init_parser() res = parser.parse(context, ParserInput("def concept hello where 1+")) return_value = res.value assert not res.status assert sheerka.isinstance(return_value, BuiltinConcepts.TOO_MANY_ERRORS) def test_i_can_parse_complex_def_concept_statement(self): text = """def concept a mult b where a,b pre isinstance(a, int) and isinstance(b, int) as res = a * b ret a if isinstance(a, Concept) else self """ sheerka, context, parser, *concepts = self.init_parser() res = parser.parse(context, ParserInput(text)) return_value = res.value expected_concept = get_def_concept( name="a mult b", where="a,b\n", pre="isinstance(a, int) and isinstance(b, int)\n", body=PN("res = a * b\n", "exec"), ret="a if isinstance(a, Concept) else self\n" ) assert res.status assert isinstance(return_value, ParserResultConcept) assert return_value.value == expected_concept def test_i_can_parse_mutilines_declarations(self): text = """ def concept add one to a as def func(x): return x+1 func(a) """ expected_concept = get_def_concept( name="add one to a ", body=PN("def func(x):\n return x+1\nfunc(a)\n", "exec") ) sheerka, context, parser, *concepts = self.init_parser() res = parser.parse(context, ParserInput(text)) return_value = res.value assert res.status assert isinstance(return_value, ParserResultConcept) assert return_value.value == expected_concept def test_i_can_use_colon_to_use_indentation(self): text = """ def concept add one to a as: def func(x): return x+1 func(a)""" expected_concept = get_def_concept( name="add one to a ", body=PythonNode( "def func(x):\n return x+1\nfunc(a)", ast.parse("def func(x):\n return x+1\nfunc(a)", mode="exec")) ) sheerka, context, parser, *concepts = self.init_parser() res = parser.parse(context, ParserInput(text)) return_value = res.value assert res.status assert isinstance(return_value, ParserResultConcept) assert return_value.value == expected_concept @pytest.mark.parametrize("text", [ "def concept name from bnf", "def concept name from bnf ", "def concept name from bnf as True", ]) def test_i_cannot_parse_empty_bnf_definition_when_no_definition(self, text): sheerka, context, parser, *concepts = self.init_parser() res = parser.parse(context, ParserInput(text)) error = res.body assert not res.status assert sheerka.isinstance(error, BuiltinConcepts.ERROR) assert error.body == [SyntaxErrorNode([], "Empty 'bnf' declaration")] def test_i_can_parse_def_concept_from_bnf(self): text = "def concept name from bnf a_concept | 'a_string' as __definition[0]" sheerka, context, parser, a_concept = self.init_parser("a_concept") res = parser.parse(context, ParserInput(text)) node = res.value.value definition = OrderedChoice(ConceptExpression(a_concept, rule_name="a_concept"), StrMatch("a_string")) parser_result = ParserResultConcept(BnfDefinitionParser(), "a_concept | 'a_string'", None, definition, definition) expected = get_def_concept(name="name", body="__definition[0]", bnf_def=parser_result) assert res.status assert res.who == parser.name assert res.value.source == text assert isinstance(res.value, ParserResultConcept) assert node == expected def test_i_can_parse_def_concept_from_bnf_when_using_concept_token(self): text = "def concept name from bnf c:a_concept: 'xxx'" sheerka, context, parser, a_concept = self.init_parser("a_concept") res = parser.parse(context, ParserInput(text)) node = res.value.value definition = Sequence(ConceptExpression(a_concept, rule_name="a_concept"), StrMatch("xxx")) parser_result = ParserResultConcept(BnfDefinitionParser(), "c:a_concept: 'xxx'", None, definition, definition) expected = get_def_concept(name="name", bnf_def=parser_result) assert res.status assert res.who == parser.name assert res.value.source == text assert isinstance(res.value, ParserResultConcept) assert node == expected def test_i_can_parse_def_concept_where_bnf_references_itself(self): text = "def concept name from bnf 'a' + name?" sheerka, context, parser, a_concept = self.init_parser("a_concept") parser.parse(context, ParserInput(text)) assert not parser.has_error @pytest.mark.parametrize("text", [ "def concept plus from bnf:\n\t'a' 'plus' 'b'", "def concept plus from bnf :\n\t'a' 'plus' 'b'", "def concept plus from bnf: \n\t'a' 'plus' 'b'", ]) def test_i_can_use_colon_and_bnf_definition_together(self, text): sheerka, context, parser, *concepts = self.init_parser() res = parser.parse(context, ParserInput(text)) defined_concept = res.body.body assert res.status assert defined_concept.definition.status assert defined_concept.definition.body.body == Sequence(StrMatch("a"), StrMatch("plus"), StrMatch("b")) @pytest.mark.parametrize("text, error", [ ("def concept name from def as True", SyntaxErrorNode([], "Empty 'from' declaration.")), ("def concept name from def", SyntaxErrorNode([], "Empty 'from' declaration.")), ("def concept name from def ", SyntaxErrorNode([], "Empty 'from' declaration.")), ("def concept name from as True", SyntaxErrorNode([], "Empty 'from' declaration.")), ("def concept name from", UnexpectedEofParsingError("While parsing keyword 'from'.")), ("def concept name from ", UnexpectedEofParsingError("While parsing keyword 'from'.")), ]) def test_i_can_detect_empty_def_declaration(self, text, error): sheerka, context, parser, *concepts = self.init_parser() res = parser.parse(context, ParserInput(text)) assert not res.status assert sheerka.isinstance(res.value, BuiltinConcepts.ERROR) assert res.value.body[0] == error @pytest.mark.parametrize("text", [ "def concept addition from a plus b as a + b", "def concept addition from def a plus b as a + b"]) def test_i_can_def_concept_from_definition(self, text): sheerka, context, parser, *concepts = self.init_parser() res = parser.parse(context, ParserInput(text)) expected = get_def_concept("addition", definition="a plus b", body="a + b") node = res.value.value assert res.status assert res.who == parser.name assert res.value.source == text assert isinstance(res.value, ParserResultConcept) assert node == expected @pytest.mark.parametrize("text", [ "def concept plus from:\n\ta plus b", "def concept plus from def:\n\ta plus b", # space before the colon "def concept plus from :\n\ta plus b", "def concept plus from def :\n\ta plus b", # space after the colon "def concept plus from: \n\ta plus b", "def concept plus from def: \n\ta plus b", ]) def test_i_can_use_colon_and_definition_together(self, text): sheerka, context, parser, *concepts = self.init_parser() res = parser.parse(context, ParserInput(text)) defined_concept = res.body.body defined_concept_tokens = [t.repr_value for t in defined_concept.definition.tokens] assert res.status assert defined_concept.definition_type == DEFINITION_TYPE_DEF assert defined_concept_tokens == [t.repr_value for t in Tokenizer("a plus b", yield_eof=False)] def test_i_can_use_colon_to_protect_keyword(self): text = """ def concept today as: from datetime import date today = date.today() from: give me the date ! """ sheerka, context, parser, *concepts = self.init_parser() res = parser.parse(context, ParserInput(text)) defined_concept = res.body.body defined_concept_tokens = [t.repr_value for t in defined_concept.definition.tokens] assert res.status assert defined_concept.definition_type == DEFINITION_TYPE_DEF assert defined_concept_tokens == [t.repr_value for t in Tokenizer("give me the date !", yield_eof=False)] assert defined_concept.body.status def test_i_can_use_colon_to_protect_keyword_2(self): text = """ def concept today as: from datetime import date today = date.today() from give me the date ! """ sheerka, context, parser, *concepts = self.init_parser() res = parser.parse(context, ParserInput(text)) defined_concept = res.body.body defined_concept_tokens = [t.repr_value for t in defined_concept.definition.tokens] assert res.status assert defined_concept.definition_type == DEFINITION_TYPE_DEF assert defined_concept_tokens == [t.repr_value for t in Tokenizer("give me the date !", yield_eof=False)] assert defined_concept.body.status @pytest.mark.parametrize("text", [ "def", "def concept_name" ]) def test_i_cannot_parse_invalid_entries(self, text): sheerka, context, parser, *concepts = self.init_parser() res = parser.parse(context, ParserInput(text)) assert not res.status assert sheerka.isinstance(res.body, BuiltinConcepts.NOT_FOR_ME) assert isinstance(res.body.reason[0], UnexpectedTokenParsingError) @pytest.mark.parametrize("text, error_msg, error_text", [ ("'name", "Missing Trailing quote", "'name"), ("foo isa 'name", "Missing Trailing quote", "'name"), ("def concept 'name", "Missing Trailing quote", "'name"), ("def concept name as 'body", "Missing Trailing quote", "'body"), ("def concept name from bnf 'expression", "Missing Trailing quote", "'expression"), ("def concept c::", "Concept identifiers not found", ""), ]) def test_i_cannot_parse_when_tokenizer_fails(self, text, error_msg, error_text): sheerka, context, parser, *concepts = self.init_parser() res = parser.parse(context, ParserInput(text)) assert not res.status assert sheerka.isinstance(res.body, BuiltinConcepts.ERROR) assert isinstance(res.body.body[0], LexerError) assert res.body.body[0].message == error_msg assert res.body.body[0].text == error_text def test_i_cannot_parse_bnf_definition_referencing_multiple_concepts_sharing_the_same_name(self): text = "def concept twenty one from bnf 'twenty' one" sheerka, context, parser, *concepts = self.init_parser(Concept("one", body="1"), Concept("one", body="1.0")) res = parser.parse(context, ParserInput(text)) assert not res.status assert context.sheerka.isinstance(res.value, BuiltinConcepts.CANNOT_RESOLVE_CONCEPT) assert res.value.body == ("key", "one") @pytest.mark.parametrize("text", [ 'def concept "def concept x"', 'def concept "def concept x" as x', ]) def test_i_can_use_double_quotes_to_protect_keywords(self, text): sheerka, context, parser, *concepts = self.init_parser() res = parser.parse(context, ParserInput(text)) concept_defined = res.value.value assert res.status assert concept_defined.name.tokens == list(Tokenizer("def concept x", yield_eof=False)) def test_i_can_parse_when_ambiguity_in_where_or_pre_clause(self): sheerka, context, parser, *concepts = self.init_parser( Concept("x is a y", pre="in_context(BuiltinConcepts.EVAL_QUESTION_REQUESTED)"), Concept("x is a y") ) text = "def concept foo x y where x is a y" res = parser.parse(context, ParserInput(text)) node = res.value.value assert res.status assert res.who == parser.name assert res.value.source == text assert isinstance(res.value, ParserResultConcept) assert isinstance(node, DefConceptNode) assert sheerka.isinstance(node.where, BuiltinConcepts.RETURN_VALUE) compare_with_test_object(node.where.body.body, CV(concepts[0], pre=True)) text = "def concept foo x y pre x is a y" res = parser.parse(context, ParserInput(text)) node = res.value.value assert res.status assert res.who == parser.name assert res.value.source == text assert isinstance(res.value, ParserResultConcept) assert isinstance(node, DefConceptNode) assert sheerka.isinstance(node.pre, BuiltinConcepts.RETURN_VALUE) compare_with_test_object(node.pre.body.body, CV(concepts[0], pre=True)) def test_i_can_parse_bnf_concept_with_regex(self): sheerka, context, parser, number = self.init_parser("number") text = "def concept sha512 from bnf r'^[a-f0-9]{128}$'" res = parser.parse(context, ParserInput(text)) assert res.status assert res.who == parser.name assert res.value.source == text assert isinstance(res.value, ParserResultConcept) node = res.value.value parsing_expression = RegExMatch("^[a-f0-9]{128}$") parser_result = ParserResultConcept(BnfDefinitionParser(), "r'^[a-f0-9]{128}$'", None, parsing_expression, parsing_expression) expected = get_def_concept(name="sha512", bnf_def=parser_result) assert node == expected def test_i_can_parse_bnf_concept_with_a_more_complicated_bnf(self): sheerka, context, parser, number = self.init_parser("number") text = "def concept foo from bnf number | r'[a-f0-9]+' | (number r'[a-f0-9]+')+" res = parser.parse(context, ParserInput(text)) assert res.status assert res.who == parser.name assert res.value.source == text assert isinstance(res.value, ParserResultConcept) node = res.value.value parsing_expression = OrderedChoice( ConceptExpression(number, rule_name="number"), RegExMatch("[a-f0-9]+"), OneOrMore(Sequence(ConceptExpression(number, rule_name="number"), RegExMatch("[a-f0-9]+"))) ) parser_result = ParserResultConcept(BnfDefinitionParser(), "number | r'[a-f0-9]+' | (number r'[a-f0-9]+')+", None, parsing_expression, parsing_expression) expected = get_def_concept(name="foo", bnf_def=parser_result) assert node == expected def test_i_can_parse_bnf_concept_definition_with_a_variable(self): sheerka, context, parser, number = self.init_parser("number") text = "def concept foo from bnf number x where x" res = parser.parse(context, ParserInput(text)) node = res.value.value definition = Sequence(ConceptExpression(number, rule_name="number"), VariableExpression("x")) parser_result = ParserResultConcept(BnfDefinitionParser(), "number x", None, definition, definition) expected = get_def_concept(name="foo", bnf_def=parser_result, where="x") assert res.status assert res.who == parser.name assert res.value.source == text assert isinstance(res.value, ParserResultConcept) assert node == expected def test_i_can_parse_bnf_definition_referencing_unknown_concept(self): text = "def concept name from bnf unknown" sheerka, context, parser, *concepts = self.init_parser() res = parser.parse(context, ParserInput(text)) node = res.value.value definition = VariableExpression("unknown") parser_result = ParserResultConcept(BnfDefinitionParser(), "unknown", None, definition, definition) expected = get_def_concept(name="name", bnf_def=parser_result) assert res.status assert res.who == parser.name assert res.value.source == text assert isinstance(res.value, ParserResultConcept) assert node == expected def test_i_can_parse_when_multiple_keyword_and_no_ambiguity(self): sheerka, context, parser, *concepts = self.init_parser() res = parser.parse(context, ParserInput("def concept x is a concept")) assert res.status @pytest.mark.parametrize("text, expected", [ ("def concept foo auto_eval True", True), ("def concept foo auto_eval true", True), ("def concept foo auto_eval False", False), ("def concept foo auto_eval false", False), ]) def test_i_can_parse_auto_eval(self, text, expected): sheerka, context, parser, *concepts = self.init_parser() res = parser.parse(context, ParserInput(text)) def_concept_node = res.value.value assert res.status assert res.who == parser.name assert res.value.source == text assert isinstance(res.value, ParserResultConcept) assert def_concept_node.auto_eval == expected @pytest.mark.parametrize("text", [ "def concept foo auto_eval", "def concept foo auto_eval as 1" ]) def test_i_cannot_parse_when_missing_auto_eval_value(self, text): sheerka, context, parser, *concepts = self.init_parser() res = parser.parse(context, ParserInput(text)) assert not res.status assert sheerka.isinstance(res.value, BuiltinConcepts.ERROR) assert sheerka.has_error(context, res, __type="SyntaxErrorNode", message="Empty 'auto_eval' declaration.") def test_i_cannot_parse_when_wrong_auto_eval_value(self): sheerka, context, parser, *concepts = self.init_parser() text = "def concept foo auto_eval wrong_value" res = parser.parse(context, ParserInput(text)) assert not res.status assert sheerka.isinstance(res.value, BuiltinConcepts.ERROR) assert isinstance(res.value.body[0], CannotHandleParsingError) @pytest.mark.parametrize("text, expected", [ ("def concept foo def_var var", ["var"]), ("def concept foo def_var var1 def_var var2", ["var1", "var2"]), ("def concept foo def_var var1 def_var long var name def_var var2", ["var1", "long var name", "var2"]), ]) def test_i_can_parse_variable_definition(self, text, expected): sheerka, context, parser, *concepts = self.init_parser() res = parser.parse(context, ParserInput(text)) def_concept_node = res.value.value assert res.status assert res.who == parser.name assert res.value.source == text assert isinstance(res.value, ParserResultConcept) assert def_concept_node.variables == expected @pytest.mark.parametrize("text", [ "def concept foo def_var", "def concept foo def_var as 1" ]) def test_i_cannot_parse_variable_definition_when_missing_value(self, text): sheerka, context, parser, *concepts = self.init_parser() res = parser.parse(context, ParserInput(text)) assert not res.status assert sheerka.isinstance(res.value, BuiltinConcepts.ERROR) assert sheerka.has_error(context, res, __type="SyntaxErrorNode", message="Empty 'def_var' declaration.")