kodjo/Sheerka-Old
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

First implementation of Debugger for SyaNodeParser

Browse Source
This commit is contained in:
Kodjo Sossouvi
2020-12-03 21:50:48 +01:00
parent 4f899280c4
commit 8b86998225
48 changed files with 1781 additions and 1795 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/parsers/test_SyaNodeParser.py
+59 -35
View File
@@ -1,10 +1,11 @@
import pytest
from core.builtin_concepts import BuiltinConcepts
from core.concept import Concept, CIO, ALL_ATTRIBUTES
from core.concept import Concept, CIO, ALL_ATTRIBUTES, CMV
from core.global_symbols import CONCEPT_COMPARISON_CONTEXT
from core.sheerka.services.SheerkaExecute import ParserInput
from core.tokenizer import Tokenizer
from parsers.BaseNodeParser import utnode, ConceptNode, cnode, short_cnode, UnrecognizedTokensNode, \
from core.utils import NextIdManager
from parsers.BaseNodeParser import utnode, cnode, short_cnode, UnrecognizedTokensNode, \
SCWC, CNC, UTN, SCN, CN
from parsers.PythonParser import PythonNode
from parsers.SyaNodeParser import SyaNodeParser, SyaConceptParserHelper, SyaAssociativity, \
@@ -916,21 +917,21 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
@pytest.mark.parametrize("expression, expected_debugs", [
("one", [[" 0:one => PUSH_UNREC"]]),
("one plus two", [[
' 0:one => PUSH_UNREC',
' 1:<ws> => PUSH_UNREC',
' 2:plus(SyaConceptDef(concept=(1005)a plus b, precedence=1, associativity=right)) => ??',
" _: => RECOG [[CN((1001)one)]]",
" _: => POP ConceptNode(concept='(1001)one', source='one', start=0, end=0)",
' 2:plus(SyaConceptDef(concept=(1005)a plus b, precedence=1, associativity=right)) => PUSH',
' 3:<ws> => EAT',
' 4:two => PUSH_UNREC',
' 5:<EOF> => ??',
" _: => RECOG [[CN((1002)two)]]",
" _: => POP ConceptNode(concept='(1002)two', source='two', start=4, end=4)",
' _: => POP SyaConceptParserHelper(concept=(1005)a plus b, start=2, error=None)']]),
("one plus two", [[' 0:one => PUSH_UNREC',
' 1:<ws> => PUSH_UNREC',
' 2:plus ((1005)a plus b, prio=1, assoc=SyaAssociativity.Right) => ??',
' _: => RECOG [[CN((1001)one)]]',
" _: => POP ConceptNode(concept='(1001)one', source='one', start=0, end=0)",
' 2:plus ((1005)a plus b, prio=1, assoc=SyaAssociativity.Right) => PUSH',
' 3:<ws> => EAT',
' 4:two => PUSH_UNREC',
' 5:<EOF> => ??',
' _: => RECOG [[CN((1002)two)]]',
" _: => POP ConceptNode(concept='(1002)two', source='two', start=4, end=4)",
' _: => POP SyaConceptParserHelper(concept=(1005)a plus b, start=2, '
'error=None)']]),
("suffixed one", [[
' 0:suffixed(SyaConceptDef(concept=(1009)suffixed a, precedence=1, associativity=right)) => PUSH',
' 0:suffixed ((1009)suffixed a, prio=1, assoc=SyaAssociativity.Right) => PUSH',
' 1:<ws> => EAT',
' 2:one => PUSH_UNREC',
' 3:<EOF> => ??',
@@ -941,10 +942,10 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
("one ? twenty one : three", [[
' 0:one => PUSH_UNREC',
' 1:<ws> => PUSH_UNREC',
' 2:?(SyaConceptDef(concept=(1011)a ? b : c, precedence=1, associativity=right)) => ??',
" _: => RECOG [[CN((1001)one)]]",
' 2:? ((1011)a ? b : c, prio=1, assoc=SyaAssociativity.Right) => ??',
' _: => RECOG [[CN((1001)one)]]',
" _: => POP ConceptNode(concept='(1001)one', source='one', start=0, end=0)",
' 2:?(SyaConceptDef(concept=(1011)a ? b : c, precedence=1, associativity=right)) => PUSH',
' 2:? ((1011)a ? b : c, prio=1, assoc=SyaAssociativity.Right) => PUSH',
' 3:<ws> => EAT',
' 4:twenty => PUSH_UNREC',
' 5:<ws> => PUSH_UNREC',
@@ -955,14 +956,13 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
" _: => POP UnrecognizedTokensNode(source='twenty ', start=4, end=5)",
" _: => POP ConceptNode(concept='(1001)one', source='one', start=6, end=6)",
" _: => => ERROR Too many parameters found for '(1011)a ? b : c' before token 'Token(:)'",
' 8:: => EAT',
], [
' 8:: => EAT'], [
' 0:one => PUSH_UNREC',
' 1:<ws> => PUSH_UNREC',
' 2:?(SyaConceptDef(concept=(1011)a ? b : c, precedence=1, associativity=right)) => ??',
' 2:? ((1011)a ? b : c, prio=1, assoc=SyaAssociativity.Right) => ??',
' _: => RECOG [[CN((1001)one)]]',
" _: => POP ConceptNode(concept='(1001)one', source='one', start=0, end=0)",
' 2:?(SyaConceptDef(concept=(1011)a ? b : c, precedence=1, associativity=right)) => PUSH',
' 2:? ((1011)a ? b : c, prio=1, assoc=SyaAssociativity.Right) => PUSH',
' 3:<ws> => EAT',
' 4:twenty => PUSH_UNREC',
' 5:<ws> => PUSH_UNREC',
@@ -976,12 +976,12 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
' 11:<EOF> => ??',
' _: => RECOG [[CN((1003)three)]]',
" _: => POP ConceptNode(concept='(1003)three', source='three', start=10, end=10)",
' _: => POP SyaConceptParserHelper(concept=(1011)a ? b : c, start=2, error=None)'
]]),
' _: => POP SyaConceptParserHelper(concept=(1011)a ? b : c, start=2, error=None)']]),
])
def test_i_can_debug(self, expression, expected_debugs):
sheerka, context, parser = self.init_parser()
context.debug_enabled = True
sheerka.set_debug(context, True)
sheerka.debug_var(context, "Sya")
res = parser.infix_to_postfix(context, ParserInput(expression))
assert len(res) == len(expected_debugs)
@@ -989,6 +989,19 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
actual_debug = [str(di) for di in res_i.debug]
assert actual_debug == expected_debug
@pytest.mark.parametrize("settings", [
"Sya.*.*",
"Sya.*.#0.can_pop"
])
def test_i_can_debug_can_pop_using_star(self, settings):
sheerka, context, parser = self.init_parser()
sheerka.set_debug(context, True)
sheerka.debug_var(context, settings)
res = parser.infix_to_postfix(context, ParserInput("one plus two mult three"))
debug = [str(di) for di in res[0].debug]
assert debug[5] == ' _: => No stack. CAN_POP false.'
def test_i_can_parse_when_concept_atom_only(self):
sheerka, context, parser = self.init_parser()
@@ -999,15 +1012,20 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
assert res.status
assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
assert lexer_nodes == [ConceptNode(cmap["plus"], 0, 8, source=text)]
assert lexer_nodes == [CN(cmap["plus"], 0, 8, source=text)]
# check the compiled
expected_concept = lexer_nodes[0].concept
assert expected_concept.get_compiled()["a"] == cmap["one"]
assert expected_concept.get_compiled()["b"] == cmap["mult"]
assert expected_concept.get_compiled()["b"] == CMV(cmap["mult"], a="two", b="three")
assert expected_concept.get_compiled()["b"].get_compiled()["a"] == cmap["two"]
assert expected_concept.get_compiled()["b"].get_compiled()["b"] == cmap["three"]
# check the metadata
expected_concept = lexer_nodes[0].concept
assert expected_concept.get_metadata().variables == [("a", "one"), ("b", "two mult three")]
assert expected_concept.get_compiled()["b"].get_metadata().variables == [("a", "two"), ("b", "three")]
def test_i_can_parse_when_python_code(self):
sheerka, context, parser = self.init_parser()
@@ -1018,7 +1036,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
assert res.status
assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
assert lexer_nodes == [ConceptNode(cmap["suffixed"], 0, 6, source=text)]
assert lexer_nodes == [CN(cmap["suffixed"], 0, 6, source=text)]
# check the compiled
expected_concept = lexer_nodes[0].concept
@@ -1031,6 +1049,9 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
assert return_value_a.body.source == "1 + 1"
assert isinstance(return_value_a.body.body, PythonNode)
# check metadata
assert expected_concept.get_metadata().variables == [("a", "1 + 1")]
def test_i_can_parse_when_bnf_concept(self):
sheerka, context, parser = self.init_parser()
@@ -1043,13 +1064,16 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
lexer_nodes = res[1].body.body
assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
assert lexer_nodes == [ConceptNode(cmap["suffixed"], 0, 4, source=text)]
assert lexer_nodes == [CN(cmap["suffixed"], 0, 4, source=text)]
# check the compiled
expected_concept = lexer_nodes[0].concept
assert sheerka.isinstance(expected_concept.get_compiled()["a"], "twenties")
assert expected_concept.get_compiled()["a"].get_compiled()["unit"] == cmap["one"]
# check metadata
assert expected_concept.get_metadata().variables == [("a", "twenty one")]
def test_i_can_parse_sequences(self):
sheerka, context, parser = self.init_parser()
@@ -1061,8 +1085,8 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
assert res.status
assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
assert lexer_nodes == [
ConceptNode(cmap["plus"], 0, 9, source="one plus 1 + 1 "),
ConceptNode(cmap["suffixed"], 10, 12, source="suffixed two")]
CN(cmap["plus"], 0, 9, source="one plus 1 + 1 "),
CN(cmap["suffixed"], 10, 12, source="suffixed two")]
# check the compiled
concept_plus_a = lexer_nodes[0].concept.get_compiled()["a"]
@@ -1198,7 +1222,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
assert not res.status
assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
assert lexer_nodes == [ConceptNode(cmap[expected_concept], 0, expected_end, source=text)]
assert lexer_nodes == [CN(cmap[expected_concept], 0, expected_end, source=text)]
concept_found = lexer_nodes[0].concept
for unrecognized in expected_unrecognized:
@@ -1278,7 +1302,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
])
def test_i_can_get_functions_names_from_unrecognized(self, expression, expected):
sheerka, context, parser = self.init_parser()
infix_to_postfix = InFixToPostFix(context)
infix_to_postfix = InFixToPostFix(context, NextIdManager())
tokens = list(Tokenizer(expression, yield_eof=False))
for pos, token in enumerate(tokens[:-1]):
@@ -1302,7 +1326,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
])
def test_i_can_get_functions_names_from_unrecognized_when_multiple_results(self, expression, expected_list):
sheerka, context, parser = self.init_parser()
infix_to_postfix = InFixToPostFix(context)
infix_to_postfix = InFixToPostFix(context, NextIdManager())
tokens = list(Tokenizer(expression, yield_eof=False))
for pos, token in enumerate(tokens[:-1]):