Refactored Caching, Refactored BnfNodeParser, Introduced Sphinx
This commit is contained in:
@@ -1,6 +1,7 @@
|
||||
from core.concept import CC, Concept
|
||||
from core.concept import CC, Concept, ConceptParts, DoNotResolve
|
||||
from core.tokenizer import Tokenizer, TokenKind, Token
|
||||
from parsers.BaseNodeParser import scnode, utnode, cnode, SCWC, CNC, short_cnode, SourceCodeWithConceptNode, CN, UTN
|
||||
from parsers.BaseNodeParser import scnode, utnode, cnode, SCWC, CNC, short_cnode, SourceCodeWithConceptNode, CN, UTN, \
|
||||
SCN
|
||||
from parsers.SyaNodeParser import SyaConceptParserHelper
|
||||
|
||||
|
||||
@@ -43,7 +44,16 @@ def compute_debug_array(res):
|
||||
return to_compare
|
||||
|
||||
|
||||
def get_node(concepts_map, expression_as_tokens, sub_expr, concept_key=None, skip=0, is_bnf=False, sya=False):
|
||||
def get_node(
|
||||
concepts_map,
|
||||
expression_as_tokens,
|
||||
sub_expr,
|
||||
concept_key=None,
|
||||
skip=0,
|
||||
is_bnf=False,
|
||||
sya=False,
|
||||
init_empty_body=False,
|
||||
exclude_body=False):
|
||||
"""
|
||||
Tries to find sub in expression
|
||||
When found, transform it to its correct type
|
||||
@@ -54,6 +64,8 @@ def get_node(concepts_map, expression_as_tokens, sub_expr, concept_key=None, ski
|
||||
:param skip: number of occurrences of sub_expr to skip
|
||||
:param is_bnf: True if the concept to search is a bnf definition
|
||||
:param sya: Return SyaConceptParserHelper instead of a ConceptNode when needed
|
||||
:param init_empty_body: if True adds the source in the body (actually in compiled.BODY)
|
||||
:param exclude_body: Ask to not compare body
|
||||
:return:
|
||||
"""
|
||||
if sub_expr == "')'":
|
||||
@@ -80,23 +92,38 @@ def get_node(concepts_map, expression_as_tokens, sub_expr, concept_key=None, ski
|
||||
content = [get_node(concepts_map, expression_as_tokens, c, sya=sya) for c in sub_expr.content]
|
||||
return SourceCodeWithConceptNode(first, last, content).pseudo_fix_source()
|
||||
|
||||
if isinstance(sub_expr, SCN):
|
||||
node = get_node(concepts_map, expression_as_tokens, sub_expr.source, sya=sya)
|
||||
sub_expr.fix_pos(node)
|
||||
return sub_expr
|
||||
|
||||
if isinstance(sub_expr, (CNC, CC, CN)):
|
||||
concept_node = get_node(
|
||||
concepts_map,
|
||||
expression_as_tokens,
|
||||
sub_expr.source or sub_expr.concept_key,
|
||||
sub_expr.concept_key, sya=sya)
|
||||
if not hasattr(concept_node, "concept"):
|
||||
raise Exception(f"'{sub_expr.concept_key}' is not a concept. Check your map.")
|
||||
concept_found = concept_node.concept
|
||||
sub_expr.concept_key = concept_found.key
|
||||
sub_expr.concept = concept_found
|
||||
sub_expr.fix_pos((concept_node.start, concept_node.end if hasattr(concept_node, "end") else concept_node.start))
|
||||
if hasattr(sub_expr, "compiled"):
|
||||
for k, v in sub_expr.compiled.items():
|
||||
node = get_node(concepts_map, expression_as_tokens, v, sya=sya) # need to get start and end positions
|
||||
new_value = CC(Concept().update_from(concepts_map[v])) if (isinstance(v, str) and v in concepts_map) \
|
||||
else node
|
||||
node = get_node(concepts_map, expression_as_tokens, v, sya=sya, exclude_body=exclude_body) # need to get start and end positions
|
||||
if isinstance(v, str) and v in concepts_map:
|
||||
new_value_concept = concepts_map[v]
|
||||
new_value = CC(Concept().update_from(new_value_concept), exclude_body=exclude_body)
|
||||
if init_empty_body:
|
||||
init_body(new_value, concept_found, v)
|
||||
else:
|
||||
new_value = node
|
||||
|
||||
sub_expr.compiled[k] = new_value
|
||||
sub_expr.fix_pos(node)
|
||||
if init_empty_body:
|
||||
init_body(sub_expr, concept_found, sub_expr.source)
|
||||
|
||||
if hasattr(sub_expr, "fix_source"):
|
||||
sub_expr.fix_source(expression_as_tokens[sub_expr.start: sub_expr.end + 1])
|
||||
@@ -119,32 +146,54 @@ def get_node(concepts_map, expression_as_tokens, sub_expr, concept_key=None, ski
|
||||
|
||||
# special case of python source code
|
||||
if "+" in sub_expr and sub_expr.strip() != "+":
|
||||
return scnode(start, start + length - 1, sub_expr)
|
||||
return SCN(sub_expr, start, start + length - 1)
|
||||
|
||||
# try to match one of the concept from the map
|
||||
concept_key = concept_key or sub_expr
|
||||
concept_found = concepts_map.get(concept_key, None)
|
||||
if concept_found:
|
||||
concept_found = Concept().update_from(concept_found) # make a copy when massively used in tests
|
||||
if not sya or len(concept_found.metadata.props) == 0 or is_bnf:
|
||||
# if it's an atom, then return a ConceptNode
|
||||
return CN(concept_found, start, start + length - 1, source=sub_expr)
|
||||
else:
|
||||
# else return a ParserHelper
|
||||
if sya and len(concept_found.metadata.variables) > 0 and not is_bnf:
|
||||
return SyaConceptParserHelper(concept_found, start)
|
||||
elif init_empty_body:
|
||||
node = CNC(concept_found, start, start + length - 1, source=sub_expr, exclude_body=exclude_body)
|
||||
init_body(node, concept_found, sub_expr)
|
||||
return node
|
||||
else:
|
||||
return CN(concept_found, start, start + length - 1, source=sub_expr)
|
||||
else:
|
||||
# else an UnrecognizedTokensNode
|
||||
return utnode(start, start + length - 1, sub_expr)
|
||||
|
||||
|
||||
def compute_expected_array(concepts_map, expression, expected, sya=False):
|
||||
def init_body(item, concept, value):
|
||||
if "body" in item.compiled:
|
||||
item.compiled[ConceptParts.BODY] = item.compiled["body"]
|
||||
del (item.compiled["body"])
|
||||
return
|
||||
|
||||
if not concept or concept.metadata.body or ConceptParts.BODY in item.compiled:
|
||||
return
|
||||
|
||||
item.compiled[ConceptParts.BODY] = DoNotResolve(value)
|
||||
|
||||
|
||||
def compute_expected_array(concepts_map, expression, expected, sya=False, init_empty_body=False, exclude_body=False):
|
||||
"""
|
||||
Computes a simple but sufficient version of the result of infix_to_postfix()
|
||||
:param concepts_map:
|
||||
:param expression:
|
||||
:param expected:
|
||||
:param sya: if true, generate an SyaConceptParserHelper instead of a cnode
|
||||
:param init_empty_body: if True adds the source in the body (actually in compiled.BODY)
|
||||
:param exclude_body: do not include ConceptParts.BODY in comparison
|
||||
:return:
|
||||
"""
|
||||
expression_as_tokens = [token.value for token in Tokenizer(expression) if token.type != TokenKind.EOF]
|
||||
return [get_node(concepts_map, expression_as_tokens, sub_expr, sya=sya) for sub_expr in expected]
|
||||
return [get_node(
|
||||
concepts_map,
|
||||
expression_as_tokens,
|
||||
sub_expr,
|
||||
sya=sya,
|
||||
init_empty_body=init_empty_body,
|
||||
exclude_body=exclude_body) for sub_expr in expected]
|
||||
|
||||
@@ -2,18 +2,21 @@ import pytest
|
||||
from core.builtin_concepts import BuiltinConcepts
|
||||
from core.concept import Concept
|
||||
from parsers.AtomNodeParser import AtomNodeParser
|
||||
from parsers.BaseNodeParser import cnode, utnode, CNC
|
||||
from parsers.BaseNodeParser import cnode, utnode, CNC, scnode, SCN
|
||||
|
||||
from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
|
||||
from tests.parsers.parsers_utils import compute_expected_array
|
||||
|
||||
|
||||
class TestAtomsParser(TestUsingMemoryBasedSheerka):
|
||||
def init_parser(self, concepts_map, **kwargs):
|
||||
sheerka, context, *updated_concepts = self.init_concepts(singleton=True, *concepts_map.values(), **kwargs)
|
||||
def init_parser(self, my_map, create_new=False, singleton=True):
|
||||
sheerka, context, *updated_concepts = self.init_concepts(
|
||||
*my_map.values(),
|
||||
create_new=create_new,
|
||||
singleton=singleton)
|
||||
|
||||
parser = AtomNodeParser()
|
||||
parser.initialize(context, updated_concepts)
|
||||
parser.init_from_concepts(context, updated_concepts)
|
||||
|
||||
return sheerka, context, parser
|
||||
|
||||
@@ -34,7 +37,7 @@ class TestAtomsParser(TestUsingMemoryBasedSheerka):
|
||||
concepts_map = {
|
||||
"foo": Concept("foo"),
|
||||
"bar": Concept("bar"),
|
||||
"twenties": Concept("twenties", definition="'twenty' ('one'|'two')=unit").def_prop("unit"),
|
||||
"twenties": Concept("twenties", definition="'twenty' ('one'|'two')=unit").def_var("unit"),
|
||||
}
|
||||
|
||||
sheerka, context, parser = self.init_parser(concepts_map)
|
||||
@@ -57,7 +60,7 @@ class TestAtomsParser(TestUsingMemoryBasedSheerka):
|
||||
concepts_map = {
|
||||
"foo bar": Concept("foo bar"),
|
||||
"one two three": Concept("one two three"),
|
||||
"twenties": Concept("twenties", definition="'twenty' ('one'|'two')=unit").def_prop("unit"),
|
||||
"twenties": Concept("twenties", definition="'twenty' ('one'|'two')=unit").def_var("unit"),
|
||||
}
|
||||
|
||||
sheerka, context, parser = self.init_parser(concepts_map)
|
||||
@@ -75,27 +78,27 @@ class TestAtomsParser(TestUsingMemoryBasedSheerka):
|
||||
("foo bar suffixed one", False, ["foo bar", " suffixed ", "one"]),
|
||||
("foo bar one prefixed", False, ["foo bar", "one", " prefixed"]),
|
||||
("foo bar one infix two", False, ["foo bar", "one", " infix ", "two"]),
|
||||
("foo bar 1 + 1", True, ["foo bar", " 1 + 1"]),
|
||||
("foo bar 1 + 1", False, ["foo bar", " 1 + 1"]),
|
||||
("foo bar twenty one", False, ["foo bar", " twenty ", "one"]),
|
||||
("foo bar x$!#", False, ["foo bar", " x$!#"]),
|
||||
|
||||
("suffixed one foo bar", False, ["suffixed ", "one", "foo bar"]),
|
||||
("one prefixed foo bar", False, ["one", " prefixed ", "foo bar"]),
|
||||
("one infix two foo bar", False, ["one", " infix ", "two", "foo bar"]),
|
||||
("1 + 1 foo bar", True, ["1 + 1 ", "foo bar"]),
|
||||
("1 + 1 foo bar", False, ["1 + 1 ", "foo bar"]),
|
||||
("twenty one foo bar", False, ["twenty ", "one", "foo bar"]),
|
||||
("x$!# foo bar", False, ["x$!# ", "foo bar"]),
|
||||
("func(one)", False, ["func(", "one", ")"]),
|
||||
])
|
||||
def test_i_can_parse_when_unrecognized(self, text, expected_status, expected):
|
||||
concepts_map = {
|
||||
"prefixed": Concept("a prefixed").def_prop("a"),
|
||||
"suffixed": Concept("prefixed a").def_prop("a"),
|
||||
"infix": Concept("a infix b").def_prop("a").def_prop("b"),
|
||||
"prefixed": Concept("a prefixed").def_var("a"),
|
||||
"suffixed": Concept("prefixed a").def_var("a"),
|
||||
"infix": Concept("a infix b").def_var("a").def_var("b"),
|
||||
"foo bar": Concept("foo bar"),
|
||||
"one": Concept("one"),
|
||||
"two": Concept("two"),
|
||||
"twenties": Concept("twenties", definition="'twenty' ('one'|'two')=unit").def_prop("unit"),
|
||||
"twenties": Concept("twenties", definition="'twenty' ('one'|'two')=unit").def_var("unit"),
|
||||
}
|
||||
|
||||
sheerka, context, parser = self.init_parser(concepts_map)
|
||||
@@ -116,13 +119,13 @@ class TestAtomsParser(TestUsingMemoryBasedSheerka):
|
||||
])
|
||||
def test_i_can_parse_when_surrounded_by_spaces(self, text, expected_status, expected):
|
||||
concepts_map = {
|
||||
"prefixed": Concept("a prefixed").def_prop("a"),
|
||||
"suffixed": Concept("prefixed a").def_prop("a"),
|
||||
"infix": Concept("a infix b").def_prop("a").def_prop("b"),
|
||||
"prefixed": Concept("a prefixed").def_var("a"),
|
||||
"suffixed": Concept("prefixed a").def_var("a"),
|
||||
"infix": Concept("a infix b").def_var("a").def_var("b"),
|
||||
"foo bar": Concept("foo bar"),
|
||||
"one": Concept("one"),
|
||||
"two": Concept("two"),
|
||||
"twenties": Concept("twenties", definition="'twenty' ('one'|'two')=unit").def_prop("unit"),
|
||||
"twenties": Concept("twenties", definition="'twenty' ('one'|'two')=unit").def_var("unit"),
|
||||
}
|
||||
|
||||
sheerka, context, parser = self.init_parser(concepts_map)
|
||||
@@ -201,8 +204,8 @@ class TestAtomsParser(TestUsingMemoryBasedSheerka):
|
||||
])
|
||||
def test_i_cannot_parse_concepts_with_property_or_bnf_or_unrecognized(self, text):
|
||||
concepts_map = {
|
||||
"foo": Concept("foo a").def_prop("a"),
|
||||
"twenties": Concept("twenties", definition="'twenty' ('one'|'two')=unit").def_prop("unit"),
|
||||
"foo": Concept("foo a").def_var("a"),
|
||||
"twenties": Concept("twenties", definition="'twenty' ('one'|'two')=unit").def_var("unit"),
|
||||
}
|
||||
|
||||
sheerka, context, parser = self.init_parser(concepts_map)
|
||||
@@ -221,12 +224,12 @@ class TestAtomsParser(TestUsingMemoryBasedSheerka):
|
||||
])
|
||||
def test_i_can_parse_when_unrecognized_yield_multiple_values(self, text, expected):
|
||||
concepts_map = {
|
||||
"hello1": Concept("hello a").def_prop("a"),
|
||||
"hello2": Concept("hello b").def_prop("b"),
|
||||
"hello1": Concept("hello a").def_var("a"),
|
||||
"hello2": Concept("hello b").def_var("b"),
|
||||
"bar": Concept("bar")
|
||||
}
|
||||
|
||||
sheerka, context, parser = self.init_parser(concepts_map, create_new=True)
|
||||
sheerka, context, parser = self.init_parser(concepts_map, create_new=True, singleton=False)
|
||||
list_of_res = parser.parse(context, text)
|
||||
|
||||
assert len(list_of_res) == len(expected)
|
||||
@@ -239,3 +242,23 @@ class TestAtomsParser(TestUsingMemoryBasedSheerka):
|
||||
expected_array = compute_expected_array(concepts_map, text, expected[1])
|
||||
assert sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
|
||||
assert lexer_nodes == expected_array
|
||||
|
||||
@pytest.mark.parametrize("text, expected", [
|
||||
("1 + twenty one", [SCN("1 + twenty "), "one"]),
|
||||
("one + twenty one", ["one", SCN(" + twenty "), ("one", 1)]),
|
||||
])
|
||||
def test_source_code_found_must_be_considered_as_potential_false_positive(self, text, expected):
|
||||
concepts_map = {
|
||||
"one": Concept("one")
|
||||
}
|
||||
|
||||
sheerka, context, parser = self.init_parser(concepts_map)
|
||||
res = parser.parse(context, text)
|
||||
wrapper = res.body
|
||||
lexer_nodes = res.body.body
|
||||
|
||||
assert not res.status
|
||||
|
||||
expected_array = compute_expected_array(concepts_map, text, expected)
|
||||
assert sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
|
||||
assert lexer_nodes == expected_array
|
||||
|
||||
@@ -0,0 +1,257 @@
|
||||
import pytest
|
||||
from core.concept import Concept
|
||||
from parsers.BaseNodeParser import BaseNodeParser
|
||||
from parsers.BnfNodeParser import StrMatch, Sequence, OrderedChoice, Optional, ZeroOrMore, OneOrMore, ConceptExpression
|
||||
|
||||
from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
|
||||
|
||||
|
||||
class TestBaseNodeParser(TestUsingMemoryBasedSheerka):
|
||||
@pytest.mark.parametrize("concept, expected", [
|
||||
(Concept("foo"), {"foo": ["1001"]}),
|
||||
(Concept("foo a").def_var("a"), {"foo": ["1001"]}),
|
||||
(Concept("a b foo").def_var("a").def_var("b"), {"foo": ["1001"]}),
|
||||
])
|
||||
def test_i_can_get_concepts_by_first_keyword(self, concept, expected):
|
||||
"""
|
||||
Given a concept, i can find the first know token
|
||||
example:
|
||||
Concept("a foo b").def_var("a").def_var("b")
|
||||
'a' and 'b' are properties
|
||||
the first 'real' token is foo
|
||||
:return:
|
||||
"""
|
||||
|
||||
sheerka, context, *updated = self.init_concepts(concept)
|
||||
|
||||
res = BaseNodeParser.get_concepts_by_first_keyword(context, updated)
|
||||
|
||||
assert res.status
|
||||
assert res.body == expected
|
||||
|
||||
@pytest.mark.parametrize("bnf, expected", [
|
||||
(StrMatch("foo"), {"foo": ["1002"]}),
|
||||
(StrMatch("bar"), {"bar": ["1002"]}),
|
||||
(ConceptExpression("bar"), {"c:|1001:": ["1002"]}),
|
||||
(Sequence(StrMatch("foo"), StrMatch("bar")), {"foo": ["1002"]}),
|
||||
(Sequence(StrMatch("foo"), ConceptExpression("bar")), {"foo": ["1002"]}),
|
||||
(Sequence(ConceptExpression("bar"), StrMatch("foo")), {"c:|1001:": ["1002"]}),
|
||||
(OrderedChoice(StrMatch("foo"), StrMatch("bar")), {"foo": ["1002"], "bar": ["1002"]}),
|
||||
(Optional(StrMatch("foo")), {"foo": ["1002"]}),
|
||||
(ZeroOrMore(StrMatch("foo")), {"foo": ["1002"]}),
|
||||
(OneOrMore(StrMatch("foo")), {"foo": ["1002"]}),
|
||||
])
|
||||
def test_i_can_get_concepts_by_first_keyword_with_bnf(self, bnf, expected):
|
||||
sheerka = self.get_sheerka()
|
||||
context = self.get_context(sheerka)
|
||||
|
||||
bar = Concept("bar").init_key()
|
||||
sheerka.set_id_if_needed(bar, False)
|
||||
sheerka.add_in_cache(bar)
|
||||
|
||||
concept = Concept("foo").init_key()
|
||||
concept.bnf = bnf
|
||||
sheerka.set_id_if_needed(concept, False)
|
||||
|
||||
res = BaseNodeParser.get_concepts_by_first_keyword(context, [concept])
|
||||
|
||||
assert res.status
|
||||
assert res.body == expected
|
||||
|
||||
def test_i_can_get_concepts_by_first_keyword_when_multiple_concepts(self):
|
||||
sheerka = self.get_sheerka()
|
||||
context = self.get_context(sheerka)
|
||||
|
||||
bar = Concept("bar").init_key()
|
||||
sheerka.set_id_if_needed(bar, False)
|
||||
sheerka.add_in_cache(bar)
|
||||
|
||||
baz = Concept("baz").init_key()
|
||||
sheerka.set_id_if_needed(baz, False)
|
||||
sheerka.add_in_cache(baz)
|
||||
|
||||
foo = Concept("foo").init_key()
|
||||
foo.bnf = OrderedChoice(ConceptExpression("bar"), ConceptExpression("baz"), StrMatch("qux"))
|
||||
sheerka.set_id_if_needed(foo, False)
|
||||
|
||||
res = BaseNodeParser.get_concepts_by_first_keyword(context, [bar, baz, foo])
|
||||
|
||||
assert res.status
|
||||
assert res.body == {
|
||||
"bar": ["1001"],
|
||||
"baz": ["1002"],
|
||||
"c:|1001:": ["1003"],
|
||||
"c:|1002:": ["1003"],
|
||||
"qux": ["1003"],
|
||||
}
|
||||
|
||||
def test_i_can_get_concepts_by_first_keyword_using_sheerka(self):
|
||||
sheerka, context, *updated = self.init_concepts(
|
||||
"one",
|
||||
"two",
|
||||
Concept("twenty", definition="'twenty' (one|two)"),
|
||||
create_new=True
|
||||
)
|
||||
|
||||
bar = Concept("bar").init_key()
|
||||
sheerka.set_id_if_needed(bar, False)
|
||||
sheerka.add_in_cache(bar)
|
||||
|
||||
foo = Concept("foo").init_key()
|
||||
foo.bnf = OrderedChoice(ConceptExpression("one"), ConceptExpression("bar"), StrMatch("qux"))
|
||||
sheerka.set_id_if_needed(foo, False)
|
||||
|
||||
res = BaseNodeParser.get_concepts_by_first_keyword(context, [bar, foo], use_sheerka=True)
|
||||
|
||||
assert res.status
|
||||
assert res.body == {
|
||||
"one": ["1001"],
|
||||
"two": ["1002"],
|
||||
"twenty": ["1003"],
|
||||
"bar": ["1004"],
|
||||
"c:|1001:": ["1005"],
|
||||
"c:|1004:": ["1005"],
|
||||
"qux": ["1005"],
|
||||
}
|
||||
|
||||
def test_i_can_resolve_concepts_by_first_keyword(self):
|
||||
sheerka, context, *updated = self.init_concepts(
|
||||
"one",
|
||||
Concept("two", definition="one"),
|
||||
Concept("three", definition="two"),
|
||||
create_new=False
|
||||
)
|
||||
|
||||
concepts_by_first_keywords = {
|
||||
"one": ["1001"],
|
||||
"c:|1001:": ["1002"],
|
||||
"c:|1002:": ["1003"],
|
||||
}
|
||||
|
||||
resolved_ret_val = BaseNodeParser.resolve_concepts_by_first_keyword(context, concepts_by_first_keywords)
|
||||
|
||||
assert resolved_ret_val.status
|
||||
assert resolved_ret_val.body == {
|
||||
"one": ["1001", "1002", "1003"],
|
||||
}
|
||||
|
||||
def test_i_can_resolve_when_concepts_are_sets(self):
|
||||
sheerka, context, one, two, three, number, foo = self.init_concepts(
|
||||
"one",
|
||||
"two",
|
||||
"three",
|
||||
"number",
|
||||
Concept("foo", definition="number three"),
|
||||
create_new=True # mandatory because set_isa() needs it
|
||||
)
|
||||
|
||||
sheerka.set_isa(context, sheerka.new("one"), number)
|
||||
sheerka.set_isa(context, sheerka.new("two"), number)
|
||||
|
||||
cbfk = BaseNodeParser.get_concepts_by_first_keyword(context, [one, two, three, number, foo]).body
|
||||
|
||||
resolved_ret_val = BaseNodeParser.resolve_concepts_by_first_keyword(context, cbfk)
|
||||
|
||||
assert resolved_ret_val.status
|
||||
assert resolved_ret_val.body == {
|
||||
"one": ["1001", "1005"],
|
||||
"two": ["1002", "1005"],
|
||||
"three": ["1003"],
|
||||
"number": ["1004"],
|
||||
}
|
||||
|
||||
def test_concepts_are_defined_once(self):
|
||||
sheerka = self.get_sheerka()
|
||||
context = self.get_context(sheerka)
|
||||
good = self.create_concept_lite(sheerka, "good")
|
||||
foo = self.create_concept_lite(sheerka, "foo", bnf=ConceptExpression("good"))
|
||||
bar = self.create_concept_lite(sheerka, "bar", bnf=ConceptExpression("good"))
|
||||
baz = self.create_concept_lite(sheerka, "baz", bnf=OrderedChoice(
|
||||
ConceptExpression("foo"),
|
||||
ConceptExpression("bar")))
|
||||
|
||||
concepts_by_first_keywords = BaseNodeParser.get_concepts_by_first_keyword(
|
||||
context, [good, foo, bar, baz]).body
|
||||
|
||||
resolved_ret_val = BaseNodeParser.resolve_concepts_by_first_keyword(context, concepts_by_first_keywords)
|
||||
assert resolved_ret_val.status
|
||||
assert resolved_ret_val.body == {
|
||||
"good": ["1001", "1002", "1003", "1004"],
|
||||
}
|
||||
|
||||
def test_i_can_resolve_more_complex(self):
|
||||
sheerka = self.get_sheerka()
|
||||
context = self.get_context(sheerka)
|
||||
|
||||
a = self.create_concept_lite(sheerka, "a", bnf=Sequence("one", "two"))
|
||||
b = self.create_concept_lite(sheerka, "b", bnf=Sequence(ConceptExpression("a"), "two"))
|
||||
|
||||
concepts_by_first_keywords = BaseNodeParser.get_concepts_by_first_keyword(
|
||||
context, [a, b]).body
|
||||
|
||||
resolved_ret_val = BaseNodeParser.resolve_concepts_by_first_keyword(context, concepts_by_first_keywords)
|
||||
assert resolved_ret_val.status
|
||||
assert resolved_ret_val.body == {
|
||||
"one": ["1001", "1002"],
|
||||
}
|
||||
|
||||
# def tests_i_can_detect_direct_recursion(self):
|
||||
# sheerka = self.get_sheerka()
|
||||
# good = self.get_concept(sheerka, "good")
|
||||
# foo = self.get_concept(sheerka, "foo", ConceptExpression("bar"))
|
||||
# bar = self.get_concept(sheerka, "bar", ConceptExpression("foo"))
|
||||
#
|
||||
# concepts_by_first_keywords = BaseNodeParser.get_concepts_by_first_keyword(sheerka, [good, foo, bar]).body
|
||||
#
|
||||
# resolved_ret_val = BaseNodeParser.resolve_concepts_by_first_keyword(sheerka, concepts_by_first_keywords)
|
||||
# assert resolved_ret_val.status
|
||||
# assert resolved_ret_val.body == {
|
||||
# "good": ["1001"],
|
||||
# BuiltinConcepts.CHICKEN_AND_EGG: ["1002", "1003"]
|
||||
# }
|
||||
#
|
||||
# def test_i_can_detect_indirect_infinite_recursion(self):
|
||||
# sheerka = self.get_sheerka()
|
||||
# good = self.get_concept(sheerka, "good")
|
||||
# one = self.get_concept(sheerka, "one", ConceptExpression("two"))
|
||||
# two = self.get_concept(sheerka, "two", ConceptExpression("three"))
|
||||
# three = self.get_concept(sheerka, "three", ConceptExpression("two"))
|
||||
#
|
||||
# concepts_by_first_keywords = BaseNodeParser.get_concepts_by_first_keyword(sheerka, [good, one, two, three]).body
|
||||
#
|
||||
# resolved_ret_val = BaseNodeParser.resolve_concepts_by_first_keyword(sheerka, concepts_by_first_keywords)
|
||||
# assert resolved_ret_val.status
|
||||
# assert resolved_ret_val.body == {
|
||||
# "good": ["1001"],
|
||||
# BuiltinConcepts.CHICKEN_AND_EGG: ["1002", "1004", "1003"]
|
||||
# }
|
||||
#
|
||||
# def test_i_can_detect_infinite_recursion_from_ordered_choice(self):
|
||||
# sheerka = self.get_sheerka()
|
||||
# good = self.get_concept(sheerka, "good")
|
||||
# one = self.get_concept(sheerka, "one", ConceptExpression("two"))
|
||||
# two = self.get_concept(sheerka, "two", OrderedChoice(ConceptExpression("one"), ConceptExpression("two")))
|
||||
#
|
||||
# concepts_by_first_keywords = BaseNodeParser.get_concepts_by_first_keyword(sheerka, [good, one, two]).body
|
||||
#
|
||||
# resolved_ret_val = BaseNodeParser.resolve_concepts_by_first_keyword(sheerka, concepts_by_first_keywords)
|
||||
# assert resolved_ret_val.status
|
||||
# assert resolved_ret_val.body == {
|
||||
# "good": ["1001"],
|
||||
# BuiltinConcepts.CHICKEN_AND_EGG: ["1002", "1003"]
|
||||
# }
|
||||
#
|
||||
# def test_i_can_detect_infinite_recursion_with_sequence(self):
|
||||
# sheerka = self.get_sheerka()
|
||||
# good = self.get_concept(sheerka, "good")
|
||||
# one = self.get_concept(sheerka, "one", ConceptExpression("two"))
|
||||
# two = self.get_concept(sheerka, "two", Sequence(StrMatch("yes"), ConceptExpression("one")))
|
||||
#
|
||||
# concepts_by_first_keywords = BaseNodeParser.get_concepts_by_first_keyword(sheerka, [good, one, two]).body
|
||||
#
|
||||
# resolved_ret_val = BaseNodeParser.resolve_concepts_by_first_keyword(sheerka, concepts_by_first_keywords)
|
||||
# assert resolved_ret_val.status
|
||||
# assert resolved_ret_val.body == {
|
||||
# "good": ["1001"],
|
||||
# BuiltinConcepts.CHICKEN_AND_EGG: ["1002", "1003"]
|
||||
# }
|
||||
+662
-1175
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -1,13 +1,12 @@
|
||||
import pytest
|
||||
|
||||
from core.builtin_concepts import BuiltinConcepts
|
||||
from core.concept import Concept
|
||||
from core.concept import Concept, DEFINITION_TYPE_BNF
|
||||
from core.tokenizer import Tokenizer, TokenKind, LexerError, Token
|
||||
from parsers.BaseNodeParser import cnode
|
||||
from parsers.BaseParser import UnexpectedTokenErrorNode
|
||||
from parsers.BnfParser import BnfParser, UnexpectedEndOfFileError
|
||||
from parsers.BnfNodeParser import StrMatch, Optional, ZeroOrMore, OrderedChoice, Sequence, OneOrMore, \
|
||||
BnfNodeParser, ConceptExpression
|
||||
from parsers.BnfParser import BnfParser, UnexpectedEndOfFileError
|
||||
|
||||
from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
|
||||
|
||||
@@ -27,6 +26,12 @@ eof_token = Token(TokenKind.EOF, "", 0, 0, 0)
|
||||
|
||||
class TestBnfParser(TestUsingMemoryBasedSheerka):
|
||||
|
||||
def init_parser(self, *concepts):
|
||||
sheerka, context, *updated = self.init_concepts(*concepts, singleton=True)
|
||||
parser = BnfParser()
|
||||
|
||||
return sheerka, context, parser, *updated
|
||||
|
||||
@pytest.mark.parametrize("expression, expected", [
|
||||
("'str'", StrMatch("str")),
|
||||
("1", StrMatch("1")),
|
||||
@@ -80,8 +85,9 @@ class TestBnfParser(TestUsingMemoryBasedSheerka):
|
||||
OneOrMore(StrMatch("b"), rule_name="y"), rule_name="z")),
|
||||
])
|
||||
def test_i_can_parse_regex(self, expression, expected):
|
||||
parser = BnfParser()
|
||||
res = parser.parse(self.get_context(), Tokenizer(expression))
|
||||
sheerka, context, parser = self.init_parser()
|
||||
|
||||
res = parser.parse(context, Tokenizer(expression))
|
||||
|
||||
assert not parser.has_error
|
||||
assert res.status
|
||||
@@ -96,16 +102,11 @@ class TestBnfParser(TestUsingMemoryBasedSheerka):
|
||||
("'str' = var", Sequence(StrMatch("str"), StrMatch("="), c("var"))),
|
||||
("'str''='var", Sequence(StrMatch("str"), StrMatch("="), c("var"))),
|
||||
("foo=f", c("foo", "f")),
|
||||
("foo=f 'constant'", Sequence(c("foo", "f"), StrMatch("constant"))),
|
||||
])
|
||||
def test_i_can_parse_regex_with_concept(self, expression, expected):
|
||||
foo = Concept("foo")
|
||||
bar = Concept("bar")
|
||||
var = Concept("var")
|
||||
context = self.get_context()
|
||||
sheerka, context, parser, foo, bar, var = self.init_parser("foo", "bar", "var")
|
||||
|
||||
for c in (foo, bar, var):
|
||||
context.sheerka.add_in_cache(c)
|
||||
parser = BnfParser()
|
||||
res = parser.parse(context, Tokenizer(expression))
|
||||
|
||||
assert not parser.has_error
|
||||
@@ -118,10 +119,9 @@ class TestBnfParser(TestUsingMemoryBasedSheerka):
|
||||
("foo=f", ConceptExpression("foo", rule_name="f")),
|
||||
])
|
||||
def test_i_can_parse_regex_with_concept_when_the_concept_is_still_under_definition(self, expression, expected):
|
||||
context = self.get_context()
|
||||
sheerka, context, parser = self.init_parser()
|
||||
context.obj = ClassWithName("foo")
|
||||
|
||||
parser = BnfParser()
|
||||
res = parser.parse(context, Tokenizer(expression))
|
||||
|
||||
assert not parser.has_error
|
||||
@@ -137,47 +137,43 @@ class TestBnfParser(TestUsingMemoryBasedSheerka):
|
||||
("'name", LexerError("Missing Trailing quote", "'name", 5, 1, 6))
|
||||
])
|
||||
def test_i_can_detect_errors(self, expression, error):
|
||||
parser = BnfParser()
|
||||
res = parser.parse(self.get_context(), Tokenizer(expression))
|
||||
sheerka, context, parser = self.init_parser()
|
||||
|
||||
res = parser.parse(context, Tokenizer(expression))
|
||||
ret_value = res.value.value
|
||||
assert parser.has_error
|
||||
assert not res.status
|
||||
assert ret_value[0] == error
|
||||
|
||||
def test_i_can_use_the_result_of_regex_parsing_to_parse_a_text(self):
|
||||
foo = Concept(name="foo")
|
||||
bar = Concept(name="bar")
|
||||
context = self.get_context()
|
||||
context.sheerka.add_in_cache(foo)
|
||||
context.sheerka.add_in_cache(bar)
|
||||
sheerka, context, regex_parser, foo, bar = self.init_parser("foo", "bar")
|
||||
|
||||
regex_parser = BnfParser()
|
||||
foo_definition = regex_parser.parse(context, "'twenty' | 'thirty'").value.value
|
||||
bar_definition = regex_parser.parse(context, "foo ('one' | 'two')").value.value
|
||||
for concept in [foo, bar]:
|
||||
concept.metadata.definition_type = DEFINITION_TYPE_BNF
|
||||
|
||||
concepts = {bar: bar_definition, foo: foo_definition}
|
||||
concept_parser = BnfNodeParser()
|
||||
concept_parser.initialize(context, concepts)
|
||||
foo.bnf = regex_parser.parse(context, "'twenty' | 'thirty'").value.value
|
||||
bar.bnf = regex_parser.parse(context, "foo ('one' | 'two')").value.value
|
||||
|
||||
res = concept_parser.parse(context, "twenty two")
|
||||
bnf_parser = BnfNodeParser()
|
||||
bnf_parser.init_from_concepts(context, [foo, bar])
|
||||
|
||||
res = bnf_parser.parse(context, "twenty two")
|
||||
assert res.status
|
||||
assert res.value.body == [cnode("bar", 0, 2, "twenty two")]
|
||||
|
||||
res = concept_parser.parse(context, "thirty one")
|
||||
res = bnf_parser.parse(context, "thirty one")
|
||||
assert res.status
|
||||
assert res.value.body == [cnode("bar", 0, 2, "thirty one")]
|
||||
|
||||
res = concept_parser.parse(context, "twenty")
|
||||
res = bnf_parser.parse(context, "twenty")
|
||||
assert res.status
|
||||
assert res.value.body == [cnode("foo", 0, 0, "twenty")]
|
||||
|
||||
def test_i_cannot_parse_when_too_many_concepts(self):
|
||||
foo1 = Concept(name="foo", body="1")
|
||||
foo2 = Concept(name="foo", body="2")
|
||||
context = self.get_context()
|
||||
context.sheerka.cache_by_key["foo"] = [foo1, foo2]
|
||||
sheerka, context, regex_parser, foo1, foo2 = self.init_parser(
|
||||
Concept(name="foo", body="1"),
|
||||
Concept(name="foo", body="2"))
|
||||
|
||||
regex_parser = BnfParser()
|
||||
res = regex_parser.parse(context, "foo")
|
||||
|
||||
assert not res.status
|
||||
@@ -185,9 +181,7 @@ class TestBnfParser(TestUsingMemoryBasedSheerka):
|
||||
assert res.value.body == ('key', 'foo')
|
||||
|
||||
def test_i_cannot_parse_when_unknown_concept(self):
|
||||
context = self.get_context()
|
||||
|
||||
regex_parser = BnfParser()
|
||||
sheerka, context, regex_parser = self.init_parser()
|
||||
res = regex_parser.parse(self.get_context(), "foo")
|
||||
|
||||
assert not res.status
|
||||
|
||||
@@ -1,194 +1,193 @@
|
||||
import ast
|
||||
|
||||
import pytest
|
||||
|
||||
from core.builtin_concepts import ParserResultConcept, ReturnValueConcept, BuiltinConcepts
|
||||
from core.concept import Concept
|
||||
from core.tokenizer import Token, TokenKind, Tokenizer
|
||||
from parsers.BaseNodeParser import SourceCodeNode
|
||||
from parsers.BnfNodeParser import ConceptNode, UnrecognizedTokensNode
|
||||
from parsers.ConceptsWithConceptsParser import ConceptsWithConceptsParser
|
||||
from parsers.MultipleConceptsParser import MultipleConceptsParser
|
||||
from parsers.PythonParser import PythonNode
|
||||
|
||||
from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
|
||||
|
||||
multiple_concepts_parser = MultipleConceptsParser()
|
||||
|
||||
|
||||
def ret_val(*args):
|
||||
result = []
|
||||
index = 0
|
||||
source = ""
|
||||
for item in args:
|
||||
if isinstance(item, Concept):
|
||||
tokens = [Token(TokenKind.IDENTIFIER, item.name, 0, 0, 0)]
|
||||
result.append(ConceptNode(item, index, index, tokens, item.name))
|
||||
index += 1
|
||||
source += item.name
|
||||
elif isinstance(item, PythonNode):
|
||||
tokens = list(Tokenizer(item.source))[:-1] # strip trailing EOF
|
||||
result.append(SourceCodeNode(item, index, index + len(tokens) - 1, tokens, item.source))
|
||||
index += len(tokens)
|
||||
source += item.source
|
||||
else:
|
||||
tokens = list(Tokenizer(item))[:-1] # strip trailing EOF
|
||||
result.append(UnrecognizedTokensNode(index, index + len(tokens) - 1, tokens))
|
||||
index += len(tokens)
|
||||
source += item
|
||||
|
||||
return ReturnValueConcept(
|
||||
"who",
|
||||
False,
|
||||
ParserResultConcept(parser=multiple_concepts_parser, value=result, source=source))
|
||||
|
||||
|
||||
class TestConceptsWithConceptsParser(TestUsingMemoryBasedSheerka):
|
||||
|
||||
def init(self, concepts, inputs):
|
||||
context = self.get_context()
|
||||
for concept in concepts:
|
||||
context.sheerka.create_new_concept(context, concept)
|
||||
|
||||
return context, ret_val(*inputs)
|
||||
|
||||
def execute(self, concepts, inputs):
|
||||
context, input_return_values = self.init(concepts, inputs)
|
||||
|
||||
parser = ConceptsWithConceptsParser()
|
||||
result = parser.parse(context, input_return_values.body)
|
||||
|
||||
wrapper = result.body
|
||||
return_value = result.body.body
|
||||
|
||||
return context, parser, result, wrapper, return_value
|
||||
|
||||
@pytest.mark.parametrize("text, interested", [
|
||||
("not parser result", False),
|
||||
(ParserResultConcept(parser="not multiple_concepts_parser"), False),
|
||||
(ParserResultConcept(parser=multiple_concepts_parser, value=[UnrecognizedTokensNode(0, 0, [])]), True),
|
||||
])
|
||||
def test_not_interested(self, text, interested):
|
||||
context = self.get_context()
|
||||
|
||||
res = ConceptsWithConceptsParser().parse(context, text)
|
||||
if interested:
|
||||
assert res is not None
|
||||
else:
|
||||
assert res is None
|
||||
|
||||
def test_i_can_parse_composition_of_concepts(self):
|
||||
foo = Concept("foo")
|
||||
bar = Concept("bar")
|
||||
plus = Concept("a plus b").def_prop("a").def_prop("b")
|
||||
|
||||
context, parser, result, wrapper, return_value = self.execute([foo, bar, plus], [foo, " plus ", bar])
|
||||
|
||||
assert result.status
|
||||
assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
|
||||
assert result.who == wrapper.parser.name
|
||||
assert wrapper.source == "foo plus bar"
|
||||
assert context.sheerka.isinstance(return_value, plus)
|
||||
|
||||
assert return_value.compiled["a"] == foo
|
||||
assert return_value.compiled["b"] == bar
|
||||
|
||||
# sanity check, I can evaluate the result
|
||||
evaluated = context.sheerka.evaluate_concept(self.get_context(context.sheerka, True), return_value)
|
||||
assert evaluated.key == return_value.key
|
||||
assert evaluated.get_prop("a") == foo.init_key()
|
||||
assert evaluated.get_prop("b") == bar.init_key()
|
||||
|
||||
def test_i_can_parse_when_composition_of_source_code(self):
|
||||
plus = Concept("a plus b", body="a + b").def_prop("a").def_prop("b")
|
||||
left = PythonNode("1+1", ast.parse("1+1", mode="eval"))
|
||||
right = PythonNode("2+2", ast.parse("2+2", mode="eval"))
|
||||
context, parser, result, wrapper, return_value = self.execute([plus], [left, " plus ", right])
|
||||
|
||||
assert result.status
|
||||
assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
|
||||
assert result.who == wrapper.parser.name
|
||||
assert wrapper.source == "1+1 plus 2+2"
|
||||
assert context.sheerka.isinstance(return_value, plus)
|
||||
|
||||
left_parser_result = ParserResultConcept(parser=parser, source="1+1", value=left)
|
||||
right_parser_result = ParserResultConcept(parser=parser, source="2+2", value=right)
|
||||
assert return_value.compiled["a"] == [ReturnValueConcept(parser.name, True, left_parser_result)]
|
||||
assert return_value.compiled["b"] == [ReturnValueConcept(parser.name, True, right_parser_result)]
|
||||
|
||||
# sanity check, I can evaluate the result
|
||||
evaluated = context.sheerka.evaluate_concept(self.get_context(context.sheerka, True), return_value)
|
||||
assert evaluated.key == return_value.key
|
||||
assert evaluated.get_prop("a") == 2
|
||||
assert evaluated.get_prop("b") == 4
|
||||
assert evaluated.body == 6
|
||||
|
||||
def test_i_can_parse_when_mix_of_concept_and_code(self):
|
||||
plus = Concept("a plus b").def_prop("a").def_prop("b")
|
||||
code = PythonNode("1+1", ast.parse("1+1", mode="eval"))
|
||||
foo = Concept("foo")
|
||||
context, parser, result, wrapper, return_value = self.execute([plus, foo], [foo, " plus ", code])
|
||||
|
||||
assert result.status
|
||||
assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
|
||||
assert result.who == wrapper.parser.name
|
||||
assert wrapper.source == "foo plus 1+1"
|
||||
assert context.sheerka.isinstance(return_value, plus)
|
||||
|
||||
code_parser_result = ParserResultConcept(parser=parser, source="1+1", value=code)
|
||||
assert return_value.compiled["a"] == foo
|
||||
assert return_value.compiled["b"] == [ReturnValueConcept(parser.name, True, code_parser_result)]
|
||||
|
||||
# sanity check, I can evaluate the result
|
||||
evaluated = context.sheerka.evaluate_concept(self.get_context(context.sheerka, True), return_value)
|
||||
assert evaluated.key == return_value.key
|
||||
assert evaluated.get_prop("a") == foo.init_key()
|
||||
assert evaluated.get_prop("b") == 2
|
||||
|
||||
def test_i_can_parse_when_multiple_concepts_are_recognized(self):
|
||||
foo = Concept("foo")
|
||||
bar = Concept("bar")
|
||||
plus_1 = Concept("a plus b", body="body1").def_prop("a").def_prop("b")
|
||||
plus_2 = Concept("a plus b", body="body2").def_prop("a").def_prop("b")
|
||||
|
||||
context, input_return_values = self.init([foo, bar, plus_1, plus_2], [foo, " plus ", bar])
|
||||
parser = ConceptsWithConceptsParser()
|
||||
result = parser.parse(context, input_return_values.body)
|
||||
|
||||
assert len(result) == 2
|
||||
|
||||
res = result[0]
|
||||
wrapper = res.value
|
||||
return_value = res.value.value
|
||||
assert res.status
|
||||
assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
|
||||
assert res.who == wrapper.parser.name
|
||||
assert wrapper.source == "foo plus bar"
|
||||
assert context.sheerka.isinstance(return_value, plus_1)
|
||||
assert return_value.compiled["a"] == foo
|
||||
assert return_value.compiled["b"] == bar
|
||||
|
||||
res = result[1]
|
||||
wrapper = res.value
|
||||
return_value = res.value.value
|
||||
assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
|
||||
assert res.who == wrapper.parser.name
|
||||
assert wrapper.source == "foo plus bar"
|
||||
assert context.sheerka.isinstance(return_value, plus_2)
|
||||
assert return_value.compiled["a"] == foo
|
||||
assert return_value.compiled["b"] == bar
|
||||
|
||||
def test_i_cannot_parse_when_unknown_concept(self):
|
||||
foo = Concept("foo")
|
||||
bar = Concept("bar")
|
||||
|
||||
context, input_return_values = self.init([foo, bar], [foo, " plus ", bar])
|
||||
parser = ConceptsWithConceptsParser()
|
||||
result = parser.parse(context, input_return_values.body)
|
||||
wrapper = result.body
|
||||
return_value = result.body.body
|
||||
|
||||
assert not result.status
|
||||
assert context.sheerka.isinstance(wrapper, BuiltinConcepts.NOT_FOR_ME)
|
||||
assert result.who == parser.name
|
||||
assert return_value == input_return_values.body.body
|
||||
# import ast
|
||||
#
|
||||
# import pytest
|
||||
#
|
||||
# from core.builtin_concepts import ParserResultConcept, ReturnValueConcept, BuiltinConcepts
|
||||
# from core.concept import Concept
|
||||
# from core.tokenizer import Token, TokenKind, Tokenizer
|
||||
# from parsers.BaseNodeParser import SourceCodeNode, ConceptNode, UnrecognizedTokensNode
|
||||
# from parsers.ConceptsWithConceptsParser import ConceptsWithConceptsParser
|
||||
# from parsers.MultipleConceptsParser import MultipleConceptsParser
|
||||
# from parsers.PythonParser import PythonNode
|
||||
#
|
||||
# from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
|
||||
#
|
||||
# multiple_concepts_parser = MultipleConceptsParser()
|
||||
#
|
||||
#
|
||||
# def ret_val(*args):
|
||||
# result = []
|
||||
# index = 0
|
||||
# source = ""
|
||||
# for item in args:
|
||||
# if isinstance(item, Concept):
|
||||
# tokens = [Token(TokenKind.IDENTIFIER, item.name, 0, 0, 0)]
|
||||
# result.append(ConceptNode(item, index, index, tokens, item.name))
|
||||
# index += 1
|
||||
# source += item.name
|
||||
# elif isinstance(item, PythonNode):
|
||||
# tokens = list(Tokenizer(item.source))[:-1] # strip trailing EOF
|
||||
# result.append(SourceCodeNode(item, index, index + len(tokens) - 1, tokens, item.source))
|
||||
# index += len(tokens)
|
||||
# source += item.source
|
||||
# else:
|
||||
# tokens = list(Tokenizer(item))[:-1] # strip trailing EOF
|
||||
# result.append(UnrecognizedTokensNode(index, index + len(tokens) - 1, tokens))
|
||||
# index += len(tokens)
|
||||
# source += item
|
||||
#
|
||||
# return ReturnValueConcept(
|
||||
# "who",
|
||||
# False,
|
||||
# ParserResultConcept(parser=multiple_concepts_parser, value=result, source=source))
|
||||
#
|
||||
#
|
||||
# class TestConceptsWithConceptsParser(TestUsingMemoryBasedSheerka):
|
||||
#
|
||||
# def init(self, concepts, inputs):
|
||||
# context = self.get_context()
|
||||
# for concept in concepts:
|
||||
# context.sheerka.create_new_concept(context, concept)
|
||||
#
|
||||
# return context, ret_val(*inputs)
|
||||
#
|
||||
# def execute(self, concepts, inputs):
|
||||
# context, input_return_values = self.init(concepts, inputs)
|
||||
#
|
||||
# parser = ConceptsWithConceptsParser()
|
||||
# result = parser.parse(context, input_return_values.body)
|
||||
#
|
||||
# wrapper = result.body
|
||||
# return_value = result.body.body
|
||||
#
|
||||
# return context, parser, result, wrapper, return_value
|
||||
#
|
||||
# @pytest.mark.parametrize("text, interested", [
|
||||
# ("not parser result", False),
|
||||
# (ParserResultConcept(parser="not multiple_concepts_parser"), False),
|
||||
# (ParserResultConcept(parser=multiple_concepts_parser, value=[UnrecognizedTokensNode(0, 0, [])]), True),
|
||||
# ])
|
||||
# def test_not_interested(self, text, interested):
|
||||
# context = self.get_context()
|
||||
#
|
||||
# res = ConceptsWithConceptsParser().parse(context, text)
|
||||
# if interested:
|
||||
# assert res is not None
|
||||
# else:
|
||||
# assert res is None
|
||||
#
|
||||
# def test_i_can_parse_composition_of_concepts(self):
|
||||
# foo = Concept("foo")
|
||||
# bar = Concept("bar")
|
||||
# plus = Concept("a plus b").def_var("a").def_var("b")
|
||||
#
|
||||
# context, parser, result, wrapper, return_value = self.execute([foo, bar, plus], [foo, " plus ", bar])
|
||||
#
|
||||
# assert result.status
|
||||
# assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
|
||||
# assert result.who == wrapper.parser.name
|
||||
# assert wrapper.source == "foo plus bar"
|
||||
# assert context.sheerka.isinstance(return_value, plus)
|
||||
#
|
||||
# assert return_value.compiled["a"] == foo
|
||||
# assert return_value.compiled["b"] == bar
|
||||
#
|
||||
# # sanity check, I can evaluate the result
|
||||
# evaluated = context.sheerka.evaluate_concept(self.get_context(context.sheerka, True), return_value)
|
||||
# assert evaluated.key == return_value.key
|
||||
# assert evaluated.get_prop("a") == foo.init_key()
|
||||
# assert evaluated.get_prop("b") == bar.init_key()
|
||||
#
|
||||
# def test_i_can_parse_when_composition_of_source_code(self):
|
||||
# plus = Concept("a plus b", body="a + b").def_var("a").def_var("b")
|
||||
# left = PythonNode("1+1", ast.parse("1+1", mode="eval"))
|
||||
# right = PythonNode("2+2", ast.parse("2+2", mode="eval"))
|
||||
# context, parser, result, wrapper, return_value = self.execute([plus], [left, " plus ", right])
|
||||
#
|
||||
# assert result.status
|
||||
# assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
|
||||
# assert result.who == wrapper.parser.name
|
||||
# assert wrapper.source == "1+1 plus 2+2"
|
||||
# assert context.sheerka.isinstance(return_value, plus)
|
||||
#
|
||||
# left_parser_result = ParserResultConcept(parser=parser, source="1+1", value=left)
|
||||
# right_parser_result = ParserResultConcept(parser=parser, source="2+2", value=right)
|
||||
# assert return_value.compiled["a"] == [ReturnValueConcept(parser.name, True, left_parser_result)]
|
||||
# assert return_value.compiled["b"] == [ReturnValueConcept(parser.name, True, right_parser_result)]
|
||||
#
|
||||
# # sanity check, I can evaluate the result
|
||||
# evaluated = context.sheerka.evaluate_concept(self.get_context(context.sheerka, True), return_value)
|
||||
# assert evaluated.key == return_value.key
|
||||
# assert evaluated.get_prop("a") == 2
|
||||
# assert evaluated.get_prop("b") == 4
|
||||
# assert evaluated.body == 6
|
||||
#
|
||||
# def test_i_can_parse_when_mix_of_concept_and_code(self):
|
||||
# plus = Concept("a plus b").def_var("a").def_var("b")
|
||||
# code = PythonNode("1+1", ast.parse("1+1", mode="eval"))
|
||||
# foo = Concept("foo")
|
||||
# context, parser, result, wrapper, return_value = self.execute([plus, foo], [foo, " plus ", code])
|
||||
#
|
||||
# assert result.status
|
||||
# assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
|
||||
# assert result.who == wrapper.parser.name
|
||||
# assert wrapper.source == "foo plus 1+1"
|
||||
# assert context.sheerka.isinstance(return_value, plus)
|
||||
#
|
||||
# code_parser_result = ParserResultConcept(parser=parser, source="1+1", value=code)
|
||||
# assert return_value.compiled["a"] == foo
|
||||
# assert return_value.compiled["b"] == [ReturnValueConcept(parser.name, True, code_parser_result)]
|
||||
#
|
||||
# # sanity check, I can evaluate the result
|
||||
# evaluated = context.sheerka.evaluate_concept(self.get_context(context.sheerka, True), return_value)
|
||||
# assert evaluated.key == return_value.key
|
||||
# assert evaluated.get_prop("a") == foo.init_key()
|
||||
# assert evaluated.get_prop("b") == 2
|
||||
#
|
||||
# def test_i_can_parse_when_multiple_concepts_are_recognized(self):
|
||||
# foo = Concept("foo")
|
||||
# bar = Concept("bar")
|
||||
# plus_1 = Concept("a plus b", body="body1").def_var("a").def_var("b")
|
||||
# plus_2 = Concept("a plus b", body="body2").def_var("a").def_var("b")
|
||||
#
|
||||
# context, input_return_values = self.init([foo, bar, plus_1, plus_2], [foo, " plus ", bar])
|
||||
# parser = ConceptsWithConceptsParser()
|
||||
# result = parser.parse(context, input_return_values.body)
|
||||
#
|
||||
# assert len(result) == 2
|
||||
#
|
||||
# res = result[0]
|
||||
# wrapper = res.value
|
||||
# return_value = res.value.value
|
||||
# assert res.status
|
||||
# assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
|
||||
# assert res.who == wrapper.parser.name
|
||||
# assert wrapper.source == "foo plus bar"
|
||||
# assert context.sheerka.isinstance(return_value, plus_1)
|
||||
# assert return_value.compiled["a"] == foo
|
||||
# assert return_value.compiled["b"] == bar
|
||||
#
|
||||
# res = result[1]
|
||||
# wrapper = res.value
|
||||
# return_value = res.value.value
|
||||
# assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
|
||||
# assert res.who == wrapper.parser.name
|
||||
# assert wrapper.source == "foo plus bar"
|
||||
# assert context.sheerka.isinstance(return_value, plus_2)
|
||||
# assert return_value.compiled["a"] == foo
|
||||
# assert return_value.compiled["b"] == bar
|
||||
#
|
||||
# def test_i_cannot_parse_when_unknown_concept(self):
|
||||
# foo = Concept("foo")
|
||||
# bar = Concept("bar")
|
||||
#
|
||||
# context, input_return_values = self.init([foo, bar], [foo, " plus ", bar])
|
||||
# parser = ConceptsWithConceptsParser()
|
||||
# result = parser.parse(context, input_return_values.body)
|
||||
# wrapper = result.body
|
||||
# return_value = result.body.body
|
||||
#
|
||||
# assert not result.status
|
||||
# assert context.sheerka.isinstance(wrapper, BuiltinConcepts.NOT_FOR_ME)
|
||||
# assert result.who == parser.name
|
||||
# assert return_value == input_return_values.body.body
|
||||
|
||||
@@ -1,14 +1,14 @@
|
||||
import pytest
|
||||
import ast
|
||||
|
||||
import pytest
|
||||
from core.builtin_concepts import ParserResultConcept, BuiltinConcepts, ReturnValueConcept
|
||||
from core.concept import Concept, DEFINITION_TYPE_BNF, DEFINITION_TYPE_DEF
|
||||
from parsers.BnfNodeParser import OrderedChoice, StrMatch, ConceptExpression
|
||||
from parsers.PythonParser import PythonParser, PythonNode
|
||||
from core.concept import DEFINITION_TYPE_BNF, DEFINITION_TYPE_DEF
|
||||
from core.tokenizer import Keywords, Tokenizer, LexerError
|
||||
from parsers.BnfNodeParser import OrderedChoice, ConceptExpression, StrMatch
|
||||
from parsers.BnfParser import BnfParser
|
||||
from parsers.DefaultParser import DefaultParser, NameNode, SyntaxErrorNode, CannotHandleErrorNode, IsaConceptNode
|
||||
from parsers.DefaultParser import UnexpectedTokenErrorNode, DefConceptNode
|
||||
from parsers.BnfParser import BnfParser
|
||||
from parsers.PythonParser import PythonParser, PythonNode
|
||||
|
||||
from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
|
||||
|
||||
@@ -63,6 +63,11 @@ def get_concept_part(part):
|
||||
|
||||
class TestDefaultParser(TestUsingMemoryBasedSheerka):
|
||||
|
||||
def init_parser(self, *concepts):
|
||||
sheerka, concept, *updated = self.init_concepts(*concepts, singleton=True)
|
||||
parser = DefaultParser()
|
||||
return sheerka, concept, parser, *updated
|
||||
|
||||
@pytest.mark.parametrize("text, expected", [
|
||||
("def concept hello", get_def_concept(name="hello")),
|
||||
("def concept hello ", get_def_concept(name="hello")),
|
||||
@@ -76,8 +81,8 @@ class TestDefaultParser(TestUsingMemoryBasedSheerka):
|
||||
("def concept hello as 1 + 1", get_def_concept(name="hello", body="1 + 1")),
|
||||
])
|
||||
def test_i_can_parse_def_concept(self, text, expected):
|
||||
parser = DefaultParser()
|
||||
res = parser.parse(self.get_context(), text)
|
||||
sheerka, context, parser = self.init_parser()
|
||||
res = parser.parse(context, text)
|
||||
node = res.value.value
|
||||
|
||||
assert res.status
|
||||
@@ -93,8 +98,8 @@ pre isinstance(a, int) and isinstance(b, float)
|
||||
post isinstance(res, int)
|
||||
as res = a + b
|
||||
"""
|
||||
parser = DefaultParser()
|
||||
res = parser.parse(self.get_context(), text)
|
||||
sheerka, context, parser = self.init_parser()
|
||||
res = parser.parse(context, text)
|
||||
return_value = res.value
|
||||
expected_concept = get_def_concept(
|
||||
name="a plus b",
|
||||
@@ -123,8 +128,8 @@ func(a)
|
||||
ast.parse("def func(x):\n return x+1\nfunc(a)", mode="exec"))
|
||||
)
|
||||
|
||||
parser = DefaultParser()
|
||||
res = parser.parse(self.get_context(), text)
|
||||
sheerka, context, parser = self.init_parser()
|
||||
res = parser.parse(context, text)
|
||||
return_value = res.value
|
||||
|
||||
assert res.status
|
||||
@@ -146,8 +151,8 @@ def concept add one to a as:
|
||||
ast.parse("def func(x):\n return x+1\nfunc(a)", mode="exec"))
|
||||
)
|
||||
|
||||
parser = DefaultParser()
|
||||
res = parser.parse(self.get_context(), text)
|
||||
sheerka, context, parser = self.init_parser()
|
||||
res = parser.parse(context, text)
|
||||
return_value = res.value
|
||||
|
||||
assert res.status
|
||||
@@ -162,8 +167,8 @@ def func(x):
|
||||
func(a)
|
||||
"""
|
||||
|
||||
parser = DefaultParser()
|
||||
res = parser.parse(self.get_context(), text)
|
||||
sheerka, context, parser = self.init_parser()
|
||||
res = parser.parse(context, text)
|
||||
return_value = res.value
|
||||
|
||||
assert not res.status
|
||||
@@ -178,9 +183,7 @@ def concept add one to a as
|
||||
return x+1
|
||||
func(a)
|
||||
"""
|
||||
context = self.get_context()
|
||||
|
||||
parser = DefaultParser()
|
||||
sheerka, context, parser = self.init_parser()
|
||||
res = parser.parse(context, text)
|
||||
return_value = res.value
|
||||
|
||||
@@ -190,8 +193,8 @@ def concept add one to a as
|
||||
def test_name_is_mandatory(self):
|
||||
text = "def concept as 'hello'"
|
||||
|
||||
parser = DefaultParser()
|
||||
res = parser.parse(self.get_context(), text)
|
||||
sheerka, context, parser = self.init_parser()
|
||||
res = parser.parse(context, text)
|
||||
return_value = res.value
|
||||
|
||||
assert not res.status
|
||||
@@ -203,8 +206,8 @@ def concept add one to a as
|
||||
text = "def hello as a where b pre c post d"
|
||||
|
||||
expected_concept = get_def_concept(name="hello", body="a", where="b", pre="c", post="d")
|
||||
parser = DefaultParser()
|
||||
res = parser.parse(self.get_context(), text)
|
||||
sheerka, context, parser = self.init_parser()
|
||||
res = parser.parse(context, text)
|
||||
return_value = res.value
|
||||
|
||||
assert not res.status
|
||||
@@ -221,10 +224,7 @@ def concept add one to a as
|
||||
"def concept hello as 1+"
|
||||
])
|
||||
def test_i_can_detect_error_in_declaration(self, text):
|
||||
context = self.get_context()
|
||||
sheerka = context.sheerka
|
||||
|
||||
parser = DefaultParser()
|
||||
sheerka, context, parser = self.init_parser()
|
||||
res = parser.parse(context, text)
|
||||
return_value = res.value
|
||||
|
||||
@@ -234,21 +234,18 @@ def concept add one to a as
|
||||
def test_new_line_is_not_allowed_in_the_name(self):
|
||||
text = "def concept hello \n my friend as 'hello'"
|
||||
|
||||
parser = DefaultParser()
|
||||
res = parser.parse(self.get_context(), text)
|
||||
sheerka, context, parser = self.init_parser()
|
||||
res = parser.parse(context, text)
|
||||
return_value = res.value
|
||||
|
||||
assert not res.status
|
||||
assert return_value.value == [SyntaxErrorNode([], "Newline are not allowed in name.")]
|
||||
|
||||
def test_i_can_parse_def_concept_from_bnf(self):
|
||||
context = self.get_context()
|
||||
a_concept = Concept("a_concept")
|
||||
context.sheerka.add_in_cache(a_concept)
|
||||
|
||||
text = "def concept name from bnf a_concept | 'a_string' as __definition[0]"
|
||||
parser = DefaultParser()
|
||||
sheerka, context, parser, a_concept = self.init_parser("a_concept")
|
||||
res = parser.parse(context, text)
|
||||
|
||||
node = res.value.value
|
||||
definition = OrderedChoice(ConceptExpression(a_concept, rule_name="a_concept"), StrMatch("a_string"))
|
||||
parser_result = ParserResultConcept(BnfParser(), "a_concept | 'a_string'", None, definition, definition)
|
||||
@@ -261,12 +258,8 @@ def concept add one to a as
|
||||
assert node == expected
|
||||
|
||||
def test_i_can_parse_def_concept_where_bnf_references_itself(self):
|
||||
context = self.get_context()
|
||||
a_concept = Concept("a_concept")
|
||||
context.sheerka.add_in_cache(a_concept)
|
||||
|
||||
text = "def concept name from bnf 'a' + name?"
|
||||
parser = DefaultParser()
|
||||
sheerka, context, parser, a_concept = self.init_parser("a_concept")
|
||||
parser.parse(context, text)
|
||||
|
||||
assert not parser.has_error
|
||||
@@ -277,9 +270,8 @@ def concept add one to a as
|
||||
"def concept name from as here is my body"
|
||||
])
|
||||
def test_i_can_detect_empty_bnf_declaration(self, text):
|
||||
|
||||
parser = DefaultParser()
|
||||
res = parser.parse(self.get_context(), text)
|
||||
sheerka, context, parser = self.init_parser()
|
||||
res = parser.parse(context, text)
|
||||
|
||||
assert not res.status
|
||||
assert res.value.value[0] == SyntaxErrorNode([], "Empty declaration")
|
||||
@@ -288,8 +280,8 @@ def concept add one to a as
|
||||
"def concept addition from a plus b as a + b",
|
||||
"def concept addition from def a plus b as a + b"])
|
||||
def test_i_can_def_concept_from_definition(self, text):
|
||||
parser = DefaultParser()
|
||||
res = parser.parse(self.get_context(), text)
|
||||
sheerka, context, parser = self.init_parser()
|
||||
res = parser.parse(context, text)
|
||||
expected = get_def_concept("addition", definition="a plus b", body="a + b")
|
||||
node = res.value.value
|
||||
|
||||
@@ -301,8 +293,7 @@ def concept add one to a as
|
||||
|
||||
def test_i_can_detect_not_for_me(self):
|
||||
text = "hello world"
|
||||
context = self.get_context()
|
||||
parser = DefaultParser()
|
||||
sheerka, context, parser = self.init_parser()
|
||||
res = parser.parse(context, text)
|
||||
|
||||
assert not res.status
|
||||
@@ -310,9 +301,9 @@ def concept add one to a as
|
||||
assert isinstance(res.value.body[0], CannotHandleErrorNode)
|
||||
|
||||
def test_i_can_parse_is_a(self):
|
||||
parser = DefaultParser()
|
||||
text = "the name of my 'concept' isa the name of the set"
|
||||
res = parser.parse(self.get_context(), text)
|
||||
sheerka, context, parser = self.init_parser()
|
||||
res = parser.parse(context, text)
|
||||
expected = IsaConceptNode([],
|
||||
concept=NameNode(list(Tokenizer("the name of my 'concept'"))),
|
||||
set=NameNode(list(Tokenizer("the name of the set"))))
|
||||
@@ -331,8 +322,8 @@ def concept add one to a as
|
||||
"def concept_name"
|
||||
])
|
||||
def test_i_cannot_parse_invalid_entries(self, text):
|
||||
parser = DefaultParser()
|
||||
res = parser.parse(self.get_context(), text)
|
||||
sheerka, context, parser = self.init_parser()
|
||||
res = parser.parse(context, text)
|
||||
|
||||
assert not res.status
|
||||
assert isinstance(res.body, ParserResultConcept)
|
||||
@@ -347,8 +338,8 @@ def concept add one to a as
|
||||
("def concept c::", "Concept identifiers not found", ""),
|
||||
])
|
||||
def test_i_cannot_parse_when_tokenizer_fails(self, text, error_msg, error_text):
|
||||
parser = DefaultParser()
|
||||
res = parser.parse(self.get_context(), text)
|
||||
sheerka, context, parser = self.init_parser()
|
||||
res = parser.parse(context, text)
|
||||
|
||||
assert not res.status
|
||||
assert isinstance(res.body, ParserResultConcept)
|
||||
@@ -357,10 +348,8 @@ def concept add one to a as
|
||||
assert res.body.body[0].text == error_text
|
||||
|
||||
def test_i_cannot_parse_bnf_definition_referencing_unknown_concept(self):
|
||||
context = self.get_context()
|
||||
text = "def concept name from bnf unknown"
|
||||
|
||||
parser = DefaultParser()
|
||||
sheerka, context, parser = self.init_parser()
|
||||
res = parser.parse(context, text)
|
||||
|
||||
assert not res.status
|
||||
|
||||
@@ -6,19 +6,19 @@ from parsers.ExactConceptParser import ExactConceptParser
|
||||
from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
|
||||
|
||||
|
||||
def metadata_prop(concept, prop_name):
|
||||
for name, value in concept.metadata.props:
|
||||
def variable_def(concept, prop_name):
|
||||
for name, value in concept.metadata.variables:
|
||||
if name == prop_name:
|
||||
return value
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def get_concept(name, variables):
|
||||
def get_concept(name, variables=None):
|
||||
c = Concept(name=name)
|
||||
if variables:
|
||||
for v in variables:
|
||||
c.def_prop(v)
|
||||
c.def_var(v)
|
||||
c.init_key()
|
||||
return c
|
||||
|
||||
@@ -50,9 +50,9 @@ class TestExactConceptParser(TestUsingMemoryBasedSheerka):
|
||||
# TODO: the last tuple is not possible, so the algo can be improved
|
||||
|
||||
def test_i_can_recognize_a_simple_concept(self):
|
||||
context = self.get_context()
|
||||
concept = get_concept("hello world", [])
|
||||
context.sheerka.add_in_cache(concept)
|
||||
sheerka = self.get_sheerka(singleton=True)
|
||||
context = self.get_context(sheerka)
|
||||
concept = self.create_concept_lite(sheerka, "hello world")
|
||||
|
||||
source = "hello world"
|
||||
results = ExactConceptParser().parse(context, source)
|
||||
@@ -64,9 +64,10 @@ class TestExactConceptParser(TestUsingMemoryBasedSheerka):
|
||||
assert not concept_found.metadata.need_validation
|
||||
|
||||
def test_i_can_recognize_concepts_defined_several_times(self):
|
||||
context = self.get_context()
|
||||
context.sheerka.add_in_cache(get_concept("hello world", []))
|
||||
context.sheerka.add_in_cache(get_concept("hello a", ["a"]))
|
||||
sheerka = self.get_sheerka(singleton=True)
|
||||
context = self.get_context(sheerka)
|
||||
self.create_concept_lite(sheerka, "hello world")
|
||||
self.create_concept_lite(sheerka, "hello a", variables=["a"])
|
||||
|
||||
source = "hello world"
|
||||
results = ExactConceptParser().parse(context, source)
|
||||
@@ -76,7 +77,7 @@ class TestExactConceptParser(TestUsingMemoryBasedSheerka):
|
||||
|
||||
assert results[0].status
|
||||
assert results[0].value.value.name == "hello a"
|
||||
assert metadata_prop(results[0].value.value, "a") == "world"
|
||||
assert variable_def(results[0].value.value, "a") == "world"
|
||||
assert results[0].value.value.metadata.need_validation
|
||||
|
||||
assert results[1].status
|
||||
@@ -84,9 +85,10 @@ class TestExactConceptParser(TestUsingMemoryBasedSheerka):
|
||||
assert not results[1].value.value.metadata.need_validation
|
||||
|
||||
def test_i_can_recognize_a_concept_with_variables(self):
|
||||
context = self.get_context()
|
||||
concept = get_concept("a + b", ["a", "b"])
|
||||
context.sheerka.add_in_cache(concept)
|
||||
sheerka = self.get_sheerka(singleton=True)
|
||||
context = self.get_context(sheerka)
|
||||
concept = self.create_concept_lite(sheerka, "a + b", ["a", "b"])
|
||||
|
||||
source = "10 + 5"
|
||||
results = ExactConceptParser().parse(context, source)
|
||||
|
||||
@@ -95,46 +97,47 @@ class TestExactConceptParser(TestUsingMemoryBasedSheerka):
|
||||
|
||||
concept_found = results[0].value.value
|
||||
assert concept_found.key == concept.key
|
||||
assert metadata_prop(concept_found, "a") == "10"
|
||||
assert metadata_prop(concept_found, "b") == "5"
|
||||
assert variable_def(concept_found, "a") == "10"
|
||||
assert variable_def(concept_found, "b") == "5"
|
||||
assert concept_found.metadata.need_validation
|
||||
|
||||
def test_i_can_recognize_a_concept_with_duplicate_variables(self):
|
||||
context = self.get_context()
|
||||
concept = get_concept("a + b + a", ["a", "b"])
|
||||
context.sheerka.cache_by_key[concept.key] = concept
|
||||
sheerka = self.get_sheerka(singleton=True)
|
||||
context = self.get_context(sheerka)
|
||||
concept = self.create_concept_lite(sheerka, "a + b + a", ["a", "b"])
|
||||
|
||||
source = "10 + 5 + 10"
|
||||
results = ExactConceptParser().parse(context, source)
|
||||
results = ExactConceptParser(max_word_size=5).parse(context, source)
|
||||
|
||||
assert len(results) == 1
|
||||
assert results[0].status
|
||||
|
||||
concept_found = results[0].value.value
|
||||
assert concept_found.key == concept.key
|
||||
assert metadata_prop(concept_found, "a") == "10"
|
||||
assert metadata_prop(concept_found, "b") == "5"
|
||||
assert variable_def(concept_found, "a") == "10"
|
||||
assert variable_def(concept_found, "b") == "5"
|
||||
assert concept_found.metadata.need_validation
|
||||
|
||||
def test_i_can_manage_unknown_concept(self):
|
||||
context = self.get_context()
|
||||
source = "def concept hello world" # this is not a concept by itself
|
||||
context = self.get_context(self.get_sheerka(singleton=True))
|
||||
source = "def concept hello" # this is not a concept by itself
|
||||
res = ExactConceptParser().parse(context, source)
|
||||
|
||||
assert not res.status
|
||||
assert context.sheerka.isinstance(res.value, BuiltinConcepts.UNKNOWN_CONCEPT)
|
||||
assert res.value.body == "def concept hello world"
|
||||
assert res.value.body == "def concept hello"
|
||||
|
||||
def test_i_can_detect_concepts_too_long(self):
|
||||
context = self.get_context()
|
||||
source = "a very very long concept that cannot be an unique one"
|
||||
context = self.get_context(self.get_sheerka(singleton=True))
|
||||
source = "a very very long concept"
|
||||
res = ExactConceptParser().parse(context, source)
|
||||
|
||||
assert not res.status
|
||||
assert context.sheerka.isinstance(res.value, BuiltinConcepts.CONCEPT_TOO_LONG)
|
||||
assert res.value.body == "a very very long concept that cannot be an unique one"
|
||||
assert res.value.body == "a very very long concept"
|
||||
|
||||
def test_i_can_detect_concept_from_tokens(self):
|
||||
context = self.get_context()
|
||||
context = self.get_context(self.get_sheerka(singleton=True))
|
||||
concept = get_concept("hello world", [])
|
||||
context.sheerka.add_in_cache(concept)
|
||||
|
||||
|
||||
@@ -1,216 +1,216 @@
|
||||
import pytest
|
||||
|
||||
from core.builtin_concepts import ParserResultConcept, BuiltinConcepts
|
||||
from core.concept import Concept
|
||||
from core.tokenizer import Tokenizer, TokenKind, Token
|
||||
from parsers.BaseNodeParser import cnode, scnode, utnode, SourceCodeNode
|
||||
from parsers.BnfNodeParser import BnfNodeParser, ConceptNode, Sequence
|
||||
from parsers.MultipleConceptsParser import MultipleConceptsParser
|
||||
from parsers.PythonParser import PythonNode
|
||||
|
||||
from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
|
||||
|
||||
|
||||
def get_return_value(context, grammar, expression):
|
||||
parser = BnfNodeParser()
|
||||
parser.initialize(context, grammar)
|
||||
|
||||
ret_val = parser.parse(context, expression)
|
||||
assert not ret_val.status
|
||||
return ret_val
|
||||
|
||||
|
||||
class TestMultipleConceptsParser(TestUsingMemoryBasedSheerka):
|
||||
|
||||
def init(self, concepts, grammar, expression):
|
||||
context = self.get_context()
|
||||
for c in concepts:
|
||||
context.sheerka.create_new_concept(context, c)
|
||||
return_value = get_return_value(context, grammar, expression)
|
||||
|
||||
return context, return_value
|
||||
|
||||
def test_not_interested_if_not_parser_result(self):
|
||||
context = self.get_context()
|
||||
text = "not parser result"
|
||||
|
||||
res = MultipleConceptsParser().parse(context, text)
|
||||
assert res is None
|
||||
|
||||
def test_not_interested_if_not_from_concept_lexer_parser(self):
|
||||
context = self.get_context()
|
||||
text = ParserResultConcept(parser="not concept lexer", value="some value")
|
||||
|
||||
res = MultipleConceptsParser().parse(context, text)
|
||||
assert res is None
|
||||
|
||||
def test_i_can_parse_exact_concepts(self):
|
||||
foo = Concept("foo", body="'foo'")
|
||||
bar = Concept("bar", body="'bar'")
|
||||
baz = Concept("baz", body="'baz'")
|
||||
grammar = {}
|
||||
context, return_value = self.init([foo, bar, baz], grammar, "bar foo baz")
|
||||
|
||||
parser = MultipleConceptsParser()
|
||||
ret_val = parser.parse(context, return_value.body)
|
||||
|
||||
assert ret_val.status
|
||||
assert ret_val.who == parser.name
|
||||
assert context.sheerka.isinstance(ret_val.value, BuiltinConcepts.PARSER_RESULT)
|
||||
assert ret_val.value.value == [
|
||||
ConceptNode(bar, 0, 0, source="bar"),
|
||||
ConceptNode(foo, 2, 2, source="foo"),
|
||||
ConceptNode(baz, 4, 4, source="baz")]
|
||||
assert ret_val.value.source == "bar foo baz"
|
||||
|
||||
def test_i_can_parse_when_ending_with_bnf(self):
|
||||
foo = Concept("foo", body="'foo'")
|
||||
bar = Concept("bar", body="'bar'")
|
||||
grammar = {foo: Sequence("foo1", "foo2", "foo3")}
|
||||
context, return_value = self.init([foo, bar], grammar, "bar foo1 foo2 foo3")
|
||||
|
||||
parser = MultipleConceptsParser()
|
||||
ret_val = parser.parse(context, return_value.body)
|
||||
|
||||
assert ret_val.status
|
||||
assert ret_val.who == parser.name
|
||||
assert context.sheerka.isinstance(ret_val.value, BuiltinConcepts.PARSER_RESULT)
|
||||
assert ret_val.value.value == [cnode("bar", 0, 0, "bar"), cnode("foo", 2, 6, "foo1 foo2 foo3")]
|
||||
assert ret_val.value.source == "bar foo1 foo2 foo3"
|
||||
|
||||
def test_i_can_parse_when_starting_with_bnf(self):
|
||||
foo = Concept("foo", body="'foo'")
|
||||
bar = Concept("bar", body="'bar'")
|
||||
grammar = {foo: Sequence("foo1", "foo2", "foo3")}
|
||||
context, return_value = self.init([foo, bar], grammar, "foo1 foo2 foo3 bar")
|
||||
|
||||
parser = MultipleConceptsParser()
|
||||
ret_val = parser.parse(context, return_value.body)
|
||||
|
||||
assert ret_val.status
|
||||
assert ret_val.who == parser.name
|
||||
assert context.sheerka.isinstance(ret_val.value, BuiltinConcepts.PARSER_RESULT)
|
||||
assert ret_val.value.value == [cnode("foo", 0, 4, "foo1 foo2 foo3"), cnode("bar", 6, 6, "bar")]
|
||||
assert ret_val.value.source == "foo1 foo2 foo3 bar"
|
||||
|
||||
def test_i_can_parse_when_concept_are_mixed(self):
|
||||
foo = Concept("foo")
|
||||
bar = Concept("bar")
|
||||
baz = Concept("baz")
|
||||
grammar = {foo: Sequence("foo1", "foo2", "foo3")}
|
||||
context, return_value = self.init([foo, bar, baz], grammar, "baz foo1 foo2 foo3 bar")
|
||||
|
||||
parser = MultipleConceptsParser()
|
||||
ret_val = parser.parse(context, return_value.body)
|
||||
|
||||
assert ret_val.status
|
||||
assert ret_val.who == parser.name
|
||||
assert context.sheerka.isinstance(ret_val.value, BuiltinConcepts.PARSER_RESULT)
|
||||
assert ret_val.value.value == [
|
||||
cnode("baz", 0, 0, "baz"),
|
||||
cnode("foo", 2, 6, "foo1 foo2 foo3"),
|
||||
cnode("bar", 8, 8, "bar")]
|
||||
assert ret_val.value.source == "baz foo1 foo2 foo3 bar"
|
||||
|
||||
def test_i_can_parse_when_multiple_concepts_are_matching(self):
|
||||
foo = Concept("foo")
|
||||
bar = Concept("bar", body="bar1")
|
||||
baz = Concept("bar", body="bar2")
|
||||
grammar = {foo: "foo"}
|
||||
context, return_value = self.init([foo, bar, baz], grammar, "foo bar")
|
||||
|
||||
parser = MultipleConceptsParser()
|
||||
ret_val = parser.parse(context, return_value.body)
|
||||
|
||||
assert len(ret_val) == 2
|
||||
assert ret_val[0].status
|
||||
assert ret_val[0].value.value == [cnode("foo", 0, 0, "foo"), cnode("bar", 2, 2, "bar")]
|
||||
assert ret_val[0].value.source == "foo bar"
|
||||
assert ret_val[0].value.value[1].concept.metadata.body == "bar1"
|
||||
|
||||
assert ret_val[1].status
|
||||
assert ret_val[1].value.value == [cnode("foo", 0, 0, "foo"), cnode("bar", 2, 2, "bar")]
|
||||
assert ret_val[1].value.source == "foo bar"
|
||||
assert ret_val[1].value.value[1].concept.metadata.body == "bar2"
|
||||
|
||||
def test_i_can_parse_when_source_code(self):
|
||||
foo = Concept("foo")
|
||||
grammar = {foo: "foo"}
|
||||
context, return_value = self.init([foo], grammar, "1 foo")
|
||||
|
||||
parser = MultipleConceptsParser()
|
||||
ret_val = parser.parse(context, return_value.body)
|
||||
wrapper = ret_val.value
|
||||
value = ret_val.value.value
|
||||
|
||||
assert ret_val.status
|
||||
assert ret_val.who == parser.name
|
||||
assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
|
||||
assert wrapper.source == "1 foo"
|
||||
assert value == [
|
||||
scnode(0, 1, "1 "),
|
||||
cnode("foo", 2, 2, "foo")]
|
||||
|
||||
def test_i_cannot_parse_when_unrecognized_token(self):
|
||||
twenty_two = Concept("twenty two")
|
||||
one = Concept("one")
|
||||
grammar = {twenty_two: Sequence("twenty", "two")}
|
||||
context, return_value = self.init([twenty_two, one], grammar, "twenty two + one")
|
||||
|
||||
parser = MultipleConceptsParser()
|
||||
ret_val = parser.parse(context, return_value.body)
|
||||
|
||||
assert not ret_val.status
|
||||
assert ret_val.who == parser.name
|
||||
assert context.sheerka.isinstance(ret_val.value, BuiltinConcepts.PARSER_RESULT)
|
||||
assert ret_val.value.value == [
|
||||
cnode("twenty two", 0, 2, "twenty two"),
|
||||
utnode(3, 5, " + "),
|
||||
cnode("one", 6, 6, "one")
|
||||
]
|
||||
assert ret_val.value.source == "twenty two + one"
|
||||
|
||||
def test_i_cannot_parse_when_unknown_concepts(self):
|
||||
twenty_two = Concept("twenty two")
|
||||
one = Concept("one")
|
||||
grammar = {twenty_two: Sequence("twenty", "two")}
|
||||
context, return_value = self.init([twenty_two, one], grammar, "twenty two plus one")
|
||||
|
||||
parser = MultipleConceptsParser()
|
||||
ret_val = parser.parse(context, return_value.body)
|
||||
|
||||
assert not ret_val.status
|
||||
assert ret_val.who == parser.name
|
||||
assert context.sheerka.isinstance(ret_val.value, BuiltinConcepts.PARSER_RESULT)
|
||||
assert ret_val.value.value == [
|
||||
cnode("twenty two", 0, 2, "twenty two"),
|
||||
utnode(3, 5, " plus "),
|
||||
cnode("one", 6, 6, "one")
|
||||
]
|
||||
assert ret_val.value.source == "twenty two plus one"
|
||||
|
||||
@pytest.mark.parametrize("text, expected_source, expected_end", [
|
||||
("True", "True", 0),
|
||||
("1 == 1", "1 == 1", 4),
|
||||
("1!xdf", "1", 0),
|
||||
("1", "1", 0),
|
||||
])
|
||||
def test_i_can_get_source_code_node(self, text, expected_source, expected_end):
|
||||
tokens = list(Tokenizer(text))[:-1] # strip trailing EOF
|
||||
|
||||
start_index = 5 # a random number different of zero
|
||||
res = MultipleConceptsParser().get_source_code_node(self.get_context(), start_index, tokens)
|
||||
|
||||
assert isinstance(res, SourceCodeNode)
|
||||
assert isinstance(res.node, PythonNode)
|
||||
assert res.source == expected_source
|
||||
assert res.start == start_index
|
||||
assert res.end == start_index + expected_end
|
||||
|
||||
def test_i_cannot_parse_null_text(self):
|
||||
res = MultipleConceptsParser().get_source_code_node(self.get_context(), 0, [])
|
||||
assert res is None
|
||||
|
||||
eof = Token(TokenKind.EOF, "", 0, 0, 0)
|
||||
res = MultipleConceptsParser().get_source_code_node(self.get_context(), 0, [eof])
|
||||
assert res is None
|
||||
# import pytest
|
||||
#
|
||||
# from core.builtin_concepts import ParserResultConcept, BuiltinConcepts
|
||||
# from core.concept import Concept
|
||||
# from core.tokenizer import Tokenizer, TokenKind, Token
|
||||
# from parsers.BaseNodeParser import cnode, scnode, utnode, SourceCodeNode, ConceptNode
|
||||
# from parsers.BnfNodeParser import BnfNodeParser, Sequence
|
||||
# from parsers.MultipleConceptsParser import MultipleConceptsParser
|
||||
# from parsers.PythonParser import PythonNode
|
||||
#
|
||||
# from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
|
||||
#
|
||||
#
|
||||
# def get_return_value(context, grammar, expression):
|
||||
# parser = BnfNodeParser()
|
||||
# parser.initialize(context, grammar)
|
||||
#
|
||||
# ret_val = parser.parse(context, expression)
|
||||
# assert not ret_val.status
|
||||
# return ret_val
|
||||
#
|
||||
#
|
||||
# class TestMultipleConceptsParser(TestUsingMemoryBasedSheerka):
|
||||
#
|
||||
# def init(self, concepts, grammar, expression):
|
||||
# context = self.get_context()
|
||||
# for c in concepts:
|
||||
# context.sheerka.create_new_concept(context, c)
|
||||
# return_value = get_return_value(context, grammar, expression)
|
||||
#
|
||||
# return context, return_value
|
||||
#
|
||||
# def test_not_interested_if_not_parser_result(self):
|
||||
# context = self.get_context()
|
||||
# text = "not parser result"
|
||||
#
|
||||
# res = MultipleConceptsParser().parse(context, text)
|
||||
# assert res is None
|
||||
#
|
||||
# def test_not_interested_if_not_from_concept_lexer_parser(self):
|
||||
# context = self.get_context()
|
||||
# text = ParserResultConcept(parser="not concept lexer", value="some value")
|
||||
#
|
||||
# res = MultipleConceptsParser().parse(context, text)
|
||||
# assert res is None
|
||||
#
|
||||
# def test_i_can_parse_exact_concepts(self):
|
||||
# foo = Concept("foo", body="'foo'")
|
||||
# bar = Concept("bar", body="'bar'")
|
||||
# baz = Concept("baz", body="'baz'")
|
||||
# grammar = {}
|
||||
# context, return_value = self.init([foo, bar, baz], grammar, "bar foo baz")
|
||||
#
|
||||
# parser = MultipleConceptsParser()
|
||||
# ret_val = parser.parse(context, return_value.body)
|
||||
#
|
||||
# assert ret_val.status
|
||||
# assert ret_val.who == parser.name
|
||||
# assert context.sheerka.isinstance(ret_val.value, BuiltinConcepts.PARSER_RESULT)
|
||||
# assert ret_val.value.value == [
|
||||
# ConceptNode(bar, 0, 0, source="bar"),
|
||||
# ConceptNode(foo, 2, 2, source="foo"),
|
||||
# ConceptNode(baz, 4, 4, source="baz")]
|
||||
# assert ret_val.value.source == "bar foo baz"
|
||||
#
|
||||
# def test_i_can_parse_when_ending_with_bnf(self):
|
||||
# foo = Concept("foo", body="'foo'")
|
||||
# bar = Concept("bar", body="'bar'")
|
||||
# grammar = {foo: Sequence("foo1", "foo2", "foo3")}
|
||||
# context, return_value = self.init([foo, bar], grammar, "bar foo1 foo2 foo3")
|
||||
#
|
||||
# parser = MultipleConceptsParser()
|
||||
# ret_val = parser.parse(context, return_value.body)
|
||||
#
|
||||
# assert ret_val.status
|
||||
# assert ret_val.who == parser.name
|
||||
# assert context.sheerka.isinstance(ret_val.value, BuiltinConcepts.PARSER_RESULT)
|
||||
# assert ret_val.value.value == [cnode("bar", 0, 0, "bar"), cnode("foo", 2, 6, "foo1 foo2 foo3")]
|
||||
# assert ret_val.value.source == "bar foo1 foo2 foo3"
|
||||
#
|
||||
# def test_i_can_parse_when_starting_with_bnf(self):
|
||||
# foo = Concept("foo", body="'foo'")
|
||||
# bar = Concept("bar", body="'bar'")
|
||||
# grammar = {foo: Sequence("foo1", "foo2", "foo3")}
|
||||
# context, return_value = self.init([foo, bar], grammar, "foo1 foo2 foo3 bar")
|
||||
#
|
||||
# parser = MultipleConceptsParser()
|
||||
# ret_val = parser.parse(context, return_value.body)
|
||||
#
|
||||
# assert ret_val.status
|
||||
# assert ret_val.who == parser.name
|
||||
# assert context.sheerka.isinstance(ret_val.value, BuiltinConcepts.PARSER_RESULT)
|
||||
# assert ret_val.value.value == [cnode("foo", 0, 4, "foo1 foo2 foo3"), cnode("bar", 6, 6, "bar")]
|
||||
# assert ret_val.value.source == "foo1 foo2 foo3 bar"
|
||||
#
|
||||
# def test_i_can_parse_when_concept_are_mixed(self):
|
||||
# foo = Concept("foo")
|
||||
# bar = Concept("bar")
|
||||
# baz = Concept("baz")
|
||||
# grammar = {foo: Sequence("foo1", "foo2", "foo3")}
|
||||
# context, return_value = self.init([foo, bar, baz], grammar, "baz foo1 foo2 foo3 bar")
|
||||
#
|
||||
# parser = MultipleConceptsParser()
|
||||
# ret_val = parser.parse(context, return_value.body)
|
||||
#
|
||||
# assert ret_val.status
|
||||
# assert ret_val.who == parser.name
|
||||
# assert context.sheerka.isinstance(ret_val.value, BuiltinConcepts.PARSER_RESULT)
|
||||
# assert ret_val.value.value == [
|
||||
# cnode("baz", 0, 0, "baz"),
|
||||
# cnode("foo", 2, 6, "foo1 foo2 foo3"),
|
||||
# cnode("bar", 8, 8, "bar")]
|
||||
# assert ret_val.value.source == "baz foo1 foo2 foo3 bar"
|
||||
#
|
||||
# def test_i_can_parse_when_multiple_concepts_are_matching(self):
|
||||
# foo = Concept("foo")
|
||||
# bar = Concept("bar", body="bar1")
|
||||
# baz = Concept("bar", body="bar2")
|
||||
# grammar = {foo: "foo"}
|
||||
# context, return_value = self.init([foo, bar, baz], grammar, "foo bar")
|
||||
#
|
||||
# parser = MultipleConceptsParser()
|
||||
# ret_val = parser.parse(context, return_value.body)
|
||||
#
|
||||
# assert len(ret_val) == 2
|
||||
# assert ret_val[0].status
|
||||
# assert ret_val[0].value.value == [cnode("foo", 0, 0, "foo"), cnode("bar", 2, 2, "bar")]
|
||||
# assert ret_val[0].value.source == "foo bar"
|
||||
# assert ret_val[0].value.value[1].concept.metadata.body == "bar1"
|
||||
#
|
||||
# assert ret_val[1].status
|
||||
# assert ret_val[1].value.value == [cnode("foo", 0, 0, "foo"), cnode("bar", 2, 2, "bar")]
|
||||
# assert ret_val[1].value.source == "foo bar"
|
||||
# assert ret_val[1].value.value[1].concept.metadata.body == "bar2"
|
||||
#
|
||||
# def test_i_can_parse_when_source_code(self):
|
||||
# foo = Concept("foo")
|
||||
# grammar = {foo: "foo"}
|
||||
# context, return_value = self.init([foo], grammar, "1 foo")
|
||||
#
|
||||
# parser = MultipleConceptsParser()
|
||||
# ret_val = parser.parse(context, return_value.body)
|
||||
# wrapper = ret_val.value
|
||||
# value = ret_val.value.value
|
||||
#
|
||||
# assert ret_val.status
|
||||
# assert ret_val.who == parser.name
|
||||
# assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
|
||||
# assert wrapper.source == "1 foo"
|
||||
# assert value == [
|
||||
# scnode(0, 1, "1 "),
|
||||
# cnode("foo", 2, 2, "foo")]
|
||||
#
|
||||
# def test_i_cannot_parse_when_unrecognized_token(self):
|
||||
# twenty_two = Concept("twenty two")
|
||||
# one = Concept("one")
|
||||
# grammar = {twenty_two: Sequence("twenty", "two")}
|
||||
# context, return_value = self.init([twenty_two, one], grammar, "twenty two + one")
|
||||
#
|
||||
# parser = MultipleConceptsParser()
|
||||
# ret_val = parser.parse(context, return_value.body)
|
||||
#
|
||||
# assert not ret_val.status
|
||||
# assert ret_val.who == parser.name
|
||||
# assert context.sheerka.isinstance(ret_val.value, BuiltinConcepts.PARSER_RESULT)
|
||||
# assert ret_val.value.value == [
|
||||
# cnode("twenty two", 0, 2, "twenty two"),
|
||||
# utnode(3, 5, " + "),
|
||||
# cnode("one", 6, 6, "one")
|
||||
# ]
|
||||
# assert ret_val.value.source == "twenty two + one"
|
||||
#
|
||||
# def test_i_cannot_parse_when_unknown_concepts(self):
|
||||
# twenty_two = Concept("twenty two")
|
||||
# one = Concept("one")
|
||||
# grammar = {twenty_two: Sequence("twenty", "two")}
|
||||
# context, return_value = self.init([twenty_two, one], grammar, "twenty two plus one")
|
||||
#
|
||||
# parser = MultipleConceptsParser()
|
||||
# ret_val = parser.parse(context, return_value.body)
|
||||
#
|
||||
# assert not ret_val.status
|
||||
# assert ret_val.who == parser.name
|
||||
# assert context.sheerka.isinstance(ret_val.value, BuiltinConcepts.PARSER_RESULT)
|
||||
# assert ret_val.value.value == [
|
||||
# cnode("twenty two", 0, 2, "twenty two"),
|
||||
# utnode(3, 5, " plus "),
|
||||
# cnode("one", 6, 6, "one")
|
||||
# ]
|
||||
# assert ret_val.value.source == "twenty two plus one"
|
||||
#
|
||||
# @pytest.mark.parametrize("text, expected_source, expected_end", [
|
||||
# ("True", "True", 0),
|
||||
# ("1 == 1", "1 == 1", 4),
|
||||
# ("1!xdf", "1", 0),
|
||||
# ("1", "1", 0),
|
||||
# ])
|
||||
# def test_i_can_get_source_code_node(self, text, expected_source, expected_end):
|
||||
# tokens = list(Tokenizer(text))[:-1] # strip trailing EOF
|
||||
#
|
||||
# start_index = 5 # a random number different of zero
|
||||
# res = MultipleConceptsParser().get_source_code_node(self.get_context(), start_index, tokens)
|
||||
#
|
||||
# assert isinstance(res, SourceCodeNode)
|
||||
# assert isinstance(res.node, PythonNode)
|
||||
# assert res.source == expected_source
|
||||
# assert res.start == start_index
|
||||
# assert res.end == start_index + expected_end
|
||||
#
|
||||
# def test_i_cannot_parse_null_text(self):
|
||||
# res = MultipleConceptsParser().get_source_code_node(self.get_context(), 0, [])
|
||||
# assert res is None
|
||||
#
|
||||
# eof = Token(TokenKind.EOF, "", 0, 0, 0)
|
||||
# res = MultipleConceptsParser().get_source_code_node(self.get_context(), 0, [eof])
|
||||
# assert res is None
|
||||
|
||||
@@ -50,9 +50,9 @@ class TestPythonParser(TestUsingMemoryBasedSheerka):
|
||||
assert res.who == parser.name
|
||||
assert isinstance(res.value, NotForMeConcept)
|
||||
assert res.value.body == text
|
||||
assert len(res.value.get_prop("reason")) == 1
|
||||
assert isinstance(res.value.get_prop("reason")[0], PythonErrorNode)
|
||||
assert isinstance(res.value.get_prop("reason")[0].exception, SyntaxError)
|
||||
assert len(res.value.get_value("reason")) == 1
|
||||
assert isinstance(res.value.get_value("reason")[0], PythonErrorNode)
|
||||
assert isinstance(res.value.get_value("reason")[0].exception, SyntaxError)
|
||||
|
||||
@pytest.mark.parametrize("text, error_msg, error_text", [
|
||||
("c::", "Concept identifiers not found", ""),
|
||||
@@ -66,10 +66,10 @@ class TestPythonParser(TestUsingMemoryBasedSheerka):
|
||||
assert isinstance(res.value, NotForMeConcept)
|
||||
assert res.value.body == text
|
||||
|
||||
assert len(res.value.get_prop("reason")) == 1
|
||||
assert isinstance(res.value.get_prop("reason")[0], LexerError)
|
||||
assert res.value.get_prop("reason")[0].message == error_msg
|
||||
assert res.value.get_prop("reason")[0].text == error_text
|
||||
assert len(res.value.get_value("reason")) == 1
|
||||
assert isinstance(res.value.get_value("reason")[0], LexerError)
|
||||
assert res.value.get_value("reason")[0].message == error_msg
|
||||
assert res.value.get_value("reason")[0].text == error_text
|
||||
|
||||
def test_i_can_parse_a_concept(self):
|
||||
text = "c:name|key: + 1"
|
||||
|
||||
@@ -4,7 +4,7 @@ import pytest
|
||||
from core.builtin_concepts import ParserResultConcept, BuiltinConcepts, ReturnValueConcept
|
||||
from core.concept import Concept
|
||||
from core.tokenizer import Token, TokenKind, Tokenizer
|
||||
from parsers.BnfNodeParser import ConceptNode, UnrecognizedTokensNode
|
||||
from parsers.BaseNodeParser import ConceptNode, UnrecognizedTokensNode
|
||||
from parsers.PythonParser import PythonNode
|
||||
from parsers.PythonWithConceptsParser import PythonWithConceptsParser
|
||||
from parsers.UnrecognizedNodeParser import UnrecognizedNodeParser
|
||||
|
||||
+317
-276
@@ -3,7 +3,7 @@ from core.builtin_concepts import BuiltinConcepts
|
||||
from core.concept import Concept, CC
|
||||
from core.tokenizer import Tokenizer
|
||||
from parsers.BaseNodeParser import utnode, ConceptNode, cnode, short_cnode, UnrecognizedTokensNode, \
|
||||
SCWC, CNC, CN, UTN, scnode
|
||||
SCWC, CNC, UTN
|
||||
from parsers.PythonParser import PythonNode
|
||||
from parsers.SyaNodeParser import SyaNodeParser, SyaConceptParserHelper, SyaAssociativity, \
|
||||
NoneAssociativeSequenceErrorNode, TooManyParametersFound
|
||||
@@ -16,14 +16,69 @@ def compute_expected_array(concepts_map, expression, expected):
|
||||
return tests.parsers.parsers_utils.compute_expected_array(concepts_map, expression, expected, sya=True)
|
||||
|
||||
|
||||
class TestSyaConceptLexerParser(TestUsingMemoryBasedSheerka):
|
||||
cmap = {
|
||||
"one": Concept("one"),
|
||||
"two": Concept("two"),
|
||||
"three": Concept("three"),
|
||||
"four": Concept("four"),
|
||||
"plus": Concept("a plus b").def_var("a").def_var("b"),
|
||||
"minus": Concept("a minus b").def_var("a").def_var("b"),
|
||||
"mult": Concept("a mult b").def_var("a").def_var("b"),
|
||||
"prefixed": Concept("a prefixed").def_var("a"),
|
||||
"suffixed": Concept("suffixed a").def_var("a"),
|
||||
"infix": Concept("a infix b").def_var("a").def_var("b"),
|
||||
"?": Concept("a ? b : c").def_var("a").def_var("b").def_var("c"),
|
||||
"if": Concept("if a then b else c end").def_var("a").def_var("b").def_var("c"),
|
||||
"square": Concept("square(a)").def_var("a"),
|
||||
"foo bar": Concept("foo bar(a)").def_var("a"),
|
||||
"twenties": Concept("twenties", definition="'twenty' (one|two)=unit").def_var("unit"),
|
||||
}
|
||||
|
||||
def init_parser(self, concepts_map, sya_def, **kwargs):
|
||||
sheerka, context, *updated_concepts = self.init_concepts(
|
||||
*concepts_map.values(),
|
||||
singleton=True,
|
||||
|
||||
class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
|
||||
sheerka = None
|
||||
|
||||
@classmethod
|
||||
def setup_class(cls):
|
||||
t = TestSyaNodeParser()
|
||||
TestSyaNodeParser.sheerka, context, _ = t.init_parser(
|
||||
cmap,
|
||||
singleton=False,
|
||||
create_new=True,
|
||||
**kwargs)
|
||||
init_from_sheerka=True)
|
||||
|
||||
TestSyaNodeParser.sheerka.force_sya_def(context, [
|
||||
(cmap["plus"].id, 5, SyaAssociativity.Right),
|
||||
(cmap["mult"].id, 10, SyaAssociativity.Right),
|
||||
(cmap["minus"].id, 10, SyaAssociativity.Right),
|
||||
(cmap["square"].id, None, SyaAssociativity.No)])
|
||||
|
||||
def init_parser(self,
|
||||
my_concepts_map=None,
|
||||
sya_def=None,
|
||||
post_init_concepts=None,
|
||||
**kwargs):
|
||||
|
||||
if my_concepts_map is not None:
|
||||
# a new concept map is given
|
||||
# use it but
|
||||
# do not instantiate a new sheerka
|
||||
# do not update / init from sheerka
|
||||
if 'singleton' not in kwargs:
|
||||
kwargs["singleton"] = True
|
||||
init_from_sheerka = kwargs.get("init_from_sheerka", False)
|
||||
sheerka, context, *concepts = self.init_concepts(*my_concepts_map.values(), **kwargs)
|
||||
else:
|
||||
# No custom concept map is given -> Use the global cmap
|
||||
# Sheerka is already initialized (the class instance)
|
||||
# Use it to initialize the parser
|
||||
init_from_sheerka = kwargs.get("init_from_sheerka", True)
|
||||
sheerka = TestSyaNodeParser.sheerka
|
||||
context = self.get_context(sheerka)
|
||||
concepts = cmap.values()
|
||||
|
||||
if post_init_concepts:
|
||||
post_init_concepts(sheerka, context)
|
||||
|
||||
if sya_def:
|
||||
sya_def_to_use = {}
|
||||
@@ -32,29 +87,15 @@ class TestSyaConceptLexerParser(TestUsingMemoryBasedSheerka):
|
||||
else:
|
||||
sya_def_to_use = None
|
||||
|
||||
parser = SyaNodeParser()
|
||||
parser.initialize(context, updated_concepts, sya_def_to_use)
|
||||
if init_from_sheerka:
|
||||
parser = SyaNodeParser(sheerka=sheerka)
|
||||
else:
|
||||
parser = SyaNodeParser()
|
||||
if my_concepts_map:
|
||||
parser.init_from_concepts(context, concepts, sya=sya_def_to_use)
|
||||
|
||||
return sheerka, context, parser
|
||||
|
||||
def test_i_can_initialize(self):
|
||||
sheerka, context, c1, c2, c3, c4, c5 = self.init_concepts(
|
||||
"foo",
|
||||
Concept("bar a").def_prop("a"),
|
||||
Concept("a baz").def_prop("a"),
|
||||
Concept("baz a qux b").def_prop("a").def_prop("b"),
|
||||
Concept("foo a bar b").def_prop("a").def_prop("b"),
|
||||
)
|
||||
|
||||
parser = SyaNodeParser()
|
||||
parser.initialize(context, [c1, c2, c3, c4, c5])
|
||||
|
||||
assert parser.concepts_by_first_keyword == {
|
||||
"foo": [c1.id, c5.id],
|
||||
"bar": [c2.id],
|
||||
"baz": [c3.id, c4.id],
|
||||
}
|
||||
|
||||
@pytest.mark.parametrize("expression, expected_sequences", [
|
||||
("one plus two", [["one", "two", "plus"]]),
|
||||
("1 + 1 plus two", [["1 + 1 ", "two", "plus"]]),
|
||||
@@ -143,21 +184,21 @@ class TestSyaConceptLexerParser(TestUsingMemoryBasedSheerka):
|
||||
("x$!# plus z$!#", [["x$!# ", " z$!#", "plus"]]),
|
||||
])
|
||||
def test_i_can_post_fix_simple_infix_concepts(self, expression, expected_sequences):
|
||||
concepts_map = {
|
||||
"plus": Concept("a plus b").def_prop("a").def_prop("b"),
|
||||
"one": Concept("one"),
|
||||
"two": Concept("two"),
|
||||
"three": Concept("three"),
|
||||
"twenties": Concept("twenties", definition="'twenty' (one|two)=unit").def_prop("unit"),
|
||||
}
|
||||
sheerka, context, parser = self.init_parser(concepts_map, None)
|
||||
# concepts_map = {
|
||||
# "plus": Concept("a plus b").def_var("a").def_var("b"),
|
||||
# "one": Concept("one"),
|
||||
# "two": Concept("two"),
|
||||
# "three": Concept("three"),
|
||||
# "twenties": Concept("twenties", definition="'twenty' (one|two)=unit").def_var("unit"),
|
||||
# }
|
||||
sheerka, context, parser = self.init_parser()
|
||||
|
||||
res = parser.infix_to_postfix(context, expression)
|
||||
|
||||
assert len(res) == len(expected_sequences)
|
||||
for res_i, expected in zip(res, expected_sequences):
|
||||
assert len(res_i.errors) == 0
|
||||
expected_array = compute_expected_array(concepts_map, expression, expected)
|
||||
expected_array = compute_expected_array(cmap, expression, expected)
|
||||
assert res_i.out == expected_array
|
||||
|
||||
@pytest.mark.parametrize("expression, expected_sequences", [
|
||||
@@ -169,14 +210,14 @@ class TestSyaConceptLexerParser(TestUsingMemoryBasedSheerka):
|
||||
])
|
||||
def test_i_can_post_fix_infix_concepts_with_long_name(self, expression, expected_sequences):
|
||||
concepts_map = {
|
||||
"plus plus plus": Concept("a plus plus plus b").def_prop("a").def_prop("b"),
|
||||
"another long name infix": Concept("a another long name infix b").def_prop("a").def_prop("b"),
|
||||
"plus plus plus": Concept("a plus plus plus b").def_var("a").def_var("b"),
|
||||
"another long name infix": Concept("a another long name infix b").def_var("a").def_var("b"),
|
||||
"one": Concept("one"),
|
||||
"two": Concept("two"),
|
||||
"three": Concept("three"),
|
||||
"twenties": Concept("twenties", definition="'twenty' (one|two)=unit").def_prop("unit"),
|
||||
"twenties": Concept("twenties", definition="'twenty' (one|two)=unit").def_var("unit"),
|
||||
}
|
||||
sheerka, context, parser = self.init_parser(concepts_map, None)
|
||||
sheerka, context, parser = self.init_parser(concepts_map, create_new=True, init_from_sheerka=True)
|
||||
|
||||
res = parser.infix_to_postfix(context, expression)
|
||||
|
||||
@@ -200,20 +241,20 @@ class TestSyaConceptLexerParser(TestUsingMemoryBasedSheerka):
|
||||
("x$!# prefixed", [["x$!# ", "prefixed"]]),
|
||||
])
|
||||
def test_i_can_post_fix_simple_prefixed_concepts(self, expression, expected_sequences):
|
||||
concepts_map = {
|
||||
"prefixed": Concept("a prefixed").def_prop("a"),
|
||||
"one": Concept("one"),
|
||||
"two": Concept("two"),
|
||||
"twenties": Concept("twenties", definition="'twenty' (one|two)=unit").def_prop("unit"),
|
||||
}
|
||||
sheerka, context, parser = self.init_parser(concepts_map, None)
|
||||
# concepts_map = {
|
||||
# "prefixed": Concept("a prefixed").def_var("a"),
|
||||
# "one": Concept("one"),
|
||||
# "two": Concept("two"),
|
||||
# "twenties": Concept("twenties", definition="'twenty' (one|two)=unit").def_var("unit"),
|
||||
# }
|
||||
sheerka, context, parser = self.init_parser()
|
||||
|
||||
res = parser.infix_to_postfix(context, expression)
|
||||
|
||||
assert len(res) == len(expected_sequences)
|
||||
for res_i, expected in zip(res, expected_sequences):
|
||||
assert len(res_i.errors) == 0
|
||||
expected_array = compute_expected_array(concepts_map, expression, expected)
|
||||
expected_array = compute_expected_array(cmap, expression, expected)
|
||||
assert res_i.out == expected_array
|
||||
|
||||
@pytest.mark.parametrize("expression, expected_sequences", [
|
||||
@@ -243,13 +284,13 @@ class TestSyaConceptLexerParser(TestUsingMemoryBasedSheerka):
|
||||
])
|
||||
def test_i_can_post_fix_prefixed_concepts_with_long_names(self, expression, expected_sequences):
|
||||
concepts_map = {
|
||||
"prefixed prefixed": Concept("a prefixed prefixed").def_prop("a"),
|
||||
"long name prefixed": Concept("a long name prefixed").def_prop("a"),
|
||||
"prefixed prefixed": Concept("a prefixed prefixed").def_var("a"),
|
||||
"long name prefixed": Concept("a long name prefixed").def_var("a"),
|
||||
"one": Concept("one"),
|
||||
"two": Concept("two"),
|
||||
"twenties": Concept("twenties", definition="'twenty' (one|two)=unit").def_prop("unit"),
|
||||
"twenties": Concept("twenties", definition="'twenty' (one|two)=unit").def_var("unit"),
|
||||
}
|
||||
sheerka, context, parser = self.init_parser(concepts_map, None)
|
||||
sheerka, context, parser = self.init_parser(concepts_map, create_new=True, init_from_sheerka=True)
|
||||
|
||||
res = parser.infix_to_postfix(context, expression)
|
||||
|
||||
@@ -273,20 +314,20 @@ class TestSyaConceptLexerParser(TestUsingMemoryBasedSheerka):
|
||||
("suffixed x$!#", [[" x$!#", "suffixed"]]),
|
||||
])
|
||||
def test_i_can_post_fix_simple_suffixed_concepts(self, expression, expected_sequences):
|
||||
concepts_map = {
|
||||
"suffixed": Concept("suffixed a").def_prop("a"),
|
||||
"one": Concept("one"),
|
||||
"two": Concept("two"),
|
||||
"twenties": Concept("twenties", definition="'twenty' (one|two)=unit").def_prop("unit"),
|
||||
}
|
||||
sheerka, context, parser = self.init_parser(concepts_map, None)
|
||||
# concepts_map = {
|
||||
# "suffixed": Concept("suffixed a").def_var("a"),
|
||||
# "one": Concept("one"),
|
||||
# "two": Concept("two"),
|
||||
# "twenties": Concept("twenties", definition="'twenty' (one|two)=unit").def_var("unit"),
|
||||
# }
|
||||
sheerka, context, parser = self.init_parser()
|
||||
|
||||
res = parser.infix_to_postfix(context, expression)
|
||||
|
||||
assert len(res) == len(expected_sequences)
|
||||
for res_i, expected in zip(res, expected_sequences):
|
||||
assert len(res_i.errors) == 0
|
||||
expected_array = compute_expected_array(concepts_map, expression, expected)
|
||||
expected_array = compute_expected_array(cmap, expression, expected)
|
||||
assert res_i.out == expected_array
|
||||
|
||||
@pytest.mark.parametrize("expression, expected", [
|
||||
@@ -295,11 +336,11 @@ class TestSyaConceptLexerParser(TestUsingMemoryBasedSheerka):
|
||||
])
|
||||
def test_i_can_post_fix_suffixed_concepts_with_long_names(self, expression, expected):
|
||||
concepts_map = {
|
||||
"suffixed suffixed": Concept("suffixed suffixed a").def_prop("a"),
|
||||
"long name suffixed": Concept("long name suffixed a").def_prop("a"),
|
||||
"suffixed suffixed": Concept("suffixed suffixed a").def_var("a"),
|
||||
"long name suffixed": Concept("long name suffixed a").def_var("a"),
|
||||
"one": Concept("one"),
|
||||
"two": Concept("two"),
|
||||
"twenties": Concept("twenties", definition="'twenty' (one|two)=unit").def_prop("unit"),
|
||||
"twenties": Concept("twenties", definition="'twenty' (one|two)=unit").def_var("unit"),
|
||||
}
|
||||
sheerka, context, parser = self.init_parser(concepts_map, None)
|
||||
|
||||
@@ -340,22 +381,22 @@ class TestSyaConceptLexerParser(TestUsingMemoryBasedSheerka):
|
||||
:return:
|
||||
"""
|
||||
|
||||
concepts_map = {
|
||||
"?": Concept("a ? b : c").def_prop("a").def_prop("b").def_prop("c"),
|
||||
"if": Concept("if a then b else c end").def_prop("a").def_prop("b").def_prop("c"),
|
||||
"one": Concept("one"),
|
||||
"two": Concept("two"),
|
||||
"three": Concept("three"),
|
||||
"twenties": Concept("twenties", definition="'twenty' (one|two)=unit").def_prop("unit"),
|
||||
}
|
||||
sheerka, context, parser = self.init_parser(concepts_map, None)
|
||||
# concepts_map = {
|
||||
# "?": Concept("a ? b : c").def_var("a").def_var("b").def_var("c"),
|
||||
# "if": Concept("if a then b else c end").def_var("a").def_var("b").def_var("c"),
|
||||
# "one": Concept("one"),
|
||||
# "two": Concept("two"),
|
||||
# "three": Concept("three"),
|
||||
# "twenties": Concept("twenties", definition="'twenty' (one|two)=unit").def_var("unit"),
|
||||
# }
|
||||
sheerka, context, parser = self.init_parser()
|
||||
|
||||
res = parser.infix_to_postfix(context, expression)
|
||||
|
||||
assert len(res) == len(expected_sequences)
|
||||
for res_i, expected in zip(res, expected_sequences):
|
||||
# assert len(res_i.errors) == 0 # Do not validate errors
|
||||
expected_array = compute_expected_array(concepts_map, expression, expected)
|
||||
expected_array = compute_expected_array(cmap, expression, expected)
|
||||
assert res_i.out == expected_array
|
||||
|
||||
@pytest.mark.parametrize("expression, expected_sequences", [
|
||||
@@ -373,14 +414,14 @@ class TestSyaConceptLexerParser(TestUsingMemoryBasedSheerka):
|
||||
])
|
||||
def test_i_can_post_fix_ternary_concept_with_long_names(self, expression, expected_sequences):
|
||||
concepts_map = {
|
||||
"? ?": Concept("a ? ? b : : c").def_prop("a").def_prop("b").def_prop("c"),
|
||||
"if if": Concept("if if a then then b else else c end end").def_prop("a").def_prop("b").def_prop("c"),
|
||||
"? ?": Concept("a ? ? b : : c").def_var("a").def_var("b").def_var("c"),
|
||||
"if if": Concept("if if a then then b else else c end end").def_var("a").def_var("b").def_var("c"),
|
||||
"one": Concept("one"),
|
||||
"two": Concept("two"),
|
||||
"three": Concept("three"),
|
||||
"twenties": Concept("twenties", definition="'twenty' (one|two)=unit").def_prop("unit"),
|
||||
"twenties": Concept("twenties", definition="'twenty' (one|two)=unit").def_var("unit"),
|
||||
}
|
||||
sheerka, context, parser = self.init_parser(concepts_map, None)
|
||||
sheerka, context, parser = self.init_parser(concepts_map, create_new=True, init_from_sheerka=True)
|
||||
|
||||
res = parser.infix_to_postfix(context, expression)
|
||||
|
||||
@@ -397,8 +438,8 @@ class TestSyaConceptLexerParser(TestUsingMemoryBasedSheerka):
|
||||
])
|
||||
def test_i_can_post_fix_suffixed_unary_composition(self, expression, expected):
|
||||
concepts_map = {
|
||||
"foo": Concept("foo a").def_prop("a"),
|
||||
"bar": Concept("bar a").def_prop("a"),
|
||||
"foo": Concept("foo a").def_var("a"),
|
||||
"bar": Concept("bar a").def_var("a"),
|
||||
"baz": Concept("baz"),
|
||||
}
|
||||
sheerka, context, parser = self.init_parser(concepts_map, None)
|
||||
@@ -416,8 +457,8 @@ class TestSyaConceptLexerParser(TestUsingMemoryBasedSheerka):
|
||||
])
|
||||
def test_i_can_post_fix_prefixed_unary_composition(self, expression, expected):
|
||||
concepts_map = {
|
||||
"foo": Concept("a foo").def_prop("a"),
|
||||
"bar": Concept("a bar").def_prop("a"),
|
||||
"foo": Concept("a foo").def_var("a"),
|
||||
"bar": Concept("a bar").def_var("a"),
|
||||
"baz": Concept("baz"),
|
||||
}
|
||||
sya_def = {
|
||||
@@ -439,29 +480,29 @@ class TestSyaConceptLexerParser(TestUsingMemoryBasedSheerka):
|
||||
("one mult (two plus three)", ["one", "two", "three", "plus", "mult"]),
|
||||
])
|
||||
def test_i_can_post_fix_binary_with_precedence(self, expression, expected):
|
||||
concepts_map = {
|
||||
"plus": Concept("a plus b").def_prop("a").def_prop("b"),
|
||||
"mult": Concept("a mult b").def_prop("a").def_prop("b"),
|
||||
"one": Concept("one"),
|
||||
"two": Concept("two"),
|
||||
"three": Concept("three"),
|
||||
}
|
||||
sya_def = {
|
||||
concepts_map["plus"]: (5, SyaAssociativity.Right),
|
||||
concepts_map["mult"]: (10, SyaAssociativity.Right), # precedence greater than plus
|
||||
}
|
||||
sheerka, context, parser = self.init_parser(concepts_map, sya_def)
|
||||
# concepts_map = {
|
||||
# "plus": Concept("a plus b").def_var("a").def_var("b"),
|
||||
# "mult": Concept("a mult b").def_var("a").def_var("b"),
|
||||
# "one": Concept("one"),
|
||||
# "two": Concept("two"),
|
||||
# "three": Concept("three"),
|
||||
# }
|
||||
# sya_def = {
|
||||
# concepts_map["plus"]: (5, SyaAssociativity.Right),
|
||||
# concepts_map["mult"]: (10, SyaAssociativity.Right), # precedence greater than plus
|
||||
# }
|
||||
sheerka, context, parser = self.init_parser()
|
||||
|
||||
res = parser.infix_to_postfix(context, expression)
|
||||
expected_array = compute_expected_array(concepts_map, expression, expected)
|
||||
expected_array = compute_expected_array(cmap, expression, expected)
|
||||
|
||||
assert len(res) == 1
|
||||
assert res[0].out == expected_array
|
||||
|
||||
def test_i_can_post_fix_unary_with_precedence(self):
|
||||
concepts_map = {
|
||||
"suffixed": Concept("suffixed a").def_prop("a"),
|
||||
"prefixed": Concept("a prefixed").def_prop("a"),
|
||||
"suffixed": Concept("suffixed a").def_var("a"),
|
||||
"prefixed": Concept("a prefixed").def_var("a"),
|
||||
"a": Concept("a"),
|
||||
}
|
||||
|
||||
@@ -496,7 +537,7 @@ class TestSyaConceptLexerParser(TestUsingMemoryBasedSheerka):
|
||||
|
||||
def test_i_can_post_fix_right_associated_binary(self):
|
||||
concepts_map = {
|
||||
"equals": Concept("a equals b").def_prop("a").def_prop("b"),
|
||||
"equals": Concept("a equals b").def_var("a").def_var("b"),
|
||||
"one": Concept("one"),
|
||||
"two": Concept("two"),
|
||||
"three": Concept("three"),
|
||||
@@ -518,7 +559,7 @@ class TestSyaConceptLexerParser(TestUsingMemoryBasedSheerka):
|
||||
|
||||
def test_i_can_post_fix_left_associated_binary(self):
|
||||
concepts_map = {
|
||||
"plus": Concept("a plus b").def_prop("a").def_prop("b"),
|
||||
"plus": Concept("a plus b").def_var("a").def_var("b"),
|
||||
"one": Concept("one"),
|
||||
"two": Concept("two"),
|
||||
"three": Concept("three"),
|
||||
@@ -550,7 +591,7 @@ class TestSyaConceptLexerParser(TestUsingMemoryBasedSheerka):
|
||||
])
|
||||
def test_i_can_post_fix_right_associated_ternary(self, expression, expected):
|
||||
concepts_map = {
|
||||
"?": Concept("a ? b : c").def_prop("a").def_prop("b").def_prop("c"),
|
||||
"?": Concept("a ? b : c").def_var("a").def_var("b").def_var("c"),
|
||||
"one": Concept("one"),
|
||||
"two": Concept("two"),
|
||||
"three": Concept("three"),
|
||||
@@ -577,7 +618,7 @@ class TestSyaConceptLexerParser(TestUsingMemoryBasedSheerka):
|
||||
])
|
||||
def test_i_can_post_fix_left_associated_ternary(self, expression, expected):
|
||||
concepts_map = {
|
||||
"?": Concept("a ? b : c").def_prop("a").def_prop("b").def_prop("c"),
|
||||
"?": Concept("a ? b : c").def_var("a").def_var("b").def_var("c"),
|
||||
"one": Concept("one"),
|
||||
"two": Concept("two"),
|
||||
"three": Concept("three"),
|
||||
@@ -594,8 +635,8 @@ class TestSyaConceptLexerParser(TestUsingMemoryBasedSheerka):
|
||||
|
||||
def test_i_can_post_fix_when_multiple_concepts_are_found(self):
|
||||
concepts_map = {
|
||||
"foo": Concept("foo a").def_prop("a"),
|
||||
"foo bar": Concept("foo bar a").def_prop("a"),
|
||||
"foo": Concept("foo a").def_var("a"),
|
||||
"foo bar": Concept("foo bar a").def_var("a"),
|
||||
"baz": Concept("baz"),
|
||||
}
|
||||
sheerka, context, parser = self.init_parser(concepts_map, None)
|
||||
@@ -603,7 +644,7 @@ class TestSyaConceptLexerParser(TestUsingMemoryBasedSheerka):
|
||||
expression = "foo bar baz"
|
||||
res = parser.infix_to_postfix(context, expression)
|
||||
expected_sequences = [
|
||||
[" bar ", "foo", "baz"],
|
||||
[UTN(" bar "), "foo", "baz"],
|
||||
["baz", "foo bar"]
|
||||
]
|
||||
|
||||
@@ -646,30 +687,30 @@ class TestSyaConceptLexerParser(TestUsingMemoryBasedSheerka):
|
||||
("foo bar ( one )", ["one", "foo bar"]),
|
||||
])
|
||||
def test_i_can_pos_fix_when_parenthesis(self, expression, expected):
|
||||
concepts_map = {
|
||||
"prefixed": Concept("a prefixed").def_prop("a"),
|
||||
"suffixed": Concept("suffixed a").def_prop("a"),
|
||||
"square": Concept("square(a)").def_prop("a"),
|
||||
"foo bar": Concept("foo bar(a)").def_prop("a"),
|
||||
"plus": Concept("a plus b").def_prop("a").def_prop("b"),
|
||||
"minus": Concept("a minus b").def_prop("a").def_prop("b"),
|
||||
"if": Concept("if a then b else c end").def_prop("a").def_prop("b").def_prop("c"),
|
||||
"?": Concept("a ? b : c").def_prop("a").def_prop("b").def_prop("c"),
|
||||
"one": Concept("one"),
|
||||
"two": Concept("two"),
|
||||
"three": Concept("three"),
|
||||
}
|
||||
# concepts_map = {
|
||||
# "prefixed": Concept("a prefixed").def_var("a"),
|
||||
# "suffixed": Concept("suffixed a").def_var("a"),
|
||||
# "square": Concept("square(a)").def_var("a"),
|
||||
# "foo bar": Concept("foo bar(a)").def_var("a"),
|
||||
# "plus": Concept("a plus b").def_var("a").def_var("b"),
|
||||
# "minus": Concept("a minus b").def_var("a").def_var("b"),
|
||||
# "if": Concept("if a then b else c end").def_var("a").def_var("b").def_var("c"),
|
||||
# "?": Concept("a ? b : c").def_var("a").def_var("b").def_var("c"),
|
||||
# "one": Concept("one"),
|
||||
# "two": Concept("two"),
|
||||
# "three": Concept("three"),
|
||||
# }
|
||||
#
|
||||
# sya_def = {
|
||||
# concepts_map["square"]: (None, SyaAssociativity.No),
|
||||
# concepts_map["plus"]: (10, SyaAssociativity.Right),
|
||||
# concepts_map["minus"]: (10, SyaAssociativity.Right),
|
||||
# }
|
||||
|
||||
sya_def = {
|
||||
concepts_map["square"]: (None, SyaAssociativity.No),
|
||||
concepts_map["plus"]: (10, SyaAssociativity.Right),
|
||||
concepts_map["minus"]: (10, SyaAssociativity.Right),
|
||||
}
|
||||
|
||||
sheerka, context, parser = self.init_parser(concepts_map, sya_def)
|
||||
sheerka, context, parser = self.init_parser()
|
||||
|
||||
res = parser.infix_to_postfix(context, expression)
|
||||
expected_array = compute_expected_array(concepts_map, expression, expected)
|
||||
expected_array = compute_expected_array(cmap, expression, expected)
|
||||
|
||||
assert len(res) == 1
|
||||
assert res[0].out == expected_array
|
||||
@@ -721,28 +762,28 @@ class TestSyaConceptLexerParser(TestUsingMemoryBasedSheerka):
|
||||
("suffixed one function(two)", [["one", "suffixed", SCWC(" function(", ")", "two")]]),
|
||||
])
|
||||
def test_i_can_post_fix_when_parenthesis_and_unknown(self, expression, expected_sequences):
|
||||
concepts_map = {
|
||||
"prefixed": Concept("a prefixed").def_prop("a"),
|
||||
"suffixed": Concept("suffixed a").def_prop("a"),
|
||||
"plus": Concept("a plus b").def_prop("a").def_prop("b"),
|
||||
"mult": Concept("a mult b").def_prop("a").def_prop("b"),
|
||||
"if": Concept("if a then b else c end").def_prop("a").def_prop("b").def_prop("c"),
|
||||
"one": Concept("one"),
|
||||
"two": Concept("two"),
|
||||
"three": Concept("three"),
|
||||
"twenties": Concept("twenties", definition="'twenty' (one|two)=unit").def_prop("unit"),
|
||||
}
|
||||
sya_def = {
|
||||
concepts_map["plus"]: (5, SyaAssociativity.Right),
|
||||
concepts_map["mult"]: (10, SyaAssociativity.Right), # precedence greater than plus
|
||||
}
|
||||
sheerka, context, parser = self.init_parser(concepts_map, sya_def)
|
||||
# concepts_map = {
|
||||
# "prefixed": Concept("a prefixed").def_var("a"),
|
||||
# "suffixed": Concept("suffixed a").def_var("a"),
|
||||
# "plus": Concept("a plus b").def_var("a").def_var("b"),
|
||||
# "mult": Concept("a mult b").def_var("a").def_var("b"),
|
||||
# "if": Concept("if a then b else c end").def_var("a").def_var("b").def_var("c"),
|
||||
# "one": Concept("one"),
|
||||
# "two": Concept("two"),
|
||||
# "three": Concept("three"),
|
||||
# "twenties": Concept("twenties", definition="'twenty' (one|two)=unit").def_var("unit"),
|
||||
# }
|
||||
# sya_def = {
|
||||
# concepts_map["plus"]: (5, SyaAssociativity.Right),
|
||||
# concepts_map["mult"]: (10, SyaAssociativity.Right), # precedence greater than plus
|
||||
# }
|
||||
sheerka, context, parser = self.init_parser()
|
||||
|
||||
res = parser.infix_to_postfix(context, expression)
|
||||
|
||||
assert len(res) == len(expected_sequences)
|
||||
for res_i, expected in zip(res, expected_sequences):
|
||||
expected_array = compute_expected_array(concepts_map, expression, expected)
|
||||
expected_array = compute_expected_array(cmap, expression, expected)
|
||||
assert res_i.out == expected_array
|
||||
|
||||
@pytest.mark.parametrize("expression, expected", [
|
||||
@@ -762,13 +803,13 @@ class TestSyaConceptLexerParser(TestUsingMemoryBasedSheerka):
|
||||
("(one plus ( 1 + )", ("(", 0)),
|
||||
])
|
||||
def test_i_can_detect_parenthesis_mismatch_error_when_post_fixing(self, expression, expected):
|
||||
concepts_map = {
|
||||
"one": Concept("one"),
|
||||
"two": Concept("two"),
|
||||
"plus": Concept("a plus b").def_prop("a").def_prop("b"),
|
||||
"?": Concept("a ? b : c").def_prop("a").def_prop("b").def_prop("c"),
|
||||
}
|
||||
sheerka, context, parser = self.init_parser(concepts_map, None)
|
||||
# concepts_map = {
|
||||
# "one": Concept("one"),
|
||||
# "two": Concept("two"),
|
||||
# "plus": Concept("a plus b").def_var("a").def_var("b"),
|
||||
# "?": Concept("a ? b : c").def_var("a").def_var("b").def_var("c"),
|
||||
# }
|
||||
sheerka, context, parser = self.init_parser()
|
||||
|
||||
res = parser.infix_to_postfix(context, expression)
|
||||
|
||||
@@ -779,13 +820,13 @@ class TestSyaConceptLexerParser(TestUsingMemoryBasedSheerka):
|
||||
("one ? one two : three", ("?", ":")),
|
||||
])
|
||||
def test_i_can_detected_when_too_many_parameters(self, expression, expected):
|
||||
concepts_map = {
|
||||
"one": Concept("one"),
|
||||
"two": Concept("two"),
|
||||
"plus": Concept("a plus b").def_prop("a").def_prop("b"),
|
||||
"?": Concept("a ? b : c").def_prop("a").def_prop("b").def_prop("c"),
|
||||
}
|
||||
sheerka, context, parser = self.init_parser(concepts_map, None)
|
||||
# concepts_map = {
|
||||
# "one": Concept("one"),
|
||||
# "two": Concept("two"),
|
||||
# "plus": Concept("a plus b").def_var("a").def_var("b"),
|
||||
# "?": Concept("a ? b : c").def_var("a").def_var("b").def_var("c"),
|
||||
# }
|
||||
sheerka, context, parser = self.init_parser(cmap, None)
|
||||
|
||||
res = parser.infix_to_postfix(context, expression)
|
||||
|
||||
@@ -793,7 +834,7 @@ class TestSyaConceptLexerParser(TestUsingMemoryBasedSheerka):
|
||||
assert len(res[0].errors) == 1
|
||||
error = res[0].errors[0]
|
||||
assert isinstance(error, TooManyParametersFound)
|
||||
assert error.concept == concepts_map[expected[0]]
|
||||
assert error.concept == cmap[expected[0]]
|
||||
assert error.token.value == expected[1]
|
||||
|
||||
@pytest.mark.parametrize("expression, expected", [
|
||||
@@ -819,30 +860,30 @@ class TestSyaConceptLexerParser(TestUsingMemoryBasedSheerka):
|
||||
("(one infix two) (three prefixed)", ["one", "two", "infix", "three", "prefixed"]),
|
||||
])
|
||||
def test_i_can_post_fix_sequences(self, expression, expected):
|
||||
concepts_map = {
|
||||
"prefixed": Concept("a prefixed").def_prop("a"),
|
||||
"suffixed": Concept("suffixed a").def_prop("a"),
|
||||
"infix": Concept("a infix b").def_prop("a").def_prop("b"),
|
||||
"?": Concept("a ? b : c").def_prop("a").def_prop("b").def_prop("c"),
|
||||
"one": Concept("one"),
|
||||
"two": Concept("two"),
|
||||
"three": Concept("three"),
|
||||
"four": Concept("four"),
|
||||
}
|
||||
# concepts_map = {
|
||||
# "prefixed": Concept("a prefixed").def_var("a"),
|
||||
# "suffixed": Concept("suffixed a").def_var("a"),
|
||||
# "infix": Concept("a infix b").def_var("a").def_var("b"),
|
||||
# "?": Concept("a ? b : c").def_var("a").def_var("b").def_var("c"),
|
||||
# "one": Concept("one"),
|
||||
# "two": Concept("two"),
|
||||
# "three": Concept("three"),
|
||||
# "four": Concept("four"),
|
||||
# }
|
||||
|
||||
sheerka, context, parser = self.init_parser(concepts_map, None)
|
||||
sheerka, context, parser = self.init_parser(cmap, None)
|
||||
|
||||
res = parser.infix_to_postfix(context, expression)
|
||||
expected_array = compute_expected_array(concepts_map, expression, expected)
|
||||
expected_array = compute_expected_array(cmap, expression, expected)
|
||||
|
||||
assert len(res) == 1
|
||||
assert res[0].out == expected_array
|
||||
|
||||
def test_the_more_concepts_the_more_results(self):
|
||||
concepts_map = {
|
||||
"plus": Concept("a plus b").def_prop("a").def_prop("b"),
|
||||
"plus plus": Concept("a plus plus").def_prop("a"),
|
||||
"plus equals": Concept("a plus equals b").def_prop("a").def_prop("b"),
|
||||
"plus": Concept("a plus b").def_var("a").def_var("b"),
|
||||
"plus plus": Concept("a plus plus").def_var("a"),
|
||||
"plus equals": Concept("a plus equals b").def_var("a").def_var("b"),
|
||||
}
|
||||
|
||||
sheerka, context, parser = self.init_parser(concepts_map, None)
|
||||
@@ -864,7 +905,7 @@ class TestSyaConceptLexerParser(TestUsingMemoryBasedSheerka):
|
||||
|
||||
def test_i_can_use_string_instead_of_identifier(self):
|
||||
concepts_map = {
|
||||
"ternary": Concept("a ? ? b '::' c").def_prop("a").def_prop("b").def_prop("c"),
|
||||
"ternary": Concept("a ? ? b '::' c").def_var("a").def_var("b").def_var("c"),
|
||||
"one": Concept("one"),
|
||||
"two": Concept("two"),
|
||||
"three": Concept("three"),
|
||||
@@ -883,7 +924,7 @@ class TestSyaConceptLexerParser(TestUsingMemoryBasedSheerka):
|
||||
|
||||
def test_i_cannot_chain_non_associative(self):
|
||||
concepts_map = {
|
||||
"less than": Concept("a less than b").def_prop("a").def_prop("b"),
|
||||
"less than": Concept("a less than b").def_var("a").def_var("b"),
|
||||
"one": Concept("one"),
|
||||
"two": Concept("two"),
|
||||
"three": Concept("three"),
|
||||
@@ -904,35 +945,35 @@ class TestSyaConceptLexerParser(TestUsingMemoryBasedSheerka):
|
||||
Not quite sure why this test is here
|
||||
:return:
|
||||
"""
|
||||
concepts_map = {
|
||||
"foo": Concept("foo a").def_prop("a"),
|
||||
"one": Concept("one"),
|
||||
"two": Concept("two"),
|
||||
"twenties": Concept("twenties", definition="'twenty' (one|two)=unit").def_prop("unit"),
|
||||
}
|
||||
sheerka, context, parser = self.init_parser(concepts_map, None)
|
||||
# concepts_map = {
|
||||
# "foo": Concept("foo a").def_var("a"),
|
||||
# "one": Concept("one"),
|
||||
# "two": Concept("two"),
|
||||
# "twenties": Concept("twenties", definition="'twenty' (one|two)=unit").def_var("unit"),
|
||||
# }
|
||||
sheerka, context, parser = self.init_parser(cmap, None)
|
||||
|
||||
expression = "foo twenties"
|
||||
expression = "suffixed twenties"
|
||||
res = parser.infix_to_postfix(context, expression)
|
||||
|
||||
expected = [cnode("twenties", 2, 2, "twenties"), "foo"]
|
||||
expected_array = compute_expected_array(concepts_map, expression, expected)
|
||||
expected = [cnode("twenties", 2, 2, "twenties"), "suffixed"]
|
||||
expected_array = compute_expected_array(cmap, expression, expected)
|
||||
assert len(res) == 1
|
||||
assert res[0].out == expected_array
|
||||
|
||||
def test_i_can_parse_when_concept_atom_only(self):
|
||||
concepts_map = {
|
||||
"plus": Concept("a plus b").def_prop("a").def_prop("b"),
|
||||
"mult": Concept("a mult b").def_prop("a").def_prop("b"),
|
||||
"one": Concept("one"),
|
||||
"two": Concept("two"),
|
||||
"three": Concept("three"),
|
||||
}
|
||||
sya_def = {
|
||||
concepts_map["plus"]: (5, SyaAssociativity.Right),
|
||||
concepts_map["mult"]: (10, SyaAssociativity.Right), # precedence greater than plus
|
||||
}
|
||||
sheerka, context, parser = self.init_parser(concepts_map, sya_def)
|
||||
# concepts_map = {
|
||||
# "plus": Concept("a plus b").def_var("a").def_var("b"),
|
||||
# "mult": Concept("a mult b").def_var("a").def_var("b"),
|
||||
# "one": Concept("one"),
|
||||
# "two": Concept("two"),
|
||||
# "three": Concept("three"),
|
||||
# }
|
||||
# sya_def = {
|
||||
# concepts_map["plus"]: (5, SyaAssociativity.Right),
|
||||
# concepts_map["mult"]: (10, SyaAssociativity.Right), # precedence greater than plus
|
||||
# }
|
||||
sheerka, context, parser = self.init_parser()
|
||||
|
||||
text = "one plus two mult three"
|
||||
res = parser.parse(context, text)
|
||||
@@ -941,29 +982,29 @@ class TestSyaConceptLexerParser(TestUsingMemoryBasedSheerka):
|
||||
|
||||
assert res.status
|
||||
assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
|
||||
assert lexer_nodes == [ConceptNode(concepts_map["plus"], 0, 8, source=text)]
|
||||
assert lexer_nodes == [ConceptNode(cmap["plus"], 0, 8, source=text)]
|
||||
|
||||
# check the compiled
|
||||
expected_concept = lexer_nodes[0].concept
|
||||
assert expected_concept.compiled["a"] == concepts_map["one"]
|
||||
assert expected_concept.compiled["b"] == concepts_map["mult"]
|
||||
assert expected_concept.compiled["b"].compiled["a"] == concepts_map["two"]
|
||||
assert expected_concept.compiled["b"].compiled["b"] == concepts_map["three"]
|
||||
assert expected_concept.compiled["a"] == cmap["one"]
|
||||
assert expected_concept.compiled["b"] == cmap["mult"]
|
||||
assert expected_concept.compiled["b"].compiled["a"] == cmap["two"]
|
||||
assert expected_concept.compiled["b"].compiled["b"] == cmap["three"]
|
||||
|
||||
def test_i_can_parse_when_python_code(self):
|
||||
concepts_map = {
|
||||
"foo": Concept("foo a").def_prop("a")
|
||||
}
|
||||
sheerka, context, parser = self.init_parser(concepts_map, None)
|
||||
# concepts_map = {
|
||||
# "foo": Concept("foo a").def_var("a")
|
||||
# }
|
||||
sheerka, context, parser = self.init_parser(cmap, None)
|
||||
|
||||
text = "foo 1 + 1"
|
||||
text = "suffixed 1 + 1"
|
||||
res = parser.parse(context, text)
|
||||
wrapper = res.body
|
||||
lexer_nodes = res.body.body
|
||||
|
||||
assert res.status
|
||||
assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
|
||||
assert lexer_nodes == [ConceptNode(concepts_map["foo"], 0, 6, source=text)]
|
||||
assert lexer_nodes == [ConceptNode(cmap["suffixed"], 0, 6, source=text)]
|
||||
|
||||
# check the compiled
|
||||
expected_concept = lexer_nodes[0].concept
|
||||
@@ -977,15 +1018,15 @@ class TestSyaConceptLexerParser(TestUsingMemoryBasedSheerka):
|
||||
assert isinstance(return_value_a.body.body, PythonNode)
|
||||
|
||||
def test_i_can_parse_when_bnf_concept(self):
|
||||
concepts_map = {
|
||||
"foo": Concept("foo a").def_prop("a"),
|
||||
"one": Concept("one"),
|
||||
"two": Concept("two"),
|
||||
"twenties": Concept("twenties", definition="'twenty' (one|two)=unit").def_prop("unit"),
|
||||
}
|
||||
sheerka, context, parser = self.init_parser(concepts_map, None)
|
||||
# concepts_map = {
|
||||
# "foo": Concept("foo a").def_var("a"),
|
||||
# "one": Concept("one"),
|
||||
# "two": Concept("two"),
|
||||
# "twenties": Concept("twenties", definition="'twenty' (one|two)=unit").def_var("unit"),
|
||||
# }
|
||||
sheerka, context, parser = self.init_parser()
|
||||
|
||||
text = "foo twenty one"
|
||||
text = "suffixed twenty one"
|
||||
res = parser.parse(context, text)
|
||||
assert len(res) == 2
|
||||
assert res[1].status
|
||||
@@ -994,23 +1035,23 @@ class TestSyaConceptLexerParser(TestUsingMemoryBasedSheerka):
|
||||
lexer_nodes = res[1].body.body
|
||||
|
||||
assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
|
||||
assert lexer_nodes == [ConceptNode(concepts_map["foo"], 0, 4, source=text)]
|
||||
assert lexer_nodes == [ConceptNode(cmap["suffixed"], 0, 4, source=text)]
|
||||
|
||||
# check the compiled
|
||||
expected_concept = lexer_nodes[0].concept
|
||||
assert sheerka.isinstance(expected_concept.compiled["a"], "twenties")
|
||||
assert expected_concept.compiled["a"].compiled["unit"] == concepts_map["one"]
|
||||
assert expected_concept.compiled["a"].compiled["unit"] == cmap["one"]
|
||||
|
||||
def test_i_can_parse_sequences(self):
|
||||
concepts_map = {
|
||||
"plus": Concept("a plus b").def_prop("a").def_prop("b"),
|
||||
"foo": Concept("foo a").def_prop("a"),
|
||||
"one": Concept("one"),
|
||||
"two": Concept("two"),
|
||||
}
|
||||
sheerka, context, parser = self.init_parser(concepts_map, None)
|
||||
# concepts_map = {
|
||||
# "plus": Concept("a plus b").def_var("a").def_var("b"),
|
||||
# "foo": Concept("foo a").def_var("a"),
|
||||
# "one": Concept("one"),
|
||||
# "two": Concept("two"),
|
||||
# }
|
||||
sheerka, context, parser = self.init_parser(cmap, None)
|
||||
|
||||
text = "one plus 1 + 1 foo two"
|
||||
text = "one plus 1 + 1 suffixed two"
|
||||
res = parser.parse(context, text)
|
||||
wrapper = res.body
|
||||
lexer_nodes = res.body.body
|
||||
@@ -1018,19 +1059,19 @@ class TestSyaConceptLexerParser(TestUsingMemoryBasedSheerka):
|
||||
assert res.status
|
||||
assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
|
||||
assert lexer_nodes == [
|
||||
ConceptNode(concepts_map["plus"], 0, 9, source="one plus 1 + 1 "),
|
||||
ConceptNode(concepts_map["foo"], 10, 12, source="foo two")]
|
||||
ConceptNode(cmap["plus"], 0, 9, source="one plus 1 + 1 "),
|
||||
ConceptNode(cmap["suffixed"], 10, 12, source="suffixed two")]
|
||||
|
||||
# check the compiled
|
||||
concept_plus_a = lexer_nodes[0].concept.compiled["a"]
|
||||
concept_plus_b = lexer_nodes[0].concept.compiled["b"]
|
||||
concept_foo_a = lexer_nodes[1].concept.compiled["a"]
|
||||
concept_suffixed_a = lexer_nodes[1].concept.compiled["a"]
|
||||
|
||||
assert concept_plus_a == concepts_map["one"]
|
||||
assert concept_plus_a == cmap["one"]
|
||||
assert len(concept_plus_b) == 1
|
||||
assert sheerka.isinstance(concept_plus_b[0], BuiltinConcepts.RETURN_VALUE)
|
||||
assert isinstance(concept_plus_b[0].body.body, PythonNode)
|
||||
assert concept_foo_a == concepts_map["two"]
|
||||
assert concept_suffixed_a == cmap["two"]
|
||||
|
||||
@pytest.mark.parametrize("text, expected_status, expected_result", [
|
||||
("function(suffixed one)", True, [
|
||||
@@ -1046,28 +1087,28 @@ class TestSyaConceptLexerParser(TestUsingMemoryBasedSheerka):
|
||||
SCWC("function(", ")", CNC("suffixed", 2, 7, a=" x$!#"))]),
|
||||
])
|
||||
def test_i_can_parse_when_one_result(self, text, expected_status, expected_result):
|
||||
concepts_map = {
|
||||
"prefixed": Concept("a prefixed").def_prop("a"),
|
||||
"suffixed": Concept("suffixed a").def_prop("a"),
|
||||
"mult": Concept("a mult b").def_prop("a").def_prop("b"),
|
||||
"plus": Concept("a plus b").def_prop("a").def_prop("b"),
|
||||
"if": Concept("if a then b else c end").def_prop("a").def_prop("b").def_prop("c"),
|
||||
"one": Concept("one"),
|
||||
"two": Concept("two"),
|
||||
"three": Concept("three"),
|
||||
"twenties": Concept("twenties", definition="'twenty' (one|two)=unit").def_prop("unit"),
|
||||
}
|
||||
sya_def = {
|
||||
concepts_map["plus"]: (5, SyaAssociativity.Right),
|
||||
concepts_map["mult"]: (10, SyaAssociativity.Right), # precedence greater than plus
|
||||
}
|
||||
sheerka, context, parser = self.init_parser(concepts_map, sya_def)
|
||||
# concepts_map = {
|
||||
# "prefixed": Concept("a prefixed").def_var("a"),
|
||||
# "suffixed": Concept("suffixed a").def_var("a"),
|
||||
# "mult": Concept("a mult b").def_var("a").def_var("b"),
|
||||
# "plus": Concept("a plus b").def_var("a").def_var("b"),
|
||||
# "if": Concept("if a then b else c end").def_var("a").def_var("b").def_var("c"),
|
||||
# "one": Concept("one"),
|
||||
# "two": Concept("two"),
|
||||
# "three": Concept("three"),
|
||||
# "twenties": Concept("twenties", definition="'twenty' (one|two)=unit").def_var("unit"),
|
||||
# }
|
||||
# sya_def = {
|
||||
# concepts_map["plus"]: (5, SyaAssociativity.Right),
|
||||
# concepts_map["mult"]: (10, SyaAssociativity.Right), # precedence greater than plus
|
||||
# }
|
||||
sheerka, context, parser = self.init_parser()
|
||||
|
||||
res = parser.parse(context, text)
|
||||
wrapper = res.body
|
||||
lexer_nodes = res.body.body
|
||||
|
||||
expected_array = compute_expected_array(concepts_map, text, expected_result)
|
||||
expected_array = compute_expected_array(cmap, text, expected_result)
|
||||
assert res.status == expected_status
|
||||
assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
|
||||
assert lexer_nodes == expected_array
|
||||
@@ -1080,15 +1121,15 @@ class TestSyaConceptLexerParser(TestUsingMemoryBasedSheerka):
|
||||
# ])
|
||||
# def test_i_can_parse_when_multiple_results(self, text, list_of_expected):
|
||||
# concepts_map = {
|
||||
# "prefixed": Concept("a prefixed").def_prop("a"),
|
||||
# "suffixed": Concept("suffixed a").def_prop("a"),
|
||||
# "mult": Concept("a mult b").def_prop("a").def_prop("b"),
|
||||
# "plus": Concept("a plus b").def_prop("a").def_prop("b"),
|
||||
# "if": Concept("if a then b else c end").def_prop("a").def_prop("b").def_prop("c"),
|
||||
# "prefixed": Concept("a prefixed").def_var("a"),
|
||||
# "suffixed": Concept("suffixed a").def_var("a"),
|
||||
# "mult": Concept("a mult b").def_var("a").def_var("b"),
|
||||
# "plus": Concept("a plus b").def_var("a").def_var("b"),
|
||||
# "if": Concept("if a then b else c end").def_var("a").def_var("b").def_var("c"),
|
||||
# "one": Concept("one"),
|
||||
# "two": Concept("two"),
|
||||
# "three": Concept("three"),
|
||||
# "twenties": Concept("twenties", definition="'twenty' (one|two)=unit").def_prop("unit"),
|
||||
# "twenties": Concept("twenties", definition="'twenty' (one|two)=unit").def_var("unit"),
|
||||
# }
|
||||
# sya_def = {
|
||||
# concepts_map["plus"]: (5, SyaAssociativity.Right),
|
||||
@@ -1116,13 +1157,13 @@ class TestSyaConceptLexerParser(TestUsingMemoryBasedSheerka):
|
||||
("x$!# infix z$!#", "infix", ["a", "b"]),
|
||||
])
|
||||
def test_i_cannot_parse_when_unrecognized(self, text, expected_concept, expected_unrecognized):
|
||||
concepts_map = {
|
||||
"suffixed": Concept("suffixed a").def_prop("a"),
|
||||
"prefixed": Concept("a prefixed").def_prop("a"),
|
||||
"infix": Concept("a infix b").def_prop("a").def_prop("b"),
|
||||
"one": Concept("one")
|
||||
}
|
||||
sheerka, context, parser = self.init_parser(concepts_map, None)
|
||||
# concepts_map = {
|
||||
# "suffixed": Concept("suffixed a").def_var("a"),
|
||||
# "prefixed": Concept("a prefixed").def_var("a"),
|
||||
# "infix": Concept("a infix b").def_var("a").def_var("b"),
|
||||
# "one": Concept("one")
|
||||
# }
|
||||
sheerka, context, parser = self.init_parser()
|
||||
|
||||
res = parser.parse(context, text)
|
||||
wrapper = res.body
|
||||
@@ -1131,7 +1172,7 @@ class TestSyaConceptLexerParser(TestUsingMemoryBasedSheerka):
|
||||
|
||||
assert not res.status
|
||||
assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
|
||||
assert lexer_nodes == [ConceptNode(concepts_map[expected_concept], 0, expected_end, source=text)]
|
||||
assert lexer_nodes == [ConceptNode(cmap[expected_concept], 0, expected_end, source=text)]
|
||||
|
||||
concept_found = lexer_nodes[0].concept
|
||||
for unrecognized in expected_unrecognized:
|
||||
@@ -1142,14 +1183,14 @@ class TestSyaConceptLexerParser(TestUsingMemoryBasedSheerka):
|
||||
("one prefixed x$!#", [cnode("__var__0 prefixed", 0, 2, "one prefixed"), utnode(3, 7, " x$!#")]),
|
||||
])
|
||||
def test_i_cannot_parse_when_part_of_the_sequence_is_not_recognized(self, text, expected):
|
||||
concepts_map = {
|
||||
"suffixed": Concept("suffixed a").def_prop("a"),
|
||||
"prefixed": Concept("a prefixed").def_prop("a"),
|
||||
"infix": Concept("a infix b").def_prop("a").def_prop("b"),
|
||||
"one": Concept("one"),
|
||||
"two": Concept("two"),
|
||||
}
|
||||
sheerka, context, parser = self.init_parser(concepts_map, None)
|
||||
# concepts_map = {
|
||||
# "suffixed": Concept("suffixed a").def_var("a"),
|
||||
# "prefixed": Concept("a prefixed").def_var("a"),
|
||||
# "infix": Concept("a infix b").def_var("a").def_var("b"),
|
||||
# "one": Concept("one"),
|
||||
# "two": Concept("two"),
|
||||
# }
|
||||
sheerka, context, parser = self.init_parser()
|
||||
|
||||
res = parser.parse(context, text)
|
||||
wrapper = res.body
|
||||
@@ -1173,14 +1214,14 @@ class TestSyaConceptLexerParser(TestUsingMemoryBasedSheerka):
|
||||
Atoms concepts, source code or BNF concepts alone are discarded by the lexer
|
||||
:return:
|
||||
"""
|
||||
concepts_map = {
|
||||
"plus": Concept("a plus b").def_prop("a").def_prop("b"),
|
||||
"one": Concept("one"),
|
||||
"two": Concept("two"),
|
||||
"three": Concept("three"),
|
||||
"twenties": Concept("twenties", definition="'twenty' (one|two)=unit").def_prop("unit"),
|
||||
}
|
||||
sheerka, context, parser = self.init_parser(concepts_map, None)
|
||||
# concepts_map = {
|
||||
# "plus": Concept("a plus b").def_var("a").def_var("b"),
|
||||
# "one": Concept("one"),
|
||||
# "two": Concept("two"),
|
||||
# "three": Concept("three"),
|
||||
# "twenties": Concept("twenties", definition="'twenty' (one|two)=unit").def_var("unit"),
|
||||
# }
|
||||
sheerka, context, parser = self.init_parser()
|
||||
|
||||
res = parser.parse(context, text)
|
||||
|
||||
@@ -42,17 +42,17 @@ def get_input_nodes_from(my_concepts_map, full_expr, *args):
|
||||
|
||||
|
||||
concepts_map = {
|
||||
"5params": Concept("5params").def_prop("a").def_prop("b").def_prop("c").def_prop("d").def_prop("e"),
|
||||
"plus": Concept("a plus b", body="a + b").def_prop("a").def_prop("b"),
|
||||
"mult": Concept("a mult b", body="a * b").def_prop("a").def_prop("b"),
|
||||
"5params": Concept("5params").def_var("a").def_var("b").def_var("c").def_var("d").def_var("e"),
|
||||
"plus": Concept("a plus b", body="a + b").def_var("a").def_var("b"),
|
||||
"mult": Concept("a mult b", body="a * b").def_var("a").def_var("b"),
|
||||
"one": Concept("one", body="1"),
|
||||
"two": Concept("two", body="2"),
|
||||
"three": Concept("three", body="3"),
|
||||
"twenties": Concept("twenties", definition="'twenty' (one|two)=unit", body="20 + unit").def_prop("unit"),
|
||||
"twenties": Concept("twenties", definition="'twenty' (one|two)=unit", body="20 + unit").def_var("unit"),
|
||||
"hello_atom": Concept("hello one"),
|
||||
"hello_sya": Concept("hello a").def_prop("a"),
|
||||
"greetings_a": Concept("greetings a").def_prop("a"),
|
||||
"greetings_b": Concept("greetings b").def_prop("b"),
|
||||
"hello_sya": Concept("hello a").def_var("a"),
|
||||
"greetings_a": Concept("greetings a").def_var("a"),
|
||||
"greetings_b": Concept("greetings b").def_var("b"),
|
||||
|
||||
}
|
||||
|
||||
@@ -64,7 +64,7 @@ class TestUnrecognizedNodeParser(TestUsingMemoryBasedSheerka):
|
||||
def setup_class(cls):
|
||||
t = TestUnrecognizedNodeParser()
|
||||
TestUnrecognizedNodeParser.sheerka, context, _ = t.init_parser(concepts_map, create_new=True)
|
||||
TestUnrecognizedNodeParser.sheerka.set_sya_def(context, [
|
||||
TestUnrecognizedNodeParser.sheerka.force_sya_def(context, [
|
||||
(concepts_map["mult"].id, 20, SyaAssociativity.Right),
|
||||
(concepts_map["plus"].id, 10, SyaAssociativity.Right),
|
||||
])
|
||||
@@ -144,13 +144,15 @@ class TestUnrecognizedNodeParser(TestUsingMemoryBasedSheerka):
|
||||
expected_nodes = compute_expected_array(
|
||||
concepts_map,
|
||||
" one plus two mult three ",
|
||||
[CNC("plus", a="one", b=CC("mult", a="two", b="three"))])
|
||||
[CNC("plus", a="one", b=CC("mult", a="two", b="three"))],
|
||||
exclude_body=True)
|
||||
|
||||
assert concept.compiled["e"][0].body.body == expected_nodes
|
||||
|
||||
# # sanity check, I can evaluate the concept
|
||||
# evaluated = sheerka.evaluate_concept(self.get_context(sheerka, eval_body=True), concept)
|
||||
# assert evaluated.key == concept.key
|
||||
# assert evaluated.get_prop("a") ==
|
||||
# assert evaluated.get_value("a") ==
|
||||
|
||||
def test_i_can_validate_with_recursion(self):
|
||||
sheerka, context, parser = self.init_parser()
|
||||
@@ -246,6 +248,22 @@ class TestUnrecognizedNodeParser(TestUsingMemoryBasedSheerka):
|
||||
assert len(actual_nodes) == 1
|
||||
assert actual_nodes[0] == scnode(0, 4, expression)
|
||||
|
||||
def test_i_cannot_parse_unrecognized_python_that_looks_like_concept(self):
|
||||
sheerka, context, parser = self.init_parser()
|
||||
|
||||
expression = "fake_concept_name" # as it's not a concept, it will be recognized as python node
|
||||
nodes = get_input_nodes_from(concepts_map, expression, UTN(expression))
|
||||
parser_input = ParserResultConcept("parsers.xxx", source=expression, value=nodes)
|
||||
|
||||
res = parser.parse(context, parser_input)
|
||||
parser_result = res.body
|
||||
actual_nodes = res.body.body
|
||||
|
||||
assert not res.status
|
||||
assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT)
|
||||
assert len(actual_nodes) == 1
|
||||
assert actual_nodes[0] == nodes[0]
|
||||
|
||||
def test_i_can_parse_unrecognized_bnf_concept_node(self):
|
||||
|
||||
sheerka, context, parser = self.init_parser()
|
||||
@@ -285,7 +303,8 @@ class TestUnrecognizedNodeParser(TestUsingMemoryBasedSheerka):
|
||||
concepts_map,
|
||||
expression, [CNC("plus",
|
||||
a="one",
|
||||
b=CC("mult", source="two mult three", a="two", b="three"))])
|
||||
b=CC("mult", source="two mult three", a="two", b="three"))],
|
||||
exclude_body=True)
|
||||
assert actual_nodes == expected_array
|
||||
|
||||
def test_i_can_parse_sequences(self):
|
||||
@@ -314,7 +333,7 @@ class TestUnrecognizedNodeParser(TestUsingMemoryBasedSheerka):
|
||||
expression = "two hello one three"
|
||||
nodes = get_input_nodes_from(concepts_map, expression,
|
||||
"two", UTN("hello one"), "three")
|
||||
parser_input = ParserResultConcept("parsers.xxx", source="one plus two hello one", value=nodes)
|
||||
parser_input = ParserResultConcept("parsers.xxx", source=expression, value=nodes)
|
||||
|
||||
res = parser.parse(context, parser_input)
|
||||
assert len(res) == 2
|
||||
@@ -332,7 +351,8 @@ class TestUnrecognizedNodeParser(TestUsingMemoryBasedSheerka):
|
||||
expected_1 = compute_expected_array(concepts_map, expression, [
|
||||
CN("two", 0, 0),
|
||||
CNC("hello_sya", source="hello one", start=2, end=4, a="one"),
|
||||
CN("three", 6, 6)])
|
||||
CN("three", 6, 6)],
|
||||
exclude_body=True)
|
||||
|
||||
assert actual_nodes1 == expected_1
|
||||
|
||||
@@ -341,7 +361,7 @@ class TestUnrecognizedNodeParser(TestUsingMemoryBasedSheerka):
|
||||
expression = "greetings two"
|
||||
nodes = get_input_nodes_from(concepts_map, expression, UTN("greetings two"))
|
||||
|
||||
parser_input = ParserResultConcept("parsers.xxx", source="greetings two", value=nodes)
|
||||
parser_input = ParserResultConcept("parsers.xxx", source=expression, value=nodes)
|
||||
|
||||
res = parser.parse(context, parser_input)
|
||||
assert len(res) == 2
|
||||
@@ -350,14 +370,25 @@ class TestUnrecognizedNodeParser(TestUsingMemoryBasedSheerka):
|
||||
|
||||
actual_nodes0 = res[0].body.body
|
||||
expected_0 = compute_expected_array(concepts_map, expression, [
|
||||
CNC("greetings_a", source="greetings two", start=0, end=2, a="two")])
|
||||
CNC("greetings_a", source="greetings two", start=0, end=2, a="two")], exclude_body=True)
|
||||
assert actual_nodes0 == expected_0
|
||||
|
||||
actual_nodes1 = res[1].body.body
|
||||
expected_1 = compute_expected_array(concepts_map, expression, [
|
||||
CNC("greetings_b", source="greetings two", start=0, end=2, b="two")])
|
||||
CNC("greetings_b", source="greetings two", start=0, end=2, b="two")], exclude_body=True)
|
||||
assert actual_nodes1 == expected_1
|
||||
|
||||
def test_i_cannot_parse_when_some_unrecognized_remain(self):
|
||||
sheerka, context, parser = self.init_parser()
|
||||
expression = "twenty one + one"
|
||||
nodes = get_input_nodes_from(concepts_map, expression, UTN("twenty "), "one", " + ", ("one", 1))
|
||||
|
||||
parser_input = ParserResultConcept("parsers.xxx", source=expression, value=nodes)
|
||||
res = parser.parse(context, parser_input)
|
||||
|
||||
assert not res.status
|
||||
assert res.body.body == nodes
|
||||
|
||||
def test_i_cannot_parse_when_i_cannot_validate(self):
|
||||
sheerka, context, parser = self.init_parser(concepts_map, create_new=True)
|
||||
expression = "one plus unknown tokens"
|
||||
|
||||
Reference in New Issue
Block a user