Refactored Caching, Refactored BnfNodeParser, Introduced Sphinx
This commit is contained in:
@@ -1,194 +1,193 @@
|
||||
import ast
|
||||
|
||||
import pytest
|
||||
|
||||
from core.builtin_concepts import ParserResultConcept, ReturnValueConcept, BuiltinConcepts
|
||||
from core.concept import Concept
|
||||
from core.tokenizer import Token, TokenKind, Tokenizer
|
||||
from parsers.BaseNodeParser import SourceCodeNode
|
||||
from parsers.BnfNodeParser import ConceptNode, UnrecognizedTokensNode
|
||||
from parsers.ConceptsWithConceptsParser import ConceptsWithConceptsParser
|
||||
from parsers.MultipleConceptsParser import MultipleConceptsParser
|
||||
from parsers.PythonParser import PythonNode
|
||||
|
||||
from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
|
||||
|
||||
multiple_concepts_parser = MultipleConceptsParser()
|
||||
|
||||
|
||||
def ret_val(*args):
|
||||
result = []
|
||||
index = 0
|
||||
source = ""
|
||||
for item in args:
|
||||
if isinstance(item, Concept):
|
||||
tokens = [Token(TokenKind.IDENTIFIER, item.name, 0, 0, 0)]
|
||||
result.append(ConceptNode(item, index, index, tokens, item.name))
|
||||
index += 1
|
||||
source += item.name
|
||||
elif isinstance(item, PythonNode):
|
||||
tokens = list(Tokenizer(item.source))[:-1] # strip trailing EOF
|
||||
result.append(SourceCodeNode(item, index, index + len(tokens) - 1, tokens, item.source))
|
||||
index += len(tokens)
|
||||
source += item.source
|
||||
else:
|
||||
tokens = list(Tokenizer(item))[:-1] # strip trailing EOF
|
||||
result.append(UnrecognizedTokensNode(index, index + len(tokens) - 1, tokens))
|
||||
index += len(tokens)
|
||||
source += item
|
||||
|
||||
return ReturnValueConcept(
|
||||
"who",
|
||||
False,
|
||||
ParserResultConcept(parser=multiple_concepts_parser, value=result, source=source))
|
||||
|
||||
|
||||
class TestConceptsWithConceptsParser(TestUsingMemoryBasedSheerka):
|
||||
|
||||
def init(self, concepts, inputs):
|
||||
context = self.get_context()
|
||||
for concept in concepts:
|
||||
context.sheerka.create_new_concept(context, concept)
|
||||
|
||||
return context, ret_val(*inputs)
|
||||
|
||||
def execute(self, concepts, inputs):
|
||||
context, input_return_values = self.init(concepts, inputs)
|
||||
|
||||
parser = ConceptsWithConceptsParser()
|
||||
result = parser.parse(context, input_return_values.body)
|
||||
|
||||
wrapper = result.body
|
||||
return_value = result.body.body
|
||||
|
||||
return context, parser, result, wrapper, return_value
|
||||
|
||||
@pytest.mark.parametrize("text, interested", [
|
||||
("not parser result", False),
|
||||
(ParserResultConcept(parser="not multiple_concepts_parser"), False),
|
||||
(ParserResultConcept(parser=multiple_concepts_parser, value=[UnrecognizedTokensNode(0, 0, [])]), True),
|
||||
])
|
||||
def test_not_interested(self, text, interested):
|
||||
context = self.get_context()
|
||||
|
||||
res = ConceptsWithConceptsParser().parse(context, text)
|
||||
if interested:
|
||||
assert res is not None
|
||||
else:
|
||||
assert res is None
|
||||
|
||||
def test_i_can_parse_composition_of_concepts(self):
|
||||
foo = Concept("foo")
|
||||
bar = Concept("bar")
|
||||
plus = Concept("a plus b").def_prop("a").def_prop("b")
|
||||
|
||||
context, parser, result, wrapper, return_value = self.execute([foo, bar, plus], [foo, " plus ", bar])
|
||||
|
||||
assert result.status
|
||||
assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
|
||||
assert result.who == wrapper.parser.name
|
||||
assert wrapper.source == "foo plus bar"
|
||||
assert context.sheerka.isinstance(return_value, plus)
|
||||
|
||||
assert return_value.compiled["a"] == foo
|
||||
assert return_value.compiled["b"] == bar
|
||||
|
||||
# sanity check, I can evaluate the result
|
||||
evaluated = context.sheerka.evaluate_concept(self.get_context(context.sheerka, True), return_value)
|
||||
assert evaluated.key == return_value.key
|
||||
assert evaluated.get_prop("a") == foo.init_key()
|
||||
assert evaluated.get_prop("b") == bar.init_key()
|
||||
|
||||
def test_i_can_parse_when_composition_of_source_code(self):
|
||||
plus = Concept("a plus b", body="a + b").def_prop("a").def_prop("b")
|
||||
left = PythonNode("1+1", ast.parse("1+1", mode="eval"))
|
||||
right = PythonNode("2+2", ast.parse("2+2", mode="eval"))
|
||||
context, parser, result, wrapper, return_value = self.execute([plus], [left, " plus ", right])
|
||||
|
||||
assert result.status
|
||||
assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
|
||||
assert result.who == wrapper.parser.name
|
||||
assert wrapper.source == "1+1 plus 2+2"
|
||||
assert context.sheerka.isinstance(return_value, plus)
|
||||
|
||||
left_parser_result = ParserResultConcept(parser=parser, source="1+1", value=left)
|
||||
right_parser_result = ParserResultConcept(parser=parser, source="2+2", value=right)
|
||||
assert return_value.compiled["a"] == [ReturnValueConcept(parser.name, True, left_parser_result)]
|
||||
assert return_value.compiled["b"] == [ReturnValueConcept(parser.name, True, right_parser_result)]
|
||||
|
||||
# sanity check, I can evaluate the result
|
||||
evaluated = context.sheerka.evaluate_concept(self.get_context(context.sheerka, True), return_value)
|
||||
assert evaluated.key == return_value.key
|
||||
assert evaluated.get_prop("a") == 2
|
||||
assert evaluated.get_prop("b") == 4
|
||||
assert evaluated.body == 6
|
||||
|
||||
def test_i_can_parse_when_mix_of_concept_and_code(self):
|
||||
plus = Concept("a plus b").def_prop("a").def_prop("b")
|
||||
code = PythonNode("1+1", ast.parse("1+1", mode="eval"))
|
||||
foo = Concept("foo")
|
||||
context, parser, result, wrapper, return_value = self.execute([plus, foo], [foo, " plus ", code])
|
||||
|
||||
assert result.status
|
||||
assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
|
||||
assert result.who == wrapper.parser.name
|
||||
assert wrapper.source == "foo plus 1+1"
|
||||
assert context.sheerka.isinstance(return_value, plus)
|
||||
|
||||
code_parser_result = ParserResultConcept(parser=parser, source="1+1", value=code)
|
||||
assert return_value.compiled["a"] == foo
|
||||
assert return_value.compiled["b"] == [ReturnValueConcept(parser.name, True, code_parser_result)]
|
||||
|
||||
# sanity check, I can evaluate the result
|
||||
evaluated = context.sheerka.evaluate_concept(self.get_context(context.sheerka, True), return_value)
|
||||
assert evaluated.key == return_value.key
|
||||
assert evaluated.get_prop("a") == foo.init_key()
|
||||
assert evaluated.get_prop("b") == 2
|
||||
|
||||
def test_i_can_parse_when_multiple_concepts_are_recognized(self):
|
||||
foo = Concept("foo")
|
||||
bar = Concept("bar")
|
||||
plus_1 = Concept("a plus b", body="body1").def_prop("a").def_prop("b")
|
||||
plus_2 = Concept("a plus b", body="body2").def_prop("a").def_prop("b")
|
||||
|
||||
context, input_return_values = self.init([foo, bar, plus_1, plus_2], [foo, " plus ", bar])
|
||||
parser = ConceptsWithConceptsParser()
|
||||
result = parser.parse(context, input_return_values.body)
|
||||
|
||||
assert len(result) == 2
|
||||
|
||||
res = result[0]
|
||||
wrapper = res.value
|
||||
return_value = res.value.value
|
||||
assert res.status
|
||||
assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
|
||||
assert res.who == wrapper.parser.name
|
||||
assert wrapper.source == "foo plus bar"
|
||||
assert context.sheerka.isinstance(return_value, plus_1)
|
||||
assert return_value.compiled["a"] == foo
|
||||
assert return_value.compiled["b"] == bar
|
||||
|
||||
res = result[1]
|
||||
wrapper = res.value
|
||||
return_value = res.value.value
|
||||
assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
|
||||
assert res.who == wrapper.parser.name
|
||||
assert wrapper.source == "foo plus bar"
|
||||
assert context.sheerka.isinstance(return_value, plus_2)
|
||||
assert return_value.compiled["a"] == foo
|
||||
assert return_value.compiled["b"] == bar
|
||||
|
||||
def test_i_cannot_parse_when_unknown_concept(self):
|
||||
foo = Concept("foo")
|
||||
bar = Concept("bar")
|
||||
|
||||
context, input_return_values = self.init([foo, bar], [foo, " plus ", bar])
|
||||
parser = ConceptsWithConceptsParser()
|
||||
result = parser.parse(context, input_return_values.body)
|
||||
wrapper = result.body
|
||||
return_value = result.body.body
|
||||
|
||||
assert not result.status
|
||||
assert context.sheerka.isinstance(wrapper, BuiltinConcepts.NOT_FOR_ME)
|
||||
assert result.who == parser.name
|
||||
assert return_value == input_return_values.body.body
|
||||
# import ast
|
||||
#
|
||||
# import pytest
|
||||
#
|
||||
# from core.builtin_concepts import ParserResultConcept, ReturnValueConcept, BuiltinConcepts
|
||||
# from core.concept import Concept
|
||||
# from core.tokenizer import Token, TokenKind, Tokenizer
|
||||
# from parsers.BaseNodeParser import SourceCodeNode, ConceptNode, UnrecognizedTokensNode
|
||||
# from parsers.ConceptsWithConceptsParser import ConceptsWithConceptsParser
|
||||
# from parsers.MultipleConceptsParser import MultipleConceptsParser
|
||||
# from parsers.PythonParser import PythonNode
|
||||
#
|
||||
# from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
|
||||
#
|
||||
# multiple_concepts_parser = MultipleConceptsParser()
|
||||
#
|
||||
#
|
||||
# def ret_val(*args):
|
||||
# result = []
|
||||
# index = 0
|
||||
# source = ""
|
||||
# for item in args:
|
||||
# if isinstance(item, Concept):
|
||||
# tokens = [Token(TokenKind.IDENTIFIER, item.name, 0, 0, 0)]
|
||||
# result.append(ConceptNode(item, index, index, tokens, item.name))
|
||||
# index += 1
|
||||
# source += item.name
|
||||
# elif isinstance(item, PythonNode):
|
||||
# tokens = list(Tokenizer(item.source))[:-1] # strip trailing EOF
|
||||
# result.append(SourceCodeNode(item, index, index + len(tokens) - 1, tokens, item.source))
|
||||
# index += len(tokens)
|
||||
# source += item.source
|
||||
# else:
|
||||
# tokens = list(Tokenizer(item))[:-1] # strip trailing EOF
|
||||
# result.append(UnrecognizedTokensNode(index, index + len(tokens) - 1, tokens))
|
||||
# index += len(tokens)
|
||||
# source += item
|
||||
#
|
||||
# return ReturnValueConcept(
|
||||
# "who",
|
||||
# False,
|
||||
# ParserResultConcept(parser=multiple_concepts_parser, value=result, source=source))
|
||||
#
|
||||
#
|
||||
# class TestConceptsWithConceptsParser(TestUsingMemoryBasedSheerka):
|
||||
#
|
||||
# def init(self, concepts, inputs):
|
||||
# context = self.get_context()
|
||||
# for concept in concepts:
|
||||
# context.sheerka.create_new_concept(context, concept)
|
||||
#
|
||||
# return context, ret_val(*inputs)
|
||||
#
|
||||
# def execute(self, concepts, inputs):
|
||||
# context, input_return_values = self.init(concepts, inputs)
|
||||
#
|
||||
# parser = ConceptsWithConceptsParser()
|
||||
# result = parser.parse(context, input_return_values.body)
|
||||
#
|
||||
# wrapper = result.body
|
||||
# return_value = result.body.body
|
||||
#
|
||||
# return context, parser, result, wrapper, return_value
|
||||
#
|
||||
# @pytest.mark.parametrize("text, interested", [
|
||||
# ("not parser result", False),
|
||||
# (ParserResultConcept(parser="not multiple_concepts_parser"), False),
|
||||
# (ParserResultConcept(parser=multiple_concepts_parser, value=[UnrecognizedTokensNode(0, 0, [])]), True),
|
||||
# ])
|
||||
# def test_not_interested(self, text, interested):
|
||||
# context = self.get_context()
|
||||
#
|
||||
# res = ConceptsWithConceptsParser().parse(context, text)
|
||||
# if interested:
|
||||
# assert res is not None
|
||||
# else:
|
||||
# assert res is None
|
||||
#
|
||||
# def test_i_can_parse_composition_of_concepts(self):
|
||||
# foo = Concept("foo")
|
||||
# bar = Concept("bar")
|
||||
# plus = Concept("a plus b").def_var("a").def_var("b")
|
||||
#
|
||||
# context, parser, result, wrapper, return_value = self.execute([foo, bar, plus], [foo, " plus ", bar])
|
||||
#
|
||||
# assert result.status
|
||||
# assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
|
||||
# assert result.who == wrapper.parser.name
|
||||
# assert wrapper.source == "foo plus bar"
|
||||
# assert context.sheerka.isinstance(return_value, plus)
|
||||
#
|
||||
# assert return_value.compiled["a"] == foo
|
||||
# assert return_value.compiled["b"] == bar
|
||||
#
|
||||
# # sanity check, I can evaluate the result
|
||||
# evaluated = context.sheerka.evaluate_concept(self.get_context(context.sheerka, True), return_value)
|
||||
# assert evaluated.key == return_value.key
|
||||
# assert evaluated.get_prop("a") == foo.init_key()
|
||||
# assert evaluated.get_prop("b") == bar.init_key()
|
||||
#
|
||||
# def test_i_can_parse_when_composition_of_source_code(self):
|
||||
# plus = Concept("a plus b", body="a + b").def_var("a").def_var("b")
|
||||
# left = PythonNode("1+1", ast.parse("1+1", mode="eval"))
|
||||
# right = PythonNode("2+2", ast.parse("2+2", mode="eval"))
|
||||
# context, parser, result, wrapper, return_value = self.execute([plus], [left, " plus ", right])
|
||||
#
|
||||
# assert result.status
|
||||
# assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
|
||||
# assert result.who == wrapper.parser.name
|
||||
# assert wrapper.source == "1+1 plus 2+2"
|
||||
# assert context.sheerka.isinstance(return_value, plus)
|
||||
#
|
||||
# left_parser_result = ParserResultConcept(parser=parser, source="1+1", value=left)
|
||||
# right_parser_result = ParserResultConcept(parser=parser, source="2+2", value=right)
|
||||
# assert return_value.compiled["a"] == [ReturnValueConcept(parser.name, True, left_parser_result)]
|
||||
# assert return_value.compiled["b"] == [ReturnValueConcept(parser.name, True, right_parser_result)]
|
||||
#
|
||||
# # sanity check, I can evaluate the result
|
||||
# evaluated = context.sheerka.evaluate_concept(self.get_context(context.sheerka, True), return_value)
|
||||
# assert evaluated.key == return_value.key
|
||||
# assert evaluated.get_prop("a") == 2
|
||||
# assert evaluated.get_prop("b") == 4
|
||||
# assert evaluated.body == 6
|
||||
#
|
||||
# def test_i_can_parse_when_mix_of_concept_and_code(self):
|
||||
# plus = Concept("a plus b").def_var("a").def_var("b")
|
||||
# code = PythonNode("1+1", ast.parse("1+1", mode="eval"))
|
||||
# foo = Concept("foo")
|
||||
# context, parser, result, wrapper, return_value = self.execute([plus, foo], [foo, " plus ", code])
|
||||
#
|
||||
# assert result.status
|
||||
# assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
|
||||
# assert result.who == wrapper.parser.name
|
||||
# assert wrapper.source == "foo plus 1+1"
|
||||
# assert context.sheerka.isinstance(return_value, plus)
|
||||
#
|
||||
# code_parser_result = ParserResultConcept(parser=parser, source="1+1", value=code)
|
||||
# assert return_value.compiled["a"] == foo
|
||||
# assert return_value.compiled["b"] == [ReturnValueConcept(parser.name, True, code_parser_result)]
|
||||
#
|
||||
# # sanity check, I can evaluate the result
|
||||
# evaluated = context.sheerka.evaluate_concept(self.get_context(context.sheerka, True), return_value)
|
||||
# assert evaluated.key == return_value.key
|
||||
# assert evaluated.get_prop("a") == foo.init_key()
|
||||
# assert evaluated.get_prop("b") == 2
|
||||
#
|
||||
# def test_i_can_parse_when_multiple_concepts_are_recognized(self):
|
||||
# foo = Concept("foo")
|
||||
# bar = Concept("bar")
|
||||
# plus_1 = Concept("a plus b", body="body1").def_var("a").def_var("b")
|
||||
# plus_2 = Concept("a plus b", body="body2").def_var("a").def_var("b")
|
||||
#
|
||||
# context, input_return_values = self.init([foo, bar, plus_1, plus_2], [foo, " plus ", bar])
|
||||
# parser = ConceptsWithConceptsParser()
|
||||
# result = parser.parse(context, input_return_values.body)
|
||||
#
|
||||
# assert len(result) == 2
|
||||
#
|
||||
# res = result[0]
|
||||
# wrapper = res.value
|
||||
# return_value = res.value.value
|
||||
# assert res.status
|
||||
# assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
|
||||
# assert res.who == wrapper.parser.name
|
||||
# assert wrapper.source == "foo plus bar"
|
||||
# assert context.sheerka.isinstance(return_value, plus_1)
|
||||
# assert return_value.compiled["a"] == foo
|
||||
# assert return_value.compiled["b"] == bar
|
||||
#
|
||||
# res = result[1]
|
||||
# wrapper = res.value
|
||||
# return_value = res.value.value
|
||||
# assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
|
||||
# assert res.who == wrapper.parser.name
|
||||
# assert wrapper.source == "foo plus bar"
|
||||
# assert context.sheerka.isinstance(return_value, plus_2)
|
||||
# assert return_value.compiled["a"] == foo
|
||||
# assert return_value.compiled["b"] == bar
|
||||
#
|
||||
# def test_i_cannot_parse_when_unknown_concept(self):
|
||||
# foo = Concept("foo")
|
||||
# bar = Concept("bar")
|
||||
#
|
||||
# context, input_return_values = self.init([foo, bar], [foo, " plus ", bar])
|
||||
# parser = ConceptsWithConceptsParser()
|
||||
# result = parser.parse(context, input_return_values.body)
|
||||
# wrapper = result.body
|
||||
# return_value = result.body.body
|
||||
#
|
||||
# assert not result.status
|
||||
# assert context.sheerka.isinstance(wrapper, BuiltinConcepts.NOT_FOR_ME)
|
||||
# assert result.who == parser.name
|
||||
# assert return_value == input_return_values.body.body
|
||||
|
||||
Reference in New Issue
Block a user