kodjo
7dcaa9c111
Fixed #77 : Parser: ShortTermMemoryParser should be called separately Fixed #78 : Remove VariableNode usage Fixed #79 : ConceptManager: Implement compile caching Fixed #80 : SheerkaExecute : parsers_key is not correctly computed Fixed #81 : ValidateConceptEvaluator : Validate concept's where and pre clauses right after the parsing Fixed #82 : SheerkaIsAManager: isa() failed when the set as a body Fixed #83 : ValidateConceptEvaluator : Support BNF and SYA Concepts Fixed #84 : ExpressionParser: Implement the parser as a standard parser Fixed #85 : Services: Give order to services Fixed #86 : cannot manage smart_get_attr(the short, color)
392 lines
18 KiB
Python
392 lines
18 KiB
Python
import ast
|
|
|
|
import pytest
|
|
|
|
from core.builtin_concepts import ReturnValueConcept, ParserResultConcept, BuiltinConcepts
|
|
from core.concept import VARIABLE_PREFIX, Concept, DEFINITION_TYPE_BNF, DEFINITION_TYPE_DEF
|
|
from core.sheerka.services.SheerkaConceptManager import NoFirstTokenError
|
|
from core.sheerka.services.SheerkaExecute import ParserInput
|
|
from core.tokenizer import Tokenizer
|
|
from evaluators.DefConceptEvaluator import DefConceptEvaluator, PossibleVariable, CertainVariable
|
|
from parsers.BaseParser import BaseParser
|
|
from parsers.BnfDefinitionParser import BnfDefinitionParser
|
|
from parsers.BnfNodeParser import Sequence, StrMatch, ZeroOrMore, ConceptExpression, VariableExpression
|
|
from parsers.DefConceptParser import DefConceptNode, NameNode, DefConceptParser
|
|
from parsers.PythonParser import PythonNode, PythonParser
|
|
from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
|
|
|
|
|
|
class TestDefConceptEvaluator(TestUsingMemoryBasedSheerka):
|
|
|
|
@staticmethod
|
|
def get_concept_part(part):
|
|
if isinstance(part, str):
|
|
node = PythonNode(part, ast.parse(part, mode="exec"))
|
|
return ReturnValueConcept(
|
|
who="parsers.Default",
|
|
status=True,
|
|
value=ParserResultConcept(
|
|
source=part,
|
|
parser=PythonParser(),
|
|
value=node))
|
|
|
|
if isinstance(part, PythonNode):
|
|
return ReturnValueConcept(
|
|
who="parsers.Default",
|
|
status=True,
|
|
value=ParserResultConcept(
|
|
source=part.source,
|
|
parser=PythonParser(),
|
|
value=part))
|
|
|
|
if isinstance(part, ReturnValueConcept):
|
|
return part
|
|
|
|
@staticmethod
|
|
def get_return_value(source, parsing_expression):
|
|
return ReturnValueConcept(
|
|
who="Parsers:RegexParser",
|
|
status=True,
|
|
value=ParserResultConcept(
|
|
source=source,
|
|
parser=BnfDefinitionParser(),
|
|
value=parsing_expression
|
|
)
|
|
)
|
|
|
|
def get_def_concept(self, name, where=None, pre=None, post=None, body=None, ret=None,
|
|
definition=None, bnf_def=None):
|
|
def_concept = DefConceptNode([], name=NameNode(list(Tokenizer(name))))
|
|
|
|
if body:
|
|
def_concept.body = self.get_concept_part(body)
|
|
if where:
|
|
def_concept.where = self.get_concept_part(where)
|
|
if pre:
|
|
def_concept.pre = self.get_concept_part(pre)
|
|
if post:
|
|
def_concept.post = self.get_concept_part(post)
|
|
if ret:
|
|
def_concept.ret = self.get_concept_part(ret)
|
|
|
|
if bnf_def:
|
|
def_concept.definition = bnf_def
|
|
def_concept.definition_type = DEFINITION_TYPE_BNF
|
|
if definition:
|
|
def_concept.definition = NameNode(list(Tokenizer(definition)))
|
|
def_concept.definition_type = DEFINITION_TYPE_DEF
|
|
|
|
return ReturnValueConcept(BaseParser.PREFIX + "some_name", True, ParserResultConcept(value=def_concept))
|
|
|
|
@staticmethod
|
|
def get_def_concept_node_from_name_only(name):
|
|
return DefConceptNode([], name=NameNode(list(Tokenizer(name))))
|
|
|
|
@pytest.mark.parametrize("ret_val, expected", [
|
|
(ReturnValueConcept(BaseParser.PREFIX + "some_name", True, ParserResultConcept(value=DefConceptNode([]))),
|
|
True),
|
|
(ReturnValueConcept(BaseParser.PREFIX + "some_name", False, ParserResultConcept(value=DefConceptNode([]))),
|
|
False),
|
|
(ReturnValueConcept(BaseParser.PREFIX + "some_name", True, "not a ParserResultConcept"), False),
|
|
(ReturnValueConcept(BaseParser.PREFIX + "some_name", True, ParserResultConcept()), False),
|
|
])
|
|
def test_i_can_match(self, ret_val, expected):
|
|
context = self.get_context()
|
|
assert DefConceptEvaluator().matches(context, ret_val) == expected
|
|
|
|
def test_that_the_source_is_correctly_set_for_bnf_concepts(self):
|
|
context = self.get_context()
|
|
def_concept_return_value = self.get_def_concept(
|
|
name="hello a",
|
|
bnf_def=self.get_return_value("hello a", Sequence(StrMatch("hello"), StrMatch("a"))),
|
|
where="isinstance(a, str )",
|
|
pre="a is not None",
|
|
body="print('hello' + a)",
|
|
ret="a")
|
|
|
|
evaluated = DefConceptEvaluator().eval(context, def_concept_return_value)
|
|
|
|
assert evaluated.status
|
|
assert context.sheerka.isinstance(evaluated.body, BuiltinConcepts.NEW_CONCEPT)
|
|
|
|
created_concept = evaluated.body.body
|
|
assert created_concept.get_metadata().name == "hello a"
|
|
assert created_concept.get_metadata().key == "hello __var__0"
|
|
assert created_concept.get_metadata().where == "isinstance(a, str )"
|
|
assert created_concept.get_metadata().pre == "a is not None"
|
|
assert created_concept.get_metadata().post is None # test that NotInitialized is mapped into None
|
|
assert created_concept.get_metadata().body == "print('hello' + a)"
|
|
assert created_concept.get_metadata().ret == "a"
|
|
assert created_concept.get_metadata().definition == "hello a"
|
|
assert created_concept.get_metadata().definition_type == "bnf"
|
|
|
|
def test_that_the_source_is_correctly_set_for_concepts_with_simple_definition(self):
|
|
context = self.get_context()
|
|
def_concept_return_value = self.get_def_concept(
|
|
name="greetings",
|
|
definition="hello a",
|
|
where="isinstance(a, str )",
|
|
pre="a is not None",
|
|
body="print('hello' + a)")
|
|
|
|
evaluated = DefConceptEvaluator().eval(context, def_concept_return_value)
|
|
|
|
assert evaluated.status
|
|
assert context.sheerka.isinstance(evaluated.body, BuiltinConcepts.NEW_CONCEPT)
|
|
|
|
created_concept = evaluated.body.body
|
|
assert created_concept.get_metadata().name == "greetings"
|
|
assert created_concept.get_metadata().key == "hello __var__0"
|
|
assert created_concept.get_metadata().where == "isinstance(a, str )"
|
|
assert created_concept.get_metadata().pre == "a is not None"
|
|
assert created_concept.get_metadata().post is None
|
|
assert created_concept.get_metadata().body == "print('hello' + a)"
|
|
assert created_concept.get_metadata().definition == "hello a"
|
|
assert created_concept.get_metadata().definition_type == "def"
|
|
|
|
@pytest.mark.parametrize("expression, name, expected", [
|
|
("isinstance(a, str)", "a b", {"a"}),
|
|
("a.location = b", "a is in b", {"a", "b"}),
|
|
("a.location = b", "'a' is in b", {"b"}),
|
|
("date.today()", "what is the date", set()),
|
|
("a.location", "where is a", {"a"})
|
|
])
|
|
def test_i_can_get_variables_from_python_node_when_long_name(self, expression, name, expected):
|
|
ret_val = self.get_concept_part(expression)
|
|
context = self.get_context()
|
|
|
|
resolved_expected = [PossibleVariable(e) for e in expected]
|
|
assert DefConceptEvaluator.get_variables(context, ret_val, name.split()) == resolved_expected
|
|
|
|
def test_i_can_get_variables_when_keywords(self):
|
|
sheerka, context = self.init_concepts()
|
|
|
|
def_concept = self.get_def_concept("condition pre").value.value
|
|
name_to_use = DefConceptEvaluator.get_name_to_use(def_concept)
|
|
concept_part = self.get_concept_part("pre")
|
|
|
|
assert DefConceptEvaluator.get_variables(context, concept_part, name_to_use) == [PossibleVariable("pre")]
|
|
|
|
def test_i_can_get_variable_when_rule_is_defined(self):
|
|
sheerka, context = self.init_test().unpack()
|
|
def_concept_parser = DefConceptParser()
|
|
|
|
parsed_ret_val = def_concept_parser.parse(context, ParserInput("def concept rule x as r:|x:"))
|
|
assert parsed_ret_val.status # sanity check
|
|
|
|
evaluated = DefConceptEvaluator().eval(context, parsed_ret_val)
|
|
assert evaluated.status
|
|
assert evaluated.body.body.key == "rule __var__0"
|
|
assert evaluated.body.body.get_metadata().variables == [("x", None)]
|
|
|
|
def test_i_cannot_get_variables_from_python_node_when_name_has_only_one_token(self):
|
|
ret_val = self.get_concept_part("isinstance(a, str)")
|
|
context = self.get_context()
|
|
|
|
assert DefConceptEvaluator.get_variables(context, ret_val, ["a"]) == []
|
|
|
|
def test_i_can_get_variables_from_definition(self):
|
|
parsing_expression = Sequence(ConceptExpression('mult'),
|
|
ZeroOrMore(Sequence(StrMatch("+"), ConceptExpression("add"))))
|
|
ret_val = self.get_return_value("mult (('+'|'-') add)?", parsing_expression)
|
|
|
|
expected = [CertainVariable("mult"), CertainVariable("add")]
|
|
assert DefConceptEvaluator.get_variables(self.get_context(), ret_val, []) == expected
|
|
|
|
def test_i_can_get_variable_from_bnf_definition_2(self):
|
|
sheerka, context, one, two = self.init_concepts("one", "two")
|
|
|
|
text = "def concept twenties from bnf 'twenty' (one|two)=unit as 20 + unit"
|
|
def_ret_val = DefConceptParser().parse(context, ParserInput(text))
|
|
concept_definition = def_ret_val.value.body.definition
|
|
|
|
expected = [CertainVariable("unit"), CertainVariable("one"), CertainVariable("two")]
|
|
assert DefConceptEvaluator.get_variables(context, concept_definition, []) == expected
|
|
|
|
def test_i_can_recognize_variables_when_referencing_other_concepts(self):
|
|
sheerka, context, isa_concept = self.init_concepts(
|
|
Concept("x is an y", pre="is_question()").def_var("x").def_var("y")
|
|
)
|
|
|
|
text = "def concept what x is y pre is_question() where x is an adjective as get_attr(x, y)"
|
|
def_ret_val = DefConceptParser().parse(context, ParserInput(text))
|
|
evaluated = DefConceptEvaluator().eval(context, def_ret_val)
|
|
new_concept = evaluated.body.body
|
|
|
|
assert evaluated.status
|
|
assert new_concept.get_metadata().variables == [('x', None), ('y', None)]
|
|
|
|
def test_i_can_recognize_variables_when_given_in_concept_definition(self):
|
|
sheerka, context = self.init_test().unpack()
|
|
|
|
text = "def concept a plus b as a + b def_var plus"
|
|
def_ret_val = DefConceptParser().parse(context, ParserInput(text))
|
|
evaluated = DefConceptEvaluator().eval(context, def_ret_val)
|
|
new_concept = evaluated.body.body
|
|
|
|
assert evaluated.status
|
|
assert new_concept.get_metadata().variables == [('plus', None)]
|
|
|
|
def test_i_can_recognize_variables_when_referencing_other_concepts_with_variable_mapping(self):
|
|
sheerka, context, number, isa, add = self.init_concepts(
|
|
"number",
|
|
Concept("u is a v").def_var("u").def_var("v"),
|
|
Concept("add a b").def_var("a").def_var("b")
|
|
)
|
|
|
|
text = "def concept x plus y where x is a number as add x y"
|
|
def_ret_val = DefConceptParser().parse(context, ParserInput(text))
|
|
evaluated = DefConceptEvaluator().eval(context, def_ret_val)
|
|
new_concept = evaluated.body.body
|
|
|
|
assert evaluated.status
|
|
assert new_concept.get_metadata().variables == [('x', None), ('y', None)]
|
|
|
|
def test_i_do_no_mixed_up_concept_and_variable_name(self):
|
|
sheerka, context, activate_debug = self.init_concepts(Concept("activate debug"))
|
|
|
|
text = "def concept debug on as activate debug"
|
|
def_ret_val = DefConceptParser().parse(context, ParserInput(text))
|
|
evaluated = DefConceptEvaluator().eval(context, def_ret_val)
|
|
new_concept = evaluated.body.body
|
|
|
|
assert evaluated.status
|
|
assert new_concept.get_metadata().variables == []
|
|
|
|
def test_other_concepts_are_not_variables(self):
|
|
sheerka, context, *concepts = self.init_test().with_concepts("little", "size", create_new=True).unpack()
|
|
|
|
def_concept_node = self.get_def_concept_node_from_name_only("little x")
|
|
name_to_use = DefConceptEvaluator.get_name_to_use(def_concept_node)
|
|
concept_part = self.get_concept_part("set_attr(x, size, little)")
|
|
|
|
assert DefConceptEvaluator.get_variables(context, concept_part, name_to_use) == [PossibleVariable("x")]
|
|
|
|
def test_that_the_new_concept_is_correctly_saved_in_db(self):
|
|
context = self.get_context()
|
|
def_concept_return_value = self.get_def_concept(
|
|
name="hello a",
|
|
bnf_def=self.get_return_value("hello a", Sequence(StrMatch("hello"), StrMatch("a"))),
|
|
where="isinstance(a, str )",
|
|
pre="a is not None",
|
|
body="print('hello' + a)")
|
|
|
|
# sanity. Make sure that the concept does not already exist
|
|
from_db = context.sheerka.get_by_key("hello " + VARIABLE_PREFIX + "0")
|
|
assert context.sheerka.isinstance(from_db, BuiltinConcepts.UNKNOWN_CONCEPT)
|
|
|
|
DefConceptEvaluator().eval(context, def_concept_return_value)
|
|
context.sheerka.concepts_cache = {} # reset cache
|
|
from_db = context.sheerka.get_by_key("hello " + VARIABLE_PREFIX + "0")
|
|
|
|
assert from_db.get_metadata().key == f"hello {VARIABLE_PREFIX}0"
|
|
assert from_db.get_metadata().name == "hello a"
|
|
assert from_db.get_metadata().where == "isinstance(a, str )"
|
|
assert from_db.get_metadata().pre == "a is not None"
|
|
assert from_db.get_metadata().post is None
|
|
assert from_db.get_metadata().body == "print('hello' + a)"
|
|
assert from_db.get_metadata().definition == "hello a"
|
|
assert from_db.get_metadata().definition_type == "bnf"
|
|
assert len(from_db.get_metadata().variables) == 1
|
|
assert from_db.get_metadata().variables[0] == ("a", None)
|
|
assert "a" in from_db.values()
|
|
|
|
assert from_db.get_compiled() == {} # ast is not saved in db
|
|
|
|
def test_concept_that_references_itself_is_correctly_created(self):
|
|
context = self.get_context()
|
|
def_concept_as_return_value = self.get_def_concept("foo", body="foo")
|
|
|
|
ret_val = DefConceptEvaluator().eval(context, def_concept_as_return_value)
|
|
|
|
assert ret_val.status
|
|
new_concept = ret_val.body.body
|
|
assert new_concept.name == 'foo'
|
|
assert new_concept.get_metadata().body == 'foo'
|
|
assert new_concept.values() == {}
|
|
assert new_concept.get_metadata().variables == []
|
|
|
|
def test_i_can_recognize_variable_when_keyword_argument(self):
|
|
sheerka, context = self.init_test().unpack()
|
|
def_concept_parser = DefConceptParser()
|
|
|
|
parsed_ret_val = def_concept_parser.parse(context, ParserInput("def concept foo1 x as func(param=x)"))
|
|
assert parsed_ret_val.status # sanity check
|
|
|
|
evaluated = DefConceptEvaluator().eval(context, parsed_ret_val)
|
|
assert evaluated.status
|
|
assert evaluated.body.body.key == "foo1 __var__0"
|
|
assert evaluated.body.body.get_metadata().variables == [("x", None)]
|
|
|
|
parsed_ret_val = def_concept_parser.parse(context, ParserInput("def concept foo2 x as func(x=value)"))
|
|
assert parsed_ret_val.status # sanity check
|
|
|
|
evaluated = DefConceptEvaluator().eval(context, parsed_ret_val)
|
|
assert evaluated.status
|
|
assert evaluated.body.body.key == "foo2 __var__0"
|
|
assert evaluated.body.body.get_metadata().variables == [("x", None)]
|
|
|
|
def test_i_can_eval_when_bnf_concept_with_regex(self):
|
|
context = self.get_context()
|
|
def_ret_val = DefConceptParser().parse(context, ParserInput("def concept hello a from bnf r'[a-z]+'=a 'hello'"))
|
|
evaluated = DefConceptEvaluator().eval(context, def_ret_val)
|
|
|
|
assert evaluated.status
|
|
assert context.sheerka.isinstance(evaluated.body, BuiltinConcepts.NEW_CONCEPT)
|
|
|
|
created_concept = evaluated.body.body
|
|
assert created_concept.get_metadata().name == "hello a"
|
|
assert created_concept.get_metadata().key == "hello __var__0"
|
|
assert created_concept.get_metadata().definition == "r'[a-z]+'=a 'hello'"
|
|
assert created_concept.get_metadata().definition_type == "bnf"
|
|
|
|
def test_i_can_eval_when_bnf_concept_with_variable(self):
|
|
context = self.get_context()
|
|
def_ret_val = DefConceptParser().parse(context, ParserInput("def concept hello x from bnf 'hello' x"))
|
|
evaluated = DefConceptEvaluator().eval(context, def_ret_val)
|
|
|
|
assert evaluated.status
|
|
assert context.sheerka.isinstance(evaluated.body, BuiltinConcepts.NEW_CONCEPT)
|
|
|
|
created_concept = evaluated.body.body
|
|
assert created_concept.get_metadata().name == "hello x"
|
|
assert created_concept.get_metadata().key == "hello __var__0"
|
|
assert created_concept.get_metadata().definition == "'hello' x"
|
|
assert created_concept.get_metadata().definition_type == "bnf"
|
|
assert created_concept.get_metadata().variables == [("x", None)]
|
|
assert created_concept._bnf == Sequence(StrMatch("hello"), VariableExpression("x"))
|
|
|
|
def test_i_can_eval_when_auto_eval_is_true(self):
|
|
sheerka, context = self.init_test().unpack()
|
|
def_ret_val = DefConceptParser().parse(context, ParserInput("def concept foo auto_eval True"))
|
|
evaluated = DefConceptEvaluator().eval(context, def_ret_val)
|
|
|
|
assert evaluated.status
|
|
assert context.sheerka.isinstance(evaluated.body, BuiltinConcepts.NEW_CONCEPT)
|
|
created_concept = evaluated.body.body
|
|
|
|
assert sheerka.get_property(created_concept, BuiltinConcepts.ISA) == {sheerka.new(BuiltinConcepts.AUTO_EVAL)}
|
|
|
|
def test_i_cannot_eval_bnf_concept_with_unknown_variable(self):
|
|
# testing MandatoryVariable
|
|
context = self.get_context()
|
|
def_ret_val = DefConceptParser().parse(context, ParserInput("def concept name from bnf unknown foo"))
|
|
evaluated = DefConceptEvaluator().eval(context, def_ret_val)
|
|
|
|
assert not evaluated.status
|
|
assert context.sheerka.isinstance(evaluated.body, BuiltinConcepts.ERROR)
|
|
unknown_concepts = [
|
|
context.sheerka.new(BuiltinConcepts.UNKNOWN_CONCEPT, body={"name": "foo"}),
|
|
context.sheerka.new(BuiltinConcepts.UNKNOWN_CONCEPT, body={"name": "unknown"}),
|
|
]
|
|
assert evaluated.body.body == unknown_concepts
|
|
|
|
def test_i_cannot_eval_bnf_concept_with_only_variable(self):
|
|
sheerka, context = self.init_test().unpack()
|
|
def_ret_val = DefConceptParser().parse(context, ParserInput("def concept foo x from bnf x"))
|
|
evaluated = DefConceptEvaluator().eval(context, def_ret_val)
|
|
|
|
assert not evaluated.status
|
|
assert sheerka.isinstance(evaluated.body, BuiltinConcepts.ERROR)
|
|
assert isinstance(evaluated.body.body, NoFirstTokenError)
|