kodjo/Sheerka-Old
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fixed variable recognition when it is a concept

Browse Source
This commit is contained in:
Kodjo Sossouvi
2020-11-20 17:24:52 +01:00
parent 315f8ea09b
commit 0e945fe0fd
8 changed files with 79 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/core/sheerka/Sheerka.py
+3
View File
@@ -691,6 +691,9 @@ class Sheerka(Concept):
return None
return new_instances(concept) if return_new else concept
def fast_get_by_name(self, name):
return self.cache_manager.get(self.CONCEPTS_BY_NAME_ENTRY, name)
def has_id(self, concept_id):
"""
Returns True if a concept with this id exists in cache
src/core/sheerka/services/SheerkaCreateNewConcept.py
+9
View File
@@ -21,6 +21,7 @@ class SheerkaCreateNewConcept(BaseService):
def initialize(self):
self.sheerka.bind_service_method(self.create_new_concept, True)
self.sheerka.bind_service_method(self.not_is_variable, False, visible=False)
def create_new_concept(self, context, concept: Concept):
"""
@@ -108,3 +109,11 @@ class SheerkaCreateNewConcept(BaseService):
refs.add(concept.id)
return refs
def not_is_variable(self, name):
"""
Given a name tells if it refers to a variable name
:param name:
:return:
"""
return not self.sheerka.cache_manager.get(self.sheerka.CONCEPTS_BY_NAME_ENTRY, name)
src/evaluators/DefConceptEvaluator.py
+7 -9
View File
@@ -8,7 +8,7 @@ from evaluators.BaseEvaluator import OneReturnValueEvaluator
from parsers.BaseParser import NotInitializedNode
from parsers.BnfNodeParser import ParsingExpression, ParsingExpressionVisitor
from parsers.DefConceptParser import DefConceptNode, NameNode
from parsers.PythonParser import PythonNode
from parsers.PythonParser import PythonNode, get_python_node
class ConceptOrRuleNameVisitor(ParsingExpressionVisitor):
@@ -129,15 +129,13 @@ class DefConceptEvaluator(OneReturnValueEvaluator):
This function can only be a draft, as there may be tons of different situations
I guess that it can only be complete when will we have access to Sheerka memory
"""
#
# Case of NameNode
#
if isinstance(ret_value, NameNode):
names = [str(t.value) for t in ret_value.tokens if t.type in (
TokenKind.IDENTIFIER, TokenKind.STRING, TokenKind.KEYWORD)]
variables = filter(lambda x: x in concept_name, names)
return set(variables)
return set(filter(lambda x: x in concept_name and context.sheerka.not_is_variable(x), names))
#
# case of BNF
@@ -150,13 +148,13 @@ class DefConceptEvaluator(OneReturnValueEvaluator):
#
# Case of python code
#
if isinstance(ret_value.value, ParserResultConcept) and isinstance(ret_value.value.value, PythonNode):
if (python_node := get_python_node(ret_value.value.value)) is not None:
if len(concept_name) > 1:
python_node = ret_value.value.value
visitor = UnreferencedVariablesVisitor(context)
names = visitor.get_names(python_node.ast_)
variables = filter(lambda x: x in concept_name, names)
return set(variables)
return set(filter(lambda x: x in concept_name and context.sheerka.not_is_variable(x), names))
else:
return set()
#
# Concept
@@ -172,4 +170,4 @@ class DefConceptEvaluator(OneReturnValueEvaluator):
variables.add(identifier)
return variables
return []
return set()
src/evaluators/PythonEvaluator.py
+2 -3
View File
@@ -12,7 +12,7 @@ from core.rule import Rule
from core.sheerka.ExecutionContext import ExecutionContext
from core.tokenizer import Token, TokenKind
from evaluators.BaseEvaluator import OneReturnValueEvaluator
from parsers.PythonParser import PythonNode
from parsers.PythonParser import PythonNode, get_python_node
TO_DISABLED = ["breakpoint", "callable", "compile", "delattr", "eval", "exec", "exit", "input", "locals", "open",
"print", "quit", "setattr"]
@@ -88,8 +88,7 @@ class PythonEvaluator(OneReturnValueEvaluator):
def eval(self, context, return_value):
sheerka = context.sheerka
node = return_value.value.value if isinstance(return_value.value.value, PythonNode) else \
return_value.value.value.python_node
node = get_python_node(return_value.value.value)
debugger = context.get_debugger(PythonEvaluator.NAME, "eval")
debugger.debug_entering(node=node)
src/parsers/BaseNodeParser.py
+7 -1
View File
@@ -18,6 +18,12 @@ class ChickenAndEggError(Exception):
concepts: Set[str]
@dataclass
class NoFirstTokenError(ErrorNode):
concept: Concept
key: str
@dataclass()
class LexerNode(Node):
start: int # starting index in the tokens list
@@ -891,7 +897,7 @@ class BaseNodeParser(BaseParser):
if keywords is None:
# no first token found for a concept ?
return sheerka.ret(sheerka.name, False, concept)
return sheerka.ret(sheerka.name, False, NoFirstTokenError(concept, concept.key))
for keyword in keywords:
res.setdefault(keyword, []).append(concept.id)
src/parsers/PythonParser.py
+8
View File
@@ -11,6 +11,14 @@ from parsers.BaseParser import BaseParser, Node, ErrorNode
log = logging.getLogger(__name__)
def get_python_node(obj):
if isinstance(obj, PythonNode):
return obj
if hasattr(obj, "python_node"):
return obj.python_node
return None
@dataclass()
class PythonErrorNode(ErrorNode):
source: str
tests/evaluators/test_DefConceptEvaluator.py
+35 -22
View File
@@ -6,8 +6,8 @@ from core.concept import VARIABLE_PREFIX, Concept, DEFINITION_TYPE_BNF, DEFINITI
from core.tokenizer import Tokenizer
from evaluators.DefConceptEvaluator import DefConceptEvaluator
from parsers.BaseParser import BaseParser
from parsers.BnfNodeParser import Sequence, StrMatch, ZeroOrMore, ConceptExpression
from parsers.BnfDefinitionParser import BnfDefinitionParser
from parsers.BnfNodeParser import Sequence, StrMatch, ZeroOrMore, ConceptExpression
from parsers.DefConceptParser import DefConceptNode, NameNode
from parsers.PythonParser import PythonNode, PythonParser
@@ -76,6 +76,10 @@ class TestDefConceptEvaluator(TestUsingMemoryBasedSheerka):
return ReturnValueConcept(BaseParser.PREFIX + "some_name", True, ParserResultConcept(value=def_concept))
@staticmethod
def get_def_concept_node_from_name_only(name):
return DefConceptNode([], name=NameNode(list(Tokenizer(name))))
@pytest.mark.parametrize("ret_val, expected", [
(ReturnValueConcept(BaseParser.PREFIX + "some_name", True, ParserResultConcept(value=DefConceptNode([]))),
True),
@@ -88,7 +92,7 @@ class TestDefConceptEvaluator(TestUsingMemoryBasedSheerka):
context = self.get_context()
assert DefConceptEvaluator().matches(context, ret_val) == expected
def test_that_the_source_is_correctly_set_for_bnf_concept(self):
def test_that_the_source_is_correctly_set_for_bnf_concepts(self):
context = self.get_context()
def_concept_return_value = self.get_def_concept(
name="hello a",
@@ -114,22 +118,7 @@ class TestDefConceptEvaluator(TestUsingMemoryBasedSheerka):
assert created_concept.get_metadata().definition == "hello a"
assert created_concept.get_metadata().definition_type == "bnf"
def test_i_can_add_concept_with_the_correct_variables_when_referencing_other_concepts(self):
context = self.get_context()
def_concept_return_value = self.get_def_concept(
name="x plus y",
where=self.pretval(Concept("u is a v").def_var("u").def_var("v"), source="x is a number"),
body=self.pretval(Concept("add a b").def_var("a").def_var("b"), source="add x y"), )
evaluated = DefConceptEvaluator().eval(context, def_concept_return_value)
assert evaluated.status
assert context.sheerka.isinstance(evaluated.body, BuiltinConcepts.NEW_CONCEPT)
created_concept = evaluated.body.body
assert created_concept.get_metadata().variables == [("x", None), ("y", None)]
def test_that_the_source_is_correctly_set_for_concept_with_simple_definition(self):
def test_that_the_source_is_correctly_set_for_concepts_with_simple_definition(self):
context = self.get_context()
def_concept_return_value = self.get_def_concept(
name="greetings",
@@ -153,6 +142,30 @@ class TestDefConceptEvaluator(TestUsingMemoryBasedSheerka):
assert created_concept.get_metadata().definition == "hello a"
assert created_concept.get_metadata().definition_type == "def"
def test_i_can_add_concept_with_the_correct_variables_when_referencing_other_concepts(self):
context = self.get_context()
def_concept_return_value = self.get_def_concept(
name="x plus y",
where=self.pretval(Concept("u is a v").def_var("u").def_var("v"), source="x is a number"),
body=self.pretval(Concept("add a b").def_var("a").def_var("b"), source="add x y"), )
evaluated = DefConceptEvaluator().eval(context, def_concept_return_value)
assert evaluated.status
assert context.sheerka.isinstance(evaluated.body, BuiltinConcepts.NEW_CONCEPT)
created_concept = evaluated.body.body
assert created_concept.get_metadata().variables == [("x", None), ("y", None)]
def test_other_concepts_are_not_variables(self):
sheerka, context, *concepts = self.init_concepts("little", "size", create_new=True)
def_concept_node = self.get_def_concept_node_from_name_only("little x")
name_to_use = DefConceptEvaluator.get_name_to_use(def_concept_node)
concept_part = self.get_concept_part("set_attr(x, size, little)")
assert DefConceptEvaluator.get_variables(context, concept_part, name_to_use) == {"x"}
def test_that_the_new_concept_is_correctly_saved_in_db(self):
context = self.get_context()
def_concept_return_value = self.get_def_concept(
@@ -195,7 +208,7 @@ class TestDefConceptEvaluator(TestUsingMemoryBasedSheerka):
ret_val = self.get_concept_part(expression)
context = self.get_context()
assert DefConceptEvaluator.get_variables(context.sheerka, ret_val, name.split()) == expected
assert DefConceptEvaluator.get_variables(context, ret_val, name.split()) == expected
def test_i_can_get_variables_when_keywords(self):
sheerka, context = self.init_concepts()
@@ -204,20 +217,20 @@ class TestDefConceptEvaluator(TestUsingMemoryBasedSheerka):
name_to_use = DefConceptEvaluator.get_name_to_use(def_concept)
concept_part = self.get_concept_part("pre")
assert DefConceptEvaluator.get_variables(context.sheerka, concept_part, name_to_use) == {"pre"}
assert DefConceptEvaluator.get_variables(context, concept_part, name_to_use) == {"pre"}
def test_i_cannot_get_variables_from_python_node_when_name_has_only_one_token(self):
ret_val = self.get_concept_part("isinstance(a, str)")
context = self.get_context()
assert DefConceptEvaluator.get_variables(context.sheerka, ret_val, ["a"]) == []
assert DefConceptEvaluator.get_variables(context, ret_val, ["a"]) == set()
def test_i_can_get_variables_from_definition(self):
parsing_expression = Sequence(ConceptExpression('mult'),
ZeroOrMore(Sequence(StrMatch("+"), ConceptExpression("add"))))
ret_val = self.get_return_value("mult (('+'|'-') add)?", parsing_expression)
assert DefConceptEvaluator.get_variables(self.get_sheerka(), ret_val, []) == {"add", "mult"}
assert DefConceptEvaluator.get_variables(self.get_context(), ret_val, []) == {"add", "mult"}
def test_concept_that_references_itself_is_correctly_created(self):
context = self.get_context()
tests/parsers/test_BaseNodeParser.py
+8 -1
View File
@@ -1,6 +1,6 @@
import pytest
from core.concept import Concept
from parsers.BaseNodeParser import BaseNodeParser
from parsers.BaseNodeParser import BaseNodeParser, NoFirstTokenError
from parsers.BnfNodeParser import StrMatch, Sequence, OrderedChoice, Optional, ZeroOrMore, OneOrMore, ConceptExpression
from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
@@ -115,6 +115,13 @@ class TestBaseNodeParser(TestUsingMemoryBasedSheerka):
"qux": ["1005"],
}
def test_i_cannot_get_concept_by_first_keyword_when_no_first_keyword(self):
sheerka, context, foo = self.init_concepts(Concept("x y", body="x y").def_var("x").def_var("y"))
res = BaseNodeParser.get_concepts_by_first_token(context, [foo])
assert not res.status
assert res.body == NoFirstTokenError(foo, foo.key)
def test_i_can_resolve_concepts_by_first_keyword(self):
sheerka, context, *updated = self.init_concepts(
"one",