diff --git a/src/core/sheerka/Sheerka.py b/src/core/sheerka/Sheerka.py index 6458ff2..e45d7e6 100644 --- a/src/core/sheerka/Sheerka.py +++ b/src/core/sheerka/Sheerka.py @@ -691,6 +691,9 @@ class Sheerka(Concept): return None return new_instances(concept) if return_new else concept + def fast_get_by_name(self, name): + return self.cache_manager.get(self.CONCEPTS_BY_NAME_ENTRY, name) + def has_id(self, concept_id): """ Returns True if a concept with this id exists in cache diff --git a/src/core/sheerka/services/SheerkaCreateNewConcept.py b/src/core/sheerka/services/SheerkaCreateNewConcept.py index 2154943..e3eb1b3 100644 --- a/src/core/sheerka/services/SheerkaCreateNewConcept.py +++ b/src/core/sheerka/services/SheerkaCreateNewConcept.py @@ -21,6 +21,7 @@ class SheerkaCreateNewConcept(BaseService): def initialize(self): self.sheerka.bind_service_method(self.create_new_concept, True) + self.sheerka.bind_service_method(self.not_is_variable, False, visible=False) def create_new_concept(self, context, concept: Concept): """ @@ -108,3 +109,11 @@ class SheerkaCreateNewConcept(BaseService): refs.add(concept.id) return refs + + def not_is_variable(self, name): + """ + Given a name tells if it refers to a variable name + :param name: + :return: + """ + return not self.sheerka.cache_manager.get(self.sheerka.CONCEPTS_BY_NAME_ENTRY, name) diff --git a/src/evaluators/DefConceptEvaluator.py b/src/evaluators/DefConceptEvaluator.py index 967311f..233a222 100644 --- a/src/evaluators/DefConceptEvaluator.py +++ b/src/evaluators/DefConceptEvaluator.py @@ -8,7 +8,7 @@ from evaluators.BaseEvaluator import OneReturnValueEvaluator from parsers.BaseParser import NotInitializedNode from parsers.BnfNodeParser import ParsingExpression, ParsingExpressionVisitor from parsers.DefConceptParser import DefConceptNode, NameNode -from parsers.PythonParser import PythonNode +from parsers.PythonParser import PythonNode, get_python_node class ConceptOrRuleNameVisitor(ParsingExpressionVisitor): @@ -129,15 +129,13 @@ class DefConceptEvaluator(OneReturnValueEvaluator): This function can only be a draft, as there may be tons of different situations I guess that it can only be complete when will we have access to Sheerka memory """ - # # Case of NameNode # if isinstance(ret_value, NameNode): names = [str(t.value) for t in ret_value.tokens if t.type in ( TokenKind.IDENTIFIER, TokenKind.STRING, TokenKind.KEYWORD)] - variables = filter(lambda x: x in concept_name, names) - return set(variables) + return set(filter(lambda x: x in concept_name and context.sheerka.not_is_variable(x), names)) # # case of BNF @@ -150,13 +148,13 @@ class DefConceptEvaluator(OneReturnValueEvaluator): # # Case of python code # - if isinstance(ret_value.value, ParserResultConcept) and isinstance(ret_value.value.value, PythonNode): + if (python_node := get_python_node(ret_value.value.value)) is not None: if len(concept_name) > 1: - python_node = ret_value.value.value visitor = UnreferencedVariablesVisitor(context) names = visitor.get_names(python_node.ast_) - variables = filter(lambda x: x in concept_name, names) - return set(variables) + return set(filter(lambda x: x in concept_name and context.sheerka.not_is_variable(x), names)) + else: + return set() # # Concept @@ -172,4 +170,4 @@ class DefConceptEvaluator(OneReturnValueEvaluator): variables.add(identifier) return variables - return [] + return set() diff --git a/src/evaluators/PythonEvaluator.py b/src/evaluators/PythonEvaluator.py index ce7c048..0671f51 100644 --- a/src/evaluators/PythonEvaluator.py +++ b/src/evaluators/PythonEvaluator.py @@ -12,7 +12,7 @@ from core.rule import Rule from core.sheerka.ExecutionContext import ExecutionContext from core.tokenizer import Token, TokenKind from evaluators.BaseEvaluator import OneReturnValueEvaluator -from parsers.PythonParser import PythonNode +from parsers.PythonParser import PythonNode, get_python_node TO_DISABLED = ["breakpoint", "callable", "compile", "delattr", "eval", "exec", "exit", "input", "locals", "open", "print", "quit", "setattr"] @@ -88,8 +88,7 @@ class PythonEvaluator(OneReturnValueEvaluator): def eval(self, context, return_value): sheerka = context.sheerka - node = return_value.value.value if isinstance(return_value.value.value, PythonNode) else \ - return_value.value.value.python_node + node = get_python_node(return_value.value.value) debugger = context.get_debugger(PythonEvaluator.NAME, "eval") debugger.debug_entering(node=node) diff --git a/src/parsers/BaseNodeParser.py b/src/parsers/BaseNodeParser.py index 331a262..ce3c0a4 100644 --- a/src/parsers/BaseNodeParser.py +++ b/src/parsers/BaseNodeParser.py @@ -18,6 +18,12 @@ class ChickenAndEggError(Exception): concepts: Set[str] +@dataclass +class NoFirstTokenError(ErrorNode): + concept: Concept + key: str + + @dataclass() class LexerNode(Node): start: int # starting index in the tokens list @@ -891,7 +897,7 @@ class BaseNodeParser(BaseParser): if keywords is None: # no first token found for a concept ? - return sheerka.ret(sheerka.name, False, concept) + return sheerka.ret(sheerka.name, False, NoFirstTokenError(concept, concept.key)) for keyword in keywords: res.setdefault(keyword, []).append(concept.id) diff --git a/src/parsers/PythonParser.py b/src/parsers/PythonParser.py index 07c57bd..4b14f41 100644 --- a/src/parsers/PythonParser.py +++ b/src/parsers/PythonParser.py @@ -11,6 +11,14 @@ from parsers.BaseParser import BaseParser, Node, ErrorNode log = logging.getLogger(__name__) +def get_python_node(obj): + if isinstance(obj, PythonNode): + return obj + if hasattr(obj, "python_node"): + return obj.python_node + return None + + @dataclass() class PythonErrorNode(ErrorNode): source: str diff --git a/tests/evaluators/test_DefConceptEvaluator.py b/tests/evaluators/test_DefConceptEvaluator.py index df12641..1dbd3c9 100644 --- a/tests/evaluators/test_DefConceptEvaluator.py +++ b/tests/evaluators/test_DefConceptEvaluator.py @@ -6,8 +6,8 @@ from core.concept import VARIABLE_PREFIX, Concept, DEFINITION_TYPE_BNF, DEFINITI from core.tokenizer import Tokenizer from evaluators.DefConceptEvaluator import DefConceptEvaluator from parsers.BaseParser import BaseParser -from parsers.BnfNodeParser import Sequence, StrMatch, ZeroOrMore, ConceptExpression from parsers.BnfDefinitionParser import BnfDefinitionParser +from parsers.BnfNodeParser import Sequence, StrMatch, ZeroOrMore, ConceptExpression from parsers.DefConceptParser import DefConceptNode, NameNode from parsers.PythonParser import PythonNode, PythonParser @@ -76,6 +76,10 @@ class TestDefConceptEvaluator(TestUsingMemoryBasedSheerka): return ReturnValueConcept(BaseParser.PREFIX + "some_name", True, ParserResultConcept(value=def_concept)) + @staticmethod + def get_def_concept_node_from_name_only(name): + return DefConceptNode([], name=NameNode(list(Tokenizer(name)))) + @pytest.mark.parametrize("ret_val, expected", [ (ReturnValueConcept(BaseParser.PREFIX + "some_name", True, ParserResultConcept(value=DefConceptNode([]))), True), @@ -88,7 +92,7 @@ class TestDefConceptEvaluator(TestUsingMemoryBasedSheerka): context = self.get_context() assert DefConceptEvaluator().matches(context, ret_val) == expected - def test_that_the_source_is_correctly_set_for_bnf_concept(self): + def test_that_the_source_is_correctly_set_for_bnf_concepts(self): context = self.get_context() def_concept_return_value = self.get_def_concept( name="hello a", @@ -114,22 +118,7 @@ class TestDefConceptEvaluator(TestUsingMemoryBasedSheerka): assert created_concept.get_metadata().definition == "hello a" assert created_concept.get_metadata().definition_type == "bnf" - def test_i_can_add_concept_with_the_correct_variables_when_referencing_other_concepts(self): - context = self.get_context() - def_concept_return_value = self.get_def_concept( - name="x plus y", - where=self.pretval(Concept("u is a v").def_var("u").def_var("v"), source="x is a number"), - body=self.pretval(Concept("add a b").def_var("a").def_var("b"), source="add x y"), ) - - evaluated = DefConceptEvaluator().eval(context, def_concept_return_value) - - assert evaluated.status - assert context.sheerka.isinstance(evaluated.body, BuiltinConcepts.NEW_CONCEPT) - - created_concept = evaluated.body.body - assert created_concept.get_metadata().variables == [("x", None), ("y", None)] - - def test_that_the_source_is_correctly_set_for_concept_with_simple_definition(self): + def test_that_the_source_is_correctly_set_for_concepts_with_simple_definition(self): context = self.get_context() def_concept_return_value = self.get_def_concept( name="greetings", @@ -153,6 +142,30 @@ class TestDefConceptEvaluator(TestUsingMemoryBasedSheerka): assert created_concept.get_metadata().definition == "hello a" assert created_concept.get_metadata().definition_type == "def" + def test_i_can_add_concept_with_the_correct_variables_when_referencing_other_concepts(self): + context = self.get_context() + def_concept_return_value = self.get_def_concept( + name="x plus y", + where=self.pretval(Concept("u is a v").def_var("u").def_var("v"), source="x is a number"), + body=self.pretval(Concept("add a b").def_var("a").def_var("b"), source="add x y"), ) + + evaluated = DefConceptEvaluator().eval(context, def_concept_return_value) + + assert evaluated.status + assert context.sheerka.isinstance(evaluated.body, BuiltinConcepts.NEW_CONCEPT) + + created_concept = evaluated.body.body + assert created_concept.get_metadata().variables == [("x", None), ("y", None)] + + def test_other_concepts_are_not_variables(self): + sheerka, context, *concepts = self.init_concepts("little", "size", create_new=True) + + def_concept_node = self.get_def_concept_node_from_name_only("little x") + name_to_use = DefConceptEvaluator.get_name_to_use(def_concept_node) + concept_part = self.get_concept_part("set_attr(x, size, little)") + + assert DefConceptEvaluator.get_variables(context, concept_part, name_to_use) == {"x"} + def test_that_the_new_concept_is_correctly_saved_in_db(self): context = self.get_context() def_concept_return_value = self.get_def_concept( @@ -195,7 +208,7 @@ class TestDefConceptEvaluator(TestUsingMemoryBasedSheerka): ret_val = self.get_concept_part(expression) context = self.get_context() - assert DefConceptEvaluator.get_variables(context.sheerka, ret_val, name.split()) == expected + assert DefConceptEvaluator.get_variables(context, ret_val, name.split()) == expected def test_i_can_get_variables_when_keywords(self): sheerka, context = self.init_concepts() @@ -204,20 +217,20 @@ class TestDefConceptEvaluator(TestUsingMemoryBasedSheerka): name_to_use = DefConceptEvaluator.get_name_to_use(def_concept) concept_part = self.get_concept_part("pre") - assert DefConceptEvaluator.get_variables(context.sheerka, concept_part, name_to_use) == {"pre"} + assert DefConceptEvaluator.get_variables(context, concept_part, name_to_use) == {"pre"} def test_i_cannot_get_variables_from_python_node_when_name_has_only_one_token(self): ret_val = self.get_concept_part("isinstance(a, str)") context = self.get_context() - assert DefConceptEvaluator.get_variables(context.sheerka, ret_val, ["a"]) == [] + assert DefConceptEvaluator.get_variables(context, ret_val, ["a"]) == set() def test_i_can_get_variables_from_definition(self): parsing_expression = Sequence(ConceptExpression('mult'), ZeroOrMore(Sequence(StrMatch("+"), ConceptExpression("add")))) ret_val = self.get_return_value("mult (('+'|'-') add)?", parsing_expression) - assert DefConceptEvaluator.get_variables(self.get_sheerka(), ret_val, []) == {"add", "mult"} + assert DefConceptEvaluator.get_variables(self.get_context(), ret_val, []) == {"add", "mult"} def test_concept_that_references_itself_is_correctly_created(self): context = self.get_context() diff --git a/tests/parsers/test_BaseNodeParser.py b/tests/parsers/test_BaseNodeParser.py index 0e3609d..16309bd 100644 --- a/tests/parsers/test_BaseNodeParser.py +++ b/tests/parsers/test_BaseNodeParser.py @@ -1,6 +1,6 @@ import pytest from core.concept import Concept -from parsers.BaseNodeParser import BaseNodeParser +from parsers.BaseNodeParser import BaseNodeParser, NoFirstTokenError from parsers.BnfNodeParser import StrMatch, Sequence, OrderedChoice, Optional, ZeroOrMore, OneOrMore, ConceptExpression from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka @@ -115,6 +115,13 @@ class TestBaseNodeParser(TestUsingMemoryBasedSheerka): "qux": ["1005"], } + def test_i_cannot_get_concept_by_first_keyword_when_no_first_keyword(self): + sheerka, context, foo = self.init_concepts(Concept("x y", body="x y").def_var("x").def_var("y")) + res = BaseNodeParser.get_concepts_by_first_token(context, [foo]) + + assert not res.status + assert res.body == NoFirstTokenError(foo, foo.key) + def test_i_can_resolve_concepts_by_first_keyword(self): sheerka, context, *updated = self.init_concepts( "one",