From a2bbd2eec20d6fef63fb9eba807d94af23ef0930 Mon Sep 17 00:00:00 2001 From: Kodjo Sossouvi Date: Tue, 10 Mar 2020 15:05:03 +0100 Subject: [PATCH] I can also get concept by name --- src/core/concept.py | 7 +- .../Services/SheerkaCreateNewConcept.py | 21 +++-- .../Services/SheerkaEvaluateConcept.py | 2 +- .../sheerka/Services/SheerkaModifyConcept.py | 20 +++- src/core/sheerka/Sheerka.py | 93 ++++++++++++------- src/evaluators/AddConceptEvaluator.py | 65 +++++++++---- src/parsers/DefaultParser.py | 54 +++++++---- tests/core/test_SheerkaCreateNewConcept.py | 31 ++++++- tests/core/test_SheerkaModifyConcept.py | 13 ++- tests/core/test_concept.py | 21 ++++- tests/evaluators/test_AddConceptEvaluator.py | 57 +++++++++--- tests/non_reg/test_sheerka_non_reg.py | 24 +++++ tests/parsers/test_DefaultParser.py | 39 ++++++-- 13 files changed, 341 insertions(+), 106 deletions(-) diff --git a/src/core/concept.py b/src/core/concept.py index 14f70b3..5c462c7 100644 --- a/src/core/concept.py +++ b/src/core/concept.py @@ -16,6 +16,8 @@ PROPERTIES_TO_SERIALIZE = PROPERTIES_FOR_DIGEST + tuple(["id"]) PROPERTIES_FOR_NEW = ("where", "pre", "post", "body", "desc") VARIABLE_PREFIX = "__var__" ORIGIN = "##origin##" # same as Serializer.ORIGIN but I don't want to include the reference +DEFINITION_TYPE_BNF = "bnf" +DEFINITION_TYPE_DEF = "def" class ConceptParts(Enum): @@ -219,7 +221,10 @@ class Concept: return self if tokens is None: - tokens = list(Tokenizer(self.metadata.name)) + if self.metadata.definition_type == DEFINITION_TYPE_DEF: + tokens = list(Tokenizer(self.metadata.definition)) + else: + tokens = list(Tokenizer(self.metadata.name)) variables = [p[0] for p in self.metadata.props] if len(core.utils.strip_tokens(tokens, True)) > 1 else [] diff --git a/src/core/sheerka/Services/SheerkaCreateNewConcept.py b/src/core/sheerka/Services/SheerkaCreateNewConcept.py index a6a0c7e..f933d35 100644 --- a/src/core/sheerka/Services/SheerkaCreateNewConcept.py +++ b/src/core/sheerka/Services/SheerkaCreateNewConcept.py @@ -27,10 +27,12 @@ class SheerkaCreateNewConcept: concepts_definitions = None init_ret_value = None + sdp = self.sheerka.sdp + # checks for duplicate concepts # TODO checks if it exists in cache first - if self.sheerka.sdp.exists(self.sheerka.CONCEPTS_BY_HASH_ENTRY, concept.get_definition_hash()): + if sdp.exists(self.sheerka.CONCEPTS_BY_HASH_ENTRY, concept.get_definition_hash()): error = SheerkaDataProviderDuplicateKeyError(self.sheerka.CONCEPTS_ENTRY + "." + concept.key, concept) return self.sheerka.ret( self.logger_name, @@ -62,7 +64,7 @@ class SheerkaCreateNewConcept: # TODO : needs to make these calls atomic (or at least one single call) # save the new concept concept.metadata.full_serialization = True - result = self.sheerka.sdp.add( + result = sdp.add( context.event.get_digest(), self.sheerka.CONCEPTS_ENTRY, concept, @@ -73,20 +75,26 @@ class SheerkaCreateNewConcept: concept.set_origin(result.digest) # save it by id - self.sheerka.sdp.add( + sdp.add( context.event.get_digest(), self.sheerka.CONCEPTS_BY_ID_ENTRY, SheerkaDataProviderRef(concept.id, result.digest)) + # save it by name + sdp.add( + context.event.get_digest(), + self.sheerka.CONCEPTS_BY_NAME_ENTRY, + SheerkaDataProviderRef(concept.name, result.digest)) + # records the hash - self.sheerka.sdp.add( + sdp.add( context.event.get_digest(), self.sheerka.CONCEPTS_BY_HASH_ENTRY, SheerkaDataProviderRef(concept.get_definition_hash(), result.digest)) # update the definition table if concepts_definitions is not None: - self.sheerka.sdp.set( + sdp.set( context.event.get_digest(), self.sheerka.CONCEPTS_DEFINITIONS_ENTRY, concept_lexer_parser.encode_grammar(init_ret_value.body), @@ -102,7 +110,8 @@ class SheerkaCreateNewConcept: # Updates the caches - self.sheerka.cache_by_key[concept.key] = self.sheerka.sdp.get_safe(self.sheerka.CONCEPTS_ENTRY, concept.key) + self.sheerka.cache_by_key[concept.key] = sdp.get_safe(self.sheerka.CONCEPTS_ENTRY, concept.key) + self.sheerka.cache_by_name[concept.name] = sdp.get_safe(self.sheerka.CONCEPTS_BY_NAME_ENTRY, concept.name) self.sheerka.cache_by_id[concept.id] = concept if init_ret_value is not None and init_ret_value.status: self.sheerka.concepts_grammars = init_ret_value.body diff --git a/src/core/sheerka/Services/SheerkaEvaluateConcept.py b/src/core/sheerka/Services/SheerkaEvaluateConcept.py index 2c02947..8184559 100644 --- a/src/core/sheerka/Services/SheerkaEvaluateConcept.py +++ b/src/core/sheerka/Services/SheerkaEvaluateConcept.py @@ -254,7 +254,7 @@ class SheerkaEvaluateConcept: # validate where clause if ConceptParts.WHERE in concept.values: where_value = concept.values[ConceptParts.WHERE] - if not (where_value is None or self.sheerka.value(where_value) is True): + if not (where_value is None or self.sheerka.value(where_value)): return self.sheerka.new(BuiltinConcepts.WHERE_CLAUSE_FAILED, body=concept) # diff --git a/src/core/sheerka/Services/SheerkaModifyConcept.py b/src/core/sheerka/Services/SheerkaModifyConcept.py index ea47603..f9033bd 100644 --- a/src/core/sheerka/Services/SheerkaModifyConcept.py +++ b/src/core/sheerka/Services/SheerkaModifyConcept.py @@ -9,25 +9,34 @@ class SheerkaModifyConcept: def modify_concept(self, context, concept): + sdp = self.sheerka.sdp + try: # modify the entry concept.metadata.full_serialization = True - result = self.sheerka.sdp.modify( + result = sdp.modify( context.event.get_digest(), self.sheerka.CONCEPTS_ENTRY, concept.key, concept) concept.metadata.full_serialization = False - # update its reference - self.sheerka.sdp.modify( + # update reference entry + sdp.modify( context.event.get_digest(), self.sheerka.CONCEPTS_BY_ID_ENTRY, concept.id, SheerkaDataProviderRef(concept.id, result.digest, concept.get_origin())) + # update name entry + sdp.modify( + context.event.get_digest(), + self.sheerka.CONCEPTS_BY_NAME_ENTRY, + concept.name, + SheerkaDataProviderRef(concept.name, result.digest, concept.get_origin())) + # update the hash entry - self.sheerka.sdp.modify( + sdp.modify( context.event.get_digest(), self.sheerka.CONCEPTS_BY_HASH_ENTRY, concept.get_original_definition_hash(), @@ -42,7 +51,8 @@ class SheerkaModifyConcept: error.args[0]) # update cache - self.sheerka.cache_by_key[concept.key] = self.sheerka.sdp.get_safe(self.sheerka.CONCEPTS_ENTRY, concept.key) + self.sheerka.cache_by_key[concept.key] = sdp.get_safe(self.sheerka.CONCEPTS_ENTRY, concept.key) + self.sheerka.cache_by_name[concept.name] = sdp.get_safe(self.sheerka.CONCEPTS_BY_NAME_ENTRY, concept.name) self.sheerka.cache_by_id[concept.id] = concept ret = self.sheerka.ret(self.logger_name, True, self.sheerka.new(BuiltinConcepts.NEW_CONCEPT, body=concept)) diff --git a/src/core/sheerka/Sheerka.py b/src/core/sheerka/Sheerka.py index 3eb3758..fe55353 100644 --- a/src/core/sheerka/Sheerka.py +++ b/src/core/sheerka/Sheerka.py @@ -34,7 +34,8 @@ class Sheerka(Concept): CONCEPTS_ENTRY = "All_Concepts" # to store all the concepts CONCEPTS_BY_ID_ENTRY = "Concepts_By_ID" - CONCEPTS_BY_HASH_ENTRY = "Concepts_By_Hash" # store hash of concepts definitions (not values) + CONCEPTS_BY_NAME_ENTRY = "Concepts_By_Name" + CONCEPTS_BY_HASH_ENTRY = "Concepts_By_Hash" # store hash of concepts definitions (not values) CONCEPTS_DEFINITIONS_ENTRY = "Concepts_Definitions" # to store definitions (bnf) of concepts BUILTIN_CONCEPTS_KEYS = "Builtins_Concepts" # sequential key for builtin concepts USER_CONCEPTS_KEYS = "User_Concepts" # sequential key for user defined concepts @@ -53,6 +54,7 @@ class Sheerka(Concept): # key is the key of the concept (not the name or the id) self.cache_by_key = {} self.cache_by_id = {} + self.cache_by_name = {} # cache for concept definitions, # Primarily used for unit test that does not have access to sdp @@ -239,7 +241,7 @@ class Sheerka(Concept): with ExecutionContext(self.key, event, self, f"Evaluating '{text}'", self.log) as execution_context: user_input = self.ret(self.name, True, self.new(BuiltinConcepts.USER_INPUT, body=text, user_name=user_name)) reduce_requested = self.ret(self.name, True, self.new(BuiltinConcepts.REDUCE_REQUESTED)) - #execution_context.local_hints.add(BuiltinConcepts.EVAL_WHERE_REQUESTED) + # execution_context.local_hints.add(BuiltinConcepts.EVAL_WHERE_REQUESTED) steps = [ BuiltinConcepts.BEFORE_PARSING, @@ -375,38 +377,22 @@ class Sheerka(Concept): :return: """ - if concept_key is None: - return ErrorConcept("Concept key is undefined.") + by_key = self.internal_get("key", concept_key, self.cache_by_key, self.CONCEPTS_ENTRY, concept_id) + if self.is_known(by_key): + return by_key - if isinstance(concept_key, BuiltinConcepts): - concept_key = str(concept_key) + # else return by name + by_name = self.internal_get("name", concept_key, self.cache_by_name, self.CONCEPTS_BY_NAME_ENTRY, concept_id) + if self.is_known(by_name): + return by_name - # first search in cache - if concept_key in self.cache_by_key: - result = self.cache_by_key[concept_key] - else: - result = self.sdp.get_safe(self.CONCEPTS_ENTRY, concept_key) - if result is None: - metadata = [("key", concept_key), ("id", concept_id)] if concept_id else ("key", concept_key) - result = self._get_unknown(metadata) - # Do not put in cache_by_key or cache_by_id unknown concept - # TODO: implement an MRU cache for them - else: - self.cache_by_key[concept_key] = result - for r in (result if isinstance(result, list) else [result]): - if r.id: - self.cache_by_id[r.id] = r + return by_key # return not found for key - if not (isinstance(result, list) and concept_id): - return result + def get_by_key(self, concept_key, concept_id=None): + return self.internal_get("key", concept_key, self.cache_by_key, self.CONCEPTS_ENTRY, concept_id) - # result is a list, but we have the concept_id to discriminate - for c in result: - if c.id == concept_id: - return c - - metadata = [("key", concept_key), ("id", concept_id)] if concept_id else ("key", concept_key) - return self._get_unknown(metadata) + def get_by_name(self, concept_name, concept_id=None): + return self.internal_get("name", concept_name, self.cache_by_name, self.CONCEPTS_BY_NAME_ENTRY, concept_id) def get_by_id(self, concept_id): if concept_id is None: @@ -419,10 +405,55 @@ class Sheerka(Concept): result = self.sdp.get_safe(self.CONCEPTS_BY_ID_ENTRY, concept_id) if result is None: result = self._get_unknown(('id', concept_id)) - self.cache_by_id[concept_id] = result + else: + self.cache_by_id[concept_id] = result return result + def internal_get(self, index_name, index_value, cache_to_use, sdp_entry, concept_id=None): + """ + Tries to find an entry + :param index_name: + :param index_value: + :param cache_to_use: + :param sdp_entry: + :param concept_id: + :return: + """ + + if index_value is None: + return ErrorConcept(f"Concept {index_name} is undefined.") + + if isinstance(index_value, BuiltinConcepts): + index_value = str(index_value) + + # first search in cache + if index_value in cache_to_use: + result = cache_to_use[index_value] + else: + result = self.sdp.get_safe(sdp_entry, index_value) + if result is None: + metadata = [(index_name, index_value), ("id", concept_id)] if concept_id else (index_name, index_value) + result = self._get_unknown(metadata) + # Do not put in cache_by_key or cache_by_id unknown concept + # TODO: implement an MRU cache for them + else: + cache_to_use[index_value] = result + for r in (result if isinstance(result, list) else [result]): + if r.id: + self.cache_by_id[r.id] = r + + if not (isinstance(result, list) and concept_id): + return result + + # result is a list, but we have the concept_id to discriminate + for c in result: + if c.id == concept_id: + return c + + metadata = [(index_name, index_value), ("id", concept_id)] if concept_id else (index_name, index_value) + return self._get_unknown(metadata) + def get_concepts_definitions(self, context): if self.concepts_definitions_cache: diff --git a/src/evaluators/AddConceptEvaluator.py b/src/evaluators/AddConceptEvaluator.py index 7b20d00..40b1266 100644 --- a/src/evaluators/AddConceptEvaluator.py +++ b/src/evaluators/AddConceptEvaluator.py @@ -1,11 +1,12 @@ from core.ast.nodes import python_to_concept from core.builtin_concepts import ParserResultConcept, ReturnValueConcept, BuiltinConcepts from core.builtin_helpers import get_names -from core.concept import Concept +from core.concept import Concept, DEFINITION_TYPE_BNF, DEFINITION_TYPE_DEF +from core.tokenizer import TokenKind from evaluators.BaseEvaluator import OneReturnValueEvaluator from parsers.BaseParser import NotInitializedNode from parsers.ConceptLexerParser import ParsingExpression, ParsingExpressionVisitor -from parsers.DefaultParser import DefConceptNode +from parsers.DefaultParser import DefConceptNode, NameNode from parsers.PythonParser import PythonNode import core.utils @@ -55,22 +56,33 @@ class AddConceptEvaluator(OneReturnValueEvaluator): props_found = set() concept = Concept(def_concept_node.name) - for prop in ("definition", "where", "pre", "post", "body"): - # put back the sources - part_ret_val = getattr(def_concept_node, prop) - if not isinstance(part_ret_val, ReturnValueConcept) or not part_ret_val.status: - continue # Nothing to do is not initialized + concept.metadata.definition_type = def_concept_node.definition_type + name_to_use = self.get_name_to_use(def_concept_node) - # update the metadata - source = self.get_source(part_ret_val) + for prop in ("definition", "where", "pre", "post", "body"): + + part_ret_val = getattr(def_concept_node, prop) + + # put back the sources + if isinstance(part_ret_val, NotInitializedNode): + continue + elif isinstance(part_ret_val, NameNode): + source = str(part_ret_val) + elif isinstance(part_ret_val, ReturnValueConcept) and part_ret_val.status: + source = part_ret_val.value.source + else: + raise Exception("Unexpected") setattr(concept.metadata, prop, source) + # Do not try to resolve variables from itself + if prop == "definition" and concept.metadata.definition_type == DEFINITION_TYPE_DEF: + continue + # try to find what can be a property - concept_name = [part.value for part in core.utils.strip_tokens(def_concept_node.name.tokens, True)] - for p in self.get_props(sheerka, part_ret_val, concept_name): + for p in self.get_props(sheerka, part_ret_val, name_to_use): props_found.add(p) - # add props order by appearance when possible + # add props by order of appearance when possible for token in def_concept_node.name.tokens: if token.value in props_found: concept.def_prop(token.value, None) @@ -80,10 +92,15 @@ class AddConceptEvaluator(OneReturnValueEvaluator): if p not in concept.props: concept.def_prop(p, None) - # finish initialisation - concept.init_key(def_concept_node.name.tokens) + # initialize the key + key_source = def_concept_node.definition.tokens if \ + def_concept_node.definition_type == DEFINITION_TYPE_DEF else \ + def_concept_node.name.tokens + concept.init_key(key_source) + + # update the bnf definition if needed if not isinstance(def_concept_node.definition, NotInitializedNode) and \ - sheerka.is_success(def_concept_node.definition): + def_concept_node.definition_type == DEFINITION_TYPE_BNF: concept.bnf = def_concept_node.definition.value.value ret = sheerka.create_new_concept(context, concept) @@ -93,8 +110,9 @@ class AddConceptEvaluator(OneReturnValueEvaluator): return sheerka.ret(self.name, ret.status, ret.value, parents=[return_value]) @staticmethod - def get_source(ret_value): - return ret_value.value.source + def get_name_to_use(node): + source = node.definition if node.definition_type == DEFINITION_TYPE_DEF else node.name + return [part.value for part in core.utils.strip_tokens(source.tokens, True)] @staticmethod def get_props(sheerka, ret_value, concept_name): @@ -104,6 +122,15 @@ class AddConceptEvaluator(OneReturnValueEvaluator): I guess that it can only be complete when will we have access to Sheerka memory """ + # + # Case of NameNode + # + if isinstance(ret_value, NameNode): + names = [str(t.value) for t in ret_value.tokens if t.type in ( + TokenKind.IDENTIFIER, TokenKind.STRING, TokenKind.KEYWORD)] + variables = filter(lambda x: x in concept_name, names) + return list(variables) + # # Case of python code # @@ -111,8 +138,8 @@ class AddConceptEvaluator(OneReturnValueEvaluator): if len(concept_name) > 1: python_node = ret_value.value.value as_concept_node = python_to_concept(python_node.ast_) - variables = get_names(sheerka, as_concept_node) - variables = filter(lambda x: x in concept_name, variables) + names = get_names(sheerka, as_concept_node) + variables = filter(lambda x: x in concept_name, names) return list(variables) # diff --git a/src/parsers/DefaultParser.py b/src/parsers/DefaultParser.py index 7bac427..1adda35 100644 --- a/src/parsers/DefaultParser.py +++ b/src/parsers/DefaultParser.py @@ -1,5 +1,5 @@ from core.builtin_concepts import BuiltinConcepts, ReturnValueConcept, ParserResultConcept -from core.concept import ConceptParts +from core.concept import ConceptParts, DEFINITION_TYPE_BNF, DEFINITION_TYPE_DEF import core.builtin_helpers import core.utils from parsers.BaseParser import BaseParser, Node, ErrorNode, NotInitializedNode @@ -84,6 +84,7 @@ class DefConceptNode(DefaultParserNode): post: ReturnValueConcept = NotInitializedNode() body: ReturnValueConcept = NotInitializedNode() definition: ReturnValueConcept = NotInitializedNode() + definition_type: str = None def get_asts(self): asts = {} @@ -93,7 +94,7 @@ class DefConceptNode(DefaultParserNode): ParserResultConcept) and hasattr( prop_value.body.body, "ast_"): asts[part_key] = prop_value - #asts[part_key] = prop_value.body.body.ast_ + # asts[part_key] = prop_value.body.body.ast_ return asts @@ -244,14 +245,15 @@ class DefaultParser(BaseParser): keywords_tokens = [def_token] concept_found = DefConceptNode(keywords_tokens) - # the definition of a concept consists of several parts - # Keywords.CONCEPT to get the name of the concept - # Keywords.FROM [Keywords.BNF] to get the definition of the concept - # Keywords.AS to get the body - # Keywords.WHERE to get the conditions to recognize for the variables - # Keywords.PRE to know if the conditions to evaluate the concept - # Keywords.POST to apply or verify once the concept is executed - # + # ## + # ## the definition of a concept consists of several parts + # ## Keywords.CONCEPT to get the name of the concept + # ## Keywords.FROM [Keywords.BNF] | [Keywords.DEF] to get the definition of the concept + # ## Keywords.AS to get the body + # ## Keywords.WHERE to get the conditions to recognize for the variables + # ## Keywords.PRE to know if the conditions to evaluate the concept + # ## Keywords.POST to apply or verify once the concept is executed + # Regroup the tokens by parts first_token, tokens_found_by_parts = self.regroup_tokens_by_parts(keywords_tokens) @@ -262,7 +264,9 @@ class DefaultParser(BaseParser): concept_found.name = self.get_concept_name(first_token, tokens_found_by_parts) # get the definition - concept_found.definition = self.get_concept_definition(concept_found, tokens_found_by_parts) + def_type, def_value = self.get_concept_definition(concept_found, tokens_found_by_parts) + concept_found.definition_type = def_type + concept_found.definition = def_value # get the ASTs for the remaining parts asts_found_by_parts = self.get_concept_parts(tokens_found_by_parts) @@ -362,16 +366,23 @@ class DefaultParser(BaseParser): def get_concept_definition(self, current_concept_def, tokens_found_by_parts): if tokens_found_by_parts[Keywords.FROM] is None: - return NotInitializedNode() + return None, NotInitializedNode() definition_tokens = tokens_found_by_parts[Keywords.FROM] - if definition_tokens[1].value != Keywords.BNF: - return NotInitializedNode() + if len(definition_tokens) == 1: + self.add_error(SyntaxErrorNode([], "Empty declaration"), False) + return None, NotInitializedNode() + if definition_tokens[1].value == Keywords.BNF: + return self.get_concept_bnf_definition(current_concept_def, definition_tokens) + + return self.get_concept_simple_definition(definition_tokens) + + def get_concept_bnf_definition(self, current_concept_def, definition_tokens): tokens = core.utils.strip_tokens(definition_tokens[2:]) if len(tokens) == 0: self.add_error(SyntaxErrorNode([definition_tokens[1]], "Empty declaration"), False) - return NotInitializedNode() + return None, NotInitializedNode() regex_parser = BnfParser() with self.context.push(self.name, obj=current_concept_def) as sub_context: @@ -380,9 +391,18 @@ class DefaultParser(BaseParser): if not parsing_result.status: self.add_error(parsing_result.value) - return NotInitializedNode() + return None, NotInitializedNode() - return parsing_result + return DEFINITION_TYPE_BNF, parsing_result + + def get_concept_simple_definition(self, definition_tokens): + start = 2 if definition_tokens[1].value == Keywords.DEF else 1 + tokens = core.utils.strip_tokens(definition_tokens[start:]) + if len(tokens) == 0: + self.add_error(SyntaxErrorNode([definition_tokens[start]], "Empty declaration"), False) + return None, NotInitializedNode() + + return DEFINITION_TYPE_DEF, NameNode(tokens) def get_concept_parts(self, tokens_found_by_parts): asts_found_by_parts = { diff --git a/tests/core/test_SheerkaCreateNewConcept.py b/tests/core/test_SheerkaCreateNewConcept.py index 2655711..0ed940c 100644 --- a/tests/core/test_SheerkaCreateNewConcept.py +++ b/tests/core/test_SheerkaCreateNewConcept.py @@ -1,5 +1,5 @@ from core.builtin_concepts import BuiltinConcepts -from core.concept import PROPERTIES_TO_SERIALIZE, Concept +from core.concept import PROPERTIES_TO_SERIALIZE, Concept, DEFINITION_TYPE_DEF from core.sheerka.Sheerka import Sheerka from sdp.sheerkaDataProvider import SheerkaDataProvider @@ -25,10 +25,38 @@ class TestSheerkaCreateNewConcept(TestUsingMemoryBasedSheerka): assert concept.key in sheerka.cache_by_key assert concept.id in sheerka.cache_by_id + assert concept.name in sheerka.cache_by_name assert sheerka.sdp.io.exists( sheerka.sdp.io.get_obj_path(SheerkaDataProvider.ObjectsFolder, concept_found.get_origin())) assert sheerka.sdp.exists(Sheerka.CONCEPTS_BY_HASH_ENTRY, concept.get_definition_hash()) assert sheerka.sdp.exists(Sheerka.CONCEPTS_BY_ID_ENTRY, concept.id) + assert sheerka.sdp.exists(Sheerka.CONCEPTS_BY_NAME_ENTRY, concept.name) + assert sheerka.sdp.exists(Sheerka.CONCEPTS_ENTRY, concept.key) + + def test_i_can_add_a_concept_when_name_differs_from_the_key(self): + sheerka = self.get_sheerka() + concept = Concept("greetings", definition="hello a", definition_type=DEFINITION_TYPE_DEF).def_prop("a") + + res = sheerka.create_new_concept(self.get_context(sheerka), concept) + + assert res.status + assert sheerka.isinstance(res.value, BuiltinConcepts.NEW_CONCEPT) + + concept_found = res.value.body + for prop in PROPERTIES_TO_SERIALIZE: + assert getattr(concept_found.metadata, prop) == getattr(concept.metadata, prop) + + assert concept_found.key == "hello __var__0" + assert concept_found.id == "1001" + + assert concept.key in sheerka.cache_by_key + assert concept.id in sheerka.cache_by_id + assert concept.name in sheerka.cache_by_name + assert sheerka.sdp.io.exists( + sheerka.sdp.io.get_obj_path(SheerkaDataProvider.ObjectsFolder, concept_found.get_origin())) + assert sheerka.sdp.exists(Sheerka.CONCEPTS_BY_HASH_ENTRY, concept.get_definition_hash()) + assert sheerka.sdp.exists(Sheerka.CONCEPTS_BY_ID_ENTRY, concept.id) + assert sheerka.sdp.exists(Sheerka.CONCEPTS_BY_NAME_ENTRY, concept.name) assert sheerka.sdp.exists(Sheerka.CONCEPTS_ENTRY, concept.key) def test_i_cannot_add_the_same_concept_twice(self): @@ -200,3 +228,4 @@ class TestSheerkaCreateNewConcept(TestUsingMemoryBasedSheerka): assert res.status + diff --git a/tests/core/test_SheerkaModifyConcept.py b/tests/core/test_SheerkaModifyConcept.py index f29ef94..c19af6c 100644 --- a/tests/core/test_SheerkaModifyConcept.py +++ b/tests/core/test_SheerkaModifyConcept.py @@ -11,9 +11,9 @@ class TestSheerkaModifyConcept(TestUsingMemoryBasedSheerka): sheerka, context, foo, bar = self.init_concepts("foo", "bar", create_new=True) foo_instance = sheerka.new("foo") - foo_instance.metadata.body = "value" - foo_instance.set_prop(BuiltinConcepts.ISA, bar) - foo_instance.set_metadata_value(ConceptParts.BODY, "body value") + foo_instance.metadata.body = "value" # modify metadata + foo_instance.set_prop(BuiltinConcepts.ISA, bar) # modify property + foo_instance.set_metadata_value(ConceptParts.BODY, "body value") # modify value res = sheerka.modify_concept(context, foo_instance) assert res.status @@ -36,6 +36,13 @@ class TestSheerkaModifyConcept(TestUsingMemoryBasedSheerka): assert foo_from_sheerka.get_prop(BuiltinConcepts.ISA) == bar assert foo_from_sheerka.body == "body value" + # test that ref by name is updated + sheerka.reset_cache() + foo_from_sheerka = sheerka.get_by_name(foo.name) + assert foo_from_sheerka.metadata.body == "value" + assert foo_from_sheerka.get_prop(BuiltinConcepts.ISA) == bar + assert foo_from_sheerka.body == "body value" + # test that ref by hash is updated foo_from_sdp = sheerka.sdp.get(Sheerka.CONCEPTS_BY_HASH_ENTRY, foo_instance.get_definition_hash()) assert foo_from_sdp.metadata.body == "value" diff --git a/tests/core/test_concept.py b/tests/core/test_concept.py index 5b27fa9..9fed84b 100644 --- a/tests/core/test_concept.py +++ b/tests/core/test_concept.py @@ -1,6 +1,6 @@ import pytest -from core.concept import Concept, ConceptParts +from core.concept import Concept, ConceptParts, DEFINITION_TYPE_DEF @pytest.mark.parametrize("name, properties, expected", [ @@ -29,6 +29,25 @@ def test_i_can_compute_the_key(name, properties, expected): assert concept.key == expected +def test_i_can_compute_the_key_when_from_definition(): + + # if definition is not defined, use the name + concept = Concept() + concept.metadata.name = "hello a" + concept.metadata.props = [("a", None)] + concept.init_key() + assert concept.key == "hello __var__0" + + # if definition is defined, use it + concept = Concept() + concept.metadata.name = "greetings" + concept.metadata.definition = "hello a" + concept.metadata.definition_type = DEFINITION_TYPE_DEF + concept.metadata.props = [("a", None)] + concept.init_key() + assert concept.key == "hello __var__0" + + def test_key_does_not_use_variable_when_definition_is_set(): concept = Concept("plus").def_prop('plus') diff --git a/tests/evaluators/test_AddConceptEvaluator.py b/tests/evaluators/test_AddConceptEvaluator.py index a05d13a..6d9110b 100644 --- a/tests/evaluators/test_AddConceptEvaluator.py +++ b/tests/evaluators/test_AddConceptEvaluator.py @@ -52,21 +52,25 @@ class TestAddConceptEvaluator(TestUsingMemoryBasedSheerka): ) ) - def get_def_concept(self, name, where=None, pre=None, post=None, body=None, definition=None): - concept = DefConceptNode([], name=NameNode(list(Tokenizer(name)))) + def get_def_concept(self, name, where=None, pre=None, post=None, body=None, definition=None, bnf_def=None): + def_concept = DefConceptNode([], name=NameNode(list(Tokenizer(name)))) if body: - concept.body = self.get_concept_part(body) + def_concept.body = self.get_concept_part(body) if where: - concept.where = self.get_concept_part(where) + def_concept.where = self.get_concept_part(where) if pre: - concept.pre = self.get_concept_part(pre) + def_concept.pre = self.get_concept_part(pre) if post: - concept.post = self.get_concept_part(post) + def_concept.post = self.get_concept_part(post) + if bnf_def: + def_concept.definition = bnf_def + def_concept.definition_type = "bnf" if definition: - concept.definition = definition + def_concept.definition = NameNode(list(Tokenizer(definition))) + def_concept.definition_type = "def" - return ReturnValueConcept(BaseParser.PREFIX + "some_name", True, ParserResultConcept(value=concept)) + return ReturnValueConcept(BaseParser.PREFIX + "some_name", True, ParserResultConcept(value=def_concept)) @pytest.mark.parametrize("ret_val, expected", [ (ReturnValueConcept(BaseParser.PREFIX + "some_name", True, ParserResultConcept(value=DefConceptNode([]))), @@ -80,11 +84,11 @@ class TestAddConceptEvaluator(TestUsingMemoryBasedSheerka): context = self.get_context() assert AddConceptEvaluator().matches(context, ret_val) == expected - def test_that_the_source_is_correctly_set(self): + def test_that_the_source_is_correctly_set_for_bnf_concept(self): context = self.get_context() def_concept_return_value = self.get_def_concept( name="hello a", - definition=self.get_return_value("hello a", Sequence(StrMatch("hello"), StrMatch("a"))), + bnf_def=self.get_return_value("hello a", Sequence(StrMatch("hello"), StrMatch("a"))), where="isinstance(a, str )", pre="a is not None", body="print('hello' + a)") @@ -96,17 +100,43 @@ class TestAddConceptEvaluator(TestUsingMemoryBasedSheerka): created_concept = evaluated.body.body assert created_concept.metadata.name == "hello a" + assert created_concept.metadata.key == "hello __var__0" assert created_concept.metadata.where == "isinstance(a, str )" assert created_concept.metadata.pre == "a is not None" assert created_concept.metadata.post is None assert created_concept.metadata.body == "print('hello' + a)" assert created_concept.metadata.definition == "hello a" + assert created_concept.metadata.definition_type == "bnf" + + def test_that_the_source_is_correctly_set_for_concept_with_simple_definition(self): + context = self.get_context() + def_concept_return_value = self.get_def_concept( + name="greetings", + definition="hello a", + where="isinstance(a, str )", + pre="a is not None", + body="print('hello' + a)") + + evaluated = AddConceptEvaluator().eval(context, def_concept_return_value) + + assert evaluated.status + assert context.sheerka.isinstance(evaluated.body, BuiltinConcepts.NEW_CONCEPT) + + created_concept = evaluated.body.body + assert created_concept.metadata.name == "greetings" + assert created_concept.metadata.key == "hello __var__0" + assert created_concept.metadata.where == "isinstance(a, str )" + assert created_concept.metadata.pre == "a is not None" + assert created_concept.metadata.post is None + assert created_concept.metadata.body == "print('hello' + a)" + assert created_concept.metadata.definition == "hello a" + assert created_concept.metadata.definition_type == "def" def test_that_the_new_concept_is_correctly_saved_in_db(self): context = self.get_context() def_concept_return_value = self.get_def_concept( name="hello a", - definition=self.get_return_value("hello a", Sequence(StrMatch("hello"), StrMatch("a"))), + bnf_def=self.get_return_value("hello a", Sequence(StrMatch("hello"), StrMatch("a"))), where="isinstance(a, str )", pre="a is not None", body="print('hello' + a)") @@ -126,6 +156,7 @@ class TestAddConceptEvaluator(TestUsingMemoryBasedSheerka): assert from_db.metadata.post is None assert from_db.metadata.body == "print('hello' + a)" assert from_db.metadata.definition == "hello a" + assert from_db.metadata.definition_type == "bnf" assert len(from_db.metadata.props) == 1 assert from_db.metadata.props[0] == ("a", None) assert "a" in from_db.props @@ -150,14 +181,14 @@ class TestAddConceptEvaluator(TestUsingMemoryBasedSheerka): status=True, value=ParserResultConcept(value=concept)) - assert AddConceptEvaluator.get_props(self.get_context(), ret_val, []) == ["a", "b"] + assert AddConceptEvaluator.get_props(self.get_sheerka(), ret_val, []) == ["a", "b"] def test_i_can_get_props_from_definition(self): parsing_expression = Sequence(ConceptExpression('mult'), ZeroOrMore(Sequence(StrMatch("+"), ConceptExpression("add")))) ret_val = self.get_return_value("mult (('+'|'-') add)?", parsing_expression) - assert AddConceptEvaluator.get_props(self.get_context(), ret_val, []) == ["add", "mult"] + assert AddConceptEvaluator.get_props(self.get_sheerka(), ret_val, []) == ["add", "mult"] def test_concept_that_references_itself_is_correctly_created(self): context = self.get_context() diff --git a/tests/non_reg/test_sheerka_non_reg.py b/tests/non_reg/test_sheerka_non_reg.py index 8c52009..3691918 100644 --- a/tests/non_reg/test_sheerka_non_reg.py +++ b/tests/non_reg/test_sheerka_non_reg.py @@ -359,6 +359,30 @@ as: assert evaluated.body == "one two three" assert evaluated.props["a"] == Property("a", sheerka.new(concept_a.key, body="one two").init_key()) + @pytest.mark.parametrize("user_input", [ + "def concept greetings from def hello a where a", + "def concept greetings from hello a where a"]) + def test_i_can_recognize_a_concept_defined_using_from_def(self, user_input): + sheerka = self.get_sheerka() + + greetings = sheerka.evaluate_user_input(user_input)[0].body.body + + res = sheerka.evaluate_user_input("hello 'foo'") + assert len(res) == 1 + assert res[0].status + concept_found = res[0].value + assert sheerka.isinstance(concept_found, greetings) + assert concept_found.get_prop("a") == "foo" + assert concept_found.metadata.need_validation + + res = sheerka.evaluate_user_input("greetings") + assert len(res) == 1 + assert res[0].status + concept_found = res[0].value + assert sheerka.isinstance(concept_found, greetings) + assert concept_found.get_prop("a") is None + assert not concept_found.metadata.need_validation + @pytest.mark.parametrize("desc, definitions", [ ("Simple form", [ "def concept one as 1", diff --git a/tests/parsers/test_DefaultParser.py b/tests/parsers/test_DefaultParser.py index bb9b99c..dc3e02b 100644 --- a/tests/parsers/test_DefaultParser.py +++ b/tests/parsers/test_DefaultParser.py @@ -2,7 +2,7 @@ import pytest import ast from core.builtin_concepts import ParserResultConcept, BuiltinConcepts, ReturnValueConcept -from core.concept import Concept +from core.concept import Concept, DEFINITION_TYPE_BNF, DEFINITION_TYPE_DEF from parsers.ConceptLexerParser import OrderedChoice, StrMatch, ConceptExpression from parsers.PythonParser import PythonParser, PythonNode from core.tokenizer import Keywords, Tokenizer, LexerError @@ -13,7 +13,7 @@ from parsers.BnfParser import BnfParser from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka -def get_def_concept(name, where=None, pre=None, post=None, body=None, definition=None): +def get_def_concept(name, where=None, pre=None, post=None, body=None, definition=None, bnf_def=None): def_concept = DefConceptNode([], name=NameNode(list(Tokenizer(name)))) if body: @@ -24,11 +24,15 @@ def get_def_concept(name, where=None, pre=None, post=None, body=None, definition def_concept.pre = get_concept_part(pre) if post: def_concept.post = get_concept_part(post) - if definition: + if bnf_def: def_concept.definition = ReturnValueConcept( "parsers.Bnf", True, - definition) + bnf_def) + def_concept.definition_type = DEFINITION_TYPE_BNF + if definition: + def_concept.definition = NameNode(list(Tokenizer(definition))) + def_concept.definition_type = DEFINITION_TYPE_DEF return def_concept @@ -237,7 +241,7 @@ def concept add one to a as assert not res.status assert return_value.value == [SyntaxErrorNode([], "Newline are not allowed in name.")] - def test_i_can_parse_def_concept_from_regex(self): + def test_i_can_parse_def_concept_from_bnf(self): context = self.get_context() a_concept = Concept("a_concept") context.sheerka.add_in_cache(a_concept) @@ -248,7 +252,7 @@ def concept add one to a as node = res.value.value definition = OrderedChoice(ConceptExpression(a_concept, rule_name="a_concept"), StrMatch("a_string")) parser_result = ParserResultConcept(BnfParser(), "a_concept | 'a_string'", definition, definition) - expected = get_def_concept(name="name", body="__definition[0]", definition=parser_result) + expected = get_def_concept(name="name", body="__definition[0]", bnf_def=parser_result) assert res.status assert res.who == parser.name @@ -267,8 +271,12 @@ def concept add one to a as assert not parser.has_error - def test_i_can_detect_empty_bnf_declaration(self): - text = "def concept name from bnf as __definition[0]" + @pytest.mark.parametrize("text", [ + "def concept name from bnf as here is my body", + "def concept name from def as here is my body", + "def concept name from as here is my body" + ]) + def test_i_can_detect_empty_bnf_declaration(self, text): parser = DefaultParser() res = parser.parse(self.get_context(), text) @@ -276,6 +284,21 @@ def concept add one to a as assert not res.status assert res.value.value[0] == SyntaxErrorNode([], "Empty declaration") + @pytest.mark.parametrize("text", [ + "def concept addition from a plus b as a + b", + "def concept addition from def a plus b as a + b"]) + def test_i_can_def_concept_from_definition(self, text): + parser = DefaultParser() + res = parser.parse(self.get_context(), text) + expected = get_def_concept("addition", definition="a plus b", body="a + b") + node = res.value.value + + assert res.status + assert res.who == parser.name + assert res.value.source == text + assert isinstance(res.value, ParserResultConcept) + assert node == expected + def test_i_can_detect_not_for_me(self): text = "hello world" context = self.get_context()