kodjo/Sheerka-Old
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

I can also get concept by name

Browse Source
This commit is contained in:
Kodjo Sossouvi
2020-03-10 15:05:03 +01:00
parent 1bde97b5e3
commit a2bbd2eec2
13 changed files with 341 additions and 106 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/core/concept.py
+5
View File
@@ -16,6 +16,8 @@ PROPERTIES_TO_SERIALIZE = PROPERTIES_FOR_DIGEST + tuple(["id"])
PROPERTIES_FOR_NEW = ("where", "pre", "post", "body", "desc")
VARIABLE_PREFIX = "__var__"
ORIGIN = "##origin##" # same as Serializer.ORIGIN but I don't want to include the reference
DEFINITION_TYPE_BNF = "bnf"
DEFINITION_TYPE_DEF = "def"
class ConceptParts(Enum):
@@ -219,6 +221,9 @@ class Concept:
return self
if tokens is None:
if self.metadata.definition_type == DEFINITION_TYPE_DEF:
tokens = list(Tokenizer(self.metadata.definition))
else:
tokens = list(Tokenizer(self.metadata.name))
variables = [p[0] for p in self.metadata.props] if len(core.utils.strip_tokens(tokens, True)) > 1 else []
src/core/sheerka/Services/SheerkaCreateNewConcept.py
+15 -6
View File
@@ -27,10 +27,12 @@ class SheerkaCreateNewConcept:
concepts_definitions = None
init_ret_value = None
sdp = self.sheerka.sdp
# checks for duplicate concepts
# TODO checks if it exists in cache first
if self.sheerka.sdp.exists(self.sheerka.CONCEPTS_BY_HASH_ENTRY, concept.get_definition_hash()):
if sdp.exists(self.sheerka.CONCEPTS_BY_HASH_ENTRY, concept.get_definition_hash()):
error = SheerkaDataProviderDuplicateKeyError(self.sheerka.CONCEPTS_ENTRY + "." + concept.key, concept)
return self.sheerka.ret(
self.logger_name,
@@ -62,7 +64,7 @@ class SheerkaCreateNewConcept:
# TODO : needs to make these calls atomic (or at least one single call)
# save the new concept
concept.metadata.full_serialization = True
result = self.sheerka.sdp.add(
result = sdp.add(
context.event.get_digest(),
self.sheerka.CONCEPTS_ENTRY,
concept,
@@ -73,20 +75,26 @@ class SheerkaCreateNewConcept:
concept.set_origin(result.digest)
# save it by id
self.sheerka.sdp.add(
sdp.add(
context.event.get_digest(),
self.sheerka.CONCEPTS_BY_ID_ENTRY,
SheerkaDataProviderRef(concept.id, result.digest))
# save it by name
sdp.add(
context.event.get_digest(),
self.sheerka.CONCEPTS_BY_NAME_ENTRY,
SheerkaDataProviderRef(concept.name, result.digest))
# records the hash
self.sheerka.sdp.add(
sdp.add(
context.event.get_digest(),
self.sheerka.CONCEPTS_BY_HASH_ENTRY,
SheerkaDataProviderRef(concept.get_definition_hash(), result.digest))
# update the definition table
if concepts_definitions is not None:
self.sheerka.sdp.set(
sdp.set(
context.event.get_digest(),
self.sheerka.CONCEPTS_DEFINITIONS_ENTRY,
concept_lexer_parser.encode_grammar(init_ret_value.body),
@@ -102,7 +110,8 @@ class SheerkaCreateNewConcept:
# Updates the caches
self.sheerka.cache_by_key[concept.key] = self.sheerka.sdp.get_safe(self.sheerka.CONCEPTS_ENTRY, concept.key)
self.sheerka.cache_by_key[concept.key] = sdp.get_safe(self.sheerka.CONCEPTS_ENTRY, concept.key)
self.sheerka.cache_by_name[concept.name] = sdp.get_safe(self.sheerka.CONCEPTS_BY_NAME_ENTRY, concept.name)
self.sheerka.cache_by_id[concept.id] = concept
if init_ret_value is not None and init_ret_value.status:
self.sheerka.concepts_grammars = init_ret_value.body
src/core/sheerka/Services/SheerkaEvaluateConcept.py
+1 -1
View File
@@ -254,7 +254,7 @@ class SheerkaEvaluateConcept:
# validate where clause
if ConceptParts.WHERE in concept.values:
where_value = concept.values[ConceptParts.WHERE]
if not (where_value is None or self.sheerka.value(where_value) is True):
if not (where_value is None or self.sheerka.value(where_value)):
return self.sheerka.new(BuiltinConcepts.WHERE_CLAUSE_FAILED, body=concept)
#
src/core/sheerka/Services/SheerkaModifyConcept.py
+15 -5
View File
@@ -9,25 +9,34 @@ class SheerkaModifyConcept:
def modify_concept(self, context, concept):
sdp = self.sheerka.sdp
try:
# modify the entry
concept.metadata.full_serialization = True
result = self.sheerka.sdp.modify(
result = sdp.modify(
context.event.get_digest(),
self.sheerka.CONCEPTS_ENTRY,
concept.key,
concept)
concept.metadata.full_serialization = False
# update its reference
self.sheerka.sdp.modify(
# update reference entry
sdp.modify(
context.event.get_digest(),
self.sheerka.CONCEPTS_BY_ID_ENTRY,
concept.id,
SheerkaDataProviderRef(concept.id, result.digest, concept.get_origin()))
# update name entry
sdp.modify(
context.event.get_digest(),
self.sheerka.CONCEPTS_BY_NAME_ENTRY,
concept.name,
SheerkaDataProviderRef(concept.name, result.digest, concept.get_origin()))
# update the hash entry
self.sheerka.sdp.modify(
sdp.modify(
context.event.get_digest(),
self.sheerka.CONCEPTS_BY_HASH_ENTRY,
concept.get_original_definition_hash(),
@@ -42,7 +51,8 @@ class SheerkaModifyConcept:
error.args[0])
# update cache
self.sheerka.cache_by_key[concept.key] = self.sheerka.sdp.get_safe(self.sheerka.CONCEPTS_ENTRY, concept.key)
self.sheerka.cache_by_key[concept.key] = sdp.get_safe(self.sheerka.CONCEPTS_ENTRY, concept.key)
self.sheerka.cache_by_name[concept.name] = sdp.get_safe(self.sheerka.CONCEPTS_BY_NAME_ENTRY, concept.name)
self.sheerka.cache_by_id[concept.id] = concept
ret = self.sheerka.ret(self.logger_name, True, self.sheerka.new(BuiltinConcepts.NEW_CONCEPT, body=concept))
src/core/sheerka/Sheerka.py
+59 -28
View File
@@ -34,6 +34,7 @@ class Sheerka(Concept):
CONCEPTS_ENTRY = "All_Concepts" # to store all the concepts
CONCEPTS_BY_ID_ENTRY = "Concepts_By_ID"
CONCEPTS_BY_NAME_ENTRY = "Concepts_By_Name"
CONCEPTS_BY_HASH_ENTRY = "Concepts_By_Hash" # store hash of concepts definitions (not values)
CONCEPTS_DEFINITIONS_ENTRY = "Concepts_Definitions" # to store definitions (bnf) of concepts
BUILTIN_CONCEPTS_KEYS = "Builtins_Concepts" # sequential key for builtin concepts
@@ -53,6 +54,7 @@ class Sheerka(Concept):
# key is the key of the concept (not the name or the id)
self.cache_by_key = {}
self.cache_by_id = {}
self.cache_by_name = {}
# cache for concept definitions,
# Primarily used for unit test that does not have access to sdp
@@ -375,38 +377,22 @@ class Sheerka(Concept):
:return:
"""
if concept_key is None:
return ErrorConcept("Concept key is undefined.")
by_key = self.internal_get("key", concept_key, self.cache_by_key, self.CONCEPTS_ENTRY, concept_id)
if self.is_known(by_key):
return by_key
if isinstance(concept_key, BuiltinConcepts):
concept_key = str(concept_key)
# else return by name
by_name = self.internal_get("name", concept_key, self.cache_by_name, self.CONCEPTS_BY_NAME_ENTRY, concept_id)
if self.is_known(by_name):
return by_name
# first search in cache
if concept_key in self.cache_by_key:
result = self.cache_by_key[concept_key]
else:
result = self.sdp.get_safe(self.CONCEPTS_ENTRY, concept_key)
if result is None:
metadata = [("key", concept_key), ("id", concept_id)] if concept_id else ("key", concept_key)
result = self._get_unknown(metadata)
# Do not put in cache_by_key or cache_by_id unknown concept
# TODO: implement an MRU cache for them
else:
self.cache_by_key[concept_key] = result
for r in (result if isinstance(result, list) else [result]):
if r.id:
self.cache_by_id[r.id] = r
return by_key # return not found for key
if not (isinstance(result, list) and concept_id):
return result
def get_by_key(self, concept_key, concept_id=None):
return self.internal_get("key", concept_key, self.cache_by_key, self.CONCEPTS_ENTRY, concept_id)
# result is a list, but we have the concept_id to discriminate
for c in result:
if c.id == concept_id:
return c
metadata = [("key", concept_key), ("id", concept_id)] if concept_id else ("key", concept_key)
return self._get_unknown(metadata)
def get_by_name(self, concept_name, concept_id=None):
return self.internal_get("name", concept_name, self.cache_by_name, self.CONCEPTS_BY_NAME_ENTRY, concept_id)
def get_by_id(self, concept_id):
if concept_id is None:
@@ -419,10 +405,55 @@ class Sheerka(Concept):
result = self.sdp.get_safe(self.CONCEPTS_BY_ID_ENTRY, concept_id)
if result is None:
result = self._get_unknown(('id', concept_id))
else:
self.cache_by_id[concept_id] = result
return result
def internal_get(self, index_name, index_value, cache_to_use, sdp_entry, concept_id=None):
"""
Tries to find an entry
:param index_name:
:param index_value:
:param cache_to_use:
:param sdp_entry:
:param concept_id:
:return:
"""
if index_value is None:
return ErrorConcept(f"Concept {index_name} is undefined.")
if isinstance(index_value, BuiltinConcepts):
index_value = str(index_value)
# first search in cache
if index_value in cache_to_use:
result = cache_to_use[index_value]
else:
result = self.sdp.get_safe(sdp_entry, index_value)
if result is None:
metadata = [(index_name, index_value), ("id", concept_id)] if concept_id else (index_name, index_value)
result = self._get_unknown(metadata)
# Do not put in cache_by_key or cache_by_id unknown concept
# TODO: implement an MRU cache for them
else:
cache_to_use[index_value] = result
for r in (result if isinstance(result, list) else [result]):
if r.id:
self.cache_by_id[r.id] = r
if not (isinstance(result, list) and concept_id):
return result
# result is a list, but we have the concept_id to discriminate
for c in result:
if c.id == concept_id:
return c
metadata = [(index_name, index_value), ("id", concept_id)] if concept_id else (index_name, index_value)
return self._get_unknown(metadata)
def get_concepts_definitions(self, context):
if self.concepts_definitions_cache:
src/evaluators/AddConceptEvaluator.py
+46 -19
View File
@@ -1,11 +1,12 @@
from core.ast.nodes import python_to_concept
from core.builtin_concepts import ParserResultConcept, ReturnValueConcept, BuiltinConcepts
from core.builtin_helpers import get_names
from core.concept import Concept
from core.concept import Concept, DEFINITION_TYPE_BNF, DEFINITION_TYPE_DEF
from core.tokenizer import TokenKind
from evaluators.BaseEvaluator import OneReturnValueEvaluator
from parsers.BaseParser import NotInitializedNode
from parsers.ConceptLexerParser import ParsingExpression, ParsingExpressionVisitor
from parsers.DefaultParser import DefConceptNode
from parsers.DefaultParser import DefConceptNode, NameNode
from parsers.PythonParser import PythonNode
import core.utils
@@ -55,22 +56,33 @@ class AddConceptEvaluator(OneReturnValueEvaluator):
props_found = set()
concept = Concept(def_concept_node.name)
for prop in ("definition", "where", "pre", "post", "body"):
# put back the sources
part_ret_val = getattr(def_concept_node, prop)
if not isinstance(part_ret_val, ReturnValueConcept) or not part_ret_val.status:
continue # Nothing to do is not initialized
concept.metadata.definition_type = def_concept_node.definition_type
name_to_use = self.get_name_to_use(def_concept_node)
# update the metadata
source = self.get_source(part_ret_val)
for prop in ("definition", "where", "pre", "post", "body"):
part_ret_val = getattr(def_concept_node, prop)
# put back the sources
if isinstance(part_ret_val, NotInitializedNode):
continue
elif isinstance(part_ret_val, NameNode):
source = str(part_ret_val)
elif isinstance(part_ret_val, ReturnValueConcept) and part_ret_val.status:
source = part_ret_val.value.source
else:
raise Exception("Unexpected")
setattr(concept.metadata, prop, source)
# Do not try to resolve variables from itself
if prop == "definition" and concept.metadata.definition_type == DEFINITION_TYPE_DEF:
continue
# try to find what can be a property
concept_name = [part.value for part in core.utils.strip_tokens(def_concept_node.name.tokens, True)]
for p in self.get_props(sheerka, part_ret_val, concept_name):
for p in self.get_props(sheerka, part_ret_val, name_to_use):
props_found.add(p)
# add props order by appearance when possible
# add props by order of appearance when possible
for token in def_concept_node.name.tokens:
if token.value in props_found:
concept.def_prop(token.value, None)
@@ -80,10 +92,15 @@ class AddConceptEvaluator(OneReturnValueEvaluator):
if p not in concept.props:
concept.def_prop(p, None)
# finish initialisation
concept.init_key(def_concept_node.name.tokens)
# initialize the key
key_source = def_concept_node.definition.tokens if \
def_concept_node.definition_type == DEFINITION_TYPE_DEF else \
def_concept_node.name.tokens
concept.init_key(key_source)
# update the bnf definition if needed
if not isinstance(def_concept_node.definition, NotInitializedNode) and \
sheerka.is_success(def_concept_node.definition):
def_concept_node.definition_type == DEFINITION_TYPE_BNF:
concept.bnf = def_concept_node.definition.value.value
ret = sheerka.create_new_concept(context, concept)
@@ -93,8 +110,9 @@ class AddConceptEvaluator(OneReturnValueEvaluator):
return sheerka.ret(self.name, ret.status, ret.value, parents=[return_value])
@staticmethod
def get_source(ret_value):
return ret_value.value.source
def get_name_to_use(node):
source = node.definition if node.definition_type == DEFINITION_TYPE_DEF else node.name
return [part.value for part in core.utils.strip_tokens(source.tokens, True)]
@staticmethod
def get_props(sheerka, ret_value, concept_name):
@@ -104,6 +122,15 @@ class AddConceptEvaluator(OneReturnValueEvaluator):
I guess that it can only be complete when will we have access to Sheerka memory
"""
#
# Case of NameNode
#
if isinstance(ret_value, NameNode):
names = [str(t.value) for t in ret_value.tokens if t.type in (
TokenKind.IDENTIFIER, TokenKind.STRING, TokenKind.KEYWORD)]
variables = filter(lambda x: x in concept_name, names)
return list(variables)
#
# Case of python code
#
@@ -111,8 +138,8 @@ class AddConceptEvaluator(OneReturnValueEvaluator):
if len(concept_name) > 1:
python_node = ret_value.value.value
as_concept_node = python_to_concept(python_node.ast_)
variables = get_names(sheerka, as_concept_node)
variables = filter(lambda x: x in concept_name, variables)
names = get_names(sheerka, as_concept_node)
variables = filter(lambda x: x in concept_name, names)
return list(variables)
#
src/parsers/DefaultParser.py
+36 -16
View File
@@ -1,5 +1,5 @@
from core.builtin_concepts import BuiltinConcepts, ReturnValueConcept, ParserResultConcept
from core.concept import ConceptParts
from core.concept import ConceptParts, DEFINITION_TYPE_BNF, DEFINITION_TYPE_DEF
import core.builtin_helpers
import core.utils
from parsers.BaseParser import BaseParser, Node, ErrorNode, NotInitializedNode
@@ -84,6 +84,7 @@ class DefConceptNode(DefaultParserNode):
post: ReturnValueConcept = NotInitializedNode()
body: ReturnValueConcept = NotInitializedNode()
definition: ReturnValueConcept = NotInitializedNode()
definition_type: str = None
def get_asts(self):
asts = {}
@@ -244,14 +245,15 @@ class DefaultParser(BaseParser):
keywords_tokens = [def_token]
concept_found = DefConceptNode(keywords_tokens)
# the definition of a concept consists of several parts
# Keywords.CONCEPT to get the name of the concept
# Keywords.FROM [Keywords.BNF] to get the definition of the concept
# Keywords.AS to get the body
# Keywords.WHERE to get the conditions to recognize for the variables
# Keywords.PRE to know if the conditions to evaluate the concept
# Keywords.POST to apply or verify once the concept is executed
#
# ##
# ## the definition of a concept consists of several parts
# ## Keywords.CONCEPT to get the name of the concept
# ## Keywords.FROM [Keywords.BNF] | [Keywords.DEF] to get the definition of the concept
# ## Keywords.AS to get the body
# ## Keywords.WHERE to get the conditions to recognize for the variables
# ## Keywords.PRE to know if the conditions to evaluate the concept
# ## Keywords.POST to apply or verify once the concept is executed
# Regroup the tokens by parts
first_token, tokens_found_by_parts = self.regroup_tokens_by_parts(keywords_tokens)
@@ -262,7 +264,9 @@ class DefaultParser(BaseParser):
concept_found.name = self.get_concept_name(first_token, tokens_found_by_parts)
# get the definition
concept_found.definition = self.get_concept_definition(concept_found, tokens_found_by_parts)
def_type, def_value = self.get_concept_definition(concept_found, tokens_found_by_parts)
concept_found.definition_type = def_type
concept_found.definition = def_value
# get the ASTs for the remaining parts
asts_found_by_parts = self.get_concept_parts(tokens_found_by_parts)
@@ -362,16 +366,23 @@ class DefaultParser(BaseParser):
def get_concept_definition(self, current_concept_def, tokens_found_by_parts):
if tokens_found_by_parts[Keywords.FROM] is None:
return NotInitializedNode()
return None, NotInitializedNode()
definition_tokens = tokens_found_by_parts[Keywords.FROM]
if definition_tokens[1].value != Keywords.BNF:
return NotInitializedNode()
if len(definition_tokens) == 1:
self.add_error(SyntaxErrorNode([], "Empty declaration"), False)
return None, NotInitializedNode()
if definition_tokens[1].value == Keywords.BNF:
return self.get_concept_bnf_definition(current_concept_def, definition_tokens)
return self.get_concept_simple_definition(definition_tokens)
def get_concept_bnf_definition(self, current_concept_def, definition_tokens):
tokens = core.utils.strip_tokens(definition_tokens[2:])
if len(tokens) == 0:
self.add_error(SyntaxErrorNode([definition_tokens[1]], "Empty declaration"), False)
return NotInitializedNode()
return None, NotInitializedNode()
regex_parser = BnfParser()
with self.context.push(self.name, obj=current_concept_def) as sub_context:
@@ -380,9 +391,18 @@ class DefaultParser(BaseParser):
if not parsing_result.status:
self.add_error(parsing_result.value)
return NotInitializedNode()
return None, NotInitializedNode()
return parsing_result
return DEFINITION_TYPE_BNF, parsing_result
def get_concept_simple_definition(self, definition_tokens):
start = 2 if definition_tokens[1].value == Keywords.DEF else 1
tokens = core.utils.strip_tokens(definition_tokens[start:])
if len(tokens) == 0:
self.add_error(SyntaxErrorNode([definition_tokens[start]], "Empty declaration"), False)
return None, NotInitializedNode()
return DEFINITION_TYPE_DEF, NameNode(tokens)
def get_concept_parts(self, tokens_found_by_parts):
asts_found_by_parts = {
tests/core/test_SheerkaCreateNewConcept.py
+30 -1
View File
@@ -1,5 +1,5 @@
from core.builtin_concepts import BuiltinConcepts
from core.concept import PROPERTIES_TO_SERIALIZE, Concept
from core.concept import PROPERTIES_TO_SERIALIZE, Concept, DEFINITION_TYPE_DEF
from core.sheerka.Sheerka import Sheerka
from sdp.sheerkaDataProvider import SheerkaDataProvider
@@ -25,10 +25,38 @@ class TestSheerkaCreateNewConcept(TestUsingMemoryBasedSheerka):
assert concept.key in sheerka.cache_by_key
assert concept.id in sheerka.cache_by_id
assert concept.name in sheerka.cache_by_name
assert sheerka.sdp.io.exists(
sheerka.sdp.io.get_obj_path(SheerkaDataProvider.ObjectsFolder, concept_found.get_origin()))
assert sheerka.sdp.exists(Sheerka.CONCEPTS_BY_HASH_ENTRY, concept.get_definition_hash())
assert sheerka.sdp.exists(Sheerka.CONCEPTS_BY_ID_ENTRY, concept.id)
assert sheerka.sdp.exists(Sheerka.CONCEPTS_BY_NAME_ENTRY, concept.name)
assert sheerka.sdp.exists(Sheerka.CONCEPTS_ENTRY, concept.key)
def test_i_can_add_a_concept_when_name_differs_from_the_key(self):
sheerka = self.get_sheerka()
concept = Concept("greetings", definition="hello a", definition_type=DEFINITION_TYPE_DEF).def_prop("a")
res = sheerka.create_new_concept(self.get_context(sheerka), concept)
assert res.status
assert sheerka.isinstance(res.value, BuiltinConcepts.NEW_CONCEPT)
concept_found = res.value.body
for prop in PROPERTIES_TO_SERIALIZE:
assert getattr(concept_found.metadata, prop) == getattr(concept.metadata, prop)
assert concept_found.key == "hello __var__0"
assert concept_found.id == "1001"
assert concept.key in sheerka.cache_by_key
assert concept.id in sheerka.cache_by_id
assert concept.name in sheerka.cache_by_name
assert sheerka.sdp.io.exists(
sheerka.sdp.io.get_obj_path(SheerkaDataProvider.ObjectsFolder, concept_found.get_origin()))
assert sheerka.sdp.exists(Sheerka.CONCEPTS_BY_HASH_ENTRY, concept.get_definition_hash())
assert sheerka.sdp.exists(Sheerka.CONCEPTS_BY_ID_ENTRY, concept.id)
assert sheerka.sdp.exists(Sheerka.CONCEPTS_BY_NAME_ENTRY, concept.name)
assert sheerka.sdp.exists(Sheerka.CONCEPTS_ENTRY, concept.key)
def test_i_cannot_add_the_same_concept_twice(self):
@@ -200,3 +228,4 @@ class TestSheerkaCreateNewConcept(TestUsingMemoryBasedSheerka):
assert res.status
tests/core/test_SheerkaModifyConcept.py
+10 -3
View File
@@ -11,9 +11,9 @@ class TestSheerkaModifyConcept(TestUsingMemoryBasedSheerka):
sheerka, context, foo, bar = self.init_concepts("foo", "bar", create_new=True)
foo_instance = sheerka.new("foo")
foo_instance.metadata.body = "value"
foo_instance.set_prop(BuiltinConcepts.ISA, bar)
foo_instance.set_metadata_value(ConceptParts.BODY, "body value")
foo_instance.metadata.body = "value" # modify metadata
foo_instance.set_prop(BuiltinConcepts.ISA, bar) # modify property
foo_instance.set_metadata_value(ConceptParts.BODY, "body value") # modify value
res = sheerka.modify_concept(context, foo_instance)
assert res.status
@@ -36,6 +36,13 @@ class TestSheerkaModifyConcept(TestUsingMemoryBasedSheerka):
assert foo_from_sheerka.get_prop(BuiltinConcepts.ISA) == bar
assert foo_from_sheerka.body == "body value"
# test that ref by name is updated
sheerka.reset_cache()
foo_from_sheerka = sheerka.get_by_name(foo.name)
assert foo_from_sheerka.metadata.body == "value"
assert foo_from_sheerka.get_prop(BuiltinConcepts.ISA) == bar
assert foo_from_sheerka.body == "body value"
# test that ref by hash is updated
foo_from_sdp = sheerka.sdp.get(Sheerka.CONCEPTS_BY_HASH_ENTRY, foo_instance.get_definition_hash())
assert foo_from_sdp.metadata.body == "value"
tests/core/test_concept.py
+20 -1
View File
@@ -1,6 +1,6 @@
import pytest
from core.concept import Concept, ConceptParts
from core.concept import Concept, ConceptParts, DEFINITION_TYPE_DEF
@pytest.mark.parametrize("name, properties, expected", [
@@ -29,6 +29,25 @@ def test_i_can_compute_the_key(name, properties, expected):
assert concept.key == expected
def test_i_can_compute_the_key_when_from_definition():
# if definition is not defined, use the name
concept = Concept()
concept.metadata.name = "hello a"
concept.metadata.props = [("a", None)]
concept.init_key()
assert concept.key == "hello __var__0"
# if definition is defined, use it
concept = Concept()
concept.metadata.name = "greetings"
concept.metadata.definition = "hello a"
concept.metadata.definition_type = DEFINITION_TYPE_DEF
concept.metadata.props = [("a", None)]
concept.init_key()
assert concept.key == "hello __var__0"
def test_key_does_not_use_variable_when_definition_is_set():
concept = Concept("plus").def_prop('plus')
tests/evaluators/test_AddConceptEvaluator.py
+44 -13
View File
@@ -52,21 +52,25 @@ class TestAddConceptEvaluator(TestUsingMemoryBasedSheerka):
)
)
def get_def_concept(self, name, where=None, pre=None, post=None, body=None, definition=None):
concept = DefConceptNode([], name=NameNode(list(Tokenizer(name))))
def get_def_concept(self, name, where=None, pre=None, post=None, body=None, definition=None, bnf_def=None):
def_concept = DefConceptNode([], name=NameNode(list(Tokenizer(name))))
if body:
concept.body = self.get_concept_part(body)
def_concept.body = self.get_concept_part(body)
if where:
concept.where = self.get_concept_part(where)
def_concept.where = self.get_concept_part(where)
if pre:
concept.pre = self.get_concept_part(pre)
def_concept.pre = self.get_concept_part(pre)
if post:
concept.post = self.get_concept_part(post)
def_concept.post = self.get_concept_part(post)
if bnf_def:
def_concept.definition = bnf_def
def_concept.definition_type = "bnf"
if definition:
concept.definition = definition
def_concept.definition = NameNode(list(Tokenizer(definition)))
def_concept.definition_type = "def"
return ReturnValueConcept(BaseParser.PREFIX + "some_name", True, ParserResultConcept(value=concept))
return ReturnValueConcept(BaseParser.PREFIX + "some_name", True, ParserResultConcept(value=def_concept))
@pytest.mark.parametrize("ret_val, expected", [
(ReturnValueConcept(BaseParser.PREFIX + "some_name", True, ParserResultConcept(value=DefConceptNode([]))),
@@ -80,11 +84,11 @@ class TestAddConceptEvaluator(TestUsingMemoryBasedSheerka):
context = self.get_context()
assert AddConceptEvaluator().matches(context, ret_val) == expected
def test_that_the_source_is_correctly_set(self):
def test_that_the_source_is_correctly_set_for_bnf_concept(self):
context = self.get_context()
def_concept_return_value = self.get_def_concept(
name="hello a",
definition=self.get_return_value("hello a", Sequence(StrMatch("hello"), StrMatch("a"))),
bnf_def=self.get_return_value("hello a", Sequence(StrMatch("hello"), StrMatch("a"))),
where="isinstance(a, str )",
pre="a is not None",
body="print('hello' + a)")
@@ -96,17 +100,43 @@ class TestAddConceptEvaluator(TestUsingMemoryBasedSheerka):
created_concept = evaluated.body.body
assert created_concept.metadata.name == "hello a"
assert created_concept.metadata.key == "hello __var__0"
assert created_concept.metadata.where == "isinstance(a, str )"
assert created_concept.metadata.pre == "a is not None"
assert created_concept.metadata.post is None
assert created_concept.metadata.body == "print('hello' + a)"
assert created_concept.metadata.definition == "hello a"
assert created_concept.metadata.definition_type == "bnf"
def test_that_the_source_is_correctly_set_for_concept_with_simple_definition(self):
context = self.get_context()
def_concept_return_value = self.get_def_concept(
name="greetings",
definition="hello a",
where="isinstance(a, str )",
pre="a is not None",
body="print('hello' + a)")
evaluated = AddConceptEvaluator().eval(context, def_concept_return_value)
assert evaluated.status
assert context.sheerka.isinstance(evaluated.body, BuiltinConcepts.NEW_CONCEPT)
created_concept = evaluated.body.body
assert created_concept.metadata.name == "greetings"
assert created_concept.metadata.key == "hello __var__0"
assert created_concept.metadata.where == "isinstance(a, str )"
assert created_concept.metadata.pre == "a is not None"
assert created_concept.metadata.post is None
assert created_concept.metadata.body == "print('hello' + a)"
assert created_concept.metadata.definition == "hello a"
assert created_concept.metadata.definition_type == "def"
def test_that_the_new_concept_is_correctly_saved_in_db(self):
context = self.get_context()
def_concept_return_value = self.get_def_concept(
name="hello a",
definition=self.get_return_value("hello a", Sequence(StrMatch("hello"), StrMatch("a"))),
bnf_def=self.get_return_value("hello a", Sequence(StrMatch("hello"), StrMatch("a"))),
where="isinstance(a, str )",
pre="a is not None",
body="print('hello' + a)")
@@ -126,6 +156,7 @@ class TestAddConceptEvaluator(TestUsingMemoryBasedSheerka):
assert from_db.metadata.post is None
assert from_db.metadata.body == "print('hello' + a)"
assert from_db.metadata.definition == "hello a"
assert from_db.metadata.definition_type == "bnf"
assert len(from_db.metadata.props) == 1
assert from_db.metadata.props[0] == ("a", None)
assert "a" in from_db.props
@@ -150,14 +181,14 @@ class TestAddConceptEvaluator(TestUsingMemoryBasedSheerka):
status=True,
value=ParserResultConcept(value=concept))
assert AddConceptEvaluator.get_props(self.get_context(), ret_val, []) == ["a", "b"]
assert AddConceptEvaluator.get_props(self.get_sheerka(), ret_val, []) == ["a", "b"]
def test_i_can_get_props_from_definition(self):
parsing_expression = Sequence(ConceptExpression('mult'),
ZeroOrMore(Sequence(StrMatch("+"), ConceptExpression("add"))))
ret_val = self.get_return_value("mult (('+'|'-') add)?", parsing_expression)
assert AddConceptEvaluator.get_props(self.get_context(), ret_val, []) == ["add", "mult"]
assert AddConceptEvaluator.get_props(self.get_sheerka(), ret_val, []) == ["add", "mult"]
def test_concept_that_references_itself_is_correctly_created(self):
context = self.get_context()
tests/non_reg/test_sheerka_non_reg.py
+24
View File
@@ -359,6 +359,30 @@ as:
assert evaluated.body == "one two three"
assert evaluated.props["a"] == Property("a", sheerka.new(concept_a.key, body="one two").init_key())
@pytest.mark.parametrize("user_input", [
"def concept greetings from def hello a where a",
"def concept greetings from hello a where a"])
def test_i_can_recognize_a_concept_defined_using_from_def(self, user_input):
sheerka = self.get_sheerka()
greetings = sheerka.evaluate_user_input(user_input)[0].body.body
res = sheerka.evaluate_user_input("hello 'foo'")
assert len(res) == 1
assert res[0].status
concept_found = res[0].value
assert sheerka.isinstance(concept_found, greetings)
assert concept_found.get_prop("a") == "foo"
assert concept_found.metadata.need_validation
res = sheerka.evaluate_user_input("greetings")
assert len(res) == 1
assert res[0].status
concept_found = res[0].value
assert sheerka.isinstance(concept_found, greetings)
assert concept_found.get_prop("a") is None
assert not concept_found.metadata.need_validation
@pytest.mark.parametrize("desc, definitions", [
("Simple form", [
"def concept one as 1",
tests/parsers/test_DefaultParser.py
+31 -8
View File
@@ -2,7 +2,7 @@ import pytest
import ast
from core.builtin_concepts import ParserResultConcept, BuiltinConcepts, ReturnValueConcept
from core.concept import Concept
from core.concept import Concept, DEFINITION_TYPE_BNF, DEFINITION_TYPE_DEF
from parsers.ConceptLexerParser import OrderedChoice, StrMatch, ConceptExpression
from parsers.PythonParser import PythonParser, PythonNode
from core.tokenizer import Keywords, Tokenizer, LexerError
@@ -13,7 +13,7 @@ from parsers.BnfParser import BnfParser
from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
def get_def_concept(name, where=None, pre=None, post=None, body=None, definition=None):
def get_def_concept(name, where=None, pre=None, post=None, body=None, definition=None, bnf_def=None):
def_concept = DefConceptNode([], name=NameNode(list(Tokenizer(name))))
if body:
@@ -24,11 +24,15 @@ def get_def_concept(name, where=None, pre=None, post=None, body=None, definition
def_concept.pre = get_concept_part(pre)
if post:
def_concept.post = get_concept_part(post)
if definition:
if bnf_def:
def_concept.definition = ReturnValueConcept(
"parsers.Bnf",
True,
definition)
bnf_def)
def_concept.definition_type = DEFINITION_TYPE_BNF
if definition:
def_concept.definition = NameNode(list(Tokenizer(definition)))
def_concept.definition_type = DEFINITION_TYPE_DEF
return def_concept
@@ -237,7 +241,7 @@ def concept add one to a as
assert not res.status
assert return_value.value == [SyntaxErrorNode([], "Newline are not allowed in name.")]
def test_i_can_parse_def_concept_from_regex(self):
def test_i_can_parse_def_concept_from_bnf(self):
context = self.get_context()
a_concept = Concept("a_concept")
context.sheerka.add_in_cache(a_concept)
@@ -248,7 +252,7 @@ def concept add one to a as
node = res.value.value
definition = OrderedChoice(ConceptExpression(a_concept, rule_name="a_concept"), StrMatch("a_string"))
parser_result = ParserResultConcept(BnfParser(), "a_concept | 'a_string'", definition, definition)
expected = get_def_concept(name="name", body="__definition[0]", definition=parser_result)
expected = get_def_concept(name="name", body="__definition[0]", bnf_def=parser_result)
assert res.status
assert res.who == parser.name
@@ -267,8 +271,12 @@ def concept add one to a as
assert not parser.has_error
def test_i_can_detect_empty_bnf_declaration(self):
text = "def concept name from bnf as __definition[0]"
@pytest.mark.parametrize("text", [
"def concept name from bnf as here is my body",
"def concept name from def as here is my body",
"def concept name from as here is my body"
])
def test_i_can_detect_empty_bnf_declaration(self, text):
parser = DefaultParser()
res = parser.parse(self.get_context(), text)
@@ -276,6 +284,21 @@ def concept add one to a as
assert not res.status
assert res.value.value[0] == SyntaxErrorNode([], "Empty declaration")
@pytest.mark.parametrize("text", [
"def concept addition from a plus b as a + b",
"def concept addition from def a plus b as a + b"])
def test_i_can_def_concept_from_definition(self, text):
parser = DefaultParser()
res = parser.parse(self.get_context(), text)
expected = get_def_concept("addition", definition="a plus b", body="a + b")
node = res.value.value
assert res.status
assert res.who == parser.name
assert res.value.source == text
assert isinstance(res.value, ParserResultConcept)
assert node == expected
def test_i_can_detect_not_for_me(self):
text = "hello world"
context = self.get_context()