Fixed #32 : concept groups are not correctly updated

Fixed #35 : Refactor test helper class (CNC, CC, CIO)
Fixed #36 : Concept values are not used when declared with variable expression
Fixed #37 : Objects in memory lose their values are restart
Fixed #38 : func(a=b, c) (which is not allowed) raise an exception
This commit is contained in:
2021-03-05 11:16:19 +01:00
parent 646c428edb
commit 05577012f3
38 changed files with 1942 additions and 1463 deletions
+13 -1
View File
@@ -35,13 +35,15 @@ class SheerkaAdmin(BaseService):
self.sheerka.bind_service_method(self.ontologies, False)
self.sheerka.bind_service_method(self.in_memory, False)
self.sheerka.bind_service_method(self.admin_history, False, as_name="history")
self.sheerka.bind_service_method(self.admin_history, False, as_name="history")
self.sheerka.bind_service_method(self.sdp, False)
def caches_names(self):
"""
Returns the name of all the caches
:return:
"""
return list(self.sheerka.om.current_cache_manager().caches.keys())
return self.sheerka.new(BuiltinConcepts.TO_LIST, body=self.sheerka.om.current_cache_manager().caches.keys())
def cache(self, name, *keys):
"""
@@ -58,6 +60,16 @@ class SheerkaAdmin(BaseService):
return {key: self.sheerka.om.get(name, key) for key in keys}
def sdp(self, name=None):
if name:
for ontology in self.sheerka.om.ontologies:
if ontology.name == name:
return ontology.cache_manager.sdp
return self.sheerka.err(self.sheerka.new(BuiltinConcepts.NOT_FOUND, {"sdp_name", name}))
return self.sheerka.om.current_sdp()
def restore(self, concept_file=CONCEPTS_FILE_TO_USE):
"""
Restore the state with all previous valid concept definitions
@@ -237,36 +237,22 @@ class SheerkaConceptManager(BaseService):
except Exception as ex:
return sheerka.ret(self.NAME, False, ex.args[0])
# compute first token and/or first regex
init_ret_value = self.compute_concepts_by_first_item(context, [concept], True)
if not init_ret_value.status:
return sheerka.ret(self.NAME, False, ErrorConcept(init_ret_value.value))
by_first_keyword, by_first_regex = init_ret_value.body
# computes resolved concepts_by_first_keyword
init_ret_value = self.resolve_concepts_by_first_keyword(context, by_first_keyword)
if not init_ret_value.status:
return sheerka.ret(self.NAME, False, ErrorConcept(init_ret_value.value))
resolved_concepts_by_first_keyword = init_ret_value.body
# compile regex
compile_ret = self.compile_concepts_by_first_regex(context, by_first_regex)
if not compile_ret.status:
return sheerka.ret(self.NAME, False, ErrorConcept(compile_ret.value))
compiled_concepts_by_first_regex = compile_ret.body
# recompute concepts by first tokens and concept by first regex
update_items_res = self.recompute_first_items(context, None, [concept])
if not update_items_res.status:
return update_items_res
by_first_keyword, by_first_regex, resolved_by_first_keyword, compiled_by_first_regex = update_items_res.body
# if everything is fine
freeze_concept_attrs(concept)
concept.freeze_definition_hash()
om.add_concept(concept)
om.put(self.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, False, by_first_keyword)
om.put(self.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY, False, resolved_concepts_by_first_keyword)
om.put(self.CONCEPTS_BY_REGEX_ENTRY, False, {k.serialize(): v for k, v in by_first_regex.items()})
# update the compiled regex
self.compiled_concepts_by_regex.clear()
self.compiled_concepts_by_regex.extend(compiled_concepts_by_first_regex)
self.update_first_items_caches(context,
by_first_keyword,
by_first_regex,
resolved_by_first_keyword,
compiled_by_first_regex)
if concept.get_metadata().definition_type == DEFINITION_TYPE_DEF and concept.get_metadata().definition != concept.name:
# allow search by definition when definition relevant
@@ -284,9 +270,7 @@ class SheerkaConceptManager(BaseService):
# publish the new concept
sheerka.publish(context, EVENT_CONCEPT_CREATED, concept)
# process the return if needed
ret = sheerka.ret(self.NAME, True, sheerka.new(BuiltinConcepts.NEW_CONCEPT, body=concept))
return ret
return sheerka.ret(self.NAME, True, sheerka.new(BuiltinConcepts.NEW_CONCEPT, body=concept))
def modify_concept(self, context, concept, to_add=None, to_remove=None, modify_source=False):
"""
@@ -324,40 +308,17 @@ class SheerkaConceptManager(BaseService):
# modify the metadata. Almost all ConceptMetadata attributes except variables and props
new_concept = sheerka.new_from_template(concept, concept.key) # reload from cache or database ?
res = self._update_concept(context, new_concept, to_add, to_remove)
if res is not None:
return res
# To update concept by first keyword and first regex
# first remove old first token and first regex entries
concepts_by_first_keyword, concepts_by_regex = self._remove_concept_first_token_and_first_regex(concept)
# recompute concepts by first tokens and concept by first regex
update_items_res = self.recompute_first_items(context, concept, [new_concept])
if not update_items_res.status:
return update_items_res
by_first_keyword, by_first_regex, resolved_by_first_keyword, compiled_by_first_regex = update_items_res.body
# and then update
init_ret_value = self.compute_concepts_by_first_item(context,
[new_concept],
False,
concepts_by_first_keyword,
concepts_by_regex)
if not init_ret_value.status:
return sheerka.ret(self.NAME, False, ErrorConcept(init_ret_value.value))
concepts_by_first_keyword, concepts_by_regex = init_ret_value.body
# computes resolved concepts_by_first_keyword
init_ret_value = self.resolve_concepts_by_first_keyword(context,
concepts_by_first_keyword,
{new_concept.id: new_concept})
if not init_ret_value.status:
return sheerka.ret(self.NAME, False, ErrorConcept(init_ret_value.value))
resolved_concepts_by_first_keyword = init_ret_value.body
# compile new regex
compile_ret = self.compile_concepts_by_first_regex(context, concepts_by_regex)
if not compile_ret.status:
return sheerka.ret(self.NAME, False, ErrorConcept(compile_ret.value))
compiled_concepts_by_first_regex = compile_ret.body
# update concept that referenced the old concept and clear old references
# update concepts that referenced the old concept and clear old references
self.update_references(context, concept, new_concept, to_add)
for ref in self.compute_references(concept):
om.delete(self.CONCEPTS_REFERENCES_ENTRY, ref, concept.id)
@@ -368,13 +329,11 @@ class SheerkaConceptManager(BaseService):
# everything is ok, update the caches
om.update_concept(concept, new_concept)
om.put(self.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, False, concepts_by_first_keyword)
om.put(self.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY, False, resolved_concepts_by_first_keyword)
om.put(self.CONCEPTS_BY_REGEX_ENTRY, False, {k.serialize(): v for k, v in concepts_by_regex.items()})
# update the compiled regex
self.compiled_concepts_by_regex.clear()
self.compiled_concepts_by_regex.extend(compiled_concepts_by_first_regex)
self.update_first_items_caches(context,
by_first_keyword,
by_first_regex,
resolved_by_first_keyword,
compiled_by_first_regex)
# everything seems to be fine. Update the list of attributes
# Caution. Must be done AFTER update_concept()
@@ -386,8 +345,7 @@ class SheerkaConceptManager(BaseService):
self._update_concept(context, concept, to_add, to_remove)
# KSI 2021-02-16 publish the modification of the concept only when someone needs it
ret = sheerka.ret(self.NAME, True, sheerka.new(BuiltinConcepts.NEW_CONCEPT, body=new_concept))
return ret
return sheerka.ret(self.NAME, True, sheerka.new(BuiltinConcepts.NEW_CONCEPT, body=new_concept))
def remove_concept(self, context, concept):
"""
@@ -410,30 +368,19 @@ class SheerkaConceptManager(BaseService):
refs_instances = [sheerka.new_from_template(c, c.key) for c in [self.get_by_id(ref) for ref in refs]]
return sheerka.ret(self.NAME, False, sheerka.err(ConceptIsReferenced(refs_instances)))
concepts_by_first_keyword, concepts_by_regex = self._remove_concept_first_token_and_first_regex(concept)
# computes resolved concepts_by_first_keyword
init_ret_value = self.resolve_concepts_by_first_keyword(context, concepts_by_first_keyword)
if not init_ret_value.status:
return sheerka.ret(self.NAME, False, ErrorConcept(init_ret_value.value))
resolved_concepts_by_first_keyword = init_ret_value.body
# compile new regex
compile_ret = self.compile_concepts_by_first_regex(context, concepts_by_regex)
if not compile_ret.status:
return sheerka.ret(self.NAME, False, ErrorConcept(compile_ret.value))
compiled_concepts_by_first_regex = compile_ret.body
# recompute concepts by first tokens and concept by first regex
update_items_res = self.recompute_first_items(context, concept, None)
if not update_items_res.status:
return update_items_res
by_first_keyword, by_first_regex, resolved_by_first_keyword, compiled_by_first_regex = update_items_res.body
# everything seems fine. I can commit the modification and remove
om.remove_concept(concept)
om.put(self.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, False, concepts_by_first_keyword)
om.put(self.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY, False, resolved_concepts_by_first_keyword)
om.put(self.CONCEPTS_BY_REGEX_ENTRY, False, {k.serialize(): v for k, v in concepts_by_regex.items()})
# update the compiled regex
self.compiled_concepts_by_regex.clear()
self.compiled_concepts_by_regex.extend(compiled_concepts_by_first_regex)
self.update_first_items_caches(context,
by_first_keyword,
by_first_regex,
resolved_by_first_keyword,
compiled_by_first_regex)
sheerka.publish(context, EVENT_CONCEPT_DELETED, concept)
return sheerka.ret(self.NAME, True, sheerka.new(BuiltinConcepts.SUCCESS))
@@ -568,9 +515,9 @@ class SheerkaConceptManager(BaseService):
"""
Updates all the concepts that reference concept
:param context:
:param concept:
:param modified_concept:
:param modifications:
:param concept: Old version of the concept
:param modified_concept: new version of the concept
:param modifications: what are the modification
:return:
"""
@@ -582,11 +529,12 @@ class SheerkaConceptManager(BaseService):
# remove the grammar entry so that it can be recreated
self.sheerka.om.delete(self.CONCEPTS_BNF_DEFINITIONS_ENTRY, concept_id)
to_update = self.get_by_id(concept_id)
metadata = to_update.get_metadata()
# reset the bnf definition if needed
if modified_concept:
if self.has_id(concept_id):
to_update = self.get_by_id(concept_id)
metadata = to_update.get_metadata()
if self.has_id(concept_id): # reset only it the bnf definition is in cache
if metadata.definition_type == DEFINITION_TYPE_BNF and self._name_has_changed(modifications):
tokens = list(Tokenizer(metadata.definition))
modified = False
@@ -601,6 +549,19 @@ class SheerkaConceptManager(BaseService):
to_update.get_metadata().definition = core.utils.get_text_from_tokens(tokens)
to_update.set_bnf(None)
# update concept_by_first_token
if metadata.definition_type == DEFINITION_TYPE_BNF:
# recompute concepts by first tokens and concept by first regex
res = self.recompute_first_items(context, concept, [concept])
if not res.status:
return res
by_first_keyword, by_first_regex, resolved_by_first_keyword, compiled_by_first_regex = res.body
self.update_first_items_caches(context,
by_first_keyword,
by_first_regex,
resolved_by_first_keyword,
compiled_by_first_regex)
def compute_references(self, concept):
"""
We need to keep a track of all concepts used by the current concept
@@ -723,6 +684,8 @@ class SheerkaConceptManager(BaseService):
else:
return sheerka.ret(self.NAME, False, sheerka.err(UnknownAttribute(k)))
core.utils.remove_list_from_list(concept.get_metadata().variables, variables_to_remove)
if concept.get_all_attributes():
core.utils.remove_list_from_list(concept.get_all_attributes(), [v[0] for v in variables_to_remove])
concept.get_metadata().key = None
if self._definition_has_changed(to_add) and concept.get_metadata().definition_type == DEFINITION_TYPE_BNF:
@@ -1046,3 +1009,78 @@ class SheerkaConceptManager(BaseService):
def get_concepts_bnf_definitions(self):
return self.sheerka.om.current_cache_manager().caches[self.CONCEPTS_BNF_DEFINITIONS_ENTRY].cache
def recompute_first_items(self, context, old_concept, new_concepts):
"""
Recompute
concepts fy first items
resolved concept by first items
concepts by first regex
compiled concepts by first regex
Do not update anything
:param context:
:param old_concept:
:param new_concepts:
:return:
"""
sheerka = context.sheerka
if old_concept and not new_concepts:
# remove
modified_concepts = None
by_first_keyword, by_first_regex = self._remove_concept_first_token_and_first_regex(old_concept)
elif old_concept and new_concepts:
# remove
by_first_keyword, by_first_regex = self._remove_concept_first_token_and_first_regex(old_concept)
# and then update
init_ret_value = self.compute_concepts_by_first_item(context,
new_concepts,
False,
by_first_keyword,
by_first_regex)
if not init_ret_value.status:
return sheerka.ret(self.NAME, False, ErrorConcept(init_ret_value.value))
by_first_keyword, by_first_regex = init_ret_value.body
modified_concepts = {new_concept.id: new_concept for new_concept in new_concepts}
elif not old_concept and new_concepts:
# only update
modified_concepts = None
init_ret_value = self.compute_concepts_by_first_item(context, new_concepts, True)
if not init_ret_value.status:
return sheerka.ret(self.NAME, False, ErrorConcept(init_ret_value.value))
by_first_keyword, by_first_regex = init_ret_value.body
# computes resolved concepts_by_first_keyword
init_ret_value = self.resolve_concepts_by_first_keyword(context, by_first_keyword, modified_concepts)
if not init_ret_value.status:
return sheerka.ret(self.NAME, False, ErrorConcept(init_ret_value.value))
resolved_by_first_keyword = init_ret_value.body
# compile new regex
compile_ret = self.compile_concepts_by_first_regex(context, by_first_regex)
if not compile_ret.status:
return sheerka.ret(self.NAME, False, ErrorConcept(compile_ret.value))
compiled_by_first_regex = compile_ret.body
return sheerka.ret(self.NAME,
True,
(by_first_keyword, by_first_regex, resolved_by_first_keyword, compiled_by_first_regex))
def update_first_items_caches(self,
context,
by_first_keyword,
by_first_regex,
resolved_by_first_keyword,
compiled_by_first_regex):
om = context.sheerka.om
om.put(self.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, False, by_first_keyword)
om.put(self.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY, False, resolved_by_first_keyword)
om.put(self.CONCEPTS_BY_REGEX_ENTRY, False, {k.serialize(): v for k, v in by_first_regex.items()})
# update the compiled regex
self.compiled_concepts_by_regex.clear()
self.compiled_concepts_by_regex.extend(compiled_by_first_regex)
@@ -4,7 +4,7 @@ from core.builtin_concepts import BuiltinConcepts
from core.builtin_helpers import expect_one, only_successful, evaluate, ensure_concept
from core.concept import Concept, DoNotResolve, ConceptParts, InfiniteRecursionResolved, AllConceptParts, \
concept_part_value
from core.global_symbols import NotInit
from core.global_symbols import NotInit, CURRENT_OBJ
from core.rule import Rule
from core.sheerka.services.SheerkaConceptManager import SheerkaConceptManager
from core.sheerka.services.SheerkaExecute import ParserInput
@@ -535,6 +535,8 @@ class SheerkaEvaluateConcept(BaseService):
if context.sheerka.isa(concept, context.sheerka.new(BuiltinConcepts.AUTO_EVAL)):
sub_context.protected_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
sub_context.add_to_short_term_memory(CURRENT_OBJ, concept)
try:
self.initialize_concept_asts(sub_context, concept)
except ChickenAndEggException as ex:
+9 -10
View File
@@ -280,17 +280,16 @@ for x in xx__concepts__xx:
sub_context.protected_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
errors = []
for element_id in ids:
concept = self.sheerka.get_by_id(element_id)
if len(concept.get_metadata().variables) == 0:
# The concepts are directly taken from Sheerka.get_by_id, so variable cannot be filled
# It's the reason why we only evaluate concept with no variable
evaluated = self.sheerka.evaluate_concept(sub_context, concept)
if context.sheerka.is_success(evaluated):
result.append(evaluated)
concept = self.sheerka.fast_resolve((None, element_id))
if concept:
if len(concept.get_metadata().variables) == 0:
evaluated = self.sheerka.evaluate_concept(sub_context, concept)
if context.sheerka.is_success(evaluated):
result.append(evaluated)
else:
errors.append(evaluated)
else:
errors.append(evaluated)
else:
result.append(concept)
result.append(concept)
sub_context.add_values(return_value=result)
sub_context.add_values(errors=errors)
return result
+32 -4
View File
@@ -12,6 +12,15 @@ from core.sheerka.services.sheerka_service import BaseService, ServiceObj
class MemoryObject(ServiceObj):
obj: object
def __eq__(self, other):
if not isinstance(other, MemoryObject):
return False
return self.obj == other.obj and self.event_id == other.event_id
def __hash__(self):
return hash((self.event_id, self.obj))
class SheerkaMemory(BaseService):
NAME = "Memory"
@@ -26,6 +35,7 @@ class SheerkaMemory(BaseService):
def initialize(self):
self.sheerka.bind_service_method(self.get_from_short_term_memory, False, visible=False)
self.sheerka.bind_service_method(self.get_all_short_term_memory, False, visible=False)
self.sheerka.bind_service_method(self.add_to_short_term_memory, True, visible=False)
self.sheerka.bind_service_method(self.remove_context, True, as_name="clear_short_term_memory", visible=False)
self.sheerka.bind_service_method(self.add_to_memory, True, visible=False)
@@ -62,8 +72,26 @@ class SheerkaMemory(BaseService):
context = context.get_parent()
def get_all_short_term_memory(self, context):
return self.short_term_objects.get(context.id)
def get_all_short_term_memory(self, context, recursive=False):
id_to_use = context.id if context else self.GLOBAL
if not recursive:
return self.short_term_objects.get(id_to_use)
all_vars = {}
while True:
try:
all_vars.update(self.short_term_objects.cache[id_to_use])
except KeyError:
pass
if id_to_use == self.GLOBAL:
break
else:
context = context.get_parent()
id_to_use = context.id if context else self.GLOBAL
return all_vars
def add_to_short_term_memory(self, context, key, value):
if context:
@@ -99,12 +127,12 @@ class SheerkaMemory(BaseService):
if last is NotFound:
self.sheerka.om.put(SheerkaMemory.OBJECTS_ENTRY, key, MemoryObject(context.event.get_digest(), concept))
return
if not isinstance(last, list) and last.obj == concept:
self.sheerka.om.delete(SheerkaMemory.OBJECTS_ENTRY, key, last)
self.sheerka.om.put(SheerkaMemory.OBJECTS_ENTRY, key, MemoryObject(context.event.get_digest(), concept))
return
if isinstance(last, list) and last[-1].obj == concept:
self.sheerka.om.delete(SheerkaMemory.OBJECTS_ENTRY, key, last[-1])
self.sheerka.om.put(SheerkaMemory.OBJECTS_ENTRY, key, MemoryObject(context.event.get_digest(), concept))