kodjo/Sheerka-Old
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Implemented SheerkaOntology

Browse Source
This commit is contained in:
Kodjo Sossouvi
2021-01-11 15:36:03 +01:00
parent e3c2adb533
commit e26c83a825
119 changed files with 6876 additions and 2002 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/core/sheerka/services/SheerkaAdmin.py
+20 -6
View File
@@ -1,5 +1,6 @@
import sys
import time
from operator import attrgetter
from os import path
from core.builtin_concepts_ids import BuiltinConcepts, BuiltinContainers
@@ -27,13 +28,16 @@ class SheerkaAdmin(BaseService):
self.sheerka.bind_service_method(self.extended_isinstance, False)
self.sheerka.bind_service_method(self.is_container, False)
self.sheerka.bind_service_method(self.format_rules, False)
self.sheerka.bind_service_method(self.admin_push_ontology, True, as_name="push_ontology")
self.sheerka.bind_service_method(self.admin_pop_ontology, True, as_name="pop_ontology")
self.sheerka.bind_service_method(self.ontologies, False)
def caches_names(self):
"""
Returns the name of all the caches
:return:
"""
return list(self.sheerka.cache_manager.caches.keys())
return list(self.sheerka.om.current_cache_manager().caches.keys())
def cache(self, name, *keys):
"""
@@ -42,13 +46,13 @@ class SheerkaAdmin(BaseService):
:param keys: look for a specific key. May ask to sdp if the key is not in cache
:return:
"""
if name not in self.sheerka.cache_manager.caches:
if name not in self.sheerka.om.current_cache_manager().caches:
return self.sheerka.new(BuiltinConcepts.NOT_FOUND, body={"cache": name})
if not keys:
return self.sheerka.cache_manager.caches[name].cache.copy()
return self.sheerka.om.current_cache_manager().caches[name].cache.copy()
return {key: self.sheerka.cache_manager.get(name, key) for key in keys}
return {key: self.sheerka.om.get(name, key) for key in keys}
def restore(self, concept_file=CONCEPTS_FILE_TO_USE):
"""
@@ -119,7 +123,8 @@ class SheerkaAdmin(BaseService):
raise e
def concepts(self):
return self.sheerka.new(BuiltinConcepts.TO_LIST, body=self.sheerka.sdp.list(self.sheerka.CONCEPTS_BY_ID_ENTRY))
concepts = sorted(self.sheerka.om.list(self.sheerka.CONCEPTS_BY_ID_ENTRY), key=lambda item: int(item.id))
return self.sheerka.new(BuiltinConcepts.TO_LIST, body=concepts)
def desc(self, *concepts):
ensure_concept(*concepts)
@@ -146,7 +151,6 @@ class SheerkaAdmin(BaseService):
def format_rules(self):
return self.sheerka.new(BuiltinConcepts.TO_LIST, items=self.sheerka.get_format_rules())
def extended_isinstance(self, a, b):
"""
switch between sheerka.isinstance and builtin.isinstance
@@ -171,3 +175,13 @@ class SheerkaAdmin(BaseService):
return False
return obj.key in BuiltinContainers
def admin_push_ontology(self, context, name):
return self.sheerka.push_ontology(context, name, False)
def admin_pop_ontology(self):
return self.sheerka.pop_ontology()
def ontologies(self):
ontologies = self.sheerka.om.ontologies_names
return self.sheerka.new(BuiltinConcepts.TO_LIST, body=ontologies)
src/core/sheerka/services/SheerkaComparisonManager.py
+28 -20
View File
@@ -3,8 +3,9 @@ from dataclasses import dataclass
from cache.Cache import Cache
from cache.ListCache import ListCache
from core.builtin_concepts import BuiltinConcepts
from core.global_symbols import EVENT_CONCEPT_PRECEDENCE_MODIFIED, EVENT_RULE_PRECEDENCE_MODIFIED, RULE_COMPARISON_CONTEXT, \
CONCEPT_COMPARISON_CONTEXT
from core.global_symbols import EVENT_CONCEPT_PRECEDENCE_MODIFIED, EVENT_RULE_PRECEDENCE_MODIFIED, \
RULE_COMPARISON_CONTEXT, \
CONCEPT_COMPARISON_CONTEXT, NotFound
from core.builtin_helpers import ensure_concept_or_rule
from core.concept import Concept
from core.sheerka.services.SheerkaRuleManager import SheerkaRuleManager
@@ -136,8 +137,8 @@ class SheerkaComparisonManager(BaseService):
def _add_comparison(self, context, comparison_obj):
key = self._compute_key(comparison_obj.property, comparison_obj.context)
previous = self.sheerka.cache_manager.get(self.COMPARISON_ENTRY, key)
new = previous.copy() if previous else []
previous = self.sheerka.om.get(self.COMPARISON_ENTRY, key)
new = previous.copy() if isinstance(previous, list) else []
for co in new:
if co.property == comparison_obj.property and \
@@ -176,10 +177,10 @@ class SheerkaComparisonManager(BaseService):
chicken_an_egg = self.sheerka.new(BuiltinConcepts.CHICKEN_AND_EGG, body=concepts_in_cycle)
return self.sheerka.ret(self.NAME, False, chicken_an_egg)
self.sheerka.cache_manager.put(self.COMPARISON_ENTRY, key, comparison_obj)
self.sheerka.cache_manager.put(self.RESOLVED_COMPARISON_ENTRY, key, self._compute_weights(new,
lesser_objs_ids,
greatest_objs_ids))
self.sheerka.om.put(self.COMPARISON_ENTRY, key, comparison_obj)
self.sheerka.om.put(self.RESOLVED_COMPARISON_ENTRY, key, self._compute_weights(new,
lesser_objs_ids,
greatest_objs_ids))
if comparison_obj.property == BuiltinConcepts.PRECEDENCE:
if comparison_obj.context == CONCEPT_COMPARISON_CONTEXT:
@@ -190,11 +191,11 @@ class SheerkaComparisonManager(BaseService):
return self.sheerka.ret(self.NAME, True, self.sheerka.new(BuiltinConcepts.SUCCESS))
def initialize(self):
cache = ListCache(default=lambda k: self.sheerka.sdp.get(self.COMPARISON_ENTRY, k))
self.sheerka.cache_manager.register_cache(self.COMPARISON_ENTRY, cache, True, True)
cache = ListCache().auto_configure(self.COMPARISON_ENTRY)
self.sheerka.om.register_cache(self.COMPARISON_ENTRY, cache, True, True)
cache = Cache()
self.sheerka.cache_manager.register_cache(self.RESOLVED_COMPARISON_ENTRY, cache, persist=False)
cache = Cache().auto_configure(self.RESOLVED_COMPARISON_ENTRY)
self.sheerka.om.register_cache(self.RESOLVED_COMPARISON_ENTRY, cache, persist=False)
self.sheerka.bind_service_method(self.set_is_greater_than, True)
self.sheerka.bind_service_method(self.set_is_less_than, True)
@@ -325,19 +326,26 @@ class SheerkaComparisonManager(BaseService):
return self._get_partition(weighted_concept)
def get_concepts_weights(self, prop_name, comparison_context="#"):
weighted_concepts = self.sheerka.cache_manager.get(
self.RESOLVED_COMPARISON_ENTRY,
self._compute_key(prop_name, comparison_context))
# KSI 2021-01-10 This implementation seems to be too complicated
# Chances are that there is a better way to implement this.
# Note that I don't want to use a DictionaryCache for the RESOLVED_COMPARISON_ENTRY
# as I don't need to have all the keys in memory at the same time
# Anyway...
if weighted_concepts is None:
key = self._compute_key(prop_name, comparison_context)
entries = self.sheerka.cache_manager.get(self.COMPARISON_ENTRY, key)
# If the weighted_concepts is in the TOP LAYER cache, we can use it
key_to_use = self._compute_key(prop_name, comparison_context)
if self.sheerka.om.current_cache_manager().has(self.RESOLVED_COMPARISON_ENTRY, key_to_use):
weighted_concepts = self.sheerka.om.get(self.RESOLVED_COMPARISON_ENTRY, key_to_use)
else:
# otherwise, either it's not computed yet or it does not include the info of the current layer
# In both case, it is safer to recompute the weights
entries = self.sheerka.om.list_by_key(self.COMPARISON_ENTRY, key_to_use)
if entries is None:
return {}
weighted_concepts = {} # Why not put it in cache ???
else:
weighted_concepts = self._compute_weights(entries)
self.sheerka.cache_manager.put(self.RESOLVED_COMPARISON_ENTRY, key, weighted_concepts)
self.sheerka.om.put(self.RESOLVED_COMPARISON_ENTRY, key_to_use, weighted_concepts)
return weighted_concepts
src/core/sheerka/services/SheerkaConceptManager.py
+47 -53
View File
@@ -8,10 +8,9 @@ from cache.SetCache import SetCache
from core.builtin_concepts import ErrorConcept
from core.builtin_concepts_ids import BuiltinConcepts, AllBuiltinConcepts, BuiltinUnique
from core.builtin_helpers import ensure_concept
from core.concept import Concept, DEFINITION_TYPE_DEF, DEFINITION_TYPE_BNF, freeze_concept_attrs, NotInit, \
ConceptMetadata
from core.concept import Concept, DEFINITION_TYPE_DEF, DEFINITION_TYPE_BNF, freeze_concept_attrs, ConceptMetadata
from core.error import ErrorObj
from core.global_symbols import EVENT_CONCEPT_CREATED
from core.global_symbols import EVENT_CONCEPT_CREATED, NotInit, NotFound
from core.sheerka.services.sheerka_service import BaseService
from core.tokenizer import Tokenizer, TokenKind
from sdp.sheerkaDataProvider import SheerkaDataProviderDuplicateKeyError
@@ -101,34 +100,28 @@ class SheerkaConceptManager(BaseService):
self.sheerka.bind_service_method(self.get_by_name, False, visible=False)
self.sheerka.bind_service_method(self.get_by_hash, False, visible=False)
self.sheerka.bind_service_method(self.get_by_id, False, visible=False)
self.sheerka.bind_service_method(self.not_is_variable, False, visible=False)
self.sheerka.bind_service_method(self.is_not_a_variable, False, visible=False)
def params(cache_name):
return {
'default': lambda k: self.sheerka.sdp.get(cache_name, k),
'extend_exists': lambda k: self.sheerka.sdp.exists(cache_name, k)
}
register_concept_cache = self.sheerka.om.register_concept_cache
register_concept_cache = self.sheerka.cache_manager.register_concept_cache
cache = Cache(**params(self.CONCEPTS_BY_ID_ENTRY))
cache = Cache().auto_configure(self.CONCEPTS_BY_ID_ENTRY)
register_concept_cache(self.CONCEPTS_BY_ID_ENTRY, cache, lambda c: c.id, True)
cache = ListIfNeededCache(**params(self.CONCEPTS_BY_KEY_ENTRY))
cache = ListIfNeededCache().auto_configure(self.CONCEPTS_BY_KEY_ENTRY)
register_concept_cache(self.CONCEPTS_BY_KEY_ENTRY, cache, lambda c: c.key, True)
cache = ListIfNeededCache(**params(self.CONCEPTS_BY_NAME_ENTRY))
cache = ListIfNeededCache().auto_configure(self.CONCEPTS_BY_NAME_ENTRY)
register_concept_cache(self.CONCEPTS_BY_NAME_ENTRY, cache, lambda c: c.name, True)
cache = ListIfNeededCache(**params(self.CONCEPTS_BY_HASH_ENTRY))
cache = ListIfNeededCache().auto_configure(self.CONCEPTS_BY_HASH_ENTRY)
register_concept_cache(self.CONCEPTS_BY_HASH_ENTRY, cache, lambda c: c.get_definition_hash(), True)
cache = SetCache(default=lambda k: self.sheerka.sdp.get(self.CONCEPTS_REFERENCES_ENTRY, k))
self.sheerka.cache_manager.register_cache(self.CONCEPTS_REFERENCES_ENTRY, cache)
cache = SetCache().auto_configure(self.CONCEPTS_REFERENCES_ENTRY)
self.sheerka.om.register_cache(self.CONCEPTS_REFERENCES_ENTRY, cache)
def initialize_deferred(self, context, is_first_time):
if is_first_time:
self.sheerka.cache_manager.put(self.sheerka.OBJECTS_IDS_ENTRY, self.USER_CONCEPTS_IDS, 1000)
self.sheerka.om.put(self.sheerka.OBJECTS_IDS_ENTRY, self.USER_CONCEPTS_IDS, 1000)
def initialize_builtin_concepts(self):
"""
@@ -146,11 +139,11 @@ class SheerkaConceptManager(BaseService):
concept.get_metadata().is_unique = True
concept.get_metadata().is_evaluated = True
from_db = self.sheerka.cache_manager.get(self.CONCEPTS_BY_KEY_ENTRY, concept.get_metadata().key)
if from_db is None:
from_db = self.sheerka.om.get(self.CONCEPTS_BY_KEY_ENTRY, concept.get_metadata().key)
if from_db is NotFound:
# self.init_log.debug(f"'{concept.name}' concept is not found in db. Adding.")
self.set_id_if_needed(concept, True)
self.sheerka.cache_manager.add_concept(concept)
self.sheerka.om.add_concept(concept)
else:
# self.init_log.debug(f"Found concept '{from_db}' in db. Updating.")
concept.update_from(from_db)
@@ -173,9 +166,9 @@ class SheerkaConceptManager(BaseService):
concept.init_key()
init_bnf_ret_value = None
cache_manager = sheerka.cache_manager
ontology = sheerka.om
if cache_manager.exists(self.CONCEPTS_BY_HASH_ENTRY, concept.get_definition_hash()):
if ontology.exists(self.CONCEPTS_BY_HASH_ENTRY, concept.get_definition_hash()):
error = SheerkaDataProviderDuplicateKeyError(self.CONCEPTS_BY_KEY_ENTRY + "." + concept.key, concept)
return sheerka.ret(
self.NAME,
@@ -186,9 +179,6 @@ class SheerkaConceptManager(BaseService):
# set id before saving in db
sheerka.set_id_if_needed(concept, False)
# freeze attributes
freeze_concept_attrs(concept)
# check if the bnf definition is correctly computed
try:
self.bnp.ensure_bnf(context, concept)
@@ -196,7 +186,7 @@ class SheerkaConceptManager(BaseService):
return sheerka.ret(self.NAME, False, ex.args[0])
# compute new concepts_by_first_keyword
init_ret_value = self.bnp.get_concepts_by_first_token(context, [concept], True)
init_ret_value = self.bnp.compute_concepts_by_first_token(context, [concept], True)
if not init_ret_value.status:
return sheerka.ret(self.NAME, False, ErrorConcept(init_ret_value.value))
concepts_by_first_keyword = init_ret_value.body
@@ -208,18 +198,20 @@ class SheerkaConceptManager(BaseService):
resolved_concepts_by_first_keyword = init_ret_value.body
# if everything is fine
freeze_concept_attrs(concept)
concept.freeze_definition_hash()
cache_manager.add_concept(concept)
cache_manager.put(sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, False, concepts_by_first_keyword)
cache_manager.put(sheerka.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY, False, resolved_concepts_by_first_keyword)
ontology.add_concept(concept)
ontology.put(sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, False, concepts_by_first_keyword)
ontology.put(sheerka.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY, False, resolved_concepts_by_first_keyword)
if concept.get_metadata().definition_type == DEFINITION_TYPE_DEF and concept.get_metadata().definition != concept.name:
# allow search by definition when definition relevant
cache_manager.put(self.sheerka.CONCEPTS_BY_NAME_ENTRY, concept.get_metadata().definition, concept)
ontology.put(self.sheerka.CONCEPTS_BY_NAME_ENTRY, concept.get_metadata().definition, concept)
# update references
for ref in self.compute_references(concept):
cache_manager.put(self.CONCEPTS_REFERENCES_ENTRY, ref, concept.id)
ontology.put(self.CONCEPTS_REFERENCES_ENTRY, ref, concept.id)
# TODO : this line seems to be useless
# The grammar is never reset
@@ -246,7 +238,7 @@ class SheerkaConceptManager(BaseService):
# to_add is a dictionary
# to_add = {
# 'meta' : {<key, value>} of metadata to add/update,
# 'meta' : {<key, value>} of metadata to update,
# 'props' : {<key, value>} of properties to add/update,
# 'variables': {<key, value>} of variables to add/update,
# }
@@ -259,12 +251,12 @@ class SheerkaConceptManager(BaseService):
# }
#
sheerka = self.sheerka
cache_manager = self.sheerka.cache_manager
cache_manager = self.sheerka.om
if not to_add and not to_remove:
return sheerka.ret(self.NAME, False, sheerka.err(NoModificationFound(concept)))
if not sheerka.cache_manager.exists(self.CONCEPTS_BY_ID_ENTRY, concept.id):
if not sheerka.om.exists(self.CONCEPTS_BY_ID_ENTRY, concept.id):
return sheerka.ret(self.NAME, False, sheerka.new(BuiltinConcepts.UNKNOWN_CONCEPT, body=concept))
# modify the metadata. Almost all ConceptMetadata attributes except variables and props
@@ -274,8 +266,6 @@ class SheerkaConceptManager(BaseService):
if res is not None:
return res
freeze_concept_attrs(new_concept)
# To update concept by first keyword
# first remove the old references
keywords = self.bnp.get_first_tokens(sheerka, concept) # keyword of the old concept
@@ -289,7 +279,7 @@ class SheerkaConceptManager(BaseService):
pass
# and then update
init_ret_value = self.bnp.get_concepts_by_first_token(context, [new_concept], False, concepts_by_first_keyword)
init_ret_value = self.bnp.compute_concepts_by_first_token(context, [new_concept], False, concepts_by_first_keyword)
if not init_ret_value.status:
return sheerka.ret(self.NAME, False, ErrorConcept(init_ret_value.value))
concepts_by_first_keyword = init_ret_value.body
@@ -316,6 +306,10 @@ class SheerkaConceptManager(BaseService):
cache_manager.put(sheerka.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY, False,
resolved_concepts_by_first_keyword)
# everything seems to be fine. Update the list of attributes
# Caution. Must be done AFTER update_concept()
freeze_concept_attrs(new_concept)
# TODO : update when definition_type = DEFINITION_TYPE_DEF : have a look at update_references() below
# TODO : Update concepts grammars : have a look at update_references() below
if modify_source:
@@ -332,13 +326,13 @@ class SheerkaConceptManager(BaseService):
:return:
"""
sheerka = context.sheerka
refs = self.sheerka.cache_manager.get(self.CONCEPTS_REFERENCES_ENTRY, concept.id)
if refs:
refs = self.sheerka.om.get(self.CONCEPTS_REFERENCES_ENTRY, concept.id)
if refs is not NotFound:
refs_instances = [sheerka.new_from_template(c, c.key) for c in [self.get_by_id(ref) for ref in refs]]
return sheerka.ret(self.NAME, False, sheerka.err(ConceptIsReferenced(refs_instances)))
try:
sheerka.cache_manager.remove_concept(concept)
sheerka.om.remove_concept(concept)
return sheerka.ret(self.NAME, True, sheerka.new(BuiltinConcepts.SUCCESS))
except ConceptNotFound as ex:
return sheerka.ret(self.NAME, False, sheerka.err(ex))
@@ -387,7 +381,7 @@ class SheerkaConceptManager(BaseService):
return
entry_key = self.BUILTIN_CONCEPTS_IDS if is_builtin else self.USER_CONCEPTS_IDS
obj.get_metadata().id = str(self.sheerka.cache_manager.get(self.sheerka.OBJECTS_IDS_ENTRY, entry_key))
obj.get_metadata().id = str(self.sheerka.om.get(self.sheerka.OBJECTS_IDS_ENTRY, entry_key))
# self.log.debug(f"Setting id '{obj.metadata.id}' to concept '{obj.metadata.name}'.")
def get_by_key(self, concept_key, concept_id=None):
@@ -412,7 +406,7 @@ class SheerkaConceptManager(BaseService):
"""
if concept_id is None:
return False
return self.sheerka.cache_manager.has(self.CONCEPTS_BY_ID_ENTRY, concept_id)
return self.sheerka.om.current_cache_manager().has(self.CONCEPTS_BY_ID_ENTRY, concept_id)
def has_key(self, concept_key):
"""
@@ -421,7 +415,7 @@ class SheerkaConceptManager(BaseService):
:param concept_key:
:return:
"""
return self.sheerka.cache_manager.has(self.CONCEPTS_BY_KEY_ENTRY, concept_key)
return self.sheerka.om.current_cache_manager().has(self.CONCEPTS_BY_KEY_ENTRY, concept_key)
def has_name(self, concept_name):
"""
@@ -430,7 +424,7 @@ class SheerkaConceptManager(BaseService):
:param concept_name:
:return:
"""
return self.sheerka.cache_manager.has(self.CONCEPTS_BY_NAME_ENTRY, concept_name)
return self.sheerka.om.current_cache_manager().has(self.CONCEPTS_BY_NAME_ENTRY, concept_name)
def has_hash(self, concept_hash):
"""
@@ -439,7 +433,7 @@ class SheerkaConceptManager(BaseService):
:param concept_hash:
:return:
"""
return self.sheerka.cache_manager.has(self.CONCEPTS_BY_HASH_ENTRY, concept_hash)
return self.sheerka.om.current_cache_manager().has(self.CONCEPTS_BY_HASH_ENTRY, concept_hash)
def internal_get(self, index_name, key, cache_name, concept_id=None):
"""
@@ -454,8 +448,8 @@ class SheerkaConceptManager(BaseService):
if key is None:
return ErrorConcept(f"Concept '{key}' is undefined.")
concepts = self.sheerka.cache_manager.get(cache_name, key)
if concepts:
concepts = self.sheerka.om.get(cache_name, key)
if concepts is not NotFound:
if concept_id is None:
return concepts
@@ -479,13 +473,13 @@ class SheerkaConceptManager(BaseService):
:return:
"""
refs = self.sheerka.cache_manager.get(self.CONCEPTS_REFERENCES_ENTRY, concept.id)
if not refs:
refs = self.sheerka.om.get(self.CONCEPTS_REFERENCES_ENTRY, concept.id)
if refs is NotFound:
return
for concept_id in refs:
# remove the grammar entry so that it can be recreated
self.sheerka.cache_manager.delete(self.sheerka.CONCEPTS_GRAMMARS_ENTRY, concept_id)
self.sheerka.om.delete(self.sheerka.CONCEPTS_GRAMMARS_ENTRY, concept_id)
# reset the bnf definition if needed
if modified_concept:
@@ -527,13 +521,13 @@ class SheerkaConceptManager(BaseService):
return refs
def not_is_variable(self, name):
def is_not_a_variable(self, name):
"""
Given a name tells if it refers to a variable name
:param name:
:return:
"""
return not self.sheerka.cache_manager.get(self.sheerka.CONCEPTS_BY_NAME_ENTRY, name)
return self.sheerka.om.get(self.sheerka.CONCEPTS_BY_NAME_ENTRY, name) is NotFound
@staticmethod
def _name_has_changed(to_add):
src/core/sheerka/services/SheerkaConceptsAlgebra.py
+2 -3
View File
@@ -119,10 +119,9 @@ class SheerkaConceptsAlgebra(BaseService):
if nb_props == 0:
return res
concepts_service = self.sheerka.services[SheerkaConceptManager.NAME]
concepts_manager = self.sheerka.services[SheerkaConceptManager.NAME]
all_concepts = self.sheerka.cache_manager.copy(concepts_service.CONCEPTS_BY_ID_ENTRY).values() \
if self.sheerka.cache_manager.cache_only else self.sheerka.sdp.list(concepts_service.CONCEPTS_BY_ID_ENTRY)
all_concepts = self.sheerka.om.list(concepts_manager.CONCEPTS_BY_ID_ENTRY)
for c in all_concepts:
score = self._compute_score(c, concept, step_b=round(1 / nb_props, 2))
src/core/sheerka/services/SheerkaDebugManager.py
+33 -14
View File
@@ -3,17 +3,17 @@ import re
from dataclasses import dataclass
from core.builtin_concepts import BuiltinConcepts
from core.concept import Concept, NotInit
from core.builtin_helpers import evaluate_expression
from core.concept import Concept
from core.global_symbols import NotInit, NotFound
from core.sheerka.ExecutionContext import ExecutionContext
from core.sheerka.services.sheerka_service import BaseService
from core.utils import CONSOLE_COLORS_MAP as CCM, CONSOLE_COLUMNS, PRIMITIVES_TYPES
from core.utils import evaluate_expression, as_bag
from core.utils import as_bag
from parsers.BaseNodeParser import SourceCodeWithConceptNode, UnrecognizedTokensNode
pp = pprint.PrettyPrinter(indent=2, width=CONSOLE_COLUMNS)
NotFound = "** Not Found **"
class ConceptDebugObj:
def __init__(self, concept, **kwargs):
@@ -278,12 +278,21 @@ class SheerkaDebugManager(BaseService):
def __init__(self, sheerka):
super().__init__(sheerka)
self.activated = False # is debug activated
self.explicit = False # No need to activate context debug when debug mode is on
self.context_cache = set() # debug for specific context
self.variable_cache = set() # debug for specific variable
self.explicit = False # No need to activate context debug when debug mode is on # to remove ?
self.context_cache = set() # debug for specific context # to remove ?
self.variable_cache = set() # debug for specific variable # to remove ?
self.debug_vars_settings = []
self.debug_rules_settings = []
self.debug_concepts_settings = []
self.state_vars = [
"activated",
"explicit", # to remove ?
"context_cache", # to remove ?
"variable_cache", # to remove ?
"debug_vars_settings",
"debug_rules_settings",
"debug_concepts_settings"
]
def initialize(self):
# TO REMOVE ???
@@ -307,13 +316,7 @@ class SheerkaDebugManager(BaseService):
# self.sheerka.bind_service_method(self.get_debug_settings, False, as_name="debug_settings")
def initialize_deferred(self, context, is_first_time):
self.restore_values("activated",
"explicit",
"context_cache",
"variable_cache",
"debug_vars_settings",
"debug_rules_settings",
"debug_concepts_settings")
self.restore_state()
def reset(self):
"""
@@ -327,6 +330,12 @@ class SheerkaDebugManager(BaseService):
self.debug_rules_settings.clear()
self.debug_concepts_settings.clear()
def save_state(self, context):
self.store_values(context, *self.state_vars)
def restore_state(self):
self.restore_values(*self.state_vars)
def set_debug(self, context, value=True):
self.activated = value
self.sheerka.record_var(context, self.NAME, "activated", self.activated)
@@ -728,6 +737,16 @@ class SheerkaDebugManager(BaseService):
@staticmethod
def parse_debug_args(item_name, *args, **kwargs):
"""
Returns
i : item to debug. It can be a Concept, Rule or a variable
s : Service to debug (so far, it is SheerkaService)
m : Method within the serice
c_id : Context id
c_children : True / False to allow debugging of context children
d : Debug id
e : enable / disabled
"""
service, method_name, context_id, context_children, item, debug_id, enabled = None, None, None, False, None, None, True
if len(args) > 0:
if args[0] is None or args[0] == "":
src/core/sheerka/services/SheerkaEvaluateConcept.py
+3 -2
View File
@@ -2,8 +2,9 @@ from dataclasses import dataclass
from core.builtin_concepts import BuiltinConcepts
from core.builtin_helpers import expect_one, only_successful, parse_unrecognized, evaluate, ensure_concept
from core.concept import Concept, DoNotResolve, ConceptParts, InfiniteRecursionResolved, NotInit, AllConceptParts, \
from core.concept import Concept, DoNotResolve, ConceptParts, InfiniteRecursionResolved, AllConceptParts, \
concept_part_value
from core.global_symbols import NotInit
from core.sheerka.services.SheerkaConceptManager import SheerkaConceptManager
from core.sheerka.services.SheerkaExecute import ParserInput
from core.sheerka.services.sheerka_service import BaseService
@@ -605,7 +606,7 @@ class SheerkaEvaluateConcept(BaseService):
# # update the cache for concepts with no variables
# Cannot use cache. See the comment at the beginning of this method
# if len(concept.get_metadata().variables) == 0:
# self.sheerka.cache_manager.put(self.sheerka.CONCEPTS_BY_ID_ENTRY, concept.id, concept)
# self.sheerka.om.put(self.sheerka.CONCEPTS_BY_ID_ENTRY, concept.id, concept)
if not concept.get_metadata().is_builtin:
self.sheerka.register_object(sub_context, concept.name, concept)
src/core/sheerka/services/SheerkaEventManager.py
+14 -3
View File
@@ -1,5 +1,5 @@
from threading import RLock
from core.global_symbols import NotFound
from core.sheerka.services.sheerka_service import BaseService
@@ -19,6 +19,16 @@ class SheerkaEventManager(BaseService):
self.sheerka.bind_service_method(self.subscribe, True, visible=False)
self.sheerka.bind_service_method(self.publish, True, visible=False)
def save_state(self, context):
with self._lock:
copy = self.subscribers.copy()
self.sheerka.record_internal_var(context, self.NAME, "subscribers", copy)
def restore_state(self):
with self._lock:
if (from_cache := self.sheerka.load_internal_var(self.NAME, "subscribers")) is not NotFound:
self.subscribers = from_cache
def subscribe(self, topic, callback):
"""
To subscribe to a topic, just give the callback to call
@@ -51,9 +61,10 @@ class SheerkaEventManager(BaseService):
except KeyError:
pass
def reset_topic(self, topic):
def test_only_reset_topic(self, topic):
"""
Remove all subsccribers from a given topic
Remove all subscribers from a given topic
TO REMOVE once sheerka ontology is fully implemented
:param topic:
:return:
"""
src/core/sheerka/services/SheerkaExecute.py
+7 -3
View File
@@ -1,6 +1,8 @@
import core.utils
from cache.Cache import Cache
from cache.FastCache import FastCache
from core.builtin_concepts import BuiltinConcepts, ReturnValueConcept
from core.global_symbols import NotFound
from core.sheerka.services.sheerka_service import BaseService
from core.tokenizer import Tokenizer, TokenKind, Token
@@ -166,7 +168,7 @@ class SheerkaExecute(BaseService):
def __init__(self, sheerka):
super().__init__(sheerka)
self.pi_cache = Cache(default=lambda key: ParserInput(key), max_size=20)
self.pi_cache = FastCache(default=lambda key: ParserInput(key), max_size=20)
self.instantiated_evaluators = None
self.evaluators_by_name = None
@@ -192,10 +194,12 @@ class SheerkaExecute(BaseService):
def initialize(self):
self.sheerka.bind_service_method(self.execute, True)
self.sheerka.cache_manager.register_cache(self.PARSERS_INPUTS_ENTRY, self.pi_cache, False)
self.reset_registered_evaluators()
self.reset_registered_parsers()
def reset_state(self):
self.pi_cache.clear()
def reset_registered_evaluators(self):
# instantiate evaluators, once for all, only keep when it's enabled
self.instantiated_evaluators = [e_class() for e_class in self.sheerka.evaluators]
@@ -340,7 +344,7 @@ class SheerkaExecute(BaseService):
if tokens is None or self.pi_cache.has(text):
pi = self.pi_cache.get(text)
if pi is None: # when CacheManager.cache_only is True
if pi is NotFound: # when CacheManager.cache_only is True
pi = ParserInput(text)
self.pi_cache.put(text, pi)
return pi
src/core/sheerka/services/SheerkaFunctionsParametersHistory.py
+11 -10
View File
@@ -3,6 +3,7 @@ from operator import itemgetter
from typing import Tuple, Dict, List
from cache.Cache import Cache
from core.global_symbols import NotFound
from core.sheerka.services.sheerka_service import BaseService, ServiceObj
@@ -29,10 +30,10 @@ class SheerkaFunctionsParametersHistory(BaseService):
def __init__(self, sheerka):
super().__init__(sheerka)
self.cache = Cache(max_size=1024, default=lambda k: self.sheerka.sdp.get(self.FUNCTIONS_PARAMETERS_ENTRY, k))
def initialize(self):
self.sheerka.cache_manager.register_cache(self.FUNCTIONS_PARAMETERS_ENTRY, self.cache, True, True)
cache = Cache(max_size=1024).auto_configure(self.FUNCTIONS_PARAMETERS_ENTRY)
self.sheerka.om.register_cache(self.FUNCTIONS_PARAMETERS_ENTRY, cache, True, True)
return self
def record_function_parameter(self, context, func_name: str, param_number: int, param_value: str):
@@ -44,8 +45,11 @@ class SheerkaFunctionsParametersHistory(BaseService):
:param param_value:
:return:
"""
old = self.cache.get(func_name)
if old is not None:
old = self.sheerka.om.get(self.FUNCTIONS_PARAMETERS_ENTRY, func_name)
if old is NotFound:
obj = FunctionParametersObj(context.event.get_digest(), func_name, {param_number: [(param_value, 1)]})
self.sheerka.om.put(self.FUNCTIONS_PARAMETERS_ENTRY, func_name, obj)
else:
if param_number in old.params:
lst = old.params[param_number]
for i, value in enumerate(lst): # value is a tuple (param_value, counter)
@@ -56,10 +60,7 @@ class SheerkaFunctionsParametersHistory(BaseService):
lst.append((param_value, 1))
else:
old.params[param_number] = [(param_value, 1)]
self.cache.put(func_name, old)
else:
obj = FunctionParametersObj(context.event.get_digest(), func_name, {param_number: [(param_value, 1)]})
self.cache.put(func_name, obj)
self.sheerka.om.put(self.FUNCTIONS_PARAMETERS_ENTRY, func_name, old)
def get_function_parameters(self, func_name: str, param_number: int):
"""
@@ -68,8 +69,8 @@ class SheerkaFunctionsParametersHistory(BaseService):
:param param_number:
:return:
"""
values = self.cache.get(func_name)
if values is None:
values = self.sheerka.om.get(self.FUNCTIONS_PARAMETERS_ENTRY, func_name)
if values is NotFound:
return []
if param_number not in values.params:
src/core/sheerka/services/SheerkaHistoryManager.py
+2 -2
View File
@@ -67,10 +67,10 @@ class SheerkaHistoryManager(BaseService):
:return:
"""
events = list(self.sheerka.sdp.load_events(depth, start))
events = list(self.sheerka.om.current_sdp().load_events(depth, start))
for event in events:
try:
result = self.sheerka.sdp.load_result(event.get_digest())
result = self.sheerka.om.current_sdp().load_result(event.get_digest())
except (IOError, KeyError):
result = None
yield History(event, result)
src/core/sheerka/services/SheerkaSetsManager.py → src/core/sheerka/services/SheerkaIsAManager.py
+24 -25
View File
@@ -4,21 +4,18 @@ from cache.SetCache import SetCache
from core.ast_helpers import UnreferencedVariablesVisitor
from core.builtin_concepts import BuiltinConcepts
from core.concept import Concept, ConceptParts, DEFINITION_TYPE_BNF
from core.global_symbols import NotFound
from core.sheerka.services.SheerkaConceptManager import SheerkaConceptManager
from core.sheerka.services.sheerka_service import BaseService
GROUP_PREFIX = 'All_'
class SheerkaSetsManager(BaseService):
NAME = "SetsManager"
CONCEPTS_GROUPS_ENTRY = "SetsManager:Concepts_Groups"
CONCEPTS_IN_GROUPS_ENTRY = "SetsManager:Concepts_In_Groups" # cache for get_set_elements()
class SheerkaIsAManager(BaseService):
NAME = "IsAManager"
CONCEPTS_GROUPS_ENTRY = "IsAManager:Concepts_Groups"
CONCEPTS_IN_GROUPS_ENTRY = "IsAManager:Concepts_In_Groups" # cache for get_set_elements()
def __init__(self, sheerka):
super().__init__(sheerka)
self.sets = SetCache(default=lambda k: self.sheerka.sdp.get(self.CONCEPTS_GROUPS_ENTRY, k))
self.concepts_in_set = Cache()
def initialize(self):
self.sheerka.bind_service_method(self.set_isa, True)
@@ -28,8 +25,10 @@ class SheerkaSetsManager(BaseService):
self.sheerka.bind_service_method(self.isa, False)
self.sheerka.bind_service_method(self.isaset, True) # concept is evaluated, need to change the code
self.sheerka.cache_manager.register_cache(self.CONCEPTS_GROUPS_ENTRY, self.sets)
self.sheerka.cache_manager.register_cache(self.CONCEPTS_IN_GROUPS_ENTRY, self.concepts_in_set, persist=False)
cache = SetCache().auto_configure(self.CONCEPTS_GROUPS_ENTRY)
self.sheerka.om.register_cache(self.CONCEPTS_GROUPS_ENTRY, cache)
cache = Cache().auto_configure(self.CONCEPTS_IN_GROUPS_ENTRY)
self.sheerka.om.register_cache(self.CONCEPTS_IN_GROUPS_ENTRY, cache, persist=False)
def set_isa(self, context, concept, concept_set):
"""
@@ -43,8 +42,8 @@ class SheerkaSetsManager(BaseService):
context.log(f"Setting concept {concept} is a {concept_set}", who=self.NAME)
core.builtin_helpers.ensure_concept(concept, concept_set)
if BuiltinConcepts.ISA in concept.get_metadata().props and concept_set in concept.get_metadata().props[
BuiltinConcepts.ISA]:
if BuiltinConcepts.ISA in concept.get_metadata().props and \
concept_set in concept.get_metadata().props[BuiltinConcepts.ISA]:
return self.sheerka.ret(
self.NAME,
False,
@@ -75,23 +74,23 @@ class SheerkaSetsManager(BaseService):
context.log(f"Adding concept {concept} to set {concept_set}", who=self.NAME)
core.builtin_helpers.ensure_concept(concept, concept_set)
set_elements = self.sets.get(concept_set.id)
if set_elements and concept.id in set_elements:
set_elements = self.sheerka.om.get(self.CONCEPTS_GROUPS_ENTRY, concept_set.id)
if set_elements is not NotFound and concept.id in set_elements:
return self.sheerka.ret(
self.NAME,
False,
self.sheerka.new(BuiltinConcepts.CONCEPT_ALREADY_IN_SET, body=concept, concept_set=concept_set))
self.sets.put(concept_set.id, concept.id)
self.sheerka.om.put(self.CONCEPTS_GROUPS_ENTRY, concept_set.id, concept.id)
# invalidate the cache of what contains concept_set
self.concepts_in_set.delete(concept_set.id)
self.sheerka.om.delete(self.CONCEPTS_IN_GROUPS_ENTRY, concept_set.id)
# update concept_set references
self.sheerka.services[SheerkaConceptManager.NAME].update_references(context, concept_set)
# remove the grammar entry so that it can be recreated
self.sheerka.cache_manager.delete(self.sheerka.CONCEPTS_GRAMMARS_ENTRY, concept_set.id)
self.sheerka.om.delete(self.sheerka.CONCEPTS_GRAMMARS_ENTRY, concept_set.id)
return self.sheerka.ret(self.NAME, True, self.sheerka.new(BuiltinConcepts.SUCCESS))
@@ -116,7 +115,7 @@ class SheerkaSetsManager(BaseService):
concept_set=concept_set)
else:
body = self.sheerka.new(BuiltinConcepts.SUCCESS)
self.concepts_in_set.delete(concept_set.id)
self.sheerka.om.delete(self.CONCEPTS_IN_GROUPS_ENTRY, concept_set.id)
return self.sheerka.ret(self.NAME, len(already_in_set) != len(concepts), body)
@@ -136,7 +135,7 @@ class SheerkaSetsManager(BaseService):
return self.sheerka.new(BuiltinConcepts.NOT_A_SET, body=concept)
# first, try to see if sub_concept has it's own group entry
ids = self.sets.get(sub_concept.id)
ids = self.sheerka.om.get(self.CONCEPTS_GROUPS_ENTRY, sub_concept.id)
concepts = self._get_concepts(context, ids, True)
# aggregate with en entries from its body
@@ -166,13 +165,13 @@ class SheerkaSetsManager(BaseService):
return concepts
# already in cache ?
if res := self.concepts_in_set.get(concept.id):
if (res := self.sheerka.om.get(self.CONCEPTS_IN_GROUPS_ENTRY, concept.id)) is not NotFound:
return res
res = _get_set_elements(concept)
# put in cache
self.concepts_in_set.put(concept.id, res)
self.sheerka.om.put(self.CONCEPTS_IN_GROUPS_ENTRY, concept.id, res)
return res
def isinset(self, a, b):
@@ -190,8 +189,8 @@ class SheerkaSetsManager(BaseService):
if not (a.id and b.id):
return False
group_elements = self.sets.get(b.id)
return group_elements and a.id in group_elements
group_elements = self.sheerka.om.get(self.CONCEPTS_GROUPS_ENTRY, b.id)
return group_elements is not NotFound and a.id in group_elements
def isa(self, a, b):
@@ -226,7 +225,7 @@ class SheerkaSetsManager(BaseService):
# check if it has a group
# TODO: use cache instead of directly requesting sdp
if self.sets.get(concept.id):
if self.sheerka.om.get(self.CONCEPTS_GROUPS_ENTRY, concept.id) is not NotFound:
return True
# it may be a concept that references a set
@@ -267,7 +266,7 @@ for x in xx__concepts__xx:
:return:
"""
if not ids:
if ids in (None, NotFound):
return []
if not evaluate:
src/core/sheerka/services/SheerkaMemory.py
+16 -14
View File
@@ -4,7 +4,7 @@ from cache.FastCache import FastCache
from cache.ListIfNeededCache import ListIfNeededCache
from core.builtin_concepts import BuiltinConcepts
from core.concept import Concept
from core.global_symbols import EVENT_CONTEXT_DISPOSED
from core.global_symbols import EVENT_CONTEXT_DISPOSED, NotFound
from core.sheerka.services.sheerka_service import BaseService, ServiceObj
@@ -17,13 +17,11 @@ class SheerkaMemory(BaseService):
NAME = "Memory"
GLOBAL = "global"
SHORT_TERM_OBJECTS_ENTRY = "Memory:ShortTermMemoryObjects"
OBJECTS_ENTRY = "Memory:Objects"
def __init__(self, sheerka):
super().__init__(sheerka)
self.short_term_objects = FastCache()
self.memory_objects = ListIfNeededCache(default=lambda k: self.sheerka.sdp.get(self.OBJECTS_ENTRY, k))
self.registration = {}
def initialize(self):
@@ -35,15 +33,20 @@ class SheerkaMemory(BaseService):
self.sheerka.bind_service_method(self.get_from_memory, False)
self.sheerka.bind_service_method(self.register_object, True, visible=False)
self.sheerka.bind_service_method(self.unregister_object, True, visible=False)
self.sheerka.bind_service_method(self.add_registered_objects, True, visible=False)
self.sheerka.bind_service_method(self.commit_registered_objects, True, visible=False)
self.sheerka.bind_service_method(self.memory, False)
self.sheerka.bind_service_method(self.mem, False)
self.sheerka.cache_manager.register_cache(self.OBJECTS_ENTRY, self.memory_objects, persist=True, use_ref=True)
cache = ListIfNeededCache().auto_configure(self.OBJECTS_ENTRY)
self.sheerka.om.register_cache(self.OBJECTS_ENTRY, cache, persist=True, use_ref=True)
def reset(self):
self.short_term_objects.clear()
self.memory_objects.clear()
self.sheerka.om.clear(self.OBJECTS_ENTRY)
def reset_state(self):
self.short_term_objects.clear()
self.registration.clear()
def initialize_deferred(self, context, is_first_time):
self.sheerka.subscribe(EVENT_CONTEXT_DISPOSED, self.remove_context)
@@ -55,7 +58,7 @@ class SheerkaMemory(BaseService):
return self.short_term_objects.cache[id_to_use][key]
except KeyError:
if context is None:
return None
return NotFound
context = context.get_parent()
@@ -92,12 +95,12 @@ class SheerkaMemory(BaseService):
:param concept:
:return:
"""
self.memory_objects.put(key, MemoryObject(context.event.get_digest(), concept))
self.sheerka.om.put(SheerkaMemory.OBJECTS_ENTRY, key, MemoryObject(context.event.get_digest(), concept))
def get_from_memory(self, context, key):
""""
"""
return self.memory_objects.get(key)
return self.sheerka.om.get(SheerkaMemory.OBJECTS_ENTRY, key)
def register_object(self, context, key, concept):
"""
@@ -126,7 +129,7 @@ class SheerkaMemory(BaseService):
except KeyError:
pass
def add_registered_objects(self, context):
def commit_registered_objects(self, context):
"""
Adds all registered memory_objects
:param context:
@@ -147,7 +150,7 @@ class SheerkaMemory(BaseService):
name_to_use = name.name if isinstance(name, Concept) else name
self.unregister_object(context, name_to_use)
obj = self.get_from_memory(context, name_to_use)
if obj is None:
if obj is NotFound:
return self.sheerka.new(BuiltinConcepts.NOT_FOUND, body={"#name": name})
if isinstance(obj, list):
@@ -156,8 +159,7 @@ class SheerkaMemory(BaseService):
return obj.obj
res = {}
for k in self.memory_objects:
obj = self.memory_objects.get(k)
for k, obj in self.sheerka.om.get_all(SheerkaMemory.OBJECTS_ENTRY).items():
if isinstance(obj, list):
obj = obj[-1]
res[k] = obj.obj
@@ -165,5 +167,5 @@ class SheerkaMemory(BaseService):
return res
def mem(self):
keys = sorted([k for k in self.memory_objects])
keys = sorted([k for k in self.sheerka.om.list(SheerkaMemory.OBJECTS_ENTRY)])
return {"keys": keys, "len": len(keys)}
src/core/sheerka/services/SheerkaQuestion.py
+2 -19
View File
@@ -10,27 +10,10 @@ class SheerkaQuestion(BaseService):
super().__init__(sheerka)
def initialize(self):
# self.sheerka.bind_service_method(self.question, False)
self.sheerka.bind_service_method(self.is_question, False)
# def question(self, context, q):
# """
# Evaluate q in the context in a question
# :param context:
# :param q:
# :return:
# """
#
# if isinstance(q, Concept):
# with context.push(BuiltinConcepts.EVALUATE_CONCEPT, q, desc=f"Evaluating question '{q}'") as sub_context:
# sub_context.global_hints.add(BuiltinConcepts.EVAL_QUESTION_REQUESTED)
# sub_context.global_hints.add(BuiltinConcepts.EVAL_UNTIL_SUCCESS_REQUESTED)
#
# evaluated = self.sheerka.evaluate_concept(sub_context, q)
#
# return evaluated
def is_question(self, context):
@staticmethod
def is_question(context):
"""
Returns True if a question is asked
:return:
src/core/sheerka/services/SheerkaResultManager.py
+25 -15
View File
@@ -2,7 +2,7 @@ import ast
from cache.Cache import Cache
from core.builtin_concepts import BuiltinConcepts
from core.global_symbols import EVENT_USER_INPUT_EVALUATED, EVENT_CONCEPT_CREATED
from core.global_symbols import EVENT_USER_INPUT_EVALUATED, EVENT_CONCEPT_CREATED, NotFound
from core.sheerka.services.sheerka_service import BaseService
from core.utils import CONSOLE_COLORS_MAP as CCM
from core.utils import as_bag
@@ -13,6 +13,9 @@ MAX_EXECUTION_HISTORY = 100
class SheerkaResultConcept(BaseService):
NAME = "Result"
# SheerkaResultConcept seems to be a concept that must not support multiple ontology layers
# We must have always access to everything that was done, whatever the ontology
def __init__(self, sheerka, page_size=30):
super().__init__(sheerka)
self.page_size = page_size
@@ -20,6 +23,7 @@ class SheerkaResultConcept(BaseService):
self.last_execution = None
self.last_created_concept = None
self.last_created_concept_id = None
self.state_vars = ["last_created_concept_id"]
def initialize(self):
self.sheerka.bind_service_method(self.get_results_by_digest, True) # digest is recorded
@@ -27,20 +31,26 @@ class SheerkaResultConcept(BaseService):
self.sheerka.bind_service_method(self.get_last_results, True) # digest is recorded
self.sheerka.bind_service_method(self.get_results, False)
self.sheerka.bind_service_method(self.get_execution_item, False)
self.sheerka.bind_service_method(self.get_last_ret, False, as_name="last_ret")
self.sheerka.bind_service_method(self.get_last_return_value, False, as_name="last_ret")
self.sheerka.bind_service_method(self.get_last_created_concept, False, as_name="last_created_concept")
def initialize_deferred(self, context, is_first_time):
self.restore_values("last_created_concept_id")
self.restore_values(*self.state_vars)
self.sheerka.subscribe(EVENT_USER_INPUT_EVALUATED, self.user_input_evaluated)
self.sheerka.subscribe(EVENT_CONCEPT_CREATED, self.new_concept_created)
def reset(self):
def test_only_reset(self):
self.executions_contexts_cache.clear()
self.last_execution = None
self.last_created_concept = None
self.last_created_concept_id = None
def save_state(self, context):
self.store_values(context, *self.state_vars)
def restore_state(self):
self.restore_values(*self.state_vars)
@staticmethod
def get_predicate(**kwargs):
if len(kwargs) == 0:
@@ -81,7 +91,7 @@ class SheerkaResultConcept(BaseService):
:param record_digest:
:return:
"""
if digest is None:
if digest is NotFound:
return None
if filter is not None:
@@ -92,8 +102,8 @@ class SheerkaResultConcept(BaseService):
result = self.executions_contexts_cache.get(digest)
event = result.event
else:
result = self.sheerka.sdp.load_result(digest)
event = self.sheerka.sdp.load_event(digest) # there is no real need for a cache of the events
result = self.sheerka.om.current_sdp().load_result(digest)
event = self.sheerka.om.current_sdp().load_event(digest) # really needed ?
if record_digest:
context.log(f"Recording digest '{digest}'")
@@ -141,7 +151,7 @@ class SheerkaResultConcept(BaseService):
start = len(self.executions_contexts_cache)
consumed = 0
while True:
for event in self.sheerka.sdp.load_events(self.page_size, start):
for event in self.sheerka.om.current_sdp().load_events(self.page_size, start):
consumed += 1
if event.message.startswith(command):
return self.get_results_by_digest(context, event.get_digest(), filter, record_digest, **kwargs)
@@ -200,14 +210,14 @@ class SheerkaResultConcept(BaseService):
:return:
"""
digest = self.sheerka.load_var(self.NAME, "digest")
if digest is None:
if digest is NotFound:
return self.sheerka.new(BuiltinConcepts.NOT_FOUND, body="no digest")
try:
if digest in self.executions_contexts_cache:
result = self.executions_contexts_cache.get(digest)
else:
result = self.sheerka.sdp.load_result(digest)
result = self.sheerka.om.current_sdp().load_result(digest)
items = list(self.as_list(result, self.get_predicate(id=item_id)))
if len(items) == 0:
@@ -227,7 +237,7 @@ class SheerkaResultConcept(BaseService):
"""
if self.sheerka.save_execution_context:
try:
self.sheerka.sdp.save_result(execution_context)
self.sheerka.om.current_sdp().save_result(execution_context)
except Exception as ex:
print(f"{CCM['red']}Failed to save execution context. Reason: {ex}{CCM['reset']}")
pass
@@ -236,7 +246,7 @@ class SheerkaResultConcept(BaseService):
self.executions_contexts_cache.put(execution_context.event.get_digest(), execution_context)
self.last_execution = execution_context
def get_last_ret(self, context):
def get_last_return_value(self, context):
"""
Return the last return value(s)
:return:
@@ -248,7 +258,7 @@ class SheerkaResultConcept(BaseService):
if event_id is not None:
try:
execution_result = self.sheerka.sdp.load_result(event_id)
execution_result = self.sheerka.om.current_sdp().load_result(event_id)
return execution_result.values["return_values"]
except FileNotFoundError as ex:
@@ -278,9 +288,9 @@ class SheerkaResultConcept(BaseService):
page_size = 2
consumed = 0
while True:
for event in self.sheerka.sdp.load_events(page_size, start):
for event in self.sheerka.om.current_sdp().load_events(page_size, start):
consumed += 1
if self.sheerka.sdp.has_result(event.get_digest()):
if self.sheerka.om.current_sdp().has_result(event.get_digest()):
return event.get_digest()
if consumed < page_size:
src/core/sheerka/services/SheerkaRuleManager.py
+53 -24
View File
@@ -4,10 +4,11 @@ from dataclasses import dataclass
from typing import Union
from cache.Cache import Cache
from cache.ListIfNeededCache import ListIfNeededCache
from core.builtin_concepts import BuiltinConcepts, ReturnValueConcept
from core.builtin_helpers import parse_unrecognized, only_successful, ensure_rule
from core.concept import Concept
from core.global_symbols import EVENT_RULE_PRECEDENCE_MODIFIED, RULE_COMPARISON_CONTEXT
from core.global_symbols import EVENT_RULE_PRECEDENCE_MODIFIED, RULE_COMPARISON_CONTEXT, NotFound
from core.rule import Rule
from core.sheerka.services.sheerka_service import BaseService
from core.tokenizer import Keywords, TokenKind, Token, IterParser
@@ -509,22 +510,25 @@ class SheerkaRuleManager(BaseService):
RULE_IDS = "Rules_Ids"
FORMAT_RULE_ENTRY = "RuleManager:FormatRules"
EXEC_RULE_ENTRY = "RuleManager:ExecRules"
RULES_BY_NAME_ENTRY = "RuleManager:Rules_By_Name"
def __init__(self, sheerka):
super().__init__(sheerka)
self.format_rule_cache = Cache(default=lambda k: self.sheerka.sdp.get(self.FORMAT_RULE_ENTRY, k))
self.exec_rule_cache = Cache(default=lambda k: self.sheerka.sdp.get(self.EXEC_RULE_ENTRY, k))
self._format_rules = None # sorted by priority
def initialize(self):
self.sheerka.bind_service_method(self.create_new_rule, True, visible=False)
self.sheerka.bind_service_method(self.get_rule_by_id, False)
self.sheerka.bind_service_method(self.get_rule_by_name, False)
self.sheerka.bind_service_method(self.dump_desc_rule, False, as_name="desc_rule")
self.sheerka.bind_service_method(self.get_format_rules, False, visible=False)
self.sheerka.cache_manager.register_cache(self.FORMAT_RULE_ENTRY, self.format_rule_cache, True, True)
self.sheerka.cache_manager.register_cache(self.EXEC_RULE_ENTRY, self.exec_rule_cache, True, True)
cache = Cache().auto_configure(self.FORMAT_RULE_ENTRY)
self.sheerka.om.register_cache(self.FORMAT_RULE_ENTRY, cache, True, True)
cache = Cache().auto_configure(self.EXEC_RULE_ENTRY)
self.sheerka.om.register_cache(self.EXEC_RULE_ENTRY, cache, True, True)
cache = ListIfNeededCache().auto_configure(self.RULES_BY_NAME_ENTRY)
self.sheerka.om.register_cache(self.RULES_BY_NAME_ENTRY, cache, True, True)
def initialize_deferred(self, context, is_first_time):
@@ -533,20 +537,29 @@ class SheerkaRuleManager(BaseService):
self.init_builtin_rules(context)
else:
# adds the other rules (when it's not the first time)
self.format_rule_cache.populate(lambda: self.sheerka.sdp.list(self.FORMAT_RULE_ENTRY), lambda rule: rule.id)
self.exec_rule_cache.populate(lambda: self.sheerka.sdp.list(self.EXEC_RULE_ENTRY), lambda rule: rule.id)
self.format_rule_cache.reset_events()
self.exec_rule_cache.reset_events()
self.sheerka.om.populate(self.FORMAT_RULE_ENTRY,
lambda sdp: sdp.list(self.FORMAT_RULE_ENTRY),
lambda rule: rule.id,
reset_events=True,
all_ontologies=True)
self.sheerka.om.populate(self.EXEC_RULE_ENTRY,
lambda sdp: sdp.list(self.EXEC_RULE_ENTRY),
lambda rule: rule.id,
reset_events=True,
all_ontologies=True)
# compile all the rules
for rule_id in self.format_rule_cache:
rule = self.init_rule(context, self.format_rule_cache.get(rule_id))
# compile all format the rules
for rule_id, rule_def in self.sheerka.om.get_all(self.FORMAT_RULE_ENTRY, cache_only=True).items():
rule = self.init_rule(context, rule_def)
# update rules priorities
self.update_rules_priorities(context)
self.sheerka.subscribe(EVENT_RULE_PRECEDENCE_MODIFIED, self.update_rules_priorities)
def reset_state(self):
self._format_rules = None
def update_rules_priorities(self, context):
"""
Ask the SheerkaComparisonManager for the priorities
@@ -555,9 +568,8 @@ class SheerkaRuleManager(BaseService):
# get the priorities
rules_weights = self.sheerka.get_concepts_weights(BuiltinConcepts.PRECEDENCE, RULE_COMPARISON_CONTEXT)
# compile all the rules
for rule_id in self.format_rule_cache:
rule = self.format_rule_cache.get(rule_id)
# update the priorities
for rule in self.sheerka.om.list(self.FORMAT_RULE_ENTRY, cache_only=True):
if rule.str_id in rules_weights:
rule.priority = rules_weights[rule.str_id]
@@ -623,7 +635,7 @@ class SheerkaRuleManager(BaseService):
if rule.metadata.id is not None:
return
rule.metadata.id = str(self.sheerka.cache_manager.get(self.sheerka.OBJECTS_IDS_ENTRY, self.RULE_IDS))
rule.metadata.id = str(self.sheerka.om.get(self.sheerka.OBJECTS_IDS_ENTRY, self.RULE_IDS))
def create_new_rule(self, context, rule):
"""
@@ -642,10 +654,14 @@ class SheerkaRuleManager(BaseService):
# save it
if rule.metadata.action_type == "print":
self.sheerka.cache_manager.put(self.FORMAT_RULE_ENTRY, rule.metadata.id, rule)
self.sheerka.om.put(self.FORMAT_RULE_ENTRY, rule.metadata.id, rule)
self._format_rules = None
else:
self.sheerka.cache_manager.put(self.EXEC_RULE_ENTRY, rule.metadata.id, rule)
self.sheerka.om.put(self.EXEC_RULE_ENTRY, rule.metadata.id, rule)
# save by name if needed
if rule.metadata.name:
self.sheerka.om.put(self.RULES_BY_NAME_ENTRY, rule.metadata.name, rule)
# process the return if needed
ret = sheerka.ret(self.NAME, True, sheerka.new(BuiltinConcepts.NEW_RULE, body=rule))
@@ -721,17 +737,28 @@ class SheerkaRuleManager(BaseService):
if rule_id is None:
return None
rule = self.format_rule_cache.get(rule_id)
if rule:
rule = self.sheerka.om.get(self.FORMAT_RULE_ENTRY, rule_id)
if rule is not NotFound:
return rule
rule = self.exec_rule_cache.get(rule_id)
if rule:
rule = self.sheerka.om.get(self.EXEC_RULE_ENTRY, rule_id)
if rule is not NotFound:
return rule
metadata = [("id", rule_id)]
return self.sheerka.new(BuiltinConcepts.UNKNOWN_RULE, body=metadata)
def get_rule_by_name(self, rule_name):
if rule_name is None:
return None
rule = self.sheerka.om.get(self.RULES_BY_NAME_ENTRY, rule_name)
if rule is NotFound:
metadata = [("name", rule_name)]
return self.sheerka.new(BuiltinConcepts.UNKNOWN_RULE, body=metadata)
return rule
def dump_desc_rule(self, rules):
"""
dumps the definition of a rule
@@ -759,7 +786,9 @@ class SheerkaRuleManager(BaseService):
if self._format_rules:
return self._format_rules
self._format_rules = sorted(self.format_rule_cache.get_all(), key=operator.attrgetter('priority'), reverse=True)
self._format_rules = sorted(self.sheerka.om.list(self.FORMAT_RULE_ENTRY, cache_only=True),
key=operator.attrgetter('priority'),
reverse=True)
return self._format_rules
def add_evaluators(self, source, ret_vals):
src/core/sheerka/services/SheerkaVariableManager.py
+58 -22
View File
@@ -3,6 +3,7 @@ from typing import List
from cache.Cache import Cache
from core.builtin_concepts import BuiltinConcepts
from core.global_symbols import NotFound
from core.sheerka.services.sheerka_service import ServiceObj, BaseService
@@ -23,32 +24,52 @@ class Variable(ServiceObj):
return f"({self.who}){self.key}={self.value}"
@dataclass
class InternalObj:
obj: object
def __deepcopy__(self, memodict={}):
return self
def __copy__(self):
return self
class SheerkaVariableManager(BaseService):
NAME = "VariableManager"
VARIABLES_ENTRY = "VariableManager:Variables" # entry for admin or internal variables
VARIABLES_ENTRY = "VariableManager:Variables" # entry for variables which will be copied in sdp
INTERNAL_VARIABLES_ENTRY = "VariableManager:InternalVariables" # internal to current process (can store lambda)
def __init__(self, sheerka):
super().__init__(sheerka)
self.bound = {
"sheerka.enable_process_return_values": "enable_process_return_values",
"sheerka.save_execution_context": "save_execution_context"
self.bound_variables = {
self.sheerka.name: {"enable_process_return_values", "save_execution_context"}
}
def initialize(self):
self.sheerka.bind_service_method(self.record_var, True, visible=False)
self.sheerka.bind_service_method(self.load_var, False, visible=False)
self.sheerka.bind_service_method(self.record_internal_var, True, visible=False)
self.sheerka.bind_service_method(self.load_internal_var, False, visible=False)
self.sheerka.bind_service_method(self.delete_var, True, visible=False)
self.sheerka.bind_service_method(self.set_var, True)
self.sheerka.bind_service_method(self.get_var, False)
self.sheerka.bind_service_method(self.list_vars, False)
cache = Cache()
cache.populate(lambda: self.sheerka.sdp.list(self.VARIABLES_ENTRY), lambda var: var.get_key())
self.sheerka.cache_manager.register_cache(self.VARIABLES_ENTRY, cache, True, True)
cache = Cache().auto_configure(self.VARIABLES_ENTRY)
self.sheerka.om.register_cache(self.VARIABLES_ENTRY, cache, True, True)
cache.populate(lambda sdp: sdp.list(self.VARIABLES_ENTRY), lambda var: var.get_key())
for variable in cache.get_all():
if variable.key in self.bound:
setattr(self.sheerka, self.bound[variable.key], variable.value)
internal_vars = Cache().auto_configure(self.INTERNAL_VARIABLES_ENTRY)
self.sheerka.om.register_cache(self.INTERNAL_VARIABLES_ENTRY, internal_vars, False, False)
def initialize_deferred(self, context, first_time):
# update bound variables
for who, keys in self.bound_variables.items():
for key in keys:
if (variable := self.sheerka.om.get(self.VARIABLES_ENTRY, f"{who}|{key}")) is not NotFound:
service = self.sheerka if who == self.sheerka.name else self.sheerka.services[who]
setattr(service, key, variable.value)
def record_var(self, context, who, key, value):
"""
@@ -61,34 +82,49 @@ class SheerkaVariableManager(BaseService):
"""
variable = Variable(context.event.get_digest(), who, key, value, None)
self.sheerka.cache_manager.put(self.VARIABLES_ENTRY, variable.get_key(), variable)
self.sheerka.om.put(self.VARIABLES_ENTRY, variable.get_key(), variable)
# TODO: manage credentials
if key in self.bound:
setattr(self.sheerka, self.bound[key], value)
return self.sheerka.ret(self.NAME, True, self.sheerka.new(BuiltinConcepts.SUCCESS))
if who in self.bound_variables and key in self.bound_variables[who]:
service = self.sheerka if who == self.sheerka.name else self.sheerka.services[who]
setattr(service, key, value)
def load_var(self, who, key):
variable = self.sheerka.cache_manager.get(self.VARIABLES_ENTRY, who + "|" + key)
if variable is None:
return None
variable = self.sheerka.om.get(self.VARIABLES_ENTRY, who + "|" + key)
if variable is NotFound:
return NotFound
return variable.value
def record_internal_var(self, context, who, key, value):
"""
Stores the value in the internal cache
This cache is not pushed to the remote repository
:param context:
:param who: entity that owns the key (acts as a namespace)
:param key:
:param value:
"""
self.sheerka.om.put(self.INTERNAL_VARIABLES_ENTRY, f"{who}|{key}", InternalObj(value))
def load_internal_var(self, who, key):
value = self.sheerka.om.get(self.INTERNAL_VARIABLES_ENTRY, f"{who}|{key}")
return NotFound if value is NotFound else value.obj
def delete_var(self, context, who, key):
self.sheerka.cache_manager.delete(self.VARIABLES_ENTRY, who + "|" + key)
self.sheerka.om.delete(self.VARIABLES_ENTRY, who + "|" + key)
def set_var(self, context, key, value):
return self.record_var(context, context.event.user_id, key, value)
self.record_var(context, context.event.user_id, key, value)
return self.sheerka.ret(self.NAME, True, self.sheerka.new(BuiltinConcepts.SUCCESS))
def get_var(self, context, key):
return self.load_var(context.event.user_id, key)
def list_vars(self, context, all_vars=False):
if all_vars:
res = [str(v) for v in self.sheerka.cache_manager.copy(self.VARIABLES_ENTRY).values()]
res = [str(v) for v in self.sheerka.om.copy(self.VARIABLES_ENTRY).values()]
else:
res = [str(v) for v in self.sheerka.cache_manager.copy(self.VARIABLES_ENTRY).values() if
res = [str(v) for v in self.sheerka.om.copy(self.VARIABLES_ENTRY).values() if
v.who == context.event.user_id]
return res
src/core/sheerka/services/sheerka_service.py
+11 -1
View File
@@ -1,5 +1,8 @@
from dataclasses import dataclass
from core.global_symbols import NotFound
from core.utils import sheerka_deepcopy
@dataclass
class ServiceObj:
@@ -21,6 +24,13 @@ class BaseService:
"""
pass
def store_values(self, context, *args):
"""
Use variable Manager to store the state of the service
"""
for prop_name in args:
self.sheerka.record_var(context, self.NAME, prop_name, sheerka_deepcopy(getattr(self, prop_name)))
def restore_values(self, *args):
"""
Use Variable Manager to restore the state of a service
@@ -28,5 +38,5 @@ class BaseService:
:return:
"""
for prop_name in args:
if (value := self.sheerka.load_var(self.NAME, prop_name)) is not None:
if (value := self.sheerka.load_var(self.NAME, prop_name)) is not NotFound:
setattr(self, prop_name, value)