kodjo/Sheerka-Old
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added first version of DebugManager. Implemented draft of the rule engine

Browse Source
This commit is contained in:
Kodjo Sossouvi
2020-11-20 13:41:45 +01:00
parent cd066881b4
commit 315f8ea09b
156 changed files with 8388 additions and 2852 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/core/sheerka/Sheerka.py
+167 -81
View File
@@ -11,8 +11,10 @@ from cache.IncCache import IncCache
from cache.ListIfNeededCache import ListIfNeededCache
from cache.SetCache import SetCache
from core.builtin_concepts import BuiltinConcepts, ErrorConcept, ReturnValueConcept, BuiltinErrors, BuiltinUnique, \
UnknownConcept
from core.concept import Concept, ConceptParts, PROPERTIES_FOR_NEW
UnknownConcept, AllBuiltinConcepts
from core.concept import Concept, ConceptParts, NotInit, get_concept_attrs
from core.error import ErrorObj
from core.profiling import profile
from core.sheerka.ExecutionContext import ExecutionContext
from core.sheerka_logger import console_handler
from core.tokenizer import Token, TokenKind
@@ -66,15 +68,18 @@ class Sheerka(Concept):
MAX_EXECUTION_HISTORY = 100
MAX_RETURN_VALUES_HISTORY = 100
ALL_ATTRIBUTES = []
def __init__(self, cache_only=False, debug=False, loggers=None):
self.init_logging(debug, loggers)
self.loggers = loggers
super().__init__(BuiltinConcepts.SHEERKA, True, True, BuiltinConcepts.SHEERKA)
self.log.debug("Starting Sheerka.")
# self.log.debug("Starting Sheerka.")
self.bnp = None # reference to the BaseNodeParser class (to compute first keyword token)
self.return_value_concept_id = None
self.error_concept_id = None
# a concept can be instantiated
# ex: File is a concept, but File('foo.txt') is an instance
@@ -85,7 +90,7 @@ class Sheerka(Concept):
# ex: hello => say('hello')
self.rules = []
self.sdp: SheerkaDataProvider = None # SheerkaDataProvider
self.sdp: SheerkaDataProvider = None
self.cache_manager = CacheManager(cache_only)
self.services = {} # sheerka plugins
@@ -103,6 +108,7 @@ class Sheerka(Concept):
self._builtins_classes_cache = None
self.save_execution_context = True
self.enable_process_return_values = False
self.methods_with_context = {"test_using_context"} # only the names, the method is defined in sheerka_methods
self.sheerka_methods = {
@@ -171,17 +177,22 @@ class Sheerka(Concept):
"""
self.sheerka_pipeables[func_name] = SheerkaMethod(function, has_side_effect)
def initialize(self, root_folder: str = None, save_execution_context=True):
def initialize(self, root_folder: str = None, save_execution_context=None, enable_process_return_values=None):
"""
Starting Sheerka
Loads the current configuration
Notes that when it's the first time, it also create the needed working folders
:param root_folder: root configuration folder
:param save_execution_context:
:param enable_process_return_values:
:return: ReturnValue(Success or Error)
"""
self.save_execution_context = save_execution_context
if save_execution_context is not None:
self.save_execution_context = save_execution_context
if enable_process_return_values is not None:
self.enable_process_return_values = enable_process_return_values
try:
from sheerkapickle.sheerka_handlers import initialize_pickle_handlers
@@ -189,7 +200,10 @@ class Sheerka(Concept):
self.sdp = SheerkaDataProvider(root_folder, self)
self.initialize_caching()
self.get_builtin_parsers()
self.get_builtin_evaluators()
self.initialize_services()
self.initialize_builtin_evaluators()
event = Event("Initializing Sheerka.", user_id=self.name)
self.sdp.save_event(event)
@@ -198,25 +212,24 @@ class Sheerka(Concept):
self,
BuiltinConcepts.INIT_SHEERKA,
None,
desc="Initializing Sheerka.",
logger=self.init_log) as exec_context:
desc="Initializing Sheerka.") as exec_context:
if self.sdp.first_time:
self.first_time_initialisation(exec_context)
self.initialize_builtin_parsers()
self.initialize_builtin_evaluators()
self.initialize_builtin_concepts()
self.initialize_concept_node_parsing(exec_context)
res = ReturnValueConcept(self, True, self)
self.initialize_services_deferred(exec_context, self.sdp.first_time)
res = ReturnValueConcept(self, True, self)
exec_context.add_values(return_values=res)
if self.cache_manager.is_dirty:
self.cache_manager.commit(exec_context)
if save_execution_context:
if self.save_execution_context:
self.sdp.save_result(exec_context, is_admin=True)
self.init_log.debug(f"Sheerka successfully initialized")
# self.init_log.debug(f"Sheerka successfully initialized")
except IOError as e:
res = ReturnValueConcept(self, False, self.get(BuiltinConcepts.ERROR), e)
@@ -276,7 +289,7 @@ class Sheerka(Concept):
Introspect to find services and bind them
:return:
"""
self.init_log.debug("Initializing services")
# self.init_log.debug("Initializing services")
core.utils.import_module_and_sub_module('core.sheerka.services')
base_class = "core.sheerka.services.sheerka_service.BaseService"
@@ -286,49 +299,63 @@ class Sheerka(Concept):
instance.initialize()
self.services[service.NAME] = instance
def initialize_services_deferred(self, context, is_first_time):
"""
Initialize part of services that may takes some time or that need the execution context
TODO: Create a separate thread for these initialisations as they may take time
:return:
"""
# self.init_log.debug("Initializing services (deferred)")
for service in self.services.values():
if hasattr(service, "initialize_deferred"):
service.initialize_deferred(context, is_first_time)
def first_time_initialisation(self, context):
self.cache_manager.put(self.CONCEPTS_KEYS_ENTRY, self.USER_CONCEPTS_KEYS, 1000)
self.record(context, self.name, "save_execution_context", True)
self.record_var(context, self.name, "save_execution_context", True)
def initialize_builtin_concepts(self):
"""
Initializes the builtin concepts
:return: None
"""
self.init_log.debug("Initializing builtin concepts")
# self.init_log.debug("Initializing builtin concepts")
builtins_classes = self.get_builtins_classes_as_dict()
# this all initialization of the builtins seems to be little bit complicated
# why do we need to update it from DB ?
for key in BuiltinConcepts:
for key in AllBuiltinConcepts:
concept = self if key == BuiltinConcepts.SHEERKA \
else builtins_classes[str(key)]() if str(key) in builtins_classes \
else Concept(key, True, False, key)
if key in BuiltinUnique:
concept.metadata.is_unique = True
concept.metadata.is_evaluated = True
concept._metadata.is_unique = True
concept._metadata.is_evaluated = True
if not concept.metadata.is_unique and str(key) in builtins_classes:
if not concept._metadata.is_unique and str(key) in builtins_classes:
self.builtin_cache[key] = builtins_classes[str(key)]
from_db = self.cache_manager.get(self.CONCEPTS_BY_KEY_ENTRY, concept.metadata.key)
from_db = self.cache_manager.get(self.CONCEPTS_BY_KEY_ENTRY, concept._metadata.key)
if from_db is None:
self.init_log.debug(f"'{concept.name}' concept is not found in db. Adding.")
# self.init_log.debug(f"'{concept.name}' concept is not found in db. Adding.")
self.set_id_if_needed(concept, True)
self.cache_manager.add_concept(concept)
if key == BuiltinConcepts.RETURN_VALUE:
self.return_value_concept_id = concept.id
elif key == BuiltinConcepts.ERROR:
self.error_concept_id = concept.id
else:
self.init_log.debug(f"Found concept '{from_db}' in db. Updating.")
# self.init_log.debug(f"Found concept '{from_db}' in db. Updating.")
concept.update_from(from_db)
return
def initialize_builtin_parsers(self):
def get_builtin_parsers(self):
"""
Init the parsers
:return:
@@ -343,7 +370,7 @@ class Sheerka(Concept):
continue
qualified_name = core.utils.get_full_qualified_name(parser)
self.init_log.debug(f"Adding builtin parser '{qualified_name}'")
# self.init_log.debug(f"Adding builtin parser '{qualified_name}'")
temp_result[qualified_name] = parser
# keep a reference to base_node_parser
@@ -361,22 +388,29 @@ class Sheerka(Concept):
self.parsers[name] = temp_result[name]
def get_builtin_evaluators(self):
"""
get all evaluators
:return:
"""
core.utils.import_module_and_sub_module("evaluators")
evaluators = core.utils.get_sub_classes("evaluators", "evaluators.BaseEvaluator.OneReturnValueEvaluator")
evaluators.extend(core.utils.get_sub_classes("evaluators", "evaluators.BaseEvaluator.AllReturnValuesEvaluator"))
for evaluator in evaluators:
self.evaluators.append(evaluator)
def initialize_builtin_evaluators(self):
"""
Init the evaluators
:return:
"""
core.utils.import_module_and_sub_module("evaluators")
for evaluator in core.utils.get_sub_classes("evaluators", "evaluators.BaseEvaluator.OneReturnValueEvaluator"):
self.init_log.debug(f"Adding builtin evaluator '{evaluator.__name__}'")
self.evaluators.append(evaluator)
for evaluator in core.utils.get_sub_classes("evaluators", "evaluators.BaseEvaluator.AllReturnValuesEvaluator"):
self.init_log.debug(f"Adding builtin evaluator '{evaluator.__name__}'")
self.evaluators.append(evaluator)
for evaluator in self.evaluators:
if hasattr(evaluator, "initialize"):
evaluator.initialize(self)
def initialize_concept_node_parsing(self, context):
self.init_log.debug("siInitializing concepts by first keyword.")
# self.init_log.debug("Initializing concepts by first keyword.")
concepts_by_first_keyword = self.cache_manager.copy(self.CONCEPTS_BY_FIRST_KEYWORD_ENTRY)
res = self.bnp.resolve_concepts_by_first_keyword(context, concepts_by_first_keyword)
@@ -391,11 +425,16 @@ class Sheerka(Concept):
service.initialize()
else:
self.cache_manager.clear()
for service in self.services.values():
if hasattr(service, "reset"):
service.reset()
self.printer_handler.reset()
self.sdp.reset()
self.locals = {}
# @profile()
# @profile(filename="profile_80")
def evaluate_user_input(self, text: str, user_name="kodjo"):
"""
Note to KSI: If you try to add execution context to this function,
@@ -404,19 +443,20 @@ class Sheerka(Concept):
:param user_name:
:return:
"""
self.log.debug(f"Processing user input '{text}', {user_name=}.")
# self.log.debug(f"Processing user input '{text}', {user_name=}.")
event = Event(text, user_name)
evt_digest = self.sdp.save_event(event)
self.log.debug(f"{evt_digest=}")
self.sdp.save_event(event)
with ExecutionContext(self.key,
event,
self,
BuiltinConcepts.PROCESS_INPUT,
text,
desc=f"Evaluating '{text}'",
logger=self.log) as execution_context:
desc=f"Evaluating '{text}'") as execution_context:
user_input = self.ret(self.name, True, self.new(BuiltinConcepts.USER_INPUT, body=text, user_name=user_name))
# TODO. Must be a context hint, not a return value
reduce_requested = self.ret(self.name, True, self.new(BuiltinConcepts.REDUCE_REQUESTED))
ret = self.execute(execution_context, [user_input, reduce_requested], EXECUTE_STEPS)
@@ -425,17 +465,21 @@ class Sheerka(Concept):
if self.cache_manager.is_dirty:
self.cache_manager.commit(execution_context)
try:
if self.save_execution_context and self.load(self.name, "save_execution_context"):
# exec_count = ExecutionContext.ids[execution_context.event.get_digest()]
# print("Execution Context Count:", exec_count)
if self.save_execution_context:
try:
# if exec_count > 3400:
# print("Saving result. digest=", execution_context.event.get_digest())
self.sdp.save_result(execution_context)
except Exception as ex:
self.log.error(f"Failed to save execution context. Reason: {ex}")
except Exception as ex:
print(f"Failed to save execution context. Reason: {ex}")
pass
# self.log.error(f"Failed to save execution context. Reason: {ex}")
# # hack to save valid concept definition
# if not self.during_restore:
# if len(ret) == 1 and ret[0].status and self.isinstance(ret[0].value, BuiltinConcepts.NEW_CONCEPT):
# with open(CONCEPTS_FILE, "a") as f:
# f.write(text + "\n")
# Do not save execution contexts from process_return_values
if self.enable_process_return_values:
self.process_return_values(execution_context, ret)
self.execution_count += 1
self._last_execution = execution_context
@@ -461,17 +505,16 @@ class Sheerka(Concept):
def set_id_if_needed(self, obj: Concept, is_builtin: bool):
"""
Set the key for the concept if needed
For test purpose only !!!!!
:param obj:
:param is_builtin:
:return:
"""
if obj.metadata.id is not None:
if obj._metadata.id is not None:
return
key = self.BUILTIN_CONCEPTS_KEYS if is_builtin else self.USER_CONCEPTS_KEYS
obj.metadata.id = str(self.cache_manager.get(self.CONCEPTS_KEYS_ENTRY, key))
self.log.debug(f"Setting id '{obj.metadata.id}' to concept '{obj.metadata.name}'.")
obj._metadata.id = str(self.cache_manager.get(self.CONCEPTS_KEYS_ENTRY, key))
# self.log.debug(f"Setting id '{obj.metadata.id}' to concept '{obj.metadata.name}'.")
def force_sya_def(self, context, list_of_def):
"""
@@ -588,9 +631,9 @@ class Sheerka(Concept):
# ##############
# if the entry is a concept token, use its values.
if isinstance(concept, Token):
if concept.type != TokenKind.CONCEPT:
if concept.type == TokenKind.RULE: # do not recognize rules !!!
return None
concept = concept.value
concept = concept.value # concept is now a tuple
if isinstance(concept, str) and \
concept.startswith("c:") and \
@@ -607,7 +650,7 @@ class Sheerka(Concept):
if concept[1]:
if self.is_known(found := self.get_by_id(concept[1])):
instance = self.new_from_template(found, found.key)
instance.metadata.is_evaluated = True
instance._metadata.is_evaluated = True
return instance
elif concept[0]:
if self.is_known(found := self.get_by_name(concept[0])):
@@ -626,6 +669,28 @@ class Sheerka(Concept):
return None
def fast_resolve(self, key, return_new=True):
def new_instances(concepts):
if hasattr(concepts, "__iter__"):
return [self.new_from_template(c, c.key) for c in concepts]
return self.new_from_template(concepts, concepts.key)
if isinstance(key, Token):
if key.type == TokenKind.RULE: # do not recognize rules !!!
return None
if key.value[1]:
concept = self.cache_manager.get(self.CONCEPTS_BY_ID_ENTRY, key.value[1])
else:
concept = self.cache_manager.get(self.CONCEPTS_BY_NAME_ENTRY, key.value[0])
else:
concept = self.cache_manager.get(self.CONCEPTS_BY_NAME_ENTRY, key)
if concept is None:
return None
return new_instances(concept) if return_new else concept
def has_id(self, concept_id):
"""
Returns True if a concept with this id exists in cache
@@ -694,13 +759,13 @@ class Sheerka(Concept):
def new_from_template(self, template, key, **kwargs):
# core.utils.my_debug(f"Created {template}, {key=}, {kwargs=}")
# manage singleton
if template.metadata.is_unique:
if template.get_metadata().is_unique:
return template
# otherwise, create another instance
concept = self.builtin_cache[key]() if key in self.builtin_cache else Concept()
concept.update_from(template, update_value=False)
concept.freeze_definition_hash()
# concept.freeze_definition_hash()
if len(kwargs) == 0:
return concept
@@ -708,17 +773,17 @@ class Sheerka(Concept):
# update the properties, values, attributes
# Not quite sure that this is the correct process order
for k, v in kwargs.items():
if k in concept.values:
if k in get_concept_attrs(concept):
concept.set_value(k, v)
elif k in PROPERTIES_FOR_NEW:
concept.set_value(ConceptParts(k), v)
elif k == "body":
concept.set_value(ConceptParts.BODY, v)
elif hasattr(concept, k):
setattr(concept, k, v)
else:
return self.new(BuiltinConcepts.UNKNOWN_PROPERTY, body=k, concept=concept)
# TODO : add the concept to the list of known concepts (self.instances)
concept.metadata.is_evaluated = True # because we have manually set the variables
concept._metadata.is_evaluated = True # because we have manually set the variables
return concept
def ret(self, who: str, status: bool, value, message=None, parents=None):
@@ -732,22 +797,16 @@ class Sheerka(Concept):
:return:
"""
# 1 second saved every twenty seconds in unit tests
return ReturnValueConcept(
who=who,
status=status,
value=value,
message=message,
parents=parents,
concept_id=self.return_value_concept_id
)
# return self.new(
# BuiltinConcepts.RETURN_VALUE,
# who=who,
# status=status,
# value=value,
# message=message,
# parents=parents)
def err(self, body):
return ErrorConcept(body, self.error_concept_id)
def objvalue(self, obj, reduce_simple_list=False):
if obj is None:
@@ -759,7 +818,7 @@ class Sheerka(Concept):
if not isinstance(obj, Concept):
return obj
if obj.body is BuiltinConcepts.NOT_INITIALIZED:
if obj.body is NotInit:
return obj
if reduce_simple_list and (isinstance(obj.body, list) or isinstance(obj.body, set)) and len(obj.body) == 1:
@@ -796,7 +855,7 @@ class Sheerka(Concept):
return self.value_by_concept(obj.body, concept)
def get_error(self, obj):
if isinstance(obj, Concept) and obj.metadata.is_builtin and obj.key in BuiltinErrors:
if isinstance(obj, Concept) and obj._metadata.is_builtin and obj.key in BuiltinErrors:
return obj
if isinstance(obj, (list, set, tuple)):
@@ -848,9 +907,9 @@ class Sheerka(Concept):
def test(self):
return f"I have access to Sheerka !"
def test_using_context(self, context, param1, param2):
def test_using_context(self, context, param):
event = context.event.get_digest()
return f"I have access to Sheerka ! {param1=}, {param2=}, {event=}."
return f"I have access to Sheerka ! {param=}, {event=}."
def test_error(self):
raise Exception("I can raise an error")
@@ -863,14 +922,17 @@ class Sheerka(Concept):
if isinstance(obj, ReturnValueConcept):
return obj.status
if isinstance(obj, ErrorObj):
return False
# other cases ?
# ...
# manage internal errors
if isinstance(obj, Concept) and obj.metadata.is_builtin and obj.key in BuiltinErrors:
if isinstance(obj, Concept) and obj._metadata.is_builtin and obj.key in BuiltinErrors:
return False
return obj
return bool(obj)
@staticmethod
def is_known(obj):
@@ -914,9 +976,7 @@ class Sheerka(Concept):
unknown_concept = UnknownConcept() # don't use new() for prevent circular reference
unknown_concept.set_value(ConceptParts.BODY, metadata)
for meta in (metadata if isinstance(metadata, list) else [metadata]):
unknown_concept.set_value(meta[0], meta[1])
unknown_concept.metadata.is_evaluated = True
unknown_concept._metadata.is_evaluated = True
return unknown_concept
@staticmethod
@@ -924,7 +984,7 @@ class Sheerka(Concept):
res = {}
for c in core.utils.get_classes("core.builtin_concepts"):
if issubclass(c, Concept) and c != Concept:
res[c().metadata.key] = c
res[c()._metadata.key] = c
return res
@@ -967,3 +1027,29 @@ class Sheerka(Concept):
logging.addLevelName(logging.ERROR, "\033[1;41m%s\033[1;0m" % logging.getLevelName(logging.ERROR))
# uncomment the following line to enable colors
# logging.StreamHandler.emit = add_coloring_to_emit_ansi(logging.StreamHandler.emit)
def to_profile():
sheerka = Sheerka()
sheerka.initialize(save_execution_context=False, enable_process_return_values=False)
event = Event("test", "kodjoko")
execution_context = ExecutionContext(sheerka.name,
event,
sheerka,
BuiltinConcepts.PROCESS_INPUT,
None)
profile_push(execution_context)
@profile(filename="profile_push")
def profile_push(execution_context):
for i in range(177942):
execution_context.push(BuiltinConcepts.NOP,
{"action": "fake"},
execution_context.sheerka.name,
desc="a proper description")
if __name__ == '__main__':
to_profile()