Fixed memory() and RET usage

This commit is contained in:
2020-09-21 21:30:38 +02:00
parent 177a6b1d5f
commit dd520c1680
37 changed files with 816 additions and 353 deletions
+4 -4
View File
@@ -15,9 +15,9 @@ class BuiltinConcepts(Enum):
"""
SHEERKA = "sheerka"
# processing instructions during sheerka.execute()
# processing instructions during sheerka.execute() or sheerka.evaluate_concept()
# The instruction may alter how the actions work
DEBUG = "debug" # activate all debug information
DEBUG = "debug" # activate all debug information
EVAL_BODY_REQUESTED = "eval body" # to evaluate the body
EVAL_WHERE_REQUESTED = "eval where" # to evaluate the where clause
RETURN_BODY_REQUESTED = "return body" # returns the body of the concept instead of the concept itself
@@ -53,7 +53,7 @@ class BuiltinConcepts(Enum):
# builtin attributes
ISA = "is a" # when a concept is an instance of another one
COMMAND = "command" # when the concept must be auto evaluated
AUTO_EVAL = "auto eval" # when the concept must be auto evaluated
# object
USER_INPUT = "user input concept" # represent an input from an user
@@ -155,7 +155,7 @@ BuiltinUnique = [
BuiltinConcepts.TESTING,
BuiltinConcepts.ISA,
BuiltinConcepts.COMMAND,
BuiltinConcepts.AUTO_EVAL,
BuiltinConcepts.INVALID_LESSER_OPERATION,
BuiltinConcepts.INVALID_GREATEST_OPERATION,
+5 -12
View File
@@ -23,7 +23,7 @@ def is_same_success(context, return_values):
Returns True if all returns values are successful and have the same value
:param context:
:param return_values:
:return:
:return: True False or None (None if the concept is not evaluated)
"""
assert isinstance(return_values, list)
@@ -31,17 +31,10 @@ def is_same_success(context, return_values):
if not ret_val.status:
raise Exception("Status is false")
if isinstance(ret_val.body, Concept):
if not ret_val.body.metadata.is_evaluated:
evaluated = context.sheerka.evaluate_concept(context, ret_val.body, eval_body=True)
if not context.sheerka.is_success(evaluated):
raise Exception("Failed to evaluate evaluate")
if isinstance(ret_val.body, Concept) and not ret_val.body.metadata.is_evaluated:
raise Exception("Concept is not evaluated")
return context.sheerka.objvalue(evaluated)
else:
return context.sheerka.objvalue(ret_val.body)
else:
return context.sheerka.objvalue(ret_val)
return context.sheerka.objvalue(ret_val)
try:
reference = _get_value(return_values[0])
@@ -53,7 +46,7 @@ def is_same_success(context, return_values):
except Exception as ex:
context.log_error(ex)
return False
return None
return True
+2 -2
View File
@@ -4,7 +4,7 @@ import time
from core.builtin_concepts import BuiltinConcepts, ParserResultConcept
from core.concept import Concept
from core.sheerka.services.SheerkaExecute import NO_MATCH
from core.sheerka.services.SheerkaShortTermMemory import SheerkaShortTermMemory
from core.sheerka.services.SheerkaMemory import SheerkaMemory
from core.sheerka_logger import get_logger
from sdp.sheerkaDataProvider import Event
@@ -130,7 +130,7 @@ class ExecutionContext:
def __exit__(self, exc_type, exc_val, exc_tb):
if self.stm:
self.sheerka.services[SheerkaShortTermMemory.NAME].remove_context(self)
self.sheerka.services[SheerkaMemory.NAME].remove_context(self)
self._stop = time.time_ns()
if self._show_stats:
+14 -6
View File
@@ -64,6 +64,7 @@ class Sheerka(Concept):
USER_CONCEPTS_KEYS = "User_Concepts" # sequential key for user defined concepts
MAX_EXECUTION_HISTORY = 100
MAX_RETURN_VALUES_HISTORY = 100
def __init__(self, cache_only=False, debug=False, loggers=None):
self.init_logging(debug, loggers)
@@ -113,6 +114,7 @@ class Sheerka(Concept):
self.locals = {}
self.last_executions = []
self.last_return_values = []
@property
def resolved_concepts_by_first_keyword(self):
@@ -138,21 +140,23 @@ class Sheerka(Concept):
def chicken_and_eggs(self):
return self.cache_manager.caches[self.CHICKEN_AND_EGG_CONCEPTS_ENTRY].cache
def bind_service_method(self, bound_method, has_side_effect, as_name=None):
def bind_service_method(self, bound_method, has_side_effect, as_name=None, visible=True):
"""
Bind service method to sheerka instance for ease of use ?
:param bound_method:
:param has_side_effect: False if the method is safe
:param as_name:
:param as_name: give another name to the method
:param visible: make the method visible to Sheerka
:return:
"""
if as_name is None:
as_name = bound_method.__name__
signature = inspect.signature(bound_method)
if len(signature.parameters) > 0 and list(signature.parameters.keys())[0] == "context":
self.methods_with_context.add(as_name)
self.sheerka_methods[as_name] = SheerkaMethod(bound_method, has_side_effect)
if visible:
signature = inspect.signature(bound_method)
if len(signature.parameters) > 0 and list(signature.parameters.keys())[0] == "context":
self.methods_with_context.add(as_name)
self.sheerka_methods[as_name] = SheerkaMethod(bound_method, has_side_effect)
setattr(self, as_name, bound_method)
@@ -437,6 +441,10 @@ class Sheerka(Concept):
del self.last_executions[0]
self.last_executions.append(execution_context)
if len(self.last_return_values) == self.MAX_RETURN_VALUES_HISTORY:
del self.last_return_values[0]
self.last_return_values.append(ret)
return ret
def print(self, result, instructions=None):
+20 -6
View File
@@ -3,9 +3,10 @@ import time
from core.builtin_concepts import BuiltinConcepts
from core.sheerka.services.sheerka_service import BaseService
CONCEPTS_FILE = "_concepts_lite.txt"
CONCEPTS_FILE_LITE = "_concepts_lite.txt"
CONCEPTS_FILE_ALL_CONCEPTS = "_concepts.txt"
CONCEPTS_FILE_TO_USE = CONCEPTS_FILE
CONCEPTS_FILE_TO_USE = CONCEPTS_FILE_ALL_CONCEPTS
class SheerkaAdmin(BaseService):
NAME = "Admin"
@@ -19,6 +20,7 @@ class SheerkaAdmin(BaseService):
self.sheerka.bind_service_method(self.restore, True)
self.sheerka.bind_service_method(self.concepts, False)
self.sheerka.bind_service_method(self.last_created_concept, False)
self.sheerka.bind_service_method(self.last_ret, False)
def caches_names(self):
"""
@@ -27,16 +29,20 @@ class SheerkaAdmin(BaseService):
"""
return list(self.sheerka.cache_manager.caches.keys())
def cache(self, name):
def cache(self, name, *keys):
"""
Returns the content of a cache
:param name:
:param keys: look for a specific key. May ask to sdp if the key is not in cache
:return:
"""
if name not in self.sheerka.cache_manager.caches:
return self.sheerka.new(BuiltinConcepts.NOT_FOUND, body={"cache": name})
return self.sheerka.cache_manager.caches[name].cache.copy()
if not keys:
return self.sheerka.cache_manager.caches[name].cache.copy()
return {key: self.sheerka.cache_manager.get(name, key) for key in keys}
def restore(self, concept_file=CONCEPTS_FILE_TO_USE):
"""
@@ -53,6 +59,7 @@ class SheerkaAdmin(BaseService):
try:
start = time.time_ns()
nb_lines = 0
nb_lines_in_error = 0
self.sheerka.during_restore = True
with open(concept_file, "r") as f:
for line in f.readlines():
@@ -63,14 +70,19 @@ class SheerkaAdmin(BaseService):
self.sheerka.log.info(line)
res = self.sheerka.evaluate_user_input(line)
if len(res) > 1 or not res[0].status:
self.sheerka.log.error("Error detected !")
nb_lines_in_error += 1
self.sheerka.log.error("\u001b[31mError detected !\u001b[0m")
self.sheerka.during_restore = False
stop = time.time_ns()
nano_sec = stop - start
dt = nano_sec / 1e6
elapsed = f"{dt} ms" if dt < 1000 else f"{dt / 1000} s"
print(f"Imported {nb_lines} line(s) in {elapsed}.")
self.sheerka.log.info(f"Imported {nb_lines} line(s) in {elapsed}.")
if nb_lines_in_error > 0:
self.sheerka.log.info(f"\u001b[31m{nb_lines_in_error} errors(s) found.\u001b[0m")
else:
self.sheerka.log.info(f"No error.")
except IOError:
pass
@@ -89,3 +101,5 @@ class SheerkaAdmin(BaseService):
return self.sheerka.new(BuiltinConcepts.NOT_FOUND)
def last_ret(self, context, index=-1):
return self.sheerka.last_return_values[index]
@@ -2,7 +2,7 @@ from dataclasses import dataclass
from core.builtin_concepts import BuiltinConcepts
from core.builtin_helpers import expect_one, only_successful, parse_unrecognized, evaluate
from core.concept import Concept, DoNotResolve, ConceptParts, InfiniteRecursionResolved, NotInit
from core.concept import Concept, DoNotResolve, ConceptParts, InfiniteRecursionResolved, NotInit, ensure_concept
from core.sheerka.services.SheerkaExecute import ParserInput
from core.sheerka.services.sheerka_service import BaseService
from core.tokenizer import Tokenizer
@@ -32,6 +32,7 @@ class SheerkaEvaluateConcept(BaseService):
def initialize(self):
self.sheerka.bind_service_method(self.evaluate_concept, True)
self.sheerka.bind_service_method(self.set_auto_eval, True)
@staticmethod
def infinite_recursion_detected(context, concept):
@@ -229,9 +230,12 @@ class SheerkaEvaluateConcept(BaseService):
continue
source = getattr(concept.metadata, part_key.value)
if source is None or not isinstance(source, str):
if source is None: # or not isinstance(source, str):
continue
if not isinstance(source, str):
raise Exception("Invalid concept init. metadata must be a string")
if source.strip() == "":
concept.compiled[part_key] = DoNotResolve(source)
else:
@@ -251,9 +255,12 @@ class SheerkaEvaluateConcept(BaseService):
if var_name in concept.compiled:
continue
if default_value is None or not isinstance(default_value, str):
if default_value is None:
continue
if not isinstance(default_value, str):
raise Exception("Invalid concept init. variable metadata must be a string")
if default_value.strip() == "":
concept.compiled[var_name] = DoNotResolve(default_value)
else:
@@ -332,12 +339,13 @@ class SheerkaEvaluateConcept(BaseService):
# when it's a concept, evaluate it
if isinstance(to_resolve, Concept) and \
not context.sheerka.isinstance(to_resolve, BuiltinConcepts.RETURN_VALUE):
evaluated = self.evaluate_concept(sub_context, to_resolve)
sub_context.add_values(return_values=evaluated)
if evaluated.key == to_resolve.key: # quicker (and dirtier) than sheerka.is_success()
return self.apply_ret(evaluated)
else:
if not context.sheerka.is_success(evaluated) and evaluated.key != to_resolve.key:
error = evaluated
else:
return evaluated
# otherwise, execute all return values to find out what is the value
else:
@@ -449,7 +457,7 @@ class SheerkaEvaluateConcept(BaseService):
sub_context.protected_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
# auto evaluate commands
if context.sheerka.isa(concept, context.sheerka.new(BuiltinConcepts.COMMAND)):
if context.sheerka.isa(concept, context.sheerka.new(BuiltinConcepts.AUTO_EVAL)):
sub_context.protected_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
self.initialize_concept_asts(sub_context, concept)
@@ -531,12 +539,19 @@ class SheerkaEvaluateConcept(BaseService):
if "body" in all_metadata_to_eval:
concept.metadata.is_evaluated = True
# # update the cache for concepts with no variables
# Cannot use cache. See the comment at the beginning of this method
# if len(concept.metadata.variables) == 0:
# self.sheerka.cache_manager.put(self.sheerka.CONCEPTS_BY_ID_ENTRY, concept.id, concept)
# # update the cache for concepts with no variables
# Cannot use cache. See the comment at the beginning of this method
# if len(concept.metadata.variables) == 0:
# self.sheerka.cache_manager.put(self.sheerka.CONCEPTS_BY_ID_ENTRY, concept.id, concept)
return concept
if not concept.metadata.is_builtin:
self.sheerka.register_object(sub_context, concept.name, concept)
# manage RET metadata
if sub_context.in_context(BuiltinConcepts.EVAL_BODY_REQUESTED) and ConceptParts.RET in concept.values:
return concept.get_value(ConceptParts.RET)
else:
return concept
def compute_metadata_to_eval(self, context, concept):
to_eval = []
@@ -553,10 +568,11 @@ class SheerkaEvaluateConcept(BaseService):
body |= b
to_eval.extend(needed)
needed, v, b = self.get_needed_metadata(concept, ConceptParts.RET, not variables, not body)
variables |= v
body |= b
to_eval.extend(needed)
if context.in_context(BuiltinConcepts.EVAL_BODY_REQUESTED):
needed, v, b = self.get_needed_metadata(concept, ConceptParts.RET, not variables, not body)
variables |= v
body |= b
to_eval.extend(needed)
needed, v, b = self.get_needed_metadata(concept, ConceptParts.POST, not variables, not body)
variables |= v
@@ -571,3 +587,13 @@ class SheerkaEvaluateConcept(BaseService):
to_eval.append("body")
return to_eval
def set_auto_eval(self, context, concept):
"""
add AUTO_EVAL to ISA
:param context:
:param concept:
:return:
"""
ensure_concept(concept)
return self.sheerka.set_isa(context, concept, self.sheerka.new(BuiltinConcepts.AUTO_EVAL))
+132
View File
@@ -0,0 +1,132 @@
from dataclasses import dataclass
from cache.ListIfNeededCache import ListIfNeededCache
from core.builtin_concepts import BuiltinConcepts
from core.concept import Concept
from core.sheerka.services.sheerka_service import BaseService, ServiceObj
@dataclass
class MemoryObject(ServiceObj):
obj: object
class SheerkaMemory(BaseService):
NAME = "Memory"
SHORT_TERM_OBJECTS_ENTRY = "Memory:ShortTermMemoryObjects"
OBJECTS_ENTRY = "Memory:Objects"
def __init__(self, sheerka):
super().__init__(sheerka)
self.short_term_objects = ListIfNeededCache()
self.objects = ListIfNeededCache(default=lambda k: self.sheerka.sdp.get(self.OBJECTS_ENTRY, k))
self.registration = {}
def initialize(self):
self.sheerka.bind_service_method(self.get_from_short_term_memory, False, visible=False)
self.sheerka.bind_service_method(self.add_to_short_term_memory, True, visible=False)
self.sheerka.bind_service_method(self.add_to_memory, True, visible=False)
self.sheerka.bind_service_method(self.get_from_memory, False)
self.sheerka.bind_service_method(self.register_object, True, visible=False)
self.sheerka.bind_service_method(self.unregister_object, True, visible=False)
self.sheerka.bind_service_method(self.add_registered_objects, True, visible=False)
self.sheerka.bind_service_method(self.memory, False)
self.sheerka.cache_manager.register_cache(self.SHORT_TERM_OBJECTS_ENTRY, self.short_term_objects, persist=False)
self.sheerka.cache_manager.register_cache(self.OBJECTS_ENTRY, self.objects, persist=True, use_ref=True)
def get_from_short_term_memory(self, context, key):
while True:
key_to_use = (str(context.id) if context else "") + ":" + key
if (obj := self.sheerka.cache_manager.get(self.SHORT_TERM_OBJECTS_ENTRY, key_to_use)) is not None:
return obj
if context is None:
return None
context = context.get_parent()
def add_to_short_term_memory(self, context, key, concept):
if context:
context.stm = True
key_to_use = (str(context.id) if context else "") + ":" + key
return self.sheerka.cache_manager.put(self.SHORT_TERM_OBJECTS_ENTRY, key_to_use, concept)
def remove_context(self, context):
self.short_term_objects.evict_by_key(lambda k: k.startswith(str(context.id) + ":"))
def add_to_memory(self, context, key, concept):
"""
Adds an object to memory
:param context:
:param key:
:param concept:
:return:
"""
self.objects.put(key, MemoryObject(context.event.get_digest(), concept))
def get_from_memory(self, context, key):
""""
"""
return self.objects.get(key)
def register_object(self, context, key, concept):
"""
Before adding objects to memory, they first need to be registered
:param context:
:param key:
:param concept:
:return:
"""
self.registration[key] = concept
def unregister_object(self, context, key):
"""
To indicate that key is no longer to be remembered
:param context:
:param key:
:return:
"""
try:
del self.registration[key]
except KeyError:
pass
def add_registered_objects(self, context):
"""
Adds all registered objects
:param context:
:return:
"""
for k, v in self.registration.items():
self.add_to_memory(context, k, v)
self.registration.clear()
def memory(self, context, name=None):
"""
Get the list of all objects in memory
:param context:
:param name:
:return:
"""
if name:
name_to_use = name.name if isinstance(name, Concept) else name
self.unregister_object(context, name_to_use)
obj = self.get_from_memory(context, name_to_use)
if obj is None:
return self.sheerka.new(BuiltinConcepts.NOT_FOUND, body={"#name": name})
if isinstance(obj, list):
obj = obj[-1]
return obj.obj
res = {}
for k in self.objects:
obj = self.objects.get(k)
if isinstance(obj, list):
obj = obj[-1]
res[k] = obj.obj
return res
@@ -1,18 +1,27 @@
from core.builtin_concepts import BuiltinConcepts
from core.concept import ensure_concept
from core.sheerka.services.sheerka_service import BaseService
from parsers.BnfParser import BnfParser
class SheerkaModifyConcept(BaseService):
NAME = "ModifyConcept"
def __init__(self, sheerka):
super().__init__(sheerka)
def initialize(self):
self.sheerka.bind_service_method(self.modify_concept, True)
self.sheerka.bind_service_method(self.set_attr, True)
self.sheerka.bind_service_method(self.get_attr, False)
def modify_concept(self, context, concept):
"""
Modify the definition of a concept
:param context:
:param concept:
:return:
"""
old_version = self.sheerka.get_by_id(concept.id)
if old_version is None:
@@ -38,16 +47,12 @@ class SheerkaModifyConcept(BaseService):
BuiltinConcepts.ALREADY_DEFINED,
body=concept))
old_references = self.sheerka.cache_manager.get(self.sheerka.CONCEPTS_REFERENCES_ENTRY, concept.id)
if old_references:
old_references = old_references.copy()
self.sheerka.cache_manager.update_concept(old_version, concept)
# TODO : update concept by first keyword
# TODO : update resolved by first keyword
# TODO : update concepts grammars
# TODO : update when definition_type = DEFINITION_TYPE_DEF
# TODO : update concept by first keyword : have a look at update_references() below
# TODO : update resolved by first keyword : have a look at update_references() below
# TODO : update when definition_type = DEFINITION_TYPE_DEF : have a look at update_references() below
# TODO : Update concepts grammars : have a look at update_references() below
ret = self.sheerka.ret(self.NAME, True, self.sheerka.new(BuiltinConcepts.NEW_CONCEPT, body=concept))
return ret
@@ -69,3 +74,30 @@ class SheerkaModifyConcept(BaseService):
if concept.bnf is not None:
BnfParser.update_recurse_id(context, concept_id, concept.bnf)
# remove the grammar entry so that it can be recreated
self.sheerka.cache_manager.delete(self.sheerka.CONCEPTS_GRAMMARS_ENTRY, concept_id)
def set_attr(self, concept, attribute, value):
"""
Modifies an attribute of a concept (concept.values)
:param context:
:param concept:
:param attribute:
:param value:
:return:
"""
ensure_concept(concept)
concept.set_value(attribute, value)
return self.sheerka.ret(self.NAME, True, self.sheerka.new(BuiltinConcepts.SUCCESS))
def get_attr(self, concept, attribute):
"""
Returns the attribute of a concept
:param context:
:param concept:
:param attribute:
:return:
"""
ensure_concept()
return concept.get_value(attribute)
@@ -59,10 +59,6 @@ class SheerkaSetsManager(BaseService):
res = self.add_concept_to_set(context, concept, concept_set)
# update concept_set references
self.sheerka.services[SheerkaModifyConcept.NAME].update_references(context, concept_set)
self.concepts_in_set.delete(concept_set.id)
return res
def add_concept_to_set(self, context, concept, concept_set):
@@ -85,11 +81,21 @@ class SheerkaSetsManager(BaseService):
self.sheerka.new(BuiltinConcepts.CONCEPT_ALREADY_IN_SET, body=concept, concept_set=concept_set))
self.sets.put(concept_set.id, concept.id)
# invalidate the cache of what contains concept_set
self.concepts_in_set.delete(concept_set.id)
# update concept_set references
self.sheerka.services[SheerkaModifyConcept.NAME].update_references(context, concept_set)
return self.sheerka.ret(self.NAME, True, self.sheerka.new(BuiltinConcepts.SUCCESS))
def add_concepts_to_set(self, context, concepts, concept_set):
"""Adding multiple concepts at the same time"""
"""
Adding multiple concepts at the same time
******** THIS METHOD IS FOR TEST ONLY *************
As it is not optimized. It needs to be rewritten in case of production usage
"""
context.log(f"Adding concepts {concepts} to set {concept_set}", who=self.NAME)
ensure_concept(concept_set)
@@ -1,37 +0,0 @@
from cache.ListIfNeededCache import ListIfNeededCache
from core.sheerka.services.sheerka_service import BaseService
class SheerkaShortTermMemory(BaseService):
NAME = "ShortTermMemory"
SHORT_TERM_MEMORY_ENTRY = "ShortTermMemory:Objects"
def __init__(self, sheerka):
super().__init__(sheerka)
self.objects = ListIfNeededCache()
def initialize(self):
self.sheerka.bind_service_method(self.get_from_short_term_memory, False)
self.sheerka.bind_service_method(self.add_to_short_term_memory, True)
self.sheerka.cache_manager.register_cache(self.SHORT_TERM_MEMORY_ENTRY, self.objects, persist=False)
def get_from_short_term_memory(self, context, key):
while True:
key_to_use = (str(context.id) if context else "") + ":" + key
if (obj := self.sheerka.cache_manager.get(self.SHORT_TERM_MEMORY_ENTRY, key_to_use)) is not None:
return obj
if context is None:
return None
context = context.get_parent()
def add_to_short_term_memory(self, context, key, concept):
if context:
context.stm = True
key_to_use = (str(context.id) if context else "") + ":" + key
return self.sheerka.cache_manager.put(self.SHORT_TERM_MEMORY_ENTRY, key_to_use, concept)
def remove_context(self, context):
self.objects.evict_by_key(lambda k: k.startswith(str(context.id) + ":"))