kodjo/Sheerka-Old
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fixed memory() and RET usage

Browse Source
This commit is contained in:
Kodjo Sossouvi
2020-09-21 21:30:38 +02:00
parent 177a6b1d5f
commit dd520c1680
37 changed files with 816 additions and 353 deletions
Download Patch File Download Diff File Expand all files Collapse all files
_concepts.txt
+45 -45
View File
@@ -19,64 +19,64 @@ def concept eighteen as 18
def concept nineteen as 19 def concept nineteen as 19
def concept twenty as 20 def concept twenty as 20
def concept number def concept number
one isa number set_isa(one, number)
two isa number set_isa(two, number)
three isa number set_isa(three, number)
four isa number set_isa(four, number)
five isa number set_isa(five, number)
six isa number set_isa(six, number)
seven isa number set_isa(seven, number)
eight isa number set_isa(eight, number)
nine isa number set_isa(nine, number)
ten isa number set_isa(ten, number)
eleven isa number set_isa(eleven, number)
twelve isa number set_isa(twelve, number)
thirteen isa number set_isa(thirteen, number)
fourteen isa number set_isa(fourteen, number)
fifteen isa number set_isa(fifteen, number)
sixteen isa number set_isa(sixteen, number)
seventeen isa number set_isa(seventeen, number)
eighteen isa number set_isa(eighteen, number)
nineteen isa number set_isa(nineteen, number)
twenty isa number set_isa(twenty, number)
def concept twenties from bnf twenty number where number < 10 as twenty + number def concept twenties from bnf twenty number where number < 10 as twenty + number
twenties isa number set_isa(twenties, number)
def concept thirty as 30 def concept thirty as 30
thirty isa number set_isa(thirty, number)
def concept thirties from bnf thirty number where number < 10 as thirty + number def concept thirties from bnf thirty number where number < 10 as thirty + number
thirties isa number set_isa(thirties, number)
def concept forty as 40 def concept forty as 40
forty isa number set_isa(forty, number)
def concept forties from bnf forty number where number < 10 as forty + number def concept forties from bnf forty number where number < 10 as forty + number
forties isa number set_isa(forties, number)
def concept fifty as 50 def concept fifty as 50
fifty isa number set_isa(fifty, number)
def concept fifties from bnf fifty number where number < 10 as fifty + number def concept fifties from bnf fifty number where number < 10 as fifty + number
fifties isa number set_isa(fifties, number)
def concept sixty as 60 def concept sixty as 60
sixty isa number set_isa(sixty, number)
def concept sixties from bnf sixty number where number < 10 as sixty + number def concept sixties from bnf sixty number where number < 10 as sixty + number
sixties isa number set_isa(sixties, number)
def concept seventy as 70 def concept seventy as 70
seventy isa number set_isa(seventy, number)
def concept seventies from bnf seventy number where number < 10 as seventy + number def concept seventies from bnf seventy number where number < 10 as seventy + number
seventies isa number set_isa(seventies, number)
def concept eighty as 80 def concept eighty as 80
eighty isa number set_isa(eighty, number)
def concept eighties from bnf eighty number where number < 10 as eighty + number def concept eighties from bnf eighty number where number < 10 as eighty + number
eighties isa number set_isa(eighties, number)
def concept ninety as 90 def concept ninety as 90
ninety isa number set_isa(ninety, number)
def concept nineties from bnf ninety number where number < 10 as ninety + number def concept nineties from bnf ninety number where number < 10 as ninety + number
nineties isa number set_isa(nineties, number)
def concept one hundred as 100 def concept one hundred as 100
one hundred isa number set_isa(one hundred, number)
def concept hundreds from bnf number=n1 'hundred' 'and' number=n2 where n1 < 10 and n2 < 100 as n1 * 100 + n2 def concept hundreds from bnf number=n1 'hundred' 'and' number=n2 where n1 < 10 and n2 < 100 as n1 * 100 + n2
hundreds isa number set_isa(hundreds, number)
def concept hundreds from bnf number 'hundred' where number < 10 as number * 100 def concept hundreds from bnf number 'hundred' where number < 10 as number * 100
last_created_concept() is number last_created_concept() is number
def concept thousands from bnf number 'thousand' where number < 1000 as number * 1000 def concept thousands from bnf number 'thousand' where number < 1000 as number * 1000
thousands isa number set_isa(thousands, number)
def concept thousands from bnf number=n1 'thousand' 'and' number=n2 as n1 * 1000 + n2 where n1 < 1000 and n2 < 1000 def concept thousands from bnf number=n1 'thousand' 'and' number=n2 as n1 * 1000 + n2 where n1 < 1000 and n2 < 1000
last_created_concept() is number last_created_concept() is number
def concept history as history() def concept history as history()
@@ -92,16 +92,16 @@ def concept explain as get_results() | filter("id == 0") | recurse(2)
def concept explain last as get_last_results() | filter("id == 0") | recurse(2) def concept explain last as get_last_results() | filter("id == 0") | recurse(2)
def concept explain x as get_results() | filter(f"id == {x}") | recurse(3) where x def concept explain x as get_results() | filter(f"id == {x}") | recurse(3) where x
def concept explain x '--recurse' y as get_results() | filter(f"id == {x}") | recurse(y) where x,y def concept explain x '--recurse' y as get_results() | filter(f"id == {x}") | recurse(y) where x,y
set_isa(c:explain:, __COMMAND) set_isa(c:explain:, __AUTO_EVAL)
set_isa(c:explain last:, __COMMAND) set_isa(c:explain last:, __AUTO_EVAL)
set_isa(c:explain x:, __COMMAND) set_isa(c:explain x:, __AUTO_EVAL)
def concept precedence a > precedence b as set_is_greater_than(__PRECEDENCE, a, b) def concept precedence a > precedence b as set_is_greater_than(__PRECEDENCE, a, b)
set_isa(c:precedence a > precedence b:, __COMMAND) set_isa(c:precedence a > precedence b:, __AUTO_EVAL)
def concept x is a command as set_isa(x, __COMMAND) def concept x is a command as set_isa(x, __AUTO_EVAL)
set_isa(c:x is a command:, __COMMAND) set_isa(c:x is a command:, __AUTO_EVAL)
def concept q from q ? as question(q) pre is_question() def concept q from q ? as question(q) pre is_question()
set_is_lesser(__PRECEDENCE, q) set_is_lesser(__PRECEDENCE, q)
def concept x is a 'concept' as isinstance(x, Concept) pre is_question() def concept x is a 'concept' as isinstance(x, Concept) pre is_question()
def concept x is a y as isa(x,y) pre is_question() def concept x is a y as isa(x,y) pre is_question()
def concept explain x values where x as get_results() | filter(f"id=={x}") | format_d def concept explain x values where x as get_results() | filter(f"id=={x}") | format_d
set_isa(c:explain x values:, __COMMAND) set_isa(c:explain x values:, __AUTO_EVAL)
_concepts_lite.txt
+7 -3
View File
@@ -1,8 +1,12 @@
def concept one as 1 def concept one as 1
def concept two as 2 def concept two as 2
def concept explain as get_results() | filter("id == 0") | recurse(2) def concept explain as get_results() | filter("id == 0") | recurse(2)
set_isa(c:explain:, __COMMAND) set_isa(c:explain:, __AUTO_EVAL)
def concept explain last as get_last_results() | filter("id == 0") | recurse(2) def concept explain last as get_last_results() | filter("id == 0") | recurse(2)
set_isa(c:explain last:, __COMMAND) set_isa(c:explain last:, __AUTO_EVAL)
def concept explain x as get_results() | filter(f"id == {x}") | recurse(3) where x def concept explain x as get_results() | filter(f"id == {x}") | recurse(3) where x
set_isa(c:explain x:, __COMMAND) set_isa(c:explain x:, __AUTO_EVAL)
def concept apple
def concept table
def concept location
def concept x is on y as set_attr(x, location, y)
src/core/builtin_concepts.py
+3 -3
View File
@@ -15,7 +15,7 @@ class BuiltinConcepts(Enum):
""" """
SHEERKA = "sheerka" SHEERKA = "sheerka"
# processing instructions during sheerka.execute() # processing instructions during sheerka.execute() or sheerka.evaluate_concept()
# The instruction may alter how the actions work # The instruction may alter how the actions work
DEBUG = "debug" # activate all debug information DEBUG = "debug" # activate all debug information
EVAL_BODY_REQUESTED = "eval body" # to evaluate the body EVAL_BODY_REQUESTED = "eval body" # to evaluate the body
@@ -53,7 +53,7 @@ class BuiltinConcepts(Enum):
# builtin attributes # builtin attributes
ISA = "is a" # when a concept is an instance of another one ISA = "is a" # when a concept is an instance of another one
COMMAND = "command" # when the concept must be auto evaluated AUTO_EVAL = "auto eval" # when the concept must be auto evaluated
# object # object
USER_INPUT = "user input concept" # represent an input from an user USER_INPUT = "user input concept" # represent an input from an user
@@ -155,7 +155,7 @@ BuiltinUnique = [
BuiltinConcepts.TESTING, BuiltinConcepts.TESTING,
BuiltinConcepts.ISA, BuiltinConcepts.ISA,
BuiltinConcepts.COMMAND, BuiltinConcepts.AUTO_EVAL,
BuiltinConcepts.INVALID_LESSER_OPERATION, BuiltinConcepts.INVALID_LESSER_OPERATION,
BuiltinConcepts.INVALID_GREATEST_OPERATION, BuiltinConcepts.INVALID_GREATEST_OPERATION,
src/core/builtin_helpers.py
+4 -11
View File
@@ -23,7 +23,7 @@ def is_same_success(context, return_values):
Returns True if all returns values are successful and have the same value Returns True if all returns values are successful and have the same value
:param context: :param context:
:param return_values: :param return_values:
:return: :return: True False or None (None if the concept is not evaluated)
""" """
assert isinstance(return_values, list) assert isinstance(return_values, list)
@@ -31,16 +31,9 @@ def is_same_success(context, return_values):
if not ret_val.status: if not ret_val.status:
raise Exception("Status is false") raise Exception("Status is false")
if isinstance(ret_val.body, Concept): if isinstance(ret_val.body, Concept) and not ret_val.body.metadata.is_evaluated:
if not ret_val.body.metadata.is_evaluated: raise Exception("Concept is not evaluated")
evaluated = context.sheerka.evaluate_concept(context, ret_val.body, eval_body=True)
if not context.sheerka.is_success(evaluated):
raise Exception("Failed to evaluate evaluate")
return context.sheerka.objvalue(evaluated)
else:
return context.sheerka.objvalue(ret_val.body)
else:
return context.sheerka.objvalue(ret_val) return context.sheerka.objvalue(ret_val)
try: try:
@@ -53,7 +46,7 @@ def is_same_success(context, return_values):
except Exception as ex: except Exception as ex:
context.log_error(ex) context.log_error(ex)
return False return None
return True return True
src/core/sheerka/ExecutionContext.py
+2 -2
View File
@@ -4,7 +4,7 @@ import time
from core.builtin_concepts import BuiltinConcepts, ParserResultConcept from core.builtin_concepts import BuiltinConcepts, ParserResultConcept
from core.concept import Concept from core.concept import Concept
from core.sheerka.services.SheerkaExecute import NO_MATCH from core.sheerka.services.SheerkaExecute import NO_MATCH
from core.sheerka.services.SheerkaShortTermMemory import SheerkaShortTermMemory from core.sheerka.services.SheerkaMemory import SheerkaMemory
from core.sheerka_logger import get_logger from core.sheerka_logger import get_logger
from sdp.sheerkaDataProvider import Event from sdp.sheerkaDataProvider import Event
@@ -130,7 +130,7 @@ class ExecutionContext:
def __exit__(self, exc_type, exc_val, exc_tb): def __exit__(self, exc_type, exc_val, exc_tb):
if self.stm: if self.stm:
self.sheerka.services[SheerkaShortTermMemory.NAME].remove_context(self) self.sheerka.services[SheerkaMemory.NAME].remove_context(self)
self._stop = time.time_ns() self._stop = time.time_ns()
if self._show_stats: if self._show_stats:
src/core/sheerka/Sheerka.py
+10 -2
View File
@@ -64,6 +64,7 @@ class Sheerka(Concept):
USER_CONCEPTS_KEYS = "User_Concepts" # sequential key for user defined concepts USER_CONCEPTS_KEYS = "User_Concepts" # sequential key for user defined concepts
MAX_EXECUTION_HISTORY = 100 MAX_EXECUTION_HISTORY = 100
MAX_RETURN_VALUES_HISTORY = 100
def __init__(self, cache_only=False, debug=False, loggers=None): def __init__(self, cache_only=False, debug=False, loggers=None):
self.init_logging(debug, loggers) self.init_logging(debug, loggers)
@@ -113,6 +114,7 @@ class Sheerka(Concept):
self.locals = {} self.locals = {}
self.last_executions = [] self.last_executions = []
self.last_return_values = []
@property @property
def resolved_concepts_by_first_keyword(self): def resolved_concepts_by_first_keyword(self):
@@ -138,17 +140,19 @@ class Sheerka(Concept):
def chicken_and_eggs(self): def chicken_and_eggs(self):
return self.cache_manager.caches[self.CHICKEN_AND_EGG_CONCEPTS_ENTRY].cache return self.cache_manager.caches[self.CHICKEN_AND_EGG_CONCEPTS_ENTRY].cache
def bind_service_method(self, bound_method, has_side_effect, as_name=None): def bind_service_method(self, bound_method, has_side_effect, as_name=None, visible=True):
""" """
Bind service method to sheerka instance for ease of use ? Bind service method to sheerka instance for ease of use ?
:param bound_method: :param bound_method:
:param has_side_effect: False if the method is safe :param has_side_effect: False if the method is safe
:param as_name: :param as_name: give another name to the method
:param visible: make the method visible to Sheerka
:return: :return:
""" """
if as_name is None: if as_name is None:
as_name = bound_method.__name__ as_name = bound_method.__name__
if visible:
signature = inspect.signature(bound_method) signature = inspect.signature(bound_method)
if len(signature.parameters) > 0 and list(signature.parameters.keys())[0] == "context": if len(signature.parameters) > 0 and list(signature.parameters.keys())[0] == "context":
self.methods_with_context.add(as_name) self.methods_with_context.add(as_name)
@@ -437,6 +441,10 @@ class Sheerka(Concept):
del self.last_executions[0] del self.last_executions[0]
self.last_executions.append(execution_context) self.last_executions.append(execution_context)
if len(self.last_return_values) == self.MAX_RETURN_VALUES_HISTORY:
del self.last_return_values[0]
self.last_return_values.append(ret)
return ret return ret
def print(self, result, instructions=None): def print(self, result, instructions=None):
src/core/sheerka/services/SheerkaAdmin.py
+19 -5
View File
@@ -3,9 +3,10 @@ import time
from core.builtin_concepts import BuiltinConcepts from core.builtin_concepts import BuiltinConcepts
from core.sheerka.services.sheerka_service import BaseService from core.sheerka.services.sheerka_service import BaseService
CONCEPTS_FILE = "_concepts_lite.txt" CONCEPTS_FILE_LITE = "_concepts_lite.txt"
CONCEPTS_FILE_ALL_CONCEPTS = "_concepts.txt" CONCEPTS_FILE_ALL_CONCEPTS = "_concepts.txt"
CONCEPTS_FILE_TO_USE = CONCEPTS_FILE CONCEPTS_FILE_TO_USE = CONCEPTS_FILE_ALL_CONCEPTS
class SheerkaAdmin(BaseService): class SheerkaAdmin(BaseService):
NAME = "Admin" NAME = "Admin"
@@ -19,6 +20,7 @@ class SheerkaAdmin(BaseService):
self.sheerka.bind_service_method(self.restore, True) self.sheerka.bind_service_method(self.restore, True)
self.sheerka.bind_service_method(self.concepts, False) self.sheerka.bind_service_method(self.concepts, False)
self.sheerka.bind_service_method(self.last_created_concept, False) self.sheerka.bind_service_method(self.last_created_concept, False)
self.sheerka.bind_service_method(self.last_ret, False)
def caches_names(self): def caches_names(self):
""" """
@@ -27,17 +29,21 @@ class SheerkaAdmin(BaseService):
""" """
return list(self.sheerka.cache_manager.caches.keys()) return list(self.sheerka.cache_manager.caches.keys())
def cache(self, name): def cache(self, name, *keys):
""" """
Returns the content of a cache Returns the content of a cache
:param name: :param name:
:param keys: look for a specific key. May ask to sdp if the key is not in cache
:return: :return:
""" """
if name not in self.sheerka.cache_manager.caches: if name not in self.sheerka.cache_manager.caches:
return self.sheerka.new(BuiltinConcepts.NOT_FOUND, body={"cache": name}) return self.sheerka.new(BuiltinConcepts.NOT_FOUND, body={"cache": name})
if not keys:
return self.sheerka.cache_manager.caches[name].cache.copy() return self.sheerka.cache_manager.caches[name].cache.copy()
return {key: self.sheerka.cache_manager.get(name, key) for key in keys}
def restore(self, concept_file=CONCEPTS_FILE_TO_USE): def restore(self, concept_file=CONCEPTS_FILE_TO_USE):
""" """
Restore the state with all previous valid concept definitions Restore the state with all previous valid concept definitions
@@ -53,6 +59,7 @@ class SheerkaAdmin(BaseService):
try: try:
start = time.time_ns() start = time.time_ns()
nb_lines = 0 nb_lines = 0
nb_lines_in_error = 0
self.sheerka.during_restore = True self.sheerka.during_restore = True
with open(concept_file, "r") as f: with open(concept_file, "r") as f:
for line in f.readlines(): for line in f.readlines():
@@ -63,14 +70,19 @@ class SheerkaAdmin(BaseService):
self.sheerka.log.info(line) self.sheerka.log.info(line)
res = self.sheerka.evaluate_user_input(line) res = self.sheerka.evaluate_user_input(line)
if len(res) > 1 or not res[0].status: if len(res) > 1 or not res[0].status:
self.sheerka.log.error("Error detected !") nb_lines_in_error += 1
self.sheerka.log.error("\u001b[31mError detected !\u001b[0m")
self.sheerka.during_restore = False self.sheerka.during_restore = False
stop = time.time_ns() stop = time.time_ns()
nano_sec = stop - start nano_sec = stop - start
dt = nano_sec / 1e6 dt = nano_sec / 1e6
elapsed = f"{dt} ms" if dt < 1000 else f"{dt / 1000} s" elapsed = f"{dt} ms" if dt < 1000 else f"{dt / 1000} s"
print(f"Imported {nb_lines} line(s) in {elapsed}.") self.sheerka.log.info(f"Imported {nb_lines} line(s) in {elapsed}.")
if nb_lines_in_error > 0:
self.sheerka.log.info(f"\u001b[31m{nb_lines_in_error} errors(s) found.\u001b[0m")
else:
self.sheerka.log.info(f"No error.")
except IOError: except IOError:
pass pass
@@ -89,3 +101,5 @@ class SheerkaAdmin(BaseService):
return self.sheerka.new(BuiltinConcepts.NOT_FOUND) return self.sheerka.new(BuiltinConcepts.NOT_FOUND)
def last_ret(self, context, index=-1):
return self.sheerka.last_return_values[index]
src/core/sheerka/services/SheerkaEvaluateConcept.py
+33 -7
View File
@@ -2,7 +2,7 @@ from dataclasses import dataclass
from core.builtin_concepts import BuiltinConcepts from core.builtin_concepts import BuiltinConcepts
from core.builtin_helpers import expect_one, only_successful, parse_unrecognized, evaluate from core.builtin_helpers import expect_one, only_successful, parse_unrecognized, evaluate
from core.concept import Concept, DoNotResolve, ConceptParts, InfiniteRecursionResolved, NotInit from core.concept import Concept, DoNotResolve, ConceptParts, InfiniteRecursionResolved, NotInit, ensure_concept
from core.sheerka.services.SheerkaExecute import ParserInput from core.sheerka.services.SheerkaExecute import ParserInput
from core.sheerka.services.sheerka_service import BaseService from core.sheerka.services.sheerka_service import BaseService
from core.tokenizer import Tokenizer from core.tokenizer import Tokenizer
@@ -32,6 +32,7 @@ class SheerkaEvaluateConcept(BaseService):
def initialize(self): def initialize(self):
self.sheerka.bind_service_method(self.evaluate_concept, True) self.sheerka.bind_service_method(self.evaluate_concept, True)
self.sheerka.bind_service_method(self.set_auto_eval, True)
@staticmethod @staticmethod
def infinite_recursion_detected(context, concept): def infinite_recursion_detected(context, concept):
@@ -229,9 +230,12 @@ class SheerkaEvaluateConcept(BaseService):
continue continue
source = getattr(concept.metadata, part_key.value) source = getattr(concept.metadata, part_key.value)
if source is None or not isinstance(source, str): if source is None: # or not isinstance(source, str):
continue continue
if not isinstance(source, str):
raise Exception("Invalid concept init. metadata must be a string")
if source.strip() == "": if source.strip() == "":
concept.compiled[part_key] = DoNotResolve(source) concept.compiled[part_key] = DoNotResolve(source)
else: else:
@@ -251,9 +255,12 @@ class SheerkaEvaluateConcept(BaseService):
if var_name in concept.compiled: if var_name in concept.compiled:
continue continue
if default_value is None or not isinstance(default_value, str): if default_value is None:
continue continue
if not isinstance(default_value, str):
raise Exception("Invalid concept init. variable metadata must be a string")
if default_value.strip() == "": if default_value.strip() == "":
concept.compiled[var_name] = DoNotResolve(default_value) concept.compiled[var_name] = DoNotResolve(default_value)
else: else:
@@ -332,12 +339,13 @@ class SheerkaEvaluateConcept(BaseService):
# when it's a concept, evaluate it # when it's a concept, evaluate it
if isinstance(to_resolve, Concept) and \ if isinstance(to_resolve, Concept) and \
not context.sheerka.isinstance(to_resolve, BuiltinConcepts.RETURN_VALUE): not context.sheerka.isinstance(to_resolve, BuiltinConcepts.RETURN_VALUE):
evaluated = self.evaluate_concept(sub_context, to_resolve) evaluated = self.evaluate_concept(sub_context, to_resolve)
sub_context.add_values(return_values=evaluated) sub_context.add_values(return_values=evaluated)
if evaluated.key == to_resolve.key: # quicker (and dirtier) than sheerka.is_success() if not context.sheerka.is_success(evaluated) and evaluated.key != to_resolve.key:
return self.apply_ret(evaluated)
else:
error = evaluated error = evaluated
else:
return evaluated
# otherwise, execute all return values to find out what is the value # otherwise, execute all return values to find out what is the value
else: else:
@@ -449,7 +457,7 @@ class SheerkaEvaluateConcept(BaseService):
sub_context.protected_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED) sub_context.protected_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
# auto evaluate commands # auto evaluate commands
if context.sheerka.isa(concept, context.sheerka.new(BuiltinConcepts.COMMAND)): if context.sheerka.isa(concept, context.sheerka.new(BuiltinConcepts.AUTO_EVAL)):
sub_context.protected_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED) sub_context.protected_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
self.initialize_concept_asts(sub_context, concept) self.initialize_concept_asts(sub_context, concept)
@@ -536,6 +544,13 @@ class SheerkaEvaluateConcept(BaseService):
# if len(concept.metadata.variables) == 0: # if len(concept.metadata.variables) == 0:
# self.sheerka.cache_manager.put(self.sheerka.CONCEPTS_BY_ID_ENTRY, concept.id, concept) # self.sheerka.cache_manager.put(self.sheerka.CONCEPTS_BY_ID_ENTRY, concept.id, concept)
if not concept.metadata.is_builtin:
self.sheerka.register_object(sub_context, concept.name, concept)
# manage RET metadata
if sub_context.in_context(BuiltinConcepts.EVAL_BODY_REQUESTED) and ConceptParts.RET in concept.values:
return concept.get_value(ConceptParts.RET)
else:
return concept return concept
def compute_metadata_to_eval(self, context, concept): def compute_metadata_to_eval(self, context, concept):
@@ -553,6 +568,7 @@ class SheerkaEvaluateConcept(BaseService):
body |= b body |= b
to_eval.extend(needed) to_eval.extend(needed)
if context.in_context(BuiltinConcepts.EVAL_BODY_REQUESTED):
needed, v, b = self.get_needed_metadata(concept, ConceptParts.RET, not variables, not body) needed, v, b = self.get_needed_metadata(concept, ConceptParts.RET, not variables, not body)
variables |= v variables |= v
body |= b body |= b
@@ -571,3 +587,13 @@ class SheerkaEvaluateConcept(BaseService):
to_eval.append("body") to_eval.append("body")
return to_eval return to_eval
def set_auto_eval(self, context, concept):
"""
add AUTO_EVAL to ISA
:param context:
:param concept:
:return:
"""
ensure_concept(concept)
return self.sheerka.set_isa(context, concept, self.sheerka.new(BuiltinConcepts.AUTO_EVAL))
src/core/sheerka/services/SheerkaMemory.py
+132
View File
@@ -0,0 +1,132 @@
from dataclasses import dataclass
from cache.ListIfNeededCache import ListIfNeededCache
from core.builtin_concepts import BuiltinConcepts
from core.concept import Concept
from core.sheerka.services.sheerka_service import BaseService, ServiceObj
@dataclass
class MemoryObject(ServiceObj):
obj: object
class SheerkaMemory(BaseService):
NAME = "Memory"
SHORT_TERM_OBJECTS_ENTRY = "Memory:ShortTermMemoryObjects"
OBJECTS_ENTRY = "Memory:Objects"
def __init__(self, sheerka):
super().__init__(sheerka)
self.short_term_objects = ListIfNeededCache()
self.objects = ListIfNeededCache(default=lambda k: self.sheerka.sdp.get(self.OBJECTS_ENTRY, k))
self.registration = {}
def initialize(self):
self.sheerka.bind_service_method(self.get_from_short_term_memory, False, visible=False)
self.sheerka.bind_service_method(self.add_to_short_term_memory, True, visible=False)
self.sheerka.bind_service_method(self.add_to_memory, True, visible=False)
self.sheerka.bind_service_method(self.get_from_memory, False)
self.sheerka.bind_service_method(self.register_object, True, visible=False)
self.sheerka.bind_service_method(self.unregister_object, True, visible=False)
self.sheerka.bind_service_method(self.add_registered_objects, True, visible=False)
self.sheerka.bind_service_method(self.memory, False)
self.sheerka.cache_manager.register_cache(self.SHORT_TERM_OBJECTS_ENTRY, self.short_term_objects, persist=False)
self.sheerka.cache_manager.register_cache(self.OBJECTS_ENTRY, self.objects, persist=True, use_ref=True)
def get_from_short_term_memory(self, context, key):
while True:
key_to_use = (str(context.id) if context else "") + ":" + key
if (obj := self.sheerka.cache_manager.get(self.SHORT_TERM_OBJECTS_ENTRY, key_to_use)) is not None:
return obj
if context is None:
return None
context = context.get_parent()
def add_to_short_term_memory(self, context, key, concept):
if context:
context.stm = True
key_to_use = (str(context.id) if context else "") + ":" + key
return self.sheerka.cache_manager.put(self.SHORT_TERM_OBJECTS_ENTRY, key_to_use, concept)
def remove_context(self, context):
self.short_term_objects.evict_by_key(lambda k: k.startswith(str(context.id) + ":"))
def add_to_memory(self, context, key, concept):
"""
Adds an object to memory
:param context:
:param key:
:param concept:
:return:
"""
self.objects.put(key, MemoryObject(context.event.get_digest(), concept))
def get_from_memory(self, context, key):
""""
"""
return self.objects.get(key)
def register_object(self, context, key, concept):
"""
Before adding objects to memory, they first need to be registered
:param context:
:param key:
:param concept:
:return:
"""
self.registration[key] = concept
def unregister_object(self, context, key):
"""
To indicate that key is no longer to be remembered
:param context:
:param key:
:return:
"""
try:
del self.registration[key]
except KeyError:
pass
def add_registered_objects(self, context):
"""
Adds all registered objects
:param context:
:return:
"""
for k, v in self.registration.items():
self.add_to_memory(context, k, v)
self.registration.clear()
def memory(self, context, name=None):
"""
Get the list of all objects in memory
:param context:
:param name:
:return:
"""
if name:
name_to_use = name.name if isinstance(name, Concept) else name
self.unregister_object(context, name_to_use)
obj = self.get_from_memory(context, name_to_use)
if obj is None:
return self.sheerka.new(BuiltinConcepts.NOT_FOUND, body={"#name": name})
if isinstance(obj, list):
obj = obj[-1]
return obj.obj
res = {}
for k in self.objects:
obj = self.objects.get(k)
if isinstance(obj, list):
obj = obj[-1]
res[k] = obj.obj
return res
src/core/sheerka/services/SheerkaModifyConcept.py
+40 -8
View File
@@ -1,4 +1,5 @@
from core.builtin_concepts import BuiltinConcepts from core.builtin_concepts import BuiltinConcepts
from core.concept import ensure_concept
from core.sheerka.services.sheerka_service import BaseService from core.sheerka.services.sheerka_service import BaseService
from parsers.BnfParser import BnfParser from parsers.BnfParser import BnfParser
@@ -11,8 +12,16 @@ class SheerkaModifyConcept(BaseService):
def initialize(self): def initialize(self):
self.sheerka.bind_service_method(self.modify_concept, True) self.sheerka.bind_service_method(self.modify_concept, True)
self.sheerka.bind_service_method(self.set_attr, True)
self.sheerka.bind_service_method(self.get_attr, False)
def modify_concept(self, context, concept): def modify_concept(self, context, concept):
"""
Modify the definition of a concept
:param context:
:param concept:
:return:
"""
old_version = self.sheerka.get_by_id(concept.id) old_version = self.sheerka.get_by_id(concept.id)
if old_version is None: if old_version is None:
@@ -38,16 +47,12 @@ class SheerkaModifyConcept(BaseService):
BuiltinConcepts.ALREADY_DEFINED, BuiltinConcepts.ALREADY_DEFINED,
body=concept)) body=concept))
old_references = self.sheerka.cache_manager.get(self.sheerka.CONCEPTS_REFERENCES_ENTRY, concept.id)
if old_references:
old_references = old_references.copy()
self.sheerka.cache_manager.update_concept(old_version, concept) self.sheerka.cache_manager.update_concept(old_version, concept)
# TODO : update concept by first keyword # TODO : update concept by first keyword : have a look at update_references() below
# TODO : update resolved by first keyword # TODO : update resolved by first keyword : have a look at update_references() below
# TODO : update concepts grammars # TODO : update when definition_type = DEFINITION_TYPE_DEF : have a look at update_references() below
# TODO : update when definition_type = DEFINITION_TYPE_DEF # TODO : Update concepts grammars : have a look at update_references() below
ret = self.sheerka.ret(self.NAME, True, self.sheerka.new(BuiltinConcepts.NEW_CONCEPT, body=concept)) ret = self.sheerka.ret(self.NAME, True, self.sheerka.new(BuiltinConcepts.NEW_CONCEPT, body=concept))
return ret return ret
@@ -69,3 +74,30 @@ class SheerkaModifyConcept(BaseService):
if concept.bnf is not None: if concept.bnf is not None:
BnfParser.update_recurse_id(context, concept_id, concept.bnf) BnfParser.update_recurse_id(context, concept_id, concept.bnf)
# remove the grammar entry so that it can be recreated
self.sheerka.cache_manager.delete(self.sheerka.CONCEPTS_GRAMMARS_ENTRY, concept_id)
def set_attr(self, concept, attribute, value):
"""
Modifies an attribute of a concept (concept.values)
:param context:
:param concept:
:param attribute:
:param value:
:return:
"""
ensure_concept(concept)
concept.set_value(attribute, value)
return self.sheerka.ret(self.NAME, True, self.sheerka.new(BuiltinConcepts.SUCCESS))
def get_attr(self, concept, attribute):
"""
Returns the attribute of a concept
:param context:
:param concept:
:param attribute:
:return:
"""
ensure_concept()
return concept.get_value(attribute)
src/core/sheerka/services/SheerkaSetsManager.py
+11 -5
View File
@@ -59,10 +59,6 @@ class SheerkaSetsManager(BaseService):
res = self.add_concept_to_set(context, concept, concept_set) res = self.add_concept_to_set(context, concept, concept_set)
# update concept_set references
self.sheerka.services[SheerkaModifyConcept.NAME].update_references(context, concept_set)
self.concepts_in_set.delete(concept_set.id)
return res return res
def add_concept_to_set(self, context, concept, concept_set): def add_concept_to_set(self, context, concept, concept_set):
@@ -85,11 +81,21 @@ class SheerkaSetsManager(BaseService):
self.sheerka.new(BuiltinConcepts.CONCEPT_ALREADY_IN_SET, body=concept, concept_set=concept_set)) self.sheerka.new(BuiltinConcepts.CONCEPT_ALREADY_IN_SET, body=concept, concept_set=concept_set))
self.sets.put(concept_set.id, concept.id) self.sets.put(concept_set.id, concept.id)
# invalidate the cache of what contains concept_set
self.concepts_in_set.delete(concept_set.id) self.concepts_in_set.delete(concept_set.id)
# update concept_set references
self.sheerka.services[SheerkaModifyConcept.NAME].update_references(context, concept_set)
return self.sheerka.ret(self.NAME, True, self.sheerka.new(BuiltinConcepts.SUCCESS)) return self.sheerka.ret(self.NAME, True, self.sheerka.new(BuiltinConcepts.SUCCESS))
def add_concepts_to_set(self, context, concepts, concept_set): def add_concepts_to_set(self, context, concepts, concept_set):
"""Adding multiple concepts at the same time""" """
Adding multiple concepts at the same time
******** THIS METHOD IS FOR TEST ONLY *************
As it is not optimized. It needs to be rewritten in case of production usage
"""
context.log(f"Adding concepts {concepts} to set {concept_set}", who=self.NAME) context.log(f"Adding concepts {concepts} to set {concept_set}", who=self.NAME)
ensure_concept(concept_set) ensure_concept(concept_set)
src/core/sheerka/services/SheerkaShortTermMemory.py
-37
View File
@@ -1,37 +0,0 @@
from cache.ListIfNeededCache import ListIfNeededCache
from core.sheerka.services.sheerka_service import BaseService
class SheerkaShortTermMemory(BaseService):
NAME = "ShortTermMemory"
SHORT_TERM_MEMORY_ENTRY = "ShortTermMemory:Objects"
def __init__(self, sheerka):
super().__init__(sheerka)
self.objects = ListIfNeededCache()
def initialize(self):
self.sheerka.bind_service_method(self.get_from_short_term_memory, False)
self.sheerka.bind_service_method(self.add_to_short_term_memory, True)
self.sheerka.cache_manager.register_cache(self.SHORT_TERM_MEMORY_ENTRY, self.objects, persist=False)
def get_from_short_term_memory(self, context, key):
while True:
key_to_use = (str(context.id) if context else "") + ":" + key
if (obj := self.sheerka.cache_manager.get(self.SHORT_TERM_MEMORY_ENTRY, key_to_use)) is not None:
return obj
if context is None:
return None
context = context.get_parent()
def add_to_short_term_memory(self, context, key, concept):
if context:
context.stm = True
key_to_use = (str(context.id) if context else "") + ":" + key
return self.sheerka.cache_manager.put(self.SHORT_TERM_MEMORY_ENTRY, key_to_use, concept)
def remove_context(self, context):
self.objects.evict_by_key(lambda k: k.startswith(str(context.id) + ":"))
src/evaluators/AddToMemoryEvaluator.py
+27
View File
@@ -0,0 +1,27 @@
from core.builtin_concepts import BuiltinConcepts
from evaluators.BaseEvaluator import OneReturnValueEvaluator
class AddToMemoryEvaluator(OneReturnValueEvaluator):
"""
Last chance to alter the return_value
This evaluator is supposed to be a generic evaluator for all rules that must be executed just before
the aggregations
"""
NAME = "AddToMemory"
def __init__(self):
super().__init__(self.NAME, [BuiltinConcepts.AFTER_EVALUATION], 10)
def matches(self, context, return_value):
evaluation_parents = context.get_parents(lambda c: c.action == BuiltinConcepts.PROCESSING)
if len(evaluation_parents) > 1:
return False # It must be executed only when the top level context
from core.sheerka.services.SheerkaMemory import SheerkaMemory
return len(context.sheerka.services[SheerkaMemory.NAME].registration) > 0
def eval(self, context, return_value):
context.sheerka.add_registered_objects(context)
return None # no need to have a second pass
src/evaluators/ConceptEvaluator.py
+2 -1
View File
@@ -53,8 +53,9 @@ class ConceptEvaluator(OneReturnValueEvaluator):
evaluated = sheerka.evaluate_concept(context, concept) evaluated = sheerka.evaluate_concept(context, concept)
if evaluated.key != concept.key: if not sheerka.is_success(evaluated) and evaluated.key != concept.key:
# evaluated.key != concept.key means that we have transformed the concept # evaluated.key != concept.key means that we have transformed the concept
# not sheerka.is_success(evaluated) means that it was transformed into an error
# When you successfully evaluate an error, the status should not be false # When you successfully evaluate an error, the status should not be false
return sheerka.ret( return sheerka.ret(
self.name, self.name,
src/evaluators/PostExecutionEvaluator.py
+1 -1
View File
@@ -22,7 +22,7 @@ class PostExecutionEvaluator(OneReturnValueEvaluator):
# only support the rule for the COMMANDS # only support the rule for the COMMANDS
value = return_value.body value = return_value.body
return isinstance(value, Concept) and context.sheerka.isa(value, context.sheerka.new(BuiltinConcepts.COMMAND)) return isinstance(value, Concept) and context.sheerka.isa(value, context.sheerka.new(BuiltinConcepts.AUTO_EVAL))
def eval(self, context, return_value): def eval(self, context, return_value):
# only support the rule for the COMMANDS # only support the rule for the COMMANDS
src/evaluators/PythonEvaluator.py
+4 -1
View File
@@ -85,6 +85,9 @@ class PythonEvaluator(OneReturnValueEvaluator):
not_for_me = context.sheerka.new(BuiltinConcepts.NOT_FOR_ME, body=node) not_for_me = context.sheerka.new(BuiltinConcepts.NOT_FOR_ME, body=node)
return sheerka.ret(self.name, False, not_for_me, parents=[return_value]) return sheerka.ret(self.name, False, not_for_me, parents=[return_value])
# If we evaluate a Concept metadata which is NOT the body ex (pre, post, where...)
# We need to disable the function that may alter the state
# It's a poor way to have source code security check
attr_under_eval = context.get_parents(lambda ec: ec.action == BuiltinConcepts.EVALUATING_ATTRIBUTE) attr_under_eval = context.get_parents(lambda ec: ec.action == BuiltinConcepts.EVALUATING_ATTRIBUTE)
if attr_under_eval: if attr_under_eval:
attr_under_eval = attr_under_eval[0] attr_under_eval = attr_under_eval[0]
@@ -256,7 +259,7 @@ class PythonEvaluator(OneReturnValueEvaluator):
else: else:
context.log(f"Evaluating '{concept}'", self.name) context.log(f"Evaluating '{concept}'", self.name)
evaluated = context.sheerka.evaluate_concept(context, concept, eval_body=True) evaluated = context.sheerka.evaluate_concept(context, concept, eval_body=True)
if evaluated.key != concept.key: if not context.sheerka.is_success(evaluated) and evaluated.key != concept.key:
context.log(f"Error while evaluating '{name}'. Skipping.", self.name) context.log(f"Error while evaluating '{name}'. Skipping.", self.name)
continue continue
concept = evaluated concept = evaluated
src/evaluators/RetEvaluator.py
+41 -40
View File
@@ -1,40 +1,41 @@
from core.builtin_concepts import BuiltinConcepts # from core.builtin_concepts import BuiltinConcepts
from core.builtin_helpers import ensure_evaluated # from core.builtin_helpers import ensure_evaluated
from core.concept import Concept, ConceptParts # from core.concept import Concept, ConceptParts
from evaluators.BaseEvaluator import OneReturnValueEvaluator # from evaluators.BaseEvaluator import OneReturnValueEvaluator
#
#
class RetEvaluator(OneReturnValueEvaluator): # class RetEvaluator(OneReturnValueEvaluator):
""" # """
The evaluator transforms the concept, using the RET metadata value # The evaluator transforms the concept, using the RET metadata value
""" # """
#
NAME = "Ret" # NAME = "Ret"
#
def __init__(self): # def __init__(self):
super().__init__(self.NAME, [BuiltinConcepts.EVALUATION], 10) # super().__init__(self.NAME, [BuiltinConcepts.EVALUATION], 10)
# self.enabled = False
def matches(self, context, return_value): #
return return_value.status and \ # def matches(self, context, return_value):
isinstance(return_value.value, Concept) and \ # return return_value.status and \
return_value.value.metadata.ret is not None # isinstance(return_value.value, Concept) and \
# return_value.value.metadata.ret is not None
def eval(self, context, return_value): #
sheerka = context.sheerka # def eval(self, context, return_value):
concept = return_value.value # sheerka = context.sheerka
context.log(f"Processing ret value for concept {concept}.", self.name) # concept = return_value.value
# context.log(f"Processing ret value for concept {concept}.", self.name)
if not concept.metadata.is_evaluated: #
evaluated = ensure_evaluated(context, concept) # if not concept.metadata.is_evaluated:
if evaluated.key != concept.key: # evaluated = ensure_evaluated(context, concept)
context.log(f"Failed to evaluate concept '{concept}'") # if evaluated.key != concept.key:
return None # context.log(f"Failed to evaluate concept '{concept}'")
ret = evaluated.get_value(ConceptParts.RET) # return None
else: # ret = evaluated.get_value(ConceptParts.RET)
ret = concept.get_value(ConceptParts.RET) # else:
# ret = concept.get_value(ConceptParts.RET)
if isinstance(ret, Concept) and sheerka.is_known(ret): #
return sheerka.ret(self.name, True, ret, parents=[return_value]) # if isinstance(ret, Concept) and sheerka.is_known(ret):
# return sheerka.ret(self.name, True, ret, parents=[return_value])
context.log(f"ret '{ret}' is not a concept!") #
return None # context.log(f"ret '{ret}' is not a concept!")
# return None
src/evaluators/TooManySuccessEvaluator.py
+7 -3
View File
@@ -1,8 +1,8 @@
import logging import logging
from core.builtin_concepts import BuiltinConcepts
import core.builtin_helpers import core.builtin_helpers
from evaluators.BaseEvaluator import AllReturnValuesEvaluator, BaseEvaluator from core.builtin_concepts import BuiltinConcepts
from evaluators.BaseEvaluator import AllReturnValuesEvaluator
from parsers.BaseParser import BaseParser from parsers.BaseParser import BaseParser
@@ -44,7 +44,11 @@ class TooManySuccessEvaluator(AllReturnValuesEvaluator):
context.log(s, self.name) context.log(s, self.name)
context.log(f"value={sheerka.value(s.value)}", self.name) context.log(f"value={sheerka.value(s.value)}", self.name)
if not core.builtin_helpers.is_same_success(context, self.success): same_success = core.builtin_helpers.is_same_success(context, self.success)
if same_success is None:
return None
if not same_success:
context.log(f"Values are different. Raising {BuiltinConcepts.TOO_MANY_SUCCESS}.", self.name) context.log(f"Values are different. Raising {BuiltinConcepts.TOO_MANY_SUCCESS}.", self.name)
too_many_success = sheerka.new(BuiltinConcepts.TOO_MANY_SUCCESS, body=self.success) too_many_success = sheerka.new(BuiltinConcepts.TOO_MANY_SUCCESS, body=self.success)
return sheerka.ret(self.name, False, too_many_success, parents=self.eaten) return sheerka.ret(self.name, False, too_many_success, parents=self.eaten)
src/parsers/BaseNodeParser.py
-2
View File
@@ -772,8 +772,6 @@ class BaseNodeParser(BaseParser):
if token.type == TokenKind.STRING: if token.type == TokenKind.STRING:
name = token.value[1:-1] if strip_quotes else token.value name = token.value[1:-1] if strip_quotes else token.value
elif token.type == TokenKind.KEYWORD:
name = token.value.value
else: else:
name = token.value name = token.value
src/parsers/ExactConceptParser.py
+4 -1
View File
@@ -48,7 +48,7 @@ class ExactConceptParser(BaseParser):
for combination in self.combinations(words): for combination in self.combinations(words):
concept_key = " ".join(combination) concept_key = " ".join(combination)
result = sheerka.new(concept_key) # use new(), not get() because we need a new instance result = sheerka.get_by_key(concept_key) # use new(), not get() because we need a new instance
if sheerka.isinstance(result, BuiltinConcepts.UNKNOWN_CONCEPT): if sheerka.isinstance(result, BuiltinConcepts.UNKNOWN_CONCEPT):
continue continue
@@ -66,6 +66,9 @@ class ExactConceptParser(BaseParser):
continue continue
context.log(f"Recognized concept {concept}.", self.name) context.log(f"Recognized concept {concept}.", self.name)
# We can ask for a new instance
concept = sheerka.new_from_template(concept, concept.key)
# update the properties if needed # update the properties if needed
for i, token in enumerate(combination): for i, token in enumerate(combination):
if token.startswith(VARIABLE_PREFIX): if token.startswith(VARIABLE_PREFIX):
src/parsers/FunctionParser.py
+12 -1
View File
@@ -38,6 +38,14 @@ class NamesNode(FunctionParserNode):
def to_unrecognized(self): def to_unrecognized(self):
return UnrecognizedTokensNode(self.start, self.end, self.tokens).fix_source() return UnrecognizedTokensNode(self.start, self.end, self.tokens).fix_source()
def to_str_unrecognized(self):
token = Token(TokenKind.STRING,
"'" + self.str_value() + "'",
self.tokens[0].index,
self.tokens[0].line,
self.tokens[0].column)
return UnrecognizedTokensNode(self.start, self.end, [token]).fix_source()
@dataclass() @dataclass()
class FunctionParameter: class FunctionParameter:
@@ -322,10 +330,13 @@ class FunctionParser(BaseParser):
scn.add_node(sep.to_unrecognized()) scn.add_node(sep.to_unrecognized())
res = [SourceCodeWithConceptNode(function_node.first.to_unrecognized(), function_node.last.to_unrecognized())] res = [SourceCodeWithConceptNode(function_node.first.to_unrecognized(), function_node.last.to_unrecognized())]
function_name = function_node.first.str_value()
for param in function_node.parameters: for param in function_node.parameters:
if isinstance(param.value, NamesNode): if isinstance(param.value, NamesNode):
unrecognized = param.value.to_unrecognized()
# try to recognize concepts # try to recognize concepts
unrecognized = param.value.to_unrecognized()
nodes_sequences = get_lexer_nodes_from_unrecognized(self.context, nodes_sequences = get_lexer_nodes_from_unrecognized(self.context,
unrecognized, unrecognized,
PARSERS) PARSERS)
src/parsers/SyaNodeParser.py
+2 -1
View File
@@ -1219,7 +1219,8 @@ class SyaNodeParser(BaseNodeParser):
start = inner_item.start start = inner_item.start
if inner_item.end > end: if inner_item.end > end:
end = inner_item.end end = inner_item.end
has_unrecognized |= isinstance(inner_item, (UnrecognizedTokensNode, SourceCodeWithConceptNode)) has_unrecognized |= isinstance(inner_item, (UnrecognizedTokensNode, SourceCodeWithConceptNode)) or \
hasattr(inner_item, "has_unrecognized") and inner_item.has_unrecognized
param_name = concept.metadata.variables[param_index][0] param_name = concept.metadata.variables[param_index][0]
param_value = inner_item.concept if hasattr(inner_item, "concept") else \ param_value = inner_item.concept if hasattr(inner_item, "concept") else \
src/sdp/readme.md
+1
View File
@@ -14,6 +14,7 @@
- D : concept definitions (no history management) - D : concept definitions (no history management)
- R : executionContext ('R' stands for Result or ReturnValue, no history management) - R : executionContext ('R' stands for Result or ReturnValue, no history management)
- O : ServiceObj (from pickle) - O : ServiceObj (from pickle)
- M : MemoryObject (using SheerkaPickle)
## How concepts are serialized ? ## How concepts are serialized ?
- get the id of the concept - get the id of the concept
src/sdp/sheerkaSerializer.py
+20 -24
View File
@@ -60,6 +60,7 @@ class Serializer:
self.register(ConceptSerializer()) self.register(ConceptSerializer())
self.register(DictionarySerializer()) self.register(DictionarySerializer())
self.register(ExecutionContextSerializer()) self.register(ExecutionContextSerializer())
self.register(MemoryObjectSerializer()) # before ServiceObjSerializer
self.register(ServiceObjSerializer()) self.register(ServiceObjSerializer())
def register(self, serializer): def register(self, serializer):
@@ -229,13 +230,14 @@ class StateSerializer(PickleSerializer):
1) 1)
class ConceptSerializer(BaseSerializer): class SheerkaPickleSerializer(BaseSerializer):
def __init__(self): def __init__(self, predicate, name, version):
BaseSerializer.__init__(self, "C", 1) BaseSerializer.__init__(self, name, version)
self.predicate = predicate
def matches(self, obj): def matches(self, obj):
return isinstance(obj, Concept) return self.predicate(obj)
def dump(self, stream, obj, context): def dump(self, stream, obj, context):
stream.write(sheerkapickle.encode(context.sheerka, obj).encode("utf-8")) stream.write(sheerkapickle.encode(context.sheerka, obj).encode("utf-8"))
@@ -248,6 +250,12 @@ class ConceptSerializer(BaseSerializer):
return obj return obj
class ConceptSerializer(SheerkaPickleSerializer):
def __init__(self):
super().__init__(lambda obj: isinstance(obj, Concept), "C", 1)
class DictionarySerializer(BaseSerializer): class DictionarySerializer(BaseSerializer):
def __init__(self): def __init__(self):
super().__init__("D", 1) super().__init__("D", 1)
@@ -267,25 +275,11 @@ class DictionarySerializer(BaseSerializer):
return obj return obj
class ExecutionContextSerializer(BaseSerializer): class ExecutionContextSerializer(SheerkaPickleSerializer):
CLASS_NAME = "core.sheerka.ExecutionContext.ExecutionContext" CLASS_NAME = "core.sheerka.ExecutionContext.ExecutionContext"
def __init__(self): def __init__(self):
BaseSerializer.__init__(self, "R", 1) super().__init__(lambda obj: get_full_qualified_name(obj) == self.CLASS_NAME, "R", 1)
def matches(self, obj):
return get_full_qualified_name(obj) == self.CLASS_NAME
def dump(self, stream, obj, context):
stream.write(sheerkapickle.encode(context.sheerka, obj).encode("utf-8"))
stream.seek(0)
return stream
def load(self, stream, context):
json_stream = stream.read().decode("utf-8")
obj = sheerkapickle.decode(context.sheerka, json_stream)
# json_message = json.loads(json_stream)
return obj
class ServiceObjSerializer(PickleSerializer): class ServiceObjSerializer(PickleSerializer):
@@ -297,7 +291,9 @@ class ServiceObjSerializer(PickleSerializer):
"O", "O",
1) 1)
#
# class SheerkaSerializer(ObjectSerializer): class MemoryObjectSerializer(SheerkaPickleSerializer):
# def __init__(self): CLASS_NAME = "core.sheerka.services.SheerkaMemory.MemoryObject"
# ObjectSerializer.__init__(self, "core.sheerka.Sheerka", "C", 1)
def __init__(self):
super().__init__(lambda obj: get_full_qualified_name(obj) == self.CLASS_NAME, "R", 1)
tests/BaseTest.py
+1
View File
@@ -90,6 +90,7 @@ class BaseTest:
instance = sheerka.new(concept.key if isinstance(concept, Concept) else concept) instance = sheerka.new(concept.key if isinstance(concept, Concept) else concept)
for i, var in enumerate(instance.metadata.variables): for i, var in enumerate(instance.metadata.variables):
if var[0] in kwargs: if var[0] in kwargs:
assert isinstance(kwargs[var[0]], str), "variables definitions must be string"
instance.metadata.variables[i] = (var[0], kwargs[var[0]]) instance.metadata.variables[i] = (var[0], kwargs[var[0]])
return instance return instance
tests/core/test_SheerkaCreateNewConcept.py
+18 -1
View File
@@ -182,7 +182,7 @@ class TestSheerkaCreateNewConcept(TestUsingMemoryBasedSheerka):
assert sheerka.cache_manager.get(sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, "-") == [bnf_concept.id] assert sheerka.cache_manager.get(sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, "-") == [bnf_concept.id]
assert sheerka.cache_manager.get(sheerka.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY, "-") == [bnf_concept.id] assert sheerka.cache_manager.get(sheerka.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY, "-") == [bnf_concept.id]
def test_concept_references_are_updated(self): def test_concept_references_are_updated_1(self):
sheerka, context, one, two, number, twenty, twenties = self.init_concepts( sheerka, context, one, two, number, twenty, twenties = self.init_concepts(
"one", "one",
"two", "two",
@@ -198,6 +198,23 @@ class TestSheerkaCreateNewConcept(TestUsingMemoryBasedSheerka):
assert sheerka.cache_manager.get(sheerka.CONCEPTS_REFERENCES_ENTRY, twenty.id) == {twenties.id} assert sheerka.cache_manager.get(sheerka.CONCEPTS_REFERENCES_ENTRY, twenty.id) == {twenties.id}
assert sheerka.cache_manager.get(sheerka.CONCEPTS_REFERENCES_ENTRY, twenties.id) is None assert sheerka.cache_manager.get(sheerka.CONCEPTS_REFERENCES_ENTRY, twenties.id) is None
def test_concept_references_are_updated_2(self):
sheerka, context, one, two, number, twenty, twenties = self.init_concepts(
"one",
"two",
"number",
"twenty",
Concept("twenties", definition="twenty number"),
create_new=True
)
assert sheerka.cache_manager.get(sheerka.CONCEPTS_REFERENCES_ENTRY, one.id) is None
assert sheerka.cache_manager.get(sheerka.CONCEPTS_REFERENCES_ENTRY, two.id) is None
assert sheerka.cache_manager.get(sheerka.CONCEPTS_REFERENCES_ENTRY, number.id) == {twenties.id}
assert sheerka.cache_manager.get(sheerka.CONCEPTS_REFERENCES_ENTRY, twenty.id) == {twenties.id}
assert sheerka.cache_manager.get(sheerka.CONCEPTS_REFERENCES_ENTRY, twenties.id) is None
class TestSheerkaCreateNewConceptFileBased(TestUsingFileBasedSheerka): class TestSheerkaCreateNewConceptFileBased(TestUsingFileBasedSheerka):
def test_i_can_add_several_concepts(self): def test_i_can_add_several_concepts(self):
tests/core/test_SheerkaEvaluateConcept.py
+70 -12
View File
@@ -2,6 +2,7 @@ import pytest
from core.builtin_concepts import BuiltinConcepts, ReturnValueConcept, ParserResultConcept from core.builtin_concepts import BuiltinConcepts, ReturnValueConcept, ParserResultConcept
from core.concept import Concept, DoNotResolve, ConceptParts, Property, InfiniteRecursionResolved, CB, NotInit from core.concept import Concept, DoNotResolve, ConceptParts, Property, InfiniteRecursionResolved, CB, NotInit
from core.sheerka.services.SheerkaEvaluateConcept import SheerkaEvaluateConcept from core.sheerka.services.SheerkaEvaluateConcept import SheerkaEvaluateConcept
from core.sheerka.services.SheerkaMemory import SheerkaMemory
from parsers.PythonParser import PythonNode from parsers.PythonParser import PythonNode
from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
@@ -34,6 +35,8 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
assert evaluated.metadata.is_evaluated assert evaluated.metadata.is_evaluated
assert len(evaluated.values) == 0 if body is None else 1 assert len(evaluated.values) == 0 if body is None else 1
assert "foo" in sheerka.services[SheerkaMemory.NAME].registration
@pytest.mark.parametrize("expr, expected", [ @pytest.mark.parametrize("expr, expected", [
("", ""), ("", ""),
("1", 1), ("1", 1),
@@ -191,7 +194,13 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
assert sheerka.objvalue(evaluated) == CB("a", BuiltinConcepts.NOT_INITIALIZED) assert sheerka.objvalue(evaluated) == CB("a", BuiltinConcepts.NOT_INITIALIZED)
assert evaluated.metadata.is_evaluated assert evaluated.metadata.is_evaluated
def test_i_can_evaluate_concept_when_variables_reference_others_concepts(self): def test_i_can_evaluate_concept_when_variables_reference_others_concepts_1(self):
"""
The body references a variable.
The variable reference a concept
The variable name is also the name of a concept
:return:
"""
sheerka, context, concept_a, concept = self.init_concepts( sheerka, context, concept_a, concept = self.init_concepts(
Concept("a"), Concept("a"),
Concept("foo", body="a").def_var("a", "a"), Concept("foo", body="a").def_var("a", "a"),
@@ -218,6 +227,35 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
assert evaluated.key == concept.key assert evaluated.key == concept.key
assert evaluated.body == concept_a assert evaluated.body == concept_a
def test_i_can_evaluate_concept_when_variables_reference_others_concepts_3(self):
"""
The body references a variable.
The variable reference a concept
The name of the variable is also the name of a concept, but the variable points to something else
:return:
"""
sheerka, context, concept_a, concept_b = self.init_concepts("a", "b", eval_body=True)
concept = Concept("foo", body="a").def_var("a", "b")
evaluated = sheerka.evaluate_concept(context, concept)
assert evaluated.key == concept.key
assert evaluated.body == concept_b
def test_i_can_evaluate_concept_when_variables_reference_others_concepts_4(self):
"""
The body references a variable.
The variable reference a concept
:return:
"""
sheerka, context, concept_b = self.init_concepts("b", eval_body=True)
concept = Concept("foo", body="a").def_var("a", "b")
evaluated = sheerka.evaluate_concept(context, concept)
assert evaluated.key == concept.key
assert evaluated.body == concept_b
def test_i_can_evaluate_concept_when_variables_reference_others_concepts_with_body(self): def test_i_can_evaluate_concept_when_variables_reference_others_concepts_with_body(self):
sheerka, context, *concepts = self.init_concepts( sheerka, context, *concepts = self.init_concepts(
Concept(name="a", body="1"), Concept(name="a", body="1"),
@@ -714,7 +752,7 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
assert evaluated.key == command.key assert evaluated.key == command.key
assert "a" not in sheerka.locals assert "a" not in sheerka.locals
sheerka.set_isa(context, command, sheerka.new(BuiltinConcepts.COMMAND)) sheerka.set_isa(context, command, sheerka.new(BuiltinConcepts.AUTO_EVAL))
evaluated = sheerka.evaluate_concept(context, sheerka.new("command")) evaluated = sheerka.evaluate_concept(context, sheerka.new("command"))
assert evaluated.key == command.key assert evaluated.key == command.key
assert "a" in sheerka.locals assert "a" in sheerka.locals
@@ -730,7 +768,7 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
setattr(foo.metadata, metadata, "a=10; print('10')") setattr(foo.metadata, metadata, "a=10; print('10')")
foo.metadata.need_validation = True foo.metadata.need_validation = True
evaluated = sheerka.evaluate_concept(context, foo) evaluated = sheerka.evaluate_concept(context, foo, eval_body=True)
captured = capsys.readouterr() captured = capsys.readouterr()
assert sheerka.isinstance(evaluated, BuiltinConcepts.CONCEPT_EVAL_ERROR) assert sheerka.isinstance(evaluated, BuiltinConcepts.CONCEPT_EVAL_ERROR)
@@ -773,19 +811,24 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
captured = capsys.readouterr() captured = capsys.readouterr()
assert captured.out == "" assert captured.out == ""
@pytest.mark.parametrize("concept, expected", [ @pytest.mark.parametrize("concept, eval_body, expected", [
(Concept("foo"), []), (Concept("foo"), False, []),
(Concept("foo", pre="pre", post="post", ret="ret", where="where"), ["pre", "ret", "post"]), (Concept("foo", pre="pre", post="post", ret="ret", where="where"), False, ["pre", "post"]),
(Concept("foo", pre="a").def_var("a"), ["variables", "pre"]), (Concept("foo", pre="pr", post="p", ret="r", where="w"), True, ["pre", "ret", "post", "variables", "body"]),
(Concept("foo", pre="self"), ["body", "pre"]), (Concept("foo", pre="a").def_var("a"), False, ["variables", "pre"]),
(Concept("foo", pre="self + a").def_var("a"), ["variables", "body", "pre"]), (Concept("foo", pre="self"), False, ["body", "pre"]),
(Concept("foo", pre="self + a", ret="ret").def_var("a"), ["variables", "body", "pre", "ret"]), (Concept("foo", pre="self + a").def_var("a"), False, ["variables", "body", "pre"]),
(Concept("foo", body="body"), []) # only if eval_body_is_set (Concept("foo", pre="self + a", ret="ret").def_var("a"), False, ["variables", "body", "pre"]),
(Concept("foo", pre="self + a", ret="ret").def_var("a"), True, ["variables", "body", "pre", "ret"]),
(Concept("foo", body="body"), False, [])
]) ])
def test_i_can_compute_metadata_to_eval(self, concept, expected): def test_i_can_compute_metadata_to_eval(self, concept, eval_body, expected):
sheerka, context, concept = self.init_concepts(concept) sheerka, context, concept = self.init_concepts(concept)
service = sheerka.services[SheerkaEvaluateConcept.NAME] service = sheerka.services[SheerkaEvaluateConcept.NAME]
if eval_body:
context.add_to_protected_hints(BuiltinConcepts.EVAL_BODY_REQUESTED)
service.initialize_concept_asts(context, concept) service.initialize_concept_asts(context, concept)
assert service.compute_metadata_to_eval(context, concept) == expected assert service.compute_metadata_to_eval(context, concept) == expected
@@ -839,6 +882,21 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
evaluated = sheerka.evaluate_concept(context, concept, metadata=['pre']) evaluated = sheerka.evaluate_concept(context, concept, metadata=['pre'])
assert evaluated.values == {"a": Property("a", NotInit), ConceptParts.PRE: Property(ConceptParts.PRE, 'pre')} assert evaluated.values == {"a": Property("a", NotInit), ConceptParts.PRE: Property(ConceptParts.PRE, 'pre')}
def test_i_can_manage_ret(self):
sheerka, context, foo, bar = self.init_concepts("foo", Concept("bar", ret="foo"))
res = sheerka.evaluate_concept(context, bar)
assert res.id == bar.id
res = sheerka.evaluate_concept(context, bar, eval_body=True)
assert res.id == foo.id
def test_ret_is_evaluated_only_is_body_is_requested(self):
sheerka, context, foo, bar = self.init_concepts("foo", Concept("bar", ret="__NOT_FOUND"))
res = sheerka.evaluate_concept(context, bar, eval_body=False)
assert res.id == bar.id
# I cannot implement value cache for now # I cannot implement value cache for now
# def test_values_when_no_variables_are_computed_only_once(self): # def test_values_when_no_variables_are_computed_only_once(self):
# sheerka, context, foo = self.init_concepts(Concept("foo", body="10")) # sheerka, context, foo = self.init_concepts(Concept("foo", body="10"))
tests/core/test_SheerkaMemory.py
+123
View File
@@ -0,0 +1,123 @@
from core.builtin_concepts import BuiltinConcepts
from core.concept import Concept
from core.sheerka.ExecutionContext import ExecutionContext
from core.sheerka.services.SheerkaMemory import SheerkaMemory, MemoryObject
from tests.TestUsingFileBasedSheerka import TestUsingFileBasedSheerka
from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
class TestSheerkaMemory(TestUsingMemoryBasedSheerka):
def test_i_can_add_to_global_short_term_memory(self):
sheerka = self.get_sheerka()
service = sheerka.services[SheerkaMemory.NAME]
foo = Concept("foo")
sheerka.add_to_short_term_memory(None, "a", foo)
assert service.short_term_objects.copy() == {":a": foo}
assert id(sheerka.get_from_short_term_memory(None, "a")) == id(foo)
def test_i_can_add_context_short_term_memory(self):
sheerka, context = self.init_concepts()
service = sheerka.services[SheerkaMemory.NAME]
foo = Concept("foo")
sheerka.add_to_short_term_memory(context, "a", foo)
context_id = ExecutionContext.ids[context.event.get_digest()]
assert service.short_term_objects.copy() == {f"{context_id}:a": foo}
assert id(sheerka.get_from_short_term_memory(context, "a")) == id(foo)
assert sheerka.get_from_short_term_memory(None, "a") is None
def test_i_can_get_obj_from_parents(self):
sheerka, context = self.init_concepts()
service = sheerka.services[SheerkaMemory.NAME]
foo = Concept("foo")
sheerka.add_to_short_term_memory(None, "a", foo)
with context.push(BuiltinConcepts.TESTING, None) as sub_context:
assert service.short_term_objects.copy() == {":a": foo}
assert id(sheerka.get_from_short_term_memory(sub_context, "a")) == id(foo)
assert id(sheerka.get_from_short_term_memory(context, "a")) == id(foo)
assert id(sheerka.get_from_short_term_memory(None, "a")) == id(foo)
def test_entry_are_removed_on_context_exit(self):
sheerka, context = self.init_concepts()
with context.push(BuiltinConcepts.TESTING, None) as sub_context:
foo = Concept("foo")
sheerka.add_to_short_term_memory(sub_context, "a", foo)
assert id(sheerka.get_from_short_term_memory(sub_context, "a")) == id(foo)
assert sheerka.get_from_short_term_memory(sub_context, "a") is None
def test_i_can_add_and_retrieve_from_memory(self):
sheerka, context = self.init_concepts()
service = sheerka.services[SheerkaMemory.NAME]
assert sheerka.get_from_memory(context, "a") is None
foo = Concept("foo")
sheerka.add_to_memory(context, "a", foo)
assert service.objects.copy() == {"a": MemoryObject(context.event.get_digest(), foo)}
assert id(sheerka.get_from_memory(context, "a").obj) == id(foo)
def test_i_can_use_memory_to_get_the_list_of_all_objects(self):
sheerka, context = self.init_concepts()
foo = Concept("foo")
bar = Concept("bar")
sheerka.add_to_memory(context, "foo", 'value that will not appear')
sheerka.add_to_memory(context, "foo", foo)
sheerka.add_to_memory(context, "bar", bar)
assert sheerka.memory(context) == {"foo": foo, "bar": bar}
def test_i_can_use_memory_with_a_string(self):
sheerka, context = self.init_concepts()
foo = Concept("foo")
sheerka.add_to_memory(context, "foo", foo)
assert sheerka.memory(context, "foo") == foo
def test_i_can_use_memory_with_a_concept(self):
sheerka, context = self.init_concepts()
foo = Concept("foo")
sheerka.add_to_memory(context, "foo", foo)
assert sheerka.memory(context, Concept("foo")) == foo
def test_concept_not_found_is_return_when_not_found(self):
sheerka, context = self.init_concepts()
res = sheerka.memory(context, "foo")
assert sheerka.isinstance(res, BuiltinConcepts.NOT_FOUND)
assert res.body == {"#name": "foo"}
def test_memory_only_returns_the_last_object(self):
sheerka, context = self.init_concepts()
foo = Concept("foo")
bar = Concept("bar")
sheerka.add_to_memory(context, "item", foo)
sheerka.add_to_memory(context, "item", bar)
assert sheerka.memory(context, "item") == bar
class TestSheerkaMemoryUsingFileBase(TestUsingFileBasedSheerka):
def test_i_can_record_memory_objects(self):
sheerka, context = self.init_concepts()
sheerka.add_to_memory(context, "item", Concept("foo"))
sheerka.cache_manager.commit(context)
sheerka = self.get_sheerka()
context = self.get_context(sheerka)
assert sheerka.get_from_memory(context, "item").obj == Concept("foo")
tests/core/test_SheerkaModifyConcept.py
+12
View File
@@ -92,6 +92,18 @@ class TestSheerkaModifyConcept(TestUsingMemoryBasedSheerka):
assert foo_from_sheerka[0].metadata.body == "1" assert foo_from_sheerka[0].metadata.body == "1"
assert foo_from_sheerka[1].metadata.body == "value" assert foo_from_sheerka[1].metadata.body == "value"
def test_i_can_get_and_set_attribute(self):
sheerka, context = self.init_concepts()
foo = Concept("foo")
prop = Concept("property")
bar = Concept("bar")
res = sheerka.set_attr(foo, prop, bar)
assert res.status
assert sheerka.isinstance(res.body, BuiltinConcepts.SUCCESS)
assert sheerka.get_attr(foo, prop) == bar
class TestSheerkaModifyConceptUsingFile(TestUsingFileBasedSheerka): class TestSheerkaModifyConceptUsingFile(TestUsingFileBasedSheerka):
tests/core/test_SheerkaShortTermMemory.py
-52
View File
@@ -1,52 +0,0 @@
from core.builtin_concepts import BuiltinConcepts
from core.concept import Concept
from core.sheerka.ExecutionContext import ExecutionContext
from core.sheerka.services.SheerkaShortTermMemory import SheerkaShortTermMemory
from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
class TestSheerkaShortTermMemory(TestUsingMemoryBasedSheerka):
def test_i_can_add_to_global_short_term_memory(self):
sheerka = self.get_sheerka()
service = sheerka.services[SheerkaShortTermMemory.NAME]
foo = Concept("foo")
sheerka.add_to_short_term_memory(None, "a", foo)
assert service.objects.copy() == {":a": foo}
assert id(sheerka.get_from_short_term_memory(None, "a")) == id(foo)
def test_i_can_add_context_short_term_memory(self):
sheerka, context = self.init_concepts()
service = sheerka.services[SheerkaShortTermMemory.NAME]
foo = Concept("foo")
sheerka.add_to_short_term_memory(context, "a", foo)
context_id = ExecutionContext.ids[context.event.get_digest()]
assert service.objects.copy() == {f"{context_id}:a": foo}
assert id(sheerka.get_from_short_term_memory(context, "a")) == id(foo)
assert sheerka.get_from_short_term_memory(None, "a") is None
def test_i_can_get_obj_from_parents(self):
sheerka, context = self.init_concepts()
service = sheerka.services[SheerkaShortTermMemory.NAME]
foo = Concept("foo")
sheerka.add_to_short_term_memory(None, "a", foo)
with context.push(BuiltinConcepts.TESTING, None) as sub_context:
assert service.objects.copy() == {":a": foo}
assert id(sheerka.get_from_short_term_memory(sub_context, "a")) == id(foo)
assert id(sheerka.get_from_short_term_memory(context, "a")) == id(foo)
assert id(sheerka.get_from_short_term_memory(None, "a")) == id(foo)
def test_entry_are_removed_on_context_exit(self):
sheerka, context = self.init_concepts()
with context.push(BuiltinConcepts.TESTING, None) as sub_context:
foo = Concept("foo")
sheerka.add_to_short_term_memory(sub_context, "a", foo)
assert id(sheerka.get_from_short_term_memory(sub_context, "a")) == id(foo)
assert sheerka.get_from_short_term_memory(sub_context, "a") is None
tests/evaluators/test_ConceptEvaluator.py
-2
View File
@@ -24,7 +24,6 @@ class TestAddConceptEvaluator(TestUsingMemoryBasedSheerka):
concept = Concept(name="foo", concept = Concept(name="foo",
where="True", where="True",
pre="2 > 1", pre="2 > 1",
ret="3",
post="4").def_var("a", "5").def_var("b", "6") post="4").def_var("a", "5").def_var("b", "6")
evaluator = ConceptEvaluator() evaluator = ConceptEvaluator()
@@ -36,7 +35,6 @@ class TestAddConceptEvaluator(TestUsingMemoryBasedSheerka):
assert result.value.name == "foo" assert result.value.name == "foo"
assert result.value.get_value(ConceptParts.WHERE) == True assert result.value.get_value(ConceptParts.WHERE) == True
assert result.value.get_value(ConceptParts.PRE) == True assert result.value.get_value(ConceptParts.PRE) == True
assert result.value.get_value(ConceptParts.RET) == 3
assert result.value.get_value(ConceptParts.POST) == 4 assert result.value.get_value(ConceptParts.POST) == 4
assert result.value.get_value("a") == 5 assert result.value.get_value("a") == 5
assert result.value.get_value("b") == 6 assert result.value.get_value("b") == 6
tests/evaluators/test_PythonEvaluator.py
+26
View File
@@ -8,6 +8,7 @@ from core.tokenizer import Tokenizer
from evaluators.PythonEvaluator import PythonEvaluator, PythonEvalError from evaluators.PythonEvaluator import PythonEvaluator, PythonEvalError
from parsers.BaseNodeParser import SourceCodeNode, SourceCodeWithConceptNode from parsers.BaseNodeParser import SourceCodeNode, SourceCodeWithConceptNode
from parsers.PythonParser import PythonNode, PythonParser from parsers.PythonParser import PythonNode, PythonParser
from parsers.PythonWithConceptsParser import PythonWithConceptsParser
from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
@@ -17,6 +18,15 @@ def get_concept_name(concept):
def get_source_code_node(source_code, concepts=None): def get_source_code_node(source_code, concepts=None):
if concepts:
for concept_name, concept in sorted(concepts.items(), key=lambda kv: len(kv[0]), reverse=True):
identifier = "__C__" + PythonWithConceptsParser.sanitize(concept.name)
if concept.id:
identifier += "__" + concept.id
identifier += "__C__"
source_code = source_code.replace(concept_name, identifier)
concepts[identifier] = concept
if source_code: if source_code:
python_node = PythonNode(source_code, ast.parse(source_code, f"<source>", 'eval')) python_node = PythonNode(source_code, ast.parse(source_code, f"<source>", 'eval'))
else: else:
@@ -32,6 +42,11 @@ def get_source_code_node(source_code, concepts=None):
return scwcn return scwcn
def get_ret_val_from_source_code(context, source_code, concepts):
parsed = get_source_code_node(source_code, concepts)
return context.sheerka.ret("parsers.??", True, ParserResultConcept(value=parsed))
class TestPythonEvaluator(TestUsingMemoryBasedSheerka): class TestPythonEvaluator(TestUsingMemoryBasedSheerka):
@pytest.mark.parametrize("ret_val, expected", [ @pytest.mark.parametrize("ret_val, expected", [
@@ -289,3 +304,14 @@ class TestPythonEvaluator(TestUsingMemoryBasedSheerka):
assert evaluated.status assert evaluated.status
assert evaluated.value == 11 assert evaluated.value == 11
def test_i_can_eval_concept_with_ret(self):
sheerka, context, one, the = self.init_concepts("one", Concept("the a", ret="a").def_var("a"))
ret_val = get_ret_val_from_source_code(context, "test_using_context(one, the one)", {
"the one": self.get_concept_instance(sheerka, the, a="one"),
"one": self.get_concept_instance(sheerka, "one")
})
evaluated = PythonEvaluator().eval(context, ret_val)
assert evaluated.status
assert evaluated.value.startswith("I have access to Sheerka ! param1=(1001)one, param2=(1001)one, event=")
tests/evaluators/test_RetEvaluator.py
+47 -47
View File
@@ -1,47 +1,47 @@
import pytest # import pytest
from core.builtin_concepts import ReturnValueConcept # from core.builtin_concepts import ReturnValueConcept
from core.concept import Concept # from core.concept import Concept
from evaluators.RetEvaluator import RetEvaluator # from evaluators.RetEvaluator import RetEvaluator
#
from tests.BaseTest import BaseTest # from tests.BaseTest import BaseTest
from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka # from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
#
#
class TestRetEvaluator(TestUsingMemoryBasedSheerka): # class TestRetEvaluator(TestUsingMemoryBasedSheerka):
#
@pytest.mark.parametrize("ret_val, expected", [ # @pytest.mark.parametrize("ret_val, expected", [
(ReturnValueConcept("who", True, Concept("foo", ret="bar")), True), # (ReturnValueConcept("who", True, Concept("foo", ret="bar")), True),
(ReturnValueConcept("who", False, Concept("foo", ret="bar")), False), # (ReturnValueConcept("who", False, Concept("foo", ret="bar")), False),
(ReturnValueConcept("who", True, Concept("foo")), False), # (ReturnValueConcept("who", True, Concept("foo")), False),
(BaseTest.pretval(Concept("foo", ret="bar")), False), # (BaseTest.pretval(Concept("foo", ret="bar")), False),
(ReturnValueConcept("who", True, "not even a concept"), False), # (ReturnValueConcept("who", True, "not even a concept"), False),
]) # ])
def test_i_can_match(self, ret_val, expected): # def test_i_can_match(self, ret_val, expected):
context = self.get_context() # context = self.get_context()
assert RetEvaluator().matches(context, ret_val) == expected # assert RetEvaluator().matches(context, ret_val) == expected
#
def test_i_can_evaluate_fully_initialized(self): # def test_i_can_evaluate_fully_initialized(self):
sheerka, context, foo, the_concept = self.init_concepts("foo", Concept("the concept", ret="a").def_var("a")) # sheerka, context, foo, the_concept = self.init_concepts("foo", Concept("the concept", ret="a").def_var("a"))
#
instance = sheerka.new("the concept") # instance = sheerka.new("the concept")
instance.set_value("a", sheerka.new("foo")) # instance.set_value("a", sheerka.new("foo"))
ret_value = self.tretval(sheerka, instance) # ret_value = self.tretval(sheerka, instance)
#
res = RetEvaluator().eval(context, ret_value) # res = RetEvaluator().eval(context, ret_value)
assert res.status # assert res.status
assert sheerka.isinstance(res.body, "foo") # assert sheerka.isinstance(res.body, "foo")
#
@pytest.mark.parametrize("instance, expected", [ # @pytest.mark.parametrize("instance, expected", [
(Concept("with ret", ret="a").def_var("a", "foo"), "foo"), # (Concept("with ret", ret="a").def_var("a", "foo"), "foo"),
(Concept("with ret", ret="a", body="10").def_var("a", "foo"), "foo"), # (Concept("with ret", ret="a", body="10").def_var("a", "foo"), "foo"),
(Concept("with ret", ret="a").def_var("a", "bar"), "bar"), # (Concept("with ret", ret="a").def_var("a", "bar"), "bar"),
]) # ])
def test_i_can_evaluate(self, instance, expected): # def test_i_can_evaluate(self, instance, expected):
sheerka, context, foo, bar = self.init_concepts("foo", Concept("bar", body="10")) # sheerka, context, foo, bar = self.init_concepts("foo", Concept("bar", body="10"))
#
ret_value = self.tretval(sheerka, instance) # ret_value = self.tretval(sheerka, instance)
#
res = RetEvaluator().eval(context, ret_value) # res = RetEvaluator().eval(context, ret_value)
assert res.status # assert res.status
assert sheerka.isinstance(res.body, expected) # assert sheerka.isinstance(res.body, expected)
#
tests/evaluators/test_TooManySucessEvaluator.py
+19 -3
View File
@@ -1,5 +1,4 @@
import pytest import pytest
from core.builtin_concepts import ReturnValueConcept, BuiltinConcepts from core.builtin_concepts import ReturnValueConcept, BuiltinConcepts
from core.concept import Concept from core.concept import Concept
from evaluators.TooManySuccessEvaluator import TooManySuccessEvaluator from evaluators.TooManySuccessEvaluator import TooManySuccessEvaluator
@@ -44,8 +43,8 @@ class TestTooManySuccessEvaluator(TestUsingMemoryBasedSheerka):
def test_i_can_eval(self): def test_i_can_eval(self):
context = self.get_context() context = self.get_context()
value1 = r("evaluators.a", value=Concept("c1", body="1")) value1 = r("evaluators.a", value=Concept("c1", body="1").auto_init())
value2 = r("evaluators.a", value=Concept("c2", body="2")) value2 = r("evaluators.a", value=Concept("c2", body="2").auto_init())
return_values = [ return_values = [
value1, value1,
value2, value2,
@@ -80,6 +79,23 @@ class TestTooManySuccessEvaluator(TestUsingMemoryBasedSheerka):
assert matches assert matches
assert res is None assert res is None
def test_i_do_not_eval_when_the_concepts_are_not_evaluated(self):
context = self.get_context()
return_values = [
r("evaluators.a", value=Concept("c1", body="1")),
r("evaluators.a", value=Concept("c2", body="2")),
r("other", False),
reduce_requested
]
evaluator = TooManySuccessEvaluator()
matches = evaluator.matches(context, return_values)
res = evaluator.eval(context, return_values)
assert matches
assert res is None
def test_other_success_are_not_reduced(self): def test_other_success_are_not_reduced(self):
context = self.get_context() context = self.get_context()
tests/non_reg/test_sheerka_non_reg.py
+31 -16
View File
@@ -1,6 +1,6 @@
import pytest import pytest
from core.builtin_concepts import BuiltinConcepts from core.builtin_concepts import BuiltinConcepts
from core.concept import Concept, PROPERTIES_TO_SERIALIZE, simplec, CMV, NotInit from core.concept import Concept, PROPERTIES_TO_SERIALIZE, simplec, CMV, NotInit, CC
from evaluators.MutipleSameSuccessEvaluator import MultipleSameSuccessEvaluator from evaluators.MutipleSameSuccessEvaluator import MultipleSameSuccessEvaluator
from evaluators.OneSuccessEvaluator import OneSuccessEvaluator from evaluators.OneSuccessEvaluator import OneSuccessEvaluator
from evaluators.PythonEvaluator import PythonEvalError from evaluators.PythonEvaluator import PythonEvalError
@@ -236,16 +236,16 @@ as:
self.create_and_add_in_cache_concept(sheerka, Concept(name="hello b", body="'hello you ' + b").def_var("b")) self.create_and_add_in_cache_concept(sheerka, Concept(name="hello b", body="'hello you ' + b").def_var("b"))
self.create_and_add_in_cache_concept(sheerka, Concept(name="foo", body="'another value'")) self.create_and_add_in_cache_concept(sheerka, Concept(name="foo", body="'another value'"))
res = sheerka.evaluate_user_input("hello foo") res = sheerka.evaluate_user_input("eval hello foo")
assert len(res) == 1 assert len(res) == 1
assert not res[0].status assert not res[0].status
assert sheerka.isinstance(res[0].value, BuiltinConcepts.TOO_MANY_SUCCESS) assert sheerka.isinstance(res[0].value, BuiltinConcepts.TOO_MANY_SUCCESS)
concepts = res[0].value.body concepts = res[0].value.body
assert len(concepts) == 2 assert len(concepts) == 2
sorted_values = sorted(concepts, key=lambda x: x.value.body) sorted_values = sorted(concepts, key=lambda x: x.value)
assert sorted_values[0].value.body == "hello another value" assert sorted_values[0].value == "hello another value"
assert sorted_values[1].value.body == "hello you another value" assert sorted_values[1].value == "hello you another value"
def test_i_can_manage_concepts_with_the_same_key_when_values_are_the_same(self): def test_i_can_manage_concepts_with_the_same_key_when_values_are_the_same(self):
sheerka = self.get_sheerka() sheerka = self.get_sheerka()
@@ -254,10 +254,10 @@ as:
sheerka.create_new_concept(context, Concept(name="hello a", body="'hello ' + a").def_var("a")) sheerka.create_new_concept(context, Concept(name="hello a", body="'hello ' + a").def_var("a"))
sheerka.create_new_concept(context, Concept(name="hello b", body="'hello ' + b").def_var("b")) sheerka.create_new_concept(context, Concept(name="hello b", body="'hello ' + b").def_var("b"))
res = sheerka.evaluate_user_input("hello 'foo'") res = sheerka.evaluate_user_input("eval hello 'foo'")
assert len(res) == 1 assert len(res) == 1
assert res[0].status assert res[0].status
assert res[0].value.body == "hello foo" # I don't know yet the one to choose assert res[0].value == "hello foo" # I don't know yet the one to choose
assert res[0].who == sheerka.get_evaluator_name(MultipleSameSuccessEvaluator.NAME) assert res[0].who == sheerka.get_evaluator_name(MultipleSameSuccessEvaluator.NAME)
def test_i_can_create_concepts_with_python_code_as_body(self): def test_i_can_create_concepts_with_python_code_as_body(self):
@@ -574,7 +574,6 @@ as:
assert res[0].status assert res[0].status
assert res[0].body == 3 assert res[0].body == 3
def test_eval_does_not_break_valid_result(self): def test_eval_does_not_break_valid_result(self):
sheerka = self.get_sheerka() sheerka = self.get_sheerka()
sheerka.evaluate_user_input("def concept one as 1") sheerka.evaluate_user_input("def concept one as 1")
@@ -981,13 +980,13 @@ as:
sheerka = self.init_scenario(init) sheerka = self.init_scenario(init)
the = sheerka.get_by_name("the a") the = sheerka.get_by_name("the a")
# res = sheerka.evaluate_user_input("one plus the one") res = sheerka.evaluate_user_input("one plus the one")
# assert res[0].status assert res[0].status
# plus = res[0].body plus = res[0].body
# assert isinstance(plus, Concept) assert isinstance(plus, Concept)
# assert plus.name == "plus" assert plus.name == "plus"
# assert plus.compiled["a"] == sheerka.new("one") assert plus.compiled["a"] == sheerka.new("one")
# assert plus.compiled["b"] == CC(the, a=sheerka.new("one")) assert plus.compiled["b"] == CC(the, a=sheerka.new("one"))
res = sheerka.evaluate_user_input("eval one plus the one") res = sheerka.evaluate_user_input("eval one plus the one")
assert res[0].status assert res[0].status
@@ -996,7 +995,7 @@ as:
def test_i_can_evaluate_command(self): def test_i_can_evaluate_command(self):
init = [ init = [
"def concept command as 'Executed !'", "def concept command as 'Executed !'",
"set_isa(c:command:, __COMMAND)", "set_isa(c:command:, __AUTO_EVAL)",
] ]
# Since command is a __COMMAND, the body is auto evaluated # Since command is a __COMMAND, the body is auto evaluated
@@ -1144,6 +1143,22 @@ as:
assert res[0].status assert res[0].status
assert sheerka.isinstance(res[0].body, BuiltinConcepts.NEW_CONCEPT) assert sheerka.isinstance(res[0].body, BuiltinConcepts.NEW_CONCEPT)
def test_bnf_node_parsers_are_updated_when_concepts_are_modified(self):
init = [
"def concept number",
"def concept one",
"def concept twenties from bnf 'twenty' number as 20 + number",
]
sheerka = self.init_scenario(init)
res = sheerka.evaluate_user_input("twenty one")
assert len(res) > 1
sheerka.evaluate_user_input("set_isa(one, number)")
res = sheerka.evaluate_user_input("twenty one")
assert len(res) == 1
assert res[0].status
class TestSheerkaNonRegFile(TestUsingFileBasedSheerka): class TestSheerkaNonRegFile(TestUsingFileBasedSheerka):
def test_i_can_def_several_concepts(self): def test_i_can_def_several_concepts(self):
tests/parsers/test_FunctionParser.py
+1
View File
@@ -174,3 +174,4 @@ class TestFunctionParser(TestUsingMemoryBasedSheerka):
assert isinstance(concept.compiled["b"], list) assert isinstance(concept.compiled["b"], list)
for item in concept.compiled["b"]: for item in concept.compiled["b"]:
assert sheerka.isinstance(item, BuiltinConcepts.RETURN_VALUE) assert sheerka.isinstance(item, BuiltinConcepts.RETURN_VALUE)
tests/parsers/test_SyaNodeParser.py
+24
View File
@@ -1154,6 +1154,30 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT) assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
assert lexer_nodes == expected_array assert lexer_nodes == expected_array
@pytest.mark.parametrize("text, expected_result", [
("a plus b", [CN("plus", source="a plus b")]),
("suffixed a plus b", [CN("suffixed", source="suffixed a plus b")]),
])
def test_i_can_almost_parse_concept_definition(self, text, expected_result):
"""
In these examples, 'a' and 'b' are not defined.
So the status of the return value cannot be True
:param text:
:param expected_result:
:return:
"""
sheerka, context, parser = self.init_parser()
res = parser.parse(context, ParserInput(text))
wrapper = res.body
lexer_nodes = res.body.body
expected_array = compute_expected_array(cmap, text, expected_result)
assert not res.status
assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
assert lexer_nodes == expected_array
@pytest.mark.parametrize("text, expected_concept, expected_unrecognized", [ @pytest.mark.parametrize("text, expected_concept, expected_unrecognized", [
("x$!# prefixed", "prefixed", ["a"]), ("x$!# prefixed", "prefixed", ["a"]),
("suffixed x$!#", "suffixed", ["a"]), ("suffixed x$!#", "suffixed", ["a"]),