Fixed #18 : Parsing and evaluating Python
This commit is contained in:
@@ -5,3 +5,4 @@ class BuiltinConcepts:
|
||||
UNKNOWN_CONCEPT = "__UNKNOWN_CONCEPT"
|
||||
USER_INPUT = "__USER_INPUT"
|
||||
PARSER_INPUT = "__PARSER_INPUT"
|
||||
PYTHON_CODE = "__PYTHON_CODE"
|
||||
|
||||
@@ -5,7 +5,7 @@ import time
|
||||
from core.Event import Event
|
||||
|
||||
|
||||
class ExecutionContextActions:
|
||||
class ContextActions:
|
||||
TESTING = "Testing"
|
||||
INIT_SHEERKA = "Init Sheerka"
|
||||
EVALUATE_USER_INPUT = "Evaluate user input"
|
||||
@@ -18,9 +18,16 @@ class ExecutionContextActions:
|
||||
EVALUATION = "Evaluation"
|
||||
AFTER_EVALUATION = "After Evaluation"
|
||||
|
||||
EVALUATING_CONCEPT = "Evaluating concept"
|
||||
BUILD_CONCEPT = "Building all attributes"
|
||||
BUILD_CONCEPT_ATTR = "Building one attribute"
|
||||
EVAL_CONCEPT = "Evaluating all attributes"
|
||||
EVAL_CONCEPT_ATTR = "Evaluating one attribute"
|
||||
|
||||
|
||||
class ContextHint:
|
||||
REDUCE_CONCEPTS = "Reduce Concepts" # to tell the process to only keep the meaningful results
|
||||
EXPRESSION_ONLY_REQUESTED = "Expression Only"
|
||||
|
||||
|
||||
ids = {} # keep track of the next execution context id, for a given event id
|
||||
@@ -51,7 +58,7 @@ class ExecutionContext:
|
||||
who: str,
|
||||
event: Event,
|
||||
sheerka,
|
||||
action: ExecutionContextActions,
|
||||
action: ContextActions,
|
||||
action_context: object,
|
||||
desc: str = None,
|
||||
logger=None,
|
||||
@@ -102,6 +109,10 @@ class ExecutionContext:
|
||||
def long_id(self):
|
||||
return f"{self.event.get_digest()}:{self._id}"
|
||||
|
||||
@property
|
||||
def medium_id(self):
|
||||
return f"{self.event.get_digest()[:8]}:{self._id}"
|
||||
|
||||
@property
|
||||
def id(self):
|
||||
return self._id
|
||||
@@ -143,7 +154,7 @@ class ExecutionContext:
|
||||
|
||||
def push(self,
|
||||
who: str,
|
||||
action: ExecutionContextActions,
|
||||
action: ContextActions,
|
||||
action_context: object,
|
||||
desc: str = None,
|
||||
logger=None):
|
||||
@@ -162,6 +173,9 @@ class ExecutionContext:
|
||||
self._children.append(child)
|
||||
return child
|
||||
|
||||
def get_parent(self):
|
||||
return self._parent
|
||||
|
||||
def get_children(self, level=-1):
|
||||
"""
|
||||
recursively look for children
|
||||
@@ -173,6 +187,30 @@ class ExecutionContext:
|
||||
if level != 1:
|
||||
yield from child.get_children(level - 1)
|
||||
|
||||
def get_parents(self, level=-1):
|
||||
"""
|
||||
recursively look for parent
|
||||
:return:
|
||||
:rtype:
|
||||
"""
|
||||
if level == 0 or self._parent is None:
|
||||
return
|
||||
|
||||
yield self._parent
|
||||
yield from self._parent.get_parents(level - 1)
|
||||
|
||||
def in_context(self, hint: ContextHint):
|
||||
return hint in self.protected_hints or \
|
||||
hint in self.global_hints or \
|
||||
hint in self.private_hints
|
||||
|
||||
def get_from_short_term_memory(self, key):
|
||||
return self.sheerka.get_from_short_term_memory(self, key)
|
||||
|
||||
def log(self, message: str, who: str = None):
|
||||
"""Send debug information to logger"""
|
||||
pass
|
||||
|
||||
def __enter__(self):
|
||||
self._start = time.time_ns()
|
||||
return self
|
||||
|
||||
+33
-13
@@ -10,21 +10,21 @@ from caching.Cache import Cache
|
||||
from caching.IncCache import IncCache
|
||||
from common.utils import get_logger_name, get_sub_classes, import_module_and_sub_module
|
||||
from core.BuiltinConcepts import BuiltinConcepts
|
||||
from core.ErrorContext import ErrorContext
|
||||
from core.Event import Event
|
||||
from core.ExecutionContext import ContextHint, ExecutionContext, ExecutionContextActions
|
||||
from core.ExecutionContext import ContextHint, ExecutionContext, ContextActions
|
||||
from core.ReturnValue import ReturnValue
|
||||
from core.concept import Concept, ConceptMetadata
|
||||
from core.error import ErrorContext
|
||||
from ontologies.SheerkaOntologyManager import SheerkaOntologyManager
|
||||
from server.authentication import User
|
||||
|
||||
EXECUTE_STEPS = [
|
||||
ExecutionContextActions.BEFORE_PARSING,
|
||||
ExecutionContextActions.PARSING,
|
||||
ExecutionContextActions.AFTER_PARSING,
|
||||
ExecutionContextActions.BEFORE_EVALUATION,
|
||||
ExecutionContextActions.EVALUATION,
|
||||
ExecutionContextActions.AFTER_EVALUATION
|
||||
ContextActions.BEFORE_PARSING,
|
||||
ContextActions.PARSING,
|
||||
ContextActions.AFTER_PARSING,
|
||||
ContextActions.BEFORE_EVALUATION,
|
||||
ContextActions.EVALUATION,
|
||||
ContextActions.AFTER_EVALUATION
|
||||
]
|
||||
|
||||
|
||||
@@ -122,6 +122,7 @@ class Sheerka:
|
||||
self.om = SheerkaOntologyManager(self, root_folder)
|
||||
# self.builtin_cache, self.builtin_cache_by_class_name = self.get_builtins_classes_as_dict()
|
||||
|
||||
self.initialize_bind_methods()
|
||||
self.initialize_caching()
|
||||
self.initialize_evaluators()
|
||||
self.initialize_services()
|
||||
@@ -133,7 +134,7 @@ class Sheerka:
|
||||
with ExecutionContext(self.name,
|
||||
event,
|
||||
self,
|
||||
ExecutionContextActions.INIT_SHEERKA,
|
||||
ContextActions.INIT_SHEERKA,
|
||||
None,
|
||||
desc="Initializing Sheerka.") as exec_context:
|
||||
if self.om.current_sdp().first_time:
|
||||
@@ -165,6 +166,14 @@ class Sheerka:
|
||||
|
||||
return res
|
||||
|
||||
def initialize_bind_methods(self):
|
||||
"""
|
||||
Add some methods to the list of available methods
|
||||
:return:
|
||||
:rtype:
|
||||
"""
|
||||
self.bind_service_method(self.name, self.echo, False)
|
||||
|
||||
@staticmethod
|
||||
def initialize_logging(is_debug, root_folder):
|
||||
if is_debug:
|
||||
@@ -206,9 +215,9 @@ class Sheerka:
|
||||
"""
|
||||
self.init_log.info("Initializing services")
|
||||
|
||||
import_module_and_sub_module('core.services')
|
||||
base_class = "core.services.BaseService.BaseService"
|
||||
services = [service(self) for service in get_sub_classes("core.services", base_class)]
|
||||
import_module_and_sub_module('services')
|
||||
base_class = "services.BaseService.BaseService"
|
||||
services = [service(self) for service in get_sub_classes("services", base_class)]
|
||||
services.sort(key=attrgetter("order"))
|
||||
for service in services:
|
||||
if hasattr(service, "initialize"):
|
||||
@@ -282,7 +291,7 @@ class Sheerka:
|
||||
with ExecutionContext(user.email,
|
||||
event,
|
||||
self,
|
||||
ExecutionContextActions.EVALUATE_USER_INPUT,
|
||||
ContextActions.EVALUATE_USER_INPUT,
|
||||
command,
|
||||
desc=f"Evaluating '{command}'",
|
||||
global_hints=self.global_context_hints.copy()) as exec_context:
|
||||
@@ -322,3 +331,14 @@ class Sheerka:
|
||||
return a.id == b[3:-1]
|
||||
|
||||
return a.key == b
|
||||
|
||||
def echo(self, msg):
|
||||
"""
|
||||
test function
|
||||
:param msg:
|
||||
:type msg:
|
||||
:return:
|
||||
:rtype:
|
||||
"""
|
||||
|
||||
return msg
|
||||
|
||||
+5
-2
@@ -14,7 +14,7 @@ class ConceptDefaultProps:
|
||||
RET = "#ret#"
|
||||
|
||||
|
||||
DefaultProps = [v for k, v in ConceptDefaultProps.__dict__.items() if not k.startswith("_")]
|
||||
ConceptDefaultPropsAttrs = [v for k, v in ConceptDefaultProps.__dict__.items() if not k.startswith("_")]
|
||||
|
||||
|
||||
class DefinitionType:
|
||||
@@ -49,6 +49,9 @@ class ConceptMetadata:
|
||||
digest: str = None
|
||||
all_attrs: tuple = None
|
||||
|
||||
def get_metadata(self):
|
||||
return self
|
||||
|
||||
|
||||
@dataclass
|
||||
class ConceptRuntimeInfo:
|
||||
@@ -57,7 +60,7 @@ class ConceptRuntimeInfo:
|
||||
They are related to the instance of the concept
|
||||
"""
|
||||
is_evaluated: bool = False # True is the concept is evaluated by sheerka.eval_concept()
|
||||
need_validation: bool = False # True if the properties of the concept need to be validated
|
||||
need_validation: bool = True # True if the properties of the concept need to be validated
|
||||
recognized_by: str = None # RECOGNIZED_BY_ID, RECOGNIZED_BY_NAME, RECOGNIZED_BY_KEY (from Sheerka.py)
|
||||
|
||||
def copy(self):
|
||||
|
||||
@@ -1,9 +1,26 @@
|
||||
from dataclasses import dataclass
|
||||
|
||||
from common.utils import compute_hash
|
||||
from core.ExecutionContext import ExecutionContext
|
||||
|
||||
|
||||
class SheerkaException(Exception):
|
||||
pass
|
||||
def get_error_msg(self) -> str:
|
||||
pass
|
||||
|
||||
|
||||
class MethodAccessError(SheerkaException):
|
||||
def __init__(self, method_name):
|
||||
self.method_name = method_name
|
||||
|
||||
def get_error_msg(self) -> str:
|
||||
return f"Cannot access method '{self.method_name}'"
|
||||
|
||||
|
||||
@dataclass
|
||||
class ErrorObj:
|
||||
def get_error_msg(self) -> str:
|
||||
pass
|
||||
|
||||
|
||||
class ErrorContext:
|
||||
@@ -18,7 +35,7 @@ class ErrorContext:
|
||||
self.parents = None
|
||||
|
||||
def __repr__(self):
|
||||
return f"Error(who={self.who}, context_id={self.context.long_id}, value={self.value})"
|
||||
return f"Error(who={self.who}, context_id={self.context.medium_id}, value={self.value})"
|
||||
|
||||
def __eq__(self, other):
|
||||
if id(self) == id(other):
|
||||
@@ -33,3 +50,16 @@ class ErrorContext:
|
||||
|
||||
def __hash__(self):
|
||||
return hash((self.who, self.context.id, compute_hash(self.value)))
|
||||
|
||||
def get_error_msg(self):
|
||||
value_as_list = self.value if isinstance(self.value, list) else [self.value]
|
||||
temp = []
|
||||
for value in value_as_list:
|
||||
if isinstance(value, str):
|
||||
temp.append(value)
|
||||
elif isinstance(value, (SheerkaException, ErrorObj)):
|
||||
temp.append(value.get_error_msg())
|
||||
else:
|
||||
temp.append(repr(value))
|
||||
|
||||
return ", ".join(temp)
|
||||
@@ -0,0 +1,59 @@
|
||||
import ast
|
||||
import copy
|
||||
|
||||
|
||||
class PythonFragment:
|
||||
|
||||
def __init__(self, source_code, ast_tree=None, original_source=None, namespace=None):
|
||||
self.source_code = source_code # what was parsed
|
||||
self.original_source = original_source or source_code # to remember source before concepts id replacements
|
||||
self.ast_tree = ast_tree # if ast_ else ast.parse(source, mode="eval") if source else None
|
||||
self.namespace = namespace or {} # when objects (mainly concepts or rules) are recognized in the expression
|
||||
self._compiled = None
|
||||
self.ast_str = self.get_dump()
|
||||
|
||||
def __repr__(self):
|
||||
ast_type = "expr" if isinstance(self.ast_tree, ast.Expression) else "module"
|
||||
return "PythonNode(" + ast_type + "='" + self.source_code + "')"
|
||||
|
||||
def __eq__(self, other):
|
||||
if not isinstance(other, PythonFragment):
|
||||
return False
|
||||
|
||||
return self.source_code == other.source_code and self.original_source == other.original_source
|
||||
|
||||
def __hash__(self):
|
||||
return hash((self.source_code, self.original_source))
|
||||
|
||||
def get_dump(self):
|
||||
if not self.ast_tree:
|
||||
return None
|
||||
|
||||
dump = ast.dump(self.ast_tree)
|
||||
for to_remove in [", ctx=Load()", ", kind=None", ", type_ignores=[]"]:
|
||||
dump = dump.replace(to_remove, "")
|
||||
return dump
|
||||
|
||||
def get_compiled(self):
|
||||
if self._compiled is None:
|
||||
if isinstance(self.ast_tree, ast.Expression):
|
||||
self._compiled = compile(self.ast_tree, "<string>", "eval")
|
||||
else:
|
||||
# in case of module, if the last expr is an expression, we want to be able to return its value
|
||||
if isinstance(self.ast_tree.body[-1], ast.Expr):
|
||||
init_ast = copy.deepcopy(self.ast_tree)
|
||||
init_ast.body = self.ast_tree.body[:-1]
|
||||
last_ast = copy.deepcopy(self.ast_tree)
|
||||
last_ast_as_expression = self.expr_to_expression(last_ast.body[0])
|
||||
self._compiled = [compile(init_ast, "<ast>", "exec"),
|
||||
compile(last_ast_as_expression, "<ast>", "eval")]
|
||||
else:
|
||||
self._compiled = compile(self.ast_tree, "<string>", "exec")
|
||||
|
||||
return self._compiled
|
||||
|
||||
@staticmethod
|
||||
def expr_to_expression(expr):
|
||||
expr.lineno = 0
|
||||
expr.col_offset = 0
|
||||
return ast.Expression(expr.value, lineno=0, col_offset=0)
|
||||
@@ -1,49 +0,0 @@
|
||||
from common.global_symbols import NotFound
|
||||
from common.utils import sheerka_deepcopy
|
||||
from core.Sheerka import Sheerka
|
||||
|
||||
|
||||
class BaseService:
|
||||
"""
|
||||
Base class for services
|
||||
"""
|
||||
|
||||
def __init__(self, sheerka: Sheerka, order=999):
|
||||
self.sheerka = sheerka
|
||||
self.order = order # initialisation order. The lowest is initialized first
|
||||
|
||||
def initialize(self):
|
||||
"""
|
||||
Adds cache or bind methods
|
||||
:return:
|
||||
"""
|
||||
pass
|
||||
|
||||
def state_properties(self):
|
||||
pass
|
||||
|
||||
def push_state(self, context):
|
||||
"""
|
||||
Use variable Manager to store the state of the service
|
||||
"""
|
||||
args = self.state_properties()
|
||||
if args:
|
||||
for prop_name in args:
|
||||
self.sheerka.record_var(context, self.NAME, prop_name, sheerka_deepcopy(getattr(self, prop_name)))
|
||||
|
||||
def pop_state(self):
|
||||
"""
|
||||
Use Variable Manager to restore the state of a service
|
||||
:return:
|
||||
"""
|
||||
args = self.state_properties()
|
||||
if args:
|
||||
for prop_name in args:
|
||||
if (value := self.sheerka.load_var(self.NAME, prop_name)) is not NotFound:
|
||||
setattr(self, prop_name, value)
|
||||
|
||||
def store_var(self, context, var_name):
|
||||
"""
|
||||
Store/record the value of an attribute
|
||||
"""
|
||||
self.sheerka.record_var(context, self.NAME, var_name, getattr(self, var_name))
|
||||
@@ -1,343 +0,0 @@
|
||||
import hashlib
|
||||
import logging
|
||||
from dataclasses import dataclass
|
||||
|
||||
from caching.Cache import Cache
|
||||
from caching.FastCache import FastCache
|
||||
from caching.ListIfNeededCache import ListIfNeededCache
|
||||
from common.global_symbols import NotFound, NotInit, VARIABLE_PREFIX
|
||||
from common.utils import get_logger_name
|
||||
from core.BuiltinConcepts import BuiltinConcepts
|
||||
from core.ErrorContext import ErrorContext, SheerkaException
|
||||
from core.ExecutionContext import ExecutionContext
|
||||
from core.ReturnValue import ReturnValue
|
||||
from core.concept import Concept, ConceptMetadata, DefaultProps, DefinitionType
|
||||
from core.services.BaseService import BaseService
|
||||
from parsers.tokenizer import TokenKind, Tokenizer, strip_tokens
|
||||
|
||||
PROPERTIES_FOR_DIGEST = ("name", "key",
|
||||
"definition", "definition_type",
|
||||
"is_builtin", "is_unique",
|
||||
"where", "pre", "post", "body", "ret",
|
||||
"desc", "bound_body", "autouse", "props", "variables", "parameters")
|
||||
|
||||
|
||||
@dataclass
|
||||
class ConceptAlreadyDefined(SheerkaException):
|
||||
concept: ConceptMetadata
|
||||
already_defined_id: str
|
||||
|
||||
|
||||
@dataclass
|
||||
class InvalidBnf(SheerkaException):
|
||||
bnf: str
|
||||
|
||||
|
||||
@dataclass
|
||||
class FirstItemError(SheerkaException):
|
||||
pass
|
||||
|
||||
|
||||
class ConceptManager(BaseService):
|
||||
"""
|
||||
The service is used for the administration of concepts
|
||||
You can define new concept, modify or delete them
|
||||
|
||||
There are also function to help retrieve them easily (like first token cache)
|
||||
Already instantiated concepts are managed by the Memory service
|
||||
"""
|
||||
|
||||
NAME = "ConceptManager"
|
||||
|
||||
USER_CONCEPTS_IDS = "User_Concepts_IDs" # incremented everytime a new concept is created
|
||||
CONCEPTS_BY_ID_ENTRY = "ConceptManager:Concepts_By_ID" # to store all the concepts
|
||||
CONCEPTS_BY_KEY_ENTRY = "ConceptManager:Concepts_By_Key"
|
||||
CONCEPTS_BY_NAME_ENTRY = "ConceptManager:Concepts_By_Name"
|
||||
CONCEPTS_BY_HASH_ENTRY = "ConceptManager:Concepts_By_Hash" # sto
|
||||
|
||||
def __init__(self, sheerka):
|
||||
super().__init__(sheerka, order=11)
|
||||
|
||||
self.log = logging.getLogger(get_logger_name(__name__))
|
||||
self.init_log = logging.getLogger(get_logger_name("init." + __name__))
|
||||
self.bnf_expr_cache = FastCache()
|
||||
|
||||
def initialize(self):
|
||||
self.init_log.debug(f"Initializing ConceptManager, order={self.order}")
|
||||
self.sheerka.bind_service_method(self.NAME, self.define_new_concept, True)
|
||||
self.sheerka.bind_service_method(self.NAME, self.newn, True)
|
||||
self.sheerka.bind_service_method(self.NAME, self.newi, True)
|
||||
|
||||
register_concept_cache = self.sheerka.om.register_concept_cache
|
||||
|
||||
# Cache of concept metadata, organized by id
|
||||
cache = Cache().auto_configure(self.CONCEPTS_BY_ID_ENTRY)
|
||||
register_concept_cache(self.CONCEPTS_BY_ID_ENTRY, cache, lambda c: c.id, True)
|
||||
|
||||
cache = ListIfNeededCache().auto_configure(self.CONCEPTS_BY_KEY_ENTRY)
|
||||
register_concept_cache(self.CONCEPTS_BY_KEY_ENTRY, cache, lambda c: c.key, True)
|
||||
|
||||
cache = ListIfNeededCache().auto_configure(self.CONCEPTS_BY_NAME_ENTRY)
|
||||
register_concept_cache(self.CONCEPTS_BY_NAME_ENTRY, cache, lambda c: c.name, True)
|
||||
|
||||
cache = ListIfNeededCache().auto_configure(self.CONCEPTS_BY_HASH_ENTRY)
|
||||
register_concept_cache(self.CONCEPTS_BY_HASH_ENTRY, cache, lambda c: c.digest, True)
|
||||
|
||||
def initialize_deferred(self, context, is_first_time):
|
||||
if is_first_time:
|
||||
self.sheerka.om.put(self.sheerka.OBJECTS_IDS_ENTRY, self.USER_CONCEPTS_IDS, 1000)
|
||||
|
||||
_ = self._create_builtin_concept
|
||||
_(1, BuiltinConcepts.SHEERKA, desc="Sheerka")
|
||||
_(2, BuiltinConcepts.NEW_CONCEPT, desc="On new concept creation", variables=("metadata",))
|
||||
_(3, BuiltinConcepts.UNKNOWN_CONCEPT, desc="Unknown concept", variables=("requested_name", "requested_id"))
|
||||
_(4, BuiltinConcepts.USER_INPUT, desc="Any external input", variables=("command",))
|
||||
_(5, BuiltinConcepts.PARSER_INPUT, desc="tokenized input", variables=("pi",))
|
||||
|
||||
self.init_log.debug('%s builtin concepts created',
|
||||
len(self.sheerka.om.current_cache_manager().concept_caches))
|
||||
|
||||
def define_new_concept(self, context: ExecutionContext,
|
||||
name: str,
|
||||
is_builtin: bool = False, # is the concept defined Sheerka
|
||||
is_unique: bool = False, # is the concept a singleton
|
||||
body: str = "", # return value of the concept
|
||||
where: str = "", # condition to recognize variables in name
|
||||
pre: str = "", # list of preconditions before calling the main function
|
||||
post: str = "", # list of post conditions after calling the main function
|
||||
ret: str = "", # variable to return when a concept is recognized
|
||||
definition: str = "", # regex used to define the concept
|
||||
definition_type: DefinitionType = DefinitionType.DEFAULT,
|
||||
autouse: bool = False, # indicate if the concept must be automatically evaluated
|
||||
bound_body: str = None, #
|
||||
desc: str = "", # possible description for the concept
|
||||
props: dict = None, # hashmap of default properties
|
||||
variables: list = None, # list of concept variables(tuple), with their default values
|
||||
parameters: list = None # list of variables that are part of the name of the concept
|
||||
) -> ReturnValue:
|
||||
"""
|
||||
Adds the definition of a new concept
|
||||
:return:
|
||||
:rtype:
|
||||
"""
|
||||
concept_key = self.create_concept_key(name, definition, variables)
|
||||
concept_id = "waiting for id"
|
||||
|
||||
metadata = ConceptMetadata(
|
||||
concept_id,
|
||||
name,
|
||||
concept_key,
|
||||
is_builtin,
|
||||
is_unique,
|
||||
body,
|
||||
where,
|
||||
pre,
|
||||
post,
|
||||
ret,
|
||||
definition,
|
||||
definition_type,
|
||||
desc,
|
||||
autouse,
|
||||
bound_body,
|
||||
props or {},
|
||||
variables or (),
|
||||
parameters or (),
|
||||
)
|
||||
|
||||
digest = self.compute_metadata_digest(metadata)
|
||||
if self.sheerka.om.exists_in_current(self.CONCEPTS_BY_HASH_ENTRY, digest):
|
||||
already_defined = self.sheerka.om.get(self.CONCEPTS_BY_HASH_ENTRY, digest)
|
||||
error = ErrorContext(self.NAME, context, ConceptAlreadyDefined(metadata, already_defined.id))
|
||||
return ReturnValue(self.NAME, False, error)
|
||||
|
||||
metadata.digest = digest
|
||||
metadata.all_attrs = self.compute_all_attrs(variables)
|
||||
|
||||
# bnf_expr = None
|
||||
# if definition_type == DefinitionType.BNF:
|
||||
# try:
|
||||
# bnf_expr = self.compute_concept_bnf(definition)
|
||||
# except InvalidBnf as ex:
|
||||
# error = ErrorContext(self.NAME, context, ex)
|
||||
# return ReturnValue(self.NAME, False, error)
|
||||
|
||||
# try:
|
||||
# first_item_res = self.recompute_first_items(context, None, [metadata])
|
||||
# except FirstItemError as ex:
|
||||
# return ReturnValue(self.NAME, False, ex)
|
||||
|
||||
# at this point everything is fine. let's get the id and save everything
|
||||
om = self.sheerka.om
|
||||
metadata.id = str(self.sheerka.om.get(self.sheerka.OBJECTS_IDS_ENTRY, self.USER_CONCEPTS_IDS))
|
||||
om.add_concept(metadata)
|
||||
# self.update_first_items_caches(context, first_item_res)
|
||||
# if bnf_expr:
|
||||
# self.bnf_expr_cache.put(metadata.id, bnf_expr)
|
||||
# # update references
|
||||
# for ref in self.compute_references(bnf_expr):
|
||||
# om.put(self.CONCEPTS_REFERENCES_ENTRY, ref, metadata.id)
|
||||
|
||||
return ReturnValue(self.NAME, True, self.newn(BuiltinConcepts.NEW_CONCEPT, metadata=metadata))
|
||||
|
||||
def newn(self, concept_name: str, **kwargs):
|
||||
"""
|
||||
new_by_name
|
||||
Creates and returns an instance of a new concept by its name
|
||||
:param concept_name:
|
||||
:type concept_name:
|
||||
:param kwargs:
|
||||
:type kwargs:
|
||||
:return:
|
||||
:rtype:
|
||||
"""
|
||||
metadata = self.get_by_name(concept_name)
|
||||
if metadata is NotFound:
|
||||
return self._inner_new(self.get_by_name(BuiltinConcepts.UNKNOWN_CONCEPT), requested_name=concept_name)
|
||||
|
||||
if isinstance(metadata, list):
|
||||
return [self._inner_new(m, **kwargs) for m in metadata]
|
||||
|
||||
return self._inner_new(metadata, **kwargs)
|
||||
|
||||
def newi(self, concept_id: str, **kwargs):
|
||||
"""
|
||||
new_by_id
|
||||
Creates and returns an instance of a new concept by its id
|
||||
:param concept_id:
|
||||
:type concept_id:
|
||||
:param kwargs:
|
||||
:type kwargs:
|
||||
:return:
|
||||
:rtype:
|
||||
"""
|
||||
metadata = self.get_by_id(concept_id)
|
||||
if metadata is NotFound:
|
||||
return self._inner_new(self.get_by_name(BuiltinConcepts.UNKNOWN_CONCEPT), requested_id=concept_id)
|
||||
return self._inner_new(metadata, **kwargs)
|
||||
|
||||
def get_by_name(self, key: str):
|
||||
"""
|
||||
Returns a concept metadata, using its name
|
||||
:param key:
|
||||
:type key:
|
||||
:return:
|
||||
:rtype:
|
||||
"""
|
||||
return self.sheerka.om.get(self.CONCEPTS_BY_NAME_ENTRY, key)
|
||||
|
||||
def get_by_id(self, concept_id: str):
|
||||
"""
|
||||
Returns a concept metadata, using its name
|
||||
:param concept_id:
|
||||
:type concept_id:
|
||||
:return:
|
||||
:rtype:
|
||||
"""
|
||||
return self.sheerka.om.get(self.CONCEPTS_BY_ID_ENTRY, concept_id)
|
||||
|
||||
def get_by_key(self, key: str):
|
||||
"""
|
||||
Returns a concept metadata, using its name
|
||||
:param key:
|
||||
:type key:
|
||||
:return:
|
||||
:rtype:
|
||||
"""
|
||||
return self.sheerka.om.get(self.CONCEPTS_BY_KEY_ENTRY, key)
|
||||
|
||||
@staticmethod
|
||||
def compute_metadata_digest(metadata: ConceptMetadata):
|
||||
"""
|
||||
Compute once for all the digest of the definition of a concept
|
||||
:param metadata:
|
||||
:type metadata:
|
||||
:return:
|
||||
:rtype:
|
||||
"""
|
||||
as_dict = {p: getattr(metadata, p) for p in PROPERTIES_FOR_DIGEST}
|
||||
return hashlib.sha256(f"{as_dict}".encode("utf-8")).hexdigest()
|
||||
|
||||
@staticmethod
|
||||
def compute_all_attrs(variables: tuple | None):
|
||||
"""
|
||||
Compute the list of available attributes for a concept
|
||||
:param variables:
|
||||
:return:
|
||||
:rtype:
|
||||
"""
|
||||
all_attrs = DefaultProps.copy()
|
||||
if variables:
|
||||
all_attrs += [k for k, v in variables]
|
||||
|
||||
return tuple(all_attrs)
|
||||
|
||||
@staticmethod
|
||||
def compute_concept_bnf(definition):
|
||||
pass
|
||||
|
||||
@staticmethod
|
||||
def create_concept_key(name: str, definition: str | None, variables: tuple | None):
|
||||
"""
|
||||
Creates the key from the definition
|
||||
:param name:
|
||||
:type name:
|
||||
:param definition:
|
||||
:type definition:
|
||||
:param variables:
|
||||
:type variables:
|
||||
:return:
|
||||
:rtype:
|
||||
"""
|
||||
|
||||
definition_to_use = definition or name
|
||||
tokens = list(Tokenizer(definition_to_use, yield_eof=False))
|
||||
|
||||
if variables is None or len(strip_tokens(tokens, True)) == 1:
|
||||
variables_to_use = []
|
||||
else:
|
||||
variables_to_use = [k for k, v in variables]
|
||||
|
||||
parts = []
|
||||
for token in tokens:
|
||||
if token.type == TokenKind.WHITESPACE:
|
||||
continue
|
||||
if token.value in variables_to_use:
|
||||
parts.append(VARIABLE_PREFIX + str(variables_to_use.index(token.value)))
|
||||
else:
|
||||
parts.append(token.value)
|
||||
|
||||
return " ".join(parts)
|
||||
|
||||
def _create_builtin_concept(self, concept_id: int, name: str, desc: str, variables: tuple = ()):
|
||||
variables_to_use = tuple((k, NotInit) for k in variables)
|
||||
concept_key = self.create_concept_key(name, None, variables_to_use)
|
||||
metadata = ConceptMetadata(
|
||||
str(concept_id),
|
||||
name,
|
||||
concept_key,
|
||||
True,
|
||||
False,
|
||||
"",
|
||||
"",
|
||||
"",
|
||||
"",
|
||||
"",
|
||||
"",
|
||||
DefinitionType.DEFAULT,
|
||||
desc,
|
||||
False,
|
||||
variables[0] if variables else "",
|
||||
{},
|
||||
variables_to_use,
|
||||
variables,
|
||||
)
|
||||
metadata.digest = self.compute_metadata_digest(metadata)
|
||||
metadata.all_attrs = self.compute_all_attrs(variables_to_use)
|
||||
self.sheerka.om.add_concept(metadata)
|
||||
|
||||
@staticmethod
|
||||
def _inner_new(_metadata_def: ConceptMetadata, **kwargs):
|
||||
concept = Concept(_metadata_def)
|
||||
for k, v in kwargs.items():
|
||||
concept.set_value(k, v)
|
||||
return concept
|
||||
@@ -1,223 +0,0 @@
|
||||
from dataclasses import dataclass
|
||||
|
||||
from common.utils import to_dict
|
||||
from core.ExecutionContext import ExecutionContext, ExecutionContextActions
|
||||
from core.ReturnValue import ReturnValue
|
||||
from core.services.BaseService import BaseService
|
||||
from evaluators.base_evaluator import AllReturnValuesEvaluator, BaseEvaluator, OneReturnValueEvaluator
|
||||
|
||||
|
||||
@dataclass
|
||||
class EvaluationPlan:
|
||||
sorted_priorities: list[int] # list of available priorities
|
||||
evaluators: dict[int, list[BaseEvaluator]]
|
||||
|
||||
|
||||
class SheerkaEngine(BaseService):
|
||||
"""
|
||||
This service is used to process user input
|
||||
It is responsible to parse and evaluate the information
|
||||
It also holds the rule engine
|
||||
"""
|
||||
NAME = "Engine"
|
||||
|
||||
def __init__(self, sheerka):
|
||||
super().__init__(sheerka, order=15)
|
||||
self.execution_plan = None # { ExecutionContextActions : { priority : [evaluators] }}
|
||||
self.no_evaluation_plan = EvaluationPlan([], {})
|
||||
|
||||
def initialize(self):
|
||||
self.execution_plan = self.compute_execution_plan(self.sheerka.evaluators.values())
|
||||
self.sheerka.bind_service_method(self.NAME, self.execute, True)
|
||||
|
||||
def call_evaluators(self,
|
||||
context: ExecutionContext,
|
||||
return_values: list[ReturnValue],
|
||||
step: ExecutionContextActions):
|
||||
"""
|
||||
Calls all evaluators defined for a given step
|
||||
:param context:
|
||||
:type context:
|
||||
:param return_values:
|
||||
:type return_values:
|
||||
:param step:
|
||||
:type step:
|
||||
:return:
|
||||
:rtype:
|
||||
"""
|
||||
|
||||
plan = self.get_evaluation_plan(context, step)
|
||||
iteration = 0
|
||||
while True:
|
||||
with context.push(self.NAME,
|
||||
ExecutionContextActions.EVALUATING_ITERATION,
|
||||
{"step": step, "iteration": iteration},
|
||||
desc=f"iteration #{iteration}") as iteration_context:
|
||||
simple_digest = return_values.copy()
|
||||
iteration_context.add_inputs(return_values=simple_digest)
|
||||
|
||||
for priority in plan.sorted_priorities:
|
||||
|
||||
return_values_copy = return_values.copy()
|
||||
new_return_values = {}
|
||||
return_values_to_delete = set()
|
||||
for evaluator in plan.evaluators[priority]:
|
||||
|
||||
sub_context_desc = f"Evaluating using {evaluator.name} ({priority=})"
|
||||
with iteration_context.push(self.NAME,
|
||||
step,
|
||||
{"step": step,
|
||||
"iteration": iteration,
|
||||
"evaluator": evaluator.name},
|
||||
desc=sub_context_desc) as evaluator_context:
|
||||
evaluator_context.add_inputs(return_values=return_values_copy)
|
||||
|
||||
# process evaluators that work on one simple return value at the time
|
||||
if isinstance(evaluator, OneReturnValueEvaluator):
|
||||
self.call_one_return_value_evaluator(evaluator_context,
|
||||
evaluator,
|
||||
return_values_copy,
|
||||
new_return_values,
|
||||
return_values_to_delete)
|
||||
|
||||
# process evaluators that work on all return values
|
||||
else:
|
||||
self.call_all_return_values_evaluator(evaluator_context,
|
||||
evaluator,
|
||||
return_values_copy,
|
||||
new_return_values,
|
||||
return_values_to_delete)
|
||||
|
||||
# Recreate the new return_value
|
||||
# Try to keep the order of what replaces what
|
||||
return_values = []
|
||||
for item in return_values_copy:
|
||||
if item not in return_values_to_delete:
|
||||
return_values.append(item)
|
||||
if item in new_return_values:
|
||||
return_values.extend(new_return_values[item])
|
||||
|
||||
iteration_context.add_values(return_values=return_values.copy())
|
||||
|
||||
iteration += 1
|
||||
if simple_digest == return_values:
|
||||
# I can use a variable like 'has_changed', but I think that this comparison is explicit
|
||||
# It explains that I stay in the loop if something was modified
|
||||
break
|
||||
|
||||
return return_values
|
||||
|
||||
def execute(self,
|
||||
context: ExecutionContext,
|
||||
return_values: list[ReturnValue],
|
||||
steps: list[ExecutionContextActions]):
|
||||
"""
|
||||
Runs the processing engine on the return_values
|
||||
:param context:
|
||||
:type context:
|
||||
:param return_values:
|
||||
:type return_values:
|
||||
:param steps:
|
||||
:type steps:
|
||||
:return:
|
||||
:rtype:
|
||||
"""
|
||||
for step in steps:
|
||||
copy = return_values.copy()
|
||||
with context.push(self.NAME, ExecutionContextActions.EVALUATING_STEP, {"step": step}) as sub_context:
|
||||
sub_context.add_inputs(return_values=copy)
|
||||
|
||||
return_values = self.call_evaluators(sub_context, return_values, step)
|
||||
|
||||
sub_context.add_values(return_values=return_values)
|
||||
sub_context.add_values(has_changed=(copy != return_values))
|
||||
|
||||
return return_values
|
||||
|
||||
def get_evaluation_plan(self, context: ExecutionContext, step: ExecutionContextActions) -> EvaluationPlan:
|
||||
if step not in self.execution_plan:
|
||||
return self.no_evaluation_plan
|
||||
|
||||
evaluators = self.execution_plan[step]
|
||||
return EvaluationPlan(sorted(evaluators.keys(), reverse=True), evaluators)
|
||||
|
||||
@staticmethod
|
||||
def call_one_return_value_evaluator(context: ExecutionContext,
|
||||
evaluator: OneReturnValueEvaluator,
|
||||
return_values: list[ReturnValue],
|
||||
new_return_values: dict[ReturnValue, list[ReturnValue]],
|
||||
return_values_to_delete: set[ReturnValue]):
|
||||
"""
|
||||
|
||||
:param context:
|
||||
:type context:
|
||||
:param evaluator:
|
||||
:type evaluator:
|
||||
:param return_values:
|
||||
:type return_values:
|
||||
:param new_return_values:
|
||||
:type new_return_values:
|
||||
:param return_values_to_delete:
|
||||
:type return_values_to_delete:
|
||||
:return:
|
||||
:rtype:
|
||||
"""
|
||||
context_trace = []
|
||||
for item in return_values:
|
||||
debug = {"item": item}
|
||||
context_trace.append(debug)
|
||||
|
||||
m = evaluator.matches(context, item)
|
||||
debug["match"] = m.status
|
||||
|
||||
if m.status:
|
||||
result = evaluator.eval(context, m.obj, item)
|
||||
return_values_to_delete.update(result.eaten)
|
||||
new_return_values.setdefault(item, []).extend(result.new)
|
||||
debug["new"] = result.new
|
||||
debug["eaten"] = result.eaten
|
||||
|
||||
context.add_values(evaluation=context_trace)
|
||||
|
||||
@staticmethod
|
||||
def call_all_return_values_evaluator(context: ExecutionContext,
|
||||
evaluator: AllReturnValuesEvaluator,
|
||||
return_values: list[ReturnValue],
|
||||
new_return_values: dict[ReturnValue, list[ReturnValue]],
|
||||
return_values_to_delete: set[ReturnValue]):
|
||||
"""
|
||||
|
||||
:param context:
|
||||
:type context:
|
||||
:param evaluator:
|
||||
:type evaluator:
|
||||
:param return_values:
|
||||
:type return_values:
|
||||
:param new_return_values:
|
||||
:type new_return_values:
|
||||
:param return_values_to_delete:
|
||||
:type return_values_to_delete:
|
||||
:return:
|
||||
:rtype:
|
||||
"""
|
||||
debug = {}
|
||||
m = evaluator.matches(context, return_values)
|
||||
debug["match"] = m.status
|
||||
|
||||
if m.status:
|
||||
result = evaluator.eval(context, m.obj, return_values)
|
||||
return_values_to_delete.update(result.eaten)
|
||||
new_return_values.setdefault(result.new[0].parents[0], []).extend(result.new)
|
||||
debug["new"] = result.new
|
||||
debug["eaten"] = result.eaten
|
||||
|
||||
context.add_values(evaluation=debug)
|
||||
|
||||
@staticmethod
|
||||
def compute_execution_plan(evaluators):
|
||||
evaluators = [e for e in evaluators if e.enabled]
|
||||
by_step = to_dict(evaluators, lambda e: e.step)
|
||||
for k, v in by_step.items():
|
||||
by_step[k] = to_dict(v, lambda e: e.priority)
|
||||
|
||||
return by_step
|
||||
Reference in New Issue
Block a user