Added first version of DebugManager. Implemented draft of the rule engine
This commit is contained in:
+147
-146
@@ -3,12 +3,14 @@ import copy
|
||||
import traceback
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
import core.ast.nodes
|
||||
import core.builtin_helpers
|
||||
import core.utils
|
||||
from core.ast.visitors import UnreferencedNamesVisitor
|
||||
from core.ast_helpers import UnreferencedNamesVisitor, NamesWithAttributesVisitor
|
||||
from core.builtin_concepts import BuiltinConcepts, ParserResultConcept
|
||||
from core.concept import ConceptParts, Concept, NotInit
|
||||
from core.sheerka.services.SheerkaFilter import Pipe
|
||||
from core.rule import Rule
|
||||
from core.sheerka.ExecutionContext import ExecutionContext
|
||||
from core.tokenizer import Token, TokenKind
|
||||
from evaluators.BaseEvaluator import OneReturnValueEvaluator
|
||||
from parsers.PythonParser import PythonNode
|
||||
|
||||
@@ -47,6 +49,20 @@ class PythonEvalError:
|
||||
traceback: str = field(repr=False)
|
||||
concepts: dict = field(repr=False)
|
||||
|
||||
def __eq__(self, other):
|
||||
if id(self) == id(other):
|
||||
return True
|
||||
|
||||
if not isinstance(other, PythonEvalError):
|
||||
return False
|
||||
|
||||
return isinstance(self.error, type(other.error)) and \
|
||||
self.traceback == other.traceback and \
|
||||
self.concepts == other.concepts
|
||||
|
||||
def __hash__(self):
|
||||
return hash(self.error)
|
||||
|
||||
|
||||
class PythonEvaluator(OneReturnValueEvaluator):
|
||||
NAME = "Python"
|
||||
@@ -54,37 +70,31 @@ class PythonEvaluator(OneReturnValueEvaluator):
|
||||
"""
|
||||
Evaluate a Python node, ie, evaluate some Python code
|
||||
"""
|
||||
isinstance = None
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(self.NAME, [BuiltinConcepts.EVALUATION], 50)
|
||||
self.globals = {}
|
||||
|
||||
@staticmethod
|
||||
def initialize(sheerka):
|
||||
from core.sheerka.services.SheerkaAdmin import SheerkaAdmin
|
||||
PythonEvaluator.isinstance = sheerka.services[SheerkaAdmin.NAME].extended_isinstance
|
||||
|
||||
def matches(self, context, return_value):
|
||||
if not return_value.status or not isinstance(return_value.value, ParserResultConcept):
|
||||
return False
|
||||
body = return_value.value.value
|
||||
return isinstance(body, PythonNode) or (
|
||||
hasattr(body, "python_node") and isinstance(body.python_node, PythonNode))
|
||||
# return return_value.status and \
|
||||
# isinstance(return_value.value, ParserResultConcept) and \
|
||||
# isinstance(return_value.value.value, PythonNode)
|
||||
return isinstance(body, PythonNode) or hasattr(body, "python_node")
|
||||
|
||||
def eval(self, context, return_value):
|
||||
sheerka = context.sheerka
|
||||
node = return_value.value.value if isinstance(return_value.value.value, PythonNode) else \
|
||||
return_value.value.value.python_node
|
||||
|
||||
debugger = context.get_debugger(PythonEvaluator.NAME, "eval")
|
||||
debugger.debug_entering(node=node)
|
||||
context.log(f"Evaluating python node {node}.", self.name)
|
||||
|
||||
# Do not evaluate if the ast refers to a concept (leave it to ConceptEvaluator)
|
||||
# TODO: Remove this section when this check will be implemented in the AFTER_PARSING step
|
||||
if isinstance(node.ast_, ast.Expression) and isinstance(node.ast_.body, ast.Name):
|
||||
c = context.sheerka.resolve(node.ast_.body.id)
|
||||
if c is not None:
|
||||
context.log("It's a simple concept. Not for me.", self.name)
|
||||
not_for_me = context.sheerka.new(BuiltinConcepts.NOT_FOR_ME, body=node)
|
||||
return sheerka.ret(self.name, False, not_for_me, parents=[return_value])
|
||||
|
||||
# If we evaluate a Concept metadata which is NOT the body ex (pre, post, where...)
|
||||
# We need to disable the function that may alter the state
|
||||
# It's a poor way to have source code security check
|
||||
@@ -104,11 +114,12 @@ class PythonEvaluator(OneReturnValueEvaluator):
|
||||
|
||||
# get globals
|
||||
my_globals = self.get_globals(context, node, expression_only)
|
||||
context.log(f"globals={my_globals}", self.name)
|
||||
|
||||
debugger.debug_var("globals", my_globals)
|
||||
|
||||
all_possible_globals = self.get_all_possible_globals(context, my_globals)
|
||||
concepts_entries = None
|
||||
evaluated = BuiltinConcepts.NOT_INITIALIZED
|
||||
concepts_entries = None # entries in globals_ that refers to Concept objects
|
||||
evaluated = NotInit
|
||||
errors = []
|
||||
expect_success = context.in_context(BuiltinConcepts.EVAL_UNTIL_SUCCESS_REQUESTED)
|
||||
for globals_ in all_possible_globals:
|
||||
@@ -116,8 +127,7 @@ class PythonEvaluator(OneReturnValueEvaluator):
|
||||
# eval
|
||||
if isinstance(node.ast_, ast.Expression):
|
||||
context.log("Evaluating using 'eval'.", self.name)
|
||||
compiled = compile(node.ast_, "<string>", "eval")
|
||||
evaluated = eval(compiled, globals_, sheerka.locals)
|
||||
evaluated = eval(node.get_compiled(), globals_, sheerka.locals)
|
||||
else:
|
||||
context.log("Evaluating using 'exec'.", self.name)
|
||||
evaluated = self.exec_with_return(node.ast_, globals_, sheerka.locals)
|
||||
@@ -128,143 +138,125 @@ class PythonEvaluator(OneReturnValueEvaluator):
|
||||
if concepts_entries is None:
|
||||
concepts_entries = self.get_concepts_entries_from_globals(my_globals)
|
||||
errors.append(PythonEvalError(ex,
|
||||
traceback.format_exc(),
|
||||
traceback.format_exc() if context.debug_enabled else None,
|
||||
self.get_concepts_values_from_globals(globals_, concepts_entries)))
|
||||
|
||||
if evaluated == BuiltinConcepts.NOT_INITIALIZED:
|
||||
if evaluated == NotInit:
|
||||
if len(errors) == 1:
|
||||
context.log_error(errors[0].error, who=self.name, exc=errors[0].traceback)
|
||||
one_error = sheerka.new(BuiltinConcepts.ERROR, body=errors[0])
|
||||
return sheerka.ret(self.name, False, one_error, parents=[return_value])
|
||||
return sheerka.ret(self.name, False, sheerka.err(errors[0]), parents=[return_value])
|
||||
|
||||
if len(errors) > 1:
|
||||
for eval_error in errors:
|
||||
context.log_error(eval_error.error, who=self.name, exc=eval_error.traceback)
|
||||
too_many_errors = sheerka.new(BuiltinConcepts.TOO_MANY_ERRORS, body=errors)
|
||||
return sheerka.ret(self.name, False, too_many_errors, parents=[return_value])
|
||||
return sheerka.ret(self.name, False, sheerka.err(errors), parents=[return_value])
|
||||
|
||||
context.log(f"{evaluated=}", self.name)
|
||||
debugger.debug_var("ret", evaluated)
|
||||
return sheerka.ret(self.name, True, evaluated, parents=[return_value])
|
||||
|
||||
def get_globals(self, context, node, expression_only):
|
||||
"""
|
||||
Creates the global variables for python source code evaluation
|
||||
Creates the globals variables
|
||||
:param context:
|
||||
:param node:
|
||||
:param expression_only: most of the commands are refused
|
||||
:param expression_only:
|
||||
:return:
|
||||
"""
|
||||
unreferenced_names_visitor = UnreferencedNamesVisitor(context)
|
||||
names = unreferenced_names_visitor.get_names(node.ast_)
|
||||
if context.debug_enabled:
|
||||
context.debug(self.NAME, "eval", "names", names)
|
||||
return self.get_globals_by_names(context, names, node, expression_only)
|
||||
|
||||
def get_sheerka_method(self, context, name, expression_only):
|
||||
try:
|
||||
method = context.sheerka.sheerka_methods[name]
|
||||
context.log(f"Resolving '{name}'. It's a sheerka method.", self.name)
|
||||
if expression_only and method.has_side_effect:
|
||||
context.log(f"...but with side effect when {expression_only=}. Discarding.", self.name)
|
||||
return None
|
||||
else:
|
||||
return inject_context(context)(method.method) if name in context.sheerka.methods_with_context \
|
||||
else method.method
|
||||
except KeyError:
|
||||
return None
|
||||
|
||||
def get_globals_by_names(self, context, names, node, expression_only):
|
||||
my_globals = {
|
||||
"Concept": core.concept.Concept,
|
||||
"BuiltinConcepts": core.builtin_concepts.BuiltinConcepts,
|
||||
"ExecutionContext": ExecutionContext,
|
||||
"in_context": context.in_context,
|
||||
}
|
||||
|
||||
if expression_only:
|
||||
# disable some builtin
|
||||
for statement in TO_DISABLED:
|
||||
my_globals[statement] = None
|
||||
|
||||
# has to be the first, to allow override
|
||||
method_from_sheerka = self.update_globals_with_sheerka_methods(my_globals, context, expression_only)
|
||||
|
||||
self.update_globals_with_context(my_globals, context)
|
||||
already_know = set(my_globals.keys())
|
||||
self.update_globals_with_node(my_globals, context, node, already_know)
|
||||
|
||||
if self.globals: # when extra values are given. Add them
|
||||
my_globals.update(self.globals)
|
||||
|
||||
my_globals["sheerka"] = Expando(method_from_sheerka) # it's the last, so I cannot be overridden
|
||||
return my_globals
|
||||
|
||||
@staticmethod
|
||||
def update_globals_with_sheerka_methods(my_locals, context, expression_only):
|
||||
methods_from_sheerka = {}
|
||||
|
||||
# Add all the methods as a direct access
|
||||
for method_name, method in context.sheerka.sheerka_methods.items():
|
||||
if expression_only and method.has_side_effect:
|
||||
for name in names:
|
||||
if name in my_globals:
|
||||
continue
|
||||
|
||||
if method_name in context.sheerka.methods_with_context:
|
||||
my_locals[method_name] = inject_context(context)(method.method)
|
||||
else:
|
||||
my_locals[method_name] = method.method
|
||||
methods_from_sheerka[method_name] = my_locals[method_name]
|
||||
|
||||
# Add pipeable functions
|
||||
for func_name, function in context.sheerka.sheerka_pipeables.items():
|
||||
if expression_only and function.has_side_effect:
|
||||
if expression_only and name in TO_DISABLED:
|
||||
my_globals[name] = None
|
||||
continue
|
||||
|
||||
my_locals[func_name] = Pipe(function.method, context)
|
||||
|
||||
return methods_from_sheerka # to allow access using prefix "sheerka."
|
||||
|
||||
def update_globals_with_context(self, my_globals, context):
|
||||
"""
|
||||
Update globals with the current object being evaluated (and its variables)
|
||||
:param my_globals:
|
||||
:param context:
|
||||
:return:
|
||||
"""
|
||||
if context.obj:
|
||||
context.log(f"Concept '{context.obj}' is in context. Adding it and its properties to locals.", self.name)
|
||||
|
||||
for prop_name in context.obj.variables():
|
||||
value = context.obj.get_value(prop_name)
|
||||
if value != NotInit:
|
||||
my_globals[prop_name] = value
|
||||
my_globals["self"] = context.obj
|
||||
|
||||
def update_globals_with_node(self, my_globals, context, node, already_known):
|
||||
"""
|
||||
Try to find concepts using the names that appear in the AST of the node.
|
||||
:param my_globals: dictionary to update
|
||||
:param context:
|
||||
:param node:
|
||||
:param already_known: if the name is in this list, do no try to instantiate it again
|
||||
:return:
|
||||
"""
|
||||
node_concept = core.ast.nodes.python_to_concept(node.ast_)
|
||||
unreferenced_names_visitor = UnreferencedNamesVisitor(context.sheerka)
|
||||
unreferenced_names_visitor.visit(node_concept)
|
||||
|
||||
for name in unreferenced_names_visitor.names:
|
||||
context.log(f"Resolving '{name}'.", self.name)
|
||||
|
||||
# get the concept
|
||||
if name in node.concepts:
|
||||
# use it, even if it already in already_known
|
||||
# This concept take precedence other the outer world
|
||||
context.log(f"Using value from node.", self.name)
|
||||
concept = self.resolve_concept(context, node.concepts[name])
|
||||
elif name in already_known:
|
||||
context.log(f"Already known. Skipping.", self.name)
|
||||
continue
|
||||
elif (concept := context.get_from_short_term_memory(name)) is not None:
|
||||
context.log(f"Using from STM known.", self.name)
|
||||
else:
|
||||
context.log(f"Instantiating new concept with {name}.", self.name)
|
||||
concept = self.resolve_concept(context, name)
|
||||
|
||||
if concept is None:
|
||||
context.log(f"Concept '{name}' is not found or cannot be instantiated. Skipping.", self.name)
|
||||
# need to add it manually to avoid conflict with sheerka.isinstance
|
||||
if name == "isinstance":
|
||||
my_globals["isinstance"] = PythonEvaluator.isinstance
|
||||
continue
|
||||
|
||||
# evaluate it if needed
|
||||
if concept.metadata.is_evaluated:
|
||||
context.log(f"Concept {name} is already evaluated.", self.name)
|
||||
else:
|
||||
context.log(f"Evaluating '{concept}'", self.name)
|
||||
evaluated = context.sheerka.evaluate_concept(context, concept, eval_body=True)
|
||||
if not context.sheerka.is_success(evaluated) and evaluated.key != concept.key:
|
||||
context.log(f"Error while evaluating '{name}'. Skipping.", self.name)
|
||||
# support reference to sheerka
|
||||
if name == "sheerka":
|
||||
bag = {}
|
||||
visitor = NamesWithAttributesVisitor()
|
||||
for sequence in visitor.get_sequences(node.ast_, "sheerka"):
|
||||
if (len(sequence) > 1 and
|
||||
(method := self.get_sheerka_method(context, sequence[1], expression_only)) is not None):
|
||||
bag[sequence[1]] = method
|
||||
my_globals["sheerka"] = Expando(bag)
|
||||
continue
|
||||
|
||||
# search in short term memory
|
||||
if (obj := context.get_from_short_term_memory(name)) is not None:
|
||||
context.log(f"Resolving '{name}'. Using value found in STM.", self.name)
|
||||
my_globals[name] = obj
|
||||
continue
|
||||
|
||||
# search in sheerka methods
|
||||
if (method := self.get_sheerka_method(context, name, expression_only)) is not None:
|
||||
my_globals[name] = method
|
||||
continue
|
||||
|
||||
# search in context.obj (to replace by short time memory ?)
|
||||
if context.obj:
|
||||
if name == "self":
|
||||
my_globals["self"] = context.obj
|
||||
continue
|
||||
concept = evaluated
|
||||
|
||||
my_globals[name] = concept
|
||||
try:
|
||||
attribute = context.obj.variables()[name]
|
||||
if attribute != NotInit:
|
||||
my_globals[name] = attribute
|
||||
continue
|
||||
context.log(f"Resolving '{name}'. It's obj attribute (obj={context.obj}).", self.name)
|
||||
except KeyError:
|
||||
pass
|
||||
|
||||
# search in current node (if the name was found during the parsing)
|
||||
if name in node.objects:
|
||||
context.log(f"Resolving '{name}'. Using value from node.", self.name)
|
||||
obj = self.resolve_object(context, node.objects[name])
|
||||
|
||||
# at last, try to instantiate a new concept
|
||||
else:
|
||||
context.log(f"Resolving '{name}'. Instantiating new concept.", self.name)
|
||||
obj = self.resolve_object(context, name)
|
||||
|
||||
if obj is None:
|
||||
context.log(f"...'{name}' is not found or cannot be instantiated. Skipping.", self.name)
|
||||
continue
|
||||
|
||||
my_globals[name] = obj
|
||||
|
||||
return my_globals
|
||||
|
||||
@staticmethod
|
||||
def get_all_possible_globals(context, my_globals):
|
||||
@@ -309,31 +301,37 @@ class PythonEvaluator(OneReturnValueEvaluator):
|
||||
return {name: my_globals[name] for name in names}
|
||||
|
||||
@staticmethod
|
||||
def resolve_concept(context, concept_hint):
|
||||
def resolve_object(context, name):
|
||||
"""
|
||||
Try to find a concept by its name, id or the pattern c:key|id:
|
||||
:param context:
|
||||
:param concept_hint:
|
||||
:param name:
|
||||
:return:
|
||||
"""
|
||||
if isinstance(concept_hint, Concept):
|
||||
return concept_hint
|
||||
if isinstance(name, Rule):
|
||||
return name
|
||||
|
||||
concept = context.sheerka.resolve(concept_hint)
|
||||
if isinstance(name, Concept):
|
||||
name = core.builtin_helpers.ensure_evaluated(context, name)
|
||||
return name
|
||||
|
||||
if isinstance(name, Token) and name.type == TokenKind.RULE:
|
||||
rule = context.sheerka.get_rule_by_id(name.value[1]) # TODO: need a resolve function for the rules
|
||||
return rule if isinstance(rule, Rule) else None
|
||||
|
||||
if isinstance(name, tuple):
|
||||
raise Exception()
|
||||
|
||||
# try to resolve by name
|
||||
concept = context.sheerka.fast_resolve(name)
|
||||
if concept is None:
|
||||
return None
|
||||
new_instance = context.sheerka.new_from_template(concept, concept.key)
|
||||
if isinstance(concept_hint, tuple):
|
||||
# It's means that it was requested by PythonParser which have found a concept token (c:xxx:)
|
||||
# So a concept was explicitly required, not its value
|
||||
# We mark the concept as already evaluated, so it's body will not be evaluated
|
||||
new_instance.metadata.is_evaluated = True
|
||||
if len(concept.metadata.variables) > 0:
|
||||
# In this situation, it means that we are dealing with the concept and not its instantiation
|
||||
# So do not try to evaluate it
|
||||
new_instance.metadata.is_evaluated = True
|
||||
|
||||
return new_instance
|
||||
if hasattr(concept, "__iter__"):
|
||||
raise NotImplementedError("Too many concepts")
|
||||
|
||||
concept = core.builtin_helpers.ensure_evaluated(context, concept)
|
||||
return concept
|
||||
|
||||
@staticmethod
|
||||
def expr_to_expression(expr):
|
||||
@@ -343,7 +341,8 @@ class PythonEvaluator(OneReturnValueEvaluator):
|
||||
|
||||
return result
|
||||
|
||||
def exec_with_return(self, code_ast, my_globals, my_locals):
|
||||
@staticmethod
|
||||
def exec_with_return(code_ast, my_globals, my_locals):
|
||||
|
||||
init_ast = copy.deepcopy(code_ast)
|
||||
init_ast.body = code_ast.body[:-1]
|
||||
@@ -353,6 +352,8 @@ class PythonEvaluator(OneReturnValueEvaluator):
|
||||
|
||||
exec(compile(init_ast, "<ast>", "exec"), my_globals, my_locals)
|
||||
if type(last_ast.body[0]) == ast.Expr:
|
||||
return eval(compile(self.expr_to_expression(last_ast.body[0]), "<ast>", "eval"), my_globals, my_locals)
|
||||
return eval(compile(PythonEvaluator.expr_to_expression(last_ast.body[0]), "<ast>", "eval"),
|
||||
my_globals,
|
||||
my_locals)
|
||||
else:
|
||||
exec(compile(last_ast, "<ast>", "exec"), my_globals, my_locals)
|
||||
|
||||
Reference in New Issue
Block a user