Added first implementation of concepts ambiguity resolution + Jenkins file test
This commit is contained in:
@@ -2,7 +2,7 @@ from core.ast.nodes import python_to_concept
|
||||
from core.builtin_concepts import ParserResultConcept, ReturnValueConcept, BuiltinConcepts
|
||||
from core.builtin_helpers import get_names
|
||||
from core.concept import Concept, DEFINITION_TYPE_BNF, DEFINITION_TYPE_DEF
|
||||
from core.tokenizer import TokenKind
|
||||
from core.tokenizer import TokenKind, Tokenizer
|
||||
from evaluators.BaseEvaluator import OneReturnValueEvaluator
|
||||
from parsers.BaseParser import NotInitializedNode
|
||||
from parsers.BnfNodeParser import ParsingExpression, ParsingExpressionVisitor
|
||||
@@ -53,7 +53,7 @@ class AddConceptEvaluator(OneReturnValueEvaluator):
|
||||
sheerka = context.sheerka
|
||||
|
||||
# validate the node
|
||||
props_found = set()
|
||||
variables_found = set()
|
||||
|
||||
concept = Concept(def_concept_node.name)
|
||||
concept.metadata.definition_type = def_concept_node.definition_type
|
||||
@@ -80,16 +80,16 @@ class AddConceptEvaluator(OneReturnValueEvaluator):
|
||||
|
||||
# try to find what can be a property
|
||||
for p in self.get_variables(sheerka, part_ret_val, name_to_use):
|
||||
props_found.add(p)
|
||||
variables_found.add(p)
|
||||
|
||||
# add variables by order of appearance when possible
|
||||
for name_part in name_to_use:
|
||||
if name_part in props_found:
|
||||
if name_part in variables_found:
|
||||
concept.def_var(name_part, None)
|
||||
|
||||
# add the remaining properties
|
||||
# They mainly come from BNF definition
|
||||
for p in props_found:
|
||||
for p in variables_found:
|
||||
if p not in concept.values:
|
||||
concept.def_var(p, None)
|
||||
|
||||
@@ -130,24 +130,31 @@ class AddConceptEvaluator(OneReturnValueEvaluator):
|
||||
names = [str(t.value) for t in ret_value.tokens if t.type in (
|
||||
TokenKind.IDENTIFIER, TokenKind.STRING, TokenKind.KEYWORD)]
|
||||
variables = filter(lambda x: x in concept_name, names)
|
||||
return list(variables)
|
||||
return set(variables)
|
||||
|
||||
#
|
||||
# Case of python code
|
||||
#
|
||||
if isinstance(ret_value.value, ParserResultConcept) and isinstance(ret_value.value.value, PythonNode):
|
||||
if len(concept_name) > 1:
|
||||
python_node = ret_value.value.value
|
||||
as_concept_node = python_to_concept(python_node.ast_)
|
||||
names = get_names(sheerka, as_concept_node)
|
||||
variables = filter(lambda x: x in concept_name, names)
|
||||
return list(variables)
|
||||
# tokens from ParserResult or source from python node
|
||||
variables = set()
|
||||
tokens = ret_value.value.tokens or list(Tokenizer(ret_value.value.value.source))
|
||||
tokens = [t.str_value for t in tokens]
|
||||
for identifier in [i for i in concept_name if str(i).isalnum()]:
|
||||
if identifier in tokens:
|
||||
variables.add(identifier)
|
||||
# python_node = ret_value.value.value
|
||||
# as_concept_node = python_to_concept(python_node.ast_)
|
||||
# names = get_names(sheerka, as_concept_node)
|
||||
# variables = filter(lambda x: x in concept_name, names)
|
||||
return variables
|
||||
|
||||
#
|
||||
# case of concept
|
||||
#
|
||||
if isinstance(ret_value.value, ParserResultConcept) and isinstance(ret_value.value.value, Concept):
|
||||
return list(ret_value.value.value.values.keys())
|
||||
return set(ret_value.value.value.values.keys())
|
||||
|
||||
#
|
||||
# case of BNF
|
||||
@@ -155,6 +162,6 @@ class AddConceptEvaluator(OneReturnValueEvaluator):
|
||||
if isinstance(ret_value.value, ParserResultConcept) and isinstance(ret_value.value.value, ParsingExpression):
|
||||
visitor = ConceptOrRuleNameVisitor()
|
||||
visitor.visit(ret_value.value.value)
|
||||
return sorted(list(visitor.names))
|
||||
return set(visitor.names)
|
||||
|
||||
return []
|
||||
|
||||
@@ -25,7 +25,7 @@ class ConceptEvaluator(OneReturnValueEvaluator):
|
||||
# self.evaluate_body = True
|
||||
#
|
||||
# for r in return_values:
|
||||
# if r.status and context.sheerka.isinstance(r.body, BuiltinConcepts.CONCEPT_VALUE_REQUESTED):
|
||||
# if r.status and context.sheerka.isinstance(r.body, BuiltinConcepts.RETURN_VALUE_REQUESTED):
|
||||
# self.evaluate_body = True
|
||||
# break
|
||||
#
|
||||
|
||||
@@ -16,7 +16,7 @@ class EvalEvaluator(AllReturnValuesEvaluator):
|
||||
def matches(self, context, return_values):
|
||||
evaluation_parents = context.get_parents(lambda c: c.action == BuiltinConcepts.PROCESSING)
|
||||
is_root = len(evaluation_parents) <= 1
|
||||
return context.in_context(BuiltinConcepts.CONCEPT_VALUE_REQUESTED) and is_root
|
||||
return context.in_context(BuiltinConcepts.RETURN_VALUE_REQUESTED) and is_root
|
||||
|
||||
def eval(self, context, return_values):
|
||||
sheerka = context.sheerka
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
from core.builtin_concepts import BuiltinConcepts
|
||||
from core.concept import Concept
|
||||
from evaluators.BaseEvaluator import OneReturnValueEvaluator
|
||||
|
||||
|
||||
class PostExecutionEvaluator(OneReturnValueEvaluator):
|
||||
"""
|
||||
Last chance to alter the return_value
|
||||
This evaluator is supposed to be a generic evaluator for all rules that must be executed just before
|
||||
the aggregations
|
||||
"""
|
||||
|
||||
NAME = "PostExecution"
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(self.NAME, [BuiltinConcepts.AFTER_EVALUATION], 90)
|
||||
|
||||
def matches(self, context, return_value):
|
||||
evaluation_parents = context.get_parents(lambda c: c.action == BuiltinConcepts.PROCESSING)
|
||||
if len(evaluation_parents) > 1:
|
||||
return False # It must be executed only when the top level context
|
||||
|
||||
# only support the rule for the COMMANDS
|
||||
value = return_value.body
|
||||
return isinstance(value, Concept) and context.sheerka.isa(value, context.sheerka.new(BuiltinConcepts.COMMAND))
|
||||
|
||||
def eval(self, context, return_value):
|
||||
# only support the rule for the COMMANDS
|
||||
body = return_value.body.body
|
||||
return context.sheerka.ret(
|
||||
self.name,
|
||||
True,
|
||||
body if body != BuiltinConcepts.NOT_INITIALIZED else return_value.body,
|
||||
parents=[return_value])
|
||||
@@ -34,6 +34,7 @@ class PrepareEvalEvaluator(OneReturnValueEvaluator):
|
||||
True, sheerka.new(BuiltinConcepts.USER_INPUT, body=self.text[5:], user_name=context.event.user_id))
|
||||
|
||||
context.global_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
|
||||
context.global_hints.add(BuiltinConcepts.CONCEPT_VALUE_REQUESTED)
|
||||
context.global_hints.add(BuiltinConcepts.EVAL_WHERE_REQUESTED)
|
||||
context.global_hints.add(BuiltinConcepts.RETURN_VALUE_REQUESTED)
|
||||
|
||||
return new_text_to_parse
|
||||
|
||||
@@ -7,7 +7,7 @@ import core.ast.nodes
|
||||
import core.utils
|
||||
from core.ast.visitors import UnreferencedNamesVisitor
|
||||
from core.builtin_concepts import BuiltinConcepts, ParserResultConcept
|
||||
from core.concept import ConceptParts, Concept
|
||||
from core.concept import ConceptParts, Concept, NotInit
|
||||
from core.sheerka.services.SheerkaFilter import Pipe
|
||||
from evaluators.BaseEvaluator import OneReturnValueEvaluator
|
||||
from parsers.PythonParser import PythonNode
|
||||
@@ -73,15 +73,29 @@ class PythonEvaluator(OneReturnValueEvaluator):
|
||||
not_for_me = context.sheerka.new(BuiltinConcepts.NOT_FOR_ME, body=node)
|
||||
return sheerka.ret(self.name, False, not_for_me, parents=[return_value])
|
||||
|
||||
attr_under_eval = context.get_parents(lambda ec: ec.action == BuiltinConcepts.EVALUATING_ATTRIBUTE)
|
||||
if attr_under_eval:
|
||||
attr_under_eval = attr_under_eval[0]
|
||||
expression_only = attr_under_eval.action_context != ConceptParts.BODY
|
||||
|
||||
if expression_only and isinstance(node.ast_, ast.Module):
|
||||
# Module execution is forbidden in where, pre, post and ret concept parts
|
||||
security_error = sheerka.new(BuiltinConcepts.PYTHON_SECURITY_ERROR,
|
||||
prop=attr_under_eval.action_context,
|
||||
body=node.source)
|
||||
return sheerka.ret(self.name, False, security_error, parents=[return_value])
|
||||
else:
|
||||
expression_only = False
|
||||
|
||||
# get globals
|
||||
my_globals = self.get_globals(context, node)
|
||||
my_globals = self.get_globals(context, node, expression_only)
|
||||
context.log(f"globals={my_globals}", self.name)
|
||||
|
||||
all_possible_globals = self.get_all_possible_globals(context, my_globals)
|
||||
concepts_entries = None
|
||||
evaluated = BuiltinConcepts.NOT_INITIALIZED
|
||||
errors = []
|
||||
expect_success = BuiltinConcepts.EVAL_SUCCESS_REQUESTED in context.local_hints
|
||||
expect_success = BuiltinConcepts.EVAL_UNTIL_SUCCESS_REQUESTED in context.local_hints
|
||||
for globals_ in all_possible_globals:
|
||||
try:
|
||||
# eval
|
||||
@@ -117,14 +131,27 @@ class PythonEvaluator(OneReturnValueEvaluator):
|
||||
context.log(f"{evaluated=}", self.name)
|
||||
return sheerka.ret(self.name, True, evaluated, parents=[return_value])
|
||||
|
||||
def get_globals(self, context, node):
|
||||
def get_globals(self, context, node, expression_only):
|
||||
"""
|
||||
Creates the global variables for python source code evaluation
|
||||
:param context:
|
||||
:param node:
|
||||
:param expression_only: most of the commands are refused
|
||||
:return:
|
||||
"""
|
||||
my_globals = {
|
||||
"Concept": core.concept.Concept,
|
||||
"BuiltinConcepts": core.builtin_concepts.BuiltinConcepts,
|
||||
"in_context": context.in_context
|
||||
}
|
||||
|
||||
if expression_only:
|
||||
# disable builtin
|
||||
my_globals["__builtins__"] = None
|
||||
|
||||
# has to be the first, to allow override
|
||||
method_from_sheerka = self.update_globals_with_sheerka_methods(my_globals, context)
|
||||
method_from_sheerka = self.update_globals_with_sheerka_methods(my_globals, context, expression_only)
|
||||
|
||||
self.update_globals_with_context(my_globals, context)
|
||||
already_know = set(my_globals.keys())
|
||||
self.update_globals_with_node(my_globals, context, node, already_know)
|
||||
@@ -136,32 +163,54 @@ class PythonEvaluator(OneReturnValueEvaluator):
|
||||
return my_globals
|
||||
|
||||
@staticmethod
|
||||
def update_globals_with_sheerka_methods(my_locals, context):
|
||||
def update_globals_with_sheerka_methods(my_locals, context, expression_only):
|
||||
# methods_from_sheerka = {}
|
||||
#
|
||||
# # Make sure that methods that need the concept are correctly wrapped
|
||||
# for method_name, method in context.sheerka.sheerka_methods.items():
|
||||
# if expression_only and method.has_side_effect:
|
||||
# continue
|
||||
#
|
||||
# if method_name in context.sheerka.methods_with_context:
|
||||
# methods_from_sheerka[method_name] = inject_context(context)(method.method)
|
||||
# else:
|
||||
# methods_from_sheerka[method_name] = method.method
|
||||
methods_from_sheerka = {}
|
||||
|
||||
# Make sure that methods that need the concept are correctly wrapped
|
||||
for method_name, method in context.sheerka.sheerka_methods.items():
|
||||
if method_name in context.sheerka.methods_with_context:
|
||||
methods_from_sheerka[method_name] = inject_context(context)(method)
|
||||
else:
|
||||
methods_from_sheerka[method_name] = method
|
||||
|
||||
# Add all the methods as a direct access
|
||||
for method_name, method in methods_from_sheerka.items():
|
||||
my_locals[method_name] = method
|
||||
for method_name, method in context.sheerka.sheerka_methods.items():
|
||||
if expression_only and method.has_side_effect:
|
||||
continue
|
||||
|
||||
if method_name in context.sheerka.methods_with_context:
|
||||
my_locals[method_name] = inject_context(context)(method.method)
|
||||
else:
|
||||
my_locals[method_name] = method.method
|
||||
methods_from_sheerka[method_name] = my_locals[method_name]
|
||||
|
||||
# Add pipeable functions
|
||||
for func_name, function in context.sheerka.sheerka_pipeables.items():
|
||||
my_locals[func_name] = Pipe(function, context)
|
||||
if expression_only and function.has_side_effect:
|
||||
continue
|
||||
|
||||
my_locals[func_name] = Pipe(function.method, context)
|
||||
|
||||
return methods_from_sheerka # to allow access using prefix "sheerka."
|
||||
|
||||
def update_globals_with_context(self, my_globals, context):
|
||||
"""
|
||||
Update globals with the current object being evaluated (and its variables)
|
||||
:param my_globals:
|
||||
:param context:
|
||||
:return:
|
||||
"""
|
||||
if context.obj:
|
||||
context.log(f"Concept '{context.obj}' is in context. Adding it and its properties to locals.", self.name)
|
||||
|
||||
for prop_name in context.obj.variables():
|
||||
my_globals[prop_name] = context.obj.get_value(prop_name)
|
||||
value = context.obj.get_value(prop_name)
|
||||
if value != NotInit:
|
||||
my_globals[prop_name] = value
|
||||
my_globals["self"] = context.obj
|
||||
|
||||
def update_globals_with_node(self, my_globals, context, node, already_known):
|
||||
@@ -202,19 +251,11 @@ class PythonEvaluator(OneReturnValueEvaluator):
|
||||
context.log(f"Concept {name} is already evaluated.", self.name)
|
||||
else:
|
||||
context.log(f"Evaluating '{concept}'", self.name)
|
||||
with context.push(BuiltinConcepts.EVALUATE_CONCEPT,
|
||||
concept,
|
||||
who=self.name,
|
||||
desc=f"Evaluating '{concept}'",
|
||||
obj=concept) as sub_context:
|
||||
sub_context.local_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
|
||||
evaluated = context.sheerka.evaluate_concept(sub_context, concept)
|
||||
sub_context.add_values(return_values=evaluated)
|
||||
|
||||
if evaluated.key != concept.key:
|
||||
context.log(f"Error while evaluating '{name}'. Skipping.", self.name)
|
||||
continue
|
||||
concept = evaluated
|
||||
evaluated = context.sheerka.evaluate_concept(context, concept, eval_body=True)
|
||||
if evaluated.key != concept.key:
|
||||
context.log(f"Error while evaluating '{name}'. Skipping.", self.name)
|
||||
continue
|
||||
concept = evaluated
|
||||
|
||||
my_globals[name] = concept
|
||||
|
||||
@@ -293,7 +334,7 @@ class PythonEvaluator(OneReturnValueEvaluator):
|
||||
last_ast = copy.deepcopy(code_ast)
|
||||
last_ast.body = code_ast.body[-1:]
|
||||
|
||||
exec(compile(init_ast, "<ast>", "exec"), {}, my_locals)
|
||||
exec(compile(init_ast, "<ast>", "exec"), my_globals, my_locals)
|
||||
if type(last_ast.body[0]) == ast.Expr:
|
||||
return eval(compile(self.expr_to_expression(last_ast.body[0]), "<ast>", "eval"), my_globals, my_locals)
|
||||
else:
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
from core.builtin_concepts import BuiltinConcepts
|
||||
from core.builtin_helpers import resolve_ambiguity
|
||||
from core.concept import Concept
|
||||
from evaluators.BaseEvaluator import AllReturnValuesEvaluator
|
||||
|
||||
|
||||
class ResolveAmbiguityEvaluator(AllReturnValuesEvaluator):
|
||||
"""
|
||||
Use when multiple concepts match the same input source
|
||||
"""
|
||||
|
||||
NAME = "ResolveAmbiguity"
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(self.NAME, [BuiltinConcepts.AFTER_PARSING], 50)
|
||||
self.sources = None
|
||||
|
||||
def matches(self, context, return_values):
|
||||
# first, arrange return_values by sources.
|
||||
# If they share the same source, that means that there are multiple results for one ParserInput
|
||||
self.sources = {}
|
||||
success = False
|
||||
for ret in [ret for ret in return_values if ret.status]:
|
||||
source = self.get_source(context, ret)
|
||||
|
||||
if source:
|
||||
self.sources.setdefault(source, []).append(ret)
|
||||
|
||||
if len(self.sources[source]) > 1:
|
||||
success = True # at least one source are more than one entry -> let's resolve it
|
||||
|
||||
return success
|
||||
|
||||
def eval(self, context, return_values):
|
||||
ret = []
|
||||
|
||||
for ret_vals in self.sources.values():
|
||||
parser_results = {id(r.body.body): r.body for r in ret_vals}
|
||||
resolved = resolve_ambiguity(context, [r.body.body for r in ret_vals])
|
||||
if len(resolved) == 0:
|
||||
ret.append(context.sheerka.ret(self.name, True, [], parents=ret_vals))
|
||||
else:
|
||||
for c in resolved:
|
||||
ret.append(context.sheerka.ret(self.name, True, parser_results[id(c)], parents=ret_vals))
|
||||
|
||||
return ret
|
||||
|
||||
@staticmethod
|
||||
def get_source(context, return_value):
|
||||
"""
|
||||
Try to get the source (the ParserInput) of the return_value
|
||||
We only consider parser result of concepts
|
||||
:param context:
|
||||
:param return_value:
|
||||
:return:
|
||||
"""
|
||||
if context.sheerka.isinstance(return_value.body, BuiltinConcepts.PARSER_RESULT) and \
|
||||
isinstance(return_value.body.body, Concept):
|
||||
return return_value.body.source
|
||||
return None
|
||||
@@ -6,7 +6,7 @@ from evaluators.BaseEvaluator import OneReturnValueEvaluator
|
||||
|
||||
class RetEvaluator(OneReturnValueEvaluator):
|
||||
"""
|
||||
The evaluator transform the a concept, using the ret value
|
||||
The evaluator transforms the concept, using the RET metadata value
|
||||
"""
|
||||
|
||||
NAME = "Ret"
|
||||
|
||||
Reference in New Issue
Block a user