Added first version of DebugManager. Implemented draft of the rule engine
This commit is contained in:
+78
-167
@@ -1,21 +1,18 @@
|
||||
import ast
|
||||
import logging
|
||||
|
||||
import core.ast.nodes
|
||||
from core.ast.nodes import CallNodeConcept
|
||||
from core.ast.visitors import UnreferencedNamesVisitor
|
||||
from core.builtin_concepts import BuiltinConcepts
|
||||
from core.concept import Concept, NotInit, ConceptParts
|
||||
from core.concept import Concept, NotInit, ConceptParts, DEFINITION_TYPE_BNF, concept_part_value
|
||||
from core.rule import Rule
|
||||
from core.sheerka.services.SheerkaExecute import SheerkaExecute
|
||||
from core.tokenizer import Keywords
|
||||
# from evaluators.BaseEvaluator import BaseEvaluator
|
||||
from parsers.BaseNodeParser import SourceCodeNode, ConceptNode, UnrecognizedTokensNode, SourceCodeWithConceptNode
|
||||
from parsers.BaseNodeParser import SourceCodeNode, ConceptNode, UnrecognizedTokensNode, SourceCodeWithConceptNode, \
|
||||
RuleNode
|
||||
from parsers.BaseParser import BaseParser, ErrorNode
|
||||
|
||||
PARSE_STEPS = [BuiltinConcepts.BEFORE_PARSING, BuiltinConcepts.PARSING, BuiltinConcepts.AFTER_PARSING]
|
||||
EVAL_STEPS = PARSE_STEPS + [BuiltinConcepts.BEFORE_EVALUATION, BuiltinConcepts.EVALUATION,
|
||||
BuiltinConcepts.AFTER_EVALUATION]
|
||||
PARSERS = ["EmptyString", "ShortTermMemory", "AtomNode", "BnfNode", "SyaNode", "Python"]
|
||||
PARSERS = ["EmptyString", "ShortTermMemory", "Sequence", "Bnf", "Sya", "Python"]
|
||||
|
||||
|
||||
def is_same_success(context, return_values):
|
||||
@@ -31,7 +28,7 @@ def is_same_success(context, return_values):
|
||||
if not ret_val.status:
|
||||
raise Exception("Status is false")
|
||||
|
||||
if isinstance(ret_val.body, Concept) and not ret_val.body.metadata.is_evaluated:
|
||||
if isinstance(ret_val.body, Concept) and not ret_val.body.get_metadata().is_evaluated:
|
||||
raise Exception("Concept is not evaluated")
|
||||
|
||||
return context.sheerka.objvalue(ret_val)
|
||||
@@ -185,7 +182,7 @@ def resolve_ambiguity(context, concepts):
|
||||
# the concept matches the context
|
||||
by_complexity = {}
|
||||
for c in concepts:
|
||||
by_complexity.setdefault(get_condition_complexity(c, "pre"), []).append(c)
|
||||
by_complexity.setdefault(get_condition_complexity(c, concept_part_value(ConceptParts.PRE)), []).append(c)
|
||||
|
||||
remaining_concepts = []
|
||||
for complexity in sorted(by_complexity.keys(), reverse=True):
|
||||
@@ -193,7 +190,7 @@ def resolve_ambiguity(context, concepts):
|
||||
remaining_concepts.extend(by_complexity[complexity])
|
||||
else:
|
||||
for c in by_complexity[complexity]:
|
||||
evaluated = context.sheerka.evaluate_concept(context, c, metadata=["pre"])
|
||||
evaluated = context.sheerka.evaluate_concept(context, c, metadata=[ConceptParts.PRE])
|
||||
if context.sheerka.is_success(evaluated) or evaluated.key == c.key:
|
||||
remaining_concepts.append(c)
|
||||
|
||||
@@ -208,21 +205,21 @@ def resolve_ambiguity(context, concepts):
|
||||
# when the input is "hello world"
|
||||
by_number_of_vars = {}
|
||||
for c in remaining_concepts:
|
||||
by_number_of_vars.setdefault(len(c.metadata.variables), []).append(c)
|
||||
by_number_of_vars.setdefault(len(c.get_metadata().variables), []).append(c)
|
||||
|
||||
return by_number_of_vars[min(by_number_of_vars.keys())]
|
||||
|
||||
|
||||
def get_condition_complexity(concept, concept_part_str):
|
||||
"""
|
||||
Need to find a proper algorithm to compute the complexity of a concept
|
||||
So far, the concept is considered as complex if it has pre
|
||||
Need to find a proper algorithm to compute the complexity of a concept metadata
|
||||
So far, the concept is considered as complex if it has concept_part_str (so far with concept_part_str='pre')
|
||||
:param concept:
|
||||
:param concept_part_str:
|
||||
:return:
|
||||
"""
|
||||
concept_part_value = getattr(concept.metadata, concept_part_str)
|
||||
if concept_part_value is None or concept_part_value.strip() == 0:
|
||||
value = getattr(concept.get_metadata(), concept_part_str)
|
||||
if value is None or value.strip() == 0:
|
||||
return 0
|
||||
|
||||
return 1 # no real computing as of now
|
||||
@@ -270,7 +267,8 @@ def only_parsers_results(context, return_values):
|
||||
return sheerka.ret(
|
||||
context.who,
|
||||
False,
|
||||
sheerka.new(BuiltinConcepts.TOO_MANY_ERRORS, body=return_values),
|
||||
sheerka.new(BuiltinConcepts.TOO_MANY_ERRORS if len(return_values) > 1 else BuiltinConcepts.ERROR,
|
||||
body=return_values),
|
||||
parents=return_values)
|
||||
|
||||
return sheerka.ret(
|
||||
@@ -291,7 +289,7 @@ def parse_unrecognized(context, source, parsers, who=None, prop=None, filter_fun
|
||||
:param parsers:
|
||||
:param who: who is asking the parsing ?
|
||||
:param prop: Extra info, when parsing a property
|
||||
:param filter_func: filter function to call is provided
|
||||
:param filter_func: Once the result are found, call this function to filter them
|
||||
:return:
|
||||
"""
|
||||
sheerka = context.sheerka
|
||||
@@ -306,11 +304,12 @@ def parse_unrecognized(context, source, parsers, who=None, prop=None, filter_fun
|
||||
with context.push(BuiltinConcepts.PARSING, action_context, who=who, desc=desc) as sub_context:
|
||||
# disable all parsers but the requested ones
|
||||
if parsers != "all":
|
||||
sub_context.add_preprocess(BaseParser.PREFIX + "*", enabled=False)
|
||||
for parser in parsers:
|
||||
sub_context.add_preprocess(BaseParser.PREFIX + parser, enabled=True)
|
||||
sub_context.preprocess_parsers = [BaseParser.PREFIX + parser for parser in parsers]
|
||||
# sub_context.add_preprocess(BaseParser.PREFIX + "*", enabled=False)
|
||||
# for parser in parsers:
|
||||
# sub_context.add_preprocess(BaseParser.PREFIX + parser, enabled=True)
|
||||
|
||||
if prop in (Keywords.WHERE, Keywords.PRE, ConceptParts.WHERE, ConceptParts.PRE):
|
||||
if prop in (Keywords.WHERE, Keywords.PRE, ConceptParts.WHERE, ConceptParts.PRE, Keywords.WHEN):
|
||||
sub_context.protected_hints.add(BuiltinConcepts.EVAL_QUESTION_REQUESTED)
|
||||
|
||||
sub_context.add_inputs(source=source)
|
||||
@@ -323,25 +322,7 @@ def parse_unrecognized(context, source, parsers, who=None, prop=None, filter_fun
|
||||
res = filter_func(sub_context, res)
|
||||
|
||||
sub_context.add_values(return_values=res)
|
||||
if not hasattr(res, "__iter__"):
|
||||
return res
|
||||
|
||||
# discard Python response if accepted by AtomNode
|
||||
is_concept = False
|
||||
for r in res:
|
||||
if r.status and r.who == "parsers.AtomNode":
|
||||
is_concept = True
|
||||
|
||||
if not is_concept:
|
||||
return res
|
||||
|
||||
no_python = []
|
||||
for r in res:
|
||||
if r.who == "parsers.Python":
|
||||
continue
|
||||
no_python.append(r)
|
||||
|
||||
return no_python
|
||||
return res
|
||||
|
||||
|
||||
def parse_function(context, source, tokens=None, start=0):
|
||||
@@ -465,7 +446,7 @@ def get_lexer_nodes(return_values, start, tokens):
|
||||
for concept in concepts:
|
||||
lexer_nodes.append([ConceptNode(concept, start, end, tokens, ret_val.body.source)])
|
||||
|
||||
elif ret_val.who in ("parsers.BnfNode", "parsers.SyaNode", "parsers.AtomNode"):
|
||||
elif ret_val.who in ("parsers.Bnf", "parsers.Sya", "parsers.Sequence"):
|
||||
nodes = [node for node in ret_val.body.body]
|
||||
for node in nodes:
|
||||
node.start += start
|
||||
@@ -474,6 +455,12 @@ def get_lexer_nodes(return_values, start, tokens):
|
||||
# but append the whole sequence if when it's a sequence
|
||||
lexer_nodes.append(nodes)
|
||||
|
||||
elif ret_val.who == "parsers.Rule":
|
||||
rules = ret_val.body.body if hasattr(ret_val.body.body, "__iter__") else [ret_val.body.body]
|
||||
end = start + len(tokens) - 1
|
||||
for rule in rules:
|
||||
lexer_nodes.append([RuleNode(rule, start, end, tokens, ret_val.body.source)])
|
||||
|
||||
else:
|
||||
raise NotImplementedError()
|
||||
|
||||
@@ -488,16 +475,16 @@ def ensure_evaluated(context, concept, eval_body=True):
|
||||
:param eval_body:
|
||||
:return:
|
||||
"""
|
||||
if concept.metadata.is_evaluated:
|
||||
if concept.get_metadata().is_evaluated:
|
||||
return concept
|
||||
|
||||
# do not try to evaluate concept that are not fully initialized
|
||||
for var in concept.metadata.variables:
|
||||
# to code
|
||||
if var[1] is None and \
|
||||
var[0] not in concept.compiled and \
|
||||
(var[0] not in concept.values or concept.get_value(var[0]) == NotInit):
|
||||
return concept
|
||||
if concept.get_metadata().definition_type != DEFINITION_TYPE_BNF:
|
||||
for var in concept.get_metadata().variables:
|
||||
if var[1] is None and \
|
||||
var[0] not in concept.get_compiled() and \
|
||||
(var[0] not in concept.values() or concept.get_value(var[0]) == NotInit):
|
||||
return concept
|
||||
|
||||
evaluated = context.sheerka.evaluate_concept(context, concept, eval_body=eval_body)
|
||||
return evaluated
|
||||
@@ -523,7 +510,7 @@ def get_lexer_nodes_from_unrecognized(context, unrecognized_tokens_node, parsers
|
||||
|
||||
def update_compiled(context, concept, errors, parsers=None):
|
||||
"""
|
||||
recursively iterate thru concept.compiled to replace LexerNode into concepts or list of ReturnValueConcept
|
||||
recursively iterate thru concept.get_compiled() to replace LexerNode into concepts or list of ReturnValueConcept
|
||||
When parsing using a LexerNodeParser (SyaNodeParser, BnfNodeParser...)
|
||||
the result will be a LexerNode.
|
||||
In the specific case of a ConceptNode, the compiled variables will also be LexerNode (UnrecognizedTokensNode...)
|
||||
@@ -534,7 +521,6 @@ def update_compiled(context, concept, errors, parsers=None):
|
||||
:param parsers: to customize the parsers to use
|
||||
:return:
|
||||
"""
|
||||
|
||||
sheerka = context.sheerka
|
||||
parsers = parsers or PARSERS
|
||||
|
||||
@@ -544,7 +530,7 @@ def update_compiled(context, concept, errors, parsers=None):
|
||||
:param c:
|
||||
:return:
|
||||
"""
|
||||
for k, v in c.compiled.items():
|
||||
for k, v in c.get_compiled().items():
|
||||
if isinstance(v, Concept):
|
||||
_validate_concept(v)
|
||||
|
||||
@@ -553,7 +539,7 @@ def update_compiled(context, concept, errors, parsers=None):
|
||||
parser_helper = PythonWithConceptsParser()
|
||||
res = parser_helper.parse_nodes(context, v.get_all_nodes())
|
||||
if res.status:
|
||||
c.compiled[k] = [res]
|
||||
c.get_compiled()[k] = [res]
|
||||
else:
|
||||
errors.append(sheerka.new(BuiltinConcepts.ERROR, body=f"Cannot parse '{v.source}'"))
|
||||
|
||||
@@ -561,7 +547,7 @@ def update_compiled(context, concept, errors, parsers=None):
|
||||
res = parse_unrecognized(context, v.source, parsers)
|
||||
res = only_successful(context, res) # only key successful parsers
|
||||
if res.status:
|
||||
c.compiled[k] = res.body.body
|
||||
c.get_compiled()[k] = res.body.body
|
||||
else:
|
||||
errors.append(sheerka.new(BuiltinConcepts.ERROR, body=f"Cannot parse '{v.source}'"))
|
||||
|
||||
@@ -588,117 +574,12 @@ def update_compiled(context, concept, errors, parsers=None):
|
||||
# and the user has entered 'a plus b'
|
||||
# Chances are that we are talking about the concept itself, and not an instantiation (like '10 plus 2')
|
||||
# This means that 'a' and 'b' don't have any real value
|
||||
if len(concept.metadata.variables) > 0:
|
||||
for name, value in concept.metadata.variables:
|
||||
if _get_source(concept.compiled, name) != name:
|
||||
if len(concept.get_metadata().variables) > 0:
|
||||
for name, value in concept.get_metadata().variables:
|
||||
if _get_source(concept.get_compiled(), name) != name:
|
||||
break
|
||||
else:
|
||||
concept.metadata.is_evaluated = True
|
||||
|
||||
|
||||
def get_names(sheerka, concept_node):
|
||||
"""
|
||||
Finds all the names referenced by the concept_node
|
||||
:param sheerka:
|
||||
:param concept_node:
|
||||
:return:
|
||||
"""
|
||||
unreferenced_names_visitor = UnreferencedNamesVisitor(sheerka)
|
||||
unreferenced_names_visitor.visit(concept_node)
|
||||
return list(unreferenced_names_visitor.names)
|
||||
|
||||
|
||||
def extract_predicates(sheerka, expression, variables_to_include, variables_to_exclude):
|
||||
"""
|
||||
from a given expression and a variable (or list of variables)
|
||||
tries to find out all the predicates referencing the(se) variable(s), and the(se) variable(s) solely
|
||||
for example
|
||||
exp : isinstance(a, int) and isinstance(b, str)
|
||||
will return 'isinstance(a, int)' if variable_name == 'a'
|
||||
:param sheerka:
|
||||
:param expression:
|
||||
:param variables_to_include:
|
||||
:param variables_to_exclude:
|
||||
:return: list of predicates
|
||||
"""
|
||||
|
||||
if len(variables_to_include) == 0:
|
||||
return []
|
||||
|
||||
def _get_predicates(_nodes):
|
||||
_predicates = []
|
||||
for _node in _nodes:
|
||||
python_node = ast.Expression(body=core.ast.nodes.concept_to_python(_node))
|
||||
python_node = ast.fix_missing_locations(python_node)
|
||||
_predicates.append(python_node)
|
||||
return _predicates
|
||||
|
||||
if isinstance(expression, str):
|
||||
node = ast.parse(expression, mode="eval")
|
||||
else:
|
||||
return NotImplementedError()
|
||||
|
||||
concept_node = core.ast.nodes.python_to_concept(node)
|
||||
main_op = concept_node.get_value("body")
|
||||
|
||||
return _get_predicates(_extract_predicates(sheerka, main_op, variables_to_include, variables_to_exclude))
|
||||
|
||||
|
||||
def _extract_predicates(sheerka, node, variables_to_include, variables_to_exclude):
|
||||
predicates = []
|
||||
|
||||
def _matches(_names, to_include, to_exclude):
|
||||
_res = None
|
||||
for n in _names:
|
||||
if n in to_include and _res is None:
|
||||
_res = True
|
||||
if n in to_exclude:
|
||||
_res = False
|
||||
return _res
|
||||
|
||||
if node.node_type == "Compare":
|
||||
if node.get_value("left").node_type == "Name":
|
||||
"""Simple case of one comparison"""
|
||||
comparison_name = sheerka.objvalue(node.get_value("left"))
|
||||
if comparison_name in variables_to_include and comparison_name not in variables_to_exclude:
|
||||
predicates.append(node)
|
||||
else:
|
||||
"""The left part is an expression"""
|
||||
res = _extract_predicates(sheerka, node.get_value("left"), variables_to_include, variables_to_exclude)
|
||||
if len(res) > 0:
|
||||
predicates.append(node)
|
||||
elif node.node_type == "Call":
|
||||
"""Simple case predicate"""
|
||||
call_node = node if isinstance(node, CallNodeConcept) else CallNodeConcept().update_from(node)
|
||||
args = list(call_node.get_args_names(sheerka))
|
||||
if _matches(args, variables_to_include, variables_to_exclude):
|
||||
predicates.append(node)
|
||||
elif node.node_type == "UnaryOp" and node.get_value("op").node_type == "Not":
|
||||
"""Simple case of negation"""
|
||||
res = _extract_predicates(sheerka, node.get_value("operand"), variables_to_include, variables_to_exclude)
|
||||
if len(res) > 0:
|
||||
predicates.append(node)
|
||||
elif node.node_type == "BinOp":
|
||||
names = get_names(sheerka, node)
|
||||
if _matches(names, variables_to_include, variables_to_exclude):
|
||||
predicates.append(node)
|
||||
elif node.node_type == "BoolOp":
|
||||
all_op = True
|
||||
temp_res = []
|
||||
for op in node.get_value("values").body:
|
||||
res = _extract_predicates(sheerka, op, variables_to_include, variables_to_exclude)
|
||||
if len(res) == 0:
|
||||
all_op = False
|
||||
else:
|
||||
temp_res.extend(res)
|
||||
|
||||
if all_op:
|
||||
predicates.append(node)
|
||||
else:
|
||||
for res in temp_res:
|
||||
predicates.append(res)
|
||||
|
||||
return predicates
|
||||
concept.get_metadata().is_evaluated = True
|
||||
|
||||
|
||||
def add_to_ret_val(sheerka, context, return_values, concept_key):
|
||||
@@ -732,8 +613,38 @@ def set_is_evaluated(concepts, check_nb_variables=False):
|
||||
|
||||
if hasattr(concepts, "__iter__"):
|
||||
for c in concepts:
|
||||
if not check_nb_variables or check_nb_variables and len(c.metadata.variables) > 0:
|
||||
c.metadata.is_evaluated = True
|
||||
if not check_nb_variables or check_nb_variables and len(c.get_metadata().variables) > 0:
|
||||
c.get_metadata().is_evaluated = True
|
||||
else:
|
||||
if not check_nb_variables or check_nb_variables and len(concepts.metadata.variables) > 0:
|
||||
concepts.metadata.is_evaluated = True
|
||||
if not check_nb_variables or check_nb_variables and len(concepts.get_metadata().variables) > 0:
|
||||
concepts.get_metadata().is_evaluated = True
|
||||
|
||||
|
||||
def ensure_concept(*concepts):
|
||||
if hasattr(concepts, "__iter__"):
|
||||
for concept in concepts:
|
||||
if not isinstance(concept, Concept):
|
||||
raise TypeError(f"'{concept}' must be a concept")
|
||||
else:
|
||||
if not isinstance(concepts, Concept):
|
||||
raise TypeError(f"'{concepts}' must be a concept")
|
||||
|
||||
|
||||
def ensure_rule(*rules):
|
||||
if hasattr(rules, "__iter__"):
|
||||
for rule in rules:
|
||||
if not isinstance(rule, Rule):
|
||||
raise TypeError(f"'{rule}' must be a rule")
|
||||
else:
|
||||
if not isinstance(rules, Rule):
|
||||
raise TypeError(f"'{rules}' must be a rule")
|
||||
|
||||
|
||||
def ensure_concept_or_rule(*items):
|
||||
if hasattr(items, "__iter__"):
|
||||
for item in items:
|
||||
if not isinstance(item, (Concept, Rule)):
|
||||
raise TypeError(f"'{item}' must be a concept or rule")
|
||||
else:
|
||||
if not isinstance(items, (Concept, Rule)):
|
||||
raise TypeError(f"'{items}' must be a concept or rule")
|
||||
|
||||
Reference in New Issue
Block a user