Fixed #109 : Mix python and concept. List comprehension
Fixed #110 : SheerkaDebugManager: add list_debug_settings Fixed #111 : SheerkaDebugManager: Implement ListDebugLogger Fixed #112 : SyaNodeParser: rewrite this parser Fixed #113 : Sheerka.: Add enable_parser_caching to disable parsers caching Fixed #114 : SyaNodeParser : Implement fast cache to resolve unrecognized tokens requests Fixed #115 : BnfNodeParser : Implement fast cache to resolve unrecognized tokens requests Fixed #116 : SequenceNodeParser : Implement fast cache to resolve unrecognized tokens requests Fixed #117 : ResolveMultiplePluralAmbiguityEvaluator: Resolve Multiple plural ambiguity
This commit is contained in:
@@ -0,0 +1,389 @@
|
||||
from dataclasses import dataclass
|
||||
from itertools import product
|
||||
from typing import Union
|
||||
|
||||
from core.builtin_helpers import is_only_successful, only_successful
|
||||
from core.global_symbols import INIT_AST_PARSERS, NotInit
|
||||
from core.sheerka.services.SheerkaEvaluateConcept import EvaluationHints
|
||||
from core.tokenizer import TokenKind
|
||||
from core.utils import merge_dicts, merge_sets
|
||||
from parsers.BaseExpressionParser import AndNode, ComparisonNode, ComparisonType, ExprNode, ExpressionVisitorWithHint, \
|
||||
FunctionNode, ListComprehensionNode, \
|
||||
ListNode, NameExprNode, NotNode, VariableNode, end_parenthesis_mapping, open_parenthesis_mapping
|
||||
from parsers.PythonParser import PythonNode
|
||||
from sheerkapython.python_wrapper import sheerka_globals
|
||||
|
||||
|
||||
@dataclass()
|
||||
class PythonExprVisitorObj:
|
||||
text: Union[str, None] # human readable
|
||||
source: Union[str, None] # python expression to compile
|
||||
objects: dict # dictionaries of object created during the visit
|
||||
variables: set # I intended to detect unbound symbols, but it's actually not used
|
||||
|
||||
|
||||
class PythonExprVisitor(ExpressionVisitorWithHint):
|
||||
def __init__(self, context, obj_counter=0):
|
||||
self.context = context
|
||||
self.obj_counter = obj_counter
|
||||
self.objects_by_id = {}
|
||||
self.objects_by_name = {}
|
||||
self.errors = {}
|
||||
self.results = []
|
||||
|
||||
def compile(self, expr_node, hint=None):
|
||||
hint = hint or EvaluationHints(eval_body=True)
|
||||
visitor_objects = self.visit(expr_node, hint)
|
||||
|
||||
for obj in visitor_objects:
|
||||
|
||||
ret = self.context.sheerka.parse_python(self.context, obj.source)
|
||||
if ret.status:
|
||||
ret.body.body.original_source = obj.text
|
||||
ret.body.body.objects = obj.objects
|
||||
self.results.append(ret)
|
||||
else:
|
||||
self.errors[obj.text] = self.context.sheerka.get_error_cause(ret.body)
|
||||
|
||||
return self.results
|
||||
|
||||
def visit_ListComprehensionNode(self, expr_node: ListComprehensionNode, hint: EvaluationHints):
|
||||
"""
|
||||
|
||||
:param expr_node:
|
||||
:param hint:
|
||||
:return:
|
||||
"""
|
||||
visitor_objects = []
|
||||
source = expr_node.get_source()
|
||||
|
||||
not_a_question_hint = EvaluationHints(eval_body=True, eval_question=False)
|
||||
is_a_question_hint = EvaluationHints(eval_body=True, eval_question=True)
|
||||
product_inputs = []
|
||||
|
||||
# add parenthesis around the element if needed
|
||||
# test case test_ExprToPython.test_i_can_compile_when_element_is_missing_its_parenthesis()
|
||||
if expr_node.element.first is None and len(expr_node.element.items) > 1:
|
||||
expr_node.element.first = NameExprNode(-1, -1, [open_parenthesis_mapping[TokenKind.LPAR]])
|
||||
expr_node.element.last = NameExprNode(-1, -1, [end_parenthesis_mapping[TokenKind.LPAR]])
|
||||
|
||||
element_objs = self.visit(expr_node.element, not_a_question_hint)
|
||||
product_inputs.append(element_objs)
|
||||
for comp in expr_node.generators:
|
||||
target_objs = self.visit(comp.target, not_a_question_hint)
|
||||
iter_objs = self.visit(comp.iterable, not_a_question_hint)
|
||||
if comp.if_expr:
|
||||
# parse it using PythonConditionExprVisitor
|
||||
res = self.context.sheerka.parse_expression(self.context, comp.if_expr.get_source())
|
||||
if not res.status:
|
||||
self.errors[comp.if_expr.get_source()] = res.body
|
||||
return None
|
||||
if_expr_objs = self.visit(res.body.body, is_a_question_hint)
|
||||
else:
|
||||
if_expr_objs = [None]
|
||||
|
||||
product_inputs.extend([target_objs, iter_objs, if_expr_objs])
|
||||
|
||||
for items in product(*product_inputs):
|
||||
visitor_objects.append(self.create_list_comprehension(source, *items))
|
||||
|
||||
return visitor_objects
|
||||
|
||||
def visit_VariableNode(self, expr_node: VariableNode, hint: EvaluationHints):
|
||||
source = expr_node.get_source()
|
||||
return self.parse_source_code(source, hint)
|
||||
|
||||
def visit_NameExprNode(self, expr_node: NameExprNode, hint: EvaluationHints):
|
||||
"""
|
||||
create visitor objects from NameExprNode
|
||||
:param expr_node:
|
||||
:param hint:
|
||||
:return:
|
||||
"""
|
||||
source = expr_node.get_source()
|
||||
return self.parse_source_code(source, hint)
|
||||
|
||||
def visit_ListNode(self, expr_node: ListNode, hint: EvaluationHints):
|
||||
visitor_objects = []
|
||||
source = expr_node.get_source()
|
||||
|
||||
items_objs = []
|
||||
for item in expr_node.items:
|
||||
items_objs.append(self.visit(item, hint))
|
||||
|
||||
for items in product(*items_objs):
|
||||
visitor_objects.append(self.create_list(source,
|
||||
expr_node.first.get_source() if expr_node.first else None,
|
||||
expr_node.last.get_source() if expr_node.last else None,
|
||||
items,
|
||||
expr_node.sep))
|
||||
return visitor_objects
|
||||
|
||||
def visit_AndNode(self, expr_node: AndNode, hint: EvaluationHints):
|
||||
"""
|
||||
|
||||
:param expr_node:
|
||||
:param hint:
|
||||
:return:
|
||||
"""
|
||||
return self.visit_or_or_and_node("and", expr_node, hint)
|
||||
|
||||
def visit_OrNode(self, expr_node: AndNode, hint: EvaluationHints):
|
||||
"""
|
||||
|
||||
:param expr_node:
|
||||
:param hint:
|
||||
:return:
|
||||
"""
|
||||
return self.visit_or_or_and_node("or", expr_node, hint)
|
||||
|
||||
def visit_NotNode(self, expr_node: NotNode, hint: EvaluationHints):
|
||||
"""
|
||||
|
||||
:param expr_node:
|
||||
:param hint:
|
||||
:return:
|
||||
"""
|
||||
visitor_objects = []
|
||||
|
||||
source = expr_node.get_source()
|
||||
objs = self.visit(expr_node.node, hint)
|
||||
|
||||
for obj in objs:
|
||||
visitor_objects.append(self.create_not(source, obj))
|
||||
|
||||
return visitor_objects
|
||||
|
||||
def visit_ComparisonNode(self, expr_node: ComparisonNode, hint: EvaluationHints):
|
||||
"""
|
||||
|
||||
:param expr_node:
|
||||
:param hint:
|
||||
:return:
|
||||
"""
|
||||
visitor_objects = []
|
||||
source = expr_node.get_source()
|
||||
|
||||
left = self.visit(expr_node.left, hint)
|
||||
right = self.visit(expr_node.right, hint)
|
||||
|
||||
for left_obj, right_obj in product(left, right):
|
||||
visitor_objects.append(self.create_comparison(source, expr_node.comp, left_obj, right_obj))
|
||||
|
||||
return visitor_objects
|
||||
|
||||
def visit_FunctionNode(self, expr_node: FunctionNode, hint: EvaluationHints):
|
||||
visitor_objects = []
|
||||
source = expr_node.get_source()
|
||||
|
||||
parameters_objects = []
|
||||
for parameter in expr_node.parameters:
|
||||
parameters_objects.append(self.visit(parameter.value, hint))
|
||||
|
||||
for parameters in product(*parameters_objects):
|
||||
visitor_objects.append(self.create_function(source,
|
||||
expr_node.first.get_source(),
|
||||
expr_node.last.get_source(),
|
||||
parameters))
|
||||
|
||||
return visitor_objects
|
||||
|
||||
def visit_or_or_and_node(self, node_type, expr_node: ExprNode, hint: EvaluationHints):
|
||||
"""
|
||||
|
||||
:param node_type:
|
||||
:param expr_node:
|
||||
:param hint:
|
||||
:return:
|
||||
"""
|
||||
visitor_objects = []
|
||||
|
||||
source = expr_node.get_source()
|
||||
objs = []
|
||||
for node in expr_node.parts:
|
||||
objs.append(self.visit(node, hint))
|
||||
|
||||
for objs_parts in product(*objs):
|
||||
visitor_objects.append(self.create_and_or(node_type, source, objs_parts))
|
||||
|
||||
return visitor_objects
|
||||
|
||||
def parse_source_code(self, source, hint):
|
||||
res = self.context.sheerka.parse_unrecognized(self.context,
|
||||
source,
|
||||
INIT_AST_PARSERS,
|
||||
filter_func=only_successful,
|
||||
is_question=hint.eval_question)
|
||||
|
||||
return_values = res.body.body if is_only_successful(self.context.sheerka, res) else [res]
|
||||
|
||||
visitor_objects = []
|
||||
for ret_val in return_values:
|
||||
if not ret_val.status:
|
||||
self.errors[source] = ret_val.body
|
||||
return
|
||||
|
||||
if isinstance(ret_val.body.body, list):
|
||||
if len(ret_val.body.body) > 1:
|
||||
raise NotImplementedError("Too many concept found. Not handled yet !")
|
||||
body = ret_val.body.body[0]
|
||||
else:
|
||||
body = ret_val.body.body
|
||||
|
||||
if hasattr(body, "get_concept"):
|
||||
visitor_objects.append(self.create_call_concept(source, body.get_concept(), hint.eval_question))
|
||||
elif hasattr(body, "get_python_node"):
|
||||
visitor_objects.append(self.create_source_code_from_python_node(body.get_python_node()))
|
||||
else:
|
||||
raise NotImplementedError(f"{body=}. Not yet implemented")
|
||||
|
||||
return visitor_objects
|
||||
|
||||
@staticmethod
|
||||
def create_source_code_from_python_node(node: PythonNode):
|
||||
return PythonExprVisitorObj(text=node.original_source,
|
||||
source=node.source,
|
||||
objects=node.objects,
|
||||
variables=set())
|
||||
|
||||
@staticmethod
|
||||
def create_list_comprehension(text, *items):
|
||||
objects = {}
|
||||
variables = set()
|
||||
|
||||
def update_objects_and_variables(*objs):
|
||||
for obj in [obj for obj in objs if obj]:
|
||||
objects.update(obj.objects)
|
||||
variables.update(obj.variables)
|
||||
|
||||
items = list(items)
|
||||
|
||||
element = items.pop(0)
|
||||
update_objects_and_variables(element)
|
||||
source = f"[ {element.source}"
|
||||
|
||||
while len(items):
|
||||
target = items.pop(0)
|
||||
iterable = items.pop(0)
|
||||
if_expr = items.pop(0)
|
||||
update_objects_and_variables(target, iterable, if_expr)
|
||||
source += f" for {target.source} in {iterable.source}"
|
||||
if if_expr:
|
||||
source += f" if {if_expr.source}"
|
||||
source += " ]"
|
||||
|
||||
return PythonExprVisitorObj(text=text,
|
||||
source=source,
|
||||
objects=objects,
|
||||
variables=variables)
|
||||
|
||||
@staticmethod
|
||||
def create_and_or(node_type, text, parts):
|
||||
return PythonExprVisitorObj(text=text,
|
||||
source=f" {node_type} ".join([p.source for p in parts]),
|
||||
objects=merge_dicts(*[p.objects for p in parts]),
|
||||
variables=merge_sets(*[p.variables for p in parts]))
|
||||
|
||||
@staticmethod
|
||||
def create_comparison(text, op, left_obj, right_obj):
|
||||
def get_source(_op, a, b):
|
||||
if _op == ComparisonType.EQUALS and b == "sheerka":
|
||||
return f"is_sheerka({a})"
|
||||
else:
|
||||
return ComparisonNode.rebuild_source(a, op, b)
|
||||
|
||||
return PythonExprVisitorObj(text=text,
|
||||
source=get_source(op, left_obj.source, right_obj.source),
|
||||
objects=merge_dicts(left_obj.objects, right_obj.objects),
|
||||
variables=merge_sets(left_obj.variables, right_obj.variables))
|
||||
|
||||
@staticmethod
|
||||
def create_not(text, node):
|
||||
return PythonExprVisitorObj(text=text,
|
||||
source=f"not {node.source}",
|
||||
objects=node.objects,
|
||||
variables=node.variables)
|
||||
|
||||
@staticmethod
|
||||
def create_function(text, first, last, parameters):
|
||||
def get_source(_first, _last, _parameters):
|
||||
return f"{_first}{', '.join(p for p in _parameters)}{_last}"
|
||||
|
||||
return PythonExprVisitorObj(text=text,
|
||||
source=get_source(first, last, [p.source for p in parameters]),
|
||||
objects=merge_dicts(*[p.objects for p in parameters]),
|
||||
variables=merge_sets(*[p.variables for p in parameters]))
|
||||
|
||||
@staticmethod
|
||||
def create_list(text, first, last, items, sep):
|
||||
def get_source(_first, _last, _items, _sep):
|
||||
res = _first or ""
|
||||
res += f"{_sep.value} ".join(item for item in _items)
|
||||
if _last:
|
||||
res += _last
|
||||
return res
|
||||
|
||||
return PythonExprVisitorObj(text=text,
|
||||
source=get_source(first, last, [p.source for p in items], sep),
|
||||
objects=merge_dicts(*[p.objects for p in items]),
|
||||
variables=merge_sets(*[p.variables for p in items]))
|
||||
|
||||
def get_object_name(self, obj):
|
||||
"""
|
||||
object found during the parsing are not serialized
|
||||
They are kept in a dictionary.
|
||||
This function returns a new name for every new object
|
||||
:param obj: object for which a name is to be created
|
||||
:param objects: already created names (it's a dictionary)
|
||||
:return: tuple(name created, dictionary of already created names)
|
||||
"""
|
||||
|
||||
if self.context.sheerka.is_sheerka(obj):
|
||||
return "sheerka"
|
||||
|
||||
try:
|
||||
return self.objects_by_id[id(obj)]
|
||||
except KeyError:
|
||||
pass
|
||||
|
||||
object_name = f"__o_{self.obj_counter:02}__"
|
||||
self.obj_counter += 1
|
||||
|
||||
self.objects_by_id[id(obj)] = object_name
|
||||
self.objects_by_name[object_name] = obj
|
||||
return object_name
|
||||
|
||||
def create_call_concept(self, source, concept, is_question):
|
||||
name = self.get_object_name(concept)
|
||||
parameters = {}
|
||||
|
||||
for var_name, default_value in concept.get_metadata().variables:
|
||||
if var_name not in concept.get_metadata().parameters:
|
||||
continue
|
||||
parameters[var_name] = default_value if default_value is not NotInit else var_name
|
||||
|
||||
function_to_call = "evaluate_question" if is_question else "call_concept"
|
||||
to_compile = f"{function_to_call}({name}"
|
||||
for p_name, p_value in parameters.items():
|
||||
to_compile += f", {p_name}={p_value}"
|
||||
to_compile += ")"
|
||||
|
||||
concept.get_hints().use_copy = True
|
||||
concept.get_hints().is_evaluated = True
|
||||
return PythonExprVisitorObj(source, to_compile, {name: concept}, set())
|
||||
|
||||
def is_a_possible_variable(self, name):
|
||||
"""
|
||||
tells whether or not the name can be a variable
|
||||
:param name:
|
||||
:return:
|
||||
"""
|
||||
if self.context.sheerka.is_a_concept_name(name):
|
||||
return False
|
||||
|
||||
try:
|
||||
eval(name, sheerka_globals)
|
||||
except:
|
||||
return True
|
||||
|
||||
return False
|
||||
Reference in New Issue
Block a user