diff --git a/app.py b/app.py deleted file mode 100644 index 25e3d2a..0000000 --- a/app.py +++ /dev/null @@ -1,3 +0,0 @@ - -f = open("MyFirstFile.txt", "w+") -f.close() \ No newline at end of file diff --git a/core/ast/__init__.py b/core/ast/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/core/ast/nodes.py b/core/ast/nodes.py new file mode 100644 index 0000000..f88c209 --- /dev/null +++ b/core/ast/nodes.py @@ -0,0 +1,110 @@ +from core.builtin_concepts import BuiltinConcepts, ListConcept +from core.concept import Concept +import ast + +import logging + +log = logging.getLogger(__name__) + + +class NodeParent: + """ + Class that represent the ancestor of a Node + For example, the 'For' nodes has three fields (target, iter and body) + So, for a node under For.iter + node -> For + field -> iter + """ + + def __init__(self, node, field): + self.node = node + self.field = field + + def __repr__(self): + if self.node is None: + return None + + if self.field is None: + return self.node.get_node_type() + + return self.node.get_node_type() + "." + self.field + + def __eq__(self, other): + # I can compare with type for simplification + if isinstance(other, tuple): + return self.node.get_node_type() == other[0] and self.field == other[1] + + # normal equals implementation + if not isinstance(other, NodeParent): + return False + + return self.node.get_node_type() == other.node.get_node_type() and self.field == other.field + + def __hash__(self): + return hash((self.node.get_node_type(), self.field)) + + +class NodeConcept(Concept): + def __init__(self, key, parent: NodeParent): + super().__init__(key, True, False, key) + self.parent = parent + + def get_node_type(self): + return self.key + + +class GenericNodeConcept(NodeConcept): + def __init__(self, node_type, parent): + super().__init__(BuiltinConcepts.GENERIC_NODE, parent) + self.node_type = node_type + + def __repr__(self): + return "Generic:" + self.node_type + + def get_node_type(self): + return self.node_type + + def get_value(self): + if self.node_type == "Name": + return self.get_prop("id") + + if self.node_type == "arg": + return self.get_prop("arg") + + return self.body + + +class IdentifierConcept(NodeConcept): + def __init__(self, parent, name): + super().__init__(BuiltinConcepts.IDENTIFIER_NODE, parent) + self.body = name + + +def transform(node): + """ + Transform Python AST node into concept nodes + for better usage + :param node: + :return: + """ + + def _transform(node, parent): + node_type = node.__class__.__name__ + concept = GenericNodeConcept(node_type, parent).init_key() + for field in node._fields: + if not hasattr(node, field): + continue + + value = getattr(node, field) + if isinstance(value, list): + lst = ListConcept().init_key() + for i in value: + lst.append(_transform(i, NodeParent(concept, field))) + concept.set_prop(field, lst) + elif isinstance(value, ast.AST): + concept.set_prop(field, _transform(value, NodeParent(concept, field))) + else: + concept.set_prop(field, value) + return concept + + return _transform(node, None) diff --git a/core/ast/visitors.py b/core/ast/visitors.py new file mode 100644 index 0000000..9af0378 --- /dev/null +++ b/core/ast/visitors.py @@ -0,0 +1,122 @@ +from core.ast.nodes import GenericNodeConcept, NodeConcept +from core.builtin_concepts import ListConcept + + +class ConceptNodeVisitor: + """ + Base class to visit NodeConcept + It is insolently inspired by python AST.Visitor class + """ + + def visit(self, node): + + """Visit a node.""" + name = node.node_type if isinstance(node, GenericNodeConcept) else node.name + name = str(name).capitalize() + + method = 'visit_' + name + visitor = getattr(self, method, self.generic_visit) + return visitor(node) + + def generic_visit(self, node): + """Called if no explicit visitor function exists for a node.""" + for field, value in iter_props(node): + if isinstance(value, ListConcept): + for item in value: + if isinstance(item, NodeConcept): + self.visit(item) + elif isinstance(value, NodeConcept): + self.visit(value) + + def visit_Constant(self, node): + value = node.get_prop("value") + type_name = _const_node_type_names.get(type(value)) + if type_name is None: + for cls, name in _const_node_type_names.items(): + if isinstance(value, cls): + type_name = name + break + if type_name is not None: + method = 'visit_' + type_name + try: + visitor = getattr(self, method) + except AttributeError: + pass + else: + import warnings + warnings.warn(f"{method} is deprecated; add visit_Constant", + PendingDeprecationWarning, 2) + return visitor(node) + return self.generic_visit(node) + + +class UnreferencedNamesVisitor(ConceptNodeVisitor): + def __init__(self, sheerka): + self.names = set() + self.sheerka = sheerka + + def visit_Name(self, node): + parents = get_parents(node) + if ("For", "target") in parents: # variable used by the 'for' iteration + return + + if ("Call", "func") in parents: # name of the function + return + + if ("Assign", "targets") in parents: # variable which is assigned + return + + if self.can_be_discarded(self.sheerka.value(node), parents): + return + + self.names.add(self.sheerka.value(node)) + + def can_be_discarded(self, variable_name, parents): + + for node in (parent.node for parent in parents): + if node is None: + return False + + if node.get_node_type() == "For" and self.sheerka.value(node.get_prop("target")) == variable_name: + # variable used by the loop + return True + + if node.get_node_type() == "FunctionDef": + # variable defined as a function parameter + args = node.get_prop("args") + args_values = list(self.sheerka.values(args.get_prop("args"))) + if variable_name in args_values: + return True + + return False + + +def get_parents(node): + if node.parent is None: + return [] + + res = [] + while True: + if node.parent is None: + break + res.append(node.parent) + node = node.parent.node + + return res + + +def iter_props(node): + for p in node.props: + yield p, node.props[p].value + + +_const_node_type_names = { + bool: 'NameConstant', # should be before int + type(None): 'NameConstant', + int: 'Num', + float: 'Num', + complex: 'Num', + str: 'Str', + bytes: 'Bytes', + type(...): 'Ellipsis', +} diff --git a/core/builtin_concepts.py b/core/builtin_concepts.py index 09fb804..6b17618 100644 --- a/core/builtin_concepts.py +++ b/core/builtin_concepts.py @@ -6,6 +6,12 @@ from core.concept import Concept class BuiltinConcepts(Enum): """ List of builtin concepts that do no need any specific implementation + Please note that the value of the enum is informal. It is not used in the system + For example, the concept 'NODE' DOES NOT have the key, the id or whatever 200 + The key if the name of the concept + The id is a sequential number given just before the concept is saved in sdp + + The values of the enum are just a convenient way for me to group the concepts """ SHEERKA = 1 SUCCESS = 2 @@ -31,6 +37,12 @@ class BuiltinConcepts(Enum): NOP = 22 # no operation concept. Does nothing PROPERTY_EVAL_ERROR = 23 # cannot evaluate a property of a concept ENUMERATION = 24 # represents a list or a set + LIST = 25 # represents a list + CANNOT_RESOLVE_VALUE_ERROR = 26 # In presence of a concept where the default value is not know + + NODE = 200 + GENERIC_NODE = 201 + IDENTIFIER_NODE = 202 """ @@ -234,3 +246,36 @@ class PropertyEvalError(Concept): @property def property_name(self): return self.body + + +class EnumerationConcept(Concept): + def __init__(self, iteration=None): + super().__init__(BuiltinConcepts.ENUMERATION, True, False, BuiltinConcepts.ENUMERATION) + self.body = iteration + + def __iter__(self): + return iter(self.body) + + +class ListConcept(Concept): + def __init__(self, items=None): + super().__init__(BuiltinConcepts.LIST, True, False, BuiltinConcepts.LIST) + self.body = items or [] + + def append(self, obj): + self.body.append(obj) + + def __len__(self): + return len(self.body) + + def __getitem__(self, key): + return self.body[key] + + def __setitem__(self, key, value): + self.body[key] = value + + def __iter__(self): + return iter(self.body) + + def __contains__(self, item): + return item in self.body diff --git a/core/builtin_helpers.py b/core/builtin_helpers.py new file mode 100644 index 0000000..144acbf --- /dev/null +++ b/core/builtin_helpers.py @@ -0,0 +1,83 @@ +from core.builtin_concepts import BuiltinConcepts + + +def is_same_success(sheerka, return_values): + """ + Returns True if all returns values are successful and have the same value + :param sheerka: + :param return_values: + :return: + """ + assert isinstance(return_values, list) + + if not return_values[0].status: + return False + + reference = sheerka.value(return_values[0].value) + + for return_value in return_values[1:]: + if not return_value.status: + return False + + actual = sheerka.value(return_value.value) + if actual != reference: + return False + + return True + + +def expect_one(context, return_values): + """ + Checks if there is at least one success return value + If there is more than one, check if it's the same value + :param context: + :param return_values: + :return: + """ + + if not isinstance(return_values, list): + return return_values + + sheerka = context.sheerka + + if len(return_values) == 0: + return sheerka.ret( + context.who, + False, + sheerka.new(BuiltinConcepts.IS_EMPTY, obj=return_values), + parents=return_values) + + successful_results = [item for item in return_values if item.status] + number_of_successful = len(successful_results) + # total_items = len(return_values) + + # remove errors when a winner is found + if number_of_successful == 1: + # log.debug(f"1 / {total_items} good item found.") + return sheerka.ret( + context.who, + True, + successful_results[0].body, + parents=return_values) + + # too many winners, which one to choose ? + if number_of_successful > 1: + if is_same_success(sheerka, successful_results): + return sheerka.ret( + context.who, + True, + successful_results[0].value, + parents=return_values) + else: + return sheerka.ret( + context.who, + False, + sheerka.new(BuiltinConcepts.TOO_MANY_SUCCESS, obj=successful_results), + parents=return_values) + + # only errors, i cannot help you + return sheerka.ret( + context.who, + False, + sheerka.new(BuiltinConcepts.TOO_MANY_ERRORS, obj=return_values), + parents=return_values) diff --git a/core/concept.py b/core/concept.py index 54449e4..f855835 100644 --- a/core/concept.py +++ b/core/concept.py @@ -190,15 +190,18 @@ class Concept: return self - def set_prop(self, prop_name, prop_value=None): + def set_prop(self, prop_name: str, prop_value=None): self.props[prop_name] = Property(prop_name, prop_value) return self - def set_prop_by_index(self, index, prop_value): + def set_prop_by_index(self, index: int, prop_value): prop_name = list(self.props.keys())[index] self.props[prop_name] = Property(prop_name, prop_value) return self + def get_prop(self, prop_name: str): + return self.props[prop_name].value + class Property: """ diff --git a/core/sheerka.py b/core/sheerka.py index 3665355..328abc7 100644 --- a/core/sheerka.py +++ b/core/sheerka.py @@ -5,11 +5,14 @@ from evaluators.BaseEvaluator import OneReturnValueEvaluator from parsers.BaseParser import BaseParser from sdp.sheerkaDataProvider import SheerkaDataProvider, Event, SheerkaDataProviderDuplicateKeyError import core.utils +import core.builtin_helpers import logging log = logging.getLogger(__name__) +concept_evaluation_steps = [BuiltinConcepts.EVALUATION, BuiltinConcepts.AFTER_EVALUATION] + class Sheerka(Concept): """ @@ -150,6 +153,12 @@ class Sheerka(Concept): logging.basicConfig(format=log_format, level=log_level) def eval(self, text): + """ + Note to KSI: If you try to add execution context to this function, + You may end in an infinite loop + :param text: + :return: + """ evt_digest = self.sdp.save_event(Event(text)) exec_context = ExecutionContext(self.key, evt_digest, self) @@ -174,32 +183,6 @@ class Sheerka(Concept): return return_values - def expect_one(self, context, items): - - if not isinstance(items, list): - items = [items] - - if len(items) == 0: - return self.ret(context.who, False, self.new(BuiltinConcepts.IS_EMPTY, obj=items)) - - successful_results = [item for item in items if item.status] - number_of_successful = len(successful_results) - total_items = len(items) - - # remove errors when a winner is found - if number_of_successful == 1: - # log.debug(f"1 / {total_items} good item found.") - return successful_results[0] - - # too many winners, which one to choose ? - if number_of_successful > 1: - log.debug(f"{number_of_successful} / {total_items} good items. Too many success") - return self.ret(context.who, False, self.new(BuiltinConcepts.TOO_MANY_SUCCESS, obj=successful_results)) - - # only errors, i cannot help you - log.debug(f"{total_items} items. Only errors") - return self.ret(context.who, False, self.new(BuiltinConcepts.TOO_MANY_ERRORS, obj=items)) - def parse(self, context, text): result = [] if log.isEnabledFor(logging.DEBUG): @@ -356,6 +339,33 @@ class Sheerka(Concept): for prop in concept.props: concept.codes[prop] = self.parse(context, concept.props[prop].value) + # updates the code of the reference when possible + if concept.key in self.concepts_cache: + entry = self.concepts_cache[concept.key] + if isinstance(entry, list): + # TODO : manage when there are multiple entries + pass + else: + self.concepts_cache[concept.key].codes = concept.codes + + def eval_concept(self, context, concept, properties_to_eval=None): + if len(concept.codes) == 0: + self.add_codes_to_concept(context, concept) + + if properties_to_eval is None: + properties_to_eval = ["where", "pre", "post", "body", "props"] + + for prop in properties_to_eval: + if prop == "props": + pass + else: + part_key = ConceptParts(prop) + if concept.codes[part_key] is None: + continue + res = self.chain_process(context, concept.codes[part_key], concept_evaluation_steps) + res = core.builtin_helpers.expect_one(context, res) + setattr(concept, prop, res.value) + def add_in_cache(self, concept): """ Adds a concept template in cache. @@ -413,16 +423,37 @@ class Sheerka(Concept): """ template = self.get(concept_key) + def new_from_template(t, k, **kwargs_): + # manage singleton + if t.is_unique: + return t + + # otherwise, create another instance + concept = self.builtin_cache[k]() if k in self.builtin_cache else Concept() + concept.update_from(t) + + # update the properties + for k, v in kwargs_.items(): + if k in concept.props: + concept.set_prop(k, v) + elif hasattr(concept, k): + setattr(concept, k, v) + else: + return self.new(BuiltinConcepts.UNKNOWN_PROPERTY, body=k, concept=concept) + + # TODO : add the concept to the list of known concepts (self.instances) + return concept + # manage concept not found if self.isinstance(template, BuiltinConcepts.UNKNOWN_CONCEPT) and \ concept_key != BuiltinConcepts.UNKNOWN_CONCEPT: return template if not isinstance(template, list): - return self._new_from_template(template, concept_key, **kwargs) + return new_from_template(template, concept_key, **kwargs) # if template is a list, it means that there a multiple concepts under the same key - concepts = [self._new_from_template(t, concept_key, **kwargs) for t in template] + concepts = [new_from_template(t, concept_key, **kwargs) for t in template] return self.new(BuiltinConcepts.ENUMERATION, body=concepts) def ret(self, who, status, value, message=None, parents=None): @@ -443,6 +474,29 @@ class Sheerka(Concept): message=message, parents=parents) + def value(self, obj, allow_none_body=False): + if obj is None: + return None + + if not isinstance(obj, Concept): + return obj + + if hasattr(obj, "get_value"): + return obj.get_value() + + if obj.body is not None: + return obj.body + + return obj if allow_none_body else self.new(BuiltinConcepts.CANNOT_RESOLVE_VALUE_ERROR, body=obj) + + def values(self, objs): + if not (isinstance(objs, list) or + self.isinstance(objs, BuiltinConcepts.LIST) or + self.isinstance(objs, BuiltinConcepts.ENUMERATION)): + objs = [objs] + + return (self.value(obj) for obj in objs) + def isinstance(self, a, b): """ return true if the concept a is an instance of the concept b @@ -463,6 +517,27 @@ class Sheerka(Concept): # for example, if a is a color, it will be found the entry 'All_Colors' return a.key == b_key + def isa(self, a, b): + """ + return true if the concept a is a b + Will handle when the keyword isa will be implemented + :param a: + :param b: + :return: + """ + + if isinstance(a, BuiltinConcepts): # common KSI error ;-) + raise SyntaxError("Remember that the first parameter of isinstance MUST be a concept") + + if not isinstance(a, Concept): + return False + + b_key = b.key if isinstance(b, Concept) else str(b) + + # TODO : manage when a is the list of all possible b + # for example, if a is a color, it will be found the entry 'All_Colors' + return a.key == b_key + def get_evaluator_name(self, name): if self.evaluators_prefix is None: base_evaluator_class = core.utils.get_class("evaluators.BaseEvaluator.BaseEvaluator") @@ -486,28 +561,7 @@ class Sheerka(Concept): else: res.append(item) - return sorted(res, key=lambda i: i.key) - - def _new_from_template(self, template, concept_key, **kwargs): - # manage singleton - if template.is_unique: - return template - - # otherwise, create another instance - concept = self.builtin_cache[concept_key]() if concept_key in self.builtin_cache else Concept() - concept.update_from(template) - - # update the properties - for k, v in kwargs.items(): - if k in concept.props: - concept.set_prop(k, v) - elif hasattr(concept, k): - setattr(concept, k, v) - else: - return self.new(BuiltinConcepts.UNKNOWN_PROPERTY, body=k, concept=concept) - - # TODO : add the concept to the list of known concepts (self.instances) - return concept + return sorted(res, key=lambda i: int(i.id)) @staticmethod def get_builtins_classes_as_dict(): diff --git a/evaluators/ConceptEvaluator.py b/evaluators/ConceptEvaluator.py index 0f0079a..508830e 100644 --- a/evaluators/ConceptEvaluator.py +++ b/evaluators/ConceptEvaluator.py @@ -1,4 +1,5 @@ from core.builtin_concepts import ParserResultConcept, BuiltinConcepts +import core.builtin_helpers from core.concept import Concept, ConceptParts from evaluators.BaseEvaluator import OneReturnValueEvaluator import logging @@ -47,13 +48,14 @@ class ConceptEvaluator(OneReturnValueEvaluator): sheerka.new(BuiltinConcepts.PROPERTY_EVAL_ERROR, body=prop, concept=concept, error=res.value), parents=[return_value]) - # Evaluate body + # Returns the concept when no body if ConceptParts.BODY not in concept.codes: return sheerka.ret(self.name, True, concept, parents=[return_value]) + # Evaluate the body otherwise body = concept.codes[ConceptParts.BODY] if body is None: - return None # seems weird + raise NotImplementedError("Seems weird !") sub_context = context.push(self.name, "Evaluating body", concept) res = self.evaluate_parsing(sheerka, sub_context, body) @@ -61,5 +63,5 @@ class ConceptEvaluator(OneReturnValueEvaluator): def evaluate_parsing(self, sheerka, context, parsing_result): res = sheerka.chain_process(context, parsing_result, self.evaluation_steps) - res = sheerka.expect_one(context, res) + res = core.builtin_helpers.expect_one(context, res) return res diff --git a/evaluators/MutipleSameSuccessEvaluator.py b/evaluators/MutipleSameSuccessEvaluator.py index 5cdab0a..b752db2 100644 --- a/evaluators/MutipleSameSuccessEvaluator.py +++ b/evaluators/MutipleSameSuccessEvaluator.py @@ -1,5 +1,6 @@ from core.builtin_concepts import BuiltinConcepts from core.concept import Concept +import core.builtin_helpers from evaluators.BaseEvaluator import AllReturnValuesEvaluator, BaseEvaluator import logging @@ -36,7 +37,7 @@ class MultipleSameSuccessEvaluator(AllReturnValuesEvaluator): elif ret.who.startswith(BaseEvaluator.PREFIX): if ret.status: nb_successful_evaluators += 1 - self.success.append(ret.value) + self.success.append(ret) elif ret.who.startswith(BaseParser.PREFIX): if ret.status: only_parsers_in_error = False @@ -46,19 +47,9 @@ class MultipleSameSuccessEvaluator(AllReturnValuesEvaluator): return after_evaluation and nb_successful_evaluators > 1 and only_parsers_in_error and not unlisted def eval(self, context, return_values): - reference = self.get_value(self.success[0]) - - for return_value in self.success[1:]: - actual = self.get_value(return_value) - if actual != reference: - return None - sheerka = context.sheerka - return sheerka.ret(self.name, True, reference, parents=return_values) + if core.builtin_helpers.is_same_success(sheerka, self.success): + reference = sheerka.value(self.success[0].value, allow_none_body=True) + return sheerka.ret(self.name, True, reference, parents=return_values) - @staticmethod - def get_value(obj): - if not isinstance(obj, Concept): - return obj - - return obj if obj.body is None else obj.body + return None diff --git a/evaluators/PythonEvaluator.py b/evaluators/PythonEvaluator.py index b857750..a20292d 100644 --- a/evaluators/PythonEvaluator.py +++ b/evaluators/PythonEvaluator.py @@ -1,14 +1,18 @@ +import copy + +from core.ast.visitors import UnreferencedNamesVisitor from core.builtin_concepts import BuiltinConcepts, ParserResultConcept from evaluators.BaseEvaluator import OneReturnValueEvaluator from parsers.PythonParser import PythonNode import ast +import core.ast.nodes + import logging log = logging.getLogger(__name__) class PythonEvaluator(OneReturnValueEvaluator): - NAME = "Python" def __init__(self): @@ -22,22 +26,62 @@ class PythonEvaluator(OneReturnValueEvaluator): def eval(self, context, return_value): sheerka = context.sheerka node = return_value.value.value - if isinstance(node.ast_, ast.Expression): - try: - log.debug(f"Evaluating python node {node}") - compiled = compile(node.ast_, "", "eval") - evaluated = eval(compiled, {}, self.get_locals(context)) - return sheerka.ret(self.name, True, evaluated, parents=[return_value]) - except Exception as error: - error = sheerka.new(BuiltinConcepts.ERROR, body=error) - return sheerka.ret(self.name, False, error, parents=[return_value]) - else: - return sheerka.ret(self.name, False, sheerka.new(BuiltinConcepts.ERROR), parents=[return_value]) + try: + log.debug(f"Evaluating python node {node}") + my_locals = self.get_locals(context, node.ast_) - @staticmethod - def get_locals(context): + if isinstance(node.ast_, ast.Expression): + compiled = compile(node.ast_, "", "eval") + evaluated = eval(compiled, {}, my_locals) + else: + evaluated = self.exec_with_return(node.ast_, my_locals) + + return sheerka.ret(self.name, True, evaluated, parents=[return_value]) + except Exception as error: + error = sheerka.new(BuiltinConcepts.ERROR, body=error) + return sheerka.ret(self.name, False, error, parents=[return_value]) + + def get_locals(self, context, ast_): my_locals = {"sheerka": context.sheerka} if context.obj: for prop_name, prop_value in context.obj.props.items(): my_locals[prop_name] = prop_value.value + + node_concept = core.ast.nodes.transform(ast_) + unreferenced_names_visitor = UnreferencedNamesVisitor(context.sheerka) + unreferenced_names_visitor.visit(node_concept) + + for name in unreferenced_names_visitor.names: + concept = context.sheerka.new(name) + if context.sheerka.isinstance(concept, BuiltinConcepts.UNKNOWN_CONCEPT): + continue + + sub_context = context.push(self.name, "Evaluating body", concept) + context.sheerka.eval_concept(sub_context, concept, ["body"]) + + if not context.sheerka.isa(concept.body, BuiltinConcepts.ERROR): + my_locals[name] = concept.body + return my_locals + + @staticmethod + def expr_to_expression(expr): + expr.lineno = 0 + expr.col_offset = 0 + result = ast.Expression(expr.value, lineno=0, col_offset=0) + + return result + + def exec_with_return(self, code_ast, my_locals): + + init_ast = copy.deepcopy(code_ast) + init_ast.body = code_ast.body[:-1] + + last_ast = copy.deepcopy(code_ast) + last_ast.body = code_ast.body[-1:] + + exec(compile(init_ast, "", "exec"), {}, my_locals) + if type(last_ast.body[0]) == ast.Expr: + return eval(compile(self.expr_to_expression(last_ast.body[0]), "", "eval"), {}, my_locals) + else: + exec(compile(last_ast, "", "exec"), {}, my_locals) diff --git a/evaluators/TooManySuccessEvaluator.py b/evaluators/TooManySuccessEvaluator.py index 4de7c66..617f258 100644 --- a/evaluators/TooManySuccessEvaluator.py +++ b/evaluators/TooManySuccessEvaluator.py @@ -1,5 +1,5 @@ from core.builtin_concepts import BuiltinConcepts -from core.concept import Concept +import core.builtin_helpers from evaluators.BaseEvaluator import AllReturnValuesEvaluator, BaseEvaluator import logging @@ -46,20 +46,9 @@ class TooManySuccessEvaluator(AllReturnValuesEvaluator): return after_evaluation and nb_successful_evaluators > 1 and only_parsers_in_error and not unlisted def eval(self, context, return_values): - reference = self.get_value(self.success[0].value) - - for return_value in self.success[1:]: - actual = self.get_value(return_value.value) - if actual != reference: - sheerka = context.sheerka - too_many_success = sheerka.new(BuiltinConcepts.TOO_MANY_SUCCESS, obj=self.success) - return sheerka.ret(self.name, False, too_many_success, parents=return_values) + sheerka = context.sheerka + if not core.builtin_helpers.is_same_success(sheerka, self.success): + too_many_success = sheerka.new(BuiltinConcepts.TOO_MANY_SUCCESS, obj=self.success) + return sheerka.ret(self.name, False, too_many_success, parents=return_values) return None - - @staticmethod - def get_value(obj): - if not isinstance(obj, Concept): - return obj - - return obj if obj.body is None else obj.body diff --git a/parsers/DefaultParser.py b/parsers/DefaultParser.py index 81b7343..6bd1f8c 100644 --- a/parsers/DefaultParser.py +++ b/parsers/DefaultParser.py @@ -1,5 +1,6 @@ from core.builtin_concepts import BuiltinConcepts, ReturnValueConcept from core.concept import ConceptParts +import core.builtin_helpers from parsers.BaseParser import BaseParser, Node, NopNode, ErrorNode, NotInitializedNode from core.tokenizer import Tokenizer, TokenKind, Token, Keywords from dataclasses import dataclass, field @@ -409,8 +410,8 @@ class DefaultParser(BaseParser): continue # ask the other parsers if they recognize the tokens - new_context = self.context.push(self) - parsing_result = self.sheerka.expect_one(new_context, self.sheerka.parse(new_context, tokens)) + new_context = self.context.push(self.name) + parsing_result = core.builtin_helpers.expect_one(new_context, self.sheerka.parse(new_context, tokens)) if not parsing_result.status: self.add_error(parsing_result.value) continue diff --git a/parsers/PythonParser.py b/parsers/PythonParser.py index 465cb2f..7becfef 100644 --- a/parsers/PythonParser.py +++ b/parsers/PythonParser.py @@ -103,28 +103,6 @@ class PythonParser(BaseParser): except Exception as error: return False, None, error - def expr_to_expression(self, expr): - expr.lineno = 0 - expr.col_offset = 0 - result = ast.Expression(expr.value, lineno=0, col_offset=0) - - return result - - def exec_with_return(self, code): - code_ast = ast.parse(code) - - init_ast = copy.deepcopy(code_ast) - init_ast.body = code_ast.body[:-1] - - last_ast = copy.deepcopy(code_ast) - last_ast.body = code_ast.body[-1:] - - exec(compile(init_ast, "", "exec"), globals()) - if type(last_ast.body[0]) == ast.Expr: - return eval(compile(self.expr_to_expression(last_ast.body[0]), "", "eval"), globals()) - else: - exec(compile(last_ast, "", "exec"), globals()) - class PythonGetNamesVisitor(ast.NodeVisitor): """ diff --git a/tests/test_ConceptEvaluator.py b/tests/test_ConceptEvaluator.py index e9d33f1..8ad3712 100644 --- a/tests/test_ConceptEvaluator.py +++ b/tests/test_ConceptEvaluator.py @@ -91,6 +91,10 @@ def test_body_is_evaluated_when_concept_body(): def test_body_is_evaluated_when_concept_body_with_a_body(): + """ + The concept refers to another concept which has a body + :return: + """ context = get_context() concept_one = Concept(name="one", body="1").init_key() context.sheerka.add_in_cache(concept_one) diff --git a/tests/test_DefaultParser.py b/tests/test_DefaultParser.py index 21997d8..2967e62 100644 --- a/tests/test_DefaultParser.py +++ b/tests/test_DefaultParser.py @@ -100,7 +100,7 @@ def get_concept_part(part): if isinstance(part, str): node = PythonNode(part, ast.parse(part, mode="eval")) return ReturnValueConcept( - who="Parsers:PythonParser", + who="Parsers:DefaultParser", status=True, value=ParserResultConcept( source=part, @@ -109,7 +109,7 @@ def get_concept_part(part): if isinstance(part, PythonNode): return ReturnValueConcept( - who="Parsers:PythonParser", + who="Parsers:DefaultParser", status=True, value=ParserResultConcept( source=part.source, diff --git a/tests/test_PyhtonEvaluator.py b/tests/test_PyhtonEvaluator.py new file mode 100644 index 0000000..2c724c2 --- /dev/null +++ b/tests/test_PyhtonEvaluator.py @@ -0,0 +1,89 @@ +import pytest +import shutil +from os import path +import os + +from core.builtin_concepts import ReturnValueConcept, ParserResultConcept +from core.sheerka import Sheerka, ExecutionContext +from core.concept import Concept +from evaluators.PythonEvaluator import PythonEvaluator +from parsers.PythonParser import PythonNode, PythonParser + +tests_root = path.abspath("../build/tests") +root_folder = "init_folder" + + +@pytest.fixture(autouse=True) +def init_test(): + if path.exists(tests_root): + shutil.rmtree(tests_root) + + if not path.exists(tests_root): + os.makedirs(tests_root) + current_pwd = os.getcwd() + os.chdir(tests_root) + + yield None + os.chdir(current_pwd) + + +def get_context(): + sheerka = Sheerka() + sheerka.initialize(root_folder) + return ExecutionContext("test", "xxx", sheerka) + + +@pytest.mark.parametrize("ret_val, expected", [ + (ReturnValueConcept("some_name", True, ParserResultConcept(value=PythonNode("", None))), True), + (ReturnValueConcept("some_name", True, ParserResultConcept(value="other thing")), False), + (ReturnValueConcept("some_name", False, "not relevant"), False), + (ReturnValueConcept("some_name", True, Concept()), False) +]) +def test_i_can_match(ret_val, expected): + context = get_context() + assert PythonEvaluator().matches(context, ret_val) == expected + + +@pytest.mark.parametrize("text, expected", [ + ("1 + 1", 2), + ("sheerka.test()", "I have access to Sheerka !"), + ("a=10\na", 10), +]) +def test_i_can_eval(text, expected): + context = get_context() + parsed = PythonParser().parse(context, text) + + evaluated = PythonEvaluator().eval(context, parsed) + + assert evaluated.status + assert evaluated.value == expected + + +def test_i_can_eval_expression_with_variables(): + """ + I can test expression with variables + :return: + """ + context = get_context() + context.sheerka.add_in_cache(Concept("foo", body="2")) + parsed = PythonParser().parse(context, "foo + 2") + + evaluated = PythonEvaluator().eval(context, parsed) + + assert evaluated.status + assert evaluated.value == 4 + + +def test_i_can_eval_module_with_variables(): + """ + I can test modules with variables + :return: + """ + context = get_context() + context.sheerka.add_in_cache(Concept("foo", body="2")) + parsed = PythonParser().parse(context, "def a(b):\n return b\na(foo)") + + evaluated = PythonEvaluator().eval(context, parsed) + + assert evaluated.status + assert evaluated.value == 2 diff --git a/tests/test_ast.py b/tests/test_ast.py new file mode 100644 index 0000000..309e343 --- /dev/null +++ b/tests/test_ast.py @@ -0,0 +1,154 @@ +import os +import shutil +from os import path + +import pytest +import ast + +from core.ast.nodes import NodeParent, GenericNodeConcept +import core.ast.nodes +from core.ast.visitors import ConceptNodeVisitor, UnreferencedNamesVisitor +from core.builtin_concepts import BuiltinConcepts +from core.sheerka import Sheerka + +tests_root = path.abspath("../build/tests") +root_folder = "init_folder" + + +@pytest.fixture(autouse=True) +def init_test(): + if path.exists(tests_root): + shutil.rmtree(tests_root) + + if not path.exists(tests_root): + os.makedirs(tests_root) + current_pwd = os.getcwd() + os.chdir(tests_root) + + yield None + + os.chdir(current_pwd) + + +def get_sheerka(): + sheerka = Sheerka() + sheerka.initialize(root_folder) + + return sheerka + + +class TestNameVisitor(ConceptNodeVisitor): + """ + Test class for a basic Visitor test + """ + + def __init__(self): + self.names = [] + + def visit_Name(self, node): + self.names.append(node) + + +def test_i_can_transform_simple_ast_using_generic_node(): + source = """ +def my_function(a,b): + for i in range(b): + a = a+b + return a +""" + tree = ast.parse(source) + tree_as_concept = core.ast.nodes.transform(tree) + sheerka = get_sheerka() + + assert tree_as_concept.node_type == "Module" + assert sheerka.isinstance(tree_as_concept.get_prop("body"), BuiltinConcepts.LIST) + + def_func = tree_as_concept.get_prop("body")[0] + assert sheerka.isinstance(def_func, BuiltinConcepts.GENERIC_NODE) + assert def_func.node_type == "FunctionDef" + assert def_func.parent == NodeParent(tree_as_concept, "body") + assert def_func.get_prop("name") == "my_function" + + def_func_args = def_func.get_prop("args") + assert sheerka.isinstance(def_func_args, BuiltinConcepts.GENERIC_NODE) + assert def_func_args.node_type == "arguments" + + def_func_args_real_args = def_func_args.get_prop("args") + assert sheerka.isinstance(def_func_args_real_args, BuiltinConcepts.LIST) + assert len(def_func_args_real_args) == 2 + + assert sheerka.isinstance(def_func_args_real_args[0], BuiltinConcepts.GENERIC_NODE) + assert def_func_args_real_args[0].node_type == "arg" + assert def_func_args_real_args[0].parent == NodeParent(def_func_args, "args") + assert def_func_args_real_args[0].get_prop("arg") == "a" + assert sheerka.isinstance(def_func_args_real_args[1], BuiltinConcepts.GENERIC_NODE) + assert def_func_args_real_args[1].node_type == "arg" + assert def_func_args_real_args[1].parent == NodeParent(def_func_args, "args") + assert def_func_args_real_args[1].get_prop("arg") == "b" + + def_fun_body = def_func.get_prop("body") + assert sheerka.isinstance(def_fun_body, BuiltinConcepts.LIST) + assert len(def_fun_body) == 2 + + def_fun_body_for = def_fun_body[0] + assert sheerka.isinstance(def_fun_body_for, BuiltinConcepts.GENERIC_NODE) + assert def_fun_body_for.node_type == "For" + assert def_fun_body_for.parent == NodeParent(def_func, "body") + + def_fun_body_return = def_fun_body[1] + assert sheerka.isinstance(def_fun_body_return, BuiltinConcepts.GENERIC_NODE) + assert def_fun_body_return.node_type == "Return" + assert def_fun_body_return.parent == NodeParent(def_func, "body") + + +def test_i_can_visit_concept_node(): + source = """ +def my_function(a,b): + for i in range(b): + a = a+b + return a + """ + + node = ast.parse(source) + concept_node = core.ast.nodes.transform(node) + + visitor = TestNameVisitor() + visitor.visit(concept_node) + + sheerka = get_sheerka() + assert sheerka.value(visitor.names[0]) == "i" + assert sheerka.value(visitor.names[1]) == "range" + assert sheerka.value(visitor.names[2]) == "b" + assert sheerka.value(visitor.names[3]) == "a" + assert sheerka.value(visitor.names[4]) == "a" + assert sheerka.value(visitor.names[5]) == "b" + assert sheerka.value(visitor.names[6]) == "a" + + +def test_i_can_get_non_referenced_variables(): + source = """ +def my_function(a,b): + for i in range(b): + a = a+b + return a + +my_function(x,y) +""" + + sheerka = get_sheerka() + + node = ast.parse(source) + concept_node = core.ast.nodes.transform(node) + + visitor = UnreferencedNamesVisitor(sheerka) + visitor.visit(concept_node) + values = visitor.names + + assert len(visitor.names) == 2 + assert "x" in values + assert "y" in values + + +def test_i_can_compare_NodeParent_with_tuple(): + node_parent = NodeParent(GenericNodeConcept("For", None), "target") + assert node_parent == ("For", "target") diff --git a/tests/test_builtin_helpers.py b/tests/test_builtin_helpers.py new file mode 100644 index 0000000..6e0ecc4 --- /dev/null +++ b/tests/test_builtin_helpers.py @@ -0,0 +1,145 @@ +import shutil +from os import path +import os + +import pytest + +from core.builtin_concepts import ReturnValueConcept, BuiltinConcepts +from core.sheerka import Sheerka, ExecutionContext +import core.builtin_helpers + +tests_root = path.abspath("../build/tests") +root_folder = "init_folder" + + +@pytest.fixture(autouse=True) +def init_test(): + if path.exists(tests_root): + shutil.rmtree(tests_root) + + if not path.exists(tests_root): + os.makedirs(tests_root) + current_pwd = os.getcwd() + os.chdir(tests_root) + + yield None + + os.chdir(current_pwd) + + +def test_i_can_use_expect_one_when_empty(): + sheerka = get_sheerka() + + res = core.builtin_helpers.expect_one(get_context(sheerka), []) + assert not res.status + assert sheerka.isinstance(res.value, BuiltinConcepts.IS_EMPTY) + + +def test_i_can_use_expect_one_when_too_many_success(): + sheerka = get_sheerka() + + items = [ + ReturnValueConcept("who", True, "value1"), + ReturnValueConcept("who", True, "value2"), + ] + res = core.builtin_helpers.expect_one(get_context(sheerka), items) + assert not res.status + assert sheerka.isinstance(res.value, BuiltinConcepts.TOO_MANY_SUCCESS) + assert res.value.obj == items + assert res.parents == items + + +def test_i_can_use_expect_one_when_same_success(): + sheerka = get_sheerka() + + items = [ + ReturnValueConcept("who", True, "value"), + ReturnValueConcept("who", True, "value"), + ] + res = core.builtin_helpers.expect_one(get_context(sheerka), items) + assert res.status + assert res.value == items[0].value + assert res.parents == items + + +def test_i_can_use_expect_when_only_errors_1(): + sheerka = get_sheerka() + + items = [ + ReturnValueConcept("who", False, None), + ] + res = core.builtin_helpers.expect_one(get_context(sheerka), items) + assert not res.status + assert sheerka.isinstance(res.value, BuiltinConcepts.TOO_MANY_ERRORS) + assert res.value.obj == items + assert res.parents == items + + +def test_i_can_use_expect_when_only_errors_2(): + sheerka = get_sheerka() + + items = [ + ReturnValueConcept("who", False, None), + ReturnValueConcept("who", False, None), + ] + res = core.builtin_helpers.expect_one(get_context(sheerka), items) + assert not res.status + assert sheerka.isinstance(res.value, BuiltinConcepts.TOO_MANY_ERRORS) + assert res.value.obj == items + assert res.parents == items + + +def test_i_can_use_expect_one_when_one_success_1(): + sheerka = get_sheerka() + + items = [ + ReturnValueConcept("who", True, None), + ] + res = core.builtin_helpers.expect_one(get_context(sheerka), items) + assert res.status + assert res.body == items[0].body + assert res.parents == items + + +def test_i_can_use_expect_one_when_one_success_2(): + sheerka = get_sheerka() + + items = [ + ReturnValueConcept("who", False, None), + ReturnValueConcept("who", True, None), + ReturnValueConcept("who", False, None), + ] + res = core.builtin_helpers.expect_one(get_context(sheerka), items) + assert res.status + assert res.body == items[1].body + assert res.parents == items + + +def test_i_can_use_expect_one_when_not_a_list_true(): + sheerka = get_sheerka() + + item = ReturnValueConcept("who", True, None) + res = core.builtin_helpers.expect_one(get_context(sheerka), item) + assert res.status + assert res == item + + +def test_i_can_use_expect_one_when_not_a_list_false(): + sheerka = get_sheerka() + + item = ReturnValueConcept("who", False, None) + res = core.builtin_helpers.expect_one(get_context(sheerka), item) + + assert not res.status + assert res == item + + +def get_sheerka(): + sheerka = Sheerka() + sheerka.initialize(root_folder) + + return sheerka + + +def get_context(sheerka): + return ExecutionContext("test", "xxx", sheerka) diff --git a/tests/test_sheerka.py b/tests/test_sheerka.py index 92ae12b..dd6bc47 100644 --- a/tests/test_sheerka.py +++ b/tests/test_sheerka.py @@ -33,6 +33,11 @@ def init_test(): os.chdir(current_pwd) +class ConceptWithGetValue(Concept): + def get_value(self): + return self.get_prop("my_prop") + + def test_root_folder_is_created_after_initialization(): return_value = Sheerka().initialize(root_folder) assert return_value.status, "initialisation should be successful" @@ -288,93 +293,33 @@ def test_i_cannot_instantiate_when_properties_are_not_recognized(): assert sheerka.isinstance(new.concept, concept) -def test_i_can_use_expect_one_when_empty(): +@pytest.mark.parametrize("concept, allow_non_body, expected", [ + (None, False, None), + (3.14, False, 3.14), + (Concept("name", body="foo"), False, "foo"), + (Concept("name"), True, Concept("name")), + (ConceptWithGetValue("name").set_prop("my_prop", "my_value"), True, "my_value"), +]) +def test_i_can_get_value(concept, allow_non_body, expected): sheerka = get_sheerka() - res = sheerka.expect_one(get_context(sheerka), []) - assert not res.status - assert sheerka.isinstance(res.value, BuiltinConcepts.IS_EMPTY) + assert sheerka.value(concept, allow_non_body) == expected -def test_i_can_use_expect_one_when_too_many_success(): +def test_i_cannot_get_value_when_no_body_and_allow_none_body_is_false(): sheerka = get_sheerka() + concept = Concept("name") + allow_none_body = False - items = [ - ReturnValueConcept("who", True, None), - ReturnValueConcept("who", True, None), - ] - res = sheerka.expect_one(get_context(sheerka), items) - assert not res.status - assert sheerka.isinstance(res.value, BuiltinConcepts.TOO_MANY_SUCCESS) - assert res.value.obj == items + assert sheerka.value(concept, allow_none_body) == sheerka.new(BuiltinConcepts.CANNOT_RESOLVE_VALUE_ERROR, + body=concept) -def test_i_can_use_expect_when_only_errors_1(): - sheerka = get_sheerka() - - items = [ - ReturnValueConcept("who", False, None), - ] - res = sheerka.expect_one(get_context(sheerka), items) - assert not res.status - assert sheerka.isinstance(res.value, BuiltinConcepts.TOO_MANY_ERRORS) - assert res.value.obj == items - - -def test_i_can_use_expect_when_only_errors_2(): - sheerka = get_sheerka() - - items = [ - ReturnValueConcept("who", False, None), - ReturnValueConcept("who", False, None), - ] - res = sheerka.expect_one(get_context(sheerka), items) - assert not res.status - assert sheerka.isinstance(res.value, BuiltinConcepts.TOO_MANY_ERRORS) - assert res.value.obj == items - - -def test_i_can_use_expect_one_when_one_success_1(): - sheerka = get_sheerka() - - items = [ - ReturnValueConcept("who", True, None), - ] - res = sheerka.expect_one(get_context(sheerka), items) - assert res.status - assert res == items[0] - - -def test_i_can_use_expect_one_when_one_success_2(): - sheerka = get_sheerka() - - items = [ - ReturnValueConcept("who", False, None), - ReturnValueConcept("who", True, None), - ReturnValueConcept("who", False, None), - ] - res = sheerka.expect_one(get_context(sheerka), items) - assert res.status - assert res == items[1] - - -def test_i_can_use_expect_one_when_not_a_list_true(): - sheerka = get_sheerka() - - res = sheerka.expect_one(get_context(sheerka), ReturnValueConcept("who", True, None)) - assert res.status - assert res == ReturnValueConcept("who", True, None) - - -def test_i_can_use_expect_one_when_not_a_list_false(): - sheerka = get_sheerka() - - res = sheerka.expect_one(get_context(sheerka), ReturnValueConcept("who", False, None)) - - assert not res.status - assert sheerka.isinstance(res.value, BuiltinConcepts.TOO_MANY_ERRORS) - assert res.value.obj == [ReturnValueConcept("who", False, None)] - +# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +# +# E V A L U A T I O N S +# +# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! @pytest.mark.parametrize("text, expected", [ ("1 + 1", 2),