kodjo/Sheerka-Old
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Enhanced PythonEvaluator to accept concepts

Browse Source
This commit is contained in:
Kodjo Sossouvi
2019-11-21 11:52:15 +01:00
parent cb6be9fec7
commit 714f4f5dd0
20 changed files with 964 additions and 208 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app.py
-3
View File
@@ -1,3 +0,0 @@
f = open("MyFirstFile.txt", "w+")
f.close()
core/ast/__init__.py
View File
core/ast/nodes.py
+110
View File
@@ -0,0 +1,110 @@
from core.builtin_concepts import BuiltinConcepts, ListConcept
from core.concept import Concept
import ast
import logging
log = logging.getLogger(__name__)
class NodeParent:
"""
Class that represent the ancestor of a Node
For example, the 'For' nodes has three fields (target, iter and body)
So, for a node under For.iter
node -> For
field -> iter
"""
def __init__(self, node, field):
self.node = node
self.field = field
def __repr__(self):
if self.node is None:
return None
if self.field is None:
return self.node.get_node_type()
return self.node.get_node_type() + "." + self.field
def __eq__(self, other):
# I can compare with type for simplification
if isinstance(other, tuple):
return self.node.get_node_type() == other[0] and self.field == other[1]
# normal equals implementation
if not isinstance(other, NodeParent):
return False
return self.node.get_node_type() == other.node.get_node_type() and self.field == other.field
def __hash__(self):
return hash((self.node.get_node_type(), self.field))
class NodeConcept(Concept):
def __init__(self, key, parent: NodeParent):
super().__init__(key, True, False, key)
self.parent = parent
def get_node_type(self):
return self.key
class GenericNodeConcept(NodeConcept):
def __init__(self, node_type, parent):
super().__init__(BuiltinConcepts.GENERIC_NODE, parent)
self.node_type = node_type
def __repr__(self):
return "Generic:" + self.node_type
def get_node_type(self):
return self.node_type
def get_value(self):
if self.node_type == "Name":
return self.get_prop("id")
if self.node_type == "arg":
return self.get_prop("arg")
return self.body
class IdentifierConcept(NodeConcept):
def __init__(self, parent, name):
super().__init__(BuiltinConcepts.IDENTIFIER_NODE, parent)
self.body = name
def transform(node):
"""
Transform Python AST node into concept nodes
for better usage
:param node:
:return:
"""
def _transform(node, parent):
node_type = node.__class__.__name__
concept = GenericNodeConcept(node_type, parent).init_key()
for field in node._fields:
if not hasattr(node, field):
continue
value = getattr(node, field)
if isinstance(value, list):
lst = ListConcept().init_key()
for i in value:
lst.append(_transform(i, NodeParent(concept, field)))
concept.set_prop(field, lst)
elif isinstance(value, ast.AST):
concept.set_prop(field, _transform(value, NodeParent(concept, field)))
else:
concept.set_prop(field, value)
return concept
return _transform(node, None)
core/ast/visitors.py
+122
View File
@@ -0,0 +1,122 @@
from core.ast.nodes import GenericNodeConcept, NodeConcept
from core.builtin_concepts import ListConcept
class ConceptNodeVisitor:
"""
Base class to visit NodeConcept
It is insolently inspired by python AST.Visitor class
"""
def visit(self, node):
"""Visit a node."""
name = node.node_type if isinstance(node, GenericNodeConcept) else node.name
name = str(name).capitalize()
method = 'visit_' + name
visitor = getattr(self, method, self.generic_visit)
return visitor(node)
def generic_visit(self, node):
"""Called if no explicit visitor function exists for a node."""
for field, value in iter_props(node):
if isinstance(value, ListConcept):
for item in value:
if isinstance(item, NodeConcept):
self.visit(item)
elif isinstance(value, NodeConcept):
self.visit(value)
def visit_Constant(self, node):
value = node.get_prop("value")
type_name = _const_node_type_names.get(type(value))
if type_name is None:
for cls, name in _const_node_type_names.items():
if isinstance(value, cls):
type_name = name
break
if type_name is not None:
method = 'visit_' + type_name
try:
visitor = getattr(self, method)
except AttributeError:
pass
else:
import warnings
warnings.warn(f"{method} is deprecated; add visit_Constant",
PendingDeprecationWarning, 2)
return visitor(node)
return self.generic_visit(node)
class UnreferencedNamesVisitor(ConceptNodeVisitor):
def __init__(self, sheerka):
self.names = set()
self.sheerka = sheerka
def visit_Name(self, node):
parents = get_parents(node)
if ("For", "target") in parents: # variable used by the 'for' iteration
return
if ("Call", "func") in parents: # name of the function
return
if ("Assign", "targets") in parents: # variable which is assigned
return
if self.can_be_discarded(self.sheerka.value(node), parents):
return
self.names.add(self.sheerka.value(node))
def can_be_discarded(self, variable_name, parents):
for node in (parent.node for parent in parents):
if node is None:
return False
if node.get_node_type() == "For" and self.sheerka.value(node.get_prop("target")) == variable_name:
# variable used by the loop
return True
if node.get_node_type() == "FunctionDef":
# variable defined as a function parameter
args = node.get_prop("args")
args_values = list(self.sheerka.values(args.get_prop("args")))
if variable_name in args_values:
return True
return False
def get_parents(node):
if node.parent is None:
return []
res = []
while True:
if node.parent is None:
break
res.append(node.parent)
node = node.parent.node
return res
def iter_props(node):
for p in node.props:
yield p, node.props[p].value
_const_node_type_names = {
bool: 'NameConstant', # should be before int
type(None): 'NameConstant',
int: 'Num',
float: 'Num',
complex: 'Num',
str: 'Str',
bytes: 'Bytes',
type(...): 'Ellipsis',
}
core/builtin_concepts.py
+45
View File
@@ -6,6 +6,12 @@ from core.concept import Concept
class BuiltinConcepts(Enum):
"""
List of builtin concepts that do no need any specific implementation
Please note that the value of the enum is informal. It is not used in the system
For example, the concept 'NODE' DOES NOT have the key, the id or whatever 200
The key if the name of the concept
The id is a sequential number given just before the concept is saved in sdp
The values of the enum are just a convenient way for me to group the concepts
"""
SHEERKA = 1
SUCCESS = 2
@@ -31,6 +37,12 @@ class BuiltinConcepts(Enum):
NOP = 22 # no operation concept. Does nothing
PROPERTY_EVAL_ERROR = 23 # cannot evaluate a property of a concept
ENUMERATION = 24 # represents a list or a set
LIST = 25 # represents a list
CANNOT_RESOLVE_VALUE_ERROR = 26 # In presence of a concept where the default value is not know
NODE = 200
GENERIC_NODE = 201
IDENTIFIER_NODE = 202
"""
@@ -234,3 +246,36 @@ class PropertyEvalError(Concept):
@property
def property_name(self):
return self.body
class EnumerationConcept(Concept):
def __init__(self, iteration=None):
super().__init__(BuiltinConcepts.ENUMERATION, True, False, BuiltinConcepts.ENUMERATION)
self.body = iteration
def __iter__(self):
return iter(self.body)
class ListConcept(Concept):
def __init__(self, items=None):
super().__init__(BuiltinConcepts.LIST, True, False, BuiltinConcepts.LIST)
self.body = items or []
def append(self, obj):
self.body.append(obj)
def __len__(self):
return len(self.body)
def __getitem__(self, key):
return self.body[key]
def __setitem__(self, key, value):
self.body[key] = value
def __iter__(self):
return iter(self.body)
def __contains__(self, item):
return item in self.body
core/builtin_helpers.py
+83
View File
@@ -0,0 +1,83 @@
from core.builtin_concepts import BuiltinConcepts
def is_same_success(sheerka, return_values):
"""
Returns True if all returns values are successful and have the same value
:param sheerka:
:param return_values:
:return:
"""
assert isinstance(return_values, list)
if not return_values[0].status:
return False
reference = sheerka.value(return_values[0].value)
for return_value in return_values[1:]:
if not return_value.status:
return False
actual = sheerka.value(return_value.value)
if actual != reference:
return False
return True
def expect_one(context, return_values):
"""
Checks if there is at least one success return value
If there is more than one, check if it's the same value
:param context:
:param return_values:
:return:
"""
if not isinstance(return_values, list):
return return_values
sheerka = context.sheerka
if len(return_values) == 0:
return sheerka.ret(
context.who,
False,
sheerka.new(BuiltinConcepts.IS_EMPTY, obj=return_values),
parents=return_values)
successful_results = [item for item in return_values if item.status]
number_of_successful = len(successful_results)
# total_items = len(return_values)
# remove errors when a winner is found
if number_of_successful == 1:
# log.debug(f"1 / {total_items} good item found.")
return sheerka.ret(
context.who,
True,
successful_results[0].body,
parents=return_values)
# too many winners, which one to choose ?
if number_of_successful > 1:
if is_same_success(sheerka, successful_results):
return sheerka.ret(
context.who,
True,
successful_results[0].value,
parents=return_values)
else:
return sheerka.ret(
context.who,
False,
sheerka.new(BuiltinConcepts.TOO_MANY_SUCCESS, obj=successful_results),
parents=return_values)
# only errors, i cannot help you
return sheerka.ret(
context.who,
False,
sheerka.new(BuiltinConcepts.TOO_MANY_ERRORS, obj=return_values),
parents=return_values)
core/concept.py
+5 -2
View File
@@ -190,15 +190,18 @@ class Concept:
return self
def set_prop(self, prop_name, prop_value=None):
def set_prop(self, prop_name: str, prop_value=None):
self.props[prop_name] = Property(prop_name, prop_value)
return self
def set_prop_by_index(self, index, prop_value):
def set_prop_by_index(self, index: int, prop_value):
prop_name = list(self.props.keys())[index]
self.props[prop_name] = Property(prop_name, prop_value)
return self
def get_prop(self, prop_name: str):
return self.props[prop_name].value
class Property:
"""
core/sheerka.py
+104 -50
View File
@@ -5,11 +5,14 @@ from evaluators.BaseEvaluator import OneReturnValueEvaluator
from parsers.BaseParser import BaseParser
from sdp.sheerkaDataProvider import SheerkaDataProvider, Event, SheerkaDataProviderDuplicateKeyError
import core.utils
import core.builtin_helpers
import logging
log = logging.getLogger(__name__)
concept_evaluation_steps = [BuiltinConcepts.EVALUATION, BuiltinConcepts.AFTER_EVALUATION]
class Sheerka(Concept):
"""
@@ -150,6 +153,12 @@ class Sheerka(Concept):
logging.basicConfig(format=log_format, level=log_level)
def eval(self, text):
"""
Note to KSI: If you try to add execution context to this function,
You may end in an infinite loop
:param text:
:return:
"""
evt_digest = self.sdp.save_event(Event(text))
exec_context = ExecutionContext(self.key, evt_digest, self)
@@ -174,32 +183,6 @@ class Sheerka(Concept):
return return_values
def expect_one(self, context, items):
if not isinstance(items, list):
items = [items]
if len(items) == 0:
return self.ret(context.who, False, self.new(BuiltinConcepts.IS_EMPTY, obj=items))
successful_results = [item for item in items if item.status]
number_of_successful = len(successful_results)
total_items = len(items)
# remove errors when a winner is found
if number_of_successful == 1:
# log.debug(f"1 / {total_items} good item found.")
return successful_results[0]
# too many winners, which one to choose ?
if number_of_successful > 1:
log.debug(f"{number_of_successful} / {total_items} good items. Too many success")
return self.ret(context.who, False, self.new(BuiltinConcepts.TOO_MANY_SUCCESS, obj=successful_results))
# only errors, i cannot help you
log.debug(f"{total_items} items. Only errors")
return self.ret(context.who, False, self.new(BuiltinConcepts.TOO_MANY_ERRORS, obj=items))
def parse(self, context, text):
result = []
if log.isEnabledFor(logging.DEBUG):
@@ -356,6 +339,33 @@ class Sheerka(Concept):
for prop in concept.props:
concept.codes[prop] = self.parse(context, concept.props[prop].value)
# updates the code of the reference when possible
if concept.key in self.concepts_cache:
entry = self.concepts_cache[concept.key]
if isinstance(entry, list):
# TODO : manage when there are multiple entries
pass
else:
self.concepts_cache[concept.key].codes = concept.codes
def eval_concept(self, context, concept, properties_to_eval=None):
if len(concept.codes) == 0:
self.add_codes_to_concept(context, concept)
if properties_to_eval is None:
properties_to_eval = ["where", "pre", "post", "body", "props"]
for prop in properties_to_eval:
if prop == "props":
pass
else:
part_key = ConceptParts(prop)
if concept.codes[part_key] is None:
continue
res = self.chain_process(context, concept.codes[part_key], concept_evaluation_steps)
res = core.builtin_helpers.expect_one(context, res)
setattr(concept, prop, res.value)
def add_in_cache(self, concept):
"""
Adds a concept template in cache.
@@ -413,16 +423,37 @@ class Sheerka(Concept):
"""
template = self.get(concept_key)
def new_from_template(t, k, **kwargs_):
# manage singleton
if t.is_unique:
return t
# otherwise, create another instance
concept = self.builtin_cache[k]() if k in self.builtin_cache else Concept()
concept.update_from(t)
# update the properties
for k, v in kwargs_.items():
if k in concept.props:
concept.set_prop(k, v)
elif hasattr(concept, k):
setattr(concept, k, v)
else:
return self.new(BuiltinConcepts.UNKNOWN_PROPERTY, body=k, concept=concept)
# TODO : add the concept to the list of known concepts (self.instances)
return concept
# manage concept not found
if self.isinstance(template, BuiltinConcepts.UNKNOWN_CONCEPT) and \
concept_key != BuiltinConcepts.UNKNOWN_CONCEPT:
return template
if not isinstance(template, list):
return self._new_from_template(template, concept_key, **kwargs)
return new_from_template(template, concept_key, **kwargs)
# if template is a list, it means that there a multiple concepts under the same key
concepts = [self._new_from_template(t, concept_key, **kwargs) for t in template]
concepts = [new_from_template(t, concept_key, **kwargs) for t in template]
return self.new(BuiltinConcepts.ENUMERATION, body=concepts)
def ret(self, who, status, value, message=None, parents=None):
@@ -443,6 +474,29 @@ class Sheerka(Concept):
message=message,
parents=parents)
def value(self, obj, allow_none_body=False):
if obj is None:
return None
if not isinstance(obj, Concept):
return obj
if hasattr(obj, "get_value"):
return obj.get_value()
if obj.body is not None:
return obj.body
return obj if allow_none_body else self.new(BuiltinConcepts.CANNOT_RESOLVE_VALUE_ERROR, body=obj)
def values(self, objs):
if not (isinstance(objs, list) or
self.isinstance(objs, BuiltinConcepts.LIST) or
self.isinstance(objs, BuiltinConcepts.ENUMERATION)):
objs = [objs]
return (self.value(obj) for obj in objs)
def isinstance(self, a, b):
"""
return true if the concept a is an instance of the concept b
@@ -463,6 +517,27 @@ class Sheerka(Concept):
# for example, if a is a color, it will be found the entry 'All_Colors'
return a.key == b_key
def isa(self, a, b):
"""
return true if the concept a is a b
Will handle when the keyword isa will be implemented
:param a:
:param b:
:return:
"""
if isinstance(a, BuiltinConcepts): # common KSI error ;-)
raise SyntaxError("Remember that the first parameter of isinstance MUST be a concept")
if not isinstance(a, Concept):
return False
b_key = b.key if isinstance(b, Concept) else str(b)
# TODO : manage when a is the list of all possible b
# for example, if a is a color, it will be found the entry 'All_Colors'
return a.key == b_key
def get_evaluator_name(self, name):
if self.evaluators_prefix is None:
base_evaluator_class = core.utils.get_class("evaluators.BaseEvaluator.BaseEvaluator")
@@ -486,28 +561,7 @@ class Sheerka(Concept):
else:
res.append(item)
return sorted(res, key=lambda i: i.key)
def _new_from_template(self, template, concept_key, **kwargs):
# manage singleton
if template.is_unique:
return template
# otherwise, create another instance
concept = self.builtin_cache[concept_key]() if concept_key in self.builtin_cache else Concept()
concept.update_from(template)
# update the properties
for k, v in kwargs.items():
if k in concept.props:
concept.set_prop(k, v)
elif hasattr(concept, k):
setattr(concept, k, v)
else:
return self.new(BuiltinConcepts.UNKNOWN_PROPERTY, body=k, concept=concept)
# TODO : add the concept to the list of known concepts (self.instances)
return concept
return sorted(res, key=lambda i: int(i.id))
@staticmethod
def get_builtins_classes_as_dict():
evaluators/ConceptEvaluator.py
+5 -3
View File
@@ -1,4 +1,5 @@
from core.builtin_concepts import ParserResultConcept, BuiltinConcepts
import core.builtin_helpers
from core.concept import Concept, ConceptParts
from evaluators.BaseEvaluator import OneReturnValueEvaluator
import logging
@@ -47,13 +48,14 @@ class ConceptEvaluator(OneReturnValueEvaluator):
sheerka.new(BuiltinConcepts.PROPERTY_EVAL_ERROR, body=prop, concept=concept, error=res.value),
parents=[return_value])
# Evaluate body
# Returns the concept when no body
if ConceptParts.BODY not in concept.codes:
return sheerka.ret(self.name, True, concept, parents=[return_value])
# Evaluate the body otherwise
body = concept.codes[ConceptParts.BODY]
if body is None:
return None # seems weird
raise NotImplementedError("Seems weird !")
sub_context = context.push(self.name, "Evaluating body", concept)
res = self.evaluate_parsing(sheerka, sub_context, body)
@@ -61,5 +63,5 @@ class ConceptEvaluator(OneReturnValueEvaluator):
def evaluate_parsing(self, sheerka, context, parsing_result):
res = sheerka.chain_process(context, parsing_result, self.evaluation_steps)
res = sheerka.expect_one(context, res)
res = core.builtin_helpers.expect_one(context, res)
return res
evaluators/MutipleSameSuccessEvaluator.py
+6 -15
View File
@@ -1,5 +1,6 @@
from core.builtin_concepts import BuiltinConcepts
from core.concept import Concept
import core.builtin_helpers
from evaluators.BaseEvaluator import AllReturnValuesEvaluator, BaseEvaluator
import logging
@@ -36,7 +37,7 @@ class MultipleSameSuccessEvaluator(AllReturnValuesEvaluator):
elif ret.who.startswith(BaseEvaluator.PREFIX):
if ret.status:
nb_successful_evaluators += 1
self.success.append(ret.value)
self.success.append(ret)
elif ret.who.startswith(BaseParser.PREFIX):
if ret.status:
only_parsers_in_error = False
@@ -46,19 +47,9 @@ class MultipleSameSuccessEvaluator(AllReturnValuesEvaluator):
return after_evaluation and nb_successful_evaluators > 1 and only_parsers_in_error and not unlisted
def eval(self, context, return_values):
reference = self.get_value(self.success[0])
for return_value in self.success[1:]:
actual = self.get_value(return_value)
if actual != reference:
return None
sheerka = context.sheerka
return sheerka.ret(self.name, True, reference, parents=return_values)
if core.builtin_helpers.is_same_success(sheerka, self.success):
reference = sheerka.value(self.success[0].value, allow_none_body=True)
return sheerka.ret(self.name, True, reference, parents=return_values)
@staticmethod
def get_value(obj):
if not isinstance(obj, Concept):
return obj
return obj if obj.body is None else obj.body
return None
evaluators/PythonEvaluator.py
+58 -14
View File
@@ -1,14 +1,18 @@
import copy
from core.ast.visitors import UnreferencedNamesVisitor
from core.builtin_concepts import BuiltinConcepts, ParserResultConcept
from evaluators.BaseEvaluator import OneReturnValueEvaluator
from parsers.PythonParser import PythonNode
import ast
import core.ast.nodes
import logging
log = logging.getLogger(__name__)
class PythonEvaluator(OneReturnValueEvaluator):
NAME = "Python"
def __init__(self):
@@ -22,22 +26,62 @@ class PythonEvaluator(OneReturnValueEvaluator):
def eval(self, context, return_value):
sheerka = context.sheerka
node = return_value.value.value
if isinstance(node.ast_, ast.Expression):
try:
log.debug(f"Evaluating python node {node}")
compiled = compile(node.ast_, "<string>", "eval")
evaluated = eval(compiled, {}, self.get_locals(context))
return sheerka.ret(self.name, True, evaluated, parents=[return_value])
except Exception as error:
error = sheerka.new(BuiltinConcepts.ERROR, body=error)
return sheerka.ret(self.name, False, error, parents=[return_value])
else:
return sheerka.ret(self.name, False, sheerka.new(BuiltinConcepts.ERROR), parents=[return_value])
try:
log.debug(f"Evaluating python node {node}")
my_locals = self.get_locals(context, node.ast_)
@staticmethod
def get_locals(context):
if isinstance(node.ast_, ast.Expression):
compiled = compile(node.ast_, "<string>", "eval")
evaluated = eval(compiled, {}, my_locals)
else:
evaluated = self.exec_with_return(node.ast_, my_locals)
return sheerka.ret(self.name, True, evaluated, parents=[return_value])
except Exception as error:
error = sheerka.new(BuiltinConcepts.ERROR, body=error)
return sheerka.ret(self.name, False, error, parents=[return_value])
def get_locals(self, context, ast_):
my_locals = {"sheerka": context.sheerka}
if context.obj:
for prop_name, prop_value in context.obj.props.items():
my_locals[prop_name] = prop_value.value
node_concept = core.ast.nodes.transform(ast_)
unreferenced_names_visitor = UnreferencedNamesVisitor(context.sheerka)
unreferenced_names_visitor.visit(node_concept)
for name in unreferenced_names_visitor.names:
concept = context.sheerka.new(name)
if context.sheerka.isinstance(concept, BuiltinConcepts.UNKNOWN_CONCEPT):
continue
sub_context = context.push(self.name, "Evaluating body", concept)
context.sheerka.eval_concept(sub_context, concept, ["body"])
if not context.sheerka.isa(concept.body, BuiltinConcepts.ERROR):
my_locals[name] = concept.body
return my_locals
@staticmethod
def expr_to_expression(expr):
expr.lineno = 0
expr.col_offset = 0
result = ast.Expression(expr.value, lineno=0, col_offset=0)
return result
def exec_with_return(self, code_ast, my_locals):
init_ast = copy.deepcopy(code_ast)
init_ast.body = code_ast.body[:-1]
last_ast = copy.deepcopy(code_ast)
last_ast.body = code_ast.body[-1:]
exec(compile(init_ast, "<ast>", "exec"), {}, my_locals)
if type(last_ast.body[0]) == ast.Expr:
return eval(compile(self.expr_to_expression(last_ast.body[0]), "<ast>", "eval"), {}, my_locals)
else:
exec(compile(last_ast, "<ast>", "exec"), {}, my_locals)
evaluators/TooManySuccessEvaluator.py
+5 -16
View File
@@ -1,5 +1,5 @@
from core.builtin_concepts import BuiltinConcepts
from core.concept import Concept
import core.builtin_helpers
from evaluators.BaseEvaluator import AllReturnValuesEvaluator, BaseEvaluator
import logging
@@ -46,20 +46,9 @@ class TooManySuccessEvaluator(AllReturnValuesEvaluator):
return after_evaluation and nb_successful_evaluators > 1 and only_parsers_in_error and not unlisted
def eval(self, context, return_values):
reference = self.get_value(self.success[0].value)
for return_value in self.success[1:]:
actual = self.get_value(return_value.value)
if actual != reference:
sheerka = context.sheerka
too_many_success = sheerka.new(BuiltinConcepts.TOO_MANY_SUCCESS, obj=self.success)
return sheerka.ret(self.name, False, too_many_success, parents=return_values)
sheerka = context.sheerka
if not core.builtin_helpers.is_same_success(sheerka, self.success):
too_many_success = sheerka.new(BuiltinConcepts.TOO_MANY_SUCCESS, obj=self.success)
return sheerka.ret(self.name, False, too_many_success, parents=return_values)
return None
@staticmethod
def get_value(obj):
if not isinstance(obj, Concept):
return obj
return obj if obj.body is None else obj.body
parsers/DefaultParser.py
+3 -2
View File
@@ -1,5 +1,6 @@
from core.builtin_concepts import BuiltinConcepts, ReturnValueConcept
from core.concept import ConceptParts
import core.builtin_helpers
from parsers.BaseParser import BaseParser, Node, NopNode, ErrorNode, NotInitializedNode
from core.tokenizer import Tokenizer, TokenKind, Token, Keywords
from dataclasses import dataclass, field
@@ -409,8 +410,8 @@ class DefaultParser(BaseParser):
continue
# ask the other parsers if they recognize the tokens
new_context = self.context.push(self)
parsing_result = self.sheerka.expect_one(new_context, self.sheerka.parse(new_context, tokens))
new_context = self.context.push(self.name)
parsing_result = core.builtin_helpers.expect_one(new_context, self.sheerka.parse(new_context, tokens))
if not parsing_result.status:
self.add_error(parsing_result.value)
continue
parsers/PythonParser.py
-22
View File
@@ -103,28 +103,6 @@ class PythonParser(BaseParser):
except Exception as error:
return False, None, error
def expr_to_expression(self, expr):
expr.lineno = 0
expr.col_offset = 0
result = ast.Expression(expr.value, lineno=0, col_offset=0)
return result
def exec_with_return(self, code):
code_ast = ast.parse(code)
init_ast = copy.deepcopy(code_ast)
init_ast.body = code_ast.body[:-1]
last_ast = copy.deepcopy(code_ast)
last_ast.body = code_ast.body[-1:]
exec(compile(init_ast, "<ast>", "exec"), globals())
if type(last_ast.body[0]) == ast.Expr:
return eval(compile(self.expr_to_expression(last_ast.body[0]), "<ast>", "eval"), globals())
else:
exec(compile(last_ast, "<ast>", "exec"), globals())
class PythonGetNamesVisitor(ast.NodeVisitor):
"""
tests/test_ConceptEvaluator.py
+4
View File
@@ -91,6 +91,10 @@ def test_body_is_evaluated_when_concept_body():
def test_body_is_evaluated_when_concept_body_with_a_body():
"""
The concept refers to another concept which has a body
:return:
"""
context = get_context()
concept_one = Concept(name="one", body="1").init_key()
context.sheerka.add_in_cache(concept_one)
tests/test_DefaultParser.py
+2 -2
View File
@@ -100,7 +100,7 @@ def get_concept_part(part):
if isinstance(part, str):
node = PythonNode(part, ast.parse(part, mode="eval"))
return ReturnValueConcept(
who="Parsers:PythonParser",
who="Parsers:DefaultParser",
status=True,
value=ParserResultConcept(
source=part,
@@ -109,7 +109,7 @@ def get_concept_part(part):
if isinstance(part, PythonNode):
return ReturnValueConcept(
who="Parsers:PythonParser",
who="Parsers:DefaultParser",
status=True,
value=ParserResultConcept(
source=part.source,
tests/test_PyhtonEvaluator.py
+89
View File
@@ -0,0 +1,89 @@
import pytest
import shutil
from os import path
import os
from core.builtin_concepts import ReturnValueConcept, ParserResultConcept
from core.sheerka import Sheerka, ExecutionContext
from core.concept import Concept
from evaluators.PythonEvaluator import PythonEvaluator
from parsers.PythonParser import PythonNode, PythonParser
tests_root = path.abspath("../build/tests")
root_folder = "init_folder"
@pytest.fixture(autouse=True)
def init_test():
if path.exists(tests_root):
shutil.rmtree(tests_root)
if not path.exists(tests_root):
os.makedirs(tests_root)
current_pwd = os.getcwd()
os.chdir(tests_root)
yield None
os.chdir(current_pwd)
def get_context():
sheerka = Sheerka()
sheerka.initialize(root_folder)
return ExecutionContext("test", "xxx", sheerka)
@pytest.mark.parametrize("ret_val, expected", [
(ReturnValueConcept("some_name", True, ParserResultConcept(value=PythonNode("", None))), True),
(ReturnValueConcept("some_name", True, ParserResultConcept(value="other thing")), False),
(ReturnValueConcept("some_name", False, "not relevant"), False),
(ReturnValueConcept("some_name", True, Concept()), False)
])
def test_i_can_match(ret_val, expected):
context = get_context()
assert PythonEvaluator().matches(context, ret_val) == expected
@pytest.mark.parametrize("text, expected", [
("1 + 1", 2),
("sheerka.test()", "I have access to Sheerka !"),
("a=10\na", 10),
])
def test_i_can_eval(text, expected):
context = get_context()
parsed = PythonParser().parse(context, text)
evaluated = PythonEvaluator().eval(context, parsed)
assert evaluated.status
assert evaluated.value == expected
def test_i_can_eval_expression_with_variables():
"""
I can test expression with variables
:return:
"""
context = get_context()
context.sheerka.add_in_cache(Concept("foo", body="2"))
parsed = PythonParser().parse(context, "foo + 2")
evaluated = PythonEvaluator().eval(context, parsed)
assert evaluated.status
assert evaluated.value == 4
def test_i_can_eval_module_with_variables():
"""
I can test modules with variables
:return:
"""
context = get_context()
context.sheerka.add_in_cache(Concept("foo", body="2"))
parsed = PythonParser().parse(context, "def a(b):\n return b\na(foo)")
evaluated = PythonEvaluator().eval(context, parsed)
assert evaluated.status
assert evaluated.value == 2
tests/test_ast.py
+154
View File
@@ -0,0 +1,154 @@
import os
import shutil
from os import path
import pytest
import ast
from core.ast.nodes import NodeParent, GenericNodeConcept
import core.ast.nodes
from core.ast.visitors import ConceptNodeVisitor, UnreferencedNamesVisitor
from core.builtin_concepts import BuiltinConcepts
from core.sheerka import Sheerka
tests_root = path.abspath("../build/tests")
root_folder = "init_folder"
@pytest.fixture(autouse=True)
def init_test():
if path.exists(tests_root):
shutil.rmtree(tests_root)
if not path.exists(tests_root):
os.makedirs(tests_root)
current_pwd = os.getcwd()
os.chdir(tests_root)
yield None
os.chdir(current_pwd)
def get_sheerka():
sheerka = Sheerka()
sheerka.initialize(root_folder)
return sheerka
class TestNameVisitor(ConceptNodeVisitor):
"""
Test class for a basic Visitor test
"""
def __init__(self):
self.names = []
def visit_Name(self, node):
self.names.append(node)
def test_i_can_transform_simple_ast_using_generic_node():
source = """
def my_function(a,b):
for i in range(b):
a = a+b
return a
"""
tree = ast.parse(source)
tree_as_concept = core.ast.nodes.transform(tree)
sheerka = get_sheerka()
assert tree_as_concept.node_type == "Module"
assert sheerka.isinstance(tree_as_concept.get_prop("body"), BuiltinConcepts.LIST)
def_func = tree_as_concept.get_prop("body")[0]
assert sheerka.isinstance(def_func, BuiltinConcepts.GENERIC_NODE)
assert def_func.node_type == "FunctionDef"
assert def_func.parent == NodeParent(tree_as_concept, "body")
assert def_func.get_prop("name") == "my_function"
def_func_args = def_func.get_prop("args")
assert sheerka.isinstance(def_func_args, BuiltinConcepts.GENERIC_NODE)
assert def_func_args.node_type == "arguments"
def_func_args_real_args = def_func_args.get_prop("args")
assert sheerka.isinstance(def_func_args_real_args, BuiltinConcepts.LIST)
assert len(def_func_args_real_args) == 2
assert sheerka.isinstance(def_func_args_real_args[0], BuiltinConcepts.GENERIC_NODE)
assert def_func_args_real_args[0].node_type == "arg"
assert def_func_args_real_args[0].parent == NodeParent(def_func_args, "args")
assert def_func_args_real_args[0].get_prop("arg") == "a"
assert sheerka.isinstance(def_func_args_real_args[1], BuiltinConcepts.GENERIC_NODE)
assert def_func_args_real_args[1].node_type == "arg"
assert def_func_args_real_args[1].parent == NodeParent(def_func_args, "args")
assert def_func_args_real_args[1].get_prop("arg") == "b"
def_fun_body = def_func.get_prop("body")
assert sheerka.isinstance(def_fun_body, BuiltinConcepts.LIST)
assert len(def_fun_body) == 2
def_fun_body_for = def_fun_body[0]
assert sheerka.isinstance(def_fun_body_for, BuiltinConcepts.GENERIC_NODE)
assert def_fun_body_for.node_type == "For"
assert def_fun_body_for.parent == NodeParent(def_func, "body")
def_fun_body_return = def_fun_body[1]
assert sheerka.isinstance(def_fun_body_return, BuiltinConcepts.GENERIC_NODE)
assert def_fun_body_return.node_type == "Return"
assert def_fun_body_return.parent == NodeParent(def_func, "body")
def test_i_can_visit_concept_node():
source = """
def my_function(a,b):
for i in range(b):
a = a+b
return a
"""
node = ast.parse(source)
concept_node = core.ast.nodes.transform(node)
visitor = TestNameVisitor()
visitor.visit(concept_node)
sheerka = get_sheerka()
assert sheerka.value(visitor.names[0]) == "i"
assert sheerka.value(visitor.names[1]) == "range"
assert sheerka.value(visitor.names[2]) == "b"
assert sheerka.value(visitor.names[3]) == "a"
assert sheerka.value(visitor.names[4]) == "a"
assert sheerka.value(visitor.names[5]) == "b"
assert sheerka.value(visitor.names[6]) == "a"
def test_i_can_get_non_referenced_variables():
source = """
def my_function(a,b):
for i in range(b):
a = a+b
return a
my_function(x,y)
"""
sheerka = get_sheerka()
node = ast.parse(source)
concept_node = core.ast.nodes.transform(node)
visitor = UnreferencedNamesVisitor(sheerka)
visitor.visit(concept_node)
values = visitor.names
assert len(visitor.names) == 2
assert "x" in values
assert "y" in values
def test_i_can_compare_NodeParent_with_tuple():
node_parent = NodeParent(GenericNodeConcept("For", None), "target")
assert node_parent == ("For", "target")
tests/test_builtin_helpers.py
+145
View File
@@ -0,0 +1,145 @@
import shutil
from os import path
import os
import pytest
from core.builtin_concepts import ReturnValueConcept, BuiltinConcepts
from core.sheerka import Sheerka, ExecutionContext
import core.builtin_helpers
tests_root = path.abspath("../build/tests")
root_folder = "init_folder"
@pytest.fixture(autouse=True)
def init_test():
if path.exists(tests_root):
shutil.rmtree(tests_root)
if not path.exists(tests_root):
os.makedirs(tests_root)
current_pwd = os.getcwd()
os.chdir(tests_root)
yield None
os.chdir(current_pwd)
def test_i_can_use_expect_one_when_empty():
sheerka = get_sheerka()
res = core.builtin_helpers.expect_one(get_context(sheerka), [])
assert not res.status
assert sheerka.isinstance(res.value, BuiltinConcepts.IS_EMPTY)
def test_i_can_use_expect_one_when_too_many_success():
sheerka = get_sheerka()
items = [
ReturnValueConcept("who", True, "value1"),
ReturnValueConcept("who", True, "value2"),
]
res = core.builtin_helpers.expect_one(get_context(sheerka), items)
assert not res.status
assert sheerka.isinstance(res.value, BuiltinConcepts.TOO_MANY_SUCCESS)
assert res.value.obj == items
assert res.parents == items
def test_i_can_use_expect_one_when_same_success():
sheerka = get_sheerka()
items = [
ReturnValueConcept("who", True, "value"),
ReturnValueConcept("who", True, "value"),
]
res = core.builtin_helpers.expect_one(get_context(sheerka), items)
assert res.status
assert res.value == items[0].value
assert res.parents == items
def test_i_can_use_expect_when_only_errors_1():
sheerka = get_sheerka()
items = [
ReturnValueConcept("who", False, None),
]
res = core.builtin_helpers.expect_one(get_context(sheerka), items)
assert not res.status
assert sheerka.isinstance(res.value, BuiltinConcepts.TOO_MANY_ERRORS)
assert res.value.obj == items
assert res.parents == items
def test_i_can_use_expect_when_only_errors_2():
sheerka = get_sheerka()
items = [
ReturnValueConcept("who", False, None),
ReturnValueConcept("who", False, None),
]
res = core.builtin_helpers.expect_one(get_context(sheerka), items)
assert not res.status
assert sheerka.isinstance(res.value, BuiltinConcepts.TOO_MANY_ERRORS)
assert res.value.obj == items
assert res.parents == items
def test_i_can_use_expect_one_when_one_success_1():
sheerka = get_sheerka()
items = [
ReturnValueConcept("who", True, None),
]
res = core.builtin_helpers.expect_one(get_context(sheerka), items)
assert res.status
assert res.body == items[0].body
assert res.parents == items
def test_i_can_use_expect_one_when_one_success_2():
sheerka = get_sheerka()
items = [
ReturnValueConcept("who", False, None),
ReturnValueConcept("who", True, None),
ReturnValueConcept("who", False, None),
]
res = core.builtin_helpers.expect_one(get_context(sheerka), items)
assert res.status
assert res.body == items[1].body
assert res.parents == items
def test_i_can_use_expect_one_when_not_a_list_true():
sheerka = get_sheerka()
item = ReturnValueConcept("who", True, None)
res = core.builtin_helpers.expect_one(get_context(sheerka), item)
assert res.status
assert res == item
def test_i_can_use_expect_one_when_not_a_list_false():
sheerka = get_sheerka()
item = ReturnValueConcept("who", False, None)
res = core.builtin_helpers.expect_one(get_context(sheerka), item)
assert not res.status
assert res == item
def get_sheerka():
sheerka = Sheerka()
sheerka.initialize(root_folder)
return sheerka
def get_context(sheerka):
return ExecutionContext("test", "xxx", sheerka)
tests/test_sheerka.py
+24 -79
View File
@@ -33,6 +33,11 @@ def init_test():
os.chdir(current_pwd)
class ConceptWithGetValue(Concept):
def get_value(self):
return self.get_prop("my_prop")
def test_root_folder_is_created_after_initialization():
return_value = Sheerka().initialize(root_folder)
assert return_value.status, "initialisation should be successful"
@@ -288,93 +293,33 @@ def test_i_cannot_instantiate_when_properties_are_not_recognized():
assert sheerka.isinstance(new.concept, concept)
def test_i_can_use_expect_one_when_empty():
@pytest.mark.parametrize("concept, allow_non_body, expected", [
(None, False, None),
(3.14, False, 3.14),
(Concept("name", body="foo"), False, "foo"),
(Concept("name"), True, Concept("name")),
(ConceptWithGetValue("name").set_prop("my_prop", "my_value"), True, "my_value"),
])
def test_i_can_get_value(concept, allow_non_body, expected):
sheerka = get_sheerka()
res = sheerka.expect_one(get_context(sheerka), [])
assert not res.status
assert sheerka.isinstance(res.value, BuiltinConcepts.IS_EMPTY)
assert sheerka.value(concept, allow_non_body) == expected
def test_i_can_use_expect_one_when_too_many_success():
def test_i_cannot_get_value_when_no_body_and_allow_none_body_is_false():
sheerka = get_sheerka()
concept = Concept("name")
allow_none_body = False
items = [
ReturnValueConcept("who", True, None),
ReturnValueConcept("who", True, None),
]
res = sheerka.expect_one(get_context(sheerka), items)
assert not res.status
assert sheerka.isinstance(res.value, BuiltinConcepts.TOO_MANY_SUCCESS)
assert res.value.obj == items
assert sheerka.value(concept, allow_none_body) == sheerka.new(BuiltinConcepts.CANNOT_RESOLVE_VALUE_ERROR,
body=concept)
def test_i_can_use_expect_when_only_errors_1():
sheerka = get_sheerka()
items = [
ReturnValueConcept("who", False, None),
]
res = sheerka.expect_one(get_context(sheerka), items)
assert not res.status
assert sheerka.isinstance(res.value, BuiltinConcepts.TOO_MANY_ERRORS)
assert res.value.obj == items
def test_i_can_use_expect_when_only_errors_2():
sheerka = get_sheerka()
items = [
ReturnValueConcept("who", False, None),
ReturnValueConcept("who", False, None),
]
res = sheerka.expect_one(get_context(sheerka), items)
assert not res.status
assert sheerka.isinstance(res.value, BuiltinConcepts.TOO_MANY_ERRORS)
assert res.value.obj == items
def test_i_can_use_expect_one_when_one_success_1():
sheerka = get_sheerka()
items = [
ReturnValueConcept("who", True, None),
]
res = sheerka.expect_one(get_context(sheerka), items)
assert res.status
assert res == items[0]
def test_i_can_use_expect_one_when_one_success_2():
sheerka = get_sheerka()
items = [
ReturnValueConcept("who", False, None),
ReturnValueConcept("who", True, None),
ReturnValueConcept("who", False, None),
]
res = sheerka.expect_one(get_context(sheerka), items)
assert res.status
assert res == items[1]
def test_i_can_use_expect_one_when_not_a_list_true():
sheerka = get_sheerka()
res = sheerka.expect_one(get_context(sheerka), ReturnValueConcept("who", True, None))
assert res.status
assert res == ReturnValueConcept("who", True, None)
def test_i_can_use_expect_one_when_not_a_list_false():
sheerka = get_sheerka()
res = sheerka.expect_one(get_context(sheerka), ReturnValueConcept("who", False, None))
assert not res.status
assert sheerka.isinstance(res.value, BuiltinConcepts.TOO_MANY_ERRORS)
assert res.value.obj == [ReturnValueConcept("who", False, None)]
# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
#
# E V A L U A T I O N S
#
# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
@pytest.mark.parametrize("text, expected", [
("1 + 1", 2),