Fixed #18 : Parsing and evaluating Python

src/client.py

@@ -7,7 +7,7 @@ from os import path
 
 import prompt_toolkit
 import requests
-from prompt_toolkit import HTML, print_formatted_text, prompt
+from prompt_toolkit import ANSI, print_formatted_text, prompt
 from prompt_toolkit.auto_suggest import AutoSuggestFromHistory
 from prompt_toolkit.history import FileHistory
 from requests import ConnectionError, HTTPError
@@ -16,6 +16,18 @@ from constants import CLIENT_OPERATION_QUIT, EXIT_COMMANDS
 
 connect_regex = re.compile("connect\(['\"](.*?)['\"]\s*,\s*['\"](.*?)['\"]\)")
 
+CONSOLE_COLORS_MAP = {
+    "reset": "\u001b[0m",
+    "black": "\u001b[30m",
+    "red": "\u001b[31m",
+    "green": "\u001b[32m",
+    "yellow": "\u001b[33m",
+    "blue": "\u001b[34m",
+    "magenta": "\u001b[35m",
+    "cyan": "\u001b[36m",
+    "white": "\u001b[37m",
+}
+
 
 @dataclass
 class TestResponse:
@@ -116,7 +128,11 @@ class SheerkaClient:
                     as_json = response.json()
 
                     # Print the response and loop
-                    self.print_info(as_json['response'])
+                    if as_json['status']:
+                        self.print_info(as_json['response'])
+                    else:
+                        self.print_error(as_json['response'])
+
                     if as_json['command'] == CLIENT_OPERATION_QUIT:
                         break
                 else:
@@ -142,7 +158,7 @@ class SheerkaClient:
         :return:
         :rtype:
         """
-        print_formatted_text(HTML(f'<ansired>{message}</ansired>'))
+        print_formatted_text(ANSI(f'{CONSOLE_COLORS_MAP["red"]}{message}{CONSOLE_COLORS_MAP["reset"]}'))
 
     @staticmethod
     def print_info(message: str):

src/common/ast_utils.py

@@ -0,0 +1,106 @@
+import ast
+
+from caching.FastCache import FastCache
+from common.global_symbols import NotFound
+
+
+class UnreferencedNamesVisitor(ast.NodeVisitor):
+    """
+    Try to find symbols that will be requested by the ast
+    It can be variable names, but also function names
+    """
+
+    cache = FastCache()
+
+    def __init__(self, context):
+        self.context = context
+        self.names = set()
+
+    def get_names(self, node):
+        key = self.__class__.__name__, node
+        names = UnreferencedNamesVisitor.cache.get(key)
+        if names is NotFound:
+            self.visit(node)
+            UnreferencedNamesVisitor.cache.put(key, self.names)
+            return self.names
+
+        return names
+
+    def visit_Name(self, node):
+        self.names.add(node.id)
+
+    def visit_For(self, node: ast.For):
+        self.visit_selected(node, ["body", "orelse"])
+
+    def visit_selected(self, node, to_visit):
+        """Called if no explicit visitor function exists for a node."""
+        for field in to_visit:
+            value = getattr(node, field)
+            if isinstance(value, list):
+                for item in value:
+                    if isinstance(item, ast.AST):
+                        self.visit(item)
+            elif isinstance(value, ast.AST):
+                self.visit(value)
+
+
+class UnreferencedVariablesVisitor(UnreferencedNamesVisitor):
+    """
+    Try to find variables names that will be requested by the ast
+    This visitor do not yield function names
+    """
+
+    def visit_Call(self, node: ast.Call):
+        self.visit_selected(node, ["args", "keywords"])
+
+    def visit_keyword(self, node: ast.keyword):
+        self.names.add(node.arg)
+        self.visit_selected(node, ["value"])
+
+
+class NamesWithAttributesVisitor(ast.NodeVisitor):
+    """
+    Looks for all attributes for a given name
+    >>> ast_ = ast.parse("foo.bar.baz", "<src>", mode="exec")
+    >>> assert NamesWithAttributesVisitor().get_sequences(ast_, "foo") == [["foo", "bar", "baz"]]
+
+    It parses all expressions / statements
+    >>> ast_ = ast.parse("foo.bar.baz; one.two.three; foo.bar", "<src>", mode="exec")
+    >>> assert NamesWithAttributesVisitor().get_sequences(ast_, "foo") == [["foo", "bar", "baz"], ["foo", "bar"]]
+    """
+
+    def __init__(self):
+        self.sequences = []
+        self.temp = []
+        self.to_lookup = None
+
+    def get_sequences(self, ast_, to_lookup):
+        self.to_lookup = to_lookup
+        self.visit(ast_)
+        return self.sequences
+
+    def visit_Attribute(self, node: ast.Attribute):
+        self.temp.append(node.attr)
+        if isinstance(node.value, ast.Attribute):
+            self.visit_Attribute(node.value)
+        if isinstance(node.value, ast.Subscript):
+            self.visit_Subscript(node.value)
+        elif isinstance(node.value, ast.Name):
+            self.visit_Name(node.value)
+
+    def visit_Subscript(self, node: ast.Subscript):
+        # TODO manage the index when it will be needed
+        # using node.slice
+        if isinstance(node.value, ast.Attribute):
+            self.visit_Attribute(node.value)
+        if isinstance(node.value, ast.Subscript):
+            self.visit_Subscript(node.value)
+        elif isinstance(node.value, ast.Name):
+            self.visit_Name(node.value)
+
+    def visit_Name(self, node: ast.Name):
+        if node.id == self.to_lookup:
+            self.temp.append(node.id)
+            self.temp.reverse()
+            self.sequences.append(self.temp.copy())
+        self.temp.clear()

src/common/utils.py

@@ -1,6 +1,7 @@
 import importlib
 import pkgutil
 from copy import deepcopy
+from typing import Any
 
 from common.global_symbols import CustomType
 
@@ -227,6 +228,25 @@ def unstr_concept(concept_repr, prefix='c:'):
     return key if key != "" else None, c_id if c_id != "" else None
 
 
+def encode_concept(t: tuple | Any, wrapper="C"):
+    """
+    Given a tuple of concept id, concept id
+    Create a valid Python identifier that can be parsed back
+
+    >>> assert encode_concept(("key", "id")) == "__C__KEY_key__ID_id__C__"
+    >>> assert encode_concept((None, "id")) == "__C__KEY_00None00__ID_id__C__"
+    >>> assert encode_concept(("key", None)) == "__C__KEY_key__ID_00None00__C__"
+
+    :param t:
+    :param wrapper:
+    :return:
+    """
+
+    key, id_ = (t[0], t[1]) if isinstance(t, tuple) else (t.key, t.id)
+    sanitized_key = "".join(c if c.isalnum() else "0" for c in key) if key else "00None00"
+    return f"__{wrapper}__KEY_{sanitized_key}__ID_{id_ or '00None00'}__{wrapper}__"
+
+
 def compute_hash(obj):
     """
     Helper to get the hash from collection
@@ -291,3 +311,55 @@ def to_dict(items, get_attr):
         res.setdefault(get_attr(item), []).append(item)
 
     return res
+
+
+def get_text_from_tokens(tokens, custom_switcher=None, tracker=None):
+    """
+    Create the source code, from the list of token
+    :param tokens: list of tokens
+    :param custom_switcher: to override the behaviour (the return value) of some token
+    :param tracker: keep track of the original token value when custom switched
+    :return:
+    """
+    if tokens is None:
+        return ""
+
+    if not hasattr(tokens, "__iter__"):
+        tokens = [tokens]
+
+    switcher = custom_switcher or {}
+
+    res = ""
+    for token in tokens:
+        value = switcher.get(token.type, lambda t: t.str_value)(token)
+        res += value
+        if tracker is not None and token.type in custom_switcher:
+            tracker[value] = token
+    return res
+
+
+def dict_product(a, b):
+    """
+    Cartesian product like where a and b are list of dictionaries
+    >>> a = [{"a": "a", "b":"b", "c":"c"}]
+    >>> b = [{"d":"d1"}, {"d":"d2"}]
+    >>>
+    >>> assert dict_product(a, b) == [{"a": "a", "b":"b", "c":"c", "d":"d1"}, {"a": "a", "b":"b", "c":"c", "d":"d2"}]
+
+    :param a:
+    :param b:
+    :return:
+    """
+    if a is None or len(a) == 0:
+        return b
+    if b is None or len(b) == 0:
+        return a
+
+    res = []
+    for item_a in a:
+        for item_b in b:
+            items = item_a.copy()
+            items.update(item_b)
+            res.append(items)
+
+    return res

src/core/BuiltinConcepts.py

@@ -5,3 +5,4 @@ class BuiltinConcepts:
     UNKNOWN_CONCEPT = "__UNKNOWN_CONCEPT"
     USER_INPUT = "__USER_INPUT"
     PARSER_INPUT = "__PARSER_INPUT"
+    PYTHON_CODE = "__PYTHON_CODE"

src/core/ExecutionContext.py

@@ -5,7 +5,7 @@ import time
 from core.Event import Event
 
 
-class ExecutionContextActions:
+class ContextActions:
     TESTING = "Testing"
     INIT_SHEERKA = "Init Sheerka"
     EVALUATE_USER_INPUT = "Evaluate user input"
@@ -18,9 +18,16 @@ class ExecutionContextActions:
     EVALUATION = "Evaluation"
     AFTER_EVALUATION = "After Evaluation"
 
+    EVALUATING_CONCEPT = "Evaluating concept"
+    BUILD_CONCEPT = "Building all attributes"
+    BUILD_CONCEPT_ATTR = "Building one attribute"
+    EVAL_CONCEPT = "Evaluating all attributes"
+    EVAL_CONCEPT_ATTR = "Evaluating one attribute"
+
 
 class ContextHint:
     REDUCE_CONCEPTS = "Reduce Concepts"  # to tell the process to only keep the meaningful results
+    EXPRESSION_ONLY_REQUESTED = "Expression Only"
 
 
 ids = {}  # keep track of the next execution context id, for a given event id
@@ -51,7 +58,7 @@ class ExecutionContext:
                  who: str,
                  event: Event,
                  sheerka,
-                 action: ExecutionContextActions,
+                 action: ContextActions,
                  action_context: object,
                  desc: str = None,
                  logger=None,
@@ -102,6 +109,10 @@ class ExecutionContext:
     def long_id(self):
         return f"{self.event.get_digest()}:{self._id}"
 
+    @property
+    def medium_id(self):
+        return f"{self.event.get_digest()[:8]}:{self._id}"
+
     @property
     def id(self):
         return self._id
@@ -143,7 +154,7 @@ class ExecutionContext:
 
     def push(self,
              who: str,
-             action: ExecutionContextActions,
+             action: ContextActions,
              action_context: object,
              desc: str = None,
              logger=None):
@@ -162,6 +173,9 @@ class ExecutionContext:
         self._children.append(child)
         return child
 
+    def get_parent(self):
+        return self._parent
+
     def get_children(self, level=-1):
         """
         recursively look for children
@@ -173,6 +187,30 @@ class ExecutionContext:
             if level != 1:
                 yield from child.get_children(level - 1)
 
+    def get_parents(self, level=-1):
+        """
+        recursively look for parent
+        :return:
+        :rtype:
+        """
+        if level == 0 or self._parent is None:
+            return
+
+        yield self._parent
+        yield from self._parent.get_parents(level - 1)
+
+    def in_context(self, hint: ContextHint):
+        return hint in self.protected_hints or \
+            hint in self.global_hints or \
+            hint in self.private_hints
+
+    def get_from_short_term_memory(self, key):
+        return self.sheerka.get_from_short_term_memory(self, key)
+
+    def log(self, message: str, who: str = None):
+        """Send debug information to logger"""
+        pass
+
     def __enter__(self):
         self._start = time.time_ns()
         return self

src/core/Sheerka.py

@@ -10,21 +10,21 @@ from caching.Cache import Cache
 from caching.IncCache import IncCache
 from common.utils import get_logger_name, get_sub_classes, import_module_and_sub_module
 from core.BuiltinConcepts import BuiltinConcepts
-from core.ErrorContext import ErrorContext
 from core.Event import Event
-from core.ExecutionContext import ContextHint, ExecutionContext, ExecutionContextActions
+from core.ExecutionContext import ContextHint, ExecutionContext, ContextActions
 from core.ReturnValue import ReturnValue
 from core.concept import Concept, ConceptMetadata
+from core.error import ErrorContext
 from ontologies.SheerkaOntologyManager import SheerkaOntologyManager
 from server.authentication import User
 
 EXECUTE_STEPS = [
-    ExecutionContextActions.BEFORE_PARSING,
-    ExecutionContextActions.PARSING,
-    ExecutionContextActions.AFTER_PARSING,
-    ExecutionContextActions.BEFORE_EVALUATION,
-    ExecutionContextActions.EVALUATION,
-    ExecutionContextActions.AFTER_EVALUATION
+    ContextActions.BEFORE_PARSING,
+    ContextActions.PARSING,
+    ContextActions.AFTER_PARSING,
+    ContextActions.BEFORE_EVALUATION,
+    ContextActions.EVALUATION,
+    ContextActions.AFTER_EVALUATION
 ]
 
 
@@ -122,6 +122,7 @@ class Sheerka:
             self.om = SheerkaOntologyManager(self, root_folder)
             # self.builtin_cache, self.builtin_cache_by_class_name = self.get_builtins_classes_as_dict()
 
+            self.initialize_bind_methods()
             self.initialize_caching()
             self.initialize_evaluators()
             self.initialize_services()
@@ -133,7 +134,7 @@ class Sheerka:
             with ExecutionContext(self.name,
                                   event,
                                   self,
-                                  ExecutionContextActions.INIT_SHEERKA,
+                                  ContextActions.INIT_SHEERKA,
                                   None,
                                   desc="Initializing Sheerka.") as exec_context:
                 if self.om.current_sdp().first_time:
@@ -165,6 +166,14 @@ class Sheerka:
 
         return res
 
+    def initialize_bind_methods(self):
+        """
+        Add some methods to the list of available methods
+        :return:
+        :rtype:
+        """
+        self.bind_service_method(self.name, self.echo, False)
+
     @staticmethod
     def initialize_logging(is_debug, root_folder):
         if is_debug:
@@ -206,9 +215,9 @@ class Sheerka:
         """
         self.init_log.info("Initializing services")
 
-        import_module_and_sub_module('core.services')
-        base_class = "core.services.BaseService.BaseService"
-        services = [service(self) for service in get_sub_classes("core.services", base_class)]
+        import_module_and_sub_module('services')
+        base_class = "services.BaseService.BaseService"
+        services = [service(self) for service in get_sub_classes("services", base_class)]
         services.sort(key=attrgetter("order"))
         for service in services:
             if hasattr(service, "initialize"):
@@ -282,7 +291,7 @@ class Sheerka:
         with ExecutionContext(user.email,
                               event,
                               self,
-                              ExecutionContextActions.EVALUATE_USER_INPUT,
+                              ContextActions.EVALUATE_USER_INPUT,
                               command,
                               desc=f"Evaluating '{command}'",
                               global_hints=self.global_context_hints.copy()) as exec_context:
@@ -322,3 +331,14 @@ class Sheerka:
             return a.id == b[3:-1]
 
         return a.key == b
+
+    def echo(self, msg):
+        """
+        test function
+        :param msg:
+        :type msg:
+        :return:
+        :rtype:
+        """
+
+        return msg

src/core/concept.py

@@ -14,7 +14,7 @@ class ConceptDefaultProps:
     RET = "#ret#"
 
 
-DefaultProps = [v for k, v in ConceptDefaultProps.__dict__.items() if not k.startswith("_")]
+ConceptDefaultPropsAttrs = [v for k, v in ConceptDefaultProps.__dict__.items() if not k.startswith("_")]
 
 
 class DefinitionType:
@@ -49,6 +49,9 @@ class ConceptMetadata:
     digest: str = None
     all_attrs: tuple = None
 
+    def get_metadata(self):
+        return self
+
 
 @dataclass
 class ConceptRuntimeInfo:
@@ -57,7 +60,7 @@ class ConceptRuntimeInfo:
     They are related to the instance of the concept
     """
     is_evaluated: bool = False  # True is the concept is evaluated by sheerka.eval_concept()
-    need_validation: bool = False  # True if the properties of the concept need to be validated
+    need_validation: bool = True  # True if the properties of the concept need to be validated
     recognized_by: str = None  # RECOGNIZED_BY_ID, RECOGNIZED_BY_NAME, RECOGNIZED_BY_KEY (from Sheerka.py)
 
     def copy(self):

src/core/error.py

@@ -1,9 +1,26 @@
+from dataclasses import dataclass
+
 from common.utils import compute_hash
 from core.ExecutionContext import ExecutionContext
 
 
 class SheerkaException(Exception):
-    pass
+    def get_error_msg(self) -> str:
+        pass
+
+
+class MethodAccessError(SheerkaException):
+    def __init__(self, method_name):
+        self.method_name = method_name
+
+    def get_error_msg(self) -> str:
+        return f"Cannot access method '{self.method_name}'"
+
+
+@dataclass
+class ErrorObj:
+    def get_error_msg(self) -> str:
+        pass
 
 
 class ErrorContext:
@@ -18,7 +35,7 @@ class ErrorContext:
         self.parents = None
 
     def __repr__(self):
-        return f"Error(who={self.who}, context_id={self.context.long_id}, value={self.value})"
+        return f"Error(who={self.who}, context_id={self.context.medium_id}, value={self.value})"
 
     def __eq__(self, other):
         if id(self) == id(other):
@@ -33,3 +50,16 @@ class ErrorContext:
 
     def __hash__(self):
         return hash((self.who, self.context.id, compute_hash(self.value)))
+
+    def get_error_msg(self):
+        value_as_list = self.value if isinstance(self.value, list) else [self.value]
+        temp = []
+        for value in value_as_list:
+            if isinstance(value, str):
+                temp.append(value)
+            elif isinstance(value, (SheerkaException, ErrorObj)):
+                temp.append(value.get_error_msg())
+            else:
+                temp.append(repr(value))
+
+        return ", ".join(temp)

src/core/python_fragment.py

@@ -0,0 +1,59 @@
+import ast
+import copy
+
+
+class PythonFragment:
+
+    def __init__(self, source_code, ast_tree=None, original_source=None, namespace=None):
+        self.source_code = source_code  # what was parsed
+        self.original_source = original_source or source_code  # to remember source before concepts id replacements
+        self.ast_tree = ast_tree  # if ast_ else ast.parse(source, mode="eval") if source else None
+        self.namespace = namespace or {}  # when objects (mainly concepts or rules) are recognized in the expression
+        self._compiled = None
+        self.ast_str = self.get_dump()
+
+    def __repr__(self):
+        ast_type = "expr" if isinstance(self.ast_tree, ast.Expression) else "module"
+        return "PythonNode(" + ast_type + "='" + self.source_code + "')"
+
+    def __eq__(self, other):
+        if not isinstance(other, PythonFragment):
+            return False
+
+        return self.source_code == other.source_code and self.original_source == other.original_source
+
+    def __hash__(self):
+        return hash((self.source_code, self.original_source))
+
+    def get_dump(self):
+        if not self.ast_tree:
+            return None
+
+        dump = ast.dump(self.ast_tree)
+        for to_remove in [", ctx=Load()", ", kind=None", ", type_ignores=[]"]:
+            dump = dump.replace(to_remove, "")
+        return dump
+
+    def get_compiled(self):
+        if self._compiled is None:
+            if isinstance(self.ast_tree, ast.Expression):
+                self._compiled = compile(self.ast_tree, "<string>", "eval")
+            else:
+                # in case of module, if the last expr is an expression, we want to be able to return its value
+                if isinstance(self.ast_tree.body[-1], ast.Expr):
+                    init_ast = copy.deepcopy(self.ast_tree)
+                    init_ast.body = self.ast_tree.body[:-1]
+                    last_ast = copy.deepcopy(self.ast_tree)
+                    last_ast_as_expression = self.expr_to_expression(last_ast.body[0])
+                    self._compiled = [compile(init_ast, "<ast>", "exec"),
+                                      compile(last_ast_as_expression, "<ast>", "eval")]
+                else:
+                    self._compiled = compile(self.ast_tree, "<string>", "exec")
+
+        return self._compiled
+
+    @staticmethod
+    def expr_to_expression(expr):
+        expr.lineno = 0
+        expr.col_offset = 0
+        return ast.Expression(expr.value, lineno=0, col_offset=0)

src/evaluators/CreateParserInput.py

@@ -1,6 +1,6 @@
 from core.BuiltinConcepts import BuiltinConcepts
-from core.ErrorContext import ErrorContext
-from core.ExecutionContext import ExecutionContext, ExecutionContextActions
+from core.error import ErrorContext
+from core.ExecutionContext import ExecutionContext, ContextActions
 from core.ReturnValue import ReturnValue
 from evaluators.base_evaluator import EvaluatorEvalResult, EvaluatorMatchResult, OneReturnValueEvaluator
 from parsers.ParserInput import ParserInput
@@ -10,7 +10,7 @@ class CreateParserInput(OneReturnValueEvaluator):
     NAME = "CreateParserInput"
 
     def __init__(self):
-        super().__init__(self.NAME, ExecutionContextActions.BEFORE_EVALUATION, 50)
+        super().__init__(self.NAME, ContextActions.BEFORE_PARSING, 50)
 
     def matches(self, context: ExecutionContext, return_value: ReturnValue) -> EvaluatorMatchResult:
         if return_value.status and \
@@ -26,5 +26,5 @@ class CreateParserInput(OneReturnValueEvaluator):
             return EvaluatorEvalResult([new_ret_val], [return_value])
         else:
             error = ErrorContext(self.NAME, context, parser_input)
-            new_ret_val = ReturnValue(self.NAME, False, error, parents=[return_value])
-            return EvaluatorEvalResult([new_ret_val], [return_value])
+            error_ret_val = ReturnValue(self.NAME, False, error, parents=[return_value])
+            return EvaluatorEvalResult([error_ret_val], [return_value])

src/evaluators/PythonEvaluator.py

@@ -0,0 +1,31 @@
+from core.BuiltinConcepts import BuiltinConcepts
+from core.ExecutionContext import ExecutionContext, ContextActions
+from core.ReturnValue import ReturnValue
+from core.error import ErrorContext
+from evaluators.base_evaluator import EvaluatorEvalResult, EvaluatorMatchResult, OneReturnValueEvaluator
+
+
+class PythonEvaluator(OneReturnValueEvaluator):
+    NAME = "PythonEvaluator"
+
+    def __init__(self):
+        super().__init__(self.NAME, ContextActions.EVALUATION, 50)
+
+    def matches(self, context: ExecutionContext, return_value: ReturnValue) -> EvaluatorMatchResult:
+        return EvaluatorMatchResult(return_value.status and
+                                    context.sheerka.isinstance(return_value.value, BuiltinConcepts.PYTHON_CODE))
+
+    def eval(self, context: ExecutionContext,
+             evaluation_context: object,
+             return_value: ReturnValue) -> EvaluatorEvalResult:
+
+        sheerka = context.sheerka
+        fragment = return_value.value.pf
+
+        evaluated = sheerka.evaluate_python(context, fragment)
+        if isinstance(evaluated, ErrorContext):
+            return EvaluatorEvalResult([ReturnValue(self.name, False, evaluated, parents=[return_value])],
+                                       [])
+        else:
+            return EvaluatorEvalResult([ReturnValue(self.name, True, evaluated, parents=[return_value])],
+                                       [return_value])

src/evaluators/PythonParser.py

@@ -0,0 +1,69 @@
+import ast
+from dataclasses import dataclass
+
+from common.utils import encode_concept
+from core.BuiltinConcepts import BuiltinConcepts
+from core.ExecutionContext import ExecutionContext, ContextActions
+from core.ReturnValue import ReturnValue
+from core.error import ErrorContext, ErrorObj
+from core.python_fragment import PythonFragment
+from evaluators.base_evaluator import EvaluatorEvalResult, EvaluatorMatchResult, OneReturnValueEvaluator
+from parsers.tokenizer import TokenKind
+
+
+@dataclass()
+class PythonErrorNode(ErrorObj):
+    source: str
+    exception: Exception
+
+    def get_error_msg(self) -> str:
+        return repr(self.exception)
+
+    def __eq__(self, other):
+        if not isinstance(other, PythonErrorNode):
+            return False
+
+        return self.source == other.source and self.exception == other.exception
+
+    def __hash__(self):
+        return hash((self.source, self.exception))
+
+
+class PythonParser(OneReturnValueEvaluator):
+    NAME = "PythonParser"
+
+    def __init__(self):
+        super().__init__(self.NAME, ContextActions.PARSING, 80)
+
+    def matches(self, context: ExecutionContext, return_value: ReturnValue) -> EvaluatorMatchResult:
+        return EvaluatorMatchResult(return_value.status and
+                                    context.sheerka.isinstance(return_value.value, BuiltinConcepts.PARSER_INPUT))
+
+    def eval(self, context: ExecutionContext,
+             evaluation_context: object,
+             return_value: ReturnValue) -> EvaluatorEvalResult:
+        parser_input = return_value.value.body
+
+        tracker = {}  # to keep track of concept tokens (c:xxx:)
+        python_switcher = {TokenKind.CONCEPT: lambda t: encode_concept(t.value),
+                           TokenKind.RULE: lambda t: encode_concept(t.value, "R")}
+        source_code = parser_input.as_text(python_switcher, tracker).lstrip()  # right side spaces must be kept
+
+        try:
+            ast_tree = ast.parse(source_code, f"<user input>", 'eval')
+        except:
+            try:
+                ast_tree = ast.parse(source_code, f"<user input>", 'exec')
+            except Exception as error:
+                error_context = ErrorContext(self.NAME, context, PythonErrorNode(parser_input.as_text(), error))
+                error_ret_val = ReturnValue(self.NAME, False, error_context, [return_value])
+                return EvaluatorEvalResult([error_ret_val], [])
+
+        # Successfully parsed some python code
+        python_code = context.sheerka.newn(BuiltinConcepts.PYTHON_CODE,
+                                           pf=PythonFragment(source_code,
+                                                             ast_tree,
+                                                             parser_input.original_text,
+                                                             tracker))
+        new = ReturnValue(self.NAME, True, python_code, parents=[return_value])
+        return EvaluatorEvalResult([new], [return_value])

src/evaluators/base_evaluator.py

@@ -1,6 +1,6 @@
 from dataclasses import dataclass
 
-from core.ExecutionContext import ExecutionContext, ExecutionContextActions
+from core.ExecutionContext import ExecutionContext, ContextActions
 from core.ReturnValue import ReturnValue
 
 
@@ -21,7 +21,7 @@ class BaseEvaluator:
     Base class to evaluate ReturnValues
     """
 
-    def __init__(self, name, step: ExecutionContextActions, priority: int, enabled=True):
+    def __init__(self, name, step: ContextActions, priority: int, enabled=True):
         self.name = name
         self.step = step
         self.priority = priority

src/parsers/ParserInput.py

@@ -1,3 +1,4 @@
+from common.utils import get_text_from_tokens
 from parsers.tokenizer import Tokenizer
 
 
@@ -5,15 +6,24 @@ class ParserInput:
     def __init__(self, text, yield_oef=True):
         self.original_text = text
         self.yield_oef = yield_oef
-        self.tokens = None
+        self.all_tokens = None
         self.exception = None
 
     def init(self) -> bool:
         try:
             # the eof if forced, but will not be yield if not set to.
-            self.tokens = list(Tokenizer(self.original_text, yield_eof=True))
+            self.all_tokens = list(Tokenizer(self.original_text, yield_eof=True))
             return True
         except Exception as ex:
-            self.tokens = None
+            self.all_tokens = None
             self.exception = ex
             return False
+
+    def as_text(self, custom_switcher=None, tracker=None):
+        if self.all_tokens is None:
+            raise Exception("You must call init() first !")
+
+        return get_text_from_tokens(self.all_tokens, custom_switcher, tracker)
+
+    def __repr__(self):
+        return f"ParserInput('{self.original_text}', len={len(self.all_tokens)})"

src/parsers/tokenizer.py

@@ -2,6 +2,7 @@ from dataclasses import dataclass, field
 from enum import Enum
 
 from common.global_symbols import VARIABLE_PREFIX
+from common.utils import str_concept
 
 
 class TokenKind(Enum):
@@ -11,7 +12,6 @@ class TokenKind(Enum):
     IDENTIFIER = "identifier"
     CONCEPT = "concept"
     RULE = "rule"
-    EXPR = "expression"
     STRING = "string"
     NUMBER = "number"
     LPAR = "lpar"

src/server/main.py

@@ -8,6 +8,8 @@ from starlette.middleware.cors import CORSMiddleware
 
 from constants import CLIENT_OPERATION_QUIT, EXIT_COMMANDS, SHEERKA_PORT
 from core.Sheerka import Sheerka
+from core.concept import Concept
+from core.error import ErrorContext
 from server.authentication import ACCESS_TOKEN_EXPIRE_MINUTES, User, authenticate_user, create_access_token, \
     fake_users_db, get_current_active_user
 
@@ -93,10 +95,17 @@ async def command(message: str, current_user: User = Depends(get_current_active_
             "response": "Take care.",
             "command": CLIENT_OPERATION_QUIT
         }
+
     res = sheerka.evaluate_user_input(message, current_user)
+    value = res[0].value
+    if isinstance(value, Concept) and value.get_runtime_info().is_evaluated:
+        value = value.body
+    if isinstance(value, ErrorContext):
+        value = value.get_error_msg()
+
     return {
         "status": res[0].status,
-        "response": res[0].value.body,
+        "response": value,
         "command": None
     }
 

src/services/SheerkaAdmin.py

@@ -0,0 +1,27 @@
+from services.BaseService import BaseService
+
+
+class SheerkaAdmin(BaseService):
+    """
+    Service for admin function, when using the CLI
+    """
+    NAME = "Admin"
+
+    def __init__(self, sheerka):
+        super().__init__(sheerka)
+
+    def initialize(self):
+        self.sheerka.bind_service_method(self.NAME, self.extended_isinstance, False)
+
+    def extended_isinstance(self, a, b):
+        """
+        switch between sheerka.isinstance and builtin.isinstance
+        :param a:
+        :param b:
+        :return:
+        """
+
+        if isinstance(b, (type, tuple)):
+            return isinstance(a, b)
+
+        return self.sheerka.isinstance(a, b)

src/services/SheerkaConceptEvaluator.py

@@ -0,0 +1,140 @@
+from dataclasses import dataclass
+
+from caching.FastCache import FastCache
+from core.BuiltinConcepts import BuiltinConcepts
+from core.ExecutionContext import ContextActions, ExecutionContext
+from core.ReturnValue import ReturnValue
+from core.concept import Concept, ConceptDefaultProps, ConceptDefaultPropsAttrs, ConceptMetadata
+from core.error import ErrorObj, SheerkaException
+from core.python_fragment import PythonFragment
+from services.BaseService import BaseService
+from services.SheerkaPython import EvaluationRef
+
+PARSING_STEPS = [
+    ContextActions.BEFORE_PARSING,
+    ContextActions.PARSING,
+]
+
+
+class ConceptCompiled:
+    """
+    Container for all PythonFragment
+    attribute will be accessed by setattr() and getattr()
+    """
+    pass
+
+
+@dataclass
+class ConceptEvaluationHints:
+    force_evaluation: bool = False
+
+
+class ConceptEvaluator(BaseService):
+    """
+    The service is used to evaluate a concept
+    """
+
+    NAME = "ConceptEvaluator"
+
+    def __init__(self, sheerka):
+        super().__init__(sheerka)
+        self.compiled_cache = FastCache()
+
+    def initialize(self):
+        self.sheerka.bind_service_method(self.NAME, self.evaluate_concept, True)
+
+    def evaluate_concept(self, context: ExecutionContext,
+                         concept: Concept,
+                         hints: ConceptEvaluationHints = None) -> Concept:
+        hints = hints or ConceptEvaluationHints()
+
+        with context.push(self.NAME, ContextActions.EVALUATING_CONCEPT, {"concept": concept}) as sub_context:
+
+            # if the concept is already evaluated, no need to do it again
+            if not hints.force_evaluation and concept.get_runtime_info().is_evaluated:
+                return concept
+
+            if concept.get_definition_digest() not in self.compiled_cache:
+                compiled = self.build(sub_context, concept.get_metadata())
+                self.compiled_cache.put(concept.get_definition_digest(), compiled)
+
+            self.inner_eval_concept(context, concept)
+            return concept
+
+    def build(self, context: ExecutionContext, metadata: ConceptMetadata):
+        sheerka = context.sheerka
+        action_context = {ConceptDefaultProps.WHERE: metadata.where,
+                          ConceptDefaultProps.PRE: metadata.pre,
+                          ConceptDefaultProps.BODY: metadata.body,
+                          ConceptDefaultProps.POST: metadata.post,
+                          ConceptDefaultProps.RET: metadata.ret}
+        for k, v in metadata.variables:
+            action_context[k] = v
+
+        compiled = ConceptCompiled()
+        with context.push(self.NAME, ContextActions.BUILD_CONCEPT, {"metadata": action_context}) as sub_context:
+
+            for attr, source_code in action_context.items():
+                if source_code is None or source_code == "":
+                    setattr(compiled, attr, None)
+                    continue
+
+                with sub_context.push(self.NAME, ContextActions.BUILD_CONCEPT_ATTR, {"attr": attr}) as attr_context:
+                    start = ReturnValue(self.NAME,
+                                        True,
+                                        sheerka.newn(BuiltinConcepts.USER_INPUT, command=source_code))
+
+                    # parse the code to get the python fragment
+                    attr_context.add_inputs(start=start)
+                    ret = sheerka.execute(attr_context, [start], PARSING_STEPS)
+                    attr_context.add_values(return_values=ret)
+
+                    # TODO : manage when the parsing fails
+
+                    # Add reference to internal variables
+                    python_fragment = ret[0].value.pf
+                    for k, v in metadata.variables:
+                        python_fragment.namespace[k] = EvaluationRef("self", k)
+
+                    setattr(compiled, attr, python_fragment)
+
+        return compiled
+
+    def inner_eval_concept(self, context, concept):
+        sheerka = context.sheerka
+        compiled = self.compiled_cache.get(concept.get_definition_digest())
+        compiled_debug = self._get_compiled_debug(compiled)
+
+        attributes = self._get_attributes_to_eval(context, concept)
+
+        with context.push(self.NAME, ContextActions.EVAL_CONCEPT, {"compiled": compiled_debug}) as sub_context:
+            # first evaluate the variables
+            for attr in attributes:
+                with context.push(self.NAME, ContextActions.EVAL_CONCEPT_ATTR, {"attr": attr}) as attr_context:
+                    res = sheerka.evaluate_python(sub_context,
+                                                  getattr(compiled, attr),
+                                                  {"self": concept})
+                    # TODO : manage errors
+                    concept.set_value(attr, res)
+
+        return concept
+
+    @staticmethod
+    def _get_attributes_to_eval(context, concept):
+        res = [v[0] for v in concept.get_metadata().variables]
+        res += ConceptDefaultPropsAttrs
+        return res
+
+    @staticmethod
+    def _get_compiled_debug(compiled):
+        ret = {}
+        for attr, value in vars(compiled).items():
+            if value is None:
+                ret[attr] = None
+            elif isinstance(value, (ErrorObj, SheerkaException)):
+                ret[attr] = value.get_error_msg()
+            elif isinstance(value, PythonFragment):
+                ret[attr] = value.original_source
+            else:
+                ret[attr] = repr(value)
+        return ret

src/services/SheerkaConceptManager.py

@@ -6,14 +6,14 @@ from caching.Cache import Cache
 from caching.FastCache import FastCache
 from caching.ListIfNeededCache import ListIfNeededCache
 from common.global_symbols import NotFound, NotInit, VARIABLE_PREFIX
-from common.utils import get_logger_name
+from common.utils import get_logger_name, unstr_concept
 from core.BuiltinConcepts import BuiltinConcepts
-from core.ErrorContext import ErrorContext, SheerkaException
 from core.ExecutionContext import ExecutionContext
 from core.ReturnValue import ReturnValue
-from core.concept import Concept, ConceptMetadata, DefaultProps, DefinitionType
-from core.services.BaseService import BaseService
+from core.concept import Concept, ConceptMetadata, ConceptDefaultPropsAttrs, DefinitionType
+from core.error import ErrorContext, SheerkaException
 from parsers.tokenizer import TokenKind, Tokenizer, strip_tokens
+from services.BaseService import BaseService
 
 PROPERTIES_FOR_DIGEST = ("name", "key",
                          "definition", "definition_type",
@@ -22,15 +22,22 @@ PROPERTIES_FOR_DIGEST = ("name", "key",
                          "desc", "bound_body", "autouse", "props", "variables", "parameters")
 
 
-@dataclass
 class ConceptAlreadyDefined(SheerkaException):
-    concept: ConceptMetadata
-    already_defined_id: str
+    def __init__(self, concept: ConceptMetadata, already_defined_id: str):
+        self.concept = concept
+        self.already_defined_id = already_defined_id
+
+    def get_error_msg(self) -> str:
+        return f"Concept {self.concept.name}, is already defined (id={self.already_defined_id})"
 
 
 @dataclass
 class InvalidBnf(SheerkaException):
-    bnf: str
+    def __init__(self, bnf: str):
+        self.bnf = bnf
+
+    def get_error_msg(self) -> str:
+        return f"Invalid bnf '{self.bnf}'"
 
 
 @dataclass
@@ -65,8 +72,12 @@ class ConceptManager(BaseService):
     def initialize(self):
         self.init_log.debug(f"Initializing ConceptManager, order={self.order}")
         self.sheerka.bind_service_method(self.NAME, self.define_new_concept, True)
+        self.sheerka.bind_service_method(self.NAME, self.new, True)
         self.sheerka.bind_service_method(self.NAME, self.newn, True)
         self.sheerka.bind_service_method(self.NAME, self.newi, True)
+        self.sheerka.bind_service_method(self.NAME, self.get_by_name, False)
+        self.sheerka.bind_service_method(self.NAME, self.get_by_id, False)
+        self.sheerka.bind_service_method(self.NAME, self.get_by_key, False)
 
         register_concept_cache = self.sheerka.om.register_concept_cache
 
@@ -93,6 +104,7 @@ class ConceptManager(BaseService):
             _(3, BuiltinConcepts.UNKNOWN_CONCEPT, desc="Unknown concept", variables=("requested_name", "requested_id"))
             _(4, BuiltinConcepts.USER_INPUT, desc="Any external input", variables=("command",))
             _(5, BuiltinConcepts.PARSER_INPUT, desc="tokenized input", variables=("pi",))
+            _(6, BuiltinConcepts.PYTHON_CODE, desc="python code", variables=("pf",))  # pf for PythonFragment
 
             self.init_log.debug('%s builtin concepts created',
                                 len(self.sheerka.om.current_cache_manager().concept_caches))
@@ -215,6 +227,31 @@ class ConceptManager(BaseService):
             return self._inner_new(self.get_by_name(BuiltinConcepts.UNKNOWN_CONCEPT), requested_id=concept_id)
         return self._inner_new(metadata, **kwargs)
 
+    def new(self, identifier, **kwargs):
+        """
+        Try to resolve the instantiation of a concept
+        :param identifier:
+        :type identifier:
+        :param kwargs:
+        :type kwargs:
+        :return:
+        :rtype:
+        """
+        if isinstance(identifier, ConceptMetadata):
+            return self._inner_new(identifier, **kwargs)
+
+        if (tmp := unstr_concept(identifier)) != (None, None):
+            # manage c:name#id:
+            identifier = tmp
+
+        if isinstance(identifier, tuple):
+            return self.newi(identifier[1], **kwargs) if identifier[1] else self.newn(identifier[0], **kwargs)
+
+        if isinstance(identifier, str):
+            return self.newn(identifier, **kwargs)
+
+        return self._inner_new(self.get_by_name(BuiltinConcepts.UNKNOWN_CONCEPT), requested_name=identifier)
+
     def get_by_name(self, key: str):
         """
         Returns a concept metadata, using its name
@@ -245,6 +282,9 @@ class ConceptManager(BaseService):
         """
         return self.sheerka.om.get(self.CONCEPTS_BY_KEY_ENTRY, key)
 
+    def get_all_concepts(self):
+        return list(sorted(self.sheerka.om.list(self.CONCEPTS_BY_ID_ENTRY), key=lambda item: int(item.id)))
+
     @staticmethod
     def compute_metadata_digest(metadata: ConceptMetadata):
         """
@@ -265,7 +305,7 @@ class ConceptManager(BaseService):
         :return:
         :rtype:
         """
-        all_attrs = DefaultProps.copy()
+        all_attrs = ConceptDefaultPropsAttrs.copy()
         if variables:
             all_attrs += [k for k, v in variables]
 

src/services/SheerkaEngine.py

@@ -1,10 +1,11 @@
 from dataclasses import dataclass
+from itertools import chain
 
 from common.utils import to_dict
-from core.ExecutionContext import ExecutionContext, ExecutionContextActions
+from core.ExecutionContext import ExecutionContext, ContextActions
 from core.ReturnValue import ReturnValue
-from core.services.BaseService import BaseService
 from evaluators.base_evaluator import AllReturnValuesEvaluator, BaseEvaluator, OneReturnValueEvaluator
+from services.BaseService import BaseService
 
 
 @dataclass
@@ -16,8 +17,7 @@ class EvaluationPlan:
 class SheerkaEngine(BaseService):
     """
     This service is used to process user input
-    It is responsible to parse and evaluate the information
-    It also holds the rule engine
+    It is responsible for parsing and evaluating the commands
     """
     NAME = "Engine"
 
@@ -33,7 +33,7 @@ class SheerkaEngine(BaseService):
     def call_evaluators(self,
                         context: ExecutionContext,
                         return_values: list[ReturnValue],
-                        step: ExecutionContextActions):
+                        step: ContextActions):
         """
         Calls all evaluators defined for a given step
         :param context:
@@ -50,7 +50,7 @@ class SheerkaEngine(BaseService):
         iteration = 0
         while True:
             with context.push(self.NAME,
-                              ExecutionContextActions.EVALUATING_ITERATION,
+                              ContextActions.EVALUATING_ITERATION,
                               {"step": step, "iteration": iteration},
                               desc=f"iteration #{iteration}") as iteration_context:
                 simple_digest = return_values.copy()
@@ -99,18 +99,24 @@ class SheerkaEngine(BaseService):
 
                 iteration_context.add_values(return_values=return_values.copy())
 
-            iteration += 1
+            # to avoid infinite loop
+            # plus already evaluated ret_val must not be evaluated a second time
+            already_evaluated = set(chain.from_iterable(r.parents for r in return_values if r.parents))
+            return_values = list(filter(lambda ret_val: ret_val not in already_evaluated, return_values))
+
             if simple_digest == return_values:
                 # I can use a variable like 'has_changed', but I think that this comparison is explicit
                 # It explains that I stay in the loop if something was modified
                 break
 
+            iteration += 1
+
         return return_values
 
     def execute(self,
                 context: ExecutionContext,
                 return_values: list[ReturnValue],
-                steps: list[ExecutionContextActions]):
+                steps: list[ContextActions]):
         """
         Runs the processing engine on the return_values
         :param context:
@@ -124,7 +130,7 @@ class SheerkaEngine(BaseService):
         """
         for step in steps:
             copy = return_values.copy()
-            with context.push(self.NAME, ExecutionContextActions.EVALUATING_STEP, {"step": step}) as sub_context:
+            with context.push(self.NAME, ContextActions.EVALUATING_STEP, {"step": step}) as sub_context:
                 sub_context.add_inputs(return_values=copy)
 
                 return_values = self.call_evaluators(sub_context, return_values, step)
@@ -134,7 +140,7 @@ class SheerkaEngine(BaseService):
 
         return return_values
 
-    def get_evaluation_plan(self, context: ExecutionContext, step: ExecutionContextActions) -> EvaluationPlan:
+    def get_evaluation_plan(self, context: ExecutionContext, step: ContextActions) -> EvaluationPlan:
         if step not in self.execution_plan:
             return self.no_evaluation_plan
 

src/services/SheerkaMemory.py

@@ -0,0 +1,73 @@
+from typing import Any
+
+from caching.FastCache import FastCache
+from common.global_symbols import NotFound
+from core.ExecutionContext import ExecutionContext
+from services.BaseService import BaseService
+
+
+class SheerkaMemory(BaseService):
+    """
+    The purpose of this service is to remember things
+    There are two types of memory
+        * short term memory : that are not persisted
+        * long term memory : that are sent to sdp (through the OntologyManager)
+    Short term memory is also use to store PythonEvaluator results
+    """
+    NAME = "Memory"
+    GLOBAL = "global"  # for short term memory with no context (global variable across user inputs)
+
+    OBJECTS_ENTRY = "Memory:Objects"
+
+    def __init__(self, sheerka):
+        super().__init__(sheerka, order=13)
+        self.short_term_objects = FastCache()
+
+    def initialize(self):
+        self.sheerka.bind_service_method(self.NAME, self.get_from_short_term_memory, False, visible=False)
+        self.sheerka.bind_service_method(self.NAME, self.add_to_short_term_memory, True, visible=False)
+        self.sheerka.bind_service_method(self.NAME, self.list_short_term_memory, False, visible=False)
+
+    def get_from_short_term_memory(self, context: ExecutionContext | None, key: str) -> Any:
+        while True:
+            try:
+                id_to_use = context.id if context else self.GLOBAL
+                return self.short_term_objects.cache[id_to_use][key]
+            except KeyError:
+                if context is None:
+                    return NotFound
+
+                context = context.get_parent()
+
+    def add_to_short_term_memory(self, context: ExecutionContext | None, key: str, value: Any):
+        if context:
+            context.stm = True
+            id_to_use = context.id
+        else:
+            id_to_use = SheerkaMemory.GLOBAL
+
+        if id_to_use in self.short_term_objects.cache:
+            self.short_term_objects.cache[id_to_use][key] = value
+        else:
+            self.short_term_objects.put(id_to_use, {key: value})
+
+    def list_short_term_memory(self, context: ExecutionContext | None):
+        """
+        list all short term memory data (stm data)
+        :param context:
+        :type context:
+        :return:
+        :rtype:
+        """
+        res = self.short_term_objects.cache[self.GLOBAL].copy() if self.GLOBAL in self.short_term_objects.cache else {}
+        if context is None:
+            return res
+
+        contexts = [context] + list(context.get_parents())
+        for ec in reversed(contexts):
+            try:
+                res.update(self.short_term_objects.cache[ec.id])
+            except KeyError:
+                pass
+
+        return res

src/services/SheerkaPython.py

@@ -0,0 +1,384 @@
+import ast
+import functools
+import traceback
+from dataclasses import dataclass, field
+
+from common.ast_utils import NamesWithAttributesVisitor, UnreferencedNamesVisitor
+from common.global_symbols import NoFirstToken, NotFound, NotInit, Removed
+from common.utils import dict_product
+from core.BuiltinConcepts import BuiltinConcepts
+from core.ExecutionContext import ContextHint, ExecutionContext
+from core.concept import Concept
+from core.error import ErrorContext, ErrorObj, MethodAccessError
+from core.python_fragment import PythonFragment
+from services.BaseService import BaseService
+
+TO_DISABLED = ["breakpoint", "callable", "compile", "delattr", "eval", "exec", "exit", "input", "locals", "open",
+               "print", "quit", "setattr"]
+
+
+class ReservedNotInitClass:
+    pass
+
+
+ReservedNotInit = ReservedNotInitClass()
+
+sheerka_globals = {
+    "Concept": Concept,
+    "BuiltinConcepts": BuiltinConcepts,
+    "NotInit": NotInit,
+    "NotFound": NotFound,
+    "Removed": Removed,
+    "NoFirstToken": NoFirstToken,
+}
+sheerka_globals.update(dict(__builtins__))
+
+
+class Expando:
+    def __init__(self, name, bag):
+        self.__name = name
+        for k, v in bag.items():
+            setattr(self, k, v)
+
+    def __repr__(self):
+        return f"{vars(self)}"
+
+    def get_name(self):
+        return self.__name
+
+    def __eq__(self, other):
+        if id(other) == id(self):
+            return True
+
+        if not isinstance(other, Expando):
+            return False
+
+        if other.get_name() != self.get_name():
+            return False
+
+        for k, v in vars(self).items():
+            if getattr(other, k) != v:
+                return False
+
+        return True
+
+    def __hash__(self):
+        hash_content = [self.__name] + list(vars(self).keys())
+        return hash(tuple(hash_content))
+
+
+@dataclass
+class PythonEvalError(ErrorObj):
+    error: Exception
+    source: str
+    traceback: str = field(repr=False)
+    concepts: dict | None = field(repr=False)
+
+    def __eq__(self, other):
+        if id(self) == id(other):
+            return True
+
+        if not isinstance(other, PythonEvalError):
+            return False
+
+        return isinstance(self.error, type(other.error)) and \
+            self.source == other.source and \
+            self.traceback == other.traceback and \
+            self.concepts == other.concepts
+
+    def __hash__(self):
+        return hash(self.error)
+
+    def get_error(self):
+        return self.error
+
+    def get_error_msg(self):
+        return ", ".join(self.error.args)
+
+
+@dataclass
+class EvaluationRef:
+    root: str
+    attr: str
+
+    def __eq__(self, other):
+        if not isinstance(other, EvaluationRef):
+            return False
+
+        return self.root == other.root and self.attr == other.attr
+
+    def __hash__(self):
+        return hash((self.root, self.attr))
+
+
+class SheerkaPython(BaseService):
+    """
+    This service manage evaluation of python fragments
+    """
+    NAME = "PythonEvaluator"
+
+    def __init__(self, sheerka):
+        super().__init__(sheerka)
+
+    def initialize(self):
+        self.sheerka.bind_service_method(self.NAME, self.evaluate_python, False, visible=False)
+
+    def evaluate_python(self, context: ExecutionContext, fragment: PythonFragment, global_namespace=None):
+        sheerka = context.sheerka
+        expression_only = False
+        global_namespace = global_namespace or {}
+
+        try:
+            my_globals = self.get_globals(context, fragment, global_namespace, expression_only)
+        except MethodAccessError as ex:
+            if context.in_context(ContextHint.EXPRESSION_ONLY_REQUESTED):
+                # Quick and dirty,
+                # When expression_only, it's normal to have some NameError exceptions
+                error = ErrorContext(self.NAME, context, ex)
+            else:
+                eval_error = PythonEvalError(ex, fragment.source_code, traceback.format_exc(), None)
+                error = ErrorContext(self.NAME, context, eval_error)
+
+            return error
+
+        all_possible_globals = self.get_all_possible_globals(context, my_globals)
+        expect_success = True
+        concepts_entries = None
+        errors = []
+        evaluated = ReservedNotInit
+        my_locals = None
+
+        for globals_ in all_possible_globals:
+            try:
+                # eval
+                tmp_locals = {}
+                evaluated = self.evaluate_ast(fragment, globals_, tmp_locals)
+                my_locals = tmp_locals
+
+                if not expect_success or evaluated:
+                    # in this first version, we stop once a success is found
+                    # it may not be the best result !
+                    break
+
+            except Exception as ex:
+                if concepts_entries is None:
+                    # I don't want to init it if no error is raised
+                    concepts_entries = self.get_concepts_entries_from_globals(my_globals)
+                eval_error = PythonEvalError(ex,
+                                             fragment.source_code,
+                                             traceback.format_exc(),
+                                             self.get_concepts_values_from_globals(globals_, concepts_entries))
+                errors.append(eval_error)
+
+        # add local namespace to stm
+        if my_locals:
+            for k, v in my_locals.items():
+                sheerka.add_to_short_term_memory(context, k, v)
+
+        return ErrorContext(self.NAME, context, errors) if evaluated == ReservedNotInit else evaluated
+
+    def get_globals(self, context, fragment, global_namespace, expression_only):
+        """
+        Creates the globals variables
+        :param context:
+        :param fragment:
+        :type fragment:
+        :param global_namespace:
+        :type global_namespace:
+        :param expression_only:
+        :return:
+        """
+        unreferenced_names_visitor = UnreferencedNamesVisitor(context)
+        names = unreferenced_names_visitor.get_names(fragment.ast_tree)
+        if "sheerka" in names:
+            sheerka_names = set()
+            visitor = NamesWithAttributesVisitor()
+            for sequence in visitor.get_sequences(fragment.ast_tree, "sheerka"):
+                if len(sequence) > 1:
+                    sheerka_names.add(sequence[1])
+        else:
+            sheerka_names = None
+
+        return self.create_namespace(context,
+                                     names,  # names to look for
+                                     sheerka_names,  # sheerka methods
+                                     fragment.namespace,  # objects from python fragment => local namespace
+                                     global_namespace,  # global namespace
+                                     expression_only)
+
+    def get_sheerka_method(self, context, who, name, expression_only):
+        try:
+            method = context.sheerka.sheerka_methods[name]
+            if expression_only and method.has_side_effect:
+                raise MethodAccessError(name)
+            else:
+                method_to_use = self.inject_context(context)(method.method) \
+                    if name in context.sheerka.methods_with_context \
+                    else method.method
+
+                return method_to_use
+        except KeyError:
+            return None
+
+    def create_namespace(self, context,
+                         names: list,
+                         sheerka_objects: dict | None,
+                         local_namespace: dict,
+                         global_namespace: dict,
+                         expression_only: bool):
+        """
+        Create a namespace for the requested names
+        :param context:
+        :param names: requested names
+        :param sheerka_objects: requested sheerka names (ex sheerka.isinstance)
+        :param local_namespace:
+        :type local_namespace:
+        :param global_namespace:
+        :type global_namespace:
+        :param expression_only: if true, discard method that can alter the global state
+        :return:
+        """
+        result = {}
+
+        for name in names:
+            if name == "in_context":
+                result[name] = context.in_context
+                continue
+
+            # need to add it manually to avoid conflict with sheerka.isinstance
+            if name == "isinstance":
+                result["isinstance"] = context.sheerka.extended_isinstance
+                continue
+
+            if not (expression_only and name in TO_DISABLED) and name in sheerka_globals:
+                result[name] = sheerka_globals[name]
+                continue
+
+            # support reference to sheerka
+            if name.lower() == "sheerka":
+                bag = {}
+                for sheerka_name in sheerka_objects:
+                    if (method := self.get_sheerka_method(context,
+                                                          context.who,
+                                                          sheerka_name,
+                                                          expression_only)) is not None:
+                        bag[sheerka_name] = method
+                result[name] = Expando("sheerka", bag)
+                continue
+
+            # search in short term memory
+            if (obj := context.get_from_short_term_memory(name)) is not NotFound:
+                context.log(f"Resolving '{name}'. Using value found in STM.")
+                result[name] = obj
+                continue
+            #
+            # # search in memory
+            # if (obj := context.sheerka.get_last_from_memory(context, name)) is not NotFound:
+            #     context.log(f"Resolving '{name}'. Using value found in Long Term Memory.", who)
+            #     result[name] = obj.obj
+            #     continue
+
+            # search in sheerka methods
+            if (method := self.get_sheerka_method(context, context.who, name, expression_only)) is not None:
+                result[name] = method
+                continue
+
+            # search in current node (if the name was found during the parsing)
+            # Local namespace references must be evaluated
+            if name in local_namespace:
+                context.log(f"Resolving '{name}'. Using value from local namespace.")
+                result[name] = self.resolve_object(context, name, local_namespace[name], global_namespace)
+                continue
+
+            # global namespace references are returned as is
+            if name in global_namespace:
+                result[name] = global_namespace[name]
+                continue
+
+            # at last, try to instantiate a new concept
+            if (metadata := context.sheerka.get_by_name(name)) != NotFound:
+                context.log(f"Resolving '{name}'. Instantiating new concept.")
+                result[name] = context.sheerka.new(metadata)
+
+            context.log(f"...'{name}' is not found or cannot be instantiated. Skipping.")
+
+        return result
+
+    @staticmethod
+    def resolve_object(context, attr_name, to_resolve, global_namespace):
+        if isinstance(to_resolve, EvaluationRef):
+            return getattr(global_namespace[to_resolve.root], to_resolve.attr)
+
+        raise AttributeError(attr_name)
+
+    @staticmethod
+    def evaluate_ast(fragment, my_globals, my_locals):
+        compiled = fragment.get_compiled()
+
+        if isinstance(compiled, list):
+            exec(compiled[0], my_globals, my_locals)
+            return eval(compiled[1], my_globals, my_locals)
+        elif isinstance(fragment.ast_tree, ast.Expression):
+            return eval(compiled, my_globals, my_locals)
+        else:
+            exec(compiled, my_globals, my_locals)
+
+    @staticmethod
+    def get_all_possible_globals(context, my_globals):
+        """
+        From a dictionary of globals (str, obj)
+        Creates as many globals as there are combination between a concept and its body
+        Example:
+            if the entry 'foo': Concept("foo", body="something")
+            2 globals will be created
+            one with foo: Concept("foo")  # we keep the concept as an object
+            one with foo: 'something'   # we substitute its value
+        :param context:
+        :param my_globals:
+        :return:
+        """
+
+        # first pass, get all the non concepts or concepts without a body
+        # Note that we consider that all concepts are evaluated
+        # In the future, it may be a good optimisation to defer the evaluation of the body
+        # until the python evaluation fails
+        fixed_values = {}
+        concepts_with_body = {}
+        for k, v in my_globals.items():
+            if not isinstance(v, Concept) or not v.get_runtime_info().is_evaluated or v.body is NotInit:
+                fixed_values[k] = v
+            else:
+                concepts_with_body[k] = v
+
+        # make the product the rest as cartesian product
+        res = [fixed_values]
+        for k, v in concepts_with_body.items():
+            res = dict_product(res, [{k: v}, {k: context.sheerka.objvalue(v)}])
+
+        return res
+
+    @staticmethod
+    def inject_context(context):
+        """
+        function Decorator used to inject the context in methods that needed
+        TODO : Maybe replace by 'partial' from functool
+        :param context:
+        :return:
+        """
+
+        def wrapped(func):
+            @functools.wraps(func)
+            def inner(*args, **kwargs):
+                return func(context, *args, **kwargs)
+
+            return inner
+
+        return wrapped
+
+    @staticmethod
+    def get_concepts_entries_from_globals(my_globals):
+        return [k for k, v in my_globals.items() if isinstance(v, Concept)]
+
+    @staticmethod
+    def get_concepts_values_from_globals(my_globals, names):
+        return {name: my_globals[name] for name in names}