fixed #18 : I can evaluate concept

src/common/ast_utils.py

@@ -1,9 +1,42 @@
 import ast
+from _ast import BoolOp
+from dataclasses import dataclass
+from typing import Any
 
 from caching.FastCache import FastCache
 from common.global_symbols import NotFound
 
 
+class NamesInExpressionVisitor(ast.NodeVisitor):
+
+    def __init__(self):
+        self._names = set()
+
+    def get_names(self, node):
+        self.visit(node)
+        return self._names
+
+    def visit_Name(self, node):
+        self._names.add(node.id)
+
+    def visit_Call(self, node: ast.Call):
+        self.visit_selected(node, ["args", "keywords"])
+
+    def visit_For(self, node: ast.For):
+        self.visit_selected(node, ["body", "orelse"])
+
+    def visit_selected(self, node, to_visit):
+        """Called if no explicit visitor function exists for a node."""
+        for field in to_visit:
+            value = getattr(node, field)
+            if isinstance(value, list):
+                for item in value:
+                    if isinstance(item, ast.AST):
+                        self.visit(item)
+            elif isinstance(value, ast.AST):
+                self.visit(value)
+
+
 class UnreferencedNamesVisitor(ast.NodeVisitor):
     """
     Try to find symbols that will be requested by the ast
@@ -12,8 +45,7 @@ class UnreferencedNamesVisitor(ast.NodeVisitor):
 
     cache = FastCache()
 
-    def __init__(self, context):
-        self.context = context
+    def __init__(self):
         self.names = set()
 
     def get_names(self, node):
@@ -46,7 +78,7 @@ class UnreferencedNamesVisitor(ast.NodeVisitor):
 
 class UnreferencedVariablesVisitor(UnreferencedNamesVisitor):
     """
-    Try to find variables names that will be requested by the ast
+    Try to find names that will be requested by the ast in module
     This visitor do not yield function names
     """
 
@@ -54,6 +86,14 @@ class UnreferencedVariablesVisitor(UnreferencedNamesVisitor):
         self.visit_selected(node, ["args", "keywords"])
 
     def visit_keyword(self, node: ast.keyword):
+        """
+        Keyword are parameters that are defined with a double star (**) in function / method definition
+        ex: def fun(positional, *args, **keywords)
+        :param node:
+        :type node:
+        :return:
+        :rtype:
+        """
         self.names.add(node.arg)
         self.visit_selected(node, ["value"])
 
@@ -104,3 +144,70 @@ class NamesWithAttributesVisitor(ast.NodeVisitor):
             self.temp.reverse()
             self.sequences.append(self.temp.copy())
         self.temp.clear()
+
+
+class WhereConstraintVisitor(ast.NodeVisitor):
+    """
+    Parse an expression (undefined behaviour for module)
+    in order to look for expr
+    ex :
+    >>> exp = "isinstance(x, Concept) and isinstance(y, Concept)"
+    >>> ast_tree = ast.parse(exp, "<user input>", 'eval')
+    >>> visitor = WhereConstraintVisitor(ast_tree)
+    >>> assert visitor.get_constraints(exp) == {"x" : WhereConstraint("isinstance(x, Concept)"),
+    >>>                                         "y" : WhereConstraint("isinstance(y, Concept)")}
+    """
+
+    @dataclass
+    class WhereConstraint:
+        source_code: str
+        ast_tree: object = None
+
+        def __repr__(self):
+            return f"WhereConstraint({self.source_code})"
+
+        def __eq__(self, other):
+            if not isinstance(other, WhereConstraintVisitor.WhereConstraint):
+                return False
+
+            return self.source_code == other.source_code
+
+        def __hash__(self):
+            return hash(self.source_code)
+
+    def __init__(self, ast_tree):
+        self.constraints = {}
+        self.ast_tree = ast_tree
+
+    def get_constraints(self):
+        if self.ast_tree is None:
+            return self.constraints
+
+        self.visit(self.ast_tree)
+
+        if self.constraints == {}:
+            self.create_constraint(self.ast_tree)
+
+        return self.constraints
+
+    def visit_BoolOp(self, node: BoolOp) -> Any:
+        if node.op.__class__.__name__ != "And":  # failed to properly compare the type !
+            raise NotImplementedError("Cannot manage other than 'and' expression")
+        for exp in node.values:
+            self.create_constraint(exp)
+
+    def create_constraint(self, node):
+        source_code = ast.unparse(node)
+        if not isinstance(node, ast.Expression):
+            node.lineno = 0
+            node.col_offset = 1
+            node_to_use = ast.Expression(node, lineno=0, col_offset=1)
+        else:
+            node_to_use = node
+
+        constraint = WhereConstraintVisitor.WhereConstraint(source_code, node_to_use)
+
+        name_visitor = NamesInExpressionVisitor()
+        names = name_visitor.get_names(node_to_use)
+        for name in names:
+            self.constraints.setdefault(name, []).append(constraint)

src/core/BuiltinConcepts.py

@@ -1,8 +1,12 @@
 class BuiltinConcepts:
     SHEERKA = "__SHEERKA"
 
-    NEW_CONCEPT = "__NEW_CONCEPT"
-    UNKNOWN_CONCEPT = "__UNKNOWN_CONCEPT"
-    USER_INPUT = "__USER_INPUT"
-    PARSER_INPUT = "__PARSER_INPUT"
-    PYTHON_CODE = "__PYTHON_CODE"
+    NEW_CONCEPT = "__NEW_CONCEPT"  # when the definition of a new concept is added
+    UNKNOWN_CONCEPT = "__UNKNOWN_CONCEPT"  # Failed to find the requested concept
+    USER_INPUT = "__USER_INPUT"  # user command
+    PARSER_INPUT = "__PARSER_INPUT"  # command that will be parsed
+    PYTHON_CODE = "__PYTHON_CODE"  # command that is parsed
+
+    INVALID_CONCEPT = "__INVALID_CONCEPT"  # failed to parse concept attributes
+    EVALUATION_ERROR = "__EVALUATION_ERROR"  # failed to evaluate concept
+

src/core/ExecutionContext.py

@@ -18,11 +18,13 @@ class ContextActions:
     EVALUATION = "Evaluation"
     AFTER_EVALUATION = "After Evaluation"
 
+    EVALUATING_PYTHON = "Evaluating python"
+
     EVALUATING_CONCEPT = "Evaluating concept"
     BUILD_CONCEPT = "Building all attributes"
-    BUILD_CONCEPT_ATTR = "Building one attribute"
+    BUILD_CONCEPT_ATTR = "Building an attribute"
     EVAL_CONCEPT = "Evaluating all attributes"
-    EVAL_CONCEPT_ATTR = "Evaluating one attribute"
+    EVAL_CONCEPT_ATTR = "Evaluating an attribute"
 
 
 class ContextHint:
@@ -204,9 +206,6 @@ class ExecutionContext:
             hint in self.global_hints or \
             hint in self.private_hints
 
-    def get_from_short_term_memory(self, key):
-        return self.sheerka.get_from_short_term_memory(self, key)
-
     def log(self, message: str, who: str = None):
         """Send debug information to logger"""
         pass

src/core/Sheerka.py

@@ -11,7 +11,7 @@ from caching.IncCache import IncCache
 from common.utils import get_logger_name, get_sub_classes, import_module_and_sub_module
 from core.BuiltinConcepts import BuiltinConcepts
 from core.Event import Event
-from core.ExecutionContext import ContextHint, ExecutionContext, ContextActions
+from core.ExecutionContext import ContextActions, ContextHint, ExecutionContext
 from core.ReturnValue import ReturnValue
 from core.concept import Concept, ConceptMetadata
 from core.error import ErrorContext
@@ -332,6 +332,18 @@ class Sheerka:
 
         return a.key == b
 
+    def objvalue(self, obj):
+        if not isinstance(obj, Concept):
+            return obj
+
+        if obj.get_runtime_info().error is not None:
+            return obj.get_runtime_info().error
+
+        if not obj.get_runtime_info().is_evaluated:
+            return obj
+
+        return self.objvalue(obj.body) if isinstance(obj.body, Concept) else obj.body
+
     def echo(self, msg):
         """
         test function

src/core/concept.py

@@ -1,4 +1,5 @@
-from dataclasses import dataclass
+from dataclasses import dataclass, field
+from typing import Any
 
 from common.global_symbols import NotFound, NotInit
 
@@ -60,13 +61,13 @@ class ConceptRuntimeInfo:
     They are related to the instance of the concept
     """
     is_evaluated: bool = False  # True is the concept is evaluated by sheerka.eval_concept()
-    need_validation: bool = True  # True if the properties of the concept need to be validated
-    recognized_by: str = None  # RECOGNIZED_BY_ID, RECOGNIZED_BY_NAME, RECOGNIZED_BY_KEY (from Sheerka.py)
+    error: Any = None  # when failed to evaluate the concept
+    info: dict = field(default_factory=dict)  # give context of why 'where' or 'pre' constraints fail
 
     def copy(self):
         return ConceptRuntimeInfo(self.is_evaluated,
-                                  self.need_validation,
-                                  self.recognized_by)
+                                  self.error,
+                                  self.info)
 
 
 class Concept:
@@ -79,21 +80,19 @@ class Concept:
     def __init__(self, metadata: ConceptMetadata):
 
         self._metadata: ConceptMetadata = metadata
-        self._compiled = {}  # cached ast for the where, pre, post and body parts and variables
-        self._compiled_context_hints = {}  # context hints to use when evaluating compiled
         self._bnf = None  # compiled bnf expression
         self._runtime_info = ConceptRuntimeInfo()  # runtime settings for the concept
         self._all_attrs = None
 
     def __repr__(self):
-        text = f"({self._metadata.id}){self._metadata.name}"
+        text = f"(Concept {self._metadata.name}#{self._metadata.id}"
         if self._metadata.pre:
             text += f", #pre={self._metadata.pre}"
 
         for attr in [attr for attr in self.all_attrs() if not attr.startswith("#")]:
             text += f", {attr}={self.get_value(attr)}"
 
-        return text
+        return text + ")"
 
     def __eq__(self, other):
         # I don't want this test to be part of the recursion
@@ -116,7 +115,7 @@ class Concept:
                 return True
 
             # 1. in order for two concepts to be equal, they must have the same definition
-            # 2. They must have the same properties and variables
+            # 2. They must have the same variables
             if left.get_definition_digest() != right.get_definition_digest():
                 return False
 

src/core/error.py

@@ -1,6 +1,7 @@
 from dataclasses import dataclass
 
 from common.utils import compute_hash
+from core.BuiltinConcepts import BuiltinConcepts
 from core.ExecutionContext import ExecutionContext
 
 
@@ -23,7 +24,7 @@ class ErrorObj:
         pass
 
 
-class ErrorContext:
+class ErrorContext(ErrorObj):
     """
     This class represents the result of a data flow processing
     """
@@ -63,3 +64,8 @@ class ErrorContext:
                 temp.append(repr(value))
 
         return ", ".join(temp)
+
+
+ErrorConcepts = {BuiltinConcepts.UNKNOWN_CONCEPT,
+                 BuiltinConcepts.EVALUATION_ERROR,
+                 BuiltinConcepts.INVALID_CONCEPT}

src/core/python_fragment.py

@@ -14,7 +14,7 @@ class PythonFragment:
 
     def __repr__(self):
         ast_type = "expr" if isinstance(self.ast_tree, ast.Expression) else "module"
-        return "PythonNode(" + ast_type + "='" + self.source_code + "')"
+        return f"PythonFragment({ast_type}='{self.source_code}', namespace={tuple(self.namespace.keys())})"
 
     def __eq__(self, other):
         if not isinstance(other, PythonFragment):

src/evaluators/PythonEvaluator.py

@@ -3,6 +3,7 @@ from core.ExecutionContext import ExecutionContext, ContextActions
 from core.ReturnValue import ReturnValue
 from core.error import ErrorContext
 from evaluators.base_evaluator import EvaluatorEvalResult, EvaluatorMatchResult, OneReturnValueEvaluator
+from services.SheerkaPython import EvaluationContext
 
 
 class PythonEvaluator(OneReturnValueEvaluator):
@@ -22,7 +23,7 @@ class PythonEvaluator(OneReturnValueEvaluator):
         sheerka = context.sheerka
         fragment = return_value.value.pf
 
-        evaluated = sheerka.evaluate_python(context, fragment)
+        evaluated = sheerka.evaluate_python(context, EvaluationContext(), fragment)
         if isinstance(evaluated, ErrorContext):
             return EvaluatorEvalResult([ReturnValue(self.name, False, evaluated, parents=[return_value])],
                                        [])

src/evaluators/PythonParser.py

@@ -3,7 +3,7 @@ from dataclasses import dataclass
 
 from common.utils import encode_concept
 from core.BuiltinConcepts import BuiltinConcepts
-from core.ExecutionContext import ExecutionContext, ContextActions
+from core.ExecutionContext import ContextActions, ExecutionContext
 from core.ReturnValue import ReturnValue
 from core.error import ErrorContext, ErrorObj
 from core.python_fragment import PythonFragment
@@ -50,10 +50,10 @@ class PythonParser(OneReturnValueEvaluator):
         source_code = parser_input.as_text(python_switcher, tracker).lstrip()  # right side spaces must be kept
 
         try:
-            ast_tree = ast.parse(source_code, f"<user input>", 'eval')
+            ast_tree = ast.parse(source_code, "<user input>", 'eval')
         except:
             try:
-                ast_tree = ast.parse(source_code, f"<user input>", 'exec')
+                ast_tree = ast.parse(source_code, "<user input>", 'exec')
             except Exception as error:
                 error_context = ErrorContext(self.NAME, context, PythonErrorNode(parser_input.as_text(), error))
                 error_ret_val = ReturnValue(self.NAME, False, error_context, [return_value])

src/ontologies/SheerkaOntologyManager.py

@@ -1,6 +1,7 @@
 from caching.Cache import Cache
 from caching.CacheManager import CacheManager
 from caching.DictionaryCache import DictionaryCache
+from caching.FastCache import FastCache
 from caching.SetCache import SetCache
 from common.global_symbols import EVENT_CONCEPT_ID_DELETED, \
     EVENT_RULE_ID_DELETED, NotFound, \
@@ -65,6 +66,7 @@ class Ontology:
         self.cache_manager = cache_manager
         self.alt_sdp = alt_sdp
         self.concepts_attributes = None
+        self.fast_cache = FastCache()
 
     def __repr__(self):
         return f"Ontology('{self.name}')"

src/services/SheerkaConceptEvaluator.py

@@ -1,6 +1,8 @@
 from dataclasses import dataclass
 
 from caching.FastCache import FastCache
+from common.ast_utils import WhereConstraintVisitor
+from common.global_symbols import CustomType, NotFound, NotInit
 from core.BuiltinConcepts import BuiltinConcepts
 from core.ExecutionContext import ContextActions, ExecutionContext
 from core.ReturnValue import ReturnValue
@@ -8,20 +10,24 @@ from core.concept import Concept, ConceptDefaultProps, ConceptDefaultPropsAttrs,
 from core.error import ErrorObj, SheerkaException
 from core.python_fragment import PythonFragment
 from services.BaseService import BaseService
-from services.SheerkaPython import EvaluationRef
+from services.SheerkaPython import EvalMethod, EvaluationContext, EvaluationRef, MultipleResults
 
 PARSING_STEPS = [
     ContextActions.BEFORE_PARSING,
     ContextActions.PARSING,
 ]
 
+CONDITIONAL_ATTR = [ConceptDefaultProps.WHERE, ConceptDefaultProps.PRE]
+
 
 class ConceptCompiled:
     """
     Container for all PythonFragment
     attribute will be accessed by setattr() and getattr()
     """
-    pass
+
+    def __init__(self):
+        self.errors = {}
 
 
 @dataclass
@@ -29,6 +35,40 @@ class ConceptEvaluationHints:
     force_evaluation: bool = False
 
 
+@dataclass
+class PredicateIsFalse(ErrorObj):
+    """
+    This error class is issued when a 'pre' or 'where' constraint fails
+    """
+    attr: str = None
+    predicate: str = None
+    namespace: dict = None
+
+    def get_error_msg(self) -> str:
+        return f"Failed to match condition '{self.predicate}' with namespace {self.namespace}."
+
+
+@dataclass
+class InfiniteRecursion(ErrorObj):
+    """
+    This error class is issued when an infinite recursion is detected during evaluation
+    """
+    ids: list
+
+
+@dataclass
+class PredicateIsTrue:
+    """
+    Information class to trace that a 'pre' or 'where' constraint passes
+    """
+    attr: str = None
+    predicate: str = None
+    namespace: dict = None
+
+    def __repr__(self):
+        return f"(PredicateIsTrue predicate='{self.predicate}', namespace={self.namespace}"
+
+
 class ConceptEvaluator(BaseService):
     """
     The service is used to evaluate a concept
@@ -39,29 +79,56 @@ class ConceptEvaluator(BaseService):
     def __init__(self, sheerka):
         super().__init__(sheerka)
         self.compiled_cache = FastCache()
+        self.where_constraints_cache = FastCache(default=None)
 
     def initialize(self):
         self.sheerka.bind_service_method(self.NAME, self.evaluate_concept, True)
 
     def evaluate_concept(self, context: ExecutionContext,
                          concept: Concept,
-                         hints: ConceptEvaluationHints = None) -> Concept:
+                         hints: ConceptEvaluationHints = None):
+        context.log(f"Evaluating concept '{concept}'")
+
         hints = hints or ConceptEvaluationHints()
 
+        # if the concept is already evaluated, no need to do it again
+        if not hints.force_evaluation and concept.get_runtime_info().is_evaluated:
+            return concept
+
+        # try to detect infinite recursion
+        if ids := self._detect_recursion(context, concept.id):
+            error = InfiniteRecursion(ids=ids)
+            concept.get_runtime_info().error = error
+            concept.get_runtime_info().is_evaluated = True
+            return context.sheerka.newn(BuiltinConcepts.EVALUATION_ERROR, concept=concept, reason=error)
+
         with context.push(self.NAME, ContextActions.EVALUATING_CONCEPT, {"concept": concept}) as sub_context:
 
-            # if the concept is already evaluated, no need to do it again
-            if not hints.force_evaluation and concept.get_runtime_info().is_evaluated:
-                return concept
-
+            # build the python fragments, if needed
             if concept.get_definition_digest() not in self.compiled_cache:
-                compiled = self.build(sub_context, concept.get_metadata())
+                # build compiled code if not done yet
+                compiled = self._build_attributes(sub_context, concept.get_metadata())
                 self.compiled_cache.put(concept.get_definition_digest(), compiled)
 
-            self.inner_eval_concept(context, concept)
-            return concept
+                # build the where constraints
+                if ((where_pf := getattr(compiled, ConceptDefaultProps.WHERE)) is not None
+                        and isinstance(where_pf, PythonFragment)):
+                    where_constraints = self._build_where_constraints(where_pf.ast_tree)
+                    self.where_constraints_cache.put(concept.get_definition_digest(), where_constraints)
 
-    def build(self, context: ExecutionContext, metadata: ConceptMetadata):
+            # eval variables and attributes
+            return self._evaluate_attributes(sub_context, concept)
+
+    def _build_attributes(self, context: ExecutionContext, metadata: ConceptMetadata):
+        """
+        get the compiled code var all variables and concept attributes
+        :param context:
+        :type context:
+        :param metadata:
+        :type metadata:
+        :return:
+        :rtype:
+        """
         sheerka = context.sheerka
         action_context = {ConceptDefaultProps.WHERE: metadata.where,
                           ConceptDefaultProps.PRE: metadata.pre,
@@ -69,7 +136,7 @@ class ConceptEvaluator(BaseService):
                           ConceptDefaultProps.POST: metadata.post,
                           ConceptDefaultProps.RET: metadata.ret}
         for k, v in metadata.variables:
-            action_context[k] = v
+            action_context[k] = self._get_source_code(v)
 
         compiled = ConceptCompiled()
         with context.push(self.NAME, ContextActions.BUILD_CONCEPT, {"metadata": action_context}) as sub_context:
@@ -89,38 +156,167 @@ class ConceptEvaluator(BaseService):
                     ret = sheerka.execute(attr_context, [start], PARSING_STEPS)
                     attr_context.add_values(return_values=ret)
 
-                    # TODO : manage when the parsing fails
+                    value = ret[0].value
+                    if isinstance(value, ErrorObj):
+                        setattr(compiled, attr, value)
+                        compiled.errors[attr] = value.get_error_msg()
+                    else:
+                        # Add reference to internal variables
+                        python_fragment = value.pf
+                        for k, v in metadata.variables:
+                            python_fragment.namespace[k] = EvaluationRef("self", k)
 
-                    # Add reference to internal variables
-                    python_fragment = ret[0].value.pf
-                    for k, v in metadata.variables:
-                        python_fragment.namespace[k] = EvaluationRef("self", k)
-
-                    setattr(compiled, attr, python_fragment)
+                        setattr(compiled, attr, python_fragment)
 
         return compiled
 
-    def inner_eval_concept(self, context, concept):
+    def _get_where_constraints(self, concept, attr):
+        constraints = self.where_constraints_cache.get(concept.get_definition_digest())
+        if constraints is NotFound:
+            return None
+
+        return getattr(constraints, attr) if hasattr(constraints, attr) else None
+
+    def _evaluate_attributes(self, context: ExecutionContext, concept: Concept):
+        """
+        Evaluate the attributes, in the correct order
+        :param context:
+        :type context:
+        :param concept:
+        :type concept:
+        :return:
+        :rtype:
+        """
         sheerka = context.sheerka
         compiled = self.compiled_cache.get(concept.get_definition_digest())
+
+        # no need to evaluate if an error was found during the parsing
+        if len(compiled.errors) > 0:
+            invalid = sheerka.newn(BuiltinConcepts.INVALID_CONCEPT, concept_id=concept.id, reason=compiled.errors)
+            concept.get_runtime_info().error = invalid
+            concept.get_runtime_info().is_evaluated = True
+            return invalid
+
+        # evaluate every attribute and variable
         compiled_debug = self._get_compiled_debug(compiled)
+        action_debug = {"concept": concept, "compiled": compiled_debug}
+        with context.push(self.NAME, ContextActions.EVAL_CONCEPT, action_debug) as sub_context:
+            attributes = self._get_attributes_to_eval(context, concept)
+            errors = {}
 
-        attributes = self._get_attributes_to_eval(context, concept)
-
-        with context.push(self.NAME, ContextActions.EVAL_CONCEPT, {"compiled": compiled_debug}) as sub_context:
-            # first evaluate the variables
             for attr in attributes:
-                with context.push(self.NAME, ContextActions.EVAL_CONCEPT_ATTR, {"attr": attr}) as attr_context:
-                    res = sheerka.evaluate_python(sub_context,
-                                                  getattr(compiled, attr),
+                compiled_attr = getattr(compiled, attr)
+
+                if compiled_attr is None:
+                    concept.set_value(attr, NotInit)
+                    continue
+
+                action_debug = {"concept": concept, "attr": attr, "compiled": compiled_debug[attr]}
+                with sub_context.push(self.NAME, ContextActions.EVAL_CONCEPT_ATTR, action_debug) as attr_context:
+                    eval_method = EvalMethod.All if attr[0] != "#" else EvalMethod.UntilSuccess
+                    eval_context = EvaluationContext(eval_method=eval_method)
+                    res = sheerka.evaluate_python(attr_context,
+                                                  eval_context,
+                                                  compiled_attr,
                                                   {"self": concept})
-                    # TODO : manage errors
+
+                    # when there are multiple result, use what was found in the 'where' constraint
+                    # to select the correct value
+                    if (attr_constraints := self._get_where_constraints(concept, attr)) is not None:
+                        res = self._apply_attr_constraints(context, attr_constraints, attr, res)
+
+                    if isinstance(res, ErrorObj):
+                        errors[attr] = res
+                        concept.set_value(attr, NotInit)
+                        res = NotInit
+
                     concept.set_value(attr, res)
 
-        return concept
+                    # stops if 'where' or 'pre' fails
+                    if attr in CONDITIONAL_ATTR:
+                        if sheerka.objvalue(res) is True:
+                            success = PredicateIsTrue(attr, compiled_attr.source_code)
+                            concept.get_runtime_info().info.setdefault(attr, []).append(success)
+                        else:
+                            error = PredicateIsFalse(attr, compiled_attr.source_code)
+                            errors[attr] = error
+                            break  # no need to continue in case of failure
+
+        concept.get_runtime_info().is_evaluated = True
+
+        if errors:
+            error_concept = sheerka.newn(BuiltinConcepts.EVALUATION_ERROR, concept=concept, reason=errors)
+            concept.get_runtime_info().error = errors
+            return error_concept
+        elif context.sheerka.isinstance(error_in_body := concept.body, BuiltinConcepts.EVALUATION_ERROR):
+            # if the body is an 'evaluation_error', it needs to be propagated
+            concept.get_runtime_info().error = error_in_body.reason
+            return error_in_body
+
+        if (ret := concept.get_value(ConceptDefaultProps.RET)) is NotInit:
+            return concept
+        else:
+            return ret
 
     @staticmethod
-    def _get_attributes_to_eval(context, concept):
+    def _detect_recursion(context, current_concept_id):
+        ids = []
+        c = context
+        while True:
+            c = c.get_parent()
+            if c is None:
+                return None
+
+            if c.action == ContextActions.EVALUATING_CONCEPT:
+                concept_id = c.action_context["concept"].id
+                ids.append(concept_id)
+                if concept_id == current_concept_id:
+                    break
+
+        return list(reversed(ids))
+
+    @staticmethod
+    def _build_where_constraints(ast_tree):
+        visitor = WhereConstraintVisitor(ast_tree)
+        result = ConceptCompiled()
+        for attr, constraints in visitor.get_constraints().items():
+            python_fragments = [PythonFragment(c.source_code, c.ast_tree) for c in constraints]
+            setattr(result, attr, python_fragments)
+
+        return result
+
+    @staticmethod
+    def _apply_attr_constraints(context, constraints: list, attr: str, res):
+        original_list = res
+        items = res.items if isinstance(res, MultipleResults) else [res]
+        validated = None
+
+        for constraint in constraints:
+            validated = []
+            for item in items:
+                res = context.sheerka.evaluate_python(context,
+                                                      EvaluationContext(expression_only=True),
+                                                      constraint,
+                                                      {attr: item})
+                if res is True:
+                    validated.append(item)
+
+            if len(validated) == 0:
+                # stop the validation if no match was found
+                break
+
+            items = validated
+
+        if len(validated) == 1:
+            return validated[0]
+        elif len(validated) == 0:
+            constraints_str = ' and '.join([c.source_code for c in constraints])
+            return PredicateIsFalse(attr, constraints_str, {attr: original_list})
+        else:
+            return MultipleResults(*validated)
+
+    @staticmethod
+    def _get_attributes_to_eval(context, concept: Concept):
         res = [v[0] for v in concept.get_metadata().variables]
         res += ConceptDefaultPropsAttrs
         return res
@@ -138,3 +334,19 @@ class ConceptEvaluator(BaseService):
             else:
                 ret[attr] = repr(value)
         return ret
+
+    @staticmethod
+    def _get_source_code(obj):
+        """
+        Use to manage when variable default value is NotInit (or other custom type)
+        :param obj:
+        :type obj:
+        :return:
+        :rtype:
+        """
+        if isinstance(obj, str):
+            return obj
+        if isinstance(obj, CustomType):
+            return repr(obj)[2:-2]
+
+        raise Exception(f"Cannot manage '{obj}' when parsing concept attributes.")

src/services/SheerkaConceptManager.py

@@ -10,7 +10,7 @@ from common.utils import get_logger_name, unstr_concept
 from core.BuiltinConcepts import BuiltinConcepts
 from core.ExecutionContext import ExecutionContext
 from core.ReturnValue import ReturnValue
-from core.concept import Concept, ConceptMetadata, ConceptDefaultPropsAttrs, DefinitionType
+from core.concept import Concept, ConceptDefaultPropsAttrs, ConceptMetadata, DefinitionType
 from core.error import ErrorContext, SheerkaException
 from parsers.tokenizer import TokenKind, Tokenizer, strip_tokens
 from services.BaseService import BaseService
@@ -105,6 +105,9 @@ class ConceptManager(BaseService):
             _(4, BuiltinConcepts.USER_INPUT, desc="Any external input", variables=("command",))
             _(5, BuiltinConcepts.PARSER_INPUT, desc="tokenized input", variables=("pi",))
             _(6, BuiltinConcepts.PYTHON_CODE, desc="python code", variables=("pf",))  # pf for PythonFragment
+            _(7, BuiltinConcepts.INVALID_CONCEPT, desc="invalid concept", variables=("concept_id", "reason"))
+            _(8, BuiltinConcepts.EVALUATION_ERROR, desc="evaluation error", variables=("concept", "reason"))
+
 
             self.init_log.debug('%s builtin concepts created',
                                 len(self.sheerka.om.current_cache_manager().concept_caches))
@@ -237,8 +240,11 @@ class ConceptManager(BaseService):
         :return:
         :rtype:
         """
-        if isinstance(identifier, ConceptMetadata):
-            return self._inner_new(identifier, **kwargs)
+        if isinstance(identifier, (ConceptMetadata, Concept)):
+            return self._inner_new(identifier.get_metadata(), **kwargs)
+
+        if isinstance(identifier, list):
+            return [self.new(item, **kwargs) for item in identifier]
 
         if (tmp := unstr_concept(identifier)) != (None, None):
             # manage c:name#id:
@@ -380,4 +386,9 @@ class ConceptManager(BaseService):
         concept = Concept(_metadata_def)
         for k, v in kwargs.items():
             concept.set_value(k, v)
+
+        if kwargs:
+            # if an attribute is set, the concept is considered as evaluated
+            concept.get_runtime_info().is_evaluated = True
+
         return concept

src/services/SheerkaMemory.py

@@ -1,8 +1,5 @@
 from typing import Any
 
-from caching.FastCache import FastCache
-from common.global_symbols import NotFound
-from core.ExecutionContext import ExecutionContext
 from services.BaseService import BaseService
 
 
@@ -21,53 +18,13 @@ class SheerkaMemory(BaseService):
 
     def __init__(self, sheerka):
         super().__init__(sheerka, order=13)
-        self.short_term_objects = FastCache()
 
     def initialize(self):
         self.sheerka.bind_service_method(self.NAME, self.get_from_short_term_memory, False, visible=False)
         self.sheerka.bind_service_method(self.NAME, self.add_to_short_term_memory, True, visible=False)
-        self.sheerka.bind_service_method(self.NAME, self.list_short_term_memory, False, visible=False)
 
-    def get_from_short_term_memory(self, context: ExecutionContext | None, key: str) -> Any:
-        while True:
-            try:
-                id_to_use = context.id if context else self.GLOBAL
-                return self.short_term_objects.cache[id_to_use][key]
-            except KeyError:
-                if context is None:
-                    return NotFound
+    def get_from_short_term_memory(self, key: str) -> Any:
+        return self.sheerka.om.current_ontology().fast_cache.get(key)
 
-                context = context.get_parent()
-
-    def add_to_short_term_memory(self, context: ExecutionContext | None, key: str, value: Any):
-        if context:
-            context.stm = True
-            id_to_use = context.id
-        else:
-            id_to_use = SheerkaMemory.GLOBAL
-
-        if id_to_use in self.short_term_objects.cache:
-            self.short_term_objects.cache[id_to_use][key] = value
-        else:
-            self.short_term_objects.put(id_to_use, {key: value})
-
-    def list_short_term_memory(self, context: ExecutionContext | None):
-        """
-        list all short term memory data (stm data)
-        :param context:
-        :type context:
-        :return:
-        :rtype:
-        """
-        res = self.short_term_objects.cache[self.GLOBAL].copy() if self.GLOBAL in self.short_term_objects.cache else {}
-        if context is None:
-            return res
-
-        contexts = [context] + list(context.get_parents())
-        for ec in reversed(contexts):
-            try:
-                res.update(self.short_term_objects.cache[ec.id])
-            except KeyError:
-                pass
-
-        return res
+    def add_to_short_term_memory(self, key: str, value: Any):
+        self.sheerka.om.current_ontology().fast_cache.put(key, value)

src/services/SheerkaPython.py

@@ -7,10 +7,11 @@ from common.ast_utils import NamesWithAttributesVisitor, UnreferencedNamesVisito
 from common.global_symbols import NoFirstToken, NotFound, NotInit, Removed
 from common.utils import dict_product
 from core.BuiltinConcepts import BuiltinConcepts
-from core.ExecutionContext import ContextHint, ExecutionContext
+from core.ExecutionContext import ContextActions, ContextHint, ExecutionContext
 from core.concept import Concept
-from core.error import ErrorContext, ErrorObj, MethodAccessError
+from core.error import ErrorConcepts, ErrorContext, ErrorObj, MethodAccessError
 from core.python_fragment import PythonFragment
+from parsers.tokenizer import Token, TokenKind
 from services.BaseService import BaseService
 
 TO_DISABLED = ["breakpoint", "callable", "compile", "delattr", "eval", "exec", "exit", "input", "locals", "open",
@@ -111,6 +112,48 @@ class EvaluationRef:
         return hash((self.root, self.attr))
 
 
+class EvalMethod:
+    UntilSuccess = "EvalUntilSuccess"
+    UntilTrue = "EvalUntilTrue"
+    All = "EvalAll"
+
+
+@dataclass
+class EvaluationContext:
+    expression_only: bool = False  # methods with side effect are not allowed
+    eval_method: EvalMethod = EvalMethod.UntilSuccess
+
+
+class MultipleResults:
+    def __init__(self, *args):
+        self.items = args
+
+    def __iter__(self):
+        return iter(self.items)
+
+    def __repr__(self):
+        return f"MultipleResults({', '.join([repr(item) for item in self.items])})"
+
+    def __eq__(self, other):
+        if not isinstance(other, MultipleResults):
+            return False
+
+        if len(other.items) != len(self.items):
+            return False
+
+        for _self, _other in zip(self.items, other.items):
+            if _self != _other:
+                return False
+
+        return True
+
+    def __hash__(self):
+        return hash(tuple(self.items))
+
+    def concepts_only(self):
+        return MultipleResults(*[item for item in self.items if isinstance(item, Concept)])
+
+
 class SheerkaPython(BaseService):
     """
     This service manage evaluation of python fragments
@@ -123,59 +166,87 @@ class SheerkaPython(BaseService):
     def initialize(self):
         self.sheerka.bind_service_method(self.NAME, self.evaluate_python, False, visible=False)
 
-    def evaluate_python(self, context: ExecutionContext, fragment: PythonFragment, global_namespace=None):
+    def evaluate_python(self, context: ExecutionContext,
+                        eval_context: EvaluationContext,
+                        fragment: PythonFragment,
+                        global_namespace=None):
         sheerka = context.sheerka
-        expression_only = False
         global_namespace = global_namespace or {}
+        context.log(f"Evaluating python fragment {fragment.source_code}'")
 
-        try:
-            my_globals = self.get_globals(context, fragment, global_namespace, expression_only)
-        except MethodAccessError as ex:
-            if context.in_context(ContextHint.EXPRESSION_ONLY_REQUESTED):
-                # Quick and dirty,
-                # When expression_only, it's normal to have some NameError exceptions
-                error = ErrorContext(self.NAME, context, ex)
-            else:
-                eval_error = PythonEvalError(ex, fragment.source_code, traceback.format_exc(), None)
-                error = ErrorContext(self.NAME, context, eval_error)
+        with context.push(self.NAME, ContextActions.EVALUATING_PYTHON, {"fragment": fragment}) as sub_context:
+            sub_context.add_inputs(eval_context=eval_context, global_namespace=global_namespace)
 
-            return error
-
-        all_possible_globals = self.get_all_possible_globals(context, my_globals)
-        expect_success = True
-        concepts_entries = None
-        errors = []
-        evaluated = ReservedNotInit
-        my_locals = None
-
-        for globals_ in all_possible_globals:
             try:
-                # eval
-                tmp_locals = {}
-                evaluated = self.evaluate_ast(fragment, globals_, tmp_locals)
-                my_locals = tmp_locals
+                my_namespace = self.get_globals(sub_context, fragment, global_namespace, eval_context.expression_only)
+            except MethodAccessError as ex:
+                if sub_context.in_context(ContextHint.EXPRESSION_ONLY_REQUESTED):
+                    # Quick and dirty,
+                    # When expression_only, it's normal to have some NameError exceptions
+                    error = ErrorContext(self.NAME, sub_context, ex)
+                else:
+                    eval_error = PythonEvalError(ex, fragment.source_code, traceback.format_exc(), None)
+                    error = ErrorContext(self.NAME, sub_context, eval_error)
 
-                if not expect_success or evaluated:
-                    # in this first version, we stop once a success is found
-                    # it may not be the best result !
-                    break
+                return error
 
-            except Exception as ex:
-                if concepts_entries is None:
-                    # I don't want to init it if no error is raised
-                    concepts_entries = self.get_concepts_entries_from_globals(my_globals)
-                eval_error = PythonEvalError(ex,
-                                             fragment.source_code,
-                                             traceback.format_exc(),
-                                             self.get_concepts_values_from_globals(globals_, concepts_entries))
-                errors.append(eval_error)
+            concepts_entries = None
+            errors = []
+            evaluated = []
+            my_locals = None
+            all_results = []
 
-        # add local namespace to stm
-        if my_locals:
-            for k, v in my_locals.items():
-                sheerka.add_to_short_term_memory(context, k, v)
+            all_possible_namespaces = self.get_all_possible_namespaces(sub_context, my_namespace)
 
-        return ErrorContext(self.NAME, context, errors) if evaluated == ReservedNotInit else evaluated
+            for globals_ in all_possible_namespaces:
+                try:
+                    # eval
+                    tmp_locals = {}
+                    res = self.evaluate_ast(fragment, globals_, tmp_locals)
+                    all_results.append(res)
+                    my_locals = tmp_locals
+
+                    if isinstance(res, ErrorObj):
+                        errors.append(res)
+                    else:
+                        evaluated.append(res)
+                        if eval_context.eval_method == EvalMethod.UntilSuccess:
+                            break
+
+                        if res is True and eval_context.eval_method == EvalMethod.UntilTrue:
+                            break
+
+                except Exception as ex:
+                    if concepts_entries is None:
+                        concepts_entries = self.get_concepts_entries_from_globals(my_namespace)
+
+                    eval_error = PythonEvalError(ex,
+                                                 fragment.source_code,
+                                                 traceback.format_exc(),
+                                                 self.get_concepts_values_from_globals(concepts_entries, globals_))
+                    all_results.append(eval_error)
+                    errors.append(eval_error)
+
+            # add local namespace to stm
+            if my_locals:
+                for k, v in my_locals.items():
+                    sheerka.add_to_short_term_memory(k, v)
+
+            sub_context.add_values(all_results=all_results,
+                                   evaluated=evaluated,
+                                   errors=errors,
+                                   my_locals=my_locals)
+
+            if not evaluated:
+                # when no practical result if found, return error
+                return ErrorContext(self.NAME, sub_context, errors)
+
+            if len(evaluated) == 1:
+                # Only one result, Yahoo !
+                return evaluated[0]
+
+            return evaluated[-1] is True if eval_context.eval_method == EvalMethod.UntilTrue \
+                else MultipleResults(*evaluated)
 
     def get_globals(self, context, fragment, global_namespace, expression_only):
         """
@@ -188,41 +259,32 @@ class SheerkaPython(BaseService):
         :param expression_only:
         :return:
         """
-        unreferenced_names_visitor = UnreferencedNamesVisitor(context)
+        unreferenced_names_visitor = UnreferencedNamesVisitor()
         names = unreferenced_names_visitor.get_names(fragment.ast_tree)
         if "sheerka" in names:
-            sheerka_names = set()
+            sheerka_requested_names = set()
             visitor = NamesWithAttributesVisitor()
             for sequence in visitor.get_sequences(fragment.ast_tree, "sheerka"):
                 if len(sequence) > 1:
-                    sheerka_names.add(sequence[1])
+                    sheerka_requested_names.add(sequence[1])
         else:
-            sheerka_names = None
+            sheerka_requested_names = None
 
         return self.create_namespace(context,
                                      names,  # names to look for
-                                     sheerka_names,  # sheerka methods
+                                     sheerka_requested_names,  # sheerka methods
                                      fragment.namespace,  # objects from python fragment => local namespace
                                      global_namespace,  # global namespace
                                      expression_only)
 
-    def get_sheerka_method(self, context, who, name, expression_only):
-        try:
-            method = context.sheerka.sheerka_methods[name]
-            if expression_only and method.has_side_effect:
-                raise MethodAccessError(name)
-            else:
-                method_to_use = self.inject_context(context)(method.method) \
-                    if name in context.sheerka.methods_with_context \
-                    else method.method
-
-                return method_to_use
-        except KeyError:
-            return None
+    def get_all_possible_namespaces(self, context, namespace):
+        by_synonym = self.manage_multiple_choices(namespace)
+        by_concept_body = self.manage_concepts_with_body(context, by_synonym)
+        return by_concept_body
 
     def create_namespace(self, context,
                          names: list,
-                         sheerka_objects: dict | None,
+                         sheerka_names: set | None,  # list of requested sheerka method or attr
                          local_namespace: dict,
                          global_namespace: dict,
                          expression_only: bool):
@@ -230,7 +292,7 @@ class SheerkaPython(BaseService):
         Create a namespace for the requested names
         :param context:
         :param names: requested names
-        :param sheerka_objects: requested sheerka names (ex sheerka.isinstance)
+        :param sheerka_names: requested sheerka names (ex sheerka.isinstance)
         :param local_namespace:
         :type local_namespace:
         :param global_namespace:
@@ -257,7 +319,7 @@ class SheerkaPython(BaseService):
             # support reference to sheerka
             if name.lower() == "sheerka":
                 bag = {}
-                for sheerka_name in sheerka_objects:
+                for sheerka_name in sheerka_names:
                     if (method := self.get_sheerka_method(context,
                                                           context.who,
                                                           sheerka_name,
@@ -267,7 +329,7 @@ class SheerkaPython(BaseService):
                 continue
 
             # search in short term memory
-            if (obj := context.get_from_short_term_memory(name)) is not NotFound:
+            if (obj := context.sheerka.get_from_short_term_memory(name)) is not NotFound:
                 context.log(f"Resolving '{name}'. Using value found in STM.")
                 result[name] = obj
                 continue
@@ -298,19 +360,36 @@ class SheerkaPython(BaseService):
             # at last, try to instantiate a new concept
             if (metadata := context.sheerka.get_by_name(name)) != NotFound:
                 context.log(f"Resolving '{name}'. Instantiating new concept.")
-                result[name] = context.sheerka.new(metadata)
+                result[name] = self.new_concept(context, metadata)
+                continue
 
             context.log(f"...'{name}' is not found or cannot be instantiated. Skipping.")
 
         return result
 
-    @staticmethod
-    def resolve_object(context, attr_name, to_resolve, global_namespace):
+    def resolve_object(self, context, attr_name, to_resolve, global_namespace):
         if isinstance(to_resolve, EvaluationRef):
             return getattr(global_namespace[to_resolve.root], to_resolve.attr)
 
+        if isinstance(to_resolve, Token) and to_resolve.type == TokenKind.CONCEPT:
+            return self.new_concept(context, to_resolve.value)
+
         raise AttributeError(attr_name)
 
+    def get_sheerka_method(self, context, who, name, expression_only):
+        try:
+            method = context.sheerka.sheerka_methods[name]
+            if expression_only and method.has_side_effect:
+                raise MethodAccessError(name)
+            else:
+                method_to_use = self.inject_context(context)(method.method) \
+                    if name in context.sheerka.methods_with_context \
+                    else method.method
+
+                return method_to_use
+        except KeyError:
+            return None
+
     @staticmethod
     def evaluate_ast(fragment, my_globals, my_locals):
         compiled = fragment.get_compiled()
@@ -324,7 +403,44 @@ class SheerkaPython(BaseService):
             exec(compiled, my_globals, my_locals)
 
     @staticmethod
-    def get_all_possible_globals(context, my_globals):
+    def manage_multiple_choices(namespace):
+        """
+        for all entry that contains synonym of concepts, create a new namespace
+        ex : {"a": 1, "b" : [Concept("foo1"), Concept("foo2")] }
+        will be transformed into two separate namespaces
+        {"a": 1, "b" : Concept("foo1") }
+        and
+        {"a": 1, "b" : Concept("foo2") }
+
+        Note that concepts in error are discarded
+
+        :param namespace:
+        :type namespace:
+        :return: list of namespaces
+        :rtype:
+        """
+
+        # I want to achieve the equivalent of a cartesian product, but with list of dictionaries
+        synonyms = {}
+        others = {}
+
+        for k, v in namespace.items():
+            if isinstance(v, MultipleResults):
+                synonyms[k] = v
+            else:
+                others[k] = v
+
+        # make the product the rest as cartesian product
+        res = [others]
+        for k, concepts in synonyms.items():
+            res = dict_product(res, [{k: c} for c in concepts
+                                     if not isinstance(c, Concept) or
+                                     c.name not in ErrorConcepts])
+
+        return res
+
+    @staticmethod
+    def manage_concepts_with_body(context, namespaces):
         """
         From a dictionary of globals (str, obj)
         Creates as many globals as there are combination between a concept and its body
@@ -334,26 +450,37 @@ class SheerkaPython(BaseService):
             one with foo: Concept("foo")  # we keep the concept as an object
             one with foo: 'something'   # we substitute its value
         :param context:
-        :param my_globals:
+        :param namespaces: list of namespaces
         :return:
         """
 
-        # first pass, get all the non concepts or concepts without a body
-        # Note that we consider that all concepts are evaluated
-        # In the future, it may be a good optimisation to defer the evaluation of the body
-        # until the python evaluation fails
-        fixed_values = {}
-        concepts_with_body = {}
-        for k, v in my_globals.items():
-            if not isinstance(v, Concept) or not v.get_runtime_info().is_evaluated or v.body is NotInit:
-                fixed_values[k] = v
-            else:
-                concepts_with_body[k] = v
+        # for each namespace in namespaces,
+        # I duplicate the namespace if I found a concept which has a value
+        # I then return a list of the aggregated namespaces
 
-        # make the product the rest as cartesian product
-        res = [fixed_values]
-        for k, v in concepts_with_body.items():
-            res = dict_product(res, [{k: v}, {k: context.sheerka.objvalue(v)}])
+        res = []
+
+        for namespace in namespaces:
+
+            # first pass, get all the non concepts or concepts without a body
+            # Note that we consider that all concepts are evaluated
+            # In the future, it may be a good optimisation to defer the evaluation of the body
+            # until the python evaluation fails
+
+            fixed_values = {}
+            concepts_with_body = {}
+            for k, v in namespace.items():
+                if not isinstance(v, Concept) or not v.get_runtime_info().is_evaluated or v.body is NotInit:
+                    fixed_values[k] = v
+                else:
+                    concepts_with_body[k] = v
+
+            # make the product the rest as cartesian product
+            temp_res = [fixed_values]
+            for k, v in concepts_with_body.items():
+                temp_res = dict_product(temp_res, [{k: v}, {k: context.sheerka.objvalue(v)}])
+
+            res.extend(temp_res)
 
         return res
 
@@ -377,8 +504,26 @@ class SheerkaPython(BaseService):
 
     @staticmethod
     def get_concepts_entries_from_globals(my_globals):
-        return [k for k, v in my_globals.items() if isinstance(v, Concept)]
+        """
+        Return the name of all concept created
+        :param my_globals:
+        :type my_globals:
+        :return:
+        :rtype:
+        """
+        return [k for k, v in my_globals.items() if isinstance(v, (Concept, MultipleResults))]
 
     @staticmethod
-    def get_concepts_values_from_globals(my_globals, names):
+    def get_concepts_values_from_globals(names, my_globals):
         return {name: my_globals[name] for name in names}
+
+    @staticmethod
+    def new_concept(context, identifier):
+        new_concept = context.sheerka.new(identifier)
+
+        if isinstance(new_concept, list):
+            evaluated = [context.sheerka.evaluate_concept(context, e) for e in new_concept]
+            return MultipleResults(*evaluated)
+        else:
+            evaluated = context.sheerka.evaluate_concept(context, new_concept)
+            return evaluated