Fixed #68: Implement SheerkaQL

Fixed #70: SheerkaFilterManager : Pipe functions
Fixed #71: SheerkaFilterManager : filter_objects
Fixed #75: SheerkaMemory: Enhance memory() to use the filtering capabilities
Fixed #76: SheerkaEvaluateConcept: Concepts that modify the state of the system must not be evaluated during question

.gitignore

@@ -13,4 +13,5 @@ tests/**/*result_test
 testingPython.ipynb
 profile*.txt
 *.prof
-new.sb
+new.sb
+lextab.py

LICENCE

@@ -22,6 +22,29 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 THE SOFTWARE.
 
+PyRete is released under the term of the MIT licence
+-----------------------------------------------------
+
+Copyright (c) 2019 GNaive, Christopher J. MacLellan
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
 PyFlwor is licensed under a BSD style license
 -------------------------------------------------------
 

Makefile

@@ -14,6 +14,7 @@ clean:
 	rm -rf tests/prof
 	rm -rf tests/build
 	rm -rf Untitled*.ipynb
+	rm -rf .ipynb_checkpoints
 	find . -name '.pytest_cache' -exec rm -rf {} +
 	find . -name '__pycache__' -exec rm -rf {} +
 	find . -name 'debug.txt' -exec rm -rf {} +

requirements.txt

@@ -20,4 +20,6 @@ six==1.13.0
 wcwidth==0.1.7
 
 ipykernel~=5.3.4
-setuptools~=41.6.0
+setuptools~=41.6.0
+ipython~=7.19.0
+ply~=3.11

sheerka_backup/default.sb

@@ -3,9 +3,9 @@ push_ontology("english")
 def concept q from q ? as question(q) pre is_question() auto_eval True
 set_is_lesser(__PRECEDENCE, q, 'Sya')
 
-def concept the x ret memory(x)
-def concept a x where 'x is a concept' ret x
-def concept an x where 'x is a concept' ret x
+def concept the x where isinstance(x, Concept) ret memory(x)
+def concept a x where isinstance(x, Concept) ret x
+def concept an x where isinstance(x, Concept) ret x
 set_is_greatest(__PRECEDENCE, c:the x:, 'Sya')
 set_is_greatest(__PRECEDENCE, c:a x:, 'Sya')
 set_is_greatest(__PRECEDENCE, c:an x:, 'Sya')
@@ -37,18 +37,24 @@ set_is_greater_than(__PRECEDENCE, c:x and y:, c:x or y:, 'Sya')
 set_is_less_than(__PRECEDENCE, c:q:, c:x or y:, 'Sya')
 
 # some words
+def concept human
 def concept male
 def concept female
 def concept man
-man is a male
 def concept woman
-woman is a female
-def concept human
-man is a human
-woman is a human
 def concept boy
-def concept boys
 def concept girl
+man is a male
+man is a human
+boy is a male
+boy is a human
+woman is a female
+woman is a human
+girl is a female
+girl is a human
+
+
+def concept boys
 def concept girls
 def concept shirt
 def concept table
@@ -66,3 +72,7 @@ def concept sunday
 def concept how is x pre is_question() as smart_get_attr(x, adjective)
 def concept what x is y pre is_question() where x is an adjective as smart_get_attr(y, x)
 def concept what is the x of y pre is_question() where x is an adjective as smart_get_attr(y, x)
+
+#
+def concept he ret memory("self is a human and self is a male")
+def concept she ret memory("self is a human and self is a female")

src/cache/FastCache.py

@@ -12,6 +12,9 @@ class FastCache:
         self.lru = []
         self.default = default
 
+    def __contains__(self, item):
+        return self.has(item)
+
     def put(self, key, value):
         if len(self.cache) == self.max_size:
             del self.cache[self.lru.pop(0)]

src/core/builtin_helpers.py

@@ -7,6 +7,7 @@ from core.builtin_concepts import BuiltinConcepts
 from core.concept import Concept, ConceptParts, DEFINITION_TYPE_BNF, concept_part_value
 from core.global_symbols import NotInit, NotFound, CURRENT_OBJ
 from core.rule import Rule
+from core.tokenizer import Tokenizer
 from core.utils import as_bag
 from parsers.BaseNodeParser import SourceCodeNode, ConceptNode, UnrecognizedTokensNode, SourceCodeWithConceptNode, \
     RuleNode, VariableNode
@@ -313,15 +314,15 @@ def only_parsers_results(context, return_values):
         parents=return_values)
 
 
-def evaluate(context,
-             source,
-             evaluators="all",
-             desc=None,
-             eval_body=True,
-             eval_where=True,
-             is_question=False,
-             expect_success=False,
-             stm=None):
+def evaluate_from_source(context,
+                         source,
+                         evaluators="all",
+                         desc=None,
+                         eval_body=True,
+                         eval_where=True,
+                         is_question=False,
+                         expect_success=False,
+                         stm=None):
     """
 
     :param context:
@@ -338,7 +339,13 @@ def evaluate(context,
 
     sheerka = context.sheerka
     desc = desc or f"Eval '{source}'"
-    with context.push(BuiltinConcepts.EVALUATE_SOURCE, source, desc=desc) as sub_context:
+    hints_to_reset = {
+        BuiltinConcepts.EVAL_BODY_REQUESTED,
+        BuiltinConcepts.EVAL_WHERE_REQUESTED,
+        BuiltinConcepts.EVAL_UNTIL_SUCCESS_REQUESTED,
+        BuiltinConcepts.EVAL_QUESTION_REQUESTED,
+    }
+    with context.push(BuiltinConcepts.EVALUATE_SOURCE, source, desc=desc, reset_hints=hints_to_reset) as sub_context:
         if eval_body:
             sub_context.protected_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
 
@@ -804,6 +811,26 @@ def get_inner_body(context, concept):
         return concept.body
 
 
+def get_possible_variables_from_concept(context, concept):
+    """
+    Given concept definition,
+    gives the variables of the concept that can be considered as a parameter in another function
+    >>> gpvfc = get_possible_variables_from_concept
+    >>> assert gpvfc(Concept("a plus b").def_var("a").def_var("b")) == {"a", "b"}
+    >>> assert gpvfc(Concept("twenties", definition="twenty (one|two)=n").def_var("n")) == set()
+    :param context:
+    :param concept:
+    :return:
+    """
+    if len(concept.name) <= 1:
+        return set()
+
+    concept_name = [t.str_value for t in Tokenizer(concept.name, yield_eof=False)]
+    names = [v_value or v_name for v_name, v_value in concept.get_metadata().variables if v_name in concept_name]
+    possible_vars = filter(lambda x: context.sheerka.is_not_a_concept_name(x), names)
+    return set(possible_vars)
+
+
 class CreateObjectIdentifiers:
     """
     Class that creates unique identifiers for Concept or Rule objects

src/core/concept.py

@@ -201,6 +201,9 @@ class Concept:
             return False
 
         for name, value in self_values.items():
+            if value == self: # not very resilient...
+                continue
+
             if value != other.get_value(name):
                 return False
 
@@ -321,6 +324,10 @@ class Concept:
     def body(self):
         return self.get_value(ConceptParts.BODY)
 
+    def get_atomic_def(self):
+        tokens = [t for t in Tokenizer(self.key, yield_eof=False) if t.type != TokenKind.VAR_DEF]
+        return core.utils.get_text_from_tokens(tokens).strip()
+
     def get_origin(self):
         """
         Return the digest used to save the concept if it exists

src/core/sheerka/ExecutionContext.py

@@ -155,10 +155,23 @@ class ExecutionContext:
 
         return True
 
-    def push(self, action: BuiltinConcepts, action_context, who=None, desc=None, logger=None, obj=None, concepts=None):
+    def push(self, action: BuiltinConcepts, action_context,
+             who=None,
+             desc=None,
+             logger=None,
+             obj=None,
+             concepts=None,
+             reset_hints=None):
         if self._push:
             return self._push
 
+        if reset_hints:
+            global_hints = self.global_hints - reset_hints
+            protected_hints = self.protected_hints - reset_hints
+        else:
+            global_hints = self.global_hints
+            protected_hints = self.protected_hints
+
         who = who or self.who
         logger = logger or self._logger
         new = ExecutionContext(
@@ -169,7 +182,7 @@ class ExecutionContext:
             action_context,
             desc,
             logger,
-            self.global_hints,
+            global_hints,
             self.errors,
             obj or self.obj,
             concepts or self.concepts)
@@ -177,7 +190,7 @@ class ExecutionContext:
         new.preprocess = self.preprocess
         new.preprocess_parsers = self.preprocess_parsers
         new.preprocess_evaluators = self.preprocess_evaluators
-        new.protected_hints.update(self.protected_hints)
+        new.protected_hints.update(protected_hints)
 
         self._children.append(new)
 

src/core/sheerka/Sheerka.py

@@ -109,6 +109,7 @@ class Sheerka(Concept):
         self.enable_commands_backup = True
 
         self.methods_with_context = {"test_using_context"}  # only the names, the method is defined in sheerka_methods
+        self.pipe_functions = set()
         self.sheerka_methods = {
             "test": SheerkaMethod(self.test, False),
             "test_using_context": SheerkaMethod(self.test_using_context, False),
@@ -145,13 +146,22 @@ class Sheerka(Concept):
         if as_name is None:
             as_name = bound_method.__name__
 
+        if as_name.startswith("pipe_"):
+            as_name = as_name[5:]
+            is_pipe_function = True
+        else:
+            is_pipe_function = False
+
         if visible:
             signature = inspect.signature(bound_method)
             if len(signature.parameters) > 0 and list(signature.parameters.keys())[0] == "context":
                 self.methods_with_context.add(as_name)
             self.sheerka_methods[as_name] = SheerkaMethod(bound_method, has_side_effect)
+            if is_pipe_function:
+                self.pipe_functions.add(as_name)
 
-        setattr(self, bound_method.__name__, bound_method)
+        if not is_pipe_function:
+            setattr(self, bound_method.__name__, bound_method)
 
     def initialize(self, root_folder: str = None, **kwargs):
         """
@@ -655,6 +665,10 @@ class Sheerka(Concept):
         if obj.body is NotInit:
             return obj
 
+        if obj.key in BuiltinErrors:
+            # KSI 2021-04-20 # errors should be returned as soon as they are detected
+            return obj
+
         if reduce_simple_list and (isinstance(obj.body, list) or isinstance(obj.body, set)) and len(obj.body) == 1:
             body_to_use = obj.body[0]
         else:
@@ -673,39 +687,15 @@ class Sheerka(Concept):
 
         return (self.objvalue(obj) for obj in objs.body)
 
-    def get_errors(self, obj, **kwargs):
+    def get_errors(self, context, obj, **kwargs):
         """
         Browse obj, looking for error
+        :param context:
         :param obj:
         :param kwargs: if defined, specialize the error
         :return:
         """
 
-        def filter_by_type(x, name):
-            if isinstance(x, Concept):
-                return x.name == name
-            else:
-                return type(x).__name__ == name
-
-        def filter_by_attribute(x, attr_name, attr_value):
-            if hasattr(x, "as_bag"):
-                try:
-                    return x.as_bag()[attr_name] == attr_value
-                except KeyError:
-                    return False
-            else:
-                try:
-                    return getattr(x, attr_name) == attr_value
-                except AttributeError:
-                    return False
-
-        def and_filter(x, cond):
-            for c in cond:
-                if not c(x):
-                    return False
-
-            return True
-
         def is_error(_obj):
             if isinstance(_obj, ErrorObj):
                 return True
@@ -715,29 +705,6 @@ class Sheerka(Concept):
 
             return False
 
-        def filter_objects(_objects):
-            if kwargs:
-                cond = []
-                for k, v in kwargs.items():
-                    if k == "__type":
-                        expected_type = v
-                        cond.append(lambda x: filter_by_type(x, expected_type))
-                    else:
-                        attr_name = k
-                        expect_value = v
-                        cond.append(lambda x: filter_by_attribute(x, attr_name, expect_value))
-
-                if len(cond) > 1:
-                    copy_of_conditions = cond.copy()
-                    full_cond = lambda x: and_filter(x, copy_of_conditions)
-
-                else:
-                    full_cond = cond[0]
-
-                return [o for o in _objects if full_cond(o)]
-
-            return _objects
-
         def inner_get_errors(_obj):
             if self.isinstance(_obj, BuiltinConcepts.RETURN_VALUE) and _obj.status:
                 return []
@@ -748,6 +715,8 @@ class Sheerka(Concept):
             if is_error(_obj):
                 if isinstance(_obj, Concept) and _obj.body not in (NotInit, None):
                     return [_obj] + inner_get_errors(_obj.body)
+                if isinstance(_obj, ErrorObj) and hasattr(_obj, "get_error"):
+                    return [_obj] + inner_get_errors(_obj.get_error())
                 else:
                     return [_obj]
 
@@ -757,10 +726,10 @@ class Sheerka(Concept):
             return []
 
         errors = inner_get_errors(obj)
-        return filter_objects([e for e in errors])
+        return self.filter_objects(context, [e for e in errors], **kwargs)
 
-    def has_error(self, obj, **kwargs):
-        errors = self.get_errors(obj, **kwargs)
+    def has_error(self, context, obj, **kwargs):
+        errors = self.get_errors(context, obj, **kwargs)
         return len(errors) > 0
 
     def get_evaluator_name(self, name):
@@ -797,13 +766,6 @@ class Sheerka(Concept):
 
         return bool(obj)
 
-    @staticmethod
-    def is_error(obj):
-        """
-        opposite of is_success
-        """
-        return not Sheerka.is_success(obj)
-
     @staticmethod
     def is_known(obj):
         if not isinstance(obj, Concept):
@@ -844,7 +806,7 @@ class Sheerka(Concept):
         if isinstance(obj, Concept) and obj.id == self.id:
             return True
 
-        from evaluators.PythonEvaluator import Expando
+        from sheerkapython.python_wrapper import Expando
         if isinstance(obj, Expando) and obj.get_name() == "sheerka":
             return True
 

src/core/sheerka/SheerkaOntologyManager.py

@@ -395,7 +395,7 @@ class SheerkaOntologyManager:
 
     def get_all(self, entry, cache_only=False):
         """
-        Return all key, value from all ontologies
+        Return all <key, value> from all ontologies
         First look in sdp, then override with the cache, for all ontologies
         :param entry: cache name / sdp entry
         :param cache_only: Do no fetch data from remote sdp

src/core/sheerka/services/SheerkaAdmin.py

@@ -3,7 +3,7 @@ import time
 from os import path
 
 from core.builtin_concepts_ids import BuiltinConcepts, BuiltinContainers
-from core.builtin_helpers import ensure_concept_or_rule
+from core.builtin_helpers import ensure_concept_or_rule, ensure_concept
 from core.concept import Concept
 from core.global_symbols import SHEERKA_BACKUP_FOLDER
 from core.sheerka.services.SheerkaHistoryManager import SheerkaHistoryManager
@@ -37,6 +37,7 @@ class SheerkaAdmin(BaseService):
         self.sheerka.bind_service_method(self.admin_history, False, as_name="history")
         self.sheerka.bind_service_method(self.admin_history, False, as_name="history")
         self.sheerka.bind_service_method(self.sdp, False)
+        self.sheerka.bind_service_method(self.atomic_def, False)
 
     def caches_names(self):
         """
@@ -218,6 +219,20 @@ class SheerkaAdmin(BaseService):
 
         return self.sheerka.isinstance(a, b)
 
+    @staticmethod
+    def atomic_def(a):
+        """
+        Return the 'atomic definition' of a concept
+        a concept key stripped from its 'var' tokens
+        >>> assert atomic_def(Concept('a plus b').def_var("a").def_var("b")) == "plus"
+        >>> assert atomic_def(Concept('x is a y').def_var("x").def_var("y")) == "is a"
+        :param a:
+        :return:
+        """
+        ensure_concept(a)
+
+        return a.get_atomic_def()
+
     @staticmethod
     def is_container(obj):
         """

src/core/sheerka/services/SheerkaConceptManager.py

@@ -126,7 +126,8 @@ class SheerkaConceptManager(BaseService):
         self.sheerka.bind_service_method(self.get_by_name, False, visible=False)
         self.sheerka.bind_service_method(self.get_by_hash, False, visible=False)
         self.sheerka.bind_service_method(self.get_by_id, False, visible=False)
-        self.sheerka.bind_service_method(self.is_not_a_variable, False, visible=False)
+        self.sheerka.bind_service_method(self.is_not_a_concept_name, False, visible=False)
+        self.sheerka.bind_service_method(self.is_a_concept_name, False, visible=False)
         self.sheerka.bind_service_method(self.get_concepts_by_first_token, False, visible=False)
         self.sheerka.bind_service_method(self.get_concepts_by_first_regex, False, visible=False)
         self.sheerka.bind_service_method(self.get_concepts_bnf_definitions, False, visible=False)
@@ -705,7 +706,7 @@ class SheerkaConceptManager(BaseService):
 
         return refs
 
-    def is_not_a_variable(self, name):
+    def is_not_a_concept_name(self, name):
         """
         Given a name tells if it refers to a variable name
         :param name:
@@ -713,6 +714,14 @@ class SheerkaConceptManager(BaseService):
         """
         return self.sheerka.om.get(self.sheerka.CONCEPTS_BY_NAME_ENTRY, name) is NotFound
 
+    def is_a_concept_name(self, name):
+        """
+        Given a name tells if it refers to a variable name
+        :param name:
+        :return:
+        """
+        return self.sheerka.om.get(self.sheerka.CONCEPTS_BY_NAME_ENTRY, name) is not NotFound
+
     def clear_bnf_definition(self, concept_id=None):
         if concept_id:
             self.sheerka.om.delete(self.CONCEPTS_BNF_DEFINITIONS_ENTRY, concept_id)

src/core/sheerka/services/SheerkaEvaluateConcept.py

@@ -1,7 +1,7 @@
 from dataclasses import dataclass
 
 from core.builtin_concepts import BuiltinConcepts
-from core.builtin_helpers import expect_one, only_successful, evaluate, ensure_concept
+from core.builtin_helpers import expect_one, only_successful, evaluate_from_source, ensure_concept
 from core.concept import Concept, DoNotResolve, ConceptParts, InfiniteRecursionResolved, AllConceptParts, \
     concept_part_value
 from core.global_symbols import NotInit, CURRENT_OBJ
@@ -11,9 +11,9 @@ from core.sheerka.services.SheerkaExecute import ParserInput
 from core.sheerka.services.sheerka_service import BaseService
 from core.tokenizer import Tokenizer
 from core.utils import unstr_concept
+from parsers.BaseExpressionParser import TrueifyVisitor
 from parsers.BaseNodeParser import ConceptNode
 from parsers.LogicalOperatorParser import LogicalOperatorParser
-from parsers.BaseExpressionParser import TrueifyVisitor
 
 CONCEPT_EVALUATION_STEPS = [
     BuiltinConcepts.BEFORE_EVALUATION,
@@ -26,6 +26,11 @@ class ChickenAndEggException(Exception):
     error: Concept
 
 
+@dataclass
+class ConceptEvalException(Exception):
+    error: Concept
+
+
 @dataclass
 class WhereClauseDef:
     concept: Concept  # concept on which the where clause is applied
@@ -43,6 +48,8 @@ class SheerkaEvaluateConcept(BaseService):
 
     def initialize(self):
         self.sheerka.bind_service_method(self.evaluate_concept, True)
+        self.sheerka.bind_service_method(self.call_concept, True)
+        self.sheerka.bind_service_method(self.call_concept, False, as_name="evaluate_question")
         self.sheerka.bind_service_method(self.set_auto_eval, True)
 
     @staticmethod
@@ -197,12 +204,12 @@ class SheerkaEvaluateConcept(BaseService):
                     ret.append(r)  # it cannot be solved unitary, let's give a chance to the global where condition
             else:
                 # it means that the where condition is an expression that needs to be executed
-                evaluation_res = evaluate(context,
-                                          where_clause_def.trueified,
-                                          desc=f"Apply where clause on '{where_clause_def.prop}'",
-                                          expect_success=True,
-                                          is_question=True,
-                                          stm={where_clause_def.prop: r.body})
+                evaluation_res = evaluate_from_source(context,
+                                                      where_clause_def.trueified,
+                                                      desc=f"Apply where clause on '{where_clause_def.prop}'",
+                                                      expect_success=True,
+                                                      is_question=True,
+                                                      stm={where_clause_def.prop: r.body})
                 one_res = expect_one(context, evaluation_res)
                 if one_res.status:
                     value = context.sheerka.objvalue(one_res)
@@ -624,6 +631,25 @@ class SheerkaEvaluateConcept(BaseService):
             else:
                 return concept
 
+    def call_concept(self, context, concept, *args, **kwargs):
+        """
+        call the concept using either args or kwargs (not both)
+        :param context:
+        :param concept:
+        :param args:
+        :param kwargs:
+        :return:
+        """
+
+        evaluated = self.evaluate_concept(context, concept)
+        if self.sheerka.has_error(context, evaluated):
+            raise ConceptEvalException(evaluated)
+
+        if ConceptParts.BODY in evaluated.get_compiled():
+            return evaluated.body
+        else:
+            return evaluated
+
     def compute_metadata_to_eval(self, context, concept):
         to_eval = []
 

src/core/sheerka/services/SheerkaEvaluateRules.py

@@ -2,7 +2,6 @@ from core.builtin_concepts import BuiltinConcepts
 from core.builtin_helpers import expect_one
 from core.global_symbols import EVENT_RULE_CREATED, EVENT_RULE_DELETED, EVENT_RULE_ID_DELETED
 from core.sheerka.services.sheerka_service import BaseService
-from evaluators.ConceptEvaluator import ConceptEvaluator
 from sheerkarete.network import ReteNetwork
 
 DISABLED_RULES = "#disabled#"
@@ -105,10 +104,18 @@ class SheerkaEvaluateRules(BaseService):
         :return:
         """
 
+        results = self.evaluate_conditions(context, rule.compiled_conditions, bag)
+
+        debugger = context.get_debugger(SheerkaEvaluateRules.NAME, "evaluate_rule", new_debug_id=False)
+        debugger.debug_rule(rule, results)
+
+        return expect_one(context, results)
+
+    def evaluate_conditions(self, context, conditions, bag):
         bag_variables = set(bag.keys())
 
         results = []
-        for compiled_condition in rule.compiled_conditions:
+        for compiled_condition in conditions:
 
             if compiled_condition.variables.intersection(bag_variables) != compiled_condition.variables:
                 continue
@@ -123,22 +130,19 @@ class SheerkaEvaluateRules(BaseService):
             else:
 
                 # do not forget to reset the 'is_evaluated' in the case of a concept
-                if compiled_condition.evaluator_type == ConceptEvaluator.NAME:
-                    compiled_condition.concept.get_metadata().is_evaluated = False
+                for concept in compiled_condition.concepts_to_reset:
+                    concept.get_metadata().is_evaluated = False
 
                 evaluator = self.evaluators_by_name[compiled_condition.evaluator_type]
                 res = evaluator.eval(context, compiled_condition.return_value)
                 if res.status and isinstance(res.body, bool) and res.body:
                     # one successful value found. No need to look any further
-                    results = [res]
+                    results = [res]  # don't we care about the other failing results ?
                     break
                 else:
                     results.append(res)
 
-        debugger = context.get_debugger(SheerkaEvaluateRules.NAME, "evaluate_rule", new_debug_id=False)
-        debugger.debug_rule(rule, results)
-
-        return expect_one(context, results)
+        return results
 
     def remove_from_rete_memory(self, lst):
         if lst is None:

src/core/sheerka/services/SheerkaHasAManager.py

@@ -43,7 +43,6 @@ class SheerkaHasAManager(BaseService):
     def hasa(self, concept_a, concept_b):
         """
         Check that concept 'a' has/owns concept 'b'
-        :param context:
         :param concept_a:
         :param concept_b:
         :return:

src/core/sheerka/services/SheerkaMemory.py

@@ -10,6 +10,7 @@ from core.sheerka.services.sheerka_service import BaseService, ServiceObj
 
 @dataclass
 class MemoryObject(ServiceObj):
+    timestamp: float
     obj: object
 
     def __eq__(self, other):
@@ -21,6 +22,9 @@ class MemoryObject(ServiceObj):
     def __hash__(self):
         return hash((self.event_id, self.obj))
 
+    def __repr__(self):
+        return f"MemoryObject({self.obj}, timestamp={self.timestamp})"
+
 
 class SheerkaMemory(BaseService):
     NAME = "Memory"
@@ -38,7 +42,7 @@ class SheerkaMemory(BaseService):
         self.sheerka.bind_service_method(self.get_all_short_term_memory, False, visible=False)
         self.sheerka.bind_service_method(self.add_to_short_term_memory, True, visible=False)
         self.sheerka.bind_service_method(self.remove_context, True, as_name="clear_short_term_memory", visible=False)
-        self.sheerka.bind_service_method(self.add_to_memory, True, visible=False)
+        self.sheerka.bind_service_method(self.add_to_memory, True)
         self.sheerka.bind_service_method(self.add_many_to_short_term_memory, True, visible=False)
         self.sheerka.bind_service_method(self.get_from_memory, False)
         self.sheerka.bind_service_method(self.get_last_from_memory, False)
@@ -125,20 +129,31 @@ class SheerkaMemory(BaseService):
         """
         last = self.sheerka.om.get(SheerkaMemory.OBJECTS_ENTRY, key)
         if last is NotFound:
-            self.sheerka.om.put(SheerkaMemory.OBJECTS_ENTRY, key, MemoryObject(context.event.get_digest(), concept))
+            self.sheerka.om.put(SheerkaMemory.OBJECTS_ENTRY, key, MemoryObject(context.event.get_digest(),
+                                                                               context.event.date.timestamp(),
+                                                                               concept))
             return
 
         if not isinstance(last, list) and last.obj == concept:
+            # replace with the new one
             self.sheerka.om.delete(SheerkaMemory.OBJECTS_ENTRY, key, last)
-            self.sheerka.om.put(SheerkaMemory.OBJECTS_ENTRY, key, MemoryObject(context.event.get_digest(), concept))
+            self.sheerka.om.put(SheerkaMemory.OBJECTS_ENTRY, key, MemoryObject(context.event.get_digest(),
+                                                                               context.event.date.timestamp(),
+                                                                               concept))
             return
 
         if isinstance(last, list) and last[-1].obj == concept:
+            # replace with the new one
             self.sheerka.om.delete(SheerkaMemory.OBJECTS_ENTRY, key, last[-1])
-            self.sheerka.om.put(SheerkaMemory.OBJECTS_ENTRY, key, MemoryObject(context.event.get_digest(), concept))
+            self.sheerka.om.put(SheerkaMemory.OBJECTS_ENTRY, key, MemoryObject(context.event.get_digest(),
+                                                                               context.event.date.timestamp(),
+                                                                               concept))
             return
 
-        self.sheerka.om.put(SheerkaMemory.OBJECTS_ENTRY, key, MemoryObject(context.event.get_digest(), concept))
+        # append the new one
+        self.sheerka.om.put(SheerkaMemory.OBJECTS_ENTRY, key, MemoryObject(context.event.get_digest(),
+                                                                           context.event.date.timestamp(),
+                                                                           concept))
 
     def get_from_memory(self, context, key):
         """"
@@ -207,14 +222,35 @@ class SheerkaMemory(BaseService):
         """
         name_to_use = name.name if isinstance(name, Concept) else name
         self.unregister_object(context, name_to_use)
-        obj = self.get_from_memory(context, name_to_use)
-        if obj is NotFound:
-            return self.sheerka.new(BuiltinConcepts.NOT_FOUND, body={"#name": name})
+        obj = self.get_last_from_memory(context, name_to_use)
+        if obj is not NotFound:
+            return obj.obj
 
-        if isinstance(obj, list):
-            obj = obj[-1]
+        all_objects = self.sheerka.om.list(SheerkaMemory.OBJECTS_ENTRY)
+        all_objects_copy = []
+        for obj in all_objects:
+            if isinstance(obj, list):
+                all_objects_copy.append(obj.copy())
+            else:
+                all_objects_copy.append([obj])
 
-        return obj.obj
+        while len(all_objects_copy) > 0:
+            current_list = []
+            temp = []
+            for obj in all_objects_copy:
+                current_list.append(obj.pop(-1))
+                if len(obj) > 0:
+                    temp.append(obj)
+
+            all_objects_copy = temp
+            current_list = sorted(current_list, key=lambda o: o.timestamp, reverse=True)
+            current_objects = [o.obj for o in current_list]
+
+            res = self.sheerka.filter_objects(context, current_objects, name)
+            if len(res) > 0:
+                return res[0]  # only the first, as it should have a better timestamp
+
+        return self.sheerka.new(BuiltinConcepts.NOT_FOUND, body={"#name": name})
 
     def mem(self):
         keys = sorted([k for k in self.sheerka.om.list(SheerkaMemory.OBJECTS_ENTRY)])

src/core/sheerka/services/SheerkaQueryManager.py

@@ -0,0 +1,251 @@
+from cache.FastCache import FastCache
+from core.builtin_concepts_ids import BuiltinContainers, BuiltinConcepts
+from core.concept import Concept, ConceptParts
+from core.sheerka.services.SheerkaEvaluateRules import SheerkaEvaluateRules
+from core.sheerka.services.sheerka_service import BaseService
+from core.tokenizer import Tokenizer, TokenKind
+from core.utils import as_bag
+from sheerkapython.python_wrapper import create_namespace, ObjectContainer, get_type
+from sheerkaql.lexer import Lexer
+from sheerkaql.parser import Parser
+
+
+class SheerkaQueryManager(BaseService):
+    """
+    This class manage the queries on objects across the system
+    """
+    NAME = "QueryManager"
+    OBJECTS_ROOT_ALIAS = "__xxx__objects__xx__"
+    QUERY_PARAMETER_PREFIX = "__xxx__query_parameter__xx__"
+
+    def __init__(self, sheerka):
+        super().__init__(sheerka)
+        self.queries = FastCache()
+        self.conditions = FastCache()
+        self.lexer = Lexer()
+        self.rule_evaluator = None
+
+    def initialize(self):
+        self.sheerka.bind_service_method(self.filter_objects, False)
+        self.sheerka.bind_service_method(self.select_objects, False)
+        self.sheerka.bind_service_method(self.collect_attributes, False)
+
+        self.sheerka.bind_service_method(self.filter_objects, False, as_name="pipe_where")
+        self.sheerka.bind_service_method(self.select_objects, False, as_name="pipe_select")
+        self.sheerka.bind_service_method(self.collect_attributes, False, as_name="pipe_props")
+
+        self.sheerka.register_debug_vars(SheerkaQueryManager.NAME, "filter_objects", "query")
+
+    def initialize_deferred(self, context, is_first_time):
+        self.rule_evaluator = self.sheerka.services[SheerkaEvaluateRules.NAME]
+
+    def get_query_by_kwargs(self, local_namespace, **kwargs):
+        """
+        Create a predicate using kwargs and filter the result
+        :param local_namespace:
+        :param kwargs:
+        :return:
+        """
+        if not kwargs:
+            return None
+
+        objects_in_context_index = 0
+        conditions = []
+        for k, v in kwargs.items():
+            current_variable_name = f"{self.QUERY_PARAMETER_PREFIX}_{objects_in_context_index:02}"
+            objects_in_context_index += 1
+            local_namespace[current_variable_name] = v
+
+            if k == "__type":
+                conditions.append(f"get_type(self) == {current_variable_name}")
+
+            elif k == "atomic_def":
+                conditions.append(f"atomic_def(self) == {current_variable_name}")
+
+            elif k in ("__self", "_"):
+                conditions.append(f"self == {current_variable_name}")
+
+            else:
+                conditions.append(f"self.{k} == {current_variable_name}")
+
+        return ' and '.join(conditions)
+
+    def filter_objects(self, context, objects, predicate=None, **kwargs):
+        """
+        filter the given objects using the conditions from kwargs
+        for each k,v in kwargs, the equality k == v is added
+        for k starting with a double underscore '__', a special treatment may be done
+            __type : get the type of object (in Sheerka world)
+        :param context:
+        :param objects:
+        :param predicate:
+        :param kwargs:
+        :return:
+        """
+
+        debugger = context.get_debugger(SheerkaQueryManager.NAME, "filter_objects")
+
+        original_container = None
+        if isinstance(objects, Concept) and objects.key in BuiltinContainers:
+            original_container = objects
+            objects = objects.body
+
+        debugger.debug_entering(nb_objects=len(objects), predicate=predicate, **kwargs)
+        local_namespace = {}
+        query_by_kwargs = self.get_query_by_kwargs(local_namespace, **kwargs)
+
+        if predicate is not None and query_by_kwargs is not None:
+            query = f"({predicate}) and ({query_by_kwargs})"
+        elif predicate is not None:
+            query = predicate
+        elif query_by_kwargs is not None:
+            query = query_by_kwargs
+        else:
+            query = None
+
+        if debugger.is_enabled():
+            debugger.debug_var("query", query)
+            for k, v in local_namespace.items():
+                debugger.debug_var("query_parameter", f"{k} = {v}")
+
+        if query and query in self.conditions:
+            # Then try using RuleManager
+            objects = self.execute_conditions(context, query, objects, local_namespace)
+        elif query:
+            try:
+                # Fist try with the FLWR parser
+                full_query = f"{self.OBJECTS_ROOT_ALIAS}.items[{query}]"
+                objects = self.execute_flwr_query(context, full_query, objects, local_namespace)
+            except SyntaxError:
+                # Then try using RuleManager
+                objects = self.execute_conditions(context, query, objects, local_namespace)
+
+        if original_container:
+            original_container.set_value(ConceptParts.BODY, objects)
+            return original_container
+        else:
+            return objects
+
+    def select_objects(self, context, objects, *props, **kwargs):
+        """
+        From the input objects, create output objects
+        The definition of these new objects can come from a flwr query
+        :param context:
+        :param objects:
+        :param query:
+        :return:
+        """
+
+        def sanitize_property(p):
+            if not isinstance(p, str):
+                raise SyntaxError(f"{p} is not the name of an attribute")
+
+            tokens = list(Tokenizer(p, yield_eof=False))
+            if len(tokens) == 1 and tokens[0].type == TokenKind.IDENTIFIER:
+                return f"self.{p}"
+            else:
+                return p
+
+        original_container = None
+        if isinstance(objects, Concept) and objects.key in BuiltinContainers:
+            original_container = objects
+            objects = objects.body
+
+        requested_properties = [sanitize_property(prop) for prop in props]
+
+        if kwargs:
+            items = [f"'{k}': {sanitize_property(v)}" for k, v in kwargs.items()]
+            requested_properties.append("{" + ", ".join(items) + "}")
+
+        query = f"for self in {self.OBJECTS_ROOT_ALIAS}.items return " + ", ".join(requested_properties)
+        objects = self.execute_flwr_query(context, query, objects, {})
+
+        if original_container:
+            original_container.set_value(ConceptParts.BODY, objects)
+            return original_container
+        else:
+            return objects
+
+    def collect_attributes(self, objects, take=10):
+        """
+        Given a list of object, returns the attributes that can be requested
+        :param objects:
+        :param take: no need to browse the whole list, you can just use a sample
+        :return:
+        """
+        if isinstance(objects, Concept) and objects.key in BuiltinContainers:
+            objects = objects.body
+
+        result = {}
+        for obj in (objects if take <= 0 else objects[:take]):
+            object_type = get_type(obj)
+            attrs = set(p for p in as_bag(obj).keys() if p != "self")
+            if object_type is result:
+                result[object_type].update(attrs)
+            else:
+                result[object_type] = attrs
+
+        result = {k: sorted(list(v)) for k, v in result.items()}
+        return self.sheerka.new(BuiltinConcepts.TO_DICT, body=result)
+
+    def execute_flwr_query(self, context, query, objects, local_namespace):
+        """
+        Execute a FLWOR query on objects
+        :param context:
+        :param query: query to execute (as a string to compile)
+        :param objects: list of objects to filter
+        :param local_namespace: objects of the namespace that are already known
+        :return:
+        """
+        if query not in self.queries:
+            parser = Parser()
+            compiled_query = parser.parse(bytes(query, 'utf-8').decode('unicode_escape'), lexer=self.lexer)
+            self.queries.put(query, (compiled_query, parser.names, parser.sheerka_names))
+
+        compiled_query, names, sheerka_names = self.queries.get(query)
+
+        namespace = create_namespace(context,
+                                     self.NAME,
+                                     names,
+                                     sheerka_names,
+                                     {self.OBJECTS_ROOT_ALIAS: ObjectContainer(objects)},
+                                     expression_only=True,
+                                     allow_builtins=True)
+        namespace.update(local_namespace)  # override if needed
+
+        return compiled_query(namespace)
+
+    def execute_conditions(self, context, query, objects, local_namespace):
+        """
+
+        :param context:
+        :param query:
+        :param objects:
+        :param local_namespace:
+        :return:
+        """
+
+        if query not in self.conditions:
+            from core.sheerka.services.SheerkaRuleManager import SheerkaRuleManager
+            rule_manager = self.sheerka.services[SheerkaRuleManager.NAME]
+            compilation_results = rule_manager.compile_when(context, self.NAME, query)
+            self.conditions.put(query, compilation_results.python_conditions)
+
+        conditions = self.conditions.get(query)
+        results = []
+        with context.push(BuiltinConcepts.EXEC_CODE, query) as sub_context:
+            sub_context.protected_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
+            sub_context.protected_hints.add(BuiltinConcepts.EVAL_WHERE_REQUESTED)
+            sub_context.protected_hints.add(BuiltinConcepts.EVAL_UNTIL_SUCCESS_REQUESTED)
+            sub_context.protected_hints.add(BuiltinConcepts.EVAL_QUESTION_REQUESTED)
+            sub_context.sheerka.add_many_to_short_term_memory(sub_context, local_namespace)
+            sub_context.deactivate_push()
+
+            for obj in objects:
+                local_namespace["self"] = obj
+                res = self.rule_evaluator.evaluate_conditions(sub_context, conditions, local_namespace)
+                successful = list(filter(lambda r: r.status and type(r.body) == bool and r.body, res))
+                if successful:
+                    results.append(obj)
+
+        return results

src/core/sheerka/services/SheerkaRuleManager.py

@@ -1,28 +1,27 @@
 import operator
 import re
 from dataclasses import dataclass
-from itertools import product
 from typing import Union, Set, List, Tuple
 
 from cache.Cache import Cache
 from cache.ListIfNeededCache import ListIfNeededCache
 from core.builtin_concepts import BuiltinConcepts, ReturnValueConcept
-from core.builtin_helpers import ensure_evaluated, expect_one, evaluate
+from core.builtin_helpers import ensure_evaluated, expect_one, evaluate_from_source, \
+    get_possible_variables_from_concept, is_a_question
 from core.concept import Concept
 from core.global_symbols import EVENT_RULE_PRECEDENCE_MODIFIED, RULE_COMPARISON_CONTEXT, NotFound, ErrorObj, \
     EVENT_RULE_CREATED, EVENT_RULE_DELETED, NotInit
 from core.rule import Rule, ACTION_TYPE_PRINT
 from core.sheerka.Sheerka import RECOGNIZED_BY_NAME, RECOGNIZED_BY_ID
-from core.sheerka.services.SheerkaExecute import ParserInput
 from core.sheerka.services.sheerka_service import BaseService, FailedToCompileError
 from core.tokenizer import Keywords, TokenKind, Token, IterParser
-from core.utils import index_tokens, COLORS, get_text_from_tokens, merge_dictionaries, merge_sets
-from evaluators.PythonEvaluator import PythonEvaluator, Expando
+from core.utils import index_tokens, COLORS, get_text_from_tokens, merge_dictionaries, merge_sets, get_safe_str_value
+from evaluators.PythonEvaluator import PythonEvaluator
 from parsers.BaseExpressionParser import AndNode, ExpressionVisitor, VariableNode, ComparisonNode, FunctionNode, \
     ComparisonType, NotNode, NameExprNode
 from parsers.BaseNodeParser import ConceptNode
 from parsers.LogicalOperatorParser import LogicalOperatorParser
-from parsers.PythonParser import PythonParser
+from sheerkapython.python_wrapper import Expando, sheerka_globals
 from sheerkarete.common import V
 from sheerkarete.conditions import AndConditions, Condition, NegatedCondition, NegatedConjunctiveConditions
 from sheerkarete.network import FACT_NAME, FACT_SELF
@@ -424,7 +423,7 @@ class FormatRuleActionParser(IterParser):
 
     def return_list(self, args, kwargs):
         """
-        Looking for variable_name, [recurse_on], [recursion_depth], [items_prop]
+        Looking for greeting_var, [recurse_on], [recursion_depth], [items_prop]
         :param args:
         :param kwargs:
         :return:
@@ -624,7 +623,8 @@ class CompiledCondition:
     return_value: Union[ReturnValueConcept, None]  # compiled source as ReturnValue
     variables: Set[str]  # variables that must be present in bag
     not_variables: Set[str]  # variables that must not be present in bag
-    concept: Union[Concept, None] = None  # compiled source as concept
+    objects: dict
+    concepts_to_reset: set  # concepts to reset before every evaluation
 
 
 @dataclass
@@ -763,12 +763,14 @@ class SheerkaRuleManager(BaseService):
         parsed = parsed_expr_ret.body.body
         rete_conditions, python_conditions = None, None
         if parsed_expr_ret.status:
+            # get the conditions as rete conditions
             try:
                 rete_visitor = ReteConditionExprVisitor(context)
                 rete_conditions = rete_visitor.get_conditions(parsed)
             except FailedToCompileError as err:
                 pass
 
+            # get the conditions as sheerka conditions
             try:
                 python_visitor = PythonConditionExprVisitor(context)
                 python_conditions = python_visitor.get_conditions(parsed)
@@ -1059,10 +1061,40 @@ class SheerkaRuleManager(BaseService):
 
 
 class GetConditionExprVisitor(ExpressionVisitor):
+    """
+    Base class for ReteConditionExprVisitor and PythonConditionExprVisitor
+    """
+
     def __init__(self, context):
         self.context = context
         self.var_counter = 0
         self.variables = {}
+        self.obj_counter = 0
+        self.objects_mapping = {}
+
+    def get_object_name(self, obj, objects=None):
+        """
+        object found during the parsing are not serialized
+        They are kept in a dictionary and this function returns a new name for every new object
+        :return:
+        """
+        if objects is None:
+            objects = {}
+
+        if self.context.sheerka.is_sheerka(obj):
+            return "sheerka", objects
+
+        try:
+            return self.objects_mapping[id(obj)], objects
+        except KeyError:
+            pass
+
+        object_name = f"__o_{self.obj_counter:02}__"
+        self.obj_counter += 1
+
+        self.objects_mapping[id(obj)] = object_name
+        objects[object_name] = obj
+        return object_name, objects
 
     def add_variable(self, target):
         """
@@ -1078,7 +1110,7 @@ class GetConditionExprVisitor(ExpressionVisitor):
     def inner_unpack_variable(self, variable_path: List[str]) -> Tuple[str, str]:
         """
         When variable_path = a.b.c.d
-        returns (x0, d) if x0 = a.b.c else (a, b.c.d)
+        returns (x0, d) if (exist x0 = a.b.c) else (a, b.c.d)
         :param variable_path:
         :return:
         """
@@ -1108,24 +1140,47 @@ class GetConditionExprVisitor(ExpressionVisitor):
             return self.variables[path]
 
         root, attr = self.inner_unpack_variable(variable_path)
-        var_name = self.add_variable(root + "." + attr)
-        return var_name, (root, attr)
+        if attr:
+            var_name = self.add_variable(root + "." + attr)
+            return var_name, (root, attr)
+        else:
+            return root, (root, None)
 
-    def evaluate(self, source, eval_body):
-        res = evaluate(self.context,
-                       source,
-                       evaluators=CONDITIONS_VISITOR_EVALUATORS,
-                       desc=None,
-                       eval_body=eval_body,
-                       eval_where=False,
-                       is_question=False,
-                       expect_success=False,
-                       stm=None)
+    def evaluate_from_source(self, source, is_question=False, return_body=False):
+        res = evaluate_from_source(self.context,
+                                   source,
+                                   evaluators=CONDITIONS_VISITOR_EVALUATORS,
+                                   desc=None,
+                                   eval_body=not is_question,
+                                   eval_where=False,
+                                   is_question=is_question,
+                                   expect_success=False,
+                                   stm=None)
         res = expect_one(self.context, res)
-        if not res.status:
-            raise FailedToCompileError([f"Failed to evaluate '{source}'"])
 
-        return res.value
+        if return_body:
+            if not res.status:
+                raise FailedToCompileError(res.body)
+
+            return res.body
+        else:
+            return res
+
+    def is_a_possible_variable(self, name):
+        """
+        tells whether or not the name can be a variable
+        :param name:
+        :return:
+        """
+        if self.context.sheerka.is_a_concept_name(name):
+            return False
+
+        try:
+            eval(name, sheerka_globals)
+        except:
+            return True
+
+        return False
 
 
 class ReteConditionExprVisitor(GetConditionExprVisitor):
@@ -1162,29 +1217,38 @@ class ReteConditionExprVisitor(GetConditionExprVisitor):
     def init_or_get_variable_from_attr(self, variable_path: List[str], conditions):
         path = ".".join(variable_path)
         if path in self.variables:
-            return self.variables[path]
+            return V(self.variables[path])
 
         root, attr = self.init_or_get_variable_from_name(variable_path, conditions)
-        var_name = self.add_variable(path)
-        variable = V(var_name)
-        conditions.append(Condition(root, attr, variable))
-        return variable
+        if attr:
+            var_name = self.add_variable(path)
+            variable = V(var_name)
+            conditions.append(Condition(root, attr, variable))
+            return variable
+        else:
+            return root
 
     def visit_VariableNode(self, expr_node: VariableNode):
         if expr_node.attributes_str is None:
             # try to recognize a concept
-            res = evaluate(self.context,
-                           expr_node.name,
-                           evaluators=CONDITIONS_VISITOR_EVALUATORS,
-                           desc=None,
-                           eval_body=True,
-                           eval_where=False,
-                           is_question=False,
-                           expect_success=False,
-                           stm=None)
-            res = expect_one(self.context, res)
-            if res.status and isinstance(res.value, Concept):
-                return self.recognize_concept(["__ret", "body"], res.value, {})
+            res = self.evaluate_from_source(expr_node.name, is_question=True)
+            if self.context.sheerka.has_error(self.context, res, __type=BuiltinConcepts.TOO_MANY_SUCCESS):
+                raise FailedToCompileError([res])
+
+        # if expr_node.attributes_str is None:
+        #     # try to recognize a concept
+        #     res = evaluate_from_source(self.context,
+        #                    expr_node.name,
+        #                    evaluators=CONDITIONS_VISITOR_EVALUATORS,
+        #                    desc=None,
+        #                    eval_body=True,
+        #                    eval_where=False,
+        #                    is_question=False,
+        #                    expect_success=False,
+        #                    stm=None)
+        #     res = expect_one(self.context, res)
+        #     if res.status and isinstance(res.value, Concept):
+        #         return self.recognize_concept(["__ret", "body"], res.value, {})
 
         conditions = []
         variable_name = expr_node.get_source()
@@ -1204,7 +1268,7 @@ class ReteConditionExprVisitor(GetConditionExprVisitor):
     def visit_ComparisonNode(self, expr_node: ComparisonNode):
         if isinstance(expr_node.left, VariableNode):
             conditions = []
-            value = self.evaluate(expr_node.right.get_source(), True)
+            value = self.evaluate_from_source(expr_node.right.get_source(), return_body=True)
             self.add_to_condition(expr_node.left.unpack(), value, conditions)
             return conditions
         else:
@@ -1268,15 +1332,15 @@ class ReteConditionExprVisitor(GetConditionExprVisitor):
             return negate_conditions(exists_condition, sub_conditions)
 
     def visit_NameExprNode(self, expr_node: NameExprNode):
-        res = evaluate(self.context,
-                       expr_node.get_source(),
-                       evaluators=CONDITIONS_VISITOR_EVALUATORS,
-                       desc=None,
-                       eval_body=True,
-                       eval_where=False,
-                       is_question=False,
-                       expect_success=False,
-                       stm=None)
+        res = evaluate_from_source(self.context,
+                                   expr_node.get_source(),
+                                   evaluators=CONDITIONS_VISITOR_EVALUATORS,
+                                   desc=None,
+                                   eval_body=True,
+                                   eval_where=False,
+                                   is_question=False,
+                                   expect_success=False,
+                                   stm=None)
         res = expect_one(self.context, res)
         if res.status and isinstance(res.value, Concept):
             return self.recognize_concept(["__ret", "body"], res.value, {})
@@ -1295,7 +1359,7 @@ class ReteConditionExprVisitor(GetConditionExprVisitor):
             if not concept_as_str:
                 return FailedToCompileError([f"Missing concept in for {variable_path}"])
 
-            concept = self.evaluate(concept_as_str, True)
+            concept = self.evaluate_from_source(concept_as_str, return_body=True)
         else:
             concept = concept_to_recognize
 
@@ -1341,9 +1405,35 @@ class PythonConditionExprVisitorObj:
     not_variables: set
 
     @staticmethod
-    def combine_with_and(left, right):
+    def create_function(first, last, parameters):
 
-        def create_and(a, b):
+        def get_function_as_text(parameter):
+            if parameter is None:
+                return f"{first}{last}"
+
+            else:
+                return f"{first}{parameter}{last}"
+
+        if parameters is None:
+            source = get_function_as_text(None)
+            return PythonConditionExprVisitorObj(source, source, {}, set(), set())
+
+        parameters_as_list = parameters if isinstance(parameters, list) else [parameters]
+
+        res = []
+        for obj in parameters_as_list:
+            res.append(PythonConditionExprVisitorObj(get_function_as_text(obj.text),
+                                                     get_function_as_text(obj.source),
+                                                     obj.objects,
+                                                     obj.variables,
+                                                     obj.not_variables))
+
+        return res[0] if len(res) == 1 else res
+
+    @staticmethod
+    def create_and(left, right):
+
+        def get_source(a, b):
             if a is None and b is None:
                 return None
 
@@ -1351,38 +1441,47 @@ class PythonConditionExprVisitorObj:
                 return b
             if b is None or b == "":
                 return a
-            return a + " and " + b
+            return a + " and " + b  # no need to protect with parenthesis
 
-        left_as_list = left if isinstance(left, list) else [left]
-        right_as_list = right if isinstance(right, list) else [right]
-
-        left_right_product = list(product(left_as_list, right_as_list))
-        res = []
-        for left_obj, right_obj in left_right_product:
-            res.append(PythonConditionExprVisitorObj(create_and(left_obj.text, right_obj.text),
-                                                     create_and(left_obj.source, right_obj.source),
-                                                     merge_dictionaries(left_obj.objects, right_obj.objects),
-                                                     merge_sets(left_obj.variables, right_obj.variables),
-                                                     merge_sets(left_obj.not_variables, right_obj.not_variables)))
-
-        return res[0] if len(res) == 1 else res
+        return PythonConditionExprVisitorObj(get_source(left.text, right.text),
+                                             get_source(left.source, right.source),
+                                             merge_dictionaries(left.objects, right.objects),
+                                             merge_sets(left.variables, right.variables),
+                                             merge_sets(left.not_variables, right.not_variables))
 
     @staticmethod
-    def combine_with_not(node):
+    def combine_with_comma(left, right):
 
-        def create_not(a):
+        def get_source(a, b):
+            if a is None and b is None:
+                return None
+
+            if a is None or a == "":
+                return b
+            if b is None or b == "":
+                return a
+            return a + ", " + b  # no need to protect with parenthesis
+
+        if left is None:
+            return right
+
+        return PythonConditionExprVisitorObj(get_source(left.text, right.text),
+                                             get_source(left.source, right.source),
+                                             merge_dictionaries(left.objects, right.objects),
+                                             merge_sets(left.variables, right.variables),
+                                             merge_sets(left.not_variables, right.not_variables))
+
+    @staticmethod
+    def create_not(node):
+
+        def get_source(a):
             return f"not ({a})"
 
-        node_as_list = node if isinstance(node, list) else [node]
-        res = []
-        for obj in node_as_list:
-            res.append(PythonConditionExprVisitorObj(create_not(obj.text),
-                                                     create_not(obj.source),
-                                                     obj.objects,
-                                                     obj.variables,
-                                                     obj.not_variables))
-
-        return res[0] if len(res) == 1 else res
+        return PythonConditionExprVisitorObj(get_source(node.text),
+                                             get_source(node.source),
+                                             node.objects,
+                                             node.variables,
+                                             node.not_variables)
 
 
 class PythonConditionExprVisitor(GetConditionExprVisitor):
@@ -1390,13 +1489,24 @@ class PythonConditionExprVisitor(GetConditionExprVisitor):
     def __init__(self, context):
         super().__init__(context)
         self.know_object_variables = {}
+        self.check_variable_existence_only = True
+        self.concepts_to_reset = set()
 
     def get_conditions(self, expr_node):
+        self.check_variable_existence_only = True
         self.var_counter = 0
         self.variables.clear()
+        self.concepts_to_reset.clear()
 
         visitor_obj = self.visit(expr_node)
-        if visitor_obj.source:
+        if self.check_variable_existence_only:
+            return [CompiledCondition(None,
+                                      None,
+                                      visitor_obj.variables,
+                                      visitor_obj.not_variables,
+                                      visitor_obj.objects,
+                                      self.concepts_to_reset)]
+        else:
             if self.variables:
                 variables_definitions = "\n".join([f"{v} = {k}" for k, v in self.variables.items()])
                 source = variables_definitions + "\n" + visitor_obj.source
@@ -1409,48 +1519,18 @@ class PythonConditionExprVisitor(GetConditionExprVisitor):
             if ret.status:
                 ret.body.body.original_source = text
                 ret.body.body.objects = visitor_obj.objects
-                return [CompiledCondition(PythonEvaluator.NAME, ret, visitor_obj.variables, visitor_obj.not_variables)]
+                return [CompiledCondition(PythonEvaluator.NAME,
+                                          ret,
+                                          visitor_obj.variables,
+                                          visitor_obj.not_variables,
+                                          visitor_obj.objects,
+                                          self.concepts_to_reset)]
 
             else:
                 errors = ret.body.reason if self.context.sheerka.isinstance(ret.body, BuiltinConcepts.NOT_FOR_ME) \
                     else ret.body.body
                 raise FailedToCompileError(errors)
 
-        else:
-            return [CompiledCondition(None, None, visitor_obj.variables, visitor_obj.not_variables)]
-
-    def get_variable(self, expr_node):
-        """
-        From a ExprNode, try to know if it refers to a bag entry or it it's a python valid name
-        :param expr_node:
-        :return:
-        """
-        if not isinstance(expr_node, VariableNode):
-            return None
-
-        var_root = expr_node.name
-        if var_root in self.know_object_variables:
-            return self.know_object_variables[var_root]
-
-        if self.context.sheerka.fast_resolve(var_root):
-            self.know_object_variables[var_root] = None
-            return None
-
-        python_parser = PythonParser()
-        ret = python_parser.parse(self.context, ParserInput(var_root))
-        if not ret.status:
-            self.know_object_variables[var_root] = var_root
-            return var_root
-
-        python_evaluator = PythonEvaluator()
-        ret = python_evaluator.eval(self.context, ret)
-        if not ret.status:
-            self.know_object_variables[var_root] = var_root
-            return var_root
-
-        self.know_object_variables[var_root] = None
-        return None
-
     def get_new_variable(self, variable_path: List[str], obj_variables):
         obj_variables.add(variable_path[0])
         var_name, var_def = self.inner_get_new_variable(variable_path)
@@ -1467,43 +1547,65 @@ class PythonConditionExprVisitor(GetConditionExprVisitor):
         return root + "." + attribute
 
     def visit_VariableNode(self, expr_node: VariableNode):
-        # try to reconize a concept
         if expr_node.attributes_str is None and not expr_node.name.startswith("__"):
             # try to recognize a concept
-            res = evaluate(self.context,
-                           expr_node.name,
-                           evaluators=CONDITIONS_VISITOR_EVALUATORS,
-                           desc=None,
-                           eval_body=True,
-                           eval_where=False,
-                           is_question=False,
-                           expect_success=False,
-                           stm=None)
-            res = expect_one(self.context, res)
+            res = self.evaluate_from_source(expr_node.name, is_question=True)
             if res.status and isinstance(res.value, Concept):
-                return self.recognize_concept(["__ret", "body"], res.value, {})
+                self.check_variable_existence_only = False
+                if is_a_question(self.context, res.value):
+                    return self.evaluate_concept_as_question(expr_node.name, res.value)
+                else:
+                    return self.evaluate_concept(expr_node.name, res.value)
+            else:
+                if self.context.sheerka.has_error(self.context, res, __type=BuiltinConcepts.TOO_MANY_SUCCESS):
+                    raise FailedToCompileError([res])
+
+                variable_name = expr_node.get_source()
+                variables = {variable_name} if not res.status else set()
+                return PythonConditionExprVisitorObj(variable_name, variable_name, {}, variables, set())
 
         variable_name = expr_node.get_source()
-        return PythonConditionExprVisitorObj(None, None, {}, {variable_name}, set())
+        variables_detected = {variable_name} if self.is_a_possible_variable(variable_name) else set()
+        return PythonConditionExprVisitorObj(variable_name, variable_name, {}, variables_detected, set())
 
     def visit_ComparisonNode(self, expr_node: ComparisonNode):
+        self.check_variable_existence_only = False
+
         if not isinstance(expr_node.left, VariableNode):
+            # KSI 2021-04-22. Not quite sure of the reason why I have this piece of code
             left = self.visit(expr_node.left)
             source = expr_node.get_source()
             return PythonConditionExprVisitorObj(source, source, {}, left.variables, left.not_variables)
 
-        value = self.evaluate(expr_node.right.get_source(), True)
-        return self.create_comparison_condition(expr_node.left.unpack(), expr_node.comp, value)
+        right = self.visit(expr_node.right)
+        if right.source in right.objects:
+            return self.create_comparison_condition(expr_node.left.unpack(),
+                                                    expr_node.comp,
+                                                    right.source,
+                                                    right.objects[right.source],
+                                                    right.objects,
+                                                    expr_node.get_source())
+        else:
+            value = self.evaluate_from_source(expr_node.right.get_source(), return_body=True)
+            object_name, objects = self.get_object_name(value)
+            return self.create_comparison_condition(expr_node.left.unpack(),
+                                                    expr_node.comp,
+                                                    object_name,
+                                                    value,
+                                                    objects,
+                                                    expr_node.get_source())
 
     def visit_AndNode(self, expr_node: AndNode):
         current_visitor_obj = self.visit(expr_node.parts[0])
         for node in expr_node.parts[1:]:
             visitor_obj = self.visit(node)
-            current_visitor_obj = PythonConditionExprVisitorObj.combine_with_and(current_visitor_obj, visitor_obj)
+            current_visitor_obj = PythonConditionExprVisitorObj.create_and(current_visitor_obj, visitor_obj)
 
         return current_visitor_obj
 
     def visit_FunctionNode(self, expr_node: FunctionNode):
+        self.check_variable_existence_only = False
+
         if expr_node.first.value == "recognize(":
             if not isinstance(expr_node.parameters[0].value, VariableNode):
                 return FailedToCompileError([f"Cannot recognize '{expr_node.parameters[0].value}'"])
@@ -1512,46 +1614,48 @@ class PythonConditionExprVisitor(GetConditionExprVisitor):
                                           expr_node.parameters[1].value,
                                           {})
         else:
-            source = expr_node.get_source()
-            obj_variables = set()
+            params_as_visitor_obj = None
             for param in expr_node.parameters:
-                if (variable := self.get_variable(param.value)) is not None:
-                    obj_variables.add(variable)
-            return PythonConditionExprVisitorObj(source, source, {}, obj_variables, set())
+                current_visitor_obj = self.visit(param.value)
+                params_as_visitor_obj = PythonConditionExprVisitorObj.combine_with_comma(params_as_visitor_obj,
+                                                                                         current_visitor_obj)
+
+            return PythonConditionExprVisitorObj.create_function(expr_node.first, expr_node.last, params_as_visitor_obj)
 
     def visit_NotNode(self, expr_node: NotNode):
         visitor_obj = self.visit(expr_node.node)
-        if visitor_obj.source is None:
-            return PythonConditionExprVisitorObj(None, None, {}, visitor_obj.not_variables, visitor_obj.variables)
+        if self.check_variable_existence_only:
+            return PythonConditionExprVisitorObj(visitor_obj.text,
+                                                 visitor_obj.source,
+                                                 visitor_obj.objects,
+                                                 visitor_obj.not_variables,
+                                                 visitor_obj.variables)
 
-        return PythonConditionExprVisitorObj.combine_with_not(visitor_obj)
+        return PythonConditionExprVisitorObj.create_not(visitor_obj)
 
     def visit_NameExprNode(self, expr_node: NameExprNode):
-        res = evaluate(self.context,
-                       expr_node.get_source(),
-                       evaluators=CONDITIONS_VISITOR_EVALUATORS,
-                       desc=None,
-                       eval_body=True,
-                       eval_where=False,
-                       is_question=False,
-                       expect_success=False,
-                       stm=None)
-        res = expect_one(self.context, res)
-        if res.status and isinstance(res.value, Concept):
-            return self.recognize_concept(["__ret", "body"], res.value, {})
+        self.check_variable_existence_only = False
+        source = expr_node.get_source()
+        res = self.evaluate_from_source(source, is_question=False)
+        if res.status:
+            obj_name, objects = self.get_object_name(res.value)
+            return PythonConditionExprVisitorObj(source, obj_name, objects, set(), set())
+        else:
+            raise FailedToCompileError([expr_node])
 
-    def recognize_concept(self, variable_path, concept_to_recognize, concept_variables: dict):
+    def recognize_concept(self, variable_path, concept_to_recognize, concept_variables: dict, original_source=None):
         if not isinstance(concept_to_recognize, Concept):
             concept_as_str = concept_to_recognize.get_source()
             if not concept_as_str:
                 return FailedToCompileError([f"Missing concept in for {variable_path}"])
 
-            concept = self.evaluate(concept_as_str, True)
+            concept = self.evaluate_from_source(concept_as_str, return_body=True)
         else:
             concept = concept_to_recognize
 
         obj_variables = set()
         var_name = self.get_new_variable(variable_path, obj_variables)
+        objects = {}
 
         source = f"isinstance({var_name}, Concept)"
 
@@ -1563,33 +1667,51 @@ class PythonConditionExprVisitor(GetConditionExprVisitor):
             source += f" and {var_name}.key == '{concept.key}'"
             concept_variables.update({k: v for k, v in concept.variables().items() if v is not NotInit})
 
+        text = source
         for var_name, var_value in concept_variables.items():
             new_var_path = variable_path.copy()
             new_var_path.append(var_name)
-            variable_condition = self.create_comparison_condition(new_var_path, ComparisonType.EQUALS, var_value)
+            obj_name, objects = self.get_object_name(var_value, objects)
+            variable_condition = self.create_comparison_condition(new_var_path,
+                                                                  ComparisonType.EQUALS,
+                                                                  obj_name,
+                                                                  var_value,
+                                                                  objects)
             source += " and " + variable_condition.source
-            obj_variables.update(variable_condition.objects)
+            text += " and " + variable_condition.text
 
-        return PythonConditionExprVisitorObj(source, source, {}, obj_variables, set())
+        return PythonConditionExprVisitorObj(original_source or text, source, objects, obj_variables, set())
+
+    def evaluate_concept_as_question(self, original_text, concept):
+        concept_var_name, objects = self.get_object_name(concept)
+        source = f"evaluate_question({concept_var_name})"
+        variables = get_possible_variables_from_concept(self.context, concept)
+        self.concepts_to_reset.add(concept)
+        return PythonConditionExprVisitorObj(original_text, source, objects, variables, set())
+
+    def evaluate_concept(self, original_text, concept):
+        ensure_evaluated(self.context, concept, eval_body=True)
+        source, objects = self.get_object_name(concept)
+        return PythonConditionExprVisitorObj(original_text, source, objects, set(), set())
+
+    def create_comparison_condition(self, left_path, op, right_name, right_value, objects, original_source=None):
+        possible_variables = set()
+        var_root, var_attr = self.unpack_variable(left_path, possible_variables)
+        left = self.construct_variable(var_root, var_attr)
+
+        if original_source is None:
+            right = get_safe_str_value(right_value)
+            original_source = ComparisonNode.rebuild_source(left, op, right)
 
-    def create_comparison_condition(self, var_path, op, value):
-        obj_variables = set()
         if op == ComparisonType.EQUALS:
-            if self.context.sheerka.is_sheerka(value):
-                var_root, var_attr = self.unpack_variable(var_path, obj_variables)
-                source = f"is_sheerka({self.construct_variable(var_root, var_attr)})"
-                return PythonConditionExprVisitorObj(source, source, {}, obj_variables, set())
-            if isinstance(value, Concept):
-                return self.recognize_concept(var_path, value, {})
+            if self.context.sheerka.is_sheerka(right_value):
+                source = f"is_sheerka({left})"
+                return PythonConditionExprVisitorObj(original_source, source, objects, possible_variables, set())
+            if isinstance(right_value, Concept):
+                return self.recognize_concept(left_path, right_value, {}, original_source)
             else:
-                if isinstance(value, str):
-                    value = "'" + value + "'"
-                var_root, var_attr = self.unpack_variable(var_path, obj_variables)
-                source = ComparisonNode.rebuild_source(self.construct_variable(var_root, var_attr), op, value)
-                return PythonConditionExprVisitorObj(source, source, {}, obj_variables, set())
+                source = ComparisonNode.rebuild_source(left, op, right_name)
+                return PythonConditionExprVisitorObj(original_source, source, objects, possible_variables, set())
         else:
-            if isinstance(value, str):
-                value = "'" + value + "'"
-            var_root, var_attr = self.unpack_variable(var_path, obj_variables)
-            source = ComparisonNode.rebuild_source(self.construct_variable(var_root, var_attr), op, value)
-            return PythonConditionExprVisitorObj(source, source, {}, obj_variables, set())
+            source = ComparisonNode.rebuild_source(left, op, right_name)
+            return PythonConditionExprVisitorObj(original_source, source, objects, possible_variables, set())

src/core/sheerka/services/SheerkaVariableManager.py

@@ -94,7 +94,6 @@ class SheerkaVariableManager(BaseService):
         if who in self.bound_variables and key in self.bound_variables[who]:
             service = self.sheerka if who == self.sheerka.name else self.sheerka.services[who]
             setattr(service, key, value)
-            print(f"{service=} is set")
 
     def load_var(self, who, key):
         variable = self.sheerka.om.get(self.VARIABLES_ENTRY, who + "|" + key)

src/core/utils.py

@@ -4,6 +4,7 @@ import inspect
 import os
 import pkgutil
 import re
+import warnings
 from copy import deepcopy
 
 # from pyparsing import *
@@ -626,6 +627,35 @@ def as_bag(obj, forced_properties=None):
     return bag
 
 
+def sheerka_getattr(obj, name):
+    """
+    Wrapper to builtins.getattr that supports objects with as_bag signature
+    :param obj:
+    :param name:
+    :return:
+    """
+    if hasattr(obj, "as_bag"):
+        try:
+            return obj.as_bag()[name]
+        except KeyError:
+            raise AttributeError(f"'{type(obj).__name__}' object has no attribute '{name}'")
+
+    return getattr(obj, name)
+
+
+def sheerka_hasattr(obj, name):
+    """
+    Wrapper to builtins.hasattr that supports objects with as_bag signature
+    :param obj:
+    :param name:
+    :return:
+    """
+    if hasattr(obj, "as_bag"):
+        return name in obj.as_bag()
+
+    return hasattr(obj, name)
+
+
 def flatten_all_children(item, get_children):
     """
     Return a list containing the current item and all its children, recursively
@@ -775,6 +805,10 @@ def escape_str(x):
     :param x:
     :return:
     """
+    warnings.warn(
+        "use get_safe_str_value() instead",
+        DeprecationWarning
+    )
     if isinstance(x, str):
         return f"'{x}'"
     return x
@@ -802,7 +836,7 @@ class NextIdManager:
 
 
 def compute_hash(obj):
-    if isinstance(obj, list):
+    if isinstance(obj, (list, tuple)):
         return hash(tuple([compute_hash(o) for o in obj]))
 
     if isinstance(obj, set):
@@ -812,3 +846,21 @@ def compute_hash(obj):
         return hash(repr(obj))
 
     return hash(obj)
+
+
+def get_safe_str_value(obj):
+    from sheerkapython.python_wrapper import Expando
+    from core.concept import Concept
+    if isinstance(obj, str):
+        if "'" in obj:
+            value = escape_char(obj, '"')
+            return f'"{value}"'
+        else:
+            value = escape_char(obj, "'")
+            return f"'{value}'"
+    elif isinstance(obj, Expando):
+        return obj.get_name()
+    elif isinstance(obj, Concept):
+        return obj.str_id
+
+    return str(obj)

src/evaluators/DefConceptEvaluator.py

@@ -241,7 +241,7 @@ class DefConceptEvaluator(OneReturnValueEvaluator):
             names = [str(t.value) for t in ret_value.tokens if t.type in (TokenKind.IDENTIFIER,
                                                                           TokenKind.STRING,
                                                                           TokenKind.KEYWORD)]
-            possible_vars = filter(lambda x: x in concept_name and context.sheerka.is_not_a_variable(x), names)
+            possible_vars = filter(lambda x: x in concept_name and context.sheerka.is_not_a_concept_name(x), names)
             debugger.debug_var("names", names, hint="from NameNode")
             debugger.debug_var("possible_vars", possible_vars, hint="from NameNode")
             return [PossibleVariable(v) for v in possible_vars]
@@ -262,7 +262,7 @@ class DefConceptEvaluator(OneReturnValueEvaluator):
             if len(concept_name) > 1:
                 visitor = UnreferencedVariablesVisitor(context)
                 names = visitor.get_names(python_node.ast_)
-                possible_vars = filter(lambda x: x in concept_name and context.sheerka.is_not_a_variable(x), names)
+                possible_vars = filter(lambda x: x in concept_name and context.sheerka.is_not_a_concept_name(x), names)
                 debugger.debug_var("names", names, hint="from python node")
                 debugger.debug_var("possible_vars", possible_vars, hint="from python node")
                 return [PossibleVariable(v) for v in possible_vars]
@@ -275,7 +275,7 @@ class DefConceptEvaluator(OneReturnValueEvaluator):
         if (concept := get_inner_concept(ret_value.value)) is not None and len(concept_name) > 1:
             # use the variables of the concept is any
             names = [var_value or var_name for var_name, var_value in concept.get_metadata().variables]
-            possible_vars = filter(lambda x: context.sheerka.is_not_a_variable(x), names)
+            possible_vars = filter(lambda x: context.sheerka.is_not_a_concept_name(x), names)
             debugger.debug_var("names", names, hint="from concept")
             debugger.debug_var("possible_vars", possible_vars, hint="from concept")
             return [PossibleVariable(v) for v in possible_vars]
@@ -296,7 +296,7 @@ class DefConceptEvaluator(OneReturnValueEvaluator):
                 else:
                     names.append(t.str_value)
 
-            possible_vars = filter(lambda x: context.sheerka.is_not_a_variable(x), names)
+            possible_vars = filter(lambda x: context.sheerka.is_not_a_concept_name(x), names)
             debugger.debug_var("names", names, hint="from source")
             debugger.debug_var("possible_vars", possible_vars, hint="from source")
             return [PossibleVariable(v) for v in possible_vars]

src/evaluators/PythonEvaluator.py

@@ -8,50 +8,15 @@ import core.utils
 from core.ast_helpers import UnreferencedNamesVisitor, NamesWithAttributesVisitor
 from core.builtin_concepts import BuiltinConcepts, ParserResultConcept
 from core.concept import ConceptParts, Concept
-from core.global_symbols import NotInit, NotFound
-from core.rule import Rule
-from core.sheerka.ExecutionContext import ExecutionContext
+from core.global_symbols import NotInit, ErrorObj
 from core.sheerka.services.SheerkaMemory import SheerkaMemory
-from core.tokenizer import Token, TokenKind
-from core.var_ref import VariableRef
 from evaluators.BaseEvaluator import OneReturnValueEvaluator
 from parsers.PythonParser import PythonNode
-
-TO_DISABLED = ["breakpoint", "callable", "compile", "delattr", "eval", "exec", "exit", "input", "locals", "open",
-               "print", "quit", "setattr"]
-
-
-def inject_context(context):
-    """
-    function Decorator used to inject the context in methods that needed
-    :param context:
-    :return:
-    """
-
-    def wrapped(func):
-        def inner(*args, **kwargs):
-            return func(context, *args, **kwargs)
-
-        return inner
-
-    return wrapped
-
-
-class Expando:
-    def __init__(self, name, bag):
-        self.__name = name
-        for k, v in bag.items():
-            setattr(self, k, v)
-
-    def __repr__(self):
-        return f"{vars(self)}"
-
-    def get_name(self):
-        return self.__name
+from sheerkapython.python_wrapper import create_namespace, MethodAccessError
 
 
 @dataclass
-class PythonEvalError:
+class PythonEvalError(ErrorObj):
     error: Exception
     source: str
     traceback: str = field(repr=False)
@@ -72,6 +37,9 @@ class PythonEvalError:
     def __hash__(self):
         return hash(self.error)
 
+    def get_error(self):
+        return self.error
+
 
 class PythonEvaluator(OneReturnValueEvaluator):
     NAME = "Python"
@@ -83,6 +51,7 @@ class PythonEvaluator(OneReturnValueEvaluator):
 
     def __init__(self):
         super().__init__(self.NAME, [BuiltinConcepts.EVALUATION], 50)
+        self.isinstance = None
 
     @staticmethod
     def initialize(sheerka):
@@ -107,26 +76,28 @@ class PythonEvaluator(OneReturnValueEvaluator):
         context.log(f"Evaluating python node {node}.", self.name)
 
         # If we evaluate a Concept metadata which is NOT the body ex (pre, post, where...)
-        # We need to disable the function that may alter the state
+        #   or if EVAL_QUESTION_REQUESTED is explicit
+        # We need to disable the functions that may alter the state
         # It's a poor way to have source code security check
-        attr_under_eval = context.get_parents(lambda ec: ec.action == BuiltinConcepts.EVALUATING_ATTRIBUTE)
-        if attr_under_eval:
-            attr_under_eval = attr_under_eval[0]
-            expression_only = attr_under_eval.action_context != ConceptParts.BODY
+        expression_only = context.in_context(BuiltinConcepts.EVAL_QUESTION_REQUESTED)
 
-            if expression_only and isinstance(node.ast_, ast.Module):
-                # Module execution is forbidden in where, pre, post and ret concept parts
-                security_error = sheerka.new(BuiltinConcepts.PYTHON_SECURITY_ERROR,
-                                             prop=attr_under_eval.action_context,
-                                             body=node.source)
-                return sheerka.ret(self.name, False, security_error, parents=[return_value])
-        else:
-            expression_only = False
+        if not expression_only:
+            attr_under_eval = context.get_parents(lambda ec: ec.action == BuiltinConcepts.EVALUATING_ATTRIBUTE)
+            if attr_under_eval:
+                attr_under_eval = attr_under_eval[0]
+                expression_only = attr_under_eval.action_context != ConceptParts.BODY
 
         # get globals
-        my_globals = self.get_globals(context, node, expression_only)
+        try:
+            my_globals = self.get_globals(context, node, expression_only)
+            debugger.debug_var("globals", my_globals)
+        except MethodAccessError as ex:
+            eval_error = PythonEvalError(ex,
+                                         node.source,
+                                         traceback.format_exc() if get_trace_back else None,
+                                         None)
 
-        debugger.debug_var("globals", my_globals)
+            return sheerka.ret(self.name, False, sheerka.err(eval_error), parents=[return_value])
 
         all_possible_globals = self.get_all_possible_globals(context, my_globals)
         concepts_entries = None  # entries in globals_ that refers to Concept objects
@@ -189,104 +160,16 @@ class PythonEvaluator(OneReturnValueEvaluator):
         """
         unreferenced_names_visitor = UnreferencedNamesVisitor(context)
         names = unreferenced_names_visitor.get_names(node.ast_)
-        return self.get_globals_by_names(context, names, node, expression_only)
+        if "sheerka" in names:
+            sheerka_names = set()
+            visitor = NamesWithAttributesVisitor()
+            for sequence in visitor.get_sequences(node.ast_, "sheerka"):
+                if len(sequence) > 1:
+                    sheerka_names.add(sequence[1])
+        else:
+            sheerka_names = None
 
-    def get_sheerka_method(self, context, name, expression_only):
-        try:
-            method = context.sheerka.sheerka_methods[name]
-            context.log(f"Resolving '{name}'. It's a sheerka method.", self.name)
-            if expression_only and method.has_side_effect:
-                context.log(f"...but with side effect when {expression_only=}. Discarding.", self.name)
-                return None
-            else:
-                return inject_context(context)(method.method) if name in context.sheerka.methods_with_context \
-                    else method.method
-        except KeyError:
-            return None
-
-    def get_globals_by_names(self, context, names, node, expression_only):
-        my_globals = {
-            "Concept": core.concept.Concept,
-            "BuiltinConcepts": core.builtin_concepts.BuiltinConcepts,
-            "Expando": Expando,
-            "ExecutionContext": ExecutionContext,
-            "in_context": context.in_context,
-            "SyaAssociativity": core.global_symbols.SyaAssociativity
-        }
-
-        for name in names:
-            if name in my_globals:
-                continue
-
-            if expression_only and name in TO_DISABLED:
-                my_globals[name] = None
-                continue
-
-            # need to add it manually to avoid conflict with sheerka.isinstance
-            if name == "isinstance":
-                my_globals["isinstance"] = PythonEvaluator.isinstance
-                continue
-
-            # support reference to sheerka
-            if name.lower() == "sheerka":
-                bag = {}
-                visitor = NamesWithAttributesVisitor()
-                for sequence in visitor.get_sequences(node.ast_, "sheerka"):
-                    if (len(sequence) > 1 and
-                        (method := self.get_sheerka_method(context, sequence[1], expression_only)) is not None):
-                        bag[sequence[1]] = method
-                my_globals[name] = Expando("sheerka", bag)
-                continue
-
-            # search in short term memory
-            if (obj := context.get_from_short_term_memory(name)) is not NotFound:
-                context.log(f"Resolving '{name}'. Using value found in STM.", self.name)
-                my_globals[name] = obj
-                continue
-
-            # search in memory
-            if (obj := context.sheerka.get_last_from_memory(context, name)) is not NotFound:
-                context.log(f"Resolving '{name}'. Using value found in Long Term Memory.", self.name)
-                my_globals[name] = obj.obj
-                continue
-
-            # search in sheerka methods
-            if (method := self.get_sheerka_method(context, name, expression_only)) is not None:
-                my_globals[name] = method
-                continue
-
-            # search in context.obj (to replace by short time memory ?)
-            if context.obj:
-                if name == "self":
-                    my_globals["self"] = context.obj
-                    continue
-
-                try:
-                    attribute = context.obj.variables()[name]
-                    if attribute != NotInit:
-                        my_globals[name] = attribute
-                        continue
-                    context.log(f"Resolving '{name}'. It's obj attribute (obj={context.obj}).", self.name)
-                except KeyError:
-                    pass
-
-            # search in current node (if the name was found during the parsing)
-            if name in node.objects:
-                context.log(f"Resolving '{name}'. Using value from node.", self.name)
-                obj = self.resolve_object(context, node.objects[name])
-
-            # at last, try to instantiate a new concept
-            else:
-                context.log(f"Resolving '{name}'. Instantiating new concept.", self.name)
-                obj = self.resolve_object(context, name)
-
-            if obj is None:
-                context.log(f"...'{name}' is not found or cannot be instantiated. Skipping.", self.name)
-                continue
-
-            my_globals[name] = obj
-
-        return my_globals
+        return create_namespace(context, self.name, names, sheerka_names, node.objects, expression_only)
 
     @staticmethod
     def get_all_possible_globals(context, my_globals):
@@ -303,7 +186,7 @@ class PythonEvaluator(OneReturnValueEvaluator):
         :return:
         """
 
-        # first pass, get all the non concept or concept with no body
+        # first pass, get all the non concepts or concepts with no body
         # Note that we consider that all concepts are evaluated
         # In the future, it may be a good optimisation to defer the evaluation of the body
         # until the python evaluation fails
@@ -330,42 +213,6 @@ class PythonEvaluator(OneReturnValueEvaluator):
     def get_concepts_values_from_globals(my_globals, names):
         return {name: my_globals[name] for name in names}
 
-    @staticmethod
-    def resolve_object(context, name):
-        """
-        Try to find a concept by its name, id or the pattern c:key|id:
-        :param context:
-        :param name:
-        :return:
-        """
-
-        if isinstance(name, VariableRef):
-            return getattr(name.obj, name.prop)
-
-        if isinstance(name, Rule):
-            return context.sheerka.resolve_rule(context, name)
-
-        if isinstance(name, Concept):
-            name = core.builtin_helpers.ensure_evaluated(context, name)
-            return name
-
-        if isinstance(name, Token) and name.type == TokenKind.RULE:
-            return context.sheerka.resolve_rule(context, name)
-
-        if isinstance(name, tuple):
-            raise Exception()
-
-        # try to resolve by name
-        concept = context.sheerka.fast_resolve(name)
-        if concept is None:
-            return None
-
-        if hasattr(concept, "__iter__"):
-            raise NotImplementedError("Too many concepts")
-
-        concept = core.builtin_helpers.ensure_evaluated(context, concept)
-        return concept
-
     @staticmethod
     def expr_to_expression(expr):
         expr.lineno = 0

src/parsers/BaseNodeParser.py

@@ -210,7 +210,7 @@ class ConceptNode(LexerNode):
     def as_bag(self):
         """
         Creates a dictionary with the useful properties of the ConceptNode
-        see Concept.as_bag() for extra informations
+        see Concept.as_bag() for extra information
         """
         bag = {}
         for k, v in self.__dict__.items():

src/parsers/PythonParser.py

@@ -6,7 +6,7 @@ import core.utils
 from core.builtin_concepts import BuiltinConcepts
 from core.sheerka.services.SheerkaExecute import ParserInput
 from core.tokenizer import TokenKind
-from parsers.BaseParser import BaseParser, Node, ParsingError, BaseParserInputParser
+from parsers.BaseParser import Node, ParsingError, BaseParserInputParser
 
 log = logging.getLogger(__name__)
 
@@ -24,8 +24,8 @@ class PythonErrorNode(ParsingError):
     source: str
     exception: Exception
 
-    # def __post_init__(self):
-    #     self.log.debug("-> PythonErrorNode: " + str(self.exception))
+    def __hash__(self):
+        return hash((self.source, self.exception))
 
 
 @dataclass()

src/sdp/sheerkaDataProvider.py

@@ -29,10 +29,10 @@ class Event(object):
     Class that represents something that modifies the state of the system
     """
 
-    def __init__(self, message="", user_id="", date=datetime.now(), parents=None):
+    def __init__(self, message="", user_id="", date=None, parents=None):
         self.version = 1  # if the class Event ever changes, to keep track of the version
         self.user_id = user_id  # id of the user that triggers the modification
-        self.date = date  # when
+        self.date = date or datetime.now()  # when
         self.message = message  # user input or whatever that modifies the system
         self.parents = parents  # digest(s) of the parent(s) of this event
         self._digest = None  # digest of the event

src/sheerkapython/python_wrapper.py

@@ -0,0 +1,279 @@
+import functools
+from dataclasses import dataclass
+
+import core.builtin_helpers
+from core.builtin_concepts_ids import BuiltinConcepts
+from core.concept import Concept
+from core.global_symbols import SyaAssociativity, NotFound, NotInit, ErrorObj
+from core.rule import Rule
+from core.sheerka.ExecutionContext import ExecutionContext
+from core.sheerka.services.SheerkaAdmin import SheerkaAdmin
+from core.tokenizer import Token, TokenKind
+from core.utils import sheerka_hasattr, sheerka_getattr
+from core.var_ref import VariableRef
+
+TO_DISABLED = ["breakpoint", "callable", "compile", "delattr", "eval", "exec", "exit", "input", "locals", "open",
+               "print", "quit", "setattr"]
+
+
+@dataclass
+class MethodAccessError(Exception, ErrorObj):
+    method_name: str
+
+
+class ObjectContainer:
+    """
+    Container for list of object (or whatever), to easily use SheerkaQueryLanguage on collection
+    """
+
+    def __init__(self, items):
+        self.items = items
+
+
+class Expando:
+    def __init__(self, name, bag):
+        self.__name = name
+        for k, v in bag.items():
+            setattr(self, k, v)
+
+    def __repr__(self):
+        return f"{vars(self)}"
+
+    def get_name(self):
+        return self.__name
+
+    def __eq__(self, other):
+        if id(other) == id(self):
+            return True
+
+        if not isinstance(other, Expando):
+            return False
+
+        if other.get_name() != self.get_name():
+            return False
+
+        for k, v in vars(self).items():
+            if getattr(other, k) != v:
+                return False
+
+        return True
+
+    def __hash__(self):
+        hash_content = [self.__name] + list(vars(self).keys())
+        return hash(tuple(hash_content))
+
+
+class Pipe:
+    """
+    https://github.com/JulienPalard/Pipe/pull/23
+    Represent a Pipeable Element :
+    Described as :
+    first = Pipe(lambda iterable: next(iter(iterable)))
+    and used as :
+    print [1, 2, 3] | first
+    printing 1
+    Or represent a Pipeable Function :
+    It's a function returning a Pipe
+    Described as :
+    select = Pipe(lambda iterable, predicate: (predicate(x) for x in iterable))
+    and used as :
+    print [1, 2, 3] | select(lambda x: x * 2)
+    # 2, 4, 6
+    """
+
+    def __init__(self, function):
+        self.function = function
+        functools.update_wrapper(self, function)
+
+    def __ror__(self, other):
+        return self.function(other)
+
+    def __call__(self, *args, **kwargs):
+        return Pipe(lambda x: self.function(x, *args, **kwargs))
+
+
+def get_type(obj):
+    if isinstance(obj, Concept):
+        return obj.name
+    else:
+        return type(obj).__name__
+
+
+sheerka_globals = {
+    "Concept": Concept,
+    "BuiltinConcepts": BuiltinConcepts,
+    "Expando": Expando,
+    "ExecutionContext": ExecutionContext,
+    "SyaAssociativity": SyaAssociativity,
+    "get_type": get_type,
+    "hasattr": sheerka_hasattr,
+    "getattr": sheerka_getattr,
+}
+
+
+def inject_context(context):
+    """
+    function Decorator used to inject the context in methods that needed
+    :param context:
+    :return:
+    """
+
+    def wrapped(func):
+        @functools.wraps(func)
+        def inner(*args, **kwargs):
+            return func(context, *args, **kwargs)
+
+        return inner
+
+    return wrapped
+
+
+def resolve_object(context, who, obj):
+    """
+    Try to find a concept by its obj, id or the pattern c:key|id:
+    :param context:
+    :param who:
+    :param obj:
+    :return:
+    """
+
+    if isinstance(obj, VariableRef):
+        return getattr(obj.obj, obj.prop)
+
+    if isinstance(obj, Rule):
+        return context.sheerka.resolve_rule(context, obj)
+
+    if isinstance(obj, Concept):
+        obj = core.builtin_helpers.ensure_evaluated(context, obj)
+        return obj
+
+    if isinstance(obj, Token) and obj.type == TokenKind.RULE:
+        return context.sheerka.resolve_rule(context, obj)
+
+    if isinstance(obj, tuple):
+        # To make sure that there is no tuple that resembles to a concept
+        raise Exception()
+
+    if (isinstance(obj, str) and obj.startswith("c:")) or isinstance(obj, Token):
+        concept = context.sheerka.fast_resolve(obj)
+        if concept is None:
+            return None
+
+        if hasattr(concept, "__iter__"):
+            raise NotImplementedError("Too many concepts")
+
+        concept = core.builtin_helpers.ensure_evaluated(context, concept)
+        return concept
+
+    return obj
+
+
+def get_sheerka_method(context, who, name, expression_only):
+    try:
+        method = context.sheerka.sheerka_methods[name]
+        context.log(f"Resolving '{name}'. It's a sheerka method.", who)
+        if expression_only and method.has_side_effect:
+            context.log(f"...but with side effect when {expression_only=}. Discarding.", who)
+            raise MethodAccessError(name)
+        else:
+            method_to_use = inject_context(context)(method.method) if name in context.sheerka.methods_with_context \
+                else method.method
+
+            if name in context.sheerka.pipe_functions:
+                return Pipe(method_to_use)
+            else:
+                return method_to_use
+    except KeyError:
+        return None
+
+
+def create_namespace(context, who, names, sheerka_names, objects, expression_only, allow_builtins=False):
+    """
+    Create a namespace for the requested names
+    :param context:
+    :param who: who is asking
+    :param names: requested names
+    :param sheerka_names: requested sheerka names (ex sheerka.isinstance)
+    :param objects: local objects that can be added
+    :param expression_only: if true, discard method that can alter the global state
+    :param allow_builtins: automatically add python builtins symbols
+    :return:
+    """
+    result = dict(__builtins__) if allow_builtins else {}
+
+    for name in names:
+        if name in sheerka_globals:
+            result[name] = sheerka_globals[name]
+            continue
+
+        if expression_only and name in TO_DISABLED:
+            result[name] = None
+            continue
+
+        if name == "in_context":
+            result[name] = context.in_context
+            continue
+
+        # need to add it manually to avoid conflict with sheerka.isinstance
+        if name == "isinstance":
+            result["isinstance"] = context.sheerka.services[SheerkaAdmin.NAME].extended_isinstance
+            continue
+
+        # support reference to sheerka
+        if name.lower() == "sheerka":
+            bag = {}
+            for sheerka_name in sheerka_names:
+                if (method := get_sheerka_method(context, who, sheerka_name, expression_only)) is not None:
+                    bag[sheerka_name] = method
+            result[name] = Expando("sheerka", bag)
+            continue
+
+        # search in short term memory
+        if (obj := context.get_from_short_term_memory(name)) is not NotFound:
+            context.log(f"Resolving '{name}'. Using value found in STM.", who)
+            result[name] = obj
+            continue
+
+        # search in memory
+        if (obj := context.sheerka.get_last_from_memory(context, name)) is not NotFound:
+            context.log(f"Resolving '{name}'. Using value found in Long Term Memory.", who)
+            result[name] = obj.obj
+            continue
+
+        # search in sheerka methods
+        if (method := get_sheerka_method(context, who, name, expression_only)) is not None:
+            result[name] = method
+            continue
+
+        # search in context.obj (to replace by short time memory ?)
+        if context.obj:
+            if name == "self":
+                result["self"] = context.obj
+                continue
+
+            try:
+                attribute = context.obj.variables()[name]
+                if attribute != NotInit:
+                    result[name] = attribute
+                    continue
+                context.log(f"Resolving '{name}'. It's obj attribute (obj={context.obj}).", who)
+            except KeyError:
+                pass
+
+        # search in current node (if the name was found during the parsing)
+        if name in objects:
+            context.log(f"Resolving '{name}'. Using value from node.", who)
+            obj = resolve_object(context, who, objects[name])
+
+        # at last, try to instantiate a new concept
+        else:
+            context.log(f"Resolving '{name}'. Instantiating new concept.", who)
+            obj = resolve_object(context, who, f"c:{name}:")
+
+        if obj is None:
+            context.log(f"...'{name}' is not found or cannot be instantiated. Skipping.", who)
+            continue
+
+        result[name] = obj
+
+    return result

src/sheerkaql/OrderedSet.py

@@ -0,0 +1,80 @@
+from __future__ import print_function
+
+import collections
+from builtins import next
+from builtins import range
+
+# OrderedSet by Raymond Hettinger
+# Active State Recipe 576694
+# Licensed Under the MIT License see the LICENSE file
+# see http://code.activestate.com/recipes/576694/
+
+KEY, PREV, NEXT = list(range(3))
+
+
+class OrderedSet(collections.MutableSet):
+
+    def __init__(self, iterable=None):
+        self.end = end = []
+        end += [None, end, end]  # sentinel node for doubly linked list
+        self.map = {}  # key --> [key, prev, next]
+        if iterable is not None:
+            self |= iterable
+
+    def __len__(self):
+        return len(self.map)
+
+    def __contains__(self, key):
+        return key in self.map
+
+    def add(self, key):
+        if key not in self.map:
+            end = self.end
+            curr = end[PREV]
+            curr[NEXT] = end[PREV] = self.map[key] = [key, curr, end]
+
+    def discard(self, key):
+        if key in self.map:
+            key, prev, next = self.map.pop(key)
+            prev[NEXT] = next
+            next[PREV] = prev
+
+    def __iter__(self):
+        end = self.end
+        curr = end[NEXT]
+        while curr is not end:
+            yield curr[KEY]
+            curr = curr[NEXT]
+
+    def __reversed__(self):
+        end = self.end
+        curr = end[PREV]
+        while curr is not end:
+            yield curr[KEY]
+            curr = curr[PREV]
+
+    def pop(self, last=True):
+        if not self:
+            raise KeyError('set is empty')
+        key = next(reversed(self)) if last else next(iter(self))
+        self.discard(key)
+        return key
+
+    def __repr__(self):
+        if not self:
+            return '%s()' % (self.__class__.__name__,)
+        return '%s(%r)' % (self.__class__.__name__, list(self))
+
+    def __eq__(self, other):
+        if other is None: return False
+        if isinstance(other, OrderedSet):
+            return len(self) == len(other) and list(self) == list(other)
+        return set(self) == set(other)
+
+    def __del__(self):
+        self.clear()  # remove circular references
+
+
+if __name__ == '__main__':
+    print(OrderedSet('abracadaba'))
+    print(OrderedSet('simsalabim'))

src/sheerkaql/SheerkaQueryLangage.py

@@ -0,0 +1,26 @@
+from sheerkaql.lexer import Lexer
+from sheerkaql.parser import Parser
+
+lexer_instance = Lexer()
+
+
+class SheerkaQueryLanguage:
+
+    def compile(self, query):
+        """
+        Compiles a query string into a python function that takes one parameter, the execution namespace.
+        The compiled function is re-usable. For information on the grammar see X.
+        """
+        return Parser().parse(bytes(query, 'utf-8').decode('unicode_escape'), lexer=lexer_instance)
+
+    def execute(self, query, namespace, allow_builtins=False):
+        """"
+        Compiles the query string and executes it with the supplied namespace. If you want to execute a
+        particular query many times, use compile to get a query function.
+        :param query: query to execute (from  self.compile())
+        :param namespace: execution's namespace
+        :param allow_builtins: auto add builtin functions and names
+        """
+        if allow_builtins:
+            namespace.update(__builtins__)
+        return self.compile(query)(namespace)

src/sheerkaql/lexer.py

@@ -0,0 +1,157 @@
+"""
+This piece of code is originally taken from pyflwor
+https://github.com/timtadh/pyflwor
+"""
+from ply import lex
+from ply.lex import Token
+
+tokens = ('NUMBER', 'STRING', 'NAME',
+          'UNION', 'INTERSECTION',
+          'AND', 'OR',
+          'STAR', 'DASH', 'PLUS', 'SLASH', 'PERCENT',
+          "IF", "THEN", "ELSE",
+          'EQEQ', 'NQ', 'LE', 'GE', 'COMMA', 'DOT', 'COLON', "NOT", 'IS',
+          'LPAREN', 'RPAREN', 'LSQUARE', 'RSQUARE', 'LANGLE', 'RANGLE', 'LCURLY', 'RCURLY',
+          'FOR', 'IN', 'LET', 'EQ', 'WHERE', 'RETURN', 'ORDER', 'BY', 'ASC', 'DESC',
+          'FUNCTION', 'FLATTEN', 'COLLECT', 'AS', 'WITH',
+          'SUBSET', 'SUPERSET', 'PROPER',
+          'SOME', 'EVERY', 'SATISFIES',
+
+          )
+
+reserved = {'some': 'SOME', 'every': 'EVERY', 'in': 'IN', 'not': 'NOT',
+            'satisfies': 'SATISFIES',
+            'and': 'AND', 'or': 'OR', 'subset': 'SUBSET',
+            'superset': 'SUPERSET',
+            'proper': 'PROPER', 'is': 'IS', 'for': 'FOR',
+            'let': 'LET', 'return': 'RETURN',
+            'where': 'WHERE', 'order': 'ORDER', 'by': 'BY',
+            'asc': 'ASC', 'desc': 'DESC',
+            'function': 'FUNCTION', 'if': 'IF', 'then': 'THEN', 'else': 'ELSE',
+            'flatten': 'FLATTEN', 'collect': 'COLLECT',
+            'as': 'AS', 'with': 'WITH'}
+
+# Common Regex Parts
+DIGIT = r'[0-9]'
+LETTER = r'[a-zA-Z_]'
+HEX = r'[a-fA-F0-9]'
+EXP = r'[Ee][+-]?(' + DIGIT + ')+'
+
+
+# Tim Henderson
+# Normally PLY works at the module level. I prefer having it encapsulated as
+# a class. Thus the strange construction of this class in the new method allows
+# PLY to do its magic.
+class Lexer:
+
+    def __new__(cls, **kwargs):
+        self = super(Lexer, cls).__new__(cls, **kwargs)
+        self.lexer = lex.lex(object=self, debug=False, optimize=True, **kwargs)
+        return self.lexer
+
+    tokens = tokens
+
+    t_EQ = r'='
+    t_EQEQ = r'=='
+    t_NQ = r'!='
+    t_GE = r'\>='
+    t_LE = r'\<='
+    t_DOT = r'\.'
+    t_STAR = r'\*'
+    t_DASH = r'\-'
+    t_PLUS = r'\+'
+    t_COMMA = r','
+    t_COLON = r'\:'
+    t_SLASH = r'/'
+    t_UNION = r'\|'
+    # t_DOLLAR = r'\$'
+    t_LPAREN = r'\('
+    t_RPAREN = r'\)'
+    t_LCURLY = r'\{'
+    t_RCURLY = r'\}'
+    t_LANGLE = r'\<'
+    t_RANGLE = r'\>'
+    t_LSQUARE = r'\['
+    t_RSQUARE = r'\]'
+    # t_DIFFERENCE = r'-'
+    t_INTERSECTION = r'&'
+    t_PERCENT = r'%'
+
+    string_literal1 = r'\"[^"]*\"'
+
+    @Token(string_literal1)
+    def t_STRING_LITERAL1(self, token):
+        token.type = 'STRING'
+        token.value = token.value[1:-1]
+        return token
+
+    string_literal2 = r"\'[^']*\'"
+
+    @Token(string_literal2)
+    def t_STRING_LITERAL2(self, token):
+        token.type = 'STRING'
+        token.value = token.value[1:-1]
+        return token
+
+    name = '(' + LETTER + ')((' + LETTER + ')|(' + DIGIT + '))*'
+
+    @Token(name)
+    def t_NAME(self, token):
+        if token.value in reserved:
+            token.type = reserved[token.value]
+        else:
+            token.type = 'NAME'
+        return token
+
+    const_hex = '0[xX](' + HEX + ')+'
+
+    @Token(const_hex)
+    def t_CONST_HEX(self, token):
+        token.type = 'NUMBER'
+        token.value = int(token.value, 16)
+        return token
+
+    const_float1 = '(' + DIGIT + ')+' + '(' + EXP + ')'
+
+    @Token(const_float1)
+    def t_CONST_FLOAT1(self, token):
+        token.type = 'NUMBER'
+        token.value = float(token.value)
+        return token
+
+    const_float2 = '(' + DIGIT + r')*\.(' + DIGIT + ')+(' + EXP + ')?'
+
+    @Token(const_float2)
+    def t_CONST_FLOAT2(self, token):
+        token.type = 'NUMBER'
+        token.value = float(token.value)
+        return token
+
+    const_dec_oct = '(' + DIGIT + ')+'
+
+    @Token(const_dec_oct)
+    def t_CONST_INTEGER_OCTAL(self, token):
+        token.type = 'NUMBER'
+        if (len(token.value) > 1 and token.value[0] == '0' or
+            (token.value[0] == '-' and token.value[1] == '0')):
+            token.value = int(token.value, 8)
+        else:
+            token.value = int(token.value, 10)
+        return token
+
+    @Token(r'(/\*([^*]|[\r\n]|(\*+([^*/]|[\r\n])))*\*+/)|(//.*)')
+    def t_COMMENT(self, token):
+        lines = len(token.value.split('\n')) - 1
+        if lines < 0:
+            lines = 0
+        token.lexer.lineno += lines
+
+    @Token(r'\n+')
+    def t_newline(self, t):
+        t.lexer.lineno += t.value.count("\n")
+
+    # Ignored characters
+    t_ignore = " \t"
+
+    def t_error(self, t):
+        raise Exception("Illegal character '%s'" % t)

src/sheerkaql/parser.py

@@ -0,0 +1,787 @@
+"""
+This piece of code is originally taken from pyflwor
+https://github.com/timtadh/pyflwor
+"""
+
+# Tim Henderson
+# The parser does not build an abstract syntax tree nor does it build
+# intermediate code, instead it composes functions and objects together. These
+# functions are defined in symbols.py file. The composed function returned
+# computes the query based on the object dictionary passed into it. This
+# dictionary (objs) is passed down through the functions (sometimes with
+# modification).
+
+# If you are confused about the syntax in this file I recommend reading the
+# documentation on the PLY website to see how this compiler compiler's syntax
+# works.
+from ply import yacc
+
+from sheerkaql import symbols
+from sheerkaql.lexer import tokens
+
+
+class SingletonMeta(type):
+    """
+    The Singleton class can be implemented in different ways in Python. Some
+    possible methods include: base class, decorator, metaclass. We will use the
+    metaclass because it is best suited for this purpose.
+    """
+
+    _instances = {}
+
+    def __call__(cls, *args, **kwargs):
+        """
+        Possible changes to the value of the `__init__` argument do not affect
+        the returned instance.
+        """
+        if cls not in cls._instances:
+            instance = super().__call__(*args, **kwargs)
+            cls._instances[cls] = instance
+        return cls._instances[cls]
+
+
+class Parser(metaclass=SingletonMeta):
+
+    def __init__(self, **kwargs):
+        self.names = None  # identifiers found
+        self.sheerka_names = None  # references to sheerka methods (sheerka.something)
+        self.yacc = yacc.yacc(module=self, debug=False, optimize=True,
+                              write_tables=False, **kwargs)
+
+    def parse(self, *args, **kwargs):
+        self.names = set()
+        self.sheerka_names = set()
+        return self.yacc.parse(*args, **kwargs)
+
+    tokens = tokens
+    precedence = (
+        ('right', 'RSQUARE'),
+        ('right', 'DASH', 'PLUS', 'SLASH', 'STAR'),
+    )
+
+    @staticmethod
+    def p_start_with_set(t):
+        """Start : Set"""
+        t[0] = t[1]
+
+    @staticmethod
+    def p_start_with_flwr_expression(t):
+        """Start : FLWRexpr"""
+        t[0] = t[1]
+
+    @staticmethod
+    def p_for_return_expr(t):
+        """FLWRexpr : ForExpr ReturnExpr"""
+        t[0] = symbols.flwr_sequence(t[2][0], for_expr=t[1], flatten=t[2][1], collecting=t[2][2])
+
+    @staticmethod
+    def p_for_let_return_expr(t):
+        """FLWRexpr : ForExpr LetExpr ReturnExpr"""
+        t[0] = symbols.flwr_sequence(t[3][0], for_expr=t[1], flatten=t[3][1], collecting=t[3][2], let_expr=t[2])
+
+    @staticmethod
+    def p_for_where_return_expr(t):
+        """FLWRexpr : ForExpr WhereExpr ReturnExpr"""
+        t[0] = symbols.flwr_sequence(t[3][0], for_expr=t[1], flatten=t[3][1], collecting=t[3][2], where_expr=t[2])
+
+    @staticmethod
+    def p_for_let_where_return_expr(t):
+        """FLWRexpr : ForExpr LetExpr WhereExpr ReturnExpr"""
+        t[0] = symbols.flwr_sequence(t[4][0], for_expr=t[1], flatten=t[4][1], collecting=t[4][2], let_expr=t[2],
+                                     where_expr=t[3])
+
+    @staticmethod
+    def p_for_order_by_return_expr(t):
+        """FLWRexpr : ForExpr OrderByExpr ReturnExpr"""
+        t[0] = symbols.flwr_sequence(t[3][0], for_expr=t[1], flatten=t[3][1], collecting=t[3][2], order_expr=t[2])
+
+    @staticmethod
+    def p_for_let_order_by_return_expr(t):
+        """FLWRexpr : ForExpr LetExpr OrderByExpr ReturnExpr"""
+        t[0] = symbols.flwr_sequence(t[4][0], for_expr=t[1], flatten=t[4][1], collecting=t[4][2], let_expr=t[2],
+                                     order_expr=t[3])
+
+    @staticmethod
+    def p_for_where_order_by_return_expr(t):
+        """FLWRexpr : ForExpr WhereExpr OrderByExpr ReturnExpr"""
+        t[0] = symbols.flwr_sequence(t[4][0], for_expr=t[1], flatten=t[4][1], collecting=t[4][2], where_expr=t[2],
+                                     order_expr=t[3])
+
+    @staticmethod
+    def p_for_let_where_oder_by_return_expr(t):
+        """FLWRexpr : ForExpr LetExpr WhereExpr OrderByExpr ReturnExpr"""
+        t[0] = symbols.flwr_sequence(t[5][0], for_expr=t[1], flatten=t[5][1], collecting=t[5][2], let_expr=t[2],
+                                     where_expr=t[3], order_expr=t[4])
+
+    @staticmethod
+    def p_return_expr(t):
+        """FLWRexpr : ReturnExpr"""
+        t[0] = symbols.flwr_sequence(t[1][0], flatten=t[1][1], collecting=t[1][2])
+
+    @staticmethod
+    def p_let_return_expr(t):
+        """FLWRexpr : LetExpr ReturnExpr"""
+        t[0] = symbols.flwr_sequence(t[2][0], flatten=t[2][1], collecting=t[2][2], let_expr=t[1])
+
+    @staticmethod
+    def p_for_expression(t):
+        """ForExpr : FOR ForList"""
+        t[0] = t[2]
+
+    @staticmethod
+    def p_for_list_with_multiple_items(t):
+        """ForList : ForList COMMA ForDefinition"""
+        t[0] = t[1] + [t[3]]
+
+    @staticmethod
+    def p_for_list(t):
+        """ForList : ForDefinition"""
+        t[0] = [t[1]]
+
+    @staticmethod
+    def p_for_definition_using_set(t):
+        """ForDefinition : NAME IN LANGLE Set RANGLE"""
+        t[0] = (t[1], t[4])
+
+    @staticmethod
+    def p_for_definition_using_flwr(t):
+        """ForDefinition : NAME IN LCURLY FLWRexpr RCURLY"""
+        t[0] = (t[1], t[4])
+
+    @staticmethod
+    def p_for_definition_using_value(t):
+        """ForDefinition : NAME IN Value"""
+        t[0] = (t[1], t[3])
+
+    @staticmethod
+    def p_multiple_let_expression(t):
+        """LetExpr : LetExpr LET LetList"""
+        t[0] = t[1] + t[3]
+
+    @staticmethod
+    def p_let_expression(t):
+        """LetExpr : LET LetList"""
+        t[0] = t[2]
+
+    @staticmethod
+    def p_let_list_from_let_definition(t):
+        """LetList : LetList COMMA LetDefinition"""
+        t[0] = t[1] + [t[3]]
+
+    @staticmethod
+    def p_let_list(t):
+        """LetList : LetDefinition"""
+        t[0] = [t[1]]
+
+    @staticmethod
+    def p_let_definition_from_set(t):
+        """LetDefinition : NAME EQ LANGLE Set RANGLE"""
+        t[0] = (t[1], t[4])
+
+    @staticmethod
+    def p_let_definition_from_flwr(t):
+        """LetDefinition : NAME EQ LCURLY FLWRexpr RCURLY"""
+        t[0] = (t[1], t[4])
+
+    @staticmethod
+    def p_let_definition_from_arith_expr(t):
+        """LetDefinition : NAME EQ ArithExpr"""
+        t[0] = (t[1], t[3])
+
+    @staticmethod
+    def p_let_definition_from_function(t):
+        """LetDefinition : NAME EQ Function"""
+        t[0] = (t[1], symbols.function_definition(*t[3]))
+
+    @staticmethod
+    def p_function_definition_no_parameter(t):
+        """Function : FUNCTION LPAREN RPAREN LCURLY FBody RCURLY"""
+        t[0] = (tuple(), t[5])
+
+    @staticmethod
+    def p_function_definition_with_parameters(t):
+        """Function : FUNCTION LPAREN FParams RPAREN LCURLY FBody RCURLY"""
+        t[0] = (tuple(t[3]), t[6])
+
+    @staticmethod
+    def p_function_parameters(t):
+        """FParams : FParams COMMA NAME"""
+        t[0] = t[1] + [t[3]]
+
+    @staticmethod
+    def p_function_parameters_from_name(t):
+        """FParams : NAME"""
+        t[0] = [t[1]]
+
+    @staticmethod
+    def p_function_body_from_flwr(t):
+        """FBody : FLWRexpr"""
+        t[0] = t[1]
+
+    @staticmethod
+    def p_function_body_from_arith_expr(t):
+        """FBody : ArithExpr"""
+        t[0] = t[1]
+
+    @staticmethod
+    def p_where_definition(t):
+        """WhereExpr : WHERE Where"""
+        t[0] = t[2]
+
+    @staticmethod
+    def p_order_by_number(t):
+        """OrderByExpr : ORDER BY NUMBER OrderDirection"""
+        t[0] = (t[3], t[4])
+
+    @staticmethod
+    def p_order_by_string(t):
+        """OrderByExpr : ORDER BY STRING OrderDirection"""
+        t[0] = (t[3], t[4])
+
+    @staticmethod
+    def p_order_direction_asc(t):
+        """OrderDirection : ASC"""
+        t[0] = 'ASC'
+
+    @staticmethod
+    def p_order_direction_desc(t):
+        """OrderDirection : DESC"""
+        t[0] = 'DESC'
+
+    @staticmethod
+    def p_return_definition_from_tuple(t):
+        """ReturnExpr : RETURN OutputTuple"""
+        t[0] = (t[2], False, False)
+
+    @staticmethod
+    def p_return_definition_from_dict(t):
+        """ReturnExpr : RETURN OutputDict"""
+        t[0] = (t[2], False, False)
+
+    @staticmethod
+    def p_return_definition_from_value(t):
+        """ReturnExpr : RETURN FLATTEN OutputValue"""
+        t[0] = ([t[3]], True, False)
+
+    @staticmethod
+    def p_return_definition_from_collection(t):
+        """ReturnExpr : CollectList"""
+        t[0] = (t[1], False, True)
+
+    @staticmethod
+    def p_collection_list(t):
+        """CollectList : CollectList Collect"""
+        t[0] = t[1] + [t[2]]
+
+    @staticmethod
+    def p_collection_list_from_collect(t):
+        """CollectList : Collect"""
+        t[0] = [t[1]]
+
+    @staticmethod
+    def p_collect_definition_from_tuple(t):
+        """Collect : COLLECT OutputTuple AS ArithExpr WITH CollectFunction"""
+        t[0] = {'value': t[2], 'as': t[4], 'with': t[6]}
+
+    @staticmethod
+    def p_collect_definition_from_dict(t):
+        """Collect : COLLECT OutputDict AS ArithExpr WITH CollectFunction"""
+        t[0] = {'value': t[2], 'as': t[4], 'with': t[6]}
+
+    @staticmethod
+    def p_collect_function_from_attribute(t):
+        """CollectFunction : AttributeValue"""
+        t[0] = symbols.attribute_value(t[1])
+
+    @staticmethod
+    def p_collect_function_from_function(t):
+        """CollectFunction : Function"""
+        t[0] = symbols.function_definition(*t[1])
+
+    @staticmethod
+    def p_output_as_tuple(t):
+        """OutputTuple : OutputTuple COMMA OutputValue"""
+        t[0] = t[1] + [t[3]]
+
+    @staticmethod
+    def p_output_as_value(t):
+        """OutputTuple : OutputValue"""
+        t[0] = [t[1]]
+
+    @staticmethod
+    def p_output_as_dict_multiple(t):
+        """OutputDict : OutputDict COMMA STRING COLON OutputValue"""
+        t[0] = t[1] + [(t[3], t[5])]
+
+    @staticmethod
+    def p_output_as_dict(t):
+        """OutputDict : STRING COLON OutputValue"""
+        t[0] = [(t[1], t[3])]
+
+    @staticmethod
+    def p_output_as_arith_expr(t):
+        """OutputValue : ArithExpr"""
+        t[0] = t[1]
+
+    @staticmethod
+    def p_output_as_set(t):
+        """OutputValue : LANGLE Set RANGLE"""
+        t[0] = t[2]
+
+    @staticmethod
+    def p_output_as_flwr(t):
+        """OutputValue : LCURLY FLWRexpr RCURLY"""
+        t[0] = t[2]
+
+    @staticmethod
+    def p_set_from_diff(t):
+        """Set : Set DASH UnionExpr"""
+        t[0] = symbols.set_value(t[1], symbols.set_operator(t[2]), t[3])
+
+    @staticmethod
+    def p_set(t):
+        """Set : UnionExpr"""
+        t[0] = t[1]
+
+    @staticmethod
+    def p_union_expr(t):
+        """UnionExpr : UnionExpr UNION IntersectionExpr"""
+        t[0] = symbols.set_value(t[1], symbols.set_operator(t[2]), t[3])
+
+    @staticmethod
+    def p_union_from_intersection(t):
+        """UnionExpr : IntersectionExpr"""
+        t[0] = t[1]
+
+    @staticmethod
+    def p_intersection(t):
+        """IntersectionExpr : IntersectionExpr INTERSECTION Collection"""
+        t[0] = symbols.set_value(t[1], symbols.set_operator(t[2]), t[3])
+
+    @staticmethod
+    def p_intersection_from_collection(t):
+        """IntersectionExpr : Collection"""
+        t[0] = t[1]
+
+    @staticmethod
+    def p_collection_from_query(t):
+        """Collection : Query"""
+        t[0] = t[1]
+
+    @staticmethod
+    def p_collection_from_set(t):
+        """Collection : LPAREN Set RPAREN"""
+        t[0] = t[2]
+
+    def p_query_start(self, t):
+        """Query : Query_"""
+        self.names.add(t[1][0][0])
+        if len(t[1]) == 2 and t[1][0][0] == "sheerka":
+            self.sheerka_names.add(t[1][1][0])
+        t[0] = symbols.query_value(t[1])
+
+    @staticmethod
+    def p_query_with_slash(t):
+        """Query_ : Query_ SLASH Entity"""
+        t[0] = t[1] + [t[3]]
+
+    @staticmethod
+    def p_query_with_dot(t):
+        """Query_ : Query_ DOT Entity"""
+        t[0] = t[1] + [t[3]]
+
+    @staticmethod
+    def p_query_from_entity(t):
+        """Query_ : Entity"""
+        t[0] = [t[1]]
+
+    @staticmethod
+    def p_entity_from_name(t):
+        """Entity : NAME"""
+        t[0] = (t[1], None)
+
+    @staticmethod
+    def p_entity_from_condition(t):
+        """Entity : NAME LSQUARE Where RSQUARE"""
+        t[0] = (t[1], symbols.where_value(t[3]))
+
+    @staticmethod
+    def p_where(t):
+        """Where : OrExpr"""
+        t[0] = t[1]
+
+    @staticmethod
+    def p_or_expr(t):
+        """OrExpr : OrExpr OR AndExpr"""
+        t[0] = symbols.boolean_expression_value(t[1], symbols.bool_operator(t[2]), t[3])
+
+    @staticmethod
+    def p_or_expr_from_and(t):
+        """OrExpr : AndExpr"""
+        t[0] = t[1]
+
+    @staticmethod
+    def p_and_expr(t):
+        """AndExpr : AndExpr AND NotExpr"""
+        t[0] = symbols.boolean_expression_value(t[1], symbols.bool_operator(t[2]), t[3])
+
+    @staticmethod
+    def p_and_from_not(t):
+        """AndExpr : NotExpr"""
+        t[0] = t[1]
+
+    @staticmethod
+    def p_not_expr(t):
+        """NotExpr : NOT BooleanExpr"""
+        t[0] = symbols.unary_expression_value(symbols.unary_operator(t[1]), t[2])
+
+    @staticmethod
+    def p_not_from_boolean(t):
+        """NotExpr : BooleanExpr"""
+        t[0] = t[1]
+
+    @staticmethod
+    def p_boolean_from_comparison(t):
+        """BooleanExpr : CmpExpr"""
+        t[0] = t[1]
+
+    @staticmethod
+    def p_boolean_from_quantified(t):
+        """BooleanExpr : QuantifiedExpr"""
+        t[0] = t[1]
+
+    @staticmethod
+    def p_boolean_from_set(t):
+        """BooleanExpr : SetExpr"""
+        t[0] = t[1]
+
+    @staticmethod
+    def p_boolean_from_arith(t):
+        """BooleanExpr : ArithExpr"""
+        t[0] = t[1]
+
+    @staticmethod
+    def p_boolean_as_where_exp(t):
+        """BooleanExpr : LPAREN Where RPAREN"""
+        t[0] = t[2]
+
+    @staticmethod
+    def p_comparison_expr(t):
+        """CmpExpr : ArithExpr CmpOp ArithExpr"""
+        t[0] = symbols.comparison_value(t[1], t[2], t[3])
+
+    @staticmethod
+    def p_comparison_operation(t):
+        """CmpOp : EQEQ
+                | NQ
+                | LANGLE
+                | LE
+                | RANGLE
+                | GE"""
+        t[0] = symbols.operator(t[1])
+
+    @staticmethod
+    def p_arith_expr(t):
+        """ArithExpr : AddSub"""
+        t[0] = t[1]
+
+    @staticmethod
+    def p_term_expr_from_add(t):
+        """AddSub : AddSub PLUS MulDiv"""
+        t[0] = symbols.arith_value(t[1], symbols.arith_operator(t[2]), t[3])
+
+    @staticmethod
+    def p_term_expr_from_sub(t):
+        """AddSub : AddSub DASH MulDiv"""
+        t[0] = symbols.arith_value(t[1], symbols.arith_operator(t[2]), t[3])
+
+    @staticmethod
+    def p_term_from_factor(t):
+        """AddSub : MulDiv"""
+        t[0] = t[1]
+
+    @staticmethod
+    def p_factor_expr_from_mult(t):
+        """MulDiv : MulDiv STAR ArithUnary"""
+        t[0] = symbols.arith_value(t[1], symbols.arith_operator(t[2]), t[3])
+
+    @staticmethod
+    def p_factor_expr_from_div(t):
+        """MulDiv : MulDiv SLASH ArithUnary"""
+        t[0] = symbols.arith_value(t[1], symbols.arith_operator(t[2]), t[3])
+
+    @staticmethod
+    def p_factor_expr_from_modulo(t):
+        """MulDiv : MulDiv PERCENT ArithUnary"""
+        t[0] = symbols.arith_value(t[1], symbols.arith_operator(t[2]), t[3])
+
+    @staticmethod
+    def p_factor_expr_from_unary(t):
+        """MulDiv : ArithUnary"""
+        t[0] = t[1]
+
+    @staticmethod
+    def p_unary_expr(t):
+        """ArithUnary : Atomic"""
+        t[0] = t[1]
+
+    @staticmethod
+    def p_unary_expr_from_negate(t):
+        """ArithUnary : DASH Atomic"""
+        t[0] = symbols.arith_value(symbols.attribute_value(-1.0, scalar=True), symbols.arith_operator('*'), t[2])
+
+    @staticmethod
+    def p_atomic_expr(t):
+        """Atomic : Value"""
+        t[0] = t[1]
+
+    @staticmethod
+    def p_atomic_expr_from_arith_expr(t):
+        """Atomic : LPAREN ArithExpr RPAREN"""
+        t[0] = t[2]
+
+    @staticmethod
+    def p_value_from_number(t):
+        """Value : NUMBER"""
+        t[0] = symbols.attribute_value(t[1], scalar=True)
+
+    @staticmethod
+    def p_value_from_number_string(t):
+        """Value : STRING"""
+        t[0] = symbols.attribute_value(t[1], scalar=True)
+
+    @staticmethod
+    def p_value_from_if_then_else(t):
+        """Value : IF Where THEN IfBody ELSE IfBody"""
+        t[0] = symbols.if_expression(t[2], t[4], t[6])
+
+    @staticmethod
+    def p_value_from_python_ternary_operator(t):
+        """Value : IfBody IF Where ELSE IfBody"""
+        t[0] = symbols.if_expression(t[3], t[1], t[5])
+
+    @staticmethod
+    def p_value_from_attritute_value(t):
+        """Value : AttributeValue"""
+        t[0] = symbols.attribute_value(t[1])
+
+    @staticmethod
+    def p_value_from_dict(t):
+        """Value : LCURLY NameValPairs RCURLY"""
+        t[0] = symbols.dict_value(t[2])
+
+    @staticmethod
+    def p_value_from_list(t):
+        """Value : LSQUARE ValueList RSQUARE"""
+        t[0] = symbols.list_value(t[2])
+
+    @staticmethod
+    def p_name_value_pairs(t):
+        """NameValPairs : NameValPairs COMMA NameValPair"""
+        t[0] = t[1] + [t[3]]
+
+    @staticmethod
+    def p_name_value_pairs_from_name_value_pair(t):
+        """NameValPairs : NameValPair"""
+        t[0] = [t[1]]
+
+    @staticmethod
+    def p_name_value_pair(t):
+        """NameValPair : ArithExpr COLON ArithExpr"""
+        t[0] = (t[1], t[3])
+
+    @staticmethod
+    def p_list_of_values(t):
+        """ValueList : ValueList COMMA ArithExpr"""
+        t[0] = t[1] + [t[3]]
+
+    @staticmethod
+    def p_list_of_values_from_arith_expr(t):
+        """ValueList : ArithExpr"""
+        t[0] = [t[1]]
+
+    @staticmethod
+    def p_if_body_from_arith_expr(t):
+        """IfBody : ArithExpr"""
+        t[0] = t[1]
+
+    @staticmethod
+    def p_if_body_from_arith_expr_from_set(t):
+        """IfBody : LANGLE Set RANGLE"""
+        t[0] = t[2]
+
+    @staticmethod
+    def p_if_body_from_arith_expr_from_flwr(t):
+        """IfBody : LCURLY FLWRexpr RCURLY"""
+        t[0] = t[2]
+
+    def p_attribute_value(self, t):
+        """AttributeValue : AttributeValue DOT Attr"""
+        self.names.add(t[1][0].name)
+        if len(t[1]) == 1 and t[1][0].name == "sheerka":
+            self.sheerka_names.add(t[3].name)
+        t[0] = t[1] + [t[3]]
+
+    def p_attribute_value_from_attr_expr(self, t):
+        """AttributeValue : Attr"""
+        self.names.add(t[1].name)
+        t[0] = [t[1]]
+
+    @staticmethod
+    def p_list_of_parameters(t):
+        """ParameterList : ParameterList COMMA Parameter"""
+        t[0] = t[1] + [t[3]]
+
+    @staticmethod
+    def p_list_of_parameters_from_parameter(t):
+        """ParameterList : Parameter"""
+        t[0] = [t[1]]
+
+    @staticmethod
+    def p_parameter_from_arith(t):
+        """Parameter : ArithExpr"""
+        t[0] = t[1]
+
+    @staticmethod
+    def p_parameter_from_set(t):
+        """Parameter : LANGLE Set RANGLE"""
+        t[0] = t[2]
+
+    @staticmethod
+    def p_parameter_from_flwr(t):
+        """Parameter : LCURLY FLWRexpr RCURLY"""
+        t[0] = t[2]
+
+    @staticmethod
+    def p_attribute_from_name(t):
+        """Attr : NAME"""
+        t[0] = symbols.Attribute(t[1])
+
+    @staticmethod
+    def p_attribute_from_call(t):
+        """Attr : NAME Call"""
+        t[0] = symbols.Attribute(t[1], t[2])
+
+    @staticmethod
+    def p_multiple_calls(t):
+        """Call : Call Call_"""
+        t[0] = t[1] + [t[2]]
+
+    @staticmethod
+    def p_call(t):
+        """Call : Call_"""
+        t[0] = [t[1]]
+
+    @staticmethod
+    def p_call_from_function_call(t):
+        """Call_ : Fcall"""
+        t[0] = t[1]
+
+    @staticmethod
+    def p_call_from_index_call(t):
+        """Call_ : Dcall"""
+        t[0] = t[1]
+
+    @staticmethod
+    def p_empty_function_call(t):
+        """Fcall : LPAREN RPAREN"""
+        t[0] = symbols.Call([])
+
+    @staticmethod
+    def p_function_call(t):
+        """Fcall : LPAREN ParameterList RPAREN"""
+        t[0] = symbols.Call(t[2])
+
+    @staticmethod
+    def p_index_call(t):
+        """Dcall : LSQUARE ArithExpr RSQUARE"""
+        t[0] = symbols.Call([t[2]], lookup=True)
+
+    @staticmethod
+    def p_quantified_expr_from_set(t):
+        """QuantifiedExpr : Quantifier NAME IN LANGLE Set RANGLE SATISFIES LPAREN Where RPAREN"""
+        t[0] = symbols.quantified_value(t[1], t[2], t[5], t[9])
+
+    @staticmethod
+    def p_quantifier_expr_from_flwr(t):
+        """QuantifiedExpr : Quantifier NAME IN LCURLY FLWRexpr RCURLY SATISFIES LPAREN Where RPAREN"""
+        t[0] = symbols.quantified_value(t[1], t[2], t[5], t[9])
+
+    @staticmethod
+    def p_quantified_from_every(t):
+        """Quantifier : EVERY"""
+        t[0] = t[1]
+
+    @staticmethod
+    def p_quantified_from_some(t):
+        """Quantifier : SOME"""
+        t[0] = t[1]
+
+    @staticmethod
+    def p_set_expr_from_in(t):
+        """SetExpr : ArithExpr IN AttributeValue"""
+        t[0] = symbols.set_expression_value(t[1], symbols.set_expression_operation1('in'), symbols.attribute_value(t[3]))
+
+    @staticmethod
+    def p_set_expr_from_not_in(t):
+        """SetExpr : ArithExpr NOT IN AttributeValue"""
+        t[0] = symbols.set_expression_value(t[1], symbols.set_expression_operation1('not in'), symbols.attribute_value(t[4]))
+
+    @staticmethod
+    def p_set_expr_from_in_set_definition(t):
+        """SetExpr : ArithExpr IN LANGLE Set RANGLE"""
+        t[0] = symbols.set_expression_value(t[1], symbols.set_expression_operation1('in'), t[4])
+
+    @staticmethod
+    def p_set_expr_from_not_in_set_definition(t):
+        """SetExpr : ArithExpr NOT IN LANGLE Set RANGLE"""
+        t[0] = symbols.set_expression_value(t[1], symbols.set_expression_operation1('not in'), t[5])
+
+    @staticmethod
+    def p_set_expr_from_subset_of_set_definition(t):
+        """SetExpr : LANGLE Set RANGLE SUBSET LANGLE Set RANGLE"""
+        t[0] = symbols.set_expression_value(t[2], symbols.set_expression_operation2('subset'), t[6])
+
+    @staticmethod
+    def p_set_expr_from_superset_of_set_definition(t):
+        """SetExpr : LANGLE Set RANGLE SUPERSET LANGLE Set RANGLE"""
+        t[0] = symbols.set_expression_value(t[2], symbols.set_expression_operation2('superset'), t[6])
+
+    @staticmethod
+    def p_set_expr_from_proper_subset_of_set_definition(t):
+        """SetExpr : LANGLE Set RANGLE PROPER SUBSET LANGLE Set RANGLE"""
+        t[0] = symbols.set_expression_value(t[2], symbols.set_expression_operation2('proper subset'), t[7])
+
+    @staticmethod
+    def p_set_expr_from_proper_superset_of_set_definition(t):
+        """SetExpr : LANGLE Set RANGLE PROPER SUPERSET LANGLE Set RANGLE"""
+        t[0] = symbols.set_expression_value(t[2], symbols.set_expression_operation2('proper superset'), t[7])
+
+    @staticmethod
+    def p_set_expr_from_is(t):
+        """SetExpr : LANGLE Set RANGLE IS LANGLE Set RANGLE"""
+        t[0] = symbols.set_expression_value(t[2], symbols.set_expression_operation2('is'), t[6])
+
+    @staticmethod
+    def p_set_expr_from_is_not(t):
+        """SetExpr : LANGLE Set RANGLE IS NOT LANGLE Set RANGLE"""
+        t[0] = symbols.set_expression_value(t[2], symbols.set_expression_operation2('is not'), t[7])
+
+    @staticmethod
+    def p_set_expr_from_in_value_list(t):
+        """SetExpr : ArithExpr IN LSQUARE ValueList RSQUARE"""
+        t[0] = symbols.set_expression_value(t[1], symbols.set_expression_operation1('in'), symbols.list_value(t[4]))
+
+    @staticmethod
+    def p_set_expr_from_not_in_value_list(t):
+        """SetExpr : ArithExpr NOT IN LSQUARE ValueList RSQUARE"""
+        t[0] = symbols.set_expression_value(t[1], symbols.set_expression_operation1('not in'), symbols.list_value(t[5]))
+
+    @staticmethod
+    def p_error(t):
+        if t is None:
+            raise SyntaxError("Unexpected end of input")
+        else:
+            raise SyntaxError("Syntax error at '%s', %s.%s" % (t,
+                                                               t.lineno,
+                                                               t.lexpos))

src/sheerkaql/symbols.py

@@ -0,0 +1,670 @@
+"""
+PyQuery - The Python Object Query System
+Author: Tim Henderson
+Contact: tim.tadh@hackthology.com
+Copyright (c) 2010 All Rights Reserved.
+Licensed under a BSD style license see the LICENSE file.
+
+File: symbols.py
+Purpose: Objects and functions representing components of a query.
+"""
+from __future__ import absolute_import
+from __future__ import division
+
+from builtins import object
+from builtins import range
+from builtins import str
+from builtins import zip
+from collections import deque
+from itertools import product
+
+from core.global_symbols import NotInit
+from core.utils import sheerka_getattr, sheerka_hasattr
+
+try:
+    from .OrderedSet import OrderedSet
+except SystemError:
+    from OrderedSet import OrderedSet
+
+
+class Attribute(object):
+    """
+    Represents an attribute. An attribute consists of a name and a
+    "callchain." The callchain represent one or more function or index lookups
+    being performed on the attribute.
+
+    eg.
+        x(1,2,3)[1]()
+
+    translates to:
+        self.name = 'x'
+        self.callchain = [Call([1,2,3]), Call([1], True), Call([])]
+    """
+
+    def __init__(self, name, callchain=None):
+        self.name = name
+        self.callchain = callchain
+
+    def __repr__(self):
+        return str(self)
+
+    def __str__(self):
+        return str(self.name) + "[" + str(self.callchain) + "]"
+
+
+class Call(object):
+    """
+    Represent a 'Call' (for context see the documentation on the Attribute
+    class). A call consists of the parameters passed into the call (themselves
+    functions which takes the namespace (objs)) and whether not this is an item
+    lookup rather than a function call.
+    """
+
+    def __init__(self, params, lookup=False):
+        self.params = params
+        self.lookup = lookup
+
+    def __repr__(self):
+        return str(self)
+
+    def __str__(self):
+        if self.lookup:
+            return "__getitem__" + str(tuple(self.params))
+        return "__call__" + str(tuple(self.params))
+
+
+class KeyValuePair(object):
+    """
+    Represents a key,value pair for use while iterating over a dictionary.
+    """
+
+    def __init__(self, key, value):
+        self.key = key
+        self.value = value
+
+    def __repr__(self):
+        return str(self)
+
+    def __str__(self):
+        return "<key:%s, value:%s>" % (self.key, self.value)
+
+
+def attribute_value(attribute_list, scalar=False, context='locals'):
+    """
+    Transforms a AttributeValue into its actual value.
+
+    eg.
+        x.y.z().q[1].r
+        attribute_list = [Attribute('x'), Attribute('y'),
+            Attribute('z',[Call([])]),  Attribute('q', [Call([1], True)]),
+            Attribute('r')]
+
+    translates into the attribute lookups, function calls, and __getitem__ calls
+    necessary to produce a value.
+
+    if scalar == True:
+        it simply returns the value stored in attribute_list
+
+    context is no longer used and should be removed.
+    """
+
+    def expand(namespace, attr, x):
+        """
+        Expands the the value of one attribute by looking the name up in the
+        namespace dict and then performing and function calls and dictionary lookups
+        specified in the callchain.
+        """
+        if attr.callchain:
+            for call in attr.callchain:
+                p = list()
+                for param in call.params:
+                    if isinstance(param, type(value)) and value.__code__ == param.__code__ or \
+                        isinstance(param, type(value)) and hasattr(param, '__objquery__'):
+                        p.append(param(namespace))
+                    else:
+                        p.append(param)
+                if call.lookup:
+                    x = x.__getitem__(p[0])
+                else:
+                    x = x.__call__(*p)
+        return x
+
+    def value(namespace):
+        """
+        The computation function returned the user. Computes the actual value
+        of the the attribute expression when @namespace is passed in.
+        """
+        if scalar:
+            return attribute_list
+
+        attr0 = attribute_list[0]
+        obj = expand(namespace, attr0, namespace[attr0.name])
+        for attr in attribute_list[1:]:
+            if sheerka_hasattr(obj, attr.name):
+                obj = expand(namespace, attr, sheerka_getattr(obj, attr.name))
+            else:
+                raise AttributeError(f"'{type(obj).__name__}' object has no attribute '{attr.name}'")
+        return obj
+
+    return value
+
+
+def operator(op):
+    """
+    Returns a function which performs comparison operations
+    """
+    if op == '==':
+        return lambda x, y: x == y
+    if op == '!=':
+        return lambda x, y: x != y
+    if op == '<=':
+        return lambda x, y: x <= y
+    if op == '>=':
+        return lambda x, y: x >= y
+    if op == '<':
+        return lambda x, y: x < y
+    if op == '>':
+        return lambda x, y: x > y
+    raise Exception("operator %s not found" % op)
+
+
+def arith_operator(op):
+    """
+    Returns a function which performs arithmetic operations
+    """
+    if op == '+':
+        return lambda x, y: x + y
+    if op == '-':
+        return lambda x, y: x - y
+    if op == '*':
+        return lambda x, y: x * y
+    if op == '/':
+        return lambda x, y: x / y
+    if op == '%':
+        return lambda x, y: x % y
+    raise Exception("operator %s not found" % op)
+
+
+def set_operator(op):
+    """
+    Returns a function which performs set operations
+    """
+    if op == '|':
+        return lambda x, y: OrderedSet(x) | OrderedSet(y)
+    if op == '&':
+        return lambda x, y: OrderedSet(x) & OrderedSet(y)
+    if op == '-':
+        return lambda x, y: OrderedSet(x) - OrderedSet(y)
+    raise Exception("operator %s not found" % op)
+
+
+def set_expression_operation1(op):
+    """
+    Returns a function which performs scalar in set operations
+    """
+    if op == 'in':
+        return lambda x, y: x in y
+    if op == 'not in':
+        return lambda x, y: x not in y
+    raise Exception("operator %s not found" % op)
+
+
+def set_expression_operation2(op):
+    """
+    Returns a function which performs set to set comparison operations
+    """
+    if op == 'is':
+        return lambda x, y: x == y
+    if op == 'is not':
+        return lambda x, y: x != y
+    if op == 'subset':
+        return lambda x, y: x <= y
+    if op == 'superset':
+        return lambda x, y: x >= y
+    if op == 'proper subset':
+        return lambda x, y: x < y
+    if op == 'proper superset':
+        return lambda x, y: x > y
+    raise Exception("operator %s not found" % op)
+
+
+def bool_operator(op):
+    """
+    Returns a function which performs basic boolean (and, or) operations
+    """
+    if op == 'and':
+        return lambda x, y, namespace: x(namespace) and y(namespace)
+    if op == 'or':
+        return lambda x, y, namespace: x(namespace) or y(namespace)
+    raise Exception("operator %s not found" % op)
+
+
+def unary_operator(op):
+    """
+    Returns a function which performs unary (not) operation
+    """
+    if op == 'not':
+        return lambda x: not x
+    raise Exception("operator %s not found" % op)
+
+
+def comparison_value(value1, op, value2):
+    """
+    Returns a function which will calculate a where expression for a basic
+    comparison operation.
+    """
+
+    def where(namespace):
+        return op(value1(namespace), value2(namespace))
+
+    object.__setattr__(where, '__objquery__', True)
+    return where
+
+
+def arith_value(value1, op, value2):
+    """
+    Returns a function which will calculate a where expression for a basic
+    arithmetic operation.
+    """
+
+    def computation(namespace):
+        return op(value1(namespace), value2(namespace))
+
+    object.__setattr__(computation, '__objquery__', True)
+    return computation
+
+
+def set_value(s1, op, s2):
+    """
+    Returns a Query function for the result of set operations (difference, union
+    etc..)
+    """
+
+    def query(namespace):
+        return op(s1(namespace), s2(namespace))
+
+    object.__setattr__(query, '__objquery__', True)
+    return query
+
+
+def set_expression_value(val, op, s):
+    """
+    Returns a where function which returns the result of a value in set
+    operation
+    """
+
+    def where(namespace):
+        return op(val(namespace), s(namespace))
+
+    object.__setattr__(where, '__objquery__', True)
+    return where
+
+
+def boolean_expression_value(value1, op, value2):
+    """
+    returns the function which computes the result of boolean (and or) operation
+    """
+
+    def where(namespace):
+        return op(value1, value2, namespace)
+
+    object.__setattr__(where, '__objquery__', True)
+    return where
+
+
+def unary_expression_value(op, val):
+    """
+    returns the function which computes the result of boolean not operation
+    """
+
+    def where(namespace):
+        return op(val(namespace))
+
+    object.__setattr__(where, '__objquery__', True)
+    return where
+
+
+def boolean_value(val):
+    """
+    returns the function which booleanizes the result of the Value function
+    """
+
+    def where(namespace):
+        return bool(val(namespace))
+
+    object.__setattr__(where, '__objquery__', True)
+    return where
+
+
+def where_value(val):
+    """
+    returns the results of a Value function.
+    """
+
+    def where(namespace):
+        return val(namespace)
+
+    object.__setattr__(where, '__objquery__', True)
+    return where
+
+
+def dict_value(pairs):
+    """
+    creates a dictionary from the passed pairs after evaluation.
+    """
+
+    def as_dict(namespace):
+        return dict((name(namespace), value(namespace)) for name, value in pairs)
+
+    object.__setattr__(as_dict, '__objquery__', True)
+    return as_dict
+
+
+def list_value(values):
+    """
+    creates a list from the pass objs after evaluation.
+    """
+
+    def as_list(objs):
+        return list(value(objs) for value in values)
+
+    object.__setattr__(as_list, '__objquery__', True)
+    return as_list
+
+
+def query_value(q):
+    """
+    Computes a path expression. The query (@q) is a list of attribute names and
+    associated where expressions. The function returned computes the result when
+    called.
+    :param q: query List[(attr_name, filter_condition)]
+    """
+    attrs = q
+
+    def query(namespace):
+        def select(namespace, attrs_path):
+            """
+            a generator which computes the actual results
+            :param namespace: dictionary of available objects
+            :param attrs_path: List[(attr_name, filter_condition)]
+            """
+
+            def add(_queue, _namespace, _index):
+                """
+                adds the object v to the queue
+                push the current namespace
+                """
+                index_to_use = _index + 1
+                try:
+                    object.__setattr__(_namespace, '_objquery__i', index_to_use)
+                except TypeError:
+                    setattr(_namespace, '_objquery__i', index_to_use)
+
+                _queue.appendleft(_namespace)
+
+            queue = deque()
+
+            # KSI 20210214: Why using type instead of the given dictionary ?
+            # -> to seamlessly use getattr() to retrieve the attribute
+            # -> to allow setattr(_namespace, '_objquery__i', index_to_use)
+            add(queue, type('base', (object,), namespace), -1)  # init with the namespace
+
+            while len(queue) > 0:
+                current_namespace = queue.pop()
+                index = object.__getattribute__(current_namespace, '_objquery__i')
+                attr_name, where = attrs_path[index]
+
+                # if sheerka_hasattr(current_namespace, attr_name):  # the current object has the attr
+                attr_value = sheerka_getattr(current_namespace, attr_name)
+                # it is iterable
+                if not isinstance(attr_value, str) and hasattr(attr_value, '__iter__'):
+                    # # try to use where clause as an indexer
+                    if where is not None:
+                        try:
+                            namespace_copy = dict(namespace)
+                            res = where(namespace_copy)
+                        except (NameError, KeyError, TypeError):
+                            res = NotInit
+
+                        if res is not NotInit and type(res) != bool:
+                            item_to_use = attr_value[res]
+
+                            # if this is the last attribute yield the obj
+                            if index + 1 == len(attrs_path):
+                                yield item_to_use
+                            else:
+                                add(queue, item_to_use, index)  # otherwise add to the queue
+                            continue
+
+                    for item in attr_value:
+                        # add each child into the processing queue
+                        if isinstance(attr_value, dict):
+                            item_to_use = KeyValuePair(item, attr_value[item])
+                        else:
+                            item_to_use = item
+
+                        # but only if its where condition is satisfied
+                        if where is not None:
+                            namespace_copy = dict(namespace)
+                            namespace_copy.update({'self': item_to_use})
+                            try:
+                                if not where(namespace_copy):
+                                    continue
+                            except (AttributeError, TypeError, KeyError):
+                                continue
+
+                        # if this is the last attribute yield the obj
+                        if index + 1 == len(attrs_path):
+                            yield item_to_use
+                        else:
+                            add(queue, item_to_use, index)  # otherwise add to the queue
+
+                else:  # it is not iterable
+                    if where is not None:
+                        namespace_copy = dict(namespace)
+                        namespace_copy.update({'self': attr_value})
+                        res = where(namespace_copy)
+                        if type(res) != bool:
+                            raise TypeError(f"'{type(attr_value).__name__}' object is not subscriptable")
+
+                        if not res:
+                            continue
+
+                    # if this is the last attribute yield the obj
+                    if index + 1 == len(attrs_path):
+                        yield attr_value
+                    else:
+                        add(queue, attr_value, index)  # otherwise add to the queue
+                # else try in the parent namespace
+
+        # return OrderedSet(select(namespace, attrs))
+        return list(select(namespace, attrs))
+
+    object.__setattr__(query, '__objquery__', True)
+    return query
+
+
+def quantified_value(mode, name, s, satisfies):
+    """
+    Processes the quantified expressions (some x in <> satisfie...) returns
+    the where function.
+    """
+
+    def where(namespace):
+        nobjs = s(namespace)  # runs the first part of the query (eg. the <path> expression)
+        if not nobjs:
+            return False  # if returns and empty set then return false
+        if mode == 'every':
+            r = True
+            for x in nobjs:
+                cobjs = dict(namespace)  # we have to copy the objects to not squash
+                # the upper namespace
+                cobjs.update({name: x})
+                if not satisfies(cobjs):
+                    r = False
+            return r
+        elif mode == 'some':
+            for x in nobjs:
+                cobjs = dict(namespace)
+                cobjs.update({name: x})
+                if satisfies(cobjs):
+                    return True
+            return False
+        raise Exception("mode '%s' is not 'every' or 'some'" % mode)
+
+    return where
+
+
+def flwr_sequence(return_expr,
+                  for_expr=None,  # list of (name, function_that_returns_a_collection)
+                  let_expr=None,
+                  where_expr=None,
+                  order_expr=None,
+                  flatten=False,
+                  collecting=False):
+    """
+    Returns the function to calculate the results of a flwr expression
+    """
+    # print order_expr
+    if flatten:
+        assert len(return_expr) == 1 and not isinstance(return_expr[0], tuple)
+        assert not collecting
+
+    # if collecting:
+    # target = return_expr['as']
+    # reduce_function = return_expr['with']
+    # return_expr = return_expr['value']
+    def sequence(namespace):
+        def _flatten_func(items):
+            if not isinstance(items, (tuple, list, set)):
+                yield items
+            else:
+                for item in items:
+                    if isinstance(item, (tuple, list, set)):
+                        for j in _flatten_func(item):
+                            yield j
+                    else:
+                        yield item
+
+        def _build_yield(_namespace):
+
+            def _build_return(obj):
+                try:
+                    if len(obj) == 1 and not isinstance(obj[0], tuple):
+                        return obj[0](_namespace)
+                    elif isinstance(obj[0], tuple):  # it has named return values
+                        return dict((name, f(_namespace)) for name, f in obj)
+                    else:  # multiple positional return values
+                        return tuple(f(_namespace) for f in obj)
+                except Exception as ex:
+                    return ex
+
+            if not collecting:
+                return _build_return(return_expr)
+
+            # return a list of collecting information
+            return [
+                {
+                    'value': _build_return(collector_def['value']),
+                    'as': collector_def['as'](_namespace),
+                    'with': collector_def['with'](_namespace)
+                } for collector_def in return_expr
+            ]
+
+        def compute_sequence(_namespace):
+            # Tim Henderson 
+            # take the cartesian product of the for expression
+            # note you cannot do this:
+            #   for x in <path>, y in <x>
+            #   :sadface: some day I will fix this.
+            #   however I will only do that when I implement and optimizer
+            #   for PyQuery otherwise it just isn't worth it.
+            if for_expr is not None:
+                obs = ([(name, obj) for obj in get_collection(_namespace)]
+                       for name, get_collection in for_expr)
+            else:
+                # Tim Henderson
+                # The goal is to get the for loop to run once. this syntax does
+                # it. We may not have a for_expr but we want everything else
+                # to execute normally.
+                obs = [[None]]
+
+            for items in product(*obs):
+                namespace_copy = dict(_namespace)
+                if for_expr is not None:
+                    for name, item in items:
+                        namespace_copy.update({name: item})
+
+                if let_expr:
+                    for name, let in let_expr:
+                        namespace_copy.update({name: let(namespace_copy)})  # calculate the let expr
+
+                if where_expr and not where_expr(namespace_copy):
+                    continue  # skip if the where fails
+
+                if not flatten:
+                    yield _build_yield(namespace_copy)  # single unamed return
+                else:
+                    for item in _flatten_func(return_expr[0](namespace_copy)):
+                        yield item
+
+        if collecting:
+            rets = tuple(dict() for _ in range(len(return_expr)))
+            for collectors in compute_sequence(namespace):
+                for i, collector in enumerate(collectors):
+                    _as = collector['as']
+                    _rf = collector['with']
+                    _value = collector['value']
+                    rets[i][_as] = _rf(rets[i].get(_as, None), _value)
+
+            return rets[0] if len(rets) == 1 else rets
+
+        else:
+            r = list(compute_sequence(namespace))
+            if not r:
+                return tuple(r)
+            elif order_expr:
+                attr, direction = order_expr
+                if isinstance(attr, str):
+                    if not isinstance(return_expr[0], tuple):
+                        raise SyntaxError("Using a name in the order by clause when not using named return values.")
+                else:
+                    if isinstance(return_expr[0], tuple):
+                        raise SyntaxError(
+                            "Using a number in the order by clause when not using positional return values.")
+                if len(return_expr) == 1 and not isinstance(return_expr[0], tuple):
+                    keyfunc = lambda x: x
+                else:
+                    keyfunc = lambda x: x[attr]
+
+                reverse_order = direction == 'DESC'
+                r = sorted(r, key=keyfunc, reverse=reverse_order)
+
+            return tuple(r)
+
+    object.__setattr__(sequence, '__objquery__', True)
+    return sequence
+
+
+def function_definition(params, query):
+    def flwr_function(namespace):
+        def function(*args):
+            if len(args) != len(params):
+                raise RuntimeError("Got wrong number of params expected %d got %d" % (len(params), len(args)))
+            namespace_copy = dict(namespace)
+            namespace_copy.update(list(zip(params, args)))
+            return query(namespace_copy)
+
+        return function
+
+    return flwr_function
+
+
+def if_expression(condition, then, otherwise):
+    def if_expr(namespace):
+        if condition(namespace):
+            return then(namespace)
+        else:
+            return otherwise(namespace)
+
+    return if_expr

src/sheerkarete/network.py

@@ -9,8 +9,8 @@ from core.concept import Concept
 from core.global_symbols import NotInit
 from core.rule import Rule, ACTION_TYPE_PRINT
 from core.utils import as_bag
-from evaluators.PythonEvaluator import Expando
 from sheerkapickle.utils import is_primitive
+from sheerkapython.python_wrapper import Expando
 from sheerkarete.alpha import AlphaMemory
 from sheerkarete.beta import ReteNode, BetaMemory
 from sheerkarete.bind_node import BindNode
@@ -439,14 +439,13 @@ class ReteNetwork:
                 self.add_wme(WME(fact_id, FACT_NAME, name))
             elif attribute == FACT_IS_CONCEPT:
                 self.add_wme(WME(fact_id, FACT_IS_CONCEPT, isinstance(obj, Concept)))
-            elif attribute == FACT_SELF:
-                if isinstance(obj, FactObj):
-                    self.add_wme(WME(fact_id, FACT_SELF, obj.value))
-                else:
-                    self.add_wme(WME(fact_id, FACT_SELF, obj))
             else:
                 try:
-                    value = getattr(obj, attribute)
+                    if attribute == FACT_SELF:
+                        value = obj.value if isinstance(obj, FactObj) else obj
+                    else:
+                        value = getattr(obj, attribute)
+
                     if (isinstance(value, Concept) and value.key == BuiltinConcepts.SHEERKA or
                         isinstance(value, Expando) and value.get_name() == "sheerka"):
                         value = FACT_SHEERKA

tests/core/test_ExecutionContext.py

@@ -12,9 +12,12 @@ class TestExecutionContext(TestUsingMemoryBasedSheerka):
     def test_id_is_incremented_by_event_digest(self):
         sheerka = self.get_sheerka()
 
-        a = ExecutionContext("foo", Event("event_1"), sheerka, BuiltinConcepts.NOP, None)
-        b = ExecutionContext("foo", Event("event_1"), sheerka, BuiltinConcepts.NOP, None)
-        c = ExecutionContext("foo", Event("event_2"), sheerka, BuiltinConcepts.NOP, None)
+        event1 = Event("event_1")
+        event2 = Event("event_2")
+
+        a = ExecutionContext("foo", event1, sheerka, BuiltinConcepts.NOP, None)
+        b = ExecutionContext("foo", event1, sheerka, BuiltinConcepts.NOP, None)
+        c = ExecutionContext("foo", event2, sheerka, BuiltinConcepts.NOP, None)
         d = b.push(BuiltinConcepts.NOP, None)
         e = c.push(BuiltinConcepts.NOP, None)
 
@@ -177,6 +180,47 @@ class TestExecutionContext(TestUsingMemoryBasedSheerka):
         assert sub2.has_parent(root.id)
         assert not sub1.has_parent(sub2.id)
         assert not sub2.has_parent(sub1.id)
+
+    def test_i_can_reset_global_hints(self):
+        sheerka = self.get_sheerka()
+
+        context = ExecutionContext("foo", Event("event_1"), sheerka, BuiltinConcepts.NOP, None)
+        context.add_to_global_hints(BuiltinConcepts.TESTING)
+        context.add_to_global_hints(BuiltinConcepts.DEBUG)
+
+        sub_context1 = context.push(BuiltinConcepts.NOP, None)
+        assert BuiltinConcepts.TESTING in sub_context1.global_hints
+        assert BuiltinConcepts.DEBUG in sub_context1.global_hints
+
+        sub_context2 = context.push(BuiltinConcepts.NOP, None, reset_hints={BuiltinConcepts.TESTING})
+        assert BuiltinConcepts.TESTING not in sub_context2.global_hints
+        assert BuiltinConcepts.DEBUG in sub_context2.global_hints
+
+        sub_context3 = context.push(BuiltinConcepts.NOP,
+                                    None,
+                                    reset_hints={BuiltinConcepts.DEBUG, BuiltinConcepts.TESTING})
+        assert sub_context3.global_hints == set()
+
+    def test_i_can_reset_protected_hints(self):
+        sheerka = self.get_sheerka()
+
+        context = ExecutionContext("foo", Event("event_1"), sheerka, BuiltinConcepts.NOP, None)
+        context.add_to_protected_hints(BuiltinConcepts.TESTING)
+        context.add_to_protected_hints(BuiltinConcepts.DEBUG)
+
+        sub_context1 = context.push(BuiltinConcepts.NOP, None)
+        assert BuiltinConcepts.TESTING in sub_context1.protected_hints
+        assert BuiltinConcepts.DEBUG in sub_context1.protected_hints
+
+        sub_context2 = context.push(BuiltinConcepts.NOP, None, reset_hints={BuiltinConcepts.TESTING})
+        assert BuiltinConcepts.TESTING not in sub_context2.protected_hints
+        assert BuiltinConcepts.DEBUG in sub_context2.protected_hints
+
+        sub_context3 = context.push(BuiltinConcepts.NOP,
+                                    None,
+                                    reset_hints={BuiltinConcepts.DEBUG, BuiltinConcepts.TESTING})
+        assert sub_context3.protected_hints == set()
+
     # def test_variables_are_passed_to_children_but_not_to_parents(self):
     #     sheerka = self.get_sheerka()
     #

tests/core/test_SheerkaEvaluateConcept.py

@@ -2,7 +2,7 @@ import pytest
 
 from core.builtin_concepts import BuiltinConcepts, ReturnValueConcept, ParserResultConcept
 from core.concept import Concept, DoNotResolve, ConceptParts, InfiniteRecursionResolved, \
-    concept_part_value, DEFINITION_TYPE_DEF
+    DEFINITION_TYPE_DEF
 from core.global_symbols import NotInit, NotFound
 from core.sheerka.services.SheerkaEvaluateConcept import SheerkaEvaluateConcept
 from core.sheerka.services.SheerkaMemory import SheerkaMemory
@@ -420,7 +420,7 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
         concept = Concept("foo", body="a")
         evaluated = sheerka.evaluate_concept(context, concept)
         assert evaluated.key == concept.key
-        compare_with_test_object(evaluated.body, CB("a", "concept_a")) # this test was already done
+        compare_with_test_object(evaluated.body, CB("a", "concept_a"))  # this test was already done
 
         # so check this one.
         concept = Concept("foo", body="a").def_var("a", "'property_a'")
@@ -784,27 +784,6 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
         assert evaluated.key == command.key
         assert sheerka.get_from_memory(context, "a").obj == 10
 
-    @pytest.mark.parametrize("metadata", [
-        ConceptParts.WHERE,
-        ConceptParts.PRE,
-        ConceptParts.POST,
-        ConceptParts.RET
-    ])
-    def test_i_cannot_evaluate_python_statement_in_where_pre_post_ret(self, metadata, capsys):
-        sheerka, context, foo = self.init_concepts("foo")
-        setattr(foo.get_metadata(), concept_part_value(metadata), "a=10; print('10')")
-        foo.get_metadata().need_validation = True
-
-        evaluated = sheerka.evaluate_concept(context, foo, eval_body=True)
-        captured = capsys.readouterr()
-
-        assert sheerka.isinstance(evaluated, BuiltinConcepts.CONCEPT_EVAL_ERROR)
-        error = evaluated.body
-        assert sheerka.isinstance(error, BuiltinConcepts.PYTHON_SECURITY_ERROR)
-        assert error.prop == metadata
-        assert error.body == "a=10; print('10')"
-        assert captured.out == ""
-
     def test_python_builtin_function_are_forbidden_in_where_pre_post_ret(self, capsys):
         # I do the test only for PRE, as it will be the same for the other ConceptPart
         sheerka, context, foo, bar = self.init_concepts(

tests/core/test_SheerkaEvaluateRules.py

@@ -9,8 +9,9 @@ from core.sheerka.Sheerka import RECOGNIZED_BY_ID, RECOGNIZED_BY_NAME
 from core.sheerka.services.SheerkaEvaluateRules import SheerkaEvaluateRules, LOW_PRIORITY_RULES, DISABLED_RULES
 from core.sheerka.services.SheerkaExecute import ParserInput
 from core.sheerka.services.SheerkaRuleManager import SheerkaRuleManager, CompiledCondition
-from evaluators.PythonEvaluator import PythonEvaluator, Expando
+from evaluators.PythonEvaluator import PythonEvaluator
 from parsers.PythonParser import PythonParser
+from sheerkapython.python_wrapper import Expando
 from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
 
 
@@ -84,7 +85,7 @@ class TestSheerkaEvaluateRules(TestUsingMemoryBasedSheerka):
         # create fake compiled predicates
         parser = PythonParser()
         my_rule.compiled_conditions = [
-            CompiledCondition(PythonEvaluator.NAME, parser.parse(context, ParserInput(exp)), set(), set(), None)
+            CompiledCondition(PythonEvaluator.NAME, parser.parse(context, ParserInput(exp)), set(), set(), None, set())
             for exp in predicates]
         my_rule.metadata.is_compiled = True
         my_rule.metadata.is_enabled = True
@@ -114,9 +115,9 @@ class TestSheerkaEvaluateRules(TestUsingMemoryBasedSheerka):
         assert res == {True: [r1, r3], False: [r2]}
 
     @pytest.mark.parametrize("predicate", [
-        "greetings",
-        "c:|1001:",
-        "hello 'kodjo'"
+        "recognize(__ret.body, greetings)",
+        "recognize(__ret.body, c:|1001:)",
+        "recognize(__ret.body, hello 'kodjo')"
     ])
     def test_i_can_evaluate_rules_when_concepts_are_not_questions(self, predicate):
         """
@@ -146,7 +147,7 @@ class TestSheerkaEvaluateRules(TestUsingMemoryBasedSheerka):
             Concept("greetings", definition="hello a", definition_type=DEFINITION_TYPE_DEF).def_var("a"),
             Concept("there"),
             create_new=True).with_format_rules(
-            Rule(predicate="hello there", action="")).unpack()
+            Rule(predicate="recognize(__ret.body, hello there)", action="")).unpack()
         service = sheerka.services[SheerkaEvaluateRules.NAME]
 
         there_instance = sheerka.new_from_template(there, there.key)
@@ -169,7 +170,7 @@ class TestSheerkaEvaluateRules(TestUsingMemoryBasedSheerka):
             Concept("greetings", definition="hello a", definition_type=DEFINITION_TYPE_DEF).def_var("a"),
             Concept("my friend"),
             create_new=True).with_format_rules(
-            Rule(predicate="hello my friend", action="")).unpack()
+            Rule(predicate="recognize(__ret.body, hello my friend)", action="")).unpack()
         service = sheerka.services[SheerkaEvaluateRules.NAME]
 
         my_friend_instance = sheerka.new_from_template(my_friend, my_friend.key)
@@ -183,7 +184,7 @@ class TestSheerkaEvaluateRules(TestUsingMemoryBasedSheerka):
         sheerka, context, greetings, rule = self.init_test().with_concepts(
             Concept("greetings", definition="hello a", definition_type=DEFINITION_TYPE_DEF).def_var("a"),
             create_new=True).with_format_rules(
-            Rule(predicate="hello sheerka", action="")).unpack()
+            Rule(predicate="recognize(__ret.body, hello sheerka)", action="")).unpack()
         service = sheerka.services[SheerkaEvaluateRules.NAME]
 
         ret = sheerka.ret("evaluator", True, sheerka.new(greetings, a=Expando("sheerka", {})))
@@ -196,7 +197,7 @@ class TestSheerkaEvaluateRules(TestUsingMemoryBasedSheerka):
             Concept("greetings", definition="hello a", definition_type=DEFINITION_TYPE_DEF).def_var("a"),
             Concept("greetings", definition="hi a", definition_type=DEFINITION_TYPE_DEF).def_var("a"),
             create_new=True).with_format_rules(
-            Rule(predicate="greetings", action="")).unpack()
+            Rule(predicate="recognize(__ret.body, greetings)", action="")).unpack()
 
         service = sheerka.services[SheerkaEvaluateRules.NAME]
         ret1 = sheerka.ret("evaluator", True, sheerka.new(g1, a="kodjo"))
@@ -210,7 +211,7 @@ class TestSheerkaEvaluateRules(TestUsingMemoryBasedSheerka):
     def test_i_can_evaluate_concept_rule_with_the_same_name_when_the_second_concept_is_declared_after(self):
         sheerka, context, g1, rule, g2 = self.init_test().with_concepts(
             Concept("greetings", definition="hello a", definition_type=DEFINITION_TYPE_DEF).def_var("a"),
-            create_new=True).with_format_rules(Rule(predicate="greetings", action="")).with_concepts(
+            create_new=True).with_format_rules(Rule(predicate="recognize(__ret.body, greetings)", action="")).with_concepts(
             Concept("greetings", definition="hi a", definition_type=DEFINITION_TYPE_DEF).def_var("a")).unpack()
 
         service = sheerka.services[SheerkaEvaluateRules.NAME]

tests/core/test_SheerkaMemory.py

@@ -87,7 +87,9 @@ class TestSheerkaMemory(TestUsingMemoryBasedSheerka):
         foo = Concept("foo")
         sheerka.add_to_memory(context, "a", foo)
 
-        assert sheerka.om.copy(SheerkaMemory.OBJECTS_ENTRY) == {"a": MemoryObject(context.event.get_digest(), foo)}
+        assert sheerka.om.copy(SheerkaMemory.OBJECTS_ENTRY) == {"a": MemoryObject(context.event.get_digest(),
+                                                                                  context.event.date.timestamp(),
+                                                                                  foo)}
         assert id(sheerka.get_from_memory(context, "a").obj) == id(foo)
 
     def test_i_can_use_memory_with_a_string(self):
@@ -106,6 +108,44 @@ class TestSheerkaMemory(TestUsingMemoryBasedSheerka):
 
         assert sheerka.memory(context, Concept("foo")) == foo
 
+    def test_i_can_use_memory_with_a_query(self):
+        sheerka, context, foo, bar = self.init_concepts("foo", "bar")
+
+        sheerka.add_to_memory(context, "x", foo)
+        sheerka.add_to_memory(context, "y", bar)
+
+        assert sheerka.memory(context, "self.name == 'foo'") == foo
+
+    def test_i_retrieve_the_last_entry_when_requesting_memory_with_a_query(self):
+        sheerka, context, foo, bar, foo2 = self.init_concepts("foo", "bar", Concept("foo", body="2"))
+
+        sheerka.add_to_memory(context, "x", foo)
+        sheerka.add_to_memory(context, "y", bar)
+
+        context2 = self.get_context(sheerka)  # timestamp is newer
+        sheerka.add_to_memory(context2, "z", foo2)
+
+        assert sheerka.memory(context, "self.name == 'foo'") == foo2
+
+    def test_i_can_look_in_the_previous_objects_when_using_query_to_request_the_memory(self):
+        sheerka, context, foo, bar, baz = self.init_concepts("foo", "bar", "baz")
+
+        sheerka.add_to_memory(context, "x", foo)
+        sheerka.add_to_memory(context, "y", bar)
+        sheerka.add_to_memory(context, "z", baz)
+
+        # another layer
+        sheerka.add_to_memory(context, "x", bar)
+        sheerka.add_to_memory(context, "y", baz)
+
+        # another layer
+        sheerka.add_to_memory(context, "x", baz)
+
+        # so under x there is [foo] -> [bar] -> [baz]
+        # so under y there is [bar] -> [baz]
+        # so under z there is [baz]
+        assert sheerka.memory(context, "self.name == 'foo'") == foo
+
     def test_concept_not_found_is_return_when_not_found(self):
         sheerka, context = self.init_test().unpack()
 

tests/core/test_SheerkaQueryManager.py

@@ -0,0 +1,205 @@
+from dataclasses import dataclass
+
+import pytest
+
+from core.builtin_concepts_ids import BuiltinConcepts
+from core.concept import Concept
+from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
+
+
+@dataclass
+class A:
+    prop1: object
+    prop2: object
+
+    def __eq__(self, other):
+        if not isinstance(other, A):
+            return False
+
+        return self.prop1 == other.prop1 and self.prop2 == other.prop2
+
+    def __hash__(self):
+        hash_res = []
+        for p in [self.prop1, self.prop2]:
+            hash_res.append(0 if isinstance(p, (list, dict, set)) else p)
+
+        return hash(tuple(hash_res))
+
+
+@dataclass
+class B(A):
+
+    def as_bag(self):
+        return {
+            "fake_prop1": self.prop1,
+            "fake_prop2": self.prop2
+        }
+
+    def __hash__(self):
+        return hash((self.prop1, self.prop2))
+
+
+class TestSheerkaQueryManager(TestUsingMemoryBasedSheerka):
+    def test_i_can_filter_objects_using_kwargs(self):
+        sheerka, context = self.init_test().unpack()
+        lst = [A("a11", 10), A("a21", 3.14), A({1, "v"}, 0xab), A([0, 1], {"key": "value"})]
+
+        assert sheerka.filter_objects(context, lst, prop1="a21") == [lst[1]]
+        assert sheerka.filter_objects(context, lst, prop2=10) == [lst[0]]
+        assert sheerka.filter_objects(context, lst, prop2=3.14) == [lst[1]]
+        assert sheerka.filter_objects(context, lst, prop2=0xab) == [lst[2]]
+        assert sheerka.filter_objects(context, lst, prop1=[0, 1]) == [lst[3]]
+        assert sheerka.filter_objects(context, lst, prop2={"key": "value"}) == [lst[3]]
+        # assert sheerka.filter_objects(context, lst, prop1={1, "v"}) == [lst[2]] set are not supported
+
+    def test_i_can_filter_by_object_type(self):
+        sheerka, context = self.init_test().unpack()
+        lst = [A("a11", "a12"), Concept("foo", body="a").auto_init(), Concept("foo", body="b").auto_init()]
+
+        assert sheerka.filter_objects(context, lst, __type="foo") == [lst[1], lst[2]]
+
+    def test_i_can_filter_on_atomic_def(self):
+        sheerka, context, isa, plus, isa2 = self.init_concepts(
+            Concept('x is a y').def_var("x").def_var("y"),
+            Concept('a plus b').def_var("a").def_var("b"),
+            Concept('u is a v').def_var("u").def_var("v"),
+        )
+
+        lst = [isa, plus, isa2]
+        assert sheerka.filter_objects(context, lst, atomic_def="is a") == [lst[0], lst[2]]
+
+    def test_i_can_filter_on_as_bag_property(self):
+        sheerka, context = self.init_test().unpack()
+        lst = [B("a11", "a12"), B("a21", "a22"), B("a31", "a32")]
+
+        assert sheerka.filter_objects(context, lst, fake_prop1="a21") == [lst[1]]
+
+    def test_i_can_filter_container(self):
+        sheerka, context = self.init_test().unpack()
+        lst = [A("a11", 10), A("a21", 3.14), A({1, "v"}, 0xab), A([0, 1], {"key": "value"})]
+        container = sheerka.new(BuiltinConcepts.EXPLANATION, body=lst, digest="xxx", command="text")
+
+        res = sheerka.filter_objects(context, container, prop1="a21")
+        assert sheerka.isinstance(res, BuiltinConcepts.EXPLANATION)
+        assert res.digest == "xxx"
+        assert res.command == "text"
+        assert res.body == [lst[1]]
+
+    def test_i_can_filter_when_property_does_not_exist(self):
+        sheerka, context = self.init_test().unpack()
+        lst = [A("a11", "a12"), B("a21", "a22"), B("a31", "a32")]
+
+        assert sheerka.filter_objects(context, lst, prop1="a11") == [lst[0]]
+
+    def test_i_can_filter_objets_using_predicate(self):
+        sheerka, context = self.init_test().unpack()
+        lst = [A("a11", 10),
+               A("a21", 3.14),
+               A({1, "v"}, 0xab),
+               A([0, 1], {"key": "value"}),
+               Concept("foo", body="a").auto_init(),
+               B("a21", "a22"),
+               Concept("foo", body="b").auto_init(),
+               B("a31", "a32")]
+
+        assert sheerka.filter_objects(context, lst, "self.prop1 == 'a21'") == [lst[1]]
+        assert sheerka.filter_objects(context, lst, "self.prop2 >= 1") == [lst[0], lst[1], lst[2]]
+        assert sheerka.filter_objects(context, lst, "get_type(self) == 'foo' ") == [lst[4], lst[6]]
+        assert sheerka.filter_objects(context, lst, "self.fake_prop1 == 'a21' ") == [lst[5]]
+        assert sheerka.filter_objects(context, lst, "hasattr(self, 'fake_prop1')") == [lst[5], lst[7]]
+
+    def test_i_can_filter_object_using_predicate_and_sheerka_objects(self):
+        sheerka, context, foo, bar = self.init_concepts("foo", "bar")
+        lst = [foo, bar, A("a21", 3.14)]
+
+        assert sheerka.filter_objects(context, lst, "self == bar") == [lst[1]]
+
+    def test_i_can_filter_objects_using_concept(self):
+        sheerka, context, foo, bar, isa = self.init_concepts(
+            "foo",
+            "bar",
+            Concept("x is a concept", body="isinstance(x, Concept)", pre="is_question()").def_var("x"),
+            create_new=True)
+
+        lst = [foo, A("a21", 3.14), bar, B("a21", 3.14)]
+        assert sheerka.filter_objects(context, lst, "self is a concept") == [foo, bar]
+
+    def test_i_can_filter_objects_when_no_kwargs_and_no_predicate(self):
+        sheerka, context, foo, bar = self.init_concepts("foo", "bar")
+        lst = [foo, bar, A("a21", 3.14)]
+
+        assert sheerka.filter_objects(context, lst) == lst
+
+    def test_i_must_select_object_property_using_string(self):
+        sheerka, context = self.init_test().unpack()
+
+        with pytest.raises(SyntaxError):
+            sheerka.select_objects(context, [], 00)
+
+    def test_i_can_select_objects_with_args(self):
+        sheerka, context = self.init_test().unpack()
+        lst = [A("a11", 10), A("a21", 3.14), A({1, "v"}, 0xab), A([0, 1], {"key": "value"})]
+
+        assert sheerka.select_objects(context, lst, "prop1", "prop2") == (
+            ('a11', 10),
+            ('a21', 3.14),
+            ({1, 'v'}, 171),
+            ([0, 1], {'key': 'value'}))
+
+    def test_i_can_select_objects_when_container(self):
+        sheerka, context = self.init_test().unpack()
+        lst = [A("a11", 10), A("a21", 3.14), A({1, "v"}, 0xab), A([0, 1], {"key": "value"})]
+        container = sheerka.new(BuiltinConcepts.EXPLANATION, body=lst, digest="xxx", command="text")
+
+        res = sheerka.select_objects(context, container, "prop1", "prop2")
+        assert sheerka.isinstance(res, BuiltinConcepts.EXPLANATION)
+        assert res.digest == "xxx"
+        assert res.command == "text"
+        assert res.body == (('a11', 10),
+                            ('a21', 3.14),
+                            ({1, 'v'}, 171),
+                            ([0, 1], {'key': 'value'}))
+
+    def test_i_can_select_objects_with_complicated_request(self):
+        sheerka, context = self.init_test().unpack()
+        lst = [A("a11", 10), A("a21", 3.14), A({1, "v"}, 0xab)]
+
+        assert sheerka.select_objects(context, lst, "self.prop2 + 5") == (15, 8.14, 0xab + 5)
+        assert sheerka.select_objects(context, lst, "isinstance(self.prop1, str)") == (True, True, False)
+
+    def test_error_when_collecting_returns_are_managed(self):
+        sheerka, context = self.init_test().unpack()
+        lst = [A("a11", 10), A("a21", {"key": "value"})]
+        res = sheerka.select_objects(context, lst, "self.prop2 + 5")
+
+        assert len(res) == 2
+        assert res[0] == 15
+        assert isinstance(res[1], TypeError)
+
+    def test_i_can_select_objects_using_kwargs(self):
+        sheerka, context = self.init_test().unpack()
+        lst = [A("a11", 10), A("a21", 3.14), A({1, "v"}, 0xab), A([0, 1], {"key": "value"})]
+
+        assert sheerka.select_objects(context, lst, p1="prop1", p2="prop2") == (
+            {"p1": "a11", "p2": 10},
+            {"p1": "a21", "p2": 3.14},
+            {"p1": {1, "v"}, "p2": 0xab},
+            {"p1": [0, 1], "p2": {"key": "value"}},
+        )
+
+    def test_i_can_collect_attributes(self):
+        sheerka = self.get_sheerka()
+        lst = [A("", ""),
+               B("", ""),
+               Concept("foo").def_var("a").auto_init(),
+               Concept("bar").def_var("y").def_var("x").auto_init()]
+
+        res = sheerka.collect_attributes(lst)
+
+        assert sheerka.isinstance(res, BuiltinConcepts.TO_DICT)
+        assert res.body == {
+            "A": ["prop1", "prop2"],
+            "B": ["fake_prop1", "fake_prop2"],
+            "foo": ["a", 'body', 'id', 'key', 'name'],
+            "bar": ['body', 'id', 'key', 'name', "x", "y"]  # attributes are sorted
+        }

tests/core/test_SheerkaRuleManager.py

@@ -1,5 +1,3 @@
-import ast
-
 import pytest
 
 from core.builtin_concepts import BuiltinConcepts, ReturnValueConcept
@@ -7,26 +5,20 @@ from core.concept import Concept, DEFINITION_TYPE_DEF
 from core.global_symbols import RULE_COMPARISON_CONTEXT, NotFound, EVENT_RULE_DELETED
 from core.rule import Rule, ACTION_TYPE_PRINT, ACTION_TYPE_EXEC
 from core.sheerka.Sheerka import RECOGNIZED_BY_ID, RECOGNIZED_BY_NAME
-from core.sheerka.services.SheerkaEvaluateRules import SheerkaEvaluateRules
 from core.sheerka.services.SheerkaExecute import ParserInput
 from core.sheerka.services.SheerkaRuleManager import SheerkaRuleManager, FormatRuleActionParser, \
     FormatAstRawText, FormatAstVariable, FormatAstSequence, FormatAstFunction, \
     FormatRuleSyntaxError, FormatAstList, UnexpectedEof, FormatAstColor, FormatAstDict, \
     FormatAstMulti, \
-    PythonCodeEmitter, FormatAstNode, ReteConditionExprVisitor, PythonConditionExprVisitor, \
-    CompiledCondition
+    PythonCodeEmitter, FormatAstNode, ReteConditionExprVisitor
 from core.tokenizer import Token, TokenKind
-from evaluators.PythonEvaluator import Expando
 from parsers.BaseParser import ErrorSink
 from parsers.ExpressionParser import ExpressionParser
-from parsers.PythonParser import PythonNode
-from sheerkarete.common import V
-from sheerkarete.conditions import Condition, AndConditions, FilterCondition
+from sheerkarete.conditions import FilterCondition
 from sheerkarete.network import ReteNetwork
 from tests.TestUsingFileBasedSheerka import TestUsingFileBasedSheerka
 from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
-from tests.parsers.parsers_utils import get_rete_conditions, NEGCOND, \
-    NCCOND
+from tests.parsers.parsers_utils import get_rete_conditions, NEGCOND, NCCOND
 
 seq = FormatAstSequence
 raw = FormatAstRawText
@@ -188,7 +180,7 @@ class TestSheerkaRuleManager(TestUsingMemoryBasedSheerka):
         rule = service.init_rule(context, rule)
 
         assert len(rule.error_sink["when"]) > 0
-        assert sheerka.is_error(rule.error_sink["when"][0])
+        assert sheerka.has_error(context, rule.error_sink["when"][0])
         assert "print" not in rule.error_sink
         assert "then" not in rule.error_sink
         assert rule.metadata.is_compiled
@@ -210,7 +202,7 @@ class TestSheerkaRuleManager(TestUsingMemoryBasedSheerka):
 
         other_action_type = ACTION_TYPE_PRINT if action_type == ACTION_TYPE_EXEC else ACTION_TYPE_EXEC
 
-        assert sheerka.is_error(rule.error_sink[action_type])
+        assert sheerka.has_error(context, rule.error_sink[action_type])
         assert other_action_type not in rule.error_sink
         assert "when" not in rule.error_sink
         assert rule.metadata.is_compiled
@@ -282,30 +274,6 @@ class TestSheerkaRuleManager(TestUsingMemoryBasedSheerka):
 
         assert parser.error_sink == expected_error
 
-    @pytest.mark.parametrize("text, compiled_text", [
-        ("a == 5", "a == 5"),
-        ("foo > 5", "foo > 5"),
-        ("func() == 5", "func() == 5"),
-        ("not a == 5", "not (a == 5)"),
-        ("not foo > 5", "not (foo > 5)"),
-        ("not func() == 5", "not (func() == 5)"),
-    ])
-    def test_i_can_compile_predicate_when_pure_python(self, text, compiled_text):
-        sheerka, context, *concepts = self.init_concepts("foo")
-        service = sheerka.services[SheerkaRuleManager.NAME]
-        ast_ = ast.parse(compiled_text, "<source>", 'eval')
-        expected_python_node = PythonNode(compiled_text, ast_)
-
-        compilation_result = service.compile_when(context, "test", text)
-        res = compilation_result.python_conditions
-
-        assert len(res) == 1
-        assert isinstance(res[0], CompiledCondition)
-        assert res[0].evaluator_type == PYTHON_EVALUATOR_NAME
-        assert sheerka.isinstance(res[0].return_value, BuiltinConcepts.RETURN_VALUE)
-        assert sheerka.objvalue(res[0].return_value) == expected_python_node
-        assert res[0].concept is None
-
     def test_i_can_get_rule_priorities(self):
         sheerka, context, rule_true, rule_false = self.init_test().with_format_rules(("True", "True"),
                                                                                      ("False", "False")).unpack()
@@ -499,20 +467,6 @@ isinstance(var, Concept) and var.key == 'hello __var__0'""" + \
         res = sheerka.get_exec_rules()
         assert res == [r2, r3, r1]
 
-    @pytest.mark.skip
-    def test_i_can_compile_rete_using_name(self):
-        sheerka, context, *concepts = self.init_test().unpack()
-        service = sheerka.services[SheerkaRuleManager.NAME]
-
-        text = "__ret"
-
-        compilation_result = service.compile_when(context, "test", text)
-        res = compilation_result.rete_disjunctions
-
-        assert len(res) == 1
-        assert isinstance(res[0], AndConditions)
-        assert res[0].conditions == [Condition(V("__x_00__"), "__name__", "__ret")]
-
     def test_i_can_properly_copy_a_rule(self):
         sheerka, context = self.init_test().unpack()
         service = sheerka.services[SheerkaRuleManager.NAME]
@@ -525,98 +479,6 @@ isinstance(var, Concept) and var.key == 'hello __var__0'""" + \
         for k, v in vars(rule).items():
             assert getattr(clone, k) == getattr(rule, k)
 
-    @pytest.mark.parametrize("expression, expected_as_str, expected_variables", [
-        (
-            "__ret",
-            ["#__x_00__|__name__|'__ret'"],
-            {"__ret"}
-        ),
-        (
-            "__ret.status == True",
-            ["#__x_00__|__name__|'__ret'", "#__x_00__|status|True"],
-            {"__ret"}
-        ),
-        (
-            "__ret.status",
-            ["#__x_00__|__name__|'__ret.status'"],
-            {"__ret.status"}
-        ),
-        (
-            "body",
-            ["#__x_00__|__name__|'body'"],
-            {"body"}
-        ),
-        (
-            "__ret and __ret.status",
-            ["#__x_00__|__name__|'__ret'", "#__x_01__|__name__|'__ret.status'"],
-            {"__ret", "__ret.status"}
-        ),
-    ])
-    def test_i_can_get_rete_conditions(self, expression, expected_as_str, expected_variables):
-        sheerka, context = self.init_test().unpack()
-        parser = ExpressionParser()
-        expected = get_rete_conditions(*expected_as_str)
-
-        error_sink = ErrorSink()
-        parser_input = ParserInput(expression)
-        parser.reset_parser_input(parser_input, error_sink)
-        parsed = parser.parse_input(context, parser_input, error_sink)
-
-        visitor = ReteConditionExprVisitor(context)
-        conditions = visitor.get_conditions(parsed)
-
-        assert conditions == [expected]
-
-        # check against a Rete network
-        network = ReteNetwork()
-        rule = Rule("test", expression, None)
-        rule.metadata.id = 9999
-        rule.metadata.is_compiled = True
-        rule.metadata.is_enabled = True
-        rule.rete_disjunctions = conditions
-        network.add_rule(rule)
-
-        return_value = ReturnValueConcept("Test", True, None)
-        if "__ret" in expected_variables:
-            network.add_obj("__ret", return_value)
-        if "__ret.status" in expected_variables:
-            network.add_obj("__ret.status", return_value.status)
-        if "body" in expected_variables:
-            network.add_obj("body", return_value.body)
-
-        matches = list(network.matches)
-        assert len(matches) == 1
-
-    def test_i_can_get_rete_conditions_when_no_attribute(self):
-        sheerka, context = self.init_test().unpack()
-        expression = "a == 10"
-        expected_as_str = ["#__x_00__|__name__|'a'", "#__x_00__|__self__|10"]
-        parser = ExpressionParser()
-        expected = get_rete_conditions(*expected_as_str)
-
-        error_sink = ErrorSink()
-        parser_input = ParserInput(expression)
-        parser.reset_parser_input(parser_input, error_sink)
-        parsed = parser.parse_input(context, parser_input, error_sink)
-
-        visitor = ReteConditionExprVisitor(context)
-        conditions = visitor.get_conditions(parsed)
-
-        assert conditions == [expected]
-
-        # check against a Rete network
-        network = ReteNetwork()
-        rule = Rule("test", expression, None)
-        rule.metadata.id = 9999
-        rule.metadata.is_compiled = True
-        rule.metadata.is_enabled = True
-        rule.rete_disjunctions = conditions
-        network.add_rule(rule)
-
-        network.add_obj("a", 10)
-        matches = list(network.matches)
-        assert len(matches) == 1
-
     @pytest.mark.skip("No ready yet for SheerkaFilterCondition")
     def test_i_can_get_rete_conditions_when_function(self):
         sheerka, context, greetings = self.init_test().with_concepts(
@@ -652,235 +514,6 @@ isinstance(var, Concept) and var.key == 'hello __var__0'""" + \
         # matches = list(network.matches)
         # assert len(matches) == 1
 
-    @pytest.mark.parametrize("test_name, expression, variable_name, expected_as_str", [
-        (
-            "recognize by name",
-            "recognize(__ret.body, greetings)",
-            None,
-            ["#__x_00__|__name__|'__ret'",
-             "#__x_00__|body|#__x_01__",
-             "#__x_01__|__is_concept__|True",
-             "#__x_01__|name|'greetings'"]
-        ),
-        (
-            "recognize by id",
-            "recognize(__ret.body, c:|1001:)",
-            None,
-            ["#__x_00__|__name__|'__ret'",
-             "#__x_00__|body|#__x_01__",
-             "#__x_01__|__is_concept__|True",
-             "#__x_01__|id|'1001'"]
-        ),
-        (
-            "recognize by name using c_str",
-            "recognize(__ret.body, c:greetings:)",
-            None,
-            ["#__x_00__|__name__|'__ret'",
-             "#__x_00__|body|#__x_01__",
-             "#__x_01__|__is_concept__|True",
-             "#__x_01__|name|'greetings'"]
-        ),
-        (
-            "recognize by name and add other conditions (str)",
-            "recognize(__ret.body, greetings) and __ret.body.a == 'my friend'",
-            "my friend",
-            ["#__x_00__|__name__|'__ret'",
-             "#__x_00__|body|#__x_01__",
-             "#__x_01__|__is_concept__|True",
-             "#__x_01__|name|'greetings'",
-             "#__x_01__|a|'my friend'"]
-        ),
-        (
-            "recognize by name and add other conditions (sheerka)",
-            "recognize(__ret.body, greetings) and __ret.body.a == sheerka",
-            "sheerka",
-            ["#__x_00__|__name__|'__ret'",
-             "#__x_00__|body|#__x_01__",
-             "#__x_01__|__is_concept__|True",
-             "#__x_01__|name|'greetings'",
-             "#__x_01__|a|'__sheerka__'"]
-        ),
-        (
-            "recognize by name and add other conditions (concept)",
-            "recognize(__ret.body, greetings) and __ret.body.a == foo",
-            "foo",
-            ["#__x_00__|__name__|'__ret'",
-             "#__x_00__|body|#__x_01__",
-             "#__x_01__|__is_concept__|True",
-             "#__x_01__|name|'greetings'",
-             "#__x_01__|a|#__x_02__",
-             "#__x_02__|__is_concept__|True",
-             "#__x_02__|key|'foo'"]
-        ),
-        (
-            "recognize by instance",
-            "recognize(__ret.body, hello sheerka)",
-            "sheerka",
-            ["#__x_00__|__name__|'__ret'",
-             "#__x_00__|body|#__x_01__",
-             "#__x_01__|__is_concept__|True",
-             "#__x_01__|key|'hello __var__0'",
-             "#__x_01__|a|'__sheerka__'"]
-        ),
-        (
-            "recognize by instance",
-            "recognize(__ret.body, hello 'my friend')",
-            "my friend",
-            ["#__x_00__|__name__|'__ret'",
-             "#__x_00__|body|#__x_01__",
-             "#__x_01__|__is_concept__|True",
-             "#__x_01__|key|'hello __var__0'",
-             "#__x_01__|a|'my friend'"]
-        ),
-        (
-            "recognize by instance",
-            "recognize(__ret.body, hello foo)",
-            "foo",
-            ["#__x_00__|__name__|'__ret'",
-             "#__x_00__|body|#__x_01__",
-             "#__x_01__|__is_concept__|True",
-             "#__x_01__|key|'hello __var__0'",
-             "#__x_01__|a|#__x_02__",
-             "#__x_02__|__is_concept__|True",
-             "#__x_02__|key|'foo'",
-             ]
-        ),
-        (
-            "recognize by instance when long concept",
-            "recognize(__ret.body, hello my best friend)",
-            "my best friend",
-            ["#__x_00__|__name__|'__ret'",
-             "#__x_00__|body|#__x_01__",
-             "#__x_01__|__is_concept__|True",
-             "#__x_01__|key|'hello __var__0'",
-             "#__x_01__|a|#__x_02__",
-             "#__x_02__|__is_concept__|True",
-             "#__x_02__|key|'my best friend'",
-             ]
-        ),
-    ])
-    def test_i_can_get_rete_using_recognize_function(self, test_name, expression, variable_name, expected_as_str):
-        sheerka, context, greetings, foo, my_best_friend = self.init_test().with_concepts(
-            Concept("greetings", definition="hello a", definition_type=DEFINITION_TYPE_DEF).def_var("a"),
-            Concept("foo"),
-            Concept("my best friend"),
-            create_new=True
-        ).unpack()
-        parser = ExpressionParser()
-        expected = get_rete_conditions(*expected_as_str)
-
-        error_sink = ErrorSink()
-        parser_input = ParserInput(expression)
-        parser.reset_parser_input(parser_input, error_sink)
-        parsed = parser.parse_input(context, parser_input, error_sink)
-
-        visitor = ReteConditionExprVisitor(context)
-        conditions = visitor.get_conditions(parsed)
-
-        assert conditions == [expected]
-
-        # check against a Rete network
-        network = ReteNetwork()
-        rule = Rule("test", expression, None)
-        rule.metadata.id = 9999
-        rule.metadata.is_compiled = True
-        rule.metadata.is_enabled = True
-        rule.rete_disjunctions = conditions
-        network.add_rule(rule)
-
-        variable_map = {
-            "foo": foo,
-            "my best friend": my_best_friend,
-            "sheerka": sheerka
-        }
-        variable = variable_map.get(variable_name, variable_name)
-        to_recognize = sheerka.new_from_template(greetings, greetings.key, a=variable)
-        network.add_obj("__ret", ReturnValueConcept("Test", True, to_recognize))
-        matches = list(network.matches)
-        assert len(matches) == 1
-
-    @pytest.mark.parametrize("expression, variable_name, expected_as_str", [
-        (
-            "greetings",
-            None,
-            ["#__x_00__|__name__|'__ret'",
-             "#__x_00__|body|#__x_01__",
-             "#__x_01__|__is_concept__|True",
-             "#__x_01__|name|'greetings'"]
-        ),
-        (
-            "c:|1001:",
-            None,
-            ["#__x_00__|__name__|'__ret'",
-             "#__x_00__|body|#__x_01__",
-             "#__x_01__|__is_concept__|True",
-             "#__x_01__|id|'1001'"]
-        ),
-        (
-            "hello foo",
-            "foo",
-            ["#__x_00__|__name__|'__ret'",
-             "#__x_00__|body|#__x_01__",
-             "#__x_01__|__is_concept__|True",
-             "#__x_01__|key|'hello __var__0'",
-             "#__x_01__|a|#__x_02__",
-             "#__x_02__|__is_concept__|True",
-             "#__x_02__|key|'foo'",
-             ]
-        ),
-        (
-            "hello my best friend",
-            "my best friend",
-            ["#__x_00__|__name__|'__ret'",
-             "#__x_00__|body|#__x_01__",
-             "#__x_01__|__is_concept__|True",
-             "#__x_01__|key|'hello __var__0'",
-             "#__x_01__|a|#__x_02__",
-             "#__x_02__|__is_concept__|True",
-             "#__x_02__|key|'my best friend'",
-             ]
-        ),
-    ])
-    def test_i_can_get_rete_when_a_concept_is_recognized(self, expression, variable_name, expected_as_str):
-        sheerka, context, greetings, foo, my_best_friend = self.init_test().with_concepts(
-            Concept("greetings", definition="hello a", definition_type=DEFINITION_TYPE_DEF).def_var("a"),
-            Concept("foo"),
-            Concept("my best friend"),
-            create_new=True
-        ).unpack()
-        parser = ExpressionParser()
-        expected = get_rete_conditions(*expected_as_str)
-
-        error_sink = ErrorSink()
-        parser_input = ParserInput(expression)
-        parser.reset_parser_input(parser_input, error_sink)
-        parsed = parser.parse_input(context, parser_input, error_sink)
-
-        visitor = ReteConditionExprVisitor(context)
-        conditions = visitor.get_conditions(parsed)
-
-        assert conditions == [expected]
-
-        # check against a Rete network
-        network = ReteNetwork()
-        rule = Rule("test", expression, None)
-        rule.metadata.id = 9999
-        rule.metadata.is_compiled = True
-        rule.metadata.is_enabled = True
-        rule.rete_disjunctions = conditions
-        network.add_rule(rule)
-
-        variable_map = {
-            "foo": foo,
-            "my best friend": my_best_friend,
-            "sheerka": sheerka
-        }
-        variable = variable_map.get(variable_name, variable_name)
-        to_recognize = sheerka.new_from_template(greetings, greetings.key, a=variable)
-        network.add_obj("__ret", ReturnValueConcept("Test", True, to_recognize))
-        matches = list(network.matches)
-        assert len(matches) == 1
-
     @pytest.mark.parametrize("expression, bag_key, expected", [
         ("not __ret", "__other", [NEGCOND("#__x_00__|__name__|'__ret'")]),
         ("not not __ret", "__ret", ["#__x_00__|__name__|'__ret'"]),
@@ -894,6 +527,10 @@ isinstance(var, Concept) and var.key == 'hello __var__0'""" + \
                                                                        "#__x_01__|key|'hello __var__0'",
                                                                        "#__x_01__|a|'__sheerka__'"])]),
         ("__ret and not __error", "__ret", ["#__x_00__|__name__|'__ret'", NEGCOND("#__x_01__|__name__|'__error'")]),
+        ("not recognize(self, hello sheerka)", "__ret", ["#__x_00__|__name__|'self'",
+                                                         NCCOND(["#__x_00__|__is_concept__|True",
+                                                                 "#__x_00__|key|'hello __var__0'",
+                                                                 "#__x_00__|a|'__sheerka__'"])]),
     ])
     def test_i_can_get_rete_using_not(self, expression, bag_key, expected):
         sheerka, context, greetings, foo = self.init_test().with_concepts(
@@ -914,7 +551,7 @@ isinstance(var, Concept) and var.key == 'hello __var__0'""" + \
         assert conditions == [expected_conditions]
 
     @pytest.mark.skip("I am not sure yet of what I want to get")
-    @pytest.mark.parametrize("expression, expected_as_str", [
+    @pytest.mark.parametrize("expression, expected_as_list_of_str", [
         (
             "eval(__ret.body, 'foo' starts with 'f')",
             ["#__x_00__|__name__|'__ret'",
@@ -926,7 +563,7 @@ isinstance(var, Concept) and var.key == 'hello __var__0'""" + \
              "$eval(__x_01__, a, b, c)"]
         ),
     ])
-    def test_i_can_get_rete_conditions_using_eval_function(self, expression, expected_as_str):
+    def test_i_can_get_rete_conditions_using_eval_function(self, expression, expected_as_list_of_str):
         sheerka, context, start_with = self.init_test().with_concepts(
             Concept("x starts with y",
                     pre="is_question",
@@ -935,7 +572,7 @@ isinstance(var, Concept) and var.key == 'hello __var__0'""" + \
         ).unpack()
 
         parser = ExpressionParser()
-        expected = get_rete_conditions(*expected_as_str)
+        expected = get_rete_conditions(*expected_as_list_of_str)
 
         error_sink = ErrorSink()
         parser_input = ParserInput(expression)
@@ -961,372 +598,6 @@ isinstance(var, Concept) and var.key == 'hello __var__0'""" + \
         matches = list(network.matches)
         assert len(matches) == 1
 
-    @pytest.mark.parametrize("expression, expected_compiled, expected_variables", [
-        ("__ret", None, {"__ret"}),
-        ("__ret.status == True", "__ret.status == True", {"__ret"}),
-        ("__ret.status", None, {"__ret.status"}),
-        ("body", None, {"body"}),
-        ("__ret and __ret.status", None, {"__ret", "__ret.status"})
-    ])
-    def test_i_can_get_compiled_conditions(self, expression, expected_compiled, expected_variables):
-        sheerka, context = self.init_test().unpack()
-
-        parser = ExpressionParser()
-        error_sink = ErrorSink()
-        parser_input = ParserInput(expression)
-        parser.reset_parser_input(parser_input, error_sink)
-        parsed = parser.parse_input(context, parser_input, error_sink)
-
-        visitor = PythonConditionExprVisitor(context)
-        conditions = visitor.get_conditions(parsed)
-
-        assert len(conditions) == 1
-        assert isinstance(conditions[0], CompiledCondition)
-        if expected_compiled:
-            ast_ = ast.parse(expected_compiled, "<source>", 'eval')
-            expected_python_node = PythonNode(expected_compiled, ast_)
-            assert conditions[0].evaluator_type == PYTHON_EVALUATOR_NAME
-            assert sheerka.isinstance(conditions[0].return_value, BuiltinConcepts.RETURN_VALUE)
-            assert sheerka.objvalue(conditions[0].return_value) == expected_python_node
-        else:
-            assert conditions[0].evaluator_type is None
-            assert conditions[0].return_value is None
-        assert conditions[0].concept is None
-        assert conditions[0].variables == expected_variables
-
-        # check against SheerkaEvaluateRules
-        evaluate_rules_service = sheerka.services[SheerkaEvaluateRules.NAME]
-        return_value = ReturnValueConcept("Test", True, None)
-        bag = {}
-        if "__ret" in expected_variables:
-            bag["__ret"] = return_value
-        if "__ret.status" in expected_variables:
-            bag["__ret.status"] = return_value.status
-        if "body" in expected_variables:
-            bag["body"] = return_value.body
-        with context.push(BuiltinConcepts.RULES_EVALUATION, bag, desc="Evaluating rules...") as sub_context:
-            sub_context.sheerka.add_many_to_short_term_memory(sub_context, bag)
-            rule = Rule(name="test_i_can_get_compiled_conditions", predicate=expression)
-            rule.compiled_conditions = conditions
-            res = evaluate_rules_service.evaluate_rule(sub_context, rule, bag)
-            assert res.status
-            assert self.sheerka.is_success(self.sheerka.objvalue(res))
-
-    def test_i_can_get_compiled_conditions_when_no_attribute(self):
-        sheerka, context = self.init_test().unpack()
-        expression = "a == 10"
-        expected_compiled = "a == 10"
-
-        parser = ExpressionParser()
-        error_sink = ErrorSink()
-        parser_input = ParserInput(expression)
-        parser.reset_parser_input(parser_input, error_sink)
-        parsed = parser.parse_input(context, parser_input, error_sink)
-
-        visitor = PythonConditionExprVisitor(context)
-        conditions = visitor.get_conditions(parsed)
-
-        assert len(conditions) == 1
-        assert isinstance(conditions[0], CompiledCondition)
-        ast_ = ast.parse(expected_compiled, "<source>", 'eval')
-        expected_python_node = PythonNode(expected_compiled, ast_)
-        assert conditions[0].evaluator_type == PYTHON_EVALUATOR_NAME
-        assert sheerka.isinstance(conditions[0].return_value, BuiltinConcepts.RETURN_VALUE)
-        assert sheerka.objvalue(conditions[0].return_value) == expected_python_node
-        assert conditions[0].concept is None
-        assert conditions[0].variables == {"a"}
-
-        # check against SheerkaEvaluateRules
-        evaluate_rules_service = sheerka.services[SheerkaEvaluateRules.NAME]
-        bag = {"a": 10}
-        with context.push(BuiltinConcepts.RULES_EVALUATION, bag, desc="Evaluating rules...") as sub_context:
-            sub_context.sheerka.add_many_to_short_term_memory(sub_context, bag)
-            rule = Rule(name="test_i_can_get_compiled_conditions", predicate=expression)
-            rule.compiled_conditions = conditions
-            res = evaluate_rules_service.evaluate_rule(sub_context, rule, bag)
-            assert res.status
-            assert self.sheerka.is_success(self.sheerka.objvalue(res))
-
-    def test_i_can_get_compiled_conditions_when_function(self):
-        sheerka, context, greetings = self.init_test().with_concepts(
-            Concept("greetings", definition="hello a", definition_type=DEFINITION_TYPE_DEF).def_var("a"),
-        ).unpack()
-        expression = "isinstance(a, greetings)"
-        expected_compiled = "isinstance(a, greetings)"
-
-        parser = ExpressionParser()
-        error_sink = ErrorSink()
-        parser_input = ParserInput(expression)
-        parser.reset_parser_input(parser_input, error_sink)
-        parsed = parser.parse_input(context, parser_input, error_sink)
-
-        visitor = PythonConditionExprVisitor(context)
-        conditions = visitor.get_conditions(parsed)
-
-        assert len(conditions) == 1
-        assert isinstance(conditions[0], CompiledCondition)
-        ast_ = ast.parse(expected_compiled, "<source>", 'eval')
-        expected_python_node = PythonNode(expected_compiled, ast_)
-        assert conditions[0].evaluator_type == PYTHON_EVALUATOR_NAME
-        assert sheerka.isinstance(conditions[0].return_value, BuiltinConcepts.RETURN_VALUE)
-        assert sheerka.objvalue(conditions[0].return_value) == expected_python_node
-        assert conditions[0].concept is None
-        assert conditions[0].variables == {"a"}
-
-        # check against SheerkaEvaluateRules
-        evaluate_rules_service = sheerka.services[SheerkaEvaluateRules.NAME]
-        bag = {"a": sheerka.new(greetings, a="my friend")}
-        with context.push(BuiltinConcepts.RULES_EVALUATION, bag, desc="Evaluating rules...") as sub_context:
-            sub_context.sheerka.add_many_to_short_term_memory(sub_context, bag)
-            rule = Rule(name="test_i_can_get_compiled_conditions", predicate=expression)
-            rule.compiled_conditions = conditions
-            res = evaluate_rules_service.evaluate_rule(sub_context, rule, bag)
-            assert res.status
-            assert self.sheerka.is_success(self.sheerka.objvalue(res))
-
-    @pytest.mark.parametrize("expression, variable_name, expected_compiled", [
-        (
-            "recognize(__ret.body, greetings)",
-            None,
-            "__x_00__ = __ret.body\nisinstance(__x_00__, Concept) and __x_00__.name == 'greetings'"
-        ),
-        (
-            "recognize(__ret.body, c:|1001:)",
-            None,
-            "__x_00__ = __ret.body\nisinstance(__x_00__, Concept) and __x_00__.id == '1001'"
-        ),
-        (
-            "recognize(__ret.body, c:greetings:)",
-            None,
-            "__x_00__ = __ret.body\nisinstance(__x_00__, Concept) and __x_00__.name == 'greetings'"
-        ),
-        (
-            "recognize(__ret.body, greetings) and __ret.body.a == 'my friend'",
-            "my friend",
-            "__x_00__ = __ret.body\nisinstance(__x_00__, Concept) and __x_00__.name == 'greetings' and __x_00__.a == 'my friend'"
-        ),
-        (
-            "recognize(__ret.body, greetings) and __ret.body.a == sheerka",
-            "sheerka",
-            """__x_00__ = __ret.body
-isinstance(__x_00__, Concept) and __x_00__.name == 'greetings' and is_sheerka(__x_00__.a)"""
-        ),
-        (
-            "recognize(__ret.body, greetings) and __ret.body.a == foo",
-            "foo",
-            """__x_00__ = __ret.body
-__x_01__ = __x_00__.a
-isinstance(__x_00__, Concept) and __x_00__.name == 'greetings' and isinstance(__x_01__, Concept) and __x_01__.key == 'foo'"""
-        ),
-        (
-            "recognize(__ret.body, hello sheerka)",
-            "sheerka",
-            """__x_00__ = __ret.body
-isinstance(__x_00__, Concept) and __x_00__.key == 'hello __var__0' and is_sheerka(__x_00__.a)"""
-        ),
-        (
-            "recognize(__ret.body, hello 'my friend')",
-            "my friend",
-            """__x_00__ = __ret.body
-isinstance(__x_00__, Concept) and __x_00__.key == 'hello __var__0' and __x_00__.a == 'my friend'"""
-        ),
-        (
-            "recognize(__ret.body, hello foo)",
-            "foo",
-            """__x_00__ = __ret.body
-__x_01__ = __x_00__.a
-isinstance(__x_00__, Concept) and __x_00__.key == 'hello __var__0' and isinstance(__x_01__, Concept) and __x_01__.key == 'foo'"""
-        ),
-        (
-            "recognize(__ret.body, hello my best friend)",
-            "my best friend",
-            """__x_00__ = __ret.body
-__x_01__ = __x_00__.a
-isinstance(__x_00__, Concept) and __x_00__.key == 'hello __var__0' and isinstance(__x_01__, Concept) and __x_01__.key == 'my best friend'"""
-        ),
-
-    ])
-    def test_i_can_get_compiled_using_recognize_function(self, expression, variable_name, expected_compiled):
-        sheerka, context, greetings, foo, my_best_friend = self.init_test().with_concepts(
-            Concept("greetings", definition="hello a", definition_type=DEFINITION_TYPE_DEF).def_var("a"),
-            Concept("foo"),
-            Concept("my best friend"),
-            create_new=True
-        ).unpack()
-
-        parser = ExpressionParser()
-        error_sink = ErrorSink()
-        parser_input = ParserInput(expression)
-        parser.reset_parser_input(parser_input, error_sink)
-        parsed = parser.parse_input(context, parser_input, error_sink)
-
-        visitor = PythonConditionExprVisitor(context)
-        conditions = visitor.get_conditions(parsed)
-
-        assert len(conditions) == 1
-        assert isinstance(conditions[0], CompiledCondition)
-        if expected_compiled:
-            ast_ = ast.parse(expected_compiled, "<source>", 'exec')
-            expected_python_node = PythonNode(expected_compiled, ast_, expected_compiled)
-            assert conditions[0].evaluator_type == PYTHON_EVALUATOR_NAME
-            assert sheerka.isinstance(conditions[0].return_value, BuiltinConcepts.RETURN_VALUE)
-            assert sheerka.objvalue(conditions[0].return_value) == expected_python_node
-        else:
-            assert conditions[0].evaluator_type is None
-            assert conditions[0].return_value is None
-        assert conditions[0].concept is None
-        assert conditions[0].variables == {"__ret"}
-
-        # check against SheerkaEvaluateRules
-        evaluate_rules_service = sheerka.services[SheerkaEvaluateRules.NAME]
-        variable_map = {
-            "foo": foo,
-            "my best friend": my_best_friend,
-            "sheerka": Expando("sheerka", {})
-        }
-        variable = variable_map.get(variable_name, variable_name)
-        to_recognize = sheerka.new_from_template(greetings, greetings.key, a=variable)
-        bag = {"__ret": ReturnValueConcept("Test", True, to_recognize)}
-        with context.push(BuiltinConcepts.RULES_EVALUATION, bag, desc="Evaluating rules...") as sub_context:
-            sub_context.sheerka.add_many_to_short_term_memory(sub_context, bag)
-            rule = Rule(name="test_i_can_get_compiled_using_recognize_function", predicate=expression)
-            rule.compiled_conditions = conditions
-            res = evaluate_rules_service.evaluate_rule(sub_context, rule, bag)
-            assert res.status
-            assert self.sheerka.is_success(self.sheerka.objvalue(res))
-
-    @pytest.mark.parametrize("expression, variable_name, expected_compiled", [
-        (
-            "greetings",
-            None,
-            "__x_00__ = __ret.body\nisinstance(__x_00__, Concept) and __x_00__.name == 'greetings'"
-        ),
-        (
-            "c:|1001:",
-            None,
-            "__x_00__ = __ret.body\nisinstance(__x_00__, Concept) and __x_00__.id == '1001'"
-        ),
-        (
-            "hello foo",
-            "foo",
-            """__x_00__ = __ret.body
-__x_01__ = __x_00__.a
-isinstance(__x_00__, Concept) and __x_00__.key == 'hello __var__0' and isinstance(__x_01__, Concept) and __x_01__.key == 'foo'"""
-        ),
-        (
-            "hello my best friend",
-            "my best friend",
-            """__x_00__ = __ret.body
-__x_01__ = __x_00__.a
-isinstance(__x_00__, Concept) and __x_00__.key == 'hello __var__0' and isinstance(__x_01__, Concept) and __x_01__.key == 'my best friend'"""
-        ),
-    ])
-    def test_i_can_get_compiled_when_a_concept_is_recognized(self, expression, variable_name, expected_compiled):
-        sheerka, context, greetings, foo, my_best_friend = self.init_test().with_concepts(
-            Concept("greetings", definition="hello a", definition_type=DEFINITION_TYPE_DEF).def_var("a"),
-            Concept("foo"),
-            Concept("my best friend"),
-            create_new=True
-        ).unpack()
-
-        parser = ExpressionParser()
-        error_sink = ErrorSink()
-        parser_input = ParserInput(expression)
-        parser.reset_parser_input(parser_input, error_sink)
-        parsed = parser.parse_input(context, parser_input, error_sink)
-
-        visitor = PythonConditionExprVisitor(context)
-        conditions = visitor.get_conditions(parsed)
-
-        assert len(conditions) == 1
-        assert isinstance(conditions[0], CompiledCondition)
-        if expected_compiled:
-            ast_ = ast.parse(expected_compiled, "<source>", 'exec')
-            expected_python_node = PythonNode(expected_compiled, ast_, expected_compiled)
-            assert conditions[0].evaluator_type == PYTHON_EVALUATOR_NAME
-            assert sheerka.isinstance(conditions[0].return_value, BuiltinConcepts.RETURN_VALUE)
-            assert sheerka.objvalue(conditions[0].return_value) == expected_python_node
-        else:
-            assert conditions[0].evaluator_type is None
-            assert conditions[0].return_value is None
-        assert conditions[0].concept is None
-        assert conditions[0].variables == {"__ret"}
-
-        # check against SheerkaEvaluateRules
-        evaluate_rules_service = sheerka.services[SheerkaEvaluateRules.NAME]
-        variable_map = {
-            "foo": foo,
-            "my best friend": my_best_friend,
-            "sheerka": Expando("sheerka", {})
-        }
-        variable = variable_map.get(variable_name, variable_name)
-        to_recognize = sheerka.new_from_template(greetings, greetings.key, a=variable)
-        bag = {"__ret": ReturnValueConcept("Test", True, to_recognize)}
-        with context.push(BuiltinConcepts.RULES_EVALUATION, bag, desc="Evaluating rules...") as sub_context:
-            sub_context.sheerka.add_many_to_short_term_memory(sub_context, bag)
-            rule = Rule(name="test_i_can_get_compiled_using_recognize_function", predicate=expression)
-            rule.compiled_conditions = conditions
-            res = evaluate_rules_service.evaluate_rule(sub_context, rule, bag)
-            assert res.status
-            assert self.sheerka.is_success(self.sheerka.objvalue(res))
-
-    @pytest.mark.parametrize("expression, expected_compiled, variables, not_variables", [
-        ("not __ret", None, set(), {"__ret"}),
-        ("not not __ret", None, {"__ret"}, set()),
-        ("not __ret.status == True", "not (__ret.status == True)", {"__ret"}, set()),
-        ("not __ret.status", None, set(), {"__ret.status"},),
-        ("__ret and not __ret.status", None, {"__ret"}, {"__ret.status"}),
-        ("not recognize(__ret.body, hello sheerka)", """__x_00__ = __ret.body
-not (isinstance(__x_00__, Concept) and __x_00__.key == 'hello __var__0' and is_sheerka(__x_00__.a))""", {"__ret"},
-         set()),
-        ("__ret and not __error", None, {"__ret"}, {"__error"}),
-    ])
-    def test_i_can_get_compiled_condition_using_not(self, expression, expected_compiled, variables, not_variables):
-        sheerka, context, greetings, foo = self.init_test().with_concepts(
-            Concept("greetings", definition="hello a", definition_type=DEFINITION_TYPE_DEF).def_var("a"),
-            Concept("foo"),
-        ).unpack()
-
-        parser = ExpressionParser()
-        error_sink = ErrorSink()
-        parser_input = ParserInput(expression)
-        parser.reset_parser_input(parser_input, error_sink)
-        parsed = parser.parse_input(context, parser_input, error_sink)
-
-        visitor = PythonConditionExprVisitor(context)
-        conditions = visitor.get_conditions(parsed)
-
-        assert len(conditions) == 1
-        assert isinstance(conditions[0], CompiledCondition)
-        if expected_compiled:
-            if "\n" in expected_compiled:
-                ast_ = ast.parse(expected_compiled, "<source>", 'exec')
-            else:
-                ast_ = ast.parse(expected_compiled, "<source>", 'eval')
-            expected_python_node = PythonNode(expected_compiled, ast_)
-            assert conditions[0].evaluator_type == PYTHON_EVALUATOR_NAME
-            assert sheerka.isinstance(conditions[0].return_value, BuiltinConcepts.RETURN_VALUE)
-            assert sheerka.objvalue(conditions[0].return_value) == expected_python_node
-
-        else:
-            assert conditions[0].evaluator_type is None
-            assert conditions[0].return_value is None
-        assert conditions[0].variables == variables
-        assert conditions[0].not_variables == not_variables
-        assert conditions[0].concept is None
-
-        # check against SheerkaEvaluateRules
-        evaluate_rules_service = sheerka.services[SheerkaEvaluateRules.NAME]
-        ret_val_key = "__ret" if "__ret" in conditions[0].variables else "__other"
-        bag = {ret_val_key: ReturnValueConcept("Test", False, None)}
-        with context.push(BuiltinConcepts.RULES_EVALUATION, bag, desc="Evaluating rules...") as sub_context:
-            sub_context.sheerka.add_many_to_short_term_memory(sub_context, bag)
-            rule = Rule(name="test_i_can_get_compiled_conditions", predicate=expression)
-            rule.compiled_conditions = conditions
-            res = evaluate_rules_service.evaluate_rule(sub_context, rule, bag)
-            assert res.status
-            assert self.sheerka.is_success(self.sheerka.objvalue(res))
-
 
 class TestSheerkaRuleManagerUsingFileBasedSheerka(TestUsingFileBasedSheerka):
     def test_rules_are_initialized_at_startup(self):

tests/core/test_builtin_helpers.py

@@ -182,6 +182,17 @@ class TestBuiltinHelpers(TestUsingMemoryBasedSheerka):
         concept = Concept("foo", pre=pre)
         assert core.builtin_helpers.is_a_question(context, concept) == expected
 
+    def test_context_hints_are_reset_when_call_evaluate_from_source(self):
+        sheerka, context, one = self.init_concepts(Concept("one", body="1"))
+        context.add_to_global_hints(BuiltinConcepts.EVAL_BODY_REQUESTED)
+        context.add_to_protected_hints(BuiltinConcepts.EVAL_BODY_REQUESTED)
+        context.add_to_private_hints(BuiltinConcepts.EVAL_BODY_REQUESTED)
+
+        res = core.builtin_helpers.evaluate_from_source(context, "one", eval_body=False)
+
+        evaluated = [r for r in res if r.status][0].body
+        assert evaluated.body is NotInit
+
     # @pytest.mark.parametrize("return_values", [
     #     None,
     #     []

tests/core/test_sheerka.py

@@ -439,27 +439,27 @@ class TestSheerkaUsingMemoryBasedSheerka(TestUsingMemoryBasedSheerka):
     def test_i_can_get_error_for_simple_objects(self, obj, expected):
         sheerka, context = self.init_test().unpack()
 
-        assert sheerka.get_errors(obj) == expected
+        assert sheerka.get_errors(context, obj) == expected
 
     def test_i_can_get_error_when_builtin_concept_in_error(self):
         sheerka, context = self.init_test().unpack()
 
         obj = sheerka.new(BuiltinConcepts.ONTOLOGY_ALREADY_DEFINED)
-        assert sheerka.get_errors(obj) == [obj]
+        assert sheerka.get_errors(context, obj) == [obj]
 
     def test_i_can_get_error_when_return_value(self):
         sheerka, context = self.init_test().unpack()
 
         error = sheerka.err("an error")
         ret_val = ReturnValueConcept("Test", False, sheerka.err("an error"))
-        assert sheerka.get_errors(ret_val) == [error]
+        assert sheerka.get_errors(context, ret_val) == [error]
 
     def test_i_can_get_inner_error(self):
         sheerka, context = self.init_test().unpack()
 
         error = sheerka.err("an error")
         ret_val = ReturnValueConcept("Test", False, sheerka.err("an error"))
-        assert sheerka.get_errors(ret_val) == [error]
+        assert sheerka.get_errors(context, ret_val) == [error]
 
     def test_i_can_get_error_when_embedded_errors(self):
         sheerka, context = self.init_test().unpack()
@@ -470,7 +470,7 @@ class TestSheerkaUsingMemoryBasedSheerka(TestUsingMemoryBasedSheerka):
         error = sheerka.err([concept_eval_error, unknown_concept, not_an_error])
 
         ret_val = ReturnValueConcept("Test", False, error)
-        errors_found = sheerka.get_errors(ret_val)
+        errors_found = sheerka.get_errors(context, ret_val)
 
         assert errors_found == [error, concept_eval_error, unknown_concept]
 
@@ -488,7 +488,7 @@ class TestSheerkaUsingMemoryBasedSheerka(TestUsingMemoryBasedSheerka):
         multiple_error = sheerka.new(BuiltinConcepts.MULTIPLE_ERRORS, body=[python_error, value_not_found])
         ret_val_2 = ReturnValueConcept("Test", False, multiple_error)
 
-        errors_found = sheerka.get_errors([ret_val_1, ret_val_2])
+        errors_found = sheerka.get_errors(context, [ret_val_1, ret_val_2])
 
         assert errors_found == [error, concept_eval_error, unknown_concept,
                                 multiple_error, python_error, value_not_found]
@@ -502,7 +502,7 @@ class TestSheerkaUsingMemoryBasedSheerka(TestUsingMemoryBasedSheerka):
         error = sheerka.err([concept_eval_error, unknown_concept, python_error])
         ret_val = ReturnValueConcept("Test", False, error)
 
-        errors_found = sheerka.get_errors(ret_val, __type=BuiltinConcepts.CONCEPT_EVAL_ERROR)
+        errors_found = sheerka.get_errors(context, ret_val, __type=BuiltinConcepts.CONCEPT_EVAL_ERROR)
         assert errors_found == [concept_eval_error]
 
     def test_i_can_filter_error_by_class_name(self):
@@ -514,7 +514,7 @@ class TestSheerkaUsingMemoryBasedSheerka(TestUsingMemoryBasedSheerka):
         error = sheerka.err([concept_eval_error, unknown_concept, python_error])
         ret_val = ReturnValueConcept("Test", False, error)
 
-        errors_found = sheerka.get_errors(ret_val, __type="PythonErrorNode")
+        errors_found = sheerka.get_errors(context, ret_val, __type="PythonErrorNode")
         assert errors_found == [python_error]
 
     def test_i_can_filter_error_by_concept_attribute(self):
@@ -526,7 +526,7 @@ class TestSheerkaUsingMemoryBasedSheerka(TestUsingMemoryBasedSheerka):
         error = sheerka.err([concept_eval_error, unknown_concept, python_error])
         ret_val = ReturnValueConcept("Test", False, error)
 
-        errors_found = sheerka.get_errors(ret_val, concept_ref="a_concept_ref")
+        errors_found = sheerka.get_errors(context, ret_val, concept_ref="a_concept_ref")
         assert errors_found == [unknown_concept]
 
     def test_i_can_filter_error_by_class_attribute(self):
@@ -538,7 +538,7 @@ class TestSheerkaUsingMemoryBasedSheerka(TestUsingMemoryBasedSheerka):
         error = sheerka.err([concept_eval_error, unknown_concept, python_error])
         ret_val = ReturnValueConcept("Test", False, error)
 
-        errors_found = sheerka.get_errors(ret_val, source="error source")
+        errors_found = sheerka.get_errors(context, ret_val, source="error source")
         assert errors_found == [python_error]
 
     def test_i_can_filter_error_on_multiple_criteria(self):
@@ -550,14 +550,14 @@ class TestSheerkaUsingMemoryBasedSheerka(TestUsingMemoryBasedSheerka):
         error = sheerka.err([concept_eval_error, unknown_concept, value_not_found])
         ret_val = ReturnValueConcept("Test", False, error)
 
-        errors_found = sheerka.get_errors(ret_val, __type="ValueNotFound", item="an_item", value="a value")
+        errors_found = sheerka.get_errors(context, ret_val, __type="ValueNotFound", item="an_item", value="a value")
         assert errors_found == [value_not_found]
 
     def test_i_cannot_get_error_when_return_value_s_status_is_true(self):
         sheerka, context = self.init_test().unpack()
 
         ret_val = ReturnValueConcept("Test", True, sheerka.err("an error"))
-        assert sheerka.get_errors(ret_val) == []
+        assert sheerka.get_errors(context, ret_val) == []
 
 
 class TestSheerkaUsingFileBasedSheerka(TestUsingFileBasedSheerka):

tests/core/test_utils.py

@@ -469,3 +469,28 @@ def test_tokens_are_matching_when_eof_differs():
     tokens2 = Tokenizer(expression2, yield_eof=False)
 
     assert core.utils.tokens_are_matching(tokens1, tokens2)
+
+
+def test_sheerka_hasattr_get_attr():
+    class A:
+        def __init__(self, property_value):
+            self.property_value = property_value
+
+        def as_bag(self):
+            return {"prop": self.property_value}
+
+    # test object with bag
+    a = A("foo")
+    assert core.utils.sheerka_hasattr(a, "prop")
+    assert core.utils.sheerka_getattr(a, "prop") == "foo"
+    assert not core.utils.sheerka_hasattr(a, "property_value")
+    with pytest.raises(AttributeError):
+        core.utils.sheerka_getattr(a, "property_value")
+
+    # test for concept
+    concept = Concept("foo").def_var("a", "value").auto_init()
+    assert core.utils.sheerka_hasattr(concept, "a")
+    assert core.utils.sheerka_getattr(concept, "a") == "value"
+    assert not core.utils.sheerka_hasattr(concept, "b")
+    with pytest.raises(AttributeError):
+        core.utils.sheerka_getattr(concept, "b")

tests/evaluators/test_PythonEvaluator.py

@@ -2,13 +2,14 @@ import ast
 
 import pytest
 
+from core.ast_helpers import NamesWithAttributesVisitor
 from core.builtin_concepts import ReturnValueConcept, ParserResultConcept, BuiltinConcepts
 from core.builtin_helpers import CreateObjectIdentifiers
 from core.concept import Concept
 from core.sheerka.services.SheerkaConceptManager import SheerkaConceptManager
 from core.sheerka.services.SheerkaExecute import ParserInput
 from core.tokenizer import Tokenizer
-from evaluators.PythonEvaluator import PythonEvaluator, PythonEvalError, NamesWithAttributesVisitor
+from evaluators.PythonEvaluator import PythonEvaluator, PythonEvalError
 from parsers.BaseNodeParser import SourceCodeNode, SourceCodeWithConceptNode
 from parsers.FunctionParser import FunctionParser
 from parsers.PythonParser import PythonNode, PythonParser
@@ -386,6 +387,16 @@ class TestPythonEvaluator(TestUsingMemoryBasedSheerka):
         assert evaluated.status
         assert evaluated.value == context.sheerka.get_rule_by_id(str(value)).name
 
+    def test_i_cannot_call_methods_with_side_effect_when_is_question_is_set(self):
+        sheerka, context, foo, bar = self.init_concepts("foo", "bar")
+        context.add_to_protected_hints(BuiltinConcepts.EVAL_QUESTION_REQUESTED)
+
+        parsed = PythonParser().parse(context, ParserInput("set_isa(foo, bar)"))
+        python_evaluator = PythonEvaluator()
+        evaluated = python_evaluator.eval(context, parsed)
+
+        assert sheerka.has_error(context, evaluated, __type="MethodAccessError")
+
     def test_something(self):
         def func(**kwargs):
             for k, v in kwargs.items():

tests/non_reg/test_sheerka_non_reg.py

@@ -7,6 +7,7 @@ from core.sheerka.services.SheerkaConceptManager import SheerkaConceptManager
 from evaluators.MutipleSameSuccessEvaluator import MultipleSameSuccessEvaluator
 from evaluators.OneSuccessEvaluator import OneSuccessEvaluator
 from evaluators.PythonEvaluator import PythonEvalError
+from sheerkapython.python_wrapper import MethodAccessError
 from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
 from tests.parsers.parsers_utils import CMV, CC, compare_with_test_object, CB
 
@@ -642,7 +643,7 @@ as:
         assert res[0].body == 21
 
     def test_i_can_use_where_in_bnf(self):
-        sheerka = self.get_sheerka()
+        sheerka, context = self.init_test().unpack()
 
         init = [
             "def concept one as 1",
@@ -683,12 +684,12 @@ as:
         assert len(res) == 1
         assert not res[0].status
         assert sheerka.isinstance(res[0].body, BuiltinConcepts.MULTIPLE_ERRORS)
-        assert str(BuiltinConcepts.CONDITION_FAILED) in [error.key for error in sheerka.get_errors(res[0].body.body)]
+        assert sheerka.has_error(context, res, __type=BuiltinConcepts.CONDITION_FAILED)
 
         res = sheerka.evaluate_user_input("eval twenty three")
         assert len(res) == 1
         assert not res[0].status
-        assert str(BuiltinConcepts.CONDITION_FAILED) in [error.key for error in sheerka.get_errors(res[0].body.body)]
+        assert sheerka.has_error(context, res, __type=BuiltinConcepts.CONDITION_FAILED)
 
     def test_i_can_manage_some_type_of_infinite_recursion(self):
         sheerka = self.get_sheerka()
@@ -1298,6 +1299,7 @@ as:
         assert sheerka.objvalue(res[0].body.get_value("qty")) == 2
 
     def test_i_can_implement_the_concept_and(self):
+        # Normally, redefining and leads to a circular ref between the concept and the python
         init = [
             "def concept x and y as x and y",
             "set_is_lesser(__PRECEDENCE, c:x and y:, 'Sya')",
@@ -1308,3 +1310,55 @@ as:
 
         assert len(res) == 1
         assert res[0].status
+
+    def test_i_can_use_result_from_memory_filtering(self):
+        init = [
+            "def concept female",
+            "def concept girl",
+            "set_isa(girl, female)",
+            "def concept she ret memory('isa(self, female)')",
+            "girl"
+        ]
+        sheerka = self.init_scenario(init)
+        context = self.get_context(sheerka)
+
+        res = sheerka.evaluate_user_input("set_attr(she, 'my_attr', 'my value')")
+
+        assert len(res) == 1
+        assert res[0].status
+
+        girl_from_memory = sheerka.get_last_from_memory(context, "girl")
+        assert girl_from_memory.obj.get_value("my_attr") == "my value"
+
+    def test_i_can_use_result_from_memory_filtering_within_other_concept(self):
+        init = [
+            "def concept female",
+            "def concept girl",
+            "set_isa(girl, female)",
+            "def concept she ret memory('isa(self, female)')",
+            "def concept x attribute y equals z as set_attr(x, y, z)",
+            "girl"
+        ]
+        sheerka = self.init_scenario(init)
+        context = self.get_context(sheerka)
+
+        res = sheerka.evaluate_user_input("eval she attribute 'my_attr' equals 'my value'")
+
+        assert len(res) == 1
+        assert res[0].status
+
+        girl_from_memory = sheerka.get_last_from_memory(context, "girl")
+        assert girl_from_memory.obj.get_value("my_attr") == "my value"
+
+    def test_i_cannot_use_method_that_alter_the_global_state_within_question(self):
+        init = [
+            "def concept foo as question(set_debug(True))",
+        ]
+        sheerka = self.init_scenario(init)
+        context = self.get_context(sheerka)
+
+        res = sheerka.evaluate_user_input("question(set_debug(True))")
+        assert sheerka.has_error(context, res, __type="MethodAccessError")
+
+        res = sheerka.evaluate_user_input("eval foo")
+        assert sheerka.has_error(context, res, __type="MethodAccessError")

tests/non_reg/test_sheerka_non_reg_pipe_functions.py

@@ -0,0 +1,85 @@
+from core.builtin_concepts_ids import BuiltinConcepts
+from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
+
+
+class TestSheerkaNonRegPipeFunctions(TestUsingMemoryBasedSheerka):
+    def test_i_can_filter_a_list_using_pipe(self):
+        init = [
+            "def concept one as 1",
+            "def concept two as 2",
+            "def concept three as 3",
+            "add_to_memory('x', [one, two, three])"
+        ]
+
+        sheerka = self.init_scenario(init)
+        res = sheerka.evaluate_user_input("x | where(body=2)")
+
+        assert len(res) == 1
+        assert res[0].status
+        assert isinstance(res[0].body, list)
+        assert len(res[0].body) == 1
+        assert sheerka.isinstance(res[0].body[0], "two")
+
+        res = sheerka.evaluate_user_input("x | where(__self=two)")
+        assert len(res) == 1
+        assert res[0].status
+        assert isinstance(res[0].body, list)
+        assert len(res[0].body) == 1
+        assert sheerka.isinstance(res[0].body[0], "two")
+
+    def test_i_can_filter_using_sheerka_methods(self):
+        init = [
+            "def concept one as 1",
+            "def concept number",
+            "set_isa(one, number)",
+            "add_to_memory('x', [one])"
+        ]
+
+        sheerka = self.init_scenario(init)
+        res = sheerka.evaluate_user_input("x | where('isa(self, number)')")
+
+        assert len(res) == 1
+        assert res[0].status
+        assert isinstance(res[0].body, list)
+        assert len(res[0].body) == 1
+        assert sheerka.isinstance(res[0].body[0], "one")
+
+    def test_i_can_select_properties(self):
+        init = [
+            "def concept one as 1",
+            "def concept two as 2",
+            "def concept three as 3",
+            "add_to_memory('x', [one, two, three])"
+        ]
+
+        sheerka = self.init_scenario(init)
+
+        res = sheerka.evaluate_user_input("x | select('id', 'name')")
+        assert len(res) == 1
+        assert res[0].status
+        assert res[0].body == (("1001", "one"), ("1002", "two"), ("1003", "three"))
+
+        res = sheerka.evaluate_user_input("x | select(p1='id', p2='name')")
+        assert len(res) == 1
+        assert res[0].status
+        assert res[0].body == ({"p1": "1001", "p2": "one"},
+                               {"p1": "1002", "p2": "two"},
+                               {"p1": "1003", "p2": "three"})
+
+    def test_i_can_collect_properties(self):
+        init = [
+            "def concept one as 1",
+            "def concept isa from x is a y def_var x def_var y",
+            "def concept plus from a plus b as a + b",
+            "add_to_memory('x', [one, isa, plus])"
+        ]
+
+        sheerka = self.init_scenario(init)
+        res = sheerka.evaluate_user_input("x | props()")
+
+        assert len(res) == 1
+        assert res[0].status
+        assert sheerka.isinstance(res[0].body, BuiltinConcepts.TO_DICT)
+        assert res[0].body.body == {'isa': ['body', 'id', 'key', 'name', 'x', 'y'],
+                                    'one': ['body', 'id', 'key', 'name'],
+                                    'plus': ['a', 'b', 'body', 'id', 'key', 'name']}

tests/non_reg/test_sheerka_non_reg_rules.py

@@ -5,7 +5,7 @@ from evaluators.PythonEvaluator import PythonEvalError
 from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
 
 
-class TestSheerkaNonRegDisplay(TestUsingMemoryBasedSheerka):
+class TestSheerkaNonRegRules(TestUsingMemoryBasedSheerka):
 
     @pytest.mark.skip
     def test_i_can_apply_simple_rule(self):

tests/parsers/parsers_utils.py

@@ -16,6 +16,7 @@ from parsers.BaseNodeParser import UnrecognizedTokensNode, SourceCodeNode, RuleN
 from parsers.FunctionParser import FunctionNode
 from parsers.PythonParser import PythonNode
 from parsers.SyaNodeParser import SyaConceptParserHelper
+from sheerkapython.python_wrapper import sheerka_globals
 from sheerkarete.common import V
 from sheerkarete.conditions import Condition, AndConditions, NegatedCondition, NegatedConjunctiveConditions
 
@@ -934,11 +935,18 @@ class FN:
 
 @dataclass()
 class NEGCOND:
+    """
+    Represents a NegatedCondition
+    """
     condition: str
 
 
 @dataclass()
 class NCCOND:
+    """
+    Represents a NegatedConjunctiveConditions
+    """
+
     conditions: List[str]
 
 
@@ -1319,11 +1327,8 @@ def get_rete_conditions(*conditions):
     def get_value(obj):
         if obj.startswith("#"):
             return V(obj[1:])
-        if obj.startswith("'"):
-            return obj[1:-1]
-        if obj in ("True", "False"):
-            return obj == "True"
-        return int(obj)
+
+        return eval(obj, sheerka_globals)
 
     res = []
     for cond in conditions:

tests/parsers/test_DefConceptParser.py

@@ -671,7 +671,7 @@ from give me the date !
 
         assert not res.status
         assert sheerka.isinstance(res.value, BuiltinConcepts.ERROR)
-        assert sheerka.has_error(res, __type="SyntaxErrorNode", message="Empty 'auto_eval' declaration.")
+        assert sheerka.has_error(context, res, __type="SyntaxErrorNode", message="Empty 'auto_eval' declaration.")
 
     def test_i_cannot_parse_when_wrong_auto_eval_value(self):
         sheerka, context, parser, *concepts = self.init_parser()
@@ -709,4 +709,4 @@ from give me the date !
 
         assert not res.status
         assert sheerka.isinstance(res.value, BuiltinConcepts.ERROR)
-        assert sheerka.has_error(res, __type="SyntaxErrorNode", message="Empty 'def_var' declaration.")
+        assert sheerka.has_error(context, res, __type="SyntaxErrorNode", message="Empty 'def_var' declaration.")

tests/sheerkapython/test_python_wrapper.py

@@ -0,0 +1,169 @@
+import pytest
+
+from core.builtin_concepts_ids import BuiltinConcepts
+from core.concept import Concept
+from core.global_symbols import SyaAssociativity
+from core.sheerka.ExecutionContext import ExecutionContext
+from core.sheerka.services.SheerkaAdmin import SheerkaAdmin
+from sheerkapython.python_wrapper import Expando, create_namespace, inject_context, get_sheerka_method, Pipe, \
+    MethodAccessError
+from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
+
+
+class TestPythonWrapper(TestUsingMemoryBasedSheerka):
+
+    @pytest.mark.parametrize("name, expected", [
+        ("Concept", Concept),
+        ("BuiltinConcepts", BuiltinConcepts),
+        ("Expando", Expando),
+        ("ExecutionContext", ExecutionContext),
+        ("SyaAssociativity", SyaAssociativity),
+    ])
+    def test_i_can_create_namespace_from_internal_references(self, name, expected):
+        context = self.get_context()
+
+        assert create_namespace(context, "TestPythonWrapper", [name], None, {}, False) == {name: expected}
+
+    def test_i_can_create_namespace_with_context_method(self):
+        context = self.get_context()
+
+        res = create_namespace(context, "TestPythonWrapper", ["in_context", "isinstance"], None, {}, False)
+        assert res["in_context"] == context.in_context
+        assert res["isinstance"] == context.sheerka.services[SheerkaAdmin.NAME].extended_isinstance
+
+    def test_i_can_create_namespace_when_sheerka_expando_object(self):
+        sheerka, context = self.init_test().unpack()
+
+        res = create_namespace(context, "TestPythonWrapper", ["sheerka"], set(), {}, False)
+        assert res["sheerka"] == Expando("sheerka", {})
+
+        res = create_namespace(context, "TestPythonWrapper", ["sheerka"], {"test", "set_debug"}, {}, False)
+        assert isinstance(res["sheerka"], Expando)
+        assert res["sheerka"].get_name() == "sheerka"
+        assert set(vars(res["sheerka"])) == {"_Expando__name", "test", "set_debug"}
+
+    def test_i_can_create_namespace_when_short_term_memory(self):
+        context = self.get_context()
+        context.add_to_short_term_memory("my_key", "my_value")
+
+        assert create_namespace(context, "TestPythonWrapper", ["my_key"], None, {}, False) == {"my_key": "my_value"}
+
+    def test_i_can_create_namespace_when_value_from_memory(self):
+        sheerka, context = self.init_test().unpack()
+
+        sheerka.add_to_memory(context, "my_key", "my_value")
+        assert create_namespace(context, "TestPythonWrapper", ["my_key"], None, {}, False) == {"my_key": "my_value"}
+
+    def test_i_can_create_namespace_when_sheerka_methods(self):
+        sheerka, context = self.init_test().unpack()
+        res = create_namespace(context, "TestPythonWrapper", ["test", "set_debug"], None, {}, False)
+
+        assert res["test"] == sheerka.test
+        assert isinstance(res["set_debug"], type(inject_context))
+        assert res["set_debug"].__name__ == "set_debug"
+
+    def test_i_can_create_namespace_when_value_from_context_obj(self):
+        sheerka, context = self.init_test().unpack()
+        obj = Concept("foo").def_var("a", "value1").auto_init()
+        context.obj = obj
+
+        res = create_namespace(context, "TestPythonWrapper", ["self", "a"], None, {}, False)
+        assert res == {"self": obj, "a": "value1"}
+
+    def test_i_can_create_namespace_when_value_from_local_objects(self):
+        sheerka, context = self.init_test().unpack()
+        obj = Concept("foo")
+        objects = {"self": obj, "a": Concept("bar")}
+
+        res = create_namespace(context, "TestPythonWrapper", ["self", "a"], None, objects, False)
+        assert res == {"self": obj, "a": objects["a"]}
+
+    def test_i_can_create_namespace_when_name_refers_to_a_concept(self):
+        sheerka, context, foo = self.init_concepts("foo")
+
+        assert create_namespace(context, "TestPythonWrapper", ["foo"], None, {}, False) == {"foo": foo}
+
+    def test_internal_references_and_context_method_take_over_short_term_memory(self):
+        context = self.get_context()
+        context.add_to_short_term_memory("Concept", "short_term_value")
+        context.add_to_short_term_memory("isinstance", "short_term_value")
+        context.add_to_short_term_memory("in_context", "short_term_value")
+
+        res = create_namespace(context, "TestPythonWrapper", ["Concept", "isinstance", "in_context"], None, {}, False)
+
+        assert res == {
+            "Concept": Concept,
+            "isinstance": context.sheerka.services[SheerkaAdmin.NAME].extended_isinstance,
+            "in_context": context.in_context,
+        }
+
+    def test_short_term_memory_takes_precedence_over_long_term_memory(self):
+        sheerka, context = self.init_test().unpack()
+
+        context.add_to_short_term_memory("my_key", "short_term")
+        sheerka.add_to_memory(context, "my_key", "long_term")
+
+        assert create_namespace(context, "TestPythonWrapper", ["my_key"], None, {}, False) == {"my_key": "short_term"}
+
+    def test_long_term_memory_takes_precedence_over_sheerka_methods(self):
+        # I am not really sure why
+        sheerka, context = self.init_test().unpack()
+
+        sheerka.add_to_memory(context, "test", "from memory")
+        assert create_namespace(context, "TestPythonWrapper", ["test"], None, {}, False) == {"test": "from memory"}
+
+    def test_sheerka_method_takes_precedence_over_context_obj(self):
+        sheerka, context = self.init_test().unpack()
+        obj = Concept("foo").def_var("test", "value1").auto_init()
+        context.obj = obj
+
+        assert create_namespace(context, "TestPythonWrapper", ["test", ], None, {}, False) == {"test": sheerka.test}
+
+    def test_context_obj_takes_precedence_over_local_objects(self):
+        sheerka, context = self.init_test().unpack()
+        obj = Concept("foo").def_var("a", "value1").auto_init()
+        context.obj = obj
+        objects = {"self": Concept("bar"), "a": Concept("bar")}
+
+        res = create_namespace(context, "TestPythonWrapper", ["self", "a"], None, objects, False)
+        assert res == {"self": obj, "a": "value1"}
+
+    def test_local_objects_take_precedence_over_object_instantiation(self):
+        sheerka, context, foo = self.init_concepts("from instantiation")
+        objects = {"foo": Concept("from local")}
+
+        res = create_namespace(context, "TestPythonWrapper", ["foo"], None, objects, False)
+        assert res == {"foo": objects["foo"]}
+
+    def test_i_can_get_sheerka_method(self):
+        context = self.get_context()
+
+        # sheerka direct method
+        assert get_sheerka_method(context, "TestPythonWrapper", "test", True) == context.sheerka.test
+
+        # sheerka indirect method
+        assert get_sheerka_method(context, "TestPythonWrapper", "get_value", True) == context.sheerka.get_value
+
+        # method that need context are wrapped
+        res = get_sheerka_method(context, "TestPythonWrapper", "test_using_context", True)
+        assert res != context.sheerka.test_using_context
+        assert type(res) == type(inject_context)
+        assert res.__name__ == "test_using_context"
+
+        # return None when the method is not found
+        assert get_sheerka_method(context, "TestPythonWrapper", "xxx", True) is None
+
+    def test_i_cannot_get_method_that_modifies_the_state_when_expression_only(self):
+        sheerka, context = self.init_test().unpack()
+
+        assert get_sheerka_method(context, "TestPythonWrapper", "set_debug", expression_only=False) is not None
+
+        with pytest.raises(MethodAccessError) as ex:
+            get_sheerka_method(context, "TestPythonWrapper", "set_debug", expression_only=True)
+        assert ex.value.method_name == "set_debug"
+
+    def test_i_can_get_method_when_pipe_function(self):
+        context = self.get_context()
+
+        res = get_sheerka_method(context, "TestPythonWrapper", "where", True)
+        assert isinstance(res, Pipe)

tests/sheerkaql/test_SheerkaQueryLanguage.py

@@ -0,0 +1,570 @@
+from dataclasses import dataclass
+
+import pytest
+
+from sheerkaql.SheerkaQueryLangage import SheerkaQueryLanguage
+from sheerkaql.symbols import flwr_sequence, attribute_value
+
+
+def oset(x):
+    return x
+
+
+class A(object):
+    def __init__(self, q):
+        self.q = q
+
+    def __repr__(self):
+        return f"A({vars(self)})"
+
+    def __eq__(self, other):
+        if not isinstance(other, A):
+            return False
+
+        return self.q == other.q
+
+    def __hash__(self):
+        return hash(str(self.q))
+
+
+@dataclass
+class BagClass:
+    property1: object
+    property2: object
+
+    def as_bag(self):
+        return {
+            "prop1": self.property1,
+            "prop2": self.property2,
+        }
+
+
+execute = SheerkaQueryLanguage().execute
+
+
+class TestSheerkaQueryLanguage:
+
+    def test_i_can_get_the_root_of_a_query(self):
+        hello = 'hello world!'
+        q = SheerkaQueryLanguage().compile('hello')
+
+        assert q(locals()) == oset([hello])
+
+    def test_i_can_traverse_object(self):
+        a = A(A(A("hello world!")))
+        q = SheerkaQueryLanguage().compile("a.q.q.q")
+
+        assert q(locals()) == oset(["hello world!"])
+
+    def test_i_can_traverse_list(self):
+        lst = [A("one"), A("two"), A("three")]
+        a = A(lst)
+
+        assert execute("a.q.q", {"a": a}) == oset(["one", "two", "three"])
+
+    def test_i_can_traverse_list_of_list(self):
+        sub_lst_number = [A("1"), A("2"), A("2")]
+        sub_lst_letter = [A("a"), A("b"), A("c")]
+        lst = [A("one"), A(sub_lst_number), A(sub_lst_letter)]
+        a = A(lst)
+
+        res = execute("a.q.q", {"a": a})
+        assert res == oset(["one", *sub_lst_number, *sub_lst_letter])
+
+    def test_i_can_traverse_object_when_where_condition_is_a_boolean(self):
+        a = A(A(A("hello world!")))
+        b_true = A(A(True))
+        b_false = A(A(False))
+        namespace = {"a": a, "hasattr": hasattr, "func": lambda x: x, "b_true": b_true, "b_false": b_false}
+
+        assert execute("a.q[1 == 1].q.q", namespace) == oset(["hello world!"])
+        assert execute("a.q[hasattr(self, 'q')].q.q", namespace) == oset(["hello world!"])
+
+        assert execute("a.q[1 == 2].q.q", namespace) == oset([])
+        assert execute("a.q[hasattr(self, 'x')].q.q", namespace) == oset([])
+
+        assert execute("a.q[True].q.q", namespace, allow_builtins=True) == oset(["hello world!"])
+        assert execute("a.q[False].q.q", namespace, allow_builtins=True) == oset([])
+
+        assert execute("a.q[func(True)].q.q", namespace) == oset(["hello world!"])
+        assert execute("a.q[func(False)].q.q", namespace) == oset([])
+
+        assert execute("a.q[b_true.q.q].q.q", namespace) == oset(["hello world!"])
+        assert execute("a.q[b_false.q.q].q.q", namespace) == oset([])
+
+    def test_i_can_request_by_list_index(self):
+        lst = [A("one"), A("two"), A("three")]
+        a = A(lst)
+
+        assert execute("a.q[1]", {"a": a}) == oset([lst[1]])
+        with pytest.raises(IndexError):
+            execute("a.q[99]", {"a": a})
+        with pytest.raises(TypeError):
+            execute("a.q['key']", {"a": a})
+
+    def test_i_can_request_by_dictionary_key(self):
+        lst = {"key1": "value1", "key2": "value2"}
+        a = A(lst)
+
+        assert execute("a.q['key1']", {"a": a}) == oset([lst["key1"]])
+        with pytest.raises(KeyError):
+            execute("a.q['key3']", {"a": a})
+
+    def test_i_can_filter(self):
+        l = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
+
+        assert execute('l[self < 5]', locals()) == [0, 1, 2, 3, 4]
+
+    def test_i_cannot_traverse_object_when_where_condition_is_not_a_boolean(self):
+        a = A("hello world!")
+        namespace = {"a": a, "func": lambda x: x, "dictionary": {"key": "value"}, "list": ["value"]}
+
+        with pytest.raises(TypeError):
+            execute("a.q[1].q.q", namespace)
+
+        with pytest.raises(TypeError):
+            execute("a.q[func(3)].q.q", namespace)
+
+        with pytest.raises(TypeError):
+            execute("a.q[dictionary['key']].q.q", namespace)
+
+        with pytest.raises(TypeError):
+            execute("a.q[list[0]].q.q", namespace)
+
+    def test_i_can_traverse_object_when_as_bag_is_defined(self):
+        a = BagClass(BagClass("sub_value1", BagClass("sub_sub_value1", "sub_sub_value1")), "value2")
+
+        assert execute("a.prop1.prop2.prop1", {"a": a}) == oset(["sub_sub_value1"])
+
+    def test_i_can_traverse_objects_when_where_condition_uses_as_bag(self):
+        hash_map = {"sub_sub_value1": "hello world!"}
+        b = BagClass(BagClass("sub_value1", BagClass("sub_sub_value1", "sub_sub_value1")), "value2")
+        a = A(hash_map)
+
+        assert execute("a.q[b.prop1.prop2.prop1]", {"a": a, "b": b}) == oset(["hello world!"])
+
+    def test_i_can_compute_set_operations(self):
+        a = [1, 2, 3, 3]
+        b = [2, 4, 3, 4]
+
+        assert execute("a | b", locals()) == [1, 2, 3, 4]
+        assert execute("a - b", locals()) == [1]
+        assert execute("b - a", locals()) == [4]
+        assert execute("a & b", locals()) == [2, 3]
+
+    def test_can_execute_set_expression(self):
+        a = [1, 2, 3, 3]
+        b = [2, 4, 3, 4]
+        c = [1, 2]
+
+        assert execute("return 1 if 1 in a else 0", locals()) == (1,)
+        assert execute("return 1 if 1 not in a else 0", locals()) == (0,)
+        assert execute("return 1 if 5 in a else 0", locals()) == (0,)
+        assert execute("return 1 if 5 not in a else 0", locals()) == (1,)
+
+        assert execute("return 1 if <c> subset <a> else 0", locals()) == (1,)
+        assert execute("return 1 if <c> superset <a> else 0", locals()) == (0,)
+
+        assert execute("return 1 if <a> subset <c> else 0", locals()) == (0,)
+        assert execute("return 1 if <a> superset <c> else 0", locals()) == (1,)
+
+        l = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
+
+        assert execute('l - l[self < 5]', locals()) == (5, 6, 7, 8, 9)
+
+    def test_i_can_traverse_a_complex_path(self):
+        answer = 'o.x.y'
+
+        o = A('top')
+        o.x = [A('asdf'), A('123')]
+        o.x[0].y = A(answer)
+        d = {'hasattr': hasattr, 'o': o}
+
+        assert execute('o/x[hasattr(self,"y")]/y/q', d) == oset([answer])
+        assert execute('o/x', d) == oset(o.x)
+
+    def test_i_can_execute_comparison_in_where_clauses(self):
+        a = A(5)
+        assert execute('a[self.q == 5]', locals()) == oset([a])
+        assert execute('a[self.q != 5]', locals()) == oset([])
+        assert execute('a[self.q >= 5]', locals()) == oset([a])
+        assert execute('a[self.q <= 5]', locals()) == oset([a])
+        assert execute('a[self.q > 5]', locals()) == oset([])
+        assert execute('a[self.q < 5]', locals()) == oset([])
+        assert execute('a[self.q == 7]', locals()) == oset([])
+        assert execute('a[self.q != 7]', locals()) == oset([a])
+        assert execute('a[self.q >= 7]', locals()) == oset([])
+        assert execute('a[self.q <= 7]', locals()) == oset([a])
+        assert execute('a[self.q > 7]', locals()) == oset([])
+        assert execute('a[self.q < 7]', locals()) == oset([a])
+        assert execute('a[self.q == 3]', locals()) == oset([])
+        assert execute('a[self.q != 3]', locals()) == oset([a])
+        assert execute('a[self.q >= 3]', locals()) == oset([a])
+        assert execute('a[self.q <= 3]', locals()) == oset([])
+        assert execute('a[self.q > 3]', locals()) == oset([a])
+        assert execute('a[self.q < 3]', locals()) == oset([])
+
+    def test_i_can_execute_boolean_expression_in_where_clauses(self):
+        a = 'hello'
+        true = True
+        false = False
+        assert execute('a[true]', locals()) == oset([a])
+        assert execute('a[false]', locals()) == oset([])
+        assert execute('a[not true]', locals()) == oset([])
+        assert execute('a[not false]', locals()) == oset([a])
+        assert execute('a[true and true]', locals()) == oset([a])
+        assert execute('a[false and true]', locals()) == oset([])
+        assert execute('a[not true and true]', locals()) == oset([])
+        assert execute('a[not false and true]', locals()) == oset([a])
+        assert execute('a[true or false]', locals()) == oset([a])
+        assert execute('a[true or true]', locals()) == oset([a])
+        assert execute('a[false or true]', locals()) == oset([a])
+        assert execute('a[false or false]', locals()) == oset([])
+        assert execute('a[not true or true]', locals()) == oset([a])
+        assert execute('a[not false or false]', locals()) == oset([a])
+        assert execute('a[true and true and true and true]', locals()) == oset([a])
+        assert execute('a[true and true and true and false]', locals()) == oset([])
+        assert execute('a[true and (false or true)]', locals()) == oset([a])
+        assert execute('a[true and (false and true)]', locals()) == oset([])
+        assert execute('a[true and (true and true)]', locals()) == oset([a])
+        assert execute('a[true and (true and (not true or false))]', locals()) == oset([])
+
+    def test_i_can_execute_comparison_in_where_clauses_using_builtin_functions(self):
+        a = 'hello'
+        true = True
+        false = False
+        d = locals()
+        assert execute('a[1 == 1]', d) == oset([a])
+        assert execute('a[-1 == -1]', d) == oset([a])
+        assert execute('a[2.2 == 2.2]', d) == oset([a])
+        assert execute('a[2.2 == float("2.2")]', d, True) == oset([a])
+        assert execute('a[2 == int(2.2)]', d, True) == oset([a])
+        assert execute('a["hello" == a]', d, True) == oset([a])
+        assert execute('a["HELLO" == a.upper()]', d, True) == oset([a])
+
+    def test_i_can_use_function_in_where_clauses(self):
+        a = 'hello'
+
+        def f():
+            return 'hello'
+
+        def g(x, y, z):
+            return x + y + z
+
+        def h(f, x):
+            return f(x)
+
+        def i(x):
+            return x ** 2
+
+        def j(f):
+            return f
+
+        true = True
+        false = False
+        d = locals()
+
+        assert execute('a[f() == "hello"]', d) == oset([a])
+        assert execute('a[g(1,2,3) == 6]', d) == oset([a])
+        assert execute('a[h(i,3) == 9]', d) == oset([a])
+        assert execute('a[i(j(j)(j)(j)(h)(i,3)) == 81]', d) == oset([a])
+        with pytest.raises(TypeError):
+            execute('a[f()]', d)
+
+    def test_i_can_use_lists_in_where_clauses(self):
+        a = 'hello'
+        l = [1, 2, 3, 4, 5, 6, 7, [1, 2, 3, 4, 5, 6, 7, [1, 2, 3, 4, 5, 6, 7, 8]]]
+        d = locals()
+
+        assert execute('a[l[0] == 1]', d) == oset([a])
+        assert execute('a[l[1] == 2]', d) == oset([a])
+        assert execute('a[l[7][0] == 1]', d) == oset([a])
+        assert execute('a[l[7][1] == 2]', d) == oset([a])
+        assert execute('a[l[7][7][0] == 1]', d) == oset([a])
+        assert execute('a[l[7][7][1] == 2]', d) == oset([a])
+        assert execute('a[l[7][7][7] == 8]', d) == oset([a])
+
+    def test_i_can_use_dicts_in_where_clauses(self):
+        a = 'hello'
+        l = {"one": 1, "two": 2, "next": {"one": 1, "two": 2, "next": {"one": 1, "two": 2}}}
+        d = locals()
+
+        assert execute('a[l["one"] == 1]', d) == oset([a])
+        assert execute('a[l["two"] == 2]', d) == oset([a])
+        assert execute('a[l["next"]["one"] == 1]', d) == oset([a])
+        assert execute('a[l["next"]["two"] == 2]', d) == oset([a])
+        assert execute('a[l["next"]["next"]["one"] == 1]', d) == oset([a])
+        assert execute('a[l["next"]["next"]["two"] == 2]', d) == oset([a])
+
+    def test_i_can_use_callable_in_where_clauses(self):
+        a = 'hello'
+
+        def f():
+            return 'hello'
+
+        def g(x, y, z):
+            return x + y + z
+
+        def h(f, x):
+            return f(x)
+
+        def i(x):
+            return x ** 2
+
+        def j(f):
+            return f
+
+        m = {"one": 1, "two": 2, "next": [1, 2, 3, 4, 5, 6, 7, j]}
+        d = locals()
+
+        assert execute('a[m["next"][7](j)(m["next"][7])(m["next"])[7](i)(m["two"]) == 4]', d) == oset([a])
+
+    def test_i_can_execute_flwr_expression(self):
+        def f():
+            return 1, 2, 3
+
+        d = locals()
+        assert execute('for x in f() return x', d) == (1, 2, 3)
+        assert execute('for x in f() let y = f() return x, y', d) == ((1, (1, 2, 3)), (2, (1, 2, 3)), (3, (1, 2, 3)))
+
+    def test_i_can_execute_flwr_with_order_by(self):
+        def f():
+            return [1, 3, 2]
+
+        d = locals()
+        with pytest.raises(SyntaxError):
+            execute('for x in f() order by "asdf" asc return x', d)
+        with pytest.raises(SyntaxError):
+            execute('for x in f() order by 0 asc return "asdf":x', d)
+        assert execute('for x in f() order by 0 asc return x', d) == (1, 2, 3)
+        assert execute('for x in f() order by 0 desc return x', d) == (3, 2, 1)
+
+    def test_i_can_execute_flwr_with_user_defined_functions(self):
+        a = 'hello'
+        l = [1, 2, 3, 4, 5, 6, 7, [1, 2, 3, 4, 5, 6, 7, [1, 2, 3, 4, 5, 6, 7, 8]]]
+        d = locals()
+
+        assert execute('''
+          for i in l
+          let f = function() { 125 }
+          return f()
+        ''', d) == (125, 125, 125, 125, 125, 125, 125, 125)
+
+        assert execute('''
+          for i in l
+            let f = function(q) {
+              for _ in <a>
+              where isinstance(q, list)
+              return {
+                for j in q
+                return f(j)
+              }
+            }
+          return f(i)
+        ''', d, True) == ((), (), (), (), (), (), (),
+                          (((), (), (), (), (), (), (),
+                            (((), (), (), (), (), (), (), ()),)),))
+
+    def test_i_can_execute_flwr_with_if_expression(self):
+        a = 'hello'
+        l = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
+        q = True
+        d = locals()
+
+        assert execute('''
+          for x in <a> return if (q) then 1 else 0
+          ''', d) == (1,)
+
+        assert execute('''
+          for x in <l> return if (x % 2 == 0) then 1 else 0
+          ''', d) == (0, 1, 0, 1, 0, 1, 0, 1, 0, 1)
+
+        assert execute('''
+          for x in <l> return if x % 2 == 0 then 1 else 0
+          ''', d) == (0, 1, 0, 1, 0, 1, 0, 1, 0, 1)
+
+        assert execute('''
+          for x in <l> return 1 if x % 2 == 0 else 0
+          ''', d) == (0, 1, 0, 1, 0, 1, 0, 1, 0, 1)
+
+        assert execute('''
+          for x in <l> return 1 if (x % 2 == 0) else 0
+          ''', d) == (0, 1, 0, 1, 0, 1, 0, 1, 0, 1)
+
+        assert execute('''
+            for x in <a> return if (True or X) then 1 else 0
+            ''', d, True) == (1,)
+
+    def test_if_short_circuit(self):
+        a = 'hello'
+        l = [1, 2, 3, 4, 5, 6, 7, [1, 2, 3, 4, 5, 6, 7, [1, 2, 3, 4, 5, 6, 7, 8]]]
+        d = locals()
+
+        assert execute('''
+            for x in <a> return if (True or X) then 1 else 0
+            ''', d, True) == (1,)
+
+        assert execute('''
+            for x in <a> return if (False and false.x) then 1 else 0
+            ''', d, True) == (0,)
+
+    def test_i_can_flatten_result(self):
+        a = 'hello'
+        l = [1, 2, 3, 4, 5, 6, 7, [1, 2, 3, 4, 5, 6, 7, [1, 2, 3, 4, 5, 6, 7, 8]]]
+        d = locals()
+
+        assert execute('''
+           for x in l return flatten x
+           ''', d, True) == (1, 2, 3, 4, 5, 6, 7, 1, 2, 3, 4, 5, 6, 7, 1, 2, 3, 4, 5, 6, 7, 8)
+
+    def test_flattened_return(self):
+        a = 'hello'
+        l = [1, 2, 3, 4, 5, 6, 7, [1, 2, 3, 4, 5, 6, 7, [1, 2, 3, 4, 5, 6, 7, 8]]]
+        d = locals()
+
+        assert execute('''
+            for i in l
+            let f = function(l) {
+              if (isinstance(l, list))
+              then {for j in l return f(j)}
+              else l
+            }
+            return flatten f(i)''', d, True) == (1, 2, 3, 4, 5, 6, 7, 1, 2, 3, 4, 5, 6, 7, 1, 2, 3, 4, 5, 6, 7, 8)
+
+        assert execute('''
+            for i in l
+            let f = function(l) {
+              if (isinstance(l, list))
+              then {for j in l return f(j)}
+              else {a:l}
+            }
+            return flatten f(i)
+            ''', d, True) == tuple({a: i} for i in (1, 2, 3, 4, 5, 6, 7, 1, 2, 3, 4, 5, 6, 7, 1, 2, 3, 4, 5, 6, 7, 8))
+
+    def test_i_can_return_none(self):
+        a = 'hello'
+        l = [1, 2, 3, 4, 5, 6, 7, [1, 2, 3, 4, 5, 6, 7, [1, 2, 3, 4, 5, 6, 7, 8]]]
+        d = locals()
+
+        assert execute('for x in l return None', d, True) == (None, None, None, None, None, None, None, None)
+
+    def test_i_can_return_a_class(self):
+        l = [1, 2, 3]
+        d = {"A": A, "l": l}
+
+        assert execute('for x in l return A(x)', d, True) == (A(1), A(2), A(3))
+
+    def test_i_can_execute_flwr_when_there_is_no_for_statement(self):
+        a = 'hello'
+        l = [1, 2, 3, 4, 5, 6, 7, [1, 2, 3, 4, 5, 6, 7, [1, 2, 3, 4, 5, 6, 7, 8]]]
+        d = locals()
+
+        flwr = flwr_sequence([attribute_value('hello', scalar=True)])
+        assert flwr(d) == ('hello',)
+        assert execute("return l", d) == (l,)
+        assert execute('''
+            let f = function(l) {
+              if (isinstance(l, list))
+              then {for j in l return f(j)}
+              else l
+            }
+            return flatten f(l)
+          ''', d, True) == (1, 2, 3, 4, 5, 6, 7, 1, 2, 3, 4, 5, 6, 7, 1, 2, 3, 4, 5, 6, 7, 8)
+
+    def test_can_collect(self):
+        l = [1, 2, 3, 4, 5, 6, 7, 3, 4, 5, 6, 7, 3, 4]
+        d = locals()
+
+        assert execute('''
+            for n in l
+            collect n as n with function(prev, next) {
+                if prev == None then 1 else prev + 1
+            }
+          ''', d, True) == {1: 1, 2: 1, 3: 3, 4: 3, 5: 2, 6: 2, 7: 2}
+
+        assert execute('''
+            for n in [1,2,3,4,5,6,7,3,4,5,6,7,3,4]
+            collect n as n with function(prev, next) {
+                if prev == None then 1 else prev + 1
+             }
+          ''', d, True) == {1: 1, 2: 1, 3: 3, 4: 3, 5: 2, 6: 2, 7: 2}
+
+    def test_i_can_perform_calculations(self):
+        d = locals()
+
+        assert execute('''
+            for n in [
+                4.0*3.0/2.0,
+                4.0/3.0*2.0,
+                (3.0+9.0)*4.0/8.0,
+                ((9.0-3.0)+(5.0-3.0))/2.0 + 2.0,
+                5.0 * 4.0 / 2.0 - 10.0 + 5.0 - 2.0 + 3.0,
+                5.0 / 4.0 * 2.0 + 10.0 - 5.0 * 2.0 / 3.0
+            ]
+            return n
+          ''', d, True) == (
+            4.0 * 3.0 / 2.0,
+            4.0 / 3.0 * 2.0,
+            (3.0 + 9.0) * 4.0 / 8.0,
+            ((9.0 - 3.0) + (5.0 - 3.0)) / 2.0 + 2.0,
+            5.0 * 4.0 / 2.0 - 10.0 + 5.0 - 2.0 + 3.0,
+            5.0 / 4.0 * 2.0 + 10.0 - 5.0 * 2.0 / 3.0
+        )
+
+    def test_i_can_execute_flwr_with_in_and_not_in(self):
+        l = [[1, 2, 3], [4, 5, 6], [7, 4, 3], [5, 6, 7], [3, 4]]
+        d = locals()
+
+        assert execute('''
+            for n in l
+            where 4 in n
+            return n
+          ''', d, True) == ([4, 5, 6], [7, 4, 3], [3, 4])
+
+        assert execute('''
+            for n in l
+            where 4 not in n
+            return n
+          ''', d, True) == ([1, 2, 3], [5, 6, 7])
+
+    def test_multi_collect(self):
+        l = [1, 2, 3, 4, 5, 6, 7, 3, 4, 5, 6, 7, 3, 4]
+        d = locals()
+
+        assert execute('''
+            for n in l
+            let counter = function(prev, next) {
+                if prev == None then 1 else prev + 1
+            }
+            where 1 in l and 12 not in l
+            collect n as n with counter
+            collect n as (int(n)/int(2)) with counter
+          ''', d, True), (
+            {1: 1, 2: 1, 3: 3, 4: 3, 5: 2, 6: 2, 7: 2},
+            {0: 1, 1: 4, 2: 5, 3: 4})
+
+    def test_i_can_execute_flwr_on_objects_attributes(self):
+        lst = [A(1), A(2), A(3)]
+        d = locals()
+
+        assert execute('for x in <lst.q> return x', d) == (1, 2, 3)
+        assert execute('for x in lst return x.q', d) == (1, 2, 3)  # another way
+
+    def test_exception_during_return_are_caught(self):
+        lst = [A(1), A([2])]
+        d = locals()
+
+        res = execute('for x in lst return x.q + [2]', d)
+        assert len(res) == 2
+        assert isinstance(res[0], TypeError)
+        assert res[1] == [2,2]
+
+    def test_i_cannot_execute_flwr_on_attributes_that_does_not_exist(self):
+        lst = [A(1), A(2), A(3)]
+        d = locals()
+
+        with pytest.raises(AttributeError):
+            execute('for x in <lst.x> return x', d)
+
+        with pytest.raises(AttributeError):
+            execute('for x in lst.q return x', d)  # AttributeError: 'list' object has no attribute 'q'

tests/sheerkaql/test_lexer.py

@@ -0,0 +1,122 @@
+from contextlib import contextmanager
+
+from ply import lex
+
+from sheerkaql import lexer
+
+
+def compare(a, b):
+    return (a.type == b.type and a.value == b.value and
+            a.lexpos == b.lexpos and a.lineno == b.lineno)
+
+
+def token(token_type, value, pos, line):
+    t = lex.LexToken()
+    t.type = token_type
+    t.value = value
+    t.lexpos = pos
+    t.lineno = line
+    return t
+
+
+@contextmanager
+def comparable_tokens():
+    eq = lex.LexToken.__eq__
+    ne = lex.LexToken.__ne__
+    setattr(lex.LexToken, "__eq__", compare)
+    setattr(lex.LexToken, "__ne__", lambda a, b: not compare(a, b))
+    yield
+    setattr(lex.LexToken, "__eq__", eq)
+    setattr(lex.LexToken, "__ne__", ne)
+
+
+class TestSheerkaQueryLanguageLexer:
+    def test_context_manager(self):
+        t1 = token("NAME", 'a', 0, 1)
+        t2 = token("NAME", 'a', 0, 1)
+        assert t1 != t2
+        with comparable_tokens():
+            assert t1 == t2
+
+    def test_NAME(self):
+        clex = lexer.Lexer()
+        clex.input('a 9a')
+        tokens = [token("NAME", 'a', 0, 1),
+                  token("NUMBER", 9, 2, 1),
+                  token("NAME", 'a', 3, 1)]
+        with comparable_tokens():
+            for t in tokens:
+                assert next(clex) == t
+
+    def test_STRING(self):
+        clex = lexer.Lexer()
+        clex.input("'asdf' \"asdf\" '\n'")
+        tokens = [token("STRING", 'asdf', 0, 1),
+                  token("STRING", 'asdf', 7, 1),
+                  token("STRING", '\n', 14, 1), ]
+        with comparable_tokens():
+            for t in tokens:
+                assert next(clex) == t
+
+    def test_HEX(self):
+        clex = lexer.Lexer()
+        clex.input("0xab 0xab")
+        tokens = [token("NUMBER", 0xab, 0, 1),
+                  token("NUMBER", 171, 5, 1)]
+        with comparable_tokens():
+            for t in tokens:
+                assert next(clex) == t
+
+    def test_FLOAT(self):
+        clex = lexer.Lexer()
+        clex.input("1.2 .2 2.3e4 .2 2.3e4")
+        tokens = [token("NUMBER", 1.2, 0, 1), token("NUMBER", .2, 4, 1),
+                  token("NUMBER", 2.3e4, 7, 1), token("NUMBER", .2, 13, 1),
+                  token("NUMBER", 2.3e4, 16, 1)]
+        with comparable_tokens():
+            for t in tokens:
+                assert next(clex) == t
+
+    def test_OCT(self):
+        clex = lexer.Lexer()
+        clex.input("073 073 073")
+        tokens = [token("NUMBER", 0o73, 0, 1),
+                  token("NUMBER", 59, 4, 1),
+                  token("NUMBER", 59, 8, 1)]
+        with comparable_tokens():
+            for t in tokens:
+                assert next(clex) == t
+
+    def test_INTEGER(self):
+        clex = lexer.Lexer()
+        clex.input("73 730 7")
+        tokens = [token("NUMBER", 73, 0, 1),
+                  token("NUMBER", 730, 3, 1),
+                  token("NUMBER", 7, 7, 1)]
+        with comparable_tokens():
+            for t in tokens:
+                assert next(clex) == t
+
+    def test_KEYWORDS(self):
+        for value, token_type in list(lexer.reserved.items()):
+            clex = lexer.Lexer()
+            clex.input(value)
+            tokens = [token(token_type, value, 0, 1)]
+            with comparable_tokens():
+                for t in tokens:
+                    assert next(clex) == t
+
+    def test_chrs(self):
+        for token_type, value in [(attr[2:], getattr(lexer.Lexer, attr))
+                                  for attr in dir(lexer.Lexer)
+                                  if attr[:2] == 't_' and
+                                     isinstance(getattr(lexer.Lexer, attr), str) and
+                                     attr[2:] != 'ignore']:
+            if value[0] == '\\':
+                value = value[1:]
+            clex = lexer.Lexer()
+            clex.input(value)
+            tokens = [token(token_type, value, 0, 1)]
+            with comparable_tokens():
+                for t in tokens:
+                    assert next(clex) == t

tests/sheerkaql/test_parser.py

@@ -0,0 +1,297 @@
+import pytest
+
+from sheerkaql.SheerkaQueryLangage import SheerkaQueryLanguage
+from sheerkaql.lexer import Lexer
+from sheerkaql.parser import Parser
+
+
+class TestSheerkaQueryLanguageParser:
+    def test_hello(self):
+        SheerkaQueryLanguage().compile('hello')
+        with pytest.raises(SyntaxError) as ex:
+            SheerkaQueryLanguage().compile('hello there')
+
+    def test_i_can_parse_with_slash(self):
+        SheerkaQueryLanguage().compile('hello/part1')
+        SheerkaQueryLanguage().compile('hello/part1/part2/part3')
+        SheerkaQueryLanguage().compile('hello/part1 / part2 / part3')
+        with pytest.raises(SyntaxError):
+            SheerkaQueryLanguage().compile('hello/part1/part3 part4/part5')
+
+    def test_i_can_parse_with_dot(self):
+        SheerkaQueryLanguage().compile('hello.part1')
+        SheerkaQueryLanguage().compile('hello.part1.part2.part3')
+        SheerkaQueryLanguage().compile('hello.part1 . part2 . part3')
+        with pytest.raises(SyntaxError):
+            SheerkaQueryLanguage().compile('hello.part1.part3 part4.part5')
+
+    def test_i_can_parse_simple_where_conditions(self):
+        SheerkaQueryLanguage().compile('hello[wheRe]/hello[asdf]')
+        SheerkaQueryLanguage().compile('hello/hello[asdf]/asdf/wef')
+        SheerkaQueryLanguage().compile('hello/hello/wewe[asdf]/wef/waef/awef/weaf')
+        SheerkaQueryLanguage().compile('hello.hello[ asdf ] . wewe. wef[asdf] .waef .awef[asdf].weaf')
+        SheerkaQueryLanguage().compile('hello["foo"]')
+        SheerkaQueryLanguage().compile('hello[123]')
+        SheerkaQueryLanguage().compile('hello[123.234]')
+        with pytest.raises(SyntaxError):
+            SheerkaQueryLanguage().compile('hello/aef[asdf] hello[adsf]')
+
+    def test_i_can_parse_where_conditions_with_comparisons(self):
+        SheerkaQueryLanguage().compile('hello[1 == 1]')
+        SheerkaQueryLanguage().compile('hello[1 != 1]')
+        SheerkaQueryLanguage().compile('hello[1 < 1]')
+        SheerkaQueryLanguage().compile('hello[1 <= 1]')
+        SheerkaQueryLanguage().compile('hello[1 > 1]')
+        SheerkaQueryLanguage().compile('hello[1 >= 1]')
+
+    def test_i_can_parse_where_conditions_with_logical_operations(self):
+        SheerkaQueryLanguage().compile('hello[a and b]')
+        SheerkaQueryLanguage().compile('hello[a or b]')
+        SheerkaQueryLanguage().compile('hello[not a or b]')
+        SheerkaQueryLanguage().compile('hello[a or not b]')
+        SheerkaQueryLanguage().compile('hello[not a and b]')
+        SheerkaQueryLanguage().compile('hello[a and not b]')
+        SheerkaQueryLanguage().compile('hello[not a or not b]')
+        SheerkaQueryLanguage().compile('hello[not a and not b]')
+
+    def test_i_can_parse_where_conditions_with_logical_operations_and_parenthesis(self):
+        SheerkaQueryLanguage().compile('hello[((a and b) and not (a or b) or not (a and b)) and not (not a or b)]')
+
+    def test_i_can_parse_where_conditions_with_arithmetic_operations(self):
+        SheerkaQueryLanguage().compile('hello[a + b]')
+        SheerkaQueryLanguage().compile('hello[a - b]')
+        SheerkaQueryLanguage().compile('hello[a * b]')
+        SheerkaQueryLanguage().compile('hello[a / b]')
+        SheerkaQueryLanguage().compile('hello[(a + b) / (a - b)]')
+        SheerkaQueryLanguage().compile('hello[-a]')
+        SheerkaQueryLanguage().compile('hello[a + b * c / e]')
+
+    def test_i_can_parse_nested_where_conditions(self):
+        SheerkaQueryLanguage().compile('hello[a]')
+        SheerkaQueryLanguage().compile("hello[a[0]['hello']]")
+        SheerkaQueryLanguage().compile("hello[a[0][\"hello\"]]")
+
+    def test_i_can_parse_where_conditions_with_function_call(self):
+        SheerkaQueryLanguage().compile("hello[a[0]['hello']()]")
+        SheerkaQueryLanguage().compile("hello[a[0]['hello'](0)]")
+        SheerkaQueryLanguage().compile("hello[a[0]['hello']('asdf')]")
+        SheerkaQueryLanguage().compile("hello[a[0]['hello'](asdf)]")
+        SheerkaQueryLanguage().compile("hello[a[0]['hello'](0, 'asdf', asdf)()()(1,2)]")
+        SheerkaQueryLanguage().compile('hello[f(1)]')
+        SheerkaQueryLanguage().compile('hello[f(1,2,3)]')
+        SheerkaQueryLanguage().compile('hello[f(a,b,c)]')
+        SheerkaQueryLanguage().compile('hello[f(a(),b(),c())]')
+        SheerkaQueryLanguage().compile('hello[f(a[a],b[b],c[c])]')
+
+    def test_i_can_parse_where_conditions_with_attributes_parsing(self):
+        SheerkaQueryLanguage().compile('hello[foo.bar.baz]')
+        SheerkaQueryLanguage().compile('hello[foo[12].bar().baz]')
+
+    def test_i_can_parse_where_conditions_with_flwr(self):
+        SheerkaQueryLanguage().compile('hello[f(1,<asdf>,{for x in <asdf> return x})]')
+
+    def test_i_can_parse_where_conditions_with_quantified_expressions(self):
+        SheerkaQueryLanguage().compile('hello[every x in <asdf> satisfies (x)]')
+        with pytest.raises(SyntaxError):
+            SheerkaQueryLanguage().compile('hello[every x in <asdf> statisfies (x)]')
+        with pytest.raises(SyntaxError):
+            SheerkaQueryLanguage().compile('hello[every x in <asdf> statisfies x]')
+        SheerkaQueryLanguage().compile('hello[some x in <asdf> satisfies (x)]')
+        SheerkaQueryLanguage().compile('hello[some x in <self/asdf> satisfies (x)]')
+        SheerkaQueryLanguage().compile('hello[some x in {for x in <asdf> return x} satisfies (x)]')
+        SheerkaQueryLanguage().compile('hello[some x in {for x in <asdf> return x} satisfies (x == y)]')
+        SheerkaQueryLanguage().compile('hello[some x in {for x in <asdf> return x} satisfies (x and not y(1,2))]')
+
+    def test_i_can_parse_where_conditions_with_set_comparisons(self):
+        SheerkaQueryLanguage().compile('hello[a in <qs>]')
+        SheerkaQueryLanguage().compile('hello[a not in <qs>]')
+        SheerkaQueryLanguage().compile('hello[not a in <qs>]')
+        SheerkaQueryLanguage().compile('hello[<a> subset <qs>]')
+        SheerkaQueryLanguage().compile('hello[<a> superset <qq>]')
+        SheerkaQueryLanguage().compile('hello[<a> proper subset <aq>]')
+        SheerkaQueryLanguage().compile('hello[<a> proper superset <aq>]')
+        SheerkaQueryLanguage().compile('hello[<a> is <qs>]')
+        SheerkaQueryLanguage().compile('hello[<a> is not <qs>]')
+
+    def test_i_can_parse_operations_on_sets(self):
+        SheerkaQueryLanguage().compile('asdf - asdf')
+        SheerkaQueryLanguage().compile('asdf & asdf')
+        SheerkaQueryLanguage().compile('asdf | asdf')
+        SheerkaQueryLanguage().compile('(asdf | asdf) & asdf - (asdf & asdf) - (asdf & (afsd | asdf))')
+        SheerkaQueryLanguage().compile('asdf/asdf - asdf/asd[erw]')
+
+    def test_i_can_parse_flwr_expression(self):
+        SheerkaQueryLanguage().compile('for x in y return x')
+        SheerkaQueryLanguage().compile('for x in <asfd> return x')
+        SheerkaQueryLanguage().compile('for x in <asfd>, y in <adsf> return x')
+        SheerkaQueryLanguage().compile('for x in {for x in <asdf> return x} return x')
+        SheerkaQueryLanguage().compile('for x in <asfd> where x == y return x')
+        SheerkaQueryLanguage().compile('for x in <asfd> let y = <x/asdf> return x')
+        SheerkaQueryLanguage().compile('for x in <asfd> let y = {for x in <asdf> return x} return x')
+        SheerkaQueryLanguage().compile('for x in <asfd> let y = <x/asdf>, x = <adf> return x')
+        SheerkaQueryLanguage().compile('for x in <asfd> let y = <x/asdf> let x = <adf> return x')
+        SheerkaQueryLanguage().compile('for x in <asfd>, z in <asdf> let y = <x/asdf> let x = <adf> return x')
+        SheerkaQueryLanguage().compile('''for x in <asfd>, z in <asdf>
+            let y = <x/asdf>
+            let x = <adf>
+            where every x in <y> satisfies (q)
+            return x''')
+        SheerkaQueryLanguage().compile('''for x in <asfd>, z in <asdf>
+            let y = <x/asdf>
+            let x = <adf>
+            where every x in <y> satisfies (q)
+            return x,y,z''')
+        SheerkaQueryLanguage().compile('''for x in <asfd>, z in <asdf>
+            let y = <x/asdf>
+            let x = <adf>
+            where every x in <y> satisfies (q)
+            return x,y.sdf.asd,z''')
+        SheerkaQueryLanguage().compile('''for x in <asfd>, z in <asdf>
+            let y = <x/asdf>
+            let x = <adf>
+            where every x in <y> satisfies (q)
+            return x,y.sdf.asd,z()()()[asdf][asfd](1,2,3)''')
+        SheerkaQueryLanguage().compile('''for x in <asfd>, z in <asdf>
+            let y = <x/asdf>
+            let x = <adf>
+            where every x in <y> satisfies (q)
+            return 'asdf':asdf''')
+        SheerkaQueryLanguage().compile('''for x in <asfd>, z in <asdf>
+            let y = <x/asdf>
+            let x = <adf>
+            where every x in <y> satisfies (q)
+            return 'asdf':asdf, "hello":"hello World!"''')
+        SheerkaQueryLanguage().compile('''for x in <asfd>, z in <asdf>
+            let y = <x/asdf>
+            let x = <adf>
+            where every x in <y> satisfies (q == z and (<y> is not <z>))
+            return 'asdf':asdf, "one":1, "2.0":2.0''')
+        SheerkaQueryLanguage().compile('''for x in <asfd>, z in <asdf>, y in <asdf/asdf>
+            let y = <x/asdf>
+            let x = <adf>
+            where every x in <y> satisfies (q == z and (<y> is not <z>))
+            return 'asdf':asdf, "one":1, "2.0":2.0''')
+        SheerkaQueryLanguage().compile('''for x in <asfd>, z in <asdf>, y in <asdf/asdf>
+            let y = <x/asdf>, y1 = <Afd>, y2 = <asdf>, y3 = <asdf>
+            let x = <adf>
+            where every x in <y> satisfies (q == z and (<y> is not <z>))
+            return 'asdf':asdf, "one":1, "2.0":2.0''')
+
+    def test_i_can_parse_flwr_with_attribute_value(self):
+        SheerkaQueryLanguage().compile('''for x in <asfd>, z in <asdf>, y in sdaf.asdf(asdf, asdf)[1]
+            let y = <x/asdf>, y1 = <Afd>, y2 = <asdf>, y3 = <asdf>
+            let x = <adf>
+            where every x in <y> satisfies (q == z and (<y> is not <z>))
+            return 'asdf':asdf, "one":1, "2.0":2.0''')
+        SheerkaQueryLanguage().compile('''for x in <asfd>, z in <asdf>, y in sdaf.asdf(asdf, asdf)[1]
+            let y = <x/asdf>, y1 = <Afd>, y2 = <asdf>, y3 = <asdf>
+            let x = <adf>
+            let q = sadf.asdf().asfd[1](1,2,3)
+            where every x in <y> satisfies (q == z and (<y> is not <z>))
+            return 'asdf':asdf, "one":1, "2.0":2.0''')
+
+    def test_flwr_orderby(self):
+        SheerkaQueryLanguage().compile('for x in <asdf> order by "adsf" desc return "adsf":x')
+        SheerkaQueryLanguage().compile('''for x in <asfd>, z in <asdf>, y in sdaf.asdf(asdf, asdf)[1]
+            let y = <x/asdf>, y1 = <Afd>, y2 = <asdf>, y3 = <asdf>
+            let x = <adf>
+            where every x in <y> satisfies (q == z and (<y> is not <z>))
+            order by "asdf" desc
+            return 'asdf':asdf, "one":1, "2.0":2.0''')
+        SheerkaQueryLanguage().compile('for x in <asdf> order by 0 asc return x')
+        SheerkaQueryLanguage().compile('''for x in <asfd>, z in <asdf>, y in sdaf.asdf(asdf, asdf)[1]
+            let y = <x/asdf>, y1 = <Afd>, y2 = <asdf>, y3 = <asdf>
+            let x = <adf>
+            where every x in <y> satisfies (q == z and (<y> is not <z>))
+            order by 1 asc
+            return asdf, 1, 2.0''')
+
+    def test_flwr_function_noargs(self):
+        SheerkaQueryLanguage().compile('''
+          for x in <asdf>
+          let f = function() { for y in <asdf> return y }
+          return f
+          ''')
+
+    def test_flwr_function_args(self):
+        SheerkaQueryLanguage().compile('''
+          for x in <asdf>
+          let f = function(q) { for y in q return y }
+          return f
+          ''')
+
+    def test_if(self):
+        SheerkaQueryLanguage().compile('''
+          for x in <asdf> return if (0) then 1 else 0
+          ''')
+
+    def test_reduce(self):
+        SheerkaQueryLanguage().compile('''
+          for x in <asdf>
+          collect x.tree as x.attr with function(prev, next) {
+            if prev == None then next else prev.combine(next)
+          }
+          ''')
+
+    def test_in_list1(self):
+        SheerkaQueryLanguage().compile("hello['foo' in ['foo','bar']]")
+
+    def test_in_list2(self):
+        SheerkaQueryLanguage().compile("hello['baz' in ['foo','bar']]")
+
+    def test_not_in_list1(self):
+        SheerkaQueryLanguage().compile("hello['foo' not in ['foo','bar']]")
+
+    def test_not_in_list2(self):
+        SheerkaQueryLanguage().compile("hello['baz' not in ['foo','bar']]")
+
+    def test_in_list3(self):
+        result = SheerkaQueryLanguage().execute("res[test_elt in ['foo','bar']]",
+                                                {'res': True, 'test_elt': 'foo'})
+        assert bool(result)
+
+    def test_in_list4(self):
+        result = SheerkaQueryLanguage().execute("res[test_elt in ['foo','bar']]",
+                                                {'res': True, 'test_elt': 'baz'})
+
+        assert not bool(result)
+
+    def test_not_in_list4(self):
+        result = SheerkaQueryLanguage().execute("res[test_elt not in ['foo','bar']]",
+                                                {'res': True, 'test_elt': 'baz'})
+        assert bool(result)
+
+    def test_not_in_list5(self):
+        result = SheerkaQueryLanguage().execute("res[test_elt not in ['foo','bar']]",
+                                                {'res': True, 'test_elt': 'foo'})
+        assert not bool(result)
+
+    @pytest.mark.parametrize("text, expected", [
+        ("hello", {"hello"}),
+        ("hello.foo.bar", {"hello"}),
+        ("hello/foo/bar", {"hello"}),
+        ("hello[foo.bar.baz]", {"hello", "foo"}),
+        ("hello[foo()]", {"hello", "foo"}),
+        ("hello[foo(bar)]", {"hello", "foo", "bar"}),
+        ("hello[foo(1, bar.baz)]", {"hello", "foo", "bar"}),
+        ("hello[foo[bar]]", {"hello", "foo", "bar"}),
+        ("hello[foo > bar]", {"hello", "foo", "bar"}),
+        ("hello[foo + bar]", {"hello", "foo", "bar"}),
+        ("hello[[a,b,c]]", {"hello", "a", "b", "c"}),
+        ("hello[{a:b}]", {"hello", "a", "b"}),
+    ])
+    def test_i_can_get_names(self, text, expected):
+        parser = Parser()
+        parser.parse(bytes(text, 'utf-8').decode('unicode_escape'), lexer=Lexer())
+        assert parser.names == expected
+
+    @pytest.mark.parametrize("text", [
+        "sheerka.method",
+        "sheerka/method",
+        "hello[sheerka.method]",
+        "hello[sheerka.method.xx]",
+    ])
+    def test_i_can_get_sheerka_methods(self, text):
+        parser = Parser()
+        parser.parse(bytes(text, 'utf-8').decode('unicode_escape'), lexer=Lexer())
+        assert parser.sheerka_names == {"method"}