diff --git a/.gitignore b/.gitignore index 4819659..6cbdd2d 100644 --- a/.gitignore +++ b/.gitignore @@ -13,4 +13,5 @@ tests/**/*result_test testingPython.ipynb profile*.txt *.prof -new.sb \ No newline at end of file +new.sb +lextab.py \ No newline at end of file diff --git a/LICENCE b/LICENCE index 5f696bc..ea8e5b3 100644 --- a/LICENCE +++ b/LICENCE @@ -22,6 +22,29 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +PyRete is released under the term of the MIT licence +----------------------------------------------------- + +Copyright (c) 2019 GNaive, Christopher J. MacLellan + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + PyFlwor is licensed under a BSD style license ------------------------------------------------------- diff --git a/Makefile b/Makefile index 7361b25..7c83aca 100644 --- a/Makefile +++ b/Makefile @@ -14,6 +14,7 @@ clean: rm -rf tests/prof rm -rf tests/build rm -rf Untitled*.ipynb + rm -rf .ipynb_checkpoints find . -name '.pytest_cache' -exec rm -rf {} + find . -name '__pycache__' -exec rm -rf {} + find . -name 'debug.txt' -exec rm -rf {} + diff --git a/requirements.txt b/requirements.txt index 7355f10..a6aa2d5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -20,4 +20,6 @@ six==1.13.0 wcwidth==0.1.7 ipykernel~=5.3.4 -setuptools~=41.6.0 \ No newline at end of file +setuptools~=41.6.0 +ipython~=7.19.0 +ply~=3.11 \ No newline at end of file diff --git a/sheerka_backup/default.sb b/sheerka_backup/default.sb index 7adebc5..aa7b53b 100644 --- a/sheerka_backup/default.sb +++ b/sheerka_backup/default.sb @@ -3,9 +3,9 @@ push_ontology("english") def concept q from q ? as question(q) pre is_question() auto_eval True set_is_lesser(__PRECEDENCE, q, 'Sya') -def concept the x ret memory(x) -def concept a x where 'x is a concept' ret x -def concept an x where 'x is a concept' ret x +def concept the x where isinstance(x, Concept) ret memory(x) +def concept a x where isinstance(x, Concept) ret x +def concept an x where isinstance(x, Concept) ret x set_is_greatest(__PRECEDENCE, c:the x:, 'Sya') set_is_greatest(__PRECEDENCE, c:a x:, 'Sya') set_is_greatest(__PRECEDENCE, c:an x:, 'Sya') @@ -37,18 +37,24 @@ set_is_greater_than(__PRECEDENCE, c:x and y:, c:x or y:, 'Sya') set_is_less_than(__PRECEDENCE, c:q:, c:x or y:, 'Sya') # some words +def concept human def concept male def concept female def concept man -man is a male def concept woman -woman is a female -def concept human -man is a human -woman is a human def concept boy -def concept boys def concept girl +man is a male +man is a human +boy is a male +boy is a human +woman is a female +woman is a human +girl is a female +girl is a human + + +def concept boys def concept girls def concept shirt def concept table @@ -66,3 +72,7 @@ def concept sunday def concept how is x pre is_question() as smart_get_attr(x, adjective) def concept what x is y pre is_question() where x is an adjective as smart_get_attr(y, x) def concept what is the x of y pre is_question() where x is an adjective as smart_get_attr(y, x) + +# +def concept he ret memory("self is a human and self is a male") +def concept she ret memory("self is a human and self is a female") \ No newline at end of file diff --git a/src/cache/FastCache.py b/src/cache/FastCache.py index 1fd9b74..a0098ed 100644 --- a/src/cache/FastCache.py +++ b/src/cache/FastCache.py @@ -12,6 +12,9 @@ class FastCache: self.lru = [] self.default = default + def __contains__(self, item): + return self.has(item) + def put(self, key, value): if len(self.cache) == self.max_size: del self.cache[self.lru.pop(0)] diff --git a/src/core/builtin_helpers.py b/src/core/builtin_helpers.py index eb39254..2436482 100644 --- a/src/core/builtin_helpers.py +++ b/src/core/builtin_helpers.py @@ -7,6 +7,7 @@ from core.builtin_concepts import BuiltinConcepts from core.concept import Concept, ConceptParts, DEFINITION_TYPE_BNF, concept_part_value from core.global_symbols import NotInit, NotFound, CURRENT_OBJ from core.rule import Rule +from core.tokenizer import Tokenizer from core.utils import as_bag from parsers.BaseNodeParser import SourceCodeNode, ConceptNode, UnrecognizedTokensNode, SourceCodeWithConceptNode, \ RuleNode, VariableNode @@ -313,15 +314,15 @@ def only_parsers_results(context, return_values): parents=return_values) -def evaluate(context, - source, - evaluators="all", - desc=None, - eval_body=True, - eval_where=True, - is_question=False, - expect_success=False, - stm=None): +def evaluate_from_source(context, + source, + evaluators="all", + desc=None, + eval_body=True, + eval_where=True, + is_question=False, + expect_success=False, + stm=None): """ :param context: @@ -338,7 +339,13 @@ def evaluate(context, sheerka = context.sheerka desc = desc or f"Eval '{source}'" - with context.push(BuiltinConcepts.EVALUATE_SOURCE, source, desc=desc) as sub_context: + hints_to_reset = { + BuiltinConcepts.EVAL_BODY_REQUESTED, + BuiltinConcepts.EVAL_WHERE_REQUESTED, + BuiltinConcepts.EVAL_UNTIL_SUCCESS_REQUESTED, + BuiltinConcepts.EVAL_QUESTION_REQUESTED, + } + with context.push(BuiltinConcepts.EVALUATE_SOURCE, source, desc=desc, reset_hints=hints_to_reset) as sub_context: if eval_body: sub_context.protected_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED) @@ -804,6 +811,26 @@ def get_inner_body(context, concept): return concept.body +def get_possible_variables_from_concept(context, concept): + """ + Given concept definition, + gives the variables of the concept that can be considered as a parameter in another function + >>> gpvfc = get_possible_variables_from_concept + >>> assert gpvfc(Concept("a plus b").def_var("a").def_var("b")) == {"a", "b"} + >>> assert gpvfc(Concept("twenties", definition="twenty (one|two)=n").def_var("n")) == set() + :param context: + :param concept: + :return: + """ + if len(concept.name) <= 1: + return set() + + concept_name = [t.str_value for t in Tokenizer(concept.name, yield_eof=False)] + names = [v_value or v_name for v_name, v_value in concept.get_metadata().variables if v_name in concept_name] + possible_vars = filter(lambda x: context.sheerka.is_not_a_concept_name(x), names) + return set(possible_vars) + + class CreateObjectIdentifiers: """ Class that creates unique identifiers for Concept or Rule objects diff --git a/src/core/concept.py b/src/core/concept.py index 78ddae5..52398b0 100644 --- a/src/core/concept.py +++ b/src/core/concept.py @@ -201,6 +201,9 @@ class Concept: return False for name, value in self_values.items(): + if value == self: # not very resilient... + continue + if value != other.get_value(name): return False @@ -321,6 +324,10 @@ class Concept: def body(self): return self.get_value(ConceptParts.BODY) + def get_atomic_def(self): + tokens = [t for t in Tokenizer(self.key, yield_eof=False) if t.type != TokenKind.VAR_DEF] + return core.utils.get_text_from_tokens(tokens).strip() + def get_origin(self): """ Return the digest used to save the concept if it exists diff --git a/src/core/sheerka/ExecutionContext.py b/src/core/sheerka/ExecutionContext.py index 352e227..d7f2614 100644 --- a/src/core/sheerka/ExecutionContext.py +++ b/src/core/sheerka/ExecutionContext.py @@ -155,10 +155,23 @@ class ExecutionContext: return True - def push(self, action: BuiltinConcepts, action_context, who=None, desc=None, logger=None, obj=None, concepts=None): + def push(self, action: BuiltinConcepts, action_context, + who=None, + desc=None, + logger=None, + obj=None, + concepts=None, + reset_hints=None): if self._push: return self._push + if reset_hints: + global_hints = self.global_hints - reset_hints + protected_hints = self.protected_hints - reset_hints + else: + global_hints = self.global_hints + protected_hints = self.protected_hints + who = who or self.who logger = logger or self._logger new = ExecutionContext( @@ -169,7 +182,7 @@ class ExecutionContext: action_context, desc, logger, - self.global_hints, + global_hints, self.errors, obj or self.obj, concepts or self.concepts) @@ -177,7 +190,7 @@ class ExecutionContext: new.preprocess = self.preprocess new.preprocess_parsers = self.preprocess_parsers new.preprocess_evaluators = self.preprocess_evaluators - new.protected_hints.update(self.protected_hints) + new.protected_hints.update(protected_hints) self._children.append(new) diff --git a/src/core/sheerka/Sheerka.py b/src/core/sheerka/Sheerka.py index 33c8f0e..66d0230 100644 --- a/src/core/sheerka/Sheerka.py +++ b/src/core/sheerka/Sheerka.py @@ -109,6 +109,7 @@ class Sheerka(Concept): self.enable_commands_backup = True self.methods_with_context = {"test_using_context"} # only the names, the method is defined in sheerka_methods + self.pipe_functions = set() self.sheerka_methods = { "test": SheerkaMethod(self.test, False), "test_using_context": SheerkaMethod(self.test_using_context, False), @@ -145,13 +146,22 @@ class Sheerka(Concept): if as_name is None: as_name = bound_method.__name__ + if as_name.startswith("pipe_"): + as_name = as_name[5:] + is_pipe_function = True + else: + is_pipe_function = False + if visible: signature = inspect.signature(bound_method) if len(signature.parameters) > 0 and list(signature.parameters.keys())[0] == "context": self.methods_with_context.add(as_name) self.sheerka_methods[as_name] = SheerkaMethod(bound_method, has_side_effect) + if is_pipe_function: + self.pipe_functions.add(as_name) - setattr(self, bound_method.__name__, bound_method) + if not is_pipe_function: + setattr(self, bound_method.__name__, bound_method) def initialize(self, root_folder: str = None, **kwargs): """ @@ -655,6 +665,10 @@ class Sheerka(Concept): if obj.body is NotInit: return obj + if obj.key in BuiltinErrors: + # KSI 2021-04-20 # errors should be returned as soon as they are detected + return obj + if reduce_simple_list and (isinstance(obj.body, list) or isinstance(obj.body, set)) and len(obj.body) == 1: body_to_use = obj.body[0] else: @@ -673,39 +687,15 @@ class Sheerka(Concept): return (self.objvalue(obj) for obj in objs.body) - def get_errors(self, obj, **kwargs): + def get_errors(self, context, obj, **kwargs): """ Browse obj, looking for error + :param context: :param obj: :param kwargs: if defined, specialize the error :return: """ - def filter_by_type(x, name): - if isinstance(x, Concept): - return x.name == name - else: - return type(x).__name__ == name - - def filter_by_attribute(x, attr_name, attr_value): - if hasattr(x, "as_bag"): - try: - return x.as_bag()[attr_name] == attr_value - except KeyError: - return False - else: - try: - return getattr(x, attr_name) == attr_value - except AttributeError: - return False - - def and_filter(x, cond): - for c in cond: - if not c(x): - return False - - return True - def is_error(_obj): if isinstance(_obj, ErrorObj): return True @@ -715,29 +705,6 @@ class Sheerka(Concept): return False - def filter_objects(_objects): - if kwargs: - cond = [] - for k, v in kwargs.items(): - if k == "__type": - expected_type = v - cond.append(lambda x: filter_by_type(x, expected_type)) - else: - attr_name = k - expect_value = v - cond.append(lambda x: filter_by_attribute(x, attr_name, expect_value)) - - if len(cond) > 1: - copy_of_conditions = cond.copy() - full_cond = lambda x: and_filter(x, copy_of_conditions) - - else: - full_cond = cond[0] - - return [o for o in _objects if full_cond(o)] - - return _objects - def inner_get_errors(_obj): if self.isinstance(_obj, BuiltinConcepts.RETURN_VALUE) and _obj.status: return [] @@ -748,6 +715,8 @@ class Sheerka(Concept): if is_error(_obj): if isinstance(_obj, Concept) and _obj.body not in (NotInit, None): return [_obj] + inner_get_errors(_obj.body) + if isinstance(_obj, ErrorObj) and hasattr(_obj, "get_error"): + return [_obj] + inner_get_errors(_obj.get_error()) else: return [_obj] @@ -757,10 +726,10 @@ class Sheerka(Concept): return [] errors = inner_get_errors(obj) - return filter_objects([e for e in errors]) + return self.filter_objects(context, [e for e in errors], **kwargs) - def has_error(self, obj, **kwargs): - errors = self.get_errors(obj, **kwargs) + def has_error(self, context, obj, **kwargs): + errors = self.get_errors(context, obj, **kwargs) return len(errors) > 0 def get_evaluator_name(self, name): @@ -797,13 +766,6 @@ class Sheerka(Concept): return bool(obj) - @staticmethod - def is_error(obj): - """ - opposite of is_success - """ - return not Sheerka.is_success(obj) - @staticmethod def is_known(obj): if not isinstance(obj, Concept): @@ -844,7 +806,7 @@ class Sheerka(Concept): if isinstance(obj, Concept) and obj.id == self.id: return True - from evaluators.PythonEvaluator import Expando + from sheerkapython.python_wrapper import Expando if isinstance(obj, Expando) and obj.get_name() == "sheerka": return True diff --git a/src/core/sheerka/SheerkaOntologyManager.py b/src/core/sheerka/SheerkaOntologyManager.py index 7f9f507..2b986cb 100644 --- a/src/core/sheerka/SheerkaOntologyManager.py +++ b/src/core/sheerka/SheerkaOntologyManager.py @@ -395,7 +395,7 @@ class SheerkaOntologyManager: def get_all(self, entry, cache_only=False): """ - Return all key, value from all ontologies + Return all from all ontologies First look in sdp, then override with the cache, for all ontologies :param entry: cache name / sdp entry :param cache_only: Do no fetch data from remote sdp diff --git a/src/core/sheerka/services/SheerkaAdmin.py b/src/core/sheerka/services/SheerkaAdmin.py index 8b7646f..00c9059 100644 --- a/src/core/sheerka/services/SheerkaAdmin.py +++ b/src/core/sheerka/services/SheerkaAdmin.py @@ -3,7 +3,7 @@ import time from os import path from core.builtin_concepts_ids import BuiltinConcepts, BuiltinContainers -from core.builtin_helpers import ensure_concept_or_rule +from core.builtin_helpers import ensure_concept_or_rule, ensure_concept from core.concept import Concept from core.global_symbols import SHEERKA_BACKUP_FOLDER from core.sheerka.services.SheerkaHistoryManager import SheerkaHistoryManager @@ -37,6 +37,7 @@ class SheerkaAdmin(BaseService): self.sheerka.bind_service_method(self.admin_history, False, as_name="history") self.sheerka.bind_service_method(self.admin_history, False, as_name="history") self.sheerka.bind_service_method(self.sdp, False) + self.sheerka.bind_service_method(self.atomic_def, False) def caches_names(self): """ @@ -218,6 +219,20 @@ class SheerkaAdmin(BaseService): return self.sheerka.isinstance(a, b) + @staticmethod + def atomic_def(a): + """ + Return the 'atomic definition' of a concept + a concept key stripped from its 'var' tokens + >>> assert atomic_def(Concept('a plus b').def_var("a").def_var("b")) == "plus" + >>> assert atomic_def(Concept('x is a y').def_var("x").def_var("y")) == "is a" + :param a: + :return: + """ + ensure_concept(a) + + return a.get_atomic_def() + @staticmethod def is_container(obj): """ diff --git a/src/core/sheerka/services/SheerkaConceptManager.py b/src/core/sheerka/services/SheerkaConceptManager.py index 13b19a5..b80c062 100644 --- a/src/core/sheerka/services/SheerkaConceptManager.py +++ b/src/core/sheerka/services/SheerkaConceptManager.py @@ -126,7 +126,8 @@ class SheerkaConceptManager(BaseService): self.sheerka.bind_service_method(self.get_by_name, False, visible=False) self.sheerka.bind_service_method(self.get_by_hash, False, visible=False) self.sheerka.bind_service_method(self.get_by_id, False, visible=False) - self.sheerka.bind_service_method(self.is_not_a_variable, False, visible=False) + self.sheerka.bind_service_method(self.is_not_a_concept_name, False, visible=False) + self.sheerka.bind_service_method(self.is_a_concept_name, False, visible=False) self.sheerka.bind_service_method(self.get_concepts_by_first_token, False, visible=False) self.sheerka.bind_service_method(self.get_concepts_by_first_regex, False, visible=False) self.sheerka.bind_service_method(self.get_concepts_bnf_definitions, False, visible=False) @@ -705,7 +706,7 @@ class SheerkaConceptManager(BaseService): return refs - def is_not_a_variable(self, name): + def is_not_a_concept_name(self, name): """ Given a name tells if it refers to a variable name :param name: @@ -713,6 +714,14 @@ class SheerkaConceptManager(BaseService): """ return self.sheerka.om.get(self.sheerka.CONCEPTS_BY_NAME_ENTRY, name) is NotFound + def is_a_concept_name(self, name): + """ + Given a name tells if it refers to a variable name + :param name: + :return: + """ + return self.sheerka.om.get(self.sheerka.CONCEPTS_BY_NAME_ENTRY, name) is not NotFound + def clear_bnf_definition(self, concept_id=None): if concept_id: self.sheerka.om.delete(self.CONCEPTS_BNF_DEFINITIONS_ENTRY, concept_id) diff --git a/src/core/sheerka/services/SheerkaEvaluateConcept.py b/src/core/sheerka/services/SheerkaEvaluateConcept.py index 9e219ab..0e2c3de 100644 --- a/src/core/sheerka/services/SheerkaEvaluateConcept.py +++ b/src/core/sheerka/services/SheerkaEvaluateConcept.py @@ -1,7 +1,7 @@ from dataclasses import dataclass from core.builtin_concepts import BuiltinConcepts -from core.builtin_helpers import expect_one, only_successful, evaluate, ensure_concept +from core.builtin_helpers import expect_one, only_successful, evaluate_from_source, ensure_concept from core.concept import Concept, DoNotResolve, ConceptParts, InfiniteRecursionResolved, AllConceptParts, \ concept_part_value from core.global_symbols import NotInit, CURRENT_OBJ @@ -11,9 +11,9 @@ from core.sheerka.services.SheerkaExecute import ParserInput from core.sheerka.services.sheerka_service import BaseService from core.tokenizer import Tokenizer from core.utils import unstr_concept +from parsers.BaseExpressionParser import TrueifyVisitor from parsers.BaseNodeParser import ConceptNode from parsers.LogicalOperatorParser import LogicalOperatorParser -from parsers.BaseExpressionParser import TrueifyVisitor CONCEPT_EVALUATION_STEPS = [ BuiltinConcepts.BEFORE_EVALUATION, @@ -26,6 +26,11 @@ class ChickenAndEggException(Exception): error: Concept +@dataclass +class ConceptEvalException(Exception): + error: Concept + + @dataclass class WhereClauseDef: concept: Concept # concept on which the where clause is applied @@ -43,6 +48,8 @@ class SheerkaEvaluateConcept(BaseService): def initialize(self): self.sheerka.bind_service_method(self.evaluate_concept, True) + self.sheerka.bind_service_method(self.call_concept, True) + self.sheerka.bind_service_method(self.call_concept, False, as_name="evaluate_question") self.sheerka.bind_service_method(self.set_auto_eval, True) @staticmethod @@ -197,12 +204,12 @@ class SheerkaEvaluateConcept(BaseService): ret.append(r) # it cannot be solved unitary, let's give a chance to the global where condition else: # it means that the where condition is an expression that needs to be executed - evaluation_res = evaluate(context, - where_clause_def.trueified, - desc=f"Apply where clause on '{where_clause_def.prop}'", - expect_success=True, - is_question=True, - stm={where_clause_def.prop: r.body}) + evaluation_res = evaluate_from_source(context, + where_clause_def.trueified, + desc=f"Apply where clause on '{where_clause_def.prop}'", + expect_success=True, + is_question=True, + stm={where_clause_def.prop: r.body}) one_res = expect_one(context, evaluation_res) if one_res.status: value = context.sheerka.objvalue(one_res) @@ -624,6 +631,25 @@ class SheerkaEvaluateConcept(BaseService): else: return concept + def call_concept(self, context, concept, *args, **kwargs): + """ + call the concept using either args or kwargs (not both) + :param context: + :param concept: + :param args: + :param kwargs: + :return: + """ + + evaluated = self.evaluate_concept(context, concept) + if self.sheerka.has_error(context, evaluated): + raise ConceptEvalException(evaluated) + + if ConceptParts.BODY in evaluated.get_compiled(): + return evaluated.body + else: + return evaluated + def compute_metadata_to_eval(self, context, concept): to_eval = [] diff --git a/src/core/sheerka/services/SheerkaEvaluateRules.py b/src/core/sheerka/services/SheerkaEvaluateRules.py index 773cde7..53ecb00 100644 --- a/src/core/sheerka/services/SheerkaEvaluateRules.py +++ b/src/core/sheerka/services/SheerkaEvaluateRules.py @@ -2,7 +2,6 @@ from core.builtin_concepts import BuiltinConcepts from core.builtin_helpers import expect_one from core.global_symbols import EVENT_RULE_CREATED, EVENT_RULE_DELETED, EVENT_RULE_ID_DELETED from core.sheerka.services.sheerka_service import BaseService -from evaluators.ConceptEvaluator import ConceptEvaluator from sheerkarete.network import ReteNetwork DISABLED_RULES = "#disabled#" @@ -105,10 +104,18 @@ class SheerkaEvaluateRules(BaseService): :return: """ + results = self.evaluate_conditions(context, rule.compiled_conditions, bag) + + debugger = context.get_debugger(SheerkaEvaluateRules.NAME, "evaluate_rule", new_debug_id=False) + debugger.debug_rule(rule, results) + + return expect_one(context, results) + + def evaluate_conditions(self, context, conditions, bag): bag_variables = set(bag.keys()) results = [] - for compiled_condition in rule.compiled_conditions: + for compiled_condition in conditions: if compiled_condition.variables.intersection(bag_variables) != compiled_condition.variables: continue @@ -123,22 +130,19 @@ class SheerkaEvaluateRules(BaseService): else: # do not forget to reset the 'is_evaluated' in the case of a concept - if compiled_condition.evaluator_type == ConceptEvaluator.NAME: - compiled_condition.concept.get_metadata().is_evaluated = False + for concept in compiled_condition.concepts_to_reset: + concept.get_metadata().is_evaluated = False evaluator = self.evaluators_by_name[compiled_condition.evaluator_type] res = evaluator.eval(context, compiled_condition.return_value) if res.status and isinstance(res.body, bool) and res.body: # one successful value found. No need to look any further - results = [res] + results = [res] # don't we care about the other failing results ? break else: results.append(res) - debugger = context.get_debugger(SheerkaEvaluateRules.NAME, "evaluate_rule", new_debug_id=False) - debugger.debug_rule(rule, results) - - return expect_one(context, results) + return results def remove_from_rete_memory(self, lst): if lst is None: diff --git a/src/core/sheerka/services/SheerkaHasAManager.py b/src/core/sheerka/services/SheerkaHasAManager.py index 914533e..74d180a 100644 --- a/src/core/sheerka/services/SheerkaHasAManager.py +++ b/src/core/sheerka/services/SheerkaHasAManager.py @@ -43,7 +43,6 @@ class SheerkaHasAManager(BaseService): def hasa(self, concept_a, concept_b): """ Check that concept 'a' has/owns concept 'b' - :param context: :param concept_a: :param concept_b: :return: diff --git a/src/core/sheerka/services/SheerkaMemory.py b/src/core/sheerka/services/SheerkaMemory.py index 7e51627..261e299 100644 --- a/src/core/sheerka/services/SheerkaMemory.py +++ b/src/core/sheerka/services/SheerkaMemory.py @@ -10,6 +10,7 @@ from core.sheerka.services.sheerka_service import BaseService, ServiceObj @dataclass class MemoryObject(ServiceObj): + timestamp: float obj: object def __eq__(self, other): @@ -21,6 +22,9 @@ class MemoryObject(ServiceObj): def __hash__(self): return hash((self.event_id, self.obj)) + def __repr__(self): + return f"MemoryObject({self.obj}, timestamp={self.timestamp})" + class SheerkaMemory(BaseService): NAME = "Memory" @@ -38,7 +42,7 @@ class SheerkaMemory(BaseService): self.sheerka.bind_service_method(self.get_all_short_term_memory, False, visible=False) self.sheerka.bind_service_method(self.add_to_short_term_memory, True, visible=False) self.sheerka.bind_service_method(self.remove_context, True, as_name="clear_short_term_memory", visible=False) - self.sheerka.bind_service_method(self.add_to_memory, True, visible=False) + self.sheerka.bind_service_method(self.add_to_memory, True) self.sheerka.bind_service_method(self.add_many_to_short_term_memory, True, visible=False) self.sheerka.bind_service_method(self.get_from_memory, False) self.sheerka.bind_service_method(self.get_last_from_memory, False) @@ -125,20 +129,31 @@ class SheerkaMemory(BaseService): """ last = self.sheerka.om.get(SheerkaMemory.OBJECTS_ENTRY, key) if last is NotFound: - self.sheerka.om.put(SheerkaMemory.OBJECTS_ENTRY, key, MemoryObject(context.event.get_digest(), concept)) + self.sheerka.om.put(SheerkaMemory.OBJECTS_ENTRY, key, MemoryObject(context.event.get_digest(), + context.event.date.timestamp(), + concept)) return if not isinstance(last, list) and last.obj == concept: + # replace with the new one self.sheerka.om.delete(SheerkaMemory.OBJECTS_ENTRY, key, last) - self.sheerka.om.put(SheerkaMemory.OBJECTS_ENTRY, key, MemoryObject(context.event.get_digest(), concept)) + self.sheerka.om.put(SheerkaMemory.OBJECTS_ENTRY, key, MemoryObject(context.event.get_digest(), + context.event.date.timestamp(), + concept)) return if isinstance(last, list) and last[-1].obj == concept: + # replace with the new one self.sheerka.om.delete(SheerkaMemory.OBJECTS_ENTRY, key, last[-1]) - self.sheerka.om.put(SheerkaMemory.OBJECTS_ENTRY, key, MemoryObject(context.event.get_digest(), concept)) + self.sheerka.om.put(SheerkaMemory.OBJECTS_ENTRY, key, MemoryObject(context.event.get_digest(), + context.event.date.timestamp(), + concept)) return - self.sheerka.om.put(SheerkaMemory.OBJECTS_ENTRY, key, MemoryObject(context.event.get_digest(), concept)) + # append the new one + self.sheerka.om.put(SheerkaMemory.OBJECTS_ENTRY, key, MemoryObject(context.event.get_digest(), + context.event.date.timestamp(), + concept)) def get_from_memory(self, context, key): """" @@ -207,14 +222,35 @@ class SheerkaMemory(BaseService): """ name_to_use = name.name if isinstance(name, Concept) else name self.unregister_object(context, name_to_use) - obj = self.get_from_memory(context, name_to_use) - if obj is NotFound: - return self.sheerka.new(BuiltinConcepts.NOT_FOUND, body={"#name": name}) + obj = self.get_last_from_memory(context, name_to_use) + if obj is not NotFound: + return obj.obj - if isinstance(obj, list): - obj = obj[-1] + all_objects = self.sheerka.om.list(SheerkaMemory.OBJECTS_ENTRY) + all_objects_copy = [] + for obj in all_objects: + if isinstance(obj, list): + all_objects_copy.append(obj.copy()) + else: + all_objects_copy.append([obj]) - return obj.obj + while len(all_objects_copy) > 0: + current_list = [] + temp = [] + for obj in all_objects_copy: + current_list.append(obj.pop(-1)) + if len(obj) > 0: + temp.append(obj) + + all_objects_copy = temp + current_list = sorted(current_list, key=lambda o: o.timestamp, reverse=True) + current_objects = [o.obj for o in current_list] + + res = self.sheerka.filter_objects(context, current_objects, name) + if len(res) > 0: + return res[0] # only the first, as it should have a better timestamp + + return self.sheerka.new(BuiltinConcepts.NOT_FOUND, body={"#name": name}) def mem(self): keys = sorted([k for k in self.sheerka.om.list(SheerkaMemory.OBJECTS_ENTRY)]) diff --git a/src/core/sheerka/services/SheerkaQueryManager.py b/src/core/sheerka/services/SheerkaQueryManager.py new file mode 100644 index 0000000..6b5205e --- /dev/null +++ b/src/core/sheerka/services/SheerkaQueryManager.py @@ -0,0 +1,251 @@ +from cache.FastCache import FastCache +from core.builtin_concepts_ids import BuiltinContainers, BuiltinConcepts +from core.concept import Concept, ConceptParts +from core.sheerka.services.SheerkaEvaluateRules import SheerkaEvaluateRules +from core.sheerka.services.sheerka_service import BaseService +from core.tokenizer import Tokenizer, TokenKind +from core.utils import as_bag +from sheerkapython.python_wrapper import create_namespace, ObjectContainer, get_type +from sheerkaql.lexer import Lexer +from sheerkaql.parser import Parser + + +class SheerkaQueryManager(BaseService): + """ + This class manage the queries on objects across the system + """ + NAME = "QueryManager" + OBJECTS_ROOT_ALIAS = "__xxx__objects__xx__" + QUERY_PARAMETER_PREFIX = "__xxx__query_parameter__xx__" + + def __init__(self, sheerka): + super().__init__(sheerka) + self.queries = FastCache() + self.conditions = FastCache() + self.lexer = Lexer() + self.rule_evaluator = None + + def initialize(self): + self.sheerka.bind_service_method(self.filter_objects, False) + self.sheerka.bind_service_method(self.select_objects, False) + self.sheerka.bind_service_method(self.collect_attributes, False) + + self.sheerka.bind_service_method(self.filter_objects, False, as_name="pipe_where") + self.sheerka.bind_service_method(self.select_objects, False, as_name="pipe_select") + self.sheerka.bind_service_method(self.collect_attributes, False, as_name="pipe_props") + + self.sheerka.register_debug_vars(SheerkaQueryManager.NAME, "filter_objects", "query") + + def initialize_deferred(self, context, is_first_time): + self.rule_evaluator = self.sheerka.services[SheerkaEvaluateRules.NAME] + + def get_query_by_kwargs(self, local_namespace, **kwargs): + """ + Create a predicate using kwargs and filter the result + :param local_namespace: + :param kwargs: + :return: + """ + if not kwargs: + return None + + objects_in_context_index = 0 + conditions = [] + for k, v in kwargs.items(): + current_variable_name = f"{self.QUERY_PARAMETER_PREFIX}_{objects_in_context_index:02}" + objects_in_context_index += 1 + local_namespace[current_variable_name] = v + + if k == "__type": + conditions.append(f"get_type(self) == {current_variable_name}") + + elif k == "atomic_def": + conditions.append(f"atomic_def(self) == {current_variable_name}") + + elif k in ("__self", "_"): + conditions.append(f"self == {current_variable_name}") + + else: + conditions.append(f"self.{k} == {current_variable_name}") + + return ' and '.join(conditions) + + def filter_objects(self, context, objects, predicate=None, **kwargs): + """ + filter the given objects using the conditions from kwargs + for each k,v in kwargs, the equality k == v is added + for k starting with a double underscore '__', a special treatment may be done + __type : get the type of object (in Sheerka world) + :param context: + :param objects: + :param predicate: + :param kwargs: + :return: + """ + + debugger = context.get_debugger(SheerkaQueryManager.NAME, "filter_objects") + + original_container = None + if isinstance(objects, Concept) and objects.key in BuiltinContainers: + original_container = objects + objects = objects.body + + debugger.debug_entering(nb_objects=len(objects), predicate=predicate, **kwargs) + local_namespace = {} + query_by_kwargs = self.get_query_by_kwargs(local_namespace, **kwargs) + + if predicate is not None and query_by_kwargs is not None: + query = f"({predicate}) and ({query_by_kwargs})" + elif predicate is not None: + query = predicate + elif query_by_kwargs is not None: + query = query_by_kwargs + else: + query = None + + if debugger.is_enabled(): + debugger.debug_var("query", query) + for k, v in local_namespace.items(): + debugger.debug_var("query_parameter", f"{k} = {v}") + + if query and query in self.conditions: + # Then try using RuleManager + objects = self.execute_conditions(context, query, objects, local_namespace) + elif query: + try: + # Fist try with the FLWR parser + full_query = f"{self.OBJECTS_ROOT_ALIAS}.items[{query}]" + objects = self.execute_flwr_query(context, full_query, objects, local_namespace) + except SyntaxError: + # Then try using RuleManager + objects = self.execute_conditions(context, query, objects, local_namespace) + + if original_container: + original_container.set_value(ConceptParts.BODY, objects) + return original_container + else: + return objects + + def select_objects(self, context, objects, *props, **kwargs): + """ + From the input objects, create output objects + The definition of these new objects can come from a flwr query + :param context: + :param objects: + :param query: + :return: + """ + + def sanitize_property(p): + if not isinstance(p, str): + raise SyntaxError(f"{p} is not the name of an attribute") + + tokens = list(Tokenizer(p, yield_eof=False)) + if len(tokens) == 1 and tokens[0].type == TokenKind.IDENTIFIER: + return f"self.{p}" + else: + return p + + original_container = None + if isinstance(objects, Concept) and objects.key in BuiltinContainers: + original_container = objects + objects = objects.body + + requested_properties = [sanitize_property(prop) for prop in props] + + if kwargs: + items = [f"'{k}': {sanitize_property(v)}" for k, v in kwargs.items()] + requested_properties.append("{" + ", ".join(items) + "}") + + query = f"for self in {self.OBJECTS_ROOT_ALIAS}.items return " + ", ".join(requested_properties) + objects = self.execute_flwr_query(context, query, objects, {}) + + if original_container: + original_container.set_value(ConceptParts.BODY, objects) + return original_container + else: + return objects + + def collect_attributes(self, objects, take=10): + """ + Given a list of object, returns the attributes that can be requested + :param objects: + :param take: no need to browse the whole list, you can just use a sample + :return: + """ + if isinstance(objects, Concept) and objects.key in BuiltinContainers: + objects = objects.body + + result = {} + for obj in (objects if take <= 0 else objects[:take]): + object_type = get_type(obj) + attrs = set(p for p in as_bag(obj).keys() if p != "self") + if object_type is result: + result[object_type].update(attrs) + else: + result[object_type] = attrs + + result = {k: sorted(list(v)) for k, v in result.items()} + return self.sheerka.new(BuiltinConcepts.TO_DICT, body=result) + + def execute_flwr_query(self, context, query, objects, local_namespace): + """ + Execute a FLWOR query on objects + :param context: + :param query: query to execute (as a string to compile) + :param objects: list of objects to filter + :param local_namespace: objects of the namespace that are already known + :return: + """ + if query not in self.queries: + parser = Parser() + compiled_query = parser.parse(bytes(query, 'utf-8').decode('unicode_escape'), lexer=self.lexer) + self.queries.put(query, (compiled_query, parser.names, parser.sheerka_names)) + + compiled_query, names, sheerka_names = self.queries.get(query) + + namespace = create_namespace(context, + self.NAME, + names, + sheerka_names, + {self.OBJECTS_ROOT_ALIAS: ObjectContainer(objects)}, + expression_only=True, + allow_builtins=True) + namespace.update(local_namespace) # override if needed + + return compiled_query(namespace) + + def execute_conditions(self, context, query, objects, local_namespace): + """ + + :param context: + :param query: + :param objects: + :param local_namespace: + :return: + """ + + if query not in self.conditions: + from core.sheerka.services.SheerkaRuleManager import SheerkaRuleManager + rule_manager = self.sheerka.services[SheerkaRuleManager.NAME] + compilation_results = rule_manager.compile_when(context, self.NAME, query) + self.conditions.put(query, compilation_results.python_conditions) + + conditions = self.conditions.get(query) + results = [] + with context.push(BuiltinConcepts.EXEC_CODE, query) as sub_context: + sub_context.protected_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED) + sub_context.protected_hints.add(BuiltinConcepts.EVAL_WHERE_REQUESTED) + sub_context.protected_hints.add(BuiltinConcepts.EVAL_UNTIL_SUCCESS_REQUESTED) + sub_context.protected_hints.add(BuiltinConcepts.EVAL_QUESTION_REQUESTED) + sub_context.sheerka.add_many_to_short_term_memory(sub_context, local_namespace) + sub_context.deactivate_push() + + for obj in objects: + local_namespace["self"] = obj + res = self.rule_evaluator.evaluate_conditions(sub_context, conditions, local_namespace) + successful = list(filter(lambda r: r.status and type(r.body) == bool and r.body, res)) + if successful: + results.append(obj) + + return results diff --git a/src/core/sheerka/services/SheerkaRuleManager.py b/src/core/sheerka/services/SheerkaRuleManager.py index 250bf22..f12e173 100644 --- a/src/core/sheerka/services/SheerkaRuleManager.py +++ b/src/core/sheerka/services/SheerkaRuleManager.py @@ -1,28 +1,27 @@ import operator import re from dataclasses import dataclass -from itertools import product from typing import Union, Set, List, Tuple from cache.Cache import Cache from cache.ListIfNeededCache import ListIfNeededCache from core.builtin_concepts import BuiltinConcepts, ReturnValueConcept -from core.builtin_helpers import ensure_evaluated, expect_one, evaluate +from core.builtin_helpers import ensure_evaluated, expect_one, evaluate_from_source, \ + get_possible_variables_from_concept, is_a_question from core.concept import Concept from core.global_symbols import EVENT_RULE_PRECEDENCE_MODIFIED, RULE_COMPARISON_CONTEXT, NotFound, ErrorObj, \ EVENT_RULE_CREATED, EVENT_RULE_DELETED, NotInit from core.rule import Rule, ACTION_TYPE_PRINT from core.sheerka.Sheerka import RECOGNIZED_BY_NAME, RECOGNIZED_BY_ID -from core.sheerka.services.SheerkaExecute import ParserInput from core.sheerka.services.sheerka_service import BaseService, FailedToCompileError from core.tokenizer import Keywords, TokenKind, Token, IterParser -from core.utils import index_tokens, COLORS, get_text_from_tokens, merge_dictionaries, merge_sets -from evaluators.PythonEvaluator import PythonEvaluator, Expando +from core.utils import index_tokens, COLORS, get_text_from_tokens, merge_dictionaries, merge_sets, get_safe_str_value +from evaluators.PythonEvaluator import PythonEvaluator from parsers.BaseExpressionParser import AndNode, ExpressionVisitor, VariableNode, ComparisonNode, FunctionNode, \ ComparisonType, NotNode, NameExprNode from parsers.BaseNodeParser import ConceptNode from parsers.LogicalOperatorParser import LogicalOperatorParser -from parsers.PythonParser import PythonParser +from sheerkapython.python_wrapper import Expando, sheerka_globals from sheerkarete.common import V from sheerkarete.conditions import AndConditions, Condition, NegatedCondition, NegatedConjunctiveConditions from sheerkarete.network import FACT_NAME, FACT_SELF @@ -424,7 +423,7 @@ class FormatRuleActionParser(IterParser): def return_list(self, args, kwargs): """ - Looking for variable_name, [recurse_on], [recursion_depth], [items_prop] + Looking for greeting_var, [recurse_on], [recursion_depth], [items_prop] :param args: :param kwargs: :return: @@ -624,7 +623,8 @@ class CompiledCondition: return_value: Union[ReturnValueConcept, None] # compiled source as ReturnValue variables: Set[str] # variables that must be present in bag not_variables: Set[str] # variables that must not be present in bag - concept: Union[Concept, None] = None # compiled source as concept + objects: dict + concepts_to_reset: set # concepts to reset before every evaluation @dataclass @@ -763,12 +763,14 @@ class SheerkaRuleManager(BaseService): parsed = parsed_expr_ret.body.body rete_conditions, python_conditions = None, None if parsed_expr_ret.status: + # get the conditions as rete conditions try: rete_visitor = ReteConditionExprVisitor(context) rete_conditions = rete_visitor.get_conditions(parsed) except FailedToCompileError as err: pass + # get the conditions as sheerka conditions try: python_visitor = PythonConditionExprVisitor(context) python_conditions = python_visitor.get_conditions(parsed) @@ -1059,10 +1061,40 @@ class SheerkaRuleManager(BaseService): class GetConditionExprVisitor(ExpressionVisitor): + """ + Base class for ReteConditionExprVisitor and PythonConditionExprVisitor + """ + def __init__(self, context): self.context = context self.var_counter = 0 self.variables = {} + self.obj_counter = 0 + self.objects_mapping = {} + + def get_object_name(self, obj, objects=None): + """ + object found during the parsing are not serialized + They are kept in a dictionary and this function returns a new name for every new object + :return: + """ + if objects is None: + objects = {} + + if self.context.sheerka.is_sheerka(obj): + return "sheerka", objects + + try: + return self.objects_mapping[id(obj)], objects + except KeyError: + pass + + object_name = f"__o_{self.obj_counter:02}__" + self.obj_counter += 1 + + self.objects_mapping[id(obj)] = object_name + objects[object_name] = obj + return object_name, objects def add_variable(self, target): """ @@ -1078,7 +1110,7 @@ class GetConditionExprVisitor(ExpressionVisitor): def inner_unpack_variable(self, variable_path: List[str]) -> Tuple[str, str]: """ When variable_path = a.b.c.d - returns (x0, d) if x0 = a.b.c else (a, b.c.d) + returns (x0, d) if (exist x0 = a.b.c) else (a, b.c.d) :param variable_path: :return: """ @@ -1108,24 +1140,47 @@ class GetConditionExprVisitor(ExpressionVisitor): return self.variables[path] root, attr = self.inner_unpack_variable(variable_path) - var_name = self.add_variable(root + "." + attr) - return var_name, (root, attr) + if attr: + var_name = self.add_variable(root + "." + attr) + return var_name, (root, attr) + else: + return root, (root, None) - def evaluate(self, source, eval_body): - res = evaluate(self.context, - source, - evaluators=CONDITIONS_VISITOR_EVALUATORS, - desc=None, - eval_body=eval_body, - eval_where=False, - is_question=False, - expect_success=False, - stm=None) + def evaluate_from_source(self, source, is_question=False, return_body=False): + res = evaluate_from_source(self.context, + source, + evaluators=CONDITIONS_VISITOR_EVALUATORS, + desc=None, + eval_body=not is_question, + eval_where=False, + is_question=is_question, + expect_success=False, + stm=None) res = expect_one(self.context, res) - if not res.status: - raise FailedToCompileError([f"Failed to evaluate '{source}'"]) - return res.value + if return_body: + if not res.status: + raise FailedToCompileError(res.body) + + return res.body + else: + return res + + def is_a_possible_variable(self, name): + """ + tells whether or not the name can be a variable + :param name: + :return: + """ + if self.context.sheerka.is_a_concept_name(name): + return False + + try: + eval(name, sheerka_globals) + except: + return True + + return False class ReteConditionExprVisitor(GetConditionExprVisitor): @@ -1162,29 +1217,38 @@ class ReteConditionExprVisitor(GetConditionExprVisitor): def init_or_get_variable_from_attr(self, variable_path: List[str], conditions): path = ".".join(variable_path) if path in self.variables: - return self.variables[path] + return V(self.variables[path]) root, attr = self.init_or_get_variable_from_name(variable_path, conditions) - var_name = self.add_variable(path) - variable = V(var_name) - conditions.append(Condition(root, attr, variable)) - return variable + if attr: + var_name = self.add_variable(path) + variable = V(var_name) + conditions.append(Condition(root, attr, variable)) + return variable + else: + return root def visit_VariableNode(self, expr_node: VariableNode): if expr_node.attributes_str is None: # try to recognize a concept - res = evaluate(self.context, - expr_node.name, - evaluators=CONDITIONS_VISITOR_EVALUATORS, - desc=None, - eval_body=True, - eval_where=False, - is_question=False, - expect_success=False, - stm=None) - res = expect_one(self.context, res) - if res.status and isinstance(res.value, Concept): - return self.recognize_concept(["__ret", "body"], res.value, {}) + res = self.evaluate_from_source(expr_node.name, is_question=True) + if self.context.sheerka.has_error(self.context, res, __type=BuiltinConcepts.TOO_MANY_SUCCESS): + raise FailedToCompileError([res]) + + # if expr_node.attributes_str is None: + # # try to recognize a concept + # res = evaluate_from_source(self.context, + # expr_node.name, + # evaluators=CONDITIONS_VISITOR_EVALUATORS, + # desc=None, + # eval_body=True, + # eval_where=False, + # is_question=False, + # expect_success=False, + # stm=None) + # res = expect_one(self.context, res) + # if res.status and isinstance(res.value, Concept): + # return self.recognize_concept(["__ret", "body"], res.value, {}) conditions = [] variable_name = expr_node.get_source() @@ -1204,7 +1268,7 @@ class ReteConditionExprVisitor(GetConditionExprVisitor): def visit_ComparisonNode(self, expr_node: ComparisonNode): if isinstance(expr_node.left, VariableNode): conditions = [] - value = self.evaluate(expr_node.right.get_source(), True) + value = self.evaluate_from_source(expr_node.right.get_source(), return_body=True) self.add_to_condition(expr_node.left.unpack(), value, conditions) return conditions else: @@ -1268,15 +1332,15 @@ class ReteConditionExprVisitor(GetConditionExprVisitor): return negate_conditions(exists_condition, sub_conditions) def visit_NameExprNode(self, expr_node: NameExprNode): - res = evaluate(self.context, - expr_node.get_source(), - evaluators=CONDITIONS_VISITOR_EVALUATORS, - desc=None, - eval_body=True, - eval_where=False, - is_question=False, - expect_success=False, - stm=None) + res = evaluate_from_source(self.context, + expr_node.get_source(), + evaluators=CONDITIONS_VISITOR_EVALUATORS, + desc=None, + eval_body=True, + eval_where=False, + is_question=False, + expect_success=False, + stm=None) res = expect_one(self.context, res) if res.status and isinstance(res.value, Concept): return self.recognize_concept(["__ret", "body"], res.value, {}) @@ -1295,7 +1359,7 @@ class ReteConditionExprVisitor(GetConditionExprVisitor): if not concept_as_str: return FailedToCompileError([f"Missing concept in for {variable_path}"]) - concept = self.evaluate(concept_as_str, True) + concept = self.evaluate_from_source(concept_as_str, return_body=True) else: concept = concept_to_recognize @@ -1341,9 +1405,35 @@ class PythonConditionExprVisitorObj: not_variables: set @staticmethod - def combine_with_and(left, right): + def create_function(first, last, parameters): - def create_and(a, b): + def get_function_as_text(parameter): + if parameter is None: + return f"{first}{last}" + + else: + return f"{first}{parameter}{last}" + + if parameters is None: + source = get_function_as_text(None) + return PythonConditionExprVisitorObj(source, source, {}, set(), set()) + + parameters_as_list = parameters if isinstance(parameters, list) else [parameters] + + res = [] + for obj in parameters_as_list: + res.append(PythonConditionExprVisitorObj(get_function_as_text(obj.text), + get_function_as_text(obj.source), + obj.objects, + obj.variables, + obj.not_variables)) + + return res[0] if len(res) == 1 else res + + @staticmethod + def create_and(left, right): + + def get_source(a, b): if a is None and b is None: return None @@ -1351,38 +1441,47 @@ class PythonConditionExprVisitorObj: return b if b is None or b == "": return a - return a + " and " + b + return a + " and " + b # no need to protect with parenthesis - left_as_list = left if isinstance(left, list) else [left] - right_as_list = right if isinstance(right, list) else [right] - - left_right_product = list(product(left_as_list, right_as_list)) - res = [] - for left_obj, right_obj in left_right_product: - res.append(PythonConditionExprVisitorObj(create_and(left_obj.text, right_obj.text), - create_and(left_obj.source, right_obj.source), - merge_dictionaries(left_obj.objects, right_obj.objects), - merge_sets(left_obj.variables, right_obj.variables), - merge_sets(left_obj.not_variables, right_obj.not_variables))) - - return res[0] if len(res) == 1 else res + return PythonConditionExprVisitorObj(get_source(left.text, right.text), + get_source(left.source, right.source), + merge_dictionaries(left.objects, right.objects), + merge_sets(left.variables, right.variables), + merge_sets(left.not_variables, right.not_variables)) @staticmethod - def combine_with_not(node): + def combine_with_comma(left, right): - def create_not(a): + def get_source(a, b): + if a is None and b is None: + return None + + if a is None or a == "": + return b + if b is None or b == "": + return a + return a + ", " + b # no need to protect with parenthesis + + if left is None: + return right + + return PythonConditionExprVisitorObj(get_source(left.text, right.text), + get_source(left.source, right.source), + merge_dictionaries(left.objects, right.objects), + merge_sets(left.variables, right.variables), + merge_sets(left.not_variables, right.not_variables)) + + @staticmethod + def create_not(node): + + def get_source(a): return f"not ({a})" - node_as_list = node if isinstance(node, list) else [node] - res = [] - for obj in node_as_list: - res.append(PythonConditionExprVisitorObj(create_not(obj.text), - create_not(obj.source), - obj.objects, - obj.variables, - obj.not_variables)) - - return res[0] if len(res) == 1 else res + return PythonConditionExprVisitorObj(get_source(node.text), + get_source(node.source), + node.objects, + node.variables, + node.not_variables) class PythonConditionExprVisitor(GetConditionExprVisitor): @@ -1390,13 +1489,24 @@ class PythonConditionExprVisitor(GetConditionExprVisitor): def __init__(self, context): super().__init__(context) self.know_object_variables = {} + self.check_variable_existence_only = True + self.concepts_to_reset = set() def get_conditions(self, expr_node): + self.check_variable_existence_only = True self.var_counter = 0 self.variables.clear() + self.concepts_to_reset.clear() visitor_obj = self.visit(expr_node) - if visitor_obj.source: + if self.check_variable_existence_only: + return [CompiledCondition(None, + None, + visitor_obj.variables, + visitor_obj.not_variables, + visitor_obj.objects, + self.concepts_to_reset)] + else: if self.variables: variables_definitions = "\n".join([f"{v} = {k}" for k, v in self.variables.items()]) source = variables_definitions + "\n" + visitor_obj.source @@ -1409,48 +1519,18 @@ class PythonConditionExprVisitor(GetConditionExprVisitor): if ret.status: ret.body.body.original_source = text ret.body.body.objects = visitor_obj.objects - return [CompiledCondition(PythonEvaluator.NAME, ret, visitor_obj.variables, visitor_obj.not_variables)] + return [CompiledCondition(PythonEvaluator.NAME, + ret, + visitor_obj.variables, + visitor_obj.not_variables, + visitor_obj.objects, + self.concepts_to_reset)] else: errors = ret.body.reason if self.context.sheerka.isinstance(ret.body, BuiltinConcepts.NOT_FOR_ME) \ else ret.body.body raise FailedToCompileError(errors) - else: - return [CompiledCondition(None, None, visitor_obj.variables, visitor_obj.not_variables)] - - def get_variable(self, expr_node): - """ - From a ExprNode, try to know if it refers to a bag entry or it it's a python valid name - :param expr_node: - :return: - """ - if not isinstance(expr_node, VariableNode): - return None - - var_root = expr_node.name - if var_root in self.know_object_variables: - return self.know_object_variables[var_root] - - if self.context.sheerka.fast_resolve(var_root): - self.know_object_variables[var_root] = None - return None - - python_parser = PythonParser() - ret = python_parser.parse(self.context, ParserInput(var_root)) - if not ret.status: - self.know_object_variables[var_root] = var_root - return var_root - - python_evaluator = PythonEvaluator() - ret = python_evaluator.eval(self.context, ret) - if not ret.status: - self.know_object_variables[var_root] = var_root - return var_root - - self.know_object_variables[var_root] = None - return None - def get_new_variable(self, variable_path: List[str], obj_variables): obj_variables.add(variable_path[0]) var_name, var_def = self.inner_get_new_variable(variable_path) @@ -1467,43 +1547,65 @@ class PythonConditionExprVisitor(GetConditionExprVisitor): return root + "." + attribute def visit_VariableNode(self, expr_node: VariableNode): - # try to reconize a concept if expr_node.attributes_str is None and not expr_node.name.startswith("__"): # try to recognize a concept - res = evaluate(self.context, - expr_node.name, - evaluators=CONDITIONS_VISITOR_EVALUATORS, - desc=None, - eval_body=True, - eval_where=False, - is_question=False, - expect_success=False, - stm=None) - res = expect_one(self.context, res) + res = self.evaluate_from_source(expr_node.name, is_question=True) if res.status and isinstance(res.value, Concept): - return self.recognize_concept(["__ret", "body"], res.value, {}) + self.check_variable_existence_only = False + if is_a_question(self.context, res.value): + return self.evaluate_concept_as_question(expr_node.name, res.value) + else: + return self.evaluate_concept(expr_node.name, res.value) + else: + if self.context.sheerka.has_error(self.context, res, __type=BuiltinConcepts.TOO_MANY_SUCCESS): + raise FailedToCompileError([res]) + + variable_name = expr_node.get_source() + variables = {variable_name} if not res.status else set() + return PythonConditionExprVisitorObj(variable_name, variable_name, {}, variables, set()) variable_name = expr_node.get_source() - return PythonConditionExprVisitorObj(None, None, {}, {variable_name}, set()) + variables_detected = {variable_name} if self.is_a_possible_variable(variable_name) else set() + return PythonConditionExprVisitorObj(variable_name, variable_name, {}, variables_detected, set()) def visit_ComparisonNode(self, expr_node: ComparisonNode): + self.check_variable_existence_only = False + if not isinstance(expr_node.left, VariableNode): + # KSI 2021-04-22. Not quite sure of the reason why I have this piece of code left = self.visit(expr_node.left) source = expr_node.get_source() return PythonConditionExprVisitorObj(source, source, {}, left.variables, left.not_variables) - value = self.evaluate(expr_node.right.get_source(), True) - return self.create_comparison_condition(expr_node.left.unpack(), expr_node.comp, value) + right = self.visit(expr_node.right) + if right.source in right.objects: + return self.create_comparison_condition(expr_node.left.unpack(), + expr_node.comp, + right.source, + right.objects[right.source], + right.objects, + expr_node.get_source()) + else: + value = self.evaluate_from_source(expr_node.right.get_source(), return_body=True) + object_name, objects = self.get_object_name(value) + return self.create_comparison_condition(expr_node.left.unpack(), + expr_node.comp, + object_name, + value, + objects, + expr_node.get_source()) def visit_AndNode(self, expr_node: AndNode): current_visitor_obj = self.visit(expr_node.parts[0]) for node in expr_node.parts[1:]: visitor_obj = self.visit(node) - current_visitor_obj = PythonConditionExprVisitorObj.combine_with_and(current_visitor_obj, visitor_obj) + current_visitor_obj = PythonConditionExprVisitorObj.create_and(current_visitor_obj, visitor_obj) return current_visitor_obj def visit_FunctionNode(self, expr_node: FunctionNode): + self.check_variable_existence_only = False + if expr_node.first.value == "recognize(": if not isinstance(expr_node.parameters[0].value, VariableNode): return FailedToCompileError([f"Cannot recognize '{expr_node.parameters[0].value}'"]) @@ -1512,46 +1614,48 @@ class PythonConditionExprVisitor(GetConditionExprVisitor): expr_node.parameters[1].value, {}) else: - source = expr_node.get_source() - obj_variables = set() + params_as_visitor_obj = None for param in expr_node.parameters: - if (variable := self.get_variable(param.value)) is not None: - obj_variables.add(variable) - return PythonConditionExprVisitorObj(source, source, {}, obj_variables, set()) + current_visitor_obj = self.visit(param.value) + params_as_visitor_obj = PythonConditionExprVisitorObj.combine_with_comma(params_as_visitor_obj, + current_visitor_obj) + + return PythonConditionExprVisitorObj.create_function(expr_node.first, expr_node.last, params_as_visitor_obj) def visit_NotNode(self, expr_node: NotNode): visitor_obj = self.visit(expr_node.node) - if visitor_obj.source is None: - return PythonConditionExprVisitorObj(None, None, {}, visitor_obj.not_variables, visitor_obj.variables) + if self.check_variable_existence_only: + return PythonConditionExprVisitorObj(visitor_obj.text, + visitor_obj.source, + visitor_obj.objects, + visitor_obj.not_variables, + visitor_obj.variables) - return PythonConditionExprVisitorObj.combine_with_not(visitor_obj) + return PythonConditionExprVisitorObj.create_not(visitor_obj) def visit_NameExprNode(self, expr_node: NameExprNode): - res = evaluate(self.context, - expr_node.get_source(), - evaluators=CONDITIONS_VISITOR_EVALUATORS, - desc=None, - eval_body=True, - eval_where=False, - is_question=False, - expect_success=False, - stm=None) - res = expect_one(self.context, res) - if res.status and isinstance(res.value, Concept): - return self.recognize_concept(["__ret", "body"], res.value, {}) + self.check_variable_existence_only = False + source = expr_node.get_source() + res = self.evaluate_from_source(source, is_question=False) + if res.status: + obj_name, objects = self.get_object_name(res.value) + return PythonConditionExprVisitorObj(source, obj_name, objects, set(), set()) + else: + raise FailedToCompileError([expr_node]) - def recognize_concept(self, variable_path, concept_to_recognize, concept_variables: dict): + def recognize_concept(self, variable_path, concept_to_recognize, concept_variables: dict, original_source=None): if not isinstance(concept_to_recognize, Concept): concept_as_str = concept_to_recognize.get_source() if not concept_as_str: return FailedToCompileError([f"Missing concept in for {variable_path}"]) - concept = self.evaluate(concept_as_str, True) + concept = self.evaluate_from_source(concept_as_str, return_body=True) else: concept = concept_to_recognize obj_variables = set() var_name = self.get_new_variable(variable_path, obj_variables) + objects = {} source = f"isinstance({var_name}, Concept)" @@ -1563,33 +1667,51 @@ class PythonConditionExprVisitor(GetConditionExprVisitor): source += f" and {var_name}.key == '{concept.key}'" concept_variables.update({k: v for k, v in concept.variables().items() if v is not NotInit}) + text = source for var_name, var_value in concept_variables.items(): new_var_path = variable_path.copy() new_var_path.append(var_name) - variable_condition = self.create_comparison_condition(new_var_path, ComparisonType.EQUALS, var_value) + obj_name, objects = self.get_object_name(var_value, objects) + variable_condition = self.create_comparison_condition(new_var_path, + ComparisonType.EQUALS, + obj_name, + var_value, + objects) source += " and " + variable_condition.source - obj_variables.update(variable_condition.objects) + text += " and " + variable_condition.text - return PythonConditionExprVisitorObj(source, source, {}, obj_variables, set()) + return PythonConditionExprVisitorObj(original_source or text, source, objects, obj_variables, set()) + + def evaluate_concept_as_question(self, original_text, concept): + concept_var_name, objects = self.get_object_name(concept) + source = f"evaluate_question({concept_var_name})" + variables = get_possible_variables_from_concept(self.context, concept) + self.concepts_to_reset.add(concept) + return PythonConditionExprVisitorObj(original_text, source, objects, variables, set()) + + def evaluate_concept(self, original_text, concept): + ensure_evaluated(self.context, concept, eval_body=True) + source, objects = self.get_object_name(concept) + return PythonConditionExprVisitorObj(original_text, source, objects, set(), set()) + + def create_comparison_condition(self, left_path, op, right_name, right_value, objects, original_source=None): + possible_variables = set() + var_root, var_attr = self.unpack_variable(left_path, possible_variables) + left = self.construct_variable(var_root, var_attr) + + if original_source is None: + right = get_safe_str_value(right_value) + original_source = ComparisonNode.rebuild_source(left, op, right) - def create_comparison_condition(self, var_path, op, value): - obj_variables = set() if op == ComparisonType.EQUALS: - if self.context.sheerka.is_sheerka(value): - var_root, var_attr = self.unpack_variable(var_path, obj_variables) - source = f"is_sheerka({self.construct_variable(var_root, var_attr)})" - return PythonConditionExprVisitorObj(source, source, {}, obj_variables, set()) - if isinstance(value, Concept): - return self.recognize_concept(var_path, value, {}) + if self.context.sheerka.is_sheerka(right_value): + source = f"is_sheerka({left})" + return PythonConditionExprVisitorObj(original_source, source, objects, possible_variables, set()) + if isinstance(right_value, Concept): + return self.recognize_concept(left_path, right_value, {}, original_source) else: - if isinstance(value, str): - value = "'" + value + "'" - var_root, var_attr = self.unpack_variable(var_path, obj_variables) - source = ComparisonNode.rebuild_source(self.construct_variable(var_root, var_attr), op, value) - return PythonConditionExprVisitorObj(source, source, {}, obj_variables, set()) + source = ComparisonNode.rebuild_source(left, op, right_name) + return PythonConditionExprVisitorObj(original_source, source, objects, possible_variables, set()) else: - if isinstance(value, str): - value = "'" + value + "'" - var_root, var_attr = self.unpack_variable(var_path, obj_variables) - source = ComparisonNode.rebuild_source(self.construct_variable(var_root, var_attr), op, value) - return PythonConditionExprVisitorObj(source, source, {}, obj_variables, set()) + source = ComparisonNode.rebuild_source(left, op, right_name) + return PythonConditionExprVisitorObj(original_source, source, objects, possible_variables, set()) diff --git a/src/core/sheerka/services/SheerkaVariableManager.py b/src/core/sheerka/services/SheerkaVariableManager.py index 7e14cde..f4d5369 100644 --- a/src/core/sheerka/services/SheerkaVariableManager.py +++ b/src/core/sheerka/services/SheerkaVariableManager.py @@ -94,7 +94,6 @@ class SheerkaVariableManager(BaseService): if who in self.bound_variables and key in self.bound_variables[who]: service = self.sheerka if who == self.sheerka.name else self.sheerka.services[who] setattr(service, key, value) - print(f"{service=} is set") def load_var(self, who, key): variable = self.sheerka.om.get(self.VARIABLES_ENTRY, who + "|" + key) diff --git a/src/core/utils.py b/src/core/utils.py index 03a615b..759eb41 100644 --- a/src/core/utils.py +++ b/src/core/utils.py @@ -4,6 +4,7 @@ import inspect import os import pkgutil import re +import warnings from copy import deepcopy # from pyparsing import * @@ -626,6 +627,35 @@ def as_bag(obj, forced_properties=None): return bag +def sheerka_getattr(obj, name): + """ + Wrapper to builtins.getattr that supports objects with as_bag signature + :param obj: + :param name: + :return: + """ + if hasattr(obj, "as_bag"): + try: + return obj.as_bag()[name] + except KeyError: + raise AttributeError(f"'{type(obj).__name__}' object has no attribute '{name}'") + + return getattr(obj, name) + + +def sheerka_hasattr(obj, name): + """ + Wrapper to builtins.hasattr that supports objects with as_bag signature + :param obj: + :param name: + :return: + """ + if hasattr(obj, "as_bag"): + return name in obj.as_bag() + + return hasattr(obj, name) + + def flatten_all_children(item, get_children): """ Return a list containing the current item and all its children, recursively @@ -775,6 +805,10 @@ def escape_str(x): :param x: :return: """ + warnings.warn( + "use get_safe_str_value() instead", + DeprecationWarning + ) if isinstance(x, str): return f"'{x}'" return x @@ -802,7 +836,7 @@ class NextIdManager: def compute_hash(obj): - if isinstance(obj, list): + if isinstance(obj, (list, tuple)): return hash(tuple([compute_hash(o) for o in obj])) if isinstance(obj, set): @@ -812,3 +846,21 @@ def compute_hash(obj): return hash(repr(obj)) return hash(obj) + + +def get_safe_str_value(obj): + from sheerkapython.python_wrapper import Expando + from core.concept import Concept + if isinstance(obj, str): + if "'" in obj: + value = escape_char(obj, '"') + return f'"{value}"' + else: + value = escape_char(obj, "'") + return f"'{value}'" + elif isinstance(obj, Expando): + return obj.get_name() + elif isinstance(obj, Concept): + return obj.str_id + + return str(obj) diff --git a/src/evaluators/DefConceptEvaluator.py b/src/evaluators/DefConceptEvaluator.py index bafdd57..8a016a5 100644 --- a/src/evaluators/DefConceptEvaluator.py +++ b/src/evaluators/DefConceptEvaluator.py @@ -241,7 +241,7 @@ class DefConceptEvaluator(OneReturnValueEvaluator): names = [str(t.value) for t in ret_value.tokens if t.type in (TokenKind.IDENTIFIER, TokenKind.STRING, TokenKind.KEYWORD)] - possible_vars = filter(lambda x: x in concept_name and context.sheerka.is_not_a_variable(x), names) + possible_vars = filter(lambda x: x in concept_name and context.sheerka.is_not_a_concept_name(x), names) debugger.debug_var("names", names, hint="from NameNode") debugger.debug_var("possible_vars", possible_vars, hint="from NameNode") return [PossibleVariable(v) for v in possible_vars] @@ -262,7 +262,7 @@ class DefConceptEvaluator(OneReturnValueEvaluator): if len(concept_name) > 1: visitor = UnreferencedVariablesVisitor(context) names = visitor.get_names(python_node.ast_) - possible_vars = filter(lambda x: x in concept_name and context.sheerka.is_not_a_variable(x), names) + possible_vars = filter(lambda x: x in concept_name and context.sheerka.is_not_a_concept_name(x), names) debugger.debug_var("names", names, hint="from python node") debugger.debug_var("possible_vars", possible_vars, hint="from python node") return [PossibleVariable(v) for v in possible_vars] @@ -275,7 +275,7 @@ class DefConceptEvaluator(OneReturnValueEvaluator): if (concept := get_inner_concept(ret_value.value)) is not None and len(concept_name) > 1: # use the variables of the concept is any names = [var_value or var_name for var_name, var_value in concept.get_metadata().variables] - possible_vars = filter(lambda x: context.sheerka.is_not_a_variable(x), names) + possible_vars = filter(lambda x: context.sheerka.is_not_a_concept_name(x), names) debugger.debug_var("names", names, hint="from concept") debugger.debug_var("possible_vars", possible_vars, hint="from concept") return [PossibleVariable(v) for v in possible_vars] @@ -296,7 +296,7 @@ class DefConceptEvaluator(OneReturnValueEvaluator): else: names.append(t.str_value) - possible_vars = filter(lambda x: context.sheerka.is_not_a_variable(x), names) + possible_vars = filter(lambda x: context.sheerka.is_not_a_concept_name(x), names) debugger.debug_var("names", names, hint="from source") debugger.debug_var("possible_vars", possible_vars, hint="from source") return [PossibleVariable(v) for v in possible_vars] diff --git a/src/evaluators/PythonEvaluator.py b/src/evaluators/PythonEvaluator.py index 36c6ac2..240025c 100644 --- a/src/evaluators/PythonEvaluator.py +++ b/src/evaluators/PythonEvaluator.py @@ -8,50 +8,15 @@ import core.utils from core.ast_helpers import UnreferencedNamesVisitor, NamesWithAttributesVisitor from core.builtin_concepts import BuiltinConcepts, ParserResultConcept from core.concept import ConceptParts, Concept -from core.global_symbols import NotInit, NotFound -from core.rule import Rule -from core.sheerka.ExecutionContext import ExecutionContext +from core.global_symbols import NotInit, ErrorObj from core.sheerka.services.SheerkaMemory import SheerkaMemory -from core.tokenizer import Token, TokenKind -from core.var_ref import VariableRef from evaluators.BaseEvaluator import OneReturnValueEvaluator from parsers.PythonParser import PythonNode - -TO_DISABLED = ["breakpoint", "callable", "compile", "delattr", "eval", "exec", "exit", "input", "locals", "open", - "print", "quit", "setattr"] - - -def inject_context(context): - """ - function Decorator used to inject the context in methods that needed - :param context: - :return: - """ - - def wrapped(func): - def inner(*args, **kwargs): - return func(context, *args, **kwargs) - - return inner - - return wrapped - - -class Expando: - def __init__(self, name, bag): - self.__name = name - for k, v in bag.items(): - setattr(self, k, v) - - def __repr__(self): - return f"{vars(self)}" - - def get_name(self): - return self.__name +from sheerkapython.python_wrapper import create_namespace, MethodAccessError @dataclass -class PythonEvalError: +class PythonEvalError(ErrorObj): error: Exception source: str traceback: str = field(repr=False) @@ -72,6 +37,9 @@ class PythonEvalError: def __hash__(self): return hash(self.error) + def get_error(self): + return self.error + class PythonEvaluator(OneReturnValueEvaluator): NAME = "Python" @@ -83,6 +51,7 @@ class PythonEvaluator(OneReturnValueEvaluator): def __init__(self): super().__init__(self.NAME, [BuiltinConcepts.EVALUATION], 50) + self.isinstance = None @staticmethod def initialize(sheerka): @@ -107,26 +76,28 @@ class PythonEvaluator(OneReturnValueEvaluator): context.log(f"Evaluating python node {node}.", self.name) # If we evaluate a Concept metadata which is NOT the body ex (pre, post, where...) - # We need to disable the function that may alter the state + # or if EVAL_QUESTION_REQUESTED is explicit + # We need to disable the functions that may alter the state # It's a poor way to have source code security check - attr_under_eval = context.get_parents(lambda ec: ec.action == BuiltinConcepts.EVALUATING_ATTRIBUTE) - if attr_under_eval: - attr_under_eval = attr_under_eval[0] - expression_only = attr_under_eval.action_context != ConceptParts.BODY + expression_only = context.in_context(BuiltinConcepts.EVAL_QUESTION_REQUESTED) - if expression_only and isinstance(node.ast_, ast.Module): - # Module execution is forbidden in where, pre, post and ret concept parts - security_error = sheerka.new(BuiltinConcepts.PYTHON_SECURITY_ERROR, - prop=attr_under_eval.action_context, - body=node.source) - return sheerka.ret(self.name, False, security_error, parents=[return_value]) - else: - expression_only = False + if not expression_only: + attr_under_eval = context.get_parents(lambda ec: ec.action == BuiltinConcepts.EVALUATING_ATTRIBUTE) + if attr_under_eval: + attr_under_eval = attr_under_eval[0] + expression_only = attr_under_eval.action_context != ConceptParts.BODY # get globals - my_globals = self.get_globals(context, node, expression_only) + try: + my_globals = self.get_globals(context, node, expression_only) + debugger.debug_var("globals", my_globals) + except MethodAccessError as ex: + eval_error = PythonEvalError(ex, + node.source, + traceback.format_exc() if get_trace_back else None, + None) - debugger.debug_var("globals", my_globals) + return sheerka.ret(self.name, False, sheerka.err(eval_error), parents=[return_value]) all_possible_globals = self.get_all_possible_globals(context, my_globals) concepts_entries = None # entries in globals_ that refers to Concept objects @@ -189,104 +160,16 @@ class PythonEvaluator(OneReturnValueEvaluator): """ unreferenced_names_visitor = UnreferencedNamesVisitor(context) names = unreferenced_names_visitor.get_names(node.ast_) - return self.get_globals_by_names(context, names, node, expression_only) + if "sheerka" in names: + sheerka_names = set() + visitor = NamesWithAttributesVisitor() + for sequence in visitor.get_sequences(node.ast_, "sheerka"): + if len(sequence) > 1: + sheerka_names.add(sequence[1]) + else: + sheerka_names = None - def get_sheerka_method(self, context, name, expression_only): - try: - method = context.sheerka.sheerka_methods[name] - context.log(f"Resolving '{name}'. It's a sheerka method.", self.name) - if expression_only and method.has_side_effect: - context.log(f"...but with side effect when {expression_only=}. Discarding.", self.name) - return None - else: - return inject_context(context)(method.method) if name in context.sheerka.methods_with_context \ - else method.method - except KeyError: - return None - - def get_globals_by_names(self, context, names, node, expression_only): - my_globals = { - "Concept": core.concept.Concept, - "BuiltinConcepts": core.builtin_concepts.BuiltinConcepts, - "Expando": Expando, - "ExecutionContext": ExecutionContext, - "in_context": context.in_context, - "SyaAssociativity": core.global_symbols.SyaAssociativity - } - - for name in names: - if name in my_globals: - continue - - if expression_only and name in TO_DISABLED: - my_globals[name] = None - continue - - # need to add it manually to avoid conflict with sheerka.isinstance - if name == "isinstance": - my_globals["isinstance"] = PythonEvaluator.isinstance - continue - - # support reference to sheerka - if name.lower() == "sheerka": - bag = {} - visitor = NamesWithAttributesVisitor() - for sequence in visitor.get_sequences(node.ast_, "sheerka"): - if (len(sequence) > 1 and - (method := self.get_sheerka_method(context, sequence[1], expression_only)) is not None): - bag[sequence[1]] = method - my_globals[name] = Expando("sheerka", bag) - continue - - # search in short term memory - if (obj := context.get_from_short_term_memory(name)) is not NotFound: - context.log(f"Resolving '{name}'. Using value found in STM.", self.name) - my_globals[name] = obj - continue - - # search in memory - if (obj := context.sheerka.get_last_from_memory(context, name)) is not NotFound: - context.log(f"Resolving '{name}'. Using value found in Long Term Memory.", self.name) - my_globals[name] = obj.obj - continue - - # search in sheerka methods - if (method := self.get_sheerka_method(context, name, expression_only)) is not None: - my_globals[name] = method - continue - - # search in context.obj (to replace by short time memory ?) - if context.obj: - if name == "self": - my_globals["self"] = context.obj - continue - - try: - attribute = context.obj.variables()[name] - if attribute != NotInit: - my_globals[name] = attribute - continue - context.log(f"Resolving '{name}'. It's obj attribute (obj={context.obj}).", self.name) - except KeyError: - pass - - # search in current node (if the name was found during the parsing) - if name in node.objects: - context.log(f"Resolving '{name}'. Using value from node.", self.name) - obj = self.resolve_object(context, node.objects[name]) - - # at last, try to instantiate a new concept - else: - context.log(f"Resolving '{name}'. Instantiating new concept.", self.name) - obj = self.resolve_object(context, name) - - if obj is None: - context.log(f"...'{name}' is not found or cannot be instantiated. Skipping.", self.name) - continue - - my_globals[name] = obj - - return my_globals + return create_namespace(context, self.name, names, sheerka_names, node.objects, expression_only) @staticmethod def get_all_possible_globals(context, my_globals): @@ -303,7 +186,7 @@ class PythonEvaluator(OneReturnValueEvaluator): :return: """ - # first pass, get all the non concept or concept with no body + # first pass, get all the non concepts or concepts with no body # Note that we consider that all concepts are evaluated # In the future, it may be a good optimisation to defer the evaluation of the body # until the python evaluation fails @@ -330,42 +213,6 @@ class PythonEvaluator(OneReturnValueEvaluator): def get_concepts_values_from_globals(my_globals, names): return {name: my_globals[name] for name in names} - @staticmethod - def resolve_object(context, name): - """ - Try to find a concept by its name, id or the pattern c:key|id: - :param context: - :param name: - :return: - """ - - if isinstance(name, VariableRef): - return getattr(name.obj, name.prop) - - if isinstance(name, Rule): - return context.sheerka.resolve_rule(context, name) - - if isinstance(name, Concept): - name = core.builtin_helpers.ensure_evaluated(context, name) - return name - - if isinstance(name, Token) and name.type == TokenKind.RULE: - return context.sheerka.resolve_rule(context, name) - - if isinstance(name, tuple): - raise Exception() - - # try to resolve by name - concept = context.sheerka.fast_resolve(name) - if concept is None: - return None - - if hasattr(concept, "__iter__"): - raise NotImplementedError("Too many concepts") - - concept = core.builtin_helpers.ensure_evaluated(context, concept) - return concept - @staticmethod def expr_to_expression(expr): expr.lineno = 0 diff --git a/src/parsers/BaseNodeParser.py b/src/parsers/BaseNodeParser.py index 3443ea8..2902dd4 100644 --- a/src/parsers/BaseNodeParser.py +++ b/src/parsers/BaseNodeParser.py @@ -210,7 +210,7 @@ class ConceptNode(LexerNode): def as_bag(self): """ Creates a dictionary with the useful properties of the ConceptNode - see Concept.as_bag() for extra informations + see Concept.as_bag() for extra information """ bag = {} for k, v in self.__dict__.items(): diff --git a/src/parsers/PythonParser.py b/src/parsers/PythonParser.py index 22864b0..96cd40f 100644 --- a/src/parsers/PythonParser.py +++ b/src/parsers/PythonParser.py @@ -6,7 +6,7 @@ import core.utils from core.builtin_concepts import BuiltinConcepts from core.sheerka.services.SheerkaExecute import ParserInput from core.tokenizer import TokenKind -from parsers.BaseParser import BaseParser, Node, ParsingError, BaseParserInputParser +from parsers.BaseParser import Node, ParsingError, BaseParserInputParser log = logging.getLogger(__name__) @@ -24,8 +24,8 @@ class PythonErrorNode(ParsingError): source: str exception: Exception - # def __post_init__(self): - # self.log.debug("-> PythonErrorNode: " + str(self.exception)) + def __hash__(self): + return hash((self.source, self.exception)) @dataclass() diff --git a/src/sdp/sheerkaDataProvider.py b/src/sdp/sheerkaDataProvider.py index 05fa215..c1a7cab 100644 --- a/src/sdp/sheerkaDataProvider.py +++ b/src/sdp/sheerkaDataProvider.py @@ -29,10 +29,10 @@ class Event(object): Class that represents something that modifies the state of the system """ - def __init__(self, message="", user_id="", date=datetime.now(), parents=None): + def __init__(self, message="", user_id="", date=None, parents=None): self.version = 1 # if the class Event ever changes, to keep track of the version self.user_id = user_id # id of the user that triggers the modification - self.date = date # when + self.date = date or datetime.now() # when self.message = message # user input or whatever that modifies the system self.parents = parents # digest(s) of the parent(s) of this event self._digest = None # digest of the event diff --git a/src/sheerkapython/__init__.py b/src/sheerkapython/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/sheerkapython/python_wrapper.py b/src/sheerkapython/python_wrapper.py new file mode 100644 index 0000000..a15cee6 --- /dev/null +++ b/src/sheerkapython/python_wrapper.py @@ -0,0 +1,279 @@ +import functools +from dataclasses import dataclass + +import core.builtin_helpers +from core.builtin_concepts_ids import BuiltinConcepts +from core.concept import Concept +from core.global_symbols import SyaAssociativity, NotFound, NotInit, ErrorObj +from core.rule import Rule +from core.sheerka.ExecutionContext import ExecutionContext +from core.sheerka.services.SheerkaAdmin import SheerkaAdmin +from core.tokenizer import Token, TokenKind +from core.utils import sheerka_hasattr, sheerka_getattr +from core.var_ref import VariableRef + +TO_DISABLED = ["breakpoint", "callable", "compile", "delattr", "eval", "exec", "exit", "input", "locals", "open", + "print", "quit", "setattr"] + + +@dataclass +class MethodAccessError(Exception, ErrorObj): + method_name: str + + +class ObjectContainer: + """ + Container for list of object (or whatever), to easily use SheerkaQueryLanguage on collection + """ + + def __init__(self, items): + self.items = items + + +class Expando: + def __init__(self, name, bag): + self.__name = name + for k, v in bag.items(): + setattr(self, k, v) + + def __repr__(self): + return f"{vars(self)}" + + def get_name(self): + return self.__name + + def __eq__(self, other): + if id(other) == id(self): + return True + + if not isinstance(other, Expando): + return False + + if other.get_name() != self.get_name(): + return False + + for k, v in vars(self).items(): + if getattr(other, k) != v: + return False + + return True + + def __hash__(self): + hash_content = [self.__name] + list(vars(self).keys()) + return hash(tuple(hash_content)) + + +class Pipe: + """ + https://github.com/JulienPalard/Pipe/pull/23 + Represent a Pipeable Element : + Described as : + first = Pipe(lambda iterable: next(iter(iterable))) + and used as : + print [1, 2, 3] | first + printing 1 + Or represent a Pipeable Function : + It's a function returning a Pipe + Described as : + select = Pipe(lambda iterable, predicate: (predicate(x) for x in iterable)) + and used as : + print [1, 2, 3] | select(lambda x: x * 2) + # 2, 4, 6 + """ + + def __init__(self, function): + self.function = function + functools.update_wrapper(self, function) + + def __ror__(self, other): + return self.function(other) + + def __call__(self, *args, **kwargs): + return Pipe(lambda x: self.function(x, *args, **kwargs)) + + +def get_type(obj): + if isinstance(obj, Concept): + return obj.name + else: + return type(obj).__name__ + + +sheerka_globals = { + "Concept": Concept, + "BuiltinConcepts": BuiltinConcepts, + "Expando": Expando, + "ExecutionContext": ExecutionContext, + "SyaAssociativity": SyaAssociativity, + "get_type": get_type, + "hasattr": sheerka_hasattr, + "getattr": sheerka_getattr, +} + + +def inject_context(context): + """ + function Decorator used to inject the context in methods that needed + :param context: + :return: + """ + + def wrapped(func): + @functools.wraps(func) + def inner(*args, **kwargs): + return func(context, *args, **kwargs) + + return inner + + return wrapped + + +def resolve_object(context, who, obj): + """ + Try to find a concept by its obj, id or the pattern c:key|id: + :param context: + :param who: + :param obj: + :return: + """ + + if isinstance(obj, VariableRef): + return getattr(obj.obj, obj.prop) + + if isinstance(obj, Rule): + return context.sheerka.resolve_rule(context, obj) + + if isinstance(obj, Concept): + obj = core.builtin_helpers.ensure_evaluated(context, obj) + return obj + + if isinstance(obj, Token) and obj.type == TokenKind.RULE: + return context.sheerka.resolve_rule(context, obj) + + if isinstance(obj, tuple): + # To make sure that there is no tuple that resembles to a concept + raise Exception() + + if (isinstance(obj, str) and obj.startswith("c:")) or isinstance(obj, Token): + concept = context.sheerka.fast_resolve(obj) + if concept is None: + return None + + if hasattr(concept, "__iter__"): + raise NotImplementedError("Too many concepts") + + concept = core.builtin_helpers.ensure_evaluated(context, concept) + return concept + + return obj + + +def get_sheerka_method(context, who, name, expression_only): + try: + method = context.sheerka.sheerka_methods[name] + context.log(f"Resolving '{name}'. It's a sheerka method.", who) + if expression_only and method.has_side_effect: + context.log(f"...but with side effect when {expression_only=}. Discarding.", who) + raise MethodAccessError(name) + else: + method_to_use = inject_context(context)(method.method) if name in context.sheerka.methods_with_context \ + else method.method + + if name in context.sheerka.pipe_functions: + return Pipe(method_to_use) + else: + return method_to_use + except KeyError: + return None + + +def create_namespace(context, who, names, sheerka_names, objects, expression_only, allow_builtins=False): + """ + Create a namespace for the requested names + :param context: + :param who: who is asking + :param names: requested names + :param sheerka_names: requested sheerka names (ex sheerka.isinstance) + :param objects: local objects that can be added + :param expression_only: if true, discard method that can alter the global state + :param allow_builtins: automatically add python builtins symbols + :return: + """ + result = dict(__builtins__) if allow_builtins else {} + + for name in names: + if name in sheerka_globals: + result[name] = sheerka_globals[name] + continue + + if expression_only and name in TO_DISABLED: + result[name] = None + continue + + if name == "in_context": + result[name] = context.in_context + continue + + # need to add it manually to avoid conflict with sheerka.isinstance + if name == "isinstance": + result["isinstance"] = context.sheerka.services[SheerkaAdmin.NAME].extended_isinstance + continue + + # support reference to sheerka + if name.lower() == "sheerka": + bag = {} + for sheerka_name in sheerka_names: + if (method := get_sheerka_method(context, who, sheerka_name, expression_only)) is not None: + bag[sheerka_name] = method + result[name] = Expando("sheerka", bag) + continue + + # search in short term memory + if (obj := context.get_from_short_term_memory(name)) is not NotFound: + context.log(f"Resolving '{name}'. Using value found in STM.", who) + result[name] = obj + continue + + # search in memory + if (obj := context.sheerka.get_last_from_memory(context, name)) is not NotFound: + context.log(f"Resolving '{name}'. Using value found in Long Term Memory.", who) + result[name] = obj.obj + continue + + # search in sheerka methods + if (method := get_sheerka_method(context, who, name, expression_only)) is not None: + result[name] = method + continue + + # search in context.obj (to replace by short time memory ?) + if context.obj: + if name == "self": + result["self"] = context.obj + continue + + try: + attribute = context.obj.variables()[name] + if attribute != NotInit: + result[name] = attribute + continue + context.log(f"Resolving '{name}'. It's obj attribute (obj={context.obj}).", who) + except KeyError: + pass + + # search in current node (if the name was found during the parsing) + if name in objects: + context.log(f"Resolving '{name}'. Using value from node.", who) + obj = resolve_object(context, who, objects[name]) + + # at last, try to instantiate a new concept + else: + context.log(f"Resolving '{name}'. Instantiating new concept.", who) + obj = resolve_object(context, who, f"c:{name}:") + + if obj is None: + context.log(f"...'{name}' is not found or cannot be instantiated. Skipping.", who) + continue + + result[name] = obj + + return result diff --git a/src/sheerkaql/OrderedSet.py b/src/sheerkaql/OrderedSet.py new file mode 100644 index 0000000..27517b4 --- /dev/null +++ b/src/sheerkaql/OrderedSet.py @@ -0,0 +1,80 @@ +from __future__ import print_function + +import collections +from builtins import next +from builtins import range + +# OrderedSet by Raymond Hettinger +# Active State Recipe 576694 +# Licensed Under the MIT License see the LICENSE file +# see http://code.activestate.com/recipes/576694/ + +KEY, PREV, NEXT = list(range(3)) + + +class OrderedSet(collections.MutableSet): + + def __init__(self, iterable=None): + self.end = end = [] + end += [None, end, end] # sentinel node for doubly linked list + self.map = {} # key --> [key, prev, next] + if iterable is not None: + self |= iterable + + def __len__(self): + return len(self.map) + + def __contains__(self, key): + return key in self.map + + def add(self, key): + if key not in self.map: + end = self.end + curr = end[PREV] + curr[NEXT] = end[PREV] = self.map[key] = [key, curr, end] + + def discard(self, key): + if key in self.map: + key, prev, next = self.map.pop(key) + prev[NEXT] = next + next[PREV] = prev + + def __iter__(self): + end = self.end + curr = end[NEXT] + while curr is not end: + yield curr[KEY] + curr = curr[NEXT] + + def __reversed__(self): + end = self.end + curr = end[PREV] + while curr is not end: + yield curr[KEY] + curr = curr[PREV] + + def pop(self, last=True): + if not self: + raise KeyError('set is empty') + key = next(reversed(self)) if last else next(iter(self)) + self.discard(key) + return key + + def __repr__(self): + if not self: + return '%s()' % (self.__class__.__name__,) + return '%s(%r)' % (self.__class__.__name__, list(self)) + + def __eq__(self, other): + if other is None: return False + if isinstance(other, OrderedSet): + return len(self) == len(other) and list(self) == list(other) + return set(self) == set(other) + + def __del__(self): + self.clear() # remove circular references + + +if __name__ == '__main__': + print(OrderedSet('abracadaba')) + print(OrderedSet('simsalabim')) diff --git a/src/sheerkaql/SheerkaQueryLangage.py b/src/sheerkaql/SheerkaQueryLangage.py new file mode 100644 index 0000000..81302cd --- /dev/null +++ b/src/sheerkaql/SheerkaQueryLangage.py @@ -0,0 +1,26 @@ +from sheerkaql.lexer import Lexer +from sheerkaql.parser import Parser + +lexer_instance = Lexer() + + +class SheerkaQueryLanguage: + + def compile(self, query): + """ + Compiles a query string into a python function that takes one parameter, the execution namespace. + The compiled function is re-usable. For information on the grammar see X. + """ + return Parser().parse(bytes(query, 'utf-8').decode('unicode_escape'), lexer=lexer_instance) + + def execute(self, query, namespace, allow_builtins=False): + """" + Compiles the query string and executes it with the supplied namespace. If you want to execute a + particular query many times, use compile to get a query function. + :param query: query to execute (from self.compile()) + :param namespace: execution's namespace + :param allow_builtins: auto add builtin functions and names + """ + if allow_builtins: + namespace.update(__builtins__) + return self.compile(query)(namespace) diff --git a/src/sheerkaql/__init__.py b/src/sheerkaql/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/sheerkaql/lexer.py b/src/sheerkaql/lexer.py new file mode 100644 index 0000000..c12d0a3 --- /dev/null +++ b/src/sheerkaql/lexer.py @@ -0,0 +1,157 @@ +""" +This piece of code is originally taken from pyflwor +https://github.com/timtadh/pyflwor +""" +from ply import lex +from ply.lex import Token + +tokens = ('NUMBER', 'STRING', 'NAME', + 'UNION', 'INTERSECTION', + 'AND', 'OR', + 'STAR', 'DASH', 'PLUS', 'SLASH', 'PERCENT', + "IF", "THEN", "ELSE", + 'EQEQ', 'NQ', 'LE', 'GE', 'COMMA', 'DOT', 'COLON', "NOT", 'IS', + 'LPAREN', 'RPAREN', 'LSQUARE', 'RSQUARE', 'LANGLE', 'RANGLE', 'LCURLY', 'RCURLY', + 'FOR', 'IN', 'LET', 'EQ', 'WHERE', 'RETURN', 'ORDER', 'BY', 'ASC', 'DESC', + 'FUNCTION', 'FLATTEN', 'COLLECT', 'AS', 'WITH', + 'SUBSET', 'SUPERSET', 'PROPER', + 'SOME', 'EVERY', 'SATISFIES', + + ) + +reserved = {'some': 'SOME', 'every': 'EVERY', 'in': 'IN', 'not': 'NOT', + 'satisfies': 'SATISFIES', + 'and': 'AND', 'or': 'OR', 'subset': 'SUBSET', + 'superset': 'SUPERSET', + 'proper': 'PROPER', 'is': 'IS', 'for': 'FOR', + 'let': 'LET', 'return': 'RETURN', + 'where': 'WHERE', 'order': 'ORDER', 'by': 'BY', + 'asc': 'ASC', 'desc': 'DESC', + 'function': 'FUNCTION', 'if': 'IF', 'then': 'THEN', 'else': 'ELSE', + 'flatten': 'FLATTEN', 'collect': 'COLLECT', + 'as': 'AS', 'with': 'WITH'} + +# Common Regex Parts +DIGIT = r'[0-9]' +LETTER = r'[a-zA-Z_]' +HEX = r'[a-fA-F0-9]' +EXP = r'[Ee][+-]?(' + DIGIT + ')+' + + +# Tim Henderson +# Normally PLY works at the module level. I prefer having it encapsulated as +# a class. Thus the strange construction of this class in the new method allows +# PLY to do its magic. +class Lexer: + + def __new__(cls, **kwargs): + self = super(Lexer, cls).__new__(cls, **kwargs) + self.lexer = lex.lex(object=self, debug=False, optimize=True, **kwargs) + return self.lexer + + tokens = tokens + + t_EQ = r'=' + t_EQEQ = r'==' + t_NQ = r'!=' + t_GE = r'\>=' + t_LE = r'\<=' + t_DOT = r'\.' + t_STAR = r'\*' + t_DASH = r'\-' + t_PLUS = r'\+' + t_COMMA = r',' + t_COLON = r'\:' + t_SLASH = r'/' + t_UNION = r'\|' + # t_DOLLAR = r'\$' + t_LPAREN = r'\(' + t_RPAREN = r'\)' + t_LCURLY = r'\{' + t_RCURLY = r'\}' + t_LANGLE = r'\<' + t_RANGLE = r'\>' + t_LSQUARE = r'\[' + t_RSQUARE = r'\]' + # t_DIFFERENCE = r'-' + t_INTERSECTION = r'&' + t_PERCENT = r'%' + + string_literal1 = r'\"[^"]*\"' + + @Token(string_literal1) + def t_STRING_LITERAL1(self, token): + token.type = 'STRING' + token.value = token.value[1:-1] + return token + + string_literal2 = r"\'[^']*\'" + + @Token(string_literal2) + def t_STRING_LITERAL2(self, token): + token.type = 'STRING' + token.value = token.value[1:-1] + return token + + name = '(' + LETTER + ')((' + LETTER + ')|(' + DIGIT + '))*' + + @Token(name) + def t_NAME(self, token): + if token.value in reserved: + token.type = reserved[token.value] + else: + token.type = 'NAME' + return token + + const_hex = '0[xX](' + HEX + ')+' + + @Token(const_hex) + def t_CONST_HEX(self, token): + token.type = 'NUMBER' + token.value = int(token.value, 16) + return token + + const_float1 = '(' + DIGIT + ')+' + '(' + EXP + ')' + + @Token(const_float1) + def t_CONST_FLOAT1(self, token): + token.type = 'NUMBER' + token.value = float(token.value) + return token + + const_float2 = '(' + DIGIT + r')*\.(' + DIGIT + ')+(' + EXP + ')?' + + @Token(const_float2) + def t_CONST_FLOAT2(self, token): + token.type = 'NUMBER' + token.value = float(token.value) + return token + + const_dec_oct = '(' + DIGIT + ')+' + + @Token(const_dec_oct) + def t_CONST_INTEGER_OCTAL(self, token): + token.type = 'NUMBER' + if (len(token.value) > 1 and token.value[0] == '0' or + (token.value[0] == '-' and token.value[1] == '0')): + token.value = int(token.value, 8) + else: + token.value = int(token.value, 10) + return token + + @Token(r'(/\*([^*]|[\r\n]|(\*+([^*/]|[\r\n])))*\*+/)|(//.*)') + def t_COMMENT(self, token): + lines = len(token.value.split('\n')) - 1 + if lines < 0: + lines = 0 + token.lexer.lineno += lines + + @Token(r'\n+') + def t_newline(self, t): + t.lexer.lineno += t.value.count("\n") + + # Ignored characters + t_ignore = " \t" + + def t_error(self, t): + raise Exception("Illegal character '%s'" % t) diff --git a/src/sheerkaql/parser.py b/src/sheerkaql/parser.py new file mode 100644 index 0000000..1d408ab --- /dev/null +++ b/src/sheerkaql/parser.py @@ -0,0 +1,787 @@ +""" +This piece of code is originally taken from pyflwor +https://github.com/timtadh/pyflwor +""" + +# Tim Henderson +# The parser does not build an abstract syntax tree nor does it build +# intermediate code, instead it composes functions and objects together. These +# functions are defined in symbols.py file. The composed function returned +# computes the query based on the object dictionary passed into it. This +# dictionary (objs) is passed down through the functions (sometimes with +# modification). + +# If you are confused about the syntax in this file I recommend reading the +# documentation on the PLY website to see how this compiler compiler's syntax +# works. +from ply import yacc + +from sheerkaql import symbols +from sheerkaql.lexer import tokens + + +class SingletonMeta(type): + """ + The Singleton class can be implemented in different ways in Python. Some + possible methods include: base class, decorator, metaclass. We will use the + metaclass because it is best suited for this purpose. + """ + + _instances = {} + + def __call__(cls, *args, **kwargs): + """ + Possible changes to the value of the `__init__` argument do not affect + the returned instance. + """ + if cls not in cls._instances: + instance = super().__call__(*args, **kwargs) + cls._instances[cls] = instance + return cls._instances[cls] + + +class Parser(metaclass=SingletonMeta): + + def __init__(self, **kwargs): + self.names = None # identifiers found + self.sheerka_names = None # references to sheerka methods (sheerka.something) + self.yacc = yacc.yacc(module=self, debug=False, optimize=True, + write_tables=False, **kwargs) + + def parse(self, *args, **kwargs): + self.names = set() + self.sheerka_names = set() + return self.yacc.parse(*args, **kwargs) + + tokens = tokens + precedence = ( + ('right', 'RSQUARE'), + ('right', 'DASH', 'PLUS', 'SLASH', 'STAR'), + ) + + @staticmethod + def p_start_with_set(t): + """Start : Set""" + t[0] = t[1] + + @staticmethod + def p_start_with_flwr_expression(t): + """Start : FLWRexpr""" + t[0] = t[1] + + @staticmethod + def p_for_return_expr(t): + """FLWRexpr : ForExpr ReturnExpr""" + t[0] = symbols.flwr_sequence(t[2][0], for_expr=t[1], flatten=t[2][1], collecting=t[2][2]) + + @staticmethod + def p_for_let_return_expr(t): + """FLWRexpr : ForExpr LetExpr ReturnExpr""" + t[0] = symbols.flwr_sequence(t[3][0], for_expr=t[1], flatten=t[3][1], collecting=t[3][2], let_expr=t[2]) + + @staticmethod + def p_for_where_return_expr(t): + """FLWRexpr : ForExpr WhereExpr ReturnExpr""" + t[0] = symbols.flwr_sequence(t[3][0], for_expr=t[1], flatten=t[3][1], collecting=t[3][2], where_expr=t[2]) + + @staticmethod + def p_for_let_where_return_expr(t): + """FLWRexpr : ForExpr LetExpr WhereExpr ReturnExpr""" + t[0] = symbols.flwr_sequence(t[4][0], for_expr=t[1], flatten=t[4][1], collecting=t[4][2], let_expr=t[2], + where_expr=t[3]) + + @staticmethod + def p_for_order_by_return_expr(t): + """FLWRexpr : ForExpr OrderByExpr ReturnExpr""" + t[0] = symbols.flwr_sequence(t[3][0], for_expr=t[1], flatten=t[3][1], collecting=t[3][2], order_expr=t[2]) + + @staticmethod + def p_for_let_order_by_return_expr(t): + """FLWRexpr : ForExpr LetExpr OrderByExpr ReturnExpr""" + t[0] = symbols.flwr_sequence(t[4][0], for_expr=t[1], flatten=t[4][1], collecting=t[4][2], let_expr=t[2], + order_expr=t[3]) + + @staticmethod + def p_for_where_order_by_return_expr(t): + """FLWRexpr : ForExpr WhereExpr OrderByExpr ReturnExpr""" + t[0] = symbols.flwr_sequence(t[4][0], for_expr=t[1], flatten=t[4][1], collecting=t[4][2], where_expr=t[2], + order_expr=t[3]) + + @staticmethod + def p_for_let_where_oder_by_return_expr(t): + """FLWRexpr : ForExpr LetExpr WhereExpr OrderByExpr ReturnExpr""" + t[0] = symbols.flwr_sequence(t[5][0], for_expr=t[1], flatten=t[5][1], collecting=t[5][2], let_expr=t[2], + where_expr=t[3], order_expr=t[4]) + + @staticmethod + def p_return_expr(t): + """FLWRexpr : ReturnExpr""" + t[0] = symbols.flwr_sequence(t[1][0], flatten=t[1][1], collecting=t[1][2]) + + @staticmethod + def p_let_return_expr(t): + """FLWRexpr : LetExpr ReturnExpr""" + t[0] = symbols.flwr_sequence(t[2][0], flatten=t[2][1], collecting=t[2][2], let_expr=t[1]) + + @staticmethod + def p_for_expression(t): + """ForExpr : FOR ForList""" + t[0] = t[2] + + @staticmethod + def p_for_list_with_multiple_items(t): + """ForList : ForList COMMA ForDefinition""" + t[0] = t[1] + [t[3]] + + @staticmethod + def p_for_list(t): + """ForList : ForDefinition""" + t[0] = [t[1]] + + @staticmethod + def p_for_definition_using_set(t): + """ForDefinition : NAME IN LANGLE Set RANGLE""" + t[0] = (t[1], t[4]) + + @staticmethod + def p_for_definition_using_flwr(t): + """ForDefinition : NAME IN LCURLY FLWRexpr RCURLY""" + t[0] = (t[1], t[4]) + + @staticmethod + def p_for_definition_using_value(t): + """ForDefinition : NAME IN Value""" + t[0] = (t[1], t[3]) + + @staticmethod + def p_multiple_let_expression(t): + """LetExpr : LetExpr LET LetList""" + t[0] = t[1] + t[3] + + @staticmethod + def p_let_expression(t): + """LetExpr : LET LetList""" + t[0] = t[2] + + @staticmethod + def p_let_list_from_let_definition(t): + """LetList : LetList COMMA LetDefinition""" + t[0] = t[1] + [t[3]] + + @staticmethod + def p_let_list(t): + """LetList : LetDefinition""" + t[0] = [t[1]] + + @staticmethod + def p_let_definition_from_set(t): + """LetDefinition : NAME EQ LANGLE Set RANGLE""" + t[0] = (t[1], t[4]) + + @staticmethod + def p_let_definition_from_flwr(t): + """LetDefinition : NAME EQ LCURLY FLWRexpr RCURLY""" + t[0] = (t[1], t[4]) + + @staticmethod + def p_let_definition_from_arith_expr(t): + """LetDefinition : NAME EQ ArithExpr""" + t[0] = (t[1], t[3]) + + @staticmethod + def p_let_definition_from_function(t): + """LetDefinition : NAME EQ Function""" + t[0] = (t[1], symbols.function_definition(*t[3])) + + @staticmethod + def p_function_definition_no_parameter(t): + """Function : FUNCTION LPAREN RPAREN LCURLY FBody RCURLY""" + t[0] = (tuple(), t[5]) + + @staticmethod + def p_function_definition_with_parameters(t): + """Function : FUNCTION LPAREN FParams RPAREN LCURLY FBody RCURLY""" + t[0] = (tuple(t[3]), t[6]) + + @staticmethod + def p_function_parameters(t): + """FParams : FParams COMMA NAME""" + t[0] = t[1] + [t[3]] + + @staticmethod + def p_function_parameters_from_name(t): + """FParams : NAME""" + t[0] = [t[1]] + + @staticmethod + def p_function_body_from_flwr(t): + """FBody : FLWRexpr""" + t[0] = t[1] + + @staticmethod + def p_function_body_from_arith_expr(t): + """FBody : ArithExpr""" + t[0] = t[1] + + @staticmethod + def p_where_definition(t): + """WhereExpr : WHERE Where""" + t[0] = t[2] + + @staticmethod + def p_order_by_number(t): + """OrderByExpr : ORDER BY NUMBER OrderDirection""" + t[0] = (t[3], t[4]) + + @staticmethod + def p_order_by_string(t): + """OrderByExpr : ORDER BY STRING OrderDirection""" + t[0] = (t[3], t[4]) + + @staticmethod + def p_order_direction_asc(t): + """OrderDirection : ASC""" + t[0] = 'ASC' + + @staticmethod + def p_order_direction_desc(t): + """OrderDirection : DESC""" + t[0] = 'DESC' + + @staticmethod + def p_return_definition_from_tuple(t): + """ReturnExpr : RETURN OutputTuple""" + t[0] = (t[2], False, False) + + @staticmethod + def p_return_definition_from_dict(t): + """ReturnExpr : RETURN OutputDict""" + t[0] = (t[2], False, False) + + @staticmethod + def p_return_definition_from_value(t): + """ReturnExpr : RETURN FLATTEN OutputValue""" + t[0] = ([t[3]], True, False) + + @staticmethod + def p_return_definition_from_collection(t): + """ReturnExpr : CollectList""" + t[0] = (t[1], False, True) + + @staticmethod + def p_collection_list(t): + """CollectList : CollectList Collect""" + t[0] = t[1] + [t[2]] + + @staticmethod + def p_collection_list_from_collect(t): + """CollectList : Collect""" + t[0] = [t[1]] + + @staticmethod + def p_collect_definition_from_tuple(t): + """Collect : COLLECT OutputTuple AS ArithExpr WITH CollectFunction""" + t[0] = {'value': t[2], 'as': t[4], 'with': t[6]} + + @staticmethod + def p_collect_definition_from_dict(t): + """Collect : COLLECT OutputDict AS ArithExpr WITH CollectFunction""" + t[0] = {'value': t[2], 'as': t[4], 'with': t[6]} + + @staticmethod + def p_collect_function_from_attribute(t): + """CollectFunction : AttributeValue""" + t[0] = symbols.attribute_value(t[1]) + + @staticmethod + def p_collect_function_from_function(t): + """CollectFunction : Function""" + t[0] = symbols.function_definition(*t[1]) + + @staticmethod + def p_output_as_tuple(t): + """OutputTuple : OutputTuple COMMA OutputValue""" + t[0] = t[1] + [t[3]] + + @staticmethod + def p_output_as_value(t): + """OutputTuple : OutputValue""" + t[0] = [t[1]] + + @staticmethod + def p_output_as_dict_multiple(t): + """OutputDict : OutputDict COMMA STRING COLON OutputValue""" + t[0] = t[1] + [(t[3], t[5])] + + @staticmethod + def p_output_as_dict(t): + """OutputDict : STRING COLON OutputValue""" + t[0] = [(t[1], t[3])] + + @staticmethod + def p_output_as_arith_expr(t): + """OutputValue : ArithExpr""" + t[0] = t[1] + + @staticmethod + def p_output_as_set(t): + """OutputValue : LANGLE Set RANGLE""" + t[0] = t[2] + + @staticmethod + def p_output_as_flwr(t): + """OutputValue : LCURLY FLWRexpr RCURLY""" + t[0] = t[2] + + @staticmethod + def p_set_from_diff(t): + """Set : Set DASH UnionExpr""" + t[0] = symbols.set_value(t[1], symbols.set_operator(t[2]), t[3]) + + @staticmethod + def p_set(t): + """Set : UnionExpr""" + t[0] = t[1] + + @staticmethod + def p_union_expr(t): + """UnionExpr : UnionExpr UNION IntersectionExpr""" + t[0] = symbols.set_value(t[1], symbols.set_operator(t[2]), t[3]) + + @staticmethod + def p_union_from_intersection(t): + """UnionExpr : IntersectionExpr""" + t[0] = t[1] + + @staticmethod + def p_intersection(t): + """IntersectionExpr : IntersectionExpr INTERSECTION Collection""" + t[0] = symbols.set_value(t[1], symbols.set_operator(t[2]), t[3]) + + @staticmethod + def p_intersection_from_collection(t): + """IntersectionExpr : Collection""" + t[0] = t[1] + + @staticmethod + def p_collection_from_query(t): + """Collection : Query""" + t[0] = t[1] + + @staticmethod + def p_collection_from_set(t): + """Collection : LPAREN Set RPAREN""" + t[0] = t[2] + + def p_query_start(self, t): + """Query : Query_""" + self.names.add(t[1][0][0]) + if len(t[1]) == 2 and t[1][0][0] == "sheerka": + self.sheerka_names.add(t[1][1][0]) + t[0] = symbols.query_value(t[1]) + + @staticmethod + def p_query_with_slash(t): + """Query_ : Query_ SLASH Entity""" + t[0] = t[1] + [t[3]] + + @staticmethod + def p_query_with_dot(t): + """Query_ : Query_ DOT Entity""" + t[0] = t[1] + [t[3]] + + @staticmethod + def p_query_from_entity(t): + """Query_ : Entity""" + t[0] = [t[1]] + + @staticmethod + def p_entity_from_name(t): + """Entity : NAME""" + t[0] = (t[1], None) + + @staticmethod + def p_entity_from_condition(t): + """Entity : NAME LSQUARE Where RSQUARE""" + t[0] = (t[1], symbols.where_value(t[3])) + + @staticmethod + def p_where(t): + """Where : OrExpr""" + t[0] = t[1] + + @staticmethod + def p_or_expr(t): + """OrExpr : OrExpr OR AndExpr""" + t[0] = symbols.boolean_expression_value(t[1], symbols.bool_operator(t[2]), t[3]) + + @staticmethod + def p_or_expr_from_and(t): + """OrExpr : AndExpr""" + t[0] = t[1] + + @staticmethod + def p_and_expr(t): + """AndExpr : AndExpr AND NotExpr""" + t[0] = symbols.boolean_expression_value(t[1], symbols.bool_operator(t[2]), t[3]) + + @staticmethod + def p_and_from_not(t): + """AndExpr : NotExpr""" + t[0] = t[1] + + @staticmethod + def p_not_expr(t): + """NotExpr : NOT BooleanExpr""" + t[0] = symbols.unary_expression_value(symbols.unary_operator(t[1]), t[2]) + + @staticmethod + def p_not_from_boolean(t): + """NotExpr : BooleanExpr""" + t[0] = t[1] + + @staticmethod + def p_boolean_from_comparison(t): + """BooleanExpr : CmpExpr""" + t[0] = t[1] + + @staticmethod + def p_boolean_from_quantified(t): + """BooleanExpr : QuantifiedExpr""" + t[0] = t[1] + + @staticmethod + def p_boolean_from_set(t): + """BooleanExpr : SetExpr""" + t[0] = t[1] + + @staticmethod + def p_boolean_from_arith(t): + """BooleanExpr : ArithExpr""" + t[0] = t[1] + + @staticmethod + def p_boolean_as_where_exp(t): + """BooleanExpr : LPAREN Where RPAREN""" + t[0] = t[2] + + @staticmethod + def p_comparison_expr(t): + """CmpExpr : ArithExpr CmpOp ArithExpr""" + t[0] = symbols.comparison_value(t[1], t[2], t[3]) + + @staticmethod + def p_comparison_operation(t): + """CmpOp : EQEQ + | NQ + | LANGLE + | LE + | RANGLE + | GE""" + t[0] = symbols.operator(t[1]) + + @staticmethod + def p_arith_expr(t): + """ArithExpr : AddSub""" + t[0] = t[1] + + @staticmethod + def p_term_expr_from_add(t): + """AddSub : AddSub PLUS MulDiv""" + t[0] = symbols.arith_value(t[1], symbols.arith_operator(t[2]), t[3]) + + @staticmethod + def p_term_expr_from_sub(t): + """AddSub : AddSub DASH MulDiv""" + t[0] = symbols.arith_value(t[1], symbols.arith_operator(t[2]), t[3]) + + @staticmethod + def p_term_from_factor(t): + """AddSub : MulDiv""" + t[0] = t[1] + + @staticmethod + def p_factor_expr_from_mult(t): + """MulDiv : MulDiv STAR ArithUnary""" + t[0] = symbols.arith_value(t[1], symbols.arith_operator(t[2]), t[3]) + + @staticmethod + def p_factor_expr_from_div(t): + """MulDiv : MulDiv SLASH ArithUnary""" + t[0] = symbols.arith_value(t[1], symbols.arith_operator(t[2]), t[3]) + + @staticmethod + def p_factor_expr_from_modulo(t): + """MulDiv : MulDiv PERCENT ArithUnary""" + t[0] = symbols.arith_value(t[1], symbols.arith_operator(t[2]), t[3]) + + @staticmethod + def p_factor_expr_from_unary(t): + """MulDiv : ArithUnary""" + t[0] = t[1] + + @staticmethod + def p_unary_expr(t): + """ArithUnary : Atomic""" + t[0] = t[1] + + @staticmethod + def p_unary_expr_from_negate(t): + """ArithUnary : DASH Atomic""" + t[0] = symbols.arith_value(symbols.attribute_value(-1.0, scalar=True), symbols.arith_operator('*'), t[2]) + + @staticmethod + def p_atomic_expr(t): + """Atomic : Value""" + t[0] = t[1] + + @staticmethod + def p_atomic_expr_from_arith_expr(t): + """Atomic : LPAREN ArithExpr RPAREN""" + t[0] = t[2] + + @staticmethod + def p_value_from_number(t): + """Value : NUMBER""" + t[0] = symbols.attribute_value(t[1], scalar=True) + + @staticmethod + def p_value_from_number_string(t): + """Value : STRING""" + t[0] = symbols.attribute_value(t[1], scalar=True) + + @staticmethod + def p_value_from_if_then_else(t): + """Value : IF Where THEN IfBody ELSE IfBody""" + t[0] = symbols.if_expression(t[2], t[4], t[6]) + + @staticmethod + def p_value_from_python_ternary_operator(t): + """Value : IfBody IF Where ELSE IfBody""" + t[0] = symbols.if_expression(t[3], t[1], t[5]) + + @staticmethod + def p_value_from_attritute_value(t): + """Value : AttributeValue""" + t[0] = symbols.attribute_value(t[1]) + + @staticmethod + def p_value_from_dict(t): + """Value : LCURLY NameValPairs RCURLY""" + t[0] = symbols.dict_value(t[2]) + + @staticmethod + def p_value_from_list(t): + """Value : LSQUARE ValueList RSQUARE""" + t[0] = symbols.list_value(t[2]) + + @staticmethod + def p_name_value_pairs(t): + """NameValPairs : NameValPairs COMMA NameValPair""" + t[0] = t[1] + [t[3]] + + @staticmethod + def p_name_value_pairs_from_name_value_pair(t): + """NameValPairs : NameValPair""" + t[0] = [t[1]] + + @staticmethod + def p_name_value_pair(t): + """NameValPair : ArithExpr COLON ArithExpr""" + t[0] = (t[1], t[3]) + + @staticmethod + def p_list_of_values(t): + """ValueList : ValueList COMMA ArithExpr""" + t[0] = t[1] + [t[3]] + + @staticmethod + def p_list_of_values_from_arith_expr(t): + """ValueList : ArithExpr""" + t[0] = [t[1]] + + @staticmethod + def p_if_body_from_arith_expr(t): + """IfBody : ArithExpr""" + t[0] = t[1] + + @staticmethod + def p_if_body_from_arith_expr_from_set(t): + """IfBody : LANGLE Set RANGLE""" + t[0] = t[2] + + @staticmethod + def p_if_body_from_arith_expr_from_flwr(t): + """IfBody : LCURLY FLWRexpr RCURLY""" + t[0] = t[2] + + def p_attribute_value(self, t): + """AttributeValue : AttributeValue DOT Attr""" + self.names.add(t[1][0].name) + if len(t[1]) == 1 and t[1][0].name == "sheerka": + self.sheerka_names.add(t[3].name) + t[0] = t[1] + [t[3]] + + def p_attribute_value_from_attr_expr(self, t): + """AttributeValue : Attr""" + self.names.add(t[1].name) + t[0] = [t[1]] + + @staticmethod + def p_list_of_parameters(t): + """ParameterList : ParameterList COMMA Parameter""" + t[0] = t[1] + [t[3]] + + @staticmethod + def p_list_of_parameters_from_parameter(t): + """ParameterList : Parameter""" + t[0] = [t[1]] + + @staticmethod + def p_parameter_from_arith(t): + """Parameter : ArithExpr""" + t[0] = t[1] + + @staticmethod + def p_parameter_from_set(t): + """Parameter : LANGLE Set RANGLE""" + t[0] = t[2] + + @staticmethod + def p_parameter_from_flwr(t): + """Parameter : LCURLY FLWRexpr RCURLY""" + t[0] = t[2] + + @staticmethod + def p_attribute_from_name(t): + """Attr : NAME""" + t[0] = symbols.Attribute(t[1]) + + @staticmethod + def p_attribute_from_call(t): + """Attr : NAME Call""" + t[0] = symbols.Attribute(t[1], t[2]) + + @staticmethod + def p_multiple_calls(t): + """Call : Call Call_""" + t[0] = t[1] + [t[2]] + + @staticmethod + def p_call(t): + """Call : Call_""" + t[0] = [t[1]] + + @staticmethod + def p_call_from_function_call(t): + """Call_ : Fcall""" + t[0] = t[1] + + @staticmethod + def p_call_from_index_call(t): + """Call_ : Dcall""" + t[0] = t[1] + + @staticmethod + def p_empty_function_call(t): + """Fcall : LPAREN RPAREN""" + t[0] = symbols.Call([]) + + @staticmethod + def p_function_call(t): + """Fcall : LPAREN ParameterList RPAREN""" + t[0] = symbols.Call(t[2]) + + @staticmethod + def p_index_call(t): + """Dcall : LSQUARE ArithExpr RSQUARE""" + t[0] = symbols.Call([t[2]], lookup=True) + + @staticmethod + def p_quantified_expr_from_set(t): + """QuantifiedExpr : Quantifier NAME IN LANGLE Set RANGLE SATISFIES LPAREN Where RPAREN""" + t[0] = symbols.quantified_value(t[1], t[2], t[5], t[9]) + + @staticmethod + def p_quantifier_expr_from_flwr(t): + """QuantifiedExpr : Quantifier NAME IN LCURLY FLWRexpr RCURLY SATISFIES LPAREN Where RPAREN""" + t[0] = symbols.quantified_value(t[1], t[2], t[5], t[9]) + + @staticmethod + def p_quantified_from_every(t): + """Quantifier : EVERY""" + t[0] = t[1] + + @staticmethod + def p_quantified_from_some(t): + """Quantifier : SOME""" + t[0] = t[1] + + @staticmethod + def p_set_expr_from_in(t): + """SetExpr : ArithExpr IN AttributeValue""" + t[0] = symbols.set_expression_value(t[1], symbols.set_expression_operation1('in'), symbols.attribute_value(t[3])) + + @staticmethod + def p_set_expr_from_not_in(t): + """SetExpr : ArithExpr NOT IN AttributeValue""" + t[0] = symbols.set_expression_value(t[1], symbols.set_expression_operation1('not in'), symbols.attribute_value(t[4])) + + @staticmethod + def p_set_expr_from_in_set_definition(t): + """SetExpr : ArithExpr IN LANGLE Set RANGLE""" + t[0] = symbols.set_expression_value(t[1], symbols.set_expression_operation1('in'), t[4]) + + @staticmethod + def p_set_expr_from_not_in_set_definition(t): + """SetExpr : ArithExpr NOT IN LANGLE Set RANGLE""" + t[0] = symbols.set_expression_value(t[1], symbols.set_expression_operation1('not in'), t[5]) + + @staticmethod + def p_set_expr_from_subset_of_set_definition(t): + """SetExpr : LANGLE Set RANGLE SUBSET LANGLE Set RANGLE""" + t[0] = symbols.set_expression_value(t[2], symbols.set_expression_operation2('subset'), t[6]) + + @staticmethod + def p_set_expr_from_superset_of_set_definition(t): + """SetExpr : LANGLE Set RANGLE SUPERSET LANGLE Set RANGLE""" + t[0] = symbols.set_expression_value(t[2], symbols.set_expression_operation2('superset'), t[6]) + + @staticmethod + def p_set_expr_from_proper_subset_of_set_definition(t): + """SetExpr : LANGLE Set RANGLE PROPER SUBSET LANGLE Set RANGLE""" + t[0] = symbols.set_expression_value(t[2], symbols.set_expression_operation2('proper subset'), t[7]) + + @staticmethod + def p_set_expr_from_proper_superset_of_set_definition(t): + """SetExpr : LANGLE Set RANGLE PROPER SUPERSET LANGLE Set RANGLE""" + t[0] = symbols.set_expression_value(t[2], symbols.set_expression_operation2('proper superset'), t[7]) + + @staticmethod + def p_set_expr_from_is(t): + """SetExpr : LANGLE Set RANGLE IS LANGLE Set RANGLE""" + t[0] = symbols.set_expression_value(t[2], symbols.set_expression_operation2('is'), t[6]) + + @staticmethod + def p_set_expr_from_is_not(t): + """SetExpr : LANGLE Set RANGLE IS NOT LANGLE Set RANGLE""" + t[0] = symbols.set_expression_value(t[2], symbols.set_expression_operation2('is not'), t[7]) + + @staticmethod + def p_set_expr_from_in_value_list(t): + """SetExpr : ArithExpr IN LSQUARE ValueList RSQUARE""" + t[0] = symbols.set_expression_value(t[1], symbols.set_expression_operation1('in'), symbols.list_value(t[4])) + + @staticmethod + def p_set_expr_from_not_in_value_list(t): + """SetExpr : ArithExpr NOT IN LSQUARE ValueList RSQUARE""" + t[0] = symbols.set_expression_value(t[1], symbols.set_expression_operation1('not in'), symbols.list_value(t[5])) + + @staticmethod + def p_error(t): + if t is None: + raise SyntaxError("Unexpected end of input") + else: + raise SyntaxError("Syntax error at '%s', %s.%s" % (t, + t.lineno, + t.lexpos)) diff --git a/src/sheerkaql/symbols.py b/src/sheerkaql/symbols.py new file mode 100644 index 0000000..dff2644 --- /dev/null +++ b/src/sheerkaql/symbols.py @@ -0,0 +1,670 @@ +""" +PyQuery - The Python Object Query System +Author: Tim Henderson +Contact: tim.tadh@hackthology.com +Copyright (c) 2010 All Rights Reserved. +Licensed under a BSD style license see the LICENSE file. + +File: symbols.py +Purpose: Objects and functions representing components of a query. +""" +from __future__ import absolute_import +from __future__ import division + +from builtins import object +from builtins import range +from builtins import str +from builtins import zip +from collections import deque +from itertools import product + +from core.global_symbols import NotInit +from core.utils import sheerka_getattr, sheerka_hasattr + +try: + from .OrderedSet import OrderedSet +except SystemError: + from OrderedSet import OrderedSet + + +class Attribute(object): + """ + Represents an attribute. An attribute consists of a name and a + "callchain." The callchain represent one or more function or index lookups + being performed on the attribute. + + eg. + x(1,2,3)[1]() + + translates to: + self.name = 'x' + self.callchain = [Call([1,2,3]), Call([1], True), Call([])] + """ + + def __init__(self, name, callchain=None): + self.name = name + self.callchain = callchain + + def __repr__(self): + return str(self) + + def __str__(self): + return str(self.name) + "[" + str(self.callchain) + "]" + + +class Call(object): + """ + Represent a 'Call' (for context see the documentation on the Attribute + class). A call consists of the parameters passed into the call (themselves + functions which takes the namespace (objs)) and whether not this is an item + lookup rather than a function call. + """ + + def __init__(self, params, lookup=False): + self.params = params + self.lookup = lookup + + def __repr__(self): + return str(self) + + def __str__(self): + if self.lookup: + return "__getitem__" + str(tuple(self.params)) + return "__call__" + str(tuple(self.params)) + + +class KeyValuePair(object): + """ + Represents a key,value pair for use while iterating over a dictionary. + """ + + def __init__(self, key, value): + self.key = key + self.value = value + + def __repr__(self): + return str(self) + + def __str__(self): + return "" % (self.key, self.value) + + +def attribute_value(attribute_list, scalar=False, context='locals'): + """ + Transforms a AttributeValue into its actual value. + + eg. + x.y.z().q[1].r + attribute_list = [Attribute('x'), Attribute('y'), + Attribute('z',[Call([])]), Attribute('q', [Call([1], True)]), + Attribute('r')] + + translates into the attribute lookups, function calls, and __getitem__ calls + necessary to produce a value. + + if scalar == True: + it simply returns the value stored in attribute_list + + context is no longer used and should be removed. + """ + + def expand(namespace, attr, x): + """ + Expands the the value of one attribute by looking the name up in the + namespace dict and then performing and function calls and dictionary lookups + specified in the callchain. + """ + if attr.callchain: + for call in attr.callchain: + p = list() + for param in call.params: + if isinstance(param, type(value)) and value.__code__ == param.__code__ or \ + isinstance(param, type(value)) and hasattr(param, '__objquery__'): + p.append(param(namespace)) + else: + p.append(param) + if call.lookup: + x = x.__getitem__(p[0]) + else: + x = x.__call__(*p) + return x + + def value(namespace): + """ + The computation function returned the user. Computes the actual value + of the the attribute expression when @namespace is passed in. + """ + if scalar: + return attribute_list + + attr0 = attribute_list[0] + obj = expand(namespace, attr0, namespace[attr0.name]) + for attr in attribute_list[1:]: + if sheerka_hasattr(obj, attr.name): + obj = expand(namespace, attr, sheerka_getattr(obj, attr.name)) + else: + raise AttributeError(f"'{type(obj).__name__}' object has no attribute '{attr.name}'") + return obj + + return value + + +def operator(op): + """ + Returns a function which performs comparison operations + """ + if op == '==': + return lambda x, y: x == y + if op == '!=': + return lambda x, y: x != y + if op == '<=': + return lambda x, y: x <= y + if op == '>=': + return lambda x, y: x >= y + if op == '<': + return lambda x, y: x < y + if op == '>': + return lambda x, y: x > y + raise Exception("operator %s not found" % op) + + +def arith_operator(op): + """ + Returns a function which performs arithmetic operations + """ + if op == '+': + return lambda x, y: x + y + if op == '-': + return lambda x, y: x - y + if op == '*': + return lambda x, y: x * y + if op == '/': + return lambda x, y: x / y + if op == '%': + return lambda x, y: x % y + raise Exception("operator %s not found" % op) + + +def set_operator(op): + """ + Returns a function which performs set operations + """ + if op == '|': + return lambda x, y: OrderedSet(x) | OrderedSet(y) + if op == '&': + return lambda x, y: OrderedSet(x) & OrderedSet(y) + if op == '-': + return lambda x, y: OrderedSet(x) - OrderedSet(y) + raise Exception("operator %s not found" % op) + + +def set_expression_operation1(op): + """ + Returns a function which performs scalar in set operations + """ + if op == 'in': + return lambda x, y: x in y + if op == 'not in': + return lambda x, y: x not in y + raise Exception("operator %s not found" % op) + + +def set_expression_operation2(op): + """ + Returns a function which performs set to set comparison operations + """ + if op == 'is': + return lambda x, y: x == y + if op == 'is not': + return lambda x, y: x != y + if op == 'subset': + return lambda x, y: x <= y + if op == 'superset': + return lambda x, y: x >= y + if op == 'proper subset': + return lambda x, y: x < y + if op == 'proper superset': + return lambda x, y: x > y + raise Exception("operator %s not found" % op) + + +def bool_operator(op): + """ + Returns a function which performs basic boolean (and, or) operations + """ + if op == 'and': + return lambda x, y, namespace: x(namespace) and y(namespace) + if op == 'or': + return lambda x, y, namespace: x(namespace) or y(namespace) + raise Exception("operator %s not found" % op) + + +def unary_operator(op): + """ + Returns a function which performs unary (not) operation + """ + if op == 'not': + return lambda x: not x + raise Exception("operator %s not found" % op) + + +def comparison_value(value1, op, value2): + """ + Returns a function which will calculate a where expression for a basic + comparison operation. + """ + + def where(namespace): + return op(value1(namespace), value2(namespace)) + + object.__setattr__(where, '__objquery__', True) + return where + + +def arith_value(value1, op, value2): + """ + Returns a function which will calculate a where expression for a basic + arithmetic operation. + """ + + def computation(namespace): + return op(value1(namespace), value2(namespace)) + + object.__setattr__(computation, '__objquery__', True) + return computation + + +def set_value(s1, op, s2): + """ + Returns a Query function for the result of set operations (difference, union + etc..) + """ + + def query(namespace): + return op(s1(namespace), s2(namespace)) + + object.__setattr__(query, '__objquery__', True) + return query + + +def set_expression_value(val, op, s): + """ + Returns a where function which returns the result of a value in set + operation + """ + + def where(namespace): + return op(val(namespace), s(namespace)) + + object.__setattr__(where, '__objquery__', True) + return where + + +def boolean_expression_value(value1, op, value2): + """ + returns the function which computes the result of boolean (and or) operation + """ + + def where(namespace): + return op(value1, value2, namespace) + + object.__setattr__(where, '__objquery__', True) + return where + + +def unary_expression_value(op, val): + """ + returns the function which computes the result of boolean not operation + """ + + def where(namespace): + return op(val(namespace)) + + object.__setattr__(where, '__objquery__', True) + return where + + +def boolean_value(val): + """ + returns the function which booleanizes the result of the Value function + """ + + def where(namespace): + return bool(val(namespace)) + + object.__setattr__(where, '__objquery__', True) + return where + + +def where_value(val): + """ + returns the results of a Value function. + """ + + def where(namespace): + return val(namespace) + + object.__setattr__(where, '__objquery__', True) + return where + + +def dict_value(pairs): + """ + creates a dictionary from the passed pairs after evaluation. + """ + + def as_dict(namespace): + return dict((name(namespace), value(namespace)) for name, value in pairs) + + object.__setattr__(as_dict, '__objquery__', True) + return as_dict + + +def list_value(values): + """ + creates a list from the pass objs after evaluation. + """ + + def as_list(objs): + return list(value(objs) for value in values) + + object.__setattr__(as_list, '__objquery__', True) + return as_list + + +def query_value(q): + """ + Computes a path expression. The query (@q) is a list of attribute names and + associated where expressions. The function returned computes the result when + called. + :param q: query List[(attr_name, filter_condition)] + """ + attrs = q + + def query(namespace): + def select(namespace, attrs_path): + """ + a generator which computes the actual results + :param namespace: dictionary of available objects + :param attrs_path: List[(attr_name, filter_condition)] + """ + + def add(_queue, _namespace, _index): + """ + adds the object v to the queue + push the current namespace + """ + index_to_use = _index + 1 + try: + object.__setattr__(_namespace, '_objquery__i', index_to_use) + except TypeError: + setattr(_namespace, '_objquery__i', index_to_use) + + _queue.appendleft(_namespace) + + queue = deque() + + # KSI 20210214: Why using type instead of the given dictionary ? + # -> to seamlessly use getattr() to retrieve the attribute + # -> to allow setattr(_namespace, '_objquery__i', index_to_use) + add(queue, type('base', (object,), namespace), -1) # init with the namespace + + while len(queue) > 0: + current_namespace = queue.pop() + index = object.__getattribute__(current_namespace, '_objquery__i') + attr_name, where = attrs_path[index] + + # if sheerka_hasattr(current_namespace, attr_name): # the current object has the attr + attr_value = sheerka_getattr(current_namespace, attr_name) + # it is iterable + if not isinstance(attr_value, str) and hasattr(attr_value, '__iter__'): + # # try to use where clause as an indexer + if where is not None: + try: + namespace_copy = dict(namespace) + res = where(namespace_copy) + except (NameError, KeyError, TypeError): + res = NotInit + + if res is not NotInit and type(res) != bool: + item_to_use = attr_value[res] + + # if this is the last attribute yield the obj + if index + 1 == len(attrs_path): + yield item_to_use + else: + add(queue, item_to_use, index) # otherwise add to the queue + continue + + for item in attr_value: + # add each child into the processing queue + if isinstance(attr_value, dict): + item_to_use = KeyValuePair(item, attr_value[item]) + else: + item_to_use = item + + # but only if its where condition is satisfied + if where is not None: + namespace_copy = dict(namespace) + namespace_copy.update({'self': item_to_use}) + try: + if not where(namespace_copy): + continue + except (AttributeError, TypeError, KeyError): + continue + + # if this is the last attribute yield the obj + if index + 1 == len(attrs_path): + yield item_to_use + else: + add(queue, item_to_use, index) # otherwise add to the queue + + else: # it is not iterable + if where is not None: + namespace_copy = dict(namespace) + namespace_copy.update({'self': attr_value}) + res = where(namespace_copy) + if type(res) != bool: + raise TypeError(f"'{type(attr_value).__name__}' object is not subscriptable") + + if not res: + continue + + # if this is the last attribute yield the obj + if index + 1 == len(attrs_path): + yield attr_value + else: + add(queue, attr_value, index) # otherwise add to the queue + # else try in the parent namespace + + # return OrderedSet(select(namespace, attrs)) + return list(select(namespace, attrs)) + + object.__setattr__(query, '__objquery__', True) + return query + + +def quantified_value(mode, name, s, satisfies): + """ + Processes the quantified expressions (some x in <> satisfie...) returns + the where function. + """ + + def where(namespace): + nobjs = s(namespace) # runs the first part of the query (eg. the expression) + if not nobjs: + return False # if returns and empty set then return false + if mode == 'every': + r = True + for x in nobjs: + cobjs = dict(namespace) # we have to copy the objects to not squash + # the upper namespace + cobjs.update({name: x}) + if not satisfies(cobjs): + r = False + return r + elif mode == 'some': + for x in nobjs: + cobjs = dict(namespace) + cobjs.update({name: x}) + if satisfies(cobjs): + return True + return False + raise Exception("mode '%s' is not 'every' or 'some'" % mode) + + return where + + +def flwr_sequence(return_expr, + for_expr=None, # list of (name, function_that_returns_a_collection) + let_expr=None, + where_expr=None, + order_expr=None, + flatten=False, + collecting=False): + """ + Returns the function to calculate the results of a flwr expression + """ + # print order_expr + if flatten: + assert len(return_expr) == 1 and not isinstance(return_expr[0], tuple) + assert not collecting + + # if collecting: + # target = return_expr['as'] + # reduce_function = return_expr['with'] + # return_expr = return_expr['value'] + def sequence(namespace): + def _flatten_func(items): + if not isinstance(items, (tuple, list, set)): + yield items + else: + for item in items: + if isinstance(item, (tuple, list, set)): + for j in _flatten_func(item): + yield j + else: + yield item + + def _build_yield(_namespace): + + def _build_return(obj): + try: + if len(obj) == 1 and not isinstance(obj[0], tuple): + return obj[0](_namespace) + elif isinstance(obj[0], tuple): # it has named return values + return dict((name, f(_namespace)) for name, f in obj) + else: # multiple positional return values + return tuple(f(_namespace) for f in obj) + except Exception as ex: + return ex + + if not collecting: + return _build_return(return_expr) + + # return a list of collecting information + return [ + { + 'value': _build_return(collector_def['value']), + 'as': collector_def['as'](_namespace), + 'with': collector_def['with'](_namespace) + } for collector_def in return_expr + ] + + def compute_sequence(_namespace): + # Tim Henderson + # take the cartesian product of the for expression + # note you cannot do this: + # for x in , y in + # :sadface: some day I will fix this. + # however I will only do that when I implement and optimizer + # for PyQuery otherwise it just isn't worth it. + if for_expr is not None: + obs = ([(name, obj) for obj in get_collection(_namespace)] + for name, get_collection in for_expr) + else: + # Tim Henderson + # The goal is to get the for loop to run once. this syntax does + # it. We may not have a for_expr but we want everything else + # to execute normally. + obs = [[None]] + + for items in product(*obs): + namespace_copy = dict(_namespace) + if for_expr is not None: + for name, item in items: + namespace_copy.update({name: item}) + + if let_expr: + for name, let in let_expr: + namespace_copy.update({name: let(namespace_copy)}) # calculate the let expr + + if where_expr and not where_expr(namespace_copy): + continue # skip if the where fails + + if not flatten: + yield _build_yield(namespace_copy) # single unamed return + else: + for item in _flatten_func(return_expr[0](namespace_copy)): + yield item + + if collecting: + rets = tuple(dict() for _ in range(len(return_expr))) + for collectors in compute_sequence(namespace): + for i, collector in enumerate(collectors): + _as = collector['as'] + _rf = collector['with'] + _value = collector['value'] + rets[i][_as] = _rf(rets[i].get(_as, None), _value) + + return rets[0] if len(rets) == 1 else rets + + else: + r = list(compute_sequence(namespace)) + if not r: + return tuple(r) + elif order_expr: + attr, direction = order_expr + if isinstance(attr, str): + if not isinstance(return_expr[0], tuple): + raise SyntaxError("Using a name in the order by clause when not using named return values.") + else: + if isinstance(return_expr[0], tuple): + raise SyntaxError( + "Using a number in the order by clause when not using positional return values.") + if len(return_expr) == 1 and not isinstance(return_expr[0], tuple): + keyfunc = lambda x: x + else: + keyfunc = lambda x: x[attr] + + reverse_order = direction == 'DESC' + r = sorted(r, key=keyfunc, reverse=reverse_order) + + return tuple(r) + + object.__setattr__(sequence, '__objquery__', True) + return sequence + + +def function_definition(params, query): + def flwr_function(namespace): + def function(*args): + if len(args) != len(params): + raise RuntimeError("Got wrong number of params expected %d got %d" % (len(params), len(args))) + namespace_copy = dict(namespace) + namespace_copy.update(list(zip(params, args))) + return query(namespace_copy) + + return function + + return flwr_function + + +def if_expression(condition, then, otherwise): + def if_expr(namespace): + if condition(namespace): + return then(namespace) + else: + return otherwise(namespace) + + return if_expr diff --git a/src/sheerkarete/network.py b/src/sheerkarete/network.py index 583b8c6..cea853c 100644 --- a/src/sheerkarete/network.py +++ b/src/sheerkarete/network.py @@ -9,8 +9,8 @@ from core.concept import Concept from core.global_symbols import NotInit from core.rule import Rule, ACTION_TYPE_PRINT from core.utils import as_bag -from evaluators.PythonEvaluator import Expando from sheerkapickle.utils import is_primitive +from sheerkapython.python_wrapper import Expando from sheerkarete.alpha import AlphaMemory from sheerkarete.beta import ReteNode, BetaMemory from sheerkarete.bind_node import BindNode @@ -439,14 +439,13 @@ class ReteNetwork: self.add_wme(WME(fact_id, FACT_NAME, name)) elif attribute == FACT_IS_CONCEPT: self.add_wme(WME(fact_id, FACT_IS_CONCEPT, isinstance(obj, Concept))) - elif attribute == FACT_SELF: - if isinstance(obj, FactObj): - self.add_wme(WME(fact_id, FACT_SELF, obj.value)) - else: - self.add_wme(WME(fact_id, FACT_SELF, obj)) else: try: - value = getattr(obj, attribute) + if attribute == FACT_SELF: + value = obj.value if isinstance(obj, FactObj) else obj + else: + value = getattr(obj, attribute) + if (isinstance(value, Concept) and value.key == BuiltinConcepts.SHEERKA or isinstance(value, Expando) and value.get_name() == "sheerka"): value = FACT_SHEERKA diff --git a/tests/core/test_ExecutionContext.py b/tests/core/test_ExecutionContext.py index 4687483..6185524 100644 --- a/tests/core/test_ExecutionContext.py +++ b/tests/core/test_ExecutionContext.py @@ -12,9 +12,12 @@ class TestExecutionContext(TestUsingMemoryBasedSheerka): def test_id_is_incremented_by_event_digest(self): sheerka = self.get_sheerka() - a = ExecutionContext("foo", Event("event_1"), sheerka, BuiltinConcepts.NOP, None) - b = ExecutionContext("foo", Event("event_1"), sheerka, BuiltinConcepts.NOP, None) - c = ExecutionContext("foo", Event("event_2"), sheerka, BuiltinConcepts.NOP, None) + event1 = Event("event_1") + event2 = Event("event_2") + + a = ExecutionContext("foo", event1, sheerka, BuiltinConcepts.NOP, None) + b = ExecutionContext("foo", event1, sheerka, BuiltinConcepts.NOP, None) + c = ExecutionContext("foo", event2, sheerka, BuiltinConcepts.NOP, None) d = b.push(BuiltinConcepts.NOP, None) e = c.push(BuiltinConcepts.NOP, None) @@ -177,6 +180,47 @@ class TestExecutionContext(TestUsingMemoryBasedSheerka): assert sub2.has_parent(root.id) assert not sub1.has_parent(sub2.id) assert not sub2.has_parent(sub1.id) + + def test_i_can_reset_global_hints(self): + sheerka = self.get_sheerka() + + context = ExecutionContext("foo", Event("event_1"), sheerka, BuiltinConcepts.NOP, None) + context.add_to_global_hints(BuiltinConcepts.TESTING) + context.add_to_global_hints(BuiltinConcepts.DEBUG) + + sub_context1 = context.push(BuiltinConcepts.NOP, None) + assert BuiltinConcepts.TESTING in sub_context1.global_hints + assert BuiltinConcepts.DEBUG in sub_context1.global_hints + + sub_context2 = context.push(BuiltinConcepts.NOP, None, reset_hints={BuiltinConcepts.TESTING}) + assert BuiltinConcepts.TESTING not in sub_context2.global_hints + assert BuiltinConcepts.DEBUG in sub_context2.global_hints + + sub_context3 = context.push(BuiltinConcepts.NOP, + None, + reset_hints={BuiltinConcepts.DEBUG, BuiltinConcepts.TESTING}) + assert sub_context3.global_hints == set() + + def test_i_can_reset_protected_hints(self): + sheerka = self.get_sheerka() + + context = ExecutionContext("foo", Event("event_1"), sheerka, BuiltinConcepts.NOP, None) + context.add_to_protected_hints(BuiltinConcepts.TESTING) + context.add_to_protected_hints(BuiltinConcepts.DEBUG) + + sub_context1 = context.push(BuiltinConcepts.NOP, None) + assert BuiltinConcepts.TESTING in sub_context1.protected_hints + assert BuiltinConcepts.DEBUG in sub_context1.protected_hints + + sub_context2 = context.push(BuiltinConcepts.NOP, None, reset_hints={BuiltinConcepts.TESTING}) + assert BuiltinConcepts.TESTING not in sub_context2.protected_hints + assert BuiltinConcepts.DEBUG in sub_context2.protected_hints + + sub_context3 = context.push(BuiltinConcepts.NOP, + None, + reset_hints={BuiltinConcepts.DEBUG, BuiltinConcepts.TESTING}) + assert sub_context3.protected_hints == set() + # def test_variables_are_passed_to_children_but_not_to_parents(self): # sheerka = self.get_sheerka() # diff --git a/tests/core/test_SheerkaEvaluateConcept.py b/tests/core/test_SheerkaEvaluateConcept.py index c4b6b3e..cbb7138 100644 --- a/tests/core/test_SheerkaEvaluateConcept.py +++ b/tests/core/test_SheerkaEvaluateConcept.py @@ -2,7 +2,7 @@ import pytest from core.builtin_concepts import BuiltinConcepts, ReturnValueConcept, ParserResultConcept from core.concept import Concept, DoNotResolve, ConceptParts, InfiniteRecursionResolved, \ - concept_part_value, DEFINITION_TYPE_DEF + DEFINITION_TYPE_DEF from core.global_symbols import NotInit, NotFound from core.sheerka.services.SheerkaEvaluateConcept import SheerkaEvaluateConcept from core.sheerka.services.SheerkaMemory import SheerkaMemory @@ -420,7 +420,7 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka): concept = Concept("foo", body="a") evaluated = sheerka.evaluate_concept(context, concept) assert evaluated.key == concept.key - compare_with_test_object(evaluated.body, CB("a", "concept_a")) # this test was already done + compare_with_test_object(evaluated.body, CB("a", "concept_a")) # this test was already done # so check this one. concept = Concept("foo", body="a").def_var("a", "'property_a'") @@ -784,27 +784,6 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka): assert evaluated.key == command.key assert sheerka.get_from_memory(context, "a").obj == 10 - @pytest.mark.parametrize("metadata", [ - ConceptParts.WHERE, - ConceptParts.PRE, - ConceptParts.POST, - ConceptParts.RET - ]) - def test_i_cannot_evaluate_python_statement_in_where_pre_post_ret(self, metadata, capsys): - sheerka, context, foo = self.init_concepts("foo") - setattr(foo.get_metadata(), concept_part_value(metadata), "a=10; print('10')") - foo.get_metadata().need_validation = True - - evaluated = sheerka.evaluate_concept(context, foo, eval_body=True) - captured = capsys.readouterr() - - assert sheerka.isinstance(evaluated, BuiltinConcepts.CONCEPT_EVAL_ERROR) - error = evaluated.body - assert sheerka.isinstance(error, BuiltinConcepts.PYTHON_SECURITY_ERROR) - assert error.prop == metadata - assert error.body == "a=10; print('10')" - assert captured.out == "" - def test_python_builtin_function_are_forbidden_in_where_pre_post_ret(self, capsys): # I do the test only for PRE, as it will be the same for the other ConceptPart sheerka, context, foo, bar = self.init_concepts( diff --git a/tests/core/test_SheerkaEvaluateRules.py b/tests/core/test_SheerkaEvaluateRules.py index 820bc18..ab5db60 100644 --- a/tests/core/test_SheerkaEvaluateRules.py +++ b/tests/core/test_SheerkaEvaluateRules.py @@ -9,8 +9,9 @@ from core.sheerka.Sheerka import RECOGNIZED_BY_ID, RECOGNIZED_BY_NAME from core.sheerka.services.SheerkaEvaluateRules import SheerkaEvaluateRules, LOW_PRIORITY_RULES, DISABLED_RULES from core.sheerka.services.SheerkaExecute import ParserInput from core.sheerka.services.SheerkaRuleManager import SheerkaRuleManager, CompiledCondition -from evaluators.PythonEvaluator import PythonEvaluator, Expando +from evaluators.PythonEvaluator import PythonEvaluator from parsers.PythonParser import PythonParser +from sheerkapython.python_wrapper import Expando from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka @@ -84,7 +85,7 @@ class TestSheerkaEvaluateRules(TestUsingMemoryBasedSheerka): # create fake compiled predicates parser = PythonParser() my_rule.compiled_conditions = [ - CompiledCondition(PythonEvaluator.NAME, parser.parse(context, ParserInput(exp)), set(), set(), None) + CompiledCondition(PythonEvaluator.NAME, parser.parse(context, ParserInput(exp)), set(), set(), None, set()) for exp in predicates] my_rule.metadata.is_compiled = True my_rule.metadata.is_enabled = True @@ -114,9 +115,9 @@ class TestSheerkaEvaluateRules(TestUsingMemoryBasedSheerka): assert res == {True: [r1, r3], False: [r2]} @pytest.mark.parametrize("predicate", [ - "greetings", - "c:|1001:", - "hello 'kodjo'" + "recognize(__ret.body, greetings)", + "recognize(__ret.body, c:|1001:)", + "recognize(__ret.body, hello 'kodjo')" ]) def test_i_can_evaluate_rules_when_concepts_are_not_questions(self, predicate): """ @@ -146,7 +147,7 @@ class TestSheerkaEvaluateRules(TestUsingMemoryBasedSheerka): Concept("greetings", definition="hello a", definition_type=DEFINITION_TYPE_DEF).def_var("a"), Concept("there"), create_new=True).with_format_rules( - Rule(predicate="hello there", action="")).unpack() + Rule(predicate="recognize(__ret.body, hello there)", action="")).unpack() service = sheerka.services[SheerkaEvaluateRules.NAME] there_instance = sheerka.new_from_template(there, there.key) @@ -169,7 +170,7 @@ class TestSheerkaEvaluateRules(TestUsingMemoryBasedSheerka): Concept("greetings", definition="hello a", definition_type=DEFINITION_TYPE_DEF).def_var("a"), Concept("my friend"), create_new=True).with_format_rules( - Rule(predicate="hello my friend", action="")).unpack() + Rule(predicate="recognize(__ret.body, hello my friend)", action="")).unpack() service = sheerka.services[SheerkaEvaluateRules.NAME] my_friend_instance = sheerka.new_from_template(my_friend, my_friend.key) @@ -183,7 +184,7 @@ class TestSheerkaEvaluateRules(TestUsingMemoryBasedSheerka): sheerka, context, greetings, rule = self.init_test().with_concepts( Concept("greetings", definition="hello a", definition_type=DEFINITION_TYPE_DEF).def_var("a"), create_new=True).with_format_rules( - Rule(predicate="hello sheerka", action="")).unpack() + Rule(predicate="recognize(__ret.body, hello sheerka)", action="")).unpack() service = sheerka.services[SheerkaEvaluateRules.NAME] ret = sheerka.ret("evaluator", True, sheerka.new(greetings, a=Expando("sheerka", {}))) @@ -196,7 +197,7 @@ class TestSheerkaEvaluateRules(TestUsingMemoryBasedSheerka): Concept("greetings", definition="hello a", definition_type=DEFINITION_TYPE_DEF).def_var("a"), Concept("greetings", definition="hi a", definition_type=DEFINITION_TYPE_DEF).def_var("a"), create_new=True).with_format_rules( - Rule(predicate="greetings", action="")).unpack() + Rule(predicate="recognize(__ret.body, greetings)", action="")).unpack() service = sheerka.services[SheerkaEvaluateRules.NAME] ret1 = sheerka.ret("evaluator", True, sheerka.new(g1, a="kodjo")) @@ -210,7 +211,7 @@ class TestSheerkaEvaluateRules(TestUsingMemoryBasedSheerka): def test_i_can_evaluate_concept_rule_with_the_same_name_when_the_second_concept_is_declared_after(self): sheerka, context, g1, rule, g2 = self.init_test().with_concepts( Concept("greetings", definition="hello a", definition_type=DEFINITION_TYPE_DEF).def_var("a"), - create_new=True).with_format_rules(Rule(predicate="greetings", action="")).with_concepts( + create_new=True).with_format_rules(Rule(predicate="recognize(__ret.body, greetings)", action="")).with_concepts( Concept("greetings", definition="hi a", definition_type=DEFINITION_TYPE_DEF).def_var("a")).unpack() service = sheerka.services[SheerkaEvaluateRules.NAME] diff --git a/tests/core/test_SheerkaMemory.py b/tests/core/test_SheerkaMemory.py index dcd4273..a7d10f1 100644 --- a/tests/core/test_SheerkaMemory.py +++ b/tests/core/test_SheerkaMemory.py @@ -87,7 +87,9 @@ class TestSheerkaMemory(TestUsingMemoryBasedSheerka): foo = Concept("foo") sheerka.add_to_memory(context, "a", foo) - assert sheerka.om.copy(SheerkaMemory.OBJECTS_ENTRY) == {"a": MemoryObject(context.event.get_digest(), foo)} + assert sheerka.om.copy(SheerkaMemory.OBJECTS_ENTRY) == {"a": MemoryObject(context.event.get_digest(), + context.event.date.timestamp(), + foo)} assert id(sheerka.get_from_memory(context, "a").obj) == id(foo) def test_i_can_use_memory_with_a_string(self): @@ -106,6 +108,44 @@ class TestSheerkaMemory(TestUsingMemoryBasedSheerka): assert sheerka.memory(context, Concept("foo")) == foo + def test_i_can_use_memory_with_a_query(self): + sheerka, context, foo, bar = self.init_concepts("foo", "bar") + + sheerka.add_to_memory(context, "x", foo) + sheerka.add_to_memory(context, "y", bar) + + assert sheerka.memory(context, "self.name == 'foo'") == foo + + def test_i_retrieve_the_last_entry_when_requesting_memory_with_a_query(self): + sheerka, context, foo, bar, foo2 = self.init_concepts("foo", "bar", Concept("foo", body="2")) + + sheerka.add_to_memory(context, "x", foo) + sheerka.add_to_memory(context, "y", bar) + + context2 = self.get_context(sheerka) # timestamp is newer + sheerka.add_to_memory(context2, "z", foo2) + + assert sheerka.memory(context, "self.name == 'foo'") == foo2 + + def test_i_can_look_in_the_previous_objects_when_using_query_to_request_the_memory(self): + sheerka, context, foo, bar, baz = self.init_concepts("foo", "bar", "baz") + + sheerka.add_to_memory(context, "x", foo) + sheerka.add_to_memory(context, "y", bar) + sheerka.add_to_memory(context, "z", baz) + + # another layer + sheerka.add_to_memory(context, "x", bar) + sheerka.add_to_memory(context, "y", baz) + + # another layer + sheerka.add_to_memory(context, "x", baz) + + # so under x there is [foo] -> [bar] -> [baz] + # so under y there is [bar] -> [baz] + # so under z there is [baz] + assert sheerka.memory(context, "self.name == 'foo'") == foo + def test_concept_not_found_is_return_when_not_found(self): sheerka, context = self.init_test().unpack() diff --git a/tests/core/test_SheerkaQueryManager.py b/tests/core/test_SheerkaQueryManager.py new file mode 100644 index 0000000..2bdfb70 --- /dev/null +++ b/tests/core/test_SheerkaQueryManager.py @@ -0,0 +1,205 @@ +from dataclasses import dataclass + +import pytest + +from core.builtin_concepts_ids import BuiltinConcepts +from core.concept import Concept +from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka + + +@dataclass +class A: + prop1: object + prop2: object + + def __eq__(self, other): + if not isinstance(other, A): + return False + + return self.prop1 == other.prop1 and self.prop2 == other.prop2 + + def __hash__(self): + hash_res = [] + for p in [self.prop1, self.prop2]: + hash_res.append(0 if isinstance(p, (list, dict, set)) else p) + + return hash(tuple(hash_res)) + + +@dataclass +class B(A): + + def as_bag(self): + return { + "fake_prop1": self.prop1, + "fake_prop2": self.prop2 + } + + def __hash__(self): + return hash((self.prop1, self.prop2)) + + +class TestSheerkaQueryManager(TestUsingMemoryBasedSheerka): + def test_i_can_filter_objects_using_kwargs(self): + sheerka, context = self.init_test().unpack() + lst = [A("a11", 10), A("a21", 3.14), A({1, "v"}, 0xab), A([0, 1], {"key": "value"})] + + assert sheerka.filter_objects(context, lst, prop1="a21") == [lst[1]] + assert sheerka.filter_objects(context, lst, prop2=10) == [lst[0]] + assert sheerka.filter_objects(context, lst, prop2=3.14) == [lst[1]] + assert sheerka.filter_objects(context, lst, prop2=0xab) == [lst[2]] + assert sheerka.filter_objects(context, lst, prop1=[0, 1]) == [lst[3]] + assert sheerka.filter_objects(context, lst, prop2={"key": "value"}) == [lst[3]] + # assert sheerka.filter_objects(context, lst, prop1={1, "v"}) == [lst[2]] set are not supported + + def test_i_can_filter_by_object_type(self): + sheerka, context = self.init_test().unpack() + lst = [A("a11", "a12"), Concept("foo", body="a").auto_init(), Concept("foo", body="b").auto_init()] + + assert sheerka.filter_objects(context, lst, __type="foo") == [lst[1], lst[2]] + + def test_i_can_filter_on_atomic_def(self): + sheerka, context, isa, plus, isa2 = self.init_concepts( + Concept('x is a y').def_var("x").def_var("y"), + Concept('a plus b').def_var("a").def_var("b"), + Concept('u is a v').def_var("u").def_var("v"), + ) + + lst = [isa, plus, isa2] + assert sheerka.filter_objects(context, lst, atomic_def="is a") == [lst[0], lst[2]] + + def test_i_can_filter_on_as_bag_property(self): + sheerka, context = self.init_test().unpack() + lst = [B("a11", "a12"), B("a21", "a22"), B("a31", "a32")] + + assert sheerka.filter_objects(context, lst, fake_prop1="a21") == [lst[1]] + + def test_i_can_filter_container(self): + sheerka, context = self.init_test().unpack() + lst = [A("a11", 10), A("a21", 3.14), A({1, "v"}, 0xab), A([0, 1], {"key": "value"})] + container = sheerka.new(BuiltinConcepts.EXPLANATION, body=lst, digest="xxx", command="text") + + res = sheerka.filter_objects(context, container, prop1="a21") + assert sheerka.isinstance(res, BuiltinConcepts.EXPLANATION) + assert res.digest == "xxx" + assert res.command == "text" + assert res.body == [lst[1]] + + def test_i_can_filter_when_property_does_not_exist(self): + sheerka, context = self.init_test().unpack() + lst = [A("a11", "a12"), B("a21", "a22"), B("a31", "a32")] + + assert sheerka.filter_objects(context, lst, prop1="a11") == [lst[0]] + + def test_i_can_filter_objets_using_predicate(self): + sheerka, context = self.init_test().unpack() + lst = [A("a11", 10), + A("a21", 3.14), + A({1, "v"}, 0xab), + A([0, 1], {"key": "value"}), + Concept("foo", body="a").auto_init(), + B("a21", "a22"), + Concept("foo", body="b").auto_init(), + B("a31", "a32")] + + assert sheerka.filter_objects(context, lst, "self.prop1 == 'a21'") == [lst[1]] + assert sheerka.filter_objects(context, lst, "self.prop2 >= 1") == [lst[0], lst[1], lst[2]] + assert sheerka.filter_objects(context, lst, "get_type(self) == 'foo' ") == [lst[4], lst[6]] + assert sheerka.filter_objects(context, lst, "self.fake_prop1 == 'a21' ") == [lst[5]] + assert sheerka.filter_objects(context, lst, "hasattr(self, 'fake_prop1')") == [lst[5], lst[7]] + + def test_i_can_filter_object_using_predicate_and_sheerka_objects(self): + sheerka, context, foo, bar = self.init_concepts("foo", "bar") + lst = [foo, bar, A("a21", 3.14)] + + assert sheerka.filter_objects(context, lst, "self == bar") == [lst[1]] + + def test_i_can_filter_objects_using_concept(self): + sheerka, context, foo, bar, isa = self.init_concepts( + "foo", + "bar", + Concept("x is a concept", body="isinstance(x, Concept)", pre="is_question()").def_var("x"), + create_new=True) + + lst = [foo, A("a21", 3.14), bar, B("a21", 3.14)] + assert sheerka.filter_objects(context, lst, "self is a concept") == [foo, bar] + + def test_i_can_filter_objects_when_no_kwargs_and_no_predicate(self): + sheerka, context, foo, bar = self.init_concepts("foo", "bar") + lst = [foo, bar, A("a21", 3.14)] + + assert sheerka.filter_objects(context, lst) == lst + + def test_i_must_select_object_property_using_string(self): + sheerka, context = self.init_test().unpack() + + with pytest.raises(SyntaxError): + sheerka.select_objects(context, [], 00) + + def test_i_can_select_objects_with_args(self): + sheerka, context = self.init_test().unpack() + lst = [A("a11", 10), A("a21", 3.14), A({1, "v"}, 0xab), A([0, 1], {"key": "value"})] + + assert sheerka.select_objects(context, lst, "prop1", "prop2") == ( + ('a11', 10), + ('a21', 3.14), + ({1, 'v'}, 171), + ([0, 1], {'key': 'value'})) + + def test_i_can_select_objects_when_container(self): + sheerka, context = self.init_test().unpack() + lst = [A("a11", 10), A("a21", 3.14), A({1, "v"}, 0xab), A([0, 1], {"key": "value"})] + container = sheerka.new(BuiltinConcepts.EXPLANATION, body=lst, digest="xxx", command="text") + + res = sheerka.select_objects(context, container, "prop1", "prop2") + assert sheerka.isinstance(res, BuiltinConcepts.EXPLANATION) + assert res.digest == "xxx" + assert res.command == "text" + assert res.body == (('a11', 10), + ('a21', 3.14), + ({1, 'v'}, 171), + ([0, 1], {'key': 'value'})) + + def test_i_can_select_objects_with_complicated_request(self): + sheerka, context = self.init_test().unpack() + lst = [A("a11", 10), A("a21", 3.14), A({1, "v"}, 0xab)] + + assert sheerka.select_objects(context, lst, "self.prop2 + 5") == (15, 8.14, 0xab + 5) + assert sheerka.select_objects(context, lst, "isinstance(self.prop1, str)") == (True, True, False) + + def test_error_when_collecting_returns_are_managed(self): + sheerka, context = self.init_test().unpack() + lst = [A("a11", 10), A("a21", {"key": "value"})] + res = sheerka.select_objects(context, lst, "self.prop2 + 5") + + assert len(res) == 2 + assert res[0] == 15 + assert isinstance(res[1], TypeError) + + def test_i_can_select_objects_using_kwargs(self): + sheerka, context = self.init_test().unpack() + lst = [A("a11", 10), A("a21", 3.14), A({1, "v"}, 0xab), A([0, 1], {"key": "value"})] + + assert sheerka.select_objects(context, lst, p1="prop1", p2="prop2") == ( + {"p1": "a11", "p2": 10}, + {"p1": "a21", "p2": 3.14}, + {"p1": {1, "v"}, "p2": 0xab}, + {"p1": [0, 1], "p2": {"key": "value"}}, + ) + + def test_i_can_collect_attributes(self): + sheerka = self.get_sheerka() + lst = [A("", ""), + B("", ""), + Concept("foo").def_var("a").auto_init(), + Concept("bar").def_var("y").def_var("x").auto_init()] + + res = sheerka.collect_attributes(lst) + + assert sheerka.isinstance(res, BuiltinConcepts.TO_DICT) + assert res.body == { + "A": ["prop1", "prop2"], + "B": ["fake_prop1", "fake_prop2"], + "foo": ["a", 'body', 'id', 'key', 'name'], + "bar": ['body', 'id', 'key', 'name', "x", "y"] # attributes are sorted + } diff --git a/tests/core/test_SheerkaRuleManager.py b/tests/core/test_SheerkaRuleManager.py index ea7911d..b1662a8 100644 --- a/tests/core/test_SheerkaRuleManager.py +++ b/tests/core/test_SheerkaRuleManager.py @@ -1,5 +1,3 @@ -import ast - import pytest from core.builtin_concepts import BuiltinConcepts, ReturnValueConcept @@ -7,26 +5,20 @@ from core.concept import Concept, DEFINITION_TYPE_DEF from core.global_symbols import RULE_COMPARISON_CONTEXT, NotFound, EVENT_RULE_DELETED from core.rule import Rule, ACTION_TYPE_PRINT, ACTION_TYPE_EXEC from core.sheerka.Sheerka import RECOGNIZED_BY_ID, RECOGNIZED_BY_NAME -from core.sheerka.services.SheerkaEvaluateRules import SheerkaEvaluateRules from core.sheerka.services.SheerkaExecute import ParserInput from core.sheerka.services.SheerkaRuleManager import SheerkaRuleManager, FormatRuleActionParser, \ FormatAstRawText, FormatAstVariable, FormatAstSequence, FormatAstFunction, \ FormatRuleSyntaxError, FormatAstList, UnexpectedEof, FormatAstColor, FormatAstDict, \ FormatAstMulti, \ - PythonCodeEmitter, FormatAstNode, ReteConditionExprVisitor, PythonConditionExprVisitor, \ - CompiledCondition + PythonCodeEmitter, FormatAstNode, ReteConditionExprVisitor from core.tokenizer import Token, TokenKind -from evaluators.PythonEvaluator import Expando from parsers.BaseParser import ErrorSink from parsers.ExpressionParser import ExpressionParser -from parsers.PythonParser import PythonNode -from sheerkarete.common import V -from sheerkarete.conditions import Condition, AndConditions, FilterCondition +from sheerkarete.conditions import FilterCondition from sheerkarete.network import ReteNetwork from tests.TestUsingFileBasedSheerka import TestUsingFileBasedSheerka from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka -from tests.parsers.parsers_utils import get_rete_conditions, NEGCOND, \ - NCCOND +from tests.parsers.parsers_utils import get_rete_conditions, NEGCOND, NCCOND seq = FormatAstSequence raw = FormatAstRawText @@ -188,7 +180,7 @@ class TestSheerkaRuleManager(TestUsingMemoryBasedSheerka): rule = service.init_rule(context, rule) assert len(rule.error_sink["when"]) > 0 - assert sheerka.is_error(rule.error_sink["when"][0]) + assert sheerka.has_error(context, rule.error_sink["when"][0]) assert "print" not in rule.error_sink assert "then" not in rule.error_sink assert rule.metadata.is_compiled @@ -210,7 +202,7 @@ class TestSheerkaRuleManager(TestUsingMemoryBasedSheerka): other_action_type = ACTION_TYPE_PRINT if action_type == ACTION_TYPE_EXEC else ACTION_TYPE_EXEC - assert sheerka.is_error(rule.error_sink[action_type]) + assert sheerka.has_error(context, rule.error_sink[action_type]) assert other_action_type not in rule.error_sink assert "when" not in rule.error_sink assert rule.metadata.is_compiled @@ -282,30 +274,6 @@ class TestSheerkaRuleManager(TestUsingMemoryBasedSheerka): assert parser.error_sink == expected_error - @pytest.mark.parametrize("text, compiled_text", [ - ("a == 5", "a == 5"), - ("foo > 5", "foo > 5"), - ("func() == 5", "func() == 5"), - ("not a == 5", "not (a == 5)"), - ("not foo > 5", "not (foo > 5)"), - ("not func() == 5", "not (func() == 5)"), - ]) - def test_i_can_compile_predicate_when_pure_python(self, text, compiled_text): - sheerka, context, *concepts = self.init_concepts("foo") - service = sheerka.services[SheerkaRuleManager.NAME] - ast_ = ast.parse(compiled_text, "", 'eval') - expected_python_node = PythonNode(compiled_text, ast_) - - compilation_result = service.compile_when(context, "test", text) - res = compilation_result.python_conditions - - assert len(res) == 1 - assert isinstance(res[0], CompiledCondition) - assert res[0].evaluator_type == PYTHON_EVALUATOR_NAME - assert sheerka.isinstance(res[0].return_value, BuiltinConcepts.RETURN_VALUE) - assert sheerka.objvalue(res[0].return_value) == expected_python_node - assert res[0].concept is None - def test_i_can_get_rule_priorities(self): sheerka, context, rule_true, rule_false = self.init_test().with_format_rules(("True", "True"), ("False", "False")).unpack() @@ -499,20 +467,6 @@ isinstance(var, Concept) and var.key == 'hello __var__0'""" + \ res = sheerka.get_exec_rules() assert res == [r2, r3, r1] - @pytest.mark.skip - def test_i_can_compile_rete_using_name(self): - sheerka, context, *concepts = self.init_test().unpack() - service = sheerka.services[SheerkaRuleManager.NAME] - - text = "__ret" - - compilation_result = service.compile_when(context, "test", text) - res = compilation_result.rete_disjunctions - - assert len(res) == 1 - assert isinstance(res[0], AndConditions) - assert res[0].conditions == [Condition(V("__x_00__"), "__name__", "__ret")] - def test_i_can_properly_copy_a_rule(self): sheerka, context = self.init_test().unpack() service = sheerka.services[SheerkaRuleManager.NAME] @@ -525,98 +479,6 @@ isinstance(var, Concept) and var.key == 'hello __var__0'""" + \ for k, v in vars(rule).items(): assert getattr(clone, k) == getattr(rule, k) - @pytest.mark.parametrize("expression, expected_as_str, expected_variables", [ - ( - "__ret", - ["#__x_00__|__name__|'__ret'"], - {"__ret"} - ), - ( - "__ret.status == True", - ["#__x_00__|__name__|'__ret'", "#__x_00__|status|True"], - {"__ret"} - ), - ( - "__ret.status", - ["#__x_00__|__name__|'__ret.status'"], - {"__ret.status"} - ), - ( - "body", - ["#__x_00__|__name__|'body'"], - {"body"} - ), - ( - "__ret and __ret.status", - ["#__x_00__|__name__|'__ret'", "#__x_01__|__name__|'__ret.status'"], - {"__ret", "__ret.status"} - ), - ]) - def test_i_can_get_rete_conditions(self, expression, expected_as_str, expected_variables): - sheerka, context = self.init_test().unpack() - parser = ExpressionParser() - expected = get_rete_conditions(*expected_as_str) - - error_sink = ErrorSink() - parser_input = ParserInput(expression) - parser.reset_parser_input(parser_input, error_sink) - parsed = parser.parse_input(context, parser_input, error_sink) - - visitor = ReteConditionExprVisitor(context) - conditions = visitor.get_conditions(parsed) - - assert conditions == [expected] - - # check against a Rete network - network = ReteNetwork() - rule = Rule("test", expression, None) - rule.metadata.id = 9999 - rule.metadata.is_compiled = True - rule.metadata.is_enabled = True - rule.rete_disjunctions = conditions - network.add_rule(rule) - - return_value = ReturnValueConcept("Test", True, None) - if "__ret" in expected_variables: - network.add_obj("__ret", return_value) - if "__ret.status" in expected_variables: - network.add_obj("__ret.status", return_value.status) - if "body" in expected_variables: - network.add_obj("body", return_value.body) - - matches = list(network.matches) - assert len(matches) == 1 - - def test_i_can_get_rete_conditions_when_no_attribute(self): - sheerka, context = self.init_test().unpack() - expression = "a == 10" - expected_as_str = ["#__x_00__|__name__|'a'", "#__x_00__|__self__|10"] - parser = ExpressionParser() - expected = get_rete_conditions(*expected_as_str) - - error_sink = ErrorSink() - parser_input = ParserInput(expression) - parser.reset_parser_input(parser_input, error_sink) - parsed = parser.parse_input(context, parser_input, error_sink) - - visitor = ReteConditionExprVisitor(context) - conditions = visitor.get_conditions(parsed) - - assert conditions == [expected] - - # check against a Rete network - network = ReteNetwork() - rule = Rule("test", expression, None) - rule.metadata.id = 9999 - rule.metadata.is_compiled = True - rule.metadata.is_enabled = True - rule.rete_disjunctions = conditions - network.add_rule(rule) - - network.add_obj("a", 10) - matches = list(network.matches) - assert len(matches) == 1 - @pytest.mark.skip("No ready yet for SheerkaFilterCondition") def test_i_can_get_rete_conditions_when_function(self): sheerka, context, greetings = self.init_test().with_concepts( @@ -652,235 +514,6 @@ isinstance(var, Concept) and var.key == 'hello __var__0'""" + \ # matches = list(network.matches) # assert len(matches) == 1 - @pytest.mark.parametrize("test_name, expression, variable_name, expected_as_str", [ - ( - "recognize by name", - "recognize(__ret.body, greetings)", - None, - ["#__x_00__|__name__|'__ret'", - "#__x_00__|body|#__x_01__", - "#__x_01__|__is_concept__|True", - "#__x_01__|name|'greetings'"] - ), - ( - "recognize by id", - "recognize(__ret.body, c:|1001:)", - None, - ["#__x_00__|__name__|'__ret'", - "#__x_00__|body|#__x_01__", - "#__x_01__|__is_concept__|True", - "#__x_01__|id|'1001'"] - ), - ( - "recognize by name using c_str", - "recognize(__ret.body, c:greetings:)", - None, - ["#__x_00__|__name__|'__ret'", - "#__x_00__|body|#__x_01__", - "#__x_01__|__is_concept__|True", - "#__x_01__|name|'greetings'"] - ), - ( - "recognize by name and add other conditions (str)", - "recognize(__ret.body, greetings) and __ret.body.a == 'my friend'", - "my friend", - ["#__x_00__|__name__|'__ret'", - "#__x_00__|body|#__x_01__", - "#__x_01__|__is_concept__|True", - "#__x_01__|name|'greetings'", - "#__x_01__|a|'my friend'"] - ), - ( - "recognize by name and add other conditions (sheerka)", - "recognize(__ret.body, greetings) and __ret.body.a == sheerka", - "sheerka", - ["#__x_00__|__name__|'__ret'", - "#__x_00__|body|#__x_01__", - "#__x_01__|__is_concept__|True", - "#__x_01__|name|'greetings'", - "#__x_01__|a|'__sheerka__'"] - ), - ( - "recognize by name and add other conditions (concept)", - "recognize(__ret.body, greetings) and __ret.body.a == foo", - "foo", - ["#__x_00__|__name__|'__ret'", - "#__x_00__|body|#__x_01__", - "#__x_01__|__is_concept__|True", - "#__x_01__|name|'greetings'", - "#__x_01__|a|#__x_02__", - "#__x_02__|__is_concept__|True", - "#__x_02__|key|'foo'"] - ), - ( - "recognize by instance", - "recognize(__ret.body, hello sheerka)", - "sheerka", - ["#__x_00__|__name__|'__ret'", - "#__x_00__|body|#__x_01__", - "#__x_01__|__is_concept__|True", - "#__x_01__|key|'hello __var__0'", - "#__x_01__|a|'__sheerka__'"] - ), - ( - "recognize by instance", - "recognize(__ret.body, hello 'my friend')", - "my friend", - ["#__x_00__|__name__|'__ret'", - "#__x_00__|body|#__x_01__", - "#__x_01__|__is_concept__|True", - "#__x_01__|key|'hello __var__0'", - "#__x_01__|a|'my friend'"] - ), - ( - "recognize by instance", - "recognize(__ret.body, hello foo)", - "foo", - ["#__x_00__|__name__|'__ret'", - "#__x_00__|body|#__x_01__", - "#__x_01__|__is_concept__|True", - "#__x_01__|key|'hello __var__0'", - "#__x_01__|a|#__x_02__", - "#__x_02__|__is_concept__|True", - "#__x_02__|key|'foo'", - ] - ), - ( - "recognize by instance when long concept", - "recognize(__ret.body, hello my best friend)", - "my best friend", - ["#__x_00__|__name__|'__ret'", - "#__x_00__|body|#__x_01__", - "#__x_01__|__is_concept__|True", - "#__x_01__|key|'hello __var__0'", - "#__x_01__|a|#__x_02__", - "#__x_02__|__is_concept__|True", - "#__x_02__|key|'my best friend'", - ] - ), - ]) - def test_i_can_get_rete_using_recognize_function(self, test_name, expression, variable_name, expected_as_str): - sheerka, context, greetings, foo, my_best_friend = self.init_test().with_concepts( - Concept("greetings", definition="hello a", definition_type=DEFINITION_TYPE_DEF).def_var("a"), - Concept("foo"), - Concept("my best friend"), - create_new=True - ).unpack() - parser = ExpressionParser() - expected = get_rete_conditions(*expected_as_str) - - error_sink = ErrorSink() - parser_input = ParserInput(expression) - parser.reset_parser_input(parser_input, error_sink) - parsed = parser.parse_input(context, parser_input, error_sink) - - visitor = ReteConditionExprVisitor(context) - conditions = visitor.get_conditions(parsed) - - assert conditions == [expected] - - # check against a Rete network - network = ReteNetwork() - rule = Rule("test", expression, None) - rule.metadata.id = 9999 - rule.metadata.is_compiled = True - rule.metadata.is_enabled = True - rule.rete_disjunctions = conditions - network.add_rule(rule) - - variable_map = { - "foo": foo, - "my best friend": my_best_friend, - "sheerka": sheerka - } - variable = variable_map.get(variable_name, variable_name) - to_recognize = sheerka.new_from_template(greetings, greetings.key, a=variable) - network.add_obj("__ret", ReturnValueConcept("Test", True, to_recognize)) - matches = list(network.matches) - assert len(matches) == 1 - - @pytest.mark.parametrize("expression, variable_name, expected_as_str", [ - ( - "greetings", - None, - ["#__x_00__|__name__|'__ret'", - "#__x_00__|body|#__x_01__", - "#__x_01__|__is_concept__|True", - "#__x_01__|name|'greetings'"] - ), - ( - "c:|1001:", - None, - ["#__x_00__|__name__|'__ret'", - "#__x_00__|body|#__x_01__", - "#__x_01__|__is_concept__|True", - "#__x_01__|id|'1001'"] - ), - ( - "hello foo", - "foo", - ["#__x_00__|__name__|'__ret'", - "#__x_00__|body|#__x_01__", - "#__x_01__|__is_concept__|True", - "#__x_01__|key|'hello __var__0'", - "#__x_01__|a|#__x_02__", - "#__x_02__|__is_concept__|True", - "#__x_02__|key|'foo'", - ] - ), - ( - "hello my best friend", - "my best friend", - ["#__x_00__|__name__|'__ret'", - "#__x_00__|body|#__x_01__", - "#__x_01__|__is_concept__|True", - "#__x_01__|key|'hello __var__0'", - "#__x_01__|a|#__x_02__", - "#__x_02__|__is_concept__|True", - "#__x_02__|key|'my best friend'", - ] - ), - ]) - def test_i_can_get_rete_when_a_concept_is_recognized(self, expression, variable_name, expected_as_str): - sheerka, context, greetings, foo, my_best_friend = self.init_test().with_concepts( - Concept("greetings", definition="hello a", definition_type=DEFINITION_TYPE_DEF).def_var("a"), - Concept("foo"), - Concept("my best friend"), - create_new=True - ).unpack() - parser = ExpressionParser() - expected = get_rete_conditions(*expected_as_str) - - error_sink = ErrorSink() - parser_input = ParserInput(expression) - parser.reset_parser_input(parser_input, error_sink) - parsed = parser.parse_input(context, parser_input, error_sink) - - visitor = ReteConditionExprVisitor(context) - conditions = visitor.get_conditions(parsed) - - assert conditions == [expected] - - # check against a Rete network - network = ReteNetwork() - rule = Rule("test", expression, None) - rule.metadata.id = 9999 - rule.metadata.is_compiled = True - rule.metadata.is_enabled = True - rule.rete_disjunctions = conditions - network.add_rule(rule) - - variable_map = { - "foo": foo, - "my best friend": my_best_friend, - "sheerka": sheerka - } - variable = variable_map.get(variable_name, variable_name) - to_recognize = sheerka.new_from_template(greetings, greetings.key, a=variable) - network.add_obj("__ret", ReturnValueConcept("Test", True, to_recognize)) - matches = list(network.matches) - assert len(matches) == 1 - @pytest.mark.parametrize("expression, bag_key, expected", [ ("not __ret", "__other", [NEGCOND("#__x_00__|__name__|'__ret'")]), ("not not __ret", "__ret", ["#__x_00__|__name__|'__ret'"]), @@ -894,6 +527,10 @@ isinstance(var, Concept) and var.key == 'hello __var__0'""" + \ "#__x_01__|key|'hello __var__0'", "#__x_01__|a|'__sheerka__'"])]), ("__ret and not __error", "__ret", ["#__x_00__|__name__|'__ret'", NEGCOND("#__x_01__|__name__|'__error'")]), + ("not recognize(self, hello sheerka)", "__ret", ["#__x_00__|__name__|'self'", + NCCOND(["#__x_00__|__is_concept__|True", + "#__x_00__|key|'hello __var__0'", + "#__x_00__|a|'__sheerka__'"])]), ]) def test_i_can_get_rete_using_not(self, expression, bag_key, expected): sheerka, context, greetings, foo = self.init_test().with_concepts( @@ -914,7 +551,7 @@ isinstance(var, Concept) and var.key == 'hello __var__0'""" + \ assert conditions == [expected_conditions] @pytest.mark.skip("I am not sure yet of what I want to get") - @pytest.mark.parametrize("expression, expected_as_str", [ + @pytest.mark.parametrize("expression, expected_as_list_of_str", [ ( "eval(__ret.body, 'foo' starts with 'f')", ["#__x_00__|__name__|'__ret'", @@ -926,7 +563,7 @@ isinstance(var, Concept) and var.key == 'hello __var__0'""" + \ "$eval(__x_01__, a, b, c)"] ), ]) - def test_i_can_get_rete_conditions_using_eval_function(self, expression, expected_as_str): + def test_i_can_get_rete_conditions_using_eval_function(self, expression, expected_as_list_of_str): sheerka, context, start_with = self.init_test().with_concepts( Concept("x starts with y", pre="is_question", @@ -935,7 +572,7 @@ isinstance(var, Concept) and var.key == 'hello __var__0'""" + \ ).unpack() parser = ExpressionParser() - expected = get_rete_conditions(*expected_as_str) + expected = get_rete_conditions(*expected_as_list_of_str) error_sink = ErrorSink() parser_input = ParserInput(expression) @@ -961,372 +598,6 @@ isinstance(var, Concept) and var.key == 'hello __var__0'""" + \ matches = list(network.matches) assert len(matches) == 1 - @pytest.mark.parametrize("expression, expected_compiled, expected_variables", [ - ("__ret", None, {"__ret"}), - ("__ret.status == True", "__ret.status == True", {"__ret"}), - ("__ret.status", None, {"__ret.status"}), - ("body", None, {"body"}), - ("__ret and __ret.status", None, {"__ret", "__ret.status"}) - ]) - def test_i_can_get_compiled_conditions(self, expression, expected_compiled, expected_variables): - sheerka, context = self.init_test().unpack() - - parser = ExpressionParser() - error_sink = ErrorSink() - parser_input = ParserInput(expression) - parser.reset_parser_input(parser_input, error_sink) - parsed = parser.parse_input(context, parser_input, error_sink) - - visitor = PythonConditionExprVisitor(context) - conditions = visitor.get_conditions(parsed) - - assert len(conditions) == 1 - assert isinstance(conditions[0], CompiledCondition) - if expected_compiled: - ast_ = ast.parse(expected_compiled, "", 'eval') - expected_python_node = PythonNode(expected_compiled, ast_) - assert conditions[0].evaluator_type == PYTHON_EVALUATOR_NAME - assert sheerka.isinstance(conditions[0].return_value, BuiltinConcepts.RETURN_VALUE) - assert sheerka.objvalue(conditions[0].return_value) == expected_python_node - else: - assert conditions[0].evaluator_type is None - assert conditions[0].return_value is None - assert conditions[0].concept is None - assert conditions[0].variables == expected_variables - - # check against SheerkaEvaluateRules - evaluate_rules_service = sheerka.services[SheerkaEvaluateRules.NAME] - return_value = ReturnValueConcept("Test", True, None) - bag = {} - if "__ret" in expected_variables: - bag["__ret"] = return_value - if "__ret.status" in expected_variables: - bag["__ret.status"] = return_value.status - if "body" in expected_variables: - bag["body"] = return_value.body - with context.push(BuiltinConcepts.RULES_EVALUATION, bag, desc="Evaluating rules...") as sub_context: - sub_context.sheerka.add_many_to_short_term_memory(sub_context, bag) - rule = Rule(name="test_i_can_get_compiled_conditions", predicate=expression) - rule.compiled_conditions = conditions - res = evaluate_rules_service.evaluate_rule(sub_context, rule, bag) - assert res.status - assert self.sheerka.is_success(self.sheerka.objvalue(res)) - - def test_i_can_get_compiled_conditions_when_no_attribute(self): - sheerka, context = self.init_test().unpack() - expression = "a == 10" - expected_compiled = "a == 10" - - parser = ExpressionParser() - error_sink = ErrorSink() - parser_input = ParserInput(expression) - parser.reset_parser_input(parser_input, error_sink) - parsed = parser.parse_input(context, parser_input, error_sink) - - visitor = PythonConditionExprVisitor(context) - conditions = visitor.get_conditions(parsed) - - assert len(conditions) == 1 - assert isinstance(conditions[0], CompiledCondition) - ast_ = ast.parse(expected_compiled, "", 'eval') - expected_python_node = PythonNode(expected_compiled, ast_) - assert conditions[0].evaluator_type == PYTHON_EVALUATOR_NAME - assert sheerka.isinstance(conditions[0].return_value, BuiltinConcepts.RETURN_VALUE) - assert sheerka.objvalue(conditions[0].return_value) == expected_python_node - assert conditions[0].concept is None - assert conditions[0].variables == {"a"} - - # check against SheerkaEvaluateRules - evaluate_rules_service = sheerka.services[SheerkaEvaluateRules.NAME] - bag = {"a": 10} - with context.push(BuiltinConcepts.RULES_EVALUATION, bag, desc="Evaluating rules...") as sub_context: - sub_context.sheerka.add_many_to_short_term_memory(sub_context, bag) - rule = Rule(name="test_i_can_get_compiled_conditions", predicate=expression) - rule.compiled_conditions = conditions - res = evaluate_rules_service.evaluate_rule(sub_context, rule, bag) - assert res.status - assert self.sheerka.is_success(self.sheerka.objvalue(res)) - - def test_i_can_get_compiled_conditions_when_function(self): - sheerka, context, greetings = self.init_test().with_concepts( - Concept("greetings", definition="hello a", definition_type=DEFINITION_TYPE_DEF).def_var("a"), - ).unpack() - expression = "isinstance(a, greetings)" - expected_compiled = "isinstance(a, greetings)" - - parser = ExpressionParser() - error_sink = ErrorSink() - parser_input = ParserInput(expression) - parser.reset_parser_input(parser_input, error_sink) - parsed = parser.parse_input(context, parser_input, error_sink) - - visitor = PythonConditionExprVisitor(context) - conditions = visitor.get_conditions(parsed) - - assert len(conditions) == 1 - assert isinstance(conditions[0], CompiledCondition) - ast_ = ast.parse(expected_compiled, "", 'eval') - expected_python_node = PythonNode(expected_compiled, ast_) - assert conditions[0].evaluator_type == PYTHON_EVALUATOR_NAME - assert sheerka.isinstance(conditions[0].return_value, BuiltinConcepts.RETURN_VALUE) - assert sheerka.objvalue(conditions[0].return_value) == expected_python_node - assert conditions[0].concept is None - assert conditions[0].variables == {"a"} - - # check against SheerkaEvaluateRules - evaluate_rules_service = sheerka.services[SheerkaEvaluateRules.NAME] - bag = {"a": sheerka.new(greetings, a="my friend")} - with context.push(BuiltinConcepts.RULES_EVALUATION, bag, desc="Evaluating rules...") as sub_context: - sub_context.sheerka.add_many_to_short_term_memory(sub_context, bag) - rule = Rule(name="test_i_can_get_compiled_conditions", predicate=expression) - rule.compiled_conditions = conditions - res = evaluate_rules_service.evaluate_rule(sub_context, rule, bag) - assert res.status - assert self.sheerka.is_success(self.sheerka.objvalue(res)) - - @pytest.mark.parametrize("expression, variable_name, expected_compiled", [ - ( - "recognize(__ret.body, greetings)", - None, - "__x_00__ = __ret.body\nisinstance(__x_00__, Concept) and __x_00__.name == 'greetings'" - ), - ( - "recognize(__ret.body, c:|1001:)", - None, - "__x_00__ = __ret.body\nisinstance(__x_00__, Concept) and __x_00__.id == '1001'" - ), - ( - "recognize(__ret.body, c:greetings:)", - None, - "__x_00__ = __ret.body\nisinstance(__x_00__, Concept) and __x_00__.name == 'greetings'" - ), - ( - "recognize(__ret.body, greetings) and __ret.body.a == 'my friend'", - "my friend", - "__x_00__ = __ret.body\nisinstance(__x_00__, Concept) and __x_00__.name == 'greetings' and __x_00__.a == 'my friend'" - ), - ( - "recognize(__ret.body, greetings) and __ret.body.a == sheerka", - "sheerka", - """__x_00__ = __ret.body -isinstance(__x_00__, Concept) and __x_00__.name == 'greetings' and is_sheerka(__x_00__.a)""" - ), - ( - "recognize(__ret.body, greetings) and __ret.body.a == foo", - "foo", - """__x_00__ = __ret.body -__x_01__ = __x_00__.a -isinstance(__x_00__, Concept) and __x_00__.name == 'greetings' and isinstance(__x_01__, Concept) and __x_01__.key == 'foo'""" - ), - ( - "recognize(__ret.body, hello sheerka)", - "sheerka", - """__x_00__ = __ret.body -isinstance(__x_00__, Concept) and __x_00__.key == 'hello __var__0' and is_sheerka(__x_00__.a)""" - ), - ( - "recognize(__ret.body, hello 'my friend')", - "my friend", - """__x_00__ = __ret.body -isinstance(__x_00__, Concept) and __x_00__.key == 'hello __var__0' and __x_00__.a == 'my friend'""" - ), - ( - "recognize(__ret.body, hello foo)", - "foo", - """__x_00__ = __ret.body -__x_01__ = __x_00__.a -isinstance(__x_00__, Concept) and __x_00__.key == 'hello __var__0' and isinstance(__x_01__, Concept) and __x_01__.key == 'foo'""" - ), - ( - "recognize(__ret.body, hello my best friend)", - "my best friend", - """__x_00__ = __ret.body -__x_01__ = __x_00__.a -isinstance(__x_00__, Concept) and __x_00__.key == 'hello __var__0' and isinstance(__x_01__, Concept) and __x_01__.key == 'my best friend'""" - ), - - ]) - def test_i_can_get_compiled_using_recognize_function(self, expression, variable_name, expected_compiled): - sheerka, context, greetings, foo, my_best_friend = self.init_test().with_concepts( - Concept("greetings", definition="hello a", definition_type=DEFINITION_TYPE_DEF).def_var("a"), - Concept("foo"), - Concept("my best friend"), - create_new=True - ).unpack() - - parser = ExpressionParser() - error_sink = ErrorSink() - parser_input = ParserInput(expression) - parser.reset_parser_input(parser_input, error_sink) - parsed = parser.parse_input(context, parser_input, error_sink) - - visitor = PythonConditionExprVisitor(context) - conditions = visitor.get_conditions(parsed) - - assert len(conditions) == 1 - assert isinstance(conditions[0], CompiledCondition) - if expected_compiled: - ast_ = ast.parse(expected_compiled, "", 'exec') - expected_python_node = PythonNode(expected_compiled, ast_, expected_compiled) - assert conditions[0].evaluator_type == PYTHON_EVALUATOR_NAME - assert sheerka.isinstance(conditions[0].return_value, BuiltinConcepts.RETURN_VALUE) - assert sheerka.objvalue(conditions[0].return_value) == expected_python_node - else: - assert conditions[0].evaluator_type is None - assert conditions[0].return_value is None - assert conditions[0].concept is None - assert conditions[0].variables == {"__ret"} - - # check against SheerkaEvaluateRules - evaluate_rules_service = sheerka.services[SheerkaEvaluateRules.NAME] - variable_map = { - "foo": foo, - "my best friend": my_best_friend, - "sheerka": Expando("sheerka", {}) - } - variable = variable_map.get(variable_name, variable_name) - to_recognize = sheerka.new_from_template(greetings, greetings.key, a=variable) - bag = {"__ret": ReturnValueConcept("Test", True, to_recognize)} - with context.push(BuiltinConcepts.RULES_EVALUATION, bag, desc="Evaluating rules...") as sub_context: - sub_context.sheerka.add_many_to_short_term_memory(sub_context, bag) - rule = Rule(name="test_i_can_get_compiled_using_recognize_function", predicate=expression) - rule.compiled_conditions = conditions - res = evaluate_rules_service.evaluate_rule(sub_context, rule, bag) - assert res.status - assert self.sheerka.is_success(self.sheerka.objvalue(res)) - - @pytest.mark.parametrize("expression, variable_name, expected_compiled", [ - ( - "greetings", - None, - "__x_00__ = __ret.body\nisinstance(__x_00__, Concept) and __x_00__.name == 'greetings'" - ), - ( - "c:|1001:", - None, - "__x_00__ = __ret.body\nisinstance(__x_00__, Concept) and __x_00__.id == '1001'" - ), - ( - "hello foo", - "foo", - """__x_00__ = __ret.body -__x_01__ = __x_00__.a -isinstance(__x_00__, Concept) and __x_00__.key == 'hello __var__0' and isinstance(__x_01__, Concept) and __x_01__.key == 'foo'""" - ), - ( - "hello my best friend", - "my best friend", - """__x_00__ = __ret.body -__x_01__ = __x_00__.a -isinstance(__x_00__, Concept) and __x_00__.key == 'hello __var__0' and isinstance(__x_01__, Concept) and __x_01__.key == 'my best friend'""" - ), - ]) - def test_i_can_get_compiled_when_a_concept_is_recognized(self, expression, variable_name, expected_compiled): - sheerka, context, greetings, foo, my_best_friend = self.init_test().with_concepts( - Concept("greetings", definition="hello a", definition_type=DEFINITION_TYPE_DEF).def_var("a"), - Concept("foo"), - Concept("my best friend"), - create_new=True - ).unpack() - - parser = ExpressionParser() - error_sink = ErrorSink() - parser_input = ParserInput(expression) - parser.reset_parser_input(parser_input, error_sink) - parsed = parser.parse_input(context, parser_input, error_sink) - - visitor = PythonConditionExprVisitor(context) - conditions = visitor.get_conditions(parsed) - - assert len(conditions) == 1 - assert isinstance(conditions[0], CompiledCondition) - if expected_compiled: - ast_ = ast.parse(expected_compiled, "", 'exec') - expected_python_node = PythonNode(expected_compiled, ast_, expected_compiled) - assert conditions[0].evaluator_type == PYTHON_EVALUATOR_NAME - assert sheerka.isinstance(conditions[0].return_value, BuiltinConcepts.RETURN_VALUE) - assert sheerka.objvalue(conditions[0].return_value) == expected_python_node - else: - assert conditions[0].evaluator_type is None - assert conditions[0].return_value is None - assert conditions[0].concept is None - assert conditions[0].variables == {"__ret"} - - # check against SheerkaEvaluateRules - evaluate_rules_service = sheerka.services[SheerkaEvaluateRules.NAME] - variable_map = { - "foo": foo, - "my best friend": my_best_friend, - "sheerka": Expando("sheerka", {}) - } - variable = variable_map.get(variable_name, variable_name) - to_recognize = sheerka.new_from_template(greetings, greetings.key, a=variable) - bag = {"__ret": ReturnValueConcept("Test", True, to_recognize)} - with context.push(BuiltinConcepts.RULES_EVALUATION, bag, desc="Evaluating rules...") as sub_context: - sub_context.sheerka.add_many_to_short_term_memory(sub_context, bag) - rule = Rule(name="test_i_can_get_compiled_using_recognize_function", predicate=expression) - rule.compiled_conditions = conditions - res = evaluate_rules_service.evaluate_rule(sub_context, rule, bag) - assert res.status - assert self.sheerka.is_success(self.sheerka.objvalue(res)) - - @pytest.mark.parametrize("expression, expected_compiled, variables, not_variables", [ - ("not __ret", None, set(), {"__ret"}), - ("not not __ret", None, {"__ret"}, set()), - ("not __ret.status == True", "not (__ret.status == True)", {"__ret"}, set()), - ("not __ret.status", None, set(), {"__ret.status"},), - ("__ret and not __ret.status", None, {"__ret"}, {"__ret.status"}), - ("not recognize(__ret.body, hello sheerka)", """__x_00__ = __ret.body -not (isinstance(__x_00__, Concept) and __x_00__.key == 'hello __var__0' and is_sheerka(__x_00__.a))""", {"__ret"}, - set()), - ("__ret and not __error", None, {"__ret"}, {"__error"}), - ]) - def test_i_can_get_compiled_condition_using_not(self, expression, expected_compiled, variables, not_variables): - sheerka, context, greetings, foo = self.init_test().with_concepts( - Concept("greetings", definition="hello a", definition_type=DEFINITION_TYPE_DEF).def_var("a"), - Concept("foo"), - ).unpack() - - parser = ExpressionParser() - error_sink = ErrorSink() - parser_input = ParserInput(expression) - parser.reset_parser_input(parser_input, error_sink) - parsed = parser.parse_input(context, parser_input, error_sink) - - visitor = PythonConditionExprVisitor(context) - conditions = visitor.get_conditions(parsed) - - assert len(conditions) == 1 - assert isinstance(conditions[0], CompiledCondition) - if expected_compiled: - if "\n" in expected_compiled: - ast_ = ast.parse(expected_compiled, "", 'exec') - else: - ast_ = ast.parse(expected_compiled, "", 'eval') - expected_python_node = PythonNode(expected_compiled, ast_) - assert conditions[0].evaluator_type == PYTHON_EVALUATOR_NAME - assert sheerka.isinstance(conditions[0].return_value, BuiltinConcepts.RETURN_VALUE) - assert sheerka.objvalue(conditions[0].return_value) == expected_python_node - - else: - assert conditions[0].evaluator_type is None - assert conditions[0].return_value is None - assert conditions[0].variables == variables - assert conditions[0].not_variables == not_variables - assert conditions[0].concept is None - - # check against SheerkaEvaluateRules - evaluate_rules_service = sheerka.services[SheerkaEvaluateRules.NAME] - ret_val_key = "__ret" if "__ret" in conditions[0].variables else "__other" - bag = {ret_val_key: ReturnValueConcept("Test", False, None)} - with context.push(BuiltinConcepts.RULES_EVALUATION, bag, desc="Evaluating rules...") as sub_context: - sub_context.sheerka.add_many_to_short_term_memory(sub_context, bag) - rule = Rule(name="test_i_can_get_compiled_conditions", predicate=expression) - rule.compiled_conditions = conditions - res = evaluate_rules_service.evaluate_rule(sub_context, rule, bag) - assert res.status - assert self.sheerka.is_success(self.sheerka.objvalue(res)) - class TestSheerkaRuleManagerUsingFileBasedSheerka(TestUsingFileBasedSheerka): def test_rules_are_initialized_at_startup(self): diff --git a/tests/core/test_SheerkaRuleManagerRulesCompilation.py b/tests/core/test_SheerkaRuleManagerRulesCompilation.py new file mode 100644 index 0000000..6e3dd97 --- /dev/null +++ b/tests/core/test_SheerkaRuleManagerRulesCompilation.py @@ -0,0 +1,1161 @@ +import ast + +import pytest + +from core.builtin_concepts import ReturnValueConcept +from core.builtin_concepts_ids import BuiltinConcepts +from core.concept import Concept, DEFINITION_TYPE_DEF +from core.rule import Rule +from core.sheerka.services.SheerkaEvaluateRules import SheerkaEvaluateRules +from core.sheerka.services.SheerkaExecute import ParserInput +from core.sheerka.services.SheerkaRuleManager import ReteConditionExprVisitor, PythonConditionExprVisitor, \ + CompiledCondition +from core.sheerka.services.sheerka_service import FailedToCompileError +from evaluators.PythonEvaluator import PythonEvaluator +from parsers.BaseParser import ErrorSink +from parsers.ExpressionParser import ExpressionParser +from parsers.PythonParser import PythonNode +from sheerkapython.python_wrapper import Expando +from sheerkarete.network import ReteNetwork +from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka +from tests.core.test_SheerkaRuleManager import PYTHON_EVALUATOR_NAME +from tests.parsers.parsers_utils import get_rete_conditions, NEGCOND + + +class BaseTestSheerkaRuleManagerRulesCompilation(TestUsingMemoryBasedSheerka): + @staticmethod + def check_against_rete(rule_expression, rule_conditions, objects): + """ + + :param rule_expression: + :param rule_conditions: + :param objects: list of tuple(name, obj) + :return: + """ + + # check against a Rete network + network = ReteNetwork() + rule = Rule("test", rule_expression, None) + rule.metadata.id = 9999 + rule.metadata.is_compiled = True + rule.metadata.is_enabled = True + rule.rete_disjunctions = rule_conditions + network.add_rule(rule) + + for name, value in objects.items(): + network.add_obj(name, value) + + matches = list(network.matches) + assert len(matches) == 1 + + @staticmethod + def check_against_python(context, rule_expression, rule_conditions, objects, expected_result=True): + bag = {} + for name, value in objects.items(): + bag[name] = value + + evaluate_rules_service = context.sheerka.services[SheerkaEvaluateRules.NAME] + rule = Rule(name="test", predicate=rule_expression) + rule.compiled_conditions = rule_conditions + rule.metadata.is_enabled = True + rule.metadata.is_compiled = True + evaluation = evaluate_rules_service.evaluate_rules(context, [rule], bag, set()) + assert expected_result in evaluation and len(evaluation[expected_result]) == 1 + + @staticmethod + def evaluate_condition(context, expression, condition, objects): + with context.push("Testing conditions SheerkaRuleManagerRulesCompilation", expression) as sub_context: + sub_context.protected_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED) + sub_context.protected_hints.add(BuiltinConcepts.EVAL_WHERE_REQUESTED) + sub_context.protected_hints.add(BuiltinConcepts.EVAL_UNTIL_SUCCESS_REQUESTED) + sub_context.protected_hints.add(BuiltinConcepts.EVAL_QUESTION_REQUESTED) + sub_context.sheerka.add_many_to_short_term_memory(sub_context, objects) + + evaluator = PythonEvaluator() + return evaluator.eval(sub_context, condition.return_value) + + @classmethod + def get_testing_objects(cls, context, objects_names, objects_mappings=None): + if objects_names is None: + return {} + + result = {} + for name in objects_names: + if isinstance(name, tuple): + name, value = name + if value == "sheerka": + value = context.sheerka + else: + value = None + + if objects_mappings and value in objects_mappings: + value = objects_mappings[value] + + if name == "__ret": + return_value = ReturnValueConcept("Test", True, value) + result["__ret"] = return_value + elif name == "__ret.status": + return_value = ReturnValueConcept("Test", True, value) + result["__ret.status"] = return_value.status + elif name == "body": + return_value = ReturnValueConcept("Test", True, value) + result["body"] = return_value.body + elif name == "a_string": + result["a_string"] = value or "hello world!" + elif name == "an_int": + result["an_int"] = value or 10 + else: + result[name] = value + + return result + + @staticmethod + def get_variables_names_from_expected_variables(expected_variables): + return {v[0] if isinstance(v, tuple) else v for v in expected_variables} + + @staticmethod + def func_true(*args, **kwargs): + return True + + @staticmethod + def func_identity(x): + return x + + def validate_python_test(self, + context, + expression, + expected_compiled, + expected_text, + expected_variables, + expected_not_variables, + expected_objects): + sheerka = context.sheerka + + parser = ExpressionParser() + error_sink = ErrorSink() + parser_input = ParserInput(expression) + parser.reset_parser_input(parser_input, error_sink) + parsed = parser.parse_input(context, parser_input, error_sink) + + visitor = PythonConditionExprVisitor(context) + conditions = visitor.get_conditions(parsed) + + assert len(conditions) == 1 + assert isinstance(conditions[0], CompiledCondition) + + if expected_compiled is None: + # manage cases where we only check for variable existence + assert conditions[0].evaluator_type is None + assert conditions[0].return_value is None + else: + # check what was compiled + ast_ = ast.parse(expected_compiled, "", 'exec' if "\n" in expected_compiled else 'eval') + expected_python_node = PythonNode(expected_compiled, ast_, expected_text) + assert conditions[0].evaluator_type == PYTHON_EVALUATOR_NAME + assert sheerka.isinstance(conditions[0].return_value, BuiltinConcepts.RETURN_VALUE) + assert sheerka.objvalue(conditions[0].return_value) == expected_python_node + + # check that variables detected + assert conditions[0].variables == self.get_variables_names_from_expected_variables(expected_variables) + + # check that variables that MUST not be present + assert conditions[0].not_variables == expected_not_variables + + # check the objects returned + assert len(expected_objects) == len(conditions[0].objects) + for obj in expected_objects: + if isinstance(obj, tuple): + assert conditions[0].objects[obj[0]] == obj[1] + else: + assert obj in conditions[0].objects + + return conditions + + +class TestSheerkaRuleManagerRulesCompilationExists(BaseTestSheerkaRuleManagerRulesCompilation): + """ + Testing variable existence: + __ret (exists) + __ret.status (exists) + body (exists) + __ret and __ret.status (both exist) + """ + + @pytest.mark.parametrize("expression, expected_as_list_of_str, expected_variables", [ + ( + "__ret", ["#__x_00__|__name__|'__ret'"], {"__ret"} + ), + ( + "__ret.status", ["#__x_00__|__name__|'__ret.status'"], {"__ret.status"} + ), + ( + "body", ["#__x_00__|__name__|'body'"], {"body"} + ), + ( + "__ret and __ret.status", + ["#__x_00__|__name__|'__ret'", "#__x_01__|__name__|'__ret.status'"], + {"__ret", "__ret.status"} + ), + ]) + def test_rete(self, expression, expected_as_list_of_str, expected_variables): + sheerka, context = self.init_test().unpack() + parser = ExpressionParser() + expected = get_rete_conditions(*expected_as_list_of_str) + + error_sink = ErrorSink() + parser_input = ParserInput(expression) + parser.reset_parser_input(parser_input, error_sink) + parsed = parser.parse_input(context, parser_input, error_sink) + + visitor = ReteConditionExprVisitor(context) + conditions = visitor.get_conditions(parsed) + + assert conditions == [expected] + + # check against a Rete network + objects = self.get_testing_objects(context, expected_variables) + self.check_against_rete(expression, conditions, objects) + + @pytest.mark.parametrize("expression, expected_variables", [ + ("__ret", {"__ret"}), + ("__ret.status", {"__ret.status"}), + ("body", {"body"}), + ("__ret and __ret.status", {"__ret", "__ret.status"}) + ]) + def test_python(self, expression, expected_variables): + sheerka, context = self.init_test().unpack() + + conditions = self.validate_python_test(context, + expression, + None, + None, + expected_variables, + set(), + set()) + + # check against SheerkaEvaluateRules + objects = self.get_testing_objects(context, expected_variables) + self.check_against_python(context, expression, conditions, objects) + + +class TestSheerkaRuleManagerRulesCompilationNotExists(BaseTestSheerkaRuleManagerRulesCompilation): + """ + Testing NOT existence + not __ret + not not __ret + not __ret.status + not body + not __ret and not __ret.status + __ret and not ret.status + __ret and not __error + """ + + @pytest.mark.parametrize("expression, expected_as_list_of_str", [ + ("not __ret", [NEGCOND("#__x_00__|__name__|'__ret'")]), + ("not not __ret", ["#__x_00__|__name__|'__ret'"]), + ("not __ret.status", [NEGCOND("#__x_00__|__name__|'__ret.status'")]), + ("not body", [NEGCOND("#__x_00__|__name__|'body'")]), + ( + "not __ret and not __ret.status", + [NEGCOND("#__x_00__|__name__|'__ret'"), NEGCOND("#__x_01__|__name__|'__ret.status'")] + ), + ( + "__ret and not __ret.status", + ["#__x_00__|__name__|'__ret'", NEGCOND("#__x_01__|__name__|'__ret.status'")] + ), + ( + "__ret and not __error", + ["#__x_00__|__name__|'__ret'", NEGCOND("#__x_01__|__name__|'__error'")] + ), + ]) + def test_rete(self, expression, expected_as_list_of_str): + sheerka, context = self.init_test().unpack() + parser = ExpressionParser() + expected = get_rete_conditions(*expected_as_list_of_str) + + error_sink = ErrorSink() + parser_input = ParserInput(expression) + parser.reset_parser_input(parser_input, error_sink) + parsed = parser.parse_input(context, parser_input, error_sink) + + visitor = ReteConditionExprVisitor(context) + conditions = visitor.get_conditions(parsed) + + assert conditions == [expected] + + # # check against a Rete network + # objects = self.get_testing_objects(context, expected_variables) + # self.check_against_rete(expression, conditions, objects) + + @pytest.mark.parametrize("expression, expected_variables, expected_not_variables", [ + ("not __ret", set(), {"__ret"}), + ("not not __ret", {"__ret"}, set()), + ("not __ret.status", set(), {"__ret.status"}), + ("not body", set(), {"body"}), + ("not __ret and not __ret.status", set(), {"__ret", "__ret.status"}), + ("__ret and not __ret.status", {"__ret"}, {"__ret.status"}), + ("__ret and not __error", {"__ret"}, {"__error"}), + ]) + def test_python(self, expression, expected_variables, expected_not_variables): + sheerka, context = self.init_test().unpack() + + conditions = self.validate_python_test(context, + expression, + None, + None, + expected_variables, + expected_not_variables, + set()) + + # check against SheerkaEvaluateRules + objects = self.get_testing_objects(context, expected_variables) + self.check_against_python(context, expression, conditions, objects) + + +class TestSheerkaRuleManagerRulesCompilationEquality(BaseTestSheerkaRuleManagerRulesCompilation): + """ + Testing simple equality: + a == 10 + __ret.status == True + self == 'a' + self == sheerka + self == BuiltinConcepts.TO_DICT + self == hello 'my friend' + """ + + @pytest.mark.parametrize("expression, expected_as_list_of_str, expected_variables", [ + ("a == 10", ["#__x_00__|__name__|'a'", "#__x_00__|__self__|10"], {("a", 10)}), + ("__ret.status == True", ["#__x_00__|__name__|'__ret'", "#__x_00__|status|True"], {"__ret"}), + ("self == 'a'", ["#__x_00__|__name__|'self'", "#__x_00__|__self__|'a'"], {("self", 'a')}), + ("self == sheerka", ["#__x_00__|__name__|'self'", "#__x_00__|__self__|'__sheerka__'"], {("self", "sheerka")}), + ( + "self == BuiltinConcepts.TO_DICT", + ["#__x_00__|__name__|'self'", "#__x_00__|__self__|BuiltinConcepts.TO_DICT"], + {("self", BuiltinConcepts.TO_DICT)} + ), + ("self == hello 'my friend'", + ["#__x_00__|__name__|'self'", + "#__x_00__|__is_concept__|True", + "#__x_00__|key|'hello __var__0'", + "#__x_00__|a|'my friend'"], + {("self", "hello_my_friend")} + ) + ]) + def test_rete(self, expression, expected_as_list_of_str, expected_variables): + sheerka, context, greetings = self.init_test().with_concepts( + Concept("greetings", definition="hello a", definition_type=DEFINITION_TYPE_DEF).def_var("a"), + create_new=True + ).unpack() + + parser = ExpressionParser() + expected = get_rete_conditions(*expected_as_list_of_str) + + error_sink = ErrorSink() + parser_input = ParserInput(expression) + parser.reset_parser_input(parser_input, error_sink) + parsed = parser.parse_input(context, parser_input, error_sink) + + visitor = ReteConditionExprVisitor(context) + conditions = visitor.get_conditions(parsed) + + assert conditions == [expected] + + # check against a Rete network + objects_mappings = {"hello_my_friend": sheerka.new(greetings, a='my friend')} + objects = self.get_testing_objects(context, expected_variables, objects_mappings) + self.check_against_rete(expression, conditions, objects) + + @pytest.mark.parametrize("expression, expected_compiled, expected_variables, expected_objects", [ + ("a == 10", "a == __o_00__", {("a", 10)}, {("__o_00__", 10)}), + ("__ret.status == True", "__ret.status == __o_00__", {"__ret"}, {("__o_00__", True)}), + ("self == 'a'", "self == __o_00__", {("self", 'a')}, {("__o_00__", 'a')}), + ("self == sheerka", "is_sheerka(self)", {("self", "sheerka")}, {}), + ( + "self == BuiltinConcepts.TO_DICT", + "self == __o_00__", + {("self", BuiltinConcepts.TO_DICT)}, + {("__o_00__", BuiltinConcepts.TO_DICT)} + ), + ( + "self == hello 'my friend'", + """isinstance(self, Concept) and self.key == 'hello __var__0' and self.a == __o_01__""", + {("self", "hello_my_friend")}, + {("__o_01__", "my friend")} + ) + ]) + def test_python(self, expression, expected_compiled, expected_variables, expected_objects): + sheerka, context, greetings = self.init_test().with_concepts( + Concept("greetings", definition="hello a", definition_type=DEFINITION_TYPE_DEF).def_var("a"), + create_new=True + ).unpack() + + conditions = self.validate_python_test(context, + expression, + expected_compiled, + expression, + expected_variables, + set(), + expected_objects) + + # check against SheerkaEvaluateRules + objects_mappings = {"hello_my_friend": sheerka.new(greetings, a='my friend')} + objects = self.get_testing_objects(context, expected_variables, objects_mappings) + self.check_against_python(context, expression, conditions, objects) + + +class TestSheerkaRuleManagerRulesCompilationFunctionsCall(BaseTestSheerkaRuleManagerRulesCompilation): + """ + Testing functions + no var : isinstance('hello', str) + with var: isinstance(a_string, str) + with and : isinstance(a_string, str) and isinstance(an_int, int) + function with concept: isinstance(self, girl) + function with concept: isinstance(self, a little boy) + function with enum: func_true(self, BuiltinConcepts.TO_DICT) + """ + + def test_rete(self): + pass + + @pytest.mark.parametrize("expression, e_compiled, e_text, e_variables, e_objects", [ + ( + "isinstance('hello', str)", + "isinstance(__o_00__, str)", + "isinstance('hello', str)", + set(), + {"__o_00__"} + ), + ( + "isinstance(a_string, str)", + "isinstance(a_string, str)", + "isinstance(a_string, str)", + {"a_string"}, + {} + ), + ( + "not isinstance(an_int, str)", + "not (isinstance(an_int, str))", + "not (isinstance(an_int, str))", + {"an_int"}, + {} + ), + ( + "isinstance(a_string, str) and isinstance(an_int, int)", + "isinstance(a_string, str) and isinstance(an_int, int)", + "isinstance(a_string, str) and isinstance(an_int, int)", + {"an_int", "a_string"}, + {} + ), + ( + "isinstance(self, girl)", + "isinstance(self, __o_00__)", + "isinstance(self, girl)", + {("self", "girl")}, + {"__o_00__"} + ), + ( + "isinstance(self, a little boy)", + "isinstance(self, __o_00__)", + "isinstance(self, a little boy)", + {("self", "a little boy")}, + {"__o_00__"} + ), + ( + "func_true(self, BuiltinConcepts.TO_DICT)", + "func_true(self, BuiltinConcepts.TO_DICT)", + "func_true(self, BuiltinConcepts.TO_DICT)", + {("self", BuiltinConcepts.TO_DICT)}, + {} + ), + ]) + def test_python(self, expression, e_compiled, e_text, e_variables, e_objects): + sheerka, context, girl, little_boy = self.init_test().with_concepts( + Concept("girl"), + Concept("a little boy"), + create_new=True + ).unpack() + + conditions = self.validate_python_test(context, + expression, + e_compiled, + e_text, + e_variables, + set(), + e_objects) + + # check against SheerkaEvaluateRules + objects_mappings = {"girl": girl, "a little boy": little_boy} + objects = self.get_testing_objects(context, e_variables, objects_mappings) + objects["func_true"] = self.func_true + self.check_against_python(context, expression, conditions, objects) + + +class TestSheerkaRuleManagerRulesCompilationRecognizeConcept(BaseTestSheerkaRuleManagerRulesCompilation): + """ + Testing recognize(path, concept) + recognize by name : recognize(__ret.body, greetings) + recognize by id : recognize(__ret.body, c:|1001:) + recognize by name using c_str : recognize(__ret.body, c:greetings:) + recognize by name + str condition : recognize(__ret.body, greetings) and __ret.body.a == 'my friend' + recognize by name + sheerka condition : recognize(__ret.body, greetings) and __ret.body.a == sheerka + recognize by name + concept condition : recognize(__ret.body, greetings) and __ret.body.a == foo + recognize by instance using sheerka : recognize(__ret.body, hello sheerka) + recognize by instance using a string : recognize(__ret.body, hello 'my friend') + recognize by instance using a concept : recognize(__ret.body, hello foo) + recognize by instance using long concept : recognize(__ret.body, hello my best friend) + recognize self : recognize(self, greetings) + """ + + @pytest.mark.parametrize("expression, expected_as_list_of_str, expected_variable, greeting_var", [ + ( + "recognize(__ret.body, greetings)", + ["#__x_00__|__name__|'__ret'", + "#__x_00__|body|#__x_01__", + "#__x_01__|__is_concept__|True", + "#__x_01__|name|'greetings'"], + "__ret", + None + ), + ( + "recognize(__ret.body, c:|1001:)", + ["#__x_00__|__name__|'__ret'", + "#__x_00__|body|#__x_01__", + "#__x_01__|__is_concept__|True", + "#__x_01__|id|'1001'"], + "__ret", + None + ), + ( + "recognize(__ret.body, c:greetings:)", + ["#__x_00__|__name__|'__ret'", + "#__x_00__|body|#__x_01__", + "#__x_01__|__is_concept__|True", + "#__x_01__|name|'greetings'"], + "__ret", + None + ), + ( + "recognize(__ret.body, greetings) and __ret.body.a == 'my friend'", + ["#__x_00__|__name__|'__ret'", + "#__x_00__|body|#__x_01__", + "#__x_01__|__is_concept__|True", + "#__x_01__|name|'greetings'", + "#__x_01__|a|'my friend'"], + "__ret", + "my friend", + ), + ( + "recognize(__ret.body, greetings) and __ret.body.a == sheerka", + ["#__x_00__|__name__|'__ret'", + "#__x_00__|body|#__x_01__", + "#__x_01__|__is_concept__|True", + "#__x_01__|name|'greetings'", + "#__x_01__|a|'__sheerka__'"], + "__ret", + "sheerka", + ), + ( + "recognize(__ret.body, greetings) and __ret.body.a == foo", + ["#__x_00__|__name__|'__ret'", + "#__x_00__|body|#__x_01__", + "#__x_01__|__is_concept__|True", + "#__x_01__|name|'greetings'", + "#__x_01__|a|#__x_02__", + "#__x_02__|__is_concept__|True", + "#__x_02__|key|'foo'"], + "__ret", + "foo", + ), + ( + "recognize(__ret.body, hello sheerka)", + ["#__x_00__|__name__|'__ret'", + "#__x_00__|body|#__x_01__", + "#__x_01__|__is_concept__|True", + "#__x_01__|key|'hello __var__0'", + "#__x_01__|a|'__sheerka__'"], + "__ret", + "sheerka", + ), + ( + "recognize(__ret.body, hello 'my friend')", + ["#__x_00__|__name__|'__ret'", + "#__x_00__|body|#__x_01__", + "#__x_01__|__is_concept__|True", + "#__x_01__|key|'hello __var__0'", + "#__x_01__|a|'my friend'"], + "__ret", + "my friend", + ), + ( + "recognize(__ret.body, hello foo)", + ["#__x_00__|__name__|'__ret'", + "#__x_00__|body|#__x_01__", + "#__x_01__|__is_concept__|True", + "#__x_01__|key|'hello __var__0'", + "#__x_01__|a|#__x_02__", + "#__x_02__|__is_concept__|True", + "#__x_02__|key|'foo'"], + "__ret", + "foo", + ), + ( + "recognize(__ret.body, hello my best friend)", + ["#__x_00__|__name__|'__ret'", + "#__x_00__|body|#__x_01__", + "#__x_01__|__is_concept__|True", + "#__x_01__|key|'hello __var__0'", + "#__x_01__|a|#__x_02__", + "#__x_02__|__is_concept__|True", + "#__x_02__|key|'my best friend'"], + "__ret", + "my best friend", + ), + ( + "recognize(self, greetings)", + ["#__x_00__|__name__|'self'", + "#__x_00__|__is_concept__|True", + "#__x_00__|name|'greetings'"], + "self", + None, + ) + ]) + def test_rete(self, expression, expected_as_list_of_str, expected_variable, greeting_var): + sheerka, context, greetings, foo, my_best_friend = self.init_test().with_concepts( + Concept("greetings", definition="hello a", definition_type=DEFINITION_TYPE_DEF).def_var("a"), + Concept("foo"), + Concept("my best friend"), + create_new=True + ).unpack() + parser = ExpressionParser() + expected = get_rete_conditions(*expected_as_list_of_str) + + error_sink = ErrorSink() + parser_input = ParserInput(expression) + parser.reset_parser_input(parser_input, error_sink) + parsed = parser.parse_input(context, parser_input, error_sink) + + visitor = ReteConditionExprVisitor(context) + conditions = visitor.get_conditions(parsed) + + assert conditions == [expected] + + # check against a Rete network + variable_map = { + "foo": foo, + "my best friend": my_best_friend, + "sheerka": sheerka + } + variable = variable_map.get(greeting_var, greeting_var) + to_recognize = sheerka.new_from_template(greetings, greetings.key, a=variable) + objects = self.get_testing_objects(context, [(expected_variable, to_recognize)]) + self.check_against_rete(expression, conditions, objects) + + @pytest.mark.parametrize("expression, e_compiled, e_text, e_variables, greeting_var, e_objects", [ + ( + "recognize(__ret.body, greetings)", + "__x_00__ = __ret.body\nisinstance(__x_00__, Concept) and __x_00__.name == 'greetings'", + "__x_00__ = __ret.body\nisinstance(__x_00__, Concept) and __x_00__.name == 'greetings'", + {"__ret"}, + None, + set() + ), + ( + "recognize(__ret.body, c:|1001:)", + "__x_00__ = __ret.body\nisinstance(__x_00__, Concept) and __x_00__.id == '1001'", + "__x_00__ = __ret.body\nisinstance(__x_00__, Concept) and __x_00__.id == '1001'", + {"__ret"}, + None, + set() + ), + ( + "recognize(__ret.body, c:greetings:)", + "__x_00__ = __ret.body\nisinstance(__x_00__, Concept) and __x_00__.name == 'greetings'", + "__x_00__ = __ret.body\nisinstance(__x_00__, Concept) and __x_00__.name == 'greetings'", + {"__ret"}, + None, + set() + ), + ( + "recognize(__ret.body, greetings) and __ret.body.a == 'my friend'", + "__x_00__ = __ret.body\nisinstance(__x_00__, Concept) and __x_00__.name == 'greetings' and __x_00__.a == __o_00__", + "__x_00__ = __ret.body\nisinstance(__x_00__, Concept) and __x_00__.name == 'greetings' and __ret.body.a == 'my friend'", + {"__ret"}, + "my friend", + {("__o_00__", "my friend")} + ), + ( + "recognize(__ret.body, greetings) and __ret.body.a == sheerka", + """__x_00__ = __ret.body +isinstance(__x_00__, Concept) and __x_00__.name == 'greetings' and is_sheerka(__x_00__.a)""", + """__x_00__ = __ret.body +isinstance(__x_00__, Concept) and __x_00__.name == 'greetings' and __ret.body.a == sheerka""", + {"__ret"}, + "sheerka", + set() + ), + ( + "recognize(__ret.body, greetings) and __ret.body.a == foo", + """__x_00__ = __ret.body +__x_01__ = __x_00__.a +isinstance(__x_00__, Concept) and __x_00__.name == 'greetings' and isinstance(__x_01__, Concept) and __x_01__.key == 'foo'""", + """__x_00__ = __ret.body +__x_01__ = __x_00__.a +isinstance(__x_00__, Concept) and __x_00__.name == 'greetings' and __ret.body.a == foo""", + {"__ret"}, + "foo", + set() + ), + ( + "recognize(__ret.body, hello sheerka)", + """__x_00__ = __ret.body +isinstance(__x_00__, Concept) and __x_00__.key == 'hello __var__0' and is_sheerka(__x_00__.a)""", + """__x_00__ = __ret.body +isinstance(__x_00__, Concept) and __x_00__.key == 'hello __var__0' and __x_00__.a == sheerka""", + {"__ret"}, + "sheerka", + set() + ), + ( + "recognize(__ret.body, hello 'my friend')", + """__x_00__ = __ret.body +isinstance(__x_00__, Concept) and __x_00__.key == 'hello __var__0' and __x_00__.a == __o_00__""", + """__x_00__ = __ret.body +isinstance(__x_00__, Concept) and __x_00__.key == 'hello __var__0' and __x_00__.a == 'my friend'""", + {"__ret"}, + "my friend", + {('__o_00__', 'my friend')} + ), + ( + "recognize(__ret.body, hello foo)", + """__x_00__ = __ret.body +__x_01__ = __x_00__.a +isinstance(__x_00__, Concept) and __x_00__.key == 'hello __var__0' and isinstance(__x_01__, Concept) and __x_01__.key == 'foo'""", + """__x_00__ = __ret.body +__x_01__ = __x_00__.a +isinstance(__x_00__, Concept) and __x_00__.key == 'hello __var__0' and __x_00__.a == c:foo|1002:""", + {"__ret"}, + "foo", + {'__o_00__'} + ), + ( + "recognize(__ret.body, hello my best friend)", + """__x_00__ = __ret.body +__x_01__ = __x_00__.a +isinstance(__x_00__, Concept) and __x_00__.key == 'hello __var__0' and isinstance(__x_01__, Concept) and __x_01__.key == 'my best friend'""", + """__x_00__ = __ret.body +__x_01__ = __x_00__.a +isinstance(__x_00__, Concept) and __x_00__.key == 'hello __var__0' and __x_00__.a == c:my best friend|1003:""", + {"__ret"}, + "my best friend", + {'__o_00__'} + ), + ( + "recognize(self, greetings)", + "isinstance(self, Concept) and self.name == 'greetings'", + "isinstance(self, Concept) and self.name == 'greetings'", + {"self"}, + None, + set() + ) + ]) + def test_python(self, expression, e_compiled, e_text, e_variables, greeting_var, e_objects): + sheerka, context, greetings, foo, my_best_friend = self.init_test().with_concepts( + Concept("greetings", definition="hello a", definition_type=DEFINITION_TYPE_DEF).def_var("a"), + Concept("foo"), + Concept("my best friend"), + create_new=True + ).unpack() + + conditions = self.validate_python_test(context, + expression, + e_compiled, + e_text, + e_variables, + set(), + e_objects) + + # check against SheerkaEvaluateRules + variable_map = { + "foo": foo, + "my best friend": my_best_friend, + "sheerka": Expando("sheerka", {}) + } + variable = variable_map.get(greeting_var, greeting_var) + to_recognize = sheerka.new_from_template(greetings, greetings.key, a=variable) + expected_variable = next(iter(e_variables)) + objects = self.get_testing_objects(context, [(expected_variable, to_recognize)]) + self.check_against_python(context, expression, conditions, objects) + + +class TestSheerkaRuleManagerRulesCompilationEvalQuestionConcept(BaseTestSheerkaRuleManagerRulesCompilation): + """ + Testing question using concepts + with no variable : girl is a human + with variable : self is a human + with long concept : the little boy is a human being + with long concept + variable : the little boy is a self + with long concept + variable : self is a human being + """ + + def test_rete(self): + pass # I don't know yet what to do + + @pytest.mark.parametrize("expression, expected_compiled, expected_variables", [ + ( + "girl is a human", + "evaluate_question(__o_00__)", + set(), + ), + ( + "self is a human", + "evaluate_question(__o_00__)", + {"self"}, + ), + ( + "the little boy is a human being", + "evaluate_question(__o_00__)", + set(), + ), + ( + "the little boy is a self", + "evaluate_question(__o_00__)", + {"self"}, + ), + ( + "self is a human being", + "evaluate_question(__o_00__)", + {"self"}, + ), + ]) + def test_python(self, expression, expected_compiled, expected_variables): + sheerka, context, girl, human, little_boy, human_being, isa = self.init_test().with_concepts( + Concept("girl"), + Concept("human"), + Concept("the little boy"), + Concept("human being"), + Concept("x is a y", pre="is_question()").def_var("x").def_var("y"), + create_new=True + ).unpack() + + parser = ExpressionParser() + error_sink = ErrorSink() + parser_input = ParserInput(expression) + parser.reset_parser_input(parser_input, error_sink) + parsed = parser.parse_input(context, parser_input, error_sink) + + visitor = PythonConditionExprVisitor(context) + conditions = visitor.get_conditions(parsed) + + assert len(conditions) == 1 + assert isinstance(conditions[0], CompiledCondition) + + ast_ = ast.parse(expected_compiled, "", 'exec' if "\n" in expected_compiled else 'eval') + expected_python_node = PythonNode(expected_compiled, ast_, expression) + assert conditions[0].evaluator_type == PYTHON_EVALUATOR_NAME + assert sheerka.isinstance(conditions[0].return_value, BuiltinConcepts.RETURN_VALUE) + assert sheerka.objvalue(conditions[0].return_value) == expected_python_node + + assert "__o_00__" in conditions[0].objects + assert conditions[0].variables == expected_variables + + assert len(conditions[0].concepts_to_reset) == 1 + assert sheerka.isinstance(next(iter(conditions[0].concepts_to_reset)), isa) + + # check against SheerkaEvaluateRules + self.check_against_python(context, expression, conditions, + self.get_testing_objects(context, expected_variables)) + + +class TestSheerkaRuleManagerRulesCompilationEvalQuestionConceptWithNot(BaseTestSheerkaRuleManagerRulesCompilation): + """ + Testing question using concepts mixed with others conditions + using not : not girl is a human + using not : not self is a human + """ + + def test_rete(self): + pass + + @pytest.mark.parametrize( + "expression, expected_compiled, expected_text, expected_variables", [ + ( + "not girl is a human", + "not (evaluate_question(__o_00__))", + "not (girl is a human)", + set(), + ), + ( + "not self is a human", + "not (evaluate_question(__o_00__))", + "not (self is a human)", + {"self"}, + ), + ]) + def test_python(self, expression, expected_compiled, expected_text, expected_variables): + sheerka, context, girl, human, little_boy, human_being, isa = self.init_test().with_concepts( + Concept("girl"), + Concept("human"), + Concept("the little boy"), + Concept("human being"), + Concept("x is a y", pre="is_question()").def_var("x").def_var("y"), + create_new=True + ).unpack() + + conditions = self.validate_python_test(context, + expression, + expected_compiled, + expected_text, + expected_variables, + set(), + {"__o_00__"}) + + # check against SheerkaEvaluateRules + self.check_against_python(context, + expression, + conditions, + self.get_testing_objects(context, expected_variables), + False) + + +class TestSheerkaRuleManagerRulesCompilationEvalConceptMixedWithOther(BaseTestSheerkaRuleManagerRulesCompilation): + """ + Testing question using concepts mixed with others conditions + using and with python : girl is a human being and isinstance(a_string, str) + using and with another concept : self is a human being and isinstance(self, girl) + starting with python : isinstance(x, str) and girl is a human being + starting with python : isinstance(self, girl) and self is a human being + multiple python + same variable : self is a human being and isinstance(self, girl) and isinstance(self, girl) + """ + + def test_rete(self): + pass # I don't know yet what to do + + @pytest.mark.parametrize( + "expression, expected_compiled, expected_variables, expected_objects", [ + ( + "girl is a human being and isinstance(a_string, str)", + "evaluate_question(__o_00__) and isinstance(a_string, str)", + {"a_string"}, + {"__o_00__"} + ), + ( + "self is a human being and isinstance(self, girl)", + "evaluate_question(__o_00__) and isinstance(self, __o_01__)", + {("self", "girl")}, + {"__o_00__", "__o_01__"} + ), + ( + "isinstance(a_string, str) and girl is a human being", + "isinstance(a_string, str) and evaluate_question(__o_00__)", + {"a_string"}, + {"__o_00__"} + ), + ( + "isinstance(self, girl) and self is a human being", + "isinstance(self, __o_00__) and evaluate_question(__o_01__)", + {("self", "girl")}, + {"__o_00__", "__o_01__"} + ), + ( + "self is a human being and isinstance(self, girl) and isinstance(self, girl)", + "evaluate_question(__o_00__) and isinstance(self, __o_01__) and isinstance(self, __o_02__)", + {("self", "girl")}, + {"__o_00__", "__o_01__", "__o_02__"} + ), + ]) + def test_python(self, expression, expected_compiled, expected_variables, expected_objects): + sheerka, context, girl, human_being, isa = self.init_test().with_concepts( + Concept("girl"), + Concept("human being"), + Concept("x is a y", pre="is_question()").def_var("x").def_var("y"), + create_new=True + ).unpack() + + conditions = self.validate_python_test(context, + expression, + expected_compiled, + expression, + expected_variables, + set(), + expected_objects) + + # check against SheerkaEvaluateRules + variable_mapping = { + "girl": girl, + "human being": human_being + } + testing_objects = self.get_testing_objects(context, expected_variables, variable_mapping) + self.check_against_python(context, + expression, + conditions, + testing_objects, + True) + + +class TestSheerkaRuleManagerRulesCompilationEvalNonQuestionConcept(BaseTestSheerkaRuleManagerRulesCompilation): + """ + Testing complex concepts composition + with BNF : twenty two + additions : twenty two + one + additions : twenty two + twenty one + additions : twenty two plus one + additions : twenty two plus twenty one + with function: func_identity(twenty two) + with function: func_identity(twenty two + one) + with function: func_identity(twenty two + twenty one) + with function: func_identity(twenty two plus one) + with function: func_identity(twenty two plus twenty one) + """ + + def test_rete(self): + pass # I don't know yet what to do + + @pytest.mark.parametrize("expression, e_compiled, e_text, e_objects, e_result", [ + ( + "twenty two", + "__o_00__", + "twenty two", + {"__o_00__"}, + 22 + ), + ( + "twenty two + one", + "__o_00__", + "twenty two + one", + {("__o_00__", 23)}, + 23 + ), + ( + "twenty two + twenty one", + "__o_00__", + "twenty two + twenty one", + {("__o_00__", 43)}, + 43 + ), + ( + "twenty two plus one", + "__o_00__", + "twenty two plus one", + {"__o_00__"}, + 23 + ), + ( + "twenty two plus twenty one", + "__o_00__", + "twenty two plus twenty one", + {"__o_00__"}, + 43 + ), + ( + "func_identity(twenty two)", + "func_identity(__o_00__)", + "func_identity(twenty two)", + {"__o_00__"}, + 22 + ), + ( + "func_identity(twenty two + one)", + "func_identity(__o_00__)", + "func_identity(twenty two + one)", + {("__o_00__", 23)}, + 23 + ), + ( + "func_identity(twenty two + twenty one)", + "func_identity(__o_00__)", + "func_identity(twenty two + twenty one)", + {("__o_00__", 43)}, + 43 + ), + ( + "func_identity(twenty two plus one)", + "func_identity(__o_00__)", + "func_identity(twenty two plus one)", + {"__o_00__"}, + 23 + ), + ( + "func_identity(twenty two plus twenty one)", + "func_identity(__o_00__)", + "func_identity(twenty two plus twenty one)", + {"__o_00__"}, + 43 + ), + ]) + def test_python(self, expression, e_compiled, e_text, e_objects, e_result): + sheerka, context, one, two, twenties, plus = self.init_test().with_concepts( + Concept("one", body="1"), + Concept("two", body="2"), + Concept("twenties", definition="'twenty' (one|two)=n", body='20 + n').def_var("n"), + Concept("a plus b", body="a + b").def_var("a").def_var("b"), + create_new=True + ).unpack() + + conditions = self.validate_python_test(context, + expression, + e_compiled, + e_text, + set(), + set(), + e_objects) + + # check against SheerkaEvaluateRules + namespace = {"func_identity": self.func_identity} + res = self.evaluate_condition(context, expression, conditions[0], namespace) + assert res.status + assert sheerka.objvalue(res) == e_result + + +class TestSheerkaRuleManagerRulesCompilationMultipleSameConcept(BaseTestSheerkaRuleManagerRulesCompilation): + """ + Test when a concept returns multiple results + The compilation should fail : No need to execute a condition if we are not sure of the meaning ? + self is a bar + """ + + def test_rete(self): + sheerka, context, bar, isa_1, isa_2 = self.init_test().with_concepts( + Concept("bar"), + Concept("x is a y").def_var("x").def_var("y"), + Concept("u is a v").def_var("u").def_var("v"), + create_new=True + ).unpack() + + with pytest.raises(FailedToCompileError): + parser = ExpressionParser() + + error_sink = ErrorSink() + parser_input = ParserInput("self is a bar") + parser.reset_parser_input(parser_input, error_sink) + parsed = parser.parse_input(context, parser_input, error_sink) + + visitor = ReteConditionExprVisitor(context) + visitor.get_conditions(parsed) + + def test_python(self): + sheerka, context, bar, isa_1, isa_2 = self.init_test().with_concepts( + Concept("bar"), + Concept("x is a y").def_var("x").def_var("y"), + Concept("u is a v").def_var("u").def_var("v"), + create_new=True + ).unpack() + + with pytest.raises(FailedToCompileError): + parser = ExpressionParser() + error_sink = ErrorSink() + parser_input = ParserInput("self is a bar") + parser.reset_parser_input(parser_input, error_sink) + parsed = parser.parse_input(context, parser_input, error_sink) + + visitor = PythonConditionExprVisitor(context) + visitor.get_conditions(parsed) + + +class TestSheerkaRuleManagerRulesCompilationNot(BaseTestSheerkaRuleManagerRulesCompilation): + """ + Testing not + not __ret.status == True + not recognize(__ret.body, hello sheerka) + + """ + pass diff --git a/tests/core/test_builtin_helpers.py b/tests/core/test_builtin_helpers.py index 883b841..dd81b9e 100644 --- a/tests/core/test_builtin_helpers.py +++ b/tests/core/test_builtin_helpers.py @@ -182,6 +182,17 @@ class TestBuiltinHelpers(TestUsingMemoryBasedSheerka): concept = Concept("foo", pre=pre) assert core.builtin_helpers.is_a_question(context, concept) == expected + def test_context_hints_are_reset_when_call_evaluate_from_source(self): + sheerka, context, one = self.init_concepts(Concept("one", body="1")) + context.add_to_global_hints(BuiltinConcepts.EVAL_BODY_REQUESTED) + context.add_to_protected_hints(BuiltinConcepts.EVAL_BODY_REQUESTED) + context.add_to_private_hints(BuiltinConcepts.EVAL_BODY_REQUESTED) + + res = core.builtin_helpers.evaluate_from_source(context, "one", eval_body=False) + + evaluated = [r for r in res if r.status][0].body + assert evaluated.body is NotInit + # @pytest.mark.parametrize("return_values", [ # None, # [] diff --git a/tests/core/test_sheerka.py b/tests/core/test_sheerka.py index 10d8033..b86bdb0 100644 --- a/tests/core/test_sheerka.py +++ b/tests/core/test_sheerka.py @@ -439,27 +439,27 @@ class TestSheerkaUsingMemoryBasedSheerka(TestUsingMemoryBasedSheerka): def test_i_can_get_error_for_simple_objects(self, obj, expected): sheerka, context = self.init_test().unpack() - assert sheerka.get_errors(obj) == expected + assert sheerka.get_errors(context, obj) == expected def test_i_can_get_error_when_builtin_concept_in_error(self): sheerka, context = self.init_test().unpack() obj = sheerka.new(BuiltinConcepts.ONTOLOGY_ALREADY_DEFINED) - assert sheerka.get_errors(obj) == [obj] + assert sheerka.get_errors(context, obj) == [obj] def test_i_can_get_error_when_return_value(self): sheerka, context = self.init_test().unpack() error = sheerka.err("an error") ret_val = ReturnValueConcept("Test", False, sheerka.err("an error")) - assert sheerka.get_errors(ret_val) == [error] + assert sheerka.get_errors(context, ret_val) == [error] def test_i_can_get_inner_error(self): sheerka, context = self.init_test().unpack() error = sheerka.err("an error") ret_val = ReturnValueConcept("Test", False, sheerka.err("an error")) - assert sheerka.get_errors(ret_val) == [error] + assert sheerka.get_errors(context, ret_val) == [error] def test_i_can_get_error_when_embedded_errors(self): sheerka, context = self.init_test().unpack() @@ -470,7 +470,7 @@ class TestSheerkaUsingMemoryBasedSheerka(TestUsingMemoryBasedSheerka): error = sheerka.err([concept_eval_error, unknown_concept, not_an_error]) ret_val = ReturnValueConcept("Test", False, error) - errors_found = sheerka.get_errors(ret_val) + errors_found = sheerka.get_errors(context, ret_val) assert errors_found == [error, concept_eval_error, unknown_concept] @@ -488,7 +488,7 @@ class TestSheerkaUsingMemoryBasedSheerka(TestUsingMemoryBasedSheerka): multiple_error = sheerka.new(BuiltinConcepts.MULTIPLE_ERRORS, body=[python_error, value_not_found]) ret_val_2 = ReturnValueConcept("Test", False, multiple_error) - errors_found = sheerka.get_errors([ret_val_1, ret_val_2]) + errors_found = sheerka.get_errors(context, [ret_val_1, ret_val_2]) assert errors_found == [error, concept_eval_error, unknown_concept, multiple_error, python_error, value_not_found] @@ -502,7 +502,7 @@ class TestSheerkaUsingMemoryBasedSheerka(TestUsingMemoryBasedSheerka): error = sheerka.err([concept_eval_error, unknown_concept, python_error]) ret_val = ReturnValueConcept("Test", False, error) - errors_found = sheerka.get_errors(ret_val, __type=BuiltinConcepts.CONCEPT_EVAL_ERROR) + errors_found = sheerka.get_errors(context, ret_val, __type=BuiltinConcepts.CONCEPT_EVAL_ERROR) assert errors_found == [concept_eval_error] def test_i_can_filter_error_by_class_name(self): @@ -514,7 +514,7 @@ class TestSheerkaUsingMemoryBasedSheerka(TestUsingMemoryBasedSheerka): error = sheerka.err([concept_eval_error, unknown_concept, python_error]) ret_val = ReturnValueConcept("Test", False, error) - errors_found = sheerka.get_errors(ret_val, __type="PythonErrorNode") + errors_found = sheerka.get_errors(context, ret_val, __type="PythonErrorNode") assert errors_found == [python_error] def test_i_can_filter_error_by_concept_attribute(self): @@ -526,7 +526,7 @@ class TestSheerkaUsingMemoryBasedSheerka(TestUsingMemoryBasedSheerka): error = sheerka.err([concept_eval_error, unknown_concept, python_error]) ret_val = ReturnValueConcept("Test", False, error) - errors_found = sheerka.get_errors(ret_val, concept_ref="a_concept_ref") + errors_found = sheerka.get_errors(context, ret_val, concept_ref="a_concept_ref") assert errors_found == [unknown_concept] def test_i_can_filter_error_by_class_attribute(self): @@ -538,7 +538,7 @@ class TestSheerkaUsingMemoryBasedSheerka(TestUsingMemoryBasedSheerka): error = sheerka.err([concept_eval_error, unknown_concept, python_error]) ret_val = ReturnValueConcept("Test", False, error) - errors_found = sheerka.get_errors(ret_val, source="error source") + errors_found = sheerka.get_errors(context, ret_val, source="error source") assert errors_found == [python_error] def test_i_can_filter_error_on_multiple_criteria(self): @@ -550,14 +550,14 @@ class TestSheerkaUsingMemoryBasedSheerka(TestUsingMemoryBasedSheerka): error = sheerka.err([concept_eval_error, unknown_concept, value_not_found]) ret_val = ReturnValueConcept("Test", False, error) - errors_found = sheerka.get_errors(ret_val, __type="ValueNotFound", item="an_item", value="a value") + errors_found = sheerka.get_errors(context, ret_val, __type="ValueNotFound", item="an_item", value="a value") assert errors_found == [value_not_found] def test_i_cannot_get_error_when_return_value_s_status_is_true(self): sheerka, context = self.init_test().unpack() ret_val = ReturnValueConcept("Test", True, sheerka.err("an error")) - assert sheerka.get_errors(ret_val) == [] + assert sheerka.get_errors(context, ret_val) == [] class TestSheerkaUsingFileBasedSheerka(TestUsingFileBasedSheerka): diff --git a/tests/core/test_utils.py b/tests/core/test_utils.py index e8a3f5d..a753a83 100644 --- a/tests/core/test_utils.py +++ b/tests/core/test_utils.py @@ -469,3 +469,28 @@ def test_tokens_are_matching_when_eof_differs(): tokens2 = Tokenizer(expression2, yield_eof=False) assert core.utils.tokens_are_matching(tokens1, tokens2) + + +def test_sheerka_hasattr_get_attr(): + class A: + def __init__(self, property_value): + self.property_value = property_value + + def as_bag(self): + return {"prop": self.property_value} + + # test object with bag + a = A("foo") + assert core.utils.sheerka_hasattr(a, "prop") + assert core.utils.sheerka_getattr(a, "prop") == "foo" + assert not core.utils.sheerka_hasattr(a, "property_value") + with pytest.raises(AttributeError): + core.utils.sheerka_getattr(a, "property_value") + + # test for concept + concept = Concept("foo").def_var("a", "value").auto_init() + assert core.utils.sheerka_hasattr(concept, "a") + assert core.utils.sheerka_getattr(concept, "a") == "value" + assert not core.utils.sheerka_hasattr(concept, "b") + with pytest.raises(AttributeError): + core.utils.sheerka_getattr(concept, "b") diff --git a/tests/evaluators/test_PythonEvaluator.py b/tests/evaluators/test_PythonEvaluator.py index 2bebeaa..a7069df 100644 --- a/tests/evaluators/test_PythonEvaluator.py +++ b/tests/evaluators/test_PythonEvaluator.py @@ -2,13 +2,14 @@ import ast import pytest +from core.ast_helpers import NamesWithAttributesVisitor from core.builtin_concepts import ReturnValueConcept, ParserResultConcept, BuiltinConcepts from core.builtin_helpers import CreateObjectIdentifiers from core.concept import Concept from core.sheerka.services.SheerkaConceptManager import SheerkaConceptManager from core.sheerka.services.SheerkaExecute import ParserInput from core.tokenizer import Tokenizer -from evaluators.PythonEvaluator import PythonEvaluator, PythonEvalError, NamesWithAttributesVisitor +from evaluators.PythonEvaluator import PythonEvaluator, PythonEvalError from parsers.BaseNodeParser import SourceCodeNode, SourceCodeWithConceptNode from parsers.FunctionParser import FunctionParser from parsers.PythonParser import PythonNode, PythonParser @@ -386,6 +387,16 @@ class TestPythonEvaluator(TestUsingMemoryBasedSheerka): assert evaluated.status assert evaluated.value == context.sheerka.get_rule_by_id(str(value)).name + def test_i_cannot_call_methods_with_side_effect_when_is_question_is_set(self): + sheerka, context, foo, bar = self.init_concepts("foo", "bar") + context.add_to_protected_hints(BuiltinConcepts.EVAL_QUESTION_REQUESTED) + + parsed = PythonParser().parse(context, ParserInput("set_isa(foo, bar)")) + python_evaluator = PythonEvaluator() + evaluated = python_evaluator.eval(context, parsed) + + assert sheerka.has_error(context, evaluated, __type="MethodAccessError") + def test_something(self): def func(**kwargs): for k, v in kwargs.items(): diff --git a/tests/non_reg/test_sheerka_non_reg.py b/tests/non_reg/test_sheerka_non_reg.py index 95cf658..249e28c 100644 --- a/tests/non_reg/test_sheerka_non_reg.py +++ b/tests/non_reg/test_sheerka_non_reg.py @@ -7,6 +7,7 @@ from core.sheerka.services.SheerkaConceptManager import SheerkaConceptManager from evaluators.MutipleSameSuccessEvaluator import MultipleSameSuccessEvaluator from evaluators.OneSuccessEvaluator import OneSuccessEvaluator from evaluators.PythonEvaluator import PythonEvalError +from sheerkapython.python_wrapper import MethodAccessError from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka from tests.parsers.parsers_utils import CMV, CC, compare_with_test_object, CB @@ -642,7 +643,7 @@ as: assert res[0].body == 21 def test_i_can_use_where_in_bnf(self): - sheerka = self.get_sheerka() + sheerka, context = self.init_test().unpack() init = [ "def concept one as 1", @@ -683,12 +684,12 @@ as: assert len(res) == 1 assert not res[0].status assert sheerka.isinstance(res[0].body, BuiltinConcepts.MULTIPLE_ERRORS) - assert str(BuiltinConcepts.CONDITION_FAILED) in [error.key for error in sheerka.get_errors(res[0].body.body)] + assert sheerka.has_error(context, res, __type=BuiltinConcepts.CONDITION_FAILED) res = sheerka.evaluate_user_input("eval twenty three") assert len(res) == 1 assert not res[0].status - assert str(BuiltinConcepts.CONDITION_FAILED) in [error.key for error in sheerka.get_errors(res[0].body.body)] + assert sheerka.has_error(context, res, __type=BuiltinConcepts.CONDITION_FAILED) def test_i_can_manage_some_type_of_infinite_recursion(self): sheerka = self.get_sheerka() @@ -1298,6 +1299,7 @@ as: assert sheerka.objvalue(res[0].body.get_value("qty")) == 2 def test_i_can_implement_the_concept_and(self): + # Normally, redefining and leads to a circular ref between the concept and the python init = [ "def concept x and y as x and y", "set_is_lesser(__PRECEDENCE, c:x and y:, 'Sya')", @@ -1308,3 +1310,55 @@ as: assert len(res) == 1 assert res[0].status + + def test_i_can_use_result_from_memory_filtering(self): + init = [ + "def concept female", + "def concept girl", + "set_isa(girl, female)", + "def concept she ret memory('isa(self, female)')", + "girl" + ] + sheerka = self.init_scenario(init) + context = self.get_context(sheerka) + + res = sheerka.evaluate_user_input("set_attr(she, 'my_attr', 'my value')") + + assert len(res) == 1 + assert res[0].status + + girl_from_memory = sheerka.get_last_from_memory(context, "girl") + assert girl_from_memory.obj.get_value("my_attr") == "my value" + + def test_i_can_use_result_from_memory_filtering_within_other_concept(self): + init = [ + "def concept female", + "def concept girl", + "set_isa(girl, female)", + "def concept she ret memory('isa(self, female)')", + "def concept x attribute y equals z as set_attr(x, y, z)", + "girl" + ] + sheerka = self.init_scenario(init) + context = self.get_context(sheerka) + + res = sheerka.evaluate_user_input("eval she attribute 'my_attr' equals 'my value'") + + assert len(res) == 1 + assert res[0].status + + girl_from_memory = sheerka.get_last_from_memory(context, "girl") + assert girl_from_memory.obj.get_value("my_attr") == "my value" + + def test_i_cannot_use_method_that_alter_the_global_state_within_question(self): + init = [ + "def concept foo as question(set_debug(True))", + ] + sheerka = self.init_scenario(init) + context = self.get_context(sheerka) + + res = sheerka.evaluate_user_input("question(set_debug(True))") + assert sheerka.has_error(context, res, __type="MethodAccessError") + + res = sheerka.evaluate_user_input("eval foo") + assert sheerka.has_error(context, res, __type="MethodAccessError") diff --git a/tests/non_reg/test_sheerka_non_reg_pipe_functions.py b/tests/non_reg/test_sheerka_non_reg_pipe_functions.py new file mode 100644 index 0000000..595723b --- /dev/null +++ b/tests/non_reg/test_sheerka_non_reg_pipe_functions.py @@ -0,0 +1,85 @@ +from core.builtin_concepts_ids import BuiltinConcepts +from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka + + +class TestSheerkaNonRegPipeFunctions(TestUsingMemoryBasedSheerka): + def test_i_can_filter_a_list_using_pipe(self): + init = [ + "def concept one as 1", + "def concept two as 2", + "def concept three as 3", + "add_to_memory('x', [one, two, three])" + ] + + sheerka = self.init_scenario(init) + res = sheerka.evaluate_user_input("x | where(body=2)") + + assert len(res) == 1 + assert res[0].status + assert isinstance(res[0].body, list) + assert len(res[0].body) == 1 + assert sheerka.isinstance(res[0].body[0], "two") + + res = sheerka.evaluate_user_input("x | where(__self=two)") + assert len(res) == 1 + assert res[0].status + assert isinstance(res[0].body, list) + assert len(res[0].body) == 1 + assert sheerka.isinstance(res[0].body[0], "two") + + def test_i_can_filter_using_sheerka_methods(self): + init = [ + "def concept one as 1", + "def concept number", + "set_isa(one, number)", + "add_to_memory('x', [one])" + ] + + sheerka = self.init_scenario(init) + res = sheerka.evaluate_user_input("x | where('isa(self, number)')") + + assert len(res) == 1 + assert res[0].status + assert isinstance(res[0].body, list) + assert len(res[0].body) == 1 + assert sheerka.isinstance(res[0].body[0], "one") + + def test_i_can_select_properties(self): + init = [ + "def concept one as 1", + "def concept two as 2", + "def concept three as 3", + "add_to_memory('x', [one, two, three])" + ] + + sheerka = self.init_scenario(init) + + res = sheerka.evaluate_user_input("x | select('id', 'name')") + assert len(res) == 1 + assert res[0].status + assert res[0].body == (("1001", "one"), ("1002", "two"), ("1003", "three")) + + res = sheerka.evaluate_user_input("x | select(p1='id', p2='name')") + assert len(res) == 1 + assert res[0].status + assert res[0].body == ({"p1": "1001", "p2": "one"}, + {"p1": "1002", "p2": "two"}, + {"p1": "1003", "p2": "three"}) + + def test_i_can_collect_properties(self): + init = [ + "def concept one as 1", + "def concept isa from x is a y def_var x def_var y", + "def concept plus from a plus b as a + b", + "add_to_memory('x', [one, isa, plus])" + ] + + sheerka = self.init_scenario(init) + res = sheerka.evaluate_user_input("x | props()") + + assert len(res) == 1 + assert res[0].status + assert sheerka.isinstance(res[0].body, BuiltinConcepts.TO_DICT) + assert res[0].body.body == {'isa': ['body', 'id', 'key', 'name', 'x', 'y'], + 'one': ['body', 'id', 'key', 'name'], + 'plus': ['a', 'b', 'body', 'id', 'key', 'name']} diff --git a/tests/non_reg/test_sheerka_non_reg_rules.py b/tests/non_reg/test_sheerka_non_reg_rules.py index 64f92b4..10deca1 100644 --- a/tests/non_reg/test_sheerka_non_reg_rules.py +++ b/tests/non_reg/test_sheerka_non_reg_rules.py @@ -5,7 +5,7 @@ from evaluators.PythonEvaluator import PythonEvalError from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka -class TestSheerkaNonRegDisplay(TestUsingMemoryBasedSheerka): +class TestSheerkaNonRegRules(TestUsingMemoryBasedSheerka): @pytest.mark.skip def test_i_can_apply_simple_rule(self): diff --git a/tests/parsers/parsers_utils.py b/tests/parsers/parsers_utils.py index b0ecad7..36d9330 100644 --- a/tests/parsers/parsers_utils.py +++ b/tests/parsers/parsers_utils.py @@ -16,6 +16,7 @@ from parsers.BaseNodeParser import UnrecognizedTokensNode, SourceCodeNode, RuleN from parsers.FunctionParser import FunctionNode from parsers.PythonParser import PythonNode from parsers.SyaNodeParser import SyaConceptParserHelper +from sheerkapython.python_wrapper import sheerka_globals from sheerkarete.common import V from sheerkarete.conditions import Condition, AndConditions, NegatedCondition, NegatedConjunctiveConditions @@ -934,11 +935,18 @@ class FN: @dataclass() class NEGCOND: + """ + Represents a NegatedCondition + """ condition: str @dataclass() class NCCOND: + """ + Represents a NegatedConjunctiveConditions + """ + conditions: List[str] @@ -1319,11 +1327,8 @@ def get_rete_conditions(*conditions): def get_value(obj): if obj.startswith("#"): return V(obj[1:]) - if obj.startswith("'"): - return obj[1:-1] - if obj in ("True", "False"): - return obj == "True" - return int(obj) + + return eval(obj, sheerka_globals) res = [] for cond in conditions: diff --git a/tests/parsers/test_DefConceptParser.py b/tests/parsers/test_DefConceptParser.py index b1990ab..204ac1d 100644 --- a/tests/parsers/test_DefConceptParser.py +++ b/tests/parsers/test_DefConceptParser.py @@ -671,7 +671,7 @@ from give me the date ! assert not res.status assert sheerka.isinstance(res.value, BuiltinConcepts.ERROR) - assert sheerka.has_error(res, __type="SyntaxErrorNode", message="Empty 'auto_eval' declaration.") + assert sheerka.has_error(context, res, __type="SyntaxErrorNode", message="Empty 'auto_eval' declaration.") def test_i_cannot_parse_when_wrong_auto_eval_value(self): sheerka, context, parser, *concepts = self.init_parser() @@ -709,4 +709,4 @@ from give me the date ! assert not res.status assert sheerka.isinstance(res.value, BuiltinConcepts.ERROR) - assert sheerka.has_error(res, __type="SyntaxErrorNode", message="Empty 'def_var' declaration.") + assert sheerka.has_error(context, res, __type="SyntaxErrorNode", message="Empty 'def_var' declaration.") diff --git a/tests/sheerkapython/__init__.py b/tests/sheerkapython/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/sheerkapython/test_python_wrapper.py b/tests/sheerkapython/test_python_wrapper.py new file mode 100644 index 0000000..04e8f5a --- /dev/null +++ b/tests/sheerkapython/test_python_wrapper.py @@ -0,0 +1,169 @@ +import pytest + +from core.builtin_concepts_ids import BuiltinConcepts +from core.concept import Concept +from core.global_symbols import SyaAssociativity +from core.sheerka.ExecutionContext import ExecutionContext +from core.sheerka.services.SheerkaAdmin import SheerkaAdmin +from sheerkapython.python_wrapper import Expando, create_namespace, inject_context, get_sheerka_method, Pipe, \ + MethodAccessError +from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka + + +class TestPythonWrapper(TestUsingMemoryBasedSheerka): + + @pytest.mark.parametrize("name, expected", [ + ("Concept", Concept), + ("BuiltinConcepts", BuiltinConcepts), + ("Expando", Expando), + ("ExecutionContext", ExecutionContext), + ("SyaAssociativity", SyaAssociativity), + ]) + def test_i_can_create_namespace_from_internal_references(self, name, expected): + context = self.get_context() + + assert create_namespace(context, "TestPythonWrapper", [name], None, {}, False) == {name: expected} + + def test_i_can_create_namespace_with_context_method(self): + context = self.get_context() + + res = create_namespace(context, "TestPythonWrapper", ["in_context", "isinstance"], None, {}, False) + assert res["in_context"] == context.in_context + assert res["isinstance"] == context.sheerka.services[SheerkaAdmin.NAME].extended_isinstance + + def test_i_can_create_namespace_when_sheerka_expando_object(self): + sheerka, context = self.init_test().unpack() + + res = create_namespace(context, "TestPythonWrapper", ["sheerka"], set(), {}, False) + assert res["sheerka"] == Expando("sheerka", {}) + + res = create_namespace(context, "TestPythonWrapper", ["sheerka"], {"test", "set_debug"}, {}, False) + assert isinstance(res["sheerka"], Expando) + assert res["sheerka"].get_name() == "sheerka" + assert set(vars(res["sheerka"])) == {"_Expando__name", "test", "set_debug"} + + def test_i_can_create_namespace_when_short_term_memory(self): + context = self.get_context() + context.add_to_short_term_memory("my_key", "my_value") + + assert create_namespace(context, "TestPythonWrapper", ["my_key"], None, {}, False) == {"my_key": "my_value"} + + def test_i_can_create_namespace_when_value_from_memory(self): + sheerka, context = self.init_test().unpack() + + sheerka.add_to_memory(context, "my_key", "my_value") + assert create_namespace(context, "TestPythonWrapper", ["my_key"], None, {}, False) == {"my_key": "my_value"} + + def test_i_can_create_namespace_when_sheerka_methods(self): + sheerka, context = self.init_test().unpack() + res = create_namespace(context, "TestPythonWrapper", ["test", "set_debug"], None, {}, False) + + assert res["test"] == sheerka.test + assert isinstance(res["set_debug"], type(inject_context)) + assert res["set_debug"].__name__ == "set_debug" + + def test_i_can_create_namespace_when_value_from_context_obj(self): + sheerka, context = self.init_test().unpack() + obj = Concept("foo").def_var("a", "value1").auto_init() + context.obj = obj + + res = create_namespace(context, "TestPythonWrapper", ["self", "a"], None, {}, False) + assert res == {"self": obj, "a": "value1"} + + def test_i_can_create_namespace_when_value_from_local_objects(self): + sheerka, context = self.init_test().unpack() + obj = Concept("foo") + objects = {"self": obj, "a": Concept("bar")} + + res = create_namespace(context, "TestPythonWrapper", ["self", "a"], None, objects, False) + assert res == {"self": obj, "a": objects["a"]} + + def test_i_can_create_namespace_when_name_refers_to_a_concept(self): + sheerka, context, foo = self.init_concepts("foo") + + assert create_namespace(context, "TestPythonWrapper", ["foo"], None, {}, False) == {"foo": foo} + + def test_internal_references_and_context_method_take_over_short_term_memory(self): + context = self.get_context() + context.add_to_short_term_memory("Concept", "short_term_value") + context.add_to_short_term_memory("isinstance", "short_term_value") + context.add_to_short_term_memory("in_context", "short_term_value") + + res = create_namespace(context, "TestPythonWrapper", ["Concept", "isinstance", "in_context"], None, {}, False) + + assert res == { + "Concept": Concept, + "isinstance": context.sheerka.services[SheerkaAdmin.NAME].extended_isinstance, + "in_context": context.in_context, + } + + def test_short_term_memory_takes_precedence_over_long_term_memory(self): + sheerka, context = self.init_test().unpack() + + context.add_to_short_term_memory("my_key", "short_term") + sheerka.add_to_memory(context, "my_key", "long_term") + + assert create_namespace(context, "TestPythonWrapper", ["my_key"], None, {}, False) == {"my_key": "short_term"} + + def test_long_term_memory_takes_precedence_over_sheerka_methods(self): + # I am not really sure why + sheerka, context = self.init_test().unpack() + + sheerka.add_to_memory(context, "test", "from memory") + assert create_namespace(context, "TestPythonWrapper", ["test"], None, {}, False) == {"test": "from memory"} + + def test_sheerka_method_takes_precedence_over_context_obj(self): + sheerka, context = self.init_test().unpack() + obj = Concept("foo").def_var("test", "value1").auto_init() + context.obj = obj + + assert create_namespace(context, "TestPythonWrapper", ["test", ], None, {}, False) == {"test": sheerka.test} + + def test_context_obj_takes_precedence_over_local_objects(self): + sheerka, context = self.init_test().unpack() + obj = Concept("foo").def_var("a", "value1").auto_init() + context.obj = obj + objects = {"self": Concept("bar"), "a": Concept("bar")} + + res = create_namespace(context, "TestPythonWrapper", ["self", "a"], None, objects, False) + assert res == {"self": obj, "a": "value1"} + + def test_local_objects_take_precedence_over_object_instantiation(self): + sheerka, context, foo = self.init_concepts("from instantiation") + objects = {"foo": Concept("from local")} + + res = create_namespace(context, "TestPythonWrapper", ["foo"], None, objects, False) + assert res == {"foo": objects["foo"]} + + def test_i_can_get_sheerka_method(self): + context = self.get_context() + + # sheerka direct method + assert get_sheerka_method(context, "TestPythonWrapper", "test", True) == context.sheerka.test + + # sheerka indirect method + assert get_sheerka_method(context, "TestPythonWrapper", "get_value", True) == context.sheerka.get_value + + # method that need context are wrapped + res = get_sheerka_method(context, "TestPythonWrapper", "test_using_context", True) + assert res != context.sheerka.test_using_context + assert type(res) == type(inject_context) + assert res.__name__ == "test_using_context" + + # return None when the method is not found + assert get_sheerka_method(context, "TestPythonWrapper", "xxx", True) is None + + def test_i_cannot_get_method_that_modifies_the_state_when_expression_only(self): + sheerka, context = self.init_test().unpack() + + assert get_sheerka_method(context, "TestPythonWrapper", "set_debug", expression_only=False) is not None + + with pytest.raises(MethodAccessError) as ex: + get_sheerka_method(context, "TestPythonWrapper", "set_debug", expression_only=True) + assert ex.value.method_name == "set_debug" + + def test_i_can_get_method_when_pipe_function(self): + context = self.get_context() + + res = get_sheerka_method(context, "TestPythonWrapper", "where", True) + assert isinstance(res, Pipe) diff --git a/tests/sheerkaql/__init__.py b/tests/sheerkaql/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/sheerkaql/test_SheerkaQueryLanguage.py b/tests/sheerkaql/test_SheerkaQueryLanguage.py new file mode 100644 index 0000000..b25e8ba --- /dev/null +++ b/tests/sheerkaql/test_SheerkaQueryLanguage.py @@ -0,0 +1,570 @@ +from dataclasses import dataclass + +import pytest + +from sheerkaql.SheerkaQueryLangage import SheerkaQueryLanguage +from sheerkaql.symbols import flwr_sequence, attribute_value + + +def oset(x): + return x + + +class A(object): + def __init__(self, q): + self.q = q + + def __repr__(self): + return f"A({vars(self)})" + + def __eq__(self, other): + if not isinstance(other, A): + return False + + return self.q == other.q + + def __hash__(self): + return hash(str(self.q)) + + +@dataclass +class BagClass: + property1: object + property2: object + + def as_bag(self): + return { + "prop1": self.property1, + "prop2": self.property2, + } + + +execute = SheerkaQueryLanguage().execute + + +class TestSheerkaQueryLanguage: + + def test_i_can_get_the_root_of_a_query(self): + hello = 'hello world!' + q = SheerkaQueryLanguage().compile('hello') + + assert q(locals()) == oset([hello]) + + def test_i_can_traverse_object(self): + a = A(A(A("hello world!"))) + q = SheerkaQueryLanguage().compile("a.q.q.q") + + assert q(locals()) == oset(["hello world!"]) + + def test_i_can_traverse_list(self): + lst = [A("one"), A("two"), A("three")] + a = A(lst) + + assert execute("a.q.q", {"a": a}) == oset(["one", "two", "three"]) + + def test_i_can_traverse_list_of_list(self): + sub_lst_number = [A("1"), A("2"), A("2")] + sub_lst_letter = [A("a"), A("b"), A("c")] + lst = [A("one"), A(sub_lst_number), A(sub_lst_letter)] + a = A(lst) + + res = execute("a.q.q", {"a": a}) + assert res == oset(["one", *sub_lst_number, *sub_lst_letter]) + + def test_i_can_traverse_object_when_where_condition_is_a_boolean(self): + a = A(A(A("hello world!"))) + b_true = A(A(True)) + b_false = A(A(False)) + namespace = {"a": a, "hasattr": hasattr, "func": lambda x: x, "b_true": b_true, "b_false": b_false} + + assert execute("a.q[1 == 1].q.q", namespace) == oset(["hello world!"]) + assert execute("a.q[hasattr(self, 'q')].q.q", namespace) == oset(["hello world!"]) + + assert execute("a.q[1 == 2].q.q", namespace) == oset([]) + assert execute("a.q[hasattr(self, 'x')].q.q", namespace) == oset([]) + + assert execute("a.q[True].q.q", namespace, allow_builtins=True) == oset(["hello world!"]) + assert execute("a.q[False].q.q", namespace, allow_builtins=True) == oset([]) + + assert execute("a.q[func(True)].q.q", namespace) == oset(["hello world!"]) + assert execute("a.q[func(False)].q.q", namespace) == oset([]) + + assert execute("a.q[b_true.q.q].q.q", namespace) == oset(["hello world!"]) + assert execute("a.q[b_false.q.q].q.q", namespace) == oset([]) + + def test_i_can_request_by_list_index(self): + lst = [A("one"), A("two"), A("three")] + a = A(lst) + + assert execute("a.q[1]", {"a": a}) == oset([lst[1]]) + with pytest.raises(IndexError): + execute("a.q[99]", {"a": a}) + with pytest.raises(TypeError): + execute("a.q['key']", {"a": a}) + + def test_i_can_request_by_dictionary_key(self): + lst = {"key1": "value1", "key2": "value2"} + a = A(lst) + + assert execute("a.q['key1']", {"a": a}) == oset([lst["key1"]]) + with pytest.raises(KeyError): + execute("a.q['key3']", {"a": a}) + + def test_i_can_filter(self): + l = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + + assert execute('l[self < 5]', locals()) == [0, 1, 2, 3, 4] + + def test_i_cannot_traverse_object_when_where_condition_is_not_a_boolean(self): + a = A("hello world!") + namespace = {"a": a, "func": lambda x: x, "dictionary": {"key": "value"}, "list": ["value"]} + + with pytest.raises(TypeError): + execute("a.q[1].q.q", namespace) + + with pytest.raises(TypeError): + execute("a.q[func(3)].q.q", namespace) + + with pytest.raises(TypeError): + execute("a.q[dictionary['key']].q.q", namespace) + + with pytest.raises(TypeError): + execute("a.q[list[0]].q.q", namespace) + + def test_i_can_traverse_object_when_as_bag_is_defined(self): + a = BagClass(BagClass("sub_value1", BagClass("sub_sub_value1", "sub_sub_value1")), "value2") + + assert execute("a.prop1.prop2.prop1", {"a": a}) == oset(["sub_sub_value1"]) + + def test_i_can_traverse_objects_when_where_condition_uses_as_bag(self): + hash_map = {"sub_sub_value1": "hello world!"} + b = BagClass(BagClass("sub_value1", BagClass("sub_sub_value1", "sub_sub_value1")), "value2") + a = A(hash_map) + + assert execute("a.q[b.prop1.prop2.prop1]", {"a": a, "b": b}) == oset(["hello world!"]) + + def test_i_can_compute_set_operations(self): + a = [1, 2, 3, 3] + b = [2, 4, 3, 4] + + assert execute("a | b", locals()) == [1, 2, 3, 4] + assert execute("a - b", locals()) == [1] + assert execute("b - a", locals()) == [4] + assert execute("a & b", locals()) == [2, 3] + + def test_can_execute_set_expression(self): + a = [1, 2, 3, 3] + b = [2, 4, 3, 4] + c = [1, 2] + + assert execute("return 1 if 1 in a else 0", locals()) == (1,) + assert execute("return 1 if 1 not in a else 0", locals()) == (0,) + assert execute("return 1 if 5 in a else 0", locals()) == (0,) + assert execute("return 1 if 5 not in a else 0", locals()) == (1,) + + assert execute("return 1 if subset else 0", locals()) == (1,) + assert execute("return 1 if superset else 0", locals()) == (0,) + + assert execute("return 1 if subset else 0", locals()) == (0,) + assert execute("return 1 if superset else 0", locals()) == (1,) + + l = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + + assert execute('l - l[self < 5]', locals()) == (5, 6, 7, 8, 9) + + def test_i_can_traverse_a_complex_path(self): + answer = 'o.x.y' + + o = A('top') + o.x = [A('asdf'), A('123')] + o.x[0].y = A(answer) + d = {'hasattr': hasattr, 'o': o} + + assert execute('o/x[hasattr(self,"y")]/y/q', d) == oset([answer]) + assert execute('o/x', d) == oset(o.x) + + def test_i_can_execute_comparison_in_where_clauses(self): + a = A(5) + assert execute('a[self.q == 5]', locals()) == oset([a]) + assert execute('a[self.q != 5]', locals()) == oset([]) + assert execute('a[self.q >= 5]', locals()) == oset([a]) + assert execute('a[self.q <= 5]', locals()) == oset([a]) + assert execute('a[self.q > 5]', locals()) == oset([]) + assert execute('a[self.q < 5]', locals()) == oset([]) + assert execute('a[self.q == 7]', locals()) == oset([]) + assert execute('a[self.q != 7]', locals()) == oset([a]) + assert execute('a[self.q >= 7]', locals()) == oset([]) + assert execute('a[self.q <= 7]', locals()) == oset([a]) + assert execute('a[self.q > 7]', locals()) == oset([]) + assert execute('a[self.q < 7]', locals()) == oset([a]) + assert execute('a[self.q == 3]', locals()) == oset([]) + assert execute('a[self.q != 3]', locals()) == oset([a]) + assert execute('a[self.q >= 3]', locals()) == oset([a]) + assert execute('a[self.q <= 3]', locals()) == oset([]) + assert execute('a[self.q > 3]', locals()) == oset([a]) + assert execute('a[self.q < 3]', locals()) == oset([]) + + def test_i_can_execute_boolean_expression_in_where_clauses(self): + a = 'hello' + true = True + false = False + assert execute('a[true]', locals()) == oset([a]) + assert execute('a[false]', locals()) == oset([]) + assert execute('a[not true]', locals()) == oset([]) + assert execute('a[not false]', locals()) == oset([a]) + assert execute('a[true and true]', locals()) == oset([a]) + assert execute('a[false and true]', locals()) == oset([]) + assert execute('a[not true and true]', locals()) == oset([]) + assert execute('a[not false and true]', locals()) == oset([a]) + assert execute('a[true or false]', locals()) == oset([a]) + assert execute('a[true or true]', locals()) == oset([a]) + assert execute('a[false or true]', locals()) == oset([a]) + assert execute('a[false or false]', locals()) == oset([]) + assert execute('a[not true or true]', locals()) == oset([a]) + assert execute('a[not false or false]', locals()) == oset([a]) + assert execute('a[true and true and true and true]', locals()) == oset([a]) + assert execute('a[true and true and true and false]', locals()) == oset([]) + assert execute('a[true and (false or true)]', locals()) == oset([a]) + assert execute('a[true and (false and true)]', locals()) == oset([]) + assert execute('a[true and (true and true)]', locals()) == oset([a]) + assert execute('a[true and (true and (not true or false))]', locals()) == oset([]) + + def test_i_can_execute_comparison_in_where_clauses_using_builtin_functions(self): + a = 'hello' + true = True + false = False + d = locals() + assert execute('a[1 == 1]', d) == oset([a]) + assert execute('a[-1 == -1]', d) == oset([a]) + assert execute('a[2.2 == 2.2]', d) == oset([a]) + assert execute('a[2.2 == float("2.2")]', d, True) == oset([a]) + assert execute('a[2 == int(2.2)]', d, True) == oset([a]) + assert execute('a["hello" == a]', d, True) == oset([a]) + assert execute('a["HELLO" == a.upper()]', d, True) == oset([a]) + + def test_i_can_use_function_in_where_clauses(self): + a = 'hello' + + def f(): + return 'hello' + + def g(x, y, z): + return x + y + z + + def h(f, x): + return f(x) + + def i(x): + return x ** 2 + + def j(f): + return f + + true = True + false = False + d = locals() + + assert execute('a[f() == "hello"]', d) == oset([a]) + assert execute('a[g(1,2,3) == 6]', d) == oset([a]) + assert execute('a[h(i,3) == 9]', d) == oset([a]) + assert execute('a[i(j(j)(j)(j)(h)(i,3)) == 81]', d) == oset([a]) + with pytest.raises(TypeError): + execute('a[f()]', d) + + def test_i_can_use_lists_in_where_clauses(self): + a = 'hello' + l = [1, 2, 3, 4, 5, 6, 7, [1, 2, 3, 4, 5, 6, 7, [1, 2, 3, 4, 5, 6, 7, 8]]] + d = locals() + + assert execute('a[l[0] == 1]', d) == oset([a]) + assert execute('a[l[1] == 2]', d) == oset([a]) + assert execute('a[l[7][0] == 1]', d) == oset([a]) + assert execute('a[l[7][1] == 2]', d) == oset([a]) + assert execute('a[l[7][7][0] == 1]', d) == oset([a]) + assert execute('a[l[7][7][1] == 2]', d) == oset([a]) + assert execute('a[l[7][7][7] == 8]', d) == oset([a]) + + def test_i_can_use_dicts_in_where_clauses(self): + a = 'hello' + l = {"one": 1, "two": 2, "next": {"one": 1, "two": 2, "next": {"one": 1, "two": 2}}} + d = locals() + + assert execute('a[l["one"] == 1]', d) == oset([a]) + assert execute('a[l["two"] == 2]', d) == oset([a]) + assert execute('a[l["next"]["one"] == 1]', d) == oset([a]) + assert execute('a[l["next"]["two"] == 2]', d) == oset([a]) + assert execute('a[l["next"]["next"]["one"] == 1]', d) == oset([a]) + assert execute('a[l["next"]["next"]["two"] == 2]', d) == oset([a]) + + def test_i_can_use_callable_in_where_clauses(self): + a = 'hello' + + def f(): + return 'hello' + + def g(x, y, z): + return x + y + z + + def h(f, x): + return f(x) + + def i(x): + return x ** 2 + + def j(f): + return f + + m = {"one": 1, "two": 2, "next": [1, 2, 3, 4, 5, 6, 7, j]} + d = locals() + + assert execute('a[m["next"][7](j)(m["next"][7])(m["next"])[7](i)(m["two"]) == 4]', d) == oset([a]) + + def test_i_can_execute_flwr_expression(self): + def f(): + return 1, 2, 3 + + d = locals() + assert execute('for x in f() return x', d) == (1, 2, 3) + assert execute('for x in f() let y = f() return x, y', d) == ((1, (1, 2, 3)), (2, (1, 2, 3)), (3, (1, 2, 3))) + + def test_i_can_execute_flwr_with_order_by(self): + def f(): + return [1, 3, 2] + + d = locals() + with pytest.raises(SyntaxError): + execute('for x in f() order by "asdf" asc return x', d) + with pytest.raises(SyntaxError): + execute('for x in f() order by 0 asc return "asdf":x', d) + assert execute('for x in f() order by 0 asc return x', d) == (1, 2, 3) + assert execute('for x in f() order by 0 desc return x', d) == (3, 2, 1) + + def test_i_can_execute_flwr_with_user_defined_functions(self): + a = 'hello' + l = [1, 2, 3, 4, 5, 6, 7, [1, 2, 3, 4, 5, 6, 7, [1, 2, 3, 4, 5, 6, 7, 8]]] + d = locals() + + assert execute(''' + for i in l + let f = function() { 125 } + return f() + ''', d) == (125, 125, 125, 125, 125, 125, 125, 125) + + assert execute(''' + for i in l + let f = function(q) { + for _ in + where isinstance(q, list) + return { + for j in q + return f(j) + } + } + return f(i) + ''', d, True) == ((), (), (), (), (), (), (), + (((), (), (), (), (), (), (), + (((), (), (), (), (), (), (), ()),)),)) + + def test_i_can_execute_flwr_with_if_expression(self): + a = 'hello' + l = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] + q = True + d = locals() + + assert execute(''' + for x in return if (q) then 1 else 0 + ''', d) == (1,) + + assert execute(''' + for x in return if (x % 2 == 0) then 1 else 0 + ''', d) == (0, 1, 0, 1, 0, 1, 0, 1, 0, 1) + + assert execute(''' + for x in return if x % 2 == 0 then 1 else 0 + ''', d) == (0, 1, 0, 1, 0, 1, 0, 1, 0, 1) + + assert execute(''' + for x in return 1 if x % 2 == 0 else 0 + ''', d) == (0, 1, 0, 1, 0, 1, 0, 1, 0, 1) + + assert execute(''' + for x in return 1 if (x % 2 == 0) else 0 + ''', d) == (0, 1, 0, 1, 0, 1, 0, 1, 0, 1) + + assert execute(''' + for x in return if (True or X) then 1 else 0 + ''', d, True) == (1,) + + def test_if_short_circuit(self): + a = 'hello' + l = [1, 2, 3, 4, 5, 6, 7, [1, 2, 3, 4, 5, 6, 7, [1, 2, 3, 4, 5, 6, 7, 8]]] + d = locals() + + assert execute(''' + for x in return if (True or X) then 1 else 0 + ''', d, True) == (1,) + + assert execute(''' + for x in return if (False and false.x) then 1 else 0 + ''', d, True) == (0,) + + def test_i_can_flatten_result(self): + a = 'hello' + l = [1, 2, 3, 4, 5, 6, 7, [1, 2, 3, 4, 5, 6, 7, [1, 2, 3, 4, 5, 6, 7, 8]]] + d = locals() + + assert execute(''' + for x in l return flatten x + ''', d, True) == (1, 2, 3, 4, 5, 6, 7, 1, 2, 3, 4, 5, 6, 7, 1, 2, 3, 4, 5, 6, 7, 8) + + def test_flattened_return(self): + a = 'hello' + l = [1, 2, 3, 4, 5, 6, 7, [1, 2, 3, 4, 5, 6, 7, [1, 2, 3, 4, 5, 6, 7, 8]]] + d = locals() + + assert execute(''' + for i in l + let f = function(l) { + if (isinstance(l, list)) + then {for j in l return f(j)} + else l + } + return flatten f(i)''', d, True) == (1, 2, 3, 4, 5, 6, 7, 1, 2, 3, 4, 5, 6, 7, 1, 2, 3, 4, 5, 6, 7, 8) + + assert execute(''' + for i in l + let f = function(l) { + if (isinstance(l, list)) + then {for j in l return f(j)} + else {a:l} + } + return flatten f(i) + ''', d, True) == tuple({a: i} for i in (1, 2, 3, 4, 5, 6, 7, 1, 2, 3, 4, 5, 6, 7, 1, 2, 3, 4, 5, 6, 7, 8)) + + def test_i_can_return_none(self): + a = 'hello' + l = [1, 2, 3, 4, 5, 6, 7, [1, 2, 3, 4, 5, 6, 7, [1, 2, 3, 4, 5, 6, 7, 8]]] + d = locals() + + assert execute('for x in l return None', d, True) == (None, None, None, None, None, None, None, None) + + def test_i_can_return_a_class(self): + l = [1, 2, 3] + d = {"A": A, "l": l} + + assert execute('for x in l return A(x)', d, True) == (A(1), A(2), A(3)) + + def test_i_can_execute_flwr_when_there_is_no_for_statement(self): + a = 'hello' + l = [1, 2, 3, 4, 5, 6, 7, [1, 2, 3, 4, 5, 6, 7, [1, 2, 3, 4, 5, 6, 7, 8]]] + d = locals() + + flwr = flwr_sequence([attribute_value('hello', scalar=True)]) + assert flwr(d) == ('hello',) + assert execute("return l", d) == (l,) + assert execute(''' + let f = function(l) { + if (isinstance(l, list)) + then {for j in l return f(j)} + else l + } + return flatten f(l) + ''', d, True) == (1, 2, 3, 4, 5, 6, 7, 1, 2, 3, 4, 5, 6, 7, 1, 2, 3, 4, 5, 6, 7, 8) + + def test_can_collect(self): + l = [1, 2, 3, 4, 5, 6, 7, 3, 4, 5, 6, 7, 3, 4] + d = locals() + + assert execute(''' + for n in l + collect n as n with function(prev, next) { + if prev == None then 1 else prev + 1 + } + ''', d, True) == {1: 1, 2: 1, 3: 3, 4: 3, 5: 2, 6: 2, 7: 2} + + assert execute(''' + for n in [1,2,3,4,5,6,7,3,4,5,6,7,3,4] + collect n as n with function(prev, next) { + if prev == None then 1 else prev + 1 + } + ''', d, True) == {1: 1, 2: 1, 3: 3, 4: 3, 5: 2, 6: 2, 7: 2} + + def test_i_can_perform_calculations(self): + d = locals() + + assert execute(''' + for n in [ + 4.0*3.0/2.0, + 4.0/3.0*2.0, + (3.0+9.0)*4.0/8.0, + ((9.0-3.0)+(5.0-3.0))/2.0 + 2.0, + 5.0 * 4.0 / 2.0 - 10.0 + 5.0 - 2.0 + 3.0, + 5.0 / 4.0 * 2.0 + 10.0 - 5.0 * 2.0 / 3.0 + ] + return n + ''', d, True) == ( + 4.0 * 3.0 / 2.0, + 4.0 / 3.0 * 2.0, + (3.0 + 9.0) * 4.0 / 8.0, + ((9.0 - 3.0) + (5.0 - 3.0)) / 2.0 + 2.0, + 5.0 * 4.0 / 2.0 - 10.0 + 5.0 - 2.0 + 3.0, + 5.0 / 4.0 * 2.0 + 10.0 - 5.0 * 2.0 / 3.0 + ) + + def test_i_can_execute_flwr_with_in_and_not_in(self): + l = [[1, 2, 3], [4, 5, 6], [7, 4, 3], [5, 6, 7], [3, 4]] + d = locals() + + assert execute(''' + for n in l + where 4 in n + return n + ''', d, True) == ([4, 5, 6], [7, 4, 3], [3, 4]) + + assert execute(''' + for n in l + where 4 not in n + return n + ''', d, True) == ([1, 2, 3], [5, 6, 7]) + + def test_multi_collect(self): + l = [1, 2, 3, 4, 5, 6, 7, 3, 4, 5, 6, 7, 3, 4] + d = locals() + + assert execute(''' + for n in l + let counter = function(prev, next) { + if prev == None then 1 else prev + 1 + } + where 1 in l and 12 not in l + collect n as n with counter + collect n as (int(n)/int(2)) with counter + ''', d, True), ( + {1: 1, 2: 1, 3: 3, 4: 3, 5: 2, 6: 2, 7: 2}, + {0: 1, 1: 4, 2: 5, 3: 4}) + + def test_i_can_execute_flwr_on_objects_attributes(self): + lst = [A(1), A(2), A(3)] + d = locals() + + assert execute('for x in return x', d) == (1, 2, 3) + assert execute('for x in lst return x.q', d) == (1, 2, 3) # another way + + def test_exception_during_return_are_caught(self): + lst = [A(1), A([2])] + d = locals() + + res = execute('for x in lst return x.q + [2]', d) + assert len(res) == 2 + assert isinstance(res[0], TypeError) + assert res[1] == [2,2] + + def test_i_cannot_execute_flwr_on_attributes_that_does_not_exist(self): + lst = [A(1), A(2), A(3)] + d = locals() + + with pytest.raises(AttributeError): + execute('for x in return x', d) + + with pytest.raises(AttributeError): + execute('for x in lst.q return x', d) # AttributeError: 'list' object has no attribute 'q' diff --git a/tests/sheerkaql/test_lexer.py b/tests/sheerkaql/test_lexer.py new file mode 100644 index 0000000..9dcc711 --- /dev/null +++ b/tests/sheerkaql/test_lexer.py @@ -0,0 +1,122 @@ +from contextlib import contextmanager + +from ply import lex + +from sheerkaql import lexer + + +def compare(a, b): + return (a.type == b.type and a.value == b.value and + a.lexpos == b.lexpos and a.lineno == b.lineno) + + +def token(token_type, value, pos, line): + t = lex.LexToken() + t.type = token_type + t.value = value + t.lexpos = pos + t.lineno = line + return t + + +@contextmanager +def comparable_tokens(): + eq = lex.LexToken.__eq__ + ne = lex.LexToken.__ne__ + setattr(lex.LexToken, "__eq__", compare) + setattr(lex.LexToken, "__ne__", lambda a, b: not compare(a, b)) + yield + setattr(lex.LexToken, "__eq__", eq) + setattr(lex.LexToken, "__ne__", ne) + + +class TestSheerkaQueryLanguageLexer: + def test_context_manager(self): + t1 = token("NAME", 'a', 0, 1) + t2 = token("NAME", 'a', 0, 1) + assert t1 != t2 + with comparable_tokens(): + assert t1 == t2 + + def test_NAME(self): + clex = lexer.Lexer() + clex.input('a 9a') + tokens = [token("NAME", 'a', 0, 1), + token("NUMBER", 9, 2, 1), + token("NAME", 'a', 3, 1)] + with comparable_tokens(): + for t in tokens: + assert next(clex) == t + + def test_STRING(self): + clex = lexer.Lexer() + clex.input("'asdf' \"asdf\" '\n'") + tokens = [token("STRING", 'asdf', 0, 1), + token("STRING", 'asdf', 7, 1), + token("STRING", '\n', 14, 1), ] + with comparable_tokens(): + for t in tokens: + assert next(clex) == t + + def test_HEX(self): + clex = lexer.Lexer() + clex.input("0xab 0xab") + tokens = [token("NUMBER", 0xab, 0, 1), + token("NUMBER", 171, 5, 1)] + with comparable_tokens(): + for t in tokens: + assert next(clex) == t + + def test_FLOAT(self): + clex = lexer.Lexer() + clex.input("1.2 .2 2.3e4 .2 2.3e4") + tokens = [token("NUMBER", 1.2, 0, 1), token("NUMBER", .2, 4, 1), + token("NUMBER", 2.3e4, 7, 1), token("NUMBER", .2, 13, 1), + token("NUMBER", 2.3e4, 16, 1)] + with comparable_tokens(): + for t in tokens: + assert next(clex) == t + + def test_OCT(self): + clex = lexer.Lexer() + clex.input("073 073 073") + tokens = [token("NUMBER", 0o73, 0, 1), + token("NUMBER", 59, 4, 1), + token("NUMBER", 59, 8, 1)] + with comparable_tokens(): + for t in tokens: + assert next(clex) == t + + def test_INTEGER(self): + clex = lexer.Lexer() + clex.input("73 730 7") + tokens = [token("NUMBER", 73, 0, 1), + token("NUMBER", 730, 3, 1), + token("NUMBER", 7, 7, 1)] + with comparable_tokens(): + for t in tokens: + assert next(clex) == t + + def test_KEYWORDS(self): + for value, token_type in list(lexer.reserved.items()): + clex = lexer.Lexer() + clex.input(value) + tokens = [token(token_type, value, 0, 1)] + with comparable_tokens(): + for t in tokens: + assert next(clex) == t + + def test_chrs(self): + for token_type, value in [(attr[2:], getattr(lexer.Lexer, attr)) + for attr in dir(lexer.Lexer) + if attr[:2] == 't_' and + isinstance(getattr(lexer.Lexer, attr), str) and + attr[2:] != 'ignore']: + if value[0] == '\\': + value = value[1:] + clex = lexer.Lexer() + clex.input(value) + tokens = [token(token_type, value, 0, 1)] + with comparable_tokens(): + for t in tokens: + assert next(clex) == t diff --git a/tests/sheerkaql/test_parser.py b/tests/sheerkaql/test_parser.py new file mode 100644 index 0000000..cbbaae9 --- /dev/null +++ b/tests/sheerkaql/test_parser.py @@ -0,0 +1,297 @@ +import pytest + +from sheerkaql.SheerkaQueryLangage import SheerkaQueryLanguage +from sheerkaql.lexer import Lexer +from sheerkaql.parser import Parser + + +class TestSheerkaQueryLanguageParser: + def test_hello(self): + SheerkaQueryLanguage().compile('hello') + with pytest.raises(SyntaxError) as ex: + SheerkaQueryLanguage().compile('hello there') + + def test_i_can_parse_with_slash(self): + SheerkaQueryLanguage().compile('hello/part1') + SheerkaQueryLanguage().compile('hello/part1/part2/part3') + SheerkaQueryLanguage().compile('hello/part1 / part2 / part3') + with pytest.raises(SyntaxError): + SheerkaQueryLanguage().compile('hello/part1/part3 part4/part5') + + def test_i_can_parse_with_dot(self): + SheerkaQueryLanguage().compile('hello.part1') + SheerkaQueryLanguage().compile('hello.part1.part2.part3') + SheerkaQueryLanguage().compile('hello.part1 . part2 . part3') + with pytest.raises(SyntaxError): + SheerkaQueryLanguage().compile('hello.part1.part3 part4.part5') + + def test_i_can_parse_simple_where_conditions(self): + SheerkaQueryLanguage().compile('hello[wheRe]/hello[asdf]') + SheerkaQueryLanguage().compile('hello/hello[asdf]/asdf/wef') + SheerkaQueryLanguage().compile('hello/hello/wewe[asdf]/wef/waef/awef/weaf') + SheerkaQueryLanguage().compile('hello.hello[ asdf ] . wewe. wef[asdf] .waef .awef[asdf].weaf') + SheerkaQueryLanguage().compile('hello["foo"]') + SheerkaQueryLanguage().compile('hello[123]') + SheerkaQueryLanguage().compile('hello[123.234]') + with pytest.raises(SyntaxError): + SheerkaQueryLanguage().compile('hello/aef[asdf] hello[adsf]') + + def test_i_can_parse_where_conditions_with_comparisons(self): + SheerkaQueryLanguage().compile('hello[1 == 1]') + SheerkaQueryLanguage().compile('hello[1 != 1]') + SheerkaQueryLanguage().compile('hello[1 < 1]') + SheerkaQueryLanguage().compile('hello[1 <= 1]') + SheerkaQueryLanguage().compile('hello[1 > 1]') + SheerkaQueryLanguage().compile('hello[1 >= 1]') + + def test_i_can_parse_where_conditions_with_logical_operations(self): + SheerkaQueryLanguage().compile('hello[a and b]') + SheerkaQueryLanguage().compile('hello[a or b]') + SheerkaQueryLanguage().compile('hello[not a or b]') + SheerkaQueryLanguage().compile('hello[a or not b]') + SheerkaQueryLanguage().compile('hello[not a and b]') + SheerkaQueryLanguage().compile('hello[a and not b]') + SheerkaQueryLanguage().compile('hello[not a or not b]') + SheerkaQueryLanguage().compile('hello[not a and not b]') + + def test_i_can_parse_where_conditions_with_logical_operations_and_parenthesis(self): + SheerkaQueryLanguage().compile('hello[((a and b) and not (a or b) or not (a and b)) and not (not a or b)]') + + def test_i_can_parse_where_conditions_with_arithmetic_operations(self): + SheerkaQueryLanguage().compile('hello[a + b]') + SheerkaQueryLanguage().compile('hello[a - b]') + SheerkaQueryLanguage().compile('hello[a * b]') + SheerkaQueryLanguage().compile('hello[a / b]') + SheerkaQueryLanguage().compile('hello[(a + b) / (a - b)]') + SheerkaQueryLanguage().compile('hello[-a]') + SheerkaQueryLanguage().compile('hello[a + b * c / e]') + + def test_i_can_parse_nested_where_conditions(self): + SheerkaQueryLanguage().compile('hello[a]') + SheerkaQueryLanguage().compile("hello[a[0]['hello']]") + SheerkaQueryLanguage().compile("hello[a[0][\"hello\"]]") + + def test_i_can_parse_where_conditions_with_function_call(self): + SheerkaQueryLanguage().compile("hello[a[0]['hello']()]") + SheerkaQueryLanguage().compile("hello[a[0]['hello'](0)]") + SheerkaQueryLanguage().compile("hello[a[0]['hello']('asdf')]") + SheerkaQueryLanguage().compile("hello[a[0]['hello'](asdf)]") + SheerkaQueryLanguage().compile("hello[a[0]['hello'](0, 'asdf', asdf)()()(1,2)]") + SheerkaQueryLanguage().compile('hello[f(1)]') + SheerkaQueryLanguage().compile('hello[f(1,2,3)]') + SheerkaQueryLanguage().compile('hello[f(a,b,c)]') + SheerkaQueryLanguage().compile('hello[f(a(),b(),c())]') + SheerkaQueryLanguage().compile('hello[f(a[a],b[b],c[c])]') + + def test_i_can_parse_where_conditions_with_attributes_parsing(self): + SheerkaQueryLanguage().compile('hello[foo.bar.baz]') + SheerkaQueryLanguage().compile('hello[foo[12].bar().baz]') + + def test_i_can_parse_where_conditions_with_flwr(self): + SheerkaQueryLanguage().compile('hello[f(1,,{for x in return x})]') + + def test_i_can_parse_where_conditions_with_quantified_expressions(self): + SheerkaQueryLanguage().compile('hello[every x in satisfies (x)]') + with pytest.raises(SyntaxError): + SheerkaQueryLanguage().compile('hello[every x in statisfies (x)]') + with pytest.raises(SyntaxError): + SheerkaQueryLanguage().compile('hello[every x in statisfies x]') + SheerkaQueryLanguage().compile('hello[some x in satisfies (x)]') + SheerkaQueryLanguage().compile('hello[some x in satisfies (x)]') + SheerkaQueryLanguage().compile('hello[some x in {for x in return x} satisfies (x)]') + SheerkaQueryLanguage().compile('hello[some x in {for x in return x} satisfies (x == y)]') + SheerkaQueryLanguage().compile('hello[some x in {for x in return x} satisfies (x and not y(1,2))]') + + def test_i_can_parse_where_conditions_with_set_comparisons(self): + SheerkaQueryLanguage().compile('hello[a in ]') + SheerkaQueryLanguage().compile('hello[a not in ]') + SheerkaQueryLanguage().compile('hello[not a in ]') + SheerkaQueryLanguage().compile('hello[ subset ]') + SheerkaQueryLanguage().compile('hello[ superset ]') + SheerkaQueryLanguage().compile('hello[ proper subset ]') + SheerkaQueryLanguage().compile('hello[ proper superset ]') + SheerkaQueryLanguage().compile('hello[ is ]') + SheerkaQueryLanguage().compile('hello[ is not ]') + + def test_i_can_parse_operations_on_sets(self): + SheerkaQueryLanguage().compile('asdf - asdf') + SheerkaQueryLanguage().compile('asdf & asdf') + SheerkaQueryLanguage().compile('asdf | asdf') + SheerkaQueryLanguage().compile('(asdf | asdf) & asdf - (asdf & asdf) - (asdf & (afsd | asdf))') + SheerkaQueryLanguage().compile('asdf/asdf - asdf/asd[erw]') + + def test_i_can_parse_flwr_expression(self): + SheerkaQueryLanguage().compile('for x in y return x') + SheerkaQueryLanguage().compile('for x in return x') + SheerkaQueryLanguage().compile('for x in , y in return x') + SheerkaQueryLanguage().compile('for x in {for x in return x} return x') + SheerkaQueryLanguage().compile('for x in where x == y return x') + SheerkaQueryLanguage().compile('for x in let y = return x') + SheerkaQueryLanguage().compile('for x in let y = {for x in return x} return x') + SheerkaQueryLanguage().compile('for x in let y = , x = return x') + SheerkaQueryLanguage().compile('for x in let y = let x = return x') + SheerkaQueryLanguage().compile('for x in , z in let y = let x = return x') + SheerkaQueryLanguage().compile('''for x in , z in + let y = + let x = + where every x in satisfies (q) + return x''') + SheerkaQueryLanguage().compile('''for x in , z in + let y = + let x = + where every x in satisfies (q) + return x,y,z''') + SheerkaQueryLanguage().compile('''for x in , z in + let y = + let x = + where every x in satisfies (q) + return x,y.sdf.asd,z''') + SheerkaQueryLanguage().compile('''for x in , z in + let y = + let x = + where every x in satisfies (q) + return x,y.sdf.asd,z()()()[asdf][asfd](1,2,3)''') + SheerkaQueryLanguage().compile('''for x in , z in + let y = + let x = + where every x in satisfies (q) + return 'asdf':asdf''') + SheerkaQueryLanguage().compile('''for x in , z in + let y = + let x = + where every x in satisfies (q) + return 'asdf':asdf, "hello":"hello World!"''') + SheerkaQueryLanguage().compile('''for x in , z in + let y = + let x = + where every x in satisfies (q == z and ( is not )) + return 'asdf':asdf, "one":1, "2.0":2.0''') + SheerkaQueryLanguage().compile('''for x in , z in , y in + let y = + let x = + where every x in satisfies (q == z and ( is not )) + return 'asdf':asdf, "one":1, "2.0":2.0''') + SheerkaQueryLanguage().compile('''for x in , z in , y in + let y = , y1 = , y2 = , y3 = + let x = + where every x in satisfies (q == z and ( is not )) + return 'asdf':asdf, "one":1, "2.0":2.0''') + + def test_i_can_parse_flwr_with_attribute_value(self): + SheerkaQueryLanguage().compile('''for x in , z in , y in sdaf.asdf(asdf, asdf)[1] + let y = , y1 = , y2 = , y3 = + let x = + where every x in satisfies (q == z and ( is not )) + return 'asdf':asdf, "one":1, "2.0":2.0''') + SheerkaQueryLanguage().compile('''for x in , z in , y in sdaf.asdf(asdf, asdf)[1] + let y = , y1 = , y2 = , y3 = + let x = + let q = sadf.asdf().asfd[1](1,2,3) + where every x in satisfies (q == z and ( is not )) + return 'asdf':asdf, "one":1, "2.0":2.0''') + + def test_flwr_orderby(self): + SheerkaQueryLanguage().compile('for x in order by "adsf" desc return "adsf":x') + SheerkaQueryLanguage().compile('''for x in , z in , y in sdaf.asdf(asdf, asdf)[1] + let y = , y1 = , y2 = , y3 = + let x = + where every x in satisfies (q == z and ( is not )) + order by "asdf" desc + return 'asdf':asdf, "one":1, "2.0":2.0''') + SheerkaQueryLanguage().compile('for x in order by 0 asc return x') + SheerkaQueryLanguage().compile('''for x in , z in , y in sdaf.asdf(asdf, asdf)[1] + let y = , y1 = , y2 = , y3 = + let x = + where every x in satisfies (q == z and ( is not )) + order by 1 asc + return asdf, 1, 2.0''') + + def test_flwr_function_noargs(self): + SheerkaQueryLanguage().compile(''' + for x in + let f = function() { for y in return y } + return f + ''') + + def test_flwr_function_args(self): + SheerkaQueryLanguage().compile(''' + for x in + let f = function(q) { for y in q return y } + return f + ''') + + def test_if(self): + SheerkaQueryLanguage().compile(''' + for x in return if (0) then 1 else 0 + ''') + + def test_reduce(self): + SheerkaQueryLanguage().compile(''' + for x in + collect x.tree as x.attr with function(prev, next) { + if prev == None then next else prev.combine(next) + } + ''') + + def test_in_list1(self): + SheerkaQueryLanguage().compile("hello['foo' in ['foo','bar']]") + + def test_in_list2(self): + SheerkaQueryLanguage().compile("hello['baz' in ['foo','bar']]") + + def test_not_in_list1(self): + SheerkaQueryLanguage().compile("hello['foo' not in ['foo','bar']]") + + def test_not_in_list2(self): + SheerkaQueryLanguage().compile("hello['baz' not in ['foo','bar']]") + + def test_in_list3(self): + result = SheerkaQueryLanguage().execute("res[test_elt in ['foo','bar']]", + {'res': True, 'test_elt': 'foo'}) + assert bool(result) + + def test_in_list4(self): + result = SheerkaQueryLanguage().execute("res[test_elt in ['foo','bar']]", + {'res': True, 'test_elt': 'baz'}) + + assert not bool(result) + + def test_not_in_list4(self): + result = SheerkaQueryLanguage().execute("res[test_elt not in ['foo','bar']]", + {'res': True, 'test_elt': 'baz'}) + assert bool(result) + + def test_not_in_list5(self): + result = SheerkaQueryLanguage().execute("res[test_elt not in ['foo','bar']]", + {'res': True, 'test_elt': 'foo'}) + assert not bool(result) + + @pytest.mark.parametrize("text, expected", [ + ("hello", {"hello"}), + ("hello.foo.bar", {"hello"}), + ("hello/foo/bar", {"hello"}), + ("hello[foo.bar.baz]", {"hello", "foo"}), + ("hello[foo()]", {"hello", "foo"}), + ("hello[foo(bar)]", {"hello", "foo", "bar"}), + ("hello[foo(1, bar.baz)]", {"hello", "foo", "bar"}), + ("hello[foo[bar]]", {"hello", "foo", "bar"}), + ("hello[foo > bar]", {"hello", "foo", "bar"}), + ("hello[foo + bar]", {"hello", "foo", "bar"}), + ("hello[[a,b,c]]", {"hello", "a", "b", "c"}), + ("hello[{a:b}]", {"hello", "a", "b"}), + ]) + def test_i_can_get_names(self, text, expected): + parser = Parser() + parser.parse(bytes(text, 'utf-8').decode('unicode_escape'), lexer=Lexer()) + assert parser.names == expected + + @pytest.mark.parametrize("text", [ + "sheerka.method", + "sheerka/method", + "hello[sheerka.method]", + "hello[sheerka.method.xx]", + ]) + def test_i_can_get_sheerka_methods(self, text): + parser = Parser() + parser.parse(bytes(text, 'utf-8').decode('unicode_escape'), lexer=Lexer()) + assert parser.sheerka_names == {"method"}