From 821dbed189380bece886fd8a0ca30a65a85b848b Mon Sep 17 00:00:00 2001 From: Kodjo Sossouvi Date: Fri, 15 Jan 2021 07:11:04 +0100 Subject: [PATCH] Fixed #3: Added sheerka.resolve_rule() Fixed #5: Refactored SheerkaComparisonManager Fixed #6: Sya parser no longer works after restart --- _concepts_admin.txt | 11 + docs/source/README.txt | 3 +- docs/source/blog/blog.rst | 8 - docs/source/conf.py | 4 +- docs/source/index.rst | 7 + docs/source/tech/debugger.rst | 19 + src/core/builtin_concepts_ids.py | 13 +- src/core/builtin_helpers.py | 24 +- src/core/rule.py | 4 +- src/core/sheerka/Sheerka.py | 17 - src/core/sheerka/SheerkaOntologyManager.py | 84 ++-- src/core/sheerka/services/SheerkaAdmin.py | 15 +- .../services/SheerkaComparisonManager.py | 472 +++++++++++------- .../sheerka/services/SheerkaConceptManager.py | 248 ++++++++- .../services/SheerkaConceptsAlgebra.py | 4 +- src/core/sheerka/services/SheerkaMemory.py | 27 +- .../sheerka/services/SheerkaResultManager.py | 35 +- .../sheerka/services/SheerkaRuleManager.py | 54 +- src/core/sheerka/services/sheerka_service.py | 6 + src/core/utils.py | 10 +- src/evaluators/PythonEvaluator.py | 5 +- src/evaluators/RuleEvaluator.py | 5 +- src/parsers/BaseNodeParser.py | 262 +--------- src/parsers/BnfNodeParser.py | 4 +- src/parsers/RuleParser.py | 4 +- src/parsers/SequenceNodeParser.py | 2 +- src/parsers/SyaNodeParser.py | 7 +- tests/cache/test_ListCache.py | 9 +- tests/core/test_SheerkaComparisonManager.py | 280 +++++++++-- tests/core/test_SheerkaConceptManager.py | 294 ++++++++++- tests/core/test_SheerkaDebugManager.py | 1 + tests/core/test_SheerkaMemory.py | 13 +- tests/core/test_SheerkaRuleManager.py | 39 +- tests/core/test_sheerka.py | 8 +- tests/core/test_sheerkaResultManager.py | 76 ++- tests/core/test_sheerka_ontology.py | 199 ++++---- tests/evaluators/test_PythonEvaluator.py | 22 +- tests/evaluators/test_RuleEvaluator.py | 8 +- tests/non_reg/test_sheerka_display.py | 17 + tests/non_reg/test_sheerka_non_reg.py | 23 +- tests/parsers/test_BaseNodeParser.py | 294 ----------- tests/parsers/test_BnfNodeParser.py | 7 +- tests/parsers/test_RuleParser.py | 2 +- tests/parsers/test_SyaNodeParser.py | 39 +- 44 files changed, 1617 insertions(+), 1068 deletions(-) create mode 100644 docs/source/tech/debugger.rst delete mode 100644 tests/parsers/test_BaseNodeParser.py diff --git a/_concepts_admin.txt b/_concepts_admin.txt index 1727684..9fb8b85 100644 --- a/_concepts_admin.txt +++ b/_concepts_admin.txt @@ -44,3 +44,14 @@ def concept deactivate return values processing as set_var("sheerka.enable_proce set_auto_eval(c:activate return values processing:) set_auto_eval(c:deactivate return values processing:) +def concept rule x where isinstance(x, int) as r:|x: +set_auto_eval(c:rule x:) +def concept rule x > rule y where isinstance(x, int) and isinstance(y, int) as set_is_greater_than(__PRECEDENCE, r:|x:, r:|y:, 'Rule') +set_auto_eval(c:rule x > rule y:) +def concept rule x is greatest where isinstance(x, int) as set_is_greatest(__PRECEDENCE, r:|x:, 'Rule') +set_auto_eval(c:rule x is greatest:) +def concept rule x < rule y where isinstance(x, int) and isinstance(y, int) as set_is_less_than(__PRECEDENCE, r:|x:, r:|y:, 'Rule') +set_auto_eval(c:rule x < rule y:) +def concept rule x is lesser where isinstance(x, int) as set_is_lesser(__PRECEDENCE, r:|x:, 'Rule') +set_auto_eval(c:rule x is lesser:) + diff --git a/docs/source/README.txt b/docs/source/README.txt index 49d2ccc..b227bb6 100644 --- a/docs/source/README.txt +++ b/docs/source/README.txt @@ -4,4 +4,5 @@ install the following extensions: * LiveServer (from Ritwick Dey) * reStructuredText (from Lextudio Inc) doc page for reStructuredText : https://docs.restructuredtext.net/ -shortcuts : https://docs.restructuredtext.net/articles/shortcuts.html \ No newline at end of file +shortcuts : https://docs.restructuredtext.net/articles/shortcuts.html +doc de sphinx : https://www.sphinx-doc.org/en/master/usage/restructuredtext/basics.html \ No newline at end of file diff --git a/docs/source/blog/blog.rst b/docs/source/blog/blog.rst index 263a4cc..50ebcc6 100644 --- a/docs/source/blog/blog.rst +++ b/docs/source/blog/blog.rst @@ -1,11 +1,3 @@ -.. toctree:: - :maxdepth: 1 - - concepts - rules - parsers - persistence - 2019-10-30 ********** diff --git a/docs/source/conf.py b/docs/source/conf.py index 74a67cb..10ad345 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -18,7 +18,7 @@ # -- Project information ----------------------------------------------------- project = 'Sheerka' -copyright = '2020, Kodjo Sossouvi' +copyright = '2020-2001, Kodjo Sossouvi' author = 'Kodjo Sossouvi' # The full version, including alpha/beta/rc tags @@ -52,4 +52,4 @@ html_theme = 'alabaster' # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] \ No newline at end of file +html_static_path = ['_static'] diff --git a/docs/source/index.rst b/docs/source/index.rst index a10f22e..a229c9d 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -1,6 +1,10 @@ Sheerka's documentation! =================================== +.. WARNING:: + + Under construction for one year ! + Hi, welcome the the documentation page of Sheerka. There will be two types of documentation @@ -15,11 +19,14 @@ There will be two types of documentation blog/blog + .. toctree:: :maxdepth: 1 :caption: Technical Design: tech/tech + tech/debugger + Indices and tables diff --git a/docs/source/tech/debugger.rst b/docs/source/tech/debugger.rst new file mode 100644 index 0000000..faa0f97 --- /dev/null +++ b/docs/source/tech/debugger.rst @@ -0,0 +1,19 @@ +Debugger +======== + +Abstract +-------- +The purpose of the debugger is to ease to understand why Sheerka does not behave like it should. +(yes, this could happen). I have have just started so, they are a lot of +functionalities that we usually found in a debugger that are not yet implemented + +I guess that the first thing to know is that I want to design a 'smart' debugger, +which can be configured in a 'declarative' way. It means that, rather that telling +it what to do, I would prefer tell it what is expected. + +As of now (2021-01-11), I am far from it, so let's explain how to use the debugger, +in an 'imperative' way. + +First, you need to enable the debugger. As it consumes resource, it is deactivated by default :: + + set_debug() diff --git a/src/core/builtin_concepts_ids.py b/src/core/builtin_concepts_ids.py index 1302e5e..bbf622d 100644 --- a/src/core/builtin_concepts_ids.py +++ b/src/core/builtin_concepts_ids.py @@ -75,6 +75,7 @@ class BuiltinConcepts: INVALID_RETURN_VALUE = "__INVALID_RETURN_VALUE" # the return value of an evaluator is not correct CONCEPT_ALREADY_DEFINED = "__CONCEPT_ALREADY_DEFINED" # when you try to add the same object twice (a concept or whatever) PROPERTY_ALREADY_DEFINED = "__PROPERTY_ALREADY_DEFINED" # When you try to add the same element in a property + COMPARISON_ALREADY_DEFINED = "__COMPARISON_ALREADY_DEFINED" # when the same comparison entry is defined several times NOP = "__NOP" # no operation concept. Does nothing CONCEPT_EVAL_ERROR = "__CONCEPT_EVAL_ERROR" # cannot evaluate a property or metadata of a concept ENUMERATION = "__ENUMERATION" # represents a list or a set @@ -91,8 +92,8 @@ class BuiltinConcepts: FORMAT_INSTRUCTIONS = "__FORMAT_INSTRUCTIONS" # to express how to print the concept NOT_IMPLEMENTED = "__NOT_IMPLEMENTED" # instead of raise an error PYTHON_SECURITY_ERROR = "__PYTHON_SECURITY_ERROR" # when trying to execute statement when only expression is allowed - INVALID_LESSER_OPERATION = "__INVALID_LESSER_OPERATION" - INVALID_GREATEST_OPERATION = "__INVALID_GREATEST_OPERATION" + IS_LESSER_CONSTRAINT_ERROR = "__IS_LESSER_CONSTRAINT_ERROR" + IS_GREATEST_CONSTRAINT_ERROR = "__IS_GREATEST_CONSTRAINT_ERROR" NEW_RULE = "__NEW_RULE" UNKNOWN_RULE = "__UNKNOWN_RULE" ONTOLOGY_ALREADY_DEFINED = "__ONTOLOGY_ALREADY_DEFINED" @@ -145,8 +146,8 @@ BuiltinUnique = [ BuiltinConcepts.ISA, BuiltinConcepts.AUTO_EVAL, - BuiltinConcepts.INVALID_LESSER_OPERATION, - BuiltinConcepts.INVALID_GREATEST_OPERATION, + BuiltinConcepts.IS_LESSER_CONSTRAINT_ERROR, + BuiltinConcepts.IS_GREATEST_CONSTRAINT_ERROR, ] BuiltinErrors = [ @@ -167,8 +168,8 @@ BuiltinErrors = [ BuiltinConcepts.CONDITION_FAILED, BuiltinConcepts.CHICKEN_AND_EGG, BuiltinConcepts.NOT_FOUND, - BuiltinConcepts.INVALID_LESSER_OPERATION, - BuiltinConcepts.INVALID_GREATEST_OPERATION, + BuiltinConcepts.IS_LESSER_CONSTRAINT_ERROR, + BuiltinConcepts.IS_GREATEST_CONSTRAINT_ERROR, BuiltinConcepts.ONTOLOGY_ALREADY_DEFINED ] diff --git a/src/core/builtin_helpers.py b/src/core/builtin_helpers.py index 2b00d65..02de8fb 100644 --- a/src/core/builtin_helpers.py +++ b/src/core/builtin_helpers.py @@ -12,7 +12,7 @@ from core.tokenizer import Keywords from core.utils import as_bag from parsers.BaseNodeParser import SourceCodeNode, ConceptNode, UnrecognizedTokensNode, SourceCodeWithConceptNode, \ RuleNode -from parsers.BaseParser import BaseParser, ParsingError +from parsers.BaseParser import ParsingError PARSE_STEPS = [BuiltinConcepts.BEFORE_PARSING, BuiltinConcepts.PARSING, BuiltinConcepts.AFTER_PARSING] EVAL_STEPS = PARSE_STEPS + [BuiltinConcepts.BEFORE_EVALUATION, BuiltinConcepts.EVALUATION, @@ -655,6 +655,28 @@ def ensure_concept_or_rule(*items): raise TypeError(f"'{items}' must be a concept or rule") +def ensure_bnf(context, concept, parser_name="BaseNodeParser"): + if concept.get_metadata().definition_type == DEFINITION_TYPE_BNF and not concept.get_bnf(): + from parsers.BnfDefinitionParser import BnfDefinitionParser + regex_parser = BnfDefinitionParser() + desc = f"Resolving BNF '{concept.get_metadata().definition}'" + with context.push(BuiltinConcepts.INIT_BNF, + concept, + who=parser_name, + obj=concept, + desc=desc) as sub_context: + sub_context.add_inputs(parser_input=concept.get_metadata().definition) + bnf_parsing_ret_val = regex_parser.parse(sub_context, concept.get_metadata().definition) + sub_context.add_values(return_values=bnf_parsing_ret_val) + + if not bnf_parsing_ret_val.status: + raise Exception(bnf_parsing_ret_val.value) + + concept.set_bnf(bnf_parsing_ret_val.body.body) + if concept.id: + context.sheerka.get_by_id(concept.id).set_bnf(concept.get_bnf()) # update bnf in cache + + expressions_cache = Cache() diff --git a/src/core/rule.py b/src/core/rule.py index 1c8a38b..84d9d89 100644 --- a/src/core/rule.py +++ b/src/core/rule.py @@ -5,17 +5,17 @@ import core.utils ACTION_TYPE_PRINT = "print" ACTION_TYPE_EXEC = "exec" -ACTION_TYPE_DEFERRED = "deferred" # KSI 2021-04-01 What is it for ? I definitely need some proper documentation @dataclass class RuleMetadata: - action_type: str # print, exec, deferred + action_type: str # print, exec name: Union[str, None] predicate: str action: str id: str = None + id_is_unresolved = False is_compiled: bool = False is_enabled: bool = False diff --git a/src/core/sheerka/Sheerka.py b/src/core/sheerka/Sheerka.py index ffa9ff8..e3b86a5 100644 --- a/src/core/sheerka/Sheerka.py +++ b/src/core/sheerka/Sheerka.py @@ -49,8 +49,6 @@ class Sheerka(Concept): CONCEPTS_BY_ID_ENTRY = "ConceptManager:Concepts_By_ID" CONCEPTS_BY_NAME_ENTRY = "ConceptManager:Concepts_By_Name" - CONCEPTS_BY_FIRST_KEYWORD_ENTRY = "Concepts_By_First_Keyword" - RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY = "Resolved_Concepts_By_First_Keyword" CONCEPTS_SYA_DEFINITION_ENTRY = "Concepts_Sya_Definitions" RESOLVED_CONCEPTS_SYA_DEFINITION_ENTRY = "Resolved_Concepts_Sya_Definitions" CONCEPTS_GRAMMARS_ENTRY = "Concepts_Grammars" @@ -183,7 +181,6 @@ class Sheerka(Concept): self.first_time_initialisation(exec_context) self.initialize_builtin_concepts() - self.initialize_concept_node_parsing(exec_context) self.initialize_services_deferred(exec_context, self.om.current_sdp().first_time) @@ -215,17 +212,10 @@ class Sheerka(Concept): cache = IncCache().auto_configure(self.OBJECTS_IDS_ENTRY) self.om.register_cache(self.OBJECTS_IDS_ENTRY, cache) - cache = DictionaryCache().auto_configure(self.CONCEPTS_BY_FIRST_KEYWORD_ENTRY) - self.om.register_cache(self.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, cache) - self.om.get(self.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, None) # to init from sdp - cache = DictionaryCache().auto_configure(self.CONCEPTS_SYA_DEFINITION_ENTRY) self.om.register_cache(self.CONCEPTS_SYA_DEFINITION_ENTRY, cache) self.om.get(self.CONCEPTS_SYA_DEFINITION_ENTRY, None) # to init from sdp - cache = DictionaryCache().auto_configure(self.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY) - self.om.register_cache(self.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY, cache, persist=False) - cache = DictionaryCache().auto_configure(self.RESOLVED_CONCEPTS_SYA_DEFINITION_ENTRY) self.om.register_cache(self.RESOLVED_CONCEPTS_SYA_DEFINITION_ENTRY, cache, persist=False) @@ -331,13 +321,6 @@ class Sheerka(Concept): if hasattr(evaluator, "initialize"): evaluator.initialize(self) - def initialize_concept_node_parsing(self, context): - # self.init_log.debug("Initializing concepts by first keyword.") - - concepts_by_first_keyword = self.om.current_cache_manager().copy(self.CONCEPTS_BY_FIRST_KEYWORD_ENTRY) - res = self.bnp.resolve_concepts_by_first_keyword(context, concepts_by_first_keyword) - self.om.put(self.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY, False, res.body) - def initialize_ontologies(self, context): ontologies = self.om.current_sdp().load_ontologies() if not ontologies: diff --git a/src/core/sheerka/SheerkaOntologyManager.py b/src/core/sheerka/SheerkaOntologyManager.py index fef677d..8927f3a 100644 --- a/src/core/sheerka/SheerkaOntologyManager.py +++ b/src/core/sheerka/SheerkaOntologyManager.py @@ -311,48 +311,48 @@ class SheerkaOntologyManager: """ return list(self.get_all(entry, cache_only).values()) - def list_by_key(self, entry, key): - """ - List all entries of a given key - If the values are lists, sets of dictionaries, they will be concatenated - Otherwise it will raise an error - """ - res = None - - def update_values(_res, values_): - if values_ is NotFound: - return _res - elif values_ is Removed: - _res.clear() - elif isinstance(values_, dict): - if _res is None: - _res = values_.copy() - elif isinstance(_res, dict): - _res.update(values_) - else: - raise ValueError(f"Expecting dict while found '{values_}'") - elif isinstance(values_, list): - if _res is None: - _res = values_.copy() - elif isinstance(_res, list): - _res.extend(values_) - else: - raise ValueError(f"Expecting list while found '{values_}'") - else: - raise NotImplementedError() - - return _res - - for ontology in reversed(self.ontologies): - - from_cache_values = ontology.cache_manager.get(entry, key) - if from_cache_values is not NotFound: - res = update_values(res, from_cache_values) - else: - from_sdp_values = ontology.cache_manager.sdp.get(entry, key) - res = update_values(res, from_sdp_values) - - return res + # def list_by_key(self, entry, key): + # """ + # List all entries of a given key + # If the values are lists, sets of dictionaries, they will be concatenated + # Otherwise it will raise an error + # """ + # res = None + # + # def update_values(_res, values_): + # if values_ is NotFound: + # return _res + # elif values_ is Removed: + # _res.clear() + # elif isinstance(values_, dict): + # if _res is None: + # _res = values_.copy() + # elif isinstance(_res, dict): + # _res.update(values_) + # else: + # raise ValueError(f"Expecting dict while found '{values_}'") + # elif isinstance(values_, list): + # if _res is None: + # _res = values_.copy() + # elif isinstance(_res, list): + # _res.extend(values_) + # else: + # raise ValueError(f"Expecting list while found '{values_}'") + # else: + # raise NotImplementedError() + # + # return _res + # + # for ontology in reversed(self.ontologies): + # + # from_cache_values = ontology.cache_manager.get(entry, key) + # if from_cache_values is not NotFound: + # res = update_values(res, from_cache_values) + # else: + # from_sdp_values = ontology.cache_manager.sdp.get(entry, key) + # res = update_values(res, from_sdp_values) + # + # return res def get_all(self, entry, cache_only=False): """ diff --git a/src/core/sheerka/services/SheerkaAdmin.py b/src/core/sheerka/services/SheerkaAdmin.py index bded521..b5edcdf 100644 --- a/src/core/sheerka/services/SheerkaAdmin.py +++ b/src/core/sheerka/services/SheerkaAdmin.py @@ -1,11 +1,11 @@ import sys import time -from operator import attrgetter from os import path from core.builtin_concepts_ids import BuiltinConcepts, BuiltinContainers from core.builtin_helpers import ensure_concept from core.concept import Concept +from core.sheerka.services.SheerkaMemory import SheerkaMemory from core.sheerka.services.sheerka_service import BaseService CONCEPTS_FILE_LITE = "_concepts_lite.txt" @@ -31,6 +31,7 @@ class SheerkaAdmin(BaseService): self.sheerka.bind_service_method(self.admin_push_ontology, True, as_name="push_ontology") self.sheerka.bind_service_method(self.admin_pop_ontology, True, as_name="pop_ontology") self.sheerka.bind_service_method(self.ontologies, False) + self.sheerka.bind_service_method(self.in_memory, False) def caches_names(self): """ @@ -185,3 +186,15 @@ class SheerkaAdmin(BaseService): def ontologies(self): ontologies = self.sheerka.om.ontologies_names return self.sheerka.new(BuiltinConcepts.TO_LIST, body=ontologies) + + def in_memory(self): + """ + Returns the list of all obj in memory + """ + res = {} + for k, obj in self.sheerka.om.get_all(SheerkaMemory.OBJECTS_ENTRY).items(): + if isinstance(obj, list): + obj = obj[-1] + res[k] = obj.obj + + return self.sheerka.ret(self.NAME, True, self.sheerka.new(BuiltinConcepts.TO_DICT, body=res)) diff --git a/src/core/sheerka/services/SheerkaComparisonManager.py b/src/core/sheerka/services/SheerkaComparisonManager.py index 9d43e2f..6cfbc04 100644 --- a/src/core/sheerka/services/SheerkaComparisonManager.py +++ b/src/core/sheerka/services/SheerkaComparisonManager.py @@ -1,15 +1,16 @@ from dataclasses import dataclass +from functools import reduce from cache.Cache import Cache from cache.ListCache import ListCache from core.builtin_concepts import BuiltinConcepts -from core.global_symbols import EVENT_CONCEPT_PRECEDENCE_MODIFIED, EVENT_RULE_PRECEDENCE_MODIFIED, \ - RULE_COMPARISON_CONTEXT, \ - CONCEPT_COMPARISON_CONTEXT, NotFound from core.builtin_helpers import ensure_concept_or_rule from core.concept import Concept -from core.sheerka.services.SheerkaRuleManager import SheerkaRuleManager +from core.global_symbols import EVENT_CONCEPT_PRECEDENCE_MODIFIED, EVENT_RULE_PRECEDENCE_MODIFIED, \ + RULE_COMPARISON_CONTEXT, CONCEPT_COMPARISON_CONTEXT, NotInit, NotFound +from core.sheerka.services.SheerkaConceptManager import ChickenAndEggError from core.sheerka.services.sheerka_service import ServiceObj, BaseService +from core.utils import flatten @dataclass @@ -18,8 +19,8 @@ class ComparisonObj(ServiceObj): Order to store """ property: str # property to compare - a: int # id of concept a - b: int # id of concept b + a: str # str_id of concept a + b: str # str_id of concept b op: str # comparison operation context: str = "#" # context when the comparison is right @@ -39,157 +40,6 @@ class SheerkaComparisonManager(BaseService): def __init__(self, sheerka): super().__init__(sheerka) - @staticmethod - def _compute_key(prop_name, comparison_context): - """ - Key to use to store the comparisons - :param prop_name: - :param comparison_context: - :return: - """ - if isinstance(prop_name, Concept): - prefix = prop_name.key if prop_name.get_metadata().is_builtin else prop_name.id - else: - prefix = prop_name - - return f"{prefix}|{comparison_context}" - - @staticmethod - def _get_weights(comparison_objs): - """ - For every element in greater_than_s, give it a weight - if weight(a) > weight(b) it means that a > b - :param comparison_objs: list of greater than objects - :return: - """ - - values = {} - for comparison_obj in comparison_objs: - values[comparison_obj.a] = SheerkaComparisonManager.DEFAULT_COMPARISON_VALUE - values[comparison_obj.b] = SheerkaComparisonManager.DEFAULT_COMPARISON_VALUE - - for _ in range(len(comparison_objs)): - for comparison_obj in comparison_objs: - if comparison_obj.op == ">": - if values[comparison_obj.a] <= values[comparison_obj.b]: - values[comparison_obj.a] = values[comparison_obj.b] + 1 - else: - if values[comparison_obj.b] <= values[comparison_obj.a]: - values[comparison_obj.b] = values[comparison_obj.a] + 1 - - return values - - def _compute_weights(self, comparison_objs, lesser_objs_ids=None, greatest_objs_ids=None): - """ - - :param comparison_objs: - :return: - """ - - def is_not_in_objs(obj, objs_ids): - return obj.op in ("<", ">") and obj.a not in objs_ids and obj.b not in objs_ids - - def is_in_objs(obj, objs_ids): - return obj.op in ("<", ">") and (obj.a in objs_ids or obj.b in objs_ids) - - lesser_objs_ids = lesser_objs_ids or {co.a for co in comparison_objs if co.op == "<<"} - greatest_objs_ids = greatest_objs_ids or {co.a for co in comparison_objs if co.op == ">>"} - - default_weight = SheerkaComparisonManager.DEFAULT_COMPARISON_VALUE - - # get the weights for all the lesser - lesser_objs = [co for co in comparison_objs if is_in_objs(co, lesser_objs_ids)] - lesser_objs_weights = self._get_weights(lesser_objs) - - # rearrange the weight to have the highest equals to DEFAULT_COMPARISON_VALUE - 1 - highest_weight = len(lesser_objs_weights) - for co_id in lesser_objs_weights: - lesser_objs_weights[co_id] = lesser_objs_weights[co_id] - highest_weight + default_weight - 1 - for concept_id in lesser_objs_ids: - if concept_id not in lesser_objs_weights: - lesser_objs_weights[concept_id] = default_weight - 1 - - # get the weights for concepts that are not lesser or greatest - in_between_objs = [o for o in comparison_objs if is_not_in_objs(o, lesser_objs_ids | greatest_objs_ids)] - in_between_weights = self._get_weights(in_between_objs) - - # get the weights for all the greatest - greatest_objs = [co for co in comparison_objs if is_in_objs(co, greatest_objs_ids)] - greatest_objs_weights = self._get_weights(greatest_objs) - - # rearrange the weight to have the lowest equals to DEFAULT_COMPARISON_VALUE + 1 - highest_weight = max(default_weight, len(in_between_weights)) - for co_id in greatest_objs_weights: - greatest_objs_weights[co_id] = greatest_objs_weights[co_id] + highest_weight - for concept_id in greatest_objs_ids: - if concept_id not in greatest_objs_weights: - greatest_objs_weights[concept_id] = highest_weight + 1 - - return {**lesser_objs_weights, **in_between_weights, **greatest_objs_weights} - - @staticmethod - def _get_partition(weighted_concepts): - - res = {} - for k, v in weighted_concepts.items(): - res.setdefault(v, []).append(k) - return res - - def _add_comparison(self, context, comparison_obj): - key = self._compute_key(comparison_obj.property, comparison_obj.context) - previous = self.sheerka.om.get(self.COMPARISON_ENTRY, key) - new = previous.copy() if isinstance(previous, list) else [] - - for co in new: - if co.property == comparison_obj.property and \ - co.a == comparison_obj.a and \ - co.b == comparison_obj.b and \ - co.op == comparison_obj.op and \ - co.context == comparison_obj.context: - return self.sheerka.ret(self.NAME, False, self.sheerka.new(BuiltinConcepts.CONCEPT_ALREADY_DEFINED)) - - new.append(comparison_obj) - - lesser_objs_ids = {co.a for co in new if co.op == "<<"} - greatest_objs_ids = {co.a for co in new if co.op == ">>"} - - # check if it is a valid operation regarding other lesser and greatest concepts - if comparison_obj.op in ("<", ">"): - a_is_lesser = comparison_obj.a in lesser_objs_ids - b_is_lesser = comparison_obj.b in lesser_objs_ids - if a_is_lesser != b_is_lesser: # XOR operation - return self.sheerka.ret(self.NAME, False, self.sheerka.new(BuiltinConcepts.INVALID_LESSER_OPERATION)) - - a_is_greatest = comparison_obj.a in greatest_objs_ids - b_is_greatest = comparison_obj.b in greatest_objs_ids - if a_is_greatest != b_is_greatest: # XOR operation - return self.sheerka.ret(self.NAME, False, self.sheerka.new(BuiltinConcepts.INVALID_GREATEST_OPERATION)) - - if comparison_obj.op == "<<" and comparison_obj.a in greatest_objs_ids: - return self.sheerka.ret(self.NAME, False, self.sheerka.new(BuiltinConcepts.INVALID_GREATEST_OPERATION)) - - if comparison_obj.op == ">>" and comparison_obj.a in lesser_objs_ids: - return self.sheerka.ret(self.NAME, False, self.sheerka.new(BuiltinConcepts.INVALID_LESSER_OPERATION)) - - cycles = self.detect_cycles(new) - if cycles: - concepts_in_cycle = [self.sheerka.fast_resolve(c) for c in cycles] - chicken_an_egg = self.sheerka.new(BuiltinConcepts.CHICKEN_AND_EGG, body=concepts_in_cycle) - return self.sheerka.ret(self.NAME, False, chicken_an_egg) - - self.sheerka.om.put(self.COMPARISON_ENTRY, key, comparison_obj) - self.sheerka.om.put(self.RESOLVED_COMPARISON_ENTRY, key, self._compute_weights(new, - lesser_objs_ids, - greatest_objs_ids)) - - if comparison_obj.property == BuiltinConcepts.PRECEDENCE: - if comparison_obj.context == CONCEPT_COMPARISON_CONTEXT: - self.sheerka.publish(context, EVENT_CONCEPT_PRECEDENCE_MODIFIED) - elif comparison_obj.context == RULE_COMPARISON_CONTEXT: - self.sheerka.publish(context, EVENT_RULE_PRECEDENCE_MODIFIED) - - return self.sheerka.ret(self.NAME, True, self.sheerka.new(BuiltinConcepts.SUCCESS)) - def initialize(self): cache = ListCache().auto_configure(self.COMPARISON_ENTRY) self.sheerka.om.register_cache(self.COMPARISON_ENTRY, cache, True, True) @@ -202,7 +52,48 @@ class SheerkaComparisonManager(BaseService): self.sheerka.bind_service_method(self.set_is_lesser, True) self.sheerka.bind_service_method(self.set_is_greatest, True) self.sheerka.bind_service_method(self.get_partition, False) - self.sheerka.bind_service_method(self.get_concepts_weights, False) + self.sheerka.bind_service_method(self.get_weights, False) + + @staticmethod + def _compute_key(prop_name, comparison_context): + """ + Key to use to store the comparisons + :param prop_name: + :param comparison_context: + :return: + """ + prefix = SheerkaComparisonManager._get_real_prop_name(prop_name) + return f"{prefix}|{comparison_context}" + + @staticmethod + def _get_real_prop_name(prop_name): + if isinstance(prop_name, Concept): + return prop_name.key if prop_name.get_metadata().is_builtin else prop_name.str_id + + return str(prop_name) + + def _add_comparison_new(self, context, comparison_obj): + key = self._compute_key(comparison_obj.property, comparison_obj.context) + previous_entries = self.sheerka.om.get(self.COMPARISON_ENTRY, key) + previous_entries = previous_entries.copy() if isinstance(previous_entries, list) else [] + + res = self.validate_new_entry(context, comparison_obj, previous_entries) + if not res.status: + return res + + previous_entries.append(comparison_obj) + weights = self.compute_weights(previous_entries) + + self.sheerka.om.put(self.COMPARISON_ENTRY, key, comparison_obj) + self.sheerka.om.put(self.RESOLVED_COMPARISON_ENTRY, key, weights) + + if comparison_obj.property == BuiltinConcepts.PRECEDENCE: + if comparison_obj.context == CONCEPT_COMPARISON_CONTEXT: + self.sheerka.publish(context, EVENT_CONCEPT_PRECEDENCE_MODIFIED) + elif comparison_obj.context == RULE_COMPARISON_CONTEXT: + self.sheerka.publish(context, EVENT_RULE_PRECEDENCE_MODIFIED) + + return self.sheerka.ret(self.NAME, True, self.sheerka.new(BuiltinConcepts.SUCCESS)) def set_is_greater_than(self, context, prop_name, item_a, item_b, comparison_context="#"): """ @@ -217,14 +108,15 @@ class SheerkaComparisonManager(BaseService): context.log(f"Setting item {item_a} is greater than {item_b}", who=self.NAME) ensure_concept_or_rule(item_a, item_b) + real_prop_name = self._get_real_prop_name(prop_name) event_digest = context.event.get_digest() comparison_obj = ComparisonObj(event_digest, - prop_name, + real_prop_name, item_a.str_id, item_b.str_id, ">", comparison_context) - return self._add_comparison(context, comparison_obj) + return self._add_comparison_new(context, comparison_obj) def set_is_less_than(self, context, prop_name, item_a, item_b, comparison_context="#"): """ @@ -239,14 +131,15 @@ class SheerkaComparisonManager(BaseService): context.log(f"Setting item {item_a} is less than {item_b}", who=self.NAME) ensure_concept_or_rule(item_a, item_b) + real_prop_name = self._get_real_prop_name(prop_name) event_digest = context.event.get_digest() comparison_obj = ComparisonObj(event_digest, - prop_name, + real_prop_name, item_a.str_id, item_b.str_id, "<", comparison_context) - return self._add_comparison(context, comparison_obj) + return self._add_comparison_new(context, comparison_obj) def set_is_lesser(self, context, prop_name, item, comparison_context="#"): """ @@ -265,14 +158,15 @@ class SheerkaComparisonManager(BaseService): context.log(f"Setting item {item} is lesser", who=self.NAME) ensure_concept_or_rule(item) + real_prop_name = self._get_real_prop_name(prop_name) event_digest = context.event.get_digest() comparison_obj = ComparisonObj(event_digest, - prop_name, + real_prop_name, item.str_id, None, "<<", comparison_context) - return self._add_comparison(context, comparison_obj) + return self._add_comparison_new(context, comparison_obj) def set_is_greatest(self, context, prop_name, item, comparison_context="#"): """ @@ -291,14 +185,15 @@ class SheerkaComparisonManager(BaseService): context.log(f"Setting item {item} is greatest", who=self.NAME) ensure_concept_or_rule(item) + real_prop_name = self._get_real_prop_name(prop_name) event_digest = context.event.get_digest() comparison_obj = ComparisonObj(event_digest, - prop_name, + real_prop_name, item.str_id, None, ">>", comparison_context) - return self._add_comparison(context, comparison_obj) + return self._add_comparison_new(context, comparison_obj) def set_are_equivalent(self, context, prop_name, concept_a, concept_b, comparison_context="#"): """ @@ -321,11 +216,14 @@ class SheerkaComparisonManager(BaseService): :param comparison_context: :return: """ - weighted_concept = self.get_concepts_weights(prop_name, comparison_context) + weighted_concept = self.get_weights(prop_name, comparison_context) - return self._get_partition(weighted_concept) + res = {} + for k, v in weighted_concept.items(): + res.setdefault(v, []).append(k) + return res - def get_concepts_weights(self, prop_name, comparison_context="#"): + def get_weights(self, prop_name, comparison_context="#"): # KSI 2021-01-10 This implementation seems to be too complicated # Chances are that there is a better way to implement this. # Note that I don't want to use a DictionaryCache for the RESOLVED_COMPARISON_ENTRY @@ -340,32 +238,245 @@ class SheerkaComparisonManager(BaseService): else: # otherwise, either it's not computed yet or it does not include the info of the current layer # In both case, it is safer to recompute the weights - entries = self.sheerka.om.list_by_key(self.COMPARISON_ENTRY, key_to_use) - if entries is None: + entries = self.sheerka.om.get(self.COMPARISON_ENTRY, key_to_use) + if entries is NotFound: weighted_concepts = {} # Why not put it in cache ??? else: - weighted_concepts = self._compute_weights(entries) + weighted_concepts = self.compute_weights(entries) self.sheerka.om.put(self.RESOLVED_COMPARISON_ENTRY, key_to_use, weighted_concepts) return weighted_concepts @staticmethod - def detect_cycles(comparison_objs): + def toposort(comparison_objs): + """ + Topological sort or topological ordering of a directed graph is a linear ordering of its vertices + such that for every directed edge uv from vertex u to vertex v, u comes before v in the ordering + https://en.wikipedia.org/wiki/Topological_sorting + This implementation is taken from https://code.activestate.com/recipes/578272-topological-sort/ + """ + if not comparison_objs: + return [] + + data = {} + for obj in comparison_objs: + data.setdefault(obj.a, set()).add(obj.b) if obj.op == ">" else data.setdefault(obj.b, set()).add(obj.a) + + # Ignore self dependencies. + for k, v in data.items(): + v.discard(k) + + # Find all items that don't depend on anything. + extra_items_in_deps = reduce(set.union, data.values()) - set(data.keys()) + + # Add empty dependencies where needed + data.update({item: set() for item in extra_items_in_deps}) + while True: + ordered = set(item for item, dep in data.items() if not dep) + if not ordered: + break + yield ordered + data = {item: (dep - ordered) + for item, dep in data.items() + if item not in ordered} + + if len(data) != 0: + raise ChickenAndEggError(set(flatten(data.values()))) + + @staticmethod + def get_objs_groups(comparison_objs): + """ + Browse all ComparisonObj definitions group the obj in three categories + * those which are defined as greatest, + * those which are defined as lowest, + * those which are defined as equivalent as another one, + We also returns the list of all object + -> Which make it a tuple of four elements + """ + greatest = set() + lowest = set() + all_items = set() + + for co in comparison_objs: + if co.op == ">>": + greatest.add(co.a) + elif co.op == "<<": + lowest.add(co.a) + + if co.a is not None: + all_items.add(co.a) + if co.b is not None: + all_items.add(co.b) + + return greatest, lowest, all_items + + @staticmethod + def group_comparison_objs(comparison_objs, greatest_objs, lowest_objs): + """ + Groups the ComparisonObj in three categories + * those which express relation between greatest objects + * those which express relation between lowest objects + * those which express relation between objects which are neither greatest or lowest + + To express a relation between greatest objects, both objects must be flagged as greatest + for example: + if the relation is 'a > b' with a is greatest, but not b, + this relation will NOT go in the greatest list + As a matter of fact, it does not fit in any list as it's an irrelevant information + + Note that for efficiency, we consider that the settings are already valid + We cannot find something like + 'a > b' with 'a' is lowest and 'b' is greatest + or 'a > b' 'b' lowest and 'a' is not + """ + greatest, lowest, equiv, other = [], [], [], [] # we keep the order + greatest_or_lowest_objs = set(greatest_objs) + greatest_or_lowest_objs.update(lowest_objs) + + for co in comparison_objs: + if co.op in ("<", ">"): + if co.a in greatest_objs and co.b in greatest_objs: + greatest.append(co) + elif co.a in lowest_objs and co.b in lowest_objs: + lowest.append(co) + elif not (co.a in greatest_or_lowest_objs or co.b in greatest_or_lowest_objs): + other.append(co) + elif co.op == "=": + equiv.append(co) + + return greatest, lowest, equiv, other + + @staticmethod + def compute_weights(comparison_objs): + """ + Computes the weights of all items that appear in the comparison objs + """ + greatest_objs, lowest_objs, all_objs = SheerkaComparisonManager.get_objs_groups(comparison_objs) + + # co stands for comparison_object + co_greatest, co_lowest, co_equiv, co_other = SheerkaComparisonManager.group_comparison_objs(comparison_objs, + greatest_objs, + lowest_objs) + + weights = {obj: NotInit for obj in all_objs} + max_weight = SheerkaComparisonManager.DEFAULT_COMPARISON_VALUE + min_weight = SheerkaComparisonManager.DEFAULT_COMPARISON_VALUE + + # manage lowest objects group + sorted_objs = list(SheerkaComparisonManager.toposort(co_lowest)) + for group in reversed(sorted_objs): + group_weight = min_weight - 1 + min_weight = group_weight + for item in group: + weights[item] = group_weight + + # manage normal objects group + sorted_objs = SheerkaComparisonManager.toposort(co_other) + for i, group in enumerate(sorted_objs): + group_weight = i + SheerkaComparisonManager.DEFAULT_COMPARISON_VALUE + max_weight = group_weight + + for item in group: + weights[item] = group_weight + + # manage greatest objects group + sorted_objs = SheerkaComparisonManager.toposort(co_greatest) + for group in sorted_objs: + group_weight = max_weight + 1 + max_weight = group_weight + + for item in group: + weights[item] = group_weight + + # manage greatest flagged objects that have not been initialized yet + for obj in greatest_objs: + if weights[obj] is NotInit: + weights[obj] = max_weight + 1 + + # manage lowest flagged objects that have not been initialized yet + for obj in lowest_objs: + if weights[obj] is NotInit: + weights[obj] = min_weight - 1 + + # manage equiv + for co in co_equiv: + if weights[co.a] is NotInit and weights[co.b] is not NotInit: + weights[co.a] = weights[co.b] + elif weights[co.b] is NotInit and weights[co.a] is not NotInit: + weights[co.b] = weights[co.a] + + # set not initialized to default value + for k, v in weights.items(): + if v is NotInit: + weights[k] = SheerkaComparisonManager.DEFAULT_COMPARISON_VALUE + + return weights + + @staticmethod + def validate_new_entry(context, new_entry, previous_entries): + sheerka = context.sheerka + name = SheerkaComparisonManager.NAME + + # detect duplicates + for co in previous_entries: + if co.property == new_entry.property and \ + co.a == new_entry.a and \ + co.b == new_entry.b and \ + co.op == new_entry.op and \ + co.context == new_entry.context: + return sheerka.ret(name, False, sheerka.new(BuiltinConcepts.COMPARISON_ALREADY_DEFINED)) + + # detect cycle + cycles = SheerkaComparisonManager.detect_cycles(previous_entries, new_entry) + if cycles: + chicken_an_egg = context.sheerka.new(BuiltinConcepts.CHICKEN_AND_EGG, body=cycles) + return sheerka.ret(name, False, chicken_an_egg) + + greatest_objs, lowest_objs, all_objs = SheerkaComparisonManager.get_objs_groups(previous_entries) + + # detect IS_GREATEST_CONSTRAINT_ERROR and IS_LESSER_CONSTRAINT_ERROR + if new_entry.op == "<<" and new_entry.a in greatest_objs: + return sheerka.ret(name, False, sheerka.new(BuiltinConcepts.IS_GREATEST_CONSTRAINT_ERROR)) + elif new_entry.op == ">>" and new_entry.a in lowest_objs: + return sheerka.ret(name, False, sheerka.new(BuiltinConcepts.IS_LESSER_CONSTRAINT_ERROR)) + + # implement the following matrix + # +-------+------+-----+-----+-----+ + # | | a<< | b<< | a>> | b>> | + # | a < b | ok | nok | nok | ok | + # | a > b | nok | ok | ok | nok | + # |-------+------+-----+-----+-----+ + # nok means that new rule cannot be accepted, unless there is a change on the other item + elif new_entry.op == "<": + if new_entry.a in greatest_objs and new_entry.b not in greatest_objs: + return sheerka.ret(name, False, sheerka.new(BuiltinConcepts.IS_GREATEST_CONSTRAINT_ERROR)) + elif new_entry.b in lowest_objs and new_entry.a not in lowest_objs: + return sheerka.ret(name, False, sheerka.new(BuiltinConcepts.IS_LESSER_CONSTRAINT_ERROR)) + elif new_entry.op == ">": + if new_entry.a in lowest_objs and new_entry.b not in lowest_objs: + return sheerka.ret(name, False, sheerka.new(BuiltinConcepts.IS_LESSER_CONSTRAINT_ERROR)) + elif new_entry.b in greatest_objs and new_entry.a not in greatest_objs: + return sheerka.ret(name, False, sheerka.new(BuiltinConcepts.IS_GREATEST_CONSTRAINT_ERROR)) + + return sheerka.ret(name, True, sheerka.new(BuiltinConcepts.SUCCESS)) + + @staticmethod + def detect_cycles(previous_comparison_objs, new_comparison_obj: ComparisonObj): """ # Thanks to Divyanshu Mehta for contributing this code # https://www.geeksforgeeks.org/detect-cycle-in-a-graph/?ref=lbp - :param comparison_objs: + :param previous_comparison_objs: + :param new_comparison_obj: :return: """ - latest = comparison_objs[-1] - if latest.op == "=": + if new_comparison_obj.op in ("<<", ">>", "="): return None def get_graph_and_vertices(): _graph = {} _vertices = set() - for obj in comparison_objs: - if obj.op == "=": + for obj in previous_comparison_objs + [new_comparison_obj]: + if obj.op in ("<<", ">>", "="): continue _vertices.add(obj.a) @@ -393,7 +504,7 @@ class SheerkaComparisonManager(BaseService): elif rec_stack[neighbour]: return True - # The node needs to be poped from + # The node needs to be popped from # recursion stack before function ends rec_stack[v] = False return False @@ -402,6 +513,7 @@ class SheerkaComparisonManager(BaseService): visited = {k: False for k in vertices} rec_stack = {k: False for k in vertices} - if is_cyclic(latest.a): # only need to check from the latest add, since the graph was not cyclic before + if is_cyclic( + new_comparison_obj.a): # only need to check from the latest add, since the graph was not cyclic before return [k for k, v in rec_stack.items() if v] return None diff --git a/src/core/sheerka/services/SheerkaConceptManager.py b/src/core/sheerka/services/SheerkaConceptManager.py index adf0637..1a39517 100644 --- a/src/core/sheerka/services/SheerkaConceptManager.py +++ b/src/core/sheerka/services/SheerkaConceptManager.py @@ -1,14 +1,17 @@ from dataclasses import dataclass +from typing import Set import core.utils from cache.Cache import Cache from cache.CacheManager import ConceptNotFound +from cache.DictionaryCache import DictionaryCache from cache.ListIfNeededCache import ListIfNeededCache from cache.SetCache import SetCache from core.builtin_concepts import ErrorConcept from core.builtin_concepts_ids import BuiltinConcepts, AllBuiltinConcepts, BuiltinUnique -from core.builtin_helpers import ensure_concept -from core.concept import Concept, DEFINITION_TYPE_DEF, DEFINITION_TYPE_BNF, freeze_concept_attrs, ConceptMetadata +from core.builtin_helpers import ensure_concept, ensure_bnf +from core.concept import Concept, DEFINITION_TYPE_DEF, DEFINITION_TYPE_BNF, freeze_concept_attrs, ConceptMetadata, \ + VARIABLE_PREFIX from core.error import ErrorObj from core.global_symbols import EVENT_CONCEPT_CREATED, NotInit, NotFound from core.sheerka.services.sheerka_service import BaseService @@ -18,6 +21,17 @@ from sdp.sheerkaDataProvider import SheerkaDataProviderDuplicateKeyError BASE_NODE_PARSER_CLASS = "parsers.BaseNodeParser.BaseNodeParser" +@dataclass +class ChickenAndEggError(Exception): + concepts: Set[str] + + +@dataclass +class NoFirstTokenError(ErrorObj): + concept: Concept + key: str + + @dataclass class NoModificationFound(ErrorObj): """ @@ -82,9 +96,11 @@ class SheerkaConceptManager(BaseService): CONCEPTS_BY_HASH_ENTRY = "ConceptManager:Concepts_By_Hash" # store hash of concepts definitions (not values) CONCEPTS_REFERENCES_ENTRY = "ConceptManager:Concepts_References" # tracks references between concepts + CONCEPTS_BY_FIRST_KEYWORD_ENTRY = "ConceptManager:Concepts_By_First_Keyword" + RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY = "ConceptManager:Resolved_Concepts_By_First_Keyword" + def __init__(self, sheerka): super().__init__(sheerka) - self.bnp = core.utils.get_class(BASE_NODE_PARSER_CLASS) # BaseNodeParser self.forbidden_meta = {"is_builtin", "key", "id", "props", "variables"} self.allowed_meta = {attr for attr in vars(ConceptMetadata) if not attr.startswith("_") and attr not in self.forbidden_meta} @@ -101,6 +117,7 @@ class SheerkaConceptManager(BaseService): self.sheerka.bind_service_method(self.get_by_hash, False, visible=False) self.sheerka.bind_service_method(self.get_by_id, False, visible=False) self.sheerka.bind_service_method(self.is_not_a_variable, False, visible=False) + self.sheerka.bind_service_method(self.get_concepts_by_first_token, False, visible=False) register_concept_cache = self.sheerka.om.register_concept_cache @@ -119,10 +136,22 @@ class SheerkaConceptManager(BaseService): cache = SetCache().auto_configure(self.CONCEPTS_REFERENCES_ENTRY) self.sheerka.om.register_cache(self.CONCEPTS_REFERENCES_ENTRY, cache) + cache = DictionaryCache().auto_configure(self.CONCEPTS_BY_FIRST_KEYWORD_ENTRY) + self.sheerka.om.register_cache(self.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, cache) + + cache = DictionaryCache().auto_configure(self.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY) + self.sheerka.om.register_cache(self.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY, cache, persist=False) + def initialize_deferred(self, context, is_first_time): if is_first_time: self.sheerka.om.put(self.sheerka.OBJECTS_IDS_ENTRY, self.USER_CONCEPTS_IDS, 1000) + # initialize the dictionary of first tokens + self.sheerka.om.get(self.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, None) # to init the cache with the values from sdp + concepts_by_first_keyword = self.sheerka.om.current_cache_manager().copy(self.CONCEPTS_BY_FIRST_KEYWORD_ENTRY) + res = self.resolve_concepts_by_first_keyword(context, concepts_by_first_keyword) + self.sheerka.om.put(self.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY, False, res.body) + def initialize_builtin_concepts(self): """ Initializes the builtin concepts @@ -181,18 +210,18 @@ class SheerkaConceptManager(BaseService): # check if the bnf definition is correctly computed try: - self.bnp.ensure_bnf(context, concept) + ensure_bnf(context, concept) except Exception as ex: return sheerka.ret(self.NAME, False, ex.args[0]) # compute new concepts_by_first_keyword - init_ret_value = self.bnp.compute_concepts_by_first_token(context, [concept], True) + init_ret_value = self.compute_concepts_by_first_token(context, [concept], True) if not init_ret_value.status: return sheerka.ret(self.NAME, False, ErrorConcept(init_ret_value.value)) concepts_by_first_keyword = init_ret_value.body # computes resolved concepts_by_first_keyword - init_ret_value = self.bnp.resolve_concepts_by_first_keyword(context, concepts_by_first_keyword) + init_ret_value = self.resolve_concepts_by_first_keyword(context, concepts_by_first_keyword) if not init_ret_value.status: return sheerka.ret(self.NAME, False, ErrorConcept(init_ret_value.value)) resolved_concepts_by_first_keyword = init_ret_value.body @@ -202,8 +231,8 @@ class SheerkaConceptManager(BaseService): concept.freeze_definition_hash() ontology.add_concept(concept) - ontology.put(sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, False, concepts_by_first_keyword) - ontology.put(sheerka.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY, False, resolved_concepts_by_first_keyword) + ontology.put(self.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, False, concepts_by_first_keyword) + ontology.put(self.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY, False, resolved_concepts_by_first_keyword) if concept.get_metadata().definition_type == DEFINITION_TYPE_DEF and concept.get_metadata().definition != concept.name: # allow search by definition when definition relevant @@ -268,8 +297,8 @@ class SheerkaConceptManager(BaseService): # To update concept by first keyword # first remove the old references - keywords = self.bnp.get_first_tokens(sheerka, concept) # keyword of the old concept - concepts_by_first_keyword = cache_manager.copy(sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY) + keywords = self.get_first_tokens(sheerka, concept) # keyword of the old concept + concepts_by_first_keyword = cache_manager.copy(self.CONCEPTS_BY_FIRST_KEYWORD_ENTRY) for keyword in keywords: try: concepts_by_first_keyword[keyword].remove(concept.id) @@ -279,15 +308,15 @@ class SheerkaConceptManager(BaseService): pass # and then update - init_ret_value = self.bnp.compute_concepts_by_first_token(context, [new_concept], False, concepts_by_first_keyword) + init_ret_value = self.compute_concepts_by_first_token(context, [new_concept], False, concepts_by_first_keyword) if not init_ret_value.status: return sheerka.ret(self.NAME, False, ErrorConcept(init_ret_value.value)) concepts_by_first_keyword = init_ret_value.body # computes resolved concepts_by_first_keyword - init_ret_value = self.bnp.resolve_concepts_by_first_keyword(context, - concepts_by_first_keyword, - {new_concept.id: new_concept}) + init_ret_value = self.resolve_concepts_by_first_keyword(context, + concepts_by_first_keyword, + {new_concept.id: new_concept}) if not init_ret_value.status: return sheerka.ret(self.NAME, False, ErrorConcept(init_ret_value.value)) resolved_concepts_by_first_keyword = init_ret_value.body @@ -302,9 +331,8 @@ class SheerkaConceptManager(BaseService): cache_manager.put(self.CONCEPTS_REFERENCES_ENTRY, ref, new_concept.id) cache_manager.update_concept(concept, new_concept) - cache_manager.put(sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, False, concepts_by_first_keyword) - cache_manager.put(sheerka.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY, False, - resolved_concepts_by_first_keyword) + cache_manager.put(self.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, False, concepts_by_first_keyword) + cache_manager.put(self.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY, False, resolved_concepts_by_first_keyword) # everything seems to be fine. Update the list of attributes # Caution. Must be done AFTER update_concept() @@ -620,8 +648,192 @@ class SheerkaConceptManager(BaseService): concept.get_metadata().key = None if self._definition_has_changed(to_add) and concept.get_metadata().definition_type == DEFINITION_TYPE_BNF: concept.set_bnf(None) - self.bnp.ensure_bnf(context, concept) + ensure_bnf(context, concept) concept.init_key() return + + @staticmethod + def get_first_tokens(sheerka, concept): + """ + + :param sheerka: + :param concept: + :return: + """ + if concept.get_bnf(): + from parsers.BnfNodeParser import BnfNodeFirstTokenVisitor + bnf_visitor = BnfNodeFirstTokenVisitor(sheerka) + bnf_visitor.visit(concept.get_bnf()) + return bnf_visitor.first_tokens + else: + keywords = concept.key.split() + for keyword in keywords: + if keyword.startswith(VARIABLE_PREFIX): + continue + + return [keyword] + + return None + + @staticmethod + def compute_concepts_by_first_token(context, concepts, use_sheerka=False, previous_entries=None): + """ + Create the map describing the first token expected by a concept + eg the dictionary that goes into CONCEPTS_BY_FIRST_KEYWORD_ENTRY + :param context: + :param concepts: lists of concepts to parse + :param use_sheerka: if True, update concepts_by_first_keyword from sheerka + :param previous_entries: + :return: + """ + sheerka = context.sheerka + res = sheerka.om.copy(SheerkaConceptManager.CONCEPTS_BY_FIRST_KEYWORD_ENTRY) if use_sheerka \ + else (previous_entries or {}) + for concept in concepts: + keywords = SheerkaConceptManager.get_first_tokens(sheerka, concept) + + if keywords is None: + # no first token found for a concept ? + return sheerka.ret(sheerka.name, False, NoFirstTokenError(concept, concept.key)) + + for keyword in keywords: + res.setdefault(keyword, []).append(concept.id) + + # 'uniquify' the lists + for k, v in res.items(): + res[k] = core.utils.make_unique(v) + + return sheerka.ret("BaseNodeParser", True, res) + + @staticmethod + def resolve_concepts_by_first_keyword(context, concepts_by_first_keyword, modified_concepts=None): + """ + From a dictionary of first tokens, create another dictionary where all references to other concepts + are resolved + fom example, from entries + { + 'c:|1001:' : 'c:|1002:' # which means than the concept 1002 starts with the concept 1001 + 'foo': 'c:|1001:' + } + It will create + { + 'foo': ['c:|1001:, 'c:|1002:'], + } + + This dictionary is supposed to go into CONCEPTS_REFERENCES_ENTRY + """ + sheerka = context.sheerka + res = {} + + def get_by_id(c_id): + if modified_concepts and c_id in modified_concepts: + return modified_concepts[c_id] + return sheerka.get_by_id(c_id) + + def resolve_concepts(concept_str): + c_key, c_id = core.utils.unstr_concept(concept_str) + if c_id in already_seen: + return ChickenAndEggError(already_seen) + + already_seen.add(c_id) + + resolved = set() + to_resolve = set() + chicken_and_egg = set() + + concept = get_by_id(c_id) + + if sheerka.isaset(context, concept): + concepts = sheerka.get_set_elements(context, concept) + else: + concepts = [concept] + + for concept in concepts: + ensure_bnf(context, concept) # need to make sure that it cannot fail + keywords = SheerkaConceptManager.get_first_tokens(sheerka, concept) + for keyword in keywords: + (to_resolve if keyword.startswith("c:|") else resolved).add(keyword) + + for concept_to_resolve_str in to_resolve: + res = resolve_concepts(concept_to_resolve_str) + if isinstance(res, ChickenAndEggError): + chicken_and_egg |= res.concepts + else: + resolved |= res + to_resolve.clear() + + if len(resolved) == 0 and len(chicken_and_egg) > 0: + raise ChickenAndEggError(chicken_and_egg) + else: + return resolved + + for k, v in concepts_by_first_keyword.items(): + if k.startswith("c:|"): + try: + already_seen = set() + resolved_keywords = resolve_concepts(k) + for resolved in resolved_keywords: + res.setdefault(resolved, []).extend(v) + except ChickenAndEggError as ex: + context.log(f"Chicken and egg detected for {k}, concepts={ex.concepts}") + concepts_in_recursion = ex.concepts + # make sure to have all the parents + for parent in v: + concepts_in_recursion.add(parent) + + for concept_id in concepts_in_recursion: + # make sure we keep the longest chain + old = sheerka.chicken_and_eggs.get(concept_id) + if old is NotFound or len(old) < len(ex.concepts): + sheerka.chicken_and_eggs.put(concept_id, concepts_in_recursion) + else: + res.setdefault(k, []).extend(v) + + # 'uniquify' the lists + for k, v in res.items(): + res[k] = core.utils.make_unique(v) + + return sheerka.ret("BaseNodeParser", True, res) + + def get_concepts_by_first_token(self, token, to_keep, custom=None, to_map=None, strip_quotes=False, parser=None): + """ + Tries to find if there are concepts that match the value of the token + Caution: Returns the actual cache, not a copy + :param token: + :param to_keep: predicate to tell if the concept is eligible + :param custom: lambda name -> List[Concepts] that gives extra concepts, according to the name + :param to_map: + :param strip_quotes: Remove quotes from strings + :param parser: If needed, parser which requested the concepts + :return: + """ + + if token.type == TokenKind.WHITESPACE: + return None + + if token.type == TokenKind.STRING: + name = token.value[1:-1] if strip_quotes else token.value + else: + name = token.value + + custom_concepts = custom(name) if custom else [] # to get extra concepts using an alternative method + + result = [] + concepts_ids = self.sheerka.om.get(self.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY, name) + if concepts_ids is NotFound: + return custom_concepts if custom else None + + for concept_id in concepts_ids: + + concept = self.sheerka.get_by_id(concept_id) + + if not to_keep(concept): + continue + + concept = to_map(concept, parser, self.sheerka) if to_map else concept + result.append(concept) + + return core.utils.make_unique(result + custom_concepts, + lambda c: c.concept.id if hasattr(c, "concept") else c.id) diff --git a/src/core/sheerka/services/SheerkaConceptsAlgebra.py b/src/core/sheerka/services/SheerkaConceptsAlgebra.py index 883fc42..5671d67 100644 --- a/src/core/sheerka/services/SheerkaConceptsAlgebra.py +++ b/src/core/sheerka/services/SheerkaConceptsAlgebra.py @@ -119,9 +119,7 @@ class SheerkaConceptsAlgebra(BaseService): if nb_props == 0: return res - concepts_manager = self.sheerka.services[SheerkaConceptManager.NAME] - - all_concepts = self.sheerka.om.list(concepts_manager.CONCEPTS_BY_ID_ENTRY) + all_concepts = self.sheerka.om.list(SheerkaConceptManager.CONCEPTS_BY_ID_ENTRY) for c in all_concepts: score = self._compute_score(c, concept, step_b=round(1 / nb_props, 2)) diff --git a/src/core/sheerka/services/SheerkaMemory.py b/src/core/sheerka/services/SheerkaMemory.py index 3413832..2e5da33 100644 --- a/src/core/sheerka/services/SheerkaMemory.py +++ b/src/core/sheerka/services/SheerkaMemory.py @@ -139,32 +139,23 @@ class SheerkaMemory(BaseService): self.add_to_memory(context, k, v) self.registration.clear() - def memory(self, context, name=None): + def memory(self, context, name): """ Get the list of all memory_objects in memory :param context: :param name: :return: """ - if name: - name_to_use = name.name if isinstance(name, Concept) else name - self.unregister_object(context, name_to_use) - obj = self.get_from_memory(context, name_to_use) - if obj is NotFound: - return self.sheerka.new(BuiltinConcepts.NOT_FOUND, body={"#name": name}) + name_to_use = name.name if isinstance(name, Concept) else name + self.unregister_object(context, name_to_use) + obj = self.get_from_memory(context, name_to_use) + if obj is NotFound: + return self.sheerka.new(BuiltinConcepts.NOT_FOUND, body={"#name": name}) - if isinstance(obj, list): - obj = obj[-1] + if isinstance(obj, list): + obj = obj[-1] - return obj.obj - - res = {} - for k, obj in self.sheerka.om.get_all(SheerkaMemory.OBJECTS_ENTRY).items(): - if isinstance(obj, list): - obj = obj[-1] - res[k] = obj.obj - - return res + return obj.obj def mem(self): keys = sorted([k for k in self.sheerka.om.list(SheerkaMemory.OBJECTS_ENTRY)]) diff --git a/src/core/sheerka/services/SheerkaResultManager.py b/src/core/sheerka/services/SheerkaResultManager.py index f285dfb..f877853 100644 --- a/src/core/sheerka/services/SheerkaResultManager.py +++ b/src/core/sheerka/services/SheerkaResultManager.py @@ -10,10 +10,10 @@ from core.utils import as_bag MAX_EXECUTION_HISTORY = 100 -class SheerkaResultConcept(BaseService): +class SheerkaResultManager(BaseService): NAME = "Result" - # SheerkaResultConcept seems to be a concept that must not support multiple ontology layers + # SheerkaResultManager seems to be a concept that must not support multiple ontology layers # We must have always access to everything that was done, whatever the ontology def __init__(self, sheerka, page_size=30): @@ -23,7 +23,9 @@ class SheerkaResultConcept(BaseService): self.last_execution = None self.last_created_concept = None self.last_created_concept_id = None - self.state_vars = ["last_created_concept_id"] + self.last_error_event_id = None + self.last_errors = None + self.state_vars = ["last_created_concept_id", "last_error_event_id"] def initialize(self): self.sheerka.bind_service_method(self.get_results_by_digest, True) # digest is recorded @@ -33,6 +35,7 @@ class SheerkaResultConcept(BaseService): self.sheerka.bind_service_method(self.get_execution_item, False) self.sheerka.bind_service_method(self.get_last_return_value, False, as_name="last_ret") self.sheerka.bind_service_method(self.get_last_created_concept, False, as_name="last_created_concept") + self.sheerka.bind_service_method(self.get_last_error, False, as_name="last_err") def initialize_deferred(self, context, is_first_time): self.restore_values(*self.state_vars) @@ -44,6 +47,8 @@ class SheerkaResultConcept(BaseService): self.last_execution = None self.last_created_concept = None self.last_created_concept_id = None + self.last_error_event_id = None + self.last_errors = None def save_state(self, context): self.store_values(context, *self.state_vars) @@ -246,6 +251,12 @@ class SheerkaResultConcept(BaseService): self.executions_contexts_cache.put(execution_context.event.get_digest(), execution_context) self.last_execution = execution_context + in_error = [ret for ret in execution_context.values["return_values"] if not ret.status] + if in_error: + self.last_error_event_id = execution_context.event.get_digest() + self.last_errors = in_error[0] if len(in_error) == 1 else in_error + self.store_var(execution_context, "last_error_event_id") + def get_last_return_value(self, context): """ Return the last return value(s) @@ -267,6 +278,24 @@ class SheerkaResultConcept(BaseService): return self.sheerka.new(BuiltinConcepts.NOT_FOUND, body={"query": "last"}) + def get_last_error(self): + if self.last_errors: + return self.last_errors + + if self.last_error_event_id is None: + return self.sheerka.new(BuiltinConcepts.NOT_FOUND, body={"query": "get_last_error"}) + + # search in history + if self.last_error_event_id in self.executions_contexts_cache: + execution_result = self.executions_contexts_cache.get(self.last_error_event_id) + else: + execution_result = self.sheerka.om.current_sdp().load_result(self.last_error_event_id) + + in_error = [ret for ret in execution_result.values["return_values"] if not ret.status] + self.last_errors = in_error[0] if len(in_error) == 1 else in_error + + return self.last_errors + def new_concept_created(self, context, concept): self.last_created_concept = concept self.last_created_concept_id = concept.id diff --git a/src/core/sheerka/services/SheerkaRuleManager.py b/src/core/sheerka/services/SheerkaRuleManager.py index 827a28b..a83021e 100644 --- a/src/core/sheerka/services/SheerkaRuleManager.py +++ b/src/core/sheerka/services/SheerkaRuleManager.py @@ -522,6 +522,7 @@ class SheerkaRuleManager(BaseService): self.sheerka.bind_service_method(self.get_rule_by_name, False) self.sheerka.bind_service_method(self.dump_desc_rule, False, as_name="desc_rule") self.sheerka.bind_service_method(self.get_format_rules, False, visible=False) + self.sheerka.bind_service_method(self.resolve_rule, False, visible=False) cache = Cache().auto_configure(self.FORMAT_RULE_ENTRY) self.sheerka.om.register_cache(self.FORMAT_RULE_ENTRY, cache, True, True) @@ -566,7 +567,7 @@ class SheerkaRuleManager(BaseService): :return: """ # get the priorities - rules_weights = self.sheerka.get_concepts_weights(BuiltinConcepts.PRECEDENCE, RULE_COMPARISON_CONTEXT) + rules_weights = self.sheerka.get_weights(BuiltinConcepts.PRECEDENCE, RULE_COMPARISON_CONTEXT) # update the priorities for rule in self.sheerka.om.list(self.FORMAT_RULE_ENTRY, cache_only=True): @@ -712,6 +713,11 @@ class SheerkaRuleManager(BaseService): Rule("print", "Display multiple success", "isinstance(__ret_container, BuiltinConcepts.MULTIPLE_SUCCESS)", "list(__ret_container.body)"), + + # # index=[9] in code, id=10 in debug + # Rule("print", "Display simple result when only one success", + # "len(__rets)==1 and __rets[0].status == True", + # "{__rets[0].body}"), ] for r in rules: @@ -737,12 +743,7 @@ class SheerkaRuleManager(BaseService): if rule_id is None: return None - rule = self.sheerka.om.get(self.FORMAT_RULE_ENTRY, rule_id) - if rule is not NotFound: - return rule - - rule = self.sheerka.om.get(self.EXEC_RULE_ENTRY, rule_id) - if rule is not NotFound: + if rule := self._inner_get_by_id(rule_id): return rule metadata = [("id", rule_id)] @@ -815,3 +816,42 @@ class SheerkaRuleManager(BaseService): else: raise NotImplementedError(r) return res + + def resolve_rule(self, context, obj): + if obj is None: + return None + + elif isinstance(obj, str): + # search by id first + if rule := self._inner_get_by_id(obj): + return rule + + elif isinstance(obj, Token) and obj.type == TokenKind.RULE: + # search by id first + if rule := self._inner_get_by_id(obj.value[1]): + return rule + # try via indirection + if (rule_id := self.sheerka.get_from_short_term_memory(context, obj.value[1])) is not NotFound and \ + (rule := self._inner_get_by_id(str(rule_id))) is not None: + return rule + + elif isinstance(obj, Rule): + if obj.metadata.id_is_unresolved: + if (rule_id := self.sheerka.get_from_short_term_memory(context, obj.id)) is not NotFound and \ + (rule := self._inner_get_by_id(str(rule_id))) is not None: + return rule + else: + return obj + + return None + + def _inner_get_by_id(self, rule_id): + rule = self.sheerka.om.get(self.FORMAT_RULE_ENTRY, rule_id) + if rule is not NotFound: + return rule + + rule = self.sheerka.om.get(self.EXEC_RULE_ENTRY, rule_id) + if rule is not NotFound: + return rule + + return None diff --git a/src/core/sheerka/services/sheerka_service.py b/src/core/sheerka/services/sheerka_service.py index e4671e3..5db8096 100644 --- a/src/core/sheerka/services/sheerka_service.py +++ b/src/core/sheerka/services/sheerka_service.py @@ -40,3 +40,9 @@ class BaseService: for prop_name in args: if (value := self.sheerka.load_var(self.NAME, prop_name)) is not NotFound: setattr(self, prop_name, value) + + def store_var(self, context, var_name): + """ + Store/record the value of an attribute + """ + self.sheerka.record_var(context, self.NAME, var_name, getattr(self, var_name)) diff --git a/src/core/utils.py b/src/core/utils.py index 8c46d20..39cdd10 100644 --- a/src/core/utils.py +++ b/src/core/utils.py @@ -5,9 +5,10 @@ import os import pkgutil from copy import deepcopy +from pyparsing import * + from core.global_symbols import CustomType from core.tokenizer import TokenKind, Tokenizer -from pyparsing import * COLORS = { "black", @@ -600,6 +601,13 @@ def flatten_all_children(item, get_children): return inner_get_all_children(item) +def flatten(list_of_lists): + """ + Flatten an list containing other lists + """ + return [item for sublist in list_of_lists for item in sublist] + + def get_text_from_tokens(tokens, custom_switcher=None, tracker=None): """ Create the source code, from the list of token diff --git a/src/evaluators/PythonEvaluator.py b/src/evaluators/PythonEvaluator.py index c0f22b7..4be43c4 100644 --- a/src/evaluators/PythonEvaluator.py +++ b/src/evaluators/PythonEvaluator.py @@ -328,15 +328,14 @@ class PythonEvaluator(OneReturnValueEvaluator): :return: """ if isinstance(name, Rule): - return name + return context.sheerka.resolve_rule(context, name) if isinstance(name, Concept): name = core.builtin_helpers.ensure_evaluated(context, name) return name if isinstance(name, Token) and name.type == TokenKind.RULE: - rule = context.sheerka.get_rule_by_id(name.value[1]) # TODO: need a resolve function for the rules - return rule if isinstance(rule, Rule) else None + return context.sheerka.resolve_rule(context, name) if isinstance(name, tuple): raise Exception() diff --git a/src/evaluators/RuleEvaluator.py b/src/evaluators/RuleEvaluator.py index 49c09a5..e760df4 100644 --- a/src/evaluators/RuleEvaluator.py +++ b/src/evaluators/RuleEvaluator.py @@ -1,6 +1,6 @@ from core.builtin_concepts import BuiltinConcepts, ParserResultConcept from core.global_symbols import NotFound -from core.rule import Rule, ACTION_TYPE_DEFERRED +from core.rule import Rule from evaluators.BaseEvaluator import OneReturnValueEvaluator @@ -34,7 +34,8 @@ class RuleEvaluator(OneReturnValueEvaluator): success = True for r in rules: # Browse the rules to find possible deferred rules - if r.metadata.action_type == ACTION_TYPE_DEFERRED: + # I don't use resolve_rule because I need to have BuiltinConcepts.UNKNOWN_RULE when not found + if r.metadata.id_is_unresolved: rule_id = sheerka.get_from_short_term_memory(context, r.id) rule = sheerka.get_rule_by_id(str(rule_id if rule_id is not NotFound else r.id)) resolved.append(rule) diff --git a/src/parsers/BaseNodeParser.py b/src/parsers/BaseNodeParser.py index 3306dde..6bd13e6 100644 --- a/src/parsers/BaseNodeParser.py +++ b/src/parsers/BaseNodeParser.py @@ -1,12 +1,9 @@ from collections import namedtuple from dataclasses import dataclass from enum import Enum -from typing import Set import core.utils -from core.builtin_concepts import BuiltinConcepts -from core.concept import VARIABLE_PREFIX, Concept, DEFINITION_TYPE_BNF, ConceptParts -from core.global_symbols import NotFound +from core.concept import Concept, ConceptParts from core.rule import Rule from core.tokenizer import TokenKind, Token from parsers.BaseParser import Node, BaseParser, ParsingError @@ -14,17 +11,6 @@ from parsers.BaseParser import Node, BaseParser, ParsingError DEBUG_COMPILED = True -@dataclass -class ChickenAndEggError(Exception): - concepts: Set[str] - - -@dataclass -class NoFirstTokenError(ParsingError): - concept: Concept - key: str - - @dataclass() class LexerNode(Node): start: int # starting index in the tokens list @@ -827,248 +813,10 @@ class BaseNodeParser(BaseParser): :param concepts :return: """ - concepts_by_first_keyword = self.compute_concepts_by_first_token(context, concepts).body - resolved = self.resolve_concepts_by_first_keyword(context, concepts_by_first_keyword).body + from core.sheerka.services.SheerkaConceptManager import SheerkaConceptManager + concepts_by_first_keyword = SheerkaConceptManager.compute_concepts_by_first_token(context, concepts).body + resolved = SheerkaConceptManager.resolve_concepts_by_first_keyword(context, concepts_by_first_keyword).body - context.sheerka.om.put(context.sheerka.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY, + context.sheerka.om.put(SheerkaConceptManager.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY, False, resolved) - - def get_concepts(self, token, to_keep, custom=None, to_map=None, strip_quotes=False): - """ - Tries to find if there are concepts that match the value of the token - Caution: Returns the actual cache, not a copy - :param token: - :param to_keep: predicate to tell if the concept is eligible - :param custom: lambda name -> List[Concepts] that gives extra concepts, according to the name - :param to_map: - :param strip_quotes: Remove quotes from strings - :return: - """ - - if token.type == TokenKind.WHITESPACE: - return None - - if token.type == TokenKind.STRING: - name = token.value[1:-1] if strip_quotes else token.value - else: - name = token.value - - custom_concepts = custom(name) if custom else [] # to get extra concepts using an alternative method - - result = [] - concepts_ids = self.sheerka.om.get(self.sheerka.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY, name) - if concepts_ids is NotFound: - return custom_concepts if custom else None - - for concept_id in concepts_ids: - - concept = self.sheerka.get_by_id(concept_id) - - if not to_keep(concept): - continue - - concept = to_map(concept, self, self.sheerka) if to_map else concept - result.append(concept) - - return core.utils.make_unique(result + custom_concepts, - lambda c: c.concept.id if hasattr(c, "concept") else c.id) - - @staticmethod - def compute_concepts_by_first_token(context, concepts, use_sheerka=False, previous_entries=None): - """ - Create the map describing the first token expected by a concept - :param context: - :param concepts: lists of concepts to parse - :param use_sheerka: if True, update concepts_by_first_keyword from sheerka - :param previous_entries: - :return: - """ - sheerka = context.sheerka - res = sheerka.om.copy(sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY) if use_sheerka else (previous_entries or {}) - for concept in concepts: - keywords = BaseNodeParser.get_first_tokens(sheerka, concept) - - if keywords is None: - # no first token found for a concept ? - return sheerka.ret(sheerka.name, False, NoFirstTokenError(concept, concept.key)) - - for keyword in keywords: - res.setdefault(keyword, []).append(concept.id) - - # 'uniquify' the lists - for k, v in res.items(): - res[k] = core.utils.make_unique(v) - - return sheerka.ret("BaseNodeParser", True, res) - - @staticmethod - def resolve_concepts_by_first_keyword(context, concepts_by_first_keyword, modified_concepts=None): - sheerka = context.sheerka - res = {} - - def get_by_id(c_id): - if modified_concepts and c_id in modified_concepts: - return modified_concepts[c_id] - return sheerka.get_by_id(c_id) - - def resolve_concepts(concept_str): - c_key, c_id = core.utils.unstr_concept(concept_str) - if c_id in already_seen: - return ChickenAndEggError(already_seen) - - already_seen.add(c_id) - - resolved = set() - to_resolve = set() - chicken_and_egg = set() - - concept = get_by_id(c_id) - - if sheerka.isaset(context, concept): - concepts = sheerka.get_set_elements(context, concept) - else: - concepts = [concept] - - for concept in concepts: - BaseNodeParser.ensure_bnf(context, concept) # need to make sure that it cannot fail - keywords = BaseNodeParser.get_first_tokens(sheerka, concept) - for keyword in keywords: - (to_resolve if keyword.startswith("c:|") else resolved).add(keyword) - - for concept_to_resolve_str in to_resolve: - res = resolve_concepts(concept_to_resolve_str) - if isinstance(res, ChickenAndEggError): - chicken_and_egg |= res.concepts - else: - resolved |= res - to_resolve.clear() - - if len(resolved) == 0 and len(chicken_and_egg) > 0: - raise ChickenAndEggError(chicken_and_egg) - else: - return resolved - - for k, v in concepts_by_first_keyword.items(): - if k.startswith("c:|"): - try: - already_seen = set() - resolved_keywords = resolve_concepts(k) - for resolved in resolved_keywords: - res.setdefault(resolved, []).extend(v) - except ChickenAndEggError as ex: - context.log(f"Chicken and egg detected for {k}, concepts={ex.concepts}") - concepts_in_recursion = ex.concepts - # make sure to have all the parents - for parent in v: - concepts_in_recursion.add(parent) - - for concept_id in concepts_in_recursion: - # make sure we keep the longest chain - old = sheerka.chicken_and_eggs.get(concept_id) - if old is NotFound or len(old) < len(ex.concepts): - sheerka.chicken_and_eggs.put(concept_id, concepts_in_recursion) - else: - res.setdefault(k, []).extend(v) - - # 'uniquify' the lists - for k, v in res.items(): - res[k] = core.utils.make_unique(v) - - return sheerka.ret("BaseNodeParser", True, res) - - @staticmethod - def get_referenced_concepts(context, concept_id, already_seen): - """ - Gets all the tokens that may allow to recognize concept concept_id - Basically, it returns all the starting tokens for concept concept_id - CHICKEN_AND_EGG is returned when a circular references are found - :param context: - :param concept_id: - :param already_seen: - :return: - """ - if concept_id in already_seen: - return ChickenAndEggError(already_seen) - - already_seen.add(concept_id) - - resolved = set() - to_resolve = set() - chicken_and_egg = set() - sheerka = context.sheerka - concept = sheerka.get_by_id(concept_id) - - if sheerka.isaset(context, concept): - concepts = sheerka.get_set_elements(context, concept) - else: - concepts = [concept] - - for concept in concepts: - BaseNodeParser.ensure_bnf(context, concept) # need to make sure that it cannot fail - keywords = BaseNodeParser.get_first_tokens(sheerka, concept) - for keyword in keywords: - (to_resolve if keyword.startswith("c:|") else resolved).add(keyword) - - for concept_to_resolve_str in to_resolve: - c_key, c_id = core.utils.unstr_concept(concept_to_resolve_str) - res = BaseNodeParser.get_referenced_concepts(context, c_id, already_seen) - if isinstance(res, ChickenAndEggError): - chicken_and_egg |= res.concepts - else: - resolved |= res - to_resolve.clear() - - if len(resolved) == 0 and len(chicken_and_egg) > 0: - raise ChickenAndEggError(chicken_and_egg) - else: - return resolved - - @staticmethod - def resolve_sya_associativity_and_precedence(context, sya): - pass - - @staticmethod - def get_first_tokens(sheerka, concept): - """ - - :param sheerka: - :param concept: - :return: - """ - if concept.get_bnf(): - from parsers.BnfNodeParser import BnfNodeFirstTokenVisitor - bnf_visitor = BnfNodeFirstTokenVisitor(sheerka) - bnf_visitor.visit(concept.get_bnf()) - return bnf_visitor.first_tokens - else: - keywords = concept.key.split() - for keyword in keywords: - if keyword.startswith(VARIABLE_PREFIX): - continue - - return [keyword] - - return None - - @staticmethod - def ensure_bnf(context, concept, parser_name="BaseNodeParser"): - if concept.get_metadata().definition_type == DEFINITION_TYPE_BNF and not concept.get_bnf(): - from parsers.BnfDefinitionParser import BnfDefinitionParser - regex_parser = BnfDefinitionParser() - desc = f"Resolving BNF '{concept.get_metadata().definition}'" - with context.push(BuiltinConcepts.INIT_BNF, - concept, - who=parser_name, - obj=concept, - desc=desc) as sub_context: - sub_context.add_inputs(parser_input=concept.get_metadata().definition) - bnf_parsing_ret_val = regex_parser.parse(sub_context, concept.get_metadata().definition) - sub_context.add_values(return_values=bnf_parsing_ret_val) - - if not bnf_parsing_ret_val.status: - raise Exception(bnf_parsing_ret_val.value) - - concept.set_bnf(bnf_parsing_ret_val.body.body) - if concept.id: - context.sheerka.get_by_id(concept.id).set_bnf(concept.get_bnf()) # update bnf in cache diff --git a/src/parsers/BnfNodeParser.py b/src/parsers/BnfNodeParser.py index d50d430..4d042b2 100644 --- a/src/parsers/BnfNodeParser.py +++ b/src/parsers/BnfNodeParser.py @@ -1291,7 +1291,7 @@ class BnfNodeParser(BaseNodeParser): debugger.debug_log(debug_prefix + ", all parsers are locked. Nothing to do.") continue - concepts = self.get_concepts(token, self._is_eligible, strip_quotes=False) + concepts = context.sheerka.get_concepts_by_first_token(token, self._is_eligible, strip_quotes=False) if not concepts: if debugger.is_enabled(): @@ -1475,7 +1475,7 @@ class BnfNodeParser(BaseNodeParser): desc = f"Resolve concept parsing expression for '{concept}'. {key_to_use=}" with context.push(BuiltinConcepts.INIT_BNF, concept, who=self.name, obj=concept, desc=desc) as sub_context: if not concept.get_bnf(): # 'if' is done outside to save a function call. Not sure it worth it. - BaseNodeParser.ensure_bnf(sub_context, concept, self.name) + core.builtin_helpers.ensure_bnf(sub_context, concept, self.name) grammar[key_to_use] = UnderConstruction(concept.id) diff --git a/src/parsers/RuleParser.py b/src/parsers/RuleParser.py index e3ee2c7..12bb114 100644 --- a/src/parsers/RuleParser.py +++ b/src/parsers/RuleParser.py @@ -1,5 +1,5 @@ from core.builtin_concepts import BuiltinConcepts -from core.rule import Rule, ACTION_TYPE_DEFERRED +from core.rule import Rule from core.sheerka.services.SheerkaExecute import ParserInput from core.tokenizer import TokenKind from parsers.BaseParser import BaseParser, ParsingError, UnexpectedTokenParsingError @@ -70,7 +70,7 @@ class RuleParser(BaseParser): rule = sheerka.get_rule_by_id(token.value[1]) else: rule = Rule().set_id(token.value[1]) - rule.metadata.action_type = ACTION_TYPE_DEFERRED + rule.metadata.id_is_unresolved = True if sheerka.isinstance(rule, BuiltinConcepts.UNKNOWN_RULE): return sheerka.ret(self.name, diff --git a/src/parsers/SequenceNodeParser.py b/src/parsers/SequenceNodeParser.py index fe47d37..c6563d1 100644 --- a/src/parsers/SequenceNodeParser.py +++ b/src/parsers/SequenceNodeParser.py @@ -250,7 +250,7 @@ class SequenceNodeParser(BaseNodeParser): return a if isinstance(a, list) else [a] concepts_by_name = as_list(self.sheerka.resolve(token)) - concepts_by_first_keyword = new_instances(super().get_concepts(token, self._is_eligible)) + concepts_by_first_keyword = new_instances(self.sheerka.get_concepts_by_first_token(token, self._is_eligible)) if concepts_by_name is None: return concepts_by_first_keyword diff --git a/src/parsers/SyaNodeParser.py b/src/parsers/SyaNodeParser.py index aeb2173..fcfc72c 100644 --- a/src/parsers/SyaNodeParser.py +++ b/src/parsers/SyaNodeParser.py @@ -136,7 +136,7 @@ class SyaConceptDef: # otherwise, use sheerka # KSI 20210109 otherwise or override ?? if sheerka: - concept_weight = parser.sheerka.get_concepts_weights(BuiltinConcepts.PRECEDENCE, CONCEPT_COMPARISON_CONTEXT) + concept_weight = parser.sheerka.get_weights(BuiltinConcepts.PRECEDENCE, CONCEPT_COMPARISON_CONTEXT) if concept.str_id in concept_weight: sya_concept_def.precedence = concept_weight[concept.str_id] @@ -1210,7 +1210,10 @@ class SyaNodeParser(BaseNodeParser): debugger.debug_log(debug_prefix + f", all parsers are locked") continue - concepts_def = self.get_concepts(token, self._is_eligible, to_map=SyaConceptDef.get_sya_concept_def) + concepts_def = context.sheerka.get_concepts_by_first_token(token, + self._is_eligible, + to_map=SyaConceptDef.get_sya_concept_def, + parser=self) if not concepts_def: if debugger.is_enabled(): debugger.debug_log(debug_prefix + f", no concept found") diff --git a/tests/cache/test_ListCache.py b/tests/cache/test_ListCache.py index 8f1c5ca..10ba695 100644 --- a/tests/cache/test_ListCache.py +++ b/tests/cache/test_ListCache.py @@ -6,7 +6,8 @@ from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka from tests.cache import FakeSdp -class TestListIfNeededCache(TestUsingMemoryBasedSheerka): +class TestListCache(TestUsingMemoryBasedSheerka): + def test_i_can_put_and_retrieve_value_from_list_cache(self): cache = ListCache() @@ -61,6 +62,12 @@ class TestListIfNeededCache(TestUsingMemoryBasedSheerka): cache.put("key", "value2", alt_sdp=FakeSdp(get_alt_value=lambda cache_name, key: "xxx")) assert cache.get("key") == ["value1", "value2"] + def test_i_can_get_when_alt_sdp(self): + cache = ListCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name") + + cache.get("key", alt_sdp=FakeSdp(get_alt_value=lambda cache_name, key: ["value1"])) + assert cache.get("key") == ["value1"] + def test_i_can_update_from_list_cache(self): cache = ListCache() diff --git a/tests/core/test_SheerkaComparisonManager.py b/tests/core/test_SheerkaComparisonManager.py index f68ebe7..e39059e 100644 --- a/tests/core/test_SheerkaComparisonManager.py +++ b/tests/core/test_SheerkaComparisonManager.py @@ -1,16 +1,35 @@ import pytest + from core.builtin_concepts import BuiltinConcepts from core.concept import Concept from core.global_symbols import EVENT_CONCEPT_PRECEDENCE_MODIFIED, CONCEPT_COMPARISON_CONTEXT, \ - EVENT_RULE_PRECEDENCE_MODIFIED, \ - RULE_COMPARISON_CONTEXT + EVENT_RULE_PRECEDENCE_MODIFIED, RULE_COMPARISON_CONTEXT from core.sheerka.services.SheerkaComparisonManager import SheerkaComparisonManager, ComparisonObj - +from core.sheerka.services.SheerkaConceptManager import ChickenAndEggError from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka class TestSheerkaGreaterThanManager(TestUsingMemoryBasedSheerka): + @staticmethod + def get_comparison_objs(lst, prop_name="prop_name", context="#"): + for item in lst: + if ">>" in item: + a = item.split(">>")[0] + yield ComparisonObj("id", prop_name, a.strip(), None, ">>", context) + elif "<<" in item: + a = item.split("<<")[0] + yield ComparisonObj("id", prop_name, a.strip(), None, "<<", context) + elif ">" in item: + a, b = item.split(">") + yield ComparisonObj("id", prop_name, a.strip(), b.strip(), ">", context) + elif "<" in item: + a, b = item.split("<") + yield ComparisonObj("id", prop_name, a.strip(), b.strip(), "<", context) + else: + a, b = item.split("=") + yield ComparisonObj("id", prop_name, a.strip(), b.strip(), "=", context) + @staticmethod def execution_definition(context, service, concepts_map, definition): if ">>" in definition: @@ -214,13 +233,13 @@ class TestSheerkaGreaterThanManager(TestUsingMemoryBasedSheerka): for entry in entries: self.execution_definition(context, service, concepts_map, entry) - assert service.get_concepts_weights("prop_name") == expected + assert service.get_weights("prop_name") == expected def test_i_can_get_concept_weight_when_no_comparison_is_defined(self): sheerka, context = self.init_concepts() service = sheerka.services[SheerkaComparisonManager.NAME] - assert service.get_concepts_weights("prop_name") == {} + assert service.get_weights("prop_name") == {} def test_i_can_recover_from_deleted_weight(self): sheerka, context, one = self.init_concepts("one") @@ -229,7 +248,7 @@ class TestSheerkaGreaterThanManager(TestUsingMemoryBasedSheerka): service.set_is_lesser(context, "prop_name", one) sheerka.om.clear(service.RESOLVED_COMPARISON_ENTRY) - assert service.get_concepts_weights("prop_name") == {"c:one|1001:": 0} + assert service.get_weights("prop_name") == {"c:one|1001:": 0} def test_i_can_get_partition(self): sheerka, context, one, two, three = self.init_concepts("one", "two", "three") @@ -246,6 +265,38 @@ class TestSheerkaGreaterThanManager(TestUsingMemoryBasedSheerka): 3: ["c:three|1003:"], } + def test_i_can_toposort(self): + # using sample test from https://code.activestate.com/recipes/578272-topological-sort/ + comparison_objs = self.get_comparison_objs([ + "2 < 11", + "11 > 9", + "9 < 8", + "10 < 11", + "3 > 10", + "11 < 7", + "11 < 5", + "7 > 8", + "3 > 8"]) + assert list(SheerkaComparisonManager.toposort(comparison_objs)) == [{'2', '9', '10'}, + {'8', '11'}, + {'3', '5', '7'}] + + def test_i_can_toposort_when_no_data(self): + assert list(SheerkaComparisonManager.toposort([])) == [] + + def test_i_cannot_toposort_when_cycle(self): + comparison_objs = self.get_comparison_objs([ + "1 < 2", + "2 < 3", + "3 < 1", + "1 < 4", # no issue with this + "2 < 5", # no issue with this + ]) + with pytest.raises(ChickenAndEggError) as ex: + list(SheerkaComparisonManager.toposort(comparison_objs)) + + assert ex.value.concepts == {"1", "2", "3"} + def test_i_can_detect_chicken_and_egg(self): sheerka, context, one, two = self.init_concepts("one", "two") service = sheerka.services[SheerkaComparisonManager.NAME] @@ -255,7 +306,7 @@ class TestSheerkaGreaterThanManager(TestUsingMemoryBasedSheerka): assert not res.status assert sheerka.isinstance(res.body, BuiltinConcepts.CHICKEN_AND_EGG) - assert set(res.body.body) == {one, two} + assert set(res.body.body) == {one.str_id, two.str_id} def test_i_can_detect_more_complex_chicken_and_egg(self): sheerka, context, one, two, three, four, five = self.init_concepts("one", "two", "three", "four", "five") @@ -270,7 +321,7 @@ class TestSheerkaGreaterThanManager(TestUsingMemoryBasedSheerka): assert not res.status assert sheerka.isinstance(res.body, BuiltinConcepts.CHICKEN_AND_EGG) - assert set(res.body.body) == {one, two, five} + assert set(res.body.body) == {one.str_id, two.str_id, five.str_id} def test_i_can_give_the_same_information_in_many_ways(self): sheerka, context, one, two = self.init_concepts("one", "two") @@ -292,10 +343,10 @@ class TestSheerkaGreaterThanManager(TestUsingMemoryBasedSheerka): service = sheerka.services[SheerkaComparisonManager.NAME] service.set_is_lesser(context, "prop_name", one) - assert service.get_concepts_weights("prop_name") == {"c:one|1001:": 0} # DEFAULT_COMPARISON_VALUE - 1 + assert service.get_weights("prop_name") == {"c:one|1001:": 0} # DEFAULT_COMPARISON_VALUE - 1 sheerka.set_is_greater_than(context, "prop_name", three, two) - assert service.get_concepts_weights("prop_name") == {"c:one|1001:": 0, "c:two|1002:": 1, "c:three|1003:": 2} + assert service.get_weights("prop_name") == {"c:one|1001:": 0, "c:two|1002:": 1, "c:three|1003:": 2} # I can commit sheerka.om.commit(context) @@ -315,7 +366,7 @@ class TestSheerkaGreaterThanManager(TestUsingMemoryBasedSheerka): sheerka.set_is_less_than(context, "prop_name", less_l, lesser) sheerka.set_is_greater_than(context, "prop_name", less_l, even_more_l) - assert service.get_concepts_weights("prop_name") == {"c:lesser|1001:": 0, + assert service.get_weights("prop_name") == {"c:lesser|1001:": 0, "c:less_l|1002:": -1, "c:even_less_l|1003:": -2} @@ -325,46 +376,46 @@ class TestSheerkaGreaterThanManager(TestUsingMemoryBasedSheerka): unless minus_one is defined as lesser itself :return: """ - sheerka, context, lesser, foo = self.init_concepts("lesser", "foo") + sheerka, context, lesser, foo, bar = self.init_concepts("lesser", "foo", "bar") service = sheerka.services[SheerkaComparisonManager.NAME] service.set_is_lesser(context, "prop_name", lesser) - res = sheerka.set_is_less_than(context, "prop_name", foo, lesser) - assert not res.status - assert sheerka.isinstance(res.body, BuiltinConcepts.INVALID_LESSER_OPERATION) - + # sanity check res = sheerka.set_is_greater_than(context, "prop_name", foo, lesser) + assert res.status + + res = sheerka.set_is_less_than(context, "prop_name", bar, lesser) assert not res.status - assert sheerka.isinstance(res.body, BuiltinConcepts.INVALID_LESSER_OPERATION) + assert sheerka.isinstance(res.body, BuiltinConcepts.IS_LESSER_CONSTRAINT_ERROR) def test_a_greatest_concept_has_the_highest_weight(self): sheerka, context, one, two, three = self.init_concepts("one", "two", "three") service = sheerka.services[SheerkaComparisonManager.NAME] service.set_is_greatest(context, "prop_name", three) - assert service.get_concepts_weights("prop_name") == {"c:three|1003:": 2} # DEFAULT_COMPARISON_VALUE + 1 + assert service.get_weights("prop_name") == {"c:three|1003:": 2} # DEFAULT_COMPARISON_VALUE + 1 sheerka.set_is_greater_than(context, "prop_name", two, one) - assert service.get_concepts_weights("prop_name") == {"c:one|1001:": 1, "c:two|1002:": 2, "c:three|1003:": 3} + assert service.get_weights("prop_name") == {"c:one|1001:": 1, "c:two|1002:": 2, "c:three|1003:": 3} def test_i_cannot_define_greater_than_a_greatest_if_not_a_greater_itself(self): - sheerka, context, greatest, foo = self.init_concepts("greatest", "foo") + sheerka, context, greatest, foo, bar = self.init_concepts("greatest", "foo", "bar") service = sheerka.services[SheerkaComparisonManager.NAME] service.set_is_greatest(context, "prop_name", greatest) + # sanity check res = sheerka.set_is_less_than(context, "prop_name", foo, greatest) - assert not res.status - assert sheerka.isinstance(res.body, BuiltinConcepts.INVALID_GREATEST_OPERATION) + assert res.status - res = sheerka.set_is_greater_than(context, "prop_name", foo, greatest) + res = sheerka.set_is_greater_than(context, "prop_name", bar, greatest) assert not res.status - assert sheerka.isinstance(res.body, BuiltinConcepts.INVALID_GREATEST_OPERATION) + assert sheerka.isinstance(res.body, BuiltinConcepts.IS_GREATEST_CONSTRAINT_ERROR) @pytest.mark.parametrize("definitions, expected", [ - (["foo >>", "foo <<"], BuiltinConcepts.INVALID_GREATEST_OPERATION), - (["foo <<", "foo >>"], BuiltinConcepts.INVALID_LESSER_OPERATION), + (["foo >>", "foo <<"], BuiltinConcepts.IS_GREATEST_CONSTRAINT_ERROR), + (["foo <<", "foo >>"], BuiltinConcepts.IS_LESSER_CONSTRAINT_ERROR), ]) def test_i_cannot_define_a_concept_as_lesser_and_greatest_at_the_same_time(self, definitions, expected): sheerka, context, foo = self.init_concepts("foo") @@ -386,7 +437,7 @@ class TestSheerkaGreaterThanManager(TestUsingMemoryBasedSheerka): sheerka.set_is_less_than(context, "prop_name", greatest, more_g) sheerka.set_is_greater_than(context, "prop_name", even_more_g, more_g) - assert service.get_concepts_weights("prop_name") == {"c:greatest|1001:": 2, + assert service.get_weights("prop_name") == {"c:greatest|1001:": 2, "c:more_g|1002:": 3, "c:even_more_g|1003:": 4} @@ -405,7 +456,7 @@ class TestSheerkaGreaterThanManager(TestUsingMemoryBasedSheerka): sheerka.set_is_less_than(context, "prop_name", three, four) - assert service.get_concepts_weights("prop_name") == { + assert service.get_weights("prop_name") == { "c:one|1001:": -1, "c:two|1002:": 0, "c:three|1003:": 1, @@ -417,7 +468,7 @@ class TestSheerkaGreaterThanManager(TestUsingMemoryBasedSheerka): sheerka.set_is_less_than(context, "prop_name", three_and_half, four) sheerka.set_is_greater_than(context, "prop_name", three_and_half, three) - assert service.get_concepts_weights("prop_name") == { + assert service.get_weights("prop_name") == { "c:one|1001:": -1, "c:two|1002:": 0, "c:three|1003:": 1, @@ -446,7 +497,7 @@ class TestSheerkaGreaterThanManager(TestUsingMemoryBasedSheerka): service.set_is_greater_than(context, "prop_name", five, one) service.set_is_greater_than(context, "prop_name", five, three) - assert service.get_concepts_weights("prop_name") == { + assert service.get_weights("prop_name") == { "c:zero|1001:": 0, "c:one|1002:": 1, "c:two|1003:": 2, @@ -471,7 +522,7 @@ class TestSheerkaGreaterThanManager(TestUsingMemoryBasedSheerka): res = self.execution_definition(context, service, concepts_map, definition) assert not res.status - assert sheerka.isinstance(res.body, BuiltinConcepts.CONCEPT_ALREADY_DEFINED) + assert sheerka.isinstance(res.body, BuiltinConcepts.COMPARISON_ALREADY_DEFINED) def test_an_event_is_fired_when_modifying_concepts_precedence(self): sheerka, context, one, two, foo = self.init_concepts("one", "two", "foo") @@ -514,3 +565,170 @@ class TestSheerkaGreaterThanManager(TestUsingMemoryBasedSheerka): sheerka.set_is_greater_than(context, BuiltinConcepts.PRECEDENCE, r1, r2, RULE_COMPARISON_CONTEXT) assert event_received + + @pytest.mark.parametrize("entries, expected", [ + ([], (set(), set(), set())), + (["a<<", "a<<", "b<<", "a>", "a>>", "b>>", "a>", "c=d", "eh"], ({"b"}, {"a"}, {"a", "b", "c", "d", "e", "f", "g", "h"})), + ]) + def test_i_can_get_objs_groups(self, entries, expected): + comparison_objs = self.get_comparison_objs(entries) + assert SheerkaComparisonManager.get_objs_groups(comparison_objs) == expected + + @pytest.mark.parametrize("entries, greatest, lowest, expected_tuple", [ + # greatest + (["a < b"], {"a", "b"}, set(), (["a < b"], [], [], [])), + (["a > b"], {"a", "b"}, set(), (["a > b"], [], [], [])), + + # lowest + (["a < b"], set(), {"a", "b"}, ([], ["a < b"], [], [])), + (["a > b"], set(), {"a", "b"}, ([], ["a > b"], [], [])), + + # equiv + (["a = b"], set(), set(), ([], [], ["a = b"], [])), + + # neither + (["a < b"], set(), set(), ([], [], [], ["a < b"])), + (["a > b"], set(), set(), ([], [], [], ["a > b"])), + + # irrelevant information + (["a > b"], {"a"}, set(), ([], [], [], [])), + (["a > b"], set(), {"b"}, ([], [], [], [])), + (["a < b"], {"b"}, set(), ([], [], [], [])), + (["a < b"], set(), {"a"}, ([], [], [], [])), + + # mixed + (["a b"], {"a": 2, "b": 1}), + (["a < b", "b > a"], {"a": 1, "b": 2}), + (["a < b", "b < c"], {"a": 1, "b": 2, "c": 3}), + (["a < b", "a < c"], {"a": 1, "b": 2, "c": 2}), + + # greatest between themselves + (["a < b", "a>>", "b>>"], {"a": 2, "b": 3}), + (["a > b", "a>>", "b>>"], {"a": 3, "b": 2}), + (["a < b", "b > a", "a>>", "b>>"], {"a": 2, "b": 3}), + (["a < b", "b < c", "a>>", "b>>", "c>>"], {"a": 2, "b": 3, "c": 4}), + (["a < b", "a < c", "a>>", "b>>", "c>>"], {"a": 2, "b": 3, "c": 3}), + + # lowest between themselves + (["a < b", "a<<", "b<<"], {"a": -1, "b": 0}), + (["a > b", "a<<", "b<<"], {"a": 0, "b": -1}), + (["a < b", "b > a", "a<<", "b<<"], {"a": -1, "b": 0}), + (["a < b", "b < c", "a<<", "b<<", "c<<"], {"a": -2, "b": -1, "c": 0}), + (["a < b", "a < c", "a<<", "b<<", "c<<"], {"a": -1, "b": 0, "c": 0}), + + # greatest that does not appear in other relations + (["a >>", "b < c"], {"a": 3, "b": 1, "c": 2}), + (["a >>", "a > b"], {"a": 2, "b": 1}), + (["a >>", "a > b", "c > d", "d > e"], {"a": 4, "b": 1, "c": 3, "d": 2, "e": 1}), + + # lowest that does not appear in other relations + (["a <<", "b < c"], {"a": 0, "b": 1, "c": 2}), + (["a <<", "a < b"], {"a": 0, "b": 1}), + (["a <<", "a < b", "c < d", "d < e"], {"a": 0, "b": 1, "c": 1, "d": 2, "e": 3}), + (["z <<", "a<<", "b<<", "c<<", "a < b", "b < c"], {"a": -2, "b": -1, "c": 0, "z": -3}), + + # eq + (["a = b"], {"a": 1, "b": 1}), + (["a = b", "b > c"], {"a": 2, "b": 2, "c": 1}), + (["a = z", "z>>", "b d"], {"a": 2, "b": 2, "c": 2, "d": 1}), + (["a = b", "b = c", "a > d"], {"a": 2, "b": 2, "c": 2, "d": 1}), + #(["a = b", "b = c", "c > d"], {"a": 2, "b": 2, "c": 2, "d": 1}), # not working + + # mix greatest and lesser + (["a >>", "b<<", "a > b"], {"a": 2, "b": 0}), + ]) + def test_i_can_compute_weight_new(self, entries, expected): + comparison_objs = list(self.get_comparison_objs(entries)) + assert SheerkaComparisonManager.compute_weights(comparison_objs) == expected + + @pytest.mark.parametrize("previous_entries, new_entry, items_in_cycle", [ + (["a > b"], "b > a", {"a", "b"}), + (["a > b", "c > d"], "b > a", {"a", "b"}), + (["a < b", "b < c"], "c < a", {"a", "b", "c"}), + ]) + def test_validate_new_entry_i_can_detect_cycle(self, previous_entries, new_entry, items_in_cycle): + sheerka, context = self.init_test().unpack() + + new_co = list(self.get_comparison_objs([new_entry]))[0] + previous_comparison_objs = list(self.get_comparison_objs(previous_entries)) + + res = SheerkaComparisonManager.validate_new_entry(context, new_co, previous_comparison_objs) + + assert not res.status + assert sheerka.isinstance(res.body, BuiltinConcepts.CHICKEN_AND_EGG) + assert set(res.body.body) == items_in_cycle + + @pytest.mark.parametrize("previous_entries, new_entry", [ + (["a > b"], "a > b"), + (["a < b"], "a < b"), + (["a = b"], "a = b"), + (["a <<"], "a <<"), + (["a >>"], "a >>"), + ]) + def test_validate_new_entry_i_can_detect_duplicate_entries(self, previous_entries, new_entry): + sheerka, context = self.init_test().unpack() + + new_co = list(self.get_comparison_objs([new_entry]))[0] + previous_comparison_objs = list(self.get_comparison_objs(previous_entries)) + + res = SheerkaComparisonManager.validate_new_entry(context, new_co, previous_comparison_objs) + + assert not res.status + assert sheerka.isinstance(res.body, BuiltinConcepts.COMPARISON_ALREADY_DEFINED) + + @pytest.mark.parametrize("previous_entries, new_entry", [ + (["a>>"], "a<<"), + (["a>>"], "a < b"), + (["a>>"], "b > a"), + ]) + def test_validate_new_entry_i_can_detect_is_greatest_constraint_error(self, previous_entries, new_entry): + sheerka, context = self.init_test().unpack() + + new_co = list(self.get_comparison_objs([new_entry]))[0] + previous_comparison_objs = list(self.get_comparison_objs(previous_entries)) + + res = SheerkaComparisonManager.validate_new_entry(context, new_co, previous_comparison_objs) + + assert not res.status + assert sheerka.isinstance(res.body, BuiltinConcepts.IS_GREATEST_CONSTRAINT_ERROR) + + @pytest.mark.parametrize("previous_entries, new_entry", [ + (["a<<"], "a>>"), + (["a<<"], "a > b"), + (["a<<"], "b < a"), + ]) + def test_validate_new_entry_i_can_detect_is_lesser_constraint_error(self, previous_entries, new_entry): + sheerka, context = self.init_test().unpack() + + new_co = list(self.get_comparison_objs([new_entry]))[0] + previous_comparison_objs = list(self.get_comparison_objs(previous_entries)) + + res = SheerkaComparisonManager.validate_new_entry(context, new_co, previous_comparison_objs) + + assert not res.status + assert sheerka.isinstance(res.body, BuiltinConcepts.IS_LESSER_CONSTRAINT_ERROR) diff --git a/tests/core/test_SheerkaConceptManager.py b/tests/core/test_SheerkaConceptManager.py index e5feafb..e313b1e 100644 --- a/tests/core/test_SheerkaConceptManager.py +++ b/tests/core/test_SheerkaConceptManager.py @@ -1,18 +1,20 @@ import pytest + from cache.CacheManager import ConceptNotFound from core.builtin_concepts import BuiltinConcepts +from core.builtin_helpers import ensure_bnf from core.concept import PROPERTIES_TO_SERIALIZE, Concept, DEFINITION_TYPE_DEF, get_concept_attrs, \ DEFINITION_TYPE_BNF from core.global_symbols import NotInit, NotFound -from core.sheerka.Sheerka import Sheerka from core.sheerka.services.SheerkaConceptManager import SheerkaConceptManager, NoModificationFound, ForbiddenAttribute, \ - UnknownAttribute, CannotRemoveMeta, ValueNotFound, ConceptIsReferenced -from parsers.BaseNodeParser import BaseNodeParser -from parsers.BnfNodeParser import Sequence, StrMatch, ConceptExpression - + UnknownAttribute, CannotRemoveMeta, ValueNotFound, ConceptIsReferenced, NoFirstTokenError +from parsers.BnfNodeParser import Sequence, StrMatch, ConceptExpression, OrderedChoice, Optional, ZeroOrMore, OneOrMore from tests.TestUsingFileBasedSheerka import TestUsingFileBasedSheerka from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka +compute_concepts_by_first_token = SheerkaConceptManager.compute_concepts_by_first_token +resolve_concepts_by_first_keyword = SheerkaConceptManager.resolve_concepts_by_first_keyword + class TestSheerkaConceptManager(TestUsingMemoryBasedSheerka): def test_i_can_create_a_concept(self): @@ -48,15 +50,15 @@ class TestSheerkaConceptManager(TestUsingMemoryBasedSheerka): assert sheerka.get_by_hash(concept.get_definition_hash()) == concept # I can get by the first entry - assert sheerka.om.get(sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, "+") == [concept.id] - assert sheerka.om.get(sheerka.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY, "+") == [concept.id] + assert sheerka.om.get(service.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, "+") == [concept.id] + assert sheerka.om.get(service.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY, "+") == [concept.id] # saved in sdp assert sheerka.om.current_sdp().exists(service.CONCEPTS_BY_ID_ENTRY, concept.id) assert sheerka.om.current_sdp().exists(service.CONCEPTS_BY_KEY_ENTRY, concept.key) assert sheerka.om.current_sdp().exists(service.CONCEPTS_BY_NAME_ENTRY, concept.name) assert sheerka.om.current_sdp().exists(service.CONCEPTS_BY_HASH_ENTRY, concept.get_definition_hash()) - assert sheerka.om.current_sdp().exists(sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, "+") + assert sheerka.om.current_sdp().exists(service.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, "+") def test_i_cannot_create_a_bnf_concept_that_references_a_concept_that_cannot_be_resolved(self): sheerka, context, one_1, one_1_0 = self.init_concepts(Concept("one", body="1"), Concept("one", body="1.0")) @@ -104,7 +106,7 @@ class TestSheerkaConceptManager(TestUsingMemoryBasedSheerka): assert sheerka.om.current_sdp().exists(service.CONCEPTS_BY_KEY_ENTRY, concept.key) assert sheerka.om.current_sdp().exists(service.CONCEPTS_BY_NAME_ENTRY, concept.name) assert sheerka.om.current_sdp().exists(service.CONCEPTS_BY_HASH_ENTRY, concept.get_definition_hash()) - assert sheerka.om.current_sdp().exists(sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, "hello") + assert sheerka.om.current_sdp().exists(service.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, "hello") def test_i_cannot_add_the_same_concept_twice(self): """ @@ -185,8 +187,8 @@ class TestSheerkaConceptManager(TestUsingMemoryBasedSheerka): assert res.status # I can get by the first entry - assert sheerka.om.get(sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, "-") == [concept.id] - assert sheerka.om.get(sheerka.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY, "-") == [concept.id] + assert sheerka.om.get(SheerkaConceptManager.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, "-") == [concept.id] + assert sheerka.om.get(SheerkaConceptManager.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY, "-") == [concept.id] @pytest.mark.parametrize("expression", [ "--'filter' ('one' | 'two') ", @@ -198,8 +200,8 @@ class TestSheerkaConceptManager(TestUsingMemoryBasedSheerka): create_new=True).unpack() # I can get by the first entry - assert sheerka.om.get(sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, "-") == [bnf_concept.id] - assert sheerka.om.get(sheerka.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY, "-") == [bnf_concept.id] + assert sheerka.om.get(SheerkaConceptManager.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, "-") == [bnf_concept.id] + assert sheerka.om.get(SheerkaConceptManager.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY, "-") == [bnf_concept.id] def test_concept_references_are_updated_1(self): sheerka, context, one, two, number, twenty, twenties = self.init_test().with_concepts( @@ -494,11 +496,11 @@ class TestSheerkaConceptManager(TestUsingMemoryBasedSheerka): Concept("baz", definition="foo"), create_new=True).unpack() - assert sheerka.om.copy(sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == { + assert sheerka.om.copy(SheerkaConceptManager.CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == { "foo": ["1001"], "bar": ["1002"], 'c:|1001:': ['1003']} - assert sheerka.om.copy(sheerka.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == { + assert sheerka.om.copy(SheerkaConceptManager.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == { 'foo': ['1001', '1003'], 'bar': ['1002']} @@ -506,10 +508,10 @@ class TestSheerkaConceptManager(TestUsingMemoryBasedSheerka): res = sheerka.modify_concept(context, foo, to_add) assert res.status - assert sheerka.om.copy(sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == { + assert sheerka.om.copy(SheerkaConceptManager.CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == { "bar": ["1002", "1001"], 'c:|1001:': ['1003']} - assert sheerka.om.copy(sheerka.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == { + assert sheerka.om.copy(SheerkaConceptManager.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == { 'bar': ['1002', '1001', '1003']} def test_references_are_updated_after_concept_modification(self): @@ -531,7 +533,7 @@ class TestSheerkaConceptManager(TestUsingMemoryBasedSheerka): assert twenty_one.get_metadata().definition == "'twenty' one" assert twenty_one.get_bnf() is None - BaseNodeParser.ensure_bnf(context, twenty_one) + ensure_bnf(context, twenty_one) assert twenty_one.get_bnf() == Sequence(StrMatch('twenty'), ConceptExpression(modified, rule_name='one')) def test_i_can_modify_on_top_of_a_new_ontology_layer(self): @@ -730,6 +732,258 @@ class TestSheerkaConceptManager(TestUsingMemoryBasedSheerka): assert res.status assert sheerka.isinstance(res.body, BuiltinConcepts.NEW_CONCEPT) + @pytest.mark.parametrize("concept, expected", [ + (Concept("foo"), {"foo": ["1001"]}), + (Concept("foo a").def_var("a"), {"foo": ["1001"]}), + (Concept("a b foo").def_var("a").def_var("b"), {"foo": ["1001"]}), + ]) + def test_i_can_get_concepts_by_first_keyword(self, concept, expected): + """ + Given a concept, i can find the first know token + example: + Concept("a foo b").def_var("a").def_var("b") + 'a' and 'b' are properties + the first 'real' token is foo + :return: + """ + + sheerka, context, *updated = self.init_concepts(concept) + + res = SheerkaConceptManager.compute_concepts_by_first_token(context, updated) + + assert res.status + assert res.body == expected + + @pytest.mark.parametrize("bnf, expected", [ + (StrMatch("foo"), {"foo": ["1002"]}), + (StrMatch("bar"), {"bar": ["1002"]}), + (ConceptExpression("bar"), {"c:|1001:": ["1002"]}), + (Sequence(StrMatch("foo"), StrMatch("bar")), {"foo": ["1002"]}), + (Sequence(StrMatch("foo"), ConceptExpression("bar")), {"foo": ["1002"]}), + (Sequence(ConceptExpression("bar"), StrMatch("foo")), {"c:|1001:": ["1002"]}), + (OrderedChoice(StrMatch("foo"), StrMatch("bar")), {"foo": ["1002"], "bar": ["1002"]}), + (Optional(StrMatch("foo")), {"foo": ["1002"]}), + (ZeroOrMore(StrMatch("foo")), {"foo": ["1002"]}), + (OneOrMore(StrMatch("foo")), {"foo": ["1002"]}), + (StrMatch("--filter"), {"--filter": ["1002"]}), # add both entries + ]) + def test_i_can_get_concepts_by_first_keyword_with_bnf(self, bnf, expected): + sheerka, context = self.init_test().unpack() + + bar = Concept("bar").init_key() + sheerka.set_id_if_needed(bar, False) + sheerka.test_only_add_in_cache(bar) + + concept = Concept("foo").init_key() + concept.set_bnf(bnf) + sheerka.set_id_if_needed(concept, False) + + res = compute_concepts_by_first_token(context, [concept]) + + assert res.status + assert res.body == expected + + def test_i_can_get_concepts_by_first_keyword_when_multiple_concepts(self): + sheerka = self.get_sheerka() + context = self.get_context(sheerka) + + bar = Concept("bar").init_key() + sheerka.set_id_if_needed(bar, False) + sheerka.test_only_add_in_cache(bar) + + baz = Concept("baz").init_key() + sheerka.set_id_if_needed(baz, False) + sheerka.test_only_add_in_cache(baz) + + foo = Concept("foo").init_key() + foo.set_bnf(OrderedChoice(ConceptExpression("bar"), ConceptExpression("baz"), StrMatch("qux"))) + sheerka.set_id_if_needed(foo, False) + + res = compute_concepts_by_first_token(context, [bar, baz, foo]) + + assert res.status + assert res.body == { + "bar": ["1001"], + "baz": ["1002"], + "c:|1001:": ["1003"], + "c:|1002:": ["1003"], + "qux": ["1003"], + } + + def test_i_can_get_concepts_by_first_keyword_using_sheerka(self): + sheerka, context, *updated = self.init_test().with_concepts( + "one", + "two", + Concept("twenty", definition="'twenty' (one|two)"), + create_new=True + ).unpack() + + bar = Concept("bar").init_key() + sheerka.set_id_if_needed(bar, False) + sheerka.test_only_add_in_cache(bar) + + foo = Concept("foo").init_key() + foo.set_bnf(OrderedChoice(ConceptExpression("one"), ConceptExpression("bar"), StrMatch("qux"))) + sheerka.set_id_if_needed(foo, False) + + res = compute_concepts_by_first_token(context, [bar, foo], use_sheerka=True) + + assert res.status + assert res.body == { + "one": ["1001"], + "two": ["1002"], + "twenty": ["1003"], + "bar": ["1004"], + "c:|1001:": ["1005"], + "c:|1004:": ["1005"], + "qux": ["1005"], + } + + def test_i_cannot_get_concept_by_first_keyword_when_no_first_keyword(self): + sheerka, context, foo = self.init_concepts(Concept("x y", body="x y").def_var("x").def_var("y")) + res = compute_concepts_by_first_token(context, [foo]) + + assert not res.status + assert res.body == NoFirstTokenError(foo, foo.key) + + def test_i_can_resolve_concepts_by_first_keyword(self): + sheerka, context, *updated = self.init_concepts( + "one", + Concept("two", definition="one"), + Concept("three", definition="two")) + + concepts_by_first_keywords = { + "one": ["1001"], + "c:|1001:": ["1002"], + "c:|1002:": ["1003"], + } + + resolved_ret_val = resolve_concepts_by_first_keyword(context, concepts_by_first_keywords) + + assert resolved_ret_val.status + assert resolved_ret_val.body == { + "one": ["1001", "1002", "1003"], + } + + def test_i_can_resolve_when_concepts_are_sets(self): + sheerka, context, number, *concepts = self.init_concepts( + "number", + "one", + "two", + "twenty", + "hundred", + Concept("twenties", definition="twenty number"), + Concept("hundreds", definition="number hundred"), + ) + + sheerka.set_isa(context, sheerka.new("one"), number) + sheerka.set_isa(context, sheerka.new("two"), number) + sheerka.set_isa(context, sheerka.new("twenty"), number) + sheerka.set_isa(context, sheerka.new("thirty"), number) + sheerka.set_isa(context, sheerka.new("hundred"), number) + sheerka.set_isa(context, sheerka.new("twenties"), number) + sheerka.set_isa(context, sheerka.new("hundreds"), number) + + sheerka.concepts_grammars.clear() # reset all the grammar to simulate Sheerka restart + + # cbft : concept_by_first_token (I usually don't use abbreviation) + cbft = compute_concepts_by_first_token(context, [number] + concepts).body + resolved_ret_val = resolve_concepts_by_first_keyword(context, cbft) + + assert resolved_ret_val.status + assert resolved_ret_val.body == { + 'number': ['1001'], + 'one': ['1002', '1007'], + 'two': ['1003', '1007'], + 'twenty': ['1004', '1006', '1007'], + 'hundred': ['1005', '1007'], + } + + def test_concepts_are_defined_once(self): + sheerka = self.get_sheerka() + context = self.get_context(sheerka) + good = self.create_and_add_in_cache_concept(sheerka, "good") + foo = self.create_and_add_in_cache_concept(sheerka, "foo", bnf=ConceptExpression("good")) + bar = self.create_and_add_in_cache_concept(sheerka, "bar", bnf=ConceptExpression("good")) + baz = self.create_and_add_in_cache_concept(sheerka, "baz", bnf=OrderedChoice( + ConceptExpression("foo"), + ConceptExpression("bar"))) + + concepts_by_first_keywords = compute_concepts_by_first_token(context, [good, foo, bar, baz]).body + + resolved_ret_val = resolve_concepts_by_first_keyword(context, concepts_by_first_keywords) + assert resolved_ret_val.status + assert resolved_ret_val.body == { + "good": ["1001", "1002", "1003", "1004"], + } + + def test_i_can_resolve_more_complex(self): + sheerka = self.get_sheerka() + context = self.get_context(sheerka) + + a = self.create_and_add_in_cache_concept(sheerka, "a", bnf=Sequence("one", "two")) + b = self.create_and_add_in_cache_concept(sheerka, "b", bnf=Sequence(ConceptExpression("a"), "two")) + + concepts_by_first_keywords = compute_concepts_by_first_token(context, [a, b]).body + + resolved_ret_val = resolve_concepts_by_first_keyword(context, concepts_by_first_keywords) + assert resolved_ret_val.status + assert resolved_ret_val.body == { + "one": ["1001", "1002"], + } + + def tests_i_can_detect_direct_recursion(self): + sheerka, context, good, foo, bar = self.init_concepts( + "good", + self.bnf_concept("foo", ConceptExpression("bar")), + self.bnf_concept("bar", ConceptExpression("foo")), + ) + + concepts_by_first_keywords = compute_concepts_by_first_token(context, [good, foo, bar]).body + resolved_ret_val = resolve_concepts_by_first_keyword(context, concepts_by_first_keywords) + assert resolved_ret_val.status + assert resolved_ret_val.body == { + "good": ["1001"], + } + assert sheerka.chicken_and_eggs.get(foo.id) == {foo.id, bar.id} + assert sheerka.chicken_and_eggs.get(bar.id) == {foo.id, bar.id} + + def test_i_can_detect_indirect_infinite_recursion(self): + sheerka, context, good, one, two, three = self.init_concepts( + "good", + self.bnf_concept("one", ConceptExpression("two")), + self.bnf_concept("two", ConceptExpression("three")), + self.bnf_concept("three", ConceptExpression("two")), + ) + + concepts_by_first_keywords = compute_concepts_by_first_token(context, [good, one, two, three]).body + resolved_ret_val = resolve_concepts_by_first_keyword(context, concepts_by_first_keywords) + assert resolved_ret_val.status + assert resolved_ret_val.body == { + "good": ["1001"], + } + assert sheerka.chicken_and_eggs.get(one.id) == {one.id, two.id, three.id} + assert sheerka.chicken_and_eggs.get(two.id) == {one.id, two.id, three.id} + assert sheerka.chicken_and_eggs.get(three.id) == {one.id, two.id, three.id} + + def test_i_can_detect_the_longest_infinite_recursion_chain(self): + sheerka, context, good, one, two, three = self.init_concepts( + "good", + self.bnf_concept("two", ConceptExpression("three")), + self.bnf_concept("three", ConceptExpression("two")), + self.bnf_concept("one", ConceptExpression("three")), + ) + + concepts_by_first_keywords = compute_concepts_by_first_token(context, [good, one, two, three]).body + resolved_ret_val = resolve_concepts_by_first_keyword(context, concepts_by_first_keywords) + assert resolved_ret_val.status + assert resolved_ret_val.body == { + "good": ["1001"], + } + assert sheerka.chicken_and_eggs.get(one.id) == {one.id, two.id, three.id} + assert sheerka.chicken_and_eggs.get(two.id) == {one.id, two.id, three.id} + assert sheerka.chicken_and_eggs.get(three.id) == {one.id, two.id, three.id} + class TestSheerkaConceptManagerUsingFileBasedSheerka(TestUsingFileBasedSheerka): def test_i_can_add_several_concepts(self): @@ -759,8 +1013,8 @@ class TestSheerkaConceptManagerUsingFileBasedSheerka(TestUsingFileBasedSheerka): assert sheerka.om.current_sdp().exists(service.CONCEPTS_BY_HASH_ENTRY, hello.get_definition_hash()) assert sheerka.om.current_sdp().exists(service.CONCEPTS_BY_HASH_ENTRY, greeting.get_definition_hash()) - assert sheerka.om.current_sdp().exists(Sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, "Hello") - assert sheerka.om.current_sdp().exists(Sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, "Greeting") + assert sheerka.om.current_sdp().exists(service.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, "Hello") + assert sheerka.om.current_sdp().exists(service.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, "Greeting") def test_i_cannot_add_the_same_concept_twice_using_sdp(self): """ diff --git a/tests/core/test_SheerkaDebugManager.py b/tests/core/test_SheerkaDebugManager.py index 59f616a..5465d41 100644 --- a/tests/core/test_SheerkaDebugManager.py +++ b/tests/core/test_SheerkaDebugManager.py @@ -973,3 +973,4 @@ class TestSheerkaDebugManager(TestUsingMemoryBasedSheerka): assert service.debug_vars_settings == [DebugItem("v_name", "v_service", "v_method", 1, True, 1, False, True)] assert service.debug_rules_settings == [DebugItem("r_name", "r_service", "r_method", 2, True, 2, False, True)] assert service.debug_concepts_settings == [DebugItem("c_name", "c_serv", "c_method", 3, True, 3, False, True)] + diff --git a/tests/core/test_SheerkaMemory.py b/tests/core/test_SheerkaMemory.py index 0b7400a..afbf07e 100644 --- a/tests/core/test_SheerkaMemory.py +++ b/tests/core/test_SheerkaMemory.py @@ -88,17 +88,6 @@ class TestSheerkaMemory(TestUsingMemoryBasedSheerka): assert sheerka.om.copy(SheerkaMemory.OBJECTS_ENTRY) == {"a": MemoryObject(context.event.get_digest(), foo)} assert id(sheerka.get_from_memory(context, "a").obj) == id(foo) - def test_i_can_use_memory_to_get_the_list_of_all_objects(self): - sheerka, context = self.init_test(cache_only=False).unpack() - foo = Concept("foo") - bar = Concept("bar") - - sheerka.add_to_memory(context, "foo", 'value that will not appear') - sheerka.add_to_memory(context, "foo", foo) - sheerka.add_to_memory(context, "bar", bar) - sheerka.om.commit(context) - - assert sheerka.memory(context) == {"foo": foo, "bar": bar} def test_i_can_use_memory_with_a_string(self): sheerka, context = self.init_test().unpack() @@ -138,7 +127,7 @@ class TestSheerkaMemory(TestUsingMemoryBasedSheerka): def test_object_are_not_added_in_memory_during_the_initialisation(self): sheerka, context = self.init_test().unpack() - assert len(sheerka.memory(context)) == 0 + assert len(sheerka.om.get_all(SheerkaMemory.OBJECTS_ENTRY)) == 0 class TestSheerkaMemoryUsingFileBase(TestUsingFileBasedSheerka): diff --git a/tests/core/test_SheerkaRuleManager.py b/tests/core/test_SheerkaRuleManager.py index a4197db..57c9d0c 100644 --- a/tests/core/test_SheerkaRuleManager.py +++ b/tests/core/test_SheerkaRuleManager.py @@ -1,9 +1,10 @@ import ast import pytest + from core.builtin_concepts import BuiltinConcepts from core.concept import Concept, CMV -from core.global_symbols import RULE_COMPARISON_CONTEXT, NotFound +from core.global_symbols import RULE_COMPARISON_CONTEXT from core.rule import Rule, ACTION_TYPE_PRINT, ACTION_TYPE_EXEC from core.sheerka.services.SheerkaRuleManager import SheerkaRuleManager, FormatRuleParser, \ FormatAstRawText, FormatAstVariable, FormatAstSequence, FormatAstFunction, \ @@ -11,7 +12,6 @@ from core.sheerka.services.SheerkaRuleManager import SheerkaRuleManager, FormatR from core.tokenizer import Token, TokenKind from parsers.BaseNodeParser import SourceCodeWithConceptNode, SourceCodeNode from parsers.PythonParser import PythonNode - from tests.TestUsingFileBasedSheerka import TestUsingFileBasedSheerka from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka @@ -335,6 +335,41 @@ class TestSheerkaRuleManager(TestUsingMemoryBasedSheerka): assert sheerka.get_rule_by_id(rule_true.id) == rule_true assert not sheerka.is_known(sheerka.get_rule_by_id(rule_false.id)) + def test_i_can_resolve_rule(self): + sheerka, context, rule = self.init_test().with_rules(("my rule", "True", "True")).unpack() + context.add_to_short_term_memory("x", rule.id) + + # direct access by id + assert sheerka.resolve_rule(context, rule.id) == rule + + # direct access by token + assert sheerka.resolve_rule(context, Token(TokenKind.RULE, ("i_do_not_care", rule.id), -1, -1, -1)) == rule + + # indirect access by token + assert sheerka.resolve_rule(context, Token(TokenKind.RULE, ("i_do_not_care", "x"), -1, -1, -1)) == rule + + # Simple returns the rule if id is resolved + assert sheerka.resolve_rule(context, rule) == rule + + # look for the correct rule when id is unresolved + unresolved = Rule(rule_id="x") + unresolved.metadata.id_is_unresolved = True + assert sheerka.resolve_rule(context, unresolved) == rule + + # look for the correct rule when id is unresolved + unresolved = Rule(rule_id="y") + unresolved.metadata.id_is_unresolved = True + assert sheerka.resolve_rule(context, unresolved) is None # no error raised + + # I still can get the value when the indirection has the wrong type + context.add_to_short_term_memory("z", int(rule.id)) + + assert sheerka.resolve_rule(context, Token(TokenKind.RULE, ("i_do_not_care", "z"), -1, -1, -1)) == rule + + unresolved = Rule(rule_id="z") + unresolved.metadata.id_is_unresolved = True + assert sheerka.resolve_rule(context, unresolved) == rule + # @pytest.mark.skip # @pytest.mark.parametrize("text, expected", [ # ("cat is an animal", set()), diff --git a/tests/core/test_sheerka.py b/tests/core/test_sheerka.py index f4af575..ccf9ea5 100644 --- a/tests/core/test_sheerka.py +++ b/tests/core/test_sheerka.py @@ -539,21 +539,21 @@ class TestSheerkaUsingFileBasedSheerka(TestUsingFileBasedSheerka): create_new=True).unpack() sheerka.om.commit(context) - assert sheerka.om.copy(sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == { + assert sheerka.om.copy(SheerkaConceptManager.CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == { 'bar': ['1002'], 'c:|1001:': ['1003'], 'foo': ['1001']} - assert sheerka.om.copy(sheerka.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == { + assert sheerka.om.copy(SheerkaConceptManager.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == { 'bar': ['1002'], 'foo': ['1001', '1003'] } sheerka = self.get_sheerka() # another instance - assert sheerka.om.copy(sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == { + assert sheerka.om.copy(SheerkaConceptManager.CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == { 'bar': ['1002'], 'c:|1001:': ['1003'], 'foo': ['1001']} - assert sheerka.om.copy(sheerka.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == { + assert sheerka.om.copy(SheerkaConceptManager.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == { 'bar': ['1002'], 'foo': ['1001', '1003'] } diff --git a/tests/core/test_sheerkaResultManager.py b/tests/core/test_sheerkaResultManager.py index 447523b..da26e84 100644 --- a/tests/core/test_sheerkaResultManager.py +++ b/tests/core/test_sheerkaResultManager.py @@ -1,12 +1,13 @@ import types import pytest + from core.builtin_concepts import BuiltinConcepts from core.sheerka.ExecutionContext import ExecutionContext from core.sheerka.SheerkaOntologyManager import SheerkaOntologyManager -from core.sheerka.services.SheerkaResultManager import SheerkaResultConcept +from core.sheerka.services.SheerkaResultManager import SheerkaResultManager from sdp.sheerkaDataProvider import Event - +from tests.TestUsingFileBasedSheerka import TestUsingFileBasedSheerka from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka @@ -25,7 +26,7 @@ class TestSheerkaResultManager(TestUsingMemoryBasedSheerka): def init_service(self): sheerka, context = self.init_test().unpack() - service = sheerka.services[SheerkaResultConcept.NAME] + service = sheerka.services[SheerkaResultManager.NAME] return sheerka, context, service @@ -64,7 +65,7 @@ class TestSheerkaResultManager(TestUsingMemoryBasedSheerka): assert isinstance(res.body, types.GeneratorType) # the digest is correctly recorded - assert sheerka.load_var(SheerkaResultConcept.NAME, "digest") == digest + assert sheerka.load_var(SheerkaResultManager.NAME, "digest") == digest previous_results = list(res.body) @@ -93,7 +94,7 @@ class TestSheerkaResultManager(TestUsingMemoryBasedSheerka): assert isinstance(res.body, types.GeneratorType) # the digest is correctly recorded - assert sheerka.load_var(SheerkaResultConcept.NAME, "digest") == digest + assert sheerka.load_var(SheerkaResultManager.NAME, "digest") == digest previous_results = list(res.body) @@ -148,7 +149,7 @@ class TestSheerkaResultManager(TestUsingMemoryBasedSheerka): sheerka.evaluate_user_input("one") sheerka.evaluate_user_input("one") - service = SheerkaResultConcept(sheerka, 2) + service = SheerkaResultManager(sheerka, 2) res = service.get_results_by_command(context, "def concept") assert sheerka.isinstance(res, BuiltinConcepts.EXPLANATION) assert res.command == "def concept one as 1" @@ -171,7 +172,7 @@ class TestSheerkaResultManager(TestUsingMemoryBasedSheerka): sheerka.evaluate_user_input("one") sheerka.evaluate_user_input("one") - service = SheerkaResultConcept(sheerka, 2) + service = SheerkaResultManager(sheerka, 2) res = service.get_results_by_command(context, "fake command") assert sheerka.isinstance(res, BuiltinConcepts.NOT_FOUND) assert res.body == {'command': 'fake command'} @@ -336,7 +337,7 @@ class TestSheerkaResultManager(TestUsingMemoryBasedSheerka): def test_i_can_manage_invalid_predicates(self): predicate = {"filter": "a b c"} with pytest.raises(SyntaxError): - SheerkaResultConcept.get_predicate(**predicate) + SheerkaResultManager.get_predicate(**predicate) def test_i_can_get_last_return_value(self): sheerka, context, service = self.init_service() @@ -368,3 +369,62 @@ class TestSheerkaResultManager(TestUsingMemoryBasedSheerka): assert service.last_created_concept_id == new_concept.id assert sheerka.get_last_created_concept(context) == new_concept + + def test_last_error_is_recorded(self): + sheerka, context, foo = self.init_test().with_concepts("foo", create_new=True).unpack() + service = sheerka.services[SheerkaResultManager.NAME] + res = sheerka.evaluate_user_input("bar") # does not exist, will cause an error + sheerka.evaluate_user_input("foo") # exists, to validate that the error is not reset + + assert service.last_error_event_id is not None + assert len(res) == 1 + assert sheerka.get_last_error() == res[0] + + def test_multiple_errors_are_recorded(self): + sheerka, context, foo = self.init_test().with_concepts("foo", create_new=True).unpack() + service = sheerka.services[SheerkaResultManager.NAME] + service.test_only_reset() + + res = sheerka.evaluate_user_input("x x") # cannot be parsed + sheerka.evaluate_user_input("foo") # exists, to validate that the error is not reset + + assert service.last_error_event_id is not None + assert len(res) > 1 + assert sheerka.get_last_error() == [r for r in res if not r.status] + + def test_i_cannot_get_last_error_when_no_error(self): + sheerka, context = self.init_test().unpack() + service = sheerka.services[SheerkaResultManager.NAME] + service.test_only_reset() + + assert service.last_error_event_id is None + assert sheerka.get_last_error() == self.sheerka.new(BuiltinConcepts.NOT_FOUND, body={"query": "get_last_error"}) + + +class TestSheerkaResultManagerFileBased(TestUsingFileBasedSheerka): + @classmethod + def setup_class(cls): + sheerka = cls().get_sheerka(cache_only=False, ontology="#TestSheerkaResultManager#") + sheerka.save_execution_context = True + cls.root_ontology_name = "#TestSheerkaResultManager#" + + @classmethod + def teardown_class(cls): + cls.sheerka.pop_ontology() + cls.root_ontology_name = SheerkaOntologyManager.ROOT_ONTOLOGY_NAME + + def test_i_can_retrieve_the_last_error_after_startup(self): + sheerka, context, foo = self.init_test().with_concepts("foo", create_new=True).unpack() + service = sheerka.services[SheerkaResultManager.NAME] + res_in_error = sheerka.evaluate_user_input("bar") # does not exist, will cause an error + sheerka.evaluate_user_input("foo") # exists, to validate that the error is not reset + + assert service.last_error_event_id is not None + assert service.get_last_error() == res_in_error[0] + + # simulate restart + sheerka = self.new_sheerka_instance(False) + service = sheerka.services[SheerkaResultManager.NAME] + + assert service.last_error_event_id is not None + assert service.get_last_error() == res_in_error[0] diff --git a/tests/core/test_sheerka_ontology.py b/tests/core/test_sheerka_ontology.py index 2f84481..43fd097 100644 --- a/tests/core/test_sheerka_ontology.py +++ b/tests/core/test_sheerka_ontology.py @@ -1167,106 +1167,107 @@ class TestSheerkaOntology(TestUsingMemoryBasedSheerka): "key5": "value5" } - def test_i_can_list_by_key_when_dictionaries(self): - sheerka = self.get_sheerka(cache_only=False) - context = self.get_context(sheerka) + # def test_i_can_list_by_key_when_dictionaries(self): + # sheerka = self.get_sheerka(cache_only=False) + # context = self.get_context(sheerka) + # + # manager = SheerkaOntologyManager(sheerka, sheerka.root_folder, sheerka.cache_only) + # manager.register_cache("cache_name", Cache().auto_configure("cache_name")) + # manager.freeze() + # + # manager.put("cache_name", "key1", {"a": "value1", "b": "value2", "c": "value3"}) + # manager.commit(context) + # + # manager.push_ontology("new ontology") + # manager.put("cache_name", "key1", {"a": "new value1", "d": "value4"}) # only in cache + # + # manager.push_ontology("another ontology") + # with manager.current_sdp().get_transaction(context.event) as transaction: + # transaction.add("cache_name", "key1", {"b": "new value2", "e": "value5"}) + # + # assert manager.list_by_key("cache_name", "key1") == { + # "a": "new value1", + # "b": "new value2", + # "c": "value3", + # "d": "value4", + # "e": "value5", + # } + # + # def test_i_can_list_by_key_when_lists(self): + # sheerka = self.get_sheerka(cache_only=False) + # context = self.get_context(sheerka) + # + # manager = SheerkaOntologyManager(sheerka, sheerka.root_folder, sheerka.cache_only) + # manager.register_cache("cache_name", Cache().auto_configure("cache_name")) + # manager.freeze() + # + # manager.put("cache_name", "key1", ["a", "b", "c"]) + # manager.commit(context) + # + # manager.push_ontology("new ontology") + # manager.put("cache_name", "key1", ["a", "d"]) # only in cache + # + # manager.push_ontology("another ontology") + # with manager.current_sdp().get_transaction(context.event) as transaction: + # transaction.add("cache_name", "key1", ["b", "e"]) + # + # assert manager.list_by_key("cache_name", "key1") == ["a", "b", "c", "a", "d", "b", "e"] + # + # def test_i_can_list_by_key_when_dictionaries_and_entries_are_removed(self): + # sheerka = self.get_sheerka(cache_only=False) + # context = self.get_context(sheerka) + # + # manager = SheerkaOntologyManager(sheerka, sheerka.root_folder, sheerka.cache_only) + # manager.register_cache("cache_name", Cache().auto_configure("cache_name")) + # manager.freeze() + # + # manager.put("cache_name", "key1", {"a": "value1", "b": "value2", "c": "value3"}) + # manager.put("cache_name", "key2", {"a": "value1", "b": "value2", "c": "value3"}) + # manager.put("cache_name", "key3", {"a": "value1", "b": "value2", "c": "value3"}) + # manager.commit(context) + # + # manager.push_ontology("new ontology") + # manager.put("cache_name", "key1", Removed) # removed in cache + # with manager.current_sdp().get_transaction(context.event) as transaction: + # transaction.add("cache_name", "key2", Removed) # removed in sdp + # + # manager.push_ontology("another ontology") + # manager.put("cache_name", "key1", {"e": "value1", "f": "value2", "g": "value3"}) + # manager.put("cache_name", "key2", {"e": "value1", "f": "value2", "g": "value3"}) + # manager.put("cache_name", "key3", {"e": "value1", "f": "value2", "g": "value3"}) + # + # assert manager.list_by_key("cache_name", "key1") == {"e": "value1", "f": "value2", "g": "value3"} + # assert manager.list_by_key("cache_name", "key2") == {"e": "value1", "f": "value2", "g": "value3"} + # assert manager.list_by_key("cache_name", "key3") == {"a": "value1", "b": "value2", "c": "value3", + # "e": "value1", "f": "value2", "g": "value3"} + # + # def test_i_can_list_by_key_when_lists_and_entries_are_removed(self): + # sheerka = self.get_sheerka(cache_only=False) + # context = self.get_context(sheerka) + # + # manager = SheerkaOntologyManager(sheerka, sheerka.root_folder, sheerka.cache_only) + # manager.register_cache("cache_name", Cache().auto_configure("cache_name")) + # manager.freeze() + # + # manager.put("cache_name", "key1", ["a", "b", "c"]) + # manager.put("cache_name", "key2", ["a", "b", "c"]) + # manager.put("cache_name", "key3", ["a", "b", "c"]) + # manager.commit(context) + # + # manager.push_ontology("new ontology") + # manager.put("cache_name", "key1", Removed) # removed in cache + # with manager.current_sdp().get_transaction(context.event) as transaction: + # transaction.add("cache_name", "key2", Removed) # removed in sdp + # + # manager.push_ontology("another ontology") + # manager.put("cache_name", "key1", ["e", "f", "g"]) + # manager.put("cache_name", "key2", ["e", "f", "g"]) + # manager.put("cache_name", "key3", ["e", "f", "g"]) + # + # assert manager.list_by_key("cache_name", "key1") == ["e", "f", "g"] + # assert manager.list_by_key("cache_name", "key2") == ["e", "f", "g"] + # assert manager.list_by_key("cache_name", "key3") == ["a", "b", "c", "e", "f", "g"] - manager = SheerkaOntologyManager(sheerka, sheerka.root_folder, sheerka.cache_only) - manager.register_cache("cache_name", Cache().auto_configure("cache_name")) - manager.freeze() - - manager.put("cache_name", "key1", {"a": "value1", "b": "value2", "c": "value3"}) - manager.commit(context) - - manager.push_ontology("new ontology") - manager.put("cache_name", "key1", {"a": "new value1", "d": "value4"}) # only in cache - - manager.push_ontology("another ontology") - with manager.current_sdp().get_transaction(context.event) as transaction: - transaction.add("cache_name", "key1", {"b": "new value2", "e": "value5"}) - - assert manager.list_by_key("cache_name", "key1") == { - "a": "new value1", - "b": "new value2", - "c": "value3", - "d": "value4", - "e": "value5", - } - - def test_i_can_list_by_key_when_lists(self): - sheerka = self.get_sheerka(cache_only=False) - context = self.get_context(sheerka) - - manager = SheerkaOntologyManager(sheerka, sheerka.root_folder, sheerka.cache_only) - manager.register_cache("cache_name", Cache().auto_configure("cache_name")) - manager.freeze() - - manager.put("cache_name", "key1", ["a", "b", "c"]) - manager.commit(context) - - manager.push_ontology("new ontology") - manager.put("cache_name", "key1", ["a", "d"]) # only in cache - - manager.push_ontology("another ontology") - with manager.current_sdp().get_transaction(context.event) as transaction: - transaction.add("cache_name", "key1", ["b", "e"]) - - assert manager.list_by_key("cache_name", "key1") == ["a", "b", "c", "a", "d", "b", "e"] - - def test_i_can_list_by_key_when_dictionaries_and_entries_are_removed(self): - sheerka = self.get_sheerka(cache_only=False) - context = self.get_context(sheerka) - - manager = SheerkaOntologyManager(sheerka, sheerka.root_folder, sheerka.cache_only) - manager.register_cache("cache_name", Cache().auto_configure("cache_name")) - manager.freeze() - - manager.put("cache_name", "key1", {"a": "value1", "b": "value2", "c": "value3"}) - manager.put("cache_name", "key2", {"a": "value1", "b": "value2", "c": "value3"}) - manager.put("cache_name", "key3", {"a": "value1", "b": "value2", "c": "value3"}) - manager.commit(context) - - manager.push_ontology("new ontology") - manager.put("cache_name", "key1", Removed) # removed in cache - with manager.current_sdp().get_transaction(context.event) as transaction: - transaction.add("cache_name", "key2", Removed) # removed in sdp - - manager.push_ontology("another ontology") - manager.put("cache_name", "key1", {"e": "value1", "f": "value2", "g": "value3"}) - manager.put("cache_name", "key2", {"e": "value1", "f": "value2", "g": "value3"}) - manager.put("cache_name", "key3", {"e": "value1", "f": "value2", "g": "value3"}) - - assert manager.list_by_key("cache_name", "key1") == {"e": "value1", "f": "value2", "g": "value3"} - assert manager.list_by_key("cache_name", "key2") == {"e": "value1", "f": "value2", "g": "value3"} - assert manager.list_by_key("cache_name", "key3") == {"a": "value1", "b": "value2", "c": "value3", - "e": "value1", "f": "value2", "g": "value3"} - - def test_i_can_list_by_key_when_lists_and_entries_are_removed(self): - sheerka = self.get_sheerka(cache_only=False) - context = self.get_context(sheerka) - - manager = SheerkaOntologyManager(sheerka, sheerka.root_folder, sheerka.cache_only) - manager.register_cache("cache_name", Cache().auto_configure("cache_name")) - manager.freeze() - - manager.put("cache_name", "key1", ["a", "b", "c"]) - manager.put("cache_name", "key2", ["a", "b", "c"]) - manager.put("cache_name", "key3", ["a", "b", "c"]) - manager.commit(context) - - manager.push_ontology("new ontology") - manager.put("cache_name", "key1", Removed) # removed in cache - with manager.current_sdp().get_transaction(context.event) as transaction: - transaction.add("cache_name", "key2", Removed) # removed in sdp - - manager.push_ontology("another ontology") - manager.put("cache_name", "key1", ["e", "f", "g"]) - manager.put("cache_name", "key2", ["e", "f", "g"]) - manager.put("cache_name", "key3", ["e", "f", "g"]) - - assert manager.list_by_key("cache_name", "key1") == ["e", "f", "g"] - assert manager.list_by_key("cache_name", "key2") == ["e", "f", "g"] - assert manager.list_by_key("cache_name", "key3") == ["a", "b", "c", "e", "f", "g"] def test_i_can_get_call_when_a_cache_is_cleared(self): sheerka = self.get_sheerka(cache_only=False) diff --git a/tests/evaluators/test_PythonEvaluator.py b/tests/evaluators/test_PythonEvaluator.py index 2b97922..4a915cf 100644 --- a/tests/evaluators/test_PythonEvaluator.py +++ b/tests/evaluators/test_PythonEvaluator.py @@ -1,6 +1,7 @@ import ast import pytest + from core.builtin_concepts import ReturnValueConcept, ParserResultConcept, BuiltinConcepts from core.concept import Concept, CB from core.sheerka.services.SheerkaConceptManager import SheerkaConceptManager @@ -11,7 +12,6 @@ from parsers.BaseNodeParser import SourceCodeNode, SourceCodeWithConceptNode from parsers.FunctionParser import FunctionParser from parsers.PythonParser import PythonNode, PythonParser from parsers.PythonWithConceptsParser import PythonWithConceptsParser - from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka @@ -226,7 +226,7 @@ class TestPythonEvaluator(TestUsingMemoryBasedSheerka): evaluated = python_evaluator.eval(context, parsed) assert evaluated.status - assert sheerka.get_concepts_weights(BuiltinConcepts.PRECEDENCE) == {'c:__var__0 plus __var__1|1001:': 1, + assert sheerka.get_weights(BuiltinConcepts.PRECEDENCE) == {'c:__var__0 plus __var__1|1001:': 1, 'c:__var__0 mult __var__1|1002:': 2} def test_i_can_define_variables(self): @@ -370,3 +370,21 @@ class TestPythonEvaluator(TestUsingMemoryBasedSheerka): sequence = visitor.get_sequences(ast_, "foo") assert sequence == expected + + @pytest.mark.parametrize("parser, value", [ + (PythonParser(), "3"), + (FunctionParser(), "3"), + (PythonParser(), 3), + (FunctionParser(), 3), + ]) + def test_i_can_eval_unresolved_rules(self, parser, value): + context = self.get_context() + context.add_to_short_term_memory("get_obj_name", get_obj_name) + context.add_to_short_term_memory("x", value) + + parsed = parser.parse(context, ParserInput("get_obj_name(r:|x:)")) + python_evaluator = PythonEvaluator() + evaluated = python_evaluator.eval(context, parsed) + + assert evaluated.status + assert evaluated.value == context.sheerka.get_rule_by_id(str(value)).name diff --git a/tests/evaluators/test_RuleEvaluator.py b/tests/evaluators/test_RuleEvaluator.py index c1bf326..f27aaba 100644 --- a/tests/evaluators/test_RuleEvaluator.py +++ b/tests/evaluators/test_RuleEvaluator.py @@ -1,6 +1,6 @@ import pytest from core.builtin_concepts import ReturnValueConcept, ParserResultConcept, BuiltinConcepts -from core.rule import Rule, ACTION_TYPE_DEFERRED +from core.rule import Rule from evaluators.RuleEvaluator import RuleEvaluator from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka @@ -37,11 +37,11 @@ class TestRuleEvaluator(TestUsingMemoryBasedSheerka): assert result.value == expected assert result.parents == [ret_val] - def test_i_can_evaluate_deferred_rules(self): + def test_i_can_evaluate_unresolved_rules(self): context = self.get_context() rule = Rule().set_id("i") - rule.metadata.action_type = ACTION_TYPE_DEFERRED + rule.metadata.id_is_unresolved = True ret_val = ReturnValueConcept("some_name", True, ParserResultConcept(value=[rule]), True) context.add_to_short_term_memory("i", 1) evaluator = RuleEvaluator() @@ -57,7 +57,7 @@ class TestRuleEvaluator(TestUsingMemoryBasedSheerka): sheerka, context = self.init_concepts() rule = Rule().set_id("unknown variable") - rule.metadata.action_type = ACTION_TYPE_DEFERRED + rule.metadata.id_is_unresolved = True ret_val = ReturnValueConcept("some_name", True, ParserResultConcept(value=[rule]), True) evaluator = RuleEvaluator() diff --git a/tests/non_reg/test_sheerka_display.py b/tests/non_reg/test_sheerka_display.py index 819fb24..ac03825 100644 --- a/tests/non_reg/test_sheerka_display.py +++ b/tests/non_reg/test_sheerka_display.py @@ -87,4 +87,21 @@ ReturnValue(who=evaluators.Concept, status=True, value=(1002)foo) test 1 #unit_test# __default__ +""" + + def test_i_can_display_objects_in_memory(self, capsys): + init = [ + "def concept one as 1", + "def concept two as 2", + "one", + "two" + ] + sheerka = self.init_scenario(init) + capsys.readouterr() + + sheerka.enable_process_return_values = True + sheerka.evaluate_user_input("in_memory()") + captured = capsys.readouterr() + assert captured.out == """one: (1001)one +two: (1002)two """ diff --git a/tests/non_reg/test_sheerka_non_reg.py b/tests/non_reg/test_sheerka_non_reg.py index 112bc87..e992107 100644 --- a/tests/non_reg/test_sheerka_non_reg.py +++ b/tests/non_reg/test_sheerka_non_reg.py @@ -116,7 +116,7 @@ as: assert service.has_id(concept_saved.id) assert service.has_name(concept_saved.name) assert service.has_hash(concept_saved.get_definition_hash()) - assert sheerka.om.copy(sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {'+': ['1001']} + assert sheerka.om.copy(SheerkaConceptManager.CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {'+': ['1001']} # sdp is up to date assert sheerka.om.current_sdp().exists(SheerkaConceptManager.CONCEPTS_BY_KEY_ENTRY, expected.key) @@ -865,7 +865,7 @@ as: res = sheerka.evaluate_user_input("set_is_less_than('some_prop', two, three)") assert res[0].status - res = sheerka.evaluate_user_input("get_concepts_weights('some_prop')") + res = sheerka.evaluate_user_input("get_weights('some_prop')") assert res[0].status assert res[0].body == {'c:one|1001:': 1, 'c:two|1002:': 2, 'c:three|1003:': 3} @@ -874,7 +874,7 @@ as: res = sheerka.evaluate_user_input("eval four > three") assert res[0].status - assert sheerka.get_concepts_weights("some_prop") == {'c:one|1001:': 1, + assert sheerka.get_weights("some_prop") == {'c:one|1001:': 1, 'c:two|1002:': 2, 'c:three|1003:': 3, 'c:four|1004:': 4} @@ -898,14 +898,14 @@ as: res = sheerka.evaluate_user_input(expression) assert res[0].status assert sheerka.isinstance(res[0].body, BuiltinConcepts.SUCCESS) - assert sheerka.get_concepts_weights("some_prop") == {'c:one|1001:': 1, 'c:two|1002:': 2} + assert sheerka.get_weights("some_prop") == {'c:one|1001:': 1, 'c:two|1002:': 2} # it now also works using the concepts names expression = "eval two < three" res = sheerka.evaluate_user_input(expression) assert res[0].status assert sheerka.isinstance(res[0].body, BuiltinConcepts.SUCCESS) - assert sheerka.get_concepts_weights("some_prop") == {'c:one|1001:': 1, 'c:two|1002:': 2, 'c:three|1003:': 3} + assert sheerka.get_weights("some_prop") == {'c:one|1001:': 1, 'c:two|1002:': 2, 'c:three|1003:': 3} def test_i_can_detect_multiple_errors_when_evaluating_a_concept(self): sheerka, context, foo, plus_one = self.init_concepts( @@ -1229,6 +1229,19 @@ as: assert res[0].status assert res[0].body == 21 + def test_i_can_define_rules_priorities(self): + sheerka, context, r1, r2 = self.init_test().with_rules(("True", "True"), ("False", "False")).unpack() + sheerka.evaluate_user_input("def concept rule x > rule y where isinstance(x, int) and isinstance(y, int) as set_is_greater_than(__PRECEDENCE, r:|x:, r:|y:, 'Rule')") + + res = sheerka.evaluate_user_input(f"eval rule {r1.id} > rule {r2.id}") + + assert len(res) == 1 + assert res[0].status + assert sheerka.isinstance(res[0].body, BuiltinConcepts.SUCCESS) + + weights = sheerka.get_weights(BuiltinConcepts.PRECEDENCE, 'Rule') + assert weights[r1.str_id] == weights[r2.str_id] + 1 + class TestSheerkaNonRegFile(TestUsingFileBasedSheerka): def test_i_can_def_several_concepts(self): diff --git a/tests/parsers/test_BaseNodeParser.py b/tests/parsers/test_BaseNodeParser.py deleted file mode 100644 index 83f8585..0000000 --- a/tests/parsers/test_BaseNodeParser.py +++ /dev/null @@ -1,294 +0,0 @@ -import pytest -from core.concept import Concept -from parsers.BaseNodeParser import BaseNodeParser, NoFirstTokenError -from parsers.BnfNodeParser import StrMatch, Sequence, OrderedChoice, Optional, ZeroOrMore, OneOrMore, ConceptExpression - -from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka - - -class TestBaseNodeParser(TestUsingMemoryBasedSheerka): - @pytest.mark.parametrize("concept, expected", [ - (Concept("foo"), {"foo": ["1001"]}), - (Concept("foo a").def_var("a"), {"foo": ["1001"]}), - (Concept("a b foo").def_var("a").def_var("b"), {"foo": ["1001"]}), - ]) - def test_i_can_get_concepts_by_first_keyword(self, concept, expected): - """ - Given a concept, i can find the first know token - example: - Concept("a foo b").def_var("a").def_var("b") - 'a' and 'b' are properties - the first 'real' token is foo - :return: - """ - - sheerka, context, *updated = self.init_concepts(concept) - - res = BaseNodeParser.compute_concepts_by_first_token(context, updated) - - assert res.status - assert res.body == expected - - @pytest.mark.parametrize("bnf, expected", [ - (StrMatch("foo"), {"foo": ["1002"]}), - (StrMatch("bar"), {"bar": ["1002"]}), - (ConceptExpression("bar"), {"c:|1001:": ["1002"]}), - (Sequence(StrMatch("foo"), StrMatch("bar")), {"foo": ["1002"]}), - (Sequence(StrMatch("foo"), ConceptExpression("bar")), {"foo": ["1002"]}), - (Sequence(ConceptExpression("bar"), StrMatch("foo")), {"c:|1001:": ["1002"]}), - (OrderedChoice(StrMatch("foo"), StrMatch("bar")), {"foo": ["1002"], "bar": ["1002"]}), - (Optional(StrMatch("foo")), {"foo": ["1002"]}), - (ZeroOrMore(StrMatch("foo")), {"foo": ["1002"]}), - (OneOrMore(StrMatch("foo")), {"foo": ["1002"]}), - (StrMatch("--filter"), {"--filter": ["1002"]}), # add both entries - ]) - def test_i_can_get_concepts_by_first_keyword_with_bnf(self, bnf, expected): - sheerka = self.get_sheerka() - context = self.get_context(sheerka) - - bar = Concept("bar").init_key() - sheerka.set_id_if_needed(bar, False) - sheerka.test_only_add_in_cache(bar) - - concept = Concept("foo").init_key() - concept.set_bnf(bnf) - sheerka.set_id_if_needed(concept, False) - - res = BaseNodeParser.compute_concepts_by_first_token(context, [concept]) - - assert res.status - assert res.body == expected - - def test_i_can_get_concepts_by_first_keyword_when_multiple_concepts(self): - sheerka = self.get_sheerka() - context = self.get_context(sheerka) - - bar = Concept("bar").init_key() - sheerka.set_id_if_needed(bar, False) - sheerka.test_only_add_in_cache(bar) - - baz = Concept("baz").init_key() - sheerka.set_id_if_needed(baz, False) - sheerka.test_only_add_in_cache(baz) - - foo = Concept("foo").init_key() - foo.set_bnf(OrderedChoice(ConceptExpression("bar"), ConceptExpression("baz"), StrMatch("qux"))) - sheerka.set_id_if_needed(foo, False) - - res = BaseNodeParser.compute_concepts_by_first_token(context, [bar, baz, foo]) - - assert res.status - assert res.body == { - "bar": ["1001"], - "baz": ["1002"], - "c:|1001:": ["1003"], - "c:|1002:": ["1003"], - "qux": ["1003"], - } - - def test_i_can_get_concepts_by_first_keyword_using_sheerka(self): - sheerka, context, *updated = self.init_test().with_concepts( - "one", - "two", - Concept("twenty", definition="'twenty' (one|two)"), - create_new=True - ).unpack() - - bar = Concept("bar").init_key() - sheerka.set_id_if_needed(bar, False) - sheerka.test_only_add_in_cache(bar) - - foo = Concept("foo").init_key() - foo.set_bnf(OrderedChoice(ConceptExpression("one"), ConceptExpression("bar"), StrMatch("qux"))) - sheerka.set_id_if_needed(foo, False) - - res = BaseNodeParser.compute_concepts_by_first_token(context, [bar, foo], use_sheerka=True) - - assert res.status - assert res.body == { - "one": ["1001"], - "two": ["1002"], - "twenty": ["1003"], - "bar": ["1004"], - "c:|1001:": ["1005"], - "c:|1004:": ["1005"], - "qux": ["1005"], - } - - def test_i_cannot_get_concept_by_first_keyword_when_no_first_keyword(self): - sheerka, context, foo = self.init_concepts(Concept("x y", body="x y").def_var("x").def_var("y")) - res = BaseNodeParser.compute_concepts_by_first_token(context, [foo]) - - assert not res.status - assert res.body == NoFirstTokenError(foo, foo.key) - - def test_i_can_resolve_concepts_by_first_keyword(self): - sheerka, context, *updated = self.init_concepts( - "one", - Concept("two", definition="one"), - Concept("three", definition="two")) - - concepts_by_first_keywords = { - "one": ["1001"], - "c:|1001:": ["1002"], - "c:|1002:": ["1003"], - } - - resolved_ret_val = BaseNodeParser.resolve_concepts_by_first_keyword(context, concepts_by_first_keywords) - - assert resolved_ret_val.status - assert resolved_ret_val.body == { - "one": ["1001", "1002", "1003"], - } - - def test_i_can_resolve_when_concepts_are_sets(self): - sheerka, context, number, *concepts = self.init_concepts( - "number", - "one", - "two", - "twenty", - "hundred", - Concept("twenties", definition="twenty number"), - Concept("hundreds", definition="number hundred"), - ) - - sheerka.set_isa(context, sheerka.new("one"), number) - sheerka.set_isa(context, sheerka.new("two"), number) - sheerka.set_isa(context, sheerka.new("twenty"), number) - sheerka.set_isa(context, sheerka.new("thirty"), number) - sheerka.set_isa(context, sheerka.new("hundred"), number) - sheerka.set_isa(context, sheerka.new("twenties"), number) - sheerka.set_isa(context, sheerka.new("hundreds"), number) - - sheerka.concepts_grammars.clear() # reset all the grammar to simulate Sheerka restart - - # cbft : concept_by_first_token (I usually don't use abbreviation) - cbft = BaseNodeParser.compute_concepts_by_first_token(context, [number] + concepts).body - resolved_ret_val = BaseNodeParser.resolve_concepts_by_first_keyword(context, cbft) - - assert resolved_ret_val.status - assert resolved_ret_val.body == { - 'number': ['1001'], - 'one': ['1002', '1007'], - 'two': ['1003', '1007'], - 'twenty': ['1004', '1006', '1007'], - 'hundred': ['1005', '1007'], - } - - def test_concepts_are_defined_once(self): - sheerka = self.get_sheerka() - context = self.get_context(sheerka) - good = self.create_and_add_in_cache_concept(sheerka, "good") - foo = self.create_and_add_in_cache_concept(sheerka, "foo", bnf=ConceptExpression("good")) - bar = self.create_and_add_in_cache_concept(sheerka, "bar", bnf=ConceptExpression("good")) - baz = self.create_and_add_in_cache_concept(sheerka, "baz", bnf=OrderedChoice( - ConceptExpression("foo"), - ConceptExpression("bar"))) - - concepts_by_first_keywords = BaseNodeParser.compute_concepts_by_first_token( - context, [good, foo, bar, baz]).body - - resolved_ret_val = BaseNodeParser.resolve_concepts_by_first_keyword(context, concepts_by_first_keywords) - assert resolved_ret_val.status - assert resolved_ret_val.body == { - "good": ["1001", "1002", "1003", "1004"], - } - - def test_i_can_resolve_more_complex(self): - sheerka = self.get_sheerka() - context = self.get_context(sheerka) - - a = self.create_and_add_in_cache_concept(sheerka, "a", bnf=Sequence("one", "two")) - b = self.create_and_add_in_cache_concept(sheerka, "b", bnf=Sequence(ConceptExpression("a"), "two")) - - concepts_by_first_keywords = BaseNodeParser.compute_concepts_by_first_token( - context, [a, b]).body - - resolved_ret_val = BaseNodeParser.resolve_concepts_by_first_keyword(context, concepts_by_first_keywords) - assert resolved_ret_val.status - assert resolved_ret_val.body == { - "one": ["1001", "1002"], - } - - def tests_i_can_detect_direct_recursion(self): - sheerka, context, good, foo, bar = self.init_concepts( - "good", - self.bnf_concept("foo", ConceptExpression("bar")), - self.bnf_concept("bar", ConceptExpression("foo")), - ) - - concepts_by_first_keywords = BaseNodeParser.compute_concepts_by_first_token(context, [good, foo, bar]).body - resolved_ret_val = BaseNodeParser.resolve_concepts_by_first_keyword(context, concepts_by_first_keywords) - assert resolved_ret_val.status - assert resolved_ret_val.body == { - "good": ["1001"], - } - assert sheerka.chicken_and_eggs.get(foo.id) == {foo.id, bar.id} - assert sheerka.chicken_and_eggs.get(bar.id) == {foo.id, bar.id} - - def test_i_can_detect_indirect_infinite_recursion(self): - sheerka, context, good, one, two, three = self.init_concepts( - "good", - self.bnf_concept("one", ConceptExpression("two")), - self.bnf_concept("two", ConceptExpression("three")), - self.bnf_concept("three", ConceptExpression("two")), - ) - - concepts_by_first_keywords = BaseNodeParser.compute_concepts_by_first_token(context, [good, one, two, three]).body - resolved_ret_val = BaseNodeParser.resolve_concepts_by_first_keyword(context, concepts_by_first_keywords) - assert resolved_ret_val.status - assert resolved_ret_val.body == { - "good": ["1001"], - } - assert sheerka.chicken_and_eggs.get(one.id) == {one.id, two.id, three.id} - assert sheerka.chicken_and_eggs.get(two.id) == {one.id, two.id, three.id} - assert sheerka.chicken_and_eggs.get(three.id) == {one.id, two.id, three.id} - - def test_i_can_detect_the_longest_infinite_recursion_chain(self): - sheerka, context, good, one, two, three = self.init_concepts( - "good", - self.bnf_concept("two", ConceptExpression("three")), - self.bnf_concept("three", ConceptExpression("two")), - self.bnf_concept("one", ConceptExpression("three")), - ) - - concepts_by_first_keywords = BaseNodeParser.compute_concepts_by_first_token(context, [good, one, two, three]).body - resolved_ret_val = BaseNodeParser.resolve_concepts_by_first_keyword(context, concepts_by_first_keywords) - assert resolved_ret_val.status - assert resolved_ret_val.body == { - "good": ["1001"], - } - assert sheerka.chicken_and_eggs.get(one.id) == {one.id, two.id, three.id} - assert sheerka.chicken_and_eggs.get(two.id) == {one.id, two.id, three.id} - assert sheerka.chicken_and_eggs.get(three.id) == {one.id, two.id, three.id} - - # - # def test_i_can_detect_infinite_recursion_from_ordered_choice(self): - # sheerka = self.get_sheerka() - # good = self.get_concept(sheerka, "good") - # one = self.get_concept(sheerka, "one", ConceptExpression("two")) - # two = self.get_concept(sheerka, "two", OrderedChoice(ConceptExpression("one"), ConceptExpression("two"))) - # - # concepts_by_first_keywords = BaseNodeParser.get_concepts_by_first_token(sheerka, [good, one, two]).body - # - # resolved_ret_val = BaseNodeParser.resolve_concepts_by_first_keyword(sheerka, concepts_by_first_keywords) - # assert resolved_ret_val.status - # assert resolved_ret_val.body == { - # "good": ["1001"], - # BuiltinConcepts.CHICKEN_AND_EGG: ["1002", "1003"] - # } - # - # def test_i_can_detect_infinite_recursion_with_sequence(self): - # sheerka = self.get_sheerka() - # good = self.get_concept(sheerka, "good") - # one = self.get_concept(sheerka, "one", ConceptExpression("two")) - # two = self.get_concept(sheerka, "two", Sequence(StrMatch("yes"), ConceptExpression("one"))) - # - # concepts_by_first_keywords = BaseNodeParser.get_concepts_by_first_token(sheerka, [good, one, two]).body - # - # resolved_ret_val = BaseNodeParser.resolve_concepts_by_first_keyword(sheerka, concepts_by_first_keywords) - # assert resolved_ret_val.status - # assert resolved_ret_val.body == { - # "good": ["1001"], - # BuiltinConcepts.CHICKEN_AND_EGG: ["1002", "1003"] - # } diff --git a/tests/parsers/test_BnfNodeParser.py b/tests/parsers/test_BnfNodeParser.py index af2799d..b958192 100644 --- a/tests/parsers/test_BnfNodeParser.py +++ b/tests/parsers/test_BnfNodeParser.py @@ -4,6 +4,7 @@ import tests.parsers.parsers_utils from core.builtin_concepts import BuiltinConcepts from core.concept import Concept, ConceptParts, DoNotResolve, CC, DEFINITION_TYPE_BNF from core.global_symbols import NotInit +from core.sheerka.services.SheerkaConceptManager import SheerkaConceptManager from core.sheerka.services.SheerkaExecute import ParserInput from parsers.BaseNodeParser import CNC, UTN, CN from parsers.BnfDefinitionParser import BnfDefinitionParser @@ -833,7 +834,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka): # every obvious cyclic recursion are removed from concept_by_first_keyword dict parser.init_from_concepts(context, my_map.values()) - assert sheerka.om.copy(sheerka.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == expected + assert sheerka.om.copy(SheerkaConceptManager.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == expected # get_parsing_expression() also returns CHICKEN_AND_EGG parsing_expression = parser.get_parsing_expression(context, my_map["foo"]) @@ -858,7 +859,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka): # every obvious cyclic recursion are removed from concept_by_first_keyword dict parser.init_from_concepts(context, my_map.values()) - assert sheerka.om.copy(sheerka.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {} + assert sheerka.om.copy(SheerkaConceptManager.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {} parsing_expression = parser.get_parsing_expression(context, my_map["foo"]) assert sheerka.isinstance(parsing_expression, BuiltinConcepts.CHICKEN_AND_EGG) @@ -884,7 +885,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka): # every obvious cyclic recursion are removed from concept_by_first_keyword dict parser.init_from_concepts(context, my_map.values()) - assert sheerka.om.copy(sheerka.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {} + assert sheerka.om.copy(SheerkaConceptManager.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {} parsing_expression = parser.get_parsing_expression(context, my_map["foo"]) assert sheerka.isinstance(parsing_expression, BuiltinConcepts.CHICKEN_AND_EGG) diff --git a/tests/parsers/test_RuleParser.py b/tests/parsers/test_RuleParser.py index 223cb67..1d729a0 100644 --- a/tests/parsers/test_RuleParser.py +++ b/tests/parsers/test_RuleParser.py @@ -87,7 +87,7 @@ class TestRuleParser(TestUsingMemoryBasedSheerka): assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT) assert len(rules) == 1 assert rules[0].id == "xxx" - assert rules[0].metadata.action_type == "deferred" + assert rules[0].metadata.id_is_unresolved @pytest.mark.parametrize("text", [ "r:|1:xxx", diff --git a/tests/parsers/test_SyaNodeParser.py b/tests/parsers/test_SyaNodeParser.py index 3cba7fc..0040ce5 100644 --- a/tests/parsers/test_SyaNodeParser.py +++ b/tests/parsers/test_SyaNodeParser.py @@ -1,6 +1,8 @@ import pytest + +import tests.parsers.parsers_utils from core.builtin_concepts import BuiltinConcepts -from core.concept import Concept, CIO, ALL_ATTRIBUTES, CMV +from core.concept import Concept, CIO, CMV from core.global_symbols import CONCEPT_COMPARISON_CONTEXT from core.sheerka.services.SheerkaExecute import ParserInput from core.tokenizer import Tokenizer @@ -10,8 +12,7 @@ from parsers.BaseNodeParser import utnode, cnode, short_cnode, UnrecognizedToken from parsers.PythonParser import PythonNode from parsers.SyaNodeParser import SyaNodeParser, SyaConceptParserHelper, SyaAssociativity, \ NoneAssociativeSequenceError, TooManyParametersFoundError, InFixToPostFix, ParenthesisMismatchError - -import tests.parsers.parsers_utils +from tests.TestUsingFileBasedSheerka import TestUsingFileBasedSheerka from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka @@ -1356,3 +1357,35 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka): assert res.status assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT) assert lexer_nodes == [CN(cmap["suffixed"], 0, 6, source=text)] + + +class TestFileBaseSyaNodeParser(TestUsingFileBasedSheerka): + def test_i_can_parse_after_restart(self): + sheerka, context, one, two, plus = self.init_test().with_concepts("one", + "two", + Concept("a plus b").def_var("a").def_var("b"), + create_new=True).unpack() + sheerka.om.commit(context) + parser = SyaNodeParser() + + # sanity check + # Remove this commented section to make sure that the nominal case still works + # res = parser.parse(context, ParserInput("one plus two")) + # assert res.status + # assert sheerka.isinstance(res.body.body[0].concept, plus.key) + + sheerka = self.new_sheerka_instance(False) + context = self.get_context(sheerka) + + res = parser.parse(context, ParserInput("one plus two")) + assert res.status + assert sheerka.isinstance(res.body.body[0].concept, plus.key) + + # adds an ontology layer and make the test again + sheerka.push_ontology(context, "new ontology") + sheerka = self.new_sheerka_instance(False) + context = self.get_context(sheerka) + + res = parser.parse(context, ParserInput("one plus two")) + assert res.status + assert sheerka.isinstance(res.body.body[0].concept, plus.key)