diff --git a/src/cache/CacheManager.py b/src/cache/CacheManager.py index ff9cfaf..783b4e9 100644 --- a/src/cache/CacheManager.py +++ b/src/cache/CacheManager.py @@ -259,3 +259,11 @@ class CacheManager: self.caches[cache_name].cache.init_from(content) return self + + def reset(self, cache_only): + """For unit test speed enhancement""" + self.cache_only = cache_only + self.caches.clear() + self.concept_caches.clear() + self.is_dirty = False + diff --git a/src/core/builtin_concepts.py b/src/core/builtin_concepts.py index 162f458..b973ffa 100644 --- a/src/core/builtin_concepts.py +++ b/src/core/builtin_concepts.py @@ -59,6 +59,8 @@ class BuiltinConcepts(Enum): CHICKEN_AND_EGG = "chicken and egg" # infinite recursion when declaring concept ISA = "is a" # builtin concept to express that a concept is an instance of another one EXPLANATION = "explanation" + PRECEDENCE = "precedence" # use to set priority among concepts when parsing + ASSOCIATIVITY = "associativity" # use to set priority among concepts when parsing NODE = "node" GENERIC_NODE = "generic node" diff --git a/src/core/concept.py b/src/core/concept.py index dcc3a92..08798a2 100644 --- a/src/core/concept.py +++ b/src/core/concept.py @@ -342,20 +342,30 @@ class Concept: return self - def add_prop(self, concept_key, value): + def add_prop(self, property_name, value): """ Set or add a behaviour to a concept A behaviour is a value from another concept (ex BuiltinConcepts.ISA - :param concept_key: Concept key + :param property_name: Concept key :param value: :return: """ - if concept_key in self.metadata.props: - self.metadata.props[concept_key].add(value) + if property_name in self.metadata.props: + self.metadata.props[property_name].add(value) else: - self.metadata.props[concept_key] = {value} # a set + self.metadata.props[property_name] = {value} # a set return self + def set_prop(self, property_name, value): + """ + Set a behaviour to a concept + A behaviour is a value from another concept (ex BuiltinConcepts.ISA + :param property_name: Concept key + :param value: + :return: + """ + self.metadata.props[property_name] = value + def get_prop(self, concept_key): """ Gets a behaviour of a concept diff --git a/src/core/sheerka/ExecutionContext.py b/src/core/sheerka/ExecutionContext.py index 838314c..73eef3a 100644 --- a/src/core/sheerka/ExecutionContext.py +++ b/src/core/sheerka/ExecutionContext.py @@ -3,7 +3,7 @@ import time from core.builtin_concepts import BuiltinConcepts from core.concept import Concept -from core.sheerka.Services.SheerkaExecute import NO_MATCH +from core.sheerka.services.SheerkaExecute import NO_MATCH from core.sheerka_logger import get_logger from sdp.sheerkaDataProvider import Event diff --git a/src/core/sheerka/Sheerka.py b/src/core/sheerka/Sheerka.py index 41ca7f0..f83c1d2 100644 --- a/src/core/sheerka/Sheerka.py +++ b/src/core/sheerka/Sheerka.py @@ -12,14 +12,14 @@ from core.builtin_concepts import BuiltinConcepts, ErrorConcept, ReturnValueConc UnknownConcept from core.concept import Concept, ConceptParts, PROPERTIES_FOR_NEW from core.sheerka.ExecutionContext import ExecutionContext -from core.sheerka.Services.SheerkaCreateNewConcept import SheerkaCreateNewConcept -from core.sheerka.Services.SheerkaDump import SheerkaDump -from core.sheerka.Services.SheerkaEvaluateConcept import SheerkaEvaluateConcept -from core.sheerka.Services.SheerkaExecute import SheerkaExecute -from core.sheerka.Services.SheerkaHistoryManager import SheerkaHistoryManager -from core.sheerka.Services.SheerkaModifyConcept import SheerkaModifyConcept -from core.sheerka.Services.SheerkaSetsManager import SheerkaSetsManager -from core.sheerka.Services.SheerkaVariableManager import SheerkaVariableManager +from core.sheerka.services.SheerkaCreateNewConcept import SheerkaCreateNewConcept +from core.sheerka.services.SheerkaDump import SheerkaDump +from core.sheerka.services.SheerkaEvaluateConcept import SheerkaEvaluateConcept +from core.sheerka.services.SheerkaExecute import SheerkaExecute +from core.sheerka.services.SheerkaHistoryManager import SheerkaHistoryManager +from core.sheerka.services.SheerkaModifyConcept import SheerkaModifyConcept +from core.sheerka.services.SheerkaSetsManager import SheerkaSetsManager +from core.sheerka.services.SheerkaVariableManager import SheerkaVariableManager from core.sheerka_logger import console_handler from printer.SheerkaPrinter import SheerkaPrinter from sdp.sheerkaDataProvider import SheerkaDataProvider, Event @@ -72,6 +72,8 @@ class Sheerka(Concept): self.sdp: SheerkaDataProvider = None # SheerkaDataProvider self.cache_manager = CacheManager(cache_only) + self.services = {} # sheerka plugins + self.builtin_cache = {} # cache for builtin concepts self.parsers = {} # cache for builtin parsers self.evaluators = [] # cache for builtin evaluators @@ -114,6 +116,20 @@ class Sheerka(Concept): def concepts_grammars(self): return self.cache_manager.caches[self.CONCEPTS_GRAMMARS_ENTRY].cache + def bind_service_method(self, instance, method, as_name=None): + """ + Bind service method to sheerka instance for ease to use + :param instance: + :param method: + :param as_name: + :return: + """ + if as_name is None: + as_name = method.__name__ + + bound_method = method.__get__(instance, instance.__class__) + setattr(self, as_name, bound_method) + def initialize(self, root_folder: str = None, save_execution_context=True): """ Starting Sheerka @@ -139,6 +155,7 @@ class Sheerka(Concept): if self.sdp.first_time: self.first_time_initialisation(exec_context) + self.initialize_services() self.initialize_builtin_parsers() self.initialize_builtin_evaluators() self.initialize_builtin_concepts() @@ -212,6 +229,21 @@ class Sheerka(Concept): self.cache_manager.put(self.CONCEPTS_KEYS_ENTRY, self.USER_CONCEPTS_KEYS, 1000) self.variable_handler.record(context, self.name, "save_execution_context", True) + def initialize_services(self): + """ + Introspect to find services and bind them + :return: + """ + self.init_log.debug("Initializing services") + + core.utils.import_module_and_sub_module('core.sheerka.services') + base_class = "core.sheerka.services.sheerka_service.BaseService" + for service in core.utils.get_sub_classes("core.sheerka.services", base_class): + instance = service(self) + if hasattr(instance, "initialize"): + instance.initialize() + self.services[service.NAME] = instance + def initialize_builtin_concepts(self): """ Initializes the builtin concepts @@ -300,8 +332,14 @@ class Sheerka(Concept): self.cache_manager.put(self.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY, False, res.body) def reset(self, cache_only=False): - self.cache_manager.clear() - self.cache_manager.cache_only = cache_only + if self.cache_manager.cache_only != cache_only: + self.cache_manager.reset(cache_only) + self.initialize_caching() + for service in self.services.values(): + if hasattr(service, "initialize"): + service.initialize() + else: + self.cache_manager.clear() self.printer_handler.reset() self.sdp.reset() diff --git a/src/core/sheerka/services/SheerkaComparisonManager.py b/src/core/sheerka/services/SheerkaComparisonManager.py new file mode 100644 index 0000000..e0d3633 --- /dev/null +++ b/src/core/sheerka/services/SheerkaComparisonManager.py @@ -0,0 +1,206 @@ +from dataclasses import dataclass + +from cache.Cache import Cache +from cache.ListCache import ListCache +from core.builtin_concepts import BuiltinConcepts +from core.sheerka.services.sheerka_service import ServiceObj, BaseService + + +@dataclass +class ComparisonObj(ServiceObj): + """ + Order to store + """ + property: str # property to compare + a: int # id of concept a + b: int # id of concept b + op: str # comparison operation + context: str = "#" # context when the comparison is right + + +class SheerkaComparisonManager(BaseService): + """ + Manage partitioning of concepts + """ + NAME = "ComparisonManager" + COMPARISON_ENTRY = "Comparison" + RESOLVED_COMPARISON_ENTRY = "Resolved_Comparison" + + def __init__(self, sheerka): + super().__init__(sheerka) + + @staticmethod + def _compute_key(prop_name, comparison_context): + return f"{prop_name}|{comparison_context}" + + @staticmethod + def _compute_weights(comparison_objs): + """ + For every element in greater_than_s, give it a weight + if weight(a) > weight(b) it means that a > b + :param comparison_objs: list of greater than objects + :return: + """ + + values = {} + for comparison_obj in comparison_objs: + values[comparison_obj.a] = 1 + values[comparison_obj.b] = 1 + + for _ in range(len(comparison_objs)): + for comparison_obj in comparison_objs: + if comparison_obj.op == ">": + values[comparison_obj.a] = values[comparison_obj.b] + 1 + else: + values[comparison_obj.b] = values[comparison_obj.a] + 1 + + return values + + @staticmethod + def _get_partition(weighted_concepts): + + res = {} + for k, v in weighted_concepts.items(): + res.setdefault(v, []).append(k) + return res + + def _inner_add_comparison(self, comparison_obj): + key = self._compute_key(comparison_obj.property, comparison_obj.context) + previous = self.sheerka.cache_manager.get(self.COMPARISON_ENTRY, key) + + new = previous.copy() if previous else [] + new.append(comparison_obj) + + cycles = self.detect_cycles(new) + if cycles: + concepts_in_cycle = [self.sheerka.get_by_id(c) for c in cycles] + chicken_an_egg = self.sheerka.new(BuiltinConcepts.CHICKEN_AND_EGG, body=concepts_in_cycle) + return self.sheerka.ret(self.NAME, False, chicken_an_egg) + + self.sheerka.cache_manager.put(self.RESOLVED_COMPARISON_ENTRY, key, self._compute_weights(new)) + self.sheerka.cache_manager.put(self.COMPARISON_ENTRY, key, comparison_obj) + + return self.sheerka.ret(self.NAME, True, self.sheerka.new(BuiltinConcepts.SUCCESS)) + + def initialize(self): + cache = ListCache(default=lambda k: self.sheerka.sdp.get(self.COMPARISON_ENTRY, k)) + self.sheerka.cache_manager.register_cache(self.COMPARISON_ENTRY, cache, True, True) + + cache = Cache() + self.sheerka.cache_manager.register_cache(self.RESOLVED_COMPARISON_ENTRY, cache, persist=False) + + self.sheerka.bind_service_method(self, SheerkaComparisonManager.is_greater_than) + self.sheerka.bind_service_method(self, SheerkaComparisonManager.is_less_than) + self.sheerka.bind_service_method(self, SheerkaComparisonManager.get_partition) + self.sheerka.bind_service_method(self, SheerkaComparisonManager.get_concepts_weights) + + def is_greater_than(self, context, prop_name, concept_a, concept_b, comparison_context="#"): + """ + Records that the property of concept a is greater than concept b's one + :param context: + :param prop_name: + :param concept_a: + :param concept_b: + :param comparison_context: + :return: + """ + context.log(f"Setting concept {concept_a} is greater than {concept_b}", who=self.NAME) + + event_digest = context.event.get_digest() + comparison_obj = ComparisonObj(event_digest, prop_name, concept_a.id, concept_b.id, ">", comparison_context) + return self._inner_add_comparison(comparison_obj) + + def is_less_than(self, context, prop_name, concept_a, concept_b, comparison_context="#"): + """ + Records that the property of concept a is lesser than concept b's one + :param context: + :param prop_name: + :param concept_a: + :param concept_b: + :param comparison_context: + :return: + """ + context.log(f"Setting concept {concept_a} is less than {concept_b}", who=self.NAME) + + event_digest = context.event.get_digest() + comparison_obj = ComparisonObj(event_digest, prop_name, concept_a.id, concept_b.id, "<", comparison_context) + return self._inner_add_comparison(comparison_obj) + + def get_partition(self, prop_name, comparison_context="#"): + weighted_concept = self.get_concepts_weights(prop_name, comparison_context) + + return self._get_partition(weighted_concept) + + def get_concepts_weights(self, prop_name, comparison_context="#"): + weighted_concept = self.sheerka.cache_manager.get( + self.RESOLVED_COMPARISON_ENTRY, + self._compute_key(prop_name, comparison_context)) + + if weighted_concept is None: + key = self._compute_key(prop_name, comparison_context) + entries = self.sheerka.cache_manager.get(self.COMPARISON_ENTRY, key) + + if entries is None: + return {} + else: + weighted_concept = self._compute_weights(entries) + self.sheerka.cache_manager.put(self.RESOLVED_COMPARISON_ENTRY, key, weighted_concept) + + return weighted_concept + + @staticmethod + def detect_cycles(comparison_objs): + """ + # Thanks to Divyanshu Mehta for contributing this code + # https://www.geeksforgeeks.org/detect-cycle-in-a-graph/?ref=lbp + :param comparison_objs: + :return: + """ + latest = comparison_objs[-1] + if latest.op == "=": + return None + + def get_graph_and_vertices(): + _graph = {} + _vertices = set() + for obj in comparison_objs: + if obj.op == "=": + continue + + _vertices.add(obj.a) + _vertices.add(obj.b) + if obj.op == ">": + _graph.setdefault(obj.a, []).append(obj.b) + else: + _graph.setdefault(obj.b, []).append(obj.a) + return _graph, _vertices + + def is_cyclic(v): + # Mark current node as visited and + # adds to recursion stack + visited[v] = True + rec_stack[v] = True + + # Recur for all neighbours + # if any neighbour is visited and in + # recStack then graph is cyclic + if v in graph: + for neighbour in graph[v]: + if not visited[neighbour]: + if is_cyclic(neighbour): + return True + elif rec_stack[neighbour]: + return True + + # The node needs to be poped from + # recursion stack before function ends + rec_stack[v] = False + return False + + graph, vertices = get_graph_and_vertices() + visited = {k: False for k in vertices} + rec_stack = {k: False for k in vertices} + + if is_cyclic(latest.a): # only need to check from the latest add, since the graph was not cyclic before + return [k for k, v in rec_stack.items() if v] + return None diff --git a/src/core/sheerka/Services/SheerkaCreateNewConcept.py b/src/core/sheerka/services/SheerkaCreateNewConcept.py similarity index 97% rename from src/core/sheerka/Services/SheerkaCreateNewConcept.py rename to src/core/sheerka/services/SheerkaCreateNewConcept.py index 7d8f159..4fdb74f 100644 --- a/src/core/sheerka/Services/SheerkaCreateNewConcept.py +++ b/src/core/sheerka/services/SheerkaCreateNewConcept.py @@ -14,7 +14,7 @@ class SheerkaCreateNewConcept: def __init__(self, sheerka): self.sheerka = sheerka - self.logger_name = self.create_new_concept.__name__ + self.logger_name = "CreateNewConcept" self.bnp = core.utils.get_class(BASE_NODE_PARSER_CLASS) # BaseNodeParser def create_new_concept(self, context, concept: Concept): diff --git a/src/core/sheerka/Services/SheerkaDump.py b/src/core/sheerka/services/SheerkaDump.py similarity index 100% rename from src/core/sheerka/Services/SheerkaDump.py rename to src/core/sheerka/services/SheerkaDump.py diff --git a/src/core/sheerka/Services/SheerkaEvaluateConcept.py b/src/core/sheerka/services/SheerkaEvaluateConcept.py similarity index 97% rename from src/core/sheerka/Services/SheerkaEvaluateConcept.py rename to src/core/sheerka/services/SheerkaEvaluateConcept.py index 15cdac9..4b00f7d 100644 --- a/src/core/sheerka/Services/SheerkaEvaluateConcept.py +++ b/src/core/sheerka/services/SheerkaEvaluateConcept.py @@ -11,7 +11,7 @@ CONCEPT_EVALUATION_STEPS = [ class SheerkaEvaluateConcept: def __init__(self, sheerka): self.sheerka = sheerka - self.logger_name = self.evaluate_concept.__name__ + self.logger_name = "EvaluateConcept" @staticmethod def infinite_recursion_detected(context, concept): @@ -88,7 +88,7 @@ class SheerkaEvaluateConcept: if source.strip() == "": concept.compiled[part_key] = DoNotResolve(source) else: - with context.push(desc=f"Initializing compiled for {part_key}") as sub_context: + with context.push(desc=f"Initializing *compiled* for {part_key}") as sub_context: sub_context.add_inputs(source=source) to_parse = self.sheerka.ret(context.who, True, self.sheerka.new(BuiltinConcepts.USER_INPUT, body=source)) @@ -107,11 +107,11 @@ class SheerkaEvaluateConcept: if default_value.strip() == "": concept.compiled[var_name] = DoNotResolve(default_value) else: - with context.push(desc=f"Initializing AST for property {var_name}") as sub_context: + with context.push(desc=f"Initializing *compiled* for property {var_name}") as sub_context: sub_context.add_inputs(source=default_value) to_parse = self.sheerka.ret(context.who, True, self.sheerka.new(BuiltinConcepts.USER_INPUT, body=default_value)) - res = self.sheerka.execute(context, to_parse, steps) + res = self.sheerka.execute(sub_context, to_parse, steps) only_success = only_successful(sub_context, res) concept.compiled[var_name] = only_success.body.body if is_only_successful(only_success) else res sub_context.add_values(return_values=res) diff --git a/src/core/sheerka/Services/SheerkaExecute.py b/src/core/sheerka/services/SheerkaExecute.py similarity index 100% rename from src/core/sheerka/Services/SheerkaExecute.py rename to src/core/sheerka/services/SheerkaExecute.py diff --git a/src/core/sheerka/Services/SheerkaHistoryManager.py b/src/core/sheerka/services/SheerkaHistoryManager.py similarity index 100% rename from src/core/sheerka/Services/SheerkaHistoryManager.py rename to src/core/sheerka/services/SheerkaHistoryManager.py diff --git a/src/core/sheerka/Services/SheerkaModifyConcept.py b/src/core/sheerka/services/SheerkaModifyConcept.py similarity index 96% rename from src/core/sheerka/Services/SheerkaModifyConcept.py rename to src/core/sheerka/services/SheerkaModifyConcept.py index 8ab8de0..f640d17 100644 --- a/src/core/sheerka/Services/SheerkaModifyConcept.py +++ b/src/core/sheerka/services/SheerkaModifyConcept.py @@ -4,7 +4,7 @@ from core.builtin_concepts import BuiltinConcepts class SheerkaModifyConcept: def __init__(self, sheerka): self.sheerka = sheerka - self.logger_name = self.modify_concept.__name__ + self.logger_name = "ModifyConcept" def modify_concept(self, context, concept): old_version = self.sheerka.get_by_id(concept.id) diff --git a/src/core/sheerka/Services/SheerkaSetsManager.py b/src/core/sheerka/services/SheerkaSetsManager.py similarity index 99% rename from src/core/sheerka/Services/SheerkaSetsManager.py rename to src/core/sheerka/services/SheerkaSetsManager.py index 8d188a2..732aca1 100644 --- a/src/core/sheerka/Services/SheerkaSetsManager.py +++ b/src/core/sheerka/services/SheerkaSetsManager.py @@ -9,7 +9,7 @@ GROUP_PREFIX = 'All_' class SheerkaSetsManager: def __init__(self, sheerka): self.sheerka = sheerka - self.logger_name = self.add_concept_to_set.__name__ + self.logger_name = "SetsManager" def set_isa(self, context, concept, concept_set): """ diff --git a/src/core/sheerka/Services/SheerkaVariableManager.py b/src/core/sheerka/services/SheerkaVariableManager.py similarity index 85% rename from src/core/sheerka/Services/SheerkaVariableManager.py rename to src/core/sheerka/services/SheerkaVariableManager.py index f7b3f6d..3fa2e95 100644 --- a/src/core/sheerka/Services/SheerkaVariableManager.py +++ b/src/core/sheerka/services/SheerkaVariableManager.py @@ -1,20 +1,21 @@ from dataclasses import dataclass from typing import List +from core.sheerka.services.sheerka_service import ServiceObj + @dataclass -class Variable: +class Variable(ServiceObj): """ Variable to store """ - event_id: str # event where the variable is modified who: str # who is the modifier key: str # key of the variable value: object # value parents: List[str] # previous references of the variable (Note that there should be only one parent) def get_key(self): - return f"{self.who}.{self.key}" + return f"{self.who}|{self.key}" class SheerkaVariableManager: @@ -36,11 +37,11 @@ class SheerkaVariableManager: self.sheerka.cache_manager.put(self.sheerka.VARIABLES_ENTRY, variable.get_key(), variable) def load(self, who, key): - variable = self.sheerka.cache_manager.get(self.sheerka.VARIABLES_ENTRY, who + "." + key) + variable = self.sheerka.cache_manager.get(self.sheerka.VARIABLES_ENTRY, who + "|" + key) if variable is None: return None return variable.value def delete(self, context, who, key): - self.sheerka.cache_manager.delete(self.sheerka.VARIABLES_ENTRY, who + "." + key) + self.sheerka.cache_manager.delete(self.sheerka.VARIABLES_ENTRY, who + "|" + key) diff --git a/src/core/sheerka/Services/__init__.py b/src/core/sheerka/services/__init__.py similarity index 100% rename from src/core/sheerka/Services/__init__.py rename to src/core/sheerka/services/__init__.py diff --git a/src/core/sheerka/services/sheerka_service.py b/src/core/sheerka/services/sheerka_service.py new file mode 100644 index 0000000..52eb297 --- /dev/null +++ b/src/core/sheerka/services/sheerka_service.py @@ -0,0 +1,14 @@ +from dataclasses import dataclass + + +@dataclass +class ServiceObj: + event_id: str # event where the object is created / modified + + +class BaseService: + """ + Base class for services + """ + def __init__(self, sheerka): + self.sheerka = sheerka diff --git a/src/core/utils.py b/src/core/utils.py index 1732e90..a718228 100644 --- a/src/core/utils.py +++ b/src/core/utils.py @@ -29,6 +29,14 @@ def sysarg_to_string(argv): return result +def get_all_loaded_modules(prefix): + import sys + if prefix: + return [m for m in sys.modules.keys() if m.startswith(prefix)] + else: + return sys.modules.keys() + + def get_class(qname): """ Loads a class from its full qualified name @@ -117,12 +125,31 @@ def get_classes_from_package(package_name): def init_package_import(package_name): pkg = __import__(package_name) prefix = pkg.__name__ + "." + # prefix = package_name + "." for (module_loader, name, ispkg) in pkgutil.iter_modules(pkg.__path__, prefix): importlib.import_module(name) + importlib.import_module(package_name) + + +def import_module_and_sub_module(module_name): + """ + Import the module, and one sub level + :param module_name: + :return: + """ + mod = get_module(module_name) + for (module_loader, name, ispkg) in pkgutil.iter_modules(mod.__path__, module_name + "."): + importlib.import_module(name) def get_sub_classes(package_name, base_class): - base_class = get_class(base_class) if isinstance(base_class, str) else base_class + def _get_class(name): + modname, _, clsname = name.rpartition('.') + mod = importlib.import_module(modname) + cls = getattr(mod, clsname) + return cls + + base_class = _get_class(base_class) if isinstance(base_class, str) else base_class all_class = set(base_class.__subclasses__()).union( [s for c in base_class.__subclasses__() for s in get_sub_classes(package_name, c)]) diff --git a/src/evaluators/PythonEvaluator.py b/src/evaluators/PythonEvaluator.py index d49074a..076eacf 100644 --- a/src/evaluators/PythonEvaluator.py +++ b/src/evaluators/PythonEvaluator.py @@ -70,7 +70,8 @@ class PythonEvaluator(OneReturnValueEvaluator): "concepts": context.sheerka.dump_handler.dump_concepts, "history": context.sheerka.dump_handler.dump_history, "state": context.sheerka.dump_handler.dump_state, - "Concept": core.concept.Concept + "Concept": core.concept.Concept, + "BuiltinConcepts": core.builtin_concepts.BuiltinConcepts, } if context.obj: context.log(f"Concept '{context.obj}' is in context. Adding it and its properties to locals.", self.name) diff --git a/src/parsers/AtomNodeParser.py b/src/parsers/AtomNodeParser.py index e1090db..8585ee5 100644 --- a/src/parsers/AtomNodeParser.py +++ b/src/parsers/AtomNodeParser.py @@ -241,7 +241,6 @@ class AtomNodeParser(BaseNodeParser): :param concept: :return: """ - # return len(concept.metadata.props) == 0 or concept.metadata.definition_type == DEFINITION_TYPE_BNF return len(concept.metadata.variables) == 0 and concept.metadata.definition_type != DEFINITION_TYPE_BNF def get_concepts_sequences(self): @@ -325,6 +324,9 @@ class AtomNodeParser(BaseNodeParser): continue for node in parser_helper.sequence: + if isinstance(node, ConceptNode): + if len(node.concept.metadata.variables) > 0: + node.concept.metadata.is_evaluated = True # Do not try to evaluate those concepts node.tokens = self.tokens[node.start:node.end + 1] node.fix_source() diff --git a/src/parsers/SyaNodeParser.py b/src/parsers/SyaNodeParser.py index 4c65245..581c427 100644 --- a/src/parsers/SyaNodeParser.py +++ b/src/parsers/SyaNodeParser.py @@ -952,107 +952,17 @@ class SyaNodeParser(BaseNodeParser): sya_concept_def.precedence = sya_def[0] if sya_def[1] is not None: sya_concept_def.associativity = sya_def[1] + + if parser.sheerka: + concept_weight = parser.sheerka.get_concepts_weights(BuiltinConcepts.PRECEDENCE) + if concept.id in concept_weight: + sya_concept_def.precedence = concept_weight[concept.id] + + if associativity := concept.get_prop(BuiltinConcepts.ASSOCIATIVITY): + sya_concept_def.associativity = SyaAssociativity(associativity) + return sya_concept_def - # def reset_parser(self, context, text): - # self.context = context - # self.sheerka = context.sheerka - # self.text = text - # - # try: - # self.tokens = list(self.get_input_as_tokens(text)) - # except LexerError as e: - # self.add_error(self.sheerka.new(BuiltinConcepts.ERROR, body=e), False) - # return False - # - # self.token = None - # self.pos = -1 - # return True - # - # def add_error(self, error, next_token=True): - # self.error_sink.append(error) - # if next_token: - # self.next_token() - # return error - # - # def get_token(self) -> Token: - # return self.token - # - # def next_token(self, skip_whitespace=True): - # if self.token and self.token.type == TokenKind.EOF: - # return False - # - # self.pos += 1 - # self.token = self.tokens[self.pos] - # - # if skip_whitespace: - # while self.token.type == TokenKind.WHITESPACE or self.token.type == TokenKind.NEWLINE: - # self.pos += 1 - # self.token = self.tokens[self.pos] - # - # return self.token.type != TokenKind.EOF - - # def initialize(self, context, concepts=None, sya_definitions=None): - # self.context = context - # self.sheerka = context.sheerka - # - # if sya_definitions: - # self.sya_definitions = sya_definitions - # - # if concepts: - # for concept in concepts: - # keywords = concept.key.split() - # for keyword in keywords: - # if keyword.startswith(VARIABLE_PREFIX): - # continue - # - # self.concepts_by_first_keyword.setdefault(keyword, []).append(concept.id) - # break - # - # return self.sheerka.ret(self.name, True, self.concepts_by_first_keyword) - # - # def get_concepts(self, token): - # """ - # Tries to find if there are concepts that match the value of the token - # :param token: - # :return: - # """ - # - # if token.type == TokenKind.STRING: - # name = token.value[1:-1] - # elif token.type == TokenKind.KEYWORD: - # name = token.value.value - # else: - # name = token.value - # - # result = [] - # if name in self.concepts_by_first_keyword: - # for concept_id in self.concepts_by_first_keyword[name]: - # - # concept = self.sheerka.get_by_id(concept_id) - # - # if len(concept.metadata.props) == 0: - # # only concepts that has parameter (refuse atoms) - # # Note that this test is needed if the definition of the concept has changed - # continue - # - # if concept.metadata.definition_type == DEFINITION_TYPE_BNF: - # # bnf definitions are not supposed to be managed by this parser - # continue - # - # sya_concept_def = SyaConceptDef(concept) - # if concept.id in self.sya_definitions: - # sya_def = self.sya_definitions[concept.id] - # if sya_def[0] is not None: - # sya_concept_def.precedence = sya_def[0] - # if sya_def[1] is not None: - # sya_concept_def.associativity = sya_def[1] - # - # result.append(sya_concept_def) - # return result - # - # return None - def infix_to_postfix(self, context, text): """ Implementing Shunting Yard Algorithm diff --git a/src/sdp/readme.md b/src/sdp/readme.md index a36597f..43895e8 100644 --- a/src/sdp/readme.md +++ b/src/sdp/readme.md @@ -13,7 +13,7 @@ - C : concept (with history management) - D : concept definitions (no history management) - R : executionContext ('R' stands for Result or ReturnValue, no history management) -- V : variable (from pickle) +- O : ServiceObj (from pickle) ## How concepts are serialized ? - get the id of the concept diff --git a/src/sdp/sheerkaSerializer.py b/src/sdp/sheerkaSerializer.py index f47007a..d8457b7 100644 --- a/src/sdp/sheerkaSerializer.py +++ b/src/sdp/sheerkaSerializer.py @@ -60,7 +60,7 @@ class Serializer: self.register(ConceptSerializer()) self.register(DictionarySerializer()) self.register(ExecutionContextSerializer()) - self.register(VariableSerializer()) + self.register(ServiceObjSerializer()) def register(self, serializer): """ @@ -288,11 +288,13 @@ class ExecutionContextSerializer(BaseSerializer): return obj -class VariableSerializer(PickleSerializer): +class ServiceObjSerializer(PickleSerializer): + base_class = get_class("core.sheerka.services.sheerka_service.ServiceObj") + def __init__(self): super().__init__( - lambda obj: get_full_qualified_name(obj) == "core.sheerka.Services.SheerkaVariableManager.Variable", - "V", + lambda obj: isinstance(obj, self.base_class), + "O", 1) # diff --git a/tests/core/test_SheerkaComparisonManager.py b/tests/core/test_SheerkaComparisonManager.py new file mode 100644 index 0000000..9cdac4d --- /dev/null +++ b/tests/core/test_SheerkaComparisonManager.py @@ -0,0 +1,149 @@ +import pytest +from core.builtin_concepts import BuiltinConcepts +from core.sheerka.services.SheerkaComparisonManager import SheerkaComparisonManager, ComparisonObj + +from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka + + +class TestSheerkaGreaterThanManager(TestUsingMemoryBasedSheerka): + def test_i_can_add_a_is_greater_than(self): + sheerka, context, one, two = self.init_concepts("one", "two", cache_only=False) + service = sheerka.services[SheerkaComparisonManager.NAME] + + res = service.is_greater_than(context, "prop_name", two, one) + assert res.status + assert sheerka.isinstance(res.body, BuiltinConcepts.SUCCESS) + + in_cache = sheerka.cache_manager.get(SheerkaComparisonManager.COMPARISON_ENTRY, "prop_name|#") + assert in_cache == [ComparisonObj(context.event.get_digest(), "prop_name", two.id, one.id, ">", "#")] + + weighted = sheerka.cache_manager.get(SheerkaComparisonManager.RESOLVED_COMPARISON_ENTRY, "prop_name|#") + assert weighted == {"1001": 1, "1002": 2} + + # I can commit + sheerka.cache_manager.commit(context) + in_db = sheerka.sdp.get(SheerkaComparisonManager.COMPARISON_ENTRY, "prop_name|#") + assert in_db == [ComparisonObj(context.event.get_digest(), "prop_name", two.id, one.id, ">", "#")] + + def test_i_can_add_a_is_less_than(self): + sheerka, context, one, two = self.init_concepts("one", "two", cache_only=False) + service = sheerka.services[SheerkaComparisonManager.NAME] + + res = service.is_less_than(context, "prop_name", one, two) + assert res.status + assert sheerka.isinstance(res.body, BuiltinConcepts.SUCCESS) + + in_cache = sheerka.cache_manager.get(SheerkaComparisonManager.COMPARISON_ENTRY, "prop_name|#") + assert in_cache == [ComparisonObj(context.event.get_digest(), "prop_name", one.id, two.id, "<", "#")] + + weighted = sheerka.cache_manager.get(SheerkaComparisonManager.RESOLVED_COMPARISON_ENTRY, "prop_name|#") + assert weighted == {"1001": 1, "1002": 2} + + # I can commit + sheerka.cache_manager.commit(context) + in_db = sheerka.sdp.get(SheerkaComparisonManager.COMPARISON_ENTRY, "prop_name|#") + assert in_db == [ComparisonObj(context.event.get_digest(), "prop_name", one.id, two.id, "<", "#")] + + def test_i_can_add_multiples_constraints(self): + sheerka, context, one, two, three, four = self.init_concepts("one", "two", "three", "four", cache_only=False) + service = sheerka.services[SheerkaComparisonManager.NAME] + + service.is_greater_than(context, "prop_name", two, one) + service.is_greater_than(context, "prop_name", three, two) + + in_cache = sheerka.cache_manager.get(SheerkaComparisonManager.COMPARISON_ENTRY, "prop_name|#") + assert in_cache == [ + ComparisonObj(context.event.get_digest(), "prop_name", two.id, one.id, ">", "#"), + ComparisonObj(context.event.get_digest(), "prop_name", three.id, two.id, ">", "#") + ] + + # I can commit + sheerka.cache_manager.commit(context) + in_db = sheerka.sdp.get(SheerkaComparisonManager.COMPARISON_ENTRY, "prop_name|#") + assert in_db == [ + ComparisonObj(context.event.get_digest(), "prop_name", two.id, one.id, ">", "#"), + ComparisonObj(context.event.get_digest(), "prop_name", three.id, two.id, ">", "#") + ] + + sheerka.cache_manager.clear(SheerkaComparisonManager.COMPARISON_ENTRY) # reset the cache + + service.is_greater_than(context, "prop_name", four, three) + in_cache = sheerka.cache_manager.get(SheerkaComparisonManager.COMPARISON_ENTRY, "prop_name|#") + assert in_cache == [ + ComparisonObj(context.event.get_digest(), "prop_name", two.id, one.id, ">", "#"), + ComparisonObj(context.event.get_digest(), "prop_name", three.id, two.id, ">", "#"), + ComparisonObj(context.event.get_digest(), "prop_name", four.id, three.id, ">", "#"), + ] + + weighted = sheerka.cache_manager.get(SheerkaComparisonManager.RESOLVED_COMPARISON_ENTRY, "prop_name|#") + assert weighted == {"1001": 1, "1002": 2, "1003": 3, "1004": 4} + + @pytest.mark.parametrize("entries, expected", [ + (["two > one"], {'1001': 1, '1002': 2}), + (["one < two"], {'1001': 1, '1002': 2}), + (["three > two", "one < two"], {'1001': 1, '1002': 2, '1003': 3}), + (["three > one", "one < two"], {'1001': 1, '1002': 2, '1003': 2}), + ]) + def test_i_can_get_concept_weight(self, entries, expected): + sheerka, context, *concepts = self.init_concepts("one", "two", "three") + service = sheerka.services[SheerkaComparisonManager.NAME] + concepts_map = dict(zip(["one", "two", "three", "four", "five", "six"], concepts)) + + for entry in entries: + if ">" in entry: + a, b = [concepts_map[e.strip()] for e in entry.split(">")] + service.is_greater_than(context, "prop_name", a, b) + else: + a, b = [concepts_map[e.strip()] for e in entry.split("<")] + service.is_less_than(context, "prop_name", a, b) + + assert service.get_concepts_weights("prop_name") == expected + + def test_i_can_get_partition(self): + sheerka, context, one, two, three = self.init_concepts("one", "two", "three") + service = sheerka.services[SheerkaComparisonManager.NAME] + + service.is_greater_than(context, "prop_name", two, one) + service.is_less_than(context, "prop_name", two, three) + + res = service.get_partition("prop_name") + + assert res == { + 1: ["1001"], + 2: ["1002"], + 3: ["1003"], + } + + def test_i_can_detect_chicken_and_egg(self): + sheerka, context, one, two = self.init_concepts("one", "two") + service = sheerka.services[SheerkaComparisonManager.NAME] + + service.is_greater_than(context, "prop_name", two, one) + res = service.is_greater_than(context, "prop_name", one, two) + + assert not res.status + assert sheerka.isinstance(res.body, BuiltinConcepts.CHICKEN_AND_EGG) + assert set(res.body.body) == {one, two} + + def test_i_can_detect_more_complex_chicken_and_egg(self): + sheerka, context, one, two, three, four, five = self.init_concepts("one", "two", "three", "four", "five") + service = sheerka.services[SheerkaComparisonManager.NAME] + + service.is_greater_than(context, "prop_name", two, one) + service.is_greater_than(context, "prop_name", five, four) + service.is_greater_than(context, "prop_name", four, three) + service.is_greater_than(context, "prop_name", five, two) + + res = service.is_greater_than(context, "prop_name", two, one) + assert res.status + + res = service.is_greater_than(context, "prop_name", one, five) + + assert not res.status + assert sheerka.isinstance(res.body, BuiltinConcepts.CHICKEN_AND_EGG) + assert set(res.body.body) == {one, two, five} + + def test_methods_are_correctly_bound(self): + sheerka, context, one, two = self.init_concepts("one", "two") + res = sheerka.is_greater_than(context, "prop_name", two, one) + assert res.status diff --git a/tests/core/test_SheerkaHistoryManager.py b/tests/core/test_SheerkaHistoryManager.py index b1dbee6..90e8878 100644 --- a/tests/core/test_SheerkaHistoryManager.py +++ b/tests/core/test_SheerkaHistoryManager.py @@ -1,4 +1,4 @@ -from core.sheerka.Services.SheerkaHistoryManager import hist +from core.sheerka.services.SheerkaHistoryManager import hist from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka diff --git a/tests/core/test_SheerkaVariableManager.py b/tests/core/test_SheerkaVariableManager.py index bba3cb7..b8982f4 100644 --- a/tests/core/test_SheerkaVariableManager.py +++ b/tests/core/test_SheerkaVariableManager.py @@ -16,8 +16,8 @@ class TestSheerkaVariable(TestUsingMemoryBasedSheerka): # I can persist in db sheerka.cache_manager.commit(context) - assert sheerka.sdp.exists(Sheerka.VARIABLES_ENTRY, "TestSheerkaVariable.my_variable") - loaded = sheerka.sdp.get(Sheerka.VARIABLES_ENTRY, "TestSheerkaVariable.my_variable") + assert sheerka.sdp.exists(Sheerka.VARIABLES_ENTRY, "TestSheerkaVariable|my_variable") + loaded = sheerka.sdp.get(Sheerka.VARIABLES_ENTRY, "TestSheerkaVariable|my_variable") assert loaded.event_id == context.event.get_digest() assert loaded.key == "my_variable" assert loaded.value == 1 diff --git a/tests/core/test_sheerka.py b/tests/core/test_sheerka.py index 36ed7a0..e889117 100644 --- a/tests/core/test_sheerka.py +++ b/tests/core/test_sheerka.py @@ -16,6 +16,13 @@ class ConceptWithGetObjValue(Concept): class TestSheerkaUsingMemoryBasedSheerka(TestUsingMemoryBasedSheerka): + def test_i_can_initialize_services(self): + sheerka = self.get_sheerka() + + assert len(sheerka.services) > 0 + assert None not in sheerka.services + assert "ComparisonManager" in sheerka.services # test at least one service + def test_i_can_initialize_builtin_parsers(self): sheerka = self.get_sheerka() diff --git a/tests/non_reg/test_sheerka_non_reg.py b/tests/non_reg/test_sheerka_non_reg.py index 85b1e45..ae13c66 100644 --- a/tests/non_reg/test_sheerka_non_reg.py +++ b/tests/non_reg/test_sheerka_non_reg.py @@ -855,6 +855,24 @@ as: assert res[0].status assert res[0].body == 64 + def test_concepts_parsed_by_atom_parser_must_not_be_evaluated(self): + definitions = [ + "def concept mult from a mult b as a * b", + "def concept a mult b as a * b", + ] + + sheerka = self.init_scenario(definitions) + + res = sheerka.evaluate_user_input("eval mult") + + assert res[0].status + assert isinstance(res[0].body, Concept) + + # res = sheerka.evaluate_user_input("eval a mult b") + # + # assert res[0].status + # assert isinstance(res[0].body, Concept) + class TestSheerkaNonRegFile(TestUsingFileBasedSheerka): def test_i_can_def_several_concepts(self): diff --git a/tests/parsers/test_AtomsParser.py b/tests/parsers/test_AtomsParser.py index 3d0e35e..0ffca77 100644 --- a/tests/parsers/test_AtomsParser.py +++ b/tests/parsers/test_AtomsParser.py @@ -1,22 +1,25 @@ import pytest from core.builtin_concepts import BuiltinConcepts -from core.concept import Concept +from core.concept import Concept, DEFINITION_TYPE_DEF from parsers.AtomNodeParser import AtomNodeParser -from parsers.BaseNodeParser import cnode, utnode, CNC, scnode, SCN +from parsers.BaseNodeParser import cnode, utnode, CNC, SCN from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka from tests.parsers.parsers_utils import compute_expected_array class TestAtomsParser(TestUsingMemoryBasedSheerka): - def init_parser(self, my_map, create_new=False, singleton=True): + def init_parser(self, my_map, create_new=False, singleton=True, use_sheerka=False): sheerka, context, *updated_concepts = self.init_concepts( *my_map.values(), create_new=create_new, singleton=singleton) - parser = AtomNodeParser() - parser.init_from_concepts(context, updated_concepts) + if use_sheerka: + parser = AtomNodeParser(sheerka=sheerka) + else: + parser = AtomNodeParser() + parser.init_from_concepts(context, updated_concepts) return sheerka, context, parser @@ -32,15 +35,20 @@ class TestAtomsParser(TestUsingMemoryBasedSheerka): ("foo", ["foo"]), ("foo bar", ["foo", "bar"]), ("foo bar twenties", ["foo", "bar", "twenties"]), + # ("plus", ["plus"]), + # ("++", ["++"]), + # ("a++ foo", ["++", "foo"]), ]) def test_i_can_parse_simple_sequences(self, text, expected): concepts_map = { "foo": Concept("foo"), "bar": Concept("bar"), + "plus": Concept("a plus b").def_var("a").def_var("b"), + "++": Concept("++", definition="a++", definition_type=DEFINITION_TYPE_DEF).def_var("a"), "twenties": Concept("twenties", definition="'twenty' ('one'|'two')=unit").def_var("unit"), } - sheerka, context, parser = self.init_parser(concepts_map) + sheerka, context, parser = self.init_parser(concepts_map, create_new=True, use_sheerka=True) res = parser.parse(context, text) wrapper = res.body lexer_nodes = res.body.body @@ -276,3 +284,27 @@ class TestAtomsParser(TestUsingMemoryBasedSheerka): expected_array = compute_expected_array(concepts_map, text, expected) assert sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT) assert lexer_nodes == expected_array + + @pytest.mark.parametrize("text, expected_is_evaluated", [ + ("foo", False), + ("bar", False ), + ("twenties", True), + ("plus", True), + # ("plus", ["plus"]), + # ("++", ["++"]), + # ("a++ foo", ["++", "foo"]), + ]) + def test_concepts_with_variables_must_not_be_evaluated(self, text, expected_is_evaluated): + concepts_map = { + "foo": Concept("foo"), + "bar": Concept("bar", body="'bar'"), + "plus": Concept("plus", definition="a plus b", definition_type=DEFINITION_TYPE_DEF).def_var("a").def_var("b"), + "twenties": Concept("twenties", definition="'twenty' ('one'|'two')=unit").def_var("unit"), + } + + sheerka, context, parser = self.init_parser(concepts_map, create_new=True, use_sheerka=True) + res = parser.parse(context, text) + lexer_nodes = res.body.body + + assert res.status + assert lexer_nodes[0].concept.metadata.is_evaluated == expected_is_evaluated diff --git a/tests/parsers/test_SyaNodeParser.py b/tests/parsers/test_SyaNodeParser.py index 682a3e9..e1c7499 100644 --- a/tests/parsers/test_SyaNodeParser.py +++ b/tests/parsers/test_SyaNodeParser.py @@ -1,6 +1,7 @@ import pytest from core.builtin_concepts import BuiltinConcepts from core.concept import Concept, CC +from core.sheerka.services.SheerkaComparisonManager import SheerkaComparisonManager from core.tokenizer import Tokenizer from parsers.BaseNodeParser import utnode, ConceptNode, cnode, short_cnode, UnrecognizedTokensNode, \ SCWC, CNC, UTN @@ -48,10 +49,20 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka): create_new=True, init_from_sheerka=True) - TestSyaNodeParser.sheerka.force_sya_def(context, [ - (cmap["plus"].id, 5, SyaAssociativity.Right), - (cmap["mult"].id, 10, SyaAssociativity.Right), - (cmap["minus"].id, 10, SyaAssociativity.Right)]) + cmap["plus"].set_prop(BuiltinConcepts.ASSOCIATIVITY, "right") + cmap["mult"].set_prop(BuiltinConcepts.ASSOCIATIVITY, "right") + cmap["minus"].set_prop(BuiltinConcepts.ASSOCIATIVITY, "right") + TestSyaNodeParser.sheerka.services[SheerkaComparisonManager.NAME].is_greater_than(context, + BuiltinConcepts.PRECEDENCE, + cmap["mult"], cmap["plus"]) + TestSyaNodeParser.sheerka.services[SheerkaComparisonManager.NAME].is_greater_than(context, + BuiltinConcepts.PRECEDENCE, + cmap["mult"], cmap["minus"]) + + # TestSyaNodeParser.sheerka.force_sya_def(context, [ + # (cmap["plus"].id, 5, SyaAssociativity.Right), + # (cmap["mult"].id, 10, SyaAssociativity.Right), + # (cmap["minus"].id, 5, SyaAssociativity.Right)]) def init_parser(self, my_concepts_map=None,