import re from dataclasses import dataclass from typing import Set, List, Union import core.utils from cache.Cache import Cache from cache.CacheManager import ConceptNotFound from cache.DictionaryCache import DictionaryCache from cache.ListIfNeededCache import ListIfNeededCache from cache.SetCache import SetCache from core.builtin_concepts import ErrorConcept from core.builtin_concepts_ids import BuiltinConcepts, AllBuiltinConcepts, BuiltinUnique from core.builtin_helpers import ensure_concept, ensure_bnf from core.concept import Concept, DEFINITION_TYPE_DEF, DEFINITION_TYPE_BNF, freeze_concept_attrs, ConceptMetadata, \ VARIABLE_PREFIX from core.global_symbols import EVENT_CONCEPT_CREATED, NotInit, NotFound, ErrorObj, EVENT_CONCEPT_DELETED, NoFirstToken, \ EVENT_CONCEPT_MODIFIED, CONCEPT_COMPARISON_CONTEXT from core.sheerka.services.sheerka_service import BaseService from core.tokenizer import Tokenizer, TokenKind from parsers.BnfNodeParser import RegExDef from sdp.sheerkaDataProvider import SheerkaDataProviderDuplicateKeyError BASE_NODE_PARSER_CLASS = "parsers.BaseNodeParser.BaseNodeParser" @dataclass class ChickenAndEggError(Exception): concepts: Set[str] @dataclass class NoFirstTokenError(ErrorObj): concept: Concept key: str @dataclass class NoModificationFound(ErrorObj): """ Trying to modify a concept without modifying it """ concept: Concept attrs: dict = None @dataclass class ForbiddenAttribute(ErrorObj): """ When trying to modify an attribute that must not be modified """ attr: str @dataclass class UnknownAttribute(ErrorObj): """ When trying to modify an attribute that does not exist """ attr: str @dataclass class CannotRemoveMeta(ErrorObj): """ When trying to remove a metadata attribute (ConceptMeta is a class, you cannot remove attr form it) """ attrs: dict @dataclass class ValueNotFound(ErrorObj): """ When trying to remove a value that does not exists (but the props/variable exists) """ item: str value: object @dataclass class ConceptIsReferenced(ErrorObj): """ When trying to remove a concept that is referenced by other concept(s) """ references: list class SheerkaConceptManager(BaseService): NAME = "ConceptManager" BUILTIN_CONCEPTS_IDS = "Builtins_Concepts_IDs" # sequential key for builtin concepts USER_CONCEPTS_IDS = "User_Concepts_IDs" # sequential key for user defined concepts CONCEPTS_IDS_ENTRY = "ConceptManager:Concepts_IDs" CONCEPTS_BY_ID_ENTRY = "ConceptManager:Concepts_By_ID" # to store all the concepts CONCEPTS_BY_KEY_ENTRY = "ConceptManager:Concepts_By_Key" CONCEPTS_BY_NAME_ENTRY = "ConceptManager:Concepts_By_Name" CONCEPTS_BY_HASH_ENTRY = "ConceptManager:Concepts_By_Hash" # store hash of concepts definitions (not values) CONCEPTS_REFERENCES_ENTRY = "ConceptManager:Concepts_References" # tracks references between concepts CONCEPTS_BY_FIRST_KEYWORD_ENTRY = "ConceptManager:Concepts_By_First_Keyword" RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY = "ConceptManager:Resolved_Concepts_By_First_Keyword" CONCEPTS_BY_REGEX_ENTRY = "ConceptManager:Concepts_By_Regex" CONCEPTS_BNF_DEFINITIONS_ENTRY = "ConceptManager:Concepts_BNF_Definitions" def __init__(self, sheerka): super().__init__(sheerka, order=11) self.forbidden_meta = {"is_builtin", "key", "id", "props", "variables"} self.allowed_meta = {attr for attr in vars(ConceptMetadata) if not attr.startswith("_") and attr not in self.forbidden_meta} self.compiled_concepts_by_regex = [] def initialize(self): self.sheerka.bind_service_method(self.NAME, self.create_new_concept, True) self.sheerka.bind_service_method(self.NAME, self.modify_concept, True) self.sheerka.bind_service_method(self.NAME, self.remove_concept, True) self.sheerka.bind_service_method(self.NAME, self.set_id_if_needed, True) self.sheerka.bind_service_method(self.NAME, self.set_attr, True) self.sheerka.bind_service_method(self.NAME, self.get_attr, False) self.sheerka.bind_service_method(self.NAME, self.smart_get_attr, False) self.sheerka.bind_service_method(self.NAME, self.set_property, True, as_name="set_prop") self.sheerka.bind_service_method(self.NAME, self.get_property, False, as_name="get_prop") self.sheerka.bind_service_method(self.NAME, self.get_by_key, False, visible=False) self.sheerka.bind_service_method(self.NAME, self.get_by_name, False, visible=False) self.sheerka.bind_service_method(self.NAME, self.get_by_hash, False, visible=False) self.sheerka.bind_service_method(self.NAME, self.get_by_id, False, visible=False) self.sheerka.bind_service_method(self.NAME, self.is_not_a_concept_name, False, visible=False) self.sheerka.bind_service_method(self.NAME, self.is_a_concept_name, False, visible=False) self.sheerka.bind_service_method(self.NAME, self.get_concepts_by_first_token, False, visible=False) self.sheerka.bind_service_method(self.NAME, self.get_concepts_by_first_regex, False, visible=False) self.sheerka.bind_service_method(self.NAME, self.get_concepts_bnf_definitions, False, visible=False) self.sheerka.bind_service_method(self.NAME, self.clear_bnf_definition, True, visible=False) self.sheerka.bind_service_method(self.NAME, self.set_precedence, True) register_concept_cache = self.sheerka.om.register_concept_cache cache = Cache().auto_configure(self.CONCEPTS_BY_ID_ENTRY) register_concept_cache(self.CONCEPTS_BY_ID_ENTRY, cache, lambda c: c.id, True) cache = ListIfNeededCache().auto_configure(self.CONCEPTS_BY_KEY_ENTRY) register_concept_cache(self.CONCEPTS_BY_KEY_ENTRY, cache, lambda c: c.key, True) cache = ListIfNeededCache().auto_configure(self.CONCEPTS_BY_NAME_ENTRY) register_concept_cache(self.CONCEPTS_BY_NAME_ENTRY, cache, lambda c: c.name, True) cache = ListIfNeededCache().auto_configure(self.CONCEPTS_BY_HASH_ENTRY) register_concept_cache(self.CONCEPTS_BY_HASH_ENTRY, cache, lambda c: c.get_definition_hash(), True) cache = SetCache().auto_configure(self.CONCEPTS_REFERENCES_ENTRY) self.sheerka.om.register_cache(self.CONCEPTS_REFERENCES_ENTRY, cache) cache = DictionaryCache().auto_configure(self.CONCEPTS_BY_FIRST_KEYWORD_ENTRY) self.sheerka.om.register_cache(self.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, cache) cache = DictionaryCache().auto_configure(self.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY) self.sheerka.om.register_cache(self.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY, cache, persist=False) cache = DictionaryCache().auto_configure(self.CONCEPTS_BY_REGEX_ENTRY) self.sheerka.om.register_cache(self.CONCEPTS_BY_REGEX_ENTRY, cache) cache = Cache().auto_configure(self.CONCEPTS_BNF_DEFINITIONS_ENTRY) self.sheerka.om.register_cache(self.CONCEPTS_BNF_DEFINITIONS_ENTRY, cache, persist=False) def initialize_deferred(self, context, is_first_time): if is_first_time: self.sheerka.om.put(self.sheerka.OBJECTS_IDS_ENTRY, self.USER_CONCEPTS_IDS, 1000) # initialize the dictionary of first tokens self.sheerka.om.get(self.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, None) # to init the cache with the values from sdp concepts_by_first_keyword = self.sheerka.om.current_cache_manager().copy(self.CONCEPTS_BY_FIRST_KEYWORD_ENTRY) res = self.resolve_concepts_by_first_keyword(context, concepts_by_first_keyword) self.sheerka.om.put(self.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY, False, res.body) # init the regular expression self.sheerka.om.get(self.CONCEPTS_BY_REGEX_ENTRY, None) from_db = self.sheerka.om.current_cache_manager().copy(self.CONCEPTS_BY_REGEX_ENTRY) concepts_by_first_regex = {RegExDef().deserialize(k): v for k, v in from_db.items()} res = self.compile_concepts_by_first_regex(context, concepts_by_first_regex) self.compiled_concepts_by_regex.clear() self.compiled_concepts_by_regex.extend(res.body) def initialize_builtin_concepts(self): """ Initializes the builtin concepts :return: None """ builtin_concepts_ids = {} for key in AllBuiltinConcepts: concept = self.sheerka if key == BuiltinConcepts.SHEERKA \ else self.sheerka.builtin_cache[str(key)]() if str(key) in self.sheerka.builtin_cache \ else Concept(key, True, False, key) if key in BuiltinUnique: concept.get_metadata().is_unique = True concept.get_hints().is_evaluated = True from_db = self.sheerka.om.get(self.CONCEPTS_BY_KEY_ENTRY, concept.get_metadata().key) if from_db is NotFound: # self.init_log.debug(f"'{concept.name}' concept is not found in db. Adding.") self.set_id_if_needed(concept, True) self.sheerka.om.add_concept(concept) else: # self.init_log.debug(f"Found concept '{from_db}' in db. Updating.") concept.update_from(from_db) builtin_concepts_ids[key] = concept.id return builtin_concepts_ids def create_new_concept(self, context, concept: Concept): """ Adds a new concept to the system :param context: :param concept: DefConceptNode :return: digest of the new concept """ ensure_concept(concept) sheerka = self.sheerka concept.init_key() init_bnf_ret_value = None om = sheerka.om if om.exists(self.CONCEPTS_BY_HASH_ENTRY, concept.get_definition_hash()): error = SheerkaDataProviderDuplicateKeyError(self.CONCEPTS_BY_KEY_ENTRY + "." + concept.key, concept) return sheerka.ret( self.NAME, False, sheerka.new(BuiltinConcepts.CONCEPT_ALREADY_DEFINED, body=concept), error.args[0]) # set id before saving in db sheerka.set_id_if_needed(concept, False) # check if the bnf definition is correctly computed try: ensure_bnf(context, concept) except Exception as ex: return sheerka.ret(self.NAME, False, ex.args[0]) # recompute concepts by first tokens and concept by first regex update_items_res = self.recompute_first_items(context, None, [concept]) if not update_items_res.status: return update_items_res by_first_keyword, by_first_regex, resolved_by_first_keyword, compiled_by_first_regex = update_items_res.body # if everything is fine freeze_concept_attrs(concept) concept.freeze_definition_hash() om.add_concept(concept) self.update_first_items_caches(context, by_first_keyword, by_first_regex, resolved_by_first_keyword, compiled_by_first_regex) if concept.get_metadata().definition_type == DEFINITION_TYPE_DEF and concept.get_metadata().definition != concept.name: # allow search by definition when definition relevant om.put(self.sheerka.CONCEPTS_BY_NAME_ENTRY, concept.get_metadata().definition, concept) # update references for ref in self.compute_references(concept): om.put(self.CONCEPTS_REFERENCES_ENTRY, ref, concept.id) # TODO : this line seems to be useless # The grammar is never reset if concept.get_bnf() and init_bnf_ret_value is not None and init_bnf_ret_value.status: sheerka.cache_manager.clear(self.CONCEPTS_BNF_DEFINITIONS_ENTRY) # publish the new concept sheerka.publish(context, EVENT_CONCEPT_CREATED, concept) return sheerka.ret(self.NAME, True, sheerka.new(BuiltinConcepts.NEW_CONCEPT, body=concept)) def modify_concept(self, context, concept, to_add=None, to_remove=None, modify_source=False): """ Modify the definition of a concept :param context: :param concept: concept to modify :param to_add: meta, props or variables to add/update :param to_remove: props or variables to remove :param modify_source: update or not the concept given in parameter :return: """ # to_add is a dictionary # to_add = { # 'meta' : {: } of metadata to update, # 'props' : {: } of properties to add/update, # 'variables': {: } of variables to add/update, # } # for variables, if the already exists, the entry is updated, otherwise a new value is created # for props, if the already exists, a new entry is added to the set # # to_remove = { # 'props' : {: [value]} entries to remove. 'value' can be a list or a single entry # 'variables': [] list of keys to remove # } # sheerka = self.sheerka om = self.sheerka.om if not to_add and not to_remove: return sheerka.ret(self.NAME, False, sheerka.err(NoModificationFound(concept))) if not sheerka.om.exists(self.CONCEPTS_BY_ID_ENTRY, concept.id): return sheerka.ret(self.NAME, False, sheerka.new(BuiltinConcepts.UNKNOWN_CONCEPT, body=concept)) # modify the metadata. Almost all ConceptMetadata attributes except variables and props new_concept = sheerka.new_from_template(concept, concept.key) # reload from cache or database ? res = self._update_concept(context, new_concept, to_add, to_remove) if res is not None: return res # recompute concepts by first tokens and concept by first regex update_items_res = self.recompute_first_items(context, concept, [new_concept]) if not update_items_res.status: return update_items_res by_first_keyword, by_first_regex, resolved_by_first_keyword, compiled_by_first_regex = update_items_res.body # update concepts that referenced the old concept and clear old references self.update_references(context, concept, new_concept, to_add) for ref in self.compute_references(concept): om.delete(self.CONCEPTS_REFERENCES_ENTRY, ref, concept.id) # compute new references for ref in self.compute_references(new_concept): om.put(self.CONCEPTS_REFERENCES_ENTRY, ref, new_concept.id) # everything is ok, update the caches om.update_concept(concept, new_concept) self.update_first_items_caches(context, by_first_keyword, by_first_regex, resolved_by_first_keyword, compiled_by_first_regex) # everything seems to be fine. Update the list of attributes # Caution. Must be done AFTER update_concept() freeze_concept_attrs(new_concept) # TODO : update when definition_type = DEFINITION_TYPE_DEF : have a look at update_references() below # TODO : Update concepts grammars : have a look at update_references() below if modify_source: self._update_concept(context, concept, to_add, to_remove) sheerka.publish(context, EVENT_CONCEPT_MODIFIED, {"old": concept, "new": new_concept}) return sheerka.ret(self.NAME, True, sheerka.new(BuiltinConcepts.NEW_CONCEPT, body=new_concept)) def remove_concept(self, context, concept): """ Remove a concept :param context: :param concept: :return: """ # TODO : resolve concept first sheerka = context.sheerka if not sheerka.is_known(concept): return sheerka.ret(self.NAME, False, sheerka.err(ConceptNotFound(concept))) om = sheerka.om refs = om.get(self.CONCEPTS_REFERENCES_ENTRY, concept.id) if refs is not NotFound: refs_instances = [sheerka.new_from_template(c, c.key) for c in [self.get_by_id(ref) for ref in refs]] return sheerka.ret(self.NAME, False, sheerka.err(ConceptIsReferenced(refs_instances))) # recompute concepts by first tokens and concept by first regex update_items_res = self.recompute_first_items(context, concept, None) if not update_items_res.status: return update_items_res by_first_keyword, by_first_regex, resolved_by_first_keyword, compiled_by_first_regex = update_items_res.body # everything seems fine. I can commit the modification and remove om.remove_concept(concept) self.update_first_items_caches(context, by_first_keyword, by_first_regex, resolved_by_first_keyword, compiled_by_first_regex) sheerka.publish(context, EVENT_CONCEPT_DELETED, concept) return sheerka.ret(self.NAME, True, sheerka.new(BuiltinConcepts.SUCCESS)) def set_attr(self, context, concept, attribute, value): """ Modifies an attribute of a concept (concept.values) :param context: :param concept: :param attribute: :param value: :return: """ ensure_concept(concept) attr = attribute.str_id if isinstance(attribute, Concept) else attribute old_value = concept.get_value(attr) if context.in_context(BuiltinConcepts.EVAL_GLOBAL_TRUTH_REQUESTED): old_value = self.sheerka.get_by_id(concept.id).get_value(attr) else: old_value = concept.get_value(attr) # Caution, creating a list when a set_attr is called for the second time is not always # what is expected if old_value is not NotInit: if old_value == value: return self.sheerka.ret(self.NAME, True, self.sheerka.new(BuiltinConcepts.SUCCESS)) if isinstance(old_value, list): old_value.append(value) value = old_value else: value = [old_value, value] if context.in_context(BuiltinConcepts.EVAL_GLOBAL_TRUTH_REQUESTED): to_add = {"variables": {attr: value}} res = self.sheerka.modify_concept(context, concept, to_add, modify_source=True) concept.set_value(attr, value) return res else: concept.set_value(attr, value) return self.sheerka.ret(self.NAME, True, self.sheerka.new(BuiltinConcepts.SUCCESS)) def get_attr(self, concept, attribute): """ Returns the attribute of a concept :param concept: :param attribute: :return: """ ensure_concept() if not self.sheerka.is_success(concept): return concept attr = attribute.str_id if isinstance(attribute, Concept) else attribute if (value := concept.get_value(attr)) == NotInit: return self.sheerka.new(BuiltinConcepts.NOT_FOUND, body={"#concept": concept, "#attr": attribute}) return value def smart_get_attr(self, concept, attribute): def get_obj_value(c, concept_to_look_for): """ Return the body of the concept c if it is an instance of concept_to_look_for Go deeper in the bodies until the concept_to_look_for is found :param c: :param concept_to_look_for: :return: """ while isinstance(c, Concept) and c.body is not NotInit: if self.sheerka.isinstance(c.body, concept_to_look_for): return c.body c = c.body return None ensure_concept() if not self.sheerka.is_success(concept): return concept value = self.get_attr(concept, attribute) if not self.sheerka.isinstance(value, BuiltinConcepts.NOT_FOUND): return value if not isinstance(attribute, Concept): return self.sheerka.new(BuiltinConcepts.NOT_FOUND, body={"#concept": concept, "#attr": attribute}) # try to be smart result = [] # first look in children for k, v in concept.variables().items(): attr_as_concept_key, attr_concept_id = core.utils.unstr_concept(k) if attr_concept_id is not None: attr_as_concept = self.sheerka.fast_resolve((attr_as_concept_key, attr_concept_id), return_new=False) if attr_as_concept is not None and self.sheerka.isa(attribute, attr_as_concept): if hasattr(v, "__iter__"): for _v in v: value = get_obj_value(_v, attribute) if value is not None: result.append(value) else: value = get_obj_value(v, attribute) if value is not None: result.append(value) if len(result) > 0: return result[0] if len(result) == 1 else result # then try the ancestors for k, v in concept.variables().items(): attr_as_concept_key, attr_concept_id = core.utils.unstr_concept(k) if attr_concept_id is not None: attr_as_concept = self.sheerka.fast_resolve((attr_as_concept_key, attr_concept_id), return_new=False) if attr_as_concept is not None and self.sheerka.isa(attr_as_concept, attribute): if isinstance(v, list): result.extend(v) else: result.append(v) if len(result) > 0: return result[0] if len(result) == 1 else result return self.sheerka.new(BuiltinConcepts.NOT_FOUND, body={"#concept": concept, "#attr": attribute}) def get_property(self, concept, prop): """ Returns the value of a concept property :param concept: :param prop: :return: """ ensure_concept() if not self.sheerka.is_success(concept): return concept if isinstance(prop, Concept) and prop.get_metadata().is_builtin: prop = prop.key if (value := concept.get_prop(prop)) is None: return self.sheerka.new(BuiltinConcepts.NOT_FOUND, body={"#concept": concept, "#prop": prop}) return value def set_property(self, context, concept, prop, value, all_concepts=False): """ Set the value of a concept property The concept is modified :param context: :param concept: :param prop: :param value: :param all_concepts: if True, updates the definitions of the concept :return: """ ensure_concept() if not self.sheerka.is_success(concept): return concept if isinstance(prop, Concept) and prop.get_metadata().is_builtin: prop = prop.key if all_concepts: return self.modify_concept(context, concept, to_add={'props': {prop: value}}, modify_source=True) else: concept.set_prop(prop, value) return self.sheerka.ret(self.NAME, True, self.sheerka.new(BuiltinConcepts.SUCCESS)) def set_id_if_needed(self, obj: Concept, is_builtin: bool): """ Set the key for the concept if needed :param obj: :param is_builtin: :return: """ if obj.get_metadata().id is not None: return entry_key = self.BUILTIN_CONCEPTS_IDS if is_builtin else self.USER_CONCEPTS_IDS obj.get_metadata().id = str(self.sheerka.om.get(self.sheerka.OBJECTS_IDS_ENTRY, entry_key)) # self.log.debug(f"Setting id '{obj.metadata.id}' to concept '{obj.metadata.name}'.") def get_by_key(self, concept_key, concept_id=None): concept_key = str(concept_key) if isinstance(concept_key, BuiltinConcepts) else concept_key return self.internal_get("key", concept_key, self.CONCEPTS_BY_KEY_ENTRY, concept_id) def get_by_name(self, concept_name, concept_id=None): return self.internal_get("name", concept_name, self.CONCEPTS_BY_NAME_ENTRY, concept_id) def get_by_hash(self, concept_hash, concept_id=None): return self.internal_get("hash", concept_hash, self.CONCEPTS_BY_HASH_ENTRY, concept_id) def get_by_id(self, concept_id): return self.internal_get("id", concept_id, self.CONCEPTS_BY_ID_ENTRY, None) def has_id(self, concept_id): """ Returns True if a concept with this id exists in cache It does not search in the remote repository :param concept_id: :return: """ if concept_id is None: return False return self.sheerka.om.current_cache_manager().has(self.CONCEPTS_BY_ID_ENTRY, concept_id) def has_key(self, concept_key): """ Returns True if concept(s) with this key exist in cache It does not search in the remote repository :param concept_key: :return: """ return self.sheerka.om.current_cache_manager().has(self.CONCEPTS_BY_KEY_ENTRY, concept_key) def has_name(self, concept_name): """ Returns True if concept(s) with this name exist in cache It does not search in the remote repository :param concept_name: :return: """ return self.sheerka.om.current_cache_manager().has(self.CONCEPTS_BY_NAME_ENTRY, concept_name) def has_hash(self, concept_hash): """ Returns True if concept(s) with this hash exist in cache It does not search in the remote repository :param concept_hash: :return: """ return self.sheerka.om.current_cache_manager().has(self.CONCEPTS_BY_HASH_ENTRY, concept_hash) def internal_get(self, index_name, key, cache_name, concept_id=None): """ Tries to find an entry :param index_name: name of the index (ex by_id, by_key...) :param key: index value :param cache_name: name of the cache (ex Concepts_By_ID...) :param concept_id: id of the concept if none, in case where there are multiple results :return: """ if key is None: return ErrorConcept(f"Concept '{key}' is undefined.") concepts = self.sheerka.om.get(cache_name, key) if concepts is not NotFound: if concept_id is None: return concepts if not hasattr(concepts, "__iter__"): return concepts for c in concepts: if c.id == concept_id: return c metadata = {index_name: key, "id": concept_id} if concept_id else {index_name: key} return self.sheerka.get_unknown(metadata) def update_references(self, context, concept, modified_concept=None, modifications=None): """ Updates all the concepts that reference concept :param context: :param concept: Old version of the concept :param modified_concept: new version of the concept :param modifications: what are the modification :return: """ refs = self.sheerka.om.get(self.CONCEPTS_REFERENCES_ENTRY, concept.id) if refs is NotFound: return for concept_id in refs: # remove the grammar entry so that it can be recreated self.sheerka.om.delete(self.CONCEPTS_BNF_DEFINITIONS_ENTRY, concept_id) to_update = self.get_by_id(concept_id) metadata = to_update.get_metadata() # reset the bnf definition if needed if modified_concept: if self.has_id(concept_id): # reset only it the bnf definition is in cache if metadata.definition_type == DEFINITION_TYPE_BNF and self._name_has_changed(modifications): tokens = list(Tokenizer(metadata.definition)) modified = False for i, token in enumerate(tokens): if token.type == TokenKind.IDENTIFIER and token.value == concept.name: clone = token.clone() clone.value = modified_concept.name tokens[i] = clone modified = True if modified: to_update.get_metadata().definition = core.utils.get_text_from_tokens(tokens) to_update.set_bnf(None) # update concept_by_first_token if metadata.definition_type == DEFINITION_TYPE_BNF: # recompute concepts by first tokens and concept by first regex res = self.recompute_first_items(context, concept, [concept]) if not res.status: return res by_first_keyword, by_first_regex, resolved_by_first_keyword, compiled_by_first_regex = res.body self.update_first_items_caches(context, by_first_keyword, by_first_regex, resolved_by_first_keyword, compiled_by_first_regex) def compute_references(self, concept): """ We need to keep a track of all concepts used by the current concept So that if one of these are modified, we can modify the current concept accordingly :param concept: :return: """ refs = set() if concept.get_metadata().definition_type == DEFINITION_TYPE_BNF: from parsers.BnfNodeParser import BnfNodeConceptExpressionVisitor other_concepts_visitor = BnfNodeConceptExpressionVisitor() other_concepts_visitor.visit(concept.get_bnf()) for concept in other_concepts_visitor.references: if isinstance(concept, str): concept = self.sheerka.get_by_key(concept) refs.add(concept.id) return refs def is_not_a_concept_name(self, name): """ Given a name tells if it refers to a variable name :param name: :return: """ return self.sheerka.om.get(self.sheerka.CONCEPTS_BY_NAME_ENTRY, name) is NotFound def is_a_concept_name(self, name): """ Given a name tells if it refers to a variable name :param name: :return: """ return self.sheerka.om.get(self.sheerka.CONCEPTS_BY_NAME_ENTRY, name) is not NotFound def clear_bnf_definition(self, concept_id=None): if concept_id: self.sheerka.om.delete(self.CONCEPTS_BNF_DEFINITIONS_ENTRY, concept_id) else: self.sheerka.om.clear(self.CONCEPTS_BNF_DEFINITIONS_ENTRY) def set_precedence(self, context, *concepts): """ Set the precedence order when parsing concept with SyaNodeParser The first concept in the list have the highest priority :param context: :param concepts: :return: """ if len(concepts) < 2: return self.sheerka.err("Not enough elements") as_iterable = iter(concepts) first = next(as_iterable) ensure_concept(first) try: while True: second = next(as_iterable) ret = self.sheerka.set_is_greater_than(context, BuiltinConcepts.PRECEDENCE, first, second, CONCEPT_COMPARISON_CONTEXT) if not ret.status: return ret first = second except StopIteration: pass return self.sheerka.new(BuiltinConcepts.SUCCESS) @staticmethod def _name_has_changed(to_add): if to_add is None or "meta" not in to_add: return False return "name" in to_add["meta"] @staticmethod def _definition_has_changed(to_add): if to_add is None or "meta" not in to_add: return False return "definition" in to_add["meta"] def _update_concept(self, context, concept, to_add, to_remove): sheerka = context.sheerka same_values = {} if to_add: if "meta" in to_add: # All modifications must be allowed metadata = concept.get_metadata() for k, v in to_add["meta"].items(): if k in self.forbidden_meta: return sheerka.ret(self.NAME, False, sheerka.err(ForbiddenAttribute(k))) try: if getattr(metadata, k) == v: same_values[k] = v else: setattr(metadata, k, v) except AttributeError: return sheerka.ret(self.NAME, False, sheerka.err(UnknownAttribute(k))) if same_values == to_add["meta"]: return sheerka.ret(self.NAME, False, sheerka.err(NoModificationFound(concept, same_values))) if "props" in to_add: for k, v in to_add["props"].items(): concept.set_prop(k, v) if "variables" in to_add: for k, v in to_add["variables"].items(): # update existing or add new for i, var in enumerate(concept.get_metadata().variables): if var[0] == k: concept.get_metadata().variables[i] = (k, v) break else: concept.def_var(k, v) if to_remove: if "meta" in to_remove: return sheerka.ret(self.NAME, False, sheerka.err(CannotRemoveMeta(to_remove["meta"]))) if "props" in to_remove: for k, v in to_remove["props"].items(): if k not in concept.get_metadata().props: return sheerka.ret(self.NAME, False, sheerka.err(UnknownAttribute(k))) props = concept.get_metadata().props[k] try: if isinstance(v, (set, list, tuple)): for item in v: props.remove(item) else: props.remove(v) # remove empty sets if len(props) == 0: del concept.get_metadata().props[k] except KeyError: return sheerka.ret(self.NAME, False, sheerka.err(ValueNotFound(k, v))) if "variables" in to_remove: variables_to_remove = [] for k in to_remove["variables"]: for var_def in concept.get_metadata().variables: if var_def[0] == k: variables_to_remove.append(var_def) delattr(concept, var_def[0]) break else: return sheerka.ret(self.NAME, False, sheerka.err(UnknownAttribute(k))) core.utils.remove_list_from_list(concept.get_metadata().variables, variables_to_remove) if concept.get_all_attributes(): core.utils.remove_list_from_list(concept.get_all_attributes(), [v[0] for v in variables_to_remove]) concept.get_metadata().key = None if self._definition_has_changed(to_add) and concept.get_metadata().definition_type == DEFINITION_TYPE_BNF: concept.set_bnf(None) ensure_bnf(context, concept) self.recompute_concept_parameters(context, concept) concept.init_key() return def _remove_concept_first_token_and_first_regex(self, concept): keywords_or_regex = self.get_first_items(self.sheerka, concept) # keyword of the old concept concepts_by_first_keyword = self.sheerka.om.copy(self.CONCEPTS_BY_FIRST_KEYWORD_ENTRY) concepts_by_regex = self.sheerka.om.copy(self.CONCEPTS_BY_REGEX_ENTRY) for item in keywords_or_regex: try: if isinstance(item, RegExDef): serialized = item.serialize() copy = concepts_by_regex[serialized].copy() copy.remove(concept.id) if len(copy) == 0: del concepts_by_regex[serialized] else: concepts_by_regex[serialized] = copy else: copy = concepts_by_first_keyword[item].copy() copy.remove(concept.id) if len(copy) == 0: del concepts_by_first_keyword[item] else: concepts_by_first_keyword[item] = copy except KeyError: # only occurs in unit tests when concepts are created without create_new() pass # return concepts_by_first_keyword, concepts_by_regex return concepts_by_first_keyword, {RegExDef().deserialize(k): v for k, v in concepts_by_regex.items()} @staticmethod def get_first_tokens(sheerka, concept): """ :param sheerka: :param concept: :return: """ if concept.get_bnf(): from parsers.BnfNodeParser import BnfNodeFirstTokenVisitor bnf_visitor = BnfNodeFirstTokenVisitor(sheerka) bnf_visitor.visit(concept.get_bnf()) return [t for t in bnf_visitor.first_tokens if t is not NoFirstToken] else: keywords = concept.key.split() for keyword in keywords: if keyword.startswith(VARIABLE_PREFIX): continue return [keyword] return None @staticmethod def get_first_items(sheerka, concept) -> List[Union[str, RegExDef]]: """ Get all the first item needed by the concept An item can either be a token, or regular expression :param sheerka: :param concept: :return: List of string (if it's token or RegExDef if it's the definition of a regex) """ if concept.get_bnf(): from parsers.BnfNodeParser import BnfNodeFirstTokenVisitor bnf_visitor = BnfNodeFirstTokenVisitor(sheerka) bnf_visitor.visit(concept.get_bnf()) return bnf_visitor.first_tokens else: keywords = concept.key.split() for keyword in keywords: if keyword.startswith(VARIABLE_PREFIX): continue return [keyword] return None @staticmethod def compute_concepts_by_first_item(context, concepts, use_sheerka=False, previous_first_keywords=None, previous_first_regex=None): """ Create two map, one for describing the first token expected by a concept one for the first regular expression eg the dictionaries that go into CONCEPTS_BY_FIRST_KEYWORD_ENTRY and CONCEPTS_BY_REGEX_ENTRY :param context: :param concepts: lists of concepts to parse :param use_sheerka: if True, updates sheerka :param previous_first_keywords: :param previous_first_regex: :return: Returns two dictionaries : on for ALL first item entries, another one for all first regex entries """ sheerka = context.sheerka if use_sheerka: previous_first_keywords = sheerka.om.copy(SheerkaConceptManager.CONCEPTS_BY_FIRST_KEYWORD_ENTRY) previous_first_regex = sheerka.om.copy(SheerkaConceptManager.CONCEPTS_BY_REGEX_ENTRY) previous_first_regex = {RegExDef().deserialize(k): v for k, v in previous_first_regex.items()} else: previous_first_keywords = previous_first_keywords or {} previous_first_regex = previous_first_regex or {} for concept in concepts: items = SheerkaConceptManager.get_first_items(sheerka, concept) if items is None: # no first token found for a concept ? return sheerka.ret(sheerka.name, False, NoFirstTokenError(concept, concept.key)) for item in items: if isinstance(item, RegExDef): previous_first_regex.setdefault(item, []).append(concept.id) else: previous_first_keywords.setdefault(item, []).append(concept.id) # 'uniquify' the lists for k, v in previous_first_keywords.items(): previous_first_keywords[k] = core.utils.make_unique(v) for k, v in previous_first_regex.items(): previous_first_regex[k] = core.utils.make_unique(v) return sheerka.ret("BaseNodeParser", True, (previous_first_keywords, previous_first_regex)) @staticmethod def compute_concepts_by_first_token(context, concepts, use_sheerka=False, previous_entries=None): """ Create the map describing the first token expected by a concept eg the dictionary that goes into CONCEPTS_BY_FIRST_KEYWORD_ENTRY :param context: :param concepts: lists of concepts to parse :param use_sheerka: if True, update concepts_by_first_keyword from sheerka :param previous_entries: :return: """ sheerka = context.sheerka res = sheerka.om.copy(SheerkaConceptManager.CONCEPTS_BY_FIRST_KEYWORD_ENTRY) if use_sheerka \ else (previous_entries or {}) for concept in concepts: keywords = SheerkaConceptManager.get_first_tokens(sheerka, concept) if keywords is None: # no first token found for a concept ? return sheerka.ret(sheerka.name, False, NoFirstTokenError(concept, concept.key)) for keyword in keywords: res.setdefault(keyword, []).append(concept.id) # 'uniquify' the lists for k, v in res.items(): res[k] = core.utils.make_unique(v) return sheerka.ret("BaseNodeParser", True, res) @staticmethod def resolve_concepts_by_first_keyword(context, concepts_by_first_keyword, modified_concepts=None): """ From a dictionary of first tokens, create another dictionary where all references to other concepts are resolved fom example, from entries { 'c:|1001:' : 'c:|1002:' # which means than the concept 1002 starts with the concept 1001 'foo': 'c:|1001:' } It will create { 'foo': ['c:|1001:, 'c:|1002:'], } This dictionary is supposed to go into CONCEPTS_REFERENCES_ENTRY """ sheerka = context.sheerka res = {} def get_by_id(c_id): if modified_concepts and c_id in modified_concepts: return modified_concepts[c_id] return sheerka.get_by_id(c_id) def resolve_concepts(concept_str): c_key, c_id = core.utils.unstr_concept(concept_str) if c_id in already_seen: return ChickenAndEggError(already_seen) already_seen.add(c_id) resolved = set() to_resolve = set() chicken_and_egg = set() concept = get_by_id(c_id) if sheerka.isaset(context, concept): concepts = sheerka.get_set_elements(context, concept) else: concepts = [concept] for c in concepts: if sheerka.isaset(context, c): to_resolve.add(c.str_id) else: ensure_bnf(context, c) # need to make sure that it cannot fail keywords = SheerkaConceptManager.get_first_tokens(sheerka, c) for keyword in keywords: (to_resolve if keyword.startswith("c:|") else resolved).add(keyword) for concept_to_resolve_str in to_resolve: res = resolve_concepts(concept_to_resolve_str) if isinstance(res, ChickenAndEggError): chicken_and_egg |= res.concepts else: resolved |= res to_resolve.clear() if len(resolved) == 0 and len(chicken_and_egg) > 0: raise ChickenAndEggError(chicken_and_egg) else: return resolved for k, v in concepts_by_first_keyword.items(): if k.startswith("c:|"): try: already_seen = set() resolved_keywords = resolve_concepts(k) for resolved in resolved_keywords: res.setdefault(resolved, []).extend(v) except ChickenAndEggError as ex: context.log(f"Chicken and egg detected for {k}, concepts={ex.concepts}") concepts_in_recursion = ex.concepts # make sure to have all the parents for parent in v: concepts_in_recursion.add(parent) for concept_id in concepts_in_recursion: # make sure we keep the longest chain old = sheerka.chicken_and_eggs.get(concept_id) if old is NotFound or len(old) < len(ex.concepts): sheerka.chicken_and_eggs.put(concept_id, concepts_in_recursion) else: res.setdefault(k, []).extend(v) # 'uniquify' the lists for k, v in res.items(): res[k] = core.utils.make_unique(v) return sheerka.ret("BaseNodeParser", True, res) @staticmethod def compile_concepts_by_first_regex(context, concepts_by_first_regex): res = [] try: for k, v in concepts_by_first_regex.items(): flags = RegExDef.compile_flags(k.ignore_case, k.multiline, k.explicit_flags) res.append((re.compile(k.to_match, flags), v)) except Exception as ex: return context.sheerka.ret("BaseNodeParser", False, ex) return context.sheerka.ret("BaseNodeParser", True, res) def get_concepts_by_first_token(self, token, to_keep, custom=None, to_map=None, strip_quotes=False, parser=None): """ Tries to find if there are concepts that match the value of the token Caution: Returns the actual cache, not a copy :param token: :param to_keep: predicate to tell if the concept is eligible :param custom: lambda name -> List[Concepts] that gives extra concepts, according to the name :param to_map: :param strip_quotes: Remove quotes from strings :param parser: If needed, parser which requested the concepts :return: """ if token.type == TokenKind.WHITESPACE: return None if token.type == TokenKind.STRING: name = token.value[1:-1] if strip_quotes else token.value else: name = token.value custom_concepts = custom(name) if custom else [] # to get extra concepts using an alternative method result = [] concepts_ids = self.sheerka.om.get(self.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY, name) if concepts_ids is NotFound: return custom_concepts if custom else None for concept_id in concepts_ids: concept = self.sheerka.get_by_id(concept_id) if not to_keep(concept): continue concept = to_map(concept, parser, self.sheerka) if to_map else concept result.append(concept) return core.utils.make_unique(result + custom_concepts, lambda c: c.concept.id if hasattr(c, "concept") else c.id) def get_concepts_by_first_regex(self, expr, pos): """ Go thru all the declared regular expressions and try to see if there is a match :param expr: :param pos: :return: """ result = [] for compiled_regex, concept_ids in self.compiled_concepts_by_regex: if compiled_regex.match(expr, pos): result.extend([self.sheerka.get_by_id(concept_id) for concept_id in concept_ids]) return result def get_concepts_bnf_definitions(self): return self.sheerka.om.current_cache_manager().caches[self.CONCEPTS_BNF_DEFINITIONS_ENTRY].cache def recompute_first_items(self, context, old_concept, new_concepts): """ Recompute concepts fy first items resolved concept by first items concepts by first regex compiled concepts by first regex Do not update anything :param context: :param old_concept: :param new_concepts: :return: """ sheerka = context.sheerka if old_concept and not new_concepts: # remove modified_concepts = None by_first_keyword, by_first_regex = self._remove_concept_first_token_and_first_regex(old_concept) elif old_concept and new_concepts: # remove by_first_keyword, by_first_regex = self._remove_concept_first_token_and_first_regex(old_concept) # and then update init_ret_value = self.compute_concepts_by_first_item(context, new_concepts, False, by_first_keyword, by_first_regex) if not init_ret_value.status: return sheerka.ret(self.NAME, False, ErrorConcept(init_ret_value.value)) by_first_keyword, by_first_regex = init_ret_value.body modified_concepts = {new_concept.id: new_concept for new_concept in new_concepts} elif not old_concept and new_concepts: # only update modified_concepts = None init_ret_value = self.compute_concepts_by_first_item(context, new_concepts, True) if not init_ret_value.status: return sheerka.ret(self.NAME, False, ErrorConcept(init_ret_value.value)) by_first_keyword, by_first_regex = init_ret_value.body # computes resolved concepts_by_first_keyword init_ret_value = self.resolve_concepts_by_first_keyword(context, by_first_keyword, modified_concepts) if not init_ret_value.status: return sheerka.ret(self.NAME, False, ErrorConcept(init_ret_value.value)) resolved_by_first_keyword = init_ret_value.body # compile new regex compile_ret = self.compile_concepts_by_first_regex(context, by_first_regex) if not compile_ret.status: return sheerka.ret(self.NAME, False, ErrorConcept(compile_ret.value)) compiled_by_first_regex = compile_ret.body return sheerka.ret(self.NAME, True, (by_first_keyword, by_first_regex, resolved_by_first_keyword, compiled_by_first_regex)) def update_first_items_caches(self, context, by_first_keyword, by_first_regex, resolved_by_first_keyword, compiled_by_first_regex): om = context.sheerka.om om.put(self.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, False, by_first_keyword) om.put(self.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY, False, resolved_by_first_keyword) om.put(self.CONCEPTS_BY_REGEX_ENTRY, False, {k.serialize(): v for k, v in by_first_regex.items()}) # update the compiled regex self.compiled_concepts_by_regex.clear() self.compiled_concepts_by_regex.extend(compiled_by_first_regex) @staticmethod def recompute_concept_parameters(context, concept): concept.get_metadata().parameters.clear() if concept.get_metadata().definition_type == DEFINITION_TYPE_DEF: tokens = list(Tokenizer(concept.get_metadata().definition, yield_eof=False)) else: tokens = list(Tokenizer(concept.get_metadata().name, yield_eof=False)) # all variables that appear in the name are concept parameters if len(tokens) > 0: variables = [p[0] for p in concept.get_metadata().variables] for token in [t for t in tokens if t.value in variables]: concept.get_metadata().parameters.append(token.value) # for bnf concept, use the visitor to extract parameters if concept.get_bnf(): from evaluators.DefConceptEvaluator import ConceptOrRuleVariableVisitor, ParameterVariable visitor = ConceptOrRuleVariableVisitor(context) visitor.visit(concept.get_bnf()) for variable in [v for v in visitor.variables if isinstance(v, ParameterVariable)]: if variable.name not in concept.get_metadata().parameters: concept.get_metadata().parameters.append(variable.name)