kodjo/Sheerka-Old
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactored Caching, Refactored BnfNodeParser, Introduced Sphinx

Browse Source
This commit is contained in:
Kodjo Sossouvi
2020-05-12 17:21:10 +02:00
parent 7d3a490bc5
commit 6e343ba996
110 changed files with 13865 additions and 7540 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/core/sheerka/Sheerka.py
+294 -229
View File
@@ -2,6 +2,12 @@ import logging
import core.builtin_helpers
import core.utils
from cache.Cache import Cache
from cache.CacheManager import CacheManager
from cache.DictionaryCache import DictionaryCache
from cache.IncCache import IncCache
from cache.ListIfNeededCache import ListIfNeededCache
from cache.SetCache import SetCache
from core.builtin_concepts import BuiltinConcepts, ErrorConcept, ReturnValueConcept, BuiltinErrors, BuiltinUnique, \
UnknownConcept
from core.concept import Concept, ConceptParts, PROPERTIES_FOR_NEW
@@ -18,8 +24,7 @@ from core.sheerka_logger import console_handler
from printer.SheerkaPrinter import SheerkaPrinter
from sdp.sheerkaDataProvider import SheerkaDataProvider, Event
CONCEPT_LEXER_PARSER_CLASS = "parsers.BnfNodeParser.BnfNodeParser"
BNF_PARSER_CLASS = "parsers.BnfParser.BnfParser"
BASE_NODE_PARSER_CLASS = "parsers.BaseNodeParser.BaseNodeParser"
CONCEPTS_FILE = "_concepts.txt"
@@ -28,45 +33,36 @@ class Sheerka(Concept):
Main controller for the project
"""
CONCEPTS_ENTRY = "All_Concepts" # to store all the concepts
CONCEPTS_BY_ID_ENTRY = "Concepts_By_ID"
CONCEPTS_BY_ID_ENTRY = "Concepts_By_ID" # to store all the concepts
CONCEPTS_BY_KEY_ENTRY = "Concepts_By_Key"
CONCEPTS_BY_NAME_ENTRY = "Concepts_By_Name"
CONCEPTS_BY_HASH_ENTRY = "Concepts_By_Hash" # store hash of concepts definitions (not values)
CONCEPTS_DEFINITIONS_ENTRY = "Concepts_Definitions" # to store definitions (bnf) of concepts
CONCEPTS_BY_FIRST_KEYWORD_ENTRY = "Concepts_By_First_Keyword"
CONCEPTS_SYA_DEFINITION_ENTRY = "Concepts_Sya_Definitions"
CONCEPTS_BY_FIRST_KEYWORD_ENTRY = "Concepts_By_First_Keyword"
RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY = "Resolved_Concepts_By_First_Keyword"
CONCEPTS_SYA_DEFINITION_ENTRY = "Concepts_Sya_Definitions"
RESOLVED_CONCEPTS_SYA_DEFINITION_ENTRY = "Resolved_Concepts_Sya_Definitions"
CONCEPTS_GRAMMARS_ENTRY = "Concepts_Grammars"
CONCEPTS_GROUPS_ENTRY = "Concepts_Groups"
VARIABLES_ENTRY = "Variables" # entry for admin or internal variables
CONCEPTS_KEYS_ENTRY = "Concepts_Keys"
BUILTIN_CONCEPTS_KEYS = "Builtins_Concepts" # sequential key for builtin concepts
USER_CONCEPTS_KEYS = "User_Concepts" # sequential key for user defined concepts
def __init__(self, skip_builtins_in_db=False, debug=False, loggers=None):
def __init__(self, cache_only=False, debug=False, loggers=None):
self.init_logging(debug, loggers)
self.loggers = loggers
super().__init__(BuiltinConcepts.SHEERKA, True, True, BuiltinConcepts.SHEERKA)
self.log.debug("Starting Sheerka.")
# cache of the most used concepts
# Note that these are only templates
# They are used as a footprint for instantiation
# Except of source when the concept is supposed to be unique
# key is the key of the concept (not the name or the id)
self.cache_by_key = {}
self.cache_by_id = {}
self.cache_by_name = {}
self.bnp = None # reference to the BaseNodeParser class (to compute first keyword token)
# cache for concept definitions,
# Primarily used for unit test that does not have access to sdp
self.concepts_definitions_cache = {}
#
# cache for concepts grammars
# a grammar is a resolved BNF
self.concepts_grammars = {}
# cache for SYA concepts
self.concepts_by_first_keyword = {}
self.sya_definitions = {}
# # Cache for concepts grammars
# # To be shared between BNFNode parsers instances
# self.concepts_grammars = {}
# a concept can be instantiated
# ex: File is a concept, but File('foo.txt') is an instance
@@ -78,6 +74,8 @@ class Sheerka(Concept):
self.rules = []
self.sdp: SheerkaDataProvider = None # SheerkaDataProvider
self.cache_manager = CacheManager(cache_only)
self.builtin_cache = {} # cache for builtin concepts
self.parsers = {} # cache for builtin parsers
self.evaluators = [] # cache for builtin evaluators
@@ -85,8 +83,6 @@ class Sheerka(Concept):
self.evaluators_prefix: str = None
self.parsers_prefix: str = None
self.skip_builtins_in_db = skip_builtins_in_db
self.execute_handler = SheerkaExecute(self)
self.create_new_concept_handler = SheerkaCreateNewConcept(self)
self.modify_concept_handler = SheerkaModifyConcept(self)
@@ -100,36 +96,65 @@ class Sheerka(Concept):
self.during_restore = False
self._builtins_classes_cache = None
def initialize(self, root_folder: str = None):
self.save_execution_context = True
@property
def resolved_concepts_by_first_keyword(self):
"""
We return the cache as we will be interested by statistics
:return:
"""
return self.cache_manager.caches[self.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY].cache
@property
def resolved_sya_def(self):
"""
:return:
"""
return self.cache_manager.caches[self.RESOLVED_CONCEPTS_SYA_DEFINITION_ENTRY].cache
@property
def concepts_grammars(self):
return self.cache_manager.caches[self.CONCEPTS_GRAMMARS_ENTRY].cache
def initialize(self, root_folder: str = None, save_execution_context=True):
"""
Starting Sheerka
Loads the current configuration
Notes that when it's the first time, it also create the needed working folders
:param root_folder: root configuration folder
:param save_execution_context:
:return: ReturnValue(Success or Error)
"""
self.save_execution_context = save_execution_context
try:
from sheerkapickle.sheerka_handlers import initialize_pickle_handlers
initialize_pickle_handlers()
self.sdp = SheerkaDataProvider(root_folder, self)
if self.sdp.first_time:
self.sdp.set_key(self.USER_CONCEPTS_KEYS, 1000)
self.initialize_caching()
event = Event("Initializing Sheerka.", user=self.name)
event = Event("Initializing Sheerka.", user_id=self.name)
self.sdp.save_event(event)
with ExecutionContext(self.key, event, self, "Initializing Sheerka.", self.init_log) as exec_context:
if self.sdp.first_time:
self.first_time_initialisation(exec_context)
self.initialize_builtin_concepts()
self.initialize_builtin_parsers()
self.initialize_builtin_evaluators()
self.initialize_bnf_parsing(exec_context)
self.initialize_sya_parsing()
self.initialize_builtin_concepts()
self.initialize_concept_node_parsing(exec_context)
res = ReturnValueConcept(self, True, self)
exec_context.add_values(return_values=res)
if not self.skip_builtins_in_db:
if self.cache_manager.is_dirty:
self.cache_manager.commit(exec_context)
if save_execution_context:
self.sdp.save_result(exec_context, is_admin=True)
self.init_log.debug(f"Sheerka successfully initialized")
@@ -138,6 +163,59 @@ class Sheerka(Concept):
return res
def initialize_caching(self):
def params(cache_name):
return {
'default': lambda k: self.sdp.get(cache_name, k),
'extend_exists': lambda k: self.sdp.exists(cache_name, k)
}
cache = IncCache(default=lambda k: self.sdp.get(self.CONCEPTS_KEYS_ENTRY, k))
self.cache_manager.register_cache(self.CONCEPTS_KEYS_ENTRY, cache)
register_concept_cache = self.cache_manager.register_concept_cache
cache = Cache(**params(self.CONCEPTS_BY_ID_ENTRY))
register_concept_cache(self.CONCEPTS_BY_ID_ENTRY, cache, lambda c: c.id, True)
cache = ListIfNeededCache(**params(self.CONCEPTS_BY_KEY_ENTRY))
register_concept_cache(self.CONCEPTS_BY_KEY_ENTRY, cache, lambda c: c.key, True)
cache = ListIfNeededCache(**params(self.CONCEPTS_BY_NAME_ENTRY))
register_concept_cache(self.CONCEPTS_BY_NAME_ENTRY, cache, lambda c: c.name, True)
cache = ListIfNeededCache(**params(self.CONCEPTS_BY_HASH_ENTRY))
register_concept_cache(self.CONCEPTS_BY_HASH_ENTRY, cache, lambda c: c.get_definition_hash(), True)
cache = DictionaryCache(default=lambda k: self.sdp.get(self.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, k))
self.cache_manager.register_cache(self.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, cache)
self.cache_manager.get(self.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, None) # to init from sdp
cache = DictionaryCache(default=lambda k: self.sdp.get(self.CONCEPTS_SYA_DEFINITION_ENTRY, k))
self.cache_manager.register_cache(self.CONCEPTS_SYA_DEFINITION_ENTRY, cache)
self.cache_manager.get(self.CONCEPTS_SYA_DEFINITION_ENTRY, None) # to init from sdp
cache = SetCache(default=lambda k: self.sdp.get(self.CONCEPTS_GROUPS_ENTRY, k))
self.cache_manager.register_cache(self.CONCEPTS_GROUPS_ENTRY, cache)
cache = Cache(default=lambda k: self.sdp.get(self.VARIABLES_ENTRY, k))
self.cache_manager.register_cache(self.VARIABLES_ENTRY, cache, True, True)
cache = DictionaryCache()
self.cache_manager.register_cache(self.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY, cache, persist=False)
cache = DictionaryCache()
self.cache_manager.register_cache(self.RESOLVED_CONCEPTS_SYA_DEFINITION_ENTRY, cache, persist=False)
cache = Cache()
self.cache_manager.register_cache(self.CONCEPTS_GRAMMARS_ENTRY, cache, persist=False)
def first_time_initialisation(self, context):
self.cache_manager.put(self.CONCEPTS_KEYS_ENTRY, self.USER_CONCEPTS_KEYS, 1000)
self.variable_handler.record(context, self.name, "save_execution_context", True)
def initialize_builtin_concepts(self):
"""
Initializes the builtin concepts
@@ -160,18 +238,16 @@ class Sheerka(Concept):
if not concept.metadata.is_unique and str(key) in builtins_classes:
self.builtin_cache[key] = builtins_classes[str(key)]
if not self.skip_builtins_in_db:
from_db = self.sdp.get_safe(self.CONCEPTS_ENTRY, concept.metadata.key)
if from_db is None:
self.init_log.debug(f"'{concept.name}' concept is not found in db. Adding.")
self.set_id_if_needed(concept, True)
concept.metadata.full_serialization = True
self.sdp.add("init", self.CONCEPTS_ENTRY, concept, use_ref=True)
else:
self.init_log.debug(f"Found concept '{from_db}' in db. Updating.")
concept.update_from(from_db)
from_db = self.cache_manager.get(self.CONCEPTS_BY_KEY_ENTRY, concept.metadata.key)
if from_db is None:
self.init_log.debug(f"'{concept.name}' concept is not found in db. Adding.")
self.set_id_if_needed(concept, True)
self.cache_manager.add_concept(concept)
else:
self.init_log.debug(f"Found concept '{from_db}' in db. Updating.")
concept.update_from(from_db)
self.add_in_cache(concept)
return
def initialize_builtin_parsers(self):
"""
@@ -187,17 +263,23 @@ class Sheerka(Concept):
if parser.__module__ == base_class.__module__:
continue
if parser.__module__ in modules_to_skip:
continue
qualified_name = core.utils.get_full_qualified_name(parser)
self.init_log.debug(f"Adding builtin parser '{qualified_name}'")
temp_result[qualified_name] = parser
# keep a reference to base_node_parser
self.bnp = temp_result[BASE_NODE_PARSER_CLASS]
# Now we sort the parser by name.
# It's not important for the logic of their usage as they have their priority anyway,
# We do that for the unit tests. They are to complicated to write otherwise
for name in sorted(temp_result.keys()):
parser = temp_result[name]
if parser.__module__ in modules_to_skip:
# base node parser module does not contains any valid parser
continue
self.parsers[name] = temp_result[name]
def initialize_builtin_evaluators(self):
@@ -214,55 +296,39 @@ class Sheerka(Concept):
self.init_log.debug(f"Adding builtin evaluator '{evaluator.__name__}'")
self.evaluators.append(evaluator)
def initialize_bnf_parsing(self, execution_context):
self.init_log.debug("Initializing concepts grammars.")
definitions = self.get_concepts_definitions(execution_context)
def initialize_concept_node_parsing(self, context):
self.init_log.debug("Initializing concept node parsing.")
if definitions is None:
self.init_log.debug("No BNF defined")
return
concepts_by_first_keyword = self.cache_manager.copy(self.CONCEPTS_BY_FIRST_KEYWORD_ENTRY)
res = self.bnp.resolve_concepts_by_first_keyword(context, concepts_by_first_keyword)
self.cache_manager.put(self.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY, False, res.body)
lexer_parser = self.parsers[CONCEPT_LEXER_PARSER_CLASS]()
ret_val = lexer_parser.initialize(execution_context, definitions)
if not ret_val.status:
self.init_log.error("Failed to initialize concepts definitions " + str(ret_val.body))
return
# sya = self.bnf.resolve_sya_associativity_and_precedence()
# self.cache_manager.put(self.RESOLVED_CONCEPTS_SYA_DEFINITION_ENTRY, sya)
#
#
# self.concepts_by_first_keyword, \
# self.resolved_concepts_by_first_keyword = \
# self.create_new_concept_handler.load_concepts_nodes_definitions(context)
self.concepts_grammars = lexer_parser.concepts_grammars
# self.concepts_by_first_keyword = self.sdp.get_safe(
# self.CONCEPTS_BY_FIRST_KEYWORD_ENTRY,
# load_origin=False) or {}
#
# self.sya_definitions = self.sdp.get_safe(
# self.CONCEPTS_SYA_DEFINITION_ENTRY,
# load_origin=False) or {}
#
# init_ret_value = self.bnp.resolve_concepts_by_first_keyword(self, self.concepts_by_first_keyword)
# if not init_ret_value.status:
# return self.sheerka.ret(self.logger_name, False, ErrorConcept(init_ret_value.value))
# self.resolved_concepts_by_first_keyword = init_ret_value.body
def initialize_sya_parsing(self):
self.init_log.debug("Initializing sya definitions.")
self.concepts_by_first_keyword = self.sdp.get_safe(
self.CONCEPTS_BY_FIRST_KEYWORD_ENTRY,
load_origin=False) or {}
self.sya_definitions = self.sdp.get_safe(
self.CONCEPTS_SYA_DEFINITION_ENTRY,
load_origin=False) or {}
def reset(self):
self.reset_cache()
self.concepts_by_first_keyword = {}
self.concepts_grammars = {}
self.sya_definitions = {}
def reset(self, cache_only=False):
self.cache_manager.clear()
self.cache_manager.cache_only = cache_only
self.printer_handler.reset()
self.sdp.reset()
self.sdp.set_key(self.USER_CONCEPTS_KEYS, 1000)
def reset_cache(self, filter_to_use=None):
"""
reset the different cache that exists
:param filter_to_use:
:return:
"""
if filter_to_use is None:
self.cache_by_key = {}
self.cache_by_id = {}
self.cache_by_name = {}
else:
raise NotImplementedError()
return self
def evaluate_user_input(self, text: str, user_name="kodjo"):
"""
@@ -294,7 +360,10 @@ class Sheerka(Concept):
ret = self.execute(execution_context, [user_input, reduce_requested], steps)
execution_context.add_values(return_values=ret)
if not self.skip_builtins_in_db:
if self.cache_manager.is_dirty:
self.cache_manager.commit(execution_context)
if self.save_execution_context and self.variable_handler.load(self.name, "save_execution_context"):
self.sdp.save_result(execution_context)
# # hack to save valid concept definition
@@ -302,6 +371,8 @@ class Sheerka(Concept):
# if len(ret) == 1 and ret[0].status and self.isinstance(ret[0].value, BuiltinConcepts.NEW_CONCEPT):
# with open(CONCEPTS_FILE, "a") as f:
# f.write(text + "\n")
self._last_execution = execution_context
return ret
def print(self, result, instructions=None):
@@ -343,8 +414,8 @@ class Sheerka(Concept):
if obj.metadata.id is not None:
return
entry = self.BUILTIN_CONCEPTS_KEYS if is_builtin else self.USER_CONCEPTS_KEYS
obj.metadata.id = self.sdp.get_next_key(entry)
key = self.BUILTIN_CONCEPTS_KEYS if is_builtin else self.USER_CONCEPTS_KEYS
obj.metadata.id = str(self.cache_manager.get(self.CONCEPTS_KEYS_ENTRY, key))
self.log.debug(f"Setting id '{obj.metadata.id}' to concept '{obj.metadata.name}'.")
def create_new_concept(self, context, concept: Concept):
@@ -380,21 +451,25 @@ class Sheerka(Concept):
"""
return self.sets_handler.set_isa(context, concept, concept_set)
def set_sya_def(self, context, list_of_def):
def force_sya_def(self, context, list_of_def):
"""
Set the precedence and/or the associativity of a concept
FOR TESTS PURPOSE. TO REMOVE EVENTUALLY
:param context:
:param list_of_def list of tuple(concept_id, precedence (int), SyaAssociativity)
:return:
"""
# validate the entries
# If one entry is an invalid concept, rollback everything
for concept_id, precedence, associativity in list_of_def:
if concept_id == BuiltinConcepts.UNKNOWN_CONCEPT:
return self.ret(self.name,
False,
self.new(BuiltinConcepts.ERROR, body=f"Concept {concept_id} is not known"))
sya_def = self.cache_manager.copy(self.RESOLVED_CONCEPTS_SYA_DEFINITION_ENTRY) or {}
# update the definitions
for concept_id, precedence, associativity in list_of_def:
if precedence is None and associativity is None:
@@ -403,12 +478,10 @@ class Sheerka(Concept):
except KeyError:
pass
else:
self.sya_definitions[concept_id] = (precedence, associativity.value)
sya_def[concept_id] = (precedence, associativity)
# then save
self.sdp.set(context.event.get_digest(),
self.CONCEPTS_SYA_DEFINITION_ENTRY,
self.sya_definitions)
# put in cache
self.cache_manager.put(self.RESOLVED_CONCEPTS_SYA_DEFINITION_ENTRY, False, sya_def)
return self.ret(self.name, True, self.new(BuiltinConcepts.SUCCESS))
@@ -448,122 +521,108 @@ class Sheerka(Concept):
if concept.key is None:
raise KeyError()
self.cache_by_key[concept.key] = concept
if concept.id:
self.cache_by_id[concept.id] = concept
self.cache_manager.add_concept(concept)
return concept
def get(self, concept_key, concept_id=None):
#
# def get(self, concept_key, concept_id=None):
# """
# Tries to find a concept
# What is return must be used a template for another concept.
# You must not modify the returned concept
# :param concept_key: key of the concept
# :param concept_id: when multiple concepts with the same key, use the id
# :return:
# """
#
# by_key = self.get_by_key(concept_key)
# if self.is_known(by_key):
# return by_key
#
# # else return by name
# by_name = self.get_by_name(concept_key)
# if self.is_known(by_name):
# return by_name
#
# return by_key # return not found for key
def get_by_key(self, concept_key, concept_id=None):
concept_key = str(concept_key) if isinstance(concept_key, BuiltinConcepts) else concept_key
return self.internal_get("key", concept_key, self.CONCEPTS_BY_KEY_ENTRY, concept_id)
def get_by_name(self, concept_name, concept_id=None):
return self.internal_get("name", concept_name, self.CONCEPTS_BY_NAME_ENTRY, concept_id)
def get_by_hash(self, concept_hash, concept_id=None):
return self.internal_get("hash", concept_hash, self.CONCEPTS_BY_HASH_ENTRY, concept_id)
def get_by_id(self, concept_id):
return self.internal_get("id", concept_id, self.CONCEPTS_BY_ID_ENTRY, None)
def internal_get(self, index_name, key, cache_name, concept_id=None):
"""
Tries to find a concept
What is return must be used a template for another concept.
You must not modify the returned concept
:param concept_key: key of the concept
:param concept_id: when multiple concepts with the same key, use the id
Tries to find an entry
:param index_name: name of the index (ex by_id, by_key...)
:param key: index value
:param cache_name: name of the cache (ex Concepts_By_ID...)
:param concept_id: id of the concept if none, in case where there are multiple results
:return:
"""
by_key = self.internal_get("key", concept_key, self.cache_by_key, self.CONCEPTS_ENTRY, concept_id)
if self.is_known(by_key):
return by_key
if key is None:
return ErrorConcept(f"Concept '{key}' is undefined.")
# else return by name
by_name = self.internal_get("name", concept_key, self.cache_by_name, self.CONCEPTS_BY_NAME_ENTRY, concept_id)
if self.is_known(by_name):
return by_name
concepts = self.cache_manager.get(cache_name, key)
if concepts:
if concept_id is None:
return concepts
return by_key # return not found for key
if not hasattr(concepts, "__iter__"):
return concepts
def get_by_key(self, concept_key, concept_id=None):
return self.internal_get("key", concept_key, self.cache_by_key, self.CONCEPTS_ENTRY, concept_id)
for c in concepts:
if c.id == concept_id:
return c
def get_by_name(self, concept_name, concept_id=None):
return self.internal_get("name", concept_name, self.cache_by_name, self.CONCEPTS_BY_NAME_ENTRY, concept_id)
metadata = [(index_name, key), ("id", concept_id)] if concept_id else (index_name, key)
return self._get_unknown(metadata)
def get_by_id(self, concept_id):
if concept_id is None:
return ErrorConcept("Concept id is undefined.")
# first search in cache
if concept_id in self.cache_by_id:
result = self.cache_by_id[concept_id]
else:
result = self.sdp.get_safe(self.CONCEPTS_BY_ID_ENTRY, concept_id)
if result is None:
result = self._get_unknown(('id', concept_id))
else:
self.cache_by_id[concept_id] = result
return result
def internal_get(self, index_name, index_value, cache_to_use, sdp_entry, concept_id=None):
def has_id(self, concept_id):
"""
Tries to find an entry
:param index_name:
:param index_value:
:param cache_to_use:
:param sdp_entry:
Returns True if a concept with this id exists in cache
It does not search in the remote repository
:param concept_id:
:return:
"""
return self.cache_manager.has(self.CONCEPTS_BY_ID_ENTRY, concept_id)
if index_value is None:
return ErrorConcept(f"Concept {index_name} is undefined.")
def has_key(self, concept_key):
"""
Returns True if concept(s) with this key exist in cache
It does not search in the remote repository
:param concept_key:
:return:
"""
return self.cache_manager.has(self.CONCEPTS_BY_KEY_ENTRY, concept_key)
if isinstance(index_value, BuiltinConcepts):
index_value = str(index_value)
def has_name(self, concept_name):
"""
Returns True if concept(s) with this name exist in cache
It does not search in the remote repository
:param concept_name:
:return:
"""
return self.cache_manager.has(self.CONCEPTS_BY_NAME_ENTRY, concept_name)
# first search in cache
if index_value in cache_to_use:
result = cache_to_use[index_value]
else:
result = self.sdp.get_safe(sdp_entry, index_value)
if result is None:
metadata = [(index_name, index_value), ("id", concept_id)] if concept_id else (index_name, index_value)
result = self._get_unknown(metadata)
# Do not put in cache_by_key or cache_by_id unknown concept
# TODO: implement an MRU cache for them
else:
cache_to_use[index_value] = result
for r in (result if isinstance(result, list) else [result]):
if r.id:
self.cache_by_id[r.id] = r
if not (isinstance(result, list) and concept_id):
return result
# result is a list, but we have the concept_id to discriminate
for c in result:
if c.id == concept_id:
return c
metadata = [(index_name, index_value), ("id", concept_id)] if concept_id else (index_name, index_value)
return self._get_unknown(metadata)
def get_concepts_definitions(self, context):
if self.concepts_definitions_cache:
return self.concepts_definitions_cache
encoded_bnf = self.sdp.get_safe(
self.CONCEPTS_DEFINITIONS_ENTRY,
load_origin=False) or {}
self.concepts_definitions_cache = {}
bnf_parser = self.parsers[BNF_PARSER_CLASS]()
for k, v in encoded_bnf.items():
key, id_ = core.utils.unstr_concept(k)
concept = self.new((key, id_))
context.log(f"Parsing BNF definition for {concept}", context.who)
rule_result = bnf_parser.parse(context, v)
if rule_result.status:
self.concepts_definitions_cache[concept] = rule_result.value.value
else:
self.log.error(f"Failed to load bnf rule for concept {key}")
return self.concepts_definitions_cache
def has_hash(self, concept_hash):
"""
Returns True if concept(s) with this hash exist in cache
It does not search in the remote repository
:param concept_hash:
:return:
"""
return self.cache_manager.has(self.CONCEPTS_BY_HASH_ENTRY, concept_hash)
def new(self, concept_key, **kwargs):
"""
@@ -578,7 +637,7 @@ class Sheerka(Concept):
else:
concept_id = None
template = self.get_by_id(concept_id) if not concept_key else self.get(concept_key, concept_id)
template = self.get_by_id(concept_id) if not concept_key else self.get_by_key(concept_key, concept_id)
# manage concept not found
if self.isinstance(template, BuiltinConcepts.UNKNOWN_CONCEPT) and \
@@ -599,7 +658,7 @@ class Sheerka(Concept):
# otherwise, create another instance
concept = self.builtin_cache[key]() if key in self.builtin_cache else Concept()
concept.update_from(template)
concept.update_from(template, update_value=False)
concept.freeze_definition_hash()
if len(kwargs) == 0:
@@ -608,10 +667,10 @@ class Sheerka(Concept):
# update the properties, values, attributes
# Not quite sure that this is the correct process order
for k, v in kwargs.items():
if k in concept.props:
concept.set_prop(k, v)
if k in concept.values:
concept.set_value(k, v)
elif k in PROPERTIES_FOR_NEW:
concept.values[ConceptParts(k)] = v
concept.set_value(ConceptParts(k), v)
elif hasattr(concept, k):
setattr(concept, k, v)
else:
@@ -639,12 +698,12 @@ class Sheerka(Concept):
message=message,
parents=parents)
def value(self, obj, reduce_simple_list=False):
def objvalue(self, obj, reduce_simple_list=False):
if obj is None:
return None
if hasattr(obj, "get_value"):
return obj.get_value()
if hasattr(obj, "get_obj_value"):
return obj.get_obj_value()
if not isinstance(obj, Concept):
return obj
@@ -657,7 +716,18 @@ class Sheerka(Concept):
else:
body_to_use = obj.body
return self.value(body_to_use)
return self.objvalue(body_to_use)
def objvalues(self, objs):
if not (isinstance(objs, list) or
self.isinstance(objs, BuiltinConcepts.LIST) or
self.isinstance(objs, BuiltinConcepts.ENUMERATION)):
objs = [objs]
if isinstance(objs, list):
return (self.objvalue(obj) for obj in objs)
return (self.objvalue(obj) for obj in objs.body)
def value_by_concept(self, obj, concept):
if obj is None:
@@ -678,8 +748,8 @@ class Sheerka(Concept):
if isinstance(obj, Concept) and obj.metadata.is_builtin and obj.key in BuiltinErrors:
return obj
if isinstance(obj, list):
return obj
if isinstance(obj, (list, set, tuple)):
return [self.get_error(o) for o in obj]
if self.isinstance(obj, BuiltinConcepts.RETURN_VALUE):
if obj.status:
@@ -687,19 +757,10 @@ class Sheerka(Concept):
if self.isinstance(obj.body, BuiltinConcepts.PARSER_RESULT):
return self.get_error(obj.body.body)
else:
return obj.body
return NotImplementedError()
def get_values(self, objs):
if not (isinstance(objs, list) or
self.isinstance(objs, BuiltinConcepts.LIST) or
self.isinstance(objs, BuiltinConcepts.ENUMERATION)):
objs = [objs]
if isinstance(objs, list):
return (self.value(obj) for obj in objs)
return (self.value(obj) for obj in objs.body)
raise NotImplementedError()
def is_success(self, obj):
if isinstance(obj, bool): # quick win
@@ -761,8 +822,12 @@ class Sheerka(Concept):
return self.parsers_prefix + name
def concepts(self):
"""
List of all known concepts (look up in sdp)
:return:
"""
res = []
lst = self.sdp.list(self.CONCEPTS_ENTRY)
lst = self.sdp.list(self.CONCEPTS_BY_ID_ENTRY)
for item in lst:
if isinstance(item, list):
res.extend(item)
@@ -818,10 +883,10 @@ class Sheerka(Concept):
# the metadata can be a list, if several attributes where given
# (key, 'not_found), (id, invalid_id)
unknown_concept = UnknownConcept()
unknown_concept.set_metadata_value(ConceptParts.BODY, metadata)
unknown_concept = UnknownConcept() # don't use new() for prevent circular reference
unknown_concept.set_value(ConceptParts.BODY, metadata)
for meta in (metadata if isinstance(metadata, list) else [metadata]):
unknown_concept.set_prop(meta[0], meta[1])
unknown_concept.set_value(meta[0], meta[1])
unknown_concept.metadata.is_evaluated = True
return unknown_concept