kodjo/Sheerka-Old
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added bnf when adding a new concept + Started logging filtering

Browse Source
This commit is contained in:
Kodjo Sossouvi
2019-12-13 20:26:11 +01:00
parent 75c8793d53
commit c668cc46d2
29 changed files with 1487 additions and 190 deletions
Download Patch File Download Diff File Expand all files Collapse all files
core/sheerka.py
+85 -24
View File
@@ -1,4 +1,6 @@
from dataclasses import dataclass
from dataclasses import dataclass, field
from functools import lru_cache
from core.builtin_concepts import BuiltinConcepts, ErrorConcept, ReturnValueConcept
from core.concept import Concept, ConceptParts, PROPERTIES_FOR_DIGEST
from evaluators.BaseEvaluator import OneReturnValueEvaluator
@@ -10,8 +12,10 @@ import core.builtin_helpers
import logging
log = logging.getLogger(__name__)
init_log = logging.getLogger(__name__ + ".init")
concept_evaluation_steps = [BuiltinConcepts.EVALUATION, BuiltinConcepts.AFTER_EVALUATION]
CONCEPT_LEXER_PARSER_CLASS = "parsers.ConceptLexerParser.ConceptLexerParser"
class Sheerka(Concept):
@@ -19,22 +23,29 @@ class Sheerka(Concept):
Main controller for the project
"""
CONCEPTS_ENTRY = "All_Concepts"
BUILTIN_CONCEPTS_KEYS = "Builtins_Concepts"
USER_CONCEPTS_KEYS = "User_Concepts"
CONCEPTS_ENTRY = "All_Concepts" # to store all the concepts
CONCEPTS_DEFINITIONS_ENTRY = "Concepts_Definitions" # to store definitions (bnf) of concepts
BUILTIN_CONCEPTS_KEYS = "Builtins_Concepts" # sequential key for builtin concepts
USER_CONCEPTS_KEYS = "User_Concepts" # sequential key for user defined concepts
def __init__(self, debug=False, skip_builtins_in_db=False):
def __init__(self, debug=False, skip_builtins_in_db=False, loggers=None):
log.debug("Starting Sheerka.")
super().__init__(BuiltinConcepts.SHEERKA, True, True, BuiltinConcepts.SHEERKA)
# cache of the most used concepts
# Note that these are only templates
# They are used as a footprint for instantiation
# Except of source when the concept is supposed to be unique
self.concepts_cache = {}
# cache for builtin types.
# It allow instantiation of a builtin clas
self.builtin_cache = {}
#
# Cache for all concepts BNF
self.concepts_definitions = {}
#
# cache for concepts grammars
# a grammar can be seen as a resolved BNF
self.concepts_grammars = {}
# a concept can be instantiated
# ex: File is a concept, but File('foo.txt') is an instance
@@ -45,14 +56,16 @@ class Sheerka(Concept):
# ex: hello => say('hello')
self.rules = []
self.sdp = None
self.parsers = []
self.evaluators = []
self.sdp: SheerkaDataProvider = None # SheerkaDataProvider
self.builtin_cache = {} # cache for builtin concepts
self.parsers = {} # cache for builtin parsers
self.evaluators = [] # cache for builtin evaluators
self.evaluators_prefix = None
self.parsers_prefix = None
self.evaluators_prefix: str = None
self.parsers_prefix: str = None
self.debug = debug
self.loggers = loggers or []
self.skip_builtins_in_db = skip_builtins_in_db
def initialize(self, root_folder: str = None):
@@ -85,7 +98,7 @@ class Sheerka(Concept):
Initializes the builtin concepts
:return: None
"""
log.debug("Initializing builtin concepts")
init_log.debug("Initializing builtin concepts")
builtins_classes = self.get_builtins_classes_as_dict()
# this all initialization of the builtins seems to be little bit complicated
@@ -101,11 +114,11 @@ class Sheerka(Concept):
if not self.skip_builtins_in_db:
from_db = self.sdp.get_safe(self.CONCEPTS_ENTRY, concept.metadata.key)
if from_db is None:
log.debug(f"'{concept.name}' concept is not found in db. Adding.")
init_log.debug(f"'{concept.name}' concept is not found in db. Adding.")
self.set_id_if_needed(concept, True)
self.sdp.add("init", self.CONCEPTS_ENTRY, concept, use_ref=True)
else:
log.debug(f"Found concept '{from_db}' in db. Updating.")
init_log.debug(f"Found concept '{from_db}' in db. Updating.")
concept.update_from(from_db)
self.add_in_cache(concept)
@@ -120,8 +133,8 @@ class Sheerka(Concept):
if parser.__module__ == base_class.__module__:
continue
log.debug(f"Adding builtin parser '{parser.__name__}'")
self.parsers.append(parser)
init_log.debug(f"Adding builtin parser '{parser.__name__}'")
self.parsers[core.utils.get_full_qualified_name(parser)] = parser
def initialize_builtin_evaluators(self):
"""
@@ -129,14 +142,26 @@ class Sheerka(Concept):
:return:
"""
for evaluator in core.utils.get_sub_classes("evaluators", "evaluators.BaseEvaluator.OneReturnValueEvaluator"):
log.debug(f"Adding builtin evaluator '{evaluator.__name__}'")
init_log.debug(f"Adding builtin evaluator '{evaluator.__name__}'")
self.evaluators.append(evaluator)
for evaluator in core.utils.get_sub_classes("evaluators", "evaluators.BaseEvaluator.AllReturnValuesEvaluator"):
log.debug(f"Adding builtin evaluator '{evaluator.__name__}'")
init_log.debug(f"Adding builtin evaluator '{evaluator.__name__}'")
self.evaluators.append(evaluator)
def logger_filter(self, record: logging.LogRecord):
if 'all' in self.loggers:
return True
ret = True
if 'init' not in self.loggers and record.name.endswith(".init"):
ret = False
return ret
def init_logging(self):
handler = logging.StreamHandler()
handler.addFilter(self.logger_filter)
if self.debug:
log_format = "%(asctime)s %(name)s [%(levelname)s] %(message)s"
log_level = logging.DEBUG
@@ -144,7 +169,7 @@ class Sheerka(Concept):
log_format = "%(message)s"
log_level = logging.INFO
logging.basicConfig(format=log_format, level=log_level)
logging.basicConfig(format=log_format, level=log_level, handlers=[handler])
def eval(self, text: str):
"""
@@ -153,7 +178,9 @@ class Sheerka(Concept):
:param text:
:return:
"""
log.debug(f"Evaluating '{text}'.")
evt_digest = self.sdp.save_event(Event(text))
log.debug(f"{evt_digest=}")
exec_context = ExecutionContext(self.key, evt_digest, self)
# Before parsing
@@ -183,7 +210,7 @@ class Sheerka(Concept):
debug_text = "'" + text + "'" if isinstance(text, str) \
else "'" + BaseParser.get_text_from_tokens(text) + "' as tokens"
log.debug(f"Parsing {debug_text}")
for parser in self.parsers:
for parser in self.parsers.values():
p = parser()
res = p.parse(context, text)
if isinstance(res, list):
@@ -193,7 +220,7 @@ class Sheerka(Concept):
return result
def process(self, context, return_values, initial_concepts=None):
log.debug(f"Processing parsing result. context concept={initial_concepts}")
log.debug(f"{initial_concepts=}. Processing " + core.utils.pp(return_values))
# return_values must be a list
if not isinstance(return_values, list):
@@ -303,6 +330,8 @@ class Sheerka(Concept):
"""
concept.init_key()
concepts_definitions = None
init_ret_value = None
# checks for duplicate concepts
if self.sdp.exists(self.CONCEPTS_ENTRY, concept.key, concept.get_digest()):
@@ -312,14 +341,33 @@ class Sheerka(Concept):
# set id before saving in db
self.set_id_if_needed(concept, False)
# add the BNF if known
if concept.bnf:
concepts_definitions = self.concepts_definitions.copy()
concepts_definitions[concept] = concept.bnf
# check if it's a valid BNF or whether it breaks the known rules
concept_lexer_parser = self.parsers[CONCEPT_LEXER_PARSER_CLASS](self.concepts_grammars.copy())
sub_context = context.push(self.name, "Initializing concept definition")
sub_context.concepts_cache[concept.key] = concept # the concept is not in the real cache yet
init_ret_value = concept_lexer_parser.initialize(sub_context, concepts_definitions)
if not init_ret_value.status:
return self.ret(self.create_new_concept.__name__, False, ErrorConcept(init_ret_value.value))
# save the new context in sdp
try:
self.sdp.add(context.event_digest, self.CONCEPTS_ENTRY, concept, use_ref=True)
if concepts_definitions is not None:
self.sdp.set(context.event_digest, self.CONCEPTS_DEFINITIONS_ENTRY, concepts_definitions, use_ref=True)
except SheerkaDataProviderDuplicateKeyError as error:
return self.ret(self.create_new_concept.__name__, False, ErrorConcept(error), error.args[0])
# add in cache for quick further reference
# Updates the caches
self.concepts_cache[concept.key] = self.sdp.get_safe(self.CONCEPTS_ENTRY, concept.key)
if concepts_definitions is not None:
self.concepts_definitions = concepts_definitions
if init_ret_value is not None and init_ret_value.status:
self.concepts_grammars = init_ret_value.body
# process the return in needed
ret = self.ret(self.create_new_concept.__name__, True, self.new(BuiltinConcepts.NEW_CONCEPT, body=concept))
@@ -514,6 +562,18 @@ class Sheerka(Concept):
return (self.value(obj) for obj in objs)
def is_success(self, obj):
if isinstance(obj, bool):
return obj
if self.isinstance(obj, BuiltinConcepts.RETURN_VALUE):
return obj.status
if self.isinstance(obj, BuiltinConcepts.ERROR):
return False
return False
def isinstance(self, a, b):
"""
return true if the concept a is an instance of the concept b
@@ -603,6 +663,7 @@ class ExecutionContext:
sheerka: Sheerka # sheerka
desc: str = None # human description of what is going on
obj: Concept = None # what is the subject of the execution context (if known)
concepts_cache: dict = field(default_factory=dict)
def push(self, who, desc=None, obj=None):
return ExecutionContext(who, self.event_digest, self.sheerka, desc=desc, obj=obj)