I can define and eval BNF definitions

core/sheerka.py

@@ -1,5 +1,4 @@
 from dataclasses import dataclass, field
-from functools import lru_cache
 
 from core.builtin_concepts import BuiltinConcepts, ErrorConcept, ReturnValueConcept
 from core.concept import Concept, ConceptParts, PROPERTIES_FOR_DIGEST
@@ -40,11 +39,12 @@ class Sheerka(Concept):
 
         #
         # Cache for all concepts BNF
+        #
         self.concepts_definitions = {}
 
         #
         # cache for concepts grammars
-        # a grammar can be seen as a resolved BNF
+        # a grammar is a resolved BNF
         self.concepts_grammars = {}
 
         # a concept can be instantiated
@@ -79,14 +79,18 @@ class Sheerka(Concept):
 
         try:
             self.init_logging()
-            self.sdp = SheerkaDataProvider(root_folder)
 
+            self.sdp = SheerkaDataProvider(root_folder)
             if self.sdp.first_time:
                 self.sdp.set_key(self.USER_CONCEPTS_KEYS, 1000)
 
+            evt_digest = self.sdp.save_event(Event("Initializing Sheerka."))
+            exec_context = ExecutionContext(self.key, evt_digest, self)
+
             self.initialize_builtin_concepts()
             self.initialize_builtin_parsers()
             self.initialize_builtin_evaluators()
+            self.initialize_concepts_definitions(exec_context)
 
         except IOError as e:
             return ReturnValueConcept(self, False, self.get(BuiltinConcepts.ERROR), e)
@@ -149,19 +153,35 @@ class Sheerka(Concept):
             init_log.debug(f"Adding builtin evaluator '{evaluator.__name__}'")
             self.evaluators.append(evaluator)
 
-    def logger_filter(self, record: logging.LogRecord):
-        if 'all' in self.loggers:
-            return True
+    def initialize_concepts_definitions(self, execution_context):
+        init_log.debug("Initializing concepts definitions")
+        definitions = self.sdp.get_safe(self.CONCEPTS_DEFINITIONS_ENTRY, load_origin=False)
 
-        ret = True
-        if 'init' not in self.loggers and record.name.endswith(".init"):
-                ret = False
+        if definitions is None:
+            init_log.debug("No BNF defined")
+            return
 
-        return ret
+        lexer_parser = self.parsers[CONCEPT_LEXER_PARSER_CLASS]()
+        ret_val = lexer_parser.initialize(execution_context, definitions)
+        if not ret_val.status:
+            init_log.error("Failed to initialize concepts definitions " + str(ret_val.body))
+            return
+
+        self.concepts_grammars = lexer_parser.concepts_grammars
 
     def init_logging(self):
+        def _logger_filter(record: logging.LogRecord):
+            if 'all' in self.loggers:
+                return True
+
+            ret = True
+            if 'init' not in self.loggers and record.name.endswith(".init"):
+                ret = False
+
+            return ret
+
         handler = logging.StreamHandler()
-        handler.addFilter(self.logger_filter)
+        handler.addFilter(_logger_filter)
         if self.debug:
             log_format = "%(asctime)s %(name)s [%(levelname)s] %(message)s"
             log_level = logging.DEBUG
@@ -211,7 +231,7 @@ class Sheerka(Concept):
                 else "'" + BaseParser.get_text_from_tokens(text) + "' as tokens"
             log.debug(f"Parsing {debug_text}")
         for parser in self.parsers.values():
-            p = parser()
+            p = parser(sheerka=self)
             res = p.parse(context, text)
             if isinstance(res, list):
                 result.extend(res)
@@ -347,7 +367,7 @@ class Sheerka(Concept):
             concepts_definitions[concept] = concept.bnf
 
             # check if it's a valid BNF or whether it breaks the known rules
-            concept_lexer_parser = self.parsers[CONCEPT_LEXER_PARSER_CLASS](self.concepts_grammars.copy())
+            concept_lexer_parser = self.parsers[CONCEPT_LEXER_PARSER_CLASS](grammars=self.concepts_grammars.copy())
             sub_context = context.push(self.name, "Initializing concept definition")
             sub_context.concepts_cache[concept.key] = concept  # the concept is not in the real cache yet
             init_ret_value = concept_lexer_parser.initialize(sub_context, concepts_definitions)

core/utils.py

@@ -1,7 +1,6 @@
 import importlib
 import inspect
 import pkgutil
-import sys
 
 from core.tokenizer import TokenKind