Implemented some enhancement request and fixed some bugs
Fixed #2 : Variables are not recognized when inside a rule token Fixed #15 : Rule: rete attributes are lost when a new ontology is created Fixed #14 : ReteNetwork: Format rules must not be added to Rete network Fixed #16 : DefConcept: Variables are not recognized when they are keyword arguments Fixed #4 : Comparison are not correctly set when comparison property is a concept Fixed #14 : Parser: merge FunctionParser.NamesNode and ExpressionParser.NamesNode Fixed #18 : Parser: Add SourceCodeNode test to UnrecognizedNodeParser Fixed #20 : At startup Number concept is saved in db a numerous number of time Fixed #21 : CacheManager: I can remove all elements from a ListIfNeededCache and fill it again Fixed #22 : CacheManager: I can remove all elements from a SetCache and fill it again Fixed #23 : HistoryManager: history() no longer works Fixed #24 : HistoryManager: history() no longer works after creating an exec rule Fixed #25 : SheerkaMemory: Use MemoryObject instead of sheerka.local Fixed #26 : Debugger: add the list all available services.. Fixed #27 : CONCEPTS_GRAMMARS_ENTRY does not seems to be in use any more Fixed #28 : Give order to services
This commit is contained in:
@@ -1,23 +1,20 @@
|
||||
import core.utils
|
||||
from cache.FastCache import FastCache
|
||||
from core.builtin_concepts import BuiltinConcepts, ReturnValueConcept
|
||||
from core.global_symbols import NotFound
|
||||
from core.concept import ConceptParts
|
||||
from core.global_symbols import NotFound, NO_MATCH
|
||||
from core.sheerka.services.sheerka_service import BaseService
|
||||
from core.tokenizer import Tokenizer, TokenKind, Token
|
||||
from core.tokenizer import Tokenizer, TokenKind, Token, Keywords
|
||||
|
||||
NO_MATCH = "** No Match **"
|
||||
EVALUATOR_STEPS = [
|
||||
BuiltinConcepts.BEFORE_PARSING,
|
||||
BuiltinConcepts.AFTER_PARSING,
|
||||
BuiltinConcepts.BEFORE_EVALUATION,
|
||||
BuiltinConcepts.EVALUATION,
|
||||
BuiltinConcepts.AFTER_EVALUATION,
|
||||
BuiltinConcepts.BEFORE_RENDERING,
|
||||
BuiltinConcepts.RENDERING,
|
||||
BuiltinConcepts.AFTER_RENDERING,
|
||||
BuiltinConcepts.BEFORE_RULES_EVALUATION,
|
||||
BuiltinConcepts.AFTER_RULES_EVALUATION,
|
||||
]
|
||||
PARSE_STEPS = [BuiltinConcepts.BEFORE_PARSING, BuiltinConcepts.PARSING, BuiltinConcepts.AFTER_PARSING]
|
||||
PARSE_AND_EVAL_STEPS = PARSE_STEPS + [BuiltinConcepts.BEFORE_EVALUATION,
|
||||
BuiltinConcepts.EVALUATION,
|
||||
BuiltinConcepts.AFTER_EVALUATION]
|
||||
ALL_STEPS = PARSE_AND_EVAL_STEPS + [BuiltinConcepts.BEFORE_RENDERING,
|
||||
BuiltinConcepts.RENDERING,
|
||||
BuiltinConcepts.AFTER_RENDERING,
|
||||
BuiltinConcepts.BEFORE_RULES_EVALUATION,
|
||||
BuiltinConcepts.AFTER_RULES_EVALUATION]
|
||||
|
||||
|
||||
class ParserInput:
|
||||
@@ -173,14 +170,15 @@ class SheerkaExecute(BaseService):
|
||||
|
||||
def __init__(self, sheerka):
|
||||
# order must be after SheerkaEvaluateRules because of self.rules_evaluation_service
|
||||
super().__init__(sheerka, order=5)
|
||||
# order must be after ConceptManager because it needs concept bnf definitions
|
||||
super().__init__(sheerka, order=15)
|
||||
self.pi_cache = FastCache(default=lambda key: ParserInput(key), max_size=20)
|
||||
self.instantiated_evaluators = None
|
||||
self.evaluators_by_name = None
|
||||
|
||||
self.instantiated_parsers = None
|
||||
self.parsers_by_name = None
|
||||
self.old_values = []
|
||||
self.preprocessed_items_old_values = []
|
||||
|
||||
# cache for all preregistered evaluator combination
|
||||
# the key is the concatenation of the step and the name of evaluators in the group
|
||||
@@ -202,6 +200,10 @@ class SheerkaExecute(BaseService):
|
||||
def initialize(self):
|
||||
self.sheerka.bind_service_method(self.execute, True, visible=False)
|
||||
self.sheerka.bind_service_method(self.execute_rules, True, visible=False)
|
||||
self.sheerka.bind_service_method(self.parse_unrecognized, False, visible=False)
|
||||
self.sheerka.bind_service_method(self.parse_function, False, visible=False)
|
||||
self.sheerka.bind_service_method(self.parse_python, False, visible=False)
|
||||
self.sheerka.bind_service_method(self.parse_expression, False, visible=False)
|
||||
|
||||
self.reset_registered_evaluators()
|
||||
self.reset_registered_parsers()
|
||||
@@ -219,7 +221,7 @@ class SheerkaExecute(BaseService):
|
||||
self.evaluators_by_name = {e.short_name: e for e in self.instantiated_evaluators}
|
||||
|
||||
# get default evaluators by process step
|
||||
for process_step in EVALUATOR_STEPS:
|
||||
for process_step in ALL_STEPS:
|
||||
self.grouped_evaluators_cache[f"{process_step}|__default"] = self.get_grouped(
|
||||
[e for e in self.instantiated_evaluators if process_step in e.steps])
|
||||
|
||||
@@ -270,7 +272,7 @@ class SheerkaExecute(BaseService):
|
||||
if var_name == "preprocess_name":
|
||||
continue
|
||||
if hasattr(item, var_name):
|
||||
self.old_values.append((item, var_name, getattr(item, var_name)))
|
||||
self.preprocessed_items_old_values.append((item, var_name, getattr(item, var_name)))
|
||||
setattr(item, var_name, value)
|
||||
|
||||
def get_evaluators(self, context, process_step):
|
||||
@@ -319,29 +321,23 @@ class SheerkaExecute(BaseService):
|
||||
for priority, parsers_classes in from_cache[0].items()}
|
||||
return grouped_instances, from_cache[1]
|
||||
|
||||
# Normal case, use all registered parsers
|
||||
if not context.preprocess_parsers and not context.preprocess:
|
||||
return get_instances(self.grouped_parsers_cache["__default"])
|
||||
|
||||
# Other case, only use a subset of parsers
|
||||
# This case is heavily used by lexer node parsers, thru parse_unrecognized
|
||||
if context.preprocess_parsers and not context.preprocess:
|
||||
key = "|".join(context.preprocess_parsers)
|
||||
key = self.get_parsers_key(context)
|
||||
if key:
|
||||
try:
|
||||
return get_instances(self.grouped_parsers_cache[key])
|
||||
return key, *get_instances(self.grouped_parsers_cache[key])
|
||||
except KeyError:
|
||||
parsers = [self.parsers_by_name[p] for p in context.preprocess_parsers if p in self.parsers_by_name]
|
||||
self.grouped_parsers_cache[key] = self.get_grouped(parsers, use_classes=True)
|
||||
return get_instances(self.grouped_parsers_cache[key])
|
||||
return key, *get_instances(self.grouped_parsers_cache[key])
|
||||
|
||||
# final case, parsers attributes are modified by the context
|
||||
# else, case where parsers attributes are modified by the context
|
||||
# This a the case when we want to disable a specific parser, or change the order of priority
|
||||
parsers = [self.parsers_by_name[p] for p in context.preprocess_parsers if p in self.parsers_by_name] \
|
||||
if context.preprocess_parsers else self.instantiated_parsers
|
||||
self.preprocess(parsers, context.preprocess)
|
||||
parsers = [p for p in parsers if p.enabled] # only keep those that are still enabled
|
||||
groups, sorted_priorities = self.get_grouped(parsers, use_classes=True)
|
||||
return get_instances((groups, sorted_priorities))
|
||||
return key, *get_instances((groups, sorted_priorities))
|
||||
|
||||
def get_parser_input(self, text, tokens=None):
|
||||
"""
|
||||
@@ -366,6 +362,22 @@ class SheerkaExecute(BaseService):
|
||||
self.pi_cache.put(key, pi)
|
||||
return pi
|
||||
|
||||
@staticmethod
|
||||
def get_parsers_key(context):
|
||||
"""
|
||||
From the context.preprocess_parsers and context.preprocess,
|
||||
try to find a key to store the further results of the parsings
|
||||
:param context:
|
||||
:return:
|
||||
"""
|
||||
if not context.preprocess_parsers and not context.preprocess:
|
||||
return "__default"
|
||||
|
||||
if context.preprocess_parsers and not context.preprocess:
|
||||
return "|".join(context.preprocess_parsers)
|
||||
|
||||
return None
|
||||
|
||||
def call_parsers(self, context, return_values):
|
||||
"""
|
||||
Call all the parsers, ordered by priority
|
||||
@@ -398,7 +410,7 @@ class SheerkaExecute(BaseService):
|
||||
# keep track of the originals user inputs, as they need to be removed at the end
|
||||
user_inputs = to_process[:]
|
||||
|
||||
grouped_parsers, sorted_priorities = self.get_parsers(context)
|
||||
parsers_key, grouped_parsers, sorted_priorities = self.get_parsers(context)
|
||||
|
||||
stop_processing = False
|
||||
for priority in sorted_priorities:
|
||||
@@ -644,10 +656,10 @@ class SheerkaExecute(BaseService):
|
||||
return return_values
|
||||
|
||||
def undo_preprocess(self):
|
||||
for item, var_name, value in self.old_values:
|
||||
for item, var_name, value in self.preprocessed_items_old_values:
|
||||
setattr(item, var_name, value)
|
||||
|
||||
self.old_values.clear()
|
||||
self.preprocessed_items_old_values.clear()
|
||||
|
||||
@staticmethod
|
||||
def matches(parser_or_evaluator_name, preprocessor_name):
|
||||
@@ -655,3 +667,107 @@ class SheerkaExecute(BaseService):
|
||||
return parser_or_evaluator_name.startswith(preprocessor_name[:-1])
|
||||
else:
|
||||
return parser_or_evaluator_name == preprocessor_name
|
||||
|
||||
def parse_unrecognized(self, context, source, parsers, who=None, prop=None, filter_func=None):
|
||||
"""
|
||||
Try to recognize concepts or code from source using the given parsers
|
||||
:param context:
|
||||
:param source: ParserInput if possible
|
||||
:param parsers:
|
||||
:param who: who is asking the parsing ?
|
||||
:param prop: Extra info, when parsing a property
|
||||
:param filter_func: Once the result are found, call this function to filter them
|
||||
:return:
|
||||
"""
|
||||
sheerka = context.sheerka
|
||||
|
||||
if prop:
|
||||
action_context = {"prop": prop, "source": source}
|
||||
desc = f"Parsing attribute '{prop}'"
|
||||
else:
|
||||
action_context = source
|
||||
desc = f"Parsing '{source}'"
|
||||
|
||||
with context.push(BuiltinConcepts.PARSING, action_context, who=who, desc=desc) as sub_context:
|
||||
# disable all parsers but the requested ones
|
||||
if parsers != "all":
|
||||
sub_context.preprocess_parsers = parsers
|
||||
# sub_context.add_preprocess(BaseParser.PREFIX + "*", enabled=False)
|
||||
# for parser in parsers:
|
||||
# sub_context.add_preprocess(BaseParser.PREFIX + parser, enabled=True)
|
||||
|
||||
if prop in (Keywords.WHERE, Keywords.PRE, ConceptParts.WHERE, ConceptParts.PRE, Keywords.WHEN):
|
||||
sub_context.protected_hints.add(BuiltinConcepts.EVAL_QUESTION_REQUESTED)
|
||||
|
||||
sub_context.add_inputs(source=source)
|
||||
to_parse = sheerka.ret(context.who,
|
||||
True,
|
||||
sheerka.new(BuiltinConcepts.USER_INPUT, body=source))
|
||||
res = sheerka.execute(sub_context, to_parse, PARSE_STEPS)
|
||||
|
||||
if filter_func:
|
||||
res = filter_func(sub_context, res)
|
||||
|
||||
sub_context.add_values(return_values=res)
|
||||
return res
|
||||
|
||||
def parse_function(self, context, source, tokens=None, start=0):
|
||||
"""
|
||||
Helper function that parses what is supposed to be a function
|
||||
:param context:
|
||||
:param source:
|
||||
:param tokens:
|
||||
:param start: start index for the source code node
|
||||
:return:
|
||||
"""
|
||||
from parsers.BaseNodeParser import SourceCodeWithConceptNode
|
||||
|
||||
sheerka = context.sheerka
|
||||
from parsers.FunctionParser import FunctionParser
|
||||
parser = FunctionParser()
|
||||
desc = f"Parsing function '{source}'"
|
||||
with context.push(BuiltinConcepts.PARSE_CODE, source, desc=desc) as sub_context:
|
||||
sheerka_execution = sheerka.services[SheerkaExecute.NAME]
|
||||
res = parser.parse(sub_context, sheerka_execution.get_parser_input(source, tokens))
|
||||
|
||||
if not isinstance(res, list):
|
||||
res = [res]
|
||||
|
||||
for r in [r for r in res if sheerka.isinstance(r.body, BuiltinConcepts.PARSER_RESULT)]:
|
||||
r.body.body.start += start
|
||||
r.body.body.end += start
|
||||
|
||||
if isinstance(r.body.body, SourceCodeWithConceptNode):
|
||||
for n in [r.body.body.first, r.body.body.last] + r.body.body.nodes:
|
||||
n.start += start
|
||||
n.end += start
|
||||
|
||||
return res
|
||||
|
||||
def parse_python(self, context, source, desc=None):
|
||||
"""
|
||||
Helper function that parses what is known to be Python source code
|
||||
:param context:
|
||||
:param source:
|
||||
:param desc: option description when creating the sub context
|
||||
"""
|
||||
from parsers.PythonParser import PythonParser
|
||||
|
||||
desc = desc or f"Compiling python '{source}'"
|
||||
with context.push(BuiltinConcepts.PARSE_CODE,
|
||||
{"language": "Python", "source": source},
|
||||
desc) as sub_context:
|
||||
parser_input = context.sheerka.services[SheerkaExecute.NAME].get_parser_input(source)
|
||||
python_parser = PythonParser()
|
||||
return python_parser.parse(sub_context, parser_input)
|
||||
|
||||
def parse_expression(self, context, source, desc=None):
|
||||
"""
|
||||
Helper function to parser expressions with AND, OR and NOT
|
||||
"""
|
||||
desc = desc or f"Parsing expression '{source}'"
|
||||
with context.push(BuiltinConcepts.PARSE_CODE, source, desc) as sub_context:
|
||||
parser_input = context.sheerka.services[SheerkaExecute.NAME].get_parser_input(source)
|
||||
from parsers.ExpressionParser import ExpressionParser
|
||||
expr_parser = ExpressionParser()
|
||||
return expr_parser.parse(sub_context, parser_input)
|
||||
|
||||
Reference in New Issue
Block a user