kodjo/Sheerka-Old
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

I can manage infinite recursion when building concept

Browse Source
This commit is contained in:
Kodjo Sossouvi
2020-12-10 21:08:10 +01:00
parent 4b6e1dd55b
commit 657c7536f7
28 changed files with 816 additions and 446 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/core/sheerka/services/SheerkaExecute.py
+100 -91
View File
@@ -169,16 +169,34 @@ class SheerkaExecute(BaseService):
self.pi_cache = Cache(default=lambda key: ParserInput(key), max_size=20)
self.instantiated_evaluators = None
self.evaluators_by_name = None
self.grouped_evaluators_cache = {} # key=step, value=tuple(evaluators for this step, sorted priorities)
self.instantiated_parsers = None
self.parsers_by_name = None
self.old_values = []
# cache for all preregistered evaluator combination
# the key is the concatenation of the step and the name of evaluators in the group
# ex : BEFORE_EVALUATION|Python|Sya|Bnf
# The value is a tuple,
# The first entry is the grouped evaluators
# ex : {60 : [PythonEvaluator(), SyaEvaluator()], 50: [BnfEvaluator()]}
# The second entry are the sorted priorities ex [60, 50]
self.grouped_evaluators_cache = {}
# cache for preregistered parsers
# Same construction than the evaluators
# Except 1 : the key does not have a step component. It is simple the list of parsers' names
# Except 2 : we store the type of the parser, not its instance
self.grouped_parsers_cache = {}
def initialize(self):
self.sheerka.bind_service_method(self.execute, True)
self.sheerka.cache_manager.register_cache(self.PARSERS_INPUTS_ENTRY, self.pi_cache, False)
self.reset_evaluators()
self.reset_registered_evaluators()
self.reset_registered_parsers()
def reset_evaluators(self):
def reset_registered_evaluators(self):
# instantiate evaluators, once for all, only keep when it's enabled
self.instantiated_evaluators = [e_class() for e_class in self.sheerka.evaluators]
self.instantiated_evaluators = [e for e in self.instantiated_evaluators if e.enabled]
@@ -186,44 +204,49 @@ class SheerkaExecute(BaseService):
# get default evaluators by process step
for process_step in EVALUATOR_STEPS:
self.grouped_evaluators_cache[f"{process_step}|__default"] = self.get_grouped_evaluators(
self.grouped_evaluators_cache[f"{process_step}|__default"] = self.get_grouped(
[e for e in self.instantiated_evaluators if process_step in e.steps])
# @staticmethod
# def get_grouped_evaluators(instantiated_evaluators, process_step):
# """
# For a given list of evaluators and a given process step
# Computes
# * the evaluators eligible for this step
# * the list of sorted priorities for theses evaluators
# :param instantiated_evaluators:
# :param process_step:
# :return:
# """
# grouped = {}
# for evaluator in [e for e in instantiated_evaluators if e.enabled and process_step in e.steps]:
# grouped.setdefault(evaluator.priority, []).append(evaluator)
#
# sorted_groups = sorted(grouped.keys(), reverse=True)
# return grouped, sorted_groups
def reset_registered_parsers(self):
"""
Browse all parsers and only keep those which are enabled
:return:
"""
self.instantiated_parsers = [parser(sheerka=self.sheerka) for parser in self.sheerka.parsers.values()]
self.instantiated_parsers = [p for p in self.instantiated_parsers if p.enabled]
self.parsers_by_name = {p.short_name: p for p in self.instantiated_parsers}
self.grouped_parsers_cache["__default"] = self.get_grouped(self.instantiated_parsers, use_classes=True)
@staticmethod
def get_grouped_evaluators(evaluators):
def get_grouped(evaluators, use_classes=False):
"""
For a given list of evaluators,
group them by priorities
sort the priorities
:param evaluators:
:param use_classes: if True, store the class (the type) of the evaluator, not its instance
:return: tuple({priority: List of evaluators with this priority}, list of sorted priorities)
"""
grouped = {}
for evaluator in evaluators:
grouped.setdefault(evaluator.priority, []).append(evaluator)
grouped.setdefault(evaluator.priority, []).append(type(evaluator) if use_classes else evaluator)
sorted_groups = sorted(grouped.keys(), reverse=True)
return grouped, sorted_groups
def preprocess(self, items, preprocess_definitions):
"""
Modifies the attributes of item
:param items: either a parser or an evaluator
:param preprocess_definitions: how to modify List of BuiltinConcepts.EVALUATOR_PRE_PROCESS
preprocess.get_value("preprocess_name") gives the name of the property to alter
preprocess.values() gives the alterations
ex:
Sheerka.new(BuiltinConcepts.EVALUATOR_PRE_PROCESS, preprocess_name="parser_name", enabled=False)
Will disable parser 'parser_name'
:return:
"""
for preprocess in preprocess_definitions:
for item in items:
if self.matches(item.name, preprocess.get_value("preprocess_name")):
@@ -234,51 +257,9 @@ class SheerkaExecute(BaseService):
self.old_values.append((item, var_name, getattr(item, var_name)))
setattr(item, var_name, value)
def preprocess_old(self, context, parsers_or_evaluators, mode):
if mode == "parsers":
if not context.preprocess and not context.preprocess_parsers:
return parsers_or_evaluators
items = context.preprocess_parsers
elif mode == "evaluators":
if not context.preprocess and not context.preprocess_evaluators:
return parsers_or_evaluators
items = context.preprocess_evaluators
else:
raise ValueError(mode)
if not hasattr(parsers_or_evaluators, "__iter__"):
single_one = True
parsers_or_evaluators = [parsers_or_evaluators]
else:
single_one = False
if items:
res = []
for item in items:
for e in parsers_or_evaluators:
if item == e.name:
res.append(e)
break
else:
raise ValueError(f"{item} not found.")
parsers_or_evaluators = res
if context.preprocess:
for preprocess in context.preprocess:
for e in parsers_or_evaluators:
if self.matches(e.name, preprocess.get_value("name")):
for var_name in preprocess.values:
if var_name == "name":
continue
if hasattr(e, var_name):
self.old_values.append((e, var_name, getattr(e, var_name)))
setattr(e, var_name, preprocess.get_value(var_name))
return parsers_or_evaluators[0] if single_one else parsers_or_evaluators
def get_evaluators(self, context, process_step):
"""
Returns the list of evaluators to use for a specific test
Returns the list of evaluators to use for a specific context need
:param context:
:param process_step:
:return:
@@ -287,30 +268,64 @@ class SheerkaExecute(BaseService):
if not context.preprocess_evaluators and not context.preprocess:
return self.grouped_evaluators_cache[f"{process_step}|__default"]
# First case, only use a subset of evaluators
if context.preprocess_evaluators and not context.preprocess:
key = str(process_step) + "|" + "|".join(context.preprocess_evaluators)
# Other case, only use a subset of evaluators
selected = context.preprocess_evaluators
if selected and not context.preprocess:
key = str(process_step) + "|" + "|".join(selected)
try:
return self.grouped_evaluators_cache[key]
except KeyError:
evaluators = [self.evaluators_by_name[e] for e in context.preprocess_evaluators]
grouped = self.get_grouped_evaluators(evaluators)
evaluators = [self.evaluators_by_name[e] for e in selected if e in self.evaluators_by_name]
evaluators = [e for e in evaluators if process_step in e.steps] # check the process step
grouped = self.get_grouped(evaluators)
self.grouped_evaluators_cache[key] = grouped
return grouped
# final case, evaluators attributes are modified by the context
# So first, get the modified evaluators
evaluators = [self.evaluators_by_name[e] for e in
context.preprocess_evaluators] if context.preprocess_evaluators else self.instantiated_evaluators
# Final case, evaluators attributes are modified by the context
evaluators = [self.evaluators_by_name[e] for e in selected if
e in self.evaluators_by_name] if selected else self.instantiated_evaluators
evaluators = [e for e in evaluators if process_step in e.steps] # check the process step
self.preprocess(evaluators, context.preprocess)
evaluators = [e for e in evaluators if e.enabled] # make sure they are still enabled
key = str(process_step) + "|" + "|".join([e.name for e in evaluators if e.enabled])
try:
return self.grouped_evaluators_cache[key]
except KeyError:
grouped = self.get_grouped_evaluators(evaluators)
self.grouped_evaluators_cache[key] = grouped
return grouped
return self.get_grouped(evaluators)
def get_parsers(self, context):
"""
We cannot use a single instance of a parser shared among executions as it's common to have a parser
calling another parser or even calling itself
So the cache holds the parser classes or sorted priorities
:param context:
:return:
"""
def get_instances(from_cache):
grouped_instances = {priority: [p(sheerka=self.sheerka) for p in parsers_classes]
for priority, parsers_classes in from_cache[0].items()}
return grouped_instances, from_cache[1]
# Normal case, use all registered parsers
if not context.preprocess_parsers and not context.preprocess:
return get_instances(self.grouped_parsers_cache["__default"])
# Other case, only use a subset of parsers
# This case is heavily used by lexer node parsers, thru parse_unrecognized
if context.preprocess_parsers and not context.preprocess:
key = "|".join(context.preprocess_parsers)
try:
return get_instances(self.grouped_parsers_cache[key])
except KeyError:
parsers = [self.parsers_by_name[p] for p in context.preprocess_parsers if p in self.parsers_by_name]
self.grouped_parsers_cache[key] = self.get_grouped(parsers, use_classes=True)
return get_instances(self.grouped_parsers_cache[key])
# final case, parsers attributes are modified by the context
# This a the case when we want to disable a specific parser, or change the order of priority
parsers = [self.parsers_by_name[p] for p in context.preprocess_parsers if p in self.parsers_by_name] \
if context.preprocess_parsers else self.instantiated_parsers
self.preprocess(parsers, context.preprocess)
parsers = [p for p in parsers if p.enabled] # only keep those that are still enabled
groups, sorted_priorities = self.get_grouped(parsers, use_classes=True)
return get_instances((groups, sorted_priorities))
def get_parser_input(self, text, tokens=None):
"""
@@ -367,14 +382,7 @@ class SheerkaExecute(BaseService):
# keep track of the originals user inputs, as they need to be removed at the end
user_inputs = to_process[:]
# group the parsers by priorities
instantiated_parsers = [parser(sheerka=self.sheerka) for parser in self.sheerka.parsers.values()]
instantiated_parsers = self.preprocess_old(context, instantiated_parsers, "parsers")
grouped_parsers = {}
for parser in [p for p in instantiated_parsers if p.enabled]:
grouped_parsers.setdefault(parser.priority, []).append(parser)
sorted_priorities = sorted(grouped_parsers.keys(), reverse=True)
grouped_parsers, sorted_priorities = self.get_parsers(context)
stop_processing = False
for priority in sorted_priorities:
@@ -427,6 +435,7 @@ class SheerkaExecute(BaseService):
break # Do not try the other priorities if a match is found
result = core.utils.remove_list_from_list(result, user_inputs)
return result
def call_evaluators(self, context, return_values, process_step):
@@ -470,7 +479,7 @@ class SheerkaExecute(BaseService):
# init the evaluator is possible
# KSI. 20201102 : Evaluators are now instantiated at startup,
# Can we move this section into reset_evaluators()
# Can we move this section into reset_registered_evaluators()
if hasattr(evaluator, "init_evaluator") and not evaluator.is_initialized:
evaluator.init_evaluator(sub_context, original_items)