kodjo/Sheerka-Old
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fixed #43 : BnfNodeParser: I can recognize when multiple level of ISA

Browse Source 
Fixed #44 : BnfNodeParser: I must simplify results when multiple levels of ISA
Fixed #45 : Dynamic variables cannot be parsed at restart
Fixed #46 : Concepts variables values are transformed into list by default
Fixed #47 : SheerkaAdmin. Add min, max, mean time when restoring files
This commit is contained in:
Kodjo Sossouvi
2021-03-08 17:35:30 +01:00
parent bd8e027827
commit 031bd0274e
20 changed files with 303 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/core/concept.py
+2 -3
View File
@@ -82,7 +82,7 @@ def get_concept_attrs(concept):
# create a class attribute
with all_attributes_lock:
all_attributes = [k for k in concept.__dict__ if k[0] != "_" and k[0] != "#"]
all_attributes = [var_name for var_name, var_value in concept.get_metadata().variables]
if concept.id and concept.key not in BuiltinDynamicAttrs:
ALL_ATTRIBUTES[concept.id] = all_attributes
return all_attributes
@@ -91,7 +91,7 @@ def get_concept_attrs(concept):
def freeze_concept_attrs(concept):
with all_attributes_lock:
if concept.key not in BuiltinDynamicAttrs:
ALL_ATTRIBUTES[concept.id] = [k for k in concept.__dict__ if k[0] != "_" and k[0] != "#"]
ALL_ATTRIBUTES[concept.id] = [var_name for var_name, var_value in concept.get_metadata().variables]
def copy_concepts_attrs():
@@ -501,7 +501,6 @@ class Concept:
def variables(self):
return {k: v for k, v in self.values().items() if not k[0] == "#"}
# return dict([(k, v) for k, v in self.values.items() if isinstance(k, str)])
def set_hint(self, name, value):
self._hints[name] = value
src/core/sheerka/services/SheerkaAdmin.py
+24 -5
View File
@@ -78,6 +78,7 @@ class SheerkaAdmin(BaseService):
def restore_from_file(file_name):
_nb_lines, _nb_instructions, _nb_lines_in_error = 0, 0, 0
_min_time, _max_time = None, None
file_path = path.join(path.dirname(sys.argv[0]), file_name)
if not path.exists(file_path):
print(f"\u001b[31mFile '{file_path}' is not found !\u001b[0m")
@@ -90,11 +91,15 @@ class SheerkaAdmin(BaseService):
if line.startswith("#import "):
to_import = "_concepts_" + line[8:] + ".txt"
print(f"Importing {to_import}")
print(f" ==== Importing {to_import} ==== ")
res = restore_from_file(to_import)
_nb_lines += res[0]
_nb_instructions += res[1]
_nb_lines_in_error += res[2]
if _min_time is None or res[3] < _min_time:
_min_time = res[3]
if _max_time is None or res[4] > _max_time:
_max_time = res[4]
continue
if line == "" or line.startswith("#"):
@@ -102,12 +107,20 @@ class SheerkaAdmin(BaseService):
print(line)
_nb_instructions += 1
stop_watch = time.time_ns()
res = self.sheerka.evaluate_user_input(line)
user_input_elapsed = time.time_ns() - stop_watch
if _min_time is None or user_input_elapsed < _min_time:
_min_time = user_input_elapsed
if _max_time is None or user_input_elapsed > _max_time:
_max_time = user_input_elapsed
if len(res) > 1 or not res[0].status:
_nb_lines_in_error += 1
print("\u001b[31mError detected !\u001b[0m")
return _nb_lines, _nb_instructions, _nb_lines_in_error
return _nb_lines, _nb_instructions, _nb_lines_in_error, _min_time, _max_time
if not concept_file.startswith("_concepts"):
concept_file = f"_concepts_{concept_file}.txt"
@@ -119,7 +132,7 @@ class SheerkaAdmin(BaseService):
enable_process_return_values_previous_value = self.sheerka.enable_process_return_values
self.sheerka.enable_process_return_values = False
nb_lines, nb_instructions, nb_lines_in_error = restore_from_file(concept_file)
nb_lines, nb_instructions, nb_lines_in_error, min_time, max_time = restore_from_file(concept_file)
self.sheerka.enable_process_return_values = enable_process_return_values_previous_value
self.sheerka.save_execution_context = True
@@ -128,8 +141,13 @@ class SheerkaAdmin(BaseService):
nano_sec = stop - start
dt = nano_sec / 1e6
elapsed = f"{dt} ms" if dt < 1000 else f"{dt / 1000} s"
print(f"Imported {nb_lines} line(s) in {elapsed}.")
elapsed = f"{dt:.3f} ms" if dt < 1000 else f"{dt / 1000:.3f} s"
min_str = f"{min_time / 1e6:.3f} ms"
max_str = f"{max_time / 1e6:.3f} ms"
mean_time = dt / nb_lines
mean_str = f"{mean_time:.3f} ms" if mean_time < 1000 else f"{mean_time / 1000:.3f} s"
print(f"Imported {nb_lines} line(s) in {elapsed}. min={min_str}, max={max_str}, mean={mean_str}")
print(f"{nb_instructions} instruction(s).")
if nb_lines_in_error > 0:
print(f"\u001b[31m{nb_lines_in_error} errors(s) found.\u001b[0m")
@@ -153,6 +171,7 @@ class SheerkaAdmin(BaseService):
"key": item.key,
"definition": item.get_metadata().definition,
"type": item.get_metadata().definition_type,
"hash": item.get_definition_hash(),
"body": item.get_metadata().body,
"where": item.get_metadata().where,
"pre": item.get_metadata().pre,
src/core/sheerka/services/SheerkaConceptManager.py
+19 -5
View File
@@ -397,6 +397,17 @@ class SheerkaConceptManager(BaseService):
ensure_concept(concept)
attr = attribute.str_id if isinstance(attribute, Concept) else attribute
if (old_value := concept.get_value(attr)) is not NotInit:
if old_value == value:
return self.sheerka.ret(self.NAME, True, self.sheerka.new(BuiltinConcepts.SUCCESS))
if isinstance(old_value, list):
old_value.append(value)
value = old_value
else:
value = [old_value, value]
concept.set_value(attr, value)
return self.sheerka.ret(self.NAME, True, self.sheerka.new(BuiltinConcepts.SUCCESS))
@@ -892,11 +903,14 @@ class SheerkaConceptManager(BaseService):
else:
concepts = [concept]
for concept in concepts:
ensure_bnf(context, concept) # need to make sure that it cannot fail
keywords = SheerkaConceptManager.get_first_tokens(sheerka, concept)
for keyword in keywords:
(to_resolve if keyword.startswith("c:|") else resolved).add(keyword)
for c in concepts:
if sheerka.isaset(context, c):
to_resolve.add(c.str_id)
else:
ensure_bnf(context, c) # need to make sure that it cannot fail
keywords = SheerkaConceptManager.get_first_tokens(sheerka, c)
for keyword in keywords:
(to_resolve if keyword.startswith("c:|") else resolved).add(keyword)
for concept_to_resolve_str in to_resolve:
res = resolve_concepts(concept_to_resolve_str)
src/core/sheerka/services/SheerkaDebugManager.py
+2
View File
@@ -10,6 +10,7 @@ from core.sheerka.ExecutionContext import ExecutionContext
from core.sheerka.services.sheerka_service import BaseService
from core.utils import CONSOLE_COLORS_MAP as CCM, CONSOLE_COLUMNS, PRIMITIVES_TYPES
from core.utils import as_bag
from evaluators.MultipleSuccessEvaluator import MultipleSuccessEvaluator
from parsers.BaseNodeParser import SourceCodeWithConceptNode, UnrecognizedTokensNode
pp = pprint.PrettyPrinter(indent=2, width=CONSOLE_COLUMNS)
@@ -349,6 +350,7 @@ class SheerkaDebugManager(BaseService):
self.register_debug_vars("Exceptions", PythonEvaluator.NAME+"-eval", "exception")
self.register_debug_vars("Exceptions", PythonEvaluator.NAME+"-eval", "trace")
self.register_debug_vars(SyaNodeParser.NAME, "parse", "*")
self.register_debug_vars(MultipleSuccessEvaluator.NAME, "matches", "return_values")
def initialize_deferred(self, context, is_first_time):
self.restore_state()
src/core/sheerka/services/SheerkaExecute.py
+4 -4
View File
@@ -368,7 +368,8 @@ class SheerkaExecute(BaseService):
if pi is NotFound: # when CacheManager.cache_only is True
pi = ParserInput(text)
self.pi_cache.put(text, pi)
return ParserInput(text, tokens=pi.tokens, length=pi.length) # new instance, but no need to tokenize the text again
return ParserInput(text, tokens=pi.tokens,
length=pi.length) # new instance, but no need to tokenize the text again
key = text or core.utils.get_text_from_tokens(tokens)
pi = ParserInput(key, tokens=tokens, length=len(tokens))
@@ -705,9 +706,8 @@ class SheerkaExecute(BaseService):
# disable all parsers but the requested ones
if parsers != "all":
sub_context.preprocess_parsers = parsers
# sub_context.add_preprocess(BaseParser.PREFIX + "*", enabled=False)
# for parser in parsers:
# sub_context.add_preprocess(BaseParser.PREFIX + parser, enabled=True)
else:
sub_context.preprocess_parsers = None
if prop in (Keywords.WHERE, Keywords.PRE, ConceptParts.WHERE, ConceptParts.PRE, Keywords.WHEN):
sub_context.protected_hints.add(BuiltinConcepts.EVAL_QUESTION_REQUESTED)
src/evaluators/MultipleSuccessEvaluator.py
+24 -1
View File
@@ -10,6 +10,7 @@ class MultipleSuccessEvaluator(AllReturnValuesEvaluator):
So we cannot decide whether it's a MultipleSameSuccess or not
All parser in error will be discarded
Cannot match if there is at least one evaluator in error
Cannot match if there is at least one successful parser
"""
NAME = "MultipleSuccess"
@@ -25,28 +26,50 @@ class MultipleSuccessEvaluator(AllReturnValuesEvaluator):
nb_evaluators_in_success = 0
to_process = False
debugger = context.get_debugger(MultipleSuccessEvaluator.NAME, "matches")
debugger.debug_entering(return_value=return_values)
for ret in return_values:
if ret.status and ret.who.startswith(BaseParser.PREFIX):
reason = "a successful parser found"
debugger.debug_log(f"Failed because {reason}. ret={ret}")
return False
elif ret.who.startswith(BaseEvaluator.PREFIX) and not ret.status:
reason = "a failed evaluator found"
debugger.debug_log(f"Failed because {reason}. ret={ret}")
return False
elif ret.status and context.sheerka.isinstance(ret.body, BuiltinConcepts.REDUCE_REQUESTED):
to_process = True
self.eaten.append(ret)
debugger.debug_log(f"BuiltinConcepts.REDUCE_REQUESTED is found. {to_process=}")
elif ret.status and ret.who.startswith(BaseEvaluator.PREFIX):
if self.already_seen(context, ret):
reason = "of duplicate return value"
debugger.debug_log(f"Failed because {reason}. ret={ret}")
return False
nb_evaluators_in_success += 1
self.successful_return_values.append(ret)
self.eaten.append(ret)
debugger.debug_log(f"Eating and keeping {ret=}. {nb_evaluators_in_success=}")
elif not ret.status and ret.who.startswith(BaseParser.PREFIX):
self.eaten.append(ret)
debugger.debug_log(f"Eating {ret=}.")
# else:
# other concepts. We do not care if there are successful or not
# They won't be part of result nor part of the parent
# --> So they will be handled by other evaluators
return to_process and nb_evaluators_in_success > 1
res = to_process and nb_evaluators_in_success > 1
if not res:
reason = "not to_process and nb_evaluators_in_success > 1"
debugger.debug_log(f"Failed because '{reason}', {to_process=}, {nb_evaluators_in_success=}")
return res
def eval(self, context, return_values):
context.log(f"{len(self.successful_return_values)} successful return values, {len(self.eaten)} item(s) eaten",
src/parsers/BnfNodeParser.py
+58 -6
View File
@@ -115,6 +115,12 @@ class NonTerminalNode(ParseTreeNode):
res = f"{self.parsing_expression.concept}=>" if isinstance(self.parsing_expression, ConceptExpression) else ""
return res + ".".join([c.get_debug() for c in self.children])
def get_depth(self):
if isinstance(self.parsing_expression, ConceptExpression):
return 1 + max([c.get_depth() for c in self.children])
else:
return max([c.get_depth() for c in self.children])
class TerminalNode(ParseTreeNode):
"""
@@ -150,6 +156,9 @@ class TerminalNode(ParseTreeNode):
def get_debug(self):
return str(self.value)
def get_depth(self):
return 0
class MultiNode:
""""
@@ -246,6 +255,12 @@ class ParsingContext:
res = f"ParsingContext('{self.node.get_debug()}', pos={self.pos})"
return res
def get_depth(self):
if isinstance(self.node, list):
return max([n.get_depth() for n in self.node])
else:
return self.node.get_depth()
class ParsingExpression:
log_sink = []
@@ -738,8 +753,9 @@ class UnOrderedChoice(ParsingExpression):
if parser_helper.debugger.is_enabled():
debug_prefix = self.debug_prefix("UnOrderedChoice", parser_helper)
debug_text = {"pos": parser_helper.pos, "text": self.debug_remaining_text(parser_helper)}
parser_helper.debug_concept(debug_prefix, raw=f"{CCM['green']}{debug_text}{CCM['reset']}")
debug_vars = {"pos": parser_helper.pos, "text": self.debug_remaining_text(parser_helper)}
debug_text = self.debug_to_raw(debug_vars)
parser_helper.debug_concept(debug_prefix, color="cyan", raw=debug_text)
debug_text = ""
for e in self.nodes:
@@ -772,16 +788,52 @@ class UnOrderedChoice(ParsingExpression):
parser_helper.seek(parsing_contexts[0].pos)
if len(parsing_contexts) == 1:
return parsing_contexts[0].node
# Try to simplify the parsing_context
simplified_parsing_contexts = self.simplify(parsing_contexts)
if parser_helper.debugger.is_enabled() and len(simplified_parsing_contexts) != len(parsing_contexts):
parser_helper.debug_concept(debug_prefix, simplified=simplified_parsing_contexts)
if len(simplified_parsing_contexts) == 1:
return simplified_parsing_contexts[0].node
else:
parsing_contexts.sort(key=attrgetter("pos"), reverse=True)
return MultiNode(parsing_contexts)
simplified_parsing_contexts.sort(key=attrgetter("pos"), reverse=True)
return MultiNode(simplified_parsing_contexts)
def __repr__(self):
to_str = "# ".join(repr(n) for n in self.elements)
return self.add_rule_name_if_needed(f"({to_str})")
@staticmethod
def simplify(parsing_contexts: List[ParsingContext]):
"""
Try to remove redundant parsing context
for example, if
color is an adjective
red is an adjective
red is a color
when parsing 'red' we will receive two parsing context
one for 'red'
one for 'color' -> 'red'
The second one should be discarded
:param parsing_contexts:
:return:
"""
if len(parsing_contexts) == 1:
return parsing_contexts
by_target = {}
for pc in parsing_contexts:
by_target.setdefault(pc.node.source, []).append((pc, pc.get_depth()))
res = []
for k, tuple_pc_pc_depth in by_target.items():
min_depth = min([pc_depth for pc, pc_depth in tuple_pc_pc_depth])
res.extend([pc for pc, pc_depth in tuple_pc_pc_depth if pc_depth == min_depth])
return res
class Optional(ParsingExpression):
"""