kodjo/Sheerka-Old
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fixed #29: Parsers: Implement parsing memoization

Browse Source 
Fixed #77 : Parser: ShortTermMemoryParser should be called separately
Fixed #78 : Remove VariableNode usage
Fixed #79 : ConceptManager: Implement compile caching
Fixed #80 : SheerkaExecute : parsers_key is not correctly computed
Fixed #81 : ValidateConceptEvaluator : Validate concept's where and pre clauses right after the parsing
Fixed #82 : SheerkaIsAManager: isa() failed when the set as a body
Fixed #83 : ValidateConceptEvaluator : Support BNF and SYA Concepts
Fixed #84 : ExpressionParser: Implement the parser as a standard parser
Fixed #85 : Services: Give order to services
Fixed #86 : cannot manage smart_get_attr(the short, color)
This commit is contained in:
Kodjo Sossouvi
2021-06-07 21:14:03 +02:00
parent 1059ce25c5
commit 7dcaa9c111
92 changed files with 4263 additions and 1890 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/core/builtin_helpers.py
+116 -49
View File
@@ -5,17 +5,17 @@ from cache.Cache import Cache
from core.ast_helpers import ast_to_props
from core.builtin_concepts import BuiltinConcepts
from core.concept import Concept, ConceptParts, DEFINITION_TYPE_BNF, concept_part_value
from core.global_symbols import NotInit, NotFound, CURRENT_OBJ
from core.global_symbols import NotInit, NotFound, INIT_AST_PARSERS, DEFAULT_EVALUATORS
from core.rule import Rule
from core.tokenizer import Tokenizer
from core.tokenizer import Tokenizer, TokenKind
from core.utils import as_bag
from parsers.BaseNodeParser import SourceCodeNode, ConceptNode, UnrecognizedTokensNode, SourceCodeWithConceptNode, \
RuleNode, VariableNode
RuleNode, LexerNode
from parsers.BaseParser import ParsingError
PARSE_STEPS = [BuiltinConcepts.BEFORE_PARSING, BuiltinConcepts.PARSING, BuiltinConcepts.AFTER_PARSING]
EVAL_STEPS = PARSE_STEPS + [BuiltinConcepts.BEFORE_EVALUATION, BuiltinConcepts.EVALUATION,
BuiltinConcepts.AFTER_EVALUATION]
EVAL_ONLY_STEPS = [BuiltinConcepts.BEFORE_EVALUATION, BuiltinConcepts.EVALUATION, BuiltinConcepts.AFTER_EVALUATION]
EVAL_STEPS = PARSE_STEPS + EVAL_ONLY_STEPS
PARSERS = ["EmptyString", "ShortTermMemory", "Sequence", "Bnf", "Sya", "Python"]
@@ -48,7 +48,7 @@ def is_same_success(context, return_values):
if not ret_val.status:
raise Exception("Status is false")
if isinstance(ret_val.body, Concept) and not ret_val.body.get_metadata().is_evaluated:
if isinstance(ret_val.body, Concept) and not ret_val.body.get_hints().is_evaluated:
raise Exception("Concept is not evaluated")
return context.sheerka.objvalue(ret_val)
@@ -223,7 +223,10 @@ def resolve_ambiguity(context, concepts):
remaining_concepts.extend(by_complexity[complexity])
else:
for c in by_complexity[complexity]:
evaluated = context.sheerka.evaluate_concept(context, c, metadata=[ConceptParts.PRE])
evaluated = context.sheerka.evaluate_concept(context, c,
eval_body=False,
validation_only=True,
metadata=[ConceptParts.PRE])
if context.sheerka.is_success(evaluated) or evaluated.key == c.key:
remaining_concepts.append(c)
@@ -316,7 +319,8 @@ def only_parsers_results(context, return_values):
def evaluate_from_source(context,
source,
evaluators="all",
parsers=INIT_AST_PARSERS,
evaluators=DEFAULT_EVALUATORS,
desc=None,
eval_body=True,
eval_where=True,
@@ -327,13 +331,14 @@ def evaluate_from_source(context,
:param context:
:param source:
:param parsers:
:param evaluators:
:param desc:
:param eval_body:
:param eval_where:
:param is_question:
:param expect_success:
:param stm: short term memories entries
:param stm: short term memories entries AKA current namespace
:return:
"""
@@ -362,12 +367,11 @@ def evaluate_from_source(context,
for k, v in stm.items():
sub_context.add_to_short_term_memory(k, v)
# disable all evaluators but the requested ones
if parsers != "all":
sub_context.preprocess_parsers = parsers
if evaluators != "all":
from evaluators.BaseEvaluator import BaseEvaluator
sub_context.add_preprocess(BaseEvaluator.PREFIX + "*", enabled=False)
for evaluator in evaluators:
sub_context.add_preprocess(BaseEvaluator.PREFIX + evaluator, enabled=True)
sub_context.preprocess_evaluators = evaluators
user_input = sheerka.ret(context.who, True, sheerka.new(BuiltinConcepts.USER_INPUT, body=source))
ret = sheerka.execute(sub_context, [user_input], EVAL_STEPS)
@@ -376,6 +380,52 @@ def evaluate_from_source(context,
return ret
def evaluate_return_values(context,
source,
return_values,
evaluators=DEFAULT_EVALUATORS,
desc=None,
eval_body=True,
eval_where=True,
is_question=False,
expect_success=False,
stm=None):
sheerka = context.sheerka
desc = desc or f"Eval '{source}' using return values"
hints_to_reset = {
BuiltinConcepts.EVAL_BODY_REQUESTED,
BuiltinConcepts.EVAL_WHERE_REQUESTED,
BuiltinConcepts.EVAL_UNTIL_SUCCESS_REQUESTED,
BuiltinConcepts.EVAL_QUESTION_REQUESTED,
}
with context.push(BuiltinConcepts.EVALUATE_SOURCE, source, desc=desc, reset_hints=hints_to_reset) as sub_context:
if eval_body:
sub_context.protected_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
if eval_where:
sub_context.protected_hints.add(BuiltinConcepts.EVAL_WHERE_REQUESTED)
if expect_success:
sub_context.protected_hints.add(BuiltinConcepts.EVAL_UNTIL_SUCCESS_REQUESTED)
if is_question:
sub_context.protected_hints.add(BuiltinConcepts.EVAL_QUESTION_REQUESTED)
if stm:
for k, v in stm.items():
sub_context.add_to_short_term_memory(k, v)
if evaluators != "all":
sub_context.preprocess_evaluators = evaluators
sub_context.add_inputs(return_values=return_values)
res = sheerka.execute(sub_context, return_values.copy(), EVAL_ONLY_STEPS)
one_r = expect_one(context, res)
sub_context.add_values(return_values=one_r)
return one_r
def get_lexer_nodes(return_values, start, tokens):
"""
Transform all elements from return_values into lexer nodes (ConceptNode, UnrecognizedTokensNode, SourceCodeNode...)
@@ -386,10 +436,14 @@ def get_lexer_nodes(return_values, start, tokens):
:param tokens:
:return: list of list (list of concept node sequence)
"""
from evaluators.BaseEvaluator import BaseEvaluator
lexer_nodes = []
for ret_val in return_values:
if ret_val.who == "parsers.Python":
# To manage AFTER_PARSING evaluators
who = ret_val.parents[0].who if ret_val.who.startswith(BaseEvaluator.PREFIX) else ret_val.who
if who == "parsers.Python":
if ret_val.body.source.strip().isidentifier():
# Discard SourceCodeNode which seems to be a concept name
@@ -404,27 +458,29 @@ def get_lexer_nodes(return_values, start, tokens):
python_node=ret_val.body.body,
return_value=ret_val)])
elif ret_val.who == "parsers.ExactConcept":
elif who == "parsers.ExactConcept":
concepts = ret_val.body.body if hasattr(ret_val.body.body, "__iter__") else [ret_val.body.body]
end = start + len(tokens) - 1
for concept in concepts:
lexer_nodes.append([ConceptNode(concept, start, end, tokens, ret_val.body.source)])
elif ret_val.who in ("parsers.Bnf", "parsers.Sya", "parsers.Sequence"):
nodes = [node for node in ret_val.body.body]
elif who in ("parsers.Bnf", "parsers.Sya", "parsers.Sequence"):
nodes = [node.clone() for node in ret_val.body.body]
for node in nodes:
node.start += start
node.end += start
if isinstance(node, ConceptNode):
for k, v in node.concept.get_compiled().items():
if hasattr(v, "start"):
if isinstance(v, LexerNode):
v = v.clone()
v.start += start
v.end += start
node.concept.get_compiled()[k] = v
# but append the whole sequence if when it's a sequence
lexer_nodes.append(nodes)
elif ret_val.who == "parsers.Rule":
elif who == "parsers.Rule":
rules = ret_val.body.body if hasattr(ret_val.body.body, "__iter__") else [ret_val.body.body]
end = start + len(tokens) - 1
for rule in rules:
@@ -447,9 +503,14 @@ def get_lexer_nodes_using_positions(return_values, positions):
:return:
"""
from evaluators.BaseEvaluator import BaseEvaluator
lexer_nodes = []
for ret_val, position in zip(return_values, positions):
if ret_val.who in ("parsers.Python", 'parsers.PythonWithConcepts'):
# To manage AFTER_PARSING evaluators
who = ret_val.parents[0].who if ret_val.who.startswith(BaseEvaluator.PREFIX) else ret_val.who
if who in ("parsers.Python", 'parsers.PythonWithConcepts'):
lexer_nodes.append(SourceCodeNode(position.start,
position.end,
@@ -458,7 +519,7 @@ def get_lexer_nodes_using_positions(return_values, positions):
python_node=ret_val.body.body,
return_value=ret_val))
elif ret_val.who == "parsers.ExactConcept":
elif who == "parsers.ExactConcept":
concepts = ret_val.body.body if hasattr(ret_val.body.body, "__iter__") else [ret_val.body.body]
for concept in concepts:
lexer_nodes.append(ConceptNode(concept,
@@ -467,21 +528,23 @@ def get_lexer_nodes_using_positions(return_values, positions):
position.tokens,
ret_val.body.source))
elif ret_val.who in ("parsers.Bnf", "parsers.Sya", "parsers.Sequence"):
nodes = [node for node in ret_val.body.body]
elif who in ("parsers.Bnf", "parsers.Sya", "parsers.Sequence"):
nodes = [node.clone() for node in ret_val.body.body]
for node in nodes:
node.start = position.start
node.end = position.end
if isinstance(node, ConceptNode):
for k, v in node.concept.get_compiled().items():
if hasattr(v, "start"):
if isinstance(v, LexerNode):
v = v.clone()
v.start += position.start
v.end += position.start
node.concept.get_compiled()[k] = v
# but append the whole sequence if when it's a sequence
lexer_nodes.extend(nodes)
elif ret_val.who == "parsers.Rule":
elif who == "parsers.Rule":
rules = ret_val.body.body if hasattr(ret_val.body.body, "__iter__") else [ret_val.body.body]
for rule in rules:
lexer_nodes.append(RuleNode(rule,
@@ -489,7 +552,7 @@ def get_lexer_nodes_using_positions(return_values, positions):
position.end,
position.tokens, ret_val.body.source))
elif ret_val.who == "parsers.Function":
elif who == "parsers.Function":
node = ret_val.body.body
node.start = position.start
node.end = position.end
@@ -510,8 +573,10 @@ def ensure_evaluated(context, concept, eval_body=True, metadata=None):
:param metadata:
:return:
"""
if concept.get_metadata().is_evaluated:
return concept
if concept.get_hints().is_evaluated:
from core.sheerka.services.SheerkaEvaluateConcept import SheerkaEvaluateConcept
return SheerkaEvaluateConcept.apply_ret(concept,
eval_body or context.in_context(BuiltinConcepts.EVAL_BODY_REQUESTED))
# do not try to evaluate concept that are not fully initialized
if concept.get_metadata().definition_type != DEFINITION_TYPE_BNF:
@@ -533,19 +598,6 @@ def get_lexer_nodes_from_unrecognized(context, unrecognized_tokens_node, parsers
:param parsers:
:return:
"""
# first look into short term memory to see if the unrecognized is not a variable of the current object
if (current_obj := context.sheerka.get_from_short_term_memory(context, CURRENT_OBJ)) is not NotFound:
if isinstance(current_obj, Concept):
source = unrecognized_tokens_node.source
if source in current_obj.get_compiled() or source in current_obj.variables():
return [[VariableNode(current_obj,
source,
unrecognized_tokens_node.start,
unrecognized_tokens_node.end,
unrecognized_tokens_node.tokens,
unrecognized_tokens_node.source)]]
res = context.sheerka.parse_unrecognized(context, unrecognized_tokens_node.source, parsers)
res = only_parsers_results(context, res)
@@ -630,7 +682,7 @@ def update_compiled(context, concept, errors, parsers=None):
if _get_source(concept.get_compiled(), name) != name:
break
else:
concept.get_metadata().is_evaluated = True
concept.get_hints().is_evaluated = True
def add_to_ret_val(sheerka, context, return_values, concept_key):
@@ -665,10 +717,10 @@ def set_is_evaluated(concepts, check_nb_variables=False):
if hasattr(concepts, "__iter__"):
for c in concepts:
if not check_nb_variables or check_nb_variables and len(c.get_metadata().variables) > 0:
c.get_metadata().is_evaluated = True
c.get_hints().is_evaluated = True
else:
if not check_nb_variables or check_nb_variables and len(concepts.get_metadata().variables) > 0:
concepts.get_metadata().is_evaluated = True
concepts.get_hints().is_evaluated = True
def ensure_concept(*concepts):
@@ -701,7 +753,7 @@ def ensure_concept_or_rule(*items):
raise TypeError(f"'{items}' must be a concept or rule")
def ensure_bnf(context, concept, parser_name="BaseNodeParser", update_bnf_for_cached_concept=True):
def ensure_bnf(context, concept, parser_name="BaseNodeParser"):
if concept.get_metadata().definition_type == DEFINITION_TYPE_BNF and not concept.get_bnf():
from parsers.BnfDefinitionParser import BnfDefinitionParser
regex_parser = BnfDefinitionParser()
@@ -719,8 +771,6 @@ def ensure_bnf(context, concept, parser_name="BaseNodeParser", update_bnf_for_ca
raise Exception(bnf_parsing_ret_val.value)
concept.set_bnf(bnf_parsing_ret_val.body.body)
if concept.id and update_bnf_for_cached_concept:
context.sheerka.get_by_id(concept.id).set_bnf(concept.get_bnf()) # update bnf in cache
expressions_cache = Cache()
@@ -828,7 +878,24 @@ def get_possible_variables_from_concept(context, concept):
concept_name = [t.str_value for t in Tokenizer(concept.name, yield_eof=False)]
names = [v_value or v_name for v_name, v_value in concept.get_metadata().variables if v_name in concept_name]
possible_vars = filter(lambda x: context.sheerka.is_not_a_concept_name(x), names)
return set(possible_vars)
to_keep = set()
for var in possible_vars:
tokens = Tokenizer(var, yield_eof=False)
for t in tokens:
if t.type in (TokenKind.IDENTIFIER, TokenKind.KEYWORD):
to_keep.add(var)
return to_keep
def is_only_successful(sheerka, return_value):
"""
:param sheerka:
:param return_value
:return:
"""
return sheerka.isinstance(return_value, BuiltinConcepts.RETURN_VALUE) and \
sheerka.isinstance(return_value.body, BuiltinConcepts.ONLY_SUCCESSFUL)
class CreateObjectIdentifiers: