Enhanced complex concepts handling
This commit is contained in:
@@ -82,7 +82,7 @@ class AddConceptEvaluator(OneReturnValueEvaluator):
|
||||
|
||||
# finish initialisation
|
||||
concept.init_key(def_concept_node.name.tokens)
|
||||
concept.add_codes(def_concept_node.get_asts())
|
||||
#concept.add_codes(def_concept_node.get_asts())
|
||||
if not isinstance(def_concept_node.definition, NotInitializedNode) and \
|
||||
sheerka.is_success(def_concept_node.definition):
|
||||
concept.bnf = def_concept_node.definition.value.value
|
||||
|
||||
@@ -1,109 +1,109 @@
|
||||
from core.builtin_concepts import BuiltinConcepts, ParserResultConcept
|
||||
from core.concept import Concept
|
||||
from core.tokenizer import TokenKind
|
||||
from evaluators.BaseEvaluator import AllReturnValuesEvaluator, BaseEvaluator
|
||||
from parsers.BaseParser import BaseParser
|
||||
from parsers.ConceptLexerParser import ConceptNode, UnrecognizedTokensNode, ConceptLexerParser
|
||||
import core.utils
|
||||
|
||||
|
||||
class ConceptComposerEvaluator(AllReturnValuesEvaluator):
|
||||
"""
|
||||
Try to reassemble parts of concepts from different evaluators
|
||||
"""
|
||||
|
||||
NAME = "ConceptComposer"
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(self.NAME, [BuiltinConcepts.EVALUATION], 40)
|
||||
|
||||
def matches(self, context, return_values):
|
||||
concept_lexer_parser_name = ConceptLexerParser().name
|
||||
|
||||
for return_value in return_values:
|
||||
if return_value.who.startswith(BaseParser.PREFIX) and return_value.status:
|
||||
return False
|
||||
|
||||
if return_value.who.startswith(BaseEvaluator.PREFIX):
|
||||
return False
|
||||
|
||||
if return_value.who != concept_lexer_parser_name:
|
||||
continue
|
||||
|
||||
if not isinstance(return_value.value, ParserResultConcept):
|
||||
return False
|
||||
|
||||
if not (
|
||||
isinstance(return_value.value.value, ConceptNode) or
|
||||
isinstance(return_value.value.value, UnrecognizedTokensNode) or
|
||||
(
|
||||
hasattr(return_value.value.value, "__iter__") and
|
||||
len(return_value.value.value) > 0 and
|
||||
(
|
||||
isinstance(return_value.value.value[0], ConceptNode) or
|
||||
isinstance(return_value.value.value[0], UnrecognizedTokensNode)
|
||||
))):
|
||||
return False
|
||||
|
||||
self.eaten = return_value
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
def eval(self, context, return_value):
|
||||
sheerka = context.sheerka
|
||||
nodes = self.eaten.value.value
|
||||
temp_res = []
|
||||
has_error = False
|
||||
concepts_only = True
|
||||
|
||||
for node in nodes:
|
||||
if isinstance(node, UnrecognizedTokensNode):
|
||||
tokens = core.utils.strip_tokens(node.tokens, True)
|
||||
for token in tokens:
|
||||
if token.type == TokenKind.IDENTIFIER:
|
||||
concept = context.new_concept(token.value)
|
||||
if sheerka.isinstance(concept, BuiltinConcepts.UNKNOWN_CONCEPT):
|
||||
has_error = True
|
||||
else:
|
||||
with context.push(self.name, desc=f"Evaluating '{concept}'") as sub_context:
|
||||
sub_context.log_new(self.verbose_log)
|
||||
concept = sheerka.evaluate_concept(sub_context, concept, self.verbose_log)
|
||||
sub_context.add_values(return_values=concept)
|
||||
temp_res.append(concept)
|
||||
|
||||
else:
|
||||
temp_res.append(core.utils.strip_quotes(token.value))
|
||||
concepts_only &= token.type == TokenKind.WHITESPACE or token.type == TokenKind.NEWLINE
|
||||
else:
|
||||
with context.push(self.name, desc=f"Evaluating '{node.concept}'") as sub_context:
|
||||
sub_context.log_new(self.verbose_log)
|
||||
concept = sheerka.evaluate_concept(sub_context, node.concept, self.verbose_log)
|
||||
sub_context.add_values(return_values=concept)
|
||||
temp_res.append(concept)
|
||||
|
||||
if has_error:
|
||||
return sheerka.ret(
|
||||
self.name,
|
||||
False,
|
||||
temp_res,
|
||||
parents=[self.eaten])
|
||||
|
||||
if concepts_only:
|
||||
res = []
|
||||
for r in temp_res:
|
||||
if isinstance(r, Concept):
|
||||
res.append(r)
|
||||
else:
|
||||
res = ""
|
||||
for r in temp_res:
|
||||
if isinstance(r, Concept):
|
||||
res += sheerka.value(r)
|
||||
else:
|
||||
res += r
|
||||
|
||||
return sheerka.ret(
|
||||
self.name,
|
||||
True,
|
||||
res,
|
||||
parents=[self.eaten])
|
||||
# from core.builtin_concepts import BuiltinConcepts, ParserResultConcept
|
||||
# from core.concept import Concept
|
||||
# from core.tokenizer import TokenKind
|
||||
# from evaluators.BaseEvaluator import AllReturnValuesEvaluator, BaseEvaluator
|
||||
# from parsers.BaseParser import BaseParser
|
||||
# from parsers.ConceptLexerParser import ConceptNode, UnrecognizedTokensNode, ConceptLexerParser
|
||||
# import core.utils
|
||||
#
|
||||
#
|
||||
# class ConceptComposerEvaluator(AllReturnValuesEvaluator):
|
||||
# """
|
||||
# Try to reassemble parts of concepts from different evaluators
|
||||
# """
|
||||
#
|
||||
# NAME = "ConceptComposer"
|
||||
#
|
||||
# def __init__(self):
|
||||
# super().__init__(self.NAME, [BuiltinConcepts.EVALUATION], 40)
|
||||
#
|
||||
# def matches(self, context, return_values):
|
||||
# concept_lexer_parser_name = ConceptLexerParser().name
|
||||
#
|
||||
# for return_value in return_values:
|
||||
# if return_value.who.startswith(BaseParser.PREFIX) and return_value.status:
|
||||
# return False
|
||||
#
|
||||
# if return_value.who.startswith(BaseEvaluator.PREFIX):
|
||||
# return False
|
||||
#
|
||||
# if return_value.who != concept_lexer_parser_name:
|
||||
# continue
|
||||
#
|
||||
# if not isinstance(return_value.value, ParserResultConcept):
|
||||
# return False
|
||||
#
|
||||
# if not (
|
||||
# isinstance(return_value.value.value, ConceptNode) or
|
||||
# isinstance(return_value.value.value, UnrecognizedTokensNode) or
|
||||
# (
|
||||
# hasattr(return_value.value.value, "__iter__") and
|
||||
# len(return_value.value.value) > 0 and
|
||||
# (
|
||||
# isinstance(return_value.value.value[0], ConceptNode) or
|
||||
# isinstance(return_value.value.value[0], UnrecognizedTokensNode)
|
||||
# ))):
|
||||
# return False
|
||||
#
|
||||
# self.eaten = return_value
|
||||
# return True
|
||||
#
|
||||
# return False
|
||||
#
|
||||
# def eval(self, context, return_value):
|
||||
# sheerka = context.sheerka
|
||||
# nodes = self.eaten.value.value
|
||||
# temp_res = []
|
||||
# has_error = False
|
||||
# concepts_only = True
|
||||
#
|
||||
# for node in nodes:
|
||||
# if isinstance(node, UnrecognizedTokensNode):
|
||||
# tokens = core.utils.strip_tokens(node.tokens, True)
|
||||
# for token in tokens:
|
||||
# if token.type == TokenKind.IDENTIFIER:
|
||||
# concept = context.new_concept(token.value)
|
||||
# if sheerka.isinstance(concept, BuiltinConcepts.UNKNOWN_CONCEPT):
|
||||
# has_error = True
|
||||
# else:
|
||||
# with context.push(self.name, desc=f"Evaluating '{concept}'") as sub_context:
|
||||
# sub_context.log_new(self.verbose_log)
|
||||
# concept = sheerka.evaluate_concept(sub_context, concept, self.verbose_log)
|
||||
# sub_context.add_values(return_values=concept)
|
||||
# temp_res.append(concept)
|
||||
#
|
||||
# else:
|
||||
# temp_res.append(core.utils.strip_quotes(token.value))
|
||||
# concepts_only &= token.type == TokenKind.WHITESPACE or token.type == TokenKind.NEWLINE
|
||||
# else:
|
||||
# with context.push(self.name, desc=f"Evaluating '{node.concept}'") as sub_context:
|
||||
# sub_context.log_new(self.verbose_log)
|
||||
# concept = sheerka.evaluate_concept(sub_context, node.concept, self.verbose_log)
|
||||
# sub_context.add_values(return_values=concept)
|
||||
# temp_res.append(concept)
|
||||
#
|
||||
# if has_error:
|
||||
# return sheerka.ret(
|
||||
# self.name,
|
||||
# False,
|
||||
# temp_res,
|
||||
# parents=[self.eaten])
|
||||
#
|
||||
# if concepts_only:
|
||||
# res = []
|
||||
# for r in temp_res:
|
||||
# if isinstance(r, Concept):
|
||||
# res.append(r)
|
||||
# else:
|
||||
# res = ""
|
||||
# for r in temp_res:
|
||||
# if isinstance(r, Concept):
|
||||
# res += sheerka.value(r)
|
||||
# else:
|
||||
# res += r
|
||||
#
|
||||
# return sheerka.ret(
|
||||
# self.name,
|
||||
# True,
|
||||
# res,
|
||||
# parents=[self.eaten])
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
from core.builtin_concepts import ParserResultConcept, BuiltinConcepts
|
||||
from evaluators.BaseEvaluator import OneReturnValueEvaluator
|
||||
import core.utils
|
||||
from parsers.ConceptLexerParser import ConceptNode, NonTerminalNode, ConceptMatch, UnrecognizedTokensNode, TerminalNode
|
||||
from parsers.ConceptLexerParser import ConceptNode, UnrecognizedTokensNode
|
||||
|
||||
|
||||
class ConceptNodeEvaluator(OneReturnValueEvaluator):
|
||||
@@ -12,7 +11,7 @@ class ConceptNodeEvaluator(OneReturnValueEvaluator):
|
||||
NAME = "ConceptNode"
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(self.NAME, [BuiltinConcepts.EVALUATION], 60) # more than the ConceptNodeEvaluator
|
||||
super().__init__(self.NAME, [BuiltinConcepts.EVALUATION], 60)
|
||||
|
||||
def matches(self, context, return_value):
|
||||
if not return_value.status:
|
||||
@@ -50,9 +49,7 @@ class ConceptNodeEvaluator(OneReturnValueEvaluator):
|
||||
for node in nodes:
|
||||
if isinstance(node, ConceptNode):
|
||||
source += node.source if source == "" else (" " + node.source)
|
||||
concept = sheerka.new(node.concept.key)
|
||||
concept = self.finalize_concept(sheerka, concept, node.underlying)
|
||||
concepts.append(concept)
|
||||
concepts.append(node.concept)
|
||||
else:
|
||||
error_found = True
|
||||
|
||||
@@ -69,118 +66,3 @@ class ConceptNodeEvaluator(OneReturnValueEvaluator):
|
||||
parents=[return_value])
|
||||
|
||||
return sheerka.ret(self.name, False, sheerka.new(BuiltinConcepts.NOT_FOR_ME), parents=[return_value])
|
||||
|
||||
def finalize_concept(self, sheerka, concept, underlying, init_empty_body=True):
|
||||
"""
|
||||
Updates the properties of the concept
|
||||
Goes in recursion if the property is a concept
|
||||
"""
|
||||
|
||||
def _add_prop(c, prop_name, value):
|
||||
"""
|
||||
Adds a new entry,
|
||||
makes a list if the property already exists
|
||||
"""
|
||||
|
||||
if prop_name not in c.props or c.props[prop_name].value is None:
|
||||
# new entry
|
||||
c.set_prop(prop_name, value)
|
||||
else:
|
||||
# make a list if there was a value
|
||||
previous_value = c.props[prop_name].value
|
||||
if isinstance(previous_value, list):
|
||||
previous_value.append(value)
|
||||
else:
|
||||
new_value = [previous_value, value]
|
||||
c.set_prop(prop_name, new_value)
|
||||
|
||||
parsing_expression = underlying.parsing_expression
|
||||
|
||||
if parsing_expression.rule_name:
|
||||
_add_prop(concept, parsing_expression.rule_name, self.get_underlying_as_string(underlying))
|
||||
|
||||
# the update of the body must come BEFORE the recursion
|
||||
# otherwise it will be updated by a children and it won't be possible to modify the value
|
||||
if init_empty_body and concept.body is None:
|
||||
concept.metadata.body = self.get_underlying_as_string(underlying) # self.escape_if_needed(underlying.source)
|
||||
|
||||
if isinstance(underlying, NonTerminalNode):
|
||||
for child in underlying.children:
|
||||
if isinstance(child.parsing_expression, ConceptMatch):
|
||||
new_concept = sheerka.new(child.parsing_expression.concept.key)
|
||||
_add_prop(concept, child.parsing_expression.rule_name, new_concept)
|
||||
if sheerka.isinstance(new_concept, BuiltinConcepts.UNKNOWN_CONCEPT):
|
||||
continue
|
||||
else:
|
||||
self.finalize_concept(sheerka, new_concept, child.children[0], init_empty_body)
|
||||
else:
|
||||
self.finalize_concept(sheerka, concept, child, init_empty_body)
|
||||
|
||||
return concept
|
||||
|
||||
@staticmethod
|
||||
def escape_if_needed(value):
|
||||
if not isinstance(value, str):
|
||||
return value
|
||||
|
||||
return "'" + core.utils.escape_char(value, "'") + "'"
|
||||
|
||||
def get_underlying_as_string(self, underlying):
|
||||
"""
|
||||
Return the sequence of the recognized character
|
||||
When a concept is recognized, return the string version of the concept eg c:concept name:
|
||||
:param underlying:
|
||||
:return:
|
||||
"""
|
||||
|
||||
# Example
|
||||
# grammar = {
|
||||
# foo: Sequence("one", "two", rule_name="var"),
|
||||
# bar: Sequence(foo, "three", rule_name="var")}
|
||||
#
|
||||
# we want bar.body and bar.prop["var"]
|
||||
# to be "foo 'three'" (no quotes surrounding foo, as it is a concept, not a string)
|
||||
|
||||
if isinstance(underlying, TerminalNode):
|
||||
return self.escape_if_needed(underlying.source)
|
||||
|
||||
res = ""
|
||||
first = True
|
||||
in_quote = ""
|
||||
for node in underlying.children:
|
||||
if isinstance(node.parsing_expression, ConceptMatch):
|
||||
if in_quote != "":
|
||||
res += in_quote + "'"
|
||||
if not first:
|
||||
res += " "
|
||||
res += node.parsing_expression.concept.key
|
||||
in_quote = ""
|
||||
else:
|
||||
if in_quote == "":
|
||||
in_quote = ("'" if first else " '") + core.utils.escape_char(node.source, "'")
|
||||
else:
|
||||
in_quote += ("" if first else " ") + core.utils.escape_char(node.source, "'")
|
||||
|
||||
first = False
|
||||
|
||||
if in_quote:
|
||||
res += in_quote + "'"
|
||||
return res
|
||||
|
||||
# - - - E X P L A N A T I O N S - - -
|
||||
# why do we need to update the body ?
|
||||
# cf test_concept_property_is_correctly_updated_when_concept_recursion_using_zero_or_more()
|
||||
# def concept number from bnf one | two | three
|
||||
# def concept add from bnf number plus number
|
||||
#
|
||||
# the expression 'one plus two plus three' will match concept add
|
||||
# add.props["number"] is a list of concepts 'number'
|
||||
# But which one is 'one', which one is 'two' which one is 'three' ?
|
||||
#
|
||||
# That's the reason why we update the body
|
||||
# add.props["number"] is a list of concepts 'number' but they won't have the same body
|
||||
#
|
||||
# !!! C A U T I O N !!!
|
||||
# In the current implementation, the body is the sequence of char found
|
||||
# If a concept is recognized, we don't put this information in the body
|
||||
# Use get_body_as_string() instead of escape_if_needed() if we need this information
|
||||
|
||||
@@ -40,7 +40,7 @@ class PythonEvaluator(OneReturnValueEvaluator):
|
||||
not_for_me = context.sheerka.new(BuiltinConcepts.NOT_FOR_ME, body=node)
|
||||
return sheerka.ret(self.name, False, not_for_me, parents=[return_value])
|
||||
|
||||
my_locals = self.get_locals(context, node.ast_)
|
||||
my_locals = self.get_locals(context, node)
|
||||
context.log(self.verbose_log, f"locals={my_locals}", self.name)
|
||||
|
||||
if isinstance(node.ast_, ast.Expression):
|
||||
@@ -58,7 +58,7 @@ class PythonEvaluator(OneReturnValueEvaluator):
|
||||
error = sheerka.new(BuiltinConcepts.ERROR, body=error)
|
||||
return sheerka.ret(self.name, False, error, parents=[return_value])
|
||||
|
||||
def get_locals(self, context, ast_):
|
||||
def get_locals(self, context, node):
|
||||
my_locals = {"sheerka": context.sheerka}
|
||||
if context.obj:
|
||||
context.log(self.verbose_log,
|
||||
@@ -70,30 +70,32 @@ class PythonEvaluator(OneReturnValueEvaluator):
|
||||
else:
|
||||
my_locals[prop_name] = context.sheerka.value(prop_value.value)
|
||||
|
||||
node_concept = core.ast.nodes.python_to_concept(ast_)
|
||||
node_concept = core.ast.nodes.python_to_concept(node.ast_)
|
||||
unreferenced_names_visitor = UnreferencedNamesVisitor(context.sheerka)
|
||||
unreferenced_names_visitor.visit(node_concept)
|
||||
|
||||
for name in unreferenced_names_visitor.names:
|
||||
context.log(self.verbose_log, f"Resolving '{name}'.", self.name)
|
||||
return_concept = False
|
||||
|
||||
if name.startswith("__C__") and name.endswith("__C__"):
|
||||
name_resolved = name[5:-5]
|
||||
return_concept = True
|
||||
if name in node.concepts:
|
||||
context.log(self.verbose_log, f"Using value from node.", self.name)
|
||||
concept = node.concepts[name]
|
||||
return_concept = False
|
||||
|
||||
else:
|
||||
name_resolved = name
|
||||
concept_key, concept_id, return_concept = self.resolve_name(context, name)
|
||||
|
||||
if name_resolved in my_locals:
|
||||
context.log(self.verbose_log, f"Using value from property.", self.name)
|
||||
continue
|
||||
if concept_key in my_locals:
|
||||
context.log(self.verbose_log, f"Using value from property.", self.name)
|
||||
continue
|
||||
|
||||
concept = context.sheerka.new(name_resolved)
|
||||
if context.sheerka.isinstance(concept, BuiltinConcepts.UNKNOWN_CONCEPT):
|
||||
context.log(self.verbose_log, f"'{name_resolved}' is not a concept. Skipping.", self.name)
|
||||
continue
|
||||
context.log(self.verbose_log, f"Instantiating new concept.", self.name)
|
||||
concept = context.sheerka.new((concept_key, concept_id))
|
||||
if context.sheerka.isinstance(concept, BuiltinConcepts.UNKNOWN_CONCEPT):
|
||||
context.log(self.verbose_log, f"'{concept_key}' is not a concept. Skipping.", self.name)
|
||||
continue
|
||||
|
||||
context.log(self.verbose_log, f"'{name_resolved}' is a concept. Evaluating.", self.name)
|
||||
context.log(self.verbose_log, f"Evaluating '{concept}'", self.name)
|
||||
with context.push(self.name, desc=f"Evaluating '{concept}'", obj=concept) as sub_context:
|
||||
sub_context.log_new(self.verbose_log)
|
||||
evaluated = context.sheerka.evaluate_concept(sub_context, concept, self.verbose_log)
|
||||
@@ -109,6 +111,58 @@ class PythonEvaluator(OneReturnValueEvaluator):
|
||||
|
||||
return my_locals
|
||||
|
||||
def resolve_name(self, context, to_resolve):
|
||||
"""
|
||||
Try to match
|
||||
__C__concept_key__C__
|
||||
or
|
||||
__C__concept_key__concept_id__C__
|
||||
|
||||
:param context:
|
||||
:param to_resolve:
|
||||
:return:
|
||||
"""
|
||||
if not to_resolve.startswith("__C__"):
|
||||
return to_resolve, None, False
|
||||
|
||||
context.log(self.verbose_log, f"Resolving name '{to_resolve}'.", self.name)
|
||||
|
||||
if len(to_resolve) >= 18 and to_resolve[:18] == "__C__USE_CONCEPT__":
|
||||
use_concept = True
|
||||
index = 18
|
||||
else:
|
||||
use_concept = False
|
||||
index = 5
|
||||
|
||||
try:
|
||||
next_index = to_resolve.index("__", index)
|
||||
if next_index == index:
|
||||
context.log(self.verbose_log, f"Error: no key between '__'.", self.name)
|
||||
return None
|
||||
concept_key = to_resolve[index: next_index]
|
||||
except ValueError:
|
||||
context.log(self.verbose_log, f"Error: Missing trailing '__'.", self.name)
|
||||
return None
|
||||
|
||||
if next_index == len(to_resolve) - 5:
|
||||
context.log(self.verbose_log, f"Recognized concept '{concept_key}'", self.name)
|
||||
return concept_key, None, use_concept
|
||||
|
||||
index = next_index + 2
|
||||
try:
|
||||
next_index = to_resolve.index("__", index)
|
||||
if next_index == index:
|
||||
context.log(self.verbose_log, f"Error: no id between '__'.", self.name)
|
||||
return None
|
||||
|
||||
concept_id = to_resolve[index: next_index]
|
||||
except ValueError:
|
||||
context.log(self.verbose_log, f"Recognized concept '{concept_key}'.", self.name)
|
||||
return concept_key, None, use_concept
|
||||
|
||||
context.log(self.verbose_log, f"Recognized concept '{concept_key}' (id='{concept_id}').", self.name)
|
||||
return concept_key, concept_id, use_concept
|
||||
|
||||
@staticmethod
|
||||
def expr_to_expression(expr):
|
||||
expr.lineno = 0
|
||||
|
||||
Reference in New Issue
Block a user