kodjo/Sheerka-Old
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Concept validation must be requested

Browse Source
This commit is contained in:
Kodjo Sossouvi
2020-03-09 12:23:53 +01:00
parent ef31a4807d
commit 1bde97b5e3
27 changed files with 346 additions and 280 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/core/builtin_concepts.py
+5 -4
View File
@@ -47,13 +47,14 @@ class BuiltinConcepts(Enum):
LIST = "list" # represents a list
CONCEPT_ALREADY_IN_SET = "concept already in set"
EVALUATOR_PRE_PROCESS = "evaluator pre process" # used modify / tweak behaviour of evaluators
CONCEPT_EVAL_REQUESTED = "concept eval requested"
EVAL_BODY_REQUESTED = "eval body requested" # to evaluate the body
EVAL_WHERE_REQUESTED = "eval where requested" # to evaluate the where clause
CONCEPT_VALUE_REQUESTED = "concept value requested" # returns the body of the concept instead of the concept itself
REDUCE_REQUESTED = "reduce requested" # remove meaningless error when possible
NOT_A_SET = "not a set" # the concept has no entry in sets
WHERE_CLAUSE_FAILED = "where clause failed" # failed to validate where clause during evaluation
CHICKEN_AND_EGG = "chicken and egg" # infinite recursion when declaring concept
ISA = "is a" # builtin concept to express that a concept is an instance of another one
CONCEPT_VALUE_REQUESTED = "concept value requested" # returns the body of the concept instead of the concept itself
NODE = "node"
GENERIC_NODE = "generic node"
@@ -78,7 +79,7 @@ BuiltinUnique = [
BuiltinConcepts.AFTER_RENDERING,
BuiltinConcepts.SUCCESS,
BuiltinConcepts.NOP,
BuiltinConcepts.CONCEPT_EVAL_REQUESTED,
BuiltinConcepts.EVAL_BODY_REQUESTED,
BuiltinConcepts.REDUCE_REQUESTED,
]
@@ -248,7 +249,7 @@ class ParserResultConcept(Concept):
Result of a parsing
"""
def __init__(self, parser=None, source=None, value=None, try_parsed=None):
def __init__(self, parser=None, source=None, value=None, try_parsed=None, validate_concept=None):
super().__init__(BuiltinConcepts.PARSER_RESULT, True, False, BuiltinConcepts.PARSER_RESULT)
self.set_metadata_value(ConceptParts.BODY, value)
self.set_prop("parser", parser)
src/core/builtin_helpers.py
+23 -21
View File
@@ -17,32 +17,34 @@ def is_same_success(context, return_values):
"""
assert isinstance(return_values, list)
if not return_values[0].status:
return False
def _get_value(ret_val):
if not ret_val.status:
raise Exception("Status is false")
if isinstance(return_values[0].body, Concept):
evaluated = context.sheerka.evaluate_concept(context, return_values[0].body, True)
if evaluated.key != return_values[0].body.key:
return False
reference = context.sheerka.value(evaluated)
else:
reference = context.sheerka.value(return_values[0])
if isinstance(ret_val.body, Concept):
if not ret_val.body.metadata.is_evaluated:
with context.push(desc=f"Evaluating concept '{ret_val.body}'") as sub_context:
sub_context.local_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
evaluated = context.sheerka.evaluate_concept(sub_context, ret_val.body)
if evaluated.key != ret_val.body.key:
raise Exception("Failed to evaluate evaluate")
return context.sheerka.value(evaluated)
else:
return context.sheerka.value(ret_val.body)
else:
return context.sheerka.value(ret_val)
for return_value in return_values[1:]:
if not return_value.status:
return False
try:
reference = _get_value(return_values[0])
if isinstance(return_value.body, Concept):
evaluated = context.sheerka.evaluate_concept(context, return_value.body, True)
if evaluated.key != return_value.body.key:
for return_value in return_values[1:]:
actual = _get_value(return_value)
if actual != reference:
return False
actual = context.sheerka.value(evaluated)
else:
actual = context.sheerka.value(return_value)
if actual != reference:
return False
except Exception as ex:
context.log_error(ex)
return False
return True
src/core/concept.py
+1
View File
@@ -48,6 +48,7 @@ class ConceptMetadata:
id: str # unique identifier for a concept. The id will never be modified (but the key can)
props: list # list properties, with their default values
is_evaluated: bool = False # True is the concept is evaluated by sheerka.eval_concept()
need_validation = False # True if the properties of the concept need to be validated
full_serialization: bool = False # If True, the full object will be serialized, rather than just the diff
src/core/sheerka/ExecutionContext.py
+14 -4
View File
@@ -21,8 +21,6 @@ PROPERTIES_TO_SERIALIZE = ("_id",
"concepts")
class ExecutionContext:
"""
To keep track of the execution of a request
@@ -44,6 +42,7 @@ class ExecutionContext:
sheerka,
desc: str = None,
logger=None,
global_hints=None,
**kwargs):
self._parent = None
@@ -60,7 +59,8 @@ class ExecutionContext:
self.children = []
self.preprocess = None
self.logger = logger
self.extra_info = []
self.local_hints = set()
self.global_hints = set() if global_hints is None else global_hints
self.inputs = {} # what was the parameters of the execution context
self.values = {} # what was produced by the execution context
@@ -211,11 +211,12 @@ class ExecutionContext:
self.sheerka,
desc,
logger,
self.global_hints,
**_kwargs)
new._parent = self
new._tab = self._tab + " " * DEBUG_TAB_SIZE
new.preprocess = self.preprocess
new.extra_info.extend(self.extra_info)
new.local_hints.update(self.local_hints)
self.children.append(new)
return new
@@ -247,6 +248,15 @@ class ExecutionContext:
def get_parent(self):
return self._parent
def in_context(self, concept_key):
if concept_key in self.local_hints:
return True
if concept_key in self.global_hints:
return True
return False
@staticmethod
def return_value_to_str(r):
value = str(r.value)
src/core/sheerka/Services/SheerkaDump.py
+2 -1
View File
@@ -53,9 +53,10 @@ class SheerkaDump:
if not first:
self.sheerka.log.info("")
self.sheerka.log.info(f"name : {c.name}")
self.sheerka.log.info(f"bnf : {c.metadata.definition}")
self.sheerka.log.info(f"key : {c.key}")
self.sheerka.log.info(f"bnf : {c.metadata.definition}")
self.sheerka.log.info(f"body : {c.metadata.body}")
self.sheerka.log.info(f"where : {c.metadata.where}")
if eval:
self.sheerka.log.info(f"value : {value}")
for p in c.props:
src/core/sheerka/Services/SheerkaEvaluateConcept.py
+48 -35
View File
@@ -118,7 +118,7 @@ class SheerkaEvaluateConcept:
else:
self.sheerka.cache_by_key[concept.key].compiled = concept.compiled
def resolve(self, context, to_resolve, current_prop, current_concept, evaluate_body):
def resolve(self, context, to_resolve, current_prop, current_concept, force_evaluation):
if isinstance(to_resolve, DoNotResolve):
return to_resolve.value
@@ -134,10 +134,13 @@ class SheerkaEvaluateConcept:
context.log(desc, self.logger_name)
with context.push(desc=desc, obj=current_concept) as sub_context:
if force_evaluation:
sub_context.local_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
# when it's a concept, evaluate it
if isinstance(to_resolve, Concept) and \
not context.sheerka.isinstance(to_resolve, BuiltinConcepts.RETURN_VALUE):
evaluated = self.evaluate_concept(sub_context, to_resolve, evaluate_body)
evaluated = self.evaluate_concept(sub_context, to_resolve)
sub_context.add_values(return_values=evaluated)
if evaluated.key == to_resolve.key:
return evaluated
@@ -147,8 +150,6 @@ class SheerkaEvaluateConcept:
# otherwise, execute all return values to find out what is the value
else:
use_copy = [r for r in to_resolve] if hasattr(to_resolve, "__iter__") else to_resolve
if evaluate_body:
sub_context.extra_info.append(BuiltinConcepts.CONCEPT_EVAL_REQUESTED)
r = self.sheerka.execute(sub_context, use_copy, CONCEPT_EVALUATION_STEPS)
one_r = expect_one(context, r)
@@ -164,7 +165,7 @@ class SheerkaEvaluateConcept:
concept=current_concept,
property_name=current_prop)
def resolve_list(self, context, list_to_resolve, current_prop, current_concept, evaluate_body):
def resolve_list(self, context, list_to_resolve, current_prop, current_concept, force_evaluation):
"""When dealing with a list, there are two possibilities"""
# It may be a list of ReturnValueConcept to execute (always the case for metadata)
# or a list of single values (may be the case for properties)
@@ -173,7 +174,7 @@ class SheerkaEvaluateConcept:
return []
if self.sheerka.isinstance(list_to_resolve[0], BuiltinConcepts.RETURN_VALUE):
return self.resolve(context, list_to_resolve, current_prop, current_concept, evaluate_body)
return self.resolve(context, list_to_resolve, current_prop, current_concept, force_evaluation)
res = []
for to_resolve in list_to_resolve:
@@ -184,14 +185,14 @@ class SheerkaEvaluateConcept:
concept=current_concept,
property_name=current_prop)
r = self.resolve(context, to_resolve, current_prop, current_concept, evaluate_body)
r = self.resolve(context, to_resolve, current_prop, current_concept, force_evaluation)
if self.sheerka.isinstance(r, BuiltinConcepts.CONCEPT_EVAL_ERROR):
return r
res.append(r)
return res
def evaluate_concept(self, context, concept: Concept, evaluate_body=False):
def evaluate_concept(self, context, concept: Concept):
"""
Evaluation a concept
It means that if the where clause is True, will evaluate the body
@@ -208,7 +209,7 @@ class SheerkaEvaluateConcept:
# to make sure of the order, it don't use ConceptParts.get_parts()
# props must be evaluated first, body must be evaluated before where
all_metadata_to_eval = self.choose_metadata_to_eval(concept, evaluate_body)
all_metadata_to_eval = self.choose_metadata_to_eval(context, concept)
for metadata_to_eval in all_metadata_to_eval:
if metadata_to_eval == "props":
@@ -221,6 +222,7 @@ class SheerkaEvaluateConcept:
else:
# Do not send the current concept for the properties
resolved = self.resolve(context, prop_ast, prop_name, None, True)
if isinstance(resolved, Concept) and not context.sheerka.is_success(resolved):
resolved.set_prop("concept", concept) # since current concept was not sent
return resolved
@@ -236,7 +238,10 @@ class SheerkaEvaluateConcept:
if part_key in concept.compiled and concept.compiled[part_key] is not None:
metadata_ast = concept.compiled[part_key]
resolved = self.resolve(context, metadata_ast, part_key, concept, evaluate_body)
# if part_key is PRE, POST or WHERE, the concepts need to be evaluated
# otherwise the predicates cannot be resolved
force_concept_eval = False if part_key == ConceptParts.BODY else True
resolved = self.resolve(context, metadata_ast, part_key, concept, force_concept_eval)
if isinstance(resolved, Concept) and not context.sheerka.is_success(resolved):
return resolved
else:
@@ -260,49 +265,57 @@ class SheerkaEvaluateConcept:
concept.metadata.is_evaluated = "body" in all_metadata_to_eval
return concept
def choose_metadata_to_eval(self, concept, evaluate_body):
if evaluate_body:
def choose_metadata_to_eval(self, context, concept):
if context.in_context(BuiltinConcepts.EVAL_BODY_REQUESTED):
return ["pre", "post", "props", "body", "where"]
metadata = ["pre", "post"] + self.needed_metadata(concept) + ["where"]
metadata = ["pre", "post"]
if context.in_context(BuiltinConcepts.EVAL_WHERE_REQUESTED) or concept.metadata.need_validation:
needed = self.needed_metadata(concept, ConceptParts.WHERE)
for e in needed:
if e not in metadata:
metadata.append(e)
if "where" not in metadata:
metadata.append("where")
return metadata
def needed_metadata(self, concept):
def needed_metadata(self, concept, metadata):
"""
Tries to find out if the evaluation of the body is necessary
It's a very basic approach that will need to be improved
:param concept:
:param metadata:
:return:
"""
if metadata not in concept.compiled:
return []
return_values = concept.compiled[metadata]
if not isinstance(return_values, list):
return []
needed = []
for metadata in (ConceptParts.PRE, ConceptParts.POST, ConceptParts.WHERE):
if metadata not in concept.compiled:
for return_value in return_values:
if not self.sheerka.isinstance(return_value, BuiltinConcepts.RETURN_VALUE):
continue
return_values = concept.compiled[metadata]
if not isinstance(return_values, list):
if not return_value.status:
continue
for return_value in return_values:
if not self.sheerka.isinstance(return_value, BuiltinConcepts.RETURN_VALUE):
continue
if not self.sheerka.isinstance(return_value.body, BuiltinConcepts.PARSER_RESULT):
continue
if not return_value.status:
continue
if not isinstance(return_value.body.source, str):
continue
if not self.sheerka.isinstance(return_value.body, BuiltinConcepts.PARSER_RESULT):
continue
for prop_name in (p[0] for p in concept.metadata.props):
if prop_name in return_value.body.source:
needed.append("props")
break
if not isinstance(return_value.body.source, str):
continue
if "self" in return_value.body.source:
needed.append("body")
for prop_name in (p[0] for p in concept.metadata.props):
if prop_name in return_value.body.source:
needed.append("props")
break
if "self" in return_value.body.source:
needed.append("body")
return needed
src/core/sheerka/Services/SheerkaExecute.py
+2 -19
View File
@@ -90,27 +90,12 @@ class SheerkaExecute:
result = core.utils.remove_list_from_list(result, user_inputs)
return result
def call_evaluators(self, execution_context, return_values, process_step, evaluation_context=None):
def call_evaluators(self, execution_context, return_values, process_step):
# return_values must be a list
if not isinstance(return_values, list):
return_values = [return_values]
# Evaluation context are contexts that may modify the behaviour of the execution
# For example, a concept to indicate that the value is not wanted
# Or a concept to indicate that we want the letter form of the response
# But first, they need to be transformed into return values
if evaluation_context is None:
evaluation_return_values = []
else:
evaluation_return_values = [self.sheerka.ret(execution_context.who, True, c) for c in evaluation_context]
# add the current step as part as the evaluation context
evaluation_return_values.append(self.sheerka.ret(execution_context.who, True, self.sheerka.new(process_step)))
# the pool of return values are the mix
return_values.extend(evaluation_return_values)
# group the evaluators by priority and sort them
# The first one to be applied will be the one with the highest priority
grouped_evaluators = {}
@@ -203,8 +188,6 @@ class SheerkaExecute:
# inc the iteration and continue
iteration += 1
# remove all evaluation context that are not reduced
return_values = core.utils.remove_list_from_list(return_values, evaluation_return_values)
return return_values
def execute(self, execution_context, return_values, execution_steps):
@@ -223,7 +206,7 @@ class SheerkaExecute:
if step == BuiltinConcepts.PARSING:
return_values = self.call_parsers(sub_context, return_values)
else:
return_values = self.call_evaluators(sub_context, return_values, step, None)
return_values = self.call_evaluators(sub_context, return_values, step)
if copy != return_values:
sub_context.log_result(return_values)
src/core/sheerka/Services/SheerkaSetsManager.py
+6 -3
View File
@@ -108,7 +108,8 @@ class SheerkaSetsManager:
# it may be a concept that references a set
if not sub_concept.metadata.is_evaluated:
with context.push(desc=f"Evaluating concept {sub_concept}") as sub_context:
evaluated = self.sheerka.evaluate_concept(sub_context, sub_concept, True)
sub_context.local_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
evaluated = self.sheerka.evaluate_concept(sub_context, sub_concept)
if evaluated.key != concept.key:
return False
return _get_set_elements(context, concept, sub_concept.body)
@@ -167,7 +168,8 @@ class SheerkaSetsManager:
# it may be a concept that references a set
if not concept.metadata.is_evaluated:
with context.push(desc=f"Evaluating concept {concept}") as sub_context:
evaluated = self.sheerka.evaluate_concept(sub_context, concept, True)
sub_context.local_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
evaluated = self.sheerka.evaluate_concept(sub_context, concept)
if evaluated.key != concept.key:
return False
@@ -216,9 +218,10 @@ for x in xx__concepts__xx:
result = []
with context.push(desc=f"Evaluating concepts of a set") as sub_context:
sub_context.local_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
for element_id in ids:
concept = self.sheerka.get_by_id(element_id)
evaluated = self.sheerka.evaluate_concept(sub_context, concept, True)
evaluated = self.sheerka.evaluate_concept(sub_context, concept)
result.append(evaluated)
return result
src/core/sheerka/Sheerka.py
+3 -4
View File
@@ -239,6 +239,7 @@ class Sheerka(Concept):
with ExecutionContext(self.key, event, self, f"Evaluating '{text}'", self.log) as execution_context:
user_input = self.ret(self.name, True, self.new(BuiltinConcepts.USER_INPUT, body=text, user_name=user_name))
reduce_requested = self.ret(self.name, True, self.new(BuiltinConcepts.REDUCE_REQUESTED))
#execution_context.local_hints.add(BuiltinConcepts.EVAL_WHERE_REQUESTED)
steps = [
BuiltinConcepts.BEFORE_PARSING,
@@ -332,17 +333,15 @@ class Sheerka(Concept):
return self.sets_handler.get_set_elements(context, concept)
def evaluate_concept(self, context, concept: Concept, evaluate_body=False):
def evaluate_concept(self, context, concept: Concept):
"""
Evaluation a concept
It means that if the where clause is True, will evaluate the body
:param evaluate_body:
:param context:
:param concept:
:param evaluate_body:
:return: value of the evaluation or error
"""
return self.evaluate_concept_handler.evaluate_concept(context, concept, evaluate_body)
return self.evaluate_concept_handler.evaluate_concept(context, concept)
def add_in_cache(self, concept: Concept):
"""
src/evaluators/ConceptEvaluator.py
+11 -13
View File
@@ -15,19 +15,17 @@ class ConceptEvaluator(OneReturnValueEvaluator):
def __init__(self, return_body=False):
super().__init__(self.NAME, [BuiltinConcepts.EVALUATION], 50)
self.return_body = return_body
self.evaluate_body = False
self.is_initialized = False
def init_evaluator(self, context, return_values):
if BuiltinConcepts.CONCEPT_EVAL_REQUESTED in context.extra_info:
self.evaluate_body = True
for r in return_values:
if r.status and context.sheerka.isinstance(r.body, BuiltinConcepts.CONCEPT_VALUE_REQUESTED):
self.evaluate_body = True
break
self.is_initialized = True
# def init_evaluator(self, context, return_values):
# if BuiltinConcepts.EVAL_BODY_REQUESTED in context.local_hints:
# self.evaluate_body = True
#
# for r in return_values:
# if r.status and context.sheerka.isinstance(r.body, BuiltinConcepts.CONCEPT_VALUE_REQUESTED):
# self.evaluate_body = True
# break
#
# self.is_initialized = True
def matches(self, context, return_value):
return return_value.status and \
@@ -49,7 +47,7 @@ class ConceptEvaluator(OneReturnValueEvaluator):
return sheerka.ret(self.name, True, value, parents=[return_value])
evaluated = sheerka.evaluate_concept(context, concept, self.evaluate_body)
evaluated = sheerka.evaluate_concept(context, concept)
if evaluated.key != concept.key:
# evaluated.key != concept.key means that we have transformed the concept
src/evaluators/EvalEvaluator.py
+5 -18
View File
@@ -12,16 +12,9 @@ class EvalEvaluator(AllReturnValuesEvaluator):
def __init__(self):
super().__init__(self.NAME, [BuiltinConcepts.AFTER_EVALUATION], 80)
self.eval_requested = None
def matches(self, context, return_values):
sheerka = context.sheerka
for ret in return_values:
if ret.status and sheerka.isinstance(ret.body, BuiltinConcepts.CONCEPT_VALUE_REQUESTED):
self.eval_requested = ret
return True
return False
return context.in_context(BuiltinConcepts.CONCEPT_VALUE_REQUESTED)
def eval(self, context, return_values):
sheerka = context.sheerka
@@ -30,22 +23,16 @@ class EvalEvaluator(AllReturnValuesEvaluator):
for ret_val in return_values:
if ret_val.status and isinstance(ret_val.body, Concept) and ret_val.body.body:
context.log(f"Evaluating {ret_val.body}", who=self)
result.append(sheerka.ret(self.name, True, ret_val.body.body, parents=[ret_val, self.eval_requested]))
result.append(sheerka.ret(self.name, True, ret_val.body.body, parents=[ret_val]))
elif ret_val.status and sheerka.isaset(context, ret_val.body):
context.log(f"Evaluating set {ret_val.body}", who=self)
result.append(sheerka.ret(
self.name,
True,
sheerka.get_set_elements(context, ret_val.body),
parents=[ret_val, self.eval_requested]))
parents=[ret_val]))
if len(result) > 0:
return result
else:
# suppress the successful BuiltinConcepts.CONCEPT_EVAL_REQUESTED
# status of CONCEPT_EVAL_REQUESTED is now set to False
return sheerka.ret(
self.name,
False,
sheerka.new(BuiltinConcepts.CONCEPT_VALUE_REQUESTED),
parents=[self.eval_requested])
return None
src/evaluators/PrepareEvalEvaluator.py
+3 -4
View File
@@ -33,8 +33,7 @@ class PrepareEvalEvaluator(OneReturnValueEvaluator):
self.name,
True, sheerka.new(BuiltinConcepts.USER_INPUT, body=self.text[5:], user_name=context.event.user))
evaluation_requested = sheerka.ret(
self.name,
True, sheerka.new(BuiltinConcepts.CONCEPT_VALUE_REQUESTED))
context.global_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
context.global_hints.add(BuiltinConcepts.CONCEPT_VALUE_REQUESTED)
return [new_text_to_parse, evaluation_requested]
return new_text_to_parse
src/evaluators/PythonEvaluator.py
+2 -1
View File
@@ -112,7 +112,8 @@ class PythonEvaluator(OneReturnValueEvaluator):
context.log(f"Evaluating '{concept}'", self.name)
with context.push(self.name, desc=f"Evaluating '{concept}'", obj=concept) as sub_context:
evaluated = context.sheerka.evaluate_concept(sub_context, concept, True)
sub_context.local_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
evaluated = context.sheerka.evaluate_concept(sub_context, concept)
sub_context.add_values(return_values=evaluated)
if evaluated.key == concept.key:
src/parsers/ConceptLexerParser.py
+1
View File
@@ -944,6 +944,7 @@ class ConceptLexerParser(BaseParser):
if _underlying.parsing_expression.rule_name:
value = _get_underlying_value(_underlying)
_add_prop(_concept, _underlying.parsing_expression.rule_name, value)
_concept.metadata.need_validation = True
if isinstance(_underlying, NonTerminalNode):
for child in _underlying.children:
src/parsers/ExactConceptParser.py
+3 -2
View File
@@ -41,21 +41,22 @@ class ExactConceptParser(BaseParser):
for combination in self.combinations(words):
concept_key = " ".join(combination)
result = sheerka.new(concept_key)
result = sheerka.new(concept_key) # use new(), not get() because we need a new instance
if sheerka.isinstance(result, BuiltinConcepts.UNKNOWN_CONCEPT):
continue
# concepts = result.body if sheerka.isinstance(result, BuiltinConcepts.ENUMERATION) else [result]
concepts = result if isinstance(result, list) else [result]
for concept in concepts:
context.log(f"Recognized concept {concept}.", self.name)
# update the properties if needed
need_validation = False
for i, token in enumerate(combination):
if token.startswith(VARIABLE_PREFIX):
index = int(token[len(VARIABLE_PREFIX):])
concept.def_prop_by_index(index, words[i])
concept.metadata.need_validation = True
if self.verbose_log.isEnabledFor(logging.DEBUG):
prop_name = list(concept.props.keys())[index]
context.log(