kodjo/Sheerka-Old
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

First implementation of questions management

Browse Source
This commit is contained in:
Kodjo Sossouvi
2020-08-14 08:16:33 +02:00
parent e84b394da2
commit 351c16f946
47 changed files with 1582 additions and 400 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/core/sheerka/services/SheerkaAdmin.py
+3 -1
View File
@@ -49,9 +49,11 @@ class SheerkaAdmin(BaseService):
try:
start = time.time_ns()
nb_lines = 0
self.sheerka.during_restore = True
with open(concept_file, "r") as f:
for line in f.readlines():
nb_lines += 1
line = line.strip()
if line == "" or line.startswith("#"):
continue
@@ -65,7 +67,7 @@ class SheerkaAdmin(BaseService):
nano_sec = stop - start
dt = nano_sec / 1e6
elapsed = f"{dt} ms" if dt < 1000 else f"{dt / 1000} s"
print(f"Execution time: {elapsed}")
print(f"Imported {nb_lines} line(s) in {elapsed}.")
except IOError:
pass
src/core/sheerka/services/SheerkaEvaluateConcept.py
+201 -78
View File
@@ -1,9 +1,13 @@
from dataclasses import dataclass
from core.builtin_concepts import BuiltinConcepts
from core.builtin_helpers import expect_one, only_successful
from core.concept import Concept, DoNotResolve, ConceptParts, InfiniteRecursionResolved
from core.builtin_helpers import expect_one, only_successful, parse_unrecognized, evaluate
from core.concept import Concept, DoNotResolve, ConceptParts, InfiniteRecursionResolved, NotInit
from core.sheerka.services.SheerkaExecute import ParserInput
from core.sheerka.services.sheerka_service import BaseService
from core.tokenizer import Tokenizer
from core.utils import unstr_concept
from parsers.ExpressionParser import ExpressionParser, TrueifyVisitor
CONCEPT_EVALUATION_STEPS = [
BuiltinConcepts.BEFORE_EVALUATION,
@@ -11,6 +15,15 @@ CONCEPT_EVALUATION_STEPS = [
BuiltinConcepts.AFTER_EVALUATION]
@dataclass
class WhereClauseDef:
concept: Concept # concept on which the where clause is applied
clause: str # original where clause
trueified: str # modified where clause (where unresolvable variables are removed)
prop: str # variable to test
compiled: object # trueified where clause Python compiled
class SheerkaEvaluateConcept(BaseService):
NAME = "EvaluateConcept"
@@ -61,6 +74,112 @@ class SheerkaEvaluateConcept(BaseService):
"""
return concept.get_value(ConceptParts.RET) if ConceptParts.RET in concept.values else concept
@staticmethod
def get_needed_metadata(concept, concept_part, check_vars, check_body):
"""
Check if the concept_part has to be evaluated
It also checks if the variables and the body need to be evaluated prior to it
:param concept:
:param concept_part:
:param check_vars:
:param check_body:
:return:
"""
ret = []
vars_needed = False
body_needed = False
if concept_part in concept.compiled and concept.compiled[concept_part] is not None:
concept_part_source = getattr(concept.metadata, concept_part.value)
assert concept_part_source is not None
tokens = [t.str_value for t in Tokenizer(concept_part_source)]
if check_vars:
for var_name in (v[0] for v in concept.metadata.variables):
if var_name in tokens:
vars_needed = True
ret.append("variables")
break
if check_body and "self" in tokens:
body_needed = True
ret.append("body")
ret.append(concept_part.value)
return ret, vars_needed, body_needed
@staticmethod
def get_where_clause_def(context, concept, var_name):
"""
Returns the compiled code to be executed
:param context:
:param concept:
:param var_name:
:return:
"""
if concept.metadata.where is None or concept.metadata.where.strip() == "":
return None
ret = ExpressionParser().parse(context, ParserInput(concept.metadata.where))
if not ret.status:
# TODO: manage invalid where clause
return None
expr = ret.body.body
to_trueify = [v[0] for v in concept.metadata.variables if v[0] != var_name]
trueified_where = str(TrueifyVisitor(to_trueify, [var_name]).visit(expr))
tokens = [t.str_value for t in Tokenizer(trueified_where)]
if var_name in tokens:
compiled = None
try:
compiled = compile(trueified_where, "<where clause>", "eval")
except Exception:
pass
return WhereClauseDef(concept, concept.metadata.where, trueified_where, var_name, compiled)
else:
return None
def apply_where_clause(self, context, where_clause_def, return_values):
"""
Apply intermediate where clause when evaluating concept variables
:param context:
:param where_clause_def:
:param return_values:
:return:
"""
ret = []
for r in [r for r in return_values if r.status]:
if where_clause_def.compiled:
try:
if eval(where_clause_def.compiled, {where_clause_def.prop: self.sheerka.objvalue(r)}):
ret.append(r)
except NameError:
ret.append(r) # it cannot be solved unitary, let's give a chance to the global where condition
else:
# it means that the where condition is an expression that needs to be executed
evaluation_res = evaluate(context,
where_clause_def.trueified,
desc=f"Apply where clause on '{where_clause_def.prop}'",
expect_success=True,
stm={where_clause_def.prop: r.body})
one_res = expect_one(context, evaluation_res)
if one_res.status:
value = context.sheerka.objvalue(one_res)
if isinstance(value, bool) and value:
ret.append(r)
if len(ret) > 0:
return ret
return self.sheerka.new(BuiltinConcepts.CONDITION_FAILED,
body=where_clause_def.clause,
concept=where_clause_def.concept,
prop=where_clause_def.prop)
def manage_infinite_recursion(self, context):
"""
We look for the fist parent that has a body that means something
@@ -105,7 +224,6 @@ class SheerkaEvaluateConcept(BaseService):
return self.sheerka.resolve(identifier)
return None
steps = [BuiltinConcepts.BEFORE_PARSING, BuiltinConcepts.PARSING, BuiltinConcepts.AFTER_PARSING]
for part_key in ConceptParts:
if part_key in concept.compiled:
continue
@@ -122,16 +240,12 @@ class SheerkaEvaluateConcept(BaseService):
context.log(f"Recognized concept '{concept_found}'", self.NAME)
concept.compiled[part_key] = concept_found
else:
with context.push(BuiltinConcepts.INIT_COMPILED,
{"part": part_key, "source": source},
desc=f"Initializing *compiled* for {part_key}") as sub_context:
sub_context.add_inputs(source=source)
to_parse = self.sheerka.ret(context.who, True,
self.sheerka.new(BuiltinConcepts.USER_INPUT, body=source))
res = self.sheerka.execute(sub_context, to_parse, steps)
only_success = only_successful(sub_context, res)
concept.compiled[part_key] = only_success.body.body if is_only_successful(only_success) else res
sub_context.add_values(return_values=res)
res = parse_unrecognized(context,
source,
parsers="all",
prop=part_key,
filter_func=only_successful)
concept.compiled[part_key] = res.body.body if is_only_successful(res) else res
for var_name, default_value in concept.metadata.variables:
if var_name in concept.compiled:
@@ -148,22 +262,36 @@ class SheerkaEvaluateConcept(BaseService):
context.log(f"Recognized concept '{concept_found}'", self.NAME)
concept.compiled[var_name] = concept_found
else:
with context.push(BuiltinConcepts.INIT_COMPILED,
{"property": var_name, "source": default_value},
desc=f"Initializing *compiled* for property {var_name}") as sub_context:
sub_context.add_inputs(source=default_value)
to_parse = self.sheerka.ret(context.who, True,
self.sheerka.new(BuiltinConcepts.USER_INPUT, body=default_value))
res = self.sheerka.execute(sub_context, to_parse, steps)
only_success = only_successful(sub_context, res)
concept.compiled[var_name] = only_success.body.body if is_only_successful(only_success) else res
sub_context.add_values(return_values=res)
res = parse_unrecognized(context,
default_value,
parsers="all",
prop=var_name,
filter_func=only_successful)
concept.compiled[var_name] = res.body.body if is_only_successful(res) else res
# Updates the cache of concepts when possible
if self.sheerka.has_id(concept.id):
self.sheerka.get_by_id(concept.id).compiled = concept.compiled
def resolve(self, context, to_resolve, current_prop, current_concept, force_evaluation, expect_success):
def resolve(self,
context,
to_resolve,
current_prop,
current_concept,
force_evaluation,
expect_success,
where_clause_def):
"""
Resolve a variable or a Concept
:param context: current execution context
:param to_resolve: Concept or list of ReturnValueConcept to resolve
:param current_prop: current property or ConceptPart
:param current_concept: current concept
:param force_evaluation: Force body evaluation
:param expect_success: for PythonEvaluator, try all possibilities to find a positive result
:param where_clause_def: intermediate where clause for variables
:return:
"""
def get_path(context_, prop_name):
concept_name = f'"{context_.action_context.name}"' if isinstance(context_.action_context, Concept) \
@@ -195,10 +323,11 @@ class SheerkaEvaluateConcept(BaseService):
path=path) as sub_context:
if force_evaluation:
sub_context.local_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
sub_context.protected_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
if expect_success:
sub_context.local_hints.add(BuiltinConcepts.EVAL_UNTIL_SUCCESS_REQUESTED)
sub_context.protected_hints.add(BuiltinConcepts.EVAL_UNTIL_SUCCESS_REQUESTED)
sub_context.protected_hints.add(BuiltinConcepts.EVAL_QUESTION_REQUESTED)
# when it's a concept, evaluate it
if isinstance(to_resolve, Concept) and \
@@ -212,15 +341,28 @@ class SheerkaEvaluateConcept(BaseService):
# otherwise, execute all return values to find out what is the value
else:
# update short term memory with current concept variables
if current_concept:
for var in current_concept.metadata.variables:
value = current_concept.get_value(var[0])
if value != NotInit:
sub_context.add_to_short_term_memory(var[0], current_concept.get_value(var[0]))
use_copy = [r for r in to_resolve] if hasattr(to_resolve, "__iter__") else to_resolve
r = self.sheerka.execute(sub_context, use_copy, CONCEPT_EVALUATION_STEPS)
one_r = expect_one(context, r)
sub_context.add_values(return_values=one_r)
if one_r.status:
return one_r.value
if where_clause_def:
# apply intermediate where clause
r = self.apply_where_clause(context, where_clause_def, r)
if self.sheerka.isinstance(r, BuiltinConcepts.CONDITION_FAILED):
return r
else:
error = one_r.value
one_r = expect_one(context, r)
sub_context.add_values(return_values=one_r)
if one_r.status:
return one_r.value
else:
error = one_r.value
return error if self.sheerka.isinstance(error, BuiltinConcepts.CHICKEN_AND_EGG) \
else self.sheerka.new(BuiltinConcepts.CONCEPT_EVAL_ERROR,
@@ -228,7 +370,14 @@ class SheerkaEvaluateConcept(BaseService):
concept=current_concept,
property_name=current_prop)
def resolve_list(self, context, list_to_resolve, current_prop, current_concept, force_evaluation, expect_success):
def resolve_list(self,
context,
list_to_resolve,
current_prop,
current_concept,
force_evaluation,
expect_success,
where_clause_def):
"""When dealing with a list, there are two possibilities"""
# It may be a list of ReturnValueConcept to execute (always the case for metadata)
# or a list of single values (may be the case for properties)
@@ -242,7 +391,8 @@ class SheerkaEvaluateConcept(BaseService):
current_prop,
current_concept,
force_evaluation,
expect_success)
expect_success,
where_clause_def)
res = []
for to_resolve in list_to_resolve:
@@ -253,7 +403,13 @@ class SheerkaEvaluateConcept(BaseService):
concept=current_concept,
property_name=current_prop)
r = self.resolve(context, to_resolve, current_prop, current_concept, force_evaluation, expect_success)
r = self.resolve(context,
to_resolve,
current_prop,
current_concept,
force_evaluation,
expect_success,
where_clause_def)
if self.sheerka.isinstance(r, BuiltinConcepts.CONCEPT_EVAL_ERROR):
return r
res.append(r)
@@ -263,7 +419,7 @@ class SheerkaEvaluateConcept(BaseService):
def evaluate_concept(self, context, concept: Concept, eval_body=False, metadata=None):
"""
Evaluation a concept
It means that if the where clause is True, will evaluate the body
ie : resolve its body
:param context:
:param concept:
:param eval_body:
@@ -275,7 +431,7 @@ class SheerkaEvaluateConcept(BaseService):
return concept
# I cannot use cache because of concept like 'number'.
# They don't have variables, but their values change every time they are instanciated
# They don't have variables, but their values change every time they are instantiated
# TODO: Need to find a way to cache despite of them
# need_body = eval_body or context.in_context(BuiltinConcepts.EVAL_BODY_REQUESTED)
# if need_body and len(concept.metadata.variables) == 0 and context.sheerka.has_id(concept.id):
@@ -290,11 +446,11 @@ class SheerkaEvaluateConcept(BaseService):
if eval_body:
# ask for body evaluation
sub_context.local_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
sub_context.protected_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
# auto evaluate commands
if context.sheerka.isa(concept, context.sheerka.new(BuiltinConcepts.COMMAND)):
sub_context.local_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
sub_context.protected_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
self.initialize_concept_asts(sub_context, concept)
@@ -307,12 +463,15 @@ class SheerkaEvaluateConcept(BaseService):
for var_name in (v for v in concept.variables() if v in concept.compiled):
prop_ast = concept.compiled[var_name]
w_clause = self.get_where_clause_def(context, concept, var_name)
# TODO, manage when the where clause cannot be parsed
if isinstance(prop_ast, list):
# Do not send the current concept for the properties
resolved = self.resolve_list(sub_context, prop_ast, var_name, None, True, False)
resolved = self.resolve_list(sub_context, prop_ast, var_name, None, True, False, w_clause)
else:
# Do not send the current concept for the properties
resolved = self.resolve(sub_context, prop_ast, var_name, None, True, False)
resolved = self.resolve(sub_context, prop_ast, var_name, None, True, False, w_clause)
if isinstance(resolved, Concept) and not sub_context.sheerka.is_success(resolved):
resolved.set_value("concept", concept) # since current concept was not sent
@@ -347,7 +506,8 @@ class SheerkaEvaluateConcept(BaseService):
part_key,
concept,
force_concept_eval,
expect_success)
expect_success,
None)
# 'FATAL' error is detected, let's stop
if isinstance(resolved, Concept) and not sub_context.sheerka.is_success(resolved):
@@ -411,40 +571,3 @@ class SheerkaEvaluateConcept(BaseService):
to_eval.append("body")
return to_eval
@staticmethod
def get_needed_metadata(concept, concept_part, check_vars, check_body):
"""
Check if the concept_part has to be evaluated
It also checks if the variables and the body need to be evaluated prior to it
:param concept:
:param concept_part:
:param check_vars:
:param check_body:
:return:
"""
ret = []
vars_needed = False
body_needed = False
if concept_part in concept.compiled and concept.compiled[concept_part] is not None:
concept_part_source = getattr(concept.metadata, concept_part.value)
assert concept_part_source is not None
tokens = [t.str_value for t in Tokenizer(concept_part_source)]
if check_vars:
for var_name in (v[0] for v in concept.metadata.variables):
if var_name in tokens:
vars_needed = True
ret.append("variables")
break
if check_body and "self" in tokens:
body_needed = True
ret.append("body")
ret.append(concept_part.value)
return ret, vars_needed, body_needed
src/core/sheerka/services/SheerkaExecute.py
+1 -1
View File
@@ -362,7 +362,7 @@ class SheerkaExecute(BaseService):
if not isinstance(results, list):
results = [results]
for result in results:
if result.body:
if result.body != BuiltinConcepts.NO_RESULT:
evaluated_items.append(result)
to_delete.extend(result.parents)
sub_context.add_values(return_values=results)
src/core/sheerka/services/SheerkaFilter.py
+2 -2
View File
@@ -404,11 +404,11 @@ class SheerkaFilter(BaseService):
yield item
@staticmethod
def pipe_recurse(iterable, depth, prop_name="children", when=None):
def pipe_recurse(iterable, depth, prop_name="_children", when=None):
"""
When printing an object that has sub properties,
indicate the depth of recursion to apply to a specific properties
Quick and dirty version because the prop name is not taken from the item (but set to 'children' by default)
Quick and dirty version because the prop name is not taken from the item (but set to '_children' by default)
:param iterable:
:param depth:
:param prop_name:
src/core/sheerka/services/SheerkaQuestion.py
+40
View File
@@ -0,0 +1,40 @@
from core.builtin_concepts import BuiltinConcepts
from core.concept import Concept
from core.sheerka.services.sheerka_service import BaseService
class SheerkaQuestion(BaseService):
NAME = "Question"
def __init__(self, sheerka):
super().__init__(sheerka)
def initialize(self):
self.sheerka.bind_service_method(self.question, False)
self.sheerka.bind_service_method(self.is_question, False)
def question(self, context, q):
"""
Evaluate q in the context in a question
:param context:
:param q:
:return:
"""
if isinstance(q, Concept):
with context.push(BuiltinConcepts.EVALUATE_CONCEPT, q, desc=f"Evaluating question '{q}'") as sub_context:
sub_context.global_hints.add(BuiltinConcepts.EVAL_QUESTION_REQUESTED)
sub_context.global_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
sub_context.global_hints.add(BuiltinConcepts.EVAL_UNTIL_SUCCESS_REQUESTED)
sub_context.protected_hints.add(BuiltinConcepts.RETURN_BODY_REQUESTED)
evaluated = self.sheerka.evaluate_concept(sub_context, q)
return evaluated
def is_question(self, context):
"""
Returns True if a question is asked
:return:
"""
return context.in_context(BuiltinConcepts.EVAL_QUESTION_REQUESTED)
src/core/sheerka/services/SheerkaResultManager.py
+2 -2
View File
@@ -119,7 +119,7 @@ class SheerkaResultConcept(BaseService):
for e in lst:
yield e
if e.children:
yield from _yield_result(e.children)
if e._children:
yield from _yield_result(e._children)
return _yield_result([execution_context])
src/core/sheerka/services/SheerkaSetsManager.py
+1 -1
View File
@@ -267,7 +267,7 @@ for x in xx__concepts__xx:
{"ids": ids},
desc=f"Evaluating concepts of a set") as sub_context:
sub_context.add_inputs(ids=ids)
sub_context.local_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
sub_context.protected_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
errors = []
for element_id in ids:
concept = self.sheerka.get_by_id(element_id)
src/core/sheerka/services/SheerkaShortTermMemory.py
+37
View File
@@ -0,0 +1,37 @@
from cache.ListIfNeededCache import ListIfNeededCache
from core.sheerka.services.sheerka_service import BaseService
class SheerkaShortTermMemory(BaseService):
NAME = "ShortTermMemory"
SHORT_TERM_MEMORY_ENTRY = "ShortTermMemory:Objects"
def __init__(self, sheerka):
super().__init__(sheerka)
self.objects = ListIfNeededCache()
def initialize(self):
self.sheerka.bind_service_method(self.get_from_short_term_memory, False)
self.sheerka.bind_service_method(self.add_to_short_term_memory, True)
self.sheerka.cache_manager.register_cache(self.SHORT_TERM_MEMORY_ENTRY, self.objects, persist=False)
def get_from_short_term_memory(self, context, key):
while True:
key_to_use = (str(context.id) if context else "") + ":" + key
if (obj := self.sheerka.cache_manager.get(self.SHORT_TERM_MEMORY_ENTRY, key_to_use)) is not None:
return obj
if context is None:
return None
context = context.get_parent()
def add_to_short_term_memory(self, context, key, concept):
if context:
context.stm = True
key_to_use = (str(context.id) if context else "") + ":" + key
return self.sheerka.cache_manager.put(self.SHORT_TERM_MEMORY_ENTRY, key_to_use, concept)
def remove_context(self, context):
self.objects.evict_by_key(lambda k: k.startswith(str(context.id) + ":"))