kodjo/Sheerka-Old
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fixed #29: Parsers: Implement parsing memoization

Browse Source 
Fixed #77 : Parser: ShortTermMemoryParser should be called separately
Fixed #78 : Remove VariableNode usage
Fixed #79 : ConceptManager: Implement compile caching
Fixed #80 : SheerkaExecute : parsers_key is not correctly computed
Fixed #81 : ValidateConceptEvaluator : Validate concept's where and pre clauses right after the parsing
Fixed #82 : SheerkaIsAManager: isa() failed when the set as a body
Fixed #83 : ValidateConceptEvaluator : Support BNF and SYA Concepts
Fixed #84 : ExpressionParser: Implement the parser as a standard parser
Fixed #85 : Services: Give order to services
Fixed #86 : cannot manage smart_get_attr(the short, color)
This commit is contained in:
Kodjo Sossouvi
2021-06-07 21:14:03 +02:00
parent 1059ce25c5
commit 7dcaa9c111
92 changed files with 4263 additions and 1890 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/core/sheerka/services/SheerkaAdmin.py
+23 -1
View File
@@ -6,6 +6,7 @@ from core.builtin_concepts_ids import BuiltinConcepts, BuiltinContainers
from core.builtin_helpers import ensure_concept_or_rule, ensure_concept
from core.concept import Concept
from core.global_symbols import SHEERKA_BACKUP_FOLDER
from core.sheerka.services.SheerkaExecute import SheerkaExecute
from core.sheerka.services.SheerkaHistoryManager import SheerkaHistoryManager
from core.sheerka.services.SheerkaMemory import SheerkaMemory
from core.sheerka.services.sheerka_service import BaseService
@@ -26,6 +27,8 @@ class SheerkaAdmin(BaseService):
self.sheerka.bind_service_method(self.restore, True)
self.sheerka.bind_service_method(self.concepts, False)
self.sheerka.bind_service_method(self.desc, False)
self.sheerka.bind_service_method(self.desc_evaluators, False)
self.sheerka.bind_service_method(self.desc_parsers, False)
self.sheerka.bind_service_method(self.extended_isinstance, False)
self.sheerka.bind_service_method(self.is_container, False)
self.sheerka.bind_service_method(self.format_rules, False)
@@ -35,7 +38,6 @@ class SheerkaAdmin(BaseService):
self.sheerka.bind_service_method(self.ontologies, False)
self.sheerka.bind_service_method(self.in_memory, False)
self.sheerka.bind_service_method(self.admin_history, False, as_name="history")
self.sheerka.bind_service_method(self.admin_history, False, as_name="history")
self.sheerka.bind_service_method(self.sdp, False)
self.sheerka.bind_service_method(self.atomic_def, False)
@@ -165,7 +167,27 @@ class SheerkaAdmin(BaseService):
concepts = sorted(self.sheerka.om.list(self.sheerka.CONCEPTS_BY_ID_ENTRY), key=lambda item: int(item.id))
return self.sheerka.new(BuiltinConcepts.TO_LIST, body=concepts)
def desc_evaluators(self):
evaluators = {k: sorted(v[0].items(), reverse=True)
for k, v in self.sheerka.services[SheerkaExecute.NAME].grouped_evaluators_cache.items()}
return self.sheerka.new(BuiltinConcepts.TO_DICT, body=evaluators)
def desc_parsers(self):
res = {}
for k, v in self.sheerka.services[SheerkaExecute.NAME].grouped_parsers_cache.items():
parsers = {k1: [p.__name__ for p in v1] for k1, v1 in v[0].items()}
sorted_parsers = sorted(parsers.items(), reverse=True)
res[k] = sorted_parsers
return self.sheerka.new(BuiltinConcepts.TO_DICT, body=res)
def desc(self, *items):
if len(items) == 1 and isinstance(items[0], str):
name = items[0].strip().lower()
if name == "parsers":
return self.desc_parsers()
elif name == "evaluators":
return self.desc_evaluators()
ensure_concept_or_rule(*items)
res = []
for item in items:
src/core/sheerka/services/SheerkaConceptManager.py
+37 -3
View File
@@ -14,7 +14,7 @@ from core.builtin_helpers import ensure_concept, ensure_bnf
from core.concept import Concept, DEFINITION_TYPE_DEF, DEFINITION_TYPE_BNF, freeze_concept_attrs, ConceptMetadata, \
VARIABLE_PREFIX
from core.global_symbols import EVENT_CONCEPT_CREATED, NotInit, NotFound, ErrorObj, EVENT_CONCEPT_DELETED, NoFirstToken, \
EVENT_CONCEPT_MODIFIED
EVENT_CONCEPT_MODIFIED, CONCEPT_COMPARISON_CONTEXT
from core.sheerka.services.sheerka_service import BaseService
from core.tokenizer import Tokenizer, TokenKind
from parsers.BnfNodeParser import RegExDef
@@ -132,6 +132,7 @@ class SheerkaConceptManager(BaseService):
self.sheerka.bind_service_method(self.get_concepts_by_first_regex, False, visible=False)
self.sheerka.bind_service_method(self.get_concepts_bnf_definitions, False, visible=False)
self.sheerka.bind_service_method(self.clear_bnf_definition, True, visible=False)
self.sheerka.bind_service_method(self.set_precedence, True)
register_concept_cache = self.sheerka.om.register_concept_cache
@@ -194,7 +195,7 @@ class SheerkaConceptManager(BaseService):
if key in BuiltinUnique:
concept.get_metadata().is_unique = True
concept.get_metadata().is_evaluated = True
concept.get_hints().is_evaluated = True
from_db = self.sheerka.om.get(self.CONCEPTS_BY_KEY_ENTRY, concept.get_metadata().key)
if from_db is NotFound:
@@ -728,6 +729,39 @@ class SheerkaConceptManager(BaseService):
else:
self.sheerka.om.clear(self.CONCEPTS_BNF_DEFINITIONS_ENTRY)
def set_precedence(self, context, *concepts):
"""
Set the precedence order when parsing concept with SyaNodeParser
The first concept in the list have the highest priority
:param context:
:param concepts:
:return:
"""
if len(concepts) < 2:
return self.sheerka.err("Not enough elements")
as_iterable = iter(concepts)
first = next(as_iterable)
ensure_concept(first)
try:
while True:
second = next(as_iterable)
ret = self.sheerka.set_is_greater_than(context,
BuiltinConcepts.PRECEDENCE,
first,
second,
CONCEPT_COMPARISON_CONTEXT)
if not ret.status:
return ret
first = second
except StopIteration:
pass
return self.sheerka.new(BuiltinConcepts.SUCCESS)
@staticmethod
def _name_has_changed(to_add):
if to_add is None or "meta" not in to_add:
@@ -821,7 +855,7 @@ class SheerkaConceptManager(BaseService):
concept.get_metadata().key = None
if self._definition_has_changed(to_add) and concept.get_metadata().definition_type == DEFINITION_TYPE_BNF:
concept.set_bnf(None)
ensure_bnf(context, concept, update_bnf_for_cached_concept=False)
ensure_bnf(context, concept)
concept.init_key()
src/core/sheerka/services/SheerkaEvaluateConcept.py
+256 -132
View File
@@ -1,18 +1,21 @@
from dataclasses import dataclass
from core.builtin_concepts import BuiltinConcepts
from core.builtin_helpers import expect_one, only_successful, evaluate_from_source, ensure_concept
from core.builtin_helpers import expect_one, only_successful, ensure_concept, is_only_successful, ensure_bnf
from core.concept import Concept, DoNotResolve, ConceptParts, InfiniteRecursionResolved, AllConceptParts, \
concept_part_value
from core.global_symbols import NotInit, CURRENT_OBJ
from core.global_symbols import NotInit, CURRENT_OBJ, INIT_AST_PARSERS, INIT_AST_QUESTION_PARSERS
from core.rule import Rule
from core.sheerka.services.SheerkaConceptManager import SheerkaConceptManager
from core.sheerka.services.SheerkaExecute import ParserInput
from core.sheerka.services.sheerka_service import BaseService
from core.sheerka.services.SheerkaEvaluateRules import SheerkaEvaluateRules
from core.sheerka.services.SheerkaExecute import ParserInput, SheerkaExecute
from core.sheerka.services.SheerkaRuleManager import PythonConditionExprVisitor
from core.sheerka.services.sheerka_service import BaseService, FailedToCompileError, ChickenAndEggException
from core.tokenizer import Tokenizer
from core.utils import unstr_concept
from parsers.BaseExpressionParser import TrueifyVisitor
from parsers.BaseNodeParser import ConceptNode
from parsers.BnfNodeParser import BnfNodeConceptExpressionVisitor
from parsers.ExpressionParser import ExpressionParser
from parsers.LogicalOperatorParser import LogicalOperatorParser
CONCEPT_EVALUATION_STEPS = [
@@ -21,11 +24,6 @@ CONCEPT_EVALUATION_STEPS = [
BuiltinConcepts.AFTER_EVALUATION]
@dataclass
class ChickenAndEggException(Exception):
error: Concept
@dataclass
class ConceptEvalException(Exception):
error: Concept
@@ -37,7 +35,7 @@ class WhereClauseDef:
clause: str # original where clause
trueified: str # modified where clause (where unresolvable variables are removed)
prop: str # variable to test
compiled: object # trueified where clause Python compiled
conditions: list # compiled trueified
class SheerkaEvaluateConcept(BaseService):
@@ -45,6 +43,7 @@ class SheerkaEvaluateConcept(BaseService):
def __init__(self, sheerka):
super().__init__(sheerka)
self.rule_evaluator = None
def initialize(self):
self.sheerka.bind_service_method(self.evaluate_concept, True)
@@ -52,6 +51,9 @@ class SheerkaEvaluateConcept(BaseService):
self.sheerka.bind_service_method(self.call_concept, False, as_name="evaluate_question")
self.sheerka.bind_service_method(self.set_auto_eval, True)
def initialize_deferred(self, context, is_first_time):
self.rule_evaluator = self.sheerka.services[SheerkaEvaluateRules.NAME]
@staticmethod
def infinite_recursion_detected(context, concept):
"""
@@ -86,14 +88,20 @@ class SheerkaEvaluateConcept(BaseService):
return None
@staticmethod
def apply_ret(concept):
def apply_ret(concept, eval_body=True):
"""
Check if a concept has its RET part defined
If True, returns it
:param concept:
:param eval_body: Do not return the ret is eval body is not requested
:return:
"""
return concept.get_value(ConceptParts.RET) if ConceptParts.RET in concept.values else concept
if (eval_body and
ConceptParts.RET in concept.values() and
(ret_value := concept.get_value(ConceptParts.RET)) != NotInit):
return ret_value
else:
return concept
@staticmethod
def get_needed_metadata(concept, concept_part, check_vars, check_body):
@@ -153,20 +161,37 @@ class SheerkaEvaluateConcept(BaseService):
to_trueify = [v[0] for v in concept.get_metadata().variables if v[0] != var_name]
trueified_where = str(TrueifyVisitor(to_trueify, [var_name]).visit(expr))
tokens = [t.str_value for t in Tokenizer(trueified_where)]
if var_name in tokens:
compiled = None
try:
compiled = compile(trueified_where, "<where clause>", "eval")
except Exception:
pass
return WhereClauseDef(concept, concept.get_metadata().where, trueified_where, var_name, compiled)
else:
try:
parser_input = context.sheerka.services[SheerkaExecute.NAME].get_parser_input(trueified_where)
parsed = ExpressionParser(auto_compile=False).parse(context, parser_input)
python_visitor = PythonConditionExprVisitor(context)
conditions = python_visitor.get_conditions(parsed.body.body)
return WhereClauseDef(concept, concept.get_metadata().where, trueified_where, var_name, conditions)
except FailedToCompileError:
# TODO: manage invalid where clause
return None
def get_recursive_definitions(self, concept, return_values):
# tokens = [t.str_value for t in Tokenizer(trueified_where)]
# if var_name in tokens:
# compiled = None
# try:
# compiled = compile(trueified_where, "<where clause>", "eval")
# except Exception:
# pass
# return WhereClauseDef(concept, concept.get_metadata().where, trueified_where, var_name, compiled)
# else:
# return None
@staticmethod
def get_recursive_definitions(context, concept, return_values):
"""
Returns the name of the parsers that will resolve to a recursive evaluation
For example, when getting ast for Concept("a and b", body="a and b")
Chances are that we will end up with two parsers
- Python parser for 'a and b'
- ExactConcept parser that point to the concept itself
The ExactConcept will be returned (to be removed from the list as it's a cyclic reference to itself)
:param context:
:param concept:
:param return_values:
:return:
@@ -176,8 +201,9 @@ class SheerkaEvaluateConcept(BaseService):
# During evaluation, inner variables take precedence other concepts
# So there won't be any cyclic reference, the variable will be picked
return
for parser in [r.body for r in return_values if
r.status and self.sheerka.isinstance(r.body, BuiltinConcepts.PARSER_RESULT)]:
r.status and context.sheerka.isinstance(r.body, BuiltinConcepts.PARSER_RESULT)]:
parsed = parser.body if isinstance(parser.body, list) else [parser.body]
for parsed_item in parsed:
if isinstance(parsed_item, Concept) and parsed_item.id == concept.id:
@@ -185,6 +211,85 @@ class SheerkaEvaluateConcept(BaseService):
elif isinstance(parsed_item, ConceptNode) and parsed_item.concept.id == concept.id:
yield parser.parser
@staticmethod
def get_asts(context, who, source, concept, part_key, possible_variables):
"""
Get the return_value_concept or the concept for a given source
:param context:
:param who:
:param concept:
:param part_key: Concept part (#body#, #pre#, #where#...) being compiled
:param source: string or parser input, it does not matter
:param possible_variables: concepts that must be considered as variables
:return:
"""
def parse_token_concept(s):
"""
The source is a direct reference / call to another concept
:param s: source
:return:
"""
if s.startswith("c:") and (identifier := unstr_concept(s)) != (None, None):
return context.sheerka.fast_resolve(identifier)
return None
def get_return_value(current_context, c, s, p):
"""
:param current_context:
:param c: concept
:param s: source
:param p: part of the concept being parsed
:return:
"""
parsers_to_use = INIT_AST_QUESTION_PARSERS if p in [ConceptParts.WHERE,
ConceptParts.PRE] else INIT_AST_PARSERS
while True:
return_value = current_context.sheerka.parse_unrecognized(current_context,
s,
parsers=parsers_to_use,
who=who,
prop=p,
filter_func=only_successful,
possible_variables=possible_variables)
if not return_value.status:
if current_context.preprocess:
raise ChickenAndEggException(context.sheerka.new(BuiltinConcepts.CHICKEN_AND_EGG, body={c}))
else:
raise FailedToCompileError([return_value.body])
return_value = return_value.body.body if is_only_successful(context.sheerka, return_value) else \
[return_value]
if c is None:
# No concept provided, we cannot look for recursive definition
return return_value
recursive_parsers = list(SheerkaEvaluateConcept.get_recursive_definitions(context, c, return_value))
if len(recursive_parsers) == 0:
return return_value
desc = f"Removing parsers {recursive_parsers}"
current_context = current_context.push(context.action, context.action_context, desc=desc)
for recursive_parser in recursive_parsers:
current_context.add_preprocess(recursive_parser.name, enabled=False)
as_str = source.as_text() if isinstance(source, ParserInput) else source
if as_str.strip() == "":
return DoNotResolve(as_str)
else:
if concept_found := parse_token_concept(as_str):
# the compiled can be a reference to another concept...
context.log(f"Recognized concept '{concept_found}'", SheerkaEvaluateConcept.NAME)
return concept_found
else:
# ...or a list of ReturnValueConcept to resolve
return get_return_value(context, concept, source, part_key)
def apply_where_clause(self, context, where_clause_def, return_values):
"""
Apply intermediate where clause when evaluating concept variables
@@ -195,26 +300,53 @@ class SheerkaEvaluateConcept(BaseService):
"""
ret = []
valid_return_values = [r for r in return_values if r.status]
with context.push(BuiltinConcepts.VALIDATING_CONCEPT,
{"concept": where_clause_def.concept, "attr": ConceptParts.WHERE},
desc=f"Apply where clause on '{where_clause_def.prop}'") as sub_context:
sub_context.protected_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
sub_context.protected_hints.add(BuiltinConcepts.EVAL_WHERE_REQUESTED)
sub_context.protected_hints.add(BuiltinConcepts.EVAL_UNTIL_SUCCESS_REQUESTED)
sub_context.protected_hints.add(BuiltinConcepts.EVAL_QUESTION_REQUESTED)
# sub_context.sheerka.add_many_to_short_term_memory(sub_context, objects)
for r in valid_return_values:
if where_clause_def.compiled:
try:
if eval(where_clause_def.compiled, {where_clause_def.prop: self.sheerka.objvalue(r)}):
ret.append(r)
except NameError:
ret.append(r) # it cannot be solved unitary, let's give a chance to the global where condition
sub_context.add_to_short_term_memory(where_clause_def.prop, r.body)
res = self.rule_evaluator.evaluate_conditions(sub_context,
where_clause_def.conditions,
{where_clause_def.prop: r.body})
if len(res) == 0:
# case where missing variables were detected
# This means that the 'where' clause can only be evaluated after all the parts are evaluated
ret.append(r)
else:
# it means that the where condition is an expression that needs to be executed
evaluation_res = evaluate_from_source(context,
where_clause_def.trueified,
desc=f"Apply where clause on '{where_clause_def.prop}'",
expect_success=True,
is_question=True,
stm={where_clause_def.prop: r.body})
one_res = expect_one(context, evaluation_res)
one_res = expect_one(context, res)
if one_res.status:
value = context.sheerka.objvalue(one_res)
if isinstance(value, bool) and value:
ret.append(r)
# value = context.sheerka.objvalue(res.body)
# if res.status and isinstance(bool, value) and value:
# ret.append(r)
# if where_clause_def.compiled:
# try:
# if eval(where_clause_def.compiled, {where_clause_def.prop: self.sheerka.objvalue(r)}):
# ret.append(r)
# except NameError:
# ret.append(r) # it cannot be solved unitary, let's give a chance to the global where condition
# else:
# # it means that the where condition is an expression that needs to be executed
# evaluation_res = evaluate_from_source(context,
# where_clause_def.trueified,
# desc=f"Apply where clause on '{where_clause_def.prop}'",
# expect_success=True,
# is_question=True,
# stm={where_clause_def.prop: r.body})
# one_res = expect_one(context, evaluation_res)
# if one_res.status:
# value = context.sheerka.objvalue(one_res)
# if isinstance(value, bool) and value:
# ret.append(r)
if len(ret) > 0:
return ret
@@ -261,57 +393,19 @@ class SheerkaEvaluateConcept(BaseService):
:return:
"""
def is_only_successful(r):
"""
# for BNF concepts, concepts are sometimes considered as variables
# example :
# def concept a from bnf 'bar' | 'baz'
# def concept foo a as 'foo' a where a == 'baz'
# In the second concept (foo) as is a still a concept, but also a variable in the where clause
:param r: return_value
:return:
"""
return context.sheerka.isinstance(r, BuiltinConcepts.RETURN_VALUE) and \
context.sheerka.isinstance(r.body, BuiltinConcepts.ONLY_SUCCESSFUL)
def parse_token_concept(s):
"""
:param s: source
:return:
"""
if s.startswith("c:") and (identifier := unstr_concept(s)) != (None, None):
return self.sheerka.fast_resolve(identifier)
return None
def get_return_value(current_context, c, s, p):
"""
:param current_context:
:param c: concept
:param s: source
:param p: part of the concept being parsed
:return:
"""
while True:
return_value = current_context.sheerka.parse_unrecognized(current_context,
s,
parsers="all",
prop=p,
filter_func=only_successful)
if not return_value.status:
if current_context.preprocess:
raise ChickenAndEggException(self.sheerka.new(BuiltinConcepts.CHICKEN_AND_EGG, body={c}))
else:
raise Exception(f"Failed to build '{s}'. But it doesn't seems to be recursion")
return_value = return_value.body.body if is_only_successful(return_value) else [return_value]
recursive_parsers = list(self.get_recursive_definitions(c, return_value))
if len(recursive_parsers) == 0:
return return_value
desc = f"Removing parsers {recursive_parsers}"
current_context = current_context.push(context.action, context.action_context, desc=desc)
for recursive_parser in recursive_parsers:
current_context.add_preprocess(recursive_parser.name, enabled=False)
ensure_bnf(context, concept)
if concept.get_bnf():
visitor = BnfNodeConceptExpressionVisitor()
visitor.visit(concept.get_bnf())
possible_variables = [c.name if isinstance(c, Concept) else c for c in visitor.references]
else:
possible_variables = None
for part_key in AllConceptParts:
if part_key in concept.get_compiled():
@@ -324,16 +418,12 @@ class SheerkaEvaluateConcept(BaseService):
if not isinstance(source, str):
raise Exception("Invalid concept init. metadata must be a string")
if source.strip() == "":
concept.get_compiled()[part_key] = DoNotResolve(source)
else:
if concept_found := parse_token_concept(source):
# the compiled can be a reference to another concept...
context.log(f"Recognized concept '{concept_found}'", self.NAME)
concept.get_compiled()[part_key] = concept_found
else:
# ...or a list of ReturnValueConcept to resolve
concept.get_compiled()[part_key] = get_return_value(context, concept, source, part_key)
concept.get_compiled()[part_key] = self.get_asts(context,
self.NAME,
source,
concept,
part_key,
possible_variables)
for var_name, default_value in concept.get_metadata().variables:
if var_name in concept.get_compiled():
@@ -345,22 +435,12 @@ class SheerkaEvaluateConcept(BaseService):
if not isinstance(default_value, str):
raise Exception("Invalid concept init. variable metadata must be a string")
if default_value.strip() == "":
concept.get_compiled()[var_name] = DoNotResolve(default_value)
else:
if concept_found := parse_token_concept(default_value):
# the compiled can be a reference to another concept...
context.log(f"Recognized concept '{concept_found}'", self.NAME)
concept.get_compiled()[var_name] = concept_found
else:
# ...or a list of ReturnValueConcept to resolve
concept.get_compiled()[var_name] = get_return_value(context, concept, default_value, var_name)
# Updates the cache of concepts when possible
# This piece of code is not used, a the compile part is removed by sheerka.new_from_template()
service = context.sheerka.services[SheerkaConceptManager.NAME]
if service.has_id(concept.id):
self.sheerka.get_by_id(concept.id).set_compiled(concept.get_compiled())
concept.get_compiled()[var_name] = self.get_asts(context,
self.NAME,
default_value,
concept,
var_name,
possible_variables)
def resolve(self,
context,
@@ -368,6 +448,7 @@ class SheerkaEvaluateConcept(BaseService):
current_prop,
current_concept,
force_evaluation,
forbid_methods_with_side_effect,
where_clause_def):
"""
Resolve a variable or a Concept
@@ -375,7 +456,8 @@ class SheerkaEvaluateConcept(BaseService):
:param to_resolve: Concept or list of ReturnValueConcept to resolve
:param current_prop: current property or ConceptPart
:param current_concept: current concept
:param force_evaluation: Force body evaluation
:param force_evaluation: force body evaluation
:param forbid_methods_with_side_effect: Do not call methods with side effect when EVAL_BODY_REQUESTED
:param where_clause_def: intermediate where clause for variables
:return:
"""
@@ -412,6 +494,9 @@ class SheerkaEvaluateConcept(BaseService):
if force_evaluation:
sub_context.protected_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
if forbid_methods_with_side_effect:
sub_context.protected_hints.add(BuiltinConcepts.VALIDATION_ONLY_REQUESTED)
if current_prop in (ConceptParts.WHERE, ConceptParts.PRE):
sub_context.protected_hints.add(BuiltinConcepts.EVAL_UNTIL_SUCCESS_REQUESTED)
@@ -440,7 +525,7 @@ class SheerkaEvaluateConcept(BaseService):
value = current_concept.get_value(var[0])
if value != NotInit:
sub_context.add_to_short_term_memory(var[0], current_concept.get_value(var[0]))
use_copy = [r for r in to_resolve] if hasattr(to_resolve, "__iter__") else to_resolve
use_copy = to_resolve.copy() if isinstance(to_resolve, list) else to_resolve
r = self.sheerka.execute(sub_context, use_copy, CONCEPT_EVALUATION_STEPS)
if where_clause_def:
@@ -469,6 +554,7 @@ class SheerkaEvaluateConcept(BaseService):
current_prop,
current_concept,
force_evaluation,
forbid_methods_with_side_effect,
where_clause_def):
"""When dealing with a list, there are two possibilities"""
# It may be a list of ReturnValueConcept to execute (always the case for metadata)
@@ -483,6 +569,7 @@ class SheerkaEvaluateConcept(BaseService):
current_prop,
current_concept,
force_evaluation,
forbid_methods_with_side_effect,
where_clause_def)
res = []
@@ -499,6 +586,7 @@ class SheerkaEvaluateConcept(BaseService):
current_prop,
current_concept,
force_evaluation,
forbid_methods_with_side_effect,
where_clause_def)
if self.sheerka.isinstance(r, BuiltinConcepts.CONCEPT_EVAL_ERROR):
return r
@@ -506,19 +594,27 @@ class SheerkaEvaluateConcept(BaseService):
return res
def evaluate_concept(self, context, concept: Concept, eval_body=False, metadata=None):
def evaluate_concept(self, context, concept: Concept, eval_body=False, validation_only=False, metadata=None):
"""
Evaluation a concept
ie : resolve its body
:param context:
:param concept:
:param eval_body:
:param validation_only: When set, the body is never evaluated, whatever eval_body or EVAL_BODY_REQUESTED
:param metadata: list of metadata to evaluate ('pre', 'post'...)
:return: value of the evaluation or error
"""
if concept.get_metadata().is_evaluated:
return concept
failed_to_evaluate_body = False
if concept.get_hints().is_evaluated:
return self.apply_ret(concept, eval_body or context.in_context(BuiltinConcepts.EVAL_BODY_REQUESTED))
# if concept.get_hints().use_copy:
# raise Exception("Use copy")
# concept = concept.copy()
# concept.get_hints().use_copy = False
# I cannot use cache because of concept like 'number'.
# They don't have variables, but their values change every time they are instantiated
@@ -526,9 +622,9 @@ class SheerkaEvaluateConcept(BaseService):
# need_body = eval_body or context.in_context(BuiltinConcepts.EVAL_BODY_REQUESTED)
# if need_body and len(concept.get_metadata().variables) == 0 and context.sheerka.has_id(concept.id):
# from_cache = context.sheerka.get_by_id(concept.id)
# if from_cache.get_metadata().is_evaluated:
# if from_cache.get_hints().is_evaluated:
# concept.set_value(ConceptParts.BODY, from_cache.body)
# concept.get_metadata().is_evaluated = True
# concept.get_hints().is_evaluated = True
# return concept
desc = f"Evaluating concept {concept}"
@@ -538,6 +634,10 @@ class SheerkaEvaluateConcept(BaseService):
# ask for body evaluation
sub_context.protected_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
if validation_only:
# Never eval the body
sub_context.protected_hints.add(BuiltinConcepts.VALIDATION_ONLY_REQUESTED)
# auto evaluate commands
if context.sheerka.isa(concept, context.sheerka.new(BuiltinConcepts.AUTO_EVAL)):
sub_context.protected_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
@@ -563,10 +663,23 @@ class SheerkaEvaluateConcept(BaseService):
if isinstance(prop_ast, list):
# Do not send the current concept for the properties
resolved = self.resolve_list(sub_context, prop_ast, var_name, None, True, w_clause)
resolved = self.resolve_list(
sub_context,
prop_ast,
var_name,
None,
True,
not sub_context.in_context(BuiltinConcepts.EVAL_BODY_REQUESTED),
w_clause)
else:
# Do not send the current concept for the properties
resolved = self.resolve(sub_context, prop_ast, var_name, None, True, w_clause)
resolved = self.resolve(sub_context,
prop_ast,
var_name,
None,
True,
not sub_context.in_context(BuiltinConcepts.EVAL_BODY_REQUESTED),
w_clause)
if isinstance(resolved, Concept) and not sub_context.sheerka.is_success(resolved):
resolved.set_value("concept", concept) # since current concept was not sent
@@ -592,11 +705,25 @@ class SheerkaEvaluateConcept(BaseService):
force_concept_eval = False if part_key == ConceptParts.BODY else True
# resolve
resolved = self.resolve(sub_context, metadata_ast, part_key, concept, force_concept_eval, None)
resolved = self.resolve(sub_context,
metadata_ast,
part_key,
concept,
force_concept_eval,
not sub_context.in_context(BuiltinConcepts.EVAL_BODY_REQUESTED),
None)
# 'FATAL' error is detected, let's stop
if isinstance(resolved, Concept) and not sub_context.sheerka.is_success(resolved):
return resolved
if not (part_key == ConceptParts.BODY and
self.sheerka.has_error(context, resolved, body=BuiltinConcepts.METHOD_ACCESS_ERROR) and
sub_context.in_context(BuiltinConcepts.VALIDATION_ONLY_REQUESTED)):
return resolved
else:
# BuiltinConcepts.METHOD_ACCESS_ERROR is returned only when the access to side effect
# method is forbidden (during validation or ast initialisation)
resolved = NotInit
failed_to_evaluate_body = True
concept.set_value(part_key, self.get_infinite_recursion_resolution(resolved) or resolved)
@@ -614,8 +741,8 @@ class SheerkaEvaluateConcept(BaseService):
concept.init_key() # Necessary for old unit tests. To remove someday
if ConceptParts.BODY in all_metadata_to_eval:
concept.get_metadata().is_evaluated = True
if ConceptParts.BODY in all_metadata_to_eval and not failed_to_evaluate_body:
concept.get_hints().is_evaluated = True
# # update the cache for concepts with no variables
# Cannot use cache. See the comment at the beginning of this method
@@ -626,10 +753,7 @@ class SheerkaEvaluateConcept(BaseService):
self.sheerka.register_object(sub_context, concept.name, concept)
# manage RET metadata
if sub_context.in_context(BuiltinConcepts.EVAL_BODY_REQUESTED) and ConceptParts.RET in concept.values():
return concept.get_value(ConceptParts.RET)
else:
return concept
return self.apply_ret(concept, sub_context.in_context(BuiltinConcepts.EVAL_BODY_REQUESTED))
def call_concept(self, context, concept, *args, **kwargs):
"""
@@ -656,7 +780,7 @@ class SheerkaEvaluateConcept(BaseService):
needed, variables, body = self.get_needed_metadata(concept, ConceptParts.PRE, True, True)
to_eval.extend(needed)
if context.in_context(BuiltinConcepts.EVAL_WHERE_REQUESTED) or concept.get_metadata().need_validation:
if context.in_context(BuiltinConcepts.EVAL_WHERE_REQUESTED) or concept.get_hints().need_validation:
# What are the cases where we do not need a validation ?
# see test_sheerka_non_reg::test_i_can_evaluate_bnf_concept_with_where_clause()
# res = sheerka.evaluate_user_input("foobar")
src/core/sheerka/services/SheerkaEvaluateRules.py
+14 -3
View File
@@ -111,13 +111,23 @@ class SheerkaEvaluateRules(BaseService):
return expect_one(context, results)
def evaluate_conditions(self, context, conditions, bag):
def evaluate_conditions(self, context, conditions, bag, missing_vars=None):
"""
Evaluate the conditions
:param context:
:param conditions:
:param bag: variables that are supposed to be in short term memory
:param missing_vars: if initialized to a set, keeps tracks of the missing variables
:return:
"""
bag_variables = set(bag.keys())
results = []
for compiled_condition in conditions:
if compiled_condition.variables.intersection(bag_variables) != compiled_condition.variables:
if isinstance(missing_vars, set):
missing_vars.update(compiled_condition.variables - bag_variables)
continue
if compiled_condition.not_variables.intersection(bag_variables):
@@ -131,11 +141,12 @@ class SheerkaEvaluateRules(BaseService):
# do not forget to reset the 'is_evaluated' in the case of a concept
for concept in compiled_condition.concepts_to_reset:
concept.get_metadata().is_evaluated = False
concept.get_hints().is_evaluated = False
evaluator = self.evaluators_by_name[compiled_condition.evaluator_type]
res = evaluator.eval(context, compiled_condition.return_value)
if res.status and isinstance(res.body, bool) and res.body:
value = context.sheerka.objvalue(res.body)
if res.status and isinstance(value, bool) and value:
# one successful value found. No need to look any further
results = [res] # don't we care about the other failing results ?
break
src/core/sheerka/services/SheerkaExecute.py
+188 -66
View File
@@ -2,7 +2,7 @@ import core.utils
from cache.FastCache import FastCache
from core.builtin_concepts import BuiltinConcepts, ReturnValueConcept
from core.concept import ConceptParts
from core.global_symbols import NotFound, NO_MATCH
from core.global_symbols import NotFound, NO_MATCH, EVENT_CONCEPT_CREATED, EVENT_CONCEPT_MODIFIED, EVENT_CONCEPT_DELETED
from core.sheerka.services.sheerka_service import BaseService
from core.tokenizer import Tokenizer, TokenKind, Token, Keywords
@@ -15,6 +15,8 @@ ALL_STEPS = PARSE_AND_EVAL_STEPS + [BuiltinConcepts.BEFORE_RENDERING,
BuiltinConcepts.AFTER_RENDERING,
BuiltinConcepts.BEFORE_RULES_EVALUATION,
BuiltinConcepts.AFTER_RULES_EVALUATION]
STM_PARSER_NAME = "ShortTermMemory"
DEFAULT = "__default"
class ParserInput:
@@ -190,17 +192,20 @@ class SheerkaExecute(BaseService):
# order must be after SheerkaEvaluateRules because of self.rules_evaluation_service
# order must be after ConceptManager because it needs concept bnf definitions
super().__init__(sheerka, order=15)
self.pi_cache = FastCache(default=lambda key: ParserInput(key), max_size=20)
self.pi_cache = FastCache(default=lambda key: ParserInput(key), max_size=200)
self.parsers_cache = FastCache(max_size=2000)
self.instantiated_evaluators = None
self.evaluators_by_name = None
self.instantiated_parsers = None
self.parsers_by_name = None
self.preprocessed_items_old_values = []
self.question_parsers = [] # parsers to use when BuiltinConcepts.EVAL_QUESTION_REQUESTED is set
# cache for all preregistered evaluator combination
# the key is the concatenation of the step and the name of evaluators in the group
# ex : BEFORE_EVALUATION|Python|Sya|Bnf
# the key is a tuple with the name of the step the names of the evaluators in the group
# ex : (BEFORE_EVALUATION, "Python|Sya|Bnf")
# The value is a tuple,
# The first entry is the grouped evaluators
# ex : {60 : [PythonEvaluator(), SyaEvaluator()], 50: [BnfEvaluator()]}
@@ -209,7 +214,7 @@ class SheerkaExecute(BaseService):
# cache for preregistered parsers
# Same construction than the evaluators
# Except 1 : the key does not have a step component. It is simple the list of parsers' names
# Except 1 : the key is a tuple (concept_hint or DEFAULT, names of the parsers)
# Except 2 : we store the type of the parser, not its instance
self.grouped_parsers_cache = {}
@@ -229,8 +234,13 @@ class SheerkaExecute(BaseService):
from core.sheerka.services.SheerkaEvaluateRules import SheerkaEvaluateRules
self.rules_eval_service = self.sheerka.services[SheerkaEvaluateRules.NAME]
self.sheerka.subscribe(EVENT_CONCEPT_CREATED, self.on_concepts_modified)
self.sheerka.subscribe(EVENT_CONCEPT_MODIFIED, self.on_concepts_modified)
self.sheerka.subscribe(EVENT_CONCEPT_DELETED, self.on_concepts_modified)
def reset_state(self):
self.pi_cache.clear()
self.parsers_cache.clear()
def reset_registered_evaluators(self):
# instantiate evaluators, once for all, only keep when it's enabled
@@ -240,7 +250,7 @@ class SheerkaExecute(BaseService):
# get default evaluators by process step
for process_step in ALL_STEPS:
self.grouped_evaluators_cache[f"{process_step}|__default"] = self.get_grouped(
self.grouped_evaluators_cache[(process_step, DEFAULT)] = self.get_grouped(
[e for e in self.instantiated_evaluators if process_step in e.steps])
def reset_registered_parsers(self):
@@ -249,10 +259,20 @@ class SheerkaExecute(BaseService):
:return:
"""
self.instantiated_parsers = [parser(sheerka=self.sheerka) for parser in self.sheerka.parsers.values()]
self.instantiated_parsers = [p for p in self.instantiated_parsers if p.enabled]
self.instantiated_parsers = [p for p in self.instantiated_parsers if p.enabled and p.name != STM_PARSER_NAME]
self.parsers_by_name = {p.short_name: p for p in self.instantiated_parsers}
self.grouped_parsers_cache["__default"] = self.get_grouped(self.instantiated_parsers, use_classes=True)
default_parsers = [p for p in self.instantiated_parsers if p.hints is None]
self.grouped_parsers_cache[(DEFAULT, DEFAULT)] = self.get_grouped(default_parsers, use_classes=True)
# By default, we use the same parsers when it's a question
question_parsers = [p for p in self.instantiated_parsers if
p.hints is not None and BuiltinConcepts.EVAL_QUESTION_REQUESTED in p.hints]
self.grouped_parsers_cache[(BuiltinConcepts.EVAL_QUESTION_REQUESTED, DEFAULT)] = self.get_grouped(
# default_parsers,
question_parsers,
use_classes=True)
self.question_parsers = [p.name for p in question_parsers]
@staticmethod
def get_grouped(evaluators, use_classes=False):
@@ -302,12 +322,12 @@ class SheerkaExecute(BaseService):
"""
# Normal case, the evaluators are the default one
if not context.preprocess_evaluators and not context.preprocess:
return self.grouped_evaluators_cache[f"{process_step}|__default"]
return self.grouped_evaluators_cache[(process_step, DEFAULT)]
# Other case, only use a subset of evaluators
selected = context.preprocess_evaluators
if selected and not context.preprocess:
key = str(process_step) + "|" + "|".join(selected)
key = (process_step, "|".join(selected))
try:
return self.grouped_evaluators_cache[key]
except KeyError:
@@ -357,6 +377,15 @@ class SheerkaExecute(BaseService):
groups, sorted_priorities = self.get_grouped(parsers, use_classes=True)
return key, *get_instances((groups, sorted_priorities))
def get_input_as_text(self, text):
if isinstance(text, str):
return text
if isinstance(text, ParserInput):
return text.as_text()
raise NotImplementedError()
def get_parser_input(self, text, tokens=None):
"""
Returns new or existing parser input
@@ -386,17 +415,44 @@ class SheerkaExecute(BaseService):
"""
From the context.preprocess_parsers and context.preprocess,
try to find a key to store the further results of the parsings
The key is a two values tuple
* The first part indicates the parsers to use (__default for the hardcoded default behaviour)
* The second part indicates some context hint, like for instance if it's question
:param context:
:return:
"""
if not context.preprocess_parsers and not context.preprocess:
return "__default"
# as of now, EVAL_QUESTION_REQUESTED is the only hint that can alter the parsing
if context.in_context(BuiltinConcepts.EVAL_QUESTION_REQUESTED):
in_context = BuiltinConcepts.EVAL_QUESTION_REQUESTED
else:
in_context = DEFAULT
if context.preprocess_parsers and not context.preprocess:
return "|".join(context.preprocess_parsers)
if context.preprocess:
from parsers.BaseParser import BaseParser
preprocess = [p for p in context.preprocess if p.preprocess_name.startswith(BaseParser.PREFIX)]
else:
preprocess = None
if not context.preprocess_parsers and not preprocess:
return in_context, DEFAULT
if context.preprocess_parsers and not preprocess:
return in_context, "|".join(context.preprocess_parsers)
return None
def add_to_parser_cache(self, parsers_key, text, return_value):
if parsers_key is None:
return
key = (parsers_key, text)
if key in self.parsers_cache:
old = self.parsers_cache.get(key)
old.append((return_value.who, return_value.status, return_value.value))
self.parsers_cache.put(key, old)
else:
self.parsers_cache.put(key, [(return_value.who, return_value.status, return_value.value)])
def call_parsers(self, context, return_values):
"""
Call all the parsers, ordered by priority
@@ -414,7 +470,7 @@ class SheerkaExecute(BaseService):
if not isinstance(return_values, list):
return_values = [return_values]
# first make the distinguish between what is for the parsers and what is not
# 1. Make the distinguish between what is for the parsers and what is not
result = []
to_process = []
for r in return_values:
@@ -431,58 +487,110 @@ class SheerkaExecute(BaseService):
parsers_key, grouped_parsers, sorted_priorities = self.get_parsers(context)
stop_processing = False
for priority in sorted_priorities:
inputs_for_this_group = to_process[:]
# 2. Try the stm parser, as it depends on the context
from parsers.ShortTermMemoryParser import ShortTermMemoryParser
if parsers_key is None or parsers_key[1] == DEFAULT or ShortTermMemoryParser.NAME in parsers_key[1]:
try:
stm_parser = self.parsers_by_name[STM_PARSER_NAME]
if stm_parser.enabled:
processed = []
for return_value in to_process:
to_parse = self.get_parser_input(return_value.body.body) \
if self.sheerka.isinstance(return_value.body, BuiltinConcepts.USER_INPUT) \
else return_value.body
for parser in grouped_parsers[priority]:
for return_value in inputs_for_this_group:
to_parse = self.get_parser_input(return_value.body.body) \
if self.sheerka.isinstance(return_value.body, BuiltinConcepts.USER_INPUT) \
else return_value.body
# if self.sheerka.log.isEnabledFor(logging.DEBUG):
# debug_text = "'" + to_parse + "'" if isinstance(to_parse, str) \
# else "'" + core.utils.get_text_from_tokens(to_parse) + "' as tokens"
# context.log(f"Parsing {debug_text}")
with context.push(BuiltinConcepts.PARSING,
{"parser": parser.name},
desc=f"Parsing using {parser.name}") as sub_context:
sub_context.add_inputs(to_parse=to_parse)
res = parser.parse(sub_context, to_parse)
if res is not None:
if hasattr(res, "__iter__"):
for r in res:
if r is None:
continue
r.parents = [return_value]
result.append(r)
if self.sheerka.isinstance(r.body, BuiltinConcepts.PARSER_RESULT):
# if a ParserResultConcept is returned, it will be used by the parsers
# of the following groups
to_process.append(r)
if r.status:
stop_processing = True
else:
with context.push(BuiltinConcepts.PARSING,
{"parser": stm_parser.name},
desc=f"Parsing using {stm_parser.name}") as sub_context:
sub_context.add_inputs(to_parse=to_parse)
res = stm_parser.parse(sub_context, to_parse)
if res.status:
res.parents = [return_value]
result.append(res)
if self.sheerka.isinstance(res.body, BuiltinConcepts.PARSER_RESULT):
# if a ParserResultConcept is returned, it will be used by the parsers
# of the following groups
to_process.append(res)
if res.status:
stop_processing = True
sub_context.add_values(return_values=res)
processed.append(return_value)
sub_context.add_values(return_values=res)
if stop_processing:
break # Do not try the other priorities if a match is found
to_process = core.utils.remove_list_from_list(to_process, processed)
except KeyError:
# stm_parser may not exist in some unit tests
pass
# 3. Try the cache
if to_process and parsers_key:
processed = []
for return_value in to_process:
to_parse_as_str = self.get_input_as_text(return_value.body.body) \
if self.sheerka.isinstance(return_value.body, BuiltinConcepts.USER_INPUT) \
else return_value.body.source
key_to_use = (parsers_key, to_parse_as_str)
if key_to_use in self.parsers_cache:
for who, status, value in self.parsers_cache.get(key_to_use):
ret = self.sheerka.ret(who, status, value)
ret.parents = [return_value]
result.append(ret)
processed.append(return_value)
to_process = core.utils.remove_list_from_list(to_process, processed)
# 4. Call the parsers
if to_process:
stop_processing = False
for priority in sorted_priorities:
inputs_for_this_group = to_process[:]
for parser in grouped_parsers[priority]:
for return_value in inputs_for_this_group:
if self.sheerka.isinstance(return_value.body, BuiltinConcepts.USER_INPUT):
to_parse_as_str = self.get_input_as_text(return_value.body.body)
to_parse = self.get_parser_input(return_value.body.body)
else:
to_parse = return_value.body
to_parse_as_str = return_value.body.source
# if self.sheerka.log.isEnabledFor(logging.DEBUG):
# debug_text = "'" + to_parse + "'" if isinstance(to_parse, str) \
# else "'" + core.utils.get_text_from_tokens(to_parse) + "' as tokens"
# context.log(f"Parsing {debug_text}")
with context.push(BuiltinConcepts.PARSING,
{"parser": parser.name},
desc=f"Parsing using {parser.name}") as sub_context:
sub_context.add_inputs(to_parse=to_parse)
res = parser.parse(sub_context, to_parse)
if res is not None:
if hasattr(res, "__iter__"):
for r in res:
if r is None:
continue
r.parents = [return_value]
result.append(r)
self.add_to_parser_cache(parsers_key, to_parse_as_str, r)
if self.sheerka.isinstance(r.body, BuiltinConcepts.PARSER_RESULT):
# if a ParserResultConcept is returned, it will be used by the parsers
# of the following groups
to_process.append(r)
if r.status:
stop_processing = True
else:
res.parents = [return_value]
result.append(res)
self.add_to_parser_cache(parsers_key, to_parse_as_str, res)
if self.sheerka.isinstance(res.body, BuiltinConcepts.PARSER_RESULT):
# if a ParserResultConcept is returned, it will be used by the parsers
# of the following groups
to_process.append(res)
if res.status:
stop_processing = True
sub_context.add_values(return_values=res)
if stop_processing:
break # Do not try the other priorities if a match is found
result = core.utils.remove_list_from_list(result, user_inputs)
return result
def call_evaluators(self, context, return_values, process_step):
@@ -687,7 +795,12 @@ class SheerkaExecute(BaseService):
else:
return parser_or_evaluator_name == preprocessor_name
def parse_unrecognized(self, context, source, parsers, who=None, prop=None, filter_func=None):
def parse_unrecognized(self, context, source, parsers,
who=None,
prop=None,
filter_func=None,
is_question=False,
possible_variables=None):
"""
Try to recognize concepts or code from source using the given parsers
:param context:
@@ -696,6 +809,8 @@ class SheerkaExecute(BaseService):
:param who: who is asking the parsing ?
:param prop: Extra info, when parsing a property
:param filter_func: Once the result are found, call this function to filter them
:param is_question: Force EVAL_QUESTION_REQUESTED
:param possible_variables: concepts that must be considered as variables
:return:
"""
sheerka = context.sheerka
@@ -708,14 +823,18 @@ class SheerkaExecute(BaseService):
desc = f"Parsing '{source}'"
with context.push(BuiltinConcepts.PARSING, action_context, who=who, desc=desc) as sub_context:
if (prop in (Keywords.WHERE, Keywords.PRE, ConceptParts.WHERE, ConceptParts.PRE, Keywords.WHEN) or
is_question):
sub_context.protected_hints.add(BuiltinConcepts.EVAL_QUESTION_REQUESTED)
# disable all parsers but the requested ones
if parsers != "all":
sub_context.preprocess_parsers = parsers
else:
sub_context.preprocess_parsers = None
if prop in (Keywords.WHERE, Keywords.PRE, ConceptParts.WHERE, ConceptParts.PRE, Keywords.WHEN):
sub_context.protected_hints.add(BuiltinConcepts.EVAL_QUESTION_REQUESTED)
sub_context.possible_variables = possible_variables
sub_context.add_inputs(source=source)
to_parse = sheerka.ret(context.who,
@@ -779,7 +898,7 @@ class SheerkaExecute(BaseService):
python_parser = PythonParser()
return python_parser.parse(sub_context, parser_input)
def parse_expression(self, context, source, desc=None):
def parse_expression(self, context, source, desc=None, auto_compile=False):
"""
Helper function to parser expressions with AND, OR and NOT
"""
@@ -787,5 +906,8 @@ class SheerkaExecute(BaseService):
with context.push(BuiltinConcepts.PARSE_CODE, source, desc) as sub_context:
parser_input = context.sheerka.services[SheerkaExecute.NAME].get_parser_input(source)
from parsers.ExpressionParser import ExpressionParser
expr_parser = ExpressionParser()
expr_parser = ExpressionParser(auto_compile=auto_compile)
return expr_parser.parse(sub_context, parser_input)
def on_concepts_modified(self, *args, **kwargs):
self.parsers_cache.clear()
src/core/sheerka/services/SheerkaIsAManager.py
+22 -34
View File
@@ -1,12 +1,12 @@
import core.builtin_helpers
from cache.Cache import Cache
from cache.SetCache import SetCache
from core.ast_helpers import UnreferencedVariablesVisitor
from core.builtin_concepts import BuiltinConcepts
from core.concept import Concept, ConceptParts, DEFINITION_TYPE_BNF
from core.concept import Concept, DEFINITION_TYPE_BNF
from core.global_symbols import NotFound
from core.sheerka.services.SheerkaConceptManager import SheerkaConceptManager
from core.sheerka.services.sheerka_service import BaseService
from core.tokenizer import Tokenizer, TokenKind
from core.utils import merge_sets
@@ -153,16 +153,7 @@ class SheerkaIsAManager(BaseService):
# apply the where clause if any
if sub_concept.get_metadata().where:
new_condition = self._validate_where_clause(context, sub_concept)
if not new_condition:
return self.sheerka.new(BuiltinConcepts.CONDITION_FAILED, body=sub_concept)
# This methods sucks, but I don't have enough tools (like proper AST manipulation functions)
# to do it properly now. It will be enhanced later
globals_ = {"xx__concepts__xx": concepts, "sheerka": self.sheerka}
locals_ = {}
exec(new_condition, globals_, locals_)
concepts = locals_["result"]
concepts = self._filter_results(context, sub_concept, concepts)
return concepts
@@ -201,7 +192,7 @@ class SheerkaIsAManager(BaseService):
return False
for c in a.get_metadata().props[BuiltinConcepts.ISA]:
if c == b:
if c.id == b.id:
return True
if self.isa(self.sheerka.get_by_id(c.id), b):
return True
@@ -237,28 +228,25 @@ class SheerkaIsAManager(BaseService):
return self.isaset(context, concept.body)
def _validate_where_clause(self, context, concept):
python_parser_result = [r for r in concept.get_compiled()[ConceptParts.WHERE] if r.who == "parsers.Python"]
if not python_parser_result or not python_parser_result[0].status:
return None
def _filter_results(self, context, concept, results):
"""
Filter the list of results, according to the specification of the concept
ex: def concept sub_concept as number where number < 4
We want to return the numbers that are < 4
:param context:
:param concept:
:param results:
:return:
"""
# first get the pivot variable
possibles_variables = [t.value for t in Tokenizer(concept.get_metadata().body, yield_eof=False) if
t.type in (TokenKind.IDENTIFIER, TokenKind.KEYWORD)]
if len(possibles_variables) != 1:
return self.sheerka.new(BuiltinConcepts.CONDITION_FAILED, body=concept)
ast_ = python_parser_result[0].body.body.ast_
visitor = UnreferencedVariablesVisitor(context)
names = list(visitor.get_names(ast_))
if len(names) != 1 or names[0] != concept.get_metadata().body:
return None
condition = concept.get_metadata().where.replace(concept.get_metadata().body, "sheerka.objvalue(x)")
expression = f"""
result=[]
for x in xx__concepts__xx:
try:
if {condition}:
result.append(x)
except Exception:
pass
"""
return expression
predicate = concept.get_metadata().where.replace(possibles_variables[0], "sheerka.objvalue(self)")
res = self.sheerka.filter_objects(context, results, predicate)
return res
def _get_concepts(self, context, ids, evaluate):
"""
src/core/sheerka/services/SheerkaMemory.py
+7 -5
View File
@@ -222,12 +222,14 @@ class SheerkaMemory(BaseService):
"""
name_to_use = name.name if isinstance(name, Concept) else name
self.unregister_object(context, name_to_use)
# first try direct access
obj = self.get_last_from_memory(context, name_to_use)
if obj is not NotFound:
return obj.obj
all_objects = self.sheerka.om.list(SheerkaMemory.OBJECTS_ENTRY)
all_objects_copy = []
all_objects = self.sheerka.om.list(SheerkaMemory.OBJECTS_ENTRY) # not always a list of list
all_objects_copy = [] # to transform into list of list
for obj in all_objects:
if isinstance(obj, list):
all_objects_copy.append(obj.copy())
@@ -242,15 +244,15 @@ class SheerkaMemory(BaseService):
if len(obj) > 0:
temp.append(obj)
all_objects_copy = temp
all_objects_copy = temp # list constructed with the last item of each item
current_list = sorted(current_list, key=lambda o: o.timestamp, reverse=True)
current_objects = [o.obj for o in current_list]
res = self.sheerka.filter_objects(context, current_objects, name)
res = self.sheerka.filter_objects(context, current_objects, name_to_use)
if len(res) > 0:
return res[0] # only the first, as it should have a better timestamp
return self.sheerka.new(BuiltinConcepts.NOT_FOUND, body={"#name": name})
return self.sheerka.new(BuiltinConcepts.NOT_FOUND, body={"#name": name_to_use})
def mem(self):
keys = sorted([k for k in self.sheerka.om.list(SheerkaMemory.OBJECTS_ENTRY)])
src/core/sheerka/services/SheerkaQueryManager.py
+1 -1
View File
@@ -19,7 +19,7 @@ class SheerkaQueryManager(BaseService):
QUERY_PARAMETER_PREFIX = "__xxx__query_parameter__xx__"
def __init__(self, sheerka):
super().__init__(sheerka)
super().__init__(sheerka, order=16)
self.queries = FastCache()
self.conditions = FastCache()
self.lexer = Lexer()
src/core/sheerka/services/SheerkaRuleManager.py
+164 -628
View File
@@ -1,610 +1,38 @@
import operator
import re
from dataclasses import dataclass
from typing import Union, Set, List, Tuple
from cache.Cache import Cache
from cache.ListIfNeededCache import ListIfNeededCache
from core.ast_helpers import UnreferencedNamesVisitor
from core.builtin_concepts import BuiltinConcepts, ReturnValueConcept
from core.builtin_helpers import ensure_evaluated, expect_one, evaluate_from_source, \
get_possible_variables_from_concept, is_a_question
get_possible_variables_from_concept, is_a_question, only_successful, is_only_successful, evaluate_return_values
from core.concept import Concept
from core.global_symbols import EVENT_RULE_PRECEDENCE_MODIFIED, RULE_COMPARISON_CONTEXT, NotFound, ErrorObj, \
EVENT_RULE_CREATED, EVENT_RULE_DELETED, NotInit
EVENT_RULE_CREATED, EVENT_RULE_DELETED, NotInit, INIT_AST_PARSERS
from core.rule import Rule, ACTION_TYPE_PRINT
from core.sheerka.Sheerka import RECOGNIZED_BY_NAME, RECOGNIZED_BY_ID
from core.sheerka.services.sheerka_service import BaseService, FailedToCompileError
from core.tokenizer import Keywords, TokenKind, Token, IterParser
from core.utils import index_tokens, COLORS, get_text_from_tokens, merge_dictionaries, merge_sets, get_safe_str_value
from core.sheerka.services.sheerka_service import BaseService, FailedToCompileError, UnknownVariableError
from core.tokenizer import Keywords, TokenKind, Token
from core.utils import merge_dictionaries, merge_sets, get_safe_str_value
from evaluators.PythonEvaluator import PythonEvaluator
from parsers.BaseExpressionParser import AndNode, ExpressionVisitor, VariableNode, ComparisonNode, FunctionNode, \
ComparisonType, NotNode, NameExprNode
from parsers.BaseNodeParser import ConceptNode
from parsers.FormatRuleActionParser import FormatRuleActionParser
from parsers.LogicalOperatorParser import LogicalOperatorParser
from sheerkapython.python_wrapper import Expando, sheerka_globals
from sheerkarete.common import V
from sheerkarete.conditions import AndConditions, Condition, NegatedCondition, NegatedConjunctiveConditions
from sheerkarete.network import FACT_NAME, FACT_SELF
CONCEPTS_ONLY_PARSERS = ["ExactConcept", "Bnf", "Sya", "Sequence"]
CONDITIONS_VISITOR_EVALUATORS = ["Python", "Concept", "LexerNode"]
identifier_regex = re.compile(r"[\w _.]+")
@dataclass
class FormatRuleError(ErrorObj):
pass
@dataclass
class BraceMismatch(FormatRuleError):
lbrace: Token
@dataclass
class UnexpectedEof(FormatRuleError):
message: str
token: Token = None
def __eq__(self, other):
if id(self) == id(other):
return True
if not isinstance(other, UnexpectedEof):
return False
return self.message == other.message and (other.token is None or other.token == self.token)
def __hash__(self):
return hash(self.message, self.token)
@dataclass
class FormatRuleSyntaxError(FormatRuleError):
message: str
token: Token
@dataclass
class FormatAstNode:
@staticmethod
def repr_value(items):
if items is None:
return ""
return ", ".join(repr(item) for item in items)
def clone(self, instance, props, **kwargs):
for prop_name in props:
setattr(instance, prop_name, getattr(self, prop_name))
for k, v in kwargs.items():
setattr(instance, k, v)
return instance
@dataclass
class FormatAstRawText(FormatAstNode):
text: str
@dataclass
class FormatAstVariable(FormatAstNode):
name: str
format: Union[str, None] = None
debug: bool = False
value: object = None
index: object = None
def clone(self, **kwargs):
return super().clone(FormatAstVariable(self.name),
("format", "debug", "value", "index"),
**kwargs)
@dataclass
class FormatAstVariableNotFound(FormatAstNode):
name: str
@dataclass
class FormatAstGrid(FormatAstNode):
pass
@dataclass
class FormatAstList(FormatAstNode):
variable: str
items_prop: str = None # where to search the list if variable does not resolve to an iterable
recurse_on: str = None
recursion_depth: int = 0
debug: bool = False
prefix: str = None
suffix: str = None
show_index: bool = False
index: object = None
items: object = None
def clone(self, **kwargs):
return super().clone(
FormatAstList(self.variable),
(
"items_prop", "recurse_on", "recursion_depth", "debug", "prefix", "suffix", "show_index", "index",
"items"),
**kwargs)
@dataclass
class FormatAstDict(FormatAstNode):
variable: str
items_prop: str = None # where to search the dict if variable does not resolve to an iterable
debug: bool = False
prefix: str = None
suffix: str = None
items: object = None
def clone(self, **kwargs):
return super().clone(
FormatAstDict(self.variable),
("items_prop", "debug", "prefix", "suffix", "items"),
**kwargs)
@dataclass
class FormatAstColor(FormatAstNode):
color: str
format_ast: FormatAstNode
def __repr__(self):
return f"{self.color}({self.format_ast})"
def clone(self, **kwargs):
return super().clone(
FormatAstColor(self.color, self.format_ast),
(),
**kwargs)
@dataclass
class FormatAstFunction(FormatAstNode):
name: str
args: list = None
kwargs: dict = None
@dataclass
class FormatAstSequence(FormatAstNode):
items: list
debug: bool = False
def __repr__(self):
return "FormatAstSequence(" + self.repr_value(self.items) + ")"
def clone(self, **kwargs):
return super().clone(
FormatAstSequence(self.items),
("debug",),
**kwargs)
@dataclass
class FormatAstMulti(FormatAstNode):
"""
Used when there are multiple out to print, but they are not related
Just print them one by one
"""
variable: str
items: list = None
def __repr__(self):
return f"FormatAstMulti({self.variable}, items={self.items})"
def clone(self, **kwargs):
return super().clone(
FormatAstMulti(self.variable),
("items",),
**kwargs)
class FormatRuleActionParser(IterParser):
@staticmethod
def to_text(list_or_dict_of_tokens):
"""
Works on list of list of tokens
or dict of list of tokens
:param list_or_dict_of_tokens:
:return:
"""
get_text = get_text_from_tokens
if isinstance(list_or_dict_of_tokens, list):
return [get_text(i) for i in list_or_dict_of_tokens]
if isinstance(list_or_dict_of_tokens, dict):
return {k: get_text(v) for k, v in list_or_dict_of_tokens.items()}
raise NotImplementedError("")
def to_value(self, tokens):
"""
Works on list of tokens
return string or numeric value of the tokens
:return:
"""
value = get_text_from_tokens(tokens)
if value[0] in ("'", '"'):
return value[1:-1]
if value in ("True", "False"):
return bool(value)
try:
return int(value)
except ValueError:
pass
try:
return float(value)
except ValueError:
self.error_sink = FormatRuleSyntaxError(f"'{value}' is not numeric", None)
def parse(self):
"""
Parses the print part of the format rule
format ::= {variable'} | function(...) | rawtext
:return:
"""
if self.source == "":
return FormatAstRawText("")
buffer = []
result = []
res = None
escaped = False
def _flush_buffer():
if len(buffer) > 0:
result.append(FormatAstRawText(get_text_from_tokens(buffer)))
buffer.clear()
while self.next_token(skip_whitespace=False):
if not escaped:
if self.token.type == TokenKind.IDENTIFIER and self.the_token_after().type == TokenKind.LPAR:
_flush_buffer()
res = self.parse_function(self.token)
elif self.token.type == TokenKind.LBRACE:
_flush_buffer()
res = self.parse_variable(self.token)
elif self.token.type == TokenKind.BACK_SLASH:
escaped = True
else:
buffer.append(self.token)
else:
escaped = False
buffer.append(self.token)
if self.error_sink:
break
if res:
result.append(res)
res = None
_flush_buffer()
return [] if len(result) == 0 else result[0] if len(result) == 1 else FormatAstSequence(result)
def parse_function(self, func_name):
self.next_token()
self.next_token()
if self.token.type == TokenKind.EOF:
self.error_sink = UnexpectedEof("while parsing function", func_name)
return None
param_buffer = []
args = []
kwargs = {}
get_text = get_text_from_tokens
def _process_parameters():
if len(param_buffer) == 0:
self.error_sink = FormatRuleSyntaxError("no parameter found", self.token)
return None
if (index := index_tokens(param_buffer, "=")) > 0:
kwargs[get_text(param_buffer[:index])] = param_buffer[index + 1:]
else:
args.append(param_buffer.copy())
param_buffer.clear()
while True:
if self.token.type == TokenKind.RPAR:
if len(param_buffer) > 0:
_process_parameters()
break
elif self.token.type == TokenKind.COMMA:
_process_parameters()
if self.error_sink:
break
else:
param_buffer.append(self.token)
if not self.next_token():
break
if self.error_sink:
return None
if self.token.type != TokenKind.RPAR:
self.error_sink = UnexpectedEof("while parsing function", func_name)
return None
if func_name.value in COLORS:
return self.return_color(func_name.value, args, kwargs)
elif func_name.value == "list":
return self.return_list(args, kwargs)
elif func_name.value == "dict":
return self.return_dict(args, kwargs)
elif func_name.value == "multi":
return self.return_multi(args, kwargs)
return FormatAstFunction(func_name.value, self.to_text(args), self.to_text(kwargs))
def parse_variable(self, lbrace):
self.next_token()
if self.token.type == TokenKind.EOF:
self.error_sink = UnexpectedEof("while parsing variable", lbrace)
return None
buffer = []
while True:
if self.token.type == TokenKind.RBRACE:
break
buffer.append(self.token)
if not self.next_token():
break
# if self.error_sink:
# return None
if self.token.type != TokenKind.RBRACE:
self.error_sink = UnexpectedEof("while parsing variable", lbrace)
return None
if len(buffer) == 0:
self.error_sink = FormatRuleSyntaxError("variable name not found", None)
return None
variable = get_text_from_tokens(buffer)
try:
index = variable.index(":")
return FormatAstVariable(variable[:index], variable[index + 1:])
except ValueError:
return FormatAstVariable(variable)
def return_color(self, color, args, kwargs):
if len(kwargs) > 0:
self.error_sink = FormatRuleSyntaxError("keyword arguments are not supported", None)
return None
if len(args) == 0:
return FormatAstColor(color, FormatAstRawText(""))
if len(args) > 1:
self.error_sink = FormatRuleSyntaxError("only one parameter supported", args[1][0])
return None
source = get_text_from_tokens(args[0])
if len(source) > 1 and source[0] in ("'", '"') and source[-1] in ("'", '"'):
source = source[1:-1]
parser = FormatRuleActionParser(source)
res = parser.parse()
self.error_sink = parser.error_sink
return FormatAstColor(color, res)
else:
try:
index = source.index(":")
variable, vformat = source[:index], source[index + 1:]
except ValueError:
variable, vformat = source, None
if not identifier_regex.fullmatch(variable):
self.error_sink = FormatRuleSyntaxError("Invalid identifier", None)
return None
return FormatAstColor(color, FormatAstVariable(variable, vformat))
def return_list(self, args, kwargs):
"""
Looking for greeting_var, [recurse_on], [recursion_depth], [items_prop]
:param args:
:param kwargs:
:return:
"""
len_args = len(args)
if len_args < 1:
self.error_sink = FormatRuleSyntaxError("variable name not found", None)
return None
if len_args > 4:
self.error_sink = FormatRuleSyntaxError("too many positional arguments", args[4][0])
return None
variable_name = get_text_from_tokens(args[0])
recurse_on, recursion_depth, items_prop = None, 0, None
if len_args == 2:
recursion_depth = self.to_value(args[1])
elif len_args == 3:
recursion_depth = self.to_value(args[1])
recurse_on = self.to_value(args[2])
elif len_args == 4:
recursion_depth = self.to_value(args[1])
recurse_on = self.to_value(args[2])
items_prop = self.to_value(args[3])
if "recurse_on" in kwargs:
recurse_on = self.to_value(kwargs["recurse_on"])
if "recursion_depth" in kwargs:
recursion_depth = self.to_value(kwargs["recursion_depth"])
if "items_prop" in kwargs:
items_prop = self.to_value(kwargs["items_prop"])
if self.error_sink:
return None
if not isinstance(recursion_depth, int):
self.error_sink = FormatRuleSyntaxError("'recursion_depth' must be an integer", None)
return None
return FormatAstList(variable_name, items_prop, recurse_on, recursion_depth)
def return_dict(self, args, kwargs):
len_args = len(args)
if len_args < 1:
self.error_sink = FormatRuleSyntaxError("variable name not found", None)
return None
if len_args > 1:
self.error_sink = FormatRuleSyntaxError("too many positional arguments", args[1][0])
return None
variable_name = get_text_from_tokens(args[0])
kwargs_parameters = {}
for prop in ("items_prop", "prefix", "suffix", "debug"):
if prop in kwargs:
kwargs_parameters[prop] = self.to_value(kwargs[prop])
if "debug" in kwargs_parameters:
if "prefix" not in kwargs_parameters:
kwargs_parameters["prefix"] = "{"
if "suffix" not in kwargs_parameters:
kwargs_parameters["suffix"] = "}"
if self.error_sink:
return None
return FormatAstDict(variable_name, **kwargs_parameters)
def return_multi(self, args, kwargs):
if len(kwargs) > 0:
self.error_sink = FormatRuleSyntaxError("keyword arguments are not supported", None)
return None
if len(args) > 1:
self.error_sink = FormatRuleSyntaxError("too many positional arguments", args[1][0])
return None
return FormatAstMulti(get_text_from_tokens(args[0]))
@dataclass
class EmitPythonCodeException(Exception):
error: object
class PythonCodeEmitter:
def __init__(self, context, text=None):
self.context = context
self.text = text or ""
self.var_counter = 0
self.variables = []
def add(self, text):
self.text += f" and {text}" if self.text else text
return self
def recognize(self, obj, as_name, root=True):
if isinstance(obj, str):
return self.recognize_str(obj, as_name)
elif isinstance(obj, (int, float)):
return self.recognize_int(obj, as_name)
elif isinstance(obj, Concept):
return self.recognize_concept(obj, as_name, root)
elif isinstance(obj, Expando):
return self.recognize_expando(obj, as_name, root)
else:
raise NotImplementedError()
def recognize_str(self, text, as_name):
if self.text:
self.text += " and "
if "'" in text and '"' in text:
self.text += f"{as_name} == '{text}'"
elif "'" in text:
self.text += f'{as_name} == "{text}"'
else:
self.text += f"{as_name} == '{text}'"
return self
def recognize_int(self, value, as_name):
if self.text:
self.text += " and "
self.text += f"{as_name} == {value}"
return self
def recognize_expando(self, value, as_name, root=True):
if self.text:
self.text += " and "
if not root:
as_name = self.add_variable(as_name)
self.text += f"isinstance({as_name}, Expando) and {as_name}.get_name() == '{value.get_name()}'"
return self
def recognize_concept(self, concept, as_name, root=True):
if self.text:
self.text += " and "
if not root:
as_name = self.add_variable(as_name)
if concept.get_hint(BuiltinConcepts.RECOGNIZED_BY) == RECOGNIZED_BY_NAME:
self.text += f"isinstance({as_name}, Concept) and {as_name}.name == '{concept.name}'"
elif concept.get_hint(BuiltinConcepts.RECOGNIZED_BY) == RECOGNIZED_BY_ID:
self.text += f"isinstance({as_name}, Concept) and {as_name}.id == '{concept.id}'"
else:
self.text += f"isinstance({as_name}, Concept) and {as_name}.key == '{concept.key}'"
if len(concept.get_metadata().variables) > 0:
# add variables constraints
evaluated = ensure_evaluated(self.context, concept, eval_body=False, metadata=["variables"])
if not self.context.sheerka.is_success(evaluated) and evaluated.key != concept.key:
raise EmitPythonCodeException(evaluated)
for k, v in concept.variables().items():
self.recognize(v, f"{as_name}.get_value('{k}')", root=False)
return self
def add_variable(self, target):
var_name = f"__x_{self.var_counter:02}__"
self.var_counter += 1
self.variables.append((var_name, target))
return var_name
def get_text(self):
if self.variables:
variables_as_str = '\n'.join([f"{k} = {v}" for k, v in self.variables])
return variables_as_str + "\n" + self.text
return self.text
class NoConditionFound(ErrorObj):
def __eq__(self, other):
return isinstance(other, NoConditionFound)
@@ -1149,7 +577,6 @@ class GetConditionExprVisitor(ExpressionVisitor):
def evaluate_from_source(self, source, is_question=False, return_body=False):
res = evaluate_from_source(self.context,
source,
evaluators=CONDITIONS_VISITOR_EVALUATORS,
desc=None,
eval_body=not is_question,
eval_where=False,
@@ -1160,6 +587,9 @@ class GetConditionExprVisitor(ExpressionVisitor):
if return_body:
if not res.status:
python_eval_error = self.context.sheerka.get_errors(self.context, res, __type="PythonEvalError")
if python_eval_error and isinstance(python_eval_error[0].error, NameError):
raise UnknownVariableError(python_eval_error[0].source)
raise FailedToCompileError(res.body)
return res.body
@@ -1268,7 +698,11 @@ class ReteConditionExprVisitor(GetConditionExprVisitor):
def visit_ComparisonNode(self, expr_node: ComparisonNode):
if isinstance(expr_node.left, VariableNode):
conditions = []
value = self.evaluate_from_source(expr_node.right.get_source(), return_body=True)
try:
value = self.evaluate_from_source(expr_node.right.get_source(), return_body=True)
except UnknownVariableError:
value = expr_node.right.unpack()
self.add_to_condition(expr_node.left.unpack(), value, conditions)
return conditions
else:
@@ -1334,7 +768,6 @@ class ReteConditionExprVisitor(GetConditionExprVisitor):
def visit_NameExprNode(self, expr_node: NameExprNode):
res = evaluate_from_source(self.context,
expr_node.get_source(),
evaluators=CONDITIONS_VISITOR_EVALUATORS,
desc=None,
eval_body=True,
eval_where=False,
@@ -1367,9 +800,9 @@ class ReteConditionExprVisitor(GetConditionExprVisitor):
variable = self.init_or_get_variable_from_attr(variable_path, conditions)
conditions.append(Condition(variable, "__is_concept__", True))
if concept.get_hint(BuiltinConcepts.RECOGNIZED_BY) == RECOGNIZED_BY_NAME:
if concept.get_hints().recognized_by == RECOGNIZED_BY_NAME:
conditions.append(Condition(variable, "name", concept.name))
elif concept.get_hint(BuiltinConcepts.RECOGNIZED_BY) == RECOGNIZED_BY_ID:
elif concept.get_hints().recognized_by == RECOGNIZED_BY_ID:
conditions.append(Condition(variable, "id", concept.id))
else:
conditions.append(Condition(variable, "key", concept.key))
@@ -1392,6 +825,9 @@ class ReteConditionExprVisitor(GetConditionExprVisitor):
elif isinstance(value, Concept):
res = self.recognize_concept(var_path, value, {})
conditions.extend(res)
elif isinstance(value, list):
var_root, var_attr = self.init_or_get_variable_from_name(value, conditions)
conditions.append(Condition(left, attr, var_root))
else:
conditions.append(Condition(left, attr, value))
@@ -1483,12 +919,25 @@ class PythonConditionExprVisitorObj:
node.variables,
node.not_variables)
@staticmethod
def create_condition(text, op, left, right):
def get_source(a, b):
if op == ComparisonType.EQUALS and b == "sheerka":
return f"is_sheerka({a})"
else:
return ComparisonNode.rebuild_source(a, op, b)
return PythonConditionExprVisitorObj(text,
get_source(left.source, right.source),
merge_dictionaries(left.objects, right.objects),
merge_sets(left.variables, right.variables),
merge_sets(left.not_variables, right.not_variables))
class PythonConditionExprVisitor(GetConditionExprVisitor):
def __init__(self, context):
super().__init__(context)
self.know_object_variables = {}
self.check_variable_existence_only = True
self.concepts_to_reset = set()
@@ -1537,7 +986,8 @@ class PythonConditionExprVisitor(GetConditionExprVisitor):
return var_name
def unpack_variable(self, variable_path: List[str], obj_variables):
obj_variables.add(variable_path[0])
if self.is_a_possible_variable(variable_path[0]):
obj_variables.add(variable_path[0])
return self.inner_unpack_variable(variable_path)
@staticmethod
@@ -1551,49 +1001,49 @@ class PythonConditionExprVisitor(GetConditionExprVisitor):
# try to recognize a concept
res = self.evaluate_from_source(expr_node.name, is_question=True)
if res.status and isinstance(res.value, Concept):
if self.context.possible_variables and res.value.name in self.context.possible_variables:
variable_name = expr_node.get_source()
return PythonConditionExprVisitorObj(variable_name, variable_name, {}, {variable_name}, set())
# else / otherwise
self.check_variable_existence_only = False
if is_a_question(self.context, res.value):
return self.evaluate_concept_as_question(expr_node.name, res.value)
else:
return self.evaluate_concept(expr_node.name, res.value)
return self.manage_concept(expr_node.get_source(), res.value)
else:
if self.context.sheerka.has_error(self.context, res, __type=BuiltinConcepts.TOO_MANY_SUCCESS):
raise FailedToCompileError([res])
variable_name = expr_node.get_source()
variables = {variable_name} if not res.status else set()
if res.status:
variables = set()
self.check_variable_existence_only = False
else:
variables = {variable_name}
return PythonConditionExprVisitorObj(variable_name, variable_name, {}, variables, set())
variable_name = expr_node.get_source()
variables_detected = {variable_name} if self.is_a_possible_variable(variable_name) else set()
return PythonConditionExprVisitorObj(variable_name, variable_name, {}, variables_detected, set())
if self.check_variable_existence_only:
# special case where we want to check the existence of the whole string
variable_name = expr_node.get_source()
possible_variables = {variable_name} if self.is_a_possible_variable(variable_name) else set()
else:
# try to detect the variable
possible_variables = set()
var_root, var_attr = self.unpack_variable(expr_node.unpack(), possible_variables)
variable_name = self.construct_variable(var_root, var_attr)
return PythonConditionExprVisitorObj(variable_name, variable_name, {}, possible_variables, set())
def visit_ComparisonNode(self, expr_node: ComparisonNode):
self.check_variable_existence_only = False
if not isinstance(expr_node.left, VariableNode):
# KSI 2021-04-22. Not quite sure of the reason why I have this piece of code
left = self.visit(expr_node.left)
source = expr_node.get_source()
return PythonConditionExprVisitorObj(source, source, {}, left.variables, left.not_variables)
left = self.visit(expr_node.left)
right = self.visit(expr_node.right)
if right.source in right.objects:
return self.create_comparison_condition(expr_node.left.unpack(),
expr_node.comp,
right.source,
right.objects[right.source],
right.objects,
expr_node.get_source())
# special case when we call recognize concept when an equality with a concept is found
right_value = right.objects[right.source] if right.source in right.objects else None
if expr_node.comp == ComparisonType.EQUALS and isinstance(right_value, Concept):
res = self.recognize_concept(expr_node.left.unpack(), right_value, {}, expr_node.get_source())
else:
value = self.evaluate_from_source(expr_node.right.get_source(), return_body=True)
object_name, objects = self.get_object_name(value)
return self.create_comparison_condition(expr_node.left.unpack(),
expr_node.comp,
object_name,
value,
objects,
expr_node.get_source())
res = PythonConditionExprVisitorObj.create_condition(expr_node.get_source(), expr_node.comp, left, right)
return res
def visit_AndNode(self, expr_node: AndNode):
current_visitor_obj = self.visit(expr_node.parts[0])
@@ -1636,13 +1086,43 @@ class PythonConditionExprVisitor(GetConditionExprVisitor):
def visit_NameExprNode(self, expr_node: NameExprNode):
self.check_variable_existence_only = False
source = expr_node.get_source()
res = self.evaluate_from_source(source, is_question=False)
if res.status:
obj_name, objects = self.get_object_name(res.value)
return PythonConditionExprVisitorObj(source, obj_name, objects, set(), set())
else:
res = self.context.sheerka.parse_unrecognized(self.context,
source,
INIT_AST_PARSERS,
filter_func=only_successful,
is_question=True)
if not res.status:
raise FailedToCompileError([expr_node])
return_values = res.body.body if is_only_successful(self.context.sheerka, res) else [res]
# if the result is a concept, create an object for it, otherwise just leave the source as it is
res = evaluate_return_values(self.context, source, return_values, is_question=True)
if res.status:
if isinstance(res.value, Concept):
return self.manage_concept(expr_node.get_source(), res.value)
else:
obj_name, objects = self.get_object_name(res.value)
return PythonConditionExprVisitorObj(source, obj_name, objects, set(), set())
else:
if len(return_values) == 1:
body = return_values[0].body.body
if hasattr(body, "get_python_node"):
python_node = return_values[0].body.body.get_python_node()
unreferenced_names_visitor = UnreferencedNamesVisitor(self.context)
variables = unreferenced_names_visitor.get_names(python_node.ast_)
variables = variables - set(python_node.objects.keys())
return PythonConditionExprVisitorObj(python_node.original_source,
python_node.source,
python_node.objects,
variables,
set())
elif isinstance(body, Concept):
return self.manage_concept(expr_node.get_source(), body)
else:
raise FailedToCompileError([expr_node])
def recognize_concept(self, variable_path, concept_to_recognize, concept_variables: dict, original_source=None):
if not isinstance(concept_to_recognize, Concept):
concept_as_str = concept_to_recognize.get_source()
@@ -1659,9 +1139,9 @@ class PythonConditionExprVisitor(GetConditionExprVisitor):
source = f"isinstance({var_name}, Concept)"
if concept.get_hint(BuiltinConcepts.RECOGNIZED_BY) == RECOGNIZED_BY_NAME:
if concept.get_hints().recognized_by == RECOGNIZED_BY_NAME:
source += f" and {var_name}.name == '{concept.name}'"
elif concept.get_hint(BuiltinConcepts.RECOGNIZED_BY) == RECOGNIZED_BY_ID:
elif concept.get_hints().recognized_by == RECOGNIZED_BY_ID:
source += f" and {var_name}.id == '{concept.id}'"
else:
source += f" and {var_name}.key == '{concept.key}'"
@@ -1676,17 +1156,24 @@ class PythonConditionExprVisitor(GetConditionExprVisitor):
ComparisonType.EQUALS,
obj_name,
var_value,
objects)
objects,
set())
source += " and " + variable_condition.source
text += " and " + variable_condition.text
return PythonConditionExprVisitorObj(original_source or text, source, objects, obj_variables, set())
def manage_concept(self, source, concept):
if is_a_question(self.context, concept):
return self.evaluate_concept_as_question(source, concept)
else:
return self.evaluate_concept(source, concept)
def evaluate_concept_as_question(self, original_text, concept):
concept_var_name, objects = self.get_object_name(concept)
source = f"evaluate_question({concept_var_name})"
variables = get_possible_variables_from_concept(self.context, concept)
self.concepts_to_reset.add(concept)
self.concepts_to_reset.update(self.get_concepts_to_reset(concept))
return PythonConditionExprVisitorObj(original_text, source, objects, variables, set())
def evaluate_concept(self, original_text, concept):
@@ -1694,13 +1181,62 @@ class PythonConditionExprVisitor(GetConditionExprVisitor):
source, objects = self.get_object_name(concept)
return PythonConditionExprVisitorObj(original_text, source, objects, set(), set())
def create_comparison_condition(self, left_path, op, right_name, right_value, objects, original_source=None):
possible_variables = set()
def get_concepts_to_reset(self, concept):
"""
Returns all the concept that might be reset before a second evaluation
The algo is empirical, there must be a theory to make sure that no concept is missed
:param concept:
:return:
"""
res = set()
def _inner_get_concept_to_reset(_concept):
if _concept in res: # prevent circular references
return
res.add(_concept)
assert not _concept.get_hints().use_copy
for part_name, asts in _concept.get_compiled().items():
if isinstance(asts, Concept):
if is_a_question(self.context, asts):
res.update(self.get_concepts_to_reset(asts))
else:
for ret_val in asts:
body_as_list = ret_val.body.body # go through the ParserResult
if not isinstance(body_as_list, list):
body_as_list = [body_as_list]
for body in body_as_list:
if hasattr(body, "get_concept"): # to manage Concept and ConceptNode
c = body.get_concept()
if is_a_question(self.context, c):
res.update(self.get_concepts_to_reset(c))
elif hasattr(body, "get_python_node"): # to manage PythonNode and SourceCodeNode like
python_node = body.get_python_node()
for obj in python_node.objects.values():
if isinstance(obj, Concept) and is_a_question(self.context, obj):
res.update(self.get_concepts_to_reset(obj))
_inner_get_concept_to_reset(concept)
return res
def create_comparison_condition(self,
left_path,
op,
right_name,
right_value,
objects,
variables,
original_source=None):
possible_variables = variables.copy()
var_root, var_attr = self.unpack_variable(left_path, possible_variables)
left = self.construct_variable(var_root, var_attr)
if original_source is None:
right = get_safe_str_value(right_value)
right = get_safe_str_value(right_value or right_name)
original_source = ComparisonNode.rebuild_source(left, op, right)
if op == ComparisonType.EQUALS:
src/core/sheerka/services/sheerka_service.py
+11
View File
@@ -1,5 +1,6 @@
from dataclasses import dataclass
from core.concept import Concept
from core.global_symbols import NotFound, ErrorObj
from core.utils import sheerka_deepcopy
@@ -52,3 +53,13 @@ class BaseService:
@dataclass()
class FailedToCompileError(Exception, ErrorObj):
cause: list
@dataclass()
class UnknownVariableError(Exception, ErrorObj):
variable: str
@dataclass
class ChickenAndEggException(Exception, ErrorObj):
error: Concept