kodjo/Sheerka-Old
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

I can manage infinite recursion when building concept

Browse Source
This commit is contained in:
Kodjo Sossouvi
2020-12-10 21:08:10 +01:00
parent 4b6e1dd55b
commit 657c7536f7
28 changed files with 816 additions and 446 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/core/builtin_concepts.py
+1 -192
View File
@@ -1,199 +1,8 @@
from core.builtin_concepts_ids import BuiltinConcepts
from core.concept import Concept, ConceptParts
from core.error import ErrorObj
class BuiltinConcepts:
"""
List of builtin concepts that do no need any specific implementation
Please note that the value of the enum is informal. It is not used in the system
For example, the concept 'NODE' DOES NOT have the key, the id or whatever 200
The key if the name of the concept
The id is a sequential number given just before the concept is saved in sdp
The values of the enum is not used the code
"""
SHEERKA = "__SHEERKA"
# processing instructions during sheerka.execute() or sheerka.evaluate_concept()
# The instructions may alter how the actions work
DEBUG = "__DEBUG" # activate all debug information
EVAL_BODY_REQUESTED = "__EVAL_BODY_REQUESTED" # to evaluate the body
EVAL_WHERE_REQUESTED = "__EVAL_WHERE_REQUESTED" # to evaluate the where clause
RETURN_BODY_REQUESTED = "__RETURN_BODY_REQUESTED" # returns the body of the concept instead of the concept itself
REDUCE_REQUESTED = "__REDUCE_REQUESTED" # remove meaningless error when possible
EVAL_UNTIL_SUCCESS_REQUESTED = "__EVAL_UNTIL_SUCCESS_REQUESTED" # PythonEvaluator tries combination until True is found
EVAL_QUESTION_REQUESTED = "__EVAL_QUESTION_REQUESTED" # the user input must be treated as question
# possible actions during sheerka.execute() or sheerka.evaluate_rules()
INIT_SHEERKA = "__INIT_SHEERKA" #
PROCESS_INPUT = "__PROCESS_INPUT" # Processing user input or other input
PROCESSING = "__PROCESSING" # Processing user input or other input
BEFORE_PARSING = "__BEFORE_PARSING" # activated before evaluation by the parsers
PARSING = "__PARSING" # activated during the parsing. It contains the text to parse
AFTER_PARSING = "__AFTER_PARSING" # after parsing
BEFORE_EVALUATION = "__BEFORE_EVALUATION" # before evaluation
EVALUATION = "__EVALUATION" # activated when the parsing process seems to be finished
AFTER_EVALUATION = "__AFTER_EVALUATION" # activated when the parsing process seems to be finished
BEFORE_RENDERING = "__BEFORE_RENDERING" # activate before the output is rendered
RENDERING = "__RENDERING" # rendering the response from sheerka
AFTER_RENDERING = "__AFTER_RENDERING" # rendering the response from sheerka
EVALUATE_SOURCE = "__EVALUATE_SOURCE" #
EVALUATE_CONCEPT = "__EVALUATE_CONCEPT" # a concept will be evaluated
EVALUATING_CONCEPT = "__EVALUATING_CONCEPT" # a concept will be evaluated
EVALUATING_ATTRIBUTE = "__EVALUATING_ATTRIBUTE" #
VALIDATE_CONCEPT = "__VALIDATE_CONCEPT"
VALIDATING_CONCEPT = "__VALIDATING_CONCEPT"
INIT_COMPILED = "__INIT_COMPILED"
INIT_BNF = "__INIT_BNF"
MANAGE_INFINITE_RECURSION = "__MANAGE_INFINITE_RECURSION"
PARSE_CODE = "__PARSE_CODE"
EXEC_CODE = "__EXEC_CODE" # to use when executing Python or other language compiled code
TESTING = "__TESTING"
EVALUATOR_PRE_PROCESS = "__EVALUATOR_PRE_PROCESS" # used modify / tweak behaviour of evaluators
EVALUATING_RULES = "__EVALUATING_RULES"
# builtin attributes
ISA = "__ISA" # when a concept is an instance of another one
HASA = "__HASA" # when a concept has/owns another concept
AUTO_EVAL = "__AUTO_EVAL" # when the concept must be auto evaluated
# object
USER_INPUT = "__USER_INPUT" # represent an input from an user
SUCCESS = "__SUCCESS"
ERROR = "__ERROR"
UNKNOWN_CONCEPT = "__UNKNOWN_CONCEPT" # the request concept is not recognized
CANNOT_RESOLVE_CONCEPT = "__CANNOT_RESOLVE_CONCEPT" # when too many concepts with the same name
RETURN_VALUE = "__RETURN_VALUE" # a value is returned
CONCEPT_TOO_LONG = "__CONCEPT_TOO_LONG" # concept cannot be processed by exactConcept parser
NEW_CONCEPT = "__NEW_CONCEPT" # when a new concept is added
UNKNOWN_PROPERTY = "__UNKNOWN_PROPERTY" # when requesting for a unknown property
PARSER_RESULT = "__PARSER_RESULT"
TOO_MANY_SUCCESS = "__TOO_MANY_SUCCESS" # when expecting a limited number of successful return value
TOO_MANY_ERRORS = "__TOO_MANY_ERRORS" # when expecting a limited number of successful return value
ONLY_SUCCESSFUL = "__ONLY_SUCCESSFUL" # filter the result, only keep successful ones
MULTIPLE_ERRORS = "__MULTIPLE_ERRORS" # filter the result, only keep evaluator in error
NOT_FOR_ME = "__NOT_FOR_ME" # a parser recognize that the entry is not meant for it
IS_EMPTY = "__IS_EMPTY" # when a set is empty
NO_RESULT = "__NO_RESULT" # no return value returned
INVALID_RETURN_VALUE = "__INVALID_RETURN_VALUE" # the return value of an evaluator is not correct
CONCEPT_ALREADY_DEFINED = "__CONCEPT_ALREADY_DEFINED" # when you try to add the same object twice (a concept or whatever)
PROPERTY_ALREADY_DEFINED = "__PROPERTY_ALREADY_DEFINED" # When you try to add the same element in a property
NOP = "__NOP" # no operation concept. Does nothing
CONCEPT_EVAL_ERROR = "__CONCEPT_EVAL_ERROR" # cannot evaluate a property or metadata of a concept
ENUMERATION = "__ENUMERATION" # represents a list or a set
LIST = "__LIST" # represents a list
FILTERED = "__FILTERED" # represents the result of a filtering
CONCEPT_ALREADY_IN_SET = "__CONCEPT_ALREADY_IN_SET"
NOT_A_SET = "__NOT_A_SET" # the concept has no entry in sets
CONDITION_FAILED = "__CONDITION_FAILED" # failed to validate where clause during evaluation
CHICKEN_AND_EGG = "__CHICKEN_AND_EGG" # infinite recursion when declaring concept
EXPLANATION = "__EXPLANATION"
PRECEDENCE = "__PRECEDENCE" # use to set priority among concepts when parsing
ASSOCIATIVITY = "__ASSOCIATIVITY" # use to set priority among concepts when parsing
NOT_FOUND = "__NOT_FOUND" # when the wanted resource is not found
FORMAT_INSTRUCTIONS = "__FORMAT_INSTRUCTIONS" # to express how to print the concept
NOT_IMPLEMENTED = "__NOT_IMPLEMENTED" # instead of raise an error
PYTHON_SECURITY_ERROR = "__PYTHON_SECURITY_ERROR" # when trying to execute statement when only expression is allowed
INVALID_LESSER_OPERATION = "__INVALID_LESSER_OPERATION"
INVALID_GREATEST_OPERATION = "__INVALID_GREATEST_OPERATION"
NEW_RULE = "__NEW_RULE"
UNKNOWN_RULE = "__UNKNOWN_RULE"
NODE = "__NODE"
GENERIC_NODE = "__GENERIC_NODE"
IDENTIFIER_NODE = "__IDENTIFIER_NODE"
# formatting
TO_LIST = "__TO_LIST"
TO_DICT = "__TO_DICT"
TO_MULTI = "__TO_MULTI" # used when there are multiple output to display
AllBuiltinConcepts = [v for n, v in BuiltinConcepts.__dict__.items() if not n.startswith("__")]
BuiltinUnique = [
BuiltinConcepts.EVAL_BODY_REQUESTED,
BuiltinConcepts.EVAL_WHERE_REQUESTED,
BuiltinConcepts.RETURN_BODY_REQUESTED,
BuiltinConcepts.REDUCE_REQUESTED,
BuiltinConcepts.EVAL_UNTIL_SUCCESS_REQUESTED,
BuiltinConcepts.EVAL_QUESTION_REQUESTED,
BuiltinConcepts.INIT_SHEERKA,
BuiltinConcepts.PROCESS_INPUT,
BuiltinConcepts.PROCESSING,
BuiltinConcepts.BEFORE_PARSING,
BuiltinConcepts.PARSING,
BuiltinConcepts.AFTER_PARSING,
BuiltinConcepts.BEFORE_EVALUATION,
BuiltinConcepts.EVALUATION,
BuiltinConcepts.AFTER_EVALUATION,
BuiltinConcepts.BEFORE_RENDERING,
BuiltinConcepts.RENDERING,
BuiltinConcepts.AFTER_RENDERING,
BuiltinConcepts.EVALUATE_CONCEPT,
BuiltinConcepts.EVALUATING_CONCEPT,
BuiltinConcepts.EVALUATING_ATTRIBUTE,
BuiltinConcepts.VALIDATE_CONCEPT,
BuiltinConcepts.VALIDATING_CONCEPT,
BuiltinConcepts.INIT_COMPILED,
BuiltinConcepts.INIT_BNF,
BuiltinConcepts.MANAGE_INFINITE_RECURSION,
BuiltinConcepts.PARSE_CODE,
BuiltinConcepts.EXEC_CODE,
BuiltinConcepts.TESTING,
BuiltinConcepts.ISA,
BuiltinConcepts.AUTO_EVAL,
BuiltinConcepts.INVALID_LESSER_OPERATION,
BuiltinConcepts.INVALID_GREATEST_OPERATION,
]
BuiltinErrors = [
BuiltinConcepts.ERROR,
BuiltinConcepts.UNKNOWN_CONCEPT,
BuiltinConcepts.CANNOT_RESOLVE_CONCEPT,
BuiltinConcepts.CONCEPT_TOO_LONG,
BuiltinConcepts.UNKNOWN_PROPERTY,
BuiltinConcepts.TOO_MANY_SUCCESS,
BuiltinConcepts.TOO_MANY_ERRORS,
BuiltinConcepts.MULTIPLE_ERRORS,
BuiltinConcepts.INVALID_RETURN_VALUE,
BuiltinConcepts.CONCEPT_ALREADY_DEFINED,
BuiltinConcepts.PROPERTY_ALREADY_DEFINED,
BuiltinConcepts.CONCEPT_EVAL_ERROR,
BuiltinConcepts.CONCEPT_ALREADY_IN_SET,
BuiltinConcepts.NOT_A_SET,
BuiltinConcepts.CONDITION_FAILED,
BuiltinConcepts.CHICKEN_AND_EGG,
BuiltinConcepts.NOT_FOUND,
BuiltinConcepts.INVALID_LESSER_OPERATION,
BuiltinConcepts.INVALID_GREATEST_OPERATION,
]
BuiltinContainers = [
BuiltinConcepts.PARSER_RESULT,
BuiltinConcepts.ONLY_SUCCESSFUL,
BuiltinConcepts.FILTERED,
BuiltinConcepts.EXPLANATION,
BuiltinConcepts.TO_LIST,
BuiltinConcepts.TO_DICT,
BuiltinConcepts.TO_MULTI,
]
BuiltinOutConcepts = [
BuiltinConcepts.TO_LIST,
BuiltinConcepts.TO_DICT,
BuiltinConcepts.TO_MULTI,
]
"""
Some concepts have a specific implementation
It's mainly to ease the usage
"""
class UserInputConcept(Concept):
ALL_ATTRIBUTES = ["text", "user_name"]
src/core/builtin_concepts_ids.py
+191
View File
@@ -0,0 +1,191 @@
class BuiltinConcepts:
"""
List of builtin concepts that do no need any specific implementation
Please note that the value of the enum is informal. It is not used in the system
For example, the concept 'NODE' DOES NOT have the key, the id or whatever 200
The key if the name of the concept
The id is a sequential number given just before the concept is saved in sdp
The values of the enum is not used the code
"""
SHEERKA = "__SHEERKA"
# processing instructions during sheerka.execute() or sheerka.evaluate_concept()
# The instructions may alter how the actions work
DEBUG = "__DEBUG" # activate all debug information
EVAL_BODY_REQUESTED = "__EVAL_BODY_REQUESTED" # to evaluate the body
EVAL_WHERE_REQUESTED = "__EVAL_WHERE_REQUESTED" # to evaluate the where clause
RETURN_BODY_REQUESTED = "__RETURN_BODY_REQUESTED" # returns the body of the concept instead of the concept itself
REDUCE_REQUESTED = "__REDUCE_REQUESTED" # remove meaningless error when possible
EVAL_UNTIL_SUCCESS_REQUESTED = "__EVAL_UNTIL_SUCCESS_REQUESTED" # PythonEvaluator tries combination until True is found
EVAL_QUESTION_REQUESTED = "__EVAL_QUESTION_REQUESTED" # the user input must be treated as question
# possible actions during sheerka.execute() or sheerka.evaluate_rules()
INIT_SHEERKA = "__INIT_SHEERKA" #
PROCESS_INPUT = "__PROCESS_INPUT" # Processing user input or other input
PROCESSING = "__PROCESSING" # Processing user input or other input
BEFORE_PARSING = "__BEFORE_PARSING" # activated before evaluation by the parsers
PARSING = "__PARSING" # activated during the parsing. It contains the text to parse
AFTER_PARSING = "__AFTER_PARSING" # after parsing
BEFORE_EVALUATION = "__BEFORE_EVALUATION" # before evaluation
EVALUATION = "__EVALUATION" # activated when the parsing process seems to be finished
AFTER_EVALUATION = "__AFTER_EVALUATION" # activated when the parsing process seems to be finished
BEFORE_RENDERING = "__BEFORE_RENDERING" # activate before the output is rendered
RENDERING = "__RENDERING" # rendering the response from sheerka
AFTER_RENDERING = "__AFTER_RENDERING" # rendering the response from sheerka
EVALUATE_SOURCE = "__EVALUATE_SOURCE" #
EVALUATE_CONCEPT = "__EVALUATE_CONCEPT" # a concept will be evaluated
EVALUATING_CONCEPT = "__EVALUATING_CONCEPT" # a concept will be evaluated
EVALUATING_ATTRIBUTE = "__EVALUATING_ATTRIBUTE" #
VALIDATE_CONCEPT = "__VALIDATE_CONCEPT"
VALIDATING_CONCEPT = "__VALIDATING_CONCEPT"
INIT_COMPILED = "__INIT_COMPILED"
INIT_BNF = "__INIT_BNF"
MANAGE_INFINITE_RECURSION = "__MANAGE_INFINITE_RECURSION"
PARSE_CODE = "__PARSE_CODE"
EXEC_CODE = "__EXEC_CODE" # to use when executing Python or other language compiled code
TESTING = "__TESTING"
EVALUATOR_PRE_PROCESS = "__EVALUATOR_PRE_PROCESS" # used modify / tweak behaviour of evaluators
EVALUATING_RULES = "__EVALUATING_RULES"
# builtin attributes
ISA = "__ISA" # when a concept is an instance of another one
HASA = "__HASA" # when a concept has/owns another concept
AUTO_EVAL = "__AUTO_EVAL" # when the concept must be auto evaluated
# object
USER_INPUT = "__USER_INPUT" # represent an input from an user
SUCCESS = "__SUCCESS"
ERROR = "__ERROR"
UNKNOWN_CONCEPT = "__UNKNOWN_CONCEPT" # the request concept is not recognized
CANNOT_RESOLVE_CONCEPT = "__CANNOT_RESOLVE_CONCEPT" # when too many concepts with the same name
RETURN_VALUE = "__RETURN_VALUE" # a value is returned
CONCEPT_TOO_LONG = "__CONCEPT_TOO_LONG" # concept cannot be processed by exactConcept parser
NEW_CONCEPT = "__NEW_CONCEPT" # when a new concept is added
UNKNOWN_PROPERTY = "__UNKNOWN_PROPERTY" # when requesting for a unknown property
PARSER_RESULT = "__PARSER_RESULT"
TOO_MANY_SUCCESS = "__TOO_MANY_SUCCESS" # when expecting a limited number of successful return value
TOO_MANY_ERRORS = "__TOO_MANY_ERRORS" # when expecting a limited number of successful return value
ONLY_SUCCESSFUL = "__ONLY_SUCCESSFUL" # filter the result, only keep successful ones
MULTIPLE_ERRORS = "__MULTIPLE_ERRORS" # filter the result, only keep evaluator in error
NOT_FOR_ME = "__NOT_FOR_ME" # a parser recognize that the entry is not meant for it
IS_EMPTY = "__IS_EMPTY" # when a set is empty
NO_RESULT = "__NO_RESULT" # no return value returned
INVALID_RETURN_VALUE = "__INVALID_RETURN_VALUE" # the return value of an evaluator is not correct
CONCEPT_ALREADY_DEFINED = "__CONCEPT_ALREADY_DEFINED" # when you try to add the same object twice (a concept or whatever)
PROPERTY_ALREADY_DEFINED = "__PROPERTY_ALREADY_DEFINED" # When you try to add the same element in a property
NOP = "__NOP" # no operation concept. Does nothing
CONCEPT_EVAL_ERROR = "__CONCEPT_EVAL_ERROR" # cannot evaluate a property or metadata of a concept
ENUMERATION = "__ENUMERATION" # represents a list or a set
LIST = "__LIST" # represents a list
FILTERED = "__FILTERED" # represents the result of a filtering
CONCEPT_ALREADY_IN_SET = "__CONCEPT_ALREADY_IN_SET"
NOT_A_SET = "__NOT_A_SET" # the concept has no entry in sets
CONDITION_FAILED = "__CONDITION_FAILED" # failed to validate where clause during evaluation
CHICKEN_AND_EGG = "__CHICKEN_AND_EGG" # infinite recursion when declaring concept
EXPLANATION = "__EXPLANATION"
PRECEDENCE = "__PRECEDENCE" # use to set priority among concepts when parsing
ASSOCIATIVITY = "__ASSOCIATIVITY" # use to set priority among concepts when parsing
NOT_FOUND = "__NOT_FOUND" # when the wanted resource is not found
FORMAT_INSTRUCTIONS = "__FORMAT_INSTRUCTIONS" # to express how to print the concept
NOT_IMPLEMENTED = "__NOT_IMPLEMENTED" # instead of raise an error
PYTHON_SECURITY_ERROR = "__PYTHON_SECURITY_ERROR" # when trying to execute statement when only expression is allowed
INVALID_LESSER_OPERATION = "__INVALID_LESSER_OPERATION"
INVALID_GREATEST_OPERATION = "__INVALID_GREATEST_OPERATION"
NEW_RULE = "__NEW_RULE"
UNKNOWN_RULE = "__UNKNOWN_RULE"
NODE = "__NODE"
GENERIC_NODE = "__GENERIC_NODE"
IDENTIFIER_NODE = "__IDENTIFIER_NODE"
# formatting
TO_LIST = "__TO_LIST"
TO_DICT = "__TO_DICT"
TO_MULTI = "__TO_MULTI" # used when there are multiple output to display
AllBuiltinConcepts = [v for n, v in BuiltinConcepts.__dict__.items() if not n.startswith("__")]
BuiltinUnique = [
BuiltinConcepts.EVAL_BODY_REQUESTED,
BuiltinConcepts.EVAL_WHERE_REQUESTED,
BuiltinConcepts.RETURN_BODY_REQUESTED,
BuiltinConcepts.REDUCE_REQUESTED,
BuiltinConcepts.EVAL_UNTIL_SUCCESS_REQUESTED,
BuiltinConcepts.EVAL_QUESTION_REQUESTED,
BuiltinConcepts.INIT_SHEERKA,
BuiltinConcepts.PROCESS_INPUT,
BuiltinConcepts.PROCESSING,
BuiltinConcepts.BEFORE_PARSING,
BuiltinConcepts.PARSING,
BuiltinConcepts.AFTER_PARSING,
BuiltinConcepts.BEFORE_EVALUATION,
BuiltinConcepts.EVALUATION,
BuiltinConcepts.AFTER_EVALUATION,
BuiltinConcepts.BEFORE_RENDERING,
BuiltinConcepts.RENDERING,
BuiltinConcepts.AFTER_RENDERING,
BuiltinConcepts.EVALUATE_CONCEPT,
BuiltinConcepts.EVALUATING_CONCEPT,
BuiltinConcepts.EVALUATING_ATTRIBUTE,
BuiltinConcepts.VALIDATE_CONCEPT,
BuiltinConcepts.VALIDATING_CONCEPT,
BuiltinConcepts.INIT_COMPILED,
BuiltinConcepts.INIT_BNF,
BuiltinConcepts.MANAGE_INFINITE_RECURSION,
BuiltinConcepts.PARSE_CODE,
BuiltinConcepts.EXEC_CODE,
BuiltinConcepts.TESTING,
BuiltinConcepts.ISA,
BuiltinConcepts.AUTO_EVAL,
BuiltinConcepts.INVALID_LESSER_OPERATION,
BuiltinConcepts.INVALID_GREATEST_OPERATION,
]
BuiltinErrors = [
BuiltinConcepts.ERROR,
BuiltinConcepts.UNKNOWN_CONCEPT,
BuiltinConcepts.CANNOT_RESOLVE_CONCEPT,
BuiltinConcepts.CONCEPT_TOO_LONG,
BuiltinConcepts.UNKNOWN_PROPERTY,
BuiltinConcepts.TOO_MANY_SUCCESS,
BuiltinConcepts.TOO_MANY_ERRORS,
BuiltinConcepts.MULTIPLE_ERRORS,
BuiltinConcepts.INVALID_RETURN_VALUE,
BuiltinConcepts.CONCEPT_ALREADY_DEFINED,
BuiltinConcepts.PROPERTY_ALREADY_DEFINED,
BuiltinConcepts.CONCEPT_EVAL_ERROR,
BuiltinConcepts.CONCEPT_ALREADY_IN_SET,
BuiltinConcepts.NOT_A_SET,
BuiltinConcepts.CONDITION_FAILED,
BuiltinConcepts.CHICKEN_AND_EGG,
BuiltinConcepts.NOT_FOUND,
BuiltinConcepts.INVALID_LESSER_OPERATION,
BuiltinConcepts.INVALID_GREATEST_OPERATION,
]
BuiltinContainers = [
BuiltinConcepts.PARSER_RESULT,
BuiltinConcepts.ONLY_SUCCESSFUL,
BuiltinConcepts.FILTERED,
BuiltinConcepts.EXPLANATION,
BuiltinConcepts.TO_LIST,
BuiltinConcepts.TO_DICT,
BuiltinConcepts.TO_MULTI,
]
BuiltinOutConcepts = [
BuiltinConcepts.TO_LIST,
BuiltinConcepts.TO_DICT,
BuiltinConcepts.TO_MULTI,
]
# BuiltinDynamicProps means that the attributes of the concept can vary for one instance to another
# So do not put them in cache
BuiltinDynamicAttrs = [
BuiltinConcepts.EVALUATOR_PRE_PROCESS
]
src/core/builtin_helpers.py
+1 -1
View File
@@ -304,7 +304,7 @@ def parse_unrecognized(context, source, parsers, who=None, prop=None, filter_fun
with context.push(BuiltinConcepts.PARSING, action_context, who=who, desc=desc) as sub_context:
# disable all parsers but the requested ones
if parsers != "all":
sub_context.preprocess_parsers = [BaseParser.PREFIX + parser for parser in parsers]
sub_context.preprocess_parsers = parsers
# sub_context.add_preprocess(BaseParser.PREFIX + "*", enabled=False)
# for parser in parsers:
# sub_context.add_preprocess(BaseParser.PREFIX + parser, enabled=True)
src/core/concept.py
+4 -2
View File
@@ -5,6 +5,7 @@ from dataclasses import dataclass
from typing import Union
import core.utils
from core.builtin_concepts_ids import BuiltinDynamicAttrs
from core.tokenizer import Tokenizer, TokenKind
PROPERTIES_FOR_DIGEST = ("name", "key",
@@ -86,13 +87,14 @@ def get_concept_attrs(concept):
pass
all_attributes = [k for k in concept.__dict__ if k[0] != "_" and k[0] != "#"]
if concept.id:
if concept.id and concept.key not in BuiltinDynamicAttrs:
ALL_ATTRIBUTES[concept.id] = all_attributes
return all_attributes
def freeze_concept_attrs(concept):
ALL_ATTRIBUTES[concept.id] = [k for k in concept.__dict__ if k[0] != "_" and k[0] != "#"]
if concept.key not in BuiltinDynamicAttrs:
ALL_ATTRIBUTES[concept.id] = [k for k in concept.__dict__ if k[0] != "_" and k[0] != "#"]
class Concept:
src/core/sheerka/Sheerka.py
+2 -1
View File
@@ -8,7 +8,8 @@ from cache.Cache import Cache
from cache.CacheManager import CacheManager
from cache.DictionaryCache import DictionaryCache
from cache.IncCache import IncCache
from core.builtin_concepts import BuiltinConcepts, ErrorConcept, ReturnValueConcept, BuiltinErrors, UnknownConcept
from core.builtin_concepts import ErrorConcept, ReturnValueConcept, UnknownConcept
from core.builtin_concepts_ids import BuiltinErrors, BuiltinConcepts
from core.concept import Concept, ConceptParts, NotInit, get_concept_attrs
from core.error import ErrorObj
from core.global_symbols import EVENT_USER_INPUT_EVALUATED
src/core/sheerka/services/SheerkaAdmin.py
+1 -1
View File
@@ -2,7 +2,7 @@ import sys
import time
from os import path
from core.builtin_concepts import BuiltinConcepts, BuiltinContainers
from core.builtin_concepts_ids import BuiltinConcepts, BuiltinContainers
from core.builtin_helpers import ensure_concept
from core.concept import Concept
from core.sheerka.services.sheerka_service import BaseService
src/core/sheerka/services/SheerkaConceptManager.py
+2 -1
View File
@@ -5,7 +5,8 @@ from cache.Cache import Cache
from cache.CacheManager import ConceptNotFound
from cache.ListIfNeededCache import ListIfNeededCache
from cache.SetCache import SetCache
from core.builtin_concepts import BuiltinConcepts, ErrorConcept, AllBuiltinConcepts, BuiltinUnique
from core.builtin_concepts import ErrorConcept
from core.builtin_concepts_ids import BuiltinConcepts, AllBuiltinConcepts, BuiltinUnique
from core.builtin_helpers import ensure_concept
from core.concept import Concept, DEFINITION_TYPE_DEF, DEFINITION_TYPE_BNF, freeze_concept_attrs, NotInit, \
ConceptMetadata
src/core/sheerka/services/SheerkaEvaluateConcept.py
+78 -14
View File
@@ -9,6 +9,7 @@ from core.sheerka.services.SheerkaExecute import ParserInput
from core.sheerka.services.sheerka_service import BaseService
from core.tokenizer import Tokenizer
from core.utils import unstr_concept
from parsers.BaseNodeParser import ConceptNode
from parsers.ExpressionParser import ExpressionParser, TrueifyVisitor
CONCEPT_EVALUATION_STEPS = [
@@ -17,6 +18,11 @@ CONCEPT_EVALUATION_STEPS = [
BuiltinConcepts.AFTER_EVALUATION]
@dataclass
class ChickenAndEggException(Exception):
error: Concept
@dataclass
class WhereClauseDef:
concept: Concept # concept on which the where clause is applied
@@ -148,6 +154,27 @@ class SheerkaEvaluateConcept(BaseService):
else:
return None
def get_recursive_definitions(self, concept, return_values):
"""
Returns the name of the parsers that will resolve to a recursive evaluation
:param concept:
:param return_values:
:return:
"""
if concept.name in concept.variables():
# There is a variable with the same name as the concept
# During evaluation, inner variables take precedence other concepts
# So there won't be any cyclic reference, the variable will be picked
return
for parser in [r.body for r in return_values if
r.status and self.sheerka.isinstance(r.body, BuiltinConcepts.PARSER_RESULT)]:
parsed = parser.body if isinstance(parser.body, list) else [parser.body]
for parsed_item in parsed:
if isinstance(parsed_item, Concept) and parsed_item.id == concept.id:
yield parser.parser
elif isinstance(parsed_item, ConceptNode) and parsed_item.concept.id == concept.id:
yield parser.parser
def apply_where_clause(self, context, where_clause_def, return_values):
"""
Apply intermediate where clause when evaluating concept variables
@@ -225,14 +252,57 @@ class SheerkaEvaluateConcept(BaseService):
"""
def is_only_successful(r):
"""
:param r: return_value
:return:
"""
return context.sheerka.isinstance(r, BuiltinConcepts.RETURN_VALUE) and \
context.sheerka.isinstance(r.body, BuiltinConcepts.ONLY_SUCCESSFUL)
def parse_token_concept(s):
"""
:param s: source
:return:
"""
if s.startswith("c:") and (identifier := unstr_concept(s)) != (None, None):
return self.sheerka.fast_resolve(identifier)
return None
def get_return_value(current_context, c, s, p):
"""
:param current_context:
:param c: concept
:param s: source
:param p: part of the concept being parsed
:return:
"""
while True:
return_value = parse_unrecognized(current_context,
s,
parsers="all",
prop=p,
filter_func=only_successful)
if not return_value.status:
if current_context.preprocess:
raise ChickenAndEggException(self.sheerka.new(BuiltinConcepts.CHICKEN_AND_EGG, body={c}))
else:
raise Exception(f"Failed to build '{s}'. But it doesn't seems to be recursion")
return_value = return_value.body.body if is_only_successful(return_value) else [return_value]
recursive_parsers = list(self.get_recursive_definitions(c, return_value))
if len(recursive_parsers) == 0:
return return_value
desc = f"Removing parsers {recursive_parsers}"
current_context = current_context.push(context.action, context.action_context, desc=desc)
for recursive_parser in recursive_parsers:
current_context.add_preprocess(recursive_parser.name, enabled=False)
for part_key in AllConceptParts:
if part_key in concept.get_compiled():
continue
@@ -253,12 +323,7 @@ class SheerkaEvaluateConcept(BaseService):
concept.get_compiled()[part_key] = concept_found
else:
# ...or a list of ReturnValueConcept to resolve
res = parse_unrecognized(context,
source,
parsers="all",
prop=part_key,
filter_func=only_successful)
concept.get_compiled()[part_key] = res.body.body if is_only_successful(res) else res
concept.get_compiled()[part_key] = get_return_value(context, concept, source, part_key)
for var_name, default_value in concept.get_metadata().variables:
if var_name in concept.get_compiled():
@@ -279,12 +344,7 @@ class SheerkaEvaluateConcept(BaseService):
concept.get_compiled()[var_name] = concept_found
else:
# ...or a list of ReturnValueConcept to resolve
res = parse_unrecognized(context,
default_value,
parsers="all",
prop=var_name,
filter_func=only_successful)
concept.get_compiled()[var_name] = res.body.body if is_only_successful(res) else res
concept.get_compiled()[var_name] = get_return_value(context, concept, default_value, var_name)
# Updates the cache of concepts when possible
# This piece of code is not used, a the compile part is removed by sheerka.new_from_template()
@@ -469,7 +529,10 @@ class SheerkaEvaluateConcept(BaseService):
if context.sheerka.isa(concept, context.sheerka.new(BuiltinConcepts.AUTO_EVAL)):
sub_context.protected_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
self.initialize_concept_asts(sub_context, concept)
try:
self.initialize_concept_asts(sub_context, concept)
except ChickenAndEggException as ex:
return ex.error
# to make sure of the order, it don't use ConceptParts.get_parts()
# variables must be evaluated first, body must be evaluated before where
@@ -525,7 +588,8 @@ class SheerkaEvaluateConcept(BaseService):
# validate PRE and WHERE condition
if part_key in (ConceptParts.PRE, ConceptParts.WHERE) and not self.sheerka.objvalue(resolved):
return self.sheerka.new(BuiltinConcepts.CONDITION_FAILED,
body=getattr(concept.get_metadata(), concept_part_value(metadata_to_eval)),
body=getattr(concept.get_metadata(),
concept_part_value(metadata_to_eval)),
concept=concept,
prop=part_key)
src/core/sheerka/services/SheerkaExecute.py
+100 -91
View File
@@ -169,16 +169,34 @@ class SheerkaExecute(BaseService):
self.pi_cache = Cache(default=lambda key: ParserInput(key), max_size=20)
self.instantiated_evaluators = None
self.evaluators_by_name = None
self.grouped_evaluators_cache = {} # key=step, value=tuple(evaluators for this step, sorted priorities)
self.instantiated_parsers = None
self.parsers_by_name = None
self.old_values = []
# cache for all preregistered evaluator combination
# the key is the concatenation of the step and the name of evaluators in the group
# ex : BEFORE_EVALUATION|Python|Sya|Bnf
# The value is a tuple,
# The first entry is the grouped evaluators
# ex : {60 : [PythonEvaluator(), SyaEvaluator()], 50: [BnfEvaluator()]}
# The second entry are the sorted priorities ex [60, 50]
self.grouped_evaluators_cache = {}
# cache for preregistered parsers
# Same construction than the evaluators
# Except 1 : the key does not have a step component. It is simple the list of parsers' names
# Except 2 : we store the type of the parser, not its instance
self.grouped_parsers_cache = {}
def initialize(self):
self.sheerka.bind_service_method(self.execute, True)
self.sheerka.cache_manager.register_cache(self.PARSERS_INPUTS_ENTRY, self.pi_cache, False)
self.reset_evaluators()
self.reset_registered_evaluators()
self.reset_registered_parsers()
def reset_evaluators(self):
def reset_registered_evaluators(self):
# instantiate evaluators, once for all, only keep when it's enabled
self.instantiated_evaluators = [e_class() for e_class in self.sheerka.evaluators]
self.instantiated_evaluators = [e for e in self.instantiated_evaluators if e.enabled]
@@ -186,44 +204,49 @@ class SheerkaExecute(BaseService):
# get default evaluators by process step
for process_step in EVALUATOR_STEPS:
self.grouped_evaluators_cache[f"{process_step}|__default"] = self.get_grouped_evaluators(
self.grouped_evaluators_cache[f"{process_step}|__default"] = self.get_grouped(
[e for e in self.instantiated_evaluators if process_step in e.steps])
# @staticmethod
# def get_grouped_evaluators(instantiated_evaluators, process_step):
# """
# For a given list of evaluators and a given process step
# Computes
# * the evaluators eligible for this step
# * the list of sorted priorities for theses evaluators
# :param instantiated_evaluators:
# :param process_step:
# :return:
# """
# grouped = {}
# for evaluator in [e for e in instantiated_evaluators if e.enabled and process_step in e.steps]:
# grouped.setdefault(evaluator.priority, []).append(evaluator)
#
# sorted_groups = sorted(grouped.keys(), reverse=True)
# return grouped, sorted_groups
def reset_registered_parsers(self):
"""
Browse all parsers and only keep those which are enabled
:return:
"""
self.instantiated_parsers = [parser(sheerka=self.sheerka) for parser in self.sheerka.parsers.values()]
self.instantiated_parsers = [p for p in self.instantiated_parsers if p.enabled]
self.parsers_by_name = {p.short_name: p for p in self.instantiated_parsers}
self.grouped_parsers_cache["__default"] = self.get_grouped(self.instantiated_parsers, use_classes=True)
@staticmethod
def get_grouped_evaluators(evaluators):
def get_grouped(evaluators, use_classes=False):
"""
For a given list of evaluators,
group them by priorities
sort the priorities
:param evaluators:
:param use_classes: if True, store the class (the type) of the evaluator, not its instance
:return: tuple({priority: List of evaluators with this priority}, list of sorted priorities)
"""
grouped = {}
for evaluator in evaluators:
grouped.setdefault(evaluator.priority, []).append(evaluator)
grouped.setdefault(evaluator.priority, []).append(type(evaluator) if use_classes else evaluator)
sorted_groups = sorted(grouped.keys(), reverse=True)
return grouped, sorted_groups
def preprocess(self, items, preprocess_definitions):
"""
Modifies the attributes of item
:param items: either a parser or an evaluator
:param preprocess_definitions: how to modify List of BuiltinConcepts.EVALUATOR_PRE_PROCESS
preprocess.get_value("preprocess_name") gives the name of the property to alter
preprocess.values() gives the alterations
ex:
Sheerka.new(BuiltinConcepts.EVALUATOR_PRE_PROCESS, preprocess_name="parser_name", enabled=False)
Will disable parser 'parser_name'
:return:
"""
for preprocess in preprocess_definitions:
for item in items:
if self.matches(item.name, preprocess.get_value("preprocess_name")):
@@ -234,51 +257,9 @@ class SheerkaExecute(BaseService):
self.old_values.append((item, var_name, getattr(item, var_name)))
setattr(item, var_name, value)
def preprocess_old(self, context, parsers_or_evaluators, mode):
if mode == "parsers":
if not context.preprocess and not context.preprocess_parsers:
return parsers_or_evaluators
items = context.preprocess_parsers
elif mode == "evaluators":
if not context.preprocess and not context.preprocess_evaluators:
return parsers_or_evaluators
items = context.preprocess_evaluators
else:
raise ValueError(mode)
if not hasattr(parsers_or_evaluators, "__iter__"):
single_one = True
parsers_or_evaluators = [parsers_or_evaluators]
else:
single_one = False
if items:
res = []
for item in items:
for e in parsers_or_evaluators:
if item == e.name:
res.append(e)
break
else:
raise ValueError(f"{item} not found.")
parsers_or_evaluators = res
if context.preprocess:
for preprocess in context.preprocess:
for e in parsers_or_evaluators:
if self.matches(e.name, preprocess.get_value("name")):
for var_name in preprocess.values:
if var_name == "name":
continue
if hasattr(e, var_name):
self.old_values.append((e, var_name, getattr(e, var_name)))
setattr(e, var_name, preprocess.get_value(var_name))
return parsers_or_evaluators[0] if single_one else parsers_or_evaluators
def get_evaluators(self, context, process_step):
"""
Returns the list of evaluators to use for a specific test
Returns the list of evaluators to use for a specific context need
:param context:
:param process_step:
:return:
@@ -287,30 +268,64 @@ class SheerkaExecute(BaseService):
if not context.preprocess_evaluators and not context.preprocess:
return self.grouped_evaluators_cache[f"{process_step}|__default"]
# First case, only use a subset of evaluators
if context.preprocess_evaluators and not context.preprocess:
key = str(process_step) + "|" + "|".join(context.preprocess_evaluators)
# Other case, only use a subset of evaluators
selected = context.preprocess_evaluators
if selected and not context.preprocess:
key = str(process_step) + "|" + "|".join(selected)
try:
return self.grouped_evaluators_cache[key]
except KeyError:
evaluators = [self.evaluators_by_name[e] for e in context.preprocess_evaluators]
grouped = self.get_grouped_evaluators(evaluators)
evaluators = [self.evaluators_by_name[e] for e in selected if e in self.evaluators_by_name]
evaluators = [e for e in evaluators if process_step in e.steps] # check the process step
grouped = self.get_grouped(evaluators)
self.grouped_evaluators_cache[key] = grouped
return grouped
# final case, evaluators attributes are modified by the context
# So first, get the modified evaluators
evaluators = [self.evaluators_by_name[e] for e in
context.preprocess_evaluators] if context.preprocess_evaluators else self.instantiated_evaluators
# Final case, evaluators attributes are modified by the context
evaluators = [self.evaluators_by_name[e] for e in selected if
e in self.evaluators_by_name] if selected else self.instantiated_evaluators
evaluators = [e for e in evaluators if process_step in e.steps] # check the process step
self.preprocess(evaluators, context.preprocess)
evaluators = [e for e in evaluators if e.enabled] # make sure they are still enabled
key = str(process_step) + "|" + "|".join([e.name for e in evaluators if e.enabled])
try:
return self.grouped_evaluators_cache[key]
except KeyError:
grouped = self.get_grouped_evaluators(evaluators)
self.grouped_evaluators_cache[key] = grouped
return grouped
return self.get_grouped(evaluators)
def get_parsers(self, context):
"""
We cannot use a single instance of a parser shared among executions as it's common to have a parser
calling another parser or even calling itself
So the cache holds the parser classes or sorted priorities
:param context:
:return:
"""
def get_instances(from_cache):
grouped_instances = {priority: [p(sheerka=self.sheerka) for p in parsers_classes]
for priority, parsers_classes in from_cache[0].items()}
return grouped_instances, from_cache[1]
# Normal case, use all registered parsers
if not context.preprocess_parsers and not context.preprocess:
return get_instances(self.grouped_parsers_cache["__default"])
# Other case, only use a subset of parsers
# This case is heavily used by lexer node parsers, thru parse_unrecognized
if context.preprocess_parsers and not context.preprocess:
key = "|".join(context.preprocess_parsers)
try:
return get_instances(self.grouped_parsers_cache[key])
except KeyError:
parsers = [self.parsers_by_name[p] for p in context.preprocess_parsers if p in self.parsers_by_name]
self.grouped_parsers_cache[key] = self.get_grouped(parsers, use_classes=True)
return get_instances(self.grouped_parsers_cache[key])
# final case, parsers attributes are modified by the context
# This a the case when we want to disable a specific parser, or change the order of priority
parsers = [self.parsers_by_name[p] for p in context.preprocess_parsers if p in self.parsers_by_name] \
if context.preprocess_parsers else self.instantiated_parsers
self.preprocess(parsers, context.preprocess)
parsers = [p for p in parsers if p.enabled] # only keep those that are still enabled
groups, sorted_priorities = self.get_grouped(parsers, use_classes=True)
return get_instances((groups, sorted_priorities))
def get_parser_input(self, text, tokens=None):
"""
@@ -367,14 +382,7 @@ class SheerkaExecute(BaseService):
# keep track of the originals user inputs, as they need to be removed at the end
user_inputs = to_process[:]
# group the parsers by priorities
instantiated_parsers = [parser(sheerka=self.sheerka) for parser in self.sheerka.parsers.values()]
instantiated_parsers = self.preprocess_old(context, instantiated_parsers, "parsers")
grouped_parsers = {}
for parser in [p for p in instantiated_parsers if p.enabled]:
grouped_parsers.setdefault(parser.priority, []).append(parser)
sorted_priorities = sorted(grouped_parsers.keys(), reverse=True)
grouped_parsers, sorted_priorities = self.get_parsers(context)
stop_processing = False
for priority in sorted_priorities:
@@ -427,6 +435,7 @@ class SheerkaExecute(BaseService):
break # Do not try the other priorities if a match is found
result = core.utils.remove_list_from_list(result, user_inputs)
return result
def call_evaluators(self, context, return_values, process_step):
@@ -470,7 +479,7 @@ class SheerkaExecute(BaseService):
# init the evaluator is possible
# KSI. 20201102 : Evaluators are now instantiated at startup,
# Can we move this section into reset_evaluators()
# Can we move this section into reset_registered_evaluators()
if hasattr(evaluator, "init_evaluator") and not evaluator.is_initialized:
evaluator.init_evaluator(sub_context, original_items)
src/evaluators/BaseEvaluator.py
+17 -2
View File
@@ -1,5 +1,4 @@
from core.sheerka.Sheerka import ExecutionContext
from core.sheerka_logger import get_logger
class BaseEvaluator:
@@ -14,7 +13,7 @@ class BaseEvaluator:
# self.init_log = get_logger("init." + self.PREFIX + self.__class__.__name__)
# self.verbose_log = get_logger("verbose." + self.PREFIX + self.__class__.__name__)
self.name = self.PREFIX + name
self.name = BaseEvaluator.get_name(name)
self.short_name = name
self.steps = steps
self.priority = priority
@@ -23,9 +22,25 @@ class BaseEvaluator:
def __repr__(self):
return f"{self.name} ({self.priority})"
def __eq__(self, other):
if not isinstance(other, BaseEvaluator):
return False
return self.name == other.name and \
self.priority == other.priority and \
self.steps == other.steps and \
self.enabled == other.enabled
def __hash__(self):
return hash((self.name, self.priority, self.steps, self.enabled))
def reset(self):
pass
@staticmethod
def get_name(name):
return BaseEvaluator.PREFIX + name
class OneReturnValueEvaluator(BaseEvaluator):
"""
src/evaluators/MultipleOutEvaluator.py
+1 -1
View File
@@ -1,4 +1,4 @@
from core.builtin_concepts import BuiltinConcepts, BuiltinOutConcepts
from core.builtin_concepts_ids import BuiltinConcepts, BuiltinOutConcepts
from evaluators.BaseEvaluator import AllReturnValuesEvaluator
from parsers.BaseParser import BaseParser
src/parsers/BaseNodeParser.py
-8
View File
@@ -824,14 +824,6 @@ class BaseNodeParser(BaseParser):
else:
self.concepts_by_first_keyword = None
# self.token = None
# self.pos = -1
# self.tokens = None
#
# self.context: ExecutionContext = None
# self.text = None
# self.sheerka = None
def init_from_concepts(self, context, concepts, **kwargs):
"""
Initialize the parser with a list of concepts
src/parsers/BaseParser.py
+6 -1
View File
@@ -94,7 +94,8 @@ class BaseParser:
# self.init_log = get_logger("init." + self.PREFIX + self.__class__.__name__)
# self.verbose_log = get_logger("verbose." + self.PREFIX + self.__class__.__name__)
self.name = self.PREFIX + name
self.name = BaseParser.get_name(name)
self.short_name = name
self.priority = priority
self.enabled = enabled
@@ -298,6 +299,10 @@ class BaseParser:
return list_a
@staticmethod
def get_name(name):
return BaseParser.PREFIX + name
class BaseTokenizerIterParser(BaseParser):
src/parsers/BnfDefinitionParser.py
+3 -1
View File
@@ -48,9 +48,11 @@ class BnfDefinitionParser(BaseParser):
return True
def reset_parser(self, context, text):
self.error_sink.clear()
self.context = context
self.sheerka = context.sheerka
self.source = ""
self.lexer_iter = iter(Tokenizer(text.strip())) if isinstance(text, str) else iter(text)
self._current = None
self.after_current = None
src/parsers/ExactConceptParser.py
+6 -7
View File
@@ -1,10 +1,8 @@
import logging
import core.builtin_helpers
from core.builtin_concepts import ReturnValueConcept, BuiltinConcepts
from core.concept import VARIABLE_PREFIX
from core.sheerka.services.SheerkaExecute import ParserInput
from core.tokenizer import TokenKind, LexerError
from core.tokenizer import TokenKind
from core.utils import str_concept
from parsers.BaseParser import BaseParser
@@ -31,12 +29,13 @@ class ExactConceptParser(BaseParser):
context.log(f"Parsing '{parser_input}'", self.name)
sheerka = context.sheerka
try:
if self.reset_parser(context, parser_input):
parser_input.reset()
words = self.get_words(parser_input)
except LexerError as e:
context.log(f"Error found in tokenizer {e}", self.name)
return sheerka.ret(self.name, False, sheerka.new(BuiltinConcepts.ERROR, body=e))
else:
error = self.error_sink[0]
context.log(f"Error found in tokenizer {error}", self.name)
return sheerka.ret(self.name, False, sheerka.new(BuiltinConcepts.ERROR, body=error))
if len(words) > (self.max_word_size or self.MAX_WORDS_SIZE):
context.log(f"Max words reached. Stopping.", self.name)
src/parsers/PythonParser.py
+10 -13
View File
@@ -5,7 +5,7 @@ from dataclasses import dataclass
import core.utils
from core.builtin_concepts import BuiltinConcepts
from core.sheerka.services.SheerkaExecute import ParserInput
from core.tokenizer import LexerError, TokenKind
from core.tokenizer import TokenKind
from parsers.BaseParser import BaseParser, Node, ErrorNode
log = logging.getLogger(__name__)
@@ -101,9 +101,11 @@ class PythonParser(BaseParser):
Parse Python scripts
"""
NAME = "Python"
def __init__(self, **kwargs):
BaseParser.__init__(self, "Python", 50)
BaseParser.__init__(self, PythonParser.NAME, 50)
self.source = kwargs.get("source", "<undef>")
def parse(self, context, parser_input: ParserInput):
@@ -119,9 +121,7 @@ class PythonParser(BaseParser):
TokenKind.RULE: lambda t: core.utils.encode_concept(t.value, "R")
}
try:
parser_input.reset()
if self.reset_parser(context, parser_input):
source_code = parser_input.as_text(python_switcher, tracker)
source_code = source_code.strip()
@@ -134,14 +134,11 @@ class PythonParser(BaseParser):
error_node = PythonErrorNode(parser_input.as_text(), error)
self.error_sink.append(error_node)
except LexerError as e:
self.error_sink.append(e)
# Python parser will refuse input that directly refers to a concept
if isinstance(tree, ast.Expression) and isinstance(tree.body, ast.Name):
if tree.body.id in tracker or context.sheerka.fast_resolve(tree.body.id, return_new=False) is not None:
context.log("It's a simple concept. Not for me.", self.name)
self.error_sink.append(ConceptDetected(tree.body.id))
# Python parser will refuse input that directly refers to a concept
if isinstance(tree, ast.Expression) and isinstance(tree.body, ast.Name):
if tree.body.id in tracker or context.sheerka.fast_resolve(tree.body.id, return_new=False) is not None:
context.log("It's a simple concept. Not for me.", self.name)
self.error_sink.append(ConceptDetected(tree.body.id))
if self.has_error:
ret = sheerka.ret(
src/parsers/PythonWithConceptsParser.py
+1
View File
@@ -34,6 +34,7 @@ class PythonWithConceptsParser(BaseParser):
yield node
def parse(self, context, parser_input):
self.error_sink.clear()
nodes = self.get_input_as_lexer_nodes(parser_input, unrecognized_nodes_parser)
return self.parse_nodes(context, nodes)
src/parsers/RuleParser.py
+38 -40
View File
@@ -1,7 +1,7 @@
from core.builtin_concepts import BuiltinConcepts
from core.rule import Rule, ACTION_TYPE_DEFERRED
from core.sheerka.services.SheerkaExecute import ParserInput
from core.tokenizer import LexerError, TokenKind
from core.tokenizer import TokenKind
from parsers.BaseParser import BaseParser, ErrorNode, UnexpectedTokenErrorNode
@@ -40,49 +40,47 @@ class RuleParser(BaseParser):
False,
sheerka.new(BuiltinConcepts.IS_EMPTY))
try:
parser_input.reset()
if not self.reset_parser(context, parser_input):
error = self.error_sink[0]
context.log(f"Error found in tokenizer {error}", self.name)
return sheerka.ret(self.name, False, sheerka.new(BuiltinConcepts.ERROR, body=error))
parser_input.next_token()
if parser_input.token.type != TokenKind.RULE:
return sheerka.ret(self.name,
False,
sheerka.new(BuiltinConcepts.NOT_FOR_ME, body=parser_input.as_text()))
parser_input.next_token()
if parser_input.token.type != TokenKind.RULE:
return sheerka.ret(self.name,
False,
sheerka.new(BuiltinConcepts.NOT_FOR_ME, body=parser_input.as_text()))
token = parser_input.token
token = parser_input.token
if parser_input.next_token():
reason = UnexpectedTokenErrorNode("Only one rule supported",
parser_input.token,
[TokenKind.EOF])
return sheerka.ret(self.name,
False,
sheerka.new(BuiltinConcepts.NOT_FOR_ME, body=parser_input.as_text(), reason=reason))
if parser_input.next_token():
reason = UnexpectedTokenErrorNode("Only one rule supported",
parser_input.token,
[TokenKind.EOF])
return sheerka.ret(self.name,
False,
sheerka.new(BuiltinConcepts.NOT_FOR_ME, body=parser_input.as_text(), reason=reason))
if token.value[1] is None:
return sheerka.ret(self.name,
False,
sheerka.new(BuiltinConcepts.NOT_IMPLEMENTED))
if token.value[1] is None:
return sheerka.ret(self.name,
False,
sheerka.new(BuiltinConcepts.NOT_IMPLEMENTED))
if token.value[1].isdigit():
rule = sheerka.get_rule_by_id(token.value[1])
else:
rule = Rule().set_id(token.value[1])
rule.metadata.action_type = ACTION_TYPE_DEFERRED
if token.value[1].isdigit():
rule = sheerka.get_rule_by_id(token.value[1])
else:
rule = Rule().set_id(token.value[1])
rule.metadata.action_type = ACTION_TYPE_DEFERRED
if sheerka.isinstance(rule, BuiltinConcepts.UNKNOWN_RULE):
return sheerka.ret(self.name,
False,
sheerka.new(BuiltinConcepts.ERROR,
body=[RuleNotFound(token.value)]))
body = sheerka.new(BuiltinConcepts.PARSER_RESULT,
parser=self,
source=parser_input.as_text(),
body=[rule],
try_parsed=[rule])
if sheerka.isinstance(rule, BuiltinConcepts.UNKNOWN_RULE):
return sheerka.ret(self.name,
False,
sheerka.new(BuiltinConcepts.ERROR,
body=[RuleNotFound(token.value)]))
body = sheerka.new(BuiltinConcepts.PARSER_RESULT,
parser=self,
source=parser_input.as_text(),
body=[rule],
try_parsed=[rule])
return sheerka.ret(self.name, True, body)
except LexerError as e:
context.log(f"Error found in tokenizer {e}", self.name)
return sheerka.ret(self.name, False, sheerka.new(BuiltinConcepts.ERROR, body=e))
return sheerka.ret(self.name, True, body)
tests/core/test_ExecutionContext.py
+1 -1
View File
@@ -1,4 +1,4 @@
from core.builtin_concepts import BuiltinConcepts
from core.builtin_concepts_ids import BuiltinConcepts
from core.concept import Concept
from core.sheerka.ExecutionContext import ExecutionContext
from core.sheerka.services.SheerkaMemory import SheerkaMemory
tests/core/test_SheerkaAdmin.py
-2
View File
@@ -1,5 +1,3 @@
from core.builtin_concepts import BuiltinConcepts
from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
tests/core/test_SheerkaEvaluateConcept.py
+57 -5
View File
@@ -1,12 +1,17 @@
from dataclasses import dataclass
import pytest
from core.builtin_concepts import BuiltinConcepts, ReturnValueConcept, ParserResultConcept
from core.concept import Concept, DoNotResolve, ConceptParts, InfiniteRecursionResolved, CB, NotInit, \
concept_part_value
concept_part_value, DEFINITION_TYPE_DEF
from core.sheerka.services.SheerkaEvaluateConcept import SheerkaEvaluateConcept
from core.sheerka.services.SheerkaMemory import SheerkaMemory
from parsers.PythonParser import PythonNode
from parsers.BaseParser import BaseParser
from parsers.PythonParser import PythonNode, PythonParser
from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
from tests.evaluators.EvaluatorTestsUtils import pr_ret_val, python_ret_val
class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
@@ -709,7 +714,7 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
assert sheerka.isinstance(evaluated, BuiltinConcepts.CHICKEN_AND_EGG)
assert evaluated.body == {foo, bar, baz, qux}
def test_i_can_detect_auto_recursion(self):
def test_i_can_detect_infinite_recursion(self):
sheerka, context, foo = self.init_concepts(
Concept("foo", body="foo"),
eval_body=True
@@ -863,11 +868,11 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
service.initialize_concept_asts(context, concept)
assert service.compute_metadata_to_eval(context, concept) == ["#where#"]
concept = Concept("foo", where="where a").def_var("a")
concept = Concept("foo", where="a").def_var("a")
service.initialize_concept_asts(context, concept)
assert service.compute_metadata_to_eval(context, concept) == ["variables", "#where#"]
concept = Concept("foo", where="where self")
concept = Concept("foo", where="self")
service.initialize_concept_asts(context, concept)
assert service.compute_metadata_to_eval(context, concept) == ["#body#", "#where#"]
@@ -925,6 +930,53 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
res = sheerka.evaluate_concept(context, foo, eval_body=True)
assert res.body == "Print return values"
def test_i_can_manage_python_concept_infinite_recursion_when_initializing_ast(self):
sheerka, context, foo = self.init_concepts(Concept("a + b", body="a + b").def_var("a").def_var("b"))
evaluator = SheerkaEvaluateConcept(sheerka)
evaluator.initialize_concept_asts(context, foo)
res = foo.get_compiled()["#body#"]
assert len(res) == 1
assert sheerka.isinstance(res[0], BuiltinConcepts.RETURN_VALUE)
assert res[0].who == BaseParser.get_name(PythonParser.NAME)
# TODO validate that a rule is created
def test_can_detect_recursive_definition_with_exact_concept(self):
sheerka, context, foo = self.init_concepts("foo")
evaluator = SheerkaEvaluateConcept(sheerka)
# 'def concept foo as foo'
return_values = [pr_ret_val(foo, parser="ExactConcept"), python_ret_val("foo")]
res = evaluator.get_recursive_definitions(foo, return_values)
assert list(res) == [BaseParser.get_name("ExactConcept")]
def test_i_can_detect_when_no_recursive_definition(self):
sheerka, context, foo, bar = self.init_concepts("foo", "bar")
evaluator = SheerkaEvaluateConcept(sheerka)
# 'def concept foo as bar'
return_values = [pr_ret_val(bar, parser="ExactConcept"), python_ret_val("foo")]
res = evaluator.get_recursive_definitions(foo, return_values)
assert list(res) == []
def test_i_can_detect_when_no_recursive_definition2(self):
sheerka, context, q = self.init_concepts(
Concept("q", definition="q ?", definition_type=DEFINITION_TYPE_DEF).def_var("q"))
evaluator = SheerkaEvaluateConcept(sheerka)
# i dunno how to construct the return value
return_values = [pr_ret_val(q, parser="ExactConcept")]
res = evaluator.get_recursive_definitions(q, return_values)
assert list(res) == []
# I cannot implement value cache for now
# def test_values_when_no_variables_are_computed_only_once(self):
# sheerka, context, foo = self.init_concepts(Concept("foo", body="10"))
tests/core/test_SheerkaSetsManager.py
+4 -2
View File
@@ -204,8 +204,10 @@ class TestSheerkaSetsManager(TestUsingMemoryBasedSheerka):
service.add_concepts_to_set(context, [one, two, twenty, twenties], number)
assert sheerka.isinset(twenties, number)
twenty_one = sheerka.evaluate_user_input("twenty one", "")[0].body
assert sheerka.isinset(twenty_one, number)
res = sheerka.evaluate_user_input("twenty one", "")
assert len(res) == 1
assert res[0].status
assert sheerka.isinset(res[0].body, number)
def test_a_concept_can_be_in_multiple_sets(self):
sheerka, context, foo, all_foo, all_bar = self.init_concepts(
tests/core/test_sheerka.py
+2 -1
View File
@@ -1,7 +1,8 @@
import os
import pytest
from core.builtin_concepts import BuiltinConcepts, ReturnValueConcept, UserInputConcept, AllBuiltinConcepts
from core.builtin_concepts import BuiltinConcepts, ReturnValueConcept, UserInputConcept
from core.builtin_concepts_ids import AllBuiltinConcepts
from core.concept import Concept, PROPERTIES_TO_SERIALIZE, ConceptParts, NotInit
from core.sheerka.Sheerka import Sheerka, BASE_NODE_PARSER_CLASS
from core.sheerka.services.SheerkaConceptManager import SheerkaConceptManager
tests/core/test_sheerka_call_evaluators.py
+149 -27
View File
@@ -72,8 +72,10 @@ class EvaluatorOneWithPriority15(EvaluatorOneWithPriority):
class EvaluatorOneWithPriority20(EvaluatorOneWithPriority):
def __init__(self):
def __init__(self, enabled=None):
super().__init__("priority20", 20)
if enabled is not None:
self.enabled = enabled
class EvaluatorAllWithPriority(AllReturnValueEvaluatorForTestingPurpose):
@@ -87,8 +89,8 @@ class EvaluatorAllWithPriority10(EvaluatorAllWithPriority):
class EvaluatorAllWithPriority15(EvaluatorAllWithPriority):
def __init__(self):
super().__init__("all_priority15", 15)
def __init__(self, priority=15):
super().__init__("all_priority15", priority)
class EvaluatorAllWithPriority20(EvaluatorAllWithPriority):
@@ -238,6 +240,12 @@ class EvaluatorOneDoNotModifyExecutionFlow(EvaluatorOneWithPriority):
return return_value
class DisabledEvaluatorOneWithPriority90(EvaluatorOneWithPriority):
def __init__(self):
super().__init__("disabled90", 90)
self.enabled = False
class TestSheerkaExecuteEvaluators(TestUsingMemoryBasedSheerka):
@classmethod
@@ -246,6 +254,127 @@ class TestSheerkaExecuteEvaluators(TestUsingMemoryBasedSheerka):
# Ask for a new one
TestUsingMemoryBasedSheerka.singleton_instance = None
def test_i_can_get_evaluators_when_context_is_not_altered(self):
sheerka, context = self.init_concepts()
sheerka.evaluators = [
EvaluatorOneWithPriority20,
EvaluatorAllWithPriority15,
EvaluatorOnePreEvaluation, # wrong step
DisabledEvaluatorOneWithPriority90, # disabled,
]
service = sheerka.services[SheerkaExecute.NAME]
service.reset_registered_evaluators()
groups, sorted_priorities = service.get_evaluators(context, BuiltinConcepts.EVALUATION)
assert groups == {20: [EvaluatorOneWithPriority20()], 15: [EvaluatorAllWithPriority15()]}
assert sorted_priorities == [20, 15]
def test_i_can_get_selected_evaluators(self):
sheerka, context = self.init_concepts()
sheerka.evaluators = [
EvaluatorOneWithPriority20,
EvaluatorAllWithPriority15,
EvaluatorOnePreEvaluation, # wrong step
DisabledEvaluatorOneWithPriority90, # disabled
EvaluatorOneWithPriority15, # not selected
]
service = sheerka.services[SheerkaExecute.NAME]
service.reset_registered_evaluators()
evaluators_names = ["priority20", "all_priority15", "preEval", "disabled90"]
context.preprocess_evaluators = evaluators_names
groups, sorted_priorities = service.get_evaluators(context, BuiltinConcepts.EVALUATION)
assert groups == {20: [EvaluatorOneWithPriority20()], 15: [EvaluatorAllWithPriority15()]}
assert sorted_priorities == [20, 15]
key = BuiltinConcepts.EVALUATION + "|" + "|".join(evaluators_names)
assert key in service.grouped_evaluators_cache
groups, sorted_priorities = service.grouped_evaluators_cache[key]
assert groups == {20: [EvaluatorOneWithPriority20()], 15: [EvaluatorAllWithPriority15()]}
assert sorted_priorities == [20, 15]
def test_i_can_get_altered_evaluators(self):
sheerka, context = self.init_concepts()
sheerka.evaluators = [
EvaluatorOneWithPriority20,
EvaluatorAllWithPriority15,
EvaluatorOnePreEvaluation, # wrong step
DisabledEvaluatorOneWithPriority90, # always disabled
EvaluatorOneWithPriority15, # dynamically disabled
]
service = sheerka.services[SheerkaExecute.NAME]
service.reset_registered_evaluators()
context.add_preprocess(BaseEvaluator.get_name("priority15"), enabled=False)
context.add_preprocess(BaseEvaluator.get_name("all_priority15"), priority=99)
groups, sorted_priorities = service.get_evaluators(context, BuiltinConcepts.EVALUATION)
assert groups == {20: [EvaluatorOneWithPriority20()], 99: [EvaluatorAllWithPriority15(99)]}
assert sorted_priorities == [99, 20]
service.undo_preprocess()
# make sure that the result in not kept in cache
another_context = self.get_context(sheerka)
another_context.add_preprocess(BaseEvaluator.get_name("all_priority15"), priority=50)
groups, sorted_priorities = service.get_evaluators(another_context, BuiltinConcepts.EVALUATION)
assert groups == {20: [EvaluatorOneWithPriority20()],
15: [EvaluatorOneWithPriority15()],
50: [EvaluatorAllWithPriority15(50)]}
assert sorted_priorities == [50, 20, 15]
def test_i_can_get_altered_and_selected_evaluators(self):
sheerka, context = self.init_concepts()
sheerka.evaluators = [
EvaluatorOneWithPriority20,
EvaluatorAllWithPriority15,
EvaluatorOnePreEvaluation, # wrong step
DisabledEvaluatorOneWithPriority90, # always disabled
EvaluatorOneWithPriority15, # dynamically disabled
]
service = sheerka.services[SheerkaExecute.NAME]
service.reset_registered_evaluators()
evaluators_names = ["all_priority15", "preEval", "disabled90", "priority15"]
context.preprocess_evaluators = evaluators_names
context.add_preprocess(BaseEvaluator.get_name("priority15"), enabled=False)
context.add_preprocess(BaseEvaluator.get_name("all_priority15"), priority=99)
context.add_preprocess(BaseEvaluator.get_name("priority20"), priority=98) # not selected
groups, sorted_priorities = service.get_evaluators(context, BuiltinConcepts.EVALUATION)
assert groups == {99: [EvaluatorAllWithPriority15(99)]}
assert sorted_priorities == [99]
key = BuiltinConcepts.EVALUATION + "|" + "|".join(evaluators_names)
assert key not in service.grouped_evaluators_cache
def test_i_can_revert_back_evaluators_alterations(self):
sheerka, context = self.init_concepts()
sheerka.evaluators = [
EvaluatorOneWithPriority20,
EvaluatorAllWithPriority15,
]
service = sheerka.services[SheerkaExecute.NAME]
service.reset_registered_evaluators()
context.add_preprocess(BaseEvaluator.get_name("priority20"), enabled=False)
context.add_preprocess(BaseEvaluator.get_name("all_priority15"), priority=99)
groups, sorted_priorities = service.get_evaluators(context, BuiltinConcepts.EVALUATION)
assert groups == {99: [EvaluatorAllWithPriority15(99)]}
assert sorted_priorities == [99]
another_context = self.get_context(sheerka)
# the result is taken from the default cache, so the priorities are okay
# But the attributes of the evaluators are not reset
groups, sorted_priorities = service.get_evaluators(another_context, BuiltinConcepts.EVALUATION)
assert groups == {15: [EvaluatorAllWithPriority15(99)], 20: [EvaluatorOneWithPriority20(enabled=False)]}
assert sorted_priorities == [20, 15]
# let's revert
service.undo_preprocess()
groups, sorted_priorities = service.get_evaluators(another_context, BuiltinConcepts.EVALUATION)
assert groups == {15: [EvaluatorAllWithPriority15()], 20: [EvaluatorOneWithPriority20()]}
assert sorted_priorities == [20, 15]
def test_that_return_values_is_unchanged_when_no_evaluator(self):
sheerka = self.get_sheerka()
sheerka.evaluators = []
@@ -299,7 +428,7 @@ class TestSheerkaExecuteEvaluators(TestUsingMemoryBasedSheerka):
EvaluatorOneWithPriority15,
EvaluatorAllWithPriority20, ]
service = sheerka.services[SheerkaExecute.NAME]
service.reset_evaluators()
service.reset_registered_evaluators()
entries = [self.tretval(sheerka, Concept("foo"))]
Out.debug_out = []
@@ -324,7 +453,7 @@ class TestSheerkaExecuteEvaluators(TestUsingMemoryBasedSheerka):
sheerka = self.get_sheerka()
sheerka.evaluators = [EvaluatorOneModifyFoo]
service = sheerka.services[SheerkaExecute.NAME]
service.reset_evaluators()
service.reset_registered_evaluators()
entries = [self.tretval(sheerka, Concept("foo")), self.tretval(sheerka, Concept("baz"))]
Out.debug_out = []
@@ -342,7 +471,7 @@ class TestSheerkaExecuteEvaluators(TestUsingMemoryBasedSheerka):
sheerka = self.get_sheerka()
sheerka.evaluators = [EvaluatorAllReduceFooBar]
service = sheerka.services[SheerkaExecute.NAME]
service.reset_evaluators()
service.reset_registered_evaluators()
entries = [self.tretval(sheerka, Concept("foo")), self.tretval(sheerka, Concept("bar"))]
Out.debug_out = []
@@ -364,7 +493,7 @@ class TestSheerkaExecuteEvaluators(TestUsingMemoryBasedSheerka):
sheerka = self.get_sheerka()
sheerka.evaluators = [EvaluatorOneModifyFoo, EvaluatorOneModifyBar]
service = sheerka.services[SheerkaExecute.NAME]
service.reset_evaluators()
service.reset_registered_evaluators()
entries = [self.tretval(sheerka, Concept("foo")), self.tretval(sheerka, Concept("baz"))]
Out.debug_out = []
@@ -391,7 +520,7 @@ class TestSheerkaExecuteEvaluators(TestUsingMemoryBasedSheerka):
sheerka = self.get_sheerka()
sheerka.evaluators = [EvaluatorOneWithPriority10, EvaluatorOnePreEvaluation]
service = sheerka.services[SheerkaExecute.NAME]
service.reset_evaluators()
service.reset_registered_evaluators()
entries = [self.tretval(sheerka, Concept("foo"))]
Out.debug_out = []
@@ -406,7 +535,7 @@ class TestSheerkaExecuteEvaluators(TestUsingMemoryBasedSheerka):
sheerka = self.get_sheerka()
sheerka.evaluators = [EvaluatorOneMultiSteps]
service = sheerka.services[SheerkaExecute.NAME]
service.reset_evaluators()
service.reset_registered_evaluators()
entries = [self.tretval(sheerka, Concept("foo"))]
Out.debug_out = []
@@ -426,8 +555,9 @@ class TestSheerkaExecuteEvaluators(TestUsingMemoryBasedSheerka):
EvaluatorAllWithPriority15,
EvaluatorAllWithPriority10, ]
service = sheerka.services[SheerkaExecute.NAME]
service.reset_evaluators()
context.preprocess_evaluators = [EvaluatorAllWithPriority10().short_name] # it will be the only one to be evaluated
service.reset_registered_evaluators()
context.preprocess_evaluators = [
EvaluatorAllWithPriority10().short_name] # it will be the only one to be evaluated
entries = [self.tretval(sheerka, Concept("foo"))]
@@ -448,7 +578,7 @@ class TestSheerkaExecuteEvaluators(TestUsingMemoryBasedSheerka):
EvaluatorOneWithPriority15,
EvaluatorOneWithPriority10, ]
service = sheerka.services[SheerkaExecute.NAME]
service.reset_evaluators()
service.reset_registered_evaluators()
# invert the priorities
context.add_preprocess(EvaluatorOneWithPriority20().name, priority=1)
@@ -468,17 +598,13 @@ class TestSheerkaExecuteEvaluators(TestUsingMemoryBasedSheerka):
'__EVALUATION [0] priority20 - eval - target=foo',
]
# grouped evaluator is in cache
service = sheerka.services[SheerkaExecute.NAME]
assert "__EVALUATION|evaluators.priority20|evaluators.priority15|evaluators.priority10" in service.grouped_evaluators_cache
def test_evaluators_enabled_can_be_tweaked_by_the_context(self):
sheerka, context = self.init_concepts()
sheerka.evaluators = [EvaluatorOneWithPriority20,
EvaluatorOneWithPriority15,
EvaluatorOneWithPriority10, ]
service = sheerka.services[SheerkaExecute.NAME]
service.reset_evaluators()
service.reset_registered_evaluators()
# invert the priorities
context.add_preprocess(EvaluatorOneWithPriority20().name, enabled=False)
@@ -493,15 +619,11 @@ class TestSheerkaExecuteEvaluators(TestUsingMemoryBasedSheerka):
"__EVALUATION [0] priority10 - eval - target=foo",
]
# grouped evaluator is in cache
service = sheerka.services[SheerkaExecute.NAME]
assert "__EVALUATION|evaluators.priority10" in service.grouped_evaluators_cache
def test_evaluators_can_be_pre_processed(self):
sheerka = self.get_sheerka()
sheerka.evaluators = [EvaluatorOneModifyFoo]
service = sheerka.services[SheerkaExecute.NAME]
service.reset_evaluators()
service.reset_registered_evaluators()
entries = [self.tretval(sheerka, Concept("foo"))]
@@ -525,7 +647,7 @@ class TestSheerkaExecuteEvaluators(TestUsingMemoryBasedSheerka):
sheerka, context, foo, bar, baz = self.init_concepts("foo", "bar", "baz")
sheerka.evaluators = [EvaluatorOneInitializationOnce]
service = sheerka.services[SheerkaExecute.NAME]
service.reset_evaluators()
service.reset_registered_evaluators()
entries = [
self.tretval(sheerka, foo),
@@ -548,7 +670,7 @@ class TestSheerkaExecuteEvaluators(TestUsingMemoryBasedSheerka):
sheerka, context, foo, bar, baz = self.init_concepts("foo", "bar", "baz")
sheerka.evaluators = [EvaluatorOneInitializationMultiple]
service = sheerka.services[SheerkaExecute.NAME]
service.reset_evaluators()
service.reset_registered_evaluators()
entries = [
self.tretval(sheerka, foo),
@@ -578,7 +700,7 @@ class TestSheerkaExecuteEvaluators(TestUsingMemoryBasedSheerka):
sheerka = self.get_sheerka()
sheerka.evaluators = [EvaluatorOneDoNotModifyExecutionFlow]
service = sheerka.services[SheerkaExecute.NAME]
service.reset_evaluators()
service.reset_registered_evaluators()
entries = [self.tretval(sheerka, Concept("foo"))]
Out.debug_out = []
@@ -600,7 +722,7 @@ class TestSheerkaExecuteEvaluators(TestUsingMemoryBasedSheerka):
sheerka = self.get_sheerka()
sheerka.evaluators = [EvaluatorOneModifyFoo]
service = sheerka.services[SheerkaExecute.NAME]
service.reset_evaluators()
service.reset_registered_evaluators()
entries = [self.tretval(sheerka, Concept("foo"))]
Out.debug_out = []
@@ -619,7 +741,7 @@ class TestSheerkaExecuteEvaluators(TestUsingMemoryBasedSheerka):
sheerka = self.get_sheerka()
sheerka.evaluators = [EvaluatorAllSuppressEntries]
service = sheerka.services[SheerkaExecute.NAME]
service.reset_evaluators()
service.reset_registered_evaluators()
entries = [self.tretval(sheerka, Concept("foo")),
self.tretval(sheerka, Concept("bar")),
tests/core/test_sheerka_call_parsers.py
+98 -17
View File
@@ -1,5 +1,5 @@
from core.builtin_concepts import ReturnValueConcept, UserInputConcept, BuiltinConcepts, ParserResultConcept
from core.sheerka.services.SheerkaExecute import ParserInput
from core.sheerka.services.SheerkaExecute import ParserInput, SheerkaExecute
from parsers.BaseParser import BaseParser
from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
@@ -142,12 +142,74 @@ class TestSheerkaExecuteParsers(TestUsingMemoryBasedSheerka):
# Ask for a new one
TestUsingMemoryBasedSheerka.singleton_instance = None
def test_i_can_get_parser_when_context_is_not_altered(self):
sheerka, context = self.init_concepts()
sheerka.parsers = {
"Enabled90False": Enabled90FalseParser,
"Enabled80False": Enabled80FalseParser,
}
service = SheerkaExecute(sheerka)
service.reset_registered_parsers()
groups, sorted_priorities = service.get_parsers(context)
assert groups == {80: [Enabled80FalseParser()], 90: [Enabled90FalseParser()]}
assert sorted_priorities == [90, 80]
def test_i_can_get_selected_parsers(self):
sheerka, context = self.init_concepts()
sheerka.parsers = {
"Enabled90False": Enabled90FalseParser,
"Enabled80False": Enabled80FalseParser,
"Enabled70False": Enabled70FalseParser,
"Enabled50True": Enabled50TrueParser,
"Disabled": Enabled50TrueParser, # <= this one is disabled. It can't be used
}
service = SheerkaExecute(sheerka)
service.reset_registered_parsers()
parsers_names = ["Enabled50True", "Enabled70False", "Disabled"]
context.preprocess_parsers = parsers_names
groups, sorted_priorities = service.get_parsers(context)
assert groups == {50: [Enabled50TrueParser()], 70: [Enabled70FalseParser()]}
assert sorted_priorities == [70, 50] # Disabled parser does not appear
key = "|".join(parsers_names)
assert key in service.grouped_parsers_cache
groups, sorted_priorities = service.grouped_parsers_cache[key]
assert groups == {50: [Enabled50TrueParser], 70: [Enabled70FalseParser]}
assert sorted_priorities == [70, 50]
def test_i_can_get_altered_parsers(self):
sheerka, context = self.init_concepts()
sheerka.parsers = {
"Enabled90False": Enabled90FalseParser,
"Enabled80False": Enabled80FalseParser,
"Enabled70False": Enabled70FalseParser,
"Enabled50True": Enabled50TrueParser,
"Disabled": Enabled50TrueParser, # <= this one is disabled. It can't be used
}
service = SheerkaExecute(sheerka)
service.reset_registered_parsers()
parsers_names = ["Enabled90False", "Enabled50True", "Enabled70False", "Disabled"]
context.preprocess_parsers = parsers_names
context.add_preprocess(BaseParser.get_name("Enabled90False"), enabled=False)
context.add_preprocess(BaseParser.get_name("Enabled50True"), priority=80)
groups, sorted_priorities = service.get_parsers(context)
assert groups == {80: [Enabled50TrueParser()], 70: [Enabled70FalseParser()]}
assert sorted_priorities == [80, 70] # Disabled parsers does not appear
key = "|".join(parsers_names)
assert key not in service.grouped_parsers_cache # not saved in cache
def test_disabled_parsers_are_not_executed(self):
sheerka = self.get_sheerka()
sheerka.parsers = {
"Enabled": Enabled10TrueParser,
"Disabled": DisabledParser
}
service = sheerka.services[SheerkaExecute.NAME]
service.reset_registered_parsers()
user_input = [get_ret_val("hello world")]
BaseTestParser.debug_out = []
@@ -162,6 +224,8 @@ class TestSheerkaExecuteParsers(TestUsingMemoryBasedSheerka):
"Enabled80False": Enabled80FalseParser,
"Enabled50True": Enabled50TrueParser,
}
service = sheerka.services[SheerkaExecute.NAME]
service.reset_registered_parsers()
user_input = [get_ret_val("hello world")]
BaseTestParser.debug_out = []
@@ -177,22 +241,25 @@ class TestSheerkaExecuteParsers(TestUsingMemoryBasedSheerka):
'name=Enabled50True, priority=50, status=True, source=Enabled80False:Enabled90False:hello world',
]
# def test_parsing_stop_at_the_first_success(self):
# sheerka = self.get_sheerka()
# sheerka.parsers = {
# "Enabled80False": Enabled80FalseParser,
# "Enabled50bisTrue": Enabled50bisTrueParser,
# "Enabled10True": Enabled10TrueParser,
# }
#
# user_input = [get_ret_val("hello world")]
# BaseTestParser.debug_out = []
# sheerka.execute(self.get_context(sheerka), user_input, [BuiltinConcepts.PARSING])
#
# assert BaseTestParser.debug_out == [
# 'name=Enabled80False, priority=80, status=False, source=hello world',
# 'name=Enabled50BisTrue, priority=50, status=True, source=hello world',
# ]
def test_parsing_stop_at_the_first_success(self):
sheerka = self.get_sheerka()
sheerka.parsers = {
"Enabled80False": Enabled80FalseParser,
"Enabled50bisTrue": Enabled50bisTrueParser,
"Enabled10True": Enabled10TrueParser,
}
service = sheerka.services[SheerkaExecute.NAME]
service.reset_registered_parsers()
user_input = [get_ret_val("hello world")]
BaseTestParser.debug_out = []
sheerka.execute(self.get_context(sheerka), user_input, [BuiltinConcepts.PARSING])
assert BaseTestParser.debug_out == [
'name=Enabled80False, priority=80, status=False, source=hello world',
'name=Enabled50BisTrue, priority=50, status=True, source=hello world',
'name=Enabled50BisTrue, priority=50, status=True, source=Enabled80False:hello world',
]
def test_parsing_stop_at_the_first_success_2(self):
"""
@@ -206,6 +273,8 @@ class TestSheerkaExecuteParsers(TestUsingMemoryBasedSheerka):
"Enabled50True": Enabled50TrueParser,
"Enabled10True": Enabled10TrueParser,
}
service = sheerka.services[SheerkaExecute.NAME]
service.reset_registered_parsers()
user_input = [get_ret_val("hello world")]
BaseTestParser.debug_out = []
@@ -235,6 +304,8 @@ class TestSheerkaExecuteParsers(TestUsingMemoryBasedSheerka):
"Enabled50False": Enabled50FalseParser,
"Enabled10True": Enabled10TrueParser,
}
service = sheerka.services[SheerkaExecute.NAME]
service.reset_registered_parsers()
user_input = [get_ret_val("hello world")]
BaseTestParser.debug_out = []
@@ -265,6 +336,8 @@ class TestSheerkaExecuteParsers(TestUsingMemoryBasedSheerka):
"Enabled80False": Enabled80FalseParser,
"Enabled50True": Enabled50TrueParser,
}
service = sheerka.services[SheerkaExecute.NAME]
service.reset_registered_parsers()
user_input = [get_ret_val("hello world")]
@@ -286,6 +359,8 @@ class TestSheerkaExecuteParsers(TestUsingMemoryBasedSheerka):
"NoneParser": NoneParser,
"ListOfNone": ListOfNoneParser,
}
service = sheerka.services[SheerkaExecute.NAME]
service.reset_registered_parsers()
user_input = [get_ret_val("hello world")]
@@ -310,6 +385,8 @@ class TestSheerkaExecuteParsers(TestUsingMemoryBasedSheerka):
"Enabled70False": Enabled70FalseParser,
"Enabled50True": Enabled50TrueParser,
}
service = sheerka.services[SheerkaExecute.NAME]
service.reset_registered_parsers()
user_input = [get_ret_val("hello world")]
BaseTestParser.debug_out = []
@@ -329,6 +406,8 @@ class TestSheerkaExecuteParsers(TestUsingMemoryBasedSheerka):
"Enabled80MultipleFalse": Enabled80MultipleFalseParser,
"Enabled50True": Enabled50TrueParser,
}
service = sheerka.services[SheerkaExecute.NAME]
service.reset_registered_parsers()
user_input = [get_ret_val("hello world")]
BaseTestParser.debug_out = []
@@ -356,6 +435,8 @@ class TestSheerkaExecuteParsers(TestUsingMemoryBasedSheerka):
"Enabled80MultipleTrue": Enabled80MultipleTrueParser,
"Enabled50True": Enabled50TrueParser,
}
service = sheerka.services[SheerkaExecute.NAME]
service.reset_registered_parsers()
user_input = [get_ret_val("hello world")]
BaseTestParser.debug_out = []
tests/evaluators/EvaluatorTestsUtils.py
+23 -2
View File
@@ -1,7 +1,10 @@
from core.builtin_concepts import ReturnValueConcept, BuiltinConcepts
import ast
from core.builtin_concepts import ReturnValueConcept, BuiltinConcepts, ParserResultConcept
from core.concept import Concept
from evaluators.BaseEvaluator import BaseEvaluator
from parsers.BaseParser import BaseParser
from parsers.PythonParser import PythonNode
reduced_requested = ReturnValueConcept("Sheerka", True, Concept(name=BuiltinConcepts.REDUCE_REQUESTED,
key=BuiltinConcepts.REDUCE_REQUESTED))
@@ -12,7 +15,7 @@ def ret_val(value="value", who="who", status=True):
def p_ret_val(value="value", parser="parser", status=True):
return ReturnValueConcept(BaseParser.PREFIX + parser, status, value)
return ReturnValueConcept(BaseParser.get_name(parser), status, value)
def e_ret_val(value="value", evaluator="evaluator", status=True):
@@ -40,6 +43,24 @@ def e_ret_val_new(key, evaluator="evaluator", status=True, **kwargs):
return e_ret_val(body, evaluator, status)
def pr_ret_val(value, parser="parser", source=None):
"""
ParserResult ReturnValue
eg: ReturnValue with a ParserResult
:param value:
:param parser:
:param source:
:return:
"""
source = source or (value.name if isinstance(value, Concept) else "source")
parser_result = ParserResultConcept(BaseParser.get_name(parser), source=source, value=value)
return p_ret_val(parser_result, parser)
def python_ret_val(source):
python_node = PythonNode(source, ast.parse(source, f"<source>", 'eval'))
return pr_ret_val(python_node, parser="Python", source=source)
def new_concept(key, **kwargs):
res = Concept(key=key, name=key, is_builtin=False, is_unique=False)
for k, v in kwargs.items():
tests/evaluators/test_ConceptEvaluator.py
+3 -5
View File
@@ -91,12 +91,10 @@ class TestAddConceptEvaluator(TestUsingMemoryBasedSheerka):
assert result.value == "'some_other_value'"
def test_i_cannot_recognize_a_concept_if_one_of_the_prop_is_unknown(self):
context = self.get_context()
sheerka, context, one, concept_plus = self.init_concepts(
"one",
Concept(name="a plus b").def_var("a", "one").def_var("b", "two"))
context.protected_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
context.sheerka.test_only_add_in_cache(Concept(name="one").init_key())
concept_plus = context.sheerka.test_only_add_in_cache(Concept(name="a plus b")
.def_var("a", "one")
.def_var("b", "two").init_key())
evaluator = ConceptEvaluator()
item = self.pretval(concept_plus)
tests/non_reg/test_sheerka_non_reg.py
+17 -8
View File
@@ -43,11 +43,7 @@ class TestSheerkaNonRegMemory(TestUsingMemoryBasedSheerka):
assert evaluated == simplec("one", 1)
def test_i_can_recognize_concept_with_concept_body(self):
sheerka = self.get_sheerka()
concept_one = Concept(name="one")
concept_un = Concept(name="un", body="one")
sheerka.test_only_add_in_cache(concept_one)
sheerka.test_only_add_in_cache(concept_un)
sheerka, context, concept_one, concept_un = self.init_concepts("one", Concept(name="un", body="one"))
res = sheerka.evaluate_user_input("un")
return_value = res[0].value
@@ -202,9 +198,10 @@ as:
assert evaluated.get_value("a") == concept_foo
def test_i_can_recognize_concept_with_variable_and_python_as_body(self):
sheerka = self.get_sheerka()
hello_a = sheerka.test_only_add_in_cache(Concept(name="hello a", body="'hello ' + a").def_var("a"))
sheerka.test_only_add_in_cache(Concept(name="foo", body="'foo'"))
sheerka, context, hello_a, foo = self.init_concepts(
Concept(name="hello a", body="'hello ' + a").def_var("a"),
Concept(name="foo", body="'foo'")
)
res = sheerka.evaluate_user_input("hello foo")
assert len(res) == 1
@@ -1199,6 +1196,18 @@ as:
assert res[0].status
assert sheerka.isinstance(res[0].body, BuiltinConcepts.TO_MULTI)
def test_i_can_evaluate_pseudo_recursive_definition(self):
init = [
"def concept a + b as a + b",
]
sheerka = self.init_scenario(init)
res = sheerka.evaluate_user_input("eval 1 + 1")
assert len(res) == 1
assert res[0].status
assert res[0].body == 2
class TestSheerkaNonRegFile(TestUsingFileBasedSheerka):
def test_i_can_def_several_concepts(self):