Fixed #109 : Mix python and concept. List comprehension

Fixed #110 : SheerkaDebugManager: add list_debug_settings
Fixed #111 : SheerkaDebugManager: Implement ListDebugLogger
Fixed #112 : SyaNodeParser: rewrite this parser
Fixed #113 : Sheerka.: Add enable_parser_caching to disable parsers caching
Fixed #114 : SyaNodeParser : Implement fast cache to resolve unrecognized tokens requests
Fixed #115 : BnfNodeParser : Implement fast cache to resolve unrecognized tokens requests
Fixed #116 : SequenceNodeParser : Implement fast cache to resolve unrecognized tokens requests
Fixed #117 : ResolveMultiplePluralAmbiguityEvaluator: Resolve Multiple plural ambiguity
This commit is contained in:
2021-09-06 11:51:50 +02:00
parent 71d1b1d1ca
commit 54e5681c5a
57 changed files with 5179 additions and 3125 deletions
+12 -1
View File
@@ -3,7 +3,7 @@ import time
from os import path
from core.builtin_concepts_ids import BuiltinConcepts, BuiltinContainers
from core.builtin_helpers import ensure_concept_or_rule, ensure_concept
from core.builtin_helpers import ensure_concept_or_rule
from core.concept import Concept
from core.global_symbols import SHEERKA_BACKUP_FOLDER
from core.sheerka.services.SheerkaExecute import SheerkaExecute
@@ -40,6 +40,7 @@ class SheerkaAdmin(BaseService):
self.sheerka.bind_service_method(self.NAME, self.in_memory, False)
self.sheerka.bind_service_method(self.NAME, self.admin_history, False, as_name="history")
self.sheerka.bind_service_method(self.NAME, self.sdp, False)
self.sheerka.bind_service_method(self.NAME, self.set_sheerka, True)
def caches_names(self):
"""
@@ -284,3 +285,13 @@ class SheerkaAdmin(BaseService):
def admin_history(self, depth=10, start=0):
history = self.sheerka.services[SheerkaHistoryManager.NAME].history(depth, start)
return self.sheerka.new(BuiltinConcepts.TO_LIST, body=history)
def set_sheerka(self, context, key, value, service=None):
"""
@param context:
@param key:
@param value:
@param service:
@return:
"""
return self.sheerka.record_var(context, service or self.sheerka.name, key, value)
+311 -53
View File
@@ -5,7 +5,7 @@ from dataclasses import dataclass
from core.builtin_concepts import BuiltinConcepts
from core.builtin_helpers import evaluate_expression
from core.concept import Concept
from core.global_symbols import NotInit, NotFound
from core.global_symbols import NotFound, NotInit
from core.sheerka.ExecutionContext import ExecutionContext
from core.sheerka.services.sheerka_service import BaseService
from core.utils import CONSOLE_COLORS_MAP as CCM, CONSOLE_COLUMNS, PRIMITIVES_TYPES
@@ -100,13 +100,22 @@ class BaseDebugLogger:
BaseDebugLogger.ids[hint] = 0
return 0
def __init__(self, debug_manager, context, who, method_name, debug_id):
pass
def __init__(self, keep_track, debug_manager, context, service_name, method_name, debug_id):
self.debug_manager = debug_manager
self.service_name = service_name
self.method_name = method_name
self.context = context
self.debug_id = debug_id
self.keep_track = keep_track # Does debug_manager need to keep track of this logger ?
def debug_entering(self, **kwargs):
pass
def debug_log(self, text, is_error=False):
def debug_leaving(self, **kwargs):
pass
def debug_log(self, text, is_error=False, args=None):
pass
def debug_var(self, name, value, is_error=False, hint=None):
@@ -122,7 +131,36 @@ class BaseDebugLogger:
pass
def get_enabled_vars(self):
pass
"""
Returns the list of all enabled variables for this console debugger
:return:
"""
return self.debug_manager.get_enabled_items("vars",
self.context,
self.service_name,
self.method_name,
self.debug_id)
def should_i_log_var(self, name, is_error=False):
return is_error or self.debug_manager.compute_debug_var(self.context,
self.service_name,
self.method_name,
name,
self.debug_id)
def should_i_log_rule(self, rule_id, is_error=False):
return is_error or self.debug_manager.compute_debug_rule(self.context,
self.service_name,
self.method_name,
rule_id,
self.debug_id)
def should_i_log_concept(self, concept_id, is_error=False):
return is_error or self.debug_manager.compute_debug_concept(self.context,
self.service_name,
self.method_name,
concept_id,
self.debug_id)
class NullDebugLogger(BaseDebugLogger):
@@ -139,12 +177,7 @@ class NullDebugLogger(BaseDebugLogger):
class ConsoleDebugLogger(BaseDebugLogger):
def __init__(self, debug_manager, context, service_name, method_name, debug_id):
BaseDebugLogger.__init__(self, debug_manager, context, service_name, method_name, debug_id)
self.debug_manager = debug_manager
self.service_name = service_name
self.method_name = method_name
self.context = context
self.debug_id = debug_id
BaseDebugLogger.__init__(self, False, debug_manager, context, service_name, method_name, debug_id)
self.is_highlighted = ""
def is_enabled(self):
@@ -154,17 +187,6 @@ class ConsoleDebugLogger(BaseDebugLogger):
"""
return True
def get_enabled_vars(self):
"""
Returns the list of all enabled variables for this console debugger
:return:
"""
return self.debug_manager.get_enabled_items("vars",
self.context,
self.service_name,
self.method_name,
self.debug_id)
def debug_entering(self, **kwargs):
"""
Log that we start debugging a method (for a specified service and context)
@@ -181,11 +203,31 @@ class ConsoleDebugLogger(BaseDebugLogger):
self.debug_manager.debug(self.prefix() + str_text)
self.debug_manager.debug(self.prefix() + str_vars)
def debug_log(self, text, is_error=False):
def debug_leaving(self, **kwargs):
"""
Log that we start debugging a method (for a specified service and context)
:param kwargs:
:return:
"""
super().debug_leaving(**kwargs)
if not kwargs:
return
str_text = f"{CCM['blue']}Leaving {self.service_name}.{self.method_name} with {CCM['reset']}"
str_vars = pp.pformat(kwargs)
if "\n" not in str(str_vars):
self.debug_manager.debug(self.prefix() + str_text + str_vars)
else:
self.debug_manager.debug(self.prefix() + str_text)
self.debug_manager.debug(self.prefix() + str_vars)
def debug_log(self, text, is_error=False, args=None):
"""
Prints a debug information (not related to a specific variable, concept or rule)
:param text:
:param is_error:
:param args:
:return:
"""
color = 'red' if is_error else 'blue'
@@ -268,8 +310,153 @@ class ConsoleDebugLogger(BaseDebugLogger):
return f"[{self.context.id:2}][{self.debug_id:2}] {self.is_highlighted}"
class ListDebugLogger(BaseDebugLogger):
ITEM_TYPE_ENTERING = "entering"
ITEM_TYPE_LEAVING = "leaving"
ITEM_TYPE_LOG = "log"
ITEM_TYPE_VAR = "var"
ITEM_TYPE_RULE = "rule"
ITEM_TYPE_CONCEPT = "concept"
class DebugItem:
def __init__(self, item_type, text, is_error=False, args=None):
self.type = item_type
self.text = text
self.is_error = is_error
self.args = args
def __repr__(self):
return self.text
def __init__(self, debug_manager, context, service_name, method_name, debug_id):
BaseDebugLogger.__init__(self, True, debug_manager, context, service_name, method_name, debug_id)
self.items = []
def is_enabled(self):
"""
True if the debug is activated for the current service, method and context
:return:
"""
return True
def debug_entering(self, **kwargs):
"""
Log that we start debugging a method (for a specified service and context)
:param kwargs:
:return:
"""
text = f"Entering {self.service_name}.{self.method_name}"
self.items.append(ListDebugLogger.DebugItem(ListDebugLogger.ITEM_TYPE_ENTERING, text, False, kwargs))
def debug_leaving(self, **kwargs):
"""
Log that we start debugging a method (for a specified service and context)
:param kwargs:
:return:
"""
super().debug_leaving(**kwargs)
if not kwargs:
return
text = f"Leaving {self.service_name}.{self.method_name}"
self.items.append(ListDebugLogger.DebugItem(ListDebugLogger.ITEM_TYPE_LEAVING, text, False, kwargs))
def debug_log(self, text, is_error=False, args=None):
"""
Prints a debug information (not related to a specific variable, concept or rule)
:param text:
:param is_error:
:param args:
:return:
"""
self.items.append(ListDebugLogger.DebugItem(ListDebugLogger.ITEM_TYPE_LOG, text, is_error, args))
def debug_var(self, name, value, is_error=False, hint=None):
"""
Prints the value of a variable
:param name:
:param value:
:param is_error:
:param hint:
:return:
"""
if not self.should_i_log_var(name, is_error):
return
text = name + hint if hint else name
self.items.append(ListDebugLogger.DebugItem(ListDebugLogger.ITEM_TYPE_VAR, text, is_error, value))
def debug_rule(self, rule, results):
"""
Prints debug information related to a specific rule id
:param rule:
:param results:
:return:
"""
if not self.should_i_log_rule(rule.id):
return
self.items.append(ListDebugLogger.DebugItem(ListDebugLogger.ITEM_TYPE_RULE, rule.id, False, results))
def debug_concept(self, concept, text=None, **kwargs):
"""
Prints debug information related to a specific concept
:param concept:
:param text:
:param kwargs:
:return:
"""
if not self.debug_manager.compute_debug_concept(self.context,
self.service_name,
self.method_name,
concept.id,
self.debug_id):
return
if not self.should_i_log_concept(concept.id):
return
concept_id = f"{concept.id}{text}" if text else f"{concept.id}"
self.items.append(ListDebugLogger.DebugItem(ListDebugLogger.ITEM_TYPE_CONCEPT, concept_id, False, kwargs))
class TeeDebugLogger(BaseDebugLogger):
def __init__(self, debug_manager, context, service_name, method_name, debug_id, loggers):
BaseDebugLogger.__init__(self, False, debug_manager, context, service_name, method_name, debug_id)
self.loggers = loggers
def is_enabled(self):
"""
True if the debug is activated for the current service, method and context
:return:
"""
return True
def debug_entering(self, **kwargs):
for logger in self.loggers:
logger.debug_entering(kwargs)
def debug_log(self, text, is_error=False, args=None):
for logger in self.loggers:
logger.debug_log(text, is_error, args=None)
def debug_var(self, name, value, is_error=False, hint=None):
for logger in self.loggers:
logger.debug_var(name, value, is_error, hint)
def debug_rule(self, rule, results):
for logger in self.loggers:
logger.debug_rule(rule, results)
def debug_concept(self, concept, text=None, **kwargs):
for logger in self.loggers:
logger.debug_concept(concept, text, **kwargs)
@dataclass
class DebugItem:
debug_type: str
item: str
service_name: str
method_name: str
@@ -280,6 +467,16 @@ class DebugItem:
enabled: bool
def __repr__(self):
text = f"type={self.debug_type}"
text += f", setting={self.service_name or '*'}.{self.method_name or '*'}.{self.item or '*'}"
text += f", context_id={self.context_id}"
text += f", debug_id={self.debug_id}"
text += f", context_children={self.context_children}"
text += f", debug_children={self.debug_children}"
text += f" (enabled={self.enabled})"
return f"DebugItem({text})"
class SheerkaDebugManager(BaseService):
NAME = "Debug"
@@ -302,7 +499,10 @@ class SheerkaDebugManager(BaseService):
self.registered_vars = [] # list of all variables that can be debugged
self.registered_rules = [] # list of all rules that can be debugged
self.registered_concepts = [] # list of all concept that can be debugged
self.debug_logger_definition = ConsoleDebugLogger # logger to use
self.instantiated_loggers = {}
# variable that needs to be reset on restart
self.state_vars = [
"activated",
"explicit", # to remove ?
@@ -323,25 +523,33 @@ class SheerkaDebugManager(BaseService):
self.sheerka.bind_service_method(self.NAME, self.set_debug, True)
self.sheerka.bind_service_method(self.NAME, self.inspect, False)
self.sheerka.bind_service_method(self.NAME, self.get_value, False)
self.sheerka.bind_service_method(self.NAME, self.get_debugger, False)
self.sheerka.bind_service_method(self.NAME, self.reset_debug, False)
self.sheerka.bind_service_method(self.NAME, self.set_debug_var, True)
self.sheerka.bind_service_method(self.NAME, self.set_debug_rule, True)
self.sheerka.bind_service_method(self.NAME, self.set_debug_concept, True)
self.sheerka.bind_service_method(self.NAME, self.list_debug_vars, True)
self.sheerka.bind_service_method(self.NAME, self.list_debug_rules, True)
self.sheerka.bind_service_method(self.NAME, self.list_debug_concepts, True)
self.sheerka.bind_service_method(self.NAME, self.list_debug_vars, False)
self.sheerka.bind_service_method(self.NAME, self.list_debug_rules, False)
self.sheerka.bind_service_method(self.NAME, self.list_debug_concepts, False)
self.sheerka.bind_service_method(self.NAME, self.list_debug_settings, False)
self.sheerka.bind_service_method(self.NAME, self.register_debug_vars, True, visible=False)
self.sheerka.bind_service_method(self.NAME, self.register_debug_rules, True, visible=False)
self.sheerka.bind_service_method(self.NAME, self.register_debug_concepts, True, visible=False)
self.sheerka.bind_service_method(self.NAME, self.get_debugger, False, visible=False)
self.sheerka.bind_service_method(self.NAME, self.get_debugger_logs, False, visible=False)
self.sheerka.bind_service_method(self.NAME, self.set_debug_logger_definition, True, visible=False)
# self.sheerka.bind_service_method(self.NAME,self.get_debug_settings, False, as_name="debug_settings")
# register what can be registered
from parsers.BnfNodeParser import BnfNodeParser
from evaluators.DefConceptEvaluator import DefConceptEvaluator
from evaluators.PythonEvaluator import PythonEvaluator
from parsers.SyaNodeParser import SyaNodeParser
from parsers.SequenceNodeParser import SequenceNodeParser
self.register_debug_vars(BnfNodeParser.NAME, "parse", "result")
self.register_debug_vars(BnfNodeParser.NAME, "parse", "stats")
self.register_debug_vars(SequenceNodeParser.NAME, "parse", "stats")
self.register_debug_concepts(BnfNodeParser.NAME, "parse", "*")
self.register_debug_vars(DefConceptEvaluator.NAME, "matches", "*")
self.register_debug_vars(DefConceptEvaluator.NAME, "eval", "*")
@@ -351,7 +559,8 @@ class SheerkaDebugManager(BaseService):
self.register_debug_vars(PythonEvaluator.NAME, "eval", "ret")
self.register_debug_vars("Exceptions", PythonEvaluator.NAME + "-eval", "exception")
self.register_debug_vars("Exceptions", PythonEvaluator.NAME + "-eval", "trace")
self.register_debug_vars(SyaNodeParser.NAME, "parse", "*")
self.register_debug_vars(SyaNodeParser.NAME, "parse", "#[number]")
self.register_debug_vars(SyaNodeParser.NAME, "parse", "stats")
self.register_debug_vars(MultipleSuccessEvaluator.NAME, "matches", "return_values")
def initialize_deferred(self, context, is_first_time):
@@ -498,14 +707,30 @@ class SheerkaDebugManager(BaseService):
self.sheerka.record_var(context, self.NAME, "activated", self.activated)
return self.sheerka.ret(SheerkaDebugManager.NAME, True, self.sheerka.new(BuiltinConcepts.SUCCESS))
def set_debug_logger_definition(self, logger_definition):
"""
Logger definition to use. By default it's the ConsoleDebugLogger
logger_definition can be a list of debug loggers
:param logger_definition:
:return:
"""
self.debug_logger_definition = logger_definition
def debug(self, *args, **kwargs):
print(*args, **kwargs)
def get_debugger(self, context, who, method_name, new_debug_id=True):
def get_debugger(self, context, who, method_name, new_debug_id=True, forced_debug_id=None):
if self.compute_debug(context, who, method_name):
debug_id = ConsoleDebugLogger.next_id(context.event.get_digest() + str(context.id)) if new_debug_id \
else ConsoleDebugLogger.current_id(context.event.get_digest() + str(context.id))
return ConsoleDebugLogger(self, context, who, method_name, debug_id)
hint = context.event.get_digest() + str(context.id)
if forced_debug_id is not None:
debug_id = forced_debug_id
BaseDebugLogger.ids[hint] = debug_id
elif new_debug_id:
debug_id = BaseDebugLogger.next_id(hint)
else:
debug_id = BaseDebugLogger.current_id(hint)
return self.get_new_debug_logger_instance(context, who, method_name, debug_id)
return NullDebugLogger()
@@ -525,16 +750,17 @@ class SheerkaDebugManager(BaseService):
items_container = getattr(self, item_type_full_name)
for setting in items_container:
if setting.item == item and \
setting.service_name == service and \
setting.method_name == method and \
setting.context_id == context_id and \
setting.context_children == context_children and \
setting.debug_id == debug_id and \
setting.debug_children == debug_children:
setting.service_name == service and \
setting.method_name == method and \
setting.context_id == context_id and \
setting.context_children == context_children and \
setting.debug_id == debug_id and \
setting.debug_children == debug_children:
setting.enabled = enabled
break
else:
items_container.append(DebugItem(item,
items_container.append(DebugItem(item_type,
item,
service,
method,
context_id,
@@ -564,9 +790,9 @@ class SheerkaDebugManager(BaseService):
continue
if (setting.service_name is None or setting.service_name == service_name) and \
(setting.method_name is None or setting.method_name == method_name) and \
(setting.context_id is None or setting.context_id == context.id or (
setting.context_children and context.has_parent(setting.context_id))):
(setting.method_name is None or setting.method_name == method_name) and \
(setting.context_id is None or setting.context_id == context.id or (
setting.context_children and context.has_parent(setting.context_id))):
selected.append(setting.enabled)
if len(selected) == 0:
@@ -599,13 +825,13 @@ class SheerkaDebugManager(BaseService):
continue
if (setting.service_name is None or setting.service_name == service_name) and \
(setting.method_name is None or setting.method_name == method_name) and \
(setting.context_id is None or setting.context_id == context.id or (
setting.context_children and context.has_parent(setting.context_id))) and \
(setting.item is None or
setting.item == "*" or
setting.item == item) and \
(setting.debug_id is None or setting.debug_id == debug_id):
(setting.method_name is None or setting.method_name == method_name) and \
(setting.context_id is None or setting.context_id == context.id or (
setting.context_children and context.has_parent(setting.context_id))) and \
(setting.item is None or
setting.item == "*" or
setting.item == item) and \
(setting.debug_id is None or setting.debug_id == debug_id):
selected.append(setting.enabled)
if len(selected) == 0:
@@ -633,10 +859,10 @@ class SheerkaDebugManager(BaseService):
continue
if (setting.service_name is None or setting.service_name == service_name) and \
(setting.method_name is None or setting.method_name == method_name) and \
(setting.context_id is None or setting.context_id == context.id or (
setting.context_children and context.has_parent(setting.context_id))) and \
(setting.debug_id is None or setting.debug_id == debug_id):
(setting.method_name is None or setting.method_name == method_name) and \
(setting.context_id is None or setting.context_id == context.id or (
setting.context_children and context.has_parent(setting.context_id))) and \
(setting.debug_id is None or setting.debug_id == debug_id):
selected.add(setting.item)
return selected
@@ -919,3 +1145,35 @@ class SheerkaDebugManager(BaseService):
del res["self"]
return res
def list_debug_settings(self):
settings = self.debug_vars_settings + self.debug_concepts_settings + self.debug_rules_settings
return self.sheerka.new(BuiltinConcepts.TO_LIST, body=settings)
def get_new_debug_logger_instance(self, context, who, method_name, debug_id):
if hasattr(self.debug_logger_definition, "__iter__"):
loggers = []
for logger_type in self.debug_logger_definition:
logger = logger_type(self, context, who, method_name, debug_id)
if logger.keep_track:
key = (who, method_name, context.id, debug_id)
self.instantiated_loggers[key] = logger
return TeeDebugLogger(self, context, who, method_name, debug_id, loggers)
logger = self.debug_logger_definition(self, context, who, method_name, debug_id)
if logger.keep_track:
key = (who, method_name, context.id, debug_id)
self.instantiated_loggers[key] = logger
return logger
def get_debugger_logs(self):
res = {}
for k, v in [(k, v) for k, v in self.instantiated_loggers.items() if isinstance(v, ListDebugLogger)]:
key = list(k)
if v.items and v.items[0].type == ListDebugLogger.ITEM_TYPE_ENTERING:
if "source" in v.items[0].args:
key.append(v.items[0].args["source"])
res[tuple(key)] = v.items
return res
@@ -1,15 +1,15 @@
from dataclasses import dataclass
from core.builtin_concepts import BuiltinConcepts
from core.builtin_helpers import expect_one, only_successful, ensure_concept, is_only_successful, ensure_bnf
from core.concept import Concept, DoNotResolve, ConceptParts, InfiniteRecursionResolved, AllConceptParts, \
from core.builtin_helpers import ensure_bnf, ensure_concept, expect_one, is_only_successful, only_successful
from core.concept import AllConceptParts, Concept, ConceptParts, DoNotResolve, InfiniteRecursionResolved, \
concept_part_value
from core.global_symbols import NotInit, CURRENT_OBJ, INIT_AST_PARSERS, INIT_AST_QUESTION_PARSERS
from core.global_symbols import CURRENT_OBJ, INIT_AST_PARSERS, INIT_AST_QUESTION_PARSERS, NotInit
from core.rule import Rule
from core.sheerka.services.SheerkaEvaluateRules import SheerkaEvaluateRules
from core.sheerka.services.SheerkaExecute import ParserInput, SheerkaExecute
from core.sheerka.services.SheerkaRuleManager import PythonConditionExprVisitor
from core.sheerka.services.sheerka_service import BaseService, FailedToCompileError, ChickenAndEggException
from core.sheerka.services.sheerka_service import BaseService, ChickenAndEggException, FailedToCompileError
from core.tokenizer import Tokenizer
from core.utils import unstr_concept
from parsers.BaseExpressionParser import TrueifyVisitor
@@ -38,6 +38,13 @@ class WhereClauseDef:
conditions: list # compiled trueified
@dataclass
class EvaluationHints:
eval_body: bool = None # true if the body must be evaluated
eval_question: bool = None # true if the eval_question must be set
expression_only: bool = None # True if function/methods to forbid functions with side effect
class SheerkaEvaluateConcept(BaseService):
NAME = "EvaluateConcept"
@@ -48,7 +55,7 @@ class SheerkaEvaluateConcept(BaseService):
def initialize(self):
self.sheerka.bind_service_method(self.NAME, self.evaluate_concept, True)
self.sheerka.bind_service_method(self.NAME, self.call_concept, True)
self.sheerka.bind_service_method(self.NAME, self.call_concept, False, as_name="evaluate_question")
self.sheerka.bind_service_method(self.NAME, self.evaluate_question, False)
self.sheerka.bind_service_method(self.NAME, self.set_auto_eval, True)
def initialize_deferred(self, context, is_first_time):
@@ -68,9 +75,9 @@ class SheerkaEvaluateConcept(BaseService):
parent = context.get_parent()
while parent is not None:
if (parent.who == context.who and
parent.action == BuiltinConcepts.EVALUATING_CONCEPT and
parent.obj == concept and
parent.obj.get_compiled() == concept.get_compiled()):
parent.action == BuiltinConcepts.EVALUATING_CONCEPT and
parent.obj == concept and
parent.obj.get_compiled() == concept.get_compiled()):
return True
parent = parent.get_parent()
@@ -97,8 +104,8 @@ class SheerkaEvaluateConcept(BaseService):
:return:
"""
if (eval_body and
ConceptParts.RET in concept.values() and
(ret_value := concept.get_value(ConceptParts.RET)) != NotInit):
ConceptParts.RET in concept.values() and
(ret_value := concept.get_value(ConceptParts.RET)) != NotInit):
return ret_value
else:
return concept
@@ -512,7 +519,7 @@ class SheerkaEvaluateConcept(BaseService):
# when it's a concept, evaluate it
if isinstance(to_resolve, Concept) and \
not context.sheerka.isinstance(to_resolve, BuiltinConcepts.RETURN_VALUE):
not context.sheerka.isinstance(to_resolve, BuiltinConcepts.RETURN_VALUE):
evaluated = self.evaluate_concept(sub_context, to_resolve)
sub_context.add_values(return_values=evaluated)
@@ -532,6 +539,8 @@ class SheerkaEvaluateConcept(BaseService):
value = current_concept.get_value(var[0])
if value != NotInit:
sub_context.add_to_short_term_memory(var[0], current_concept.get_value(var[0]))
# KSI 2021-08-10 It seems that a copy is not needed here, as it's the first thing done ine execute()
use_copy = to_resolve.copy() if isinstance(to_resolve, list) else to_resolve
r = self.sheerka.execute(sub_context, use_copy, CONCEPT_EVALUATION_STEPS)
@@ -601,22 +610,30 @@ class SheerkaEvaluateConcept(BaseService):
return res
def evaluate_concept(self, context, concept: Concept, eval_body=False, validation_only=False, metadata=None):
def evaluate_concept(self, context, concept: Concept, hints: EvaluationHints = None, metadata=None):
"""
Evaluation a concept
ie : resolve its body
:param context:
:param concept:
:param eval_body:
:param validation_only: When set, the body is never evaluated, whatever eval_body or EVAL_BODY_REQUESTED
:param hints:
:param metadata: list of metadata to evaluate ('pre', 'post'...)
:return: value of the evaluation or error
"""
failed_to_evaluate_body = False
hints = hints or EvaluationHints()
if concept.get_hints().is_evaluated:
return self.apply_ret(concept, eval_body or context.in_context(BuiltinConcepts.EVAL_BODY_REQUESTED))
return self.apply_ret(concept, hints.eval_body or context.in_context(BuiltinConcepts.EVAL_BODY_REQUESTED))
to_reset = set()
if isinstance(hints.eval_body, bool) and not hints.eval_body:
to_reset.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
if isinstance(hints.eval_question, bool) and not hints.eval_question:
to_reset.add(BuiltinConcepts.EVAL_QUESTION_REQUESTED)
if isinstance(hints.expression_only, bool) and not hints.expression_only:
to_reset.add(BuiltinConcepts.EXPRESSION_ONLY_REQUESTED)
# if concept.get_hints().use_copy:
# raise Exception("Use copy")
@@ -635,17 +652,21 @@ class SheerkaEvaluateConcept(BaseService):
# return concept
desc = f"Evaluating concept {concept}"
with context.push(BuiltinConcepts.EVALUATING_CONCEPT, concept, desc=desc) as sub_context:
sub_context.add_inputs(eval_body=eval_body)
if eval_body:
# ask for body evaluation
with context.push(BuiltinConcepts.EVALUATING_CONCEPT, concept, desc=desc, reset_hints=to_reset) as sub_context:
sub_context.add_inputs(hints=hints)
# update context with evaluate_concept parameters
if hints.eval_body:
sub_context.protected_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
if validation_only:
if hints.eval_question:
sub_context.protected_hints.add(BuiltinConcepts.EVAL_QUESTION_REQUESTED)
if hints.expression_only:
# Never call methods with side effect in this concept or sub concepts
sub_context.protected_hints.add(BuiltinConcepts.EXPRESSION_ONLY_REQUESTED)
# auto evaluate commands
# update context with evaluate_concept parameters
if context.sheerka.isa(concept, context.sheerka.new(BuiltinConcepts.AUTO_EVAL)):
sub_context.protected_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
@@ -659,10 +680,10 @@ class SheerkaEvaluateConcept(BaseService):
except ChickenAndEggException as ex:
return ex.error
# to make sure of the order, it don't use ConceptParts.get_parts()
# to make sure of the order, it does not use ConceptParts.get_parts()
# variables must be evaluated first, body must be evaluated before where
all_metadata_to_eval = metadata or self.compute_metadata_to_eval(sub_context, concept)
if validation_only and ConceptParts.BODY in all_metadata_to_eval:
if hints.expression_only and ConceptParts.BODY in all_metadata_to_eval:
all_metadata_to_eval.remove(ConceptParts.BODY)
for metadata_to_eval in all_metadata_to_eval:
@@ -681,7 +702,7 @@ class SheerkaEvaluateConcept(BaseService):
var_name,
concept,
True,
not sub_context.in_context(BuiltinConcepts.EVAL_BODY_REQUESTED),
sub_context.in_context(BuiltinConcepts.EXPRESSION_ONLY_REQUESTED),
w_clause)
else:
# Do not send the current concept for the properties
@@ -690,7 +711,7 @@ class SheerkaEvaluateConcept(BaseService):
var_name,
concept,
True,
not sub_context.in_context(BuiltinConcepts.EVAL_BODY_REQUESTED),
sub_context.in_context(BuiltinConcepts.EXPRESSION_ONLY_REQUESTED),
w_clause)
if isinstance(resolved, Concept) and not sub_context.sheerka.is_success(resolved):
@@ -722,7 +743,7 @@ class SheerkaEvaluateConcept(BaseService):
part_key,
concept,
force_concept_eval,
not sub_context.in_context(BuiltinConcepts.EVAL_BODY_REQUESTED),
sub_context.in_context(BuiltinConcepts.EXPRESSION_ONLY_REQUESTED),
None)
# 'FATAL' error is detected, let's stop
@@ -766,10 +787,25 @@ class SheerkaEvaluateConcept(BaseService):
return self.apply_ret(concept, sub_context.in_context(BuiltinConcepts.EVAL_BODY_REQUESTED))
def call_concept(self, context, concept, *args, **kwargs):
return self.call_concept_with_args(context,
concept,
hints=EvaluationHints(eval_body=True, eval_question=False),
*args,
**kwargs)
def evaluate_question(self, context, concept, *args, **kwargs):
return self.call_concept_with_args(context,
concept,
hints=EvaluationHints(eval_body=True, eval_question=True),
*args,
**kwargs)
def call_concept_with_args(self, context, concept, hints, *args, **kwargs):
"""
call the concept using either args or kwargs (not both)
:param context:
:param concept:
:param hints:
:param args:
:param kwargs:
:return:
@@ -780,9 +816,13 @@ class SheerkaEvaluateConcept(BaseService):
concept.get_hints().is_instance = True
concept.get_hints().is_evaluated = False # force evaluation
# TODO : update the variables before calling the concept
for param_name, arg in zip(concept.get_metadata().parameters, args):
concept.get_compiled()[param_name] = DoNotResolve(arg)
evaluated = self.evaluate_concept(context, concept)
for param_name, param_value in kwargs.items():
concept.get_compiled()[param_name] = DoNotResolve(param_value)
evaluated = self.evaluate_concept(context, concept, hints=hints)
if self.sheerka.has_error(context, evaluated):
raise ConceptEvalException(evaluated)
+27 -5
View File
@@ -2,9 +2,9 @@ import core.utils
from cache.FastCache import FastCache
from core.builtin_concepts import BuiltinConcepts, ReturnValueConcept
from core.concept import ConceptParts
from core.global_symbols import NotFound, NO_MATCH, EVENT_CONCEPT_CREATED, EVENT_CONCEPT_MODIFIED, EVENT_CONCEPT_DELETED
from core.global_symbols import EVENT_CONCEPT_CREATED, EVENT_CONCEPT_DELETED, EVENT_CONCEPT_MODIFIED, NO_MATCH, NotFound
from core.sheerka.services.sheerka_service import BaseService
from core.tokenizer import Tokenizer, TokenKind, Token, Keywords
from core.tokenizer import Keywords, Token, TokenKind, Tokenizer
PARSE_STEPS = [BuiltinConcepts.BEFORE_PARSING, BuiltinConcepts.PARSING, BuiltinConcepts.AFTER_PARSING]
PARSE_AND_EVAL_STEPS = PARSE_STEPS + [BuiltinConcepts.BEFORE_EVALUATION,
@@ -179,6 +179,24 @@ class ParserInput:
return True
return False
def clone(self):
clone = ParserInput(self.text)
clone.tokens = self.tokens
clone.length = self.length
clone.yield_oef = self.yield_oef
clone.start = self.start
clone.end = self.end
clone.sub_text = self.sub_text
clone.sub_tokens = self.sub_tokens
clone.from_tokens = self.from_tokens
clone.pos = self.pos
clone.token = self.token
return clone
def sub_part(self, start, end, yield_oef=None):
return ParserInput(self.text, self.tokens, self.length, start, end, yield_oef)
class SheerkaExecute(BaseService):
"""
@@ -227,6 +245,7 @@ class SheerkaExecute(BaseService):
self.sheerka.bind_service_method(self.NAME, self.parse_function, False, visible=False)
self.sheerka.bind_service_method(self.NAME, self.parse_python, False, visible=False)
self.sheerka.bind_service_method(self.NAME, self.parse_expression, False, visible=False)
self.sheerka.bind_service_method(self.NAME, self.clear_parser_cache, True)
self.reset_registered_evaluators()
self.reset_registered_parsers()
@@ -274,6 +293,9 @@ class SheerkaExecute(BaseService):
use_classes=True)
self.question_parsers = [p.name for p in question_parsers]
def clear_parser_cache(self):
self.parsers_cache.clear()
@staticmethod
def get_grouped(evaluators, use_classes=False):
"""
@@ -442,7 +464,7 @@ class SheerkaExecute(BaseService):
return None
def add_to_parser_cache(self, parsers_key, text, return_value):
if parsers_key is None:
if not self.sheerka.enable_parser_caching or parsers_key is None:
return
key = (parsers_key, text)
@@ -517,7 +539,7 @@ class SheerkaExecute(BaseService):
pass
# 3. Try the cache
if to_process and parsers_key:
if self.sheerka.enable_parser_caching and to_process and parsers_key:
processed = []
for return_value in to_process:
to_parse_as_str = self.get_input_as_text(return_value.body.body) \
@@ -825,7 +847,7 @@ class SheerkaExecute(BaseService):
with context.push(BuiltinConcepts.PARSING, action_context, who=who, desc=desc) as sub_context:
if (prop in (Keywords.WHERE, Keywords.PRE, ConceptParts.WHERE, ConceptParts.PRE, Keywords.WHEN) or
is_question):
is_question):
sub_context.protected_hints.add(BuiltinConcepts.EVAL_QUESTION_REQUESTED)
# disable all parsers but the requested ones
+10 -7
View File
@@ -6,7 +6,7 @@ from core.concept import Concept, DEFINITION_TYPE_BNF
from core.global_symbols import NotFound
from core.sheerka.services.SheerkaConceptManager import SheerkaConceptManager
from core.sheerka.services.sheerka_service import BaseService
from core.tokenizer import Tokenizer, TokenKind
from core.tokenizer import TokenKind, Tokenizer
from core.utils import merge_sets
@@ -49,7 +49,7 @@ class SheerkaIsAManager(BaseService):
concept_to_use = concept
if BuiltinConcepts.ISA in concept_to_use.get_metadata().props and \
concept_set in concept_to_use.get_metadata().props[BuiltinConcepts.ISA]:
concept_set in concept_to_use.get_metadata().props[BuiltinConcepts.ISA]:
return self.sheerka.ret(
self.NAME,
False,
@@ -66,7 +66,8 @@ class SheerkaIsAManager(BaseService):
return res
concept.set_prop(BuiltinConcepts.ISA, new_concept_set)
res = self.add_concept_to_set(context, concept, concept_set)
# KSI 2021-08-12. Make sure to use the newly created concept to put in cache
res = self.add_concept_to_set(context, res.body.body, concept_set)
return res
else:
concept.set_prop(BuiltinConcepts.ISA, new_concept_set)
@@ -144,15 +145,16 @@ class SheerkaIsAManager(BaseService):
if not self.isaset(context, sub_concept):
return self.sheerka.new(BuiltinConcepts.NOT_A_SET, body=concept)
# is it a valid concept ?
sub_concept = core.builtin_helpers.ensure_evaluated(context, sub_concept)
if not self.sheerka.is_success(sub_concept):
return sub_concept
# first, try to see if sub_concept has it's own group entry
ids = self.sheerka.om.get(self.CONCEPTS_GROUPS_ENTRY, sub_concept.id)
concepts = self._get_concepts(context, ids, True)
# aggregate with en entries from its body
sub_concept = core.builtin_helpers.ensure_evaluated(context, sub_concept)
if not self.sheerka.is_success(sub_concept):
return sub_concept
if self.isaset(context, sub_concept.body):
other_concepts = _get_set_elements(sub_concept.body)
if not self.sheerka.is_success(other_concepts):
@@ -169,6 +171,7 @@ class SheerkaIsAManager(BaseService):
if (res := self.sheerka.om.get(self.CONCEPTS_IN_GROUPS_ENTRY, concept.id)) is not NotFound:
return res
# get the elements that are in the set
res = _get_set_elements(concept)
# put in cache
@@ -15,7 +15,7 @@ from core.rule import Rule, ACTION_TYPE_PRINT
from core.sheerka.Sheerka import RECOGNIZED_BY_NAME, RECOGNIZED_BY_ID
from core.sheerka.services.sheerka_service import BaseService, FailedToCompileError, UnknownVariableError
from core.tokenizer import Keywords, TokenKind, Token
from core.utils import merge_dictionaries, merge_sets, get_safe_str_value
from core.utils import merge_dicts, merge_sets, get_safe_str_value
from evaluators.PythonEvaluator import PythonEvaluator
from parsers.BaseExpressionParser import AndNode, ExpressionVisitor, VariableNode, ComparisonNode, FunctionNode, \
ComparisonType, NotNode, NameExprNode
@@ -503,8 +503,11 @@ class GetConditionExprVisitor(ExpressionVisitor):
def get_object_name(self, obj, objects=None):
"""
object found during the parsing are not serialized
They are kept in a dictionary and this function returns a new name for every new object
:return:
They are kept in a dictionary.
This function returns a new name for every new object
:param obj: object for which a name is to be created
:param objects: already created names (it's a dictionary)
:return: tuple(name created, dictionary of already created names)
"""
if objects is None:
objects = {}
@@ -526,9 +529,9 @@ class GetConditionExprVisitor(ExpressionVisitor):
def add_variable(self, target):
"""
Create a new variable
Create a new variable name of the object 'target'
:param target:
:return:
:return: generated name
"""
var_name = f"__x_{self.var_counter:02}__"
self.var_counter += 1
@@ -881,7 +884,7 @@ class PythonConditionExprVisitorObj:
return PythonConditionExprVisitorObj(get_source(left.text, right.text),
get_source(left.source, right.source),
merge_dictionaries(left.objects, right.objects),
merge_dicts(left.objects, right.objects),
merge_sets(left.variables, right.variables),
merge_sets(left.not_variables, right.not_variables))
@@ -903,7 +906,7 @@ class PythonConditionExprVisitorObj:
return PythonConditionExprVisitorObj(get_source(left.text, right.text),
get_source(left.source, right.source),
merge_dictionaries(left.objects, right.objects),
merge_dicts(left.objects, right.objects),
merge_sets(left.variables, right.variables),
merge_sets(left.not_variables, right.not_variables))
@@ -929,7 +932,7 @@ class PythonConditionExprVisitorObj:
return PythonConditionExprVisitorObj(text,
get_source(left.source, right.source),
merge_dictionaries(left.objects, right.objects),
merge_dicts(left.objects, right.objects),
merge_sets(left.variables, right.variables),
merge_sets(left.not_variables, right.not_variables))
@@ -20,9 +20,12 @@ class Variable(ServiceObj):
def get_key(self):
return f"{self.who}|{self.key}"
def __str__(self):
def __repr__(self):
return f"({self.who}){self.key}={self.value}"
def __str__(self):
return f"{self.who}.{self.key}={self.value}"
@dataclass
class InternalObj:
@@ -49,7 +52,8 @@ class SheerkaVariableManager(BaseService):
self.sheerka.name: {"enable_process_return_values",
"save_execution_context",
"enable_process_rules",
"enable_commands_backup"}
"enable_commands_backup",
"enable_parser_caching"}
}
def initialize(self):
@@ -79,7 +83,7 @@ class SheerkaVariableManager(BaseService):
def record_var(self, context, who, key, value):
"""
Internal set
:param context:
:param who: entity that owns the key (acts as a namespace)
:param key:
@@ -96,6 +100,9 @@ class SheerkaVariableManager(BaseService):
setattr(service, key, value)
def load_var(self, who, key):
"""
Internal get
"""
variable = self.sheerka.om.get(self.VARIABLES_ENTRY, who + "|" + key)
if variable is NotFound:
return NotFound