Fixed some misbehaviours regarding question() + added #import functionality when restoring

This commit is contained in:
2020-09-23 20:08:15 +02:00
parent 17c74d3808
commit eeeed0f110
20 changed files with 164 additions and 83 deletions
+12
View File
@@ -0,0 +1,12 @@
# admin helpers
def concept explain as get_results() | filter("id == 0") | recurse(2)
set_isa(c:explain:, __AUTO_EVAL)
def concept explain last as get_last_results() | filter("id == 0") | recurse(2)
set_isa(c:explain last:, __AUTO_EVAL)
def concept explain x as get_results() | filter(f"id == {x}") | recurse(3) where x
set_isa(c:explain x:, __AUTO_EVAL)
def concept explain x values where x as get_results() | filter(f"id=={x}") | format_d
set_isa(c:explain x values:, __AUTO_EVAL)
+4 -6
View File
@@ -1,12 +1,10 @@
#import admin
def concept one as 1
def concept two as 2
def concept explain as get_results() | filter("id == 0") | recurse(2)
set_isa(c:explain:, __AUTO_EVAL)
def concept explain last as get_last_results() | filter("id == 0") | recurse(2)
set_isa(c:explain last:, __AUTO_EVAL)
def concept explain x as get_results() | filter(f"id == {x}") | recurse(3) where x
set_isa(c:explain x:, __AUTO_EVAL)
def concept number
def concept apple
def concept table
def concept location
def concept x is on y as set_attr(x, location, y)
def concept x is a y as set_isa(x, y)
def concept x is a y as isa(x, y) pre is_question()
+2 -2
View File
@@ -63,8 +63,8 @@ class UnreferencedNamesVisitor(ConceptNodeVisitor):
if ("Call", "func") in parents: # name of the function
return
if ("Assign", "targets") in parents: # variable which is assigned
return
# if ("Assign", "targets") in parents: # variable which is assigned
# return
if self.can_be_discarded(self.sheerka.objvalue(node), parents):
return
+13
View File
@@ -181,6 +181,7 @@ BuiltinErrors = [str(e) for e in {
BuiltinConcepts.NOT_FOUND,
BuiltinConcepts.INVALID_LESSER_OPERATION,
BuiltinConcepts.INVALID_GREATEST_OPERATION,
# DO NOT PUT NOT_INITIALIZED. It's not an error
}]
"""
@@ -534,3 +535,15 @@ class PythonSecurityError(Concept):
self.set_value("column", column) # column number
self.set_value(ConceptParts.BODY, source_code) # code being executed
self.metadata.is_evaluated = True
class NotFound(Concept):
def __init__(self, body=None):
super().__init__(BuiltinConcepts.NOT_FOUND,
True,
False,
BuiltinConcepts.NOT_FOUND)
self.set_value(ConceptParts.BODY, body)
def __repr__(self):
return f"({self.metadata.id}){self.metadata.name}, body={self.get_value(ConceptParts.BODY)}"
+12 -1
View File
@@ -194,7 +194,7 @@ def resolve_ambiguity(context, concepts):
else:
for c in by_complexity[complexity]:
evaluated = context.sheerka.evaluate_concept(context, c, metadata=["pre"])
if evaluated.key == c.key:
if context.sheerka.is_success(evaluated) or evaluated.key == c.key:
remaining_concepts.append(c)
if len(remaining_concepts) > 0:
@@ -214,6 +214,13 @@ def resolve_ambiguity(context, concepts):
def get_condition_complexity(concept, concept_part_str):
"""
Need to find a proper algorithm to compute the complexity of a concept
So far, the concept is considered as complex if it has pre
:param concept:
:param concept_part_str:
:return:
"""
concept_part_value = getattr(concept.metadata, concept_part_str)
if concept_part_value is None or concept_part_value.strip() == 0:
return 0
@@ -374,6 +381,7 @@ def evaluate(context,
desc=None,
eval_body=True,
eval_where=True,
is_question=False,
expect_success=False,
stm=None):
"""
@@ -384,6 +392,7 @@ def evaluate(context,
:param desc:
:param eval_body:
:param eval_where:
:param is_question:
:param expect_success:
:param stm: short term memories entries
:return:
@@ -400,6 +409,8 @@ def evaluate(context,
if expect_success:
sub_context.protected_hints.add(BuiltinConcepts.EVAL_UNTIL_SUCCESS_REQUESTED)
if is_question:
sub_context.protected_hints.add(BuiltinConcepts.EVAL_QUESTION_REQUESTED)
if stm:
+5 -1
View File
@@ -191,6 +191,7 @@ class ExecutionContext:
new.protected_hints.update(self.protected_hints)
self._children.append(new)
return new
def add_preprocess(self, name, **kwargs):
@@ -295,6 +296,9 @@ class ExecutionContext:
to_str = self.return_value_to_str(r)
self._logger.debug(f"[{self._id:2}]" + self._tab + "-> " + to_str)
def debug(self, text):
print(text)
def get_parent(self):
return self._parent
@@ -309,7 +313,7 @@ class ExecutionContext:
def in_private_context(self, concept_key):
return concept_key in self.private_hints
def add_to_private_hints (self, concept_key):
def add_to_private_hints(self, concept_key):
self.private_hints.add(concept_key)
def add_to_protected_hints(self, concept_key):
+37 -15
View File
@@ -1,4 +1,5 @@
import time
from os import path
from core.builtin_concepts import BuiltinConcepts
from core.sheerka.services.sheerka_service import BaseService
@@ -50,6 +51,38 @@ class SheerkaAdmin(BaseService):
:return:
"""
def restore_from_file(file_name):
_nb_lines, _nb_instructions, _nb_lines_in_error = 0, 0, 0
if not path.exists(file_name):
self.sheerka.log.error(f"\u001b[31mFile '{file_name}' is not found !\u001b[0m")
return 0, 0, 1
with open(file_name, "r") as f:
for line in f.readlines():
_nb_lines += 1
line = line.strip()
if line.startswith("#import "):
to_import = "_concepts_" + line[8:] + ".txt"
self.sheerka.log.info(f"Importing {to_import}")
res = restore_from_file(to_import)
_nb_lines += res[0]
_nb_instructions += res[1]
_nb_lines_in_error += res[2]
continue
if line == "" or line.startswith("#"):
continue
self.sheerka.log.info(line)
_nb_instructions += 1
res = self.sheerka.evaluate_user_input(line)
if len(res) > 1 or not res[0].status:
_nb_lines_in_error += 1
self.sheerka.log.error("\u001b[31mError detected !\u001b[0m")
return _nb_lines, _nb_instructions, _nb_lines_in_error
if concept_file == "full":
concept_file = CONCEPTS_FILE_ALL_CONCEPTS
@@ -58,20 +91,8 @@ class SheerkaAdmin(BaseService):
try:
start = time.time_ns()
nb_lines = 0
nb_lines_in_error = 0
self.sheerka.during_restore = True
with open(concept_file, "r") as f:
for line in f.readlines():
nb_lines += 1
line = line.strip()
if line == "" or line.startswith("#"):
continue
self.sheerka.log.info(line)
res = self.sheerka.evaluate_user_input(line)
if len(res) > 1 or not res[0].status:
nb_lines_in_error += 1
self.sheerka.log.error("\u001b[31mError detected !\u001b[0m")
nb_lines, nb_instructions, nb_lines_in_error = restore_from_file(concept_file)
self.sheerka.during_restore = False
stop = time.time_ns()
@@ -79,12 +100,13 @@ class SheerkaAdmin(BaseService):
dt = nano_sec / 1e6
elapsed = f"{dt} ms" if dt < 1000 else f"{dt / 1000} s"
self.sheerka.log.info(f"Imported {nb_lines} line(s) in {elapsed}.")
self.sheerka.log.info(f"{nb_instructions} instruction(s).")
if nb_lines_in_error > 0:
self.sheerka.log.info(f"\u001b[31m{nb_lines_in_error} errors(s) found.\u001b[0m")
else:
self.sheerka.log.info(f"No error.")
except IOError:
pass
except IOError as e:
raise e
def concepts(self):
return self.sheerka.sdp.list(self.sheerka.CONCEPTS_BY_ID_ENTRY)
@@ -4,13 +4,12 @@ from core.concept import Concept, DEFINITION_TYPE_DEF, ensure_concept, DEFINITIO
from core.sheerka.services.sheerka_service import BaseService
from sdp.sheerkaDataProvider import SheerkaDataProviderDuplicateKeyError
BNF_NODE_PARSER_CLASS = "parsers.BnfNodeParser_Old.BnfNodeParser"
BASE_NODE_PARSER_CLASS = "parsers.BaseNodeParser.BaseNodeParser"
class SheerkaCreateNewConcept(BaseService):
"""
Manage the creation of a new concept
Manages the creation of a new concept
"""
NAME = "CreateNewConcept"
@@ -166,6 +166,7 @@ class SheerkaEvaluateConcept(BaseService):
where_clause_def.trueified,
desc=f"Apply where clause on '{where_clause_def.prop}'",
expect_success=True,
is_question=True,
stm={where_clause_def.prop: r.body})
one_res = expect_one(context, evaluation_res)
if one_res.status:
@@ -286,7 +287,6 @@ class SheerkaEvaluateConcept(BaseService):
current_prop,
current_concept,
force_evaluation,
expect_success,
where_clause_def):
"""
Resolve a variable or a Concept
@@ -295,7 +295,6 @@ class SheerkaEvaluateConcept(BaseService):
:param current_prop: current property or ConceptPart
:param current_concept: current concept
:param force_evaluation: Force body evaluation
:param expect_success: for PythonEvaluator, try all possibilities to find a positive result
:param where_clause_def: intermediate where clause for variables
:return:
"""
@@ -332,8 +331,10 @@ class SheerkaEvaluateConcept(BaseService):
if force_evaluation:
sub_context.protected_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
if expect_success:
if current_prop in (ConceptParts.WHERE, ConceptParts.PRE):
sub_context.protected_hints.add(BuiltinConcepts.EVAL_UNTIL_SUCCESS_REQUESTED)
if current_prop == ConceptParts.WHERE:
sub_context.protected_hints.add(BuiltinConcepts.EVAL_QUESTION_REQUESTED)
# when it's a concept, evaluate it
@@ -384,7 +385,6 @@ class SheerkaEvaluateConcept(BaseService):
current_prop,
current_concept,
force_evaluation,
expect_success,
where_clause_def):
"""When dealing with a list, there are two possibilities"""
# It may be a list of ReturnValueConcept to execute (always the case for metadata)
@@ -399,7 +399,6 @@ class SheerkaEvaluateConcept(BaseService):
current_prop,
current_concept,
force_evaluation,
expect_success,
where_clause_def)
res = []
@@ -416,7 +415,6 @@ class SheerkaEvaluateConcept(BaseService):
current_prop,
current_concept,
force_evaluation,
expect_success,
where_clause_def)
if self.sheerka.isinstance(r, BuiltinConcepts.CONCEPT_EVAL_ERROR):
return r
@@ -476,10 +474,10 @@ class SheerkaEvaluateConcept(BaseService):
if isinstance(prop_ast, list):
# Do not send the current concept for the properties
resolved = self.resolve_list(sub_context, prop_ast, var_name, None, True, False, w_clause)
resolved = self.resolve_list(sub_context, prop_ast, var_name, None, True, w_clause)
else:
# Do not send the current concept for the properties
resolved = self.resolve(sub_context, prop_ast, var_name, None, True, False, w_clause)
resolved = self.resolve(sub_context, prop_ast, var_name, None, True, w_clause)
if isinstance(resolved, Concept) and not sub_context.sheerka.is_success(resolved):
resolved.set_value("concept", concept) # since current concept was not sent
@@ -504,18 +502,8 @@ class SheerkaEvaluateConcept(BaseService):
# otherwise no need to force
force_concept_eval = False if part_key == ConceptParts.BODY else True
# when resolving predicate (where or pre), we need to make sure that PythonEvaluator
# will try every possibilities before returning False
expect_success = part_key in (ConceptParts.WHERE, ConceptParts.PRE)
# resolve
resolved = self.resolve(sub_context,
metadata_ast,
part_key,
concept,
force_concept_eval,
expect_success,
None)
resolved = self.resolve(sub_context, metadata_ast, part_key, concept, force_concept_eval, None)
# 'FATAL' error is detected, let's stop
if isinstance(resolved, Concept) and not sub_context.sheerka.is_success(resolved):
@@ -100,4 +100,9 @@ class SheerkaModifyConcept(BaseService):
:return:
"""
ensure_concept()
return concept.get_value(attribute)
if not self.sheerka.is_success(concept):
return concept
if (value := concept.get_value(attribute)) == BuiltinConcepts.NOT_INITIALIZED:
return self.sheerka.new(BuiltinConcepts.NOT_FOUND, body={"#concept": concept, "#attr": attribute})
return value
+17 -19
View File
@@ -10,27 +10,25 @@ class SheerkaQuestion(BaseService):
super().__init__(sheerka)
def initialize(self):
self.sheerka.bind_service_method(self.question, False)
# self.sheerka.bind_service_method(self.question, False)
self.sheerka.bind_service_method(self.is_question, False)
def question(self, context, q):
"""
Evaluate q in the context in a question
:param context:
:param q:
:return:
"""
if isinstance(q, Concept):
with context.push(BuiltinConcepts.EVALUATE_CONCEPT, q, desc=f"Evaluating question '{q}'") as sub_context:
sub_context.global_hints.add(BuiltinConcepts.EVAL_QUESTION_REQUESTED)
sub_context.global_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
sub_context.global_hints.add(BuiltinConcepts.EVAL_UNTIL_SUCCESS_REQUESTED)
sub_context.protected_hints.add(BuiltinConcepts.RETURN_BODY_REQUESTED)
evaluated = self.sheerka.evaluate_concept(sub_context, q)
return evaluated
# def question(self, context, q):
# """
# Evaluate q in the context in a question
# :param context:
# :param q:
# :return:
# """
#
# if isinstance(q, Concept):
# with context.push(BuiltinConcepts.EVALUATE_CONCEPT, q, desc=f"Evaluating question '{q}'") as sub_context:
# sub_context.global_hints.add(BuiltinConcepts.EVAL_QUESTION_REQUESTED)
# sub_context.global_hints.add(BuiltinConcepts.EVAL_UNTIL_SUCCESS_REQUESTED)
#
# evaluated = self.sheerka.evaluate_concept(sub_context, q)
#
# return evaluated
def is_question(self, context):
"""
+15 -1
View File
@@ -1,5 +1,7 @@
import core.utils
from core.ast.nodes import python_to_concept
from core.builtin_concepts import ParserResultConcept, ReturnValueConcept, BuiltinConcepts
from core.builtin_helpers import get_names
from core.concept import Concept, DEFINITION_TYPE_BNF, DEFINITION_TYPE_DEF
from core.sheerka.services.SheerkaExecute import ParserInput
from core.tokenizer import TokenKind, Tokenizer
@@ -7,6 +9,7 @@ from evaluators.BaseEvaluator import OneReturnValueEvaluator
from parsers.BaseParser import NotInitializedNode
from parsers.BnfNodeParser import ParsingExpression, ParsingExpressionVisitor
from parsers.DefaultParser import DefConceptNode, NameNode
from parsers.PythonParser import PythonNode
class ConceptOrRuleNameVisitor(ParsingExpressionVisitor):
@@ -141,7 +144,18 @@ class AddConceptEvaluator(OneReturnValueEvaluator):
return set(visitor.names)
#
# other (python code and concept)
# Case of python code
#
if isinstance(ret_value.value, ParserResultConcept) and isinstance(ret_value.value.value, PythonNode):
if len(concept_name) > 1:
python_node = ret_value.value.value
as_concept_node = python_to_concept(python_node.ast_)
names = get_names(sheerka, as_concept_node)
variables = filter(lambda x: x in concept_name, names)
return set(variables)
#
# Concept
#
if isinstance(ret_value.value, ParserResultConcept) and len(concept_name) > 1:
variables = set()
+2 -1
View File
@@ -26,7 +26,8 @@ class PostExecutionEvaluator(OneReturnValueEvaluator):
def eval(self, context, return_value):
# only support the rule for the COMMANDS
body = return_value.body.body
#body = return_value.body.body
body = context.sheerka.objvalue(return_value)
return context.sheerka.ret(
self.name,
True,
@@ -36,10 +36,12 @@ class PrepareEvalQuestionEvaluator(OneReturnValueEvaluator):
self.name,
True, sheerka.new(BuiltinConcepts.USER_INPUT, body=self.question, user_name=context.event.user_id))
root = context.get_parents(lambda ec: ec.action == BuiltinConcepts.PROCESS_INPUT)
root = context.get_parents(lambda ec: ec.action in (BuiltinConcepts.EVALUATING_CONCEPT,
BuiltinConcepts.PROCESS_INPUT))
root = root[0] if root else context
root.protected_hints.add(BuiltinConcepts.EVAL_QUESTION_REQUESTED)
root.protected_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
root.protected_hints.add(BuiltinConcepts.RETURN_BODY_REQUESTED)
root.add_to_protected_hints(BuiltinConcepts.EVAL_QUESTION_REQUESTED)
root.add_to_protected_hints(BuiltinConcepts.EVAL_UNTIL_SUCCESS_REQUESTED)
root.add_to_protected_hints(BuiltinConcepts.EVAL_BODY_REQUESTED)
root.add_to_protected_hints(BuiltinConcepts.RETURN_BODY_REQUESTED)
return new_text_to_parse
+8
View File
@@ -1184,6 +1184,14 @@ class SyaNodeParser(BaseNodeParser):
infix_to_postfix.finalize(self.parser_input.pos)
_add_forked_to_res()
if context.in_context(BuiltinConcepts.DEBUG):
context.debug(f"Parsing {parser_input}")
context.debug(f"{len(res)} InfixToPostFix(s) found")
for i, r in enumerate(res):
context.debug(f"#{i}")
for line in r.debug:
context.debug(line)
return res
def postfix_to_item(self, sheerka, postfixed):
@@ -215,7 +215,6 @@ class TestSheerkaCreateNewConcept(TestUsingMemoryBasedSheerka):
assert sheerka.cache_manager.get(sheerka.CONCEPTS_REFERENCES_ENTRY, twenties.id) is None
class TestSheerkaCreateNewConceptFileBased(TestUsingFileBasedSheerka):
def test_i_can_add_several_concepts(self):
sheerka = self.get_sheerka()
+4 -1
View File
@@ -146,8 +146,11 @@ class TestBuiltinHelpers(TestUsingMemoryBasedSheerka):
@pytest.mark.parametrize("concepts, expected", [
([], []),
([Concept("foo", pre="False"), Concept("bar")], ["bar"]),
([Concept("foo", pre="True"), Concept("bar")], ["foo"]),
([Concept("foo").def_var("a"), Concept("bar")], ["bar"]),
([Concept("foo").def_var("a"), Concept("bar")], ["bar"]), # less variables is better
([Concept("foo"), Concept("bar")], ["foo", "bar"]),
([Concept("foo", pre="is_question()"), Concept("bar")], ["bar"]),
])
def test_i_can_resolve_ambiguity_when_empty(self, concepts, expected):
context = self.get_context()
+5 -3
View File
@@ -119,7 +119,7 @@ class TestAddConceptEvaluator(TestUsingMemoryBasedSheerka):
def_concept_return_value = self.get_def_concept(
name="x plus y",
where=self.pretval(Concept("u is a v").def_var("u").def_var("v"), source="x is a number"),
body=self.pretval(Concept("add a b").def_var("a").def_var("b"), source="add x y"),)
body=self.pretval(Concept("add a b").def_var("a").def_var("b"), source="add x y"), )
evaluated = AddConceptEvaluator().eval(context, def_concept_return_value)
@@ -186,8 +186,10 @@ class TestAddConceptEvaluator(TestUsingMemoryBasedSheerka):
@pytest.mark.parametrize("expression, name, expected", [
("isinstance(a, str)", "a b", {"a"}),
("a.location=b", "a is in b", {"a", "b"}),
("a.location=b", "'a' is in b", {"b"}),
("a.location = b", "a is in b", {"a", "b"}),
("a.location = b", "'a' is in b", {"b"}),
("date.today()", "what is the date", set()),
("a.location", "where is a", {"a"})
])
def test_i_can_get_variables_from_python_node_when_long_name(self, expression, name, expected):
ret_val = self.get_concept_part(expression)
@@ -45,5 +45,6 @@ class TestPrepareEvalQuestionEvaluator(TestUsingMemoryBasedSheerka):
assert res.body.body == expected
assert BuiltinConcepts.EVAL_QUESTION_REQUESTED in context.protected_hints
assert BuiltinConcepts.EVAL_UNTIL_SUCCESS_REQUESTED in context.protected_hints
assert BuiltinConcepts.EVAL_BODY_REQUESTED in context.protected_hints
assert BuiltinConcepts.RETURN_BODY_REQUESTED in context.protected_hints
+7 -6
View File
@@ -1061,26 +1061,27 @@ as:
"def concept number",
"set_isa(one, number)",
"def concept q from q ? as question(q)",
"set_auto_eval(q)",
"def concept is_a from x is a y as isa(x,y) pre in_context(BuiltinConcepts.EVAL_QUESTION_REQUESTED)",
"set_is_greater_than(BuiltinConcepts.PRECEDENCE, c:is_a:, c:q:)"
"set_is_greater_than(BuiltinConcepts.PRECEDENCE, c:is_a:, c:q:)",
]
sheerka = self.init_scenario(init)
res = sheerka.evaluate_user_input("one is a number ?") # automatically evaluated
assert len(res) == 1
assert res[0].status
assert res[0].body
assert res[0].body == True # the body MUST be a boolean
res = sheerka.evaluate_user_input("foo is a number ?") # automatically evaluated
assert len(res) == 1
assert res[0].status
assert not res[0].body
assert res[0].body == False # the body MUST be a boolean
# Sanity, when there is only one 'is a' concept. It's chosen regardless of the PRE condition
# x is a y is supposed to be a question. It cannot be used if not in a context of a question
res = sheerka.evaluate_user_input("one is a number")
assert len(res) == 1
assert res[0].status
assert res[0].body
assert not res[0].status
assert sheerka.isinstance(res[0].body, BuiltinConcepts.CONDITION_FAILED)
def test_i_can_evaluate_source_code_with_concept(self):
init = [