kodjo/Sheerka-Old
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactored Caching, Refactored BnfNodeParser, Introduced Sphinx

Browse Source
This commit is contained in:
Kodjo Sossouvi
2020-05-12 17:21:10 +02:00
parent 7d3a490bc5
commit 6e343ba996
110 changed files with 13865 additions and 7540 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/core/ast/nodes.py
+12 -12
View File
@@ -1,5 +1,5 @@
from core.builtin_concepts import BuiltinConcepts, ListConcept
from core.concept import Concept
from core.concept import Concept, ConceptParts
import ast
import core.utils
@@ -65,12 +65,12 @@ class GenericNodeConcept(NodeConcept):
def get_node_type(self):
return self.node_type
def get_value(self):
def get_obj_value(self):
if self.node_type == "Name":
return self.get_prop("id")
return self.get_value("id")
if self.node_type == "arg":
return self.get_prop("arg")
return self.get_value("arg")
return self.body
@@ -78,7 +78,7 @@ class GenericNodeConcept(NodeConcept):
class IdentifierNodeConcept(NodeConcept):
def __init__(self, parent, name):
super().__init__(BuiltinConcepts.IDENTIFIER_NODE, "Name", parent)
self.body = name
self.set_value(ConceptParts.BODY, name)
class CallNodeConcept(NodeConcept):
@@ -86,7 +86,7 @@ class CallNodeConcept(NodeConcept):
super().__init__(BuiltinConcepts.IDENTIFIER_NODE, "Call", parent)
def get_args_names(self, sheerka):
return sheerka.get_values(self.get_prop("args"))
return sheerka.objvalues(self.get_value("args"))
def python_to_concept(python_node):
@@ -105,16 +105,16 @@ def python_to_concept(python_node):
continue
value = getattr(node, field)
concept.def_prop(field)
concept.def_var(field)
if isinstance(value, list):
lst = ListConcept().init_key()
for i in value:
lst.append(_transform(i, NodeParent(concept, field)))
concept.set_prop(field, lst)
concept.set_value(field, lst)
elif isinstance(value, ast.AST):
concept.set_prop(field, _transform(value, NodeParent(concept, field)))
concept.set_value(field, _transform(value, NodeParent(concept, field)))
else:
concept.set_prop(field, value)
concept.set_value(field, value)
concept.metadata.is_evaluated = True
return concept
@@ -132,11 +132,11 @@ def concept_to_python(concept_node):
def _transform(node):
node_type = node.get_node_type()
ast_object = core.utils.new_object("_ast." + node_type)
for field in node.props:
for field in node.values:
if field not in ast_object._fields:
continue
value = node.get_prop(field)
value = node.get_value(field)
if isinstance(value, list) or isinstance(value, Concept) and value.key == str(BuiltinConcepts.LIST):
lst = []
for i in value.body:
src/core/ast/visitors.py
+8 -8
View File
@@ -29,7 +29,7 @@ class ConceptNodeVisitor:
self.visit(value)
def visit_Constant(self, node):
value = node.get_prop("value")
value = node.get_value("value")
type_name = _const_node_type_names.get(type(value))
if type_name is None:
for cls, name in _const_node_type_names.items():
@@ -66,10 +66,10 @@ class UnreferencedNamesVisitor(ConceptNodeVisitor):
if ("Assign", "targets") in parents: # variable which is assigned
return
if self.can_be_discarded(self.sheerka.value(node), parents):
if self.can_be_discarded(self.sheerka.objvalue(node), parents):
return
self.names.add(self.sheerka.value(node))
self.names.add(self.sheerka.objvalue(node))
def can_be_discarded(self, variable_name, parents):
@@ -77,14 +77,14 @@ class UnreferencedNamesVisitor(ConceptNodeVisitor):
if node is None:
return False
if node.get_node_type() == "For" and self.sheerka.value(node.get_prop("target")) == variable_name:
if node.get_node_type() == "For" and self.sheerka.objvalue(node.get_value("target")) == variable_name:
# variable used by the loop
return True
if node.get_node_type() == "FunctionDef":
# variable defined as a function parameter
args = node.get_prop("args")
args_values = list(self.sheerka.get_values(args.get_prop("args")))
args = node.get_value("args")
args_values = list(self.sheerka.objvalues(args.get_value("args")))
if variable_name in args_values:
return True
@@ -112,8 +112,8 @@ def get_parents(node):
def iter_props(node):
for p in node.props:
yield p, node.props[p].value
for p in [p for p in node.values if isinstance(p, str)]:
yield p, node.get_value(p)
_const_node_type_names = {
src/core/builtin_concepts.py
+57 -56
View File
@@ -114,8 +114,8 @@ It's mainly to ease the usage
class UserInputConcept(Concept):
def __init__(self, text=None, user_name=None):
super().__init__(BuiltinConcepts.USER_INPUT, True, False, BuiltinConcepts.USER_INPUT)
self.set_metadata_value(ConceptParts.BODY, text)
self.set_prop("user_name", user_name)
self.set_value(ConceptParts.BODY, text)
self.set_value("user_name", user_name)
self.metadata.is_evaluated = True
@property
@@ -124,7 +124,7 @@ class UserInputConcept(Concept):
@property
def user_name(self):
return self.props["user_name"].value
return self.get_value("user_name")
def __repr__(self):
return f"({self.id}){self.name}: '{self.body}'"
@@ -133,7 +133,7 @@ class UserInputConcept(Concept):
class ErrorConcept(Concept):
def __init__(self, error=None):
super().__init__(BuiltinConcepts.ERROR, True, False, BuiltinConcepts.ERROR)
self.set_metadata_value(ConceptParts.BODY, error)
self.set_value(ConceptParts.BODY, error)
self.metadata.is_evaluated = True
def __repr__(self):
@@ -143,7 +143,7 @@ class ErrorConcept(Concept):
class UnknownConcept(Concept):
def __init__(self, metadata=None):
super().__init__(BuiltinConcepts.UNKNOWN_CONCEPT, True, False, BuiltinConcepts.UNKNOWN_CONCEPT)
self.set_metadata_value(ConceptParts.BODY, metadata)
self.set_value(ConceptParts.BODY, metadata)
self.metadata.is_evaluated = True
def __repr__(self):
@@ -158,28 +158,28 @@ class ReturnValueConcept(Concept):
def __init__(self, who=None, status=None, value=None, message=None, parents=None):
super().__init__(BuiltinConcepts.RETURN_VALUE, True, False, BuiltinConcepts.RETURN_VALUE)
self.set_metadata_value(ConceptParts.BODY, value)
self.set_prop("who", who)
self.set_prop("status", status)
self.set_prop("message", message)
self.set_prop("parents", parents)
self.set_value(ConceptParts.BODY, value)
self.set_value("who", who)
self.set_value("status", status)
self.set_value("message", message)
self.set_value("parents", parents)
self.metadata.is_evaluated = True
@property
def who(self):
return self.props["who"].value
return self.get_value("who")
@who.setter
def who(self, value):
self.set_prop("who", value)
self.set_value("who", value)
@property
def status(self):
return self.props["status"].value
return self.get_value("status")
@status.setter
def status(self, value):
self.set_prop("status", value)
self.set_value("status", value)
@property
def value(self):
@@ -187,23 +187,23 @@ class ReturnValueConcept(Concept):
@value.setter
def value(self, value):
self.set_metadata_value(ConceptParts.BODY, value)
self.set_value(ConceptParts.BODY, value)
@property
def message(self):
return self.props["message"].value
return self.get_value("message")
@message.setter
def message(self, value):
self.set_prop("message", value)
self.set_value("message", value)
@property
def parents(self):
return self.props["parents"].value
return self.get_value("parents")
@parents.setter
def parents(self, value):
self.set_prop("parents", value)
self.set_value("parents", value)
def __repr__(self):
return f"ReturnValue(who={self.who}, status={self.status}, value={self.value}, message={self.message})"
@@ -233,8 +233,8 @@ class UnknownPropertyConcept(Concept):
def __init__(self, property_name=None, concept=None):
super().__init__(BuiltinConcepts.UNKNOWN_PROPERTY, True, False, BuiltinConcepts.UNKNOWN_PROPERTY)
self.set_metadata_value(ConceptParts.BODY, property_name)
self.set_prop("concept", concept)
self.set_value(ConceptParts.BODY, property_name)
self.set_value("concept", concept)
self.metadata.is_evaluated = True
def __repr__(self):
@@ -242,7 +242,7 @@ class UnknownPropertyConcept(Concept):
@property
def concept(self):
return self.props["concept"].value
return self.get_value("concept")
@property
def property_name(self):
@@ -256,16 +256,16 @@ class ParserResultConcept(Concept):
def __init__(self, parser=None, source=None, tokens=None, value=None, try_parsed=None):
super().__init__(BuiltinConcepts.PARSER_RESULT, True, False, BuiltinConcepts.PARSER_RESULT)
self.set_metadata_value(ConceptParts.BODY, value)
self.set_prop("parser", parser)
self.set_prop("source", source)
self.set_prop("tokens", tokens)
self.set_prop("try_parsed", try_parsed) # in case of error, what was found before the error
self.set_value(ConceptParts.BODY, value)
self.set_value("parser", parser)
self.set_value("source", source)
self.set_value("tokens", tokens)
self.set_value("try_parsed", try_parsed) # in case of error, what was found before the error
self.metadata.is_evaluated = True
def __repr__(self):
text = f"ParserResult(parser={self.props['parser'].value}"
source = self.props['source'].value
text = f"ParserResult(parser={self.get_value('parser')}"
source = self.get_value('source')
text += f", source='{source}')" if source else f", body='{self.body}')"
return text
@@ -287,15 +287,15 @@ class ParserResultConcept(Concept):
@property
def try_parsed(self):
return self.props["try_parsed"].value
return self.get_value("try_parsed")
@property
def source(self):
return self.props["source"].value
return self.get_value("source")
@property
def parser(self):
return self.props["parser"].value
return self.get_value("parser")
class InvalidReturnValueConcept(Concept):
@@ -311,8 +311,8 @@ class InvalidReturnValueConcept(Concept):
True,
False,
BuiltinConcepts.INVALID_RETURN_VALUE)
self.set_metadata_value(ConceptParts.BODY, return_value)
self.set_prop("evaluator", evaluator)
self.set_value(ConceptParts.BODY, return_value)
self.set_value("evaluator", evaluator)
self.metadata.is_evaluated = True
@@ -322,9 +322,9 @@ class ConceptEvalError(Concept):
True,
False,
BuiltinConcepts.CONCEPT_EVAL_ERROR)
self.set_metadata_value(ConceptParts.BODY, error)
self.set_prop("concept", concept)
self.set_prop("property_name", property_name)
self.set_value(ConceptParts.BODY, error)
self.set_value("concept", concept)
self.set_value("property_name", property_name)
self.metadata.is_evaluated = True
def __repr__(self):
@@ -336,17 +336,17 @@ class ConceptEvalError(Concept):
@property
def concept(self):
return self.props["concept"].value
return self.get_value("concept")
@property
def property_name(self):
return self.props["property_name"].value
return self.get_value("property_name")
class EnumerationConcept(Concept):
def __init__(self, iteration=None):
super().__init__(BuiltinConcepts.ENUMERATION, True, False, BuiltinConcepts.ENUMERATION)
self.set_metadata_value(ConceptParts.BODY, iteration)
self.set_value(ConceptParts.BODY, iteration)
self.metadata.is_evaluated = True
# def __iter__(self):
@@ -356,7 +356,7 @@ class EnumerationConcept(Concept):
class ListConcept(Concept):
def __init__(self, items=None):
super().__init__(BuiltinConcepts.LIST, True, False, BuiltinConcepts.LIST)
self.set_metadata_value(ConceptParts.BODY, items or [])
self.set_value(ConceptParts.BODY, items or [])
self.metadata.is_evaluated = True
def append(self, obj):
@@ -381,9 +381,10 @@ class ListConcept(Concept):
class FilteredConcept(Concept):
def __init__(self, filtered=None, iterable=None, predicate=None):
super().__init__(BuiltinConcepts.FILTERED, True, False, BuiltinConcepts.FILTERED)
self.set_metadata_value(ConceptParts.BODY, filtered)
self.def_prop("iterable", iterable)
self.def_prop("predicate", predicate)
self.set_value(ConceptParts.BODY, filtered)
self.set_value("iterable", iterable)
self.set_value("predicate", predicate)
self.metadata.is_evaluated = True
class ConceptAlreadyInSet(Concept):
@@ -392,8 +393,8 @@ class ConceptAlreadyInSet(Concept):
True,
False,
BuiltinConcepts.CONCEPT_ALREADY_IN_SET)
self.set_metadata_value(ConceptParts.BODY, concept)
self.set_prop("concept_set", concept_set)
self.set_value(ConceptParts.BODY, concept)
self.set_value("concept_set", concept_set)
self.metadata.is_evaluated = True
def __repr__(self):
@@ -405,7 +406,7 @@ class ConceptAlreadyInSet(Concept):
@property
def concept_set(self):
return self.props["concept_set"].value
return self.get_value("concept_set")
class WhereClauseFailed(Concept):
@@ -414,7 +415,7 @@ class WhereClauseFailed(Concept):
True,
False,
BuiltinConcepts.WHERE_CLAUSE_FAILED)
self.set_metadata_value(ConceptParts.BODY, concept)
self.set_value(ConceptParts.BODY, concept)
self.metadata.is_evaluated = True
def __repr__(self):
@@ -431,12 +432,12 @@ class NotForMeConcept(Concept):
True,
False,
BuiltinConcepts.NOT_FOR_ME)
self.set_metadata_value(ConceptParts.BODY, source)
self.def_prop("reason", reason)
self.set_value(ConceptParts.BODY, source)
self.set_value("reason", reason)
self.metadata.is_evaluated = True
def __repr__(self):
return f"NotForMeConcept(source={self.body}, reason={self.get_prop('reason')})"
return f"NotForMeConcept(source={self.body}, reason={self.get_value('reason')})"
class ExplanationConcept(Concept):
@@ -445,9 +446,9 @@ class ExplanationConcept(Concept):
True,
False,
BuiltinConcepts.EXPLANATION)
self.def_prop("digest", digest) # event digest
self.def_prop("command", command) # explain command parameters
self.def_prop("title", title) # a title to the explanation
self.def_prop("instructions", instructions) # instructions for SheerkaPrint
self.set_metadata_value(ConceptParts.BODY, execution_result) # list of results
self.set_value("digest", digest) # event digest
self.set_value("command", command) # explain command parameters
self.set_value("title", title) # a title to the explanation
self.set_value("instructions", instructions) # instructions for SheerkaPrint
self.set_value(ConceptParts.BODY, execution_result) # list of results
self.metadata.is_evaluated = True
src/core/builtin_helpers.py
+47 -12
View File
@@ -30,11 +30,11 @@ def is_same_success(context, return_values):
evaluated = context.sheerka.evaluate_concept(sub_context, ret_val.body)
if evaluated.key != ret_val.body.key:
raise Exception("Failed to evaluate evaluate")
return context.sheerka.value(evaluated)
return context.sheerka.objvalue(evaluated)
else:
return context.sheerka.value(ret_val.body)
return context.sheerka.objvalue(ret_val.body)
else:
return context.sheerka.value(ret_val)
return context.sheerka.objvalue(ret_val)
try:
reference = _get_value(return_values[0])
@@ -280,8 +280,8 @@ def get_lexer_nodes(return_values, start, tokens):
for ret_val in return_values:
if ret_val.who == "parsers.Python":
if ret_val.body.source.strip().isalnum() and not ret_val.body.source.strip().isnumeric():
# Discard SourceCodeNode which seems to be a concept
if ret_val.body.source.strip().isidentifier():
# Discard SourceCodeNode which seems to be a concept name
# It may be a wrong idea, so let's see
continue
@@ -309,6 +309,41 @@ def get_lexer_nodes(return_values, start, tokens):
return lexer_nodes
def ensure_evaluated(context, concept):
"""
Evaluate a concept is not already evaluated
:param context:
:param concept:
:return:
"""
if concept.metadata.is_evaluated:
return concept
with context.push(desc=f"Evaluating concept {concept}") as sub_context:
sub_context.local_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
evaluated = context.sheerka.evaluate_concept(sub_context, concept)
sub_context.add_values(return_values=evaluated)
return evaluated
def get_lexer_nodes_from_unrecognized(context, unrecognized_tokens_node, parsers):
"""
Using parsers, try to recognize concepts from source
:param context:
:param unrecognized_tokens_node:
:param parsers:
:return:
"""
res = parse_unrecognized(context, unrecognized_tokens_node.source, parsers)
res = only_parsers_results(context, res)
if not res.status:
return None
return get_lexer_nodes(res.body.body, unrecognized_tokens_node.start, unrecognized_tokens_node.tokens)
def get_names(sheerka, concept_node):
"""
Finds all the names referenced by the concept_node
@@ -352,7 +387,7 @@ def extract_predicates(sheerka, expression, variables_to_include, variables_to_e
return NotImplementedError()
concept_node = core.ast.nodes.python_to_concept(node)
main_op = concept_node.get_prop("body")
main_op = concept_node.get_value("body")
return _get_predicates(_extract_predicates(sheerka, main_op, variables_to_include, variables_to_exclude))
@@ -370,14 +405,14 @@ def _extract_predicates(sheerka, node, variables_to_include, variables_to_exclud
return _res
if node.node_type == "Compare":
if node.get_prop("left").node_type == "Name":
if node.get_value("left").node_type == "Name":
"""Simple case of one comparison"""
comparison_name = sheerka.value(node.get_prop("left"))
comparison_name = sheerka.objvalue(node.get_value("left"))
if comparison_name in variables_to_include and comparison_name not in variables_to_exclude:
predicates.append(node)
else:
"""The left part is an expression"""
res = _extract_predicates(sheerka, node.get_prop("left"), variables_to_include, variables_to_exclude)
res = _extract_predicates(sheerka, node.get_value("left"), variables_to_include, variables_to_exclude)
if len(res) > 0:
predicates.append(node)
elif node.node_type == "Call":
@@ -386,9 +421,9 @@ def _extract_predicates(sheerka, node, variables_to_include, variables_to_exclud
args = list(call_node.get_args_names(sheerka))
if _matches(args, variables_to_include, variables_to_exclude):
predicates.append(node)
elif node.node_type == "UnaryOp" and node.get_prop("op").node_type == "Not":
elif node.node_type == "UnaryOp" and node.get_value("op").node_type == "Not":
"""Simple case of negation"""
res = _extract_predicates(sheerka, node.get_prop("operand"), variables_to_include, variables_to_exclude)
res = _extract_predicates(sheerka, node.get_value("operand"), variables_to_include, variables_to_exclude)
if len(res) > 0:
predicates.append(node)
elif node.node_type == "BinOp":
@@ -398,7 +433,7 @@ def _extract_predicates(sheerka, node, variables_to_include, variables_to_exclud
elif node.node_type == "BoolOp":
all_op = True
temp_res = []
for op in node.get_prop("values").body:
for op in node.get_value("values").body:
res = _extract_predicates(sheerka, op, variables_to_include, variables_to_exclude)
if len(res) == 0:
all_op = False
src/core/concept.py
+136 -89
View File
@@ -1,17 +1,19 @@
import hashlib
from collections import namedtuple
from copy import deepcopy
from dataclasses import dataclass
from enum import Enum
from core.sheerka_logger import get_logger
from typing import Union
import core.utils
from core.sheerka_logger import get_logger
from core.tokenizer import Tokenizer, TokenKind
PROPERTIES_FOR_DIGEST = ("name", "key",
"definition", "definition_type",
"is_builtin", "is_unique",
"where", "pre", "post", "body",
"desc", "props")
"desc", "props", "variables")
PROPERTIES_TO_SERIALIZE = PROPERTIES_FOR_DIGEST + tuple(["id"])
PROPERTIES_FOR_NEW = ("where", "pre", "post", "body", "desc")
VARIABLE_PREFIX = "__var__"
@@ -48,15 +50,13 @@ class ConceptMetadata:
definition_type: str # definition can be done with something else than regex
desc: str # possible description for the concept
id: str # unique identifier for a concept. The id will never be modified (but the key can)
props: list # list properties, with their default values
props: dict # hashmap of properties, values
variables: list # list of concept variables, with their default values
is_evaluated: bool = False # True is the concept is evaluated by sheerka.eval_concept()
need_validation = False # True if the properties of the concept need to be validated
full_serialization: bool = False # If True, the full object will be serialized, rather than just the diff
simplec = namedtuple("concept", "name body") # for simple concept (tests purposes only)
class Concept:
"""
Default concept object
@@ -76,7 +76,8 @@ class Concept:
definition_type=None,
desc=None,
id=None,
props=None):
props=None,
variables=None):
metadata = ConceptMetadata(
str(name) if name else None,
@@ -91,17 +92,17 @@ class Concept:
definition_type,
desc,
id,
props or []
props or {},
variables or []
)
self.metadata = metadata
self.compiled = {} # cached ast for the where, pre, post and body parts
self.values = {} # values of metadata once resolved
self.props = {} # resolved properties of this concept
self.bnf = None
self.compiled = {} # cached ast for the where, pre, post and body parts and variables
self.values = {} # resolved values. As compiled, it's used both for metadata and variables
self.bnf = None # parsing expression
self.log = get_logger("core." + self.__class__.__name__)
self.init_log = get_logger("init.core." + self.__class__.__name__)
self.original_definition_hash = None
self.original_definition_hash = None # concept hash before any alteration of the metadata
def __repr__(self):
return f"({self.metadata.id}){self.metadata.name}"
@@ -117,6 +118,9 @@ class Concept:
if isinstance(other, CC):
return other == self
if isinstance(other, CB):
return other == self
if not isinstance(other, Concept):
return False
@@ -147,15 +151,8 @@ class Concept:
if len(self.values) != len(other.values):
return False
for metadata in self.values:
if self.get_metadata_value(metadata) != other.get_metadata_value(metadata):
return False
if len(self.props) != len(other.props):
return False
for prop in self.props:
if self.get_prop(prop) != other.get_prop(prop):
for name in self.values:
if self.get_value(name) != other.get_value(name):
return False
return True
@@ -166,28 +163,36 @@ class Concept:
def __getattr__(self, item):
# I have this complicated implementation because of the usage of Pickle
if 'props' in vars(self) and item in self.props:
return self.props[item].value
if 'values' in vars(self) and item in self.values:
return self.get_value(item)
name = self.name if 'metadata' in vars(self) else 'Concept'
raise AttributeError(f"'{name}' concept has no attribute '{item}'")
def def_prop(self, prop_name, default_value=None):
def def_var(self, var_name, default_value=None):
"""
Adds a property to the metadata
:param prop_name: name or concept
:param var_name: name or concept
:param default_value:
:return:
"""
assert default_value is None or isinstance(default_value, str) # default properties will have to be evaluated
self.metadata.props.append((prop_name, default_value))
self.props[prop_name] = Property(prop_name, None) # do not set the default value
# why not setting props to the default values ?
# Because it may not be the real values, as metadata.props need to be evaluated
# this assert in not a functional requirement
# It's just to control what I put in the default value of properties
# You can allow more type if it's REALLY needed.
# - str are for standard definition
# - list of concepts is used by ISA
assert default_value is None or isinstance(default_value, str)
self.metadata.variables.append((var_name, default_value))
self.set_value(var_name, None) # do not set the default value
# why not setting variables to the default values ?
# Because it may not be the real values, as metadata.variables need to be evaluated
return self
def def_prop_by_index(self, index: int, value):
def def_var_by_index(self, index: int, value):
"""
Re-assign a value to a property (mainly used by ExactConceptParser)
:param index:
@@ -195,8 +200,8 @@ class Concept:
:return:
"""
assert value is None or isinstance(value, str) # default properties will have to be evaluated
prop = self.metadata.props[index]
self.metadata.props[index] = (prop[0], value)
var_name = self.metadata.variables[index]
self.metadata.variables[index] = (var_name[0], value) # change the default value
return self
@property
@@ -229,7 +234,7 @@ class Concept:
else:
tokens = list(Tokenizer(self.metadata.name))
variables = [p[0] for p in self.metadata.props] if len(core.utils.strip_tokens(tokens, True)) > 1 else []
variables = [p[0] for p in self.metadata.variables] if len(core.utils.strip_tokens(tokens, True)) > 1 else []
key = ""
first = True
@@ -252,7 +257,7 @@ class Concept:
@property
def body(self):
return self.values[ConceptParts.BODY] if ConceptParts.BODY in self.values else None
return self.get_value(ConceptParts.BODY)
def get_origin(self):
"""
@@ -284,8 +289,12 @@ class Concept:
"""
props_to_use = props_to_use or PROPERTIES_TO_SERIALIZE
props_as_dict = dict((prop, getattr(self.metadata, prop)) for prop in props_to_use)
props_as_dict = {}
for prop in props_to_use:
if prop == "props": # no need to copy variables as the ref won't be used in from_dict
props_as_dict[prop] = deepcopy(getattr(self.metadata, prop))
else:
props_as_dict[prop] = getattr(self.metadata, prop)
return props_as_dict
def from_dict(self, as_dict):
@@ -296,19 +305,20 @@ class Concept:
"""
for prop in PROPERTIES_TO_SERIALIZE:
if prop in as_dict:
if prop == "props":
if prop == "variables":
for name, value in as_dict[prop]:
self.def_prop(name, value)
self.def_var(name, value)
else:
setattr(self.metadata, prop, as_dict[prop])
return self
def update_from(self, other):
def update_from(self, other, update_value=True):
"""
Update self using the properties of another concept
This method is to mimic the class to instance pattern
'other' is the class, the template, and 'self' is a new instance
:param other:
:param update_value:
:return:
"""
if other is None:
@@ -321,12 +331,9 @@ class Concept:
self.from_dict(other.to_dict())
# update values
for k, v in other.values.items():
self.values[k] = v
# update properties
for k, v in other.props.items():
self.set_prop(k, v.value)
if update_value:
for k in other.values:
self.set_value(k, other.get_value(k))
# origin
from sdp.sheerkaSerializer import Serializer
@@ -335,54 +342,53 @@ class Concept:
return self
def set_prop(self, prop_name, prop_value):
def add_prop(self, concept_key, value):
"""
Set the value of a property (not the metadata)
:param prop_name: Name the property or another concept
:param prop_value:
:return:
"""
self.props[prop_name] = Property(prop_name, prop_value)
return self
def get_prop(self, prop_name: str):
"""
Gets the value of a property
:param prop_name: name or concept
:return:
"""
return self.props[prop_name].value
def set_prop_by_index(self, index: int, value):
"""
Set the value of a property (not the metadata) using the index
:param index: Name the property or another concept
Set or add a behaviour to a concept
A behaviour is a value from another concept (ex BuiltinConcepts.ISA
:param concept_key: Concept key
:param value:
:return:
"""
prop_name = list(self.props.keys())[index]
self.props[prop_name].value = value
if concept_key in self.metadata.props:
self.metadata.props[concept_key].add(value)
else:
self.metadata.props[concept_key] = {value} # a set
return self
def set_metadata_value(self, metadata: ConceptParts, value):
def get_prop(self, concept_key):
"""
Set the resolved value of a metadata (not the metadata itself)
:param metadata:
Gets a behaviour of a concept
:param concept_key: name of the behaviour
:return:
"""
return self.metadata.props[concept_key] if concept_key in self.metadata.props else None
def set_value(self, name, value):
"""
Set the resolved value of a metadata or a variable (not the metadata itself)
:param name:
:param value:
:return:
"""
self.values[metadata] = value
if name in self.values:
self.values[name].value = value
else:
self.values[name] = Property(name, value)
return self
def get_metadata_value(self, metadata: ConceptParts):
def get_value(self, prop_name):
"""
Gets the resolved value of a metadata
:param metadata:
:param prop_name:
:return:
"""
if metadata not in self.values:
if prop_name not in self.values:
return None
return self.values[metadata]
return self.values[prop_name].value
def variables(self):
return dict([(k, v) for k, v in self.values.items() if isinstance(k, str)])
def auto_init(self):
"""
@@ -398,10 +404,10 @@ class Concept:
for metadata in ConceptParts:
value = getattr(self.metadata, metadata.value)
if value is not None:
self.values[metadata] = value
self.set_value(metadata, value)
for prop, value in self.metadata.props:
self.set_prop(prop, value)
for var, value in self.metadata.variables:
self.set_value(var, value)
self.metadata.is_evaluated = True
return self
@@ -419,9 +425,10 @@ class Concept:
And it removes the visibility from the other attributes/methods
"""
bag = {}
for prop in self.props:
bag[prop] = self.get_prop(prop)
bag["prop." + prop] = self.get_prop(prop)
for var in self.values:
if isinstance(var, str):
bag[var] = self.get_value(var)
bag["var." + var] = self.get_value(var)
for prop in ("id", "name", "key", "body"):
bag[prop] = getattr(self, prop)
return bag
@@ -469,10 +476,17 @@ class InfiniteRecursionResolved:
"""This class is used to when we managed to break an infinite recursion concept definition"""
value: object
def get_value(self):
def get_obj_value(self):
return self.value
# ################################
#
# Class created for tests purpose
#
# ################################
class CC:
"""
Concept class for test purpose
@@ -484,13 +498,14 @@ class CC:
# The other properties (concept, source, start and end)
# are used in tests/parsers/parsers_utils.py to help creating helper objects
def __init__(self, concept, source=None, **kwargs):
def __init__(self, concept, source=None, exclude_body=False, **kwargs):
self.concept_key = concept.key if isinstance(concept, Concept) else concept
self.compiled = kwargs
self.concept = concept if isinstance(concept, Concept) else None
self.source = source # to use when the key is different from the sub str to search when filling start and stop
self.start = None # for debug purpose, indicate where the concept starts
self.end = None # for debug purpose, indicate where the concept ends
self.exclude_body = exclude_body
def __eq__(self, other):
if id(self) == id(other):
@@ -499,13 +514,19 @@ class CC:
if isinstance(other, Concept):
if other.key != self.concept_key:
return False
return self.compiled == other.compiled
if self.exclude_body:
to_compare = {k: v for k, v in other.compiled.items() if k != ConceptParts.BODY}
else:
to_compare = other.compiled
return self.compiled == to_compare
if not isinstance(other, CC):
return False
return self.concept_key == other.concept_key and \
self.compiled == other.compiled
if self.concept_key != other.concept_key:
return False
return self.compiled == other.compiled
def __hash__(self):
if self.concept:
@@ -536,3 +557,29 @@ class CC:
if self.end is None or end > self.end:
self.end = end
return self
@dataclass()
class CB:
"""
Concept with body only
Test class that test only the body of the concept
"""
concept: Union[str, Concept]
body: object
def __eq__(self, other):
if isinstance(other, Concept):
key = self.concept if isinstance(self.concept, str) else self.concept.key
return key == other.key and self.body == other.body
if not isinstance(other, CB):
return False
return self.concept == other.concept and self.body == other.body
def __hash__(self):
return hash((self.concept, self.body))
simplec = namedtuple("concept", "name body") # for simple concept (tests purposes only)
src/core/profiling.py
+26
View File
@@ -0,0 +1,26 @@
# ############################
# from github: nealtodd/decorator.py
# ############################
import pstats
from cProfile import Profile
def profile(sort_args=None, print_args=None):
sort_args = sort_args or ['cumulative']
print_args = print_args or [10]
profiler = Profile()
def decorator(fn):
def inner(*args, **kwargs):
result = None
try:
result = profiler.runcall(fn, *args, **kwargs)
finally:
stats = pstats.Stats(profiler)
stats.strip_dirs().sort_stats(*sort_args).print_stats(*print_args)
return result
return inner
return decorator
src/core/sheerka/ExecutionContext.py
+10 -9
View File
@@ -144,9 +144,9 @@ class ExecutionContext:
def add_preprocess(self, name, **kwargs):
preprocess = self.sheerka.new(BuiltinConcepts.EVALUATOR_PRE_PROCESS)
preprocess.set_prop("name", name)
preprocess.set_value("name", name)
for k, v in kwargs.items():
preprocess.set_prop(k, v)
preprocess.set_value(k, v)
if not self.preprocess:
self.preprocess = []
@@ -168,9 +168,9 @@ class ExecutionContext:
if isinstance(self.obj, Concept):
if self.obj.key == key:
return self.obj
for prop in self.obj.props:
if prop == key:
value = self.obj.props[prop].value
for var_name in self.obj.values:
if var_name == key:
value = self.obj.get_value(var_name)
if isinstance(value, Concept):
return value
@@ -180,16 +180,16 @@ class ExecutionContext:
if k == key:
return c
return self.sheerka.get(key)
return self.sheerka.get_by_key(key)
def new_concept(self, key, **kwargs):
# search in obj
if self.obj:
if self.obj.key == key:
return self.sheerka.new_from_template(self.obj, key, **kwargs)
for prop in self.obj.props:
if prop == key:
value = self.obj.props[prop].value
for var_name in self.obj.values:
if var_name == key:
value = self.obj.get_value(var_name)
if isinstance(value, Concept):
return self.sheerka.new_from_template(value, key, **kwargs)
else:
@@ -327,6 +327,7 @@ class ExecutionContext:
bag[prop] = getattr(self, prop)
bag["status"] = self.get_status()
bag["elapsed"] = self.elapsed
bag["elapsed_str"] = self.elapsed_str
bag["digest"] = self.event.get_digest() if self.event else None
return bag
src/core/sheerka/Services/SheerkaCreateNewConcept.py
+51 -93
View File
@@ -1,9 +1,9 @@
import core.utils
from core.builtin_concepts import BuiltinConcepts, ErrorConcept
from core.concept import Concept
from sdp.sheerkaDataProvider import SheerkaDataProviderDuplicateKeyError, SheerkaDataProviderRef
import core.utils
from sdp.sheerkaDataProvider_Old import SheerkaDataProviderDuplicateKeyError
BNF_NODE_PARSER_CLASS = "parsers.BnfNodeParser.BnfNodeParser"
BNF_NODE_PARSER_CLASS = "parsers.BnfNodeParser_Old.BnfNodeParser"
BASE_NODE_PARSER_CLASS = "parsers.BaseNodeParser.BaseNodeParser"
@@ -15,122 +15,80 @@ class SheerkaCreateNewConcept:
def __init__(self, sheerka):
self.sheerka = sheerka
self.logger_name = self.create_new_concept.__name__
self.base_lexer_parser = core.utils.get_class(BASE_NODE_PARSER_CLASS)("BaseNodeParser", 0)
self.bnp = core.utils.get_class(BASE_NODE_PARSER_CLASS) # BaseNodeParser
def create_new_concept(self, context, concept: Concept):
"""
Adds a new concept to the system
:param context:
:param concept: DefConceptNode
:param logger
:return: digest of the new concept
"""
sheerka = self.sheerka
concept.init_key()
concepts_definitions = None
init_bnf_ret_value = None
sdp = self.sheerka.sdp
cache_manager = sheerka.cache_manager
# checks for duplicate concepts
# TODO checks if it exists in cache first
if sdp.exists(self.sheerka.CONCEPTS_BY_HASH_ENTRY, concept.get_definition_hash()):
error = SheerkaDataProviderDuplicateKeyError(self.sheerka.CONCEPTS_ENTRY + "." + concept.key, concept)
return self.sheerka.ret(
if cache_manager.exists(sheerka.CONCEPTS_BY_HASH_ENTRY, concept.get_definition_hash()):
error = SheerkaDataProviderDuplicateKeyError(sheerka.CONCEPTS_BY_KEY_ENTRY + "." + concept.key,
concept)
return sheerka.ret(
self.logger_name,
False,
self.sheerka.new(BuiltinConcepts.CONCEPT_ALREADY_DEFINED, body=concept),
sheerka.new(BuiltinConcepts.CONCEPT_ALREADY_DEFINED, body=concept),
error.args[0])
# set id before saving in db
self.sheerka.set_id_if_needed(concept, False)
sheerka.set_id_if_needed(concept, False)
# add the BNF if known
if concept.bnf:
concepts_definitions = self.sheerka.get_concepts_definitions(context)
concepts_definitions[concept] = concept.bnf
# update the dictionary of concepts by first key
init_ret_value = self.bnp.get_concepts_by_first_keyword(context, [concept], True)
if not init_ret_value.status:
return sheerka.ret(self.logger_name, False, ErrorConcept(init_ret_value.value))
concepts_by_first_keyword = init_ret_value.body
# check if it's a valid BNF or whether it breaks the known rules
bnf_lexer_parser = self.sheerka.parsers[BNF_NODE_PARSER_CLASS]()
with context.push(self.sheerka.name, desc=f"Initializing concept definition for {concept}") as sub_context:
sub_context.concepts[concept.key] = concept # the concept is not in the real cache yet
init_bnf_ret_value = bnf_lexer_parser.initialize(sub_context, concepts_definitions)
sub_context.add_values(return_values=init_bnf_ret_value)
if not init_bnf_ret_value.status:
return self.sheerka.ret(self.logger_name, False, ErrorConcept(init_bnf_ret_value.value))
# update resolved dictionary
init_ret_value = self.bnp.resolve_concepts_by_first_keyword(context, concepts_by_first_keyword)
if not init_ret_value.status:
return sheerka.ret(self.logger_name, False, ErrorConcept(init_ret_value.value))
resolved_concepts_by_first_keyword = init_ret_value.body
# update concept definition by key
init_sya_ret_value = self.base_lexer_parser.initialize(context, [concept], use_sheerka=True)
if not init_sya_ret_value.status:
return self.sheerka.ret(self.logger_name, False, ErrorConcept(init_sya_ret_value.value))
concepts_by_first_keyword = init_sya_ret_value.body
# init_sya_ret_value = self.bnp.initialize(context, [concept], use_sheerka=True)
# if not init_sya_ret_value.status:
# return sheerka.ret(self.logger_name, False, ErrorConcept(init_sya_ret_value.value))
# concepts_by_first_keyword = init_sya_ret_value.body
concept.freeze_definition_hash()
# save the new concept in sdp
try:
# TODO : needs to make these calls atomic (or at least one single call)
# save the new concept
concept.metadata.full_serialization = True
result = sdp.add(
context.event.get_digest(),
self.sheerka.CONCEPTS_ENTRY,
concept,
use_ref=True)
concept.metadata.full_serialization = False
cache_manager.add_concept(concept)
cache_manager.put(sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, False, concepts_by_first_keyword)
cache_manager.put(sheerka.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY, False, resolved_concepts_by_first_keyword)
# update the concept (I hope that it's enough)
concept.set_origin(result.digest)
# save it by id
sdp.add(
context.event.get_digest(),
self.sheerka.CONCEPTS_BY_ID_ENTRY,
SheerkaDataProviderRef(concept.id, result.digest))
# save it by name
sdp.add(
context.event.get_digest(),
self.sheerka.CONCEPTS_BY_NAME_ENTRY,
SheerkaDataProviderRef(concept.name, result.digest))
# records the hash
sdp.add(
context.event.get_digest(),
self.sheerka.CONCEPTS_BY_HASH_ENTRY,
SheerkaDataProviderRef(concept.get_definition_hash(), result.digest))
# update the definition table
if concepts_definitions is not None:
sdp.set(
context.event.get_digest(),
self.sheerka.CONCEPTS_DEFINITIONS_ENTRY,
bnf_lexer_parser.encode_grammar(init_bnf_ret_value.body),
use_ref=True)
self.sheerka.concepts_definitions_cache = None # invalidate cache
# update the concepts by first keyword
sdp.set(context.event.get_digest(),
self.sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY,
concepts_by_first_keyword)
except SheerkaDataProviderDuplicateKeyError as error:
context.log_error("Failed to create a new concept.", who=self.logger_name)
return self.sheerka.ret(
self.logger_name,
False,
self.sheerka.new(BuiltinConcepts.CONCEPT_ALREADY_DEFINED, body=concept),
error.args[0])
# Updates the caches
self.sheerka.cache_by_key[concept.key] = sdp.get_safe(self.sheerka.CONCEPTS_ENTRY, concept.key)
self.sheerka.cache_by_name[concept.name] = sdp.get_safe(self.sheerka.CONCEPTS_BY_NAME_ENTRY, concept.name)
self.sheerka.cache_by_id[concept.id] = concept
if init_bnf_ret_value is not None and init_bnf_ret_value.status:
self.sheerka.concepts_grammars = init_bnf_ret_value.body
self.sheerka.concepts_by_first_keyword = concepts_by_first_keyword
if concept.bnf and init_bnf_ret_value is not None and init_bnf_ret_value.status:
sheerka.cache_manager.clear(sheerka.CONCEPTS_GRAMMARS_ENTRY)
# process the return if needed
ret = self.sheerka.ret(self.logger_name, True, self.sheerka.new(BuiltinConcepts.NEW_CONCEPT, body=concept))
ret = sheerka.ret(self.logger_name, True, sheerka.new(BuiltinConcepts.NEW_CONCEPT, body=concept))
return ret
# def load_concepts_nodes_definitions(self, context):
# """
# Gets from sdp what is need to parse nodes
# :return:
# """
# sdp = self.sheerka.sdp
#
# concepts_by_first_keyword = sdp.get(
# self.sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY,
# load_origin=False) or {}
#
# init_ret_value = self.bnp.resolve_concepts_by_first_keyword(context, concepts_by_first_keyword)
# if not init_ret_value.status:
# return self.sheerka.ret(self.logger_name, False, ErrorConcept(init_ret_value.value))
# resolved_concepts_by_first_keyword = init_ret_value.body
#
# return concepts_by_first_keyword, resolved_concepts_by_first_keyword
src/core/sheerka/Services/SheerkaDump.py
+7 -10
View File
@@ -1,9 +1,10 @@
import os
import pprint
from core.builtin_concepts import BuiltinConcepts
from core.concept import Concept
from core.sheerka.ExecutionContext import ExecutionContext
from sdp.sheerkaDataProvider import SheerkaDataProvider, Event
import pprint
import os
def get_pp():
@@ -17,7 +18,7 @@ class SheerkaDump:
self.sheerka = sheerka
def dump_concepts(self):
lst = self.sheerka.sdp.list(self.sheerka.CONCEPTS_ENTRY)
lst = self.sheerka.sdp.list(self.sheerka.CONCEPTS_BY_KEY_ENTRY)
for item in lst:
if hasattr(item, "__iter__"):
for i in item:
@@ -25,10 +26,6 @@ class SheerkaDump:
else:
self.sheerka.log.info(item)
def dump_definitions(self):
defs = self.sheerka.sdp.get(self.sheerka.CONCEPTS_DEFINITIONS_ENTRY)
self.sheerka.log.info(defs)
def dump_desc(self, *concept_names, eval=False):
first = True
event = Event(f"Dumping description", "")
@@ -37,7 +34,7 @@ class SheerkaDump:
if isinstance(concept_name, Concept):
concepts = concept_name
else:
concepts = self.sheerka.get(concept_name)
concepts = self.sheerka.get_by_key(concept_name)
if self.sheerka.isinstance(concepts, BuiltinConcepts.UNKNOWN_CONCEPT):
self.sheerka.log.error(f"Concept '{concept_name}' is unknown")
return False
@@ -59,8 +56,8 @@ class SheerkaDump:
self.sheerka.log.info(f"where : {c.metadata.where}")
if eval:
self.sheerka.log.info(f"value : {value}")
for p in c.props:
self.sheerka.log.info(f"{p}: {c.get_prop(p)}")
for v in c.values:
self.sheerka.log.info(f"{v}: {c.get_value(v)}")
else:
self.sheerka.log.info("No property")
src/core/sheerka/Services/SheerkaEvaluateConcept.py
+23 -29
View File
@@ -1,6 +1,6 @@
from core.builtin_concepts import BuiltinConcepts
from core.concept import Concept, DoNotResolve, ConceptParts, InfiniteRecursionResolved
from core.builtin_helpers import expect_one
from core.concept import Concept, DoNotResolve, ConceptParts, InfiniteRecursionResolved
CONCEPT_EVALUATION_STEPS = [
BuiltinConcepts.BEFORE_EVALUATION,
@@ -91,32 +91,27 @@ class SheerkaEvaluateConcept:
concept.compiled[part_key] = res
sub_context.add_values(return_values=res)
for prop, default_value in concept.metadata.props:
if prop in concept.compiled:
for var_name, default_value in concept.metadata.variables:
if var_name in concept.compiled:
continue
if default_value is None or not isinstance(default_value, str):
continue
if default_value.strip() == "":
concept.compiled[prop] = DoNotResolve(default_value)
concept.compiled[var_name] = DoNotResolve(default_value)
else:
with context.push(desc=f"Initializing AST for property {prop}") as sub_context:
with context.push(desc=f"Initializing AST for property {var_name}") as sub_context:
sub_context.add_inputs(source=default_value)
to_parse = self.sheerka.ret(context.who, True,
self.sheerka.new(BuiltinConcepts.USER_INPUT, body=default_value))
res = self.sheerka.execute(context, to_parse, steps)
concept.compiled[prop] = res
concept.compiled[var_name] = res
sub_context.add_values(return_values=res)
# Updates the cache of concepts when possible
if concept.key in self.sheerka.cache_by_key:
entry = self.sheerka.cache_by_key[concept.key]
if isinstance(entry, list):
# TODO : manage when there are multiple entries
pass
else:
self.sheerka.cache_by_key[concept.key].compiled = concept.compiled
if self.sheerka.has_id(concept.id):
self.sheerka.get_by_id(concept.id).compiled = concept.compiled
def resolve(self, context, to_resolve, current_prop, current_concept, force_evaluation):
if isinstance(to_resolve, DoNotResolve):
@@ -198,7 +193,6 @@ class SheerkaEvaluateConcept:
It means that if the where clause is True, will evaluate the body
:param context:
:param concept:
:param evaluate_body: If false, only evaluate body when necessary
:return: value of the evaluation or error
"""
@@ -208,26 +202,26 @@ class SheerkaEvaluateConcept:
self.initialize_concept_asts(context, concept)
# to make sure of the order, it don't use ConceptParts.get_parts()
# props must be evaluated first, body must be evaluated before where
# variables must be evaluated first, body must be evaluated before where
all_metadata_to_eval = self.choose_metadata_to_eval(context, concept)
for metadata_to_eval in all_metadata_to_eval:
if metadata_to_eval == "props":
for prop_name in (p for p in concept.props if p in concept.compiled):
prop_ast = concept.compiled[prop_name]
if metadata_to_eval == "variables":
for var_name in (v for v in concept.variables() if v in concept.compiled):
prop_ast = concept.compiled[var_name]
if isinstance(prop_ast, list):
# Do not send the current concept for the properties
resolved = self.resolve_list(context, prop_ast, prop_name, None, True)
resolved = self.resolve_list(context, prop_ast, var_name, None, True)
else:
# Do not send the current concept for the properties
resolved = self.resolve(context, prop_ast, prop_name, None, True)
resolved = self.resolve(context, prop_ast, var_name, None, True)
if isinstance(resolved, Concept) and not context.sheerka.is_success(resolved):
resolved.set_prop("concept", concept) # since current concept was not sent
resolved.set_value("concept", concept) # since current concept was not sent
return resolved
else:
concept.set_prop(prop_name, resolved)
concept.set_value(var_name, resolved)
else:
part_key = ConceptParts(metadata_to_eval)
@@ -245,7 +239,7 @@ class SheerkaEvaluateConcept:
if isinstance(resolved, Concept) and not context.sheerka.is_success(resolved):
return resolved
else:
concept.values[part_key] = self.get_infinite_recursion_resolution(resolved) or resolved
concept.set_value(part_key, self.get_infinite_recursion_resolution(resolved) or resolved)
#
# TODO : Validate the PRE condition
@@ -253,8 +247,8 @@ class SheerkaEvaluateConcept:
# validate where clause
if ConceptParts.WHERE in concept.values:
where_value = concept.values[ConceptParts.WHERE]
if not (where_value is None or self.sheerka.value(where_value)):
where_value = concept.get_value(ConceptParts.WHERE)
if not (where_value is None or self.sheerka.objvalue(where_value)):
return self.sheerka.new(BuiltinConcepts.WHERE_CLAUSE_FAILED, body=concept)
#
@@ -267,7 +261,7 @@ class SheerkaEvaluateConcept:
def choose_metadata_to_eval(self, context, concept):
if context.in_context(BuiltinConcepts.EVAL_BODY_REQUESTED):
return ["pre", "post", "props", "body", "where"]
return ["pre", "post", "variables", "body", "where"]
metadata = ["pre", "post"]
if context.in_context(BuiltinConcepts.EVAL_WHERE_REQUESTED) or concept.metadata.need_validation:
@@ -310,9 +304,9 @@ class SheerkaEvaluateConcept:
if not isinstance(return_value.body.source, str):
continue
for prop_name in (p[0] for p in concept.metadata.props):
if prop_name in return_value.body.source:
needed.append("props")
for var_name in (p[0] for p in concept.metadata.variables):
if var_name in return_value.body.source:
needed.append("variables")
break
if "self" in return_value.body.source:
src/core/sheerka/Services/SheerkaExecute.py
+5 -5
View File
@@ -230,12 +230,12 @@ class SheerkaExecute:
for preprocess in context.preprocess:
for e in parsers_or_evaluators:
if self.matches(e.name, preprocess.get_prop("name")):
for prop, value in preprocess.props.items():
if prop == "name":
if self.matches(e.name, preprocess.get_value("name")):
for var_name in preprocess.values:
if var_name == "name":
continue
if hasattr(e, prop):
setattr(e, prop, value.value)
if hasattr(e, var_name):
setattr(e, var_name, preprocess.get_value(var_name))
return parsers_or_evaluators[0] if single_one else parsers_or_evaluators
@staticmethod
src/core/sheerka/Services/SheerkaModifyConcept.py
+26 -44
View File
@@ -1,5 +1,4 @@
from core.builtin_concepts import BuiltinConcepts
from sdp.sheerkaDataProvider import SheerkaDataProviderRef
class SheerkaModifyConcept:
@@ -8,52 +7,35 @@ class SheerkaModifyConcept:
self.logger_name = self.modify_concept.__name__
def modify_concept(self, context, concept):
old_version = self.sheerka.get_by_id(concept.id)
sdp = self.sheerka.sdp
try:
# modify the entry
concept.metadata.full_serialization = True
result = sdp.modify(
context.event.get_digest(),
self.sheerka.CONCEPTS_ENTRY,
concept.key,
concept)
concept.metadata.full_serialization = False
# update reference entry
sdp.modify(
context.event.get_digest(),
self.sheerka.CONCEPTS_BY_ID_ENTRY,
concept.id,
SheerkaDataProviderRef(concept.id, result.digest, concept.get_origin()))
# update name entry
sdp.modify(
context.event.get_digest(),
self.sheerka.CONCEPTS_BY_NAME_ENTRY,
concept.name,
SheerkaDataProviderRef(concept.name, result.digest, concept.get_origin()))
# update the hash entry
sdp.modify(
context.event.get_digest(),
self.sheerka.CONCEPTS_BY_HASH_ENTRY,
concept.get_original_definition_hash(),
SheerkaDataProviderRef(concept.get_definition_hash(), result.digest, concept.get_origin()))
except IndexError as error:
context.log_error(f"Failed to update concept '{concept}'.", who=self.logger_name)
if old_version is None:
# nothing found in cache
return self.sheerka.ret(
self.logger_name,
False,
self.sheerka.new(BuiltinConcepts.UNKNOWN_CONCEPT, body=concept),
error.args[0])
self.logger_name, False,
self.sheerka.new(
BuiltinConcepts.UNKNOWN_CONCEPT,
body=[("key", concept.key), ("id", concept.id)]))
# update cache
self.sheerka.cache_by_key[concept.key] = sdp.get_safe(self.sheerka.CONCEPTS_ENTRY, concept.key)
self.sheerka.cache_by_name[concept.name] = sdp.get_safe(self.sheerka.CONCEPTS_BY_NAME_ENTRY, concept.name)
self.sheerka.cache_by_id[concept.id] = concept
if not self.sheerka.is_success(old_version) and concept.key != old_version.key:
# an error concept is returned
return self.sheerka.ret(
self.logger_name, False,
old_version)
if old_version == concept:
# the concept is not modified
return self.sheerka.ret(
self.logger_name, False,
self.sheerka.new(
BuiltinConcepts.CONCEPT_ALREADY_DEFINED,
body=concept))
self.sheerka.cache_manager.update_concept(old_version, concept)
# TODO : update concept by first keyword
# TODO : update resolved by first keyword
# TODO : update concets grammars
ret = self.sheerka.ret(self.logger_name, True, self.sheerka.new(BuiltinConcepts.NEW_CONCEPT, body=concept))
return ret
src/core/sheerka/Services/SheerkaSetsManager.py
+81 -76
View File
@@ -1,7 +1,7 @@
from core.ast.nodes import python_to_concept
from core.builtin_concepts import BuiltinConcepts, ErrorConcept
from core.concept import Concept, ConceptParts
import core.builtin_helpers
from core.ast.nodes import python_to_concept
from core.builtin_concepts import BuiltinConcepts
from core.concept import Concept, ConceptParts
GROUP_PREFIX = 'All_'
@@ -20,13 +20,16 @@ class SheerkaSetsManager:
:return:
"""
context.log(f"Setting that concept {concept} is a {concept_set}", who=self.logger_name)
context.log(f"Setting concept {concept} is a {concept_set}", who=self.logger_name)
isa = [] if BuiltinConcepts.ISA not in concept.props else concept.get_prop(BuiltinConcepts.ISA)
if concept_set not in isa:
isa.append(concept_set)
if BuiltinConcepts.ISA in concept.metadata.props and concept_set in concept.metadata.props[BuiltinConcepts.ISA]:
return self.sheerka.ret(
self.logger_name,
False,
self.sheerka.new(BuiltinConcepts.CONCEPT_ALREADY_IN_SET, body=concept, concept_set=concept_set))
concept.add_prop(BuiltinConcepts.ISA, concept_set)
concept.set_prop(BuiltinConcepts.ISA, isa)
res = self.sheerka.modify_concept(context, concept)
if not res.status:
return res
@@ -47,32 +50,34 @@ class SheerkaSetsManager:
assert concept.id
assert concept_set.id
try:
result = self.sheerka.sdp.add_unique(context.event.get_digest(), GROUP_PREFIX + concept_set.id, concept.id)
if result.already_exists: # concept already in set
return self.sheerka.ret(
self.logger_name,
False,
self.sheerka.new(BuiltinConcepts.CONCEPT_ALREADY_IN_SET, body=concept, concept_set=concept_set))
else:
return self.sheerka.ret(self.logger_name, True, self.sheerka.new(BuiltinConcepts.SUCCESS))
except Exception as error:
context.log_error("Failed to add to set.", who=self.logger_name)
return self.sheerka.ret(self.logger_name, False, ErrorConcept(error), error.args[0])
set_elements = self.sheerka.cache_manager.get(self.sheerka.CONCEPTS_GROUPS_ENTRY, concept_set.id)
if set_elements and concept.id in set_elements:
return self.sheerka.ret(
self.logger_name,
False,
self.sheerka.new(BuiltinConcepts.CONCEPT_ALREADY_IN_SET, body=concept, concept_set=concept_set))
self.sheerka.cache_manager.put(self.sheerka.CONCEPTS_GROUPS_ENTRY, concept_set.id, concept.id)
return self.sheerka.ret(self.logger_name, True, self.sheerka.new(BuiltinConcepts.SUCCESS))
def add_concepts_to_set(self, context, concepts, concept_set):
"""Adding multiple concepts at the same time"""
context.log(f"Adding concepts {concepts} to set {concept_set}", who=self.logger_name)
previous = self.sheerka.sdp.get_safe(GROUP_PREFIX + concept_set.id)
already_in_set = []
for concept in concepts:
res = self.add_concept_to_set(context, concept, concept_set)
if self.sheerka.isinstance(res.body, BuiltinConcepts.CONCEPT_ALREADY_IN_SET):
already_in_set.append(res.body.body)
new_ids = [c.id for c in concepts] if previous is None else previous + [c.id for c in concepts]
try:
self.sheerka.sdp.set(context.event.get_digest(), GROUP_PREFIX + concept_set.id, new_ids)
return self.sheerka.ret(self.logger_name, True, self.sheerka.new(BuiltinConcepts.SUCCESS))
except Exception as error:
context.log_error("Failed to add to set.", who=self.logger_name)
return self.sheerka.ret(self.logger_name, False, ErrorConcept(error), error.args[0])
if already_in_set:
body = self.sheerka.new(BuiltinConcepts.CONCEPT_ALREADY_IN_SET,
body=already_in_set,
concept_set=concept_set)
else:
body = self.sheerka.new(BuiltinConcepts.SUCCESS)
return self.sheerka.ret(self.logger_name, len(already_in_set) != len(concepts), body)
def get_set_elements(self, context, concept):
"""
@@ -83,38 +88,41 @@ class SheerkaSetsManager:
:return:
"""
# noinspection PyShadowingNames
def _get_set_elements(context, concept, sub_concept):
if not (isinstance(sub_concept, Concept) and sub_concept.id):
def _get_set_elements(sub_concept):
if not self.isaset(context, sub_concept):
return self.sheerka.new(BuiltinConcepts.NOT_A_SET, body=concept)
ids = self.sheerka.sdp.get_safe(GROUP_PREFIX + sub_concept.id)
if ids:
if concept.metadata.where:
new_condition = self._validate_where_clause(concept)
if not new_condition:
return self.sheerka.new(BuiltinConcepts.WHERE_CLAUSE_FAILED, body=concept)
else:
# This methods sucks, but I don't have enough tools (like proper AST manipulation functions)
# to do it properly now. It will be enhanced later
concepts = self._get_concepts(context, ids, True)
globals_ = {"xx__concepts__xx": concepts, "sheerka": self.sheerka}
locals_ = {}
exec(new_condition, globals_, locals_)
return locals_["result"]
else:
return self._get_concepts(context, ids, False)
# first, try to see if sub_context has it's own group entry
ids = self.sheerka.cache_manager.get(self.sheerka.CONCEPTS_GROUPS_ENTRY, sub_concept.id)
concepts = self._get_concepts(context, ids, True)
# it may be a concept that references a set
if not sub_concept.metadata.is_evaluated:
with context.push(desc=f"Evaluating concept {sub_concept}") as sub_context:
sub_context.local_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
evaluated = self.sheerka.evaluate_concept(sub_context, sub_concept)
if evaluated.key != concept.key:
return False
return _get_set_elements(context, concept, sub_concept.body)
# aggregate with en entries from its body
sub_concept = core.builtin_helpers.ensure_evaluated(context, sub_concept)
if not self.sheerka.is_success(sub_concept):
return sub_concept
return _get_set_elements(context, concept, concept)
if self.isaset(context, sub_concept.body):
other_concepts = _get_set_elements(sub_concept.body)
if not self.sheerka.is_success(other_concepts):
return other_concepts
concepts.extend(other_concepts)
# apply the where clause if any
if sub_concept.metadata.where:
new_condition = self._validate_where_clause(sub_concept)
if not new_condition:
return self.sheerka.new(BuiltinConcepts.WHERE_CLAUSE_FAILED, body=sub_concept)
# This methods sucks, but I don't have enough tools (like proper AST manipulation functions)
# to do it properly now. It will be enhanced later
globals_ = {"xx__concepts__xx": concepts, "sheerka": self.sheerka}
locals_ = {}
exec(new_condition, globals_, locals_)
concepts = locals_["result"]
return concepts
return _get_set_elements(concept)
def isinset(self, a, b):
"""
@@ -135,17 +143,15 @@ class SheerkaSetsManager:
if not (a.id and b.id):
return False
if self.sheerka.sdp.exists(GROUP_PREFIX + b.id, a.id):
return True
return False
group_elements = self.sheerka.cache_manager.get(self.sheerka.CONCEPTS_GROUPS_ENTRY, b.id)
return group_elements and a.id in group_elements
def isa(self, a, b):
if BuiltinConcepts.ISA not in a.props:
if BuiltinConcepts.ISA not in a.metadata.props:
return False
for c in a.get_prop(BuiltinConcepts.ISA):
for c in a.metadata.props[BuiltinConcepts.ISA]:
if c == b:
return True
if self.isa(c, b):
@@ -163,21 +169,19 @@ class SheerkaSetsManager:
""""""
if not (isinstance(concept, Concept) and concept.id):
return None
return False
# check if it has a group
# TODO: use cache instead of directly requesting sdp
if self.sheerka.cache_manager.get(self.sheerka.CONCEPTS_GROUPS_ENTRY, concept.id):
return True
# it may be a concept that references a set
if not concept.metadata.is_evaluated:
with context.push(desc=f"Evaluating concept {concept}") as sub_context:
sub_context.local_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
evaluated = self.sheerka.evaluate_concept(sub_context, concept)
if evaluated.key != concept.key:
return False
concept = core.builtin_helpers.ensure_evaluated(context, concept)
if not context.sheerka.is_success(concept):
return False
if concept.body:
return self.isaset(context, concept.body)
res = self.sheerka.sdp.get_safe(GROUP_PREFIX + concept.id)
return res is not None
return self.isaset(context, concept.body)
def _validate_where_clause(self, concept):
python_parser_result = [r for r in concept.compiled[ConceptParts.WHERE] if r.who == "parsers.Python"]
@@ -190,7 +194,7 @@ class SheerkaSetsManager:
if len(names) != 1 or names[0] != concept.metadata.body:
return None
condition = concept.metadata.where.replace(concept.metadata.body, "sheerka.value(x)")
condition = concept.metadata.where.replace(concept.metadata.body, "sheerka.objvalue(x)")
expression = f"""
result=[]
for x in xx__concepts__xx:
@@ -218,10 +222,11 @@ for x in xx__concepts__xx:
result = []
with context.push(desc=f"Evaluating concepts of a set") as sub_context:
sub_context.add_inputs(ids=ids)
sub_context.local_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
for element_id in ids:
concept = self.sheerka.get_by_id(element_id)
evaluated = self.sheerka.evaluate_concept(sub_context, concept)
result.append(evaluated)
sub_context.add_inputs(return_value=result)
return result
src/core/sheerka/Services/SheerkaVariableManager.py
+11 -17
View File
@@ -20,35 +20,29 @@ class Variable:
class SheerkaVariableManager:
VARIABLES_ENTRY = "All_Variables" # to store all the concepts
def __init__(self, sheerka):
self.sheerka = sheerka
def record(self, context, who, key, value):
"""Persist a variable"""
# first check if there is a previous version of the variable
try:
old = self.sheerka.sdp.get(self.VARIABLES_ENTRY, who + "." + key)
if old.value == value:
return
"""
parent = getattr(old, Serializer.ORIGIN)
except IndexError:
parent = None
:param context:
:param who: entity that owns the key (acts as a namespace)
:param key:
:param value:
:return:
"""
variable = Variable(context.event.get_digest(), who, key, value, [parent] if parent else None)
self.sheerka.sdp.set(context.event.get_digest(), self.VARIABLES_ENTRY, variable, use_ref=True)
variable = Variable(context.event.get_digest(), who, key, value, None)
self.sheerka.cache_manager.put(self.sheerka.VARIABLES_ENTRY, variable.get_key(), variable)
def load(self, who, key):
variable = self.sheerka.sdp.get_safe(self.VARIABLES_ENTRY, who + "." + key)
variable = self.sheerka.cache_manager.get(self.sheerka.VARIABLES_ENTRY, who + "." + key)
if variable is None:
return None
return variable.value
def delete(self, context, who, key):
self.sheerka.sdp.remove(
context.event.get_digest(),
self.VARIABLES_ENTRY,
lambda _key, _var: _key == who + "." + key)
self.sheerka.cache_manager.delete(self.sheerka.VARIABLES_ENTRY, who + "." + key)
src/core/sheerka/Sheerka.py
+294 -229
View File
@@ -2,6 +2,12 @@ import logging
import core.builtin_helpers
import core.utils
from cache.Cache import Cache
from cache.CacheManager import CacheManager
from cache.DictionaryCache import DictionaryCache
from cache.IncCache import IncCache
from cache.ListIfNeededCache import ListIfNeededCache
from cache.SetCache import SetCache
from core.builtin_concepts import BuiltinConcepts, ErrorConcept, ReturnValueConcept, BuiltinErrors, BuiltinUnique, \
UnknownConcept
from core.concept import Concept, ConceptParts, PROPERTIES_FOR_NEW
@@ -18,8 +24,7 @@ from core.sheerka_logger import console_handler
from printer.SheerkaPrinter import SheerkaPrinter
from sdp.sheerkaDataProvider import SheerkaDataProvider, Event
CONCEPT_LEXER_PARSER_CLASS = "parsers.BnfNodeParser.BnfNodeParser"
BNF_PARSER_CLASS = "parsers.BnfParser.BnfParser"
BASE_NODE_PARSER_CLASS = "parsers.BaseNodeParser.BaseNodeParser"
CONCEPTS_FILE = "_concepts.txt"
@@ -28,45 +33,36 @@ class Sheerka(Concept):
Main controller for the project
"""
CONCEPTS_ENTRY = "All_Concepts" # to store all the concepts
CONCEPTS_BY_ID_ENTRY = "Concepts_By_ID"
CONCEPTS_BY_ID_ENTRY = "Concepts_By_ID" # to store all the concepts
CONCEPTS_BY_KEY_ENTRY = "Concepts_By_Key"
CONCEPTS_BY_NAME_ENTRY = "Concepts_By_Name"
CONCEPTS_BY_HASH_ENTRY = "Concepts_By_Hash" # store hash of concepts definitions (not values)
CONCEPTS_DEFINITIONS_ENTRY = "Concepts_Definitions" # to store definitions (bnf) of concepts
CONCEPTS_BY_FIRST_KEYWORD_ENTRY = "Concepts_By_First_Keyword"
CONCEPTS_SYA_DEFINITION_ENTRY = "Concepts_Sya_Definitions"
CONCEPTS_BY_FIRST_KEYWORD_ENTRY = "Concepts_By_First_Keyword"
RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY = "Resolved_Concepts_By_First_Keyword"
CONCEPTS_SYA_DEFINITION_ENTRY = "Concepts_Sya_Definitions"
RESOLVED_CONCEPTS_SYA_DEFINITION_ENTRY = "Resolved_Concepts_Sya_Definitions"
CONCEPTS_GRAMMARS_ENTRY = "Concepts_Grammars"
CONCEPTS_GROUPS_ENTRY = "Concepts_Groups"
VARIABLES_ENTRY = "Variables" # entry for admin or internal variables
CONCEPTS_KEYS_ENTRY = "Concepts_Keys"
BUILTIN_CONCEPTS_KEYS = "Builtins_Concepts" # sequential key for builtin concepts
USER_CONCEPTS_KEYS = "User_Concepts" # sequential key for user defined concepts
def __init__(self, skip_builtins_in_db=False, debug=False, loggers=None):
def __init__(self, cache_only=False, debug=False, loggers=None):
self.init_logging(debug, loggers)
self.loggers = loggers
super().__init__(BuiltinConcepts.SHEERKA, True, True, BuiltinConcepts.SHEERKA)
self.log.debug("Starting Sheerka.")
# cache of the most used concepts
# Note that these are only templates
# They are used as a footprint for instantiation
# Except of source when the concept is supposed to be unique
# key is the key of the concept (not the name or the id)
self.cache_by_key = {}
self.cache_by_id = {}
self.cache_by_name = {}
self.bnp = None # reference to the BaseNodeParser class (to compute first keyword token)
# cache for concept definitions,
# Primarily used for unit test that does not have access to sdp
self.concepts_definitions_cache = {}
#
# cache for concepts grammars
# a grammar is a resolved BNF
self.concepts_grammars = {}
# cache for SYA concepts
self.concepts_by_first_keyword = {}
self.sya_definitions = {}
# # Cache for concepts grammars
# # To be shared between BNFNode parsers instances
# self.concepts_grammars = {}
# a concept can be instantiated
# ex: File is a concept, but File('foo.txt') is an instance
@@ -78,6 +74,8 @@ class Sheerka(Concept):
self.rules = []
self.sdp: SheerkaDataProvider = None # SheerkaDataProvider
self.cache_manager = CacheManager(cache_only)
self.builtin_cache = {} # cache for builtin concepts
self.parsers = {} # cache for builtin parsers
self.evaluators = [] # cache for builtin evaluators
@@ -85,8 +83,6 @@ class Sheerka(Concept):
self.evaluators_prefix: str = None
self.parsers_prefix: str = None
self.skip_builtins_in_db = skip_builtins_in_db
self.execute_handler = SheerkaExecute(self)
self.create_new_concept_handler = SheerkaCreateNewConcept(self)
self.modify_concept_handler = SheerkaModifyConcept(self)
@@ -100,36 +96,65 @@ class Sheerka(Concept):
self.during_restore = False
self._builtins_classes_cache = None
def initialize(self, root_folder: str = None):
self.save_execution_context = True
@property
def resolved_concepts_by_first_keyword(self):
"""
We return the cache as we will be interested by statistics
:return:
"""
return self.cache_manager.caches[self.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY].cache
@property
def resolved_sya_def(self):
"""
:return:
"""
return self.cache_manager.caches[self.RESOLVED_CONCEPTS_SYA_DEFINITION_ENTRY].cache
@property
def concepts_grammars(self):
return self.cache_manager.caches[self.CONCEPTS_GRAMMARS_ENTRY].cache
def initialize(self, root_folder: str = None, save_execution_context=True):
"""
Starting Sheerka
Loads the current configuration
Notes that when it's the first time, it also create the needed working folders
:param root_folder: root configuration folder
:param save_execution_context:
:return: ReturnValue(Success or Error)
"""
self.save_execution_context = save_execution_context
try:
from sheerkapickle.sheerka_handlers import initialize_pickle_handlers
initialize_pickle_handlers()
self.sdp = SheerkaDataProvider(root_folder, self)
if self.sdp.first_time:
self.sdp.set_key(self.USER_CONCEPTS_KEYS, 1000)
self.initialize_caching()
event = Event("Initializing Sheerka.", user=self.name)
event = Event("Initializing Sheerka.", user_id=self.name)
self.sdp.save_event(event)
with ExecutionContext(self.key, event, self, "Initializing Sheerka.", self.init_log) as exec_context:
if self.sdp.first_time:
self.first_time_initialisation(exec_context)
self.initialize_builtin_concepts()
self.initialize_builtin_parsers()
self.initialize_builtin_evaluators()
self.initialize_bnf_parsing(exec_context)
self.initialize_sya_parsing()
self.initialize_builtin_concepts()
self.initialize_concept_node_parsing(exec_context)
res = ReturnValueConcept(self, True, self)
exec_context.add_values(return_values=res)
if not self.skip_builtins_in_db:
if self.cache_manager.is_dirty:
self.cache_manager.commit(exec_context)
if save_execution_context:
self.sdp.save_result(exec_context, is_admin=True)
self.init_log.debug(f"Sheerka successfully initialized")
@@ -138,6 +163,59 @@ class Sheerka(Concept):
return res
def initialize_caching(self):
def params(cache_name):
return {
'default': lambda k: self.sdp.get(cache_name, k),
'extend_exists': lambda k: self.sdp.exists(cache_name, k)
}
cache = IncCache(default=lambda k: self.sdp.get(self.CONCEPTS_KEYS_ENTRY, k))
self.cache_manager.register_cache(self.CONCEPTS_KEYS_ENTRY, cache)
register_concept_cache = self.cache_manager.register_concept_cache
cache = Cache(**params(self.CONCEPTS_BY_ID_ENTRY))
register_concept_cache(self.CONCEPTS_BY_ID_ENTRY, cache, lambda c: c.id, True)
cache = ListIfNeededCache(**params(self.CONCEPTS_BY_KEY_ENTRY))
register_concept_cache(self.CONCEPTS_BY_KEY_ENTRY, cache, lambda c: c.key, True)
cache = ListIfNeededCache(**params(self.CONCEPTS_BY_NAME_ENTRY))
register_concept_cache(self.CONCEPTS_BY_NAME_ENTRY, cache, lambda c: c.name, True)
cache = ListIfNeededCache(**params(self.CONCEPTS_BY_HASH_ENTRY))
register_concept_cache(self.CONCEPTS_BY_HASH_ENTRY, cache, lambda c: c.get_definition_hash(), True)
cache = DictionaryCache(default=lambda k: self.sdp.get(self.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, k))
self.cache_manager.register_cache(self.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, cache)
self.cache_manager.get(self.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, None) # to init from sdp
cache = DictionaryCache(default=lambda k: self.sdp.get(self.CONCEPTS_SYA_DEFINITION_ENTRY, k))
self.cache_manager.register_cache(self.CONCEPTS_SYA_DEFINITION_ENTRY, cache)
self.cache_manager.get(self.CONCEPTS_SYA_DEFINITION_ENTRY, None) # to init from sdp
cache = SetCache(default=lambda k: self.sdp.get(self.CONCEPTS_GROUPS_ENTRY, k))
self.cache_manager.register_cache(self.CONCEPTS_GROUPS_ENTRY, cache)
cache = Cache(default=lambda k: self.sdp.get(self.VARIABLES_ENTRY, k))
self.cache_manager.register_cache(self.VARIABLES_ENTRY, cache, True, True)
cache = DictionaryCache()
self.cache_manager.register_cache(self.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY, cache, persist=False)
cache = DictionaryCache()
self.cache_manager.register_cache(self.RESOLVED_CONCEPTS_SYA_DEFINITION_ENTRY, cache, persist=False)
cache = Cache()
self.cache_manager.register_cache(self.CONCEPTS_GRAMMARS_ENTRY, cache, persist=False)
def first_time_initialisation(self, context):
self.cache_manager.put(self.CONCEPTS_KEYS_ENTRY, self.USER_CONCEPTS_KEYS, 1000)
self.variable_handler.record(context, self.name, "save_execution_context", True)
def initialize_builtin_concepts(self):
"""
Initializes the builtin concepts
@@ -160,18 +238,16 @@ class Sheerka(Concept):
if not concept.metadata.is_unique and str(key) in builtins_classes:
self.builtin_cache[key] = builtins_classes[str(key)]
if not self.skip_builtins_in_db:
from_db = self.sdp.get_safe(self.CONCEPTS_ENTRY, concept.metadata.key)
if from_db is None:
self.init_log.debug(f"'{concept.name}' concept is not found in db. Adding.")
self.set_id_if_needed(concept, True)
concept.metadata.full_serialization = True
self.sdp.add("init", self.CONCEPTS_ENTRY, concept, use_ref=True)
else:
self.init_log.debug(f"Found concept '{from_db}' in db. Updating.")
concept.update_from(from_db)
from_db = self.cache_manager.get(self.CONCEPTS_BY_KEY_ENTRY, concept.metadata.key)
if from_db is None:
self.init_log.debug(f"'{concept.name}' concept is not found in db. Adding.")
self.set_id_if_needed(concept, True)
self.cache_manager.add_concept(concept)
else:
self.init_log.debug(f"Found concept '{from_db}' in db. Updating.")
concept.update_from(from_db)
self.add_in_cache(concept)
return
def initialize_builtin_parsers(self):
"""
@@ -187,17 +263,23 @@ class Sheerka(Concept):
if parser.__module__ == base_class.__module__:
continue
if parser.__module__ in modules_to_skip:
continue
qualified_name = core.utils.get_full_qualified_name(parser)
self.init_log.debug(f"Adding builtin parser '{qualified_name}'")
temp_result[qualified_name] = parser
# keep a reference to base_node_parser
self.bnp = temp_result[BASE_NODE_PARSER_CLASS]
# Now we sort the parser by name.
# It's not important for the logic of their usage as they have their priority anyway,
# We do that for the unit tests. They are to complicated to write otherwise
for name in sorted(temp_result.keys()):
parser = temp_result[name]
if parser.__module__ in modules_to_skip:
# base node parser module does not contains any valid parser
continue
self.parsers[name] = temp_result[name]
def initialize_builtin_evaluators(self):
@@ -214,55 +296,39 @@ class Sheerka(Concept):
self.init_log.debug(f"Adding builtin evaluator '{evaluator.__name__}'")
self.evaluators.append(evaluator)
def initialize_bnf_parsing(self, execution_context):
self.init_log.debug("Initializing concepts grammars.")
definitions = self.get_concepts_definitions(execution_context)
def initialize_concept_node_parsing(self, context):
self.init_log.debug("Initializing concept node parsing.")
if definitions is None:
self.init_log.debug("No BNF defined")
return
concepts_by_first_keyword = self.cache_manager.copy(self.CONCEPTS_BY_FIRST_KEYWORD_ENTRY)
res = self.bnp.resolve_concepts_by_first_keyword(context, concepts_by_first_keyword)
self.cache_manager.put(self.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY, False, res.body)
lexer_parser = self.parsers[CONCEPT_LEXER_PARSER_CLASS]()
ret_val = lexer_parser.initialize(execution_context, definitions)
if not ret_val.status:
self.init_log.error("Failed to initialize concepts definitions " + str(ret_val.body))
return
# sya = self.bnf.resolve_sya_associativity_and_precedence()
# self.cache_manager.put(self.RESOLVED_CONCEPTS_SYA_DEFINITION_ENTRY, sya)
#
#
# self.concepts_by_first_keyword, \
# self.resolved_concepts_by_first_keyword = \
# self.create_new_concept_handler.load_concepts_nodes_definitions(context)
self.concepts_grammars = lexer_parser.concepts_grammars
# self.concepts_by_first_keyword = self.sdp.get_safe(
# self.CONCEPTS_BY_FIRST_KEYWORD_ENTRY,
# load_origin=False) or {}
#
# self.sya_definitions = self.sdp.get_safe(
# self.CONCEPTS_SYA_DEFINITION_ENTRY,
# load_origin=False) or {}
#
# init_ret_value = self.bnp.resolve_concepts_by_first_keyword(self, self.concepts_by_first_keyword)
# if not init_ret_value.status:
# return self.sheerka.ret(self.logger_name, False, ErrorConcept(init_ret_value.value))
# self.resolved_concepts_by_first_keyword = init_ret_value.body
def initialize_sya_parsing(self):
self.init_log.debug("Initializing sya definitions.")
self.concepts_by_first_keyword = self.sdp.get_safe(
self.CONCEPTS_BY_FIRST_KEYWORD_ENTRY,
load_origin=False) or {}
self.sya_definitions = self.sdp.get_safe(
self.CONCEPTS_SYA_DEFINITION_ENTRY,
load_origin=False) or {}
def reset(self):
self.reset_cache()
self.concepts_by_first_keyword = {}
self.concepts_grammars = {}
self.sya_definitions = {}
def reset(self, cache_only=False):
self.cache_manager.clear()
self.cache_manager.cache_only = cache_only
self.printer_handler.reset()
self.sdp.reset()
self.sdp.set_key(self.USER_CONCEPTS_KEYS, 1000)
def reset_cache(self, filter_to_use=None):
"""
reset the different cache that exists
:param filter_to_use:
:return:
"""
if filter_to_use is None:
self.cache_by_key = {}
self.cache_by_id = {}
self.cache_by_name = {}
else:
raise NotImplementedError()
return self
def evaluate_user_input(self, text: str, user_name="kodjo"):
"""
@@ -294,7 +360,10 @@ class Sheerka(Concept):
ret = self.execute(execution_context, [user_input, reduce_requested], steps)
execution_context.add_values(return_values=ret)
if not self.skip_builtins_in_db:
if self.cache_manager.is_dirty:
self.cache_manager.commit(execution_context)
if self.save_execution_context and self.variable_handler.load(self.name, "save_execution_context"):
self.sdp.save_result(execution_context)
# # hack to save valid concept definition
@@ -302,6 +371,8 @@ class Sheerka(Concept):
# if len(ret) == 1 and ret[0].status and self.isinstance(ret[0].value, BuiltinConcepts.NEW_CONCEPT):
# with open(CONCEPTS_FILE, "a") as f:
# f.write(text + "\n")
self._last_execution = execution_context
return ret
def print(self, result, instructions=None):
@@ -343,8 +414,8 @@ class Sheerka(Concept):
if obj.metadata.id is not None:
return
entry = self.BUILTIN_CONCEPTS_KEYS if is_builtin else self.USER_CONCEPTS_KEYS
obj.metadata.id = self.sdp.get_next_key(entry)
key = self.BUILTIN_CONCEPTS_KEYS if is_builtin else self.USER_CONCEPTS_KEYS
obj.metadata.id = str(self.cache_manager.get(self.CONCEPTS_KEYS_ENTRY, key))
self.log.debug(f"Setting id '{obj.metadata.id}' to concept '{obj.metadata.name}'.")
def create_new_concept(self, context, concept: Concept):
@@ -380,21 +451,25 @@ class Sheerka(Concept):
"""
return self.sets_handler.set_isa(context, concept, concept_set)
def set_sya_def(self, context, list_of_def):
def force_sya_def(self, context, list_of_def):
"""
Set the precedence and/or the associativity of a concept
FOR TESTS PURPOSE. TO REMOVE EVENTUALLY
:param context:
:param list_of_def list of tuple(concept_id, precedence (int), SyaAssociativity)
:return:
"""
# validate the entries
# If one entry is an invalid concept, rollback everything
for concept_id, precedence, associativity in list_of_def:
if concept_id == BuiltinConcepts.UNKNOWN_CONCEPT:
return self.ret(self.name,
False,
self.new(BuiltinConcepts.ERROR, body=f"Concept {concept_id} is not known"))
sya_def = self.cache_manager.copy(self.RESOLVED_CONCEPTS_SYA_DEFINITION_ENTRY) or {}
# update the definitions
for concept_id, precedence, associativity in list_of_def:
if precedence is None and associativity is None:
@@ -403,12 +478,10 @@ class Sheerka(Concept):
except KeyError:
pass
else:
self.sya_definitions[concept_id] = (precedence, associativity.value)
sya_def[concept_id] = (precedence, associativity)
# then save
self.sdp.set(context.event.get_digest(),
self.CONCEPTS_SYA_DEFINITION_ENTRY,
self.sya_definitions)
# put in cache
self.cache_manager.put(self.RESOLVED_CONCEPTS_SYA_DEFINITION_ENTRY, False, sya_def)
return self.ret(self.name, True, self.new(BuiltinConcepts.SUCCESS))
@@ -448,122 +521,108 @@ class Sheerka(Concept):
if concept.key is None:
raise KeyError()
self.cache_by_key[concept.key] = concept
if concept.id:
self.cache_by_id[concept.id] = concept
self.cache_manager.add_concept(concept)
return concept
def get(self, concept_key, concept_id=None):
#
# def get(self, concept_key, concept_id=None):
# """
# Tries to find a concept
# What is return must be used a template for another concept.
# You must not modify the returned concept
# :param concept_key: key of the concept
# :param concept_id: when multiple concepts with the same key, use the id
# :return:
# """
#
# by_key = self.get_by_key(concept_key)
# if self.is_known(by_key):
# return by_key
#
# # else return by name
# by_name = self.get_by_name(concept_key)
# if self.is_known(by_name):
# return by_name
#
# return by_key # return not found for key
def get_by_key(self, concept_key, concept_id=None):
concept_key = str(concept_key) if isinstance(concept_key, BuiltinConcepts) else concept_key
return self.internal_get("key", concept_key, self.CONCEPTS_BY_KEY_ENTRY, concept_id)
def get_by_name(self, concept_name, concept_id=None):
return self.internal_get("name", concept_name, self.CONCEPTS_BY_NAME_ENTRY, concept_id)
def get_by_hash(self, concept_hash, concept_id=None):
return self.internal_get("hash", concept_hash, self.CONCEPTS_BY_HASH_ENTRY, concept_id)
def get_by_id(self, concept_id):
return self.internal_get("id", concept_id, self.CONCEPTS_BY_ID_ENTRY, None)
def internal_get(self, index_name, key, cache_name, concept_id=None):
"""
Tries to find a concept
What is return must be used a template for another concept.
You must not modify the returned concept
:param concept_key: key of the concept
:param concept_id: when multiple concepts with the same key, use the id
Tries to find an entry
:param index_name: name of the index (ex by_id, by_key...)
:param key: index value
:param cache_name: name of the cache (ex Concepts_By_ID...)
:param concept_id: id of the concept if none, in case where there are multiple results
:return:
"""
by_key = self.internal_get("key", concept_key, self.cache_by_key, self.CONCEPTS_ENTRY, concept_id)
if self.is_known(by_key):
return by_key
if key is None:
return ErrorConcept(f"Concept '{key}' is undefined.")
# else return by name
by_name = self.internal_get("name", concept_key, self.cache_by_name, self.CONCEPTS_BY_NAME_ENTRY, concept_id)
if self.is_known(by_name):
return by_name
concepts = self.cache_manager.get(cache_name, key)
if concepts:
if concept_id is None:
return concepts
return by_key # return not found for key
if not hasattr(concepts, "__iter__"):
return concepts
def get_by_key(self, concept_key, concept_id=None):
return self.internal_get("key", concept_key, self.cache_by_key, self.CONCEPTS_ENTRY, concept_id)
for c in concepts:
if c.id == concept_id:
return c
def get_by_name(self, concept_name, concept_id=None):
return self.internal_get("name", concept_name, self.cache_by_name, self.CONCEPTS_BY_NAME_ENTRY, concept_id)
metadata = [(index_name, key), ("id", concept_id)] if concept_id else (index_name, key)
return self._get_unknown(metadata)
def get_by_id(self, concept_id):
if concept_id is None:
return ErrorConcept("Concept id is undefined.")
# first search in cache
if concept_id in self.cache_by_id:
result = self.cache_by_id[concept_id]
else:
result = self.sdp.get_safe(self.CONCEPTS_BY_ID_ENTRY, concept_id)
if result is None:
result = self._get_unknown(('id', concept_id))
else:
self.cache_by_id[concept_id] = result
return result
def internal_get(self, index_name, index_value, cache_to_use, sdp_entry, concept_id=None):
def has_id(self, concept_id):
"""
Tries to find an entry
:param index_name:
:param index_value:
:param cache_to_use:
:param sdp_entry:
Returns True if a concept with this id exists in cache
It does not search in the remote repository
:param concept_id:
:return:
"""
return self.cache_manager.has(self.CONCEPTS_BY_ID_ENTRY, concept_id)
if index_value is None:
return ErrorConcept(f"Concept {index_name} is undefined.")
def has_key(self, concept_key):
"""
Returns True if concept(s) with this key exist in cache
It does not search in the remote repository
:param concept_key:
:return:
"""
return self.cache_manager.has(self.CONCEPTS_BY_KEY_ENTRY, concept_key)
if isinstance(index_value, BuiltinConcepts):
index_value = str(index_value)
def has_name(self, concept_name):
"""
Returns True if concept(s) with this name exist in cache
It does not search in the remote repository
:param concept_name:
:return:
"""
return self.cache_manager.has(self.CONCEPTS_BY_NAME_ENTRY, concept_name)
# first search in cache
if index_value in cache_to_use:
result = cache_to_use[index_value]
else:
result = self.sdp.get_safe(sdp_entry, index_value)
if result is None:
metadata = [(index_name, index_value), ("id", concept_id)] if concept_id else (index_name, index_value)
result = self._get_unknown(metadata)
# Do not put in cache_by_key or cache_by_id unknown concept
# TODO: implement an MRU cache for them
else:
cache_to_use[index_value] = result
for r in (result if isinstance(result, list) else [result]):
if r.id:
self.cache_by_id[r.id] = r
if not (isinstance(result, list) and concept_id):
return result
# result is a list, but we have the concept_id to discriminate
for c in result:
if c.id == concept_id:
return c
metadata = [(index_name, index_value), ("id", concept_id)] if concept_id else (index_name, index_value)
return self._get_unknown(metadata)
def get_concepts_definitions(self, context):
if self.concepts_definitions_cache:
return self.concepts_definitions_cache
encoded_bnf = self.sdp.get_safe(
self.CONCEPTS_DEFINITIONS_ENTRY,
load_origin=False) or {}
self.concepts_definitions_cache = {}
bnf_parser = self.parsers[BNF_PARSER_CLASS]()
for k, v in encoded_bnf.items():
key, id_ = core.utils.unstr_concept(k)
concept = self.new((key, id_))
context.log(f"Parsing BNF definition for {concept}", context.who)
rule_result = bnf_parser.parse(context, v)
if rule_result.status:
self.concepts_definitions_cache[concept] = rule_result.value.value
else:
self.log.error(f"Failed to load bnf rule for concept {key}")
return self.concepts_definitions_cache
def has_hash(self, concept_hash):
"""
Returns True if concept(s) with this hash exist in cache
It does not search in the remote repository
:param concept_hash:
:return:
"""
return self.cache_manager.has(self.CONCEPTS_BY_HASH_ENTRY, concept_hash)
def new(self, concept_key, **kwargs):
"""
@@ -578,7 +637,7 @@ class Sheerka(Concept):
else:
concept_id = None
template = self.get_by_id(concept_id) if not concept_key else self.get(concept_key, concept_id)
template = self.get_by_id(concept_id) if not concept_key else self.get_by_key(concept_key, concept_id)
# manage concept not found
if self.isinstance(template, BuiltinConcepts.UNKNOWN_CONCEPT) and \
@@ -599,7 +658,7 @@ class Sheerka(Concept):
# otherwise, create another instance
concept = self.builtin_cache[key]() if key in self.builtin_cache else Concept()
concept.update_from(template)
concept.update_from(template, update_value=False)
concept.freeze_definition_hash()
if len(kwargs) == 0:
@@ -608,10 +667,10 @@ class Sheerka(Concept):
# update the properties, values, attributes
# Not quite sure that this is the correct process order
for k, v in kwargs.items():
if k in concept.props:
concept.set_prop(k, v)
if k in concept.values:
concept.set_value(k, v)
elif k in PROPERTIES_FOR_NEW:
concept.values[ConceptParts(k)] = v
concept.set_value(ConceptParts(k), v)
elif hasattr(concept, k):
setattr(concept, k, v)
else:
@@ -639,12 +698,12 @@ class Sheerka(Concept):
message=message,
parents=parents)
def value(self, obj, reduce_simple_list=False):
def objvalue(self, obj, reduce_simple_list=False):
if obj is None:
return None
if hasattr(obj, "get_value"):
return obj.get_value()
if hasattr(obj, "get_obj_value"):
return obj.get_obj_value()
if not isinstance(obj, Concept):
return obj
@@ -657,7 +716,18 @@ class Sheerka(Concept):
else:
body_to_use = obj.body
return self.value(body_to_use)
return self.objvalue(body_to_use)
def objvalues(self, objs):
if not (isinstance(objs, list) or
self.isinstance(objs, BuiltinConcepts.LIST) or
self.isinstance(objs, BuiltinConcepts.ENUMERATION)):
objs = [objs]
if isinstance(objs, list):
return (self.objvalue(obj) for obj in objs)
return (self.objvalue(obj) for obj in objs.body)
def value_by_concept(self, obj, concept):
if obj is None:
@@ -678,8 +748,8 @@ class Sheerka(Concept):
if isinstance(obj, Concept) and obj.metadata.is_builtin and obj.key in BuiltinErrors:
return obj
if isinstance(obj, list):
return obj
if isinstance(obj, (list, set, tuple)):
return [self.get_error(o) for o in obj]
if self.isinstance(obj, BuiltinConcepts.RETURN_VALUE):
if obj.status:
@@ -687,19 +757,10 @@ class Sheerka(Concept):
if self.isinstance(obj.body, BuiltinConcepts.PARSER_RESULT):
return self.get_error(obj.body.body)
else:
return obj.body
return NotImplementedError()
def get_values(self, objs):
if not (isinstance(objs, list) or
self.isinstance(objs, BuiltinConcepts.LIST) or
self.isinstance(objs, BuiltinConcepts.ENUMERATION)):
objs = [objs]
if isinstance(objs, list):
return (self.value(obj) for obj in objs)
return (self.value(obj) for obj in objs.body)
raise NotImplementedError()
def is_success(self, obj):
if isinstance(obj, bool): # quick win
@@ -761,8 +822,12 @@ class Sheerka(Concept):
return self.parsers_prefix + name
def concepts(self):
"""
List of all known concepts (look up in sdp)
:return:
"""
res = []
lst = self.sdp.list(self.CONCEPTS_ENTRY)
lst = self.sdp.list(self.CONCEPTS_BY_ID_ENTRY)
for item in lst:
if isinstance(item, list):
res.extend(item)
@@ -818,10 +883,10 @@ class Sheerka(Concept):
# the metadata can be a list, if several attributes where given
# (key, 'not_found), (id, invalid_id)
unknown_concept = UnknownConcept()
unknown_concept.set_metadata_value(ConceptParts.BODY, metadata)
unknown_concept = UnknownConcept() # don't use new() for prevent circular reference
unknown_concept.set_value(ConceptParts.BODY, metadata)
for meta in (metadata if isinstance(metadata, list) else [metadata]):
unknown_concept.set_prop(meta[0], meta[1])
unknown_concept.set_value(meta[0], meta[1])
unknown_concept.metadata.is_evaluated = True
return unknown_concept
src/core/utils.py
+4 -2
View File
@@ -261,7 +261,7 @@ def decode_enum(enum_repr: str):
return None
def str_concept(t):
def str_concept(t, skip_key=None):
"""
The key,id identifiers of a concept are stored in a tuple
we want to return the key and the id, separated by a pipe
@@ -272,7 +272,9 @@ def str_concept(t):
>>> assert str_concept(("key", None)) == "c:key:"
>>> assert str_concept((None, None)) == ""
>>> assert str_concept(Concept(key="foo", id="bar")) == "c:foo|bar:"
>>> assert str_concept(Concept(key="foo", id="bar"), skip_key=True) == "c:|bar:"
:param t:
:param skip_key: True if we only want the id (and not the key)
:return:
"""
if isinstance(t, tuple):
@@ -283,7 +285,7 @@ def str_concept(t):
if key is None and id_ is None:
return ""
result = 'c:' if key is None else "c:" + key
result = 'c:' if (key is None or skip_key) else "c:" + key
if id_:
result += "|" + id_
return result + ":"