kodjo/Sheerka-Old
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactored Concept class for better separation of metadata, compiled and values

Browse Source
This commit is contained in:
Kodjo Sossouvi
2020-01-17 17:27:54 +01:00
parent 3789ef25d1
commit a7b239c167
27 changed files with 614 additions and 349 deletions
Download Patch File Download Diff File Expand all files Collapse all files
core/ast/nodes.py
+4 -1
View File
@@ -86,7 +86,7 @@ class CallNodeConcept(NodeConcept):
super().__init__(BuiltinConcepts.IDENTIFIER_NODE, "Call", parent)
def get_args_names(self, sheerka):
return sheerka.values(self.get_prop("args"))
return sheerka.get_values(self.get_prop("args"))
def python_to_concept(python_node):
@@ -105,6 +105,7 @@ def python_to_concept(python_node):
continue
value = getattr(node, field)
concept.def_prop(field)
if isinstance(value, list):
lst = ListConcept().init_key()
for i in value:
@@ -114,6 +115,8 @@ def python_to_concept(python_node):
concept.set_prop(field, _transform(value, NodeParent(concept, field)))
else:
concept.set_prop(field, value)
concept.metadata.is_evaluated = True
return concept
return _transform(python_node, None)
core/ast/visitors.py
+1 -1
View File
@@ -84,7 +84,7 @@ class UnreferencedNamesVisitor(ConceptNodeVisitor):
if node.get_node_type() == "FunctionDef":
# variable defined as a function parameter
args = node.get_prop("args")
args_values = list(self.sheerka.values(args.get_prop("args")))
args_values = list(self.sheerka.get_values(args.get_prop("args")))
if variable_name in args_values:
return True
core/builtin_concepts.py
+32 -15
View File
@@ -1,6 +1,6 @@
from enum import Enum
from core.concept import Concept
from core.concept import Concept, ConceptParts
class BuiltinConcepts(Enum):
@@ -97,8 +97,10 @@ It's mainly to ease the usage
class UserInputConcept(Concept):
def __init__(self, text=None, user_name=None):
super().__init__(BuiltinConcepts.USER_INPUT, True, False, BuiltinConcepts.USER_INPUT, text)
super().__init__(BuiltinConcepts.USER_INPUT, True, False, BuiltinConcepts.USER_INPUT)
self.set_metadata_value(ConceptParts.BODY, text)
self.set_prop("user_name", user_name)
self.metadata.is_evaluated = True
@property
def text(self):
@@ -114,7 +116,9 @@ class UserInputConcept(Concept):
class ErrorConcept(Concept):
def __init__(self, error=None):
super().__init__(BuiltinConcepts.ERROR, True, False, BuiltinConcepts.ERROR, error)
super().__init__(BuiltinConcepts.ERROR, True, False, BuiltinConcepts.ERROR)
self.set_metadata_value(ConceptParts.BODY, error)
self.metadata.is_evaluated = True
def __repr__(self):
return f"({self.id}){self.name}: {self.body}"
@@ -127,11 +131,13 @@ class ReturnValueConcept(Concept):
"""
def __init__(self, who=None, status=None, value=None, message=None, parents=None):
super().__init__(BuiltinConcepts.RETURN_VALUE, True, False, BuiltinConcepts.RETURN_VALUE, value)
super().__init__(BuiltinConcepts.RETURN_VALUE, True, False, BuiltinConcepts.RETURN_VALUE)
self.set_metadata_value(ConceptParts.BODY, value)
self.set_prop("who", who)
self.set_prop("status", status)
self.set_prop("message", message)
self.set_prop("parents", parents)
self.metadata.is_evaluated = True
@property
def who(self):
@@ -155,7 +161,7 @@ class ReturnValueConcept(Concept):
@value.setter
def value(self, value):
self.metadata.body = value
self.set_metadata_value(ConceptParts.BODY, value)
@property
def message(self):
@@ -200,8 +206,10 @@ class UnknownPropertyConcept(Concept):
"""
def __init__(self, property_name=None, concept=None):
super().__init__(BuiltinConcepts.UNKNOWN_PROPERTY, True, False, BuiltinConcepts.UNKNOWN_PROPERTY, property_name)
super().__init__(BuiltinConcepts.UNKNOWN_PROPERTY, True, False, BuiltinConcepts.UNKNOWN_PROPERTY)
self.set_metadata_value(ConceptParts.BODY, property_name)
self.set_prop("concept", concept)
self.metadata.is_evaluated = True
def __repr__(self):
return f"UnknownProperty(property={self.property_name}, concept={self.concept})"
@@ -221,10 +229,12 @@ class ParserResultConcept(Concept):
"""
def __init__(self, parser=None, source=None, value=None, try_parsed=None):
super().__init__(BuiltinConcepts.PARSER_RESULT, True, False, BuiltinConcepts.PARSER_RESULT, value)
super().__init__(BuiltinConcepts.PARSER_RESULT, True, False, BuiltinConcepts.PARSER_RESULT)
self.set_metadata_value(ConceptParts.BODY, value)
self.set_prop("parser", parser)
self.set_prop("source", source)
self.set_prop("try_parsed", try_parsed) # in case of error, what was found before the error
self.metadata.is_evaluated = True
def __repr__(self):
text = f"ParserResult(parser={self.props['parser'].value}"
@@ -273,9 +283,10 @@ class InvalidReturnValueConcept(Concept):
BuiltinConcepts.INVALID_RETURN_VALUE,
True,
False,
BuiltinConcepts.INVALID_RETURN_VALUE,
return_value)
BuiltinConcepts.INVALID_RETURN_VALUE)
self.set_metadata_value(ConceptParts.BODY, return_value)
self.set_prop("evaluator", evaluator)
self.metadata.is_evaluated = True
class ConceptEvalError(Concept):
@@ -283,10 +294,11 @@ class ConceptEvalError(Concept):
super().__init__(BuiltinConcepts.CONCEPT_EVAL_ERROR,
True,
False,
BuiltinConcepts.CONCEPT_EVAL_ERROR,
error)
BuiltinConcepts.CONCEPT_EVAL_ERROR)
self.set_metadata_value(ConceptParts.BODY, error)
self.set_prop("concept", concept)
self.set_prop("property_name", property_name)
self.metadata.is_evaluated = True
def __repr__(self):
return f"ConceptEvalError(error={self.error}, concept={self.concept}, property={self.property_name})"
@@ -306,7 +318,9 @@ class ConceptEvalError(Concept):
class EnumerationConcept(Concept):
def __init__(self, iteration=None):
super().__init__(BuiltinConcepts.ENUMERATION, True, False, BuiltinConcepts.ENUMERATION, iteration)
super().__init__(BuiltinConcepts.ENUMERATION, True, False, BuiltinConcepts.ENUMERATION)
self.set_metadata_value(ConceptParts.BODY, iteration)
self.metadata.is_evaluated = True
def __iter__(self):
return iter(self.body)
@@ -314,7 +328,9 @@ class EnumerationConcept(Concept):
class ListConcept(Concept):
def __init__(self, items=None):
super().__init__(BuiltinConcepts.LIST, True, False, BuiltinConcepts.LIST, items or [])
super().__init__(BuiltinConcepts.LIST, True, False, BuiltinConcepts.LIST)
self.set_metadata_value(ConceptParts.BODY, items or [])
self.metadata.is_evaluated = True
def append(self, obj):
self.body.append(obj)
@@ -340,9 +356,10 @@ class ConceptAlreadyInSet(Concept):
super().__init__(BuiltinConcepts.CONCEPT_ALREADY_IN_SET,
True,
False,
BuiltinConcepts.CONCEPT_ALREADY_IN_SET,
concept)
BuiltinConcepts.CONCEPT_ALREADY_IN_SET)
self.set_metadata_value(ConceptParts.BODY, concept)
self.set_prop("concept_set", concept_set)
self.metadata.is_evaluated = True
def __repr__(self):
return f"ConceptAlreadyInSet(concept={self.concept}, concept_set={self.concept_set})"
core/builtin_helpers.py
+4 -3
View File
@@ -121,10 +121,11 @@ def get_names(sheerka, concept_node):
def extract_predicates(sheerka, expression, variables_to_include, variables_to_exclude):
"""
from expression, tries to find all the predicates referencing a variable, and the variable only
from a given expression and a variable (or list of variables)
tries to find out all the predicates referencing the(se) variable(s), and the(se) variable(s) solely
for example
exp : isinstance(a, int) and isinstance(b, str)
will return 'isinstance(a, int)' if variable_name == 'a'
exp : isinstance(a, int) and isinstance(b, str)
will return 'isinstance(a, int)' if variable_name == 'a'
:param sheerka:
:param expression:
:param variables_to_include:
core/concept.py
+126 -29
View File
@@ -1,5 +1,6 @@
import hashlib
from dataclasses import dataclass
from collections import namedtuple
from dataclasses import dataclass, field
from enum import Enum
from core.sheerka_logger import get_logger
@@ -10,7 +11,7 @@ PROPERTIES_FOR_DIGEST = ("name", "key",
"definition", "definition_type",
"is_builtin", "is_unique",
"where", "pre", "post", "body",
"desc")
"desc", "props")
PROPERTIES_TO_SERIALIZE = PROPERTIES_FOR_DIGEST + tuple(["id"])
PROPERTIES_FOR_NEW = ("where", "pre", "post", "body", "desc")
VARIABLE_PREFIX = "__var__"
@@ -44,9 +45,13 @@ class ConceptMetadata:
definition_type: str # definition can be done with something else than regex
desc: str # possible description for the concept
id: str # unique identifier for a concept. The id will never be modified (but the key can)
props: list # list properties, with their default values
is_evaluated: bool = False # True is the concept is evaluated by sheerka.eval_concept()
simplec = namedtuple("concept", "name body") # for simple concept (tests purposes only)
class Concept:
"""
Default concept object
@@ -65,7 +70,8 @@ class Concept:
definition=None,
definition_type=None,
desc=None,
id=None):
id=None,
props=None):
metadata = ConceptMetadata(
str(name) if name else None,
@@ -79,12 +85,14 @@ class Concept:
definition,
definition_type,
desc,
id
id,
props or []
)
self.metadata = metadata
self.props = {} # list of Property for this concept
self.cached_asts = {} # cached ast for the where, pre, post and body parts
self.compiled = {} # cached ast for the where, pre, post and body parts
self.values = {} # values of metadata once resolved
self.props = {} # resolved properties of this concept
self.bnf = None
self.log = get_logger("core." + self.__class__.__name__)
self.init_log = get_logger("init.core." + self.__class__.__name__)
@@ -93,10 +101,17 @@ class Concept:
return f"({self.metadata.id}){self.metadata.name}"
def __eq__(self, other):
if isinstance(other, simplec):
return self.name == other.name and self.body == other.body
if id(self) == id(other):
return True
if not isinstance(other, Concept):
return False
# check the attributes
# check the metadata
for prop in PROPERTIES_TO_SERIALIZE:
# print(prop) # use full to know which id does not match
my_value = getattr(self.metadata, prop)
@@ -119,9 +134,19 @@ class Concept:
if my_value != other_value:
return False
# check the props (Concept variables)
for var_name, p in self.props.items():
if p != other.props[var_name]:
# checks the values
if len(self.values) != len(other.values):
return False
for metadata in self.values:
if self.get_metadata_value(metadata) != other.get_metadata_value(metadata):
return False
if len(self.props) != len(other.props):
return False
for prop in self.props:
if self.get_prop(prop) != other.get_prop(prop):
return False
return True
@@ -138,6 +163,33 @@ class Concept:
name = self.name if 'metadata' in vars(self) else 'Concept'
raise AttributeError(f"'{name}' concept has no attribute '{item}'")
def def_prop(self, prop_name: str, default_value=None):
"""
Adds a property to the metadata
:param prop_name:
:param default_value:
:return:
"""
assert default_value is None or isinstance(default_value, str) # default properties will have to be evaluated
self.metadata.props.append((prop_name, default_value))
self.props[prop_name] = Property(prop_name, None) # do not set the default value
# why not setting props to the default values ?
# Because it may not be the real values, as metadata.props need to be evaluated
return self
def def_prop_by_index(self, index: int, value):
"""
Re-assign a value to a property (mainly used by ExactConceptParser)
:param index:
:param value:
:return:
"""
assert value is None or isinstance(value, str) # default properties will have to be evaluated
prop = self.metadata.props[index]
self.metadata.props[index] = (prop[0], value)
return self
@property
def name(self):
return self.metadata.name
@@ -165,7 +217,7 @@ class Concept:
if tokens is None:
tokens = list(Tokenizer(self.metadata.name))
variables = list(self.props.keys()) if len(core.utils.strip_tokens(tokens, True)) > 1 else []
variables = [p[0] for p in self.metadata.props] if len(core.utils.strip_tokens(tokens, True)) > 1 else []
key = ""
first = True
@@ -175,8 +227,8 @@ class Concept:
if token.type == TokenKind.WHITESPACE:
continue
if not first:
key += " " # spaces are normalized
if variables is not None and token.value in variables:
key += " " # spaces are normalized
if token.value in variables:
key += VARIABLE_PREFIX + str(variables.index(token.value))
else:
key += token.value[1:-1] if token.type == TokenKind.STRING else token.value
@@ -187,7 +239,7 @@ class Concept:
@property
def body(self):
return self.metadata.body
return self.values[ConceptParts.BODY] if ConceptParts.BODY in self.values else None
def add_codes(self, codes):
"""
@@ -204,7 +256,7 @@ class Concept:
return
for key in codes:
self.cached_asts[key] = codes[key]
self.compiled[key] = codes[key]
return self
@@ -224,7 +276,6 @@ class Concept:
props_to_use = props_to_use or PROPERTIES_TO_SERIALIZE
props_as_dict = dict((prop, getattr(self.metadata, prop)) for prop in props_to_use)
props_as_dict["props"] = [(p, self.props[p].value) for p in self.props]
return props_as_dict
def from_dict(self, as_dict):
@@ -235,10 +286,11 @@ class Concept:
"""
for prop in PROPERTIES_TO_SERIALIZE:
if prop in as_dict:
setattr(self.metadata, prop, as_dict[prop])
if "props" in as_dict:
for n, v in as_dict["props"]:
self.set_prop(n, v)
if prop == "props":
for name, value in as_dict[prop]:
self.def_prop(name, value)
else:
setattr(self.metadata, prop, as_dict[prop])
return self
def update_from(self, other):
@@ -252,24 +304,69 @@ class Concept:
if other is None:
return self
if id(other) == id(self):
return self
# update metadata
self.from_dict(other.to_dict())
# for prop in self.props_to_serialize:
# setattr(self, prop, getattr(other, prop))
# update values
for k, v in other.values.items():
self.values[k] = v
# update properties
for k, v in other.props.items():
self.set_prop(k, v.value)
return self
def set_prop(self, prop_name: str, prop_value=None):
self.props[prop_name] = Property(prop_name, prop_value) # Python 3.x order is kept in dictionaries
return self
def set_prop_by_index(self, index: int, prop_value):
prop_name = list(self.props.keys())[index]
def set_prop(self, prop_name: str, prop_value):
"""Directly sets a value to a property"""
self.props[prop_name] = Property(prop_name, prop_value)
return self
def get_prop(self, prop_name: str):
return self.props[prop_name].value
def set_metadata_value(self, metadata: ConceptParts, value):
"""
Set the resolved value of a metadata (not the metadata itself)
:param metadata:
:param value:
:return:
"""
self.values[metadata] = value
def get_metadata_value(self, metadata: ConceptParts):
"""
Gets the resolved value of a metadata
:param metadata:
:return:
"""
return self.values[metadata]
def auto_init(self):
"""
Sometimes (for tests purposes)
You don't need the full process of evaluation to to get the values of the concept
Directly use the values of the metadata
:return:
"""
if self.metadata.is_evaluated:
return self
for metadata in ConceptParts:
value = getattr(self.metadata, metadata.value)
if value is not None:
self.values[metadata] = value
for prop, value in self.metadata.props:
self.set_prop(prop, value)
self.metadata.is_evaluated = True
return self
class Property:
"""
@@ -303,6 +400,6 @@ class DoNotResolve:
For example, if you want to set a value to the BODY that will not change when
when the concept will be evaluated,
set concept.cached_asts[BODY] to DoNotResolve(value)
set concept.compiled[BODY] to DoNotResolve(value)
"""
value: object
core/sheerka.py
+44 -37
View File
@@ -571,40 +571,41 @@ class Sheerka(Concept):
"""
steps = [BuiltinConcepts.BEFORE_PARSING, BuiltinConcepts.PARSING, BuiltinConcepts.AFTER_PARSING]
for part_key in ConceptParts:
if part_key in concept.cached_asts:
if part_key in concept.compiled:
continue
source = getattr(concept.metadata, part_key.value)
if source is None or not isinstance(source, str) or source == "":
# the only sources that I am sure to parse are strings
# I refuse empty strings for performance matters, I don't want to handle useless NOPConcepts
if source is None or not isinstance(source, str):
continue
if source.strip() == "":
concept.compiled[part_key] = DoNotResolve(source)
else:
with context.push(desc=f"Initializing AST for {part_key}") as sub_context:
with context.push(desc=f"Initializing compiled for {part_key}") as sub_context:
sub_context.log_new(logger)
sub_context.add_inputs(source=source)
to_parse = self.ret(context.who, True, self.new(BuiltinConcepts.USER_INPUT, body=source))
res = self.execute(sub_context, to_parse, steps, logger)
concept.cached_asts[part_key] = res
concept.compiled[part_key] = res
sub_context.add_values(return_values=res)
for prop in concept.props:
if prop in concept.cached_asts:
for prop, default_value in concept.metadata.props:
if prop in concept.compiled:
continue
value = concept.props[prop].value
if value:
if isinstance(value, Concept):
concept.cached_asts[prop] = value
else:
to_parse = self.ret(
context.who,
True,
self.new(BuiltinConcepts.USER_INPUT, body=value))
with context.push(desc=f"Initializing AST for property {prop}") as sub_context:
sub_context.log_new(logger)
res = self.execute(context, to_parse, steps)
concept.cached_asts[prop] = res
sub_context.add_values(return_values=res)
if default_value is None or not isinstance(default_value, str):
continue
if default_value.strip() == "":
concept.compiled[prop] = DoNotResolve(default_value)
else:
with context.push(desc=f"Initializing AST for property {prop}") as sub_context:
sub_context.log_new(logger)
sub_context.add_inputs(source=default_value)
to_parse = self.ret(context.who, True, self.new(BuiltinConcepts.USER_INPUT, body=default_value))
res = self.execute(context, to_parse, steps)
concept.compiled[prop] = res
sub_context.add_values(return_values=res)
# Updates the cache of concepts when possible
if concept.key in self.concepts_cache:
@@ -613,7 +614,7 @@ class Sheerka(Concept):
# TODO : manage when there are multiple entries
pass
else:
self.concepts_cache[concept.key].cached_asts = concept.cached_asts
self.concepts_cache[concept.key].compiled = concept.compiled
def evaluate_concept(self, context, concept: Concept, logger=None):
"""
@@ -706,8 +707,8 @@ class Sheerka(Concept):
for metadata_to_eval in all_metadata_to_eval:
if metadata_to_eval == "props":
for prop_name in (p for p in concept.props if p in concept.cached_asts):
prop_ast = concept.cached_asts[prop_name]
for prop_name in (p for p in concept.props if p in concept.compiled):
prop_ast = concept.compiled[prop_name]
if isinstance(prop_ast, list):
resolved = _resolve_list(context.sheerka, prop_ast, prop_name, None)
@@ -719,13 +720,13 @@ class Sheerka(Concept):
concept.set_prop(prop_name, resolved)
else:
part_key = ConceptParts(metadata_to_eval)
if part_key in concept.cached_asts and concept.cached_asts[part_key] is not None:
metadata_ast = concept.cached_asts[part_key]
if part_key in concept.compiled and concept.compiled[part_key] is not None:
metadata_ast = concept.compiled[part_key]
resolved = _resolve(metadata_ast, part_key, concept)
if context.sheerka.isinstance(resolved, BuiltinConcepts.CONCEPT_EVAL_ERROR):
return resolved
else:
setattr(concept.metadata, metadata_to_eval, resolved)
concept.values[part_key] = resolved
#
# TODO : Validate the POST condition
@@ -789,7 +790,8 @@ class Sheerka(Concept):
unknown_concept = Concept()
template = self.concepts_cache[str(BuiltinConcepts.UNKNOWN_CONCEPT)]
unknown_concept.update_from(template)
unknown_concept.metadata.body = concept_key
unknown_concept.set_metadata_value(ConceptParts.BODY, concept_key)
unknown_concept.metadata.is_evaluated = True
return unknown_concept
def new(self, concept_key, **kwargs):
@@ -812,13 +814,13 @@ class Sheerka(Concept):
concept_key != BuiltinConcepts.UNKNOWN_CONCEPT:
return template
if not isinstance(template, list):
if isinstance(template, list):
# if template is a list, it means that there a multiple concepts under the same key
concepts = [self.new_from_template(t, concept_key, **kwargs) for t in template]
return concepts
else:
return self.new_from_template(template, concept_key, **kwargs)
# if template is a list, it means that there a multiple concepts under the same key
concepts = [self.new_from_template(t, concept_key, **kwargs) for t in template]
return concepts
def new_from_template(self, template, key, **kwargs):
# manage singleton
if template.metadata.is_unique:
@@ -828,18 +830,23 @@ class Sheerka(Concept):
concept = self.builtin_cache[key]() if key in self.builtin_cache else Concept()
concept.update_from(template)
# update the properties
if len(kwargs) == 0:
return concept
# update the properties, values, attributes
# Not quite sure that this is the correct process order
for k, v in kwargs.items():
if k in concept.props:
concept.set_prop(k, v)
elif k in PROPERTIES_FOR_NEW:
setattr(concept.metadata, k, v)
concept.values[ConceptParts(k)] = v
elif hasattr(concept, k):
setattr(concept, k, v)
else:
return self.new(BuiltinConcepts.UNKNOWN_PROPERTY, body=k, concept=concept)
# TODO : add the concept to the list of known concepts (self.instances)
concept.metadata.is_evaluated = True
return concept
def ret(self, who: str, status: bool, value, message=None, parents=None):
@@ -880,7 +887,7 @@ class Sheerka(Concept):
return self.value(body_to_use)
def values(self, objs):
def get_values(self, objs):
if not (isinstance(objs, list) or
self.isinstance(objs, BuiltinConcepts.LIST) or
self.isinstance(objs, BuiltinConcepts.ENUMERATION)):
core/sheerka_transform.py
+13 -4
View File
@@ -26,6 +26,9 @@ class SheerkaTransformType(Enum):
Node = 5
Exception = 6
def __repr__(self):
return self.__class__.__name__ + "." + self.name
class SheerkaTransform:
@@ -117,18 +120,24 @@ class SheerkaTransform:
# transform metadata
for prop in PROPERTIES_TO_SERIALIZE:
value = self.to_dict(getattr(obj.metadata, prop))
value = getattr(obj.metadata, prop)
ref_value = getattr(ref.metadata, prop)
if value != ref_value:
to_dict[prop] = value
to_dict["meta." + prop] = self.to_dict(value)
# transform value
for metadata, value in obj.values.items():
ref_value = ref.values[metadata] if metadata in ref.values else None
if value != ref_value:
to_dict[metadata.value] = self.to_dict(value)
# transform properties
for prop in obj.props:
value = self.to_dict(obj.props[prop].value)
value = obj.props[prop].value
if prop not in ref.props or value != ref.props[prop].value:
if "props" not in to_dict:
to_dict["props"] = []
to_dict["props"].append((prop, value))
to_dict["props"].append((prop, self.to_dict(value)))
return to_dict