kodjo/Sheerka-Old
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactored Caching, Refactored BnfNodeParser, Introduced Sphinx

Browse Source
This commit is contained in:
Kodjo Sossouvi
2020-05-12 17:21:10 +02:00
parent 7d3a490bc5
commit 6e343ba996
110 changed files with 13865 additions and 7540 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/core/concept.py
+136 -89
View File
@@ -1,17 +1,19 @@
import hashlib
from collections import namedtuple
from copy import deepcopy
from dataclasses import dataclass
from enum import Enum
from core.sheerka_logger import get_logger
from typing import Union
import core.utils
from core.sheerka_logger import get_logger
from core.tokenizer import Tokenizer, TokenKind
PROPERTIES_FOR_DIGEST = ("name", "key",
"definition", "definition_type",
"is_builtin", "is_unique",
"where", "pre", "post", "body",
"desc", "props")
"desc", "props", "variables")
PROPERTIES_TO_SERIALIZE = PROPERTIES_FOR_DIGEST + tuple(["id"])
PROPERTIES_FOR_NEW = ("where", "pre", "post", "body", "desc")
VARIABLE_PREFIX = "__var__"
@@ -48,15 +50,13 @@ class ConceptMetadata:
definition_type: str # definition can be done with something else than regex
desc: str # possible description for the concept
id: str # unique identifier for a concept. The id will never be modified (but the key can)
props: list # list properties, with their default values
props: dict # hashmap of properties, values
variables: list # list of concept variables, with their default values
is_evaluated: bool = False # True is the concept is evaluated by sheerka.eval_concept()
need_validation = False # True if the properties of the concept need to be validated
full_serialization: bool = False # If True, the full object will be serialized, rather than just the diff
simplec = namedtuple("concept", "name body") # for simple concept (tests purposes only)
class Concept:
"""
Default concept object
@@ -76,7 +76,8 @@ class Concept:
definition_type=None,
desc=None,
id=None,
props=None):
props=None,
variables=None):
metadata = ConceptMetadata(
str(name) if name else None,
@@ -91,17 +92,17 @@ class Concept:
definition_type,
desc,
id,
props or []
props or {},
variables or []
)
self.metadata = metadata
self.compiled = {} # cached ast for the where, pre, post and body parts
self.values = {} # values of metadata once resolved
self.props = {} # resolved properties of this concept
self.bnf = None
self.compiled = {} # cached ast for the where, pre, post and body parts and variables
self.values = {} # resolved values. As compiled, it's used both for metadata and variables
self.bnf = None # parsing expression
self.log = get_logger("core." + self.__class__.__name__)
self.init_log = get_logger("init.core." + self.__class__.__name__)
self.original_definition_hash = None
self.original_definition_hash = None # concept hash before any alteration of the metadata
def __repr__(self):
return f"({self.metadata.id}){self.metadata.name}"
@@ -117,6 +118,9 @@ class Concept:
if isinstance(other, CC):
return other == self
if isinstance(other, CB):
return other == self
if not isinstance(other, Concept):
return False
@@ -147,15 +151,8 @@ class Concept:
if len(self.values) != len(other.values):
return False
for metadata in self.values:
if self.get_metadata_value(metadata) != other.get_metadata_value(metadata):
return False
if len(self.props) != len(other.props):
return False
for prop in self.props:
if self.get_prop(prop) != other.get_prop(prop):
for name in self.values:
if self.get_value(name) != other.get_value(name):
return False
return True
@@ -166,28 +163,36 @@ class Concept:
def __getattr__(self, item):
# I have this complicated implementation because of the usage of Pickle
if 'props' in vars(self) and item in self.props:
return self.props[item].value
if 'values' in vars(self) and item in self.values:
return self.get_value(item)
name = self.name if 'metadata' in vars(self) else 'Concept'
raise AttributeError(f"'{name}' concept has no attribute '{item}'")
def def_prop(self, prop_name, default_value=None):
def def_var(self, var_name, default_value=None):
"""
Adds a property to the metadata
:param prop_name: name or concept
:param var_name: name or concept
:param default_value:
:return:
"""
assert default_value is None or isinstance(default_value, str) # default properties will have to be evaluated
self.metadata.props.append((prop_name, default_value))
self.props[prop_name] = Property(prop_name, None) # do not set the default value
# why not setting props to the default values ?
# Because it may not be the real values, as metadata.props need to be evaluated
# this assert in not a functional requirement
# It's just to control what I put in the default value of properties
# You can allow more type if it's REALLY needed.
# - str are for standard definition
# - list of concepts is used by ISA
assert default_value is None or isinstance(default_value, str)
self.metadata.variables.append((var_name, default_value))
self.set_value(var_name, None) # do not set the default value
# why not setting variables to the default values ?
# Because it may not be the real values, as metadata.variables need to be evaluated
return self
def def_prop_by_index(self, index: int, value):
def def_var_by_index(self, index: int, value):
"""
Re-assign a value to a property (mainly used by ExactConceptParser)
:param index:
@@ -195,8 +200,8 @@ class Concept:
:return:
"""
assert value is None or isinstance(value, str) # default properties will have to be evaluated
prop = self.metadata.props[index]
self.metadata.props[index] = (prop[0], value)
var_name = self.metadata.variables[index]
self.metadata.variables[index] = (var_name[0], value) # change the default value
return self
@property
@@ -229,7 +234,7 @@ class Concept:
else:
tokens = list(Tokenizer(self.metadata.name))
variables = [p[0] for p in self.metadata.props] if len(core.utils.strip_tokens(tokens, True)) > 1 else []
variables = [p[0] for p in self.metadata.variables] if len(core.utils.strip_tokens(tokens, True)) > 1 else []
key = ""
first = True
@@ -252,7 +257,7 @@ class Concept:
@property
def body(self):
return self.values[ConceptParts.BODY] if ConceptParts.BODY in self.values else None
return self.get_value(ConceptParts.BODY)
def get_origin(self):
"""
@@ -284,8 +289,12 @@ class Concept:
"""
props_to_use = props_to_use or PROPERTIES_TO_SERIALIZE
props_as_dict = dict((prop, getattr(self.metadata, prop)) for prop in props_to_use)
props_as_dict = {}
for prop in props_to_use:
if prop == "props": # no need to copy variables as the ref won't be used in from_dict
props_as_dict[prop] = deepcopy(getattr(self.metadata, prop))
else:
props_as_dict[prop] = getattr(self.metadata, prop)
return props_as_dict
def from_dict(self, as_dict):
@@ -296,19 +305,20 @@ class Concept:
"""
for prop in PROPERTIES_TO_SERIALIZE:
if prop in as_dict:
if prop == "props":
if prop == "variables":
for name, value in as_dict[prop]:
self.def_prop(name, value)
self.def_var(name, value)
else:
setattr(self.metadata, prop, as_dict[prop])
return self
def update_from(self, other):
def update_from(self, other, update_value=True):
"""
Update self using the properties of another concept
This method is to mimic the class to instance pattern
'other' is the class, the template, and 'self' is a new instance
:param other:
:param update_value:
:return:
"""
if other is None:
@@ -321,12 +331,9 @@ class Concept:
self.from_dict(other.to_dict())
# update values
for k, v in other.values.items():
self.values[k] = v
# update properties
for k, v in other.props.items():
self.set_prop(k, v.value)
if update_value:
for k in other.values:
self.set_value(k, other.get_value(k))
# origin
from sdp.sheerkaSerializer import Serializer
@@ -335,54 +342,53 @@ class Concept:
return self
def set_prop(self, prop_name, prop_value):
def add_prop(self, concept_key, value):
"""
Set the value of a property (not the metadata)
:param prop_name: Name the property or another concept
:param prop_value:
:return:
"""
self.props[prop_name] = Property(prop_name, prop_value)
return self
def get_prop(self, prop_name: str):
"""
Gets the value of a property
:param prop_name: name or concept
:return:
"""
return self.props[prop_name].value
def set_prop_by_index(self, index: int, value):
"""
Set the value of a property (not the metadata) using the index
:param index: Name the property or another concept
Set or add a behaviour to a concept
A behaviour is a value from another concept (ex BuiltinConcepts.ISA
:param concept_key: Concept key
:param value:
:return:
"""
prop_name = list(self.props.keys())[index]
self.props[prop_name].value = value
if concept_key in self.metadata.props:
self.metadata.props[concept_key].add(value)
else:
self.metadata.props[concept_key] = {value} # a set
return self
def set_metadata_value(self, metadata: ConceptParts, value):
def get_prop(self, concept_key):
"""
Set the resolved value of a metadata (not the metadata itself)
:param metadata:
Gets a behaviour of a concept
:param concept_key: name of the behaviour
:return:
"""
return self.metadata.props[concept_key] if concept_key in self.metadata.props else None
def set_value(self, name, value):
"""
Set the resolved value of a metadata or a variable (not the metadata itself)
:param name:
:param value:
:return:
"""
self.values[metadata] = value
if name in self.values:
self.values[name].value = value
else:
self.values[name] = Property(name, value)
return self
def get_metadata_value(self, metadata: ConceptParts):
def get_value(self, prop_name):
"""
Gets the resolved value of a metadata
:param metadata:
:param prop_name:
:return:
"""
if metadata not in self.values:
if prop_name not in self.values:
return None
return self.values[metadata]
return self.values[prop_name].value
def variables(self):
return dict([(k, v) for k, v in self.values.items() if isinstance(k, str)])
def auto_init(self):
"""
@@ -398,10 +404,10 @@ class Concept:
for metadata in ConceptParts:
value = getattr(self.metadata, metadata.value)
if value is not None:
self.values[metadata] = value
self.set_value(metadata, value)
for prop, value in self.metadata.props:
self.set_prop(prop, value)
for var, value in self.metadata.variables:
self.set_value(var, value)
self.metadata.is_evaluated = True
return self
@@ -419,9 +425,10 @@ class Concept:
And it removes the visibility from the other attributes/methods
"""
bag = {}
for prop in self.props:
bag[prop] = self.get_prop(prop)
bag["prop." + prop] = self.get_prop(prop)
for var in self.values:
if isinstance(var, str):
bag[var] = self.get_value(var)
bag["var." + var] = self.get_value(var)
for prop in ("id", "name", "key", "body"):
bag[prop] = getattr(self, prop)
return bag
@@ -469,10 +476,17 @@ class InfiniteRecursionResolved:
"""This class is used to when we managed to break an infinite recursion concept definition"""
value: object
def get_value(self):
def get_obj_value(self):
return self.value
# ################################
#
# Class created for tests purpose
#
# ################################
class CC:
"""
Concept class for test purpose
@@ -484,13 +498,14 @@ class CC:
# The other properties (concept, source, start and end)
# are used in tests/parsers/parsers_utils.py to help creating helper objects
def __init__(self, concept, source=None, **kwargs):
def __init__(self, concept, source=None, exclude_body=False, **kwargs):
self.concept_key = concept.key if isinstance(concept, Concept) else concept
self.compiled = kwargs
self.concept = concept if isinstance(concept, Concept) else None
self.source = source # to use when the key is different from the sub str to search when filling start and stop
self.start = None # for debug purpose, indicate where the concept starts
self.end = None # for debug purpose, indicate where the concept ends
self.exclude_body = exclude_body
def __eq__(self, other):
if id(self) == id(other):
@@ -499,13 +514,19 @@ class CC:
if isinstance(other, Concept):
if other.key != self.concept_key:
return False
return self.compiled == other.compiled
if self.exclude_body:
to_compare = {k: v for k, v in other.compiled.items() if k != ConceptParts.BODY}
else:
to_compare = other.compiled
return self.compiled == to_compare
if not isinstance(other, CC):
return False
return self.concept_key == other.concept_key and \
self.compiled == other.compiled
if self.concept_key != other.concept_key:
return False
return self.compiled == other.compiled
def __hash__(self):
if self.concept:
@@ -536,3 +557,29 @@ class CC:
if self.end is None or end > self.end:
self.end = end
return self
@dataclass()
class CB:
"""
Concept with body only
Test class that test only the body of the concept
"""
concept: Union[str, Concept]
body: object
def __eq__(self, other):
if isinstance(other, Concept):
key = self.concept if isinstance(self.concept, str) else self.concept.key
return key == other.key and self.body == other.body
if not isinstance(other, CB):
return False
return self.concept == other.concept and self.body == other.body
def __hash__(self):
return hash((self.concept, self.body))
simplec = namedtuple("concept", "name body") # for simple concept (tests purposes only)