kodjo/Sheerka-Old
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactored ExecutionContext serialization (added sheerkapickle) and added History management

Browse Source
This commit is contained in:
Kodjo Sossouvi
2020-01-31 18:58:03 +01:00
parent fed0735eb9
commit b9afcba61f
31 changed files with 1546 additions and 518 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/core/concept.py
+2
View File
@@ -343,6 +343,8 @@ class Concept:
:param metadata:
:return:
"""
if metadata not in self.values:
return None
return self.values[metadata]
def auto_init(self):
src/core/sheerka/ExecutionContext.py
+34 -5
View File
@@ -4,9 +4,22 @@ import time
from core.builtin_concepts import BuiltinConcepts
from core.concept import Concept
from sdp.sheerkaDataProvider import Event
from sheerkapickle.SheerkaPickler import SheerkaPickler
DEBUG_TAB_SIZE = 4
PROPERTIES_TO_SERIALIZE = ("_id",
"_bag",
"_start",
"_stop",
"who",
"desc",
"children",
"inputs",
"values",
"obj",
"concepts")
class ExecutionContext:
"""
@@ -31,7 +44,7 @@ class ExecutionContext:
**kwargs):
self._parent = None
self._id = ExecutionContext.get_id(event.get_digest())
self._id = ExecutionContext.get_id(event.get_digest()) if event else None
self._tab = ""
self._bag = {} # other variables
self._start = 0
@@ -90,6 +103,26 @@ class ExecutionContext:
msg += ")"
return msg
def __eq__(self, other):
if id(self) == id(other):
return True
if not isinstance(other, ExecutionContext):
return False
for prop in PROPERTIES_TO_SERIALIZE:
if prop == "who":
value = str(getattr(self, prop))
other_value = str(getattr(other, prop))
else:
value = getattr(self, prop)
other_value = getattr(other, prop)
if value != other_value:
return False
return True
def add_preprocess(self, name, **kwargs):
preprocess = self.sheerka.new(BuiltinConcepts.EVALUATOR_PRE_PROCESS)
preprocess.set_prop("name", name)
@@ -189,10 +222,6 @@ class ExecutionContext:
to_str = self.return_value_to_str(r)
logger.debug(f"[{self._id:2}]" + self._tab + "-> " + to_str)
def to_dict(self):
from core.sheerka_transform import SheerkaTransform
st = SheerkaTransform(self.sheerka)
return st.to_dict(self)
@staticmethod
def return_value_to_str(r):
src/core/sheerka/SheerkaCreateNewConcept.py → src/core/sheerka/Services/SheerkaCreateNewConcept.py
View File
src/core/sheerka/SheerkaDump.py → src/core/sheerka/Services/SheerkaDump.py
+3 -2
View File
@@ -39,7 +39,8 @@ class SheerkaDump:
self.sheerka.log.info(f"name : {c.name}")
self.sheerka.log.info(f"bnf : {c.metadata.definition}")
self.sheerka.log.info(f"key : {c.key}")
self.sheerka.log.info(f"body : {c.body}")
self.sheerka.log.info(f"body : {c.metadata.body}")
self.sheerka.log.info(f"value : {c.body}")
self.sheerka.log.info(f"digest : {c.get_digest()}")
first = False
@@ -57,7 +58,7 @@ class SheerkaDump:
while True:
try:
if h.user != self.sheerka.name:
if h.event.user != self.sheerka.name:
self.sheerka.log.info(h)
count += 1
h = next(history)
src/core/sheerka/SheerkaEvaluateConcept.py → src/core/sheerka/Services/SheerkaEvaluateConcept.py
View File
src/core/sheerka/SheerkaExecute.py → src/core/sheerka/Services/SheerkaExecute.py
+1 -1
View File
@@ -218,7 +218,7 @@ class SheerkaExecute:
for step in execution_steps:
copy = return_values[:] if hasattr(return_values, "__iter__") else [return_values]
with execution_context.push(step=step, iteration=0, desc=f"{step=}", return_values=copy) as sub_context:
with execution_context.push(step=step, iteration=0, desc=f"{step=}") as sub_context:
sub_context.log(logger or self.sheerka.log, f"{step=}, context='{sub_context}'")
if step == BuiltinConcepts.PARSING:
src/core/sheerka/Services/SheerkaHistoryManager.py
+74
View File
@@ -0,0 +1,74 @@
from collections import namedtuple
from sdp.sheerkaDataProvider import Event
hist = namedtuple("History", "text status") # tests purposes only
class History:
def __init__(self, event: Event, result):
self.event = event
self.result = result
self._status = None
def __str__(self):
msg = f"{self.event.get_digest()} {self.event.date.strftime('%d/%m/%Y %H:%M:%S')} : {self.event.message}"
status = self.status
if status is not None:
msg += f" => {status}"
return msg
def __repr__(self):
return f"event={self.event!r}, status={self.status}, result={self.result}"
def __eq__(self, other):
if id(self) == id(other):
return True
if isinstance(other, hist):
return self.event.message == other.text and self.status == other.status
if not isinstance(other, History):
return False
return self.event == other.event and self.result == other.result
@property
def status(self):
if self._status:
return self._status
if not self.result or "return_values" not in self.result.values:
return
if hasattr(self.result.values["return_values"], "__iter__"):
if len(self.result.values["return_values"]) != 1:
self._status = False
return self._status
else:
self._status = self.result.values["return_values"][0].status
return self._status
else:
self._status = self.result.values["return_values"].status
return self._status
class SheerkaHistoryManager:
def __init__(self, sheerka):
self.sheerka = sheerka
def history(self, depth_or_digest, start):
"""
Load history
:param depth_or_digest: number of items or digest
:param start:
:return:
"""
events = list(self.sheerka.sdp.load_events(depth_or_digest, start))
for event in events:
try:
result = self.sheerka.sdp.load_result(self.sheerka, event.get_digest())
except (IOError, KeyError):
result = None
yield History(event, result)
src/core/sheerka/SheerkaSetsManager.py → src/core/sheerka/Services/SheerkaSetsManager.py
View File
src/core/sheerka/Services/__init__.py
View File
src/core/sheerka/Sheerka.py
+36 -10
View File
@@ -2,11 +2,12 @@ from core.builtin_concepts import BuiltinConcepts, ErrorConcept, ReturnValueConc
UnknownConcept
from core.concept import Concept, ConceptParts, PROPERTIES_FOR_NEW
from core.sheerka.ExecutionContext import ExecutionContext
from core.sheerka.SheerkaCreateNewConcept import SheerkaCreateNewConcept
from core.sheerka.SheerkaDump import SheerkaDump
from core.sheerka.SheerkaEvaluateConcept import SheerkaEvaluateConcept
from core.sheerka.SheerkaExecute import SheerkaExecute
from core.sheerka.SheerkaSetsManager import SheerkaSetsManager
from core.sheerka.Services.SheerkaCreateNewConcept import SheerkaCreateNewConcept
from core.sheerka.Services.SheerkaDump import SheerkaDump
from core.sheerka.Services.SheerkaEvaluateConcept import SheerkaEvaluateConcept
from core.sheerka.Services.SheerkaExecute import SheerkaExecute
from core.sheerka.Services.SheerkaHistoryManager import SheerkaHistoryManager
from core.sheerka.Services.SheerkaSetsManager import SheerkaSetsManager
from sdp.sheerkaDataProvider import SheerkaDataProvider, Event
import core.utils
import core.builtin_helpers
@@ -21,7 +22,7 @@ import logging
# BuiltinConcepts.AFTER_EVALUATION]
CONCEPT_LEXER_PARSER_CLASS = "parsers.ConceptLexerParser.ConceptLexerParser"
CONCEPTS_FILE = "_concepts.txt"
class Sheerka(Concept):
"""
@@ -81,6 +82,7 @@ class Sheerka(Concept):
self.dump_handler = SheerkaDump(self)
self.sets_handler = SheerkaSetsManager(self)
self.evaluate_concept_handler = SheerkaEvaluateConcept(self)
self.history_handler = SheerkaHistoryManager(self)
def initialize(self, root_folder: str = None):
"""
@@ -92,6 +94,9 @@ class Sheerka(Concept):
"""
try:
from sheerkapickle.sheerka_handlers import initialize_pickle_handlers
initialize_pickle_handlers()
self.sdp = SheerkaDataProvider(root_folder)
if self.sdp.first_time:
self.sdp.set_key(self.USER_CONCEPTS_KEYS, 1000)
@@ -104,11 +109,16 @@ class Sheerka(Concept):
self.initialize_builtin_parsers()
self.initialize_builtin_evaluators()
self.initialize_concepts_definitions(exec_context)
res = ReturnValueConcept(self, True, self)
exec_context.add_values(return_values=res)
if not self.skip_builtins_in_db:
self.sdp.save_result(self, exec_context)
except IOError as e:
return ReturnValueConcept(self, False, self.get(BuiltinConcepts.ERROR), e)
res = ReturnValueConcept(self, False, self.get(BuiltinConcepts.ERROR), e)
return ReturnValueConcept(self, True, self)
return res
def initialize_builtin_concepts(self):
"""
@@ -232,7 +242,12 @@ class Sheerka(Concept):
execution_context.add_values(return_values=ret)
if not self.skip_builtins_in_db:
self.sdp.save_result(execution_context)
self.sdp.save_result(self, execution_context)
#hack to save valid concept definition
if len(ret) == 1 and ret[0].status and self.isinstance(ret[0].value, BuiltinConcepts.NEW_CONCEPT):
with open(CONCEPTS_FILE, "a") as f:
f.write(text + "\n")
return ret
def execute(self, execution_context, return_values, execution_steps, logger=None):
@@ -549,8 +564,19 @@ class Sheerka(Concept):
def history(self, page=10, start=0):
"""Gets the history of all commands"""
return self.sdp.load_events(page, start)
return self.history_handler.history(page, start)
def restore(self):
"""
Restore the state with all previous valid concept definitions
:return:
"""
try:
with open(CONCEPTS_FILE, "r") as f:
for line in f.readlines():
self.evaluate_user_input(line)
except IOError:
pass
def test(self):
return f"I have access to Sheerka !"
src/core/sheerka_transform.py
-161
View File
@@ -1,161 +0,0 @@
import dataclasses
from enum import Enum
from core.concept import Concept, PROPERTIES_TO_SERIALIZE
from core.sheerka.Sheerka import ExecutionContext
from core.tokenizer import Token
from evaluators.BaseEvaluator import BaseEvaluator
from parsers.BaseParser import BaseParser, Node
from parsers.BnfParser import BnfParser
from parsers.ConceptLexerParser import UnrecognizedTokensNode, ParsingExpression
from parsers.PythonParser import PythonNode
from sdp.sheerkaDataProvider import Event
OBJ_TYPE_KEY = "__type__"
OBJ_ID_KEY = "__id__"
OBJ_NAME_KEY = "__name__"
default_concept = Concept()
class SheerkaTransformType(Enum):
Concept = 1
Reference = 2
ExecutionContext = 3
Event = 4
Node = 5
Exception = 6
def __repr__(self):
return self.__class__.__name__ + "." + self.name
class SheerkaTransform:
def __init__(self, sheerka):
self.ids = {}
self.sheerka = sheerka
self.id_count = -1
def to_dict(self, obj):
if isinstance(obj, (Concept, ExecutionContext, Event)):
exists, _id = self.exist(obj)
if exists:
return {
OBJ_TYPE_KEY: SheerkaTransformType.Reference,
OBJ_ID_KEY: _id
}
else:
self.id_count = self.id_count + 1
self.ids[obj] = self.id_count
if isinstance(obj, Concept):
return self.concept_to_dict(obj)
elif isinstance(obj, ExecutionContext):
return self.execution_context_to_dict(obj)
elif isinstance(obj, Event):
return {
OBJ_TYPE_KEY: SheerkaTransformType.Event,
OBJ_ID_KEY: self.id_count,
'digest': obj.get_digest()}
elif isinstance(obj, (BaseParser, BaseEvaluator, BnfParser)):
return obj.name
elif isinstance(obj, Token):
return obj.__dict__
elif isinstance(obj, PythonNode):
return {
OBJ_TYPE_KEY: SheerkaTransformType.Node,
OBJ_NAME_KEY: "PythonNode",
'source': obj.source,
'ast_': obj.get_dump(obj.ast_)
}
elif isinstance(obj, Node):
to_dict = {
OBJ_TYPE_KEY: SheerkaTransformType.Node,
OBJ_NAME_KEY: obj.__class__.__name__,
}
for k, v in obj.__dict__.items():
to_dict[k] = self.to_dict(v)
return to_dict
elif isinstance(obj, Exception):
to_dict = {
OBJ_TYPE_KEY: SheerkaTransformType.Exception,
OBJ_NAME_KEY: obj.__class__.__name__,
}
for k, v in obj.__dict__.items():
to_dict[k] = self.to_dict(v)
return to_dict
elif isinstance(obj, ParsingExpression):
return obj.__repr__()
elif isinstance(obj, dict):
return dict((str(k) if isinstance(k, Concept) else k, self.to_dict(v)) for k, v in obj.items())
elif hasattr(obj, "__iter__") and not isinstance(obj, str):
return list(self.to_dict(o) for o in obj)
else:
return obj
def concept_to_dict(self, obj: Concept):
to_dict = {
OBJ_TYPE_KEY: SheerkaTransformType.Concept,
OBJ_ID_KEY: self.id_count,
}
if obj.id:
ref = self.sheerka.get(obj.key, obj.id)
to_dict["id"] = obj.id
else:
ref = default_concept
# transform metadata
for prop in PROPERTIES_TO_SERIALIZE:
value = getattr(obj.metadata, prop)
ref_value = getattr(ref.metadata, prop)
if value != ref_value:
to_dict["meta." + prop] = self.to_dict(value)
# transform value
for metadata, value in obj.values.items():
ref_value = ref.values[metadata] if metadata in ref.values else None
if value != ref_value:
to_dict[metadata.value] = self.to_dict(value)
# transform properties
for prop in obj.props:
value = obj.props[prop].value
if prop not in ref.props or value != ref.props[prop].value:
if "props" not in to_dict:
to_dict["props"] = []
to_dict["props"].append((prop, self.to_dict(value)))
return to_dict
def execution_context_to_dict(self, obj: ExecutionContext):
to_dict = {
OBJ_TYPE_KEY: SheerkaTransformType.ExecutionContext,
OBJ_ID_KEY: self.id_count
}
for property_name in obj.__dict__:
if property_name == "sheerka":
continue
to_dict[property_name] = self.to_dict(getattr(obj, property_name))
return to_dict
def exist(self, obj):
for k, v in self.ids.items():
if id(k) == id(obj) or k == obj:
return True, v
return False, None
src/core/utils.py
+64
View File
@@ -228,6 +228,7 @@ def escape_char(text, to_escape):
return res
def pp(items):
if not hasattr(items, "__iter__"):
return str(items)
@@ -236,3 +237,66 @@ def pp(items):
return str(items)
return " \n" + " \n".join(str(item) for item in items)
def decode_concept(concept_repr):
"""
if concept_repr is like :c:key:id:
return the key and the id
:param concept_repr:
:return:
"""
if not (concept_repr and isinstance(concept_repr, str) and concept_repr.startswith(":c:")):
return None, None
i = 3
length = len(concept_repr)
key = ""
while i < length:
if concept_repr[i] == ":":
break
key += concept_repr[i]
i += 1
else:
return None, None
i += 1
if i >= length:
return key, None
id = ""
while i < length:
if concept_repr[i] == ":":
break
id += concept_repr[i]
i += 1
else:
return None, None
return key, id
def decode_enum(enum_repr: str):
"""
Tries to transform ClassName.Name into an enum
:param enum_repr:
:return:
"""
if not (enum_repr and isinstance(enum_repr, str)):
return None
try:
idx = enum_repr.rindex(".")
if idx == len(enum_repr):
return None
cls_name = enum_repr[:idx]
cls = get_class(cls_name)
name = enum_repr[idx + 1:]
return cls[name]
except ValueError:
return None
except TypeError:
return None