kodjo/Sheerka-Old
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactored Caching, Refactored BnfNodeParser, Introduced Sphinx

Browse Source
This commit is contained in:
Kodjo Sossouvi
2020-05-12 17:21:10 +02:00
parent 7d3a490bc5
commit 6e343ba996
110 changed files with 13865 additions and 7540 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/cache/BaseCache.py
Vendored
+241
View File
@@ -0,0 +1,241 @@
from threading import RLock
class BaseCache:
"""
An in memory FIFO cache object
When the max_size is reach the first element that was put is removed
When you put the same key twice, the previous element is overridden
"""
def __init__(self, max_size=None, default=None, extend_exists=None):
self._cache = {}
self._max_size = max_size
self._default = default # default value to return when key is not found. It can be a callable of key
self._extend_exists = extend_exists # search in remote
self._lock = RLock()
self._current_size = 0
self._initialized_keys = set()
self.to_add = set()
self.to_remove = set()
def __len__(self):
"""
Return the number of items in the cache
:return:
"""
with self._lock:
return self._current_size
def __contains__(self, key):
with self._lock:
return key in self._cache
def __iter__(self):
with self._lock:
keys = self._cache.copy()
yield from keys
def __next__(self):
return next(iter(self._cache))
def __repr__(self):
return f"{self.__class__.__name__}(size={self._current_size}, #keys={len(self._cache)})"
def configure(self, max_size=None, default=None, extend_exists=None):
if max_size is not None:
self._max_size = max_size
if default is not None:
self._default = default
if extend_exists is not None:
self._extend_exists = extend_exists
def disable_default(self):
self._default = None
def put(self, key, value):
"""
Add a new entry in cache
:param key:
:param value:
:return:
"""
with self._lock:
if self._max_size and self._current_size >= self._max_size:
self.evict(self._max_size - self._current_size + 1)
if self._put(key, value):
self._current_size += 1
def get(self, key):
"""
Retrieve an entry from the cache
If the entry does not exist, will use the 'default' value or delegate
:param key:
:return:
"""
with self._lock:
self._initialized_keys.add(key)
return self._get(key)
def inner_get(self, key):
return self._cache[key]
def update(self, old_key, old_value, new_key, new_value):
"""
Update an entry in the cache
:param old_key: key of the previous version of the entry
:param old_value: previous version of the entry
:param new_key: key of the entry
:param new_value: new value
:return:
"""
with self._lock:
self._update(old_key, old_value, new_key, new_value)
def delete(self, key, value=None):
with self._lock:
try:
self._delete(key, value)
except KeyError:
pass
def has(self, key):
"""
Return True if the key is in the cache
Never use extend_exist
:param key:
:return:
"""
with self._lock:
return key in self._cache
def exists(self, key):
"""
Return True if the key is in the cache
Can use extend_exist
:param key:
:return:
"""
with self._lock:
if key in self._cache:
return True
return self._extend_exists(key) if self._extend_exists else False
def evict(self, nb_items):
"""
Remove nb_items from the cache, using the replacement policy
:return:
"""
with self._lock:
nb_items = self._current_size if self._current_size < nb_items else nb_items
nb_to_delete = nb_items
while nb_items > 0:
key = next(iter(self._cache))
del (self._cache[key])
try:
self._initialized_keys.remove(key)
except KeyError:
pass
nb_items -= 1
self._current_size -= nb_to_delete
return nb_to_delete
def clear(self):
with self._lock:
self._cache.clear()
self._current_size = 0
self._initialized_keys.clear()
self.to_add.clear()
self.to_remove.clear()
def dump(self):
with self._lock:
return {
"current_size": self._current_size,
"cache": self._cache.copy()
}
def copy(self):
with self._lock:
return self._cache.copy()
def init_from(self, dump):
with self._lock:
self._current_size = dump["current_size"]
self._cache = dump["cache"].copy()
return self
def reset_events(self):
with self._lock:
self.to_add.clear()
self.to_remove.clear()
def _sync(self, *keys):
for key in keys:
if key not in self._initialized_keys and self._default:
# to keep sync with the remote repo is needed
self.get(key)
def _add_to_add(self, key):
self.to_add.add(key)
try:
self.to_remove.remove(key)
except KeyError:
pass
def _add_to_remove(self, key):
self.to_remove.add(key)
try:
self.to_add.remove(key)
except KeyError:
pass
def _get(self, key):
try:
value = self._cache[key]
except KeyError:
if callable(self._default):
value = self._default(key)
if value is not None:
self._cache[key] = value
# update _current_size
if isinstance(value, (list, set)):
self._current_size += len(value)
else:
self._current_size += 1
else:
value = self._default
return value
def _put(self, key, value):
pass
def _update(self, old_key, old_value, new_key, new_value):
pass
def _delete(self, key, value):
raise NotImplementedError()
# def _put(self, key, value):
# self._cache[key] = value
# self._add_to_add(key)
# return True
#
#
# def _update(self, old_key, old_value, new_key, new_value):
# self._cache[new_key] = new_value
# self._add_to_add(new_key)
#
# if new_key != old_key:
# del (self._cache[old_key])
# self._add_to_remove(old_key)
src/cache/Cache.py
Vendored
+31
View File
@@ -0,0 +1,31 @@
from threading import RLock
from cache.BaseCache import BaseCache
class Cache(BaseCache):
"""
An in memory FIFO cache object
When the max_size is reach the first element that was put is removed
When you put the same key twice, the previous element is overridden
"""
def _put(self, key, value):
res = key not in self._cache
self._cache[key] = value
self._add_to_add(key)
return res
def _update(self, old_key, old_value, new_key, new_value):
self._cache[new_key] = new_value
self._add_to_add(new_key)
if new_key != old_key:
self._sync(old_key)
del (self._cache[old_key])
self._add_to_remove(old_key)
def _delete(self, key, value):
del(self._cache[key])
self._add_to_remove(key)
src/cache/CacheManager.py
Vendored
+261
View File
@@ -0,0 +1,261 @@
from dataclasses import dataclass, field
from threading import RLock
from typing import Callable
from cache.Cache import Cache
from core.concept import Concept
class MultipleEntryError(Exception):
"""
Exception raised when trying to alter an entry with multiple element
without giving the origin of the element
"""
def __init__(self, key):
self.key = key
@dataclass
class CacheDefinition:
cache: Cache
use_ref: bool
get_key: Callable[[Concept], str] = field(repr=False)
persist: bool = True
class CacheManager:
"""
Single class to manage all the caches
"""
def __init__(self, cache_only):
self.cache_only = cache_only # if true disable all remote access when key not found
self.caches = {}
self.concept_caches = []
self.is_dirty = False # to indicate that the value of a cache has changed
self._lock = RLock()
def register_concept_cache(self, name, cache, get_key, use_ref):
"""
Define which type of cache along with how to compute the key
:param name:
:param cache:
:param get_key:
:param use_ref:
:return:
"""
with self._lock:
if self.cache_only:
cache.disable_default()
self.caches[name] = CacheDefinition(cache, use_ref, get_key)
self.concept_caches.append(name)
def register_cache(self, name, cache, persist=True, use_ref=False):
"""
Define which type of cache along with how to compute the key
:param name:
:param cache:
:param persist:
:param use_ref:
:return:
"""
with self._lock:
if self.cache_only:
cache.disable_default()
self.caches[name] = CacheDefinition(cache, use_ref, None, persist)
def add_concept(self, concept):
"""
We need multiple indexes to retrieve a concept
So the new concept is dispatched into multiple caches
:param concept:
:return:
"""
with self._lock:
for name in self.concept_caches:
cache_def = self.caches[name]
key = cache_def.get_key(concept)
cache_def.cache.put(key, concept)
self.is_dirty = True
def update_concept(self, old, new):
"""
Update a concept.
:param old: old version of the concept
:param new: new version of the concept
:return:
"""
with self._lock:
for cache_name in self.concept_caches:
cache_def = self.caches[cache_name]
old_key = cache_def.get_key(old)
new_key = cache_def.get_key(new)
cache_def.cache.update(old_key, old, new_key, new)
self.is_dirty = True
# how can you update an entry it the key may have changed ?
# You need to have an invariant. By convention the keys in the first cache cannot change
# with self._lock:
# iter_cache_def = iter(self.caches)
#
# cache_def = next(iter_cache_def)
# old_key = cache_def.get_key(concept)
#
# try:
# while True:
# items = cache_def.cache[old_key]
# if isinstance(items, (list, set)):
# for item in items:
# if item.id == concept.id:
# break
# else:
# raise IndexError(f"{old_key=}, id={concept.id}")
#
# cache_def.cache.update(old_key, item, cache_def.get_key(concept), concept)
#
# else:
# cache_def.cache.update(old_key, items, cache_def.get_key(concept), concept)
#
# cache_def = next(iter_cache_def)
# except StopIteration:
# pass
# self.is_dirty = True
def get(self, cache_name, key):
"""
From concept cache, get an entry
:param cache_name:
:param key:
:return:
"""
with self._lock:
return self.caches[cache_name].cache.get(key)
def copy(self, cache_name):
"""
get a copy the content of the whole cache as a dictionary
:param self:
:param cache_name:
:return:
"""
return self.caches[cache_name].cache.copy()
def put(self, cache_name, key, value):
"""
Add to a cache
:param cache_name:
:param key:
:param value:
:return:
"""
with self._lock:
self.caches[cache_name].cache.put(key, value)
self.is_dirty = True
def delete(self, cache_name, key, value=None):
"""
Delete an entry from the cache
:param cache_name:
:param key:
:param value:
:return:
"""
with self._lock:
self.caches[cache_name].cache.delete(key, value)
self.is_dirty = True
def has(self, cache_name, key):
"""
True if the value is in cache only. Never try to look in a remote repository
:param cache_name:
:param key:
:return:
"""
with self._lock:
return self.caches[cache_name].cache.has(key)
def exists(self, cache_name, key):
"""
True if the value is in cache.
If not found, may search in a remote repository
:param cache_name:
:param key:
:return:
"""
if self.cache_only:
return self.has(cache_name, key)
with self._lock:
return self.caches[cache_name].cache.exists(key)
def commit(self, context):
"""
Persist all the caches into a physical persistence storage
:param context:
:return:
"""
def update_full_serialisation(items, value):
# Take care, infinite recursion is not handled !!
if isinstance(items, (list, set, tuple)):
for item in items:
update_full_serialisation(item, value)
elif isinstance(items, dict):
for values in items.values():
update_full_serialisation(values, value)
elif isinstance(items, Concept):
items.metadata.full_serialization = value
if self.cache_only:
return
with self._lock:
with context.sheerka.sdp.get_transaction(context.event.get_digest()) as transaction:
for cache_name, cache_def in self.caches.items():
if not cache_def.persist:
continue
for key in cache_def.cache.to_remove:
transaction.remove(cache_name, key)
for key in cache_def.cache.to_add:
if key == "*self*":
transaction.add(cache_name, None, cache_def.cache.dump()["cache"])
else:
to_save = cache_def.cache.inner_get(key)
update_full_serialisation(to_save, True)
transaction.add(cache_name, key, to_save, cache_def.use_ref)
update_full_serialisation(to_save, False)
cache_def.cache.reset_events()
self.is_dirty = False
def clear(self, cache_name=None):
with self._lock:
if cache_name:
self.caches[cache_name].cache.clear()
else:
for cache_def in self.caches.values():
cache_def.cache.clear()
def dump(self):
with self._lock:
res = {}
for cache_name, cache_def in self.caches.items():
res[cache_name] = cache_def.cache.dump()
return res
def init_from(self, dump):
with self._lock:
for cache_name, content in dump.items():
if cache_name in self.caches:
self.caches[cache_name].cache.init_from(content)
return self
src/cache/DictionaryCache.py
Vendored
+53
View File
@@ -0,0 +1,53 @@
from cache.BaseCache import BaseCache
class DictionaryCache(BaseCache):
def _get(self, key):
"""
Management of the default is different
:param key:
:return:
"""
try:
value = self._cache[key]
return value
except KeyError:
if callable(self._default):
self._cache = self._default(key) or {}
else:
self._cache = self._default.copy() if self._default else {}
self._count_items()
return self._cache[key] if key in self._cache else None
def _put(self, key, value):
"""
Adds a whole dictionary
:param key: True to append, false to reset
:param value: dictionary
:return:
"""
if not isinstance(key, bool):
raise KeyError
if not isinstance(value, dict):
raise ValueError
if key:
if self._cache is None:
self._cache = value.copy()
else:
self._cache.update(value)
else:
self._cache = value
self._count_items()
# special meaning for to_add
self._add_to_add("*self*")
return False
def _count_items(self):
self._current_size = 0
for v in self._cache.values():
self._current_size += len(v) if hasattr(v, "__len__") and not isinstance(v, str) else 1
src/cache/IncCache.py
Vendored
+18
View File
@@ -0,0 +1,18 @@
from cache.Cache import Cache
class IncCache(Cache):
"""
Increment the value of the key every time it's accessed
"""
def _get(self, key):
value = super()._get(key) or 0
value += 1
self._put(key, value)
return value
def _put(self, key, value):
self._cache[key] = value
self._add_to_add(key)
return True
src/cache/ListCache.py
Vendored
+43
View File
@@ -0,0 +1,43 @@
from cache.Cache import BaseCache
class ListCache(BaseCache):
"""
An in memory FIFO cache object
When the max_size is reach the first element that was put is removed
Items of this cache are list
"""
def _put(self, key, value):
if key in self._cache:
self._cache[key].append(value)
else:
self._sync(key)
if key in self._cache:
self._cache[key].append(value)
else:
self._cache[key] = [value]
self._add_to_add(key)
return True
def _update(self, old_key, old_value, new_key, new_value):
self._sync(old_key, new_key)
if old_key != new_key:
self._cache[old_key].remove(old_value)
if len(self._cache[old_key]) == 0:
del (self._cache[old_key])
self._add_to_remove(old_key)
else:
self._add_to_add(old_key)
self._put(new_key, new_value)
self._add_to_add(new_key)
else:
for i in range(len(self._cache[new_key])):
if self._cache[new_key][i] == old_value:
self._cache[new_key][i] = new_value # avoid add and remove in dict
break # only the first one is affected
self._add_to_add(new_key)
src/cache/ListIfNeededCache.py
Vendored
+56
View File
@@ -0,0 +1,56 @@
from cache.Cache import BaseCache
class ListIfNeededCache(BaseCache):
"""
An in memory FIFO cache object
When the max_size is reach the first element that was put is removed
When you put the same key twice, you now have a list of two elements
"""
def _put(self, key, value):
if key in self._cache:
if isinstance(self._cache[key], list):
self._cache[key].append(value)
else:
self._cache[key] = [self._cache[key], value]
else:
self._sync(key)
if key in self._cache:
if isinstance(self._cache[key], list):
self._cache[key].append(value)
else:
self._cache[key] = [self._cache[key], value]
else:
self._cache[key] = value
self._add_to_add(key)
return True
def _update(self, old_key, old_value, new_key, new_value):
self._sync(old_key, new_key)
if old_key != new_key:
if isinstance(self._cache[old_key], list):
self._cache[old_key].remove(old_value)
if len(self._cache[old_key]) == 0:
del (self._cache[old_key])
self._add_to_remove(old_key)
else:
self._add_to_add(old_key)
else:
del (self._cache[old_key])
self._add_to_remove(old_key)
self._put(new_key, new_value)
self._add_to_add(new_key)
else:
if isinstance(self._cache[new_key], list):
for i in range(len(self._cache[new_key])):
if self._cache[new_key][i] == old_value:
self._cache[new_key][i] = new_value # avoid add and remove in dict
break
else:
self._cache[new_key] = new_value
self._add_to_add(new_key)
src/cache/SetCache.py
Vendored
+45
View File
@@ -0,0 +1,45 @@
from cache.Cache import BaseCache
class SetCache(BaseCache):
"""
An in memory FIFO cache object
When the max_size is reach the first element that was put is removed
You can use the same key multiple times, but the elements under this key will be unique
When there are multiple elements, a python set is used
"""
def _put(self, key, value):
if key in self._cache:
if value in self._cache[key]:
return False
self._cache[key].add(value)
else:
self._sync(key)
if key in self._cache:
self._cache[key].add(value)
else:
self._cache[key] = {value}
self._add_to_add(key)
return True
def _update(self, old_key, old_value, new_key, new_value):
self._sync(old_key, new_key)
if old_key != new_key:
if isinstance(self._cache[old_key], set):
self._cache[old_key].remove(old_value)
if len(self._cache[old_key]) == 0:
del (self._cache[old_key])
self._add_to_remove(old_key)
else:
self._add_to_add(old_key)
self._put(new_key, new_value)
self._add_to_add(new_key)
else:
self._cache[new_key].remove(old_value)
self._put(new_key, new_value)
self._add_to_add(new_key)
src/cache/__init__.py
Vendored
View File
src/core/ast/nodes.py
+12 -12
View File
@@ -1,5 +1,5 @@
from core.builtin_concepts import BuiltinConcepts, ListConcept
from core.concept import Concept
from core.concept import Concept, ConceptParts
import ast
import core.utils
@@ -65,12 +65,12 @@ class GenericNodeConcept(NodeConcept):
def get_node_type(self):
return self.node_type
def get_value(self):
def get_obj_value(self):
if self.node_type == "Name":
return self.get_prop("id")
return self.get_value("id")
if self.node_type == "arg":
return self.get_prop("arg")
return self.get_value("arg")
return self.body
@@ -78,7 +78,7 @@ class GenericNodeConcept(NodeConcept):
class IdentifierNodeConcept(NodeConcept):
def __init__(self, parent, name):
super().__init__(BuiltinConcepts.IDENTIFIER_NODE, "Name", parent)
self.body = name
self.set_value(ConceptParts.BODY, name)
class CallNodeConcept(NodeConcept):
@@ -86,7 +86,7 @@ class CallNodeConcept(NodeConcept):
super().__init__(BuiltinConcepts.IDENTIFIER_NODE, "Call", parent)
def get_args_names(self, sheerka):
return sheerka.get_values(self.get_prop("args"))
return sheerka.objvalues(self.get_value("args"))
def python_to_concept(python_node):
@@ -105,16 +105,16 @@ def python_to_concept(python_node):
continue
value = getattr(node, field)
concept.def_prop(field)
concept.def_var(field)
if isinstance(value, list):
lst = ListConcept().init_key()
for i in value:
lst.append(_transform(i, NodeParent(concept, field)))
concept.set_prop(field, lst)
concept.set_value(field, lst)
elif isinstance(value, ast.AST):
concept.set_prop(field, _transform(value, NodeParent(concept, field)))
concept.set_value(field, _transform(value, NodeParent(concept, field)))
else:
concept.set_prop(field, value)
concept.set_value(field, value)
concept.metadata.is_evaluated = True
return concept
@@ -132,11 +132,11 @@ def concept_to_python(concept_node):
def _transform(node):
node_type = node.get_node_type()
ast_object = core.utils.new_object("_ast." + node_type)
for field in node.props:
for field in node.values:
if field not in ast_object._fields:
continue
value = node.get_prop(field)
value = node.get_value(field)
if isinstance(value, list) or isinstance(value, Concept) and value.key == str(BuiltinConcepts.LIST):
lst = []
for i in value.body:
src/core/ast/visitors.py
+8 -8
View File
@@ -29,7 +29,7 @@ class ConceptNodeVisitor:
self.visit(value)
def visit_Constant(self, node):
value = node.get_prop("value")
value = node.get_value("value")
type_name = _const_node_type_names.get(type(value))
if type_name is None:
for cls, name in _const_node_type_names.items():
@@ -66,10 +66,10 @@ class UnreferencedNamesVisitor(ConceptNodeVisitor):
if ("Assign", "targets") in parents: # variable which is assigned
return
if self.can_be_discarded(self.sheerka.value(node), parents):
if self.can_be_discarded(self.sheerka.objvalue(node), parents):
return
self.names.add(self.sheerka.value(node))
self.names.add(self.sheerka.objvalue(node))
def can_be_discarded(self, variable_name, parents):
@@ -77,14 +77,14 @@ class UnreferencedNamesVisitor(ConceptNodeVisitor):
if node is None:
return False
if node.get_node_type() == "For" and self.sheerka.value(node.get_prop("target")) == variable_name:
if node.get_node_type() == "For" and self.sheerka.objvalue(node.get_value("target")) == variable_name:
# variable used by the loop
return True
if node.get_node_type() == "FunctionDef":
# variable defined as a function parameter
args = node.get_prop("args")
args_values = list(self.sheerka.get_values(args.get_prop("args")))
args = node.get_value("args")
args_values = list(self.sheerka.objvalues(args.get_value("args")))
if variable_name in args_values:
return True
@@ -112,8 +112,8 @@ def get_parents(node):
def iter_props(node):
for p in node.props:
yield p, node.props[p].value
for p in [p for p in node.values if isinstance(p, str)]:
yield p, node.get_value(p)
_const_node_type_names = {
src/core/builtin_concepts.py
+57 -56
View File
@@ -114,8 +114,8 @@ It's mainly to ease the usage
class UserInputConcept(Concept):
def __init__(self, text=None, user_name=None):
super().__init__(BuiltinConcepts.USER_INPUT, True, False, BuiltinConcepts.USER_INPUT)
self.set_metadata_value(ConceptParts.BODY, text)
self.set_prop("user_name", user_name)
self.set_value(ConceptParts.BODY, text)
self.set_value("user_name", user_name)
self.metadata.is_evaluated = True
@property
@@ -124,7 +124,7 @@ class UserInputConcept(Concept):
@property
def user_name(self):
return self.props["user_name"].value
return self.get_value("user_name")
def __repr__(self):
return f"({self.id}){self.name}: '{self.body}'"
@@ -133,7 +133,7 @@ class UserInputConcept(Concept):
class ErrorConcept(Concept):
def __init__(self, error=None):
super().__init__(BuiltinConcepts.ERROR, True, False, BuiltinConcepts.ERROR)
self.set_metadata_value(ConceptParts.BODY, error)
self.set_value(ConceptParts.BODY, error)
self.metadata.is_evaluated = True
def __repr__(self):
@@ -143,7 +143,7 @@ class ErrorConcept(Concept):
class UnknownConcept(Concept):
def __init__(self, metadata=None):
super().__init__(BuiltinConcepts.UNKNOWN_CONCEPT, True, False, BuiltinConcepts.UNKNOWN_CONCEPT)
self.set_metadata_value(ConceptParts.BODY, metadata)
self.set_value(ConceptParts.BODY, metadata)
self.metadata.is_evaluated = True
def __repr__(self):
@@ -158,28 +158,28 @@ class ReturnValueConcept(Concept):
def __init__(self, who=None, status=None, value=None, message=None, parents=None):
super().__init__(BuiltinConcepts.RETURN_VALUE, True, False, BuiltinConcepts.RETURN_VALUE)
self.set_metadata_value(ConceptParts.BODY, value)
self.set_prop("who", who)
self.set_prop("status", status)
self.set_prop("message", message)
self.set_prop("parents", parents)
self.set_value(ConceptParts.BODY, value)
self.set_value("who", who)
self.set_value("status", status)
self.set_value("message", message)
self.set_value("parents", parents)
self.metadata.is_evaluated = True
@property
def who(self):
return self.props["who"].value
return self.get_value("who")
@who.setter
def who(self, value):
self.set_prop("who", value)
self.set_value("who", value)
@property
def status(self):
return self.props["status"].value
return self.get_value("status")
@status.setter
def status(self, value):
self.set_prop("status", value)
self.set_value("status", value)
@property
def value(self):
@@ -187,23 +187,23 @@ class ReturnValueConcept(Concept):
@value.setter
def value(self, value):
self.set_metadata_value(ConceptParts.BODY, value)
self.set_value(ConceptParts.BODY, value)
@property
def message(self):
return self.props["message"].value
return self.get_value("message")
@message.setter
def message(self, value):
self.set_prop("message", value)
self.set_value("message", value)
@property
def parents(self):
return self.props["parents"].value
return self.get_value("parents")
@parents.setter
def parents(self, value):
self.set_prop("parents", value)
self.set_value("parents", value)
def __repr__(self):
return f"ReturnValue(who={self.who}, status={self.status}, value={self.value}, message={self.message})"
@@ -233,8 +233,8 @@ class UnknownPropertyConcept(Concept):
def __init__(self, property_name=None, concept=None):
super().__init__(BuiltinConcepts.UNKNOWN_PROPERTY, True, False, BuiltinConcepts.UNKNOWN_PROPERTY)
self.set_metadata_value(ConceptParts.BODY, property_name)
self.set_prop("concept", concept)
self.set_value(ConceptParts.BODY, property_name)
self.set_value("concept", concept)
self.metadata.is_evaluated = True
def __repr__(self):
@@ -242,7 +242,7 @@ class UnknownPropertyConcept(Concept):
@property
def concept(self):
return self.props["concept"].value
return self.get_value("concept")
@property
def property_name(self):
@@ -256,16 +256,16 @@ class ParserResultConcept(Concept):
def __init__(self, parser=None, source=None, tokens=None, value=None, try_parsed=None):
super().__init__(BuiltinConcepts.PARSER_RESULT, True, False, BuiltinConcepts.PARSER_RESULT)
self.set_metadata_value(ConceptParts.BODY, value)
self.set_prop("parser", parser)
self.set_prop("source", source)
self.set_prop("tokens", tokens)
self.set_prop("try_parsed", try_parsed) # in case of error, what was found before the error
self.set_value(ConceptParts.BODY, value)
self.set_value("parser", parser)
self.set_value("source", source)
self.set_value("tokens", tokens)
self.set_value("try_parsed", try_parsed) # in case of error, what was found before the error
self.metadata.is_evaluated = True
def __repr__(self):
text = f"ParserResult(parser={self.props['parser'].value}"
source = self.props['source'].value
text = f"ParserResult(parser={self.get_value('parser')}"
source = self.get_value('source')
text += f", source='{source}')" if source else f", body='{self.body}')"
return text
@@ -287,15 +287,15 @@ class ParserResultConcept(Concept):
@property
def try_parsed(self):
return self.props["try_parsed"].value
return self.get_value("try_parsed")
@property
def source(self):
return self.props["source"].value
return self.get_value("source")
@property
def parser(self):
return self.props["parser"].value
return self.get_value("parser")
class InvalidReturnValueConcept(Concept):
@@ -311,8 +311,8 @@ class InvalidReturnValueConcept(Concept):
True,
False,
BuiltinConcepts.INVALID_RETURN_VALUE)
self.set_metadata_value(ConceptParts.BODY, return_value)
self.set_prop("evaluator", evaluator)
self.set_value(ConceptParts.BODY, return_value)
self.set_value("evaluator", evaluator)
self.metadata.is_evaluated = True
@@ -322,9 +322,9 @@ class ConceptEvalError(Concept):
True,
False,
BuiltinConcepts.CONCEPT_EVAL_ERROR)
self.set_metadata_value(ConceptParts.BODY, error)
self.set_prop("concept", concept)
self.set_prop("property_name", property_name)
self.set_value(ConceptParts.BODY, error)
self.set_value("concept", concept)
self.set_value("property_name", property_name)
self.metadata.is_evaluated = True
def __repr__(self):
@@ -336,17 +336,17 @@ class ConceptEvalError(Concept):
@property
def concept(self):
return self.props["concept"].value
return self.get_value("concept")
@property
def property_name(self):
return self.props["property_name"].value
return self.get_value("property_name")
class EnumerationConcept(Concept):
def __init__(self, iteration=None):
super().__init__(BuiltinConcepts.ENUMERATION, True, False, BuiltinConcepts.ENUMERATION)
self.set_metadata_value(ConceptParts.BODY, iteration)
self.set_value(ConceptParts.BODY, iteration)
self.metadata.is_evaluated = True
# def __iter__(self):
@@ -356,7 +356,7 @@ class EnumerationConcept(Concept):
class ListConcept(Concept):
def __init__(self, items=None):
super().__init__(BuiltinConcepts.LIST, True, False, BuiltinConcepts.LIST)
self.set_metadata_value(ConceptParts.BODY, items or [])
self.set_value(ConceptParts.BODY, items or [])
self.metadata.is_evaluated = True
def append(self, obj):
@@ -381,9 +381,10 @@ class ListConcept(Concept):
class FilteredConcept(Concept):
def __init__(self, filtered=None, iterable=None, predicate=None):
super().__init__(BuiltinConcepts.FILTERED, True, False, BuiltinConcepts.FILTERED)
self.set_metadata_value(ConceptParts.BODY, filtered)
self.def_prop("iterable", iterable)
self.def_prop("predicate", predicate)
self.set_value(ConceptParts.BODY, filtered)
self.set_value("iterable", iterable)
self.set_value("predicate", predicate)
self.metadata.is_evaluated = True
class ConceptAlreadyInSet(Concept):
@@ -392,8 +393,8 @@ class ConceptAlreadyInSet(Concept):
True,
False,
BuiltinConcepts.CONCEPT_ALREADY_IN_SET)
self.set_metadata_value(ConceptParts.BODY, concept)
self.set_prop("concept_set", concept_set)
self.set_value(ConceptParts.BODY, concept)
self.set_value("concept_set", concept_set)
self.metadata.is_evaluated = True
def __repr__(self):
@@ -405,7 +406,7 @@ class ConceptAlreadyInSet(Concept):
@property
def concept_set(self):
return self.props["concept_set"].value
return self.get_value("concept_set")
class WhereClauseFailed(Concept):
@@ -414,7 +415,7 @@ class WhereClauseFailed(Concept):
True,
False,
BuiltinConcepts.WHERE_CLAUSE_FAILED)
self.set_metadata_value(ConceptParts.BODY, concept)
self.set_value(ConceptParts.BODY, concept)
self.metadata.is_evaluated = True
def __repr__(self):
@@ -431,12 +432,12 @@ class NotForMeConcept(Concept):
True,
False,
BuiltinConcepts.NOT_FOR_ME)
self.set_metadata_value(ConceptParts.BODY, source)
self.def_prop("reason", reason)
self.set_value(ConceptParts.BODY, source)
self.set_value("reason", reason)
self.metadata.is_evaluated = True
def __repr__(self):
return f"NotForMeConcept(source={self.body}, reason={self.get_prop('reason')})"
return f"NotForMeConcept(source={self.body}, reason={self.get_value('reason')})"
class ExplanationConcept(Concept):
@@ -445,9 +446,9 @@ class ExplanationConcept(Concept):
True,
False,
BuiltinConcepts.EXPLANATION)
self.def_prop("digest", digest) # event digest
self.def_prop("command", command) # explain command parameters
self.def_prop("title", title) # a title to the explanation
self.def_prop("instructions", instructions) # instructions for SheerkaPrint
self.set_metadata_value(ConceptParts.BODY, execution_result) # list of results
self.set_value("digest", digest) # event digest
self.set_value("command", command) # explain command parameters
self.set_value("title", title) # a title to the explanation
self.set_value("instructions", instructions) # instructions for SheerkaPrint
self.set_value(ConceptParts.BODY, execution_result) # list of results
self.metadata.is_evaluated = True
src/core/builtin_helpers.py
+47 -12
View File
@@ -30,11 +30,11 @@ def is_same_success(context, return_values):
evaluated = context.sheerka.evaluate_concept(sub_context, ret_val.body)
if evaluated.key != ret_val.body.key:
raise Exception("Failed to evaluate evaluate")
return context.sheerka.value(evaluated)
return context.sheerka.objvalue(evaluated)
else:
return context.sheerka.value(ret_val.body)
return context.sheerka.objvalue(ret_val.body)
else:
return context.sheerka.value(ret_val)
return context.sheerka.objvalue(ret_val)
try:
reference = _get_value(return_values[0])
@@ -280,8 +280,8 @@ def get_lexer_nodes(return_values, start, tokens):
for ret_val in return_values:
if ret_val.who == "parsers.Python":
if ret_val.body.source.strip().isalnum() and not ret_val.body.source.strip().isnumeric():
# Discard SourceCodeNode which seems to be a concept
if ret_val.body.source.strip().isidentifier():
# Discard SourceCodeNode which seems to be a concept name
# It may be a wrong idea, so let's see
continue
@@ -309,6 +309,41 @@ def get_lexer_nodes(return_values, start, tokens):
return lexer_nodes
def ensure_evaluated(context, concept):
"""
Evaluate a concept is not already evaluated
:param context:
:param concept:
:return:
"""
if concept.metadata.is_evaluated:
return concept
with context.push(desc=f"Evaluating concept {concept}") as sub_context:
sub_context.local_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
evaluated = context.sheerka.evaluate_concept(sub_context, concept)
sub_context.add_values(return_values=evaluated)
return evaluated
def get_lexer_nodes_from_unrecognized(context, unrecognized_tokens_node, parsers):
"""
Using parsers, try to recognize concepts from source
:param context:
:param unrecognized_tokens_node:
:param parsers:
:return:
"""
res = parse_unrecognized(context, unrecognized_tokens_node.source, parsers)
res = only_parsers_results(context, res)
if not res.status:
return None
return get_lexer_nodes(res.body.body, unrecognized_tokens_node.start, unrecognized_tokens_node.tokens)
def get_names(sheerka, concept_node):
"""
Finds all the names referenced by the concept_node
@@ -352,7 +387,7 @@ def extract_predicates(sheerka, expression, variables_to_include, variables_to_e
return NotImplementedError()
concept_node = core.ast.nodes.python_to_concept(node)
main_op = concept_node.get_prop("body")
main_op = concept_node.get_value("body")
return _get_predicates(_extract_predicates(sheerka, main_op, variables_to_include, variables_to_exclude))
@@ -370,14 +405,14 @@ def _extract_predicates(sheerka, node, variables_to_include, variables_to_exclud
return _res
if node.node_type == "Compare":
if node.get_prop("left").node_type == "Name":
if node.get_value("left").node_type == "Name":
"""Simple case of one comparison"""
comparison_name = sheerka.value(node.get_prop("left"))
comparison_name = sheerka.objvalue(node.get_value("left"))
if comparison_name in variables_to_include and comparison_name not in variables_to_exclude:
predicates.append(node)
else:
"""The left part is an expression"""
res = _extract_predicates(sheerka, node.get_prop("left"), variables_to_include, variables_to_exclude)
res = _extract_predicates(sheerka, node.get_value("left"), variables_to_include, variables_to_exclude)
if len(res) > 0:
predicates.append(node)
elif node.node_type == "Call":
@@ -386,9 +421,9 @@ def _extract_predicates(sheerka, node, variables_to_include, variables_to_exclud
args = list(call_node.get_args_names(sheerka))
if _matches(args, variables_to_include, variables_to_exclude):
predicates.append(node)
elif node.node_type == "UnaryOp" and node.get_prop("op").node_type == "Not":
elif node.node_type == "UnaryOp" and node.get_value("op").node_type == "Not":
"""Simple case of negation"""
res = _extract_predicates(sheerka, node.get_prop("operand"), variables_to_include, variables_to_exclude)
res = _extract_predicates(sheerka, node.get_value("operand"), variables_to_include, variables_to_exclude)
if len(res) > 0:
predicates.append(node)
elif node.node_type == "BinOp":
@@ -398,7 +433,7 @@ def _extract_predicates(sheerka, node, variables_to_include, variables_to_exclud
elif node.node_type == "BoolOp":
all_op = True
temp_res = []
for op in node.get_prop("values").body:
for op in node.get_value("values").body:
res = _extract_predicates(sheerka, op, variables_to_include, variables_to_exclude)
if len(res) == 0:
all_op = False
src/core/concept.py
+136 -89
View File
@@ -1,17 +1,19 @@
import hashlib
from collections import namedtuple
from copy import deepcopy
from dataclasses import dataclass
from enum import Enum
from core.sheerka_logger import get_logger
from typing import Union
import core.utils
from core.sheerka_logger import get_logger
from core.tokenizer import Tokenizer, TokenKind
PROPERTIES_FOR_DIGEST = ("name", "key",
"definition", "definition_type",
"is_builtin", "is_unique",
"where", "pre", "post", "body",
"desc", "props")
"desc", "props", "variables")
PROPERTIES_TO_SERIALIZE = PROPERTIES_FOR_DIGEST + tuple(["id"])
PROPERTIES_FOR_NEW = ("where", "pre", "post", "body", "desc")
VARIABLE_PREFIX = "__var__"
@@ -48,15 +50,13 @@ class ConceptMetadata:
definition_type: str # definition can be done with something else than regex
desc: str # possible description for the concept
id: str # unique identifier for a concept. The id will never be modified (but the key can)
props: list # list properties, with their default values
props: dict # hashmap of properties, values
variables: list # list of concept variables, with their default values
is_evaluated: bool = False # True is the concept is evaluated by sheerka.eval_concept()
need_validation = False # True if the properties of the concept need to be validated
full_serialization: bool = False # If True, the full object will be serialized, rather than just the diff
simplec = namedtuple("concept", "name body") # for simple concept (tests purposes only)
class Concept:
"""
Default concept object
@@ -76,7 +76,8 @@ class Concept:
definition_type=None,
desc=None,
id=None,
props=None):
props=None,
variables=None):
metadata = ConceptMetadata(
str(name) if name else None,
@@ -91,17 +92,17 @@ class Concept:
definition_type,
desc,
id,
props or []
props or {},
variables or []
)
self.metadata = metadata
self.compiled = {} # cached ast for the where, pre, post and body parts
self.values = {} # values of metadata once resolved
self.props = {} # resolved properties of this concept
self.bnf = None
self.compiled = {} # cached ast for the where, pre, post and body parts and variables
self.values = {} # resolved values. As compiled, it's used both for metadata and variables
self.bnf = None # parsing expression
self.log = get_logger("core." + self.__class__.__name__)
self.init_log = get_logger("init.core." + self.__class__.__name__)
self.original_definition_hash = None
self.original_definition_hash = None # concept hash before any alteration of the metadata
def __repr__(self):
return f"({self.metadata.id}){self.metadata.name}"
@@ -117,6 +118,9 @@ class Concept:
if isinstance(other, CC):
return other == self
if isinstance(other, CB):
return other == self
if not isinstance(other, Concept):
return False
@@ -147,15 +151,8 @@ class Concept:
if len(self.values) != len(other.values):
return False
for metadata in self.values:
if self.get_metadata_value(metadata) != other.get_metadata_value(metadata):
return False
if len(self.props) != len(other.props):
return False
for prop in self.props:
if self.get_prop(prop) != other.get_prop(prop):
for name in self.values:
if self.get_value(name) != other.get_value(name):
return False
return True
@@ -166,28 +163,36 @@ class Concept:
def __getattr__(self, item):
# I have this complicated implementation because of the usage of Pickle
if 'props' in vars(self) and item in self.props:
return self.props[item].value
if 'values' in vars(self) and item in self.values:
return self.get_value(item)
name = self.name if 'metadata' in vars(self) else 'Concept'
raise AttributeError(f"'{name}' concept has no attribute '{item}'")
def def_prop(self, prop_name, default_value=None):
def def_var(self, var_name, default_value=None):
"""
Adds a property to the metadata
:param prop_name: name or concept
:param var_name: name or concept
:param default_value:
:return:
"""
assert default_value is None or isinstance(default_value, str) # default properties will have to be evaluated
self.metadata.props.append((prop_name, default_value))
self.props[prop_name] = Property(prop_name, None) # do not set the default value
# why not setting props to the default values ?
# Because it may not be the real values, as metadata.props need to be evaluated
# this assert in not a functional requirement
# It's just to control what I put in the default value of properties
# You can allow more type if it's REALLY needed.
# - str are for standard definition
# - list of concepts is used by ISA
assert default_value is None or isinstance(default_value, str)
self.metadata.variables.append((var_name, default_value))
self.set_value(var_name, None) # do not set the default value
# why not setting variables to the default values ?
# Because it may not be the real values, as metadata.variables need to be evaluated
return self
def def_prop_by_index(self, index: int, value):
def def_var_by_index(self, index: int, value):
"""
Re-assign a value to a property (mainly used by ExactConceptParser)
:param index:
@@ -195,8 +200,8 @@ class Concept:
:return:
"""
assert value is None or isinstance(value, str) # default properties will have to be evaluated
prop = self.metadata.props[index]
self.metadata.props[index] = (prop[0], value)
var_name = self.metadata.variables[index]
self.metadata.variables[index] = (var_name[0], value) # change the default value
return self
@property
@@ -229,7 +234,7 @@ class Concept:
else:
tokens = list(Tokenizer(self.metadata.name))
variables = [p[0] for p in self.metadata.props] if len(core.utils.strip_tokens(tokens, True)) > 1 else []
variables = [p[0] for p in self.metadata.variables] if len(core.utils.strip_tokens(tokens, True)) > 1 else []
key = ""
first = True
@@ -252,7 +257,7 @@ class Concept:
@property
def body(self):
return self.values[ConceptParts.BODY] if ConceptParts.BODY in self.values else None
return self.get_value(ConceptParts.BODY)
def get_origin(self):
"""
@@ -284,8 +289,12 @@ class Concept:
"""
props_to_use = props_to_use or PROPERTIES_TO_SERIALIZE
props_as_dict = dict((prop, getattr(self.metadata, prop)) for prop in props_to_use)
props_as_dict = {}
for prop in props_to_use:
if prop == "props": # no need to copy variables as the ref won't be used in from_dict
props_as_dict[prop] = deepcopy(getattr(self.metadata, prop))
else:
props_as_dict[prop] = getattr(self.metadata, prop)
return props_as_dict
def from_dict(self, as_dict):
@@ -296,19 +305,20 @@ class Concept:
"""
for prop in PROPERTIES_TO_SERIALIZE:
if prop in as_dict:
if prop == "props":
if prop == "variables":
for name, value in as_dict[prop]:
self.def_prop(name, value)
self.def_var(name, value)
else:
setattr(self.metadata, prop, as_dict[prop])
return self
def update_from(self, other):
def update_from(self, other, update_value=True):
"""
Update self using the properties of another concept
This method is to mimic the class to instance pattern
'other' is the class, the template, and 'self' is a new instance
:param other:
:param update_value:
:return:
"""
if other is None:
@@ -321,12 +331,9 @@ class Concept:
self.from_dict(other.to_dict())
# update values
for k, v in other.values.items():
self.values[k] = v
# update properties
for k, v in other.props.items():
self.set_prop(k, v.value)
if update_value:
for k in other.values:
self.set_value(k, other.get_value(k))
# origin
from sdp.sheerkaSerializer import Serializer
@@ -335,54 +342,53 @@ class Concept:
return self
def set_prop(self, prop_name, prop_value):
def add_prop(self, concept_key, value):
"""
Set the value of a property (not the metadata)
:param prop_name: Name the property or another concept
:param prop_value:
:return:
"""
self.props[prop_name] = Property(prop_name, prop_value)
return self
def get_prop(self, prop_name: str):
"""
Gets the value of a property
:param prop_name: name or concept
:return:
"""
return self.props[prop_name].value
def set_prop_by_index(self, index: int, value):
"""
Set the value of a property (not the metadata) using the index
:param index: Name the property or another concept
Set or add a behaviour to a concept
A behaviour is a value from another concept (ex BuiltinConcepts.ISA
:param concept_key: Concept key
:param value:
:return:
"""
prop_name = list(self.props.keys())[index]
self.props[prop_name].value = value
if concept_key in self.metadata.props:
self.metadata.props[concept_key].add(value)
else:
self.metadata.props[concept_key] = {value} # a set
return self
def set_metadata_value(self, metadata: ConceptParts, value):
def get_prop(self, concept_key):
"""
Set the resolved value of a metadata (not the metadata itself)
:param metadata:
Gets a behaviour of a concept
:param concept_key: name of the behaviour
:return:
"""
return self.metadata.props[concept_key] if concept_key in self.metadata.props else None
def set_value(self, name, value):
"""
Set the resolved value of a metadata or a variable (not the metadata itself)
:param name:
:param value:
:return:
"""
self.values[metadata] = value
if name in self.values:
self.values[name].value = value
else:
self.values[name] = Property(name, value)
return self
def get_metadata_value(self, metadata: ConceptParts):
def get_value(self, prop_name):
"""
Gets the resolved value of a metadata
:param metadata:
:param prop_name:
:return:
"""
if metadata not in self.values:
if prop_name not in self.values:
return None
return self.values[metadata]
return self.values[prop_name].value
def variables(self):
return dict([(k, v) for k, v in self.values.items() if isinstance(k, str)])
def auto_init(self):
"""
@@ -398,10 +404,10 @@ class Concept:
for metadata in ConceptParts:
value = getattr(self.metadata, metadata.value)
if value is not None:
self.values[metadata] = value
self.set_value(metadata, value)
for prop, value in self.metadata.props:
self.set_prop(prop, value)
for var, value in self.metadata.variables:
self.set_value(var, value)
self.metadata.is_evaluated = True
return self
@@ -419,9 +425,10 @@ class Concept:
And it removes the visibility from the other attributes/methods
"""
bag = {}
for prop in self.props:
bag[prop] = self.get_prop(prop)
bag["prop." + prop] = self.get_prop(prop)
for var in self.values:
if isinstance(var, str):
bag[var] = self.get_value(var)
bag["var." + var] = self.get_value(var)
for prop in ("id", "name", "key", "body"):
bag[prop] = getattr(self, prop)
return bag
@@ -469,10 +476,17 @@ class InfiniteRecursionResolved:
"""This class is used to when we managed to break an infinite recursion concept definition"""
value: object
def get_value(self):
def get_obj_value(self):
return self.value
# ################################
#
# Class created for tests purpose
#
# ################################
class CC:
"""
Concept class for test purpose
@@ -484,13 +498,14 @@ class CC:
# The other properties (concept, source, start and end)
# are used in tests/parsers/parsers_utils.py to help creating helper objects
def __init__(self, concept, source=None, **kwargs):
def __init__(self, concept, source=None, exclude_body=False, **kwargs):
self.concept_key = concept.key if isinstance(concept, Concept) else concept
self.compiled = kwargs
self.concept = concept if isinstance(concept, Concept) else None
self.source = source # to use when the key is different from the sub str to search when filling start and stop
self.start = None # for debug purpose, indicate where the concept starts
self.end = None # for debug purpose, indicate where the concept ends
self.exclude_body = exclude_body
def __eq__(self, other):
if id(self) == id(other):
@@ -499,13 +514,19 @@ class CC:
if isinstance(other, Concept):
if other.key != self.concept_key:
return False
return self.compiled == other.compiled
if self.exclude_body:
to_compare = {k: v for k, v in other.compiled.items() if k != ConceptParts.BODY}
else:
to_compare = other.compiled
return self.compiled == to_compare
if not isinstance(other, CC):
return False
return self.concept_key == other.concept_key and \
self.compiled == other.compiled
if self.concept_key != other.concept_key:
return False
return self.compiled == other.compiled
def __hash__(self):
if self.concept:
@@ -536,3 +557,29 @@ class CC:
if self.end is None or end > self.end:
self.end = end
return self
@dataclass()
class CB:
"""
Concept with body only
Test class that test only the body of the concept
"""
concept: Union[str, Concept]
body: object
def __eq__(self, other):
if isinstance(other, Concept):
key = self.concept if isinstance(self.concept, str) else self.concept.key
return key == other.key and self.body == other.body
if not isinstance(other, CB):
return False
return self.concept == other.concept and self.body == other.body
def __hash__(self):
return hash((self.concept, self.body))
simplec = namedtuple("concept", "name body") # for simple concept (tests purposes only)
src/core/profiling.py
+26
View File
@@ -0,0 +1,26 @@
# ############################
# from github: nealtodd/decorator.py
# ############################
import pstats
from cProfile import Profile
def profile(sort_args=None, print_args=None):
sort_args = sort_args or ['cumulative']
print_args = print_args or [10]
profiler = Profile()
def decorator(fn):
def inner(*args, **kwargs):
result = None
try:
result = profiler.runcall(fn, *args, **kwargs)
finally:
stats = pstats.Stats(profiler)
stats.strip_dirs().sort_stats(*sort_args).print_stats(*print_args)
return result
return inner
return decorator
src/core/sheerka/ExecutionContext.py
+10 -9
View File
@@ -144,9 +144,9 @@ class ExecutionContext:
def add_preprocess(self, name, **kwargs):
preprocess = self.sheerka.new(BuiltinConcepts.EVALUATOR_PRE_PROCESS)
preprocess.set_prop("name", name)
preprocess.set_value("name", name)
for k, v in kwargs.items():
preprocess.set_prop(k, v)
preprocess.set_value(k, v)
if not self.preprocess:
self.preprocess = []
@@ -168,9 +168,9 @@ class ExecutionContext:
if isinstance(self.obj, Concept):
if self.obj.key == key:
return self.obj
for prop in self.obj.props:
if prop == key:
value = self.obj.props[prop].value
for var_name in self.obj.values:
if var_name == key:
value = self.obj.get_value(var_name)
if isinstance(value, Concept):
return value
@@ -180,16 +180,16 @@ class ExecutionContext:
if k == key:
return c
return self.sheerka.get(key)
return self.sheerka.get_by_key(key)
def new_concept(self, key, **kwargs):
# search in obj
if self.obj:
if self.obj.key == key:
return self.sheerka.new_from_template(self.obj, key, **kwargs)
for prop in self.obj.props:
if prop == key:
value = self.obj.props[prop].value
for var_name in self.obj.values:
if var_name == key:
value = self.obj.get_value(var_name)
if isinstance(value, Concept):
return self.sheerka.new_from_template(value, key, **kwargs)
else:
@@ -327,6 +327,7 @@ class ExecutionContext:
bag[prop] = getattr(self, prop)
bag["status"] = self.get_status()
bag["elapsed"] = self.elapsed
bag["elapsed_str"] = self.elapsed_str
bag["digest"] = self.event.get_digest() if self.event else None
return bag
src/core/sheerka/Services/SheerkaCreateNewConcept.py
+51 -93
View File
@@ -1,9 +1,9 @@
import core.utils
from core.builtin_concepts import BuiltinConcepts, ErrorConcept
from core.concept import Concept
from sdp.sheerkaDataProvider import SheerkaDataProviderDuplicateKeyError, SheerkaDataProviderRef
import core.utils
from sdp.sheerkaDataProvider_Old import SheerkaDataProviderDuplicateKeyError
BNF_NODE_PARSER_CLASS = "parsers.BnfNodeParser.BnfNodeParser"
BNF_NODE_PARSER_CLASS = "parsers.BnfNodeParser_Old.BnfNodeParser"
BASE_NODE_PARSER_CLASS = "parsers.BaseNodeParser.BaseNodeParser"
@@ -15,122 +15,80 @@ class SheerkaCreateNewConcept:
def __init__(self, sheerka):
self.sheerka = sheerka
self.logger_name = self.create_new_concept.__name__
self.base_lexer_parser = core.utils.get_class(BASE_NODE_PARSER_CLASS)("BaseNodeParser", 0)
self.bnp = core.utils.get_class(BASE_NODE_PARSER_CLASS) # BaseNodeParser
def create_new_concept(self, context, concept: Concept):
"""
Adds a new concept to the system
:param context:
:param concept: DefConceptNode
:param logger
:return: digest of the new concept
"""
sheerka = self.sheerka
concept.init_key()
concepts_definitions = None
init_bnf_ret_value = None
sdp = self.sheerka.sdp
cache_manager = sheerka.cache_manager
# checks for duplicate concepts
# TODO checks if it exists in cache first
if sdp.exists(self.sheerka.CONCEPTS_BY_HASH_ENTRY, concept.get_definition_hash()):
error = SheerkaDataProviderDuplicateKeyError(self.sheerka.CONCEPTS_ENTRY + "." + concept.key, concept)
return self.sheerka.ret(
if cache_manager.exists(sheerka.CONCEPTS_BY_HASH_ENTRY, concept.get_definition_hash()):
error = SheerkaDataProviderDuplicateKeyError(sheerka.CONCEPTS_BY_KEY_ENTRY + "." + concept.key,
concept)
return sheerka.ret(
self.logger_name,
False,
self.sheerka.new(BuiltinConcepts.CONCEPT_ALREADY_DEFINED, body=concept),
sheerka.new(BuiltinConcepts.CONCEPT_ALREADY_DEFINED, body=concept),
error.args[0])
# set id before saving in db
self.sheerka.set_id_if_needed(concept, False)
sheerka.set_id_if_needed(concept, False)
# add the BNF if known
if concept.bnf:
concepts_definitions = self.sheerka.get_concepts_definitions(context)
concepts_definitions[concept] = concept.bnf
# update the dictionary of concepts by first key
init_ret_value = self.bnp.get_concepts_by_first_keyword(context, [concept], True)
if not init_ret_value.status:
return sheerka.ret(self.logger_name, False, ErrorConcept(init_ret_value.value))
concepts_by_first_keyword = init_ret_value.body
# check if it's a valid BNF or whether it breaks the known rules
bnf_lexer_parser = self.sheerka.parsers[BNF_NODE_PARSER_CLASS]()
with context.push(self.sheerka.name, desc=f"Initializing concept definition for {concept}") as sub_context:
sub_context.concepts[concept.key] = concept # the concept is not in the real cache yet
init_bnf_ret_value = bnf_lexer_parser.initialize(sub_context, concepts_definitions)
sub_context.add_values(return_values=init_bnf_ret_value)
if not init_bnf_ret_value.status:
return self.sheerka.ret(self.logger_name, False, ErrorConcept(init_bnf_ret_value.value))
# update resolved dictionary
init_ret_value = self.bnp.resolve_concepts_by_first_keyword(context, concepts_by_first_keyword)
if not init_ret_value.status:
return sheerka.ret(self.logger_name, False, ErrorConcept(init_ret_value.value))
resolved_concepts_by_first_keyword = init_ret_value.body
# update concept definition by key
init_sya_ret_value = self.base_lexer_parser.initialize(context, [concept], use_sheerka=True)
if not init_sya_ret_value.status:
return self.sheerka.ret(self.logger_name, False, ErrorConcept(init_sya_ret_value.value))
concepts_by_first_keyword = init_sya_ret_value.body
# init_sya_ret_value = self.bnp.initialize(context, [concept], use_sheerka=True)
# if not init_sya_ret_value.status:
# return sheerka.ret(self.logger_name, False, ErrorConcept(init_sya_ret_value.value))
# concepts_by_first_keyword = init_sya_ret_value.body
concept.freeze_definition_hash()
# save the new concept in sdp
try:
# TODO : needs to make these calls atomic (or at least one single call)
# save the new concept
concept.metadata.full_serialization = True
result = sdp.add(
context.event.get_digest(),
self.sheerka.CONCEPTS_ENTRY,
concept,
use_ref=True)
concept.metadata.full_serialization = False
cache_manager.add_concept(concept)
cache_manager.put(sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, False, concepts_by_first_keyword)
cache_manager.put(sheerka.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY, False, resolved_concepts_by_first_keyword)
# update the concept (I hope that it's enough)
concept.set_origin(result.digest)
# save it by id
sdp.add(
context.event.get_digest(),
self.sheerka.CONCEPTS_BY_ID_ENTRY,
SheerkaDataProviderRef(concept.id, result.digest))
# save it by name
sdp.add(
context.event.get_digest(),
self.sheerka.CONCEPTS_BY_NAME_ENTRY,
SheerkaDataProviderRef(concept.name, result.digest))
# records the hash
sdp.add(
context.event.get_digest(),
self.sheerka.CONCEPTS_BY_HASH_ENTRY,
SheerkaDataProviderRef(concept.get_definition_hash(), result.digest))
# update the definition table
if concepts_definitions is not None:
sdp.set(
context.event.get_digest(),
self.sheerka.CONCEPTS_DEFINITIONS_ENTRY,
bnf_lexer_parser.encode_grammar(init_bnf_ret_value.body),
use_ref=True)
self.sheerka.concepts_definitions_cache = None # invalidate cache
# update the concepts by first keyword
sdp.set(context.event.get_digest(),
self.sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY,
concepts_by_first_keyword)
except SheerkaDataProviderDuplicateKeyError as error:
context.log_error("Failed to create a new concept.", who=self.logger_name)
return self.sheerka.ret(
self.logger_name,
False,
self.sheerka.new(BuiltinConcepts.CONCEPT_ALREADY_DEFINED, body=concept),
error.args[0])
# Updates the caches
self.sheerka.cache_by_key[concept.key] = sdp.get_safe(self.sheerka.CONCEPTS_ENTRY, concept.key)
self.sheerka.cache_by_name[concept.name] = sdp.get_safe(self.sheerka.CONCEPTS_BY_NAME_ENTRY, concept.name)
self.sheerka.cache_by_id[concept.id] = concept
if init_bnf_ret_value is not None and init_bnf_ret_value.status:
self.sheerka.concepts_grammars = init_bnf_ret_value.body
self.sheerka.concepts_by_first_keyword = concepts_by_first_keyword
if concept.bnf and init_bnf_ret_value is not None and init_bnf_ret_value.status:
sheerka.cache_manager.clear(sheerka.CONCEPTS_GRAMMARS_ENTRY)
# process the return if needed
ret = self.sheerka.ret(self.logger_name, True, self.sheerka.new(BuiltinConcepts.NEW_CONCEPT, body=concept))
ret = sheerka.ret(self.logger_name, True, sheerka.new(BuiltinConcepts.NEW_CONCEPT, body=concept))
return ret
# def load_concepts_nodes_definitions(self, context):
# """
# Gets from sdp what is need to parse nodes
# :return:
# """
# sdp = self.sheerka.sdp
#
# concepts_by_first_keyword = sdp.get(
# self.sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY,
# load_origin=False) or {}
#
# init_ret_value = self.bnp.resolve_concepts_by_first_keyword(context, concepts_by_first_keyword)
# if not init_ret_value.status:
# return self.sheerka.ret(self.logger_name, False, ErrorConcept(init_ret_value.value))
# resolved_concepts_by_first_keyword = init_ret_value.body
#
# return concepts_by_first_keyword, resolved_concepts_by_first_keyword
src/core/sheerka/Services/SheerkaDump.py
+7 -10
View File
@@ -1,9 +1,10 @@
import os
import pprint
from core.builtin_concepts import BuiltinConcepts
from core.concept import Concept
from core.sheerka.ExecutionContext import ExecutionContext
from sdp.sheerkaDataProvider import SheerkaDataProvider, Event
import pprint
import os
def get_pp():
@@ -17,7 +18,7 @@ class SheerkaDump:
self.sheerka = sheerka
def dump_concepts(self):
lst = self.sheerka.sdp.list(self.sheerka.CONCEPTS_ENTRY)
lst = self.sheerka.sdp.list(self.sheerka.CONCEPTS_BY_KEY_ENTRY)
for item in lst:
if hasattr(item, "__iter__"):
for i in item:
@@ -25,10 +26,6 @@ class SheerkaDump:
else:
self.sheerka.log.info(item)
def dump_definitions(self):
defs = self.sheerka.sdp.get(self.sheerka.CONCEPTS_DEFINITIONS_ENTRY)
self.sheerka.log.info(defs)
def dump_desc(self, *concept_names, eval=False):
first = True
event = Event(f"Dumping description", "")
@@ -37,7 +34,7 @@ class SheerkaDump:
if isinstance(concept_name, Concept):
concepts = concept_name
else:
concepts = self.sheerka.get(concept_name)
concepts = self.sheerka.get_by_key(concept_name)
if self.sheerka.isinstance(concepts, BuiltinConcepts.UNKNOWN_CONCEPT):
self.sheerka.log.error(f"Concept '{concept_name}' is unknown")
return False
@@ -59,8 +56,8 @@ class SheerkaDump:
self.sheerka.log.info(f"where : {c.metadata.where}")
if eval:
self.sheerka.log.info(f"value : {value}")
for p in c.props:
self.sheerka.log.info(f"{p}: {c.get_prop(p)}")
for v in c.values:
self.sheerka.log.info(f"{v}: {c.get_value(v)}")
else:
self.sheerka.log.info("No property")
src/core/sheerka/Services/SheerkaEvaluateConcept.py
+23 -29
View File
@@ -1,6 +1,6 @@
from core.builtin_concepts import BuiltinConcepts
from core.concept import Concept, DoNotResolve, ConceptParts, InfiniteRecursionResolved
from core.builtin_helpers import expect_one
from core.concept import Concept, DoNotResolve, ConceptParts, InfiniteRecursionResolved
CONCEPT_EVALUATION_STEPS = [
BuiltinConcepts.BEFORE_EVALUATION,
@@ -91,32 +91,27 @@ class SheerkaEvaluateConcept:
concept.compiled[part_key] = res
sub_context.add_values(return_values=res)
for prop, default_value in concept.metadata.props:
if prop in concept.compiled:
for var_name, default_value in concept.metadata.variables:
if var_name in concept.compiled:
continue
if default_value is None or not isinstance(default_value, str):
continue
if default_value.strip() == "":
concept.compiled[prop] = DoNotResolve(default_value)
concept.compiled[var_name] = DoNotResolve(default_value)
else:
with context.push(desc=f"Initializing AST for property {prop}") as sub_context:
with context.push(desc=f"Initializing AST for property {var_name}") as sub_context:
sub_context.add_inputs(source=default_value)
to_parse = self.sheerka.ret(context.who, True,
self.sheerka.new(BuiltinConcepts.USER_INPUT, body=default_value))
res = self.sheerka.execute(context, to_parse, steps)
concept.compiled[prop] = res
concept.compiled[var_name] = res
sub_context.add_values(return_values=res)
# Updates the cache of concepts when possible
if concept.key in self.sheerka.cache_by_key:
entry = self.sheerka.cache_by_key[concept.key]
if isinstance(entry, list):
# TODO : manage when there are multiple entries
pass
else:
self.sheerka.cache_by_key[concept.key].compiled = concept.compiled
if self.sheerka.has_id(concept.id):
self.sheerka.get_by_id(concept.id).compiled = concept.compiled
def resolve(self, context, to_resolve, current_prop, current_concept, force_evaluation):
if isinstance(to_resolve, DoNotResolve):
@@ -198,7 +193,6 @@ class SheerkaEvaluateConcept:
It means that if the where clause is True, will evaluate the body
:param context:
:param concept:
:param evaluate_body: If false, only evaluate body when necessary
:return: value of the evaluation or error
"""
@@ -208,26 +202,26 @@ class SheerkaEvaluateConcept:
self.initialize_concept_asts(context, concept)
# to make sure of the order, it don't use ConceptParts.get_parts()
# props must be evaluated first, body must be evaluated before where
# variables must be evaluated first, body must be evaluated before where
all_metadata_to_eval = self.choose_metadata_to_eval(context, concept)
for metadata_to_eval in all_metadata_to_eval:
if metadata_to_eval == "props":
for prop_name in (p for p in concept.props if p in concept.compiled):
prop_ast = concept.compiled[prop_name]
if metadata_to_eval == "variables":
for var_name in (v for v in concept.variables() if v in concept.compiled):
prop_ast = concept.compiled[var_name]
if isinstance(prop_ast, list):
# Do not send the current concept for the properties
resolved = self.resolve_list(context, prop_ast, prop_name, None, True)
resolved = self.resolve_list(context, prop_ast, var_name, None, True)
else:
# Do not send the current concept for the properties
resolved = self.resolve(context, prop_ast, prop_name, None, True)
resolved = self.resolve(context, prop_ast, var_name, None, True)
if isinstance(resolved, Concept) and not context.sheerka.is_success(resolved):
resolved.set_prop("concept", concept) # since current concept was not sent
resolved.set_value("concept", concept) # since current concept was not sent
return resolved
else:
concept.set_prop(prop_name, resolved)
concept.set_value(var_name, resolved)
else:
part_key = ConceptParts(metadata_to_eval)
@@ -245,7 +239,7 @@ class SheerkaEvaluateConcept:
if isinstance(resolved, Concept) and not context.sheerka.is_success(resolved):
return resolved
else:
concept.values[part_key] = self.get_infinite_recursion_resolution(resolved) or resolved
concept.set_value(part_key, self.get_infinite_recursion_resolution(resolved) or resolved)
#
# TODO : Validate the PRE condition
@@ -253,8 +247,8 @@ class SheerkaEvaluateConcept:
# validate where clause
if ConceptParts.WHERE in concept.values:
where_value = concept.values[ConceptParts.WHERE]
if not (where_value is None or self.sheerka.value(where_value)):
where_value = concept.get_value(ConceptParts.WHERE)
if not (where_value is None or self.sheerka.objvalue(where_value)):
return self.sheerka.new(BuiltinConcepts.WHERE_CLAUSE_FAILED, body=concept)
#
@@ -267,7 +261,7 @@ class SheerkaEvaluateConcept:
def choose_metadata_to_eval(self, context, concept):
if context.in_context(BuiltinConcepts.EVAL_BODY_REQUESTED):
return ["pre", "post", "props", "body", "where"]
return ["pre", "post", "variables", "body", "where"]
metadata = ["pre", "post"]
if context.in_context(BuiltinConcepts.EVAL_WHERE_REQUESTED) or concept.metadata.need_validation:
@@ -310,9 +304,9 @@ class SheerkaEvaluateConcept:
if not isinstance(return_value.body.source, str):
continue
for prop_name in (p[0] for p in concept.metadata.props):
if prop_name in return_value.body.source:
needed.append("props")
for var_name in (p[0] for p in concept.metadata.variables):
if var_name in return_value.body.source:
needed.append("variables")
break
if "self" in return_value.body.source:
src/core/sheerka/Services/SheerkaExecute.py
+5 -5
View File
@@ -230,12 +230,12 @@ class SheerkaExecute:
for preprocess in context.preprocess:
for e in parsers_or_evaluators:
if self.matches(e.name, preprocess.get_prop("name")):
for prop, value in preprocess.props.items():
if prop == "name":
if self.matches(e.name, preprocess.get_value("name")):
for var_name in preprocess.values:
if var_name == "name":
continue
if hasattr(e, prop):
setattr(e, prop, value.value)
if hasattr(e, var_name):
setattr(e, var_name, preprocess.get_value(var_name))
return parsers_or_evaluators[0] if single_one else parsers_or_evaluators
@staticmethod
src/core/sheerka/Services/SheerkaModifyConcept.py
+26 -44
View File
@@ -1,5 +1,4 @@
from core.builtin_concepts import BuiltinConcepts
from sdp.sheerkaDataProvider import SheerkaDataProviderRef
class SheerkaModifyConcept:
@@ -8,52 +7,35 @@ class SheerkaModifyConcept:
self.logger_name = self.modify_concept.__name__
def modify_concept(self, context, concept):
old_version = self.sheerka.get_by_id(concept.id)
sdp = self.sheerka.sdp
try:
# modify the entry
concept.metadata.full_serialization = True
result = sdp.modify(
context.event.get_digest(),
self.sheerka.CONCEPTS_ENTRY,
concept.key,
concept)
concept.metadata.full_serialization = False
# update reference entry
sdp.modify(
context.event.get_digest(),
self.sheerka.CONCEPTS_BY_ID_ENTRY,
concept.id,
SheerkaDataProviderRef(concept.id, result.digest, concept.get_origin()))
# update name entry
sdp.modify(
context.event.get_digest(),
self.sheerka.CONCEPTS_BY_NAME_ENTRY,
concept.name,
SheerkaDataProviderRef(concept.name, result.digest, concept.get_origin()))
# update the hash entry
sdp.modify(
context.event.get_digest(),
self.sheerka.CONCEPTS_BY_HASH_ENTRY,
concept.get_original_definition_hash(),
SheerkaDataProviderRef(concept.get_definition_hash(), result.digest, concept.get_origin()))
except IndexError as error:
context.log_error(f"Failed to update concept '{concept}'.", who=self.logger_name)
if old_version is None:
# nothing found in cache
return self.sheerka.ret(
self.logger_name,
False,
self.sheerka.new(BuiltinConcepts.UNKNOWN_CONCEPT, body=concept),
error.args[0])
self.logger_name, False,
self.sheerka.new(
BuiltinConcepts.UNKNOWN_CONCEPT,
body=[("key", concept.key), ("id", concept.id)]))
# update cache
self.sheerka.cache_by_key[concept.key] = sdp.get_safe(self.sheerka.CONCEPTS_ENTRY, concept.key)
self.sheerka.cache_by_name[concept.name] = sdp.get_safe(self.sheerka.CONCEPTS_BY_NAME_ENTRY, concept.name)
self.sheerka.cache_by_id[concept.id] = concept
if not self.sheerka.is_success(old_version) and concept.key != old_version.key:
# an error concept is returned
return self.sheerka.ret(
self.logger_name, False,
old_version)
if old_version == concept:
# the concept is not modified
return self.sheerka.ret(
self.logger_name, False,
self.sheerka.new(
BuiltinConcepts.CONCEPT_ALREADY_DEFINED,
body=concept))
self.sheerka.cache_manager.update_concept(old_version, concept)
# TODO : update concept by first keyword
# TODO : update resolved by first keyword
# TODO : update concets grammars
ret = self.sheerka.ret(self.logger_name, True, self.sheerka.new(BuiltinConcepts.NEW_CONCEPT, body=concept))
return ret
src/core/sheerka/Services/SheerkaSetsManager.py
+81 -76
View File
@@ -1,7 +1,7 @@
from core.ast.nodes import python_to_concept
from core.builtin_concepts import BuiltinConcepts, ErrorConcept
from core.concept import Concept, ConceptParts
import core.builtin_helpers
from core.ast.nodes import python_to_concept
from core.builtin_concepts import BuiltinConcepts
from core.concept import Concept, ConceptParts
GROUP_PREFIX = 'All_'
@@ -20,13 +20,16 @@ class SheerkaSetsManager:
:return:
"""
context.log(f"Setting that concept {concept} is a {concept_set}", who=self.logger_name)
context.log(f"Setting concept {concept} is a {concept_set}", who=self.logger_name)
isa = [] if BuiltinConcepts.ISA not in concept.props else concept.get_prop(BuiltinConcepts.ISA)
if concept_set not in isa:
isa.append(concept_set)
if BuiltinConcepts.ISA in concept.metadata.props and concept_set in concept.metadata.props[BuiltinConcepts.ISA]:
return self.sheerka.ret(
self.logger_name,
False,
self.sheerka.new(BuiltinConcepts.CONCEPT_ALREADY_IN_SET, body=concept, concept_set=concept_set))
concept.add_prop(BuiltinConcepts.ISA, concept_set)
concept.set_prop(BuiltinConcepts.ISA, isa)
res = self.sheerka.modify_concept(context, concept)
if not res.status:
return res
@@ -47,32 +50,34 @@ class SheerkaSetsManager:
assert concept.id
assert concept_set.id
try:
result = self.sheerka.sdp.add_unique(context.event.get_digest(), GROUP_PREFIX + concept_set.id, concept.id)
if result.already_exists: # concept already in set
return self.sheerka.ret(
self.logger_name,
False,
self.sheerka.new(BuiltinConcepts.CONCEPT_ALREADY_IN_SET, body=concept, concept_set=concept_set))
else:
return self.sheerka.ret(self.logger_name, True, self.sheerka.new(BuiltinConcepts.SUCCESS))
except Exception as error:
context.log_error("Failed to add to set.", who=self.logger_name)
return self.sheerka.ret(self.logger_name, False, ErrorConcept(error), error.args[0])
set_elements = self.sheerka.cache_manager.get(self.sheerka.CONCEPTS_GROUPS_ENTRY, concept_set.id)
if set_elements and concept.id in set_elements:
return self.sheerka.ret(
self.logger_name,
False,
self.sheerka.new(BuiltinConcepts.CONCEPT_ALREADY_IN_SET, body=concept, concept_set=concept_set))
self.sheerka.cache_manager.put(self.sheerka.CONCEPTS_GROUPS_ENTRY, concept_set.id, concept.id)
return self.sheerka.ret(self.logger_name, True, self.sheerka.new(BuiltinConcepts.SUCCESS))
def add_concepts_to_set(self, context, concepts, concept_set):
"""Adding multiple concepts at the same time"""
context.log(f"Adding concepts {concepts} to set {concept_set}", who=self.logger_name)
previous = self.sheerka.sdp.get_safe(GROUP_PREFIX + concept_set.id)
already_in_set = []
for concept in concepts:
res = self.add_concept_to_set(context, concept, concept_set)
if self.sheerka.isinstance(res.body, BuiltinConcepts.CONCEPT_ALREADY_IN_SET):
already_in_set.append(res.body.body)
new_ids = [c.id for c in concepts] if previous is None else previous + [c.id for c in concepts]
try:
self.sheerka.sdp.set(context.event.get_digest(), GROUP_PREFIX + concept_set.id, new_ids)
return self.sheerka.ret(self.logger_name, True, self.sheerka.new(BuiltinConcepts.SUCCESS))
except Exception as error:
context.log_error("Failed to add to set.", who=self.logger_name)
return self.sheerka.ret(self.logger_name, False, ErrorConcept(error), error.args[0])
if already_in_set:
body = self.sheerka.new(BuiltinConcepts.CONCEPT_ALREADY_IN_SET,
body=already_in_set,
concept_set=concept_set)
else:
body = self.sheerka.new(BuiltinConcepts.SUCCESS)
return self.sheerka.ret(self.logger_name, len(already_in_set) != len(concepts), body)
def get_set_elements(self, context, concept):
"""
@@ -83,38 +88,41 @@ class SheerkaSetsManager:
:return:
"""
# noinspection PyShadowingNames
def _get_set_elements(context, concept, sub_concept):
if not (isinstance(sub_concept, Concept) and sub_concept.id):
def _get_set_elements(sub_concept):
if not self.isaset(context, sub_concept):
return self.sheerka.new(BuiltinConcepts.NOT_A_SET, body=concept)
ids = self.sheerka.sdp.get_safe(GROUP_PREFIX + sub_concept.id)
if ids:
if concept.metadata.where:
new_condition = self._validate_where_clause(concept)
if not new_condition:
return self.sheerka.new(BuiltinConcepts.WHERE_CLAUSE_FAILED, body=concept)
else:
# This methods sucks, but I don't have enough tools (like proper AST manipulation functions)
# to do it properly now. It will be enhanced later
concepts = self._get_concepts(context, ids, True)
globals_ = {"xx__concepts__xx": concepts, "sheerka": self.sheerka}
locals_ = {}
exec(new_condition, globals_, locals_)
return locals_["result"]
else:
return self._get_concepts(context, ids, False)
# first, try to see if sub_context has it's own group entry
ids = self.sheerka.cache_manager.get(self.sheerka.CONCEPTS_GROUPS_ENTRY, sub_concept.id)
concepts = self._get_concepts(context, ids, True)
# it may be a concept that references a set
if not sub_concept.metadata.is_evaluated:
with context.push(desc=f"Evaluating concept {sub_concept}") as sub_context:
sub_context.local_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
evaluated = self.sheerka.evaluate_concept(sub_context, sub_concept)
if evaluated.key != concept.key:
return False
return _get_set_elements(context, concept, sub_concept.body)
# aggregate with en entries from its body
sub_concept = core.builtin_helpers.ensure_evaluated(context, sub_concept)
if not self.sheerka.is_success(sub_concept):
return sub_concept
return _get_set_elements(context, concept, concept)
if self.isaset(context, sub_concept.body):
other_concepts = _get_set_elements(sub_concept.body)
if not self.sheerka.is_success(other_concepts):
return other_concepts
concepts.extend(other_concepts)
# apply the where clause if any
if sub_concept.metadata.where:
new_condition = self._validate_where_clause(sub_concept)
if not new_condition:
return self.sheerka.new(BuiltinConcepts.WHERE_CLAUSE_FAILED, body=sub_concept)
# This methods sucks, but I don't have enough tools (like proper AST manipulation functions)
# to do it properly now. It will be enhanced later
globals_ = {"xx__concepts__xx": concepts, "sheerka": self.sheerka}
locals_ = {}
exec(new_condition, globals_, locals_)
concepts = locals_["result"]
return concepts
return _get_set_elements(concept)
def isinset(self, a, b):
"""
@@ -135,17 +143,15 @@ class SheerkaSetsManager:
if not (a.id and b.id):
return False
if self.sheerka.sdp.exists(GROUP_PREFIX + b.id, a.id):
return True
return False
group_elements = self.sheerka.cache_manager.get(self.sheerka.CONCEPTS_GROUPS_ENTRY, b.id)
return group_elements and a.id in group_elements
def isa(self, a, b):
if BuiltinConcepts.ISA not in a.props:
if BuiltinConcepts.ISA not in a.metadata.props:
return False
for c in a.get_prop(BuiltinConcepts.ISA):
for c in a.metadata.props[BuiltinConcepts.ISA]:
if c == b:
return True
if self.isa(c, b):
@@ -163,21 +169,19 @@ class SheerkaSetsManager:
""""""
if not (isinstance(concept, Concept) and concept.id):
return None
return False
# check if it has a group
# TODO: use cache instead of directly requesting sdp
if self.sheerka.cache_manager.get(self.sheerka.CONCEPTS_GROUPS_ENTRY, concept.id):
return True
# it may be a concept that references a set
if not concept.metadata.is_evaluated:
with context.push(desc=f"Evaluating concept {concept}") as sub_context:
sub_context.local_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
evaluated = self.sheerka.evaluate_concept(sub_context, concept)
if evaluated.key != concept.key:
return False
concept = core.builtin_helpers.ensure_evaluated(context, concept)
if not context.sheerka.is_success(concept):
return False
if concept.body:
return self.isaset(context, concept.body)
res = self.sheerka.sdp.get_safe(GROUP_PREFIX + concept.id)
return res is not None
return self.isaset(context, concept.body)
def _validate_where_clause(self, concept):
python_parser_result = [r for r in concept.compiled[ConceptParts.WHERE] if r.who == "parsers.Python"]
@@ -190,7 +194,7 @@ class SheerkaSetsManager:
if len(names) != 1 or names[0] != concept.metadata.body:
return None
condition = concept.metadata.where.replace(concept.metadata.body, "sheerka.value(x)")
condition = concept.metadata.where.replace(concept.metadata.body, "sheerka.objvalue(x)")
expression = f"""
result=[]
for x in xx__concepts__xx:
@@ -218,10 +222,11 @@ for x in xx__concepts__xx:
result = []
with context.push(desc=f"Evaluating concepts of a set") as sub_context:
sub_context.add_inputs(ids=ids)
sub_context.local_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
for element_id in ids:
concept = self.sheerka.get_by_id(element_id)
evaluated = self.sheerka.evaluate_concept(sub_context, concept)
result.append(evaluated)
sub_context.add_inputs(return_value=result)
return result
src/core/sheerka/Services/SheerkaVariableManager.py
+11 -17
View File
@@ -20,35 +20,29 @@ class Variable:
class SheerkaVariableManager:
VARIABLES_ENTRY = "All_Variables" # to store all the concepts
def __init__(self, sheerka):
self.sheerka = sheerka
def record(self, context, who, key, value):
"""Persist a variable"""
# first check if there is a previous version of the variable
try:
old = self.sheerka.sdp.get(self.VARIABLES_ENTRY, who + "." + key)
if old.value == value:
return
"""
parent = getattr(old, Serializer.ORIGIN)
except IndexError:
parent = None
:param context:
:param who: entity that owns the key (acts as a namespace)
:param key:
:param value:
:return:
"""
variable = Variable(context.event.get_digest(), who, key, value, [parent] if parent else None)
self.sheerka.sdp.set(context.event.get_digest(), self.VARIABLES_ENTRY, variable, use_ref=True)
variable = Variable(context.event.get_digest(), who, key, value, None)
self.sheerka.cache_manager.put(self.sheerka.VARIABLES_ENTRY, variable.get_key(), variable)
def load(self, who, key):
variable = self.sheerka.sdp.get_safe(self.VARIABLES_ENTRY, who + "." + key)
variable = self.sheerka.cache_manager.get(self.sheerka.VARIABLES_ENTRY, who + "." + key)
if variable is None:
return None
return variable.value
def delete(self, context, who, key):
self.sheerka.sdp.remove(
context.event.get_digest(),
self.VARIABLES_ENTRY,
lambda _key, _var: _key == who + "." + key)
self.sheerka.cache_manager.delete(self.sheerka.VARIABLES_ENTRY, who + "." + key)
src/core/sheerka/Sheerka.py
+294 -229
View File
@@ -2,6 +2,12 @@ import logging
import core.builtin_helpers
import core.utils
from cache.Cache import Cache
from cache.CacheManager import CacheManager
from cache.DictionaryCache import DictionaryCache
from cache.IncCache import IncCache
from cache.ListIfNeededCache import ListIfNeededCache
from cache.SetCache import SetCache
from core.builtin_concepts import BuiltinConcepts, ErrorConcept, ReturnValueConcept, BuiltinErrors, BuiltinUnique, \
UnknownConcept
from core.concept import Concept, ConceptParts, PROPERTIES_FOR_NEW
@@ -18,8 +24,7 @@ from core.sheerka_logger import console_handler
from printer.SheerkaPrinter import SheerkaPrinter
from sdp.sheerkaDataProvider import SheerkaDataProvider, Event
CONCEPT_LEXER_PARSER_CLASS = "parsers.BnfNodeParser.BnfNodeParser"
BNF_PARSER_CLASS = "parsers.BnfParser.BnfParser"
BASE_NODE_PARSER_CLASS = "parsers.BaseNodeParser.BaseNodeParser"
CONCEPTS_FILE = "_concepts.txt"
@@ -28,45 +33,36 @@ class Sheerka(Concept):
Main controller for the project
"""
CONCEPTS_ENTRY = "All_Concepts" # to store all the concepts
CONCEPTS_BY_ID_ENTRY = "Concepts_By_ID"
CONCEPTS_BY_ID_ENTRY = "Concepts_By_ID" # to store all the concepts
CONCEPTS_BY_KEY_ENTRY = "Concepts_By_Key"
CONCEPTS_BY_NAME_ENTRY = "Concepts_By_Name"
CONCEPTS_BY_HASH_ENTRY = "Concepts_By_Hash" # store hash of concepts definitions (not values)
CONCEPTS_DEFINITIONS_ENTRY = "Concepts_Definitions" # to store definitions (bnf) of concepts
CONCEPTS_BY_FIRST_KEYWORD_ENTRY = "Concepts_By_First_Keyword"
CONCEPTS_SYA_DEFINITION_ENTRY = "Concepts_Sya_Definitions"
CONCEPTS_BY_FIRST_KEYWORD_ENTRY = "Concepts_By_First_Keyword"
RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY = "Resolved_Concepts_By_First_Keyword"
CONCEPTS_SYA_DEFINITION_ENTRY = "Concepts_Sya_Definitions"
RESOLVED_CONCEPTS_SYA_DEFINITION_ENTRY = "Resolved_Concepts_Sya_Definitions"
CONCEPTS_GRAMMARS_ENTRY = "Concepts_Grammars"
CONCEPTS_GROUPS_ENTRY = "Concepts_Groups"
VARIABLES_ENTRY = "Variables" # entry for admin or internal variables
CONCEPTS_KEYS_ENTRY = "Concepts_Keys"
BUILTIN_CONCEPTS_KEYS = "Builtins_Concepts" # sequential key for builtin concepts
USER_CONCEPTS_KEYS = "User_Concepts" # sequential key for user defined concepts
def __init__(self, skip_builtins_in_db=False, debug=False, loggers=None):
def __init__(self, cache_only=False, debug=False, loggers=None):
self.init_logging(debug, loggers)
self.loggers = loggers
super().__init__(BuiltinConcepts.SHEERKA, True, True, BuiltinConcepts.SHEERKA)
self.log.debug("Starting Sheerka.")
# cache of the most used concepts
# Note that these are only templates
# They are used as a footprint for instantiation
# Except of source when the concept is supposed to be unique
# key is the key of the concept (not the name or the id)
self.cache_by_key = {}
self.cache_by_id = {}
self.cache_by_name = {}
self.bnp = None # reference to the BaseNodeParser class (to compute first keyword token)
# cache for concept definitions,
# Primarily used for unit test that does not have access to sdp
self.concepts_definitions_cache = {}
#
# cache for concepts grammars
# a grammar is a resolved BNF
self.concepts_grammars = {}
# cache for SYA concepts
self.concepts_by_first_keyword = {}
self.sya_definitions = {}
# # Cache for concepts grammars
# # To be shared between BNFNode parsers instances
# self.concepts_grammars = {}
# a concept can be instantiated
# ex: File is a concept, but File('foo.txt') is an instance
@@ -78,6 +74,8 @@ class Sheerka(Concept):
self.rules = []
self.sdp: SheerkaDataProvider = None # SheerkaDataProvider
self.cache_manager = CacheManager(cache_only)
self.builtin_cache = {} # cache for builtin concepts
self.parsers = {} # cache for builtin parsers
self.evaluators = [] # cache for builtin evaluators
@@ -85,8 +83,6 @@ class Sheerka(Concept):
self.evaluators_prefix: str = None
self.parsers_prefix: str = None
self.skip_builtins_in_db = skip_builtins_in_db
self.execute_handler = SheerkaExecute(self)
self.create_new_concept_handler = SheerkaCreateNewConcept(self)
self.modify_concept_handler = SheerkaModifyConcept(self)
@@ -100,36 +96,65 @@ class Sheerka(Concept):
self.during_restore = False
self._builtins_classes_cache = None
def initialize(self, root_folder: str = None):
self.save_execution_context = True
@property
def resolved_concepts_by_first_keyword(self):
"""
We return the cache as we will be interested by statistics
:return:
"""
return self.cache_manager.caches[self.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY].cache
@property
def resolved_sya_def(self):
"""
:return:
"""
return self.cache_manager.caches[self.RESOLVED_CONCEPTS_SYA_DEFINITION_ENTRY].cache
@property
def concepts_grammars(self):
return self.cache_manager.caches[self.CONCEPTS_GRAMMARS_ENTRY].cache
def initialize(self, root_folder: str = None, save_execution_context=True):
"""
Starting Sheerka
Loads the current configuration
Notes that when it's the first time, it also create the needed working folders
:param root_folder: root configuration folder
:param save_execution_context:
:return: ReturnValue(Success or Error)
"""
self.save_execution_context = save_execution_context
try:
from sheerkapickle.sheerka_handlers import initialize_pickle_handlers
initialize_pickle_handlers()
self.sdp = SheerkaDataProvider(root_folder, self)
if self.sdp.first_time:
self.sdp.set_key(self.USER_CONCEPTS_KEYS, 1000)
self.initialize_caching()
event = Event("Initializing Sheerka.", user=self.name)
event = Event("Initializing Sheerka.", user_id=self.name)
self.sdp.save_event(event)
with ExecutionContext(self.key, event, self, "Initializing Sheerka.", self.init_log) as exec_context:
if self.sdp.first_time:
self.first_time_initialisation(exec_context)
self.initialize_builtin_concepts()
self.initialize_builtin_parsers()
self.initialize_builtin_evaluators()
self.initialize_bnf_parsing(exec_context)
self.initialize_sya_parsing()
self.initialize_builtin_concepts()
self.initialize_concept_node_parsing(exec_context)
res = ReturnValueConcept(self, True, self)
exec_context.add_values(return_values=res)
if not self.skip_builtins_in_db:
if self.cache_manager.is_dirty:
self.cache_manager.commit(exec_context)
if save_execution_context:
self.sdp.save_result(exec_context, is_admin=True)
self.init_log.debug(f"Sheerka successfully initialized")
@@ -138,6 +163,59 @@ class Sheerka(Concept):
return res
def initialize_caching(self):
def params(cache_name):
return {
'default': lambda k: self.sdp.get(cache_name, k),
'extend_exists': lambda k: self.sdp.exists(cache_name, k)
}
cache = IncCache(default=lambda k: self.sdp.get(self.CONCEPTS_KEYS_ENTRY, k))
self.cache_manager.register_cache(self.CONCEPTS_KEYS_ENTRY, cache)
register_concept_cache = self.cache_manager.register_concept_cache
cache = Cache(**params(self.CONCEPTS_BY_ID_ENTRY))
register_concept_cache(self.CONCEPTS_BY_ID_ENTRY, cache, lambda c: c.id, True)
cache = ListIfNeededCache(**params(self.CONCEPTS_BY_KEY_ENTRY))
register_concept_cache(self.CONCEPTS_BY_KEY_ENTRY, cache, lambda c: c.key, True)
cache = ListIfNeededCache(**params(self.CONCEPTS_BY_NAME_ENTRY))
register_concept_cache(self.CONCEPTS_BY_NAME_ENTRY, cache, lambda c: c.name, True)
cache = ListIfNeededCache(**params(self.CONCEPTS_BY_HASH_ENTRY))
register_concept_cache(self.CONCEPTS_BY_HASH_ENTRY, cache, lambda c: c.get_definition_hash(), True)
cache = DictionaryCache(default=lambda k: self.sdp.get(self.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, k))
self.cache_manager.register_cache(self.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, cache)
self.cache_manager.get(self.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, None) # to init from sdp
cache = DictionaryCache(default=lambda k: self.sdp.get(self.CONCEPTS_SYA_DEFINITION_ENTRY, k))
self.cache_manager.register_cache(self.CONCEPTS_SYA_DEFINITION_ENTRY, cache)
self.cache_manager.get(self.CONCEPTS_SYA_DEFINITION_ENTRY, None) # to init from sdp
cache = SetCache(default=lambda k: self.sdp.get(self.CONCEPTS_GROUPS_ENTRY, k))
self.cache_manager.register_cache(self.CONCEPTS_GROUPS_ENTRY, cache)
cache = Cache(default=lambda k: self.sdp.get(self.VARIABLES_ENTRY, k))
self.cache_manager.register_cache(self.VARIABLES_ENTRY, cache, True, True)
cache = DictionaryCache()
self.cache_manager.register_cache(self.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY, cache, persist=False)
cache = DictionaryCache()
self.cache_manager.register_cache(self.RESOLVED_CONCEPTS_SYA_DEFINITION_ENTRY, cache, persist=False)
cache = Cache()
self.cache_manager.register_cache(self.CONCEPTS_GRAMMARS_ENTRY, cache, persist=False)
def first_time_initialisation(self, context):
self.cache_manager.put(self.CONCEPTS_KEYS_ENTRY, self.USER_CONCEPTS_KEYS, 1000)
self.variable_handler.record(context, self.name, "save_execution_context", True)
def initialize_builtin_concepts(self):
"""
Initializes the builtin concepts
@@ -160,18 +238,16 @@ class Sheerka(Concept):
if not concept.metadata.is_unique and str(key) in builtins_classes:
self.builtin_cache[key] = builtins_classes[str(key)]
if not self.skip_builtins_in_db:
from_db = self.sdp.get_safe(self.CONCEPTS_ENTRY, concept.metadata.key)
if from_db is None:
self.init_log.debug(f"'{concept.name}' concept is not found in db. Adding.")
self.set_id_if_needed(concept, True)
concept.metadata.full_serialization = True
self.sdp.add("init", self.CONCEPTS_ENTRY, concept, use_ref=True)
else:
self.init_log.debug(f"Found concept '{from_db}' in db. Updating.")
concept.update_from(from_db)
from_db = self.cache_manager.get(self.CONCEPTS_BY_KEY_ENTRY, concept.metadata.key)
if from_db is None:
self.init_log.debug(f"'{concept.name}' concept is not found in db. Adding.")
self.set_id_if_needed(concept, True)
self.cache_manager.add_concept(concept)
else:
self.init_log.debug(f"Found concept '{from_db}' in db. Updating.")
concept.update_from(from_db)
self.add_in_cache(concept)
return
def initialize_builtin_parsers(self):
"""
@@ -187,17 +263,23 @@ class Sheerka(Concept):
if parser.__module__ == base_class.__module__:
continue
if parser.__module__ in modules_to_skip:
continue
qualified_name = core.utils.get_full_qualified_name(parser)
self.init_log.debug(f"Adding builtin parser '{qualified_name}'")
temp_result[qualified_name] = parser
# keep a reference to base_node_parser
self.bnp = temp_result[BASE_NODE_PARSER_CLASS]
# Now we sort the parser by name.
# It's not important for the logic of their usage as they have their priority anyway,
# We do that for the unit tests. They are to complicated to write otherwise
for name in sorted(temp_result.keys()):
parser = temp_result[name]
if parser.__module__ in modules_to_skip:
# base node parser module does not contains any valid parser
continue
self.parsers[name] = temp_result[name]
def initialize_builtin_evaluators(self):
@@ -214,55 +296,39 @@ class Sheerka(Concept):
self.init_log.debug(f"Adding builtin evaluator '{evaluator.__name__}'")
self.evaluators.append(evaluator)
def initialize_bnf_parsing(self, execution_context):
self.init_log.debug("Initializing concepts grammars.")
definitions = self.get_concepts_definitions(execution_context)
def initialize_concept_node_parsing(self, context):
self.init_log.debug("Initializing concept node parsing.")
if definitions is None:
self.init_log.debug("No BNF defined")
return
concepts_by_first_keyword = self.cache_manager.copy(self.CONCEPTS_BY_FIRST_KEYWORD_ENTRY)
res = self.bnp.resolve_concepts_by_first_keyword(context, concepts_by_first_keyword)
self.cache_manager.put(self.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY, False, res.body)
lexer_parser = self.parsers[CONCEPT_LEXER_PARSER_CLASS]()
ret_val = lexer_parser.initialize(execution_context, definitions)
if not ret_val.status:
self.init_log.error("Failed to initialize concepts definitions " + str(ret_val.body))
return
# sya = self.bnf.resolve_sya_associativity_and_precedence()
# self.cache_manager.put(self.RESOLVED_CONCEPTS_SYA_DEFINITION_ENTRY, sya)
#
#
# self.concepts_by_first_keyword, \
# self.resolved_concepts_by_first_keyword = \
# self.create_new_concept_handler.load_concepts_nodes_definitions(context)
self.concepts_grammars = lexer_parser.concepts_grammars
# self.concepts_by_first_keyword = self.sdp.get_safe(
# self.CONCEPTS_BY_FIRST_KEYWORD_ENTRY,
# load_origin=False) or {}
#
# self.sya_definitions = self.sdp.get_safe(
# self.CONCEPTS_SYA_DEFINITION_ENTRY,
# load_origin=False) or {}
#
# init_ret_value = self.bnp.resolve_concepts_by_first_keyword(self, self.concepts_by_first_keyword)
# if not init_ret_value.status:
# return self.sheerka.ret(self.logger_name, False, ErrorConcept(init_ret_value.value))
# self.resolved_concepts_by_first_keyword = init_ret_value.body
def initialize_sya_parsing(self):
self.init_log.debug("Initializing sya definitions.")
self.concepts_by_first_keyword = self.sdp.get_safe(
self.CONCEPTS_BY_FIRST_KEYWORD_ENTRY,
load_origin=False) or {}
self.sya_definitions = self.sdp.get_safe(
self.CONCEPTS_SYA_DEFINITION_ENTRY,
load_origin=False) or {}
def reset(self):
self.reset_cache()
self.concepts_by_first_keyword = {}
self.concepts_grammars = {}
self.sya_definitions = {}
def reset(self, cache_only=False):
self.cache_manager.clear()
self.cache_manager.cache_only = cache_only
self.printer_handler.reset()
self.sdp.reset()
self.sdp.set_key(self.USER_CONCEPTS_KEYS, 1000)
def reset_cache(self, filter_to_use=None):
"""
reset the different cache that exists
:param filter_to_use:
:return:
"""
if filter_to_use is None:
self.cache_by_key = {}
self.cache_by_id = {}
self.cache_by_name = {}
else:
raise NotImplementedError()
return self
def evaluate_user_input(self, text: str, user_name="kodjo"):
"""
@@ -294,7 +360,10 @@ class Sheerka(Concept):
ret = self.execute(execution_context, [user_input, reduce_requested], steps)
execution_context.add_values(return_values=ret)
if not self.skip_builtins_in_db:
if self.cache_manager.is_dirty:
self.cache_manager.commit(execution_context)
if self.save_execution_context and self.variable_handler.load(self.name, "save_execution_context"):
self.sdp.save_result(execution_context)
# # hack to save valid concept definition
@@ -302,6 +371,8 @@ class Sheerka(Concept):
# if len(ret) == 1 and ret[0].status and self.isinstance(ret[0].value, BuiltinConcepts.NEW_CONCEPT):
# with open(CONCEPTS_FILE, "a") as f:
# f.write(text + "\n")
self._last_execution = execution_context
return ret
def print(self, result, instructions=None):
@@ -343,8 +414,8 @@ class Sheerka(Concept):
if obj.metadata.id is not None:
return
entry = self.BUILTIN_CONCEPTS_KEYS if is_builtin else self.USER_CONCEPTS_KEYS
obj.metadata.id = self.sdp.get_next_key(entry)
key = self.BUILTIN_CONCEPTS_KEYS if is_builtin else self.USER_CONCEPTS_KEYS
obj.metadata.id = str(self.cache_manager.get(self.CONCEPTS_KEYS_ENTRY, key))
self.log.debug(f"Setting id '{obj.metadata.id}' to concept '{obj.metadata.name}'.")
def create_new_concept(self, context, concept: Concept):
@@ -380,21 +451,25 @@ class Sheerka(Concept):
"""
return self.sets_handler.set_isa(context, concept, concept_set)
def set_sya_def(self, context, list_of_def):
def force_sya_def(self, context, list_of_def):
"""
Set the precedence and/or the associativity of a concept
FOR TESTS PURPOSE. TO REMOVE EVENTUALLY
:param context:
:param list_of_def list of tuple(concept_id, precedence (int), SyaAssociativity)
:return:
"""
# validate the entries
# If one entry is an invalid concept, rollback everything
for concept_id, precedence, associativity in list_of_def:
if concept_id == BuiltinConcepts.UNKNOWN_CONCEPT:
return self.ret(self.name,
False,
self.new(BuiltinConcepts.ERROR, body=f"Concept {concept_id} is not known"))
sya_def = self.cache_manager.copy(self.RESOLVED_CONCEPTS_SYA_DEFINITION_ENTRY) or {}
# update the definitions
for concept_id, precedence, associativity in list_of_def:
if precedence is None and associativity is None:
@@ -403,12 +478,10 @@ class Sheerka(Concept):
except KeyError:
pass
else:
self.sya_definitions[concept_id] = (precedence, associativity.value)
sya_def[concept_id] = (precedence, associativity)
# then save
self.sdp.set(context.event.get_digest(),
self.CONCEPTS_SYA_DEFINITION_ENTRY,
self.sya_definitions)
# put in cache
self.cache_manager.put(self.RESOLVED_CONCEPTS_SYA_DEFINITION_ENTRY, False, sya_def)
return self.ret(self.name, True, self.new(BuiltinConcepts.SUCCESS))
@@ -448,122 +521,108 @@ class Sheerka(Concept):
if concept.key is None:
raise KeyError()
self.cache_by_key[concept.key] = concept
if concept.id:
self.cache_by_id[concept.id] = concept
self.cache_manager.add_concept(concept)
return concept
def get(self, concept_key, concept_id=None):
#
# def get(self, concept_key, concept_id=None):
# """
# Tries to find a concept
# What is return must be used a template for another concept.
# You must not modify the returned concept
# :param concept_key: key of the concept
# :param concept_id: when multiple concepts with the same key, use the id
# :return:
# """
#
# by_key = self.get_by_key(concept_key)
# if self.is_known(by_key):
# return by_key
#
# # else return by name
# by_name = self.get_by_name(concept_key)
# if self.is_known(by_name):
# return by_name
#
# return by_key # return not found for key
def get_by_key(self, concept_key, concept_id=None):
concept_key = str(concept_key) if isinstance(concept_key, BuiltinConcepts) else concept_key
return self.internal_get("key", concept_key, self.CONCEPTS_BY_KEY_ENTRY, concept_id)
def get_by_name(self, concept_name, concept_id=None):
return self.internal_get("name", concept_name, self.CONCEPTS_BY_NAME_ENTRY, concept_id)
def get_by_hash(self, concept_hash, concept_id=None):
return self.internal_get("hash", concept_hash, self.CONCEPTS_BY_HASH_ENTRY, concept_id)
def get_by_id(self, concept_id):
return self.internal_get("id", concept_id, self.CONCEPTS_BY_ID_ENTRY, None)
def internal_get(self, index_name, key, cache_name, concept_id=None):
"""
Tries to find a concept
What is return must be used a template for another concept.
You must not modify the returned concept
:param concept_key: key of the concept
:param concept_id: when multiple concepts with the same key, use the id
Tries to find an entry
:param index_name: name of the index (ex by_id, by_key...)
:param key: index value
:param cache_name: name of the cache (ex Concepts_By_ID...)
:param concept_id: id of the concept if none, in case where there are multiple results
:return:
"""
by_key = self.internal_get("key", concept_key, self.cache_by_key, self.CONCEPTS_ENTRY, concept_id)
if self.is_known(by_key):
return by_key
if key is None:
return ErrorConcept(f"Concept '{key}' is undefined.")
# else return by name
by_name = self.internal_get("name", concept_key, self.cache_by_name, self.CONCEPTS_BY_NAME_ENTRY, concept_id)
if self.is_known(by_name):
return by_name
concepts = self.cache_manager.get(cache_name, key)
if concepts:
if concept_id is None:
return concepts
return by_key # return not found for key
if not hasattr(concepts, "__iter__"):
return concepts
def get_by_key(self, concept_key, concept_id=None):
return self.internal_get("key", concept_key, self.cache_by_key, self.CONCEPTS_ENTRY, concept_id)
for c in concepts:
if c.id == concept_id:
return c
def get_by_name(self, concept_name, concept_id=None):
return self.internal_get("name", concept_name, self.cache_by_name, self.CONCEPTS_BY_NAME_ENTRY, concept_id)
metadata = [(index_name, key), ("id", concept_id)] if concept_id else (index_name, key)
return self._get_unknown(metadata)
def get_by_id(self, concept_id):
if concept_id is None:
return ErrorConcept("Concept id is undefined.")
# first search in cache
if concept_id in self.cache_by_id:
result = self.cache_by_id[concept_id]
else:
result = self.sdp.get_safe(self.CONCEPTS_BY_ID_ENTRY, concept_id)
if result is None:
result = self._get_unknown(('id', concept_id))
else:
self.cache_by_id[concept_id] = result
return result
def internal_get(self, index_name, index_value, cache_to_use, sdp_entry, concept_id=None):
def has_id(self, concept_id):
"""
Tries to find an entry
:param index_name:
:param index_value:
:param cache_to_use:
:param sdp_entry:
Returns True if a concept with this id exists in cache
It does not search in the remote repository
:param concept_id:
:return:
"""
return self.cache_manager.has(self.CONCEPTS_BY_ID_ENTRY, concept_id)
if index_value is None:
return ErrorConcept(f"Concept {index_name} is undefined.")
def has_key(self, concept_key):
"""
Returns True if concept(s) with this key exist in cache
It does not search in the remote repository
:param concept_key:
:return:
"""
return self.cache_manager.has(self.CONCEPTS_BY_KEY_ENTRY, concept_key)
if isinstance(index_value, BuiltinConcepts):
index_value = str(index_value)
def has_name(self, concept_name):
"""
Returns True if concept(s) with this name exist in cache
It does not search in the remote repository
:param concept_name:
:return:
"""
return self.cache_manager.has(self.CONCEPTS_BY_NAME_ENTRY, concept_name)
# first search in cache
if index_value in cache_to_use:
result = cache_to_use[index_value]
else:
result = self.sdp.get_safe(sdp_entry, index_value)
if result is None:
metadata = [(index_name, index_value), ("id", concept_id)] if concept_id else (index_name, index_value)
result = self._get_unknown(metadata)
# Do not put in cache_by_key or cache_by_id unknown concept
# TODO: implement an MRU cache for them
else:
cache_to_use[index_value] = result
for r in (result if isinstance(result, list) else [result]):
if r.id:
self.cache_by_id[r.id] = r
if not (isinstance(result, list) and concept_id):
return result
# result is a list, but we have the concept_id to discriminate
for c in result:
if c.id == concept_id:
return c
metadata = [(index_name, index_value), ("id", concept_id)] if concept_id else (index_name, index_value)
return self._get_unknown(metadata)
def get_concepts_definitions(self, context):
if self.concepts_definitions_cache:
return self.concepts_definitions_cache
encoded_bnf = self.sdp.get_safe(
self.CONCEPTS_DEFINITIONS_ENTRY,
load_origin=False) or {}
self.concepts_definitions_cache = {}
bnf_parser = self.parsers[BNF_PARSER_CLASS]()
for k, v in encoded_bnf.items():
key, id_ = core.utils.unstr_concept(k)
concept = self.new((key, id_))
context.log(f"Parsing BNF definition for {concept}", context.who)
rule_result = bnf_parser.parse(context, v)
if rule_result.status:
self.concepts_definitions_cache[concept] = rule_result.value.value
else:
self.log.error(f"Failed to load bnf rule for concept {key}")
return self.concepts_definitions_cache
def has_hash(self, concept_hash):
"""
Returns True if concept(s) with this hash exist in cache
It does not search in the remote repository
:param concept_hash:
:return:
"""
return self.cache_manager.has(self.CONCEPTS_BY_HASH_ENTRY, concept_hash)
def new(self, concept_key, **kwargs):
"""
@@ -578,7 +637,7 @@ class Sheerka(Concept):
else:
concept_id = None
template = self.get_by_id(concept_id) if not concept_key else self.get(concept_key, concept_id)
template = self.get_by_id(concept_id) if not concept_key else self.get_by_key(concept_key, concept_id)
# manage concept not found
if self.isinstance(template, BuiltinConcepts.UNKNOWN_CONCEPT) and \
@@ -599,7 +658,7 @@ class Sheerka(Concept):
# otherwise, create another instance
concept = self.builtin_cache[key]() if key in self.builtin_cache else Concept()
concept.update_from(template)
concept.update_from(template, update_value=False)
concept.freeze_definition_hash()
if len(kwargs) == 0:
@@ -608,10 +667,10 @@ class Sheerka(Concept):
# update the properties, values, attributes
# Not quite sure that this is the correct process order
for k, v in kwargs.items():
if k in concept.props:
concept.set_prop(k, v)
if k in concept.values:
concept.set_value(k, v)
elif k in PROPERTIES_FOR_NEW:
concept.values[ConceptParts(k)] = v
concept.set_value(ConceptParts(k), v)
elif hasattr(concept, k):
setattr(concept, k, v)
else:
@@ -639,12 +698,12 @@ class Sheerka(Concept):
message=message,
parents=parents)
def value(self, obj, reduce_simple_list=False):
def objvalue(self, obj, reduce_simple_list=False):
if obj is None:
return None
if hasattr(obj, "get_value"):
return obj.get_value()
if hasattr(obj, "get_obj_value"):
return obj.get_obj_value()
if not isinstance(obj, Concept):
return obj
@@ -657,7 +716,18 @@ class Sheerka(Concept):
else:
body_to_use = obj.body
return self.value(body_to_use)
return self.objvalue(body_to_use)
def objvalues(self, objs):
if not (isinstance(objs, list) or
self.isinstance(objs, BuiltinConcepts.LIST) or
self.isinstance(objs, BuiltinConcepts.ENUMERATION)):
objs = [objs]
if isinstance(objs, list):
return (self.objvalue(obj) for obj in objs)
return (self.objvalue(obj) for obj in objs.body)
def value_by_concept(self, obj, concept):
if obj is None:
@@ -678,8 +748,8 @@ class Sheerka(Concept):
if isinstance(obj, Concept) and obj.metadata.is_builtin and obj.key in BuiltinErrors:
return obj
if isinstance(obj, list):
return obj
if isinstance(obj, (list, set, tuple)):
return [self.get_error(o) for o in obj]
if self.isinstance(obj, BuiltinConcepts.RETURN_VALUE):
if obj.status:
@@ -687,19 +757,10 @@ class Sheerka(Concept):
if self.isinstance(obj.body, BuiltinConcepts.PARSER_RESULT):
return self.get_error(obj.body.body)
else:
return obj.body
return NotImplementedError()
def get_values(self, objs):
if not (isinstance(objs, list) or
self.isinstance(objs, BuiltinConcepts.LIST) or
self.isinstance(objs, BuiltinConcepts.ENUMERATION)):
objs = [objs]
if isinstance(objs, list):
return (self.value(obj) for obj in objs)
return (self.value(obj) for obj in objs.body)
raise NotImplementedError()
def is_success(self, obj):
if isinstance(obj, bool): # quick win
@@ -761,8 +822,12 @@ class Sheerka(Concept):
return self.parsers_prefix + name
def concepts(self):
"""
List of all known concepts (look up in sdp)
:return:
"""
res = []
lst = self.sdp.list(self.CONCEPTS_ENTRY)
lst = self.sdp.list(self.CONCEPTS_BY_ID_ENTRY)
for item in lst:
if isinstance(item, list):
res.extend(item)
@@ -818,10 +883,10 @@ class Sheerka(Concept):
# the metadata can be a list, if several attributes where given
# (key, 'not_found), (id, invalid_id)
unknown_concept = UnknownConcept()
unknown_concept.set_metadata_value(ConceptParts.BODY, metadata)
unknown_concept = UnknownConcept() # don't use new() for prevent circular reference
unknown_concept.set_value(ConceptParts.BODY, metadata)
for meta in (metadata if isinstance(metadata, list) else [metadata]):
unknown_concept.set_prop(meta[0], meta[1])
unknown_concept.set_value(meta[0], meta[1])
unknown_concept.metadata.is_evaluated = True
return unknown_concept
src/core/utils.py
+4 -2
View File
@@ -261,7 +261,7 @@ def decode_enum(enum_repr: str):
return None
def str_concept(t):
def str_concept(t, skip_key=None):
"""
The key,id identifiers of a concept are stored in a tuple
we want to return the key and the id, separated by a pipe
@@ -272,7 +272,9 @@ def str_concept(t):
>>> assert str_concept(("key", None)) == "c:key:"
>>> assert str_concept((None, None)) == ""
>>> assert str_concept(Concept(key="foo", id="bar")) == "c:foo|bar:"
>>> assert str_concept(Concept(key="foo", id="bar"), skip_key=True) == "c:|bar:"
:param t:
:param skip_key: True if we only want the id (and not the key)
:return:
"""
if isinstance(t, tuple):
@@ -283,7 +285,7 @@ def str_concept(t):
if key is None and id_ is None:
return ""
result = 'c:' if key is None else "c:" + key
result = 'c:' if (key is None or skip_key) else "c:" + key
if id_:
result += "|" + id_
return result + ":"
src/evaluators/AddConceptEvaluator.py
+8 -8
View File
@@ -79,18 +79,18 @@ class AddConceptEvaluator(OneReturnValueEvaluator):
continue
# try to find what can be a property
for p in self.get_props(sheerka, part_ret_val, name_to_use):
for p in self.get_variables(sheerka, part_ret_val, name_to_use):
props_found.add(p)
# add props by order of appearance when possible
# add variables by order of appearance when possible
for token in def_concept_node.name.tokens:
if token.value in props_found:
concept.def_prop(token.value, None)
concept.def_var(token.value, None)
# add the remaining properties
for p in props_found:
if p not in concept.props:
concept.def_prop(p, None)
if p not in concept.values:
concept.def_var(p, None)
# initialize the key
key_source = def_concept_node.definition.tokens if \
@@ -105,7 +105,7 @@ class AddConceptEvaluator(OneReturnValueEvaluator):
ret = sheerka.create_new_concept(context, concept)
if not ret.status:
error_cause = sheerka.value(ret.body)
error_cause = sheerka.objvalue(ret.body)
context.log(f"Failed to add concept '{concept.name}'. Reason: {error_cause}", self.name)
return sheerka.ret(self.name, ret.status, ret.value, parents=[return_value])
@@ -115,7 +115,7 @@ class AddConceptEvaluator(OneReturnValueEvaluator):
return [part.value for part in core.utils.strip_tokens(source.tokens, True)]
@staticmethod
def get_props(sheerka, ret_value, concept_name):
def get_variables(sheerka, ret_value, concept_name):
"""
Try to find out the variables
This function can only be a draft, as there may be tons of different situations
@@ -146,7 +146,7 @@ class AddConceptEvaluator(OneReturnValueEvaluator):
# case of concept
#
if isinstance(ret_value.value, ParserResultConcept) and isinstance(ret_value.value.value, Concept):
return list(ret_value.value.value.props.keys())
return list(ret_value.value.value.values.keys())
#
# case of BNF
src/evaluators/AddConceptInSetEvaluator.py
+1 -1
View File
@@ -64,7 +64,7 @@ class AddConceptInSetEvaluator(OneReturnValueEvaluator):
res = sheerka.set_isa(context, concept, concept_set)
if not res.status:
context.log(f"Failed. Reason: {sheerka.value(res.body)}.", self.name)
context.log(f"Failed. Reason: {sheerka.objvalue(res.body)}.", self.name)
else:
context.log(f"Concept added.", self.name)
src/evaluators/ConceptEvaluator.py
+2 -2
View File
@@ -41,8 +41,8 @@ class ConceptEvaluator(OneReturnValueEvaluator):
# Why ?
# If we evaluate Concept("foo", body="a").set_prop("a", "'property_a'")
# The body should be 'property_a', and not a concept called 'a'
if context.obj and concept.name in context.obj.props:
value = context.obj.props[concept.name].value
if context.obj and concept.name in context.obj.values:
value = context.obj.get_value(concept.name)
context.log(f"{concept.name} is a property. Returning value '{value}'.", self.name)
return sheerka.ret(self.name, True, value, parents=[return_value])
src/evaluators/PrepareEvalEvaluator.py
+1 -1
View File
@@ -31,7 +31,7 @@ class PrepareEvalEvaluator(OneReturnValueEvaluator):
new_text_to_parse = sheerka.ret(
self.name,
True, sheerka.new(BuiltinConcepts.USER_INPUT, body=self.text[5:], user_name=context.event.user))
True, sheerka.new(BuiltinConcepts.USER_INPUT, body=self.text[5:], user_name=context.event.user_id))
context.global_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
context.global_hints.add(BuiltinConcepts.CONCEPT_VALUE_REQUESTED)
src/evaluators/PythonEvaluator.py
+10 -12
View File
@@ -1,15 +1,14 @@
import ast
import copy
import traceback
from enum import Enum
import core.ast.nodes
import core.utils
from core.ast.visitors import UnreferencedNamesVisitor
from core.builtin_concepts import BuiltinConcepts, ParserResultConcept
from core.concept import ConceptParts, Concept
from evaluators.BaseEvaluator import OneReturnValueEvaluator
from parsers.PythonParser import PythonNode
import ast
import core.ast.nodes
import core.utils
class PythonEvaluator(OneReturnValueEvaluator):
@@ -36,7 +35,7 @@ class PythonEvaluator(OneReturnValueEvaluator):
# Do not evaluate if the ast refers to a concept (leave it to ConceptEvaluator)
if isinstance(node.ast_, ast.Expression) and isinstance(node.ast_.body, ast.Name):
c = context.sheerka.get(node.ast_.body.id)
c = context.sheerka.get_by_key(node.ast_.body.id)
if not context.sheerka.isinstance(c, BuiltinConcepts.UNKNOWN_CONCEPT):
context.log("It's a simple concept. Not for me.", self.name)
not_for_me = context.sheerka.new(BuiltinConcepts.NOT_FOR_ME, body=node)
@@ -69,7 +68,6 @@ class PythonEvaluator(OneReturnValueEvaluator):
"sheerka": context.sheerka,
"desc": context.sheerka.dump_handler.dump_desc,
"concepts": context.sheerka.dump_handler.dump_concepts,
"definitions": context.sheerka.dump_handler.dump_definitions,
"history": context.sheerka.dump_handler.dump_history,
"state": context.sheerka.dump_handler.dump_state,
"Concept": core.concept.Concept
@@ -77,11 +75,12 @@ class PythonEvaluator(OneReturnValueEvaluator):
if context.obj:
context.log(f"Concept '{context.obj}' is in context. Adding it and its properties to locals.", self.name)
for prop_name, prop_value in context.obj.props.items():
if isinstance(prop_value.value, Concept):
my_locals[prop_name] = context.sheerka.value(prop_value.value)
for prop_name in context.obj.variables():
prop_value = context.obj.get_value(prop_name)
if isinstance(prop_value, Concept):
my_locals[prop_name] = context.sheerka.objvalue(prop_value)
else:
my_locals[prop_name] = prop_value.value
my_locals[prop_name] = prop_value
my_locals["self"] = context.obj.body
@@ -118,7 +117,7 @@ class PythonEvaluator(OneReturnValueEvaluator):
sub_context.add_values(return_values=evaluated)
if evaluated.key == concept.key:
my_locals[name] = evaluated if return_concept else context.sheerka.value(evaluated)
my_locals[name] = evaluated if return_concept else context.sheerka.objvalue(evaluated)
if self.locals: # when exta values are given. Add them
my_locals.update(self.locals)
@@ -142,7 +141,6 @@ class PythonEvaluator(OneReturnValueEvaluator):
else:
return to_resolve, None, False
@staticmethod
def expr_to_expression(expr):
expr.lineno = 0
src/parsers/AtomNodeParser.py
+36 -26
View File
@@ -1,12 +1,11 @@
import copy
from dataclasses import dataclass
from core import builtin_helpers
from core.builtin_concepts import BuiltinConcepts
from core.concept import Concept, DEFINITION_TYPE_BNF
from core.tokenizer import TokenKind, Tokenizer
from parsers.BaseNodeParser import BaseNodeParser, ConceptNode, UnrecognizedTokensNode
from parsers.BaseParser import BaseParser, UnexpectedTokenErrorNode, ErrorNode
from core.concept import DEFINITION_TYPE_BNF
from core.tokenizer import Tokenizer
from parsers.BaseNodeParser import BaseNodeParser, ConceptNode, UnrecognizedTokensNode, SourceCodeNode
from parsers.BaseParser import UnexpectedTokenErrorNode, ErrorNode
PARSERS = ["BnfNode", "SyaNode", "Python"]
@@ -141,7 +140,11 @@ class AtomConceptParserHelper:
self.unrecognized_tokens.fix_source()
# try to recognize concepts
nodes_sequences = self._get_lexer_nodes_from_unrecognized()
nodes_sequences = builtin_helpers.get_lexer_nodes_from_unrecognized(
self.context,
self.unrecognized_tokens,
PARSERS)
if nodes_sequences:
instances = [self]
for i in range(len(nodes_sequences) - 1):
@@ -152,7 +155,7 @@ class AtomConceptParserHelper:
for instance, node_sequence in zip(instances, nodes_sequences):
for node in node_sequence:
instance.sequence.append(node)
if isinstance(node, UnrecognizedTokensNode) or \
if isinstance(node, (UnrecognizedTokensNode, SourceCodeNode)) or \
hasattr(node, "unrecognized_tokens") and node.unrecognized_tokens:
instance.has_unrecognized = True
instance.unrecognized_tokens = UnrecognizedTokensNode(-1, -1, [])
@@ -193,22 +196,22 @@ class AtomConceptParserHelper:
clone.has_unrecognized = self.has_unrecognized
return clone
def _get_lexer_nodes_from_unrecognized(self):
"""
Use the source of self.unrecognized_tokens gto find concepts or source code
:return:
"""
res = builtin_helpers.parse_unrecognized(self.context, self.unrecognized_tokens.source, PARSERS)
only_parsers_results = builtin_helpers.only_parsers_results(self.context, res)
if not only_parsers_results.status:
return None
return builtin_helpers.get_lexer_nodes(
only_parsers_results.body.body,
self.unrecognized_tokens.start,
self.unrecognized_tokens.tokens)
# def _get_lexer_nodes_from_unrecognized(self):
# """
# Use the source of self.unrecognized_tokens gto find concepts or source code
# :return:
# """
#
# res = builtin_helpers.parse_unrecognized(self.context, self.unrecognized_tokens.source, PARSERS)
# only_parsers_results = builtin_helpers.only_parsers_results(self.context, res)
#
# if not only_parsers_results.status:
# return None
#
# return builtin_helpers.get_lexer_nodes(
# only_parsers_results.body.body,
# self.unrecognized_tokens.start,
# self.unrecognized_tokens.tokens)
class AtomNodeParser(BaseNodeParser):
@@ -230,7 +233,6 @@ class AtomNodeParser(BaseNodeParser):
def __init__(self, **kwargs):
super().__init__("AtomNode", 50, **kwargs)
self.enabled = False
@staticmethod
def _is_eligible(concept):
@@ -239,7 +241,8 @@ class AtomNodeParser(BaseNodeParser):
:param concept:
:return:
"""
return len(concept.metadata.props) == 0 or concept.metadata.definition_type == DEFINITION_TYPE_BNF
# return len(concept.metadata.props) == 0 or concept.metadata.definition_type == DEFINITION_TYPE_BNF
return len(concept.metadata.variables) == 0 and concept.metadata.definition_type != DEFINITION_TYPE_BNF
def get_concepts_sequences(self):
@@ -255,6 +258,13 @@ class AtomNodeParser(BaseNodeParser):
concept_parser_helpers.extend(forked)
forked.clear()
def _get_concepts_by_name(name):
other_concepts = self.sheerka.get_by_name(name)
if isinstance(other_concepts, list):
return other_concepts
return [other_concepts] if self.sheerka.is_known(other_concepts) else []
concept_parser_helpers = [AtomConceptParserHelper(self.context)]
while self.next_token(False):
@@ -268,7 +278,7 @@ class AtomNodeParser(BaseNodeParser):
if concept_parser.eat_token(self.token, self.pos):
concept_parser.lock()
concepts = self.get_concepts(token, self._is_eligible)
concepts = self.get_concepts(token, self._is_eligible, custom=_get_concepts_by_name)
if not concepts:
for concept_parser in concept_parser_helpers:
concept_parser.eat_unrecognized(token, self.pos)
src/parsers/BaseNodeParser.py
+195 -65
View File
@@ -2,8 +2,9 @@ from collections import namedtuple
from dataclasses import dataclass
from enum import Enum
import core.utils
from core.builtin_concepts import BuiltinConcepts
from core.concept import VARIABLE_PREFIX, Concept
from core.concept import VARIABLE_PREFIX, Concept, DEFINITION_TYPE_BNF, ConceptParts
from core.sheerka.ExecutionContext import ExecutionContext
from core.tokenizer import TokenKind, LexerError, Token
from parsers.BaseParser import Node, BaseParser, ErrorNode
@@ -187,6 +188,9 @@ class SourceCodeNode(LexerNode):
self.end == other.end and \
self.source == other.source
if isinstance(other, SCN):
return other == self
if not isinstance(other, SourceCodeNode):
return False
@@ -352,6 +356,51 @@ class HelperWithPos:
return self
class SCN(HelperWithPos):
"""
SourceCodeNode tester class
It matches with SourceCodeNode but with less constraints
SCN == SourceCodeNode if source, start, end (start and end are not validated when None)
"""
def __init__(self, source, start=None, end=None):
super().__init__(start, end)
self.source = source
def __eq__(self, other):
if id(self) == id(other):
return True
if isinstance(other, SourceCodeNode):
if self.source != other.source:
return False
if self.start is not None and self.start != other.start:
return False
if self.end is not None and self.end != other.end:
return False
return True
if not isinstance(other, CN):
return False
return self.source == other.source and \
self.start == other.start and \
self.end == other.end
def __hash__(self):
return hash((self.source, self.start, self.end))
def __repr__(self):
txt = f"SCN(source='{self.source}'"
if self.start is not None:
txt += f", start={self.start}"
if self.end is not None:
txt += f", end={self.end}"
return txt + ")"
class CN(HelperWithPos):
"""
ConceptNode tester class
@@ -390,6 +439,8 @@ class CN(HelperWithPos):
return False
if self.end is not None and self.end != other.end:
return False
if self.source is not None and self.source != other.source:
return False
return True
if not isinstance(other, CN):
@@ -425,9 +476,10 @@ class CNC(CN):
CNC == ConceptNode if CNC.compiled == ConceptNode.concept.compiled
"""
def __init__(self, concept_key, start=None, end=None, source=None, **kwargs):
def __init__(self, concept_key, start=None, end=None, source=None, exclude_body=False, **kwargs):
super().__init__(concept_key, start, end, source)
self.compiled = kwargs
self.exclude_body = exclude_body
def __eq__(self, other):
if id(self) == id(other):
@@ -442,7 +494,13 @@ class CNC(CN):
return False
if self.end is not None and self.end != other.end:
return False
return self.compiled == other.concept.compiled # assert instead of return to help debugging tests
if self.source is not None and self.source != other.source:
return False
if self.exclude_body:
to_compare = {k: v for k, v in other.concept.compiled.items() if k != ConceptParts.BODY}
else:
to_compare = other.concept.compiled
return self.compiled == to_compare
if not isinstance(other, CNC):
return False
@@ -518,11 +576,10 @@ class BaseNodeParser(BaseParser):
super().__init__(name, priority)
if 'sheerka' in kwargs:
sheerka = kwargs.get("sheerka")
self.init_from_sheerka(sheerka)
self.concepts_by_first_keyword = sheerka.resolved_concepts_by_first_keyword
else:
self.concepts_by_first_keyword = None
self.sya_definitions = None
self.token = None
self.pos = -1
@@ -532,17 +589,16 @@ class BaseNodeParser(BaseParser):
self.text = None
self.sheerka = None
def init_from_sheerka(self, sheerka):
def init_from_concepts(self, context, concepts, **kwargs):
"""
Use the definitons from Sheerka to initialize
:param sheerka:
Initialize the parser with a list of concepts
For unit tests convenience
:param context
:param concepts
:return:
"""
self.concepts_by_first_keyword = sheerka.concepts_by_first_keyword
if sheerka.sya_definitions:
self.sya_definitions = {}
for k, v in sheerka.sya_definitions.items():
self.sya_definitions[k] = (v[0], SyaAssociativity(v[1]))
concepts_by_first_keyword = self.get_concepts_by_first_keyword(context, concepts).body
self.concepts_by_first_keyword = self.resolve_concepts_by_first_keyword(context, concepts_by_first_keyword).body
def reset_parser(self, context, text):
self.context = context
@@ -582,82 +638,43 @@ class BaseNodeParser(BaseParser):
return self.token.type != TokenKind.EOF
def initialize(self, context, concepts, sya_definitions=None, use_sheerka=False):
"""
To quickly find a concept, we store them in an hash where the key is the first token of the concept
example :
Concept("foo a").def_prop("a"), "foo" is a token, "a" is a variable
So the key to use will be "foo"
Concept("a foo").def_prop("a") -> first token is "foo"
Concept("Hello my dear a").def_prop("a") -> first token is "Hello"
Note that under the same key, there will be multiple entry
a B-Tree may be a better implementation in the future
We also store sya_definition which a is tuple (concept_precedence:int, concept_associativity:SyaAssociativity)
:param context:
:param concepts: list[Concept]
:param sya_definitions: hash[concept_id, tuple(precedence:int, associativity:SyaAssociativity)]
:param use_sheerka: first init with the definitions from Sheerka
:return:
"""
self.context = context
self.sheerka = context.sheerka
if use_sheerka:
self.init_from_sheerka(self.sheerka)
if sya_definitions:
if self.sya_definitions:
self.sya_definitions.update(sya_definitions)
else:
self.sya_definitions = sya_definitions
if self.concepts_by_first_keyword is None:
self.concepts_by_first_keyword = {}
for concept in concepts:
keywords = concept.key.split()
for keyword in keywords:
if keyword.startswith(VARIABLE_PREFIX):
continue
self.concepts_by_first_keyword.setdefault(keyword, []).append(concept.id)
break
return self.sheerka.ret(self.name, True, self.concepts_by_first_keyword)
def get_concepts(self, token, to_keep, to_map=None):
def get_concepts(self, token, to_keep, custom=None, to_map=None, strip_quotes=False):
"""
Tries to find if there are concepts that match the value of the token
:param token:
:param to_keep: predicate to tell if the concept is eligible
:param custom: lambda name -> List[Concepts] that gives extra concepts, according to the name
:param to_map:
:param strip_quotes: Remove quotes from strings
:return:
"""
if token.type == TokenKind.WHITESPACE:
return None
if token.type == TokenKind.STRING:
name = token.value[1:-1]
name = token.value[1:-1] if strip_quotes else token.value
elif token.type == TokenKind.KEYWORD:
name = token.value.value
else:
name = token.value
custom_concepts = custom(name) if custom else []
result = []
if name in self.concepts_by_first_keyword:
for concept_id in self.concepts_by_first_keyword[name]:
for concept_id in self.concepts_by_first_keyword.get(name):
concept = self.sheerka.get_by_id(concept_id)
if not to_keep(concept):
continue
concept = to_map(concept) if to_map else concept
concept = to_map(self, concept) if to_map else concept
result.append(concept)
return result
return result + custom_concepts
return None
return custom_concepts if custom else None
@staticmethod
def get_token_value(token):
@@ -667,3 +684,116 @@ class BaseNodeParser(BaseParser):
return token.value.value
else:
return token.value
@staticmethod
def get_concepts_by_first_keyword(context, concepts, use_sheerka=False):
"""
Create the map describing the first token expected by a concept
:param context:
:param concepts: lists of concepts to parse
:param use_sheerka: if True, update concepts_by_first_keyword from sheerka
:return:
"""
sheerka = context.sheerka
res = sheerka.cache_manager.copy(sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY) if use_sheerka else {}
for concept in concepts:
keywords = BaseNodeParser.get_first_tokens(sheerka, concept)
if keywords is None:
# no first token found for a concept ?
return sheerka.ret(sheerka.name, False, concept)
for keyword in keywords:
res.setdefault(keyword, []).append(concept.id)
return sheerka.ret("BaseNodeParser", True, res)
@staticmethod
def resolve_concepts_by_first_keyword(context, concepts_by_first_keyword):
sheerka = context.sheerka
def _make_unique(elements):
keys = {}
for e in elements:
keys[e] = 1
return list(keys.keys())
def _resolve_concepts(concept_str):
resolved = []
to_resolve = []
concept = sheerka.get_by_id(core.utils.unstr_concept(concept_str)[1])
if sheerka.isaset(context, concept):
concepts = sheerka.get_set_elements(context, concept)
else:
concepts = [concept]
for concept in concepts:
BaseNodeParser.ensure_bnf(context, concept) # need to make sure that it cannot fail
keywords = BaseNodeParser.get_first_tokens(sheerka, concept)
for keyword in keywords:
(to_resolve if keyword.startswith("c:|") else resolved).append(keyword)
for concept_to_resolve_str in to_resolve:
resolved += _resolve_concepts(concept_to_resolve_str)
return resolved
res = {}
for k, v in concepts_by_first_keyword.items():
if k.startswith("c:|"):
resolved_keywords = _resolve_concepts(k)
for resolved in resolved_keywords:
res.setdefault(resolved, []).extend(v)
else:
res.setdefault(k, []).extend(v)
# 'uniquify' the lists
for k, v in res.items():
res[k] = _make_unique(v)
return sheerka.ret("BaseNodeParser", True, res)
@staticmethod
def resolve_sya_associativity_and_precedence(context, sya):
pass
@staticmethod
def get_first_tokens(sheerka, concept):
"""
:param sheerka:
:param concept:
:return:
"""
if concept.bnf:
from parsers.BnfNodeParser import BnfNodeFirstTokenVisitor
bnf_visitor = BnfNodeFirstTokenVisitor(sheerka)
bnf_visitor.visit(concept.bnf)
return bnf_visitor.first_tokens
else:
keywords = concept.key.split()
for keyword in keywords:
if keyword.startswith(VARIABLE_PREFIX):
continue
return [keyword]
return None
@staticmethod
def ensure_bnf(context, concept, parser_name="BaseNodeParser"):
if concept.metadata.definition_type == DEFINITION_TYPE_BNF and not concept.bnf:
from parsers.BnfParser import BnfParser
regex_parser = BnfParser()
desc = f"Resolving BNF {concept.metadata.definition}"
with context.push(parser_name, obj=concept, desc=desc) as sub_context:
sub_context.add_inputs(parser_input=concept.metadata.definition)
bnf_parsing_ret_val = regex_parser.parse(sub_context, concept.metadata.definition)
sub_context.add_values(return_values=bnf_parsing_ret_val)
if not bnf_parsing_ret_val.status:
raise Exception(bnf_parsing_ret_val.value)
concept.bnf = bnf_parsing_ret_val.body.body
if concept.id:
context.sheerka.get_by_id(concept.id).bnf = concept.bnf # update bnf in cache
src/parsers/BnfNodeParser.py
+510 -432
View File
File diff suppressed because it is too large Load Diff
src/parsers/BnfParser.py
+5 -5
View File
@@ -6,7 +6,7 @@ from core.sheerka.Sheerka import ExecutionContext
from core.tokenizer import Tokenizer, Token, TokenKind, LexerError
from parsers.BaseParser import BaseParser, ErrorNode, UnexpectedTokenErrorNode
from parsers.BnfNodeParser import OrderedChoice, Sequence, Optional, ZeroOrMore, OneOrMore, ConceptExpression, \
StrMatch, ConceptGroupExpression
StrMatch
@dataclass()
@@ -234,8 +234,9 @@ class BnfParser(BaseParser):
if token.type == TokenKind.CONCEPT:
self.next_token()
concept = self.sheerka.new((token.value[0], token.value[1]))
expr = ConceptGroupExpression(concept) if self.sheerka.isaset(self.context, concept) \
else ConceptExpression(concept)
expr = ConceptExpression(concept)
# expr = ConceptGroupExpression(concept) if self.sheerka.isaset(self.context, concept) \
# else ConceptExpression(concept)
return self.eat_rule_name_if_needed(expr)
if token.type == TokenKind.IDENTIFIER:
@@ -259,8 +260,7 @@ class BnfParser(BaseParser):
body=("key", concept_name)))
return None
else:
expr = ConceptGroupExpression(concept) if self.sheerka.isaset(self.context, concept) \
else ConceptExpression(concept)
expr = ConceptExpression(concept)
expr.rule_name = concept.name
return self.eat_rule_name_if_needed(expr)
src/parsers/ConceptsWithConceptsParser.py
-109
View File
@@ -1,109 +0,0 @@
# try to match something like
# ConceptNode 'plus' ConceptNode
#
# Replaced by SyaNodeParser
from core.builtin_concepts import BuiltinConcepts
from core.tokenizer import TokenKind, Token
from parsers.BaseNodeParser import SourceCodeNode
from parsers.BaseParser import BaseParser
from parsers.BnfNodeParser import ConceptNode, UnrecognizedTokensNode
from parsers.MultipleConceptsParser import MultipleConceptsParser
from core.concept import VARIABLE_PREFIX
multiple_concepts_parser = MultipleConceptsParser()
class ConceptsWithConceptsParser(BaseParser):
def __init__(self, **kwargs):
super().__init__("ConceptsWithConcepts", 25)
self.enabled = False
@staticmethod
def get_tokens(nodes):
tokens = []
for node in nodes:
if isinstance(node, ConceptNode):
index, line, column = node.tokens[0].index, node.tokens[0].line, node.tokens[0].column
tokens.append(Token(TokenKind.CONCEPT, node.concept, index, line, column))
else:
for token in node.tokens:
if token.type == TokenKind.EOF:
break
elif token.type in (TokenKind.NEWLINE, TokenKind.WHITESPACE):
continue
else:
tokens.append(token)
return tokens
@staticmethod
def get_key(nodes):
key = ""
index = 0
for node in nodes:
if key:
key += " "
if isinstance(node, UnrecognizedTokensNode):
key += node.source.strip()
else:
key += f"{VARIABLE_PREFIX}{index}"
index += 1
return key
def finalize_concept(self, context, concept, nodes):
index = 0
for node in nodes:
if isinstance(node, ConceptNode):
prop_name = list(concept.props.keys())[index]
concept.compiled[prop_name] = node.concept
context.log(
f"Setting property '{prop_name}='{node.concept}'.",
self.name)
index += 1
elif isinstance(node, SourceCodeNode):
prop_name = list(concept.props.keys())[index]
sheerka = context.sheerka
value = sheerka.new(BuiltinConcepts.PARSER_RESULT, parser=self, source=node.source, body=node.node)
concept.compiled[prop_name] = [context.sheerka.ret(self.name, True, value)]
context.log(
f"Setting property '{prop_name}'='Python({node.source})'.",
self.name)
index += 1
return concept
def parse(self, context, parser_input):
sheerka = context.sheerka
nodes = self.get_input_as_lexer_nodes(parser_input, multiple_concepts_parser)
if not nodes:
return None
concept_key = self.get_key(nodes)
concept = sheerka.new(concept_key)
if sheerka.isinstance(concept, BuiltinConcepts.UNKNOWN_CONCEPT):
return sheerka.ret(
self.name,
False,
sheerka.new(BuiltinConcepts.NOT_FOR_ME, body=parser_input.body))
concepts = concept if hasattr(concept, "__iter__") else [concept]
for concept in concepts:
self.finalize_concept(context, concept, nodes)
res = []
for concept in concepts:
res.append(sheerka.ret(
self.name,
True,
sheerka.new(
BuiltinConcepts.PARSER_RESULT,
parser=self,
source=parser_input.source,
body=concept,
try_parsed=None)))
return res[0] if len(res) == 1 else res
src/parsers/DefaultParser.py
+2 -1
View File
@@ -384,7 +384,8 @@ class DefaultParser(BaseParser):
return None, NotInitializedNode()
regex_parser = BnfParser()
with self.context.push(self.name, obj=current_concept_def) as sub_context:
desc = f"Resolving BNF {current_concept_def.definition}"
with self.context.push(self.name, obj=current_concept_def, desc=desc) as sub_context:
parsing_result = regex_parser.parse(sub_context, tokens)
sub_context.add_values(return_values=parsing_result)
src/parsers/ExactConceptParser.py
+21 -12
View File
@@ -1,9 +1,9 @@
import logging
from core.builtin_concepts import ReturnValueConcept, BuiltinConcepts
from parsers.BaseParser import BaseParser
from core.tokenizer import Tokenizer, Keywords, TokenKind, LexerError
from core.concept import VARIABLE_PREFIX
from core.tokenizer import Keywords, TokenKind, LexerError
from parsers.BaseParser import BaseParser
class ExactConceptParser(BaseParser):
@@ -11,10 +11,11 @@ class ExactConceptParser(BaseParser):
Tries to recognize a single concept
"""
MAX_WORDS_SIZE = 10
MAX_WORDS_SIZE = 3
def __init__(self, **kwargs):
def __init__(self, max_word_size=None, **kwargs):
BaseParser.__init__(self, "ExactConcept", 80)
self.max_word_size = max_word_size
def parse(self, context, parser_input):
"""
@@ -33,11 +34,11 @@ class ExactConceptParser(BaseParser):
context.log(f"Error found in tokenizer {e}", self.name)
return sheerka.ret(self.name, False, sheerka.new(BuiltinConcepts.ERROR, body=e))
if len(words) > self.MAX_WORDS_SIZE:
if len(words) > (self.max_word_size or self.MAX_WORDS_SIZE):
context.log(f"Max words reached. Stopping.", self.name)
return sheerka.ret(self.name, False, sheerka.new(BuiltinConcepts.CONCEPT_TOO_LONG, body=parser_input))
recognized = False
recognized = [] # keep track of the concepts founds
for combination in self.combinations(words):
concept_key = " ".join(combination)
@@ -49,16 +50,23 @@ class ExactConceptParser(BaseParser):
concepts = result if isinstance(result, list) else [result]
for concept in concepts:
if concept.id in recognized:
context.log(f"Recognized concept {concept} again. Skipping.", self.name)
# example
# if the input is foo a and a concept is defined as foo a
# The will be two matches. One for 'foo a' and 'foo _var_0'
# but it's the same concept foo a
continue
context.log(f"Recognized concept {concept}.", self.name)
# update the properties if needed
need_validation = False
for i, token in enumerate(combination):
if token.startswith(VARIABLE_PREFIX):
index = int(token[len(VARIABLE_PREFIX):])
concept.def_prop_by_index(index, words[i])
concept.def_var_by_index(index, words[i])
concept.metadata.need_validation = True
if self.verbose_log.isEnabledFor(logging.DEBUG):
prop_name = list(concept.props.keys())[index]
prop_name = concept.metadata.variables[index][0]
context.log(
f"Added property {index}: {prop_name}='{words[i]}'.",
self.name)
@@ -69,12 +77,13 @@ class ExactConceptParser(BaseParser):
context.sheerka.new(
BuiltinConcepts.PARSER_RESULT,
parser=self,
source=parser_input if isinstance(parser_input, str) else self.get_text_from_tokens(parser_input),
source=parser_input if isinstance(parser_input, str) else self.get_text_from_tokens(
parser_input),
body=concept,
try_parsed=concept)))
recognized = True
recognized.append(concept.id)
if recognized:
if len(recognized) > 0:
if len(res) == 1:
self.log_result(context, parser_input, res[0])
else:
src/parsers/ExplainParser.py
+1 -1
View File
@@ -318,7 +318,7 @@ class ExplainParser(BaseSplitIterParser):
def parse(self, context, parser_input):
"""
text can be string, but text can also be an list of tokens
parser_input can be string, but text can also be an list of tokens
:param context:
:param parser_input:
:return:
src/parsers/MultipleConceptsParser.py
-163
View File
@@ -1,163 +0,0 @@
# to be replaced by SyaNodeParser
import ast
from core.builtin_concepts import BuiltinConcepts
from core.tokenizer import TokenKind
from parsers.BaseNodeParser import SourceCodeNode
from parsers.BaseParser import BaseParser
from parsers.BnfNodeParser import BnfNodeParser, UnrecognizedTokensNode, ConceptNode
import core.utils
from parsers.PythonParser import PythonParser
concept_lexer_parser = BnfNodeParser()
class MultipleConceptsParser(BaseParser):
"""
Parser that will take the result of BnfNodeParser and
try to resolve the unrecognized tokens token by token
It is a success when it returns a list ConceptNode exclusively
"""
def __init__(self, **kwargs):
BaseParser.__init__(self, "MultipleConcepts", 45)
self.enabled = False
@staticmethod
def finalize(nodes_found, unrecognized_tokens):
if not unrecognized_tokens:
return nodes_found, unrecognized_tokens
unrecognized_tokens.fix_source()
if unrecognized_tokens.not_whitespace():
nodes_found = core.utils.product(nodes_found, [unrecognized_tokens])
return nodes_found, None
@staticmethod
def create_or_add(unrecognized_tokens, token, index):
if unrecognized_tokens:
unrecognized_tokens.add_token(token, index)
else:
unrecognized_tokens = UnrecognizedTokensNode(index, index, [token])
return unrecognized_tokens
def parse(self, context, parser_input):
sheerka = context.sheerka
nodes = self.get_input_as_lexer_nodes(parser_input, concept_lexer_parser)
if not nodes:
return None
nodes_found = [[]]
concepts_only = True
for node in nodes:
if isinstance(node, UnrecognizedTokensNode):
unrecognized_tokens = None
i = 0
while i < len(node.tokens):
token_index = node.start + i
token = node.tokens[i]
concepts_nodes = self.get_concepts_nodes(context, token_index, token)
if concepts_nodes is not None:
nodes_found, unrecognized_tokens = self.finalize(nodes_found, unrecognized_tokens)
nodes_found = core.utils.product(nodes_found, concepts_nodes)
i += 1
continue
source_code_node = self.get_source_code_node(context, token_index, node.tokens[i:])
if source_code_node:
nodes_found, unrecognized_tokens = self.finalize(nodes_found, unrecognized_tokens)
nodes_found = core.utils.product(nodes_found, [source_code_node])
i += len(source_code_node.tokens)
continue
# not a concept nor some source code
unrecognized_tokens = self.create_or_add(unrecognized_tokens, token, token_index)
concepts_only &= token.type in (TokenKind.WHITESPACE, TokenKind.NEWLINE)
i += 1
# finish processing if needed
nodes_found, unrecognized_tokens = self.finalize(nodes_found, unrecognized_tokens)
else:
nodes_found = core.utils.product(nodes_found, [node])
ret = []
for choice in nodes_found:
ret.append(
sheerka.ret(
self.name,
concepts_only,
sheerka.new(
BuiltinConcepts.PARSER_RESULT,
parser=self,
source=parser_input.source,
body=choice,
try_parsed=None))
)
if len(ret) == 1:
self.log_result(context, parser_input.source, ret[0])
return ret[0]
else:
self.log_multiple_results(context, parser_input.source, ret)
return ret
@staticmethod
def get_concepts_nodes(context, index, token):
"""
Tries to recognize a concept
from the univers of all known concepts
"""
if token.type != TokenKind.IDENTIFIER:
return None
concept = context.new_concept(token.value)
if hasattr(concept, "__iter__") or context.sheerka.is_known(concept):
concepts = concept if hasattr(concept, "__iter__") else [concept]
concepts_nodes = [ConceptNode(c, index, index, [token], token.value) for c in concepts]
return concepts_nodes
return None
@staticmethod
def get_source_code_node(context, index, tokens):
"""
Tries to recognize source code.
For the time being, only Python is supported
:param context:
:param tokens:
:param index:
:return:
"""
if len(tokens) == 0 or (len(tokens) == 1 and tokens[0].type == TokenKind.EOF):
return None
end_index = len(tokens)
while end_index > 0:
parser = PythonParser()
tokens_to_parse = tokens[:end_index]
res = parser.parse(context, tokens_to_parse)
if res.status:
# only expression are accepted
ast_ = res.value.value.ast_
if not isinstance(ast_, ast.Expression):
return None
try:
compiled = compile(ast_, "<string>", "eval")
eval(compiled, {}, {})
except Exception:
return None
source = BaseParser.get_text_from_tokens(tokens_to_parse)
return SourceCodeNode(res.value.value, index, index + end_index - 1, tokens_to_parse, source)
end_index -= 1
return None
src/parsers/PythonParser.py
+5 -5
View File
@@ -1,11 +1,11 @@
from core.builtin_concepts import BuiltinConcepts, ParserResultConcept
from core.tokenizer import Tokenizer, LexerError, TokenKind
from parsers.BaseParser import BaseParser, Node, ErrorNode
from dataclasses import dataclass
import ast
import logging
import core.utils
from dataclasses import dataclass
import core.utils
from core.builtin_concepts import BuiltinConcepts
from core.tokenizer import LexerError, TokenKind
from parsers.BaseParser import BaseParser, Node, ErrorNode
from parsers.BnfNodeParser import ConceptNode
log = logging.getLogger(__name__)
src/parsers/PythonWithConceptsParser.py
-1
View File
@@ -1,7 +1,6 @@
from core.builtin_concepts import BuiltinConcepts
from parsers.BaseParser import BaseParser
from parsers.BnfNodeParser import ConceptNode
from parsers.MultipleConceptsParser import MultipleConceptsParser
from parsers.PythonParser import PythonParser
from parsers.UnrecognizedNodeParser import UnrecognizedNodeParser
src/parsers/SyaNodeParser.py
+166 -118
View File
@@ -1,4 +1,3 @@
import copy
from collections import namedtuple
from dataclasses import dataclass, field
from typing import List
@@ -7,10 +6,10 @@ from core import builtin_helpers
from core.builtin_concepts import BuiltinConcepts
from core.concept import VARIABLE_PREFIX, Concept, DEFINITION_TYPE_BNF
from core.sheerka.ExecutionContext import ExecutionContext
from core.tokenizer import LexerError, Token, TokenKind
from core.tokenizer import Token, TokenKind
from parsers.BaseNodeParser import UnrecognizedTokensNode, ConceptNode, SourceCodeNode, SyaAssociativity, \
SourceCodeWithConceptNode
from parsers.BaseParser import BaseParser, ErrorNode
SourceCodeWithConceptNode, BaseNodeParser
from parsers.BaseParser import ErrorNode, UnexpectedTokenErrorNode
PARSERS = ["BnfNode", "AtomNode", "Python"]
@@ -116,13 +115,13 @@ class SyaConceptParserHelper:
return len(self.expected) == 0
def is_atom(self):
return len(self.concept.concept.metadata.props) == 0 and len(self.expected) == 0
return len(self.concept.concept.metadata.variables) == 0 and len(self.expected) == 0
def is_expected(self, token):
if self.is_matched():
return False
token_value = self._get_token_value(token)
token_value = BaseNodeParser.get_token_value(token)
for expected in self.expected:
if not expected.startswith(VARIABLE_PREFIX) and expected == token_value:
@@ -139,7 +138,7 @@ class SyaConceptParserHelper:
# return True is a whole sequence of keyword is eaten
# example
# Concept("foo a bar baz qux b").def_prop("a").def_prop("b")
# Concept("foo a bar baz qux b").def_var("a").def_var("b")
# 'bar' is just eaten. We will return False because 'baz' and 'qux' are still waiting
if len(self.expected) == 0:
return True
@@ -169,14 +168,14 @@ class SyaConceptParserHelper:
self.concept = self.concept.concept
return self
@staticmethod
def _get_token_value(token):
if token.type == TokenKind.STRING:
return token.value[1:-1]
elif token.type == TokenKind.KEYWORD:
return token.value.value
else:
return token.value
# @staticmethod
# def _get_token_value(token):
# if token.type == TokenKind.STRING:
# return token.value[1:-1]
# elif token.type == TokenKind.KEYWORD:
# return token.value.value
# else:
# return token.value
def clone(self):
clone = SyaConceptParserHelper(self.concept, self.start, self.end)
@@ -215,7 +214,10 @@ class InFixToPostFix:
if not isinstance(other, InFixToPostFix):
return False
return self.out == other.out
return self.out == other.out and self.errors == other.errors
def __hash__(self):
return len(self.sequence) + len(self.errors)
def _add_error(self, error):
self.errors.append(error)
@@ -396,6 +398,7 @@ class InFixToPostFix:
del current_concept.expected[0]
def manage_unrecognized(self):
if self.unrecognized_tokens.is_empty():
return
@@ -514,10 +517,10 @@ class InFixToPostFix:
def handle_expected_token(self, token, pos):
"""
True if the token is part of the concept being parsed and the last token in a sequence is eaten
Example : Concept("foo a bar b").def_prop("a").def_prop("b")
Example : Concept("foo a bar b").def_var("a").def_var("b")
The expected tokens are 'foo' and 'bar' (as a and b are parameters)
Example: Concept("foo a bar baz b").def_prop("a").def_prop("b")
Example: Concept("foo a bar baz b").def_var("a").def_var("b")
If the token is 'bar', it will be eaten but handle_expected_token() will return False
as we still expect 'baz'
:param token:
@@ -565,6 +568,18 @@ class InFixToPostFix:
return True
# else:
# if token.type != TokenKind.WHITESPACE:
# # hack, because whitespaces are not correctly parsed in self.expected
# # KSI 2020/04/25
# # I no longer understand why we are in a loop (the reverse one)
# # if we are parsing a concept and the expected token does not match
# # The whole class should be in error
# self._add_error(UnexpectedTokenErrorNode(
# f"Failed to parse '{current_concept.concept.concept}'",
# token, current_concept.expected))
# return False
return False
def eat_token(self, token, pos):
@@ -581,7 +596,7 @@ class InFixToPostFix:
if self.handle_expected_token(token, pos):
# a token is found, let's check if it's part of a concepts being parsed
# example Concept(name="foo", definition="foo a bar b").def_prop("a").def_prop("b")
# example Concept(name="foo", definition="foo a bar b").def_var("a").def_var("b")
# if the token 'bar' is found, it has to be considered as part of the concept foo
self.debug.append(token)
return True
@@ -780,16 +795,13 @@ class PostFixToItem:
has_unrecognized: bool
class SyaNodeParser(BaseParser):
class SyaNodeParser(BaseNodeParser):
def __init__(self, **kwargs):
BaseParser.__init__(self, "SyaNode", 50)
super().__init__("SyaNode", 50, **kwargs)
if 'sheerka' in kwargs:
sheerka = kwargs.get("sheerka")
self.concepts_by_first_keyword = sheerka.concepts_by_first_keyword
self.sya_definitions = {}
if sheerka.sya_definitions:
for k, v in sheerka.sya_definitions.items():
self.sya_definitions[k] = (v[0], SyaAssociativity(v[1]))
self.sya_definitions = sheerka.resolved_sya_def
else:
self.concepts_by_first_keyword = {}
@@ -803,104 +815,133 @@ class SyaNodeParser(BaseParser):
self.text = None
self.sheerka = None
def reset_parser(self, context, text):
self.context = context
self.sheerka = context.sheerka
self.text = text
try:
self.tokens = list(self.get_input_as_tokens(text))
except LexerError as e:
self.add_error(self.sheerka.new(BuiltinConcepts.ERROR, body=e), False)
return False
self.token = None
self.pos = -1
return True
def add_error(self, error, next_token=True):
self.error_sink.append(error)
if next_token:
self.next_token()
return error
def get_token(self) -> Token:
return self.token
def next_token(self, skip_whitespace=True):
if self.token and self.token.type == TokenKind.EOF:
return False
self.pos += 1
self.token = self.tokens[self.pos]
if skip_whitespace:
while self.token.type == TokenKind.WHITESPACE or self.token.type == TokenKind.NEWLINE:
self.pos += 1
self.token = self.tokens[self.pos]
return self.token.type != TokenKind.EOF
def initialize(self, context, concepts=None, sya_definitions=None):
self.context = context
self.sheerka = context.sheerka
def init_from_concepts(self, context, concepts, **kwargs):
super().init_from_concepts(context, concepts)
sya_definitions = kwargs.get("sya", None)
if sya_definitions:
self.sya_definitions = sya_definitions
if concepts:
for concept in concepts:
keywords = concept.key.split()
for keyword in keywords:
if keyword.startswith(VARIABLE_PREFIX):
continue
self.concepts_by_first_keyword.setdefault(keyword, []).append(concept.id)
break
return self.sheerka.ret(self.name, True, self.concepts_by_first_keyword)
def get_concepts(self, token):
@staticmethod
def _is_eligible(concept):
"""
Tries to find if there are concepts that match the value of the token
:param token:
Predicate that select concepts that must handled by AtomNodeParser
:param concept:
:return:
"""
# We only concepts that has parameter (refuse atoms)
# Bnf definitions are not supposed to be managed by this parser either
return len(concept.metadata.variables) > 0 and concept.metadata.definition_type != DEFINITION_TYPE_BNF
if token.type == TokenKind.STRING:
name = token.value[1:-1]
elif token.type == TokenKind.KEYWORD:
name = token.value.value
else:
name = token.value
@staticmethod
def _get_sya_concept_def(parser, concept):
sya_concept_def = SyaConceptDef(concept)
if concept.id in parser.sya_definitions:
sya_def = parser.sya_definitions.get(concept.id)
if sya_def[0] is not None:
sya_concept_def.precedence = sya_def[0]
if sya_def[1] is not None:
sya_concept_def.associativity = sya_def[1]
return sya_concept_def
result = []
if name in self.concepts_by_first_keyword:
for concept_id in self.concepts_by_first_keyword[name]:
# def reset_parser(self, context, text):
# self.context = context
# self.sheerka = context.sheerka
# self.text = text
#
# try:
# self.tokens = list(self.get_input_as_tokens(text))
# except LexerError as e:
# self.add_error(self.sheerka.new(BuiltinConcepts.ERROR, body=e), False)
# return False
#
# self.token = None
# self.pos = -1
# return True
#
# def add_error(self, error, next_token=True):
# self.error_sink.append(error)
# if next_token:
# self.next_token()
# return error
#
# def get_token(self) -> Token:
# return self.token
#
# def next_token(self, skip_whitespace=True):
# if self.token and self.token.type == TokenKind.EOF:
# return False
#
# self.pos += 1
# self.token = self.tokens[self.pos]
#
# if skip_whitespace:
# while self.token.type == TokenKind.WHITESPACE or self.token.type == TokenKind.NEWLINE:
# self.pos += 1
# self.token = self.tokens[self.pos]
#
# return self.token.type != TokenKind.EOF
concept = self.sheerka.get_by_id(concept_id)
if len(concept.metadata.props) == 0:
# only concepts that has parameter (refuse atoms)
# Note that this test is needed if the definition of the concept has changed
continue
if concept.metadata.definition_type == DEFINITION_TYPE_BNF:
# bnf definitions are not supposed to be managed by this parser
continue
sya_concept_def = SyaConceptDef(concept)
if concept.id in self.sya_definitions:
sya_def = self.sya_definitions[concept.id]
if sya_def[0] is not None:
sya_concept_def.precedence = sya_def[0]
if sya_def[1] is not None:
sya_concept_def.associativity = sya_def[1]
result.append(sya_concept_def)
return result
return None
# def initialize(self, context, concepts=None, sya_definitions=None):
# self.context = context
# self.sheerka = context.sheerka
#
# if sya_definitions:
# self.sya_definitions = sya_definitions
#
# if concepts:
# for concept in concepts:
# keywords = concept.key.split()
# for keyword in keywords:
# if keyword.startswith(VARIABLE_PREFIX):
# continue
#
# self.concepts_by_first_keyword.setdefault(keyword, []).append(concept.id)
# break
#
# return self.sheerka.ret(self.name, True, self.concepts_by_first_keyword)
#
# def get_concepts(self, token):
# """
# Tries to find if there are concepts that match the value of the token
# :param token:
# :return:
# """
#
# if token.type == TokenKind.STRING:
# name = token.value[1:-1]
# elif token.type == TokenKind.KEYWORD:
# name = token.value.value
# else:
# name = token.value
#
# result = []
# if name in self.concepts_by_first_keyword:
# for concept_id in self.concepts_by_first_keyword[name]:
#
# concept = self.sheerka.get_by_id(concept_id)
#
# if len(concept.metadata.props) == 0:
# # only concepts that has parameter (refuse atoms)
# # Note that this test is needed if the definition of the concept has changed
# continue
#
# if concept.metadata.definition_type == DEFINITION_TYPE_BNF:
# # bnf definitions are not supposed to be managed by this parser
# continue
#
# sya_concept_def = SyaConceptDef(concept)
# if concept.id in self.sya_definitions:
# sya_def = self.sya_definitions[concept.id]
# if sya_def[0] is not None:
# sya_concept_def.precedence = sya_def[0]
# if sya_def[1] is not None:
# sya_concept_def.associativity = sya_def[1]
#
# result.append(sya_concept_def)
# return result
#
# return None
def infix_to_postfix(self, context, text):
"""
@@ -943,7 +984,7 @@ class SyaNodeParser(BaseParser):
if infix_to_postfix.eat_token(token, self.pos):
infix_to_postfix.lock()
concepts = self.get_concepts(token)
concepts = self.get_concepts(token, self._is_eligible, to_map=self._get_sya_concept_def)
if not concepts:
for infix_to_postfix in res:
infix_to_postfix.eat_unrecognized(token, self.pos)
@@ -988,7 +1029,7 @@ class SyaNodeParser(BaseParser):
else:
items.append(res)
item.has_unrecognized |= hasattr(res, "has_unrecognized") and res.has_unrecognized or \
isinstance(res, UnrecognizedTokensNode)
isinstance(res, UnrecognizedTokensNode)
item.nodes = items
item.fix_all_pos()
item.tokens = self.tokens[item.start:item.end + 1]
@@ -1000,7 +1041,7 @@ class SyaNodeParser(BaseParser):
end = item.end
has_unrecognized = False
concept = sheerka.new_from_template(item.concept, item.concept.id)
for param_index in reversed(range(len(concept.metadata.props))):
for param_index in reversed(range(len(concept.metadata.variables))):
inner_item = self.postfix_to_item(sheerka, postfixed)
if inner_item.start < start:
start = inner_item.start
@@ -1008,7 +1049,7 @@ class SyaNodeParser(BaseParser):
end = inner_item.end
has_unrecognized |= isinstance(inner_item, UnrecognizedTokensNode)
param_name = concept.metadata.props[param_index][0]
param_name = concept.metadata.variables[param_index][0]
param_value = inner_item.concept if hasattr(inner_item, "concept") else \
[inner_item.return_value] if isinstance(inner_item, SourceCodeNode) else \
inner_item
@@ -1115,3 +1156,10 @@ class SyaNodeParser(BaseParser):
result.append(infix_to_postfix)
return result
# @staticmethod
# def init_sheerka(self, sheerka):
# if hasattr(BaseNodeParser, "init_sheerka"):
# BaseNodeParser.init_sheerka(sheerka)
#
# # init syadefinitins
src/parsers/UnrecognizedNodeParser.py
+10 -1
View File
@@ -52,11 +52,20 @@ class UnrecognizedNodeParser(BaseParser):
res = only_successful(context, res)
if res.status:
lexer_nodes = get_lexer_nodes(res.body.body, node.start, node.tokens)
sequences_found = core.utils.product(sequences_found, lexer_nodes)
if lexer_nodes:
# make lexer_nodes is not empty (for example, some Python result are discarded)
sequences_found = core.utils.product(sequences_found, lexer_nodes)
else:
sequences_found = core.utils.product(sequences_found, [node])
has_unrecognized = True
else:
sequences_found = core.utils.product(sequences_found, [node])
has_unrecognized = True
elif isinstance(node, SourceCodeNode):
sequences_found = core.utils.product(sequences_found, [node])
has_unrecognized = True # never trust source code not. I may be an invalid source code
else: # cannot happen as of today :-)
raise NotImplementedError()
src/parsers/_BnfNodeParser_Old.py
+912
View File
@@ -0,0 +1,912 @@
# #####################################################################################################
# # This implementation of the parser is highly inspired by the arpeggio project (https://github.com/textX/Arpeggio)
# # I don't directly use the project, but it helped me figure out
# # what to do.
# # Dejanović I., Milosavljević G., Vaderna R.:
# # Arpeggio: A flexible PEG parser for Python,
# # Knowledge-Based Systems, 2016, 95, 71 - 74, doi:10.1016/j.knosys.2015.12.004
# #####################################################################################################
# from collections import namedtuple
# from dataclasses import dataclass
# from collections import defaultdict
# from core.builtin_concepts import BuiltinConcepts, ParserResultConcept
# from core.concept import Concept, ConceptParts, DoNotResolve
# from core.tokenizer import TokenKind, Tokenizer, Token
# from parsers.BaseNodeParser import LexerNode, GrammarErrorNode, ConceptNode, UnrecognizedTokensNode
# from parsers.BaseParser import BaseParser, ErrorNode
# import core.utils
#
#
# class NonTerminalNode(LexerNode):
# """
# Returned by the BnfNodeParser
# """
#
# def __init__(self, parsing_expression, start, end, tokens, children=None):
# super().__init__(start, end, tokens)
# self.parsing_expression = parsing_expression
# self.children = children
#
# def __repr__(self):
# name = self.parsing_expression.rule_name or self.parsing_expression.__class__.__name__
# if len(self.children) > 0:
# sub_names = "(" + ",".join([repr(child) for child in self.children]) + ")"
# else:
# sub_names = ""
# return name + sub_names
#
# def __eq__(self, other):
# if not isinstance(other, NonTerminalNode):
# return False
#
# return self.parsing_expression == other.parsing_expression and \
# self.start == other.start and \
# self.end == other.end and \
# self.children == other.children
#
# def __hash__(self):
# return hash((self.parsing_expression, self.start, self.end, self.children))
#
#
# class TerminalNode(LexerNode):
# """
# Returned by the BnfNodeParser
# """
#
# def __init__(self, parsing_expression, start, end, value):
# super().__init__(start, end, source=value)
# self.parsing_expression = parsing_expression
# self.value = value
#
# def __repr__(self):
# name = self.parsing_expression.rule_name or ""
# return name + f"'{self.value}'"
#
# def __eq__(self, other):
# if not isinstance(other, TerminalNode):
# return False
#
# return self.parsing_expression == other.parsing_expression and \
# self.start == other.start and \
# self.end == other.end and \
# self.value == other.value
#
# def __hash__(self):
# return hash((self.parsing_expression, self.start, self.end, self.value))
#
#
# @dataclass()
# class UnknownConceptNode(ErrorNode):
# concept_key: str
#
#
# @dataclass()
# class TooManyConceptNode(ErrorNode):
# concept_key: str
#
#
# class ParsingExpression:
# def __init__(self, *args, **kwargs):
# self.elements = args
#
# nodes = kwargs.get('nodes', [])
# if not hasattr(nodes, '__iter__'):
# nodes = [nodes]
# self.nodes = nodes
#
# self.rule_name = kwargs.get('rule_name', '')
#
# def __eq__(self, other):
# if not isinstance(other, ParsingExpression):
# return False
#
# return self.rule_name == other.rule_name and self.elements == other.elements
#
# def __hash__(self):
# return hash((self.rule_name, self.elements))
#
# def parse(self, parser):
# return self._parse(parser)
#
# def add_rule_name_if_needed(self, text):
# return text + "=" + self.rule_name if self.rule_name else text
#
#
# class ConceptExpression(ParsingExpression):
# """
# Will match a concept
# It used only for rule definition
#
# When the grammar is created, it is replaced by the actual concept
# """
#
# def __init__(self, concept, rule_name=""):
# super().__init__(rule_name=rule_name)
# self.concept = concept
#
# def __repr__(self):
# return self.add_rule_name_if_needed(f"{self.concept}")
#
# def __eq__(self, other):
# if not super().__eq__(other):
# return False
#
# if not isinstance(other, ConceptExpression):
# return False
#
# if isinstance(self.concept, Concept):
# return self.concept.name == other.concept.name
#
# # when it's only the name of the concept
# return self.concept == other.concept
#
# def __hash__(self):
# return hash((self.concept, self.rule_name))
#
# @staticmethod
# def get_parsing_expression_from_name(name):
# tokens = Tokenizer(name)
# nodes = [StrMatch(core.utils.strip_quotes(token.value)) for token in list(tokens)[:-1]]
# if len(nodes) == 1:
# return nodes[0]
# else:
# sequence = Sequence(nodes)
# sequence.nodes = nodes
# return sequence
#
# def _parse(self, parser):
# to_match = parser.get_concept(self.concept) if isinstance(self.concept, str) else self.concept
# if parser.sheerka.isinstance(to_match, BuiltinConcepts.UNKNOWN_CONCEPT):
# return None
#
# self.concept = to_match # Memoize
#
# if to_match not in parser.concepts_grammars:
# # Try to match the concept using its name
# expr = self.get_parsing_expression_from_name(to_match.name)
# node = expr.parse(parser)
# else:
# node = parser.concepts_grammars[to_match].parse(parser)
#
# if node is None:
# return None
#
# return NonTerminalNode(self, node.start, node.end, parser.tokens[node.start: node.end + 1], [node])
#
#
# class ConceptGroupExpression(ConceptExpression):
# def _parse(self, parser):
# to_match = parser.get_concept(self.concept) if isinstance(self.concept, str) else self.concept
# if parser.sheerka.isinstance(to_match, BuiltinConcepts.UNKNOWN_CONCEPT):
# return None
#
# self.concept = to_match # Memoize
#
# if to_match not in parser.concepts_grammars:
# concepts_in_group = parser.sheerka.get_set_elements(parser.context, self.concept)
# nodes = [ConceptExpression(c, rule_name=c.name) for c in concepts_in_group]
# expr = OrderedChoice(nodes)
# expr.nodes = nodes
# node = expr.parse(parser)
# else:
# node = parser.concepts_grammars[to_match].parse(parser)
#
# if node is None:
# return None
#
# return NonTerminalNode(self, node.start, node.end, parser.tokens[node.start: node.end + 1], [node])
#
#
# class Sequence(ParsingExpression):
# """
# Will match sequence of parser expressions in exact order they are defined.
# """
#
# def _parse(self, parser):
# init_pos = parser.pos
# end_pos = parser.pos
#
# children = []
# for e in self.nodes:
# node = e.parse(parser)
# if node is None:
# return None
# else:
# if node.end != -1: # because returns -1 when no match
# children.append(node)
# end_pos = node.end
#
# return NonTerminalNode(self, init_pos, end_pos, parser.tokens[init_pos: end_pos + 1], children)
#
# def __repr__(self):
# to_str = ", ".join(repr(n) for n in self.elements)
# return self.add_rule_name_if_needed(f"({to_str})")
#
#
# class OrderedChoice(ParsingExpression):
# """
# Will match one among multiple
# It will stop at the first match (so the order of definition is important)
# """
#
# def _parse(self, parser):
# init_pos = parser.pos
#
# for e in self.nodes:
# node = e.parse(parser)
# if node:
# return NonTerminalNode(self, init_pos, node.end, parser.tokens[init_pos: node.end + 1], [node])
#
# parser.seek(init_pos) # backtrack
#
# return None
#
# def __repr__(self):
# to_str = "| ".join(repr(n) for n in self.elements)
# return self.add_rule_name_if_needed(f"({to_str})")
#
#
# class Optional(ParsingExpression):
# """
# Will match or not the elements
# if many matches, will choose longest one
# If you need order, use Optional(OrderedChoice)
# """
#
# def _parse(self, parser):
# init_pos = parser.pos
# selected_node = NonTerminalNode(self, parser.pos, -1, [], []) # means that nothing is found
#
# for e in self.nodes:
# node = e.parse(parser)
# if node:
# if node.end > selected_node.end:
# selected_node = NonTerminalNode(
# self,
# node.start,
# node.end,
# parser.tokens[node.start: node.end + 1],
# [node])
#
# parser.seek(init_pos) # backtrack
#
# if selected_node.end != -1:
# parser.seek(selected_node.end)
# parser.next_token() # eat the tokens found
#
# return selected_node
#
# def __repr__(self):
# if len(self.elements) == 1:
# return f"{self.elements[0]}?"
# else:
# to_str = ", ".join(repr(n) for n in self.elements)
# return self.add_rule_name_if_needed(f"({to_str})?")
#
#
# class Repetition(ParsingExpression):
# """
# Base class for all repetition-like parser expressions (?,*,+)
# Args:
# eolterm(bool): Flag that indicates that end of line should
# terminate repetition match.
# """
#
# def __init__(self, *elements, **kwargs):
# super(Repetition, self).__init__(*elements, **kwargs)
# self.sep = kwargs.get('sep', None)
#
#
# class ZeroOrMore(Repetition):
# """
# ZeroOrMore will try to match parser expression specified zero or more
# times. It will never fail.
# """
#
# def _parse(self, parser):
# init_pos = parser.pos
# end_pos = -1
# children = []
#
# while True:
# current_pos = parser.pos
#
# # maybe eat the separator if needed
# if self.sep and children:
# sep_result = self.sep.parse(parser)
# if sep_result is None:
# parser.seek(current_pos)
# break
#
# # eat the ZeroOrMore
# node = self.nodes[0].parse(parser)
# if node is None:
# parser.seek(current_pos)
# break
# else:
# if node.end != -1: # because returns -1 when no match
# children.append(node)
# end_pos = node.end
#
# if len(children) == 0:
# return NonTerminalNode(self, init_pos, -1, [], [])
#
# return NonTerminalNode(self, init_pos, end_pos, parser.tokens[init_pos: end_pos + 1], children)
#
# def __repr__(self):
# to_str = ", ".join(repr(n) for n in self.elements)
# return self.add_rule_name_if_needed(f"({to_str})*")
#
#
# class OneOrMore(Repetition):
# """
# OneOrMore will try to match parser expression specified one or more times.
# """
#
# def _parse(self, parser):
# init_pos = parser.pos
# end_pos = -1
# children = []
#
# while True:
# current_pos = parser.pos
#
# # maybe eat the separator if needed
# if self.sep and children:
# sep_result = self.sep.parse(parser)
# if sep_result is None:
# parser.seek(current_pos)
# break
#
# # eat the ZeroOrMore
# node = self.nodes[0].parse(parser)
# if node is None:
# parser.seek(current_pos)
# break
# else:
# if node.end != -1: # because returns -1 when no match
# children.append(node)
# end_pos = node.end
#
# if len(children) == 0: # if nothing is found, it's an error
# return None
#
# return NonTerminalNode(self, init_pos, end_pos, parser.tokens[init_pos: end_pos + 1], children)
#
# def __repr__(self):
# to_str = ", ".join(repr(n) for n in self.elements)
# return self.add_rule_name_if_needed(f"({to_str})+")
#
#
# class UnorderedGroup(Repetition):
# """
# Will try to match all of the parsing expression in any order.
# """
#
# def _parse(self, parser):
# raise NotImplementedError()
#
# # def __repr__(self):
# # to_str = ", ".join(repr(n) for n in self.elements)
# # return f"({to_str})#"
#
#
# class Match(ParsingExpression):
# """
# Base class for all classes that will try to match something from the input.
# """
#
# def __init__(self, rule_name, root=False):
# super(Match, self).__init__(rule_name=rule_name, root=root)
#
# def parse(self, parser):
# result = self._parse(parser)
# return result
#
#
# class StrMatch(Match):
# """
# Matches a literal
# """
#
# def __init__(self, to_match, rule_name="", ignore_case=True):
# super(Match, self).__init__(rule_name=rule_name)
# self.to_match = to_match
# self.ignore_case = ignore_case
#
# def __repr__(self):
# return self.add_rule_name_if_needed(f"'{self.to_match}'")
#
# def __eq__(self, other):
# if not super().__eq__(other):
# return False
#
# if not isinstance(other, StrMatch):
# return False
#
# return self.to_match == other.to_match and self.ignore_case == other.ignore_case
#
# def _parse(self, parser):
# token = parser.get_token()
# m = str(token.value).lower() == self.to_match.lower() if self.ignore_case \
# else token.value == self.to_match
#
# if m:
# node = TerminalNode(self, parser.pos, parser.pos, token.value)
# parser.next_token()
# return node
#
# return None
#
#
# class BnfNodeParser(BaseParser):
# def __init__(self, **kwargs):
# super().__init__("BnfNode_old", 50)
# self.enabled = False
# if 'grammars' in kwargs:
# self.concepts_grammars = kwargs.get("grammars")
# elif 'sheerka' in kwargs:
# self.concepts_grammars = kwargs.get("sheerka").concepts_grammars
# else:
# self.concepts_grammars = {}
#
# self.ignore_case = True
#
# self.token = None
# self.pos = -1
# self.tokens = None
#
# self.context = None
# self.text = None
# self.sheerka = None
#
# def add_error(self, error, next_token=True):
# self.error_sink.append(error)
# if next_token:
# self.next_token()
# return error
#
# def reset_parser(self, context, text):
# self.context = context
# self.sheerka = context.sheerka
# self.text = text
#
# try:
# self.tokens = list(self.get_input_as_tokens(text))
# except core.tokenizer.LexerError as e:
# self.add_error(self.sheerka.new(BuiltinConcepts.ERROR, body=e), False)
# return False
#
# self.token = None
# self.pos = -1
# self.next_token(False)
# return True
#
# def get_token(self) -> Token:
# return self.token
#
# def next_token(self, skip_whitespace=True):
# if self.token and self.token.type == TokenKind.EOF:
# return False
#
# self.pos += 1
# self.token = self.tokens[self.pos]
#
# if skip_whitespace:
# while self.token.type == TokenKind.WHITESPACE or self.token.type == TokenKind.NEWLINE:
# self.pos += 1
# self.token = self.tokens[self.pos]
#
# return self.token.type != TokenKind.EOF
#
# def seek(self, pos):
# self.pos = pos
# self.token = self.tokens[self.pos]
# return True
#
# def rewind(self, offset, skip_whitespace=True):
# self.pos += offset
# self.token = self.tokens[self.pos]
#
# if skip_whitespace:
# while self.pos > 0 and (self.token.type == TokenKind.WHITESPACE or self.token.type == TokenKind.NEWLINE):
# self.pos -= 1
# self.token = self.tokens[self.pos]
#
# def initialize(self, context, concepts_definitions):
# """
# Adds a bunch of concepts, and how they can be recognized
# :param context: execution context
# :param concepts_definitions: dictionary of concept, concept_definition
# :return:
# """
#
# self.context = context
# self.sheerka = context.sheerka
# concepts_to_resolve = set()
#
# for concept, concept_def in concepts_definitions.items():
# # ## Gets the grammars
# context.log(f"Resolving grammar for '{concept}'", context.who)
# concept.init_key() # make sure that the key is initialized
# grammar = self.get_model(concept_def, concepts_to_resolve)
# self.concepts_grammars[concept] = grammar
#
# if self.has_error:
# return self.sheerka.ret(self.name, False, self.error_sink)
#
# # ## Removes concepts with infinite recursions
# concepts_to_remove = self.detect_infinite_recursion(concepts_to_resolve)
# for concept in concepts_to_remove:
# concepts_to_resolve.remove(concept)
# del self.concepts_grammars[concept]
#
# if self.has_error:
# return self.sheerka.ret(self.name, False, self.error_sink)
# else:
# return self.sheerka.ret(self.name, True, self.concepts_grammars)
#
# def get_concept(self, concept_name):
# if concept_name in self.context.concepts:
# return self.context.concepts[concept_name]
# return self.sheerka.get_by_key(concept_name)
#
# def get_model(self, concept_def, concepts_to_resolve):
#
# # TODO
# # inner_get_model must not modify the initial ParsingExpression
# # A copy must be created
# def inner_get_model(expression):
# if isinstance(expression, Concept):
# if self.sheerka.isaset(self.context, expression):
# ret = ConceptGroupExpression(expression, rule_name=expression.name)
# else:
# ret = ConceptExpression(expression, rule_name=expression.name)
# concepts_to_resolve.add(expression)
# elif isinstance(expression, ConceptExpression): # it includes ConceptGroupExpression
# if expression.rule_name is None or expression.rule_name == "":
# expression.rule_name = expression.concept.name if isinstance(expression.concept, Concept) \
# else expression.concept
# if isinstance(expression.concept, str):
# concept = self.get_concept(expression.concept)
# if self.sheerka.is_known(concept):
# expression.concept = concept
# concepts_to_resolve.add(expression.concept)
# ret = expression
# elif isinstance(expression, str):
# ret = StrMatch(expression, ignore_case=self.ignore_case)
# elif isinstance(expression, StrMatch):
# ret = expression
# if ret.ignore_case is None:
# ret.ignore_case = self.ignore_case
# elif isinstance(expression, Sequence) or \
# isinstance(expression, OrderedChoice) or \
# isinstance(expression, ZeroOrMore) or \
# isinstance(expression, OneOrMore) or \
# isinstance(expression, Optional):
# ret = expression
# ret.nodes = [inner_get_model(e) for e in ret.elements]
# else:
# ret = self.add_error(GrammarErrorNode(f"Unrecognized grammar element '{expression}'."), False)
#
# # Translate separator expression.
# if isinstance(expression, Repetition) and expression.sep:
# expression.sep = inner_get_model(expression.sep)
#
# return ret
#
# model = inner_get_model(concept_def)
#
# return model
#
# def detect_infinite_recursion(self, concepts_to_resolve):
#
# # infinite recursion matcher
# def _is_infinite_recursion(ref_concept, node):
# if isinstance(node, ConceptExpression):
# if node.concept == ref_concept:
# return True
#
# if isinstance(node.concept, str):
# to_match = self.get_concept(node.concept)
# if self.sheerka.isinstance(to_match, BuiltinConcepts.UNKNOWN_CONCEPT):
# return False
# else:
# to_match = node.concept
#
# if to_match not in self.concepts_grammars:
# return False
#
# return _is_infinite_recursion(ref_concept, self.concepts_grammars[to_match])
#
# if isinstance(node, OrderedChoice):
# return _is_infinite_recursion(ref_concept, node.nodes[0])
#
# if isinstance(node, Sequence):
# for node in node.nodes:
# if _is_infinite_recursion(ref_concept, node):
# return True
# return False
#
# return False
#
# removed_concepts = []
# for e in concepts_to_resolve:
# if isinstance(e, str):
# e = self.get_concept(e)
# if self.sheerka.isinstance(e, BuiltinConcepts.UNKNOWN_CONCEPT):
# continue
#
# if e not in self.concepts_grammars:
# continue
#
# to_resolve = self.concepts_grammars[e]
# if _is_infinite_recursion(e, to_resolve):
# removed_concepts.append(e)
# return removed_concepts
#
# def parse(self, context, parser_input):
# if parser_input == "":
# return context.sheerka.ret(
# self.name,
# False,
# context.sheerka.new(BuiltinConcepts.IS_EMPTY)
# )
#
# if not self.reset_parser(context, parser_input):
# return self.sheerka.ret(
# self.name,
# False,
# context.sheerka.new(BuiltinConcepts.ERROR, body=self.error_sink))
#
# concepts_found = [[]]
# unrecognized_tokens = None
# has_unrecognized = False
#
# # actually list of list
# # The first dimension is the number of possibilities found
# # The second dimension is the number of concepts found, under one possibility
# #
# # Example 1
# # concept foo : 'one' 'two'
# # concept bar : 'one' 'two'
# # input 'one two' -> will produce two possibilities (foo and bar).
# #
# # Example 2
# # concept foo : 'one'
# # concept bar : 'two'
# # input 'one two' -> will produce one possibility which is (foo, bar) (foo then bar)
#
# while True:
# init_pos = self.pos
# res = []
#
# for concept, grammar in self.concepts_grammars.items():
# self.seek(init_pos)
# node = grammar.parse(self) # a node is TerminalNode or NonTerminalNode
# if node is not None and node.end != -1:
# updated_concept = self.finalize_concept(context.sheerka, concept, node)
# concept_node = ConceptNode(
# updated_concept,
# node.start,
# node.end,
# self.tokens[node.start: node.end + 1],
# None,
# node)
# res.append(concept_node)
#
# if len(res) == 0: # not recognized
# self.seek(init_pos)
# if unrecognized_tokens:
# unrecognized_tokens.add_token(self.get_token(), init_pos)
# else:
# unrecognized_tokens = UnrecognizedTokensNode(init_pos, init_pos, [self.get_token()])
#
# if not self.next_token(False):
# break
#
# else: # some concepts are recognized
# if unrecognized_tokens and unrecognized_tokens.not_whitespace():
# unrecognized_tokens.fix_source()
# concepts_found = core.utils.product(concepts_found, [unrecognized_tokens])
# has_unrecognized = True
# unrecognized_tokens = None
#
# res = self.get_bests(res) # only keep the concepts that eat the more tokens
# concepts_found = core.utils.product(concepts_found, res)
#
# # loop
# self.seek(res[0].end)
# if not self.next_token(False):
# break
#
# # Fix the source for unrecognized tokens
# if unrecognized_tokens and unrecognized_tokens.not_whitespace():
# unrecognized_tokens.fix_source()
# concepts_found = core.utils.product(concepts_found, [unrecognized_tokens])
# has_unrecognized = True
#
# # else
# # returns as many ReturnValue than choices found
# ret = []
# for choice in concepts_found:
# ret.append(
# self.sheerka.ret(
# self.name,
# not has_unrecognized,
# self.sheerka.new(
# BuiltinConcepts.PARSER_RESULT,
# parser=self,
# source=parser_input,
# body=choice,
# try_parsed=choice)))
#
# if len(ret) == 1:
# self.log_result(context, parser_input, ret[0])
# return ret[0]
# else:
# self.log_multiple_results(context, parser_input, ret)
# return ret
#
# def finalize_concept(self, sheerka, template, underlying, init_empty_body=True):
# """
# Updates the properties of the concept
# Goes in recursion if the property is a concept
# """
#
# # this cache is to make sure that we return the same concept for the same ConceptExpression
# _underlying_value_cache = {}
#
# def _add_prop(_concept, prop_name, value):
# """
# Adds a new entry,
# makes a list if the property already exists
# """
# if prop_name not in _concept.compiled or _concept.compiled[prop_name] is None:
# # new entry
# _concept.compiled[prop_name] = value
# else:
# # make a list if there was a value
# previous_value = _concept.compiled[prop_name]
# if isinstance(previous_value, list):
# previous_value.append(value)
# else:
# new_value = [previous_value, value]
# _concept.compiled[prop_name] = new_value
#
# def _look_for_concept_match(_underlying):
# """
# At some point, there is either an StrMatch or a ConceptMatch,
# that allowed the recognition.
# Look for the ConceptMatch, with recursion if needed
# """
# if isinstance(_underlying.parsing_expression, ConceptExpression):
# return _underlying
#
# if not isinstance(_underlying, NonTerminalNode):
# return None
#
# if len(_underlying.children) != 1:
# return None
#
# return _look_for_concept_match(_underlying.children[0])
#
# def _get_underlying_value(_underlying):
# concept_match_node = _look_for_concept_match(_underlying)
# if concept_match_node:
# # the value is a concept
# if id(concept_match_node) in _underlying_value_cache:
# result = _underlying_value_cache[id(concept_match_node)]
# else:
# ref_tpl = concept_match_node.parsing_expression.concept
# result = self.finalize_concept(sheerka, ref_tpl, concept_match_node.children[0], init_empty_body)
# _underlying_value_cache[id(concept_match_node)] = result
# else:
# # the value is a string
# result = DoNotResolve(_underlying.source)
#
# return result
#
# def _process_rule_name(_concept, _underlying):
# if _underlying.parsing_expression.rule_name:
# value = _get_underlying_value(_underlying)
# _add_prop(_concept, _underlying.parsing_expression.rule_name, value)
# _concept.metadata.need_validation = True
#
# if isinstance(_underlying, NonTerminalNode):
# for child in _underlying.children:
# _process_rule_name(_concept, child)
#
# key = (template.key, template.id) if template.id else template.key
# concept = sheerka.new(key)
# if init_empty_body and concept.metadata.body is None:
# value = _get_underlying_value(underlying)
# concept.compiled[ConceptParts.BODY] = value
# if underlying.parsing_expression.rule_name:
# _add_prop(concept, underlying.parsing_expression.rule_name, value)
# # KSI : Why don't we set concept.metadata.need_validation to True ?
#
# if isinstance(underlying, NonTerminalNode):
# for node in underlying.children:
# _process_rule_name(concept, node)
#
# return concept
#
# def encode_grammar(self, grammar):
# """
# Transform the grammar into something that can easily can be serialized
# :param grammar:
# :return:
# """
#
# def _encode(expression):
# if isinstance(expression, StrMatch):
# res = f"'{expression.to_match}'"
#
# elif isinstance(expression, ConceptExpression):
# res = core.utils.str_concept(expression.concept)
#
# elif isinstance(expression, Sequence):
# res = "(" + " ".join(_encode(c) for c in expression.nodes) + ")"
#
# elif isinstance(expression, OrderedChoice):
# res = "(" + "|".join(_encode(c) for c in expression.nodes) + ")"
#
# elif isinstance(expression, Optional):
# res = _encode(expression.nodes[0]) + "?"
#
# elif isinstance(expression, ZeroOrMore):
# res = _encode(expression.nodes[0]) + "*"
#
# elif isinstance(expression, OneOrMore):
# res = _encode(expression.nodes[0]) + "+"
#
# if expression.rule_name:
# res += "=" + expression.rule_name
#
# return res
#
# result = {}
# for k, v in grammar.items():
# key = core.utils.str_concept(k)
# value = _encode(v)
# result[key] = value
# return result
#
# @staticmethod
# def get_bests(results):
# """
# Returns the result that is the longest
# :param results:
# :return:
# """
# by_end_pos = defaultdict(list)
# for result in results:
# by_end_pos[result.end].append(result)
#
# return by_end_pos[max(by_end_pos)]
#
#
# class ParsingExpressionVisitor:
# """
# visit ParsingExpression
# """
#
# def visit(self, parsing_expression):
# name = parsing_expression.__class__.__name__
#
# method = 'visit_' + name
# visitor = getattr(self, method, self.generic_visit)
# return visitor(parsing_expression)
#
# def generic_visit(self, parsing_expression):
# if hasattr(self, "visit_all"):
# self.visit_all(parsing_expression)
#
# for node in parsing_expression.elements:
# if isinstance(node, Concept):
# self.visit(ConceptExpression(node.key or node.name))
# elif isinstance(node, str):
# self.visit(StrMatch(node))
# else:
# self.visit(node)
src/parsers/_ConceptsWithConceptsParser.py
+108
View File
@@ -0,0 +1,108 @@
# # try to match something like
# # ConceptNode 'plus' ConceptNode
# #
# # Replaced by SyaNodeParser
# from core.builtin_concepts import BuiltinConcepts
# from core.tokenizer import TokenKind, Token
# from parsers.BaseNodeParser import SourceCodeNode, ConceptNode, UnrecognizedTokensNode
# from parsers.BaseParser import BaseParser
# from parsers.MultipleConceptsParser import MultipleConceptsParser
# from core.concept import VARIABLE_PREFIX
#
# multiple_concepts_parser = MultipleConceptsParser()
#
#
# class ConceptsWithConceptsParser(BaseParser):
# def __init__(self, **kwargs):
# super().__init__("ConceptsWithConcepts", 25)
# self.enabled = False
#
# @staticmethod
# def get_tokens(nodes):
# tokens = []
#
# for node in nodes:
# if isinstance(node, ConceptNode):
# index, line, column = node.tokens[0].index, node.tokens[0].line, node.tokens[0].column
# tokens.append(Token(TokenKind.CONCEPT, node.concept, index, line, column))
# else:
# for token in node.tokens:
# if token.type == TokenKind.EOF:
# break
# elif token.type in (TokenKind.NEWLINE, TokenKind.WHITESPACE):
# continue
# else:
# tokens.append(token)
#
# return tokens
#
# @staticmethod
# def get_key(nodes):
# key = ""
# index = 0
# for node in nodes:
# if key:
# key += " "
#
# if isinstance(node, UnrecognizedTokensNode):
# key += node.source.strip()
# else:
# key += f"{VARIABLE_PREFIX}{index}"
# index += 1
#
# return key
#
# def finalize_concept(self, context, concept, nodes):
# index = 0
# for node in nodes:
#
# if isinstance(node, ConceptNode):
# prop_name = list(concept.props.keys())[index]
# concept.compiled[prop_name] = node.concept
# context.log(
# f"Setting property '{prop_name}='{node.concept}'.",
# self.name)
# index += 1
# elif isinstance(node, SourceCodeNode):
# prop_name = list(concept.props.keys())[index]
# sheerka = context.sheerka
# value = sheerka.new(BuiltinConcepts.PARSER_RESULT, parser=self, source=node.source, body=node.node)
# concept.compiled[prop_name] = [context.sheerka.ret(self.name, True, value)]
# context.log(
# f"Setting property '{prop_name}'='Python({node.source})'.",
# self.name)
# index += 1
#
# return concept
#
# def parse(self, context, parser_input):
# sheerka = context.sheerka
# nodes = self.get_input_as_lexer_nodes(parser_input, multiple_concepts_parser)
# if not nodes:
# return None
#
# concept_key = self.get_key(nodes)
# concept = sheerka.new(concept_key)
# if sheerka.isinstance(concept, BuiltinConcepts.UNKNOWN_CONCEPT):
# return sheerka.ret(
# self.name,
# False,
# sheerka.new(BuiltinConcepts.NOT_FOR_ME, body=parser_input.body))
#
# concepts = concept if hasattr(concept, "__iter__") else [concept]
# for concept in concepts:
# self.finalize_concept(context, concept, nodes)
#
# res = []
# for concept in concepts:
# res.append(sheerka.ret(
# self.name,
# True,
# sheerka.new(
# BuiltinConcepts.PARSER_RESULT,
# parser=self,
# source=parser_input.source,
# body=concept,
# try_parsed=None)))
#
# return res[0] if len(res) == 1 else res
src/parsers/_MultipleConceptsParser.py
+163
View File
@@ -0,0 +1,163 @@
# # to be replaced by SyaNodeParser
# import ast
#
# from core.builtin_concepts import BuiltinConcepts
# from core.tokenizer import TokenKind
# from parsers.BaseNodeParser import SourceCodeNode
# from parsers.BaseParser import BaseParser
# from parsers.BnfNodeParser import BnfNodeParser, UnrecognizedTokensNode, ConceptNode
# import core.utils
# from parsers.PythonParser import PythonParser
#
# concept_lexer_parser = BnfNodeParser()
#
#
# class MultipleConceptsParser(BaseParser):
# """
# Parser that will take the result of BnfNodeParser and
# try to resolve the unrecognized tokens token by token
#
# It is a success when it returns a list ConceptNode exclusively
# """
#
# def __init__(self, **kwargs):
# BaseParser.__init__(self, "MultipleConcepts", 45)
# self.enabled = False
#
# @staticmethod
# def finalize(nodes_found, unrecognized_tokens):
# if not unrecognized_tokens:
# return nodes_found, unrecognized_tokens
#
# unrecognized_tokens.fix_source()
# if unrecognized_tokens.not_whitespace():
# nodes_found = core.utils.product(nodes_found, [unrecognized_tokens])
#
# return nodes_found, None
#
# @staticmethod
# def create_or_add(unrecognized_tokens, token, index):
# if unrecognized_tokens:
# unrecognized_tokens.add_token(token, index)
# else:
# unrecognized_tokens = UnrecognizedTokensNode(index, index, [token])
# return unrecognized_tokens
#
# def parse(self, context, parser_input):
# sheerka = context.sheerka
# nodes = self.get_input_as_lexer_nodes(parser_input, concept_lexer_parser)
# if not nodes:
# return None
#
# nodes_found = [[]]
# concepts_only = True
#
# for node in nodes:
# if isinstance(node, UnrecognizedTokensNode):
# unrecognized_tokens = None
# i = 0
#
# while i < len(node.tokens):
#
# token_index = node.start + i
# token = node.tokens[i]
#
# concepts_nodes = self.get_concepts_nodes(context, token_index, token)
# if concepts_nodes is not None:
# nodes_found, unrecognized_tokens = self.finalize(nodes_found, unrecognized_tokens)
# nodes_found = core.utils.product(nodes_found, concepts_nodes)
# i += 1
# continue
#
# source_code_node = self.get_source_code_node(context, token_index, node.tokens[i:])
# if source_code_node:
# nodes_found, unrecognized_tokens = self.finalize(nodes_found, unrecognized_tokens)
# nodes_found = core.utils.product(nodes_found, [source_code_node])
# i += len(source_code_node.tokens)
# continue
#
# # not a concept nor some source code
# unrecognized_tokens = self.create_or_add(unrecognized_tokens, token, token_index)
# concepts_only &= token.type in (TokenKind.WHITESPACE, TokenKind.NEWLINE)
# i += 1
#
# # finish processing if needed
# nodes_found, unrecognized_tokens = self.finalize(nodes_found, unrecognized_tokens)
#
# else:
# nodes_found = core.utils.product(nodes_found, [node])
#
# ret = []
# for choice in nodes_found:
# ret.append(
# sheerka.ret(
# self.name,
# concepts_only,
# sheerka.new(
# BuiltinConcepts.PARSER_RESULT,
# parser=self,
# source=parser_input.source,
# body=choice,
# try_parsed=None))
# )
#
# if len(ret) == 1:
# self.log_result(context, parser_input.source, ret[0])
# return ret[0]
# else:
# self.log_multiple_results(context, parser_input.source, ret)
# return ret
#
# @staticmethod
# def get_concepts_nodes(context, index, token):
# """
# Tries to recognize a concept
# from the univers of all known concepts
# """
#
# if token.type != TokenKind.IDENTIFIER:
# return None
#
# concept = context.new_concept(token.value)
# if hasattr(concept, "__iter__") or context.sheerka.is_known(concept):
# concepts = concept if hasattr(concept, "__iter__") else [concept]
# concepts_nodes = [ConceptNode(c, index, index, [token], token.value) for c in concepts]
# return concepts_nodes
#
# return None
#
# @staticmethod
# def get_source_code_node(context, index, tokens):
# """
# Tries to recognize source code.
# For the time being, only Python is supported
# :param context:
# :param tokens:
# :param index:
# :return:
# """
#
# if len(tokens) == 0 or (len(tokens) == 1 and tokens[0].type == TokenKind.EOF):
# return None
#
# end_index = len(tokens)
# while end_index > 0:
# parser = PythonParser()
# tokens_to_parse = tokens[:end_index]
# res = parser.parse(context, tokens_to_parse)
# if res.status:
# # only expression are accepted
# ast_ = res.value.value.ast_
# if not isinstance(ast_, ast.Expression):
# return None
# try:
# compiled = compile(ast_, "<string>", "eval")
# eval(compiled, {}, {})
# except Exception:
# return None
#
# source = BaseParser.get_text_from_tokens(tokens_to_parse)
# return SourceCodeNode(res.value.value, index, index + end_index - 1, tokens_to_parse, source)
# end_index -= 1
#
# return None
src/printer/Formatter.py
+5
View File
@@ -4,6 +4,11 @@ from printer.FormatInstructions import FormatDetailDesc, FormatDetailType, Forma
class Formatter:
def __init__(self):
self.custom_l_formats = None
self.custom_d_formats = None
self.reset_formats()
def reset_formats(self):
self.custom_l_formats = {}
self.custom_d_formats = []
src/printer/SheerkaPrinter.py
+5
View File
@@ -29,10 +29,15 @@ class SheerkaPrinter:
self.sheerka = sheerka
self.formatter = Formatter()
self.formatter.register_format_l(EXECUTION_CONTEXT_CLASS, "[{id:3}] %tab%{desc} ({status})")
self.custom_concepts_printers = None
self.reset()
def reset(self):
self.custom_concepts_printers = {
str(BuiltinConcepts.EXPLANATION): self.print_explanation,
str(BuiltinConcepts.RETURN_VALUE): self.print_return_value,
}
self.formatter.reset_formats()
def register_custom_printer(self, concept, custom_format):
key = concept.key if isinstance(concept, Concept) else concept
src/sdp/sheerkaDataProvider.py
+155 -792
View File
File diff suppressed because it is too large Load Diff
src/sdp/sheerkaDataProviderIO.py
+6
View File
@@ -77,9 +77,11 @@ class SheerkaDataProviderFileIO(SheerkaDataProviderIO):
def write_text(self, file_path, content):
self._write(file_path, content, "w")
return len(content)
def write_binary(self, file_path, content):
self._write(file_path, content, "wb")
return len(content)
def exists(self, file_path):
return path.exists(file_path)
@@ -120,10 +122,12 @@ class SheerkaDataProviderMemoryIO(SheerkaDataProviderIO):
def write_binary(self, file_path, content):
self._ensure_parent_folder(file_path)
self.mem_fs.writebytes(file_path, content)
return len(content)
def write_text(self, file_path, content):
self._ensure_parent_folder(file_path)
self.mem_fs.writetext(file_path, content)
return len(content)
def remove(self, file_path):
self.mem_fs.remove(file_path)
@@ -155,9 +159,11 @@ class SheerkaDataProviderDictionaryIO(SheerkaDataProviderIO):
def write_binary(self, file_path, content):
self.cache[file_path] = content
return len(content)
def write_text(self, file_path, content):
self.cache[file_path] = content
return len(content)
def remove(self, file_path):
del (self.cache[file_path])
src/sdp/sheerkaDataProvider_Old.py
+1087
View File
File diff suppressed because it is too large Load Diff
src/sheerkapickle/sheerka_handlers.py
+19 -23
View File
@@ -23,25 +23,21 @@ class ConceptHandler(BaseHandler):
data[CONCEPT_ID] = (obj.key, obj.id)
# transform metadata
for prop in CONCEPT_PROPERTIES_TO_SERIALIZE:
value = getattr(obj.metadata, prop)
ref_value = getattr(ref.metadata, prop)
for name in CONCEPT_PROPERTIES_TO_SERIALIZE:
value = getattr(obj.metadata, name)
ref_value = getattr(ref.metadata, name)
if value != ref_value:
data["meta." + prop] = pickler.flatten(value)
value_to_use = [list(t) for t in value] if name == "variables" else value
data["meta." + name] = pickler.flatten(value_to_use)
# transform value
for metadata, value in obj.values.items():
ref_value = ref.values[metadata] if metadata in ref.values else None
if value != ref_value:
data[metadata.value] = pickler.flatten(value)
# transform properties
for prop in obj.props:
value = obj.props[prop].value
if prop not in ref.props or value != ref.props[prop].value:
if "props" not in data:
data["props"] = []
data["props"].append((pickler.flatten(prop), pickler.flatten(value)))
# # transform values
for name in obj.values:
value = obj.get_value(name)
if name not in ref.values or value != ref.get_value(name):
if "values" not in data:
data["values"] = []
key_to_use = "cParts." + name.value if isinstance(name, ConceptParts) else name
data["values"].append((pickler.flatten(key_to_use), pickler.flatten(value)))
return data
@@ -61,18 +57,18 @@ class ConceptHandler(BaseHandler):
if key.startswith("meta."):
# get metadata
resolved_prop = key[5:]
if resolved_prop == "props":
if resolved_prop == "variables":
for prop_name, prop_value in resolved_value:
instance.def_prop(prop_name, prop_value)
instance.def_var(prop_name, prop_value)
else:
setattr(instance.metadata, resolved_prop, resolved_value)
elif key == "props":
elif key == "values":
# get properties
for prop_name, prop_value in resolved_value:
instance.set_prop(prop_name, prop_value)
key_to_use = ConceptParts(prop_name[7:]) if isinstance(prop_name, str) and prop_name.startswith("cParts.") else prop_name
instance.set_value(key_to_use, prop_value)
else:
# get value
instance.set_metadata_value(ConceptParts(key), resolved_value)
raise Exception("Sanity check as it's not possible yet")
instance.freeze_definition_hash()
return instance