kodjo/Sheerka-Old
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fixed #3: Added sheerka.resolve_rule()

Browse Source 
Fixed #5: Refactored SheerkaComparisonManager
Fixed #6: Sya parser no longer works after restart
This commit is contained in:
Kodjo Sossouvi
2021-01-15 07:11:04 +01:00
parent e26c83a825
commit 821dbed189
44 changed files with 1617 additions and 1068 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/core/builtin_concepts_ids.py
+7 -6
View File
@@ -75,6 +75,7 @@ class BuiltinConcepts:
INVALID_RETURN_VALUE = "__INVALID_RETURN_VALUE" # the return value of an evaluator is not correct
CONCEPT_ALREADY_DEFINED = "__CONCEPT_ALREADY_DEFINED" # when you try to add the same object twice (a concept or whatever)
PROPERTY_ALREADY_DEFINED = "__PROPERTY_ALREADY_DEFINED" # When you try to add the same element in a property
COMPARISON_ALREADY_DEFINED = "__COMPARISON_ALREADY_DEFINED" # when the same comparison entry is defined several times
NOP = "__NOP" # no operation concept. Does nothing
CONCEPT_EVAL_ERROR = "__CONCEPT_EVAL_ERROR" # cannot evaluate a property or metadata of a concept
ENUMERATION = "__ENUMERATION" # represents a list or a set
@@ -91,8 +92,8 @@ class BuiltinConcepts:
FORMAT_INSTRUCTIONS = "__FORMAT_INSTRUCTIONS" # to express how to print the concept
NOT_IMPLEMENTED = "__NOT_IMPLEMENTED" # instead of raise an error
PYTHON_SECURITY_ERROR = "__PYTHON_SECURITY_ERROR" # when trying to execute statement when only expression is allowed
INVALID_LESSER_OPERATION = "__INVALID_LESSER_OPERATION"
INVALID_GREATEST_OPERATION = "__INVALID_GREATEST_OPERATION"
IS_LESSER_CONSTRAINT_ERROR = "__IS_LESSER_CONSTRAINT_ERROR"
IS_GREATEST_CONSTRAINT_ERROR = "__IS_GREATEST_CONSTRAINT_ERROR"
NEW_RULE = "__NEW_RULE"
UNKNOWN_RULE = "__UNKNOWN_RULE"
ONTOLOGY_ALREADY_DEFINED = "__ONTOLOGY_ALREADY_DEFINED"
@@ -145,8 +146,8 @@ BuiltinUnique = [
BuiltinConcepts.ISA,
BuiltinConcepts.AUTO_EVAL,
BuiltinConcepts.INVALID_LESSER_OPERATION,
BuiltinConcepts.INVALID_GREATEST_OPERATION,
BuiltinConcepts.IS_LESSER_CONSTRAINT_ERROR,
BuiltinConcepts.IS_GREATEST_CONSTRAINT_ERROR,
]
BuiltinErrors = [
@@ -167,8 +168,8 @@ BuiltinErrors = [
BuiltinConcepts.CONDITION_FAILED,
BuiltinConcepts.CHICKEN_AND_EGG,
BuiltinConcepts.NOT_FOUND,
BuiltinConcepts.INVALID_LESSER_OPERATION,
BuiltinConcepts.INVALID_GREATEST_OPERATION,
BuiltinConcepts.IS_LESSER_CONSTRAINT_ERROR,
BuiltinConcepts.IS_GREATEST_CONSTRAINT_ERROR,
BuiltinConcepts.ONTOLOGY_ALREADY_DEFINED
]
src/core/builtin_helpers.py
+23 -1
View File
@@ -12,7 +12,7 @@ from core.tokenizer import Keywords
from core.utils import as_bag
from parsers.BaseNodeParser import SourceCodeNode, ConceptNode, UnrecognizedTokensNode, SourceCodeWithConceptNode, \
RuleNode
from parsers.BaseParser import BaseParser, ParsingError
from parsers.BaseParser import ParsingError
PARSE_STEPS = [BuiltinConcepts.BEFORE_PARSING, BuiltinConcepts.PARSING, BuiltinConcepts.AFTER_PARSING]
EVAL_STEPS = PARSE_STEPS + [BuiltinConcepts.BEFORE_EVALUATION, BuiltinConcepts.EVALUATION,
@@ -655,6 +655,28 @@ def ensure_concept_or_rule(*items):
raise TypeError(f"'{items}' must be a concept or rule")
def ensure_bnf(context, concept, parser_name="BaseNodeParser"):
if concept.get_metadata().definition_type == DEFINITION_TYPE_BNF and not concept.get_bnf():
from parsers.BnfDefinitionParser import BnfDefinitionParser
regex_parser = BnfDefinitionParser()
desc = f"Resolving BNF '{concept.get_metadata().definition}'"
with context.push(BuiltinConcepts.INIT_BNF,
concept,
who=parser_name,
obj=concept,
desc=desc) as sub_context:
sub_context.add_inputs(parser_input=concept.get_metadata().definition)
bnf_parsing_ret_val = regex_parser.parse(sub_context, concept.get_metadata().definition)
sub_context.add_values(return_values=bnf_parsing_ret_val)
if not bnf_parsing_ret_val.status:
raise Exception(bnf_parsing_ret_val.value)
concept.set_bnf(bnf_parsing_ret_val.body.body)
if concept.id:
context.sheerka.get_by_id(concept.id).set_bnf(concept.get_bnf()) # update bnf in cache
expressions_cache = Cache()
src/core/rule.py
+2 -2
View File
@@ -5,17 +5,17 @@ import core.utils
ACTION_TYPE_PRINT = "print"
ACTION_TYPE_EXEC = "exec"
ACTION_TYPE_DEFERRED = "deferred" # KSI 2021-04-01 What is it for ? I definitely need some proper documentation
@dataclass
class RuleMetadata:
action_type: str # print, exec, deferred
action_type: str # print, exec
name: Union[str, None]
predicate: str
action: str
id: str = None
id_is_unresolved = False
is_compiled: bool = False
is_enabled: bool = False
src/core/sheerka/Sheerka.py
-17
View File
@@ -49,8 +49,6 @@ class Sheerka(Concept):
CONCEPTS_BY_ID_ENTRY = "ConceptManager:Concepts_By_ID"
CONCEPTS_BY_NAME_ENTRY = "ConceptManager:Concepts_By_Name"
CONCEPTS_BY_FIRST_KEYWORD_ENTRY = "Concepts_By_First_Keyword"
RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY = "Resolved_Concepts_By_First_Keyword"
CONCEPTS_SYA_DEFINITION_ENTRY = "Concepts_Sya_Definitions"
RESOLVED_CONCEPTS_SYA_DEFINITION_ENTRY = "Resolved_Concepts_Sya_Definitions"
CONCEPTS_GRAMMARS_ENTRY = "Concepts_Grammars"
@@ -183,7 +181,6 @@ class Sheerka(Concept):
self.first_time_initialisation(exec_context)
self.initialize_builtin_concepts()
self.initialize_concept_node_parsing(exec_context)
self.initialize_services_deferred(exec_context, self.om.current_sdp().first_time)
@@ -215,17 +212,10 @@ class Sheerka(Concept):
cache = IncCache().auto_configure(self.OBJECTS_IDS_ENTRY)
self.om.register_cache(self.OBJECTS_IDS_ENTRY, cache)
cache = DictionaryCache().auto_configure(self.CONCEPTS_BY_FIRST_KEYWORD_ENTRY)
self.om.register_cache(self.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, cache)
self.om.get(self.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, None) # to init from sdp
cache = DictionaryCache().auto_configure(self.CONCEPTS_SYA_DEFINITION_ENTRY)
self.om.register_cache(self.CONCEPTS_SYA_DEFINITION_ENTRY, cache)
self.om.get(self.CONCEPTS_SYA_DEFINITION_ENTRY, None) # to init from sdp
cache = DictionaryCache().auto_configure(self.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY)
self.om.register_cache(self.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY, cache, persist=False)
cache = DictionaryCache().auto_configure(self.RESOLVED_CONCEPTS_SYA_DEFINITION_ENTRY)
self.om.register_cache(self.RESOLVED_CONCEPTS_SYA_DEFINITION_ENTRY, cache, persist=False)
@@ -331,13 +321,6 @@ class Sheerka(Concept):
if hasattr(evaluator, "initialize"):
evaluator.initialize(self)
def initialize_concept_node_parsing(self, context):
# self.init_log.debug("Initializing concepts by first keyword.")
concepts_by_first_keyword = self.om.current_cache_manager().copy(self.CONCEPTS_BY_FIRST_KEYWORD_ENTRY)
res = self.bnp.resolve_concepts_by_first_keyword(context, concepts_by_first_keyword)
self.om.put(self.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY, False, res.body)
def initialize_ontologies(self, context):
ontologies = self.om.current_sdp().load_ontologies()
if not ontologies:
src/core/sheerka/SheerkaOntologyManager.py
+42 -42
View File
@@ -311,48 +311,48 @@ class SheerkaOntologyManager:
"""
return list(self.get_all(entry, cache_only).values())
def list_by_key(self, entry, key):
"""
List all entries of a given key
If the values are lists, sets of dictionaries, they will be concatenated
Otherwise it will raise an error
"""
res = None
def update_values(_res, values_):
if values_ is NotFound:
return _res
elif values_ is Removed:
_res.clear()
elif isinstance(values_, dict):
if _res is None:
_res = values_.copy()
elif isinstance(_res, dict):
_res.update(values_)
else:
raise ValueError(f"Expecting dict while found '{values_}'")
elif isinstance(values_, list):
if _res is None:
_res = values_.copy()
elif isinstance(_res, list):
_res.extend(values_)
else:
raise ValueError(f"Expecting list while found '{values_}'")
else:
raise NotImplementedError()
return _res
for ontology in reversed(self.ontologies):
from_cache_values = ontology.cache_manager.get(entry, key)
if from_cache_values is not NotFound:
res = update_values(res, from_cache_values)
else:
from_sdp_values = ontology.cache_manager.sdp.get(entry, key)
res = update_values(res, from_sdp_values)
return res
# def list_by_key(self, entry, key):
# """
# List all entries of a given key
# If the values are lists, sets of dictionaries, they will be concatenated
# Otherwise it will raise an error
# """
# res = None
#
# def update_values(_res, values_):
# if values_ is NotFound:
# return _res
# elif values_ is Removed:
# _res.clear()
# elif isinstance(values_, dict):
# if _res is None:
# _res = values_.copy()
# elif isinstance(_res, dict):
# _res.update(values_)
# else:
# raise ValueError(f"Expecting dict while found '{values_}'")
# elif isinstance(values_, list):
# if _res is None:
# _res = values_.copy()
# elif isinstance(_res, list):
# _res.extend(values_)
# else:
# raise ValueError(f"Expecting list while found '{values_}'")
# else:
# raise NotImplementedError()
#
# return _res
#
# for ontology in reversed(self.ontologies):
#
# from_cache_values = ontology.cache_manager.get(entry, key)
# if from_cache_values is not NotFound:
# res = update_values(res, from_cache_values)
# else:
# from_sdp_values = ontology.cache_manager.sdp.get(entry, key)
# res = update_values(res, from_sdp_values)
#
# return res
def get_all(self, entry, cache_only=False):
"""
src/core/sheerka/services/SheerkaAdmin.py
+14 -1
View File
@@ -1,11 +1,11 @@
import sys
import time
from operator import attrgetter
from os import path
from core.builtin_concepts_ids import BuiltinConcepts, BuiltinContainers
from core.builtin_helpers import ensure_concept
from core.concept import Concept
from core.sheerka.services.SheerkaMemory import SheerkaMemory
from core.sheerka.services.sheerka_service import BaseService
CONCEPTS_FILE_LITE = "_concepts_lite.txt"
@@ -31,6 +31,7 @@ class SheerkaAdmin(BaseService):
self.sheerka.bind_service_method(self.admin_push_ontology, True, as_name="push_ontology")
self.sheerka.bind_service_method(self.admin_pop_ontology, True, as_name="pop_ontology")
self.sheerka.bind_service_method(self.ontologies, False)
self.sheerka.bind_service_method(self.in_memory, False)
def caches_names(self):
"""
@@ -185,3 +186,15 @@ class SheerkaAdmin(BaseService):
def ontologies(self):
ontologies = self.sheerka.om.ontologies_names
return self.sheerka.new(BuiltinConcepts.TO_LIST, body=ontologies)
def in_memory(self):
"""
Returns the list of all obj in memory
"""
res = {}
for k, obj in self.sheerka.om.get_all(SheerkaMemory.OBJECTS_ENTRY).items():
if isinstance(obj, list):
obj = obj[-1]
res[k] = obj.obj
return self.sheerka.ret(self.NAME, True, self.sheerka.new(BuiltinConcepts.TO_DICT, body=res))
src/core/sheerka/services/SheerkaComparisonManager.py
+292 -180
View File
@@ -1,15 +1,16 @@
from dataclasses import dataclass
from functools import reduce
from cache.Cache import Cache
from cache.ListCache import ListCache
from core.builtin_concepts import BuiltinConcepts
from core.global_symbols import EVENT_CONCEPT_PRECEDENCE_MODIFIED, EVENT_RULE_PRECEDENCE_MODIFIED, \
RULE_COMPARISON_CONTEXT, \
CONCEPT_COMPARISON_CONTEXT, NotFound
from core.builtin_helpers import ensure_concept_or_rule
from core.concept import Concept
from core.sheerka.services.SheerkaRuleManager import SheerkaRuleManager
from core.global_symbols import EVENT_CONCEPT_PRECEDENCE_MODIFIED, EVENT_RULE_PRECEDENCE_MODIFIED, \
RULE_COMPARISON_CONTEXT, CONCEPT_COMPARISON_CONTEXT, NotInit, NotFound
from core.sheerka.services.SheerkaConceptManager import ChickenAndEggError
from core.sheerka.services.sheerka_service import ServiceObj, BaseService
from core.utils import flatten
@dataclass
@@ -18,8 +19,8 @@ class ComparisonObj(ServiceObj):
Order to store
"""
property: str # property to compare
a: int # id of concept a
b: int # id of concept b
a: str # str_id of concept a
b: str # str_id of concept b
op: str # comparison operation
context: str = "#" # context when the comparison is right
@@ -39,157 +40,6 @@ class SheerkaComparisonManager(BaseService):
def __init__(self, sheerka):
super().__init__(sheerka)
@staticmethod
def _compute_key(prop_name, comparison_context):
"""
Key to use to store the comparisons
:param prop_name:
:param comparison_context:
:return:
"""
if isinstance(prop_name, Concept):
prefix = prop_name.key if prop_name.get_metadata().is_builtin else prop_name.id
else:
prefix = prop_name
return f"{prefix}|{comparison_context}"
@staticmethod
def _get_weights(comparison_objs):
"""
For every element in greater_than_s, give it a weight
if weight(a) > weight(b) it means that a > b
:param comparison_objs: list of greater than objects
:return:
"""
values = {}
for comparison_obj in comparison_objs:
values[comparison_obj.a] = SheerkaComparisonManager.DEFAULT_COMPARISON_VALUE
values[comparison_obj.b] = SheerkaComparisonManager.DEFAULT_COMPARISON_VALUE
for _ in range(len(comparison_objs)):
for comparison_obj in comparison_objs:
if comparison_obj.op == ">":
if values[comparison_obj.a] <= values[comparison_obj.b]:
values[comparison_obj.a] = values[comparison_obj.b] + 1
else:
if values[comparison_obj.b] <= values[comparison_obj.a]:
values[comparison_obj.b] = values[comparison_obj.a] + 1
return values
def _compute_weights(self, comparison_objs, lesser_objs_ids=None, greatest_objs_ids=None):
"""
:param comparison_objs:
:return:
"""
def is_not_in_objs(obj, objs_ids):
return obj.op in ("<", ">") and obj.a not in objs_ids and obj.b not in objs_ids
def is_in_objs(obj, objs_ids):
return obj.op in ("<", ">") and (obj.a in objs_ids or obj.b in objs_ids)
lesser_objs_ids = lesser_objs_ids or {co.a for co in comparison_objs if co.op == "<<"}
greatest_objs_ids = greatest_objs_ids or {co.a for co in comparison_objs if co.op == ">>"}
default_weight = SheerkaComparisonManager.DEFAULT_COMPARISON_VALUE
# get the weights for all the lesser
lesser_objs = [co for co in comparison_objs if is_in_objs(co, lesser_objs_ids)]
lesser_objs_weights = self._get_weights(lesser_objs)
# rearrange the weight to have the highest equals to DEFAULT_COMPARISON_VALUE - 1
highest_weight = len(lesser_objs_weights)
for co_id in lesser_objs_weights:
lesser_objs_weights[co_id] = lesser_objs_weights[co_id] - highest_weight + default_weight - 1
for concept_id in lesser_objs_ids:
if concept_id not in lesser_objs_weights:
lesser_objs_weights[concept_id] = default_weight - 1
# get the weights for concepts that are not lesser or greatest
in_between_objs = [o for o in comparison_objs if is_not_in_objs(o, lesser_objs_ids | greatest_objs_ids)]
in_between_weights = self._get_weights(in_between_objs)
# get the weights for all the greatest
greatest_objs = [co for co in comparison_objs if is_in_objs(co, greatest_objs_ids)]
greatest_objs_weights = self._get_weights(greatest_objs)
# rearrange the weight to have the lowest equals to DEFAULT_COMPARISON_VALUE + 1
highest_weight = max(default_weight, len(in_between_weights))
for co_id in greatest_objs_weights:
greatest_objs_weights[co_id] = greatest_objs_weights[co_id] + highest_weight
for concept_id in greatest_objs_ids:
if concept_id not in greatest_objs_weights:
greatest_objs_weights[concept_id] = highest_weight + 1
return {**lesser_objs_weights, **in_between_weights, **greatest_objs_weights}
@staticmethod
def _get_partition(weighted_concepts):
res = {}
for k, v in weighted_concepts.items():
res.setdefault(v, []).append(k)
return res
def _add_comparison(self, context, comparison_obj):
key = self._compute_key(comparison_obj.property, comparison_obj.context)
previous = self.sheerka.om.get(self.COMPARISON_ENTRY, key)
new = previous.copy() if isinstance(previous, list) else []
for co in new:
if co.property == comparison_obj.property and \
co.a == comparison_obj.a and \
co.b == comparison_obj.b and \
co.op == comparison_obj.op and \
co.context == comparison_obj.context:
return self.sheerka.ret(self.NAME, False, self.sheerka.new(BuiltinConcepts.CONCEPT_ALREADY_DEFINED))
new.append(comparison_obj)
lesser_objs_ids = {co.a for co in new if co.op == "<<"}
greatest_objs_ids = {co.a for co in new if co.op == ">>"}
# check if it is a valid operation regarding other lesser and greatest concepts
if comparison_obj.op in ("<", ">"):
a_is_lesser = comparison_obj.a in lesser_objs_ids
b_is_lesser = comparison_obj.b in lesser_objs_ids
if a_is_lesser != b_is_lesser: # XOR operation
return self.sheerka.ret(self.NAME, False, self.sheerka.new(BuiltinConcepts.INVALID_LESSER_OPERATION))
a_is_greatest = comparison_obj.a in greatest_objs_ids
b_is_greatest = comparison_obj.b in greatest_objs_ids
if a_is_greatest != b_is_greatest: # XOR operation
return self.sheerka.ret(self.NAME, False, self.sheerka.new(BuiltinConcepts.INVALID_GREATEST_OPERATION))
if comparison_obj.op == "<<" and comparison_obj.a in greatest_objs_ids:
return self.sheerka.ret(self.NAME, False, self.sheerka.new(BuiltinConcepts.INVALID_GREATEST_OPERATION))
if comparison_obj.op == ">>" and comparison_obj.a in lesser_objs_ids:
return self.sheerka.ret(self.NAME, False, self.sheerka.new(BuiltinConcepts.INVALID_LESSER_OPERATION))
cycles = self.detect_cycles(new)
if cycles:
concepts_in_cycle = [self.sheerka.fast_resolve(c) for c in cycles]
chicken_an_egg = self.sheerka.new(BuiltinConcepts.CHICKEN_AND_EGG, body=concepts_in_cycle)
return self.sheerka.ret(self.NAME, False, chicken_an_egg)
self.sheerka.om.put(self.COMPARISON_ENTRY, key, comparison_obj)
self.sheerka.om.put(self.RESOLVED_COMPARISON_ENTRY, key, self._compute_weights(new,
lesser_objs_ids,
greatest_objs_ids))
if comparison_obj.property == BuiltinConcepts.PRECEDENCE:
if comparison_obj.context == CONCEPT_COMPARISON_CONTEXT:
self.sheerka.publish(context, EVENT_CONCEPT_PRECEDENCE_MODIFIED)
elif comparison_obj.context == RULE_COMPARISON_CONTEXT:
self.sheerka.publish(context, EVENT_RULE_PRECEDENCE_MODIFIED)
return self.sheerka.ret(self.NAME, True, self.sheerka.new(BuiltinConcepts.SUCCESS))
def initialize(self):
cache = ListCache().auto_configure(self.COMPARISON_ENTRY)
self.sheerka.om.register_cache(self.COMPARISON_ENTRY, cache, True, True)
@@ -202,7 +52,48 @@ class SheerkaComparisonManager(BaseService):
self.sheerka.bind_service_method(self.set_is_lesser, True)
self.sheerka.bind_service_method(self.set_is_greatest, True)
self.sheerka.bind_service_method(self.get_partition, False)
self.sheerka.bind_service_method(self.get_concepts_weights, False)
self.sheerka.bind_service_method(self.get_weights, False)
@staticmethod
def _compute_key(prop_name, comparison_context):
"""
Key to use to store the comparisons
:param prop_name:
:param comparison_context:
:return:
"""
prefix = SheerkaComparisonManager._get_real_prop_name(prop_name)
return f"{prefix}|{comparison_context}"
@staticmethod
def _get_real_prop_name(prop_name):
if isinstance(prop_name, Concept):
return prop_name.key if prop_name.get_metadata().is_builtin else prop_name.str_id
return str(prop_name)
def _add_comparison_new(self, context, comparison_obj):
key = self._compute_key(comparison_obj.property, comparison_obj.context)
previous_entries = self.sheerka.om.get(self.COMPARISON_ENTRY, key)
previous_entries = previous_entries.copy() if isinstance(previous_entries, list) else []
res = self.validate_new_entry(context, comparison_obj, previous_entries)
if not res.status:
return res
previous_entries.append(comparison_obj)
weights = self.compute_weights(previous_entries)
self.sheerka.om.put(self.COMPARISON_ENTRY, key, comparison_obj)
self.sheerka.om.put(self.RESOLVED_COMPARISON_ENTRY, key, weights)
if comparison_obj.property == BuiltinConcepts.PRECEDENCE:
if comparison_obj.context == CONCEPT_COMPARISON_CONTEXT:
self.sheerka.publish(context, EVENT_CONCEPT_PRECEDENCE_MODIFIED)
elif comparison_obj.context == RULE_COMPARISON_CONTEXT:
self.sheerka.publish(context, EVENT_RULE_PRECEDENCE_MODIFIED)
return self.sheerka.ret(self.NAME, True, self.sheerka.new(BuiltinConcepts.SUCCESS))
def set_is_greater_than(self, context, prop_name, item_a, item_b, comparison_context="#"):
"""
@@ -217,14 +108,15 @@ class SheerkaComparisonManager(BaseService):
context.log(f"Setting item {item_a} is greater than {item_b}", who=self.NAME)
ensure_concept_or_rule(item_a, item_b)
real_prop_name = self._get_real_prop_name(prop_name)
event_digest = context.event.get_digest()
comparison_obj = ComparisonObj(event_digest,
prop_name,
real_prop_name,
item_a.str_id,
item_b.str_id,
">",
comparison_context)
return self._add_comparison(context, comparison_obj)
return self._add_comparison_new(context, comparison_obj)
def set_is_less_than(self, context, prop_name, item_a, item_b, comparison_context="#"):
"""
@@ -239,14 +131,15 @@ class SheerkaComparisonManager(BaseService):
context.log(f"Setting item {item_a} is less than {item_b}", who=self.NAME)
ensure_concept_or_rule(item_a, item_b)
real_prop_name = self._get_real_prop_name(prop_name)
event_digest = context.event.get_digest()
comparison_obj = ComparisonObj(event_digest,
prop_name,
real_prop_name,
item_a.str_id,
item_b.str_id,
"<",
comparison_context)
return self._add_comparison(context, comparison_obj)
return self._add_comparison_new(context, comparison_obj)
def set_is_lesser(self, context, prop_name, item, comparison_context="#"):
"""
@@ -265,14 +158,15 @@ class SheerkaComparisonManager(BaseService):
context.log(f"Setting item {item} is lesser", who=self.NAME)
ensure_concept_or_rule(item)
real_prop_name = self._get_real_prop_name(prop_name)
event_digest = context.event.get_digest()
comparison_obj = ComparisonObj(event_digest,
prop_name,
real_prop_name,
item.str_id,
None,
"<<",
comparison_context)
return self._add_comparison(context, comparison_obj)
return self._add_comparison_new(context, comparison_obj)
def set_is_greatest(self, context, prop_name, item, comparison_context="#"):
"""
@@ -291,14 +185,15 @@ class SheerkaComparisonManager(BaseService):
context.log(f"Setting item {item} is greatest", who=self.NAME)
ensure_concept_or_rule(item)
real_prop_name = self._get_real_prop_name(prop_name)
event_digest = context.event.get_digest()
comparison_obj = ComparisonObj(event_digest,
prop_name,
real_prop_name,
item.str_id,
None,
">>",
comparison_context)
return self._add_comparison(context, comparison_obj)
return self._add_comparison_new(context, comparison_obj)
def set_are_equivalent(self, context, prop_name, concept_a, concept_b, comparison_context="#"):
"""
@@ -321,11 +216,14 @@ class SheerkaComparisonManager(BaseService):
:param comparison_context:
:return:
"""
weighted_concept = self.get_concepts_weights(prop_name, comparison_context)
weighted_concept = self.get_weights(prop_name, comparison_context)
return self._get_partition(weighted_concept)
res = {}
for k, v in weighted_concept.items():
res.setdefault(v, []).append(k)
return res
def get_concepts_weights(self, prop_name, comparison_context="#"):
def get_weights(self, prop_name, comparison_context="#"):
# KSI 2021-01-10 This implementation seems to be too complicated
# Chances are that there is a better way to implement this.
# Note that I don't want to use a DictionaryCache for the RESOLVED_COMPARISON_ENTRY
@@ -340,32 +238,245 @@ class SheerkaComparisonManager(BaseService):
else:
# otherwise, either it's not computed yet or it does not include the info of the current layer
# In both case, it is safer to recompute the weights
entries = self.sheerka.om.list_by_key(self.COMPARISON_ENTRY, key_to_use)
if entries is None:
entries = self.sheerka.om.get(self.COMPARISON_ENTRY, key_to_use)
if entries is NotFound:
weighted_concepts = {} # Why not put it in cache ???
else:
weighted_concepts = self._compute_weights(entries)
weighted_concepts = self.compute_weights(entries)
self.sheerka.om.put(self.RESOLVED_COMPARISON_ENTRY, key_to_use, weighted_concepts)
return weighted_concepts
@staticmethod
def detect_cycles(comparison_objs):
def toposort(comparison_objs):
"""
Topological sort or topological ordering of a directed graph is a linear ordering of its vertices
such that for every directed edge uv from vertex u to vertex v, u comes before v in the ordering
https://en.wikipedia.org/wiki/Topological_sorting
This implementation is taken from https://code.activestate.com/recipes/578272-topological-sort/
"""
if not comparison_objs:
return []
data = {}
for obj in comparison_objs:
data.setdefault(obj.a, set()).add(obj.b) if obj.op == ">" else data.setdefault(obj.b, set()).add(obj.a)
# Ignore self dependencies.
for k, v in data.items():
v.discard(k)
# Find all items that don't depend on anything.
extra_items_in_deps = reduce(set.union, data.values()) - set(data.keys())
# Add empty dependencies where needed
data.update({item: set() for item in extra_items_in_deps})
while True:
ordered = set(item for item, dep in data.items() if not dep)
if not ordered:
break
yield ordered
data = {item: (dep - ordered)
for item, dep in data.items()
if item not in ordered}
if len(data) != 0:
raise ChickenAndEggError(set(flatten(data.values())))
@staticmethod
def get_objs_groups(comparison_objs):
"""
Browse all ComparisonObj definitions group the obj in three categories
* those which are defined as greatest,
* those which are defined as lowest,
* those which are defined as equivalent as another one,
We also returns the list of all object
-> Which make it a tuple of four elements
"""
greatest = set()
lowest = set()
all_items = set()
for co in comparison_objs:
if co.op == ">>":
greatest.add(co.a)
elif co.op == "<<":
lowest.add(co.a)
if co.a is not None:
all_items.add(co.a)
if co.b is not None:
all_items.add(co.b)
return greatest, lowest, all_items
@staticmethod
def group_comparison_objs(comparison_objs, greatest_objs, lowest_objs):
"""
Groups the ComparisonObj in three categories
* those which express relation between greatest objects
* those which express relation between lowest objects
* those which express relation between objects which are neither greatest or lowest
To express a relation between greatest objects, both objects must be flagged as greatest
for example:
if the relation is 'a > b' with a is greatest, but not b,
this relation will NOT go in the greatest list
As a matter of fact, it does not fit in any list as it's an irrelevant information
Note that for efficiency, we consider that the settings are already valid
We cannot find something like
'a > b' with 'a' is lowest and 'b' is greatest
or 'a > b' 'b' lowest and 'a' is not
"""
greatest, lowest, equiv, other = [], [], [], [] # we keep the order
greatest_or_lowest_objs = set(greatest_objs)
greatest_or_lowest_objs.update(lowest_objs)
for co in comparison_objs:
if co.op in ("<", ">"):
if co.a in greatest_objs and co.b in greatest_objs:
greatest.append(co)
elif co.a in lowest_objs and co.b in lowest_objs:
lowest.append(co)
elif not (co.a in greatest_or_lowest_objs or co.b in greatest_or_lowest_objs):
other.append(co)
elif co.op == "=":
equiv.append(co)
return greatest, lowest, equiv, other
@staticmethod
def compute_weights(comparison_objs):
"""
Computes the weights of all items that appear in the comparison objs
"""
greatest_objs, lowest_objs, all_objs = SheerkaComparisonManager.get_objs_groups(comparison_objs)
# co stands for comparison_object
co_greatest, co_lowest, co_equiv, co_other = SheerkaComparisonManager.group_comparison_objs(comparison_objs,
greatest_objs,
lowest_objs)
weights = {obj: NotInit for obj in all_objs}
max_weight = SheerkaComparisonManager.DEFAULT_COMPARISON_VALUE
min_weight = SheerkaComparisonManager.DEFAULT_COMPARISON_VALUE
# manage lowest objects group
sorted_objs = list(SheerkaComparisonManager.toposort(co_lowest))
for group in reversed(sorted_objs):
group_weight = min_weight - 1
min_weight = group_weight
for item in group:
weights[item] = group_weight
# manage normal objects group
sorted_objs = SheerkaComparisonManager.toposort(co_other)
for i, group in enumerate(sorted_objs):
group_weight = i + SheerkaComparisonManager.DEFAULT_COMPARISON_VALUE
max_weight = group_weight
for item in group:
weights[item] = group_weight
# manage greatest objects group
sorted_objs = SheerkaComparisonManager.toposort(co_greatest)
for group in sorted_objs:
group_weight = max_weight + 1
max_weight = group_weight
for item in group:
weights[item] = group_weight
# manage greatest flagged objects that have not been initialized yet
for obj in greatest_objs:
if weights[obj] is NotInit:
weights[obj] = max_weight + 1
# manage lowest flagged objects that have not been initialized yet
for obj in lowest_objs:
if weights[obj] is NotInit:
weights[obj] = min_weight - 1
# manage equiv
for co in co_equiv:
if weights[co.a] is NotInit and weights[co.b] is not NotInit:
weights[co.a] = weights[co.b]
elif weights[co.b] is NotInit and weights[co.a] is not NotInit:
weights[co.b] = weights[co.a]
# set not initialized to default value
for k, v in weights.items():
if v is NotInit:
weights[k] = SheerkaComparisonManager.DEFAULT_COMPARISON_VALUE
return weights
@staticmethod
def validate_new_entry(context, new_entry, previous_entries):
sheerka = context.sheerka
name = SheerkaComparisonManager.NAME
# detect duplicates
for co in previous_entries:
if co.property == new_entry.property and \
co.a == new_entry.a and \
co.b == new_entry.b and \
co.op == new_entry.op and \
co.context == new_entry.context:
return sheerka.ret(name, False, sheerka.new(BuiltinConcepts.COMPARISON_ALREADY_DEFINED))
# detect cycle
cycles = SheerkaComparisonManager.detect_cycles(previous_entries, new_entry)
if cycles:
chicken_an_egg = context.sheerka.new(BuiltinConcepts.CHICKEN_AND_EGG, body=cycles)
return sheerka.ret(name, False, chicken_an_egg)
greatest_objs, lowest_objs, all_objs = SheerkaComparisonManager.get_objs_groups(previous_entries)
# detect IS_GREATEST_CONSTRAINT_ERROR and IS_LESSER_CONSTRAINT_ERROR
if new_entry.op == "<<" and new_entry.a in greatest_objs:
return sheerka.ret(name, False, sheerka.new(BuiltinConcepts.IS_GREATEST_CONSTRAINT_ERROR))
elif new_entry.op == ">>" and new_entry.a in lowest_objs:
return sheerka.ret(name, False, sheerka.new(BuiltinConcepts.IS_LESSER_CONSTRAINT_ERROR))
# implement the following matrix
# +-------+------+-----+-----+-----+
# | | a<< | b<< | a>> | b>> |
# | a < b | ok | nok | nok | ok |
# | a > b | nok | ok | ok | nok |
# |-------+------+-----+-----+-----+
# nok means that new rule cannot be accepted, unless there is a change on the other item
elif new_entry.op == "<":
if new_entry.a in greatest_objs and new_entry.b not in greatest_objs:
return sheerka.ret(name, False, sheerka.new(BuiltinConcepts.IS_GREATEST_CONSTRAINT_ERROR))
elif new_entry.b in lowest_objs and new_entry.a not in lowest_objs:
return sheerka.ret(name, False, sheerka.new(BuiltinConcepts.IS_LESSER_CONSTRAINT_ERROR))
elif new_entry.op == ">":
if new_entry.a in lowest_objs and new_entry.b not in lowest_objs:
return sheerka.ret(name, False, sheerka.new(BuiltinConcepts.IS_LESSER_CONSTRAINT_ERROR))
elif new_entry.b in greatest_objs and new_entry.a not in greatest_objs:
return sheerka.ret(name, False, sheerka.new(BuiltinConcepts.IS_GREATEST_CONSTRAINT_ERROR))
return sheerka.ret(name, True, sheerka.new(BuiltinConcepts.SUCCESS))
@staticmethod
def detect_cycles(previous_comparison_objs, new_comparison_obj: ComparisonObj):
"""
# Thanks to Divyanshu Mehta for contributing this code
# https://www.geeksforgeeks.org/detect-cycle-in-a-graph/?ref=lbp
:param comparison_objs:
:param previous_comparison_objs:
:param new_comparison_obj:
:return:
"""
latest = comparison_objs[-1]
if latest.op == "=":
if new_comparison_obj.op in ("<<", ">>", "="):
return None
def get_graph_and_vertices():
_graph = {}
_vertices = set()
for obj in comparison_objs:
if obj.op == "=":
for obj in previous_comparison_objs + [new_comparison_obj]:
if obj.op in ("<<", ">>", "="):
continue
_vertices.add(obj.a)
@@ -393,7 +504,7 @@ class SheerkaComparisonManager(BaseService):
elif rec_stack[neighbour]:
return True
# The node needs to be poped from
# The node needs to be popped from
# recursion stack before function ends
rec_stack[v] = False
return False
@@ -402,6 +513,7 @@ class SheerkaComparisonManager(BaseService):
visited = {k: False for k in vertices}
rec_stack = {k: False for k in vertices}
if is_cyclic(latest.a): # only need to check from the latest add, since the graph was not cyclic before
if is_cyclic(
new_comparison_obj.a): # only need to check from the latest add, since the graph was not cyclic before
return [k for k, v in rec_stack.items() if v]
return None
src/core/sheerka/services/SheerkaConceptManager.py
+230 -18
View File
@@ -1,14 +1,17 @@
from dataclasses import dataclass
from typing import Set
import core.utils
from cache.Cache import Cache
from cache.CacheManager import ConceptNotFound
from cache.DictionaryCache import DictionaryCache
from cache.ListIfNeededCache import ListIfNeededCache
from cache.SetCache import SetCache
from core.builtin_concepts import ErrorConcept
from core.builtin_concepts_ids import BuiltinConcepts, AllBuiltinConcepts, BuiltinUnique
from core.builtin_helpers import ensure_concept
from core.concept import Concept, DEFINITION_TYPE_DEF, DEFINITION_TYPE_BNF, freeze_concept_attrs, ConceptMetadata
from core.builtin_helpers import ensure_concept, ensure_bnf
from core.concept import Concept, DEFINITION_TYPE_DEF, DEFINITION_TYPE_BNF, freeze_concept_attrs, ConceptMetadata, \
VARIABLE_PREFIX
from core.error import ErrorObj
from core.global_symbols import EVENT_CONCEPT_CREATED, NotInit, NotFound
from core.sheerka.services.sheerka_service import BaseService
@@ -18,6 +21,17 @@ from sdp.sheerkaDataProvider import SheerkaDataProviderDuplicateKeyError
BASE_NODE_PARSER_CLASS = "parsers.BaseNodeParser.BaseNodeParser"
@dataclass
class ChickenAndEggError(Exception):
concepts: Set[str]
@dataclass
class NoFirstTokenError(ErrorObj):
concept: Concept
key: str
@dataclass
class NoModificationFound(ErrorObj):
"""
@@ -82,9 +96,11 @@ class SheerkaConceptManager(BaseService):
CONCEPTS_BY_HASH_ENTRY = "ConceptManager:Concepts_By_Hash" # store hash of concepts definitions (not values)
CONCEPTS_REFERENCES_ENTRY = "ConceptManager:Concepts_References" # tracks references between concepts
CONCEPTS_BY_FIRST_KEYWORD_ENTRY = "ConceptManager:Concepts_By_First_Keyword"
RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY = "ConceptManager:Resolved_Concepts_By_First_Keyword"
def __init__(self, sheerka):
super().__init__(sheerka)
self.bnp = core.utils.get_class(BASE_NODE_PARSER_CLASS) # BaseNodeParser
self.forbidden_meta = {"is_builtin", "key", "id", "props", "variables"}
self.allowed_meta = {attr for attr in vars(ConceptMetadata) if
not attr.startswith("_") and attr not in self.forbidden_meta}
@@ -101,6 +117,7 @@ class SheerkaConceptManager(BaseService):
self.sheerka.bind_service_method(self.get_by_hash, False, visible=False)
self.sheerka.bind_service_method(self.get_by_id, False, visible=False)
self.sheerka.bind_service_method(self.is_not_a_variable, False, visible=False)
self.sheerka.bind_service_method(self.get_concepts_by_first_token, False, visible=False)
register_concept_cache = self.sheerka.om.register_concept_cache
@@ -119,10 +136,22 @@ class SheerkaConceptManager(BaseService):
cache = SetCache().auto_configure(self.CONCEPTS_REFERENCES_ENTRY)
self.sheerka.om.register_cache(self.CONCEPTS_REFERENCES_ENTRY, cache)
cache = DictionaryCache().auto_configure(self.CONCEPTS_BY_FIRST_KEYWORD_ENTRY)
self.sheerka.om.register_cache(self.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, cache)
cache = DictionaryCache().auto_configure(self.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY)
self.sheerka.om.register_cache(self.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY, cache, persist=False)
def initialize_deferred(self, context, is_first_time):
if is_first_time:
self.sheerka.om.put(self.sheerka.OBJECTS_IDS_ENTRY, self.USER_CONCEPTS_IDS, 1000)
# initialize the dictionary of first tokens
self.sheerka.om.get(self.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, None) # to init the cache with the values from sdp
concepts_by_first_keyword = self.sheerka.om.current_cache_manager().copy(self.CONCEPTS_BY_FIRST_KEYWORD_ENTRY)
res = self.resolve_concepts_by_first_keyword(context, concepts_by_first_keyword)
self.sheerka.om.put(self.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY, False, res.body)
def initialize_builtin_concepts(self):
"""
Initializes the builtin concepts
@@ -181,18 +210,18 @@ class SheerkaConceptManager(BaseService):
# check if the bnf definition is correctly computed
try:
self.bnp.ensure_bnf(context, concept)
ensure_bnf(context, concept)
except Exception as ex:
return sheerka.ret(self.NAME, False, ex.args[0])
# compute new concepts_by_first_keyword
init_ret_value = self.bnp.compute_concepts_by_first_token(context, [concept], True)
init_ret_value = self.compute_concepts_by_first_token(context, [concept], True)
if not init_ret_value.status:
return sheerka.ret(self.NAME, False, ErrorConcept(init_ret_value.value))
concepts_by_first_keyword = init_ret_value.body
# computes resolved concepts_by_first_keyword
init_ret_value = self.bnp.resolve_concepts_by_first_keyword(context, concepts_by_first_keyword)
init_ret_value = self.resolve_concepts_by_first_keyword(context, concepts_by_first_keyword)
if not init_ret_value.status:
return sheerka.ret(self.NAME, False, ErrorConcept(init_ret_value.value))
resolved_concepts_by_first_keyword = init_ret_value.body
@@ -202,8 +231,8 @@ class SheerkaConceptManager(BaseService):
concept.freeze_definition_hash()
ontology.add_concept(concept)
ontology.put(sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, False, concepts_by_first_keyword)
ontology.put(sheerka.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY, False, resolved_concepts_by_first_keyword)
ontology.put(self.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, False, concepts_by_first_keyword)
ontology.put(self.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY, False, resolved_concepts_by_first_keyword)
if concept.get_metadata().definition_type == DEFINITION_TYPE_DEF and concept.get_metadata().definition != concept.name:
# allow search by definition when definition relevant
@@ -268,8 +297,8 @@ class SheerkaConceptManager(BaseService):
# To update concept by first keyword
# first remove the old references
keywords = self.bnp.get_first_tokens(sheerka, concept) # keyword of the old concept
concepts_by_first_keyword = cache_manager.copy(sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY)
keywords = self.get_first_tokens(sheerka, concept) # keyword of the old concept
concepts_by_first_keyword = cache_manager.copy(self.CONCEPTS_BY_FIRST_KEYWORD_ENTRY)
for keyword in keywords:
try:
concepts_by_first_keyword[keyword].remove(concept.id)
@@ -279,15 +308,15 @@ class SheerkaConceptManager(BaseService):
pass
# and then update
init_ret_value = self.bnp.compute_concepts_by_first_token(context, [new_concept], False, concepts_by_first_keyword)
init_ret_value = self.compute_concepts_by_first_token(context, [new_concept], False, concepts_by_first_keyword)
if not init_ret_value.status:
return sheerka.ret(self.NAME, False, ErrorConcept(init_ret_value.value))
concepts_by_first_keyword = init_ret_value.body
# computes resolved concepts_by_first_keyword
init_ret_value = self.bnp.resolve_concepts_by_first_keyword(context,
concepts_by_first_keyword,
{new_concept.id: new_concept})
init_ret_value = self.resolve_concepts_by_first_keyword(context,
concepts_by_first_keyword,
{new_concept.id: new_concept})
if not init_ret_value.status:
return sheerka.ret(self.NAME, False, ErrorConcept(init_ret_value.value))
resolved_concepts_by_first_keyword = init_ret_value.body
@@ -302,9 +331,8 @@ class SheerkaConceptManager(BaseService):
cache_manager.put(self.CONCEPTS_REFERENCES_ENTRY, ref, new_concept.id)
cache_manager.update_concept(concept, new_concept)
cache_manager.put(sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, False, concepts_by_first_keyword)
cache_manager.put(sheerka.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY, False,
resolved_concepts_by_first_keyword)
cache_manager.put(self.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, False, concepts_by_first_keyword)
cache_manager.put(self.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY, False, resolved_concepts_by_first_keyword)
# everything seems to be fine. Update the list of attributes
# Caution. Must be done AFTER update_concept()
@@ -620,8 +648,192 @@ class SheerkaConceptManager(BaseService):
concept.get_metadata().key = None
if self._definition_has_changed(to_add) and concept.get_metadata().definition_type == DEFINITION_TYPE_BNF:
concept.set_bnf(None)
self.bnp.ensure_bnf(context, concept)
ensure_bnf(context, concept)
concept.init_key()
return
@staticmethod
def get_first_tokens(sheerka, concept):
"""
:param sheerka:
:param concept:
:return:
"""
if concept.get_bnf():
from parsers.BnfNodeParser import BnfNodeFirstTokenVisitor
bnf_visitor = BnfNodeFirstTokenVisitor(sheerka)
bnf_visitor.visit(concept.get_bnf())
return bnf_visitor.first_tokens
else:
keywords = concept.key.split()
for keyword in keywords:
if keyword.startswith(VARIABLE_PREFIX):
continue
return [keyword]
return None
@staticmethod
def compute_concepts_by_first_token(context, concepts, use_sheerka=False, previous_entries=None):
"""
Create the map describing the first token expected by a concept
eg the dictionary that goes into CONCEPTS_BY_FIRST_KEYWORD_ENTRY
:param context:
:param concepts: lists of concepts to parse
:param use_sheerka: if True, update concepts_by_first_keyword from sheerka
:param previous_entries:
:return:
"""
sheerka = context.sheerka
res = sheerka.om.copy(SheerkaConceptManager.CONCEPTS_BY_FIRST_KEYWORD_ENTRY) if use_sheerka \
else (previous_entries or {})
for concept in concepts:
keywords = SheerkaConceptManager.get_first_tokens(sheerka, concept)
if keywords is None:
# no first token found for a concept ?
return sheerka.ret(sheerka.name, False, NoFirstTokenError(concept, concept.key))
for keyword in keywords:
res.setdefault(keyword, []).append(concept.id)
# 'uniquify' the lists
for k, v in res.items():
res[k] = core.utils.make_unique(v)
return sheerka.ret("BaseNodeParser", True, res)
@staticmethod
def resolve_concepts_by_first_keyword(context, concepts_by_first_keyword, modified_concepts=None):
"""
From a dictionary of first tokens, create another dictionary where all references to other concepts
are resolved
fom example, from entries
{
'c:|1001:' : 'c:|1002:' # which means than the concept 1002 starts with the concept 1001
'foo': 'c:|1001:'
}
It will create
{
'foo': ['c:|1001:, 'c:|1002:'],
}
This dictionary is supposed to go into CONCEPTS_REFERENCES_ENTRY
"""
sheerka = context.sheerka
res = {}
def get_by_id(c_id):
if modified_concepts and c_id in modified_concepts:
return modified_concepts[c_id]
return sheerka.get_by_id(c_id)
def resolve_concepts(concept_str):
c_key, c_id = core.utils.unstr_concept(concept_str)
if c_id in already_seen:
return ChickenAndEggError(already_seen)
already_seen.add(c_id)
resolved = set()
to_resolve = set()
chicken_and_egg = set()
concept = get_by_id(c_id)
if sheerka.isaset(context, concept):
concepts = sheerka.get_set_elements(context, concept)
else:
concepts = [concept]
for concept in concepts:
ensure_bnf(context, concept) # need to make sure that it cannot fail
keywords = SheerkaConceptManager.get_first_tokens(sheerka, concept)
for keyword in keywords:
(to_resolve if keyword.startswith("c:|") else resolved).add(keyword)
for concept_to_resolve_str in to_resolve:
res = resolve_concepts(concept_to_resolve_str)
if isinstance(res, ChickenAndEggError):
chicken_and_egg |= res.concepts
else:
resolved |= res
to_resolve.clear()
if len(resolved) == 0 and len(chicken_and_egg) > 0:
raise ChickenAndEggError(chicken_and_egg)
else:
return resolved
for k, v in concepts_by_first_keyword.items():
if k.startswith("c:|"):
try:
already_seen = set()
resolved_keywords = resolve_concepts(k)
for resolved in resolved_keywords:
res.setdefault(resolved, []).extend(v)
except ChickenAndEggError as ex:
context.log(f"Chicken and egg detected for {k}, concepts={ex.concepts}")
concepts_in_recursion = ex.concepts
# make sure to have all the parents
for parent in v:
concepts_in_recursion.add(parent)
for concept_id in concepts_in_recursion:
# make sure we keep the longest chain
old = sheerka.chicken_and_eggs.get(concept_id)
if old is NotFound or len(old) < len(ex.concepts):
sheerka.chicken_and_eggs.put(concept_id, concepts_in_recursion)
else:
res.setdefault(k, []).extend(v)
# 'uniquify' the lists
for k, v in res.items():
res[k] = core.utils.make_unique(v)
return sheerka.ret("BaseNodeParser", True, res)
def get_concepts_by_first_token(self, token, to_keep, custom=None, to_map=None, strip_quotes=False, parser=None):
"""
Tries to find if there are concepts that match the value of the token
Caution: Returns the actual cache, not a copy
:param token:
:param to_keep: predicate to tell if the concept is eligible
:param custom: lambda name -> List[Concepts] that gives extra concepts, according to the name
:param to_map:
:param strip_quotes: Remove quotes from strings
:param parser: If needed, parser which requested the concepts
:return:
"""
if token.type == TokenKind.WHITESPACE:
return None
if token.type == TokenKind.STRING:
name = token.value[1:-1] if strip_quotes else token.value
else:
name = token.value
custom_concepts = custom(name) if custom else [] # to get extra concepts using an alternative method
result = []
concepts_ids = self.sheerka.om.get(self.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY, name)
if concepts_ids is NotFound:
return custom_concepts if custom else None
for concept_id in concepts_ids:
concept = self.sheerka.get_by_id(concept_id)
if not to_keep(concept):
continue
concept = to_map(concept, parser, self.sheerka) if to_map else concept
result.append(concept)
return core.utils.make_unique(result + custom_concepts,
lambda c: c.concept.id if hasattr(c, "concept") else c.id)
src/core/sheerka/services/SheerkaConceptsAlgebra.py
+1 -3
View File
@@ -119,9 +119,7 @@ class SheerkaConceptsAlgebra(BaseService):
if nb_props == 0:
return res
concepts_manager = self.sheerka.services[SheerkaConceptManager.NAME]
all_concepts = self.sheerka.om.list(concepts_manager.CONCEPTS_BY_ID_ENTRY)
all_concepts = self.sheerka.om.list(SheerkaConceptManager.CONCEPTS_BY_ID_ENTRY)
for c in all_concepts:
score = self._compute_score(c, concept, step_b=round(1 / nb_props, 2))
src/core/sheerka/services/SheerkaMemory.py
+9 -18
View File
@@ -139,32 +139,23 @@ class SheerkaMemory(BaseService):
self.add_to_memory(context, k, v)
self.registration.clear()
def memory(self, context, name=None):
def memory(self, context, name):
"""
Get the list of all memory_objects in memory
:param context:
:param name:
:return:
"""
if name:
name_to_use = name.name if isinstance(name, Concept) else name
self.unregister_object(context, name_to_use)
obj = self.get_from_memory(context, name_to_use)
if obj is NotFound:
return self.sheerka.new(BuiltinConcepts.NOT_FOUND, body={"#name": name})
name_to_use = name.name if isinstance(name, Concept) else name
self.unregister_object(context, name_to_use)
obj = self.get_from_memory(context, name_to_use)
if obj is NotFound:
return self.sheerka.new(BuiltinConcepts.NOT_FOUND, body={"#name": name})
if isinstance(obj, list):
obj = obj[-1]
if isinstance(obj, list):
obj = obj[-1]
return obj.obj
res = {}
for k, obj in self.sheerka.om.get_all(SheerkaMemory.OBJECTS_ENTRY).items():
if isinstance(obj, list):
obj = obj[-1]
res[k] = obj.obj
return res
return obj.obj
def mem(self):
keys = sorted([k for k in self.sheerka.om.list(SheerkaMemory.OBJECTS_ENTRY)])
src/core/sheerka/services/SheerkaResultManager.py
+32 -3
View File
@@ -10,10 +10,10 @@ from core.utils import as_bag
MAX_EXECUTION_HISTORY = 100
class SheerkaResultConcept(BaseService):
class SheerkaResultManager(BaseService):
NAME = "Result"
# SheerkaResultConcept seems to be a concept that must not support multiple ontology layers
# SheerkaResultManager seems to be a concept that must not support multiple ontology layers
# We must have always access to everything that was done, whatever the ontology
def __init__(self, sheerka, page_size=30):
@@ -23,7 +23,9 @@ class SheerkaResultConcept(BaseService):
self.last_execution = None
self.last_created_concept = None
self.last_created_concept_id = None
self.state_vars = ["last_created_concept_id"]
self.last_error_event_id = None
self.last_errors = None
self.state_vars = ["last_created_concept_id", "last_error_event_id"]
def initialize(self):
self.sheerka.bind_service_method(self.get_results_by_digest, True) # digest is recorded
@@ -33,6 +35,7 @@ class SheerkaResultConcept(BaseService):
self.sheerka.bind_service_method(self.get_execution_item, False)
self.sheerka.bind_service_method(self.get_last_return_value, False, as_name="last_ret")
self.sheerka.bind_service_method(self.get_last_created_concept, False, as_name="last_created_concept")
self.sheerka.bind_service_method(self.get_last_error, False, as_name="last_err")
def initialize_deferred(self, context, is_first_time):
self.restore_values(*self.state_vars)
@@ -44,6 +47,8 @@ class SheerkaResultConcept(BaseService):
self.last_execution = None
self.last_created_concept = None
self.last_created_concept_id = None
self.last_error_event_id = None
self.last_errors = None
def save_state(self, context):
self.store_values(context, *self.state_vars)
@@ -246,6 +251,12 @@ class SheerkaResultConcept(BaseService):
self.executions_contexts_cache.put(execution_context.event.get_digest(), execution_context)
self.last_execution = execution_context
in_error = [ret for ret in execution_context.values["return_values"] if not ret.status]
if in_error:
self.last_error_event_id = execution_context.event.get_digest()
self.last_errors = in_error[0] if len(in_error) == 1 else in_error
self.store_var(execution_context, "last_error_event_id")
def get_last_return_value(self, context):
"""
Return the last return value(s)
@@ -267,6 +278,24 @@ class SheerkaResultConcept(BaseService):
return self.sheerka.new(BuiltinConcepts.NOT_FOUND, body={"query": "last"})
def get_last_error(self):
if self.last_errors:
return self.last_errors
if self.last_error_event_id is None:
return self.sheerka.new(BuiltinConcepts.NOT_FOUND, body={"query": "get_last_error"})
# search in history
if self.last_error_event_id in self.executions_contexts_cache:
execution_result = self.executions_contexts_cache.get(self.last_error_event_id)
else:
execution_result = self.sheerka.om.current_sdp().load_result(self.last_error_event_id)
in_error = [ret for ret in execution_result.values["return_values"] if not ret.status]
self.last_errors = in_error[0] if len(in_error) == 1 else in_error
return self.last_errors
def new_concept_created(self, context, concept):
self.last_created_concept = concept
self.last_created_concept_id = concept.id
src/core/sheerka/services/SheerkaRuleManager.py
+47 -7
View File
@@ -522,6 +522,7 @@ class SheerkaRuleManager(BaseService):
self.sheerka.bind_service_method(self.get_rule_by_name, False)
self.sheerka.bind_service_method(self.dump_desc_rule, False, as_name="desc_rule")
self.sheerka.bind_service_method(self.get_format_rules, False, visible=False)
self.sheerka.bind_service_method(self.resolve_rule, False, visible=False)
cache = Cache().auto_configure(self.FORMAT_RULE_ENTRY)
self.sheerka.om.register_cache(self.FORMAT_RULE_ENTRY, cache, True, True)
@@ -566,7 +567,7 @@ class SheerkaRuleManager(BaseService):
:return:
"""
# get the priorities
rules_weights = self.sheerka.get_concepts_weights(BuiltinConcepts.PRECEDENCE, RULE_COMPARISON_CONTEXT)
rules_weights = self.sheerka.get_weights(BuiltinConcepts.PRECEDENCE, RULE_COMPARISON_CONTEXT)
# update the priorities
for rule in self.sheerka.om.list(self.FORMAT_RULE_ENTRY, cache_only=True):
@@ -712,6 +713,11 @@ class SheerkaRuleManager(BaseService):
Rule("print", "Display multiple success",
"isinstance(__ret_container, BuiltinConcepts.MULTIPLE_SUCCESS)",
"list(__ret_container.body)"),
# # index=[9] in code, id=10 in debug
# Rule("print", "Display simple result when only one success",
# "len(__rets)==1 and __rets[0].status == True",
# "{__rets[0].body}"),
]
for r in rules:
@@ -737,12 +743,7 @@ class SheerkaRuleManager(BaseService):
if rule_id is None:
return None
rule = self.sheerka.om.get(self.FORMAT_RULE_ENTRY, rule_id)
if rule is not NotFound:
return rule
rule = self.sheerka.om.get(self.EXEC_RULE_ENTRY, rule_id)
if rule is not NotFound:
if rule := self._inner_get_by_id(rule_id):
return rule
metadata = [("id", rule_id)]
@@ -815,3 +816,42 @@ class SheerkaRuleManager(BaseService):
else:
raise NotImplementedError(r)
return res
def resolve_rule(self, context, obj):
if obj is None:
return None
elif isinstance(obj, str):
# search by id first
if rule := self._inner_get_by_id(obj):
return rule
elif isinstance(obj, Token) and obj.type == TokenKind.RULE:
# search by id first
if rule := self._inner_get_by_id(obj.value[1]):
return rule
# try via indirection
if (rule_id := self.sheerka.get_from_short_term_memory(context, obj.value[1])) is not NotFound and \
(rule := self._inner_get_by_id(str(rule_id))) is not None:
return rule
elif isinstance(obj, Rule):
if obj.metadata.id_is_unresolved:
if (rule_id := self.sheerka.get_from_short_term_memory(context, obj.id)) is not NotFound and \
(rule := self._inner_get_by_id(str(rule_id))) is not None:
return rule
else:
return obj
return None
def _inner_get_by_id(self, rule_id):
rule = self.sheerka.om.get(self.FORMAT_RULE_ENTRY, rule_id)
if rule is not NotFound:
return rule
rule = self.sheerka.om.get(self.EXEC_RULE_ENTRY, rule_id)
if rule is not NotFound:
return rule
return None
src/core/sheerka/services/sheerka_service.py
+6
View File
@@ -40,3 +40,9 @@ class BaseService:
for prop_name in args:
if (value := self.sheerka.load_var(self.NAME, prop_name)) is not NotFound:
setattr(self, prop_name, value)
def store_var(self, context, var_name):
"""
Store/record the value of an attribute
"""
self.sheerka.record_var(context, self.NAME, var_name, getattr(self, var_name))
src/core/utils.py
+9 -1
View File
@@ -5,9 +5,10 @@ import os
import pkgutil
from copy import deepcopy
from pyparsing import *
from core.global_symbols import CustomType
from core.tokenizer import TokenKind, Tokenizer
from pyparsing import *
COLORS = {
"black",
@@ -600,6 +601,13 @@ def flatten_all_children(item, get_children):
return inner_get_all_children(item)
def flatten(list_of_lists):
"""
Flatten an list containing other lists
"""
return [item for sublist in list_of_lists for item in sublist]
def get_text_from_tokens(tokens, custom_switcher=None, tracker=None):
"""
Create the source code, from the list of token