kodjo/Sheerka-Old
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fixed #3: Added sheerka.resolve_rule()

Browse Source 
Fixed #5: Refactored SheerkaComparisonManager
Fixed #6: Sya parser no longer works after restart
This commit is contained in:
Kodjo Sossouvi
2021-01-15 07:11:04 +01:00
parent e26c83a825
commit 821dbed189
44 changed files with 1617 additions and 1068 deletions
Download Patch File Download Diff File Expand all files Collapse all files
_concepts_admin.txt
+11
View File
@@ -44,3 +44,14 @@ def concept deactivate return values processing as set_var("sheerka.enable_proce
set_auto_eval(c:activate return values processing:)
set_auto_eval(c:deactivate return values processing:)
def concept rule x where isinstance(x, int) as r:|x:
set_auto_eval(c:rule x:)
def concept rule x > rule y where isinstance(x, int) and isinstance(y, int) as set_is_greater_than(__PRECEDENCE, r:|x:, r:|y:, 'Rule')
set_auto_eval(c:rule x > rule y:)
def concept rule x is greatest where isinstance(x, int) as set_is_greatest(__PRECEDENCE, r:|x:, 'Rule')
set_auto_eval(c:rule x is greatest:)
def concept rule x < rule y where isinstance(x, int) and isinstance(y, int) as set_is_less_than(__PRECEDENCE, r:|x:, r:|y:, 'Rule')
set_auto_eval(c:rule x < rule y:)
def concept rule x is lesser where isinstance(x, int) as set_is_lesser(__PRECEDENCE, r:|x:, 'Rule')
set_auto_eval(c:rule x is lesser:)
docs/source/README.txt
+1
View File
@@ -5,3 +5,4 @@ install the following extensions:
* reStructuredText (from Lextudio Inc)
doc page for reStructuredText : https://docs.restructuredtext.net/
shortcuts : https://docs.restructuredtext.net/articles/shortcuts.html
doc de sphinx : https://www.sphinx-doc.org/en/master/usage/restructuredtext/basics.html
docs/source/blog/blog.rst
-8
View File
@@ -1,11 +1,3 @@
.. toctree::
:maxdepth: 1
concepts
rules
parsers
persistence
2019-10-30
**********
docs/source/conf.py
+1 -1
View File
@@ -18,7 +18,7 @@
# -- Project information -----------------------------------------------------
project = 'Sheerka'
copyright = '2020, Kodjo Sossouvi'
copyright = '2020-2001, Kodjo Sossouvi'
author = 'Kodjo Sossouvi'
# The full version, including alpha/beta/rc tags
docs/source/index.rst
+7
View File
@@ -1,6 +1,10 @@
Sheerka's documentation!
===================================
.. WARNING::
Under construction for one year !
Hi, welcome the the documentation page of Sheerka.
There will be two types of documentation
@@ -15,11 +19,14 @@ There will be two types of documentation
blog/blog
.. toctree::
:maxdepth: 1
:caption: Technical Design:
tech/tech
tech/debugger
Indices and tables
docs/source/tech/debugger.rst
+19
View File
@@ -0,0 +1,19 @@
Debugger
========
Abstract
--------
The purpose of the debugger is to ease to understand why Sheerka does not behave like it should.
(yes, this could happen). I have have just started so, they are a lot of
functionalities that we usually found in a debugger that are not yet implemented
I guess that the first thing to know is that I want to design a 'smart' debugger,
which can be configured in a 'declarative' way. It means that, rather that telling
it what to do, I would prefer tell it what is expected.
As of now (2021-01-11), I am far from it, so let's explain how to use the debugger,
in an 'imperative' way.
First, you need to enable the debugger. As it consumes resource, it is deactivated by default ::
set_debug()
src/core/builtin_concepts_ids.py
+7 -6
View File
@@ -75,6 +75,7 @@ class BuiltinConcepts:
INVALID_RETURN_VALUE = "__INVALID_RETURN_VALUE" # the return value of an evaluator is not correct
CONCEPT_ALREADY_DEFINED = "__CONCEPT_ALREADY_DEFINED" # when you try to add the same object twice (a concept or whatever)
PROPERTY_ALREADY_DEFINED = "__PROPERTY_ALREADY_DEFINED" # When you try to add the same element in a property
COMPARISON_ALREADY_DEFINED = "__COMPARISON_ALREADY_DEFINED" # when the same comparison entry is defined several times
NOP = "__NOP" # no operation concept. Does nothing
CONCEPT_EVAL_ERROR = "__CONCEPT_EVAL_ERROR" # cannot evaluate a property or metadata of a concept
ENUMERATION = "__ENUMERATION" # represents a list or a set
@@ -91,8 +92,8 @@ class BuiltinConcepts:
FORMAT_INSTRUCTIONS = "__FORMAT_INSTRUCTIONS" # to express how to print the concept
NOT_IMPLEMENTED = "__NOT_IMPLEMENTED" # instead of raise an error
PYTHON_SECURITY_ERROR = "__PYTHON_SECURITY_ERROR" # when trying to execute statement when only expression is allowed
INVALID_LESSER_OPERATION = "__INVALID_LESSER_OPERATION"
INVALID_GREATEST_OPERATION = "__INVALID_GREATEST_OPERATION"
IS_LESSER_CONSTRAINT_ERROR = "__IS_LESSER_CONSTRAINT_ERROR"
IS_GREATEST_CONSTRAINT_ERROR = "__IS_GREATEST_CONSTRAINT_ERROR"
NEW_RULE = "__NEW_RULE"
UNKNOWN_RULE = "__UNKNOWN_RULE"
ONTOLOGY_ALREADY_DEFINED = "__ONTOLOGY_ALREADY_DEFINED"
@@ -145,8 +146,8 @@ BuiltinUnique = [
BuiltinConcepts.ISA,
BuiltinConcepts.AUTO_EVAL,
BuiltinConcepts.INVALID_LESSER_OPERATION,
BuiltinConcepts.INVALID_GREATEST_OPERATION,
BuiltinConcepts.IS_LESSER_CONSTRAINT_ERROR,
BuiltinConcepts.IS_GREATEST_CONSTRAINT_ERROR,
]
BuiltinErrors = [
@@ -167,8 +168,8 @@ BuiltinErrors = [
BuiltinConcepts.CONDITION_FAILED,
BuiltinConcepts.CHICKEN_AND_EGG,
BuiltinConcepts.NOT_FOUND,
BuiltinConcepts.INVALID_LESSER_OPERATION,
BuiltinConcepts.INVALID_GREATEST_OPERATION,
BuiltinConcepts.IS_LESSER_CONSTRAINT_ERROR,
BuiltinConcepts.IS_GREATEST_CONSTRAINT_ERROR,
BuiltinConcepts.ONTOLOGY_ALREADY_DEFINED
]
src/core/builtin_helpers.py
+23 -1
View File
@@ -12,7 +12,7 @@ from core.tokenizer import Keywords
from core.utils import as_bag
from parsers.BaseNodeParser import SourceCodeNode, ConceptNode, UnrecognizedTokensNode, SourceCodeWithConceptNode, \
RuleNode
from parsers.BaseParser import BaseParser, ParsingError
from parsers.BaseParser import ParsingError
PARSE_STEPS = [BuiltinConcepts.BEFORE_PARSING, BuiltinConcepts.PARSING, BuiltinConcepts.AFTER_PARSING]
EVAL_STEPS = PARSE_STEPS + [BuiltinConcepts.BEFORE_EVALUATION, BuiltinConcepts.EVALUATION,
@@ -655,6 +655,28 @@ def ensure_concept_or_rule(*items):
raise TypeError(f"'{items}' must be a concept or rule")
def ensure_bnf(context, concept, parser_name="BaseNodeParser"):
if concept.get_metadata().definition_type == DEFINITION_TYPE_BNF and not concept.get_bnf():
from parsers.BnfDefinitionParser import BnfDefinitionParser
regex_parser = BnfDefinitionParser()
desc = f"Resolving BNF '{concept.get_metadata().definition}'"
with context.push(BuiltinConcepts.INIT_BNF,
concept,
who=parser_name,
obj=concept,
desc=desc) as sub_context:
sub_context.add_inputs(parser_input=concept.get_metadata().definition)
bnf_parsing_ret_val = regex_parser.parse(sub_context, concept.get_metadata().definition)
sub_context.add_values(return_values=bnf_parsing_ret_val)
if not bnf_parsing_ret_val.status:
raise Exception(bnf_parsing_ret_val.value)
concept.set_bnf(bnf_parsing_ret_val.body.body)
if concept.id:
context.sheerka.get_by_id(concept.id).set_bnf(concept.get_bnf()) # update bnf in cache
expressions_cache = Cache()
src/core/rule.py
+2 -2
View File
@@ -5,17 +5,17 @@ import core.utils
ACTION_TYPE_PRINT = "print"
ACTION_TYPE_EXEC = "exec"
ACTION_TYPE_DEFERRED = "deferred" # KSI 2021-04-01 What is it for ? I definitely need some proper documentation
@dataclass
class RuleMetadata:
action_type: str # print, exec, deferred
action_type: str # print, exec
name: Union[str, None]
predicate: str
action: str
id: str = None
id_is_unresolved = False
is_compiled: bool = False
is_enabled: bool = False
src/core/sheerka/Sheerka.py
-17
View File
@@ -49,8 +49,6 @@ class Sheerka(Concept):
CONCEPTS_BY_ID_ENTRY = "ConceptManager:Concepts_By_ID"
CONCEPTS_BY_NAME_ENTRY = "ConceptManager:Concepts_By_Name"
CONCEPTS_BY_FIRST_KEYWORD_ENTRY = "Concepts_By_First_Keyword"
RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY = "Resolved_Concepts_By_First_Keyword"
CONCEPTS_SYA_DEFINITION_ENTRY = "Concepts_Sya_Definitions"
RESOLVED_CONCEPTS_SYA_DEFINITION_ENTRY = "Resolved_Concepts_Sya_Definitions"
CONCEPTS_GRAMMARS_ENTRY = "Concepts_Grammars"
@@ -183,7 +181,6 @@ class Sheerka(Concept):
self.first_time_initialisation(exec_context)
self.initialize_builtin_concepts()
self.initialize_concept_node_parsing(exec_context)
self.initialize_services_deferred(exec_context, self.om.current_sdp().first_time)
@@ -215,17 +212,10 @@ class Sheerka(Concept):
cache = IncCache().auto_configure(self.OBJECTS_IDS_ENTRY)
self.om.register_cache(self.OBJECTS_IDS_ENTRY, cache)
cache = DictionaryCache().auto_configure(self.CONCEPTS_BY_FIRST_KEYWORD_ENTRY)
self.om.register_cache(self.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, cache)
self.om.get(self.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, None) # to init from sdp
cache = DictionaryCache().auto_configure(self.CONCEPTS_SYA_DEFINITION_ENTRY)
self.om.register_cache(self.CONCEPTS_SYA_DEFINITION_ENTRY, cache)
self.om.get(self.CONCEPTS_SYA_DEFINITION_ENTRY, None) # to init from sdp
cache = DictionaryCache().auto_configure(self.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY)
self.om.register_cache(self.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY, cache, persist=False)
cache = DictionaryCache().auto_configure(self.RESOLVED_CONCEPTS_SYA_DEFINITION_ENTRY)
self.om.register_cache(self.RESOLVED_CONCEPTS_SYA_DEFINITION_ENTRY, cache, persist=False)
@@ -331,13 +321,6 @@ class Sheerka(Concept):
if hasattr(evaluator, "initialize"):
evaluator.initialize(self)
def initialize_concept_node_parsing(self, context):
# self.init_log.debug("Initializing concepts by first keyword.")
concepts_by_first_keyword = self.om.current_cache_manager().copy(self.CONCEPTS_BY_FIRST_KEYWORD_ENTRY)
res = self.bnp.resolve_concepts_by_first_keyword(context, concepts_by_first_keyword)
self.om.put(self.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY, False, res.body)
def initialize_ontologies(self, context):
ontologies = self.om.current_sdp().load_ontologies()
if not ontologies:
src/core/sheerka/SheerkaOntologyManager.py
+42 -42
View File
@@ -311,48 +311,48 @@ class SheerkaOntologyManager:
"""
return list(self.get_all(entry, cache_only).values())
def list_by_key(self, entry, key):
"""
List all entries of a given key
If the values are lists, sets of dictionaries, they will be concatenated
Otherwise it will raise an error
"""
res = None
def update_values(_res, values_):
if values_ is NotFound:
return _res
elif values_ is Removed:
_res.clear()
elif isinstance(values_, dict):
if _res is None:
_res = values_.copy()
elif isinstance(_res, dict):
_res.update(values_)
else:
raise ValueError(f"Expecting dict while found '{values_}'")
elif isinstance(values_, list):
if _res is None:
_res = values_.copy()
elif isinstance(_res, list):
_res.extend(values_)
else:
raise ValueError(f"Expecting list while found '{values_}'")
else:
raise NotImplementedError()
return _res
for ontology in reversed(self.ontologies):
from_cache_values = ontology.cache_manager.get(entry, key)
if from_cache_values is not NotFound:
res = update_values(res, from_cache_values)
else:
from_sdp_values = ontology.cache_manager.sdp.get(entry, key)
res = update_values(res, from_sdp_values)
return res
# def list_by_key(self, entry, key):
# """
# List all entries of a given key
# If the values are lists, sets of dictionaries, they will be concatenated
# Otherwise it will raise an error
# """
# res = None
#
# def update_values(_res, values_):
# if values_ is NotFound:
# return _res
# elif values_ is Removed:
# _res.clear()
# elif isinstance(values_, dict):
# if _res is None:
# _res = values_.copy()
# elif isinstance(_res, dict):
# _res.update(values_)
# else:
# raise ValueError(f"Expecting dict while found '{values_}'")
# elif isinstance(values_, list):
# if _res is None:
# _res = values_.copy()
# elif isinstance(_res, list):
# _res.extend(values_)
# else:
# raise ValueError(f"Expecting list while found '{values_}'")
# else:
# raise NotImplementedError()
#
# return _res
#
# for ontology in reversed(self.ontologies):
#
# from_cache_values = ontology.cache_manager.get(entry, key)
# if from_cache_values is not NotFound:
# res = update_values(res, from_cache_values)
# else:
# from_sdp_values = ontology.cache_manager.sdp.get(entry, key)
# res = update_values(res, from_sdp_values)
#
# return res
def get_all(self, entry, cache_only=False):
"""
src/core/sheerka/services/SheerkaAdmin.py
+14 -1
View File
@@ -1,11 +1,11 @@
import sys
import time
from operator import attrgetter
from os import path
from core.builtin_concepts_ids import BuiltinConcepts, BuiltinContainers
from core.builtin_helpers import ensure_concept
from core.concept import Concept
from core.sheerka.services.SheerkaMemory import SheerkaMemory
from core.sheerka.services.sheerka_service import BaseService
CONCEPTS_FILE_LITE = "_concepts_lite.txt"
@@ -31,6 +31,7 @@ class SheerkaAdmin(BaseService):
self.sheerka.bind_service_method(self.admin_push_ontology, True, as_name="push_ontology")
self.sheerka.bind_service_method(self.admin_pop_ontology, True, as_name="pop_ontology")
self.sheerka.bind_service_method(self.ontologies, False)
self.sheerka.bind_service_method(self.in_memory, False)
def caches_names(self):
"""
@@ -185,3 +186,15 @@ class SheerkaAdmin(BaseService):
def ontologies(self):
ontologies = self.sheerka.om.ontologies_names
return self.sheerka.new(BuiltinConcepts.TO_LIST, body=ontologies)
def in_memory(self):
"""
Returns the list of all obj in memory
"""
res = {}
for k, obj in self.sheerka.om.get_all(SheerkaMemory.OBJECTS_ENTRY).items():
if isinstance(obj, list):
obj = obj[-1]
res[k] = obj.obj
return self.sheerka.ret(self.NAME, True, self.sheerka.new(BuiltinConcepts.TO_DICT, body=res))
src/core/sheerka/services/SheerkaComparisonManager.py
+292 -180
View File
@@ -1,15 +1,16 @@
from dataclasses import dataclass
from functools import reduce
from cache.Cache import Cache
from cache.ListCache import ListCache
from core.builtin_concepts import BuiltinConcepts
from core.global_symbols import EVENT_CONCEPT_PRECEDENCE_MODIFIED, EVENT_RULE_PRECEDENCE_MODIFIED, \
RULE_COMPARISON_CONTEXT, \
CONCEPT_COMPARISON_CONTEXT, NotFound
from core.builtin_helpers import ensure_concept_or_rule
from core.concept import Concept
from core.sheerka.services.SheerkaRuleManager import SheerkaRuleManager
from core.global_symbols import EVENT_CONCEPT_PRECEDENCE_MODIFIED, EVENT_RULE_PRECEDENCE_MODIFIED, \
RULE_COMPARISON_CONTEXT, CONCEPT_COMPARISON_CONTEXT, NotInit, NotFound
from core.sheerka.services.SheerkaConceptManager import ChickenAndEggError
from core.sheerka.services.sheerka_service import ServiceObj, BaseService
from core.utils import flatten
@dataclass
@@ -18,8 +19,8 @@ class ComparisonObj(ServiceObj):
Order to store
"""
property: str # property to compare
a: int # id of concept a
b: int # id of concept b
a: str # str_id of concept a
b: str # str_id of concept b
op: str # comparison operation
context: str = "#" # context when the comparison is right
@@ -39,157 +40,6 @@ class SheerkaComparisonManager(BaseService):
def __init__(self, sheerka):
super().__init__(sheerka)
@staticmethod
def _compute_key(prop_name, comparison_context):
"""
Key to use to store the comparisons
:param prop_name:
:param comparison_context:
:return:
"""
if isinstance(prop_name, Concept):
prefix = prop_name.key if prop_name.get_metadata().is_builtin else prop_name.id
else:
prefix = prop_name
return f"{prefix}|{comparison_context}"
@staticmethod
def _get_weights(comparison_objs):
"""
For every element in greater_than_s, give it a weight
if weight(a) > weight(b) it means that a > b
:param comparison_objs: list of greater than objects
:return:
"""
values = {}
for comparison_obj in comparison_objs:
values[comparison_obj.a] = SheerkaComparisonManager.DEFAULT_COMPARISON_VALUE
values[comparison_obj.b] = SheerkaComparisonManager.DEFAULT_COMPARISON_VALUE
for _ in range(len(comparison_objs)):
for comparison_obj in comparison_objs:
if comparison_obj.op == ">":
if values[comparison_obj.a] <= values[comparison_obj.b]:
values[comparison_obj.a] = values[comparison_obj.b] + 1
else:
if values[comparison_obj.b] <= values[comparison_obj.a]:
values[comparison_obj.b] = values[comparison_obj.a] + 1
return values
def _compute_weights(self, comparison_objs, lesser_objs_ids=None, greatest_objs_ids=None):
"""
:param comparison_objs:
:return:
"""
def is_not_in_objs(obj, objs_ids):
return obj.op in ("<", ">") and obj.a not in objs_ids and obj.b not in objs_ids
def is_in_objs(obj, objs_ids):
return obj.op in ("<", ">") and (obj.a in objs_ids or obj.b in objs_ids)
lesser_objs_ids = lesser_objs_ids or {co.a for co in comparison_objs if co.op == "<<"}
greatest_objs_ids = greatest_objs_ids or {co.a for co in comparison_objs if co.op == ">>"}
default_weight = SheerkaComparisonManager.DEFAULT_COMPARISON_VALUE
# get the weights for all the lesser
lesser_objs = [co for co in comparison_objs if is_in_objs(co, lesser_objs_ids)]
lesser_objs_weights = self._get_weights(lesser_objs)
# rearrange the weight to have the highest equals to DEFAULT_COMPARISON_VALUE - 1
highest_weight = len(lesser_objs_weights)
for co_id in lesser_objs_weights:
lesser_objs_weights[co_id] = lesser_objs_weights[co_id] - highest_weight + default_weight - 1
for concept_id in lesser_objs_ids:
if concept_id not in lesser_objs_weights:
lesser_objs_weights[concept_id] = default_weight - 1
# get the weights for concepts that are not lesser or greatest
in_between_objs = [o for o in comparison_objs if is_not_in_objs(o, lesser_objs_ids | greatest_objs_ids)]
in_between_weights = self._get_weights(in_between_objs)
# get the weights for all the greatest
greatest_objs = [co for co in comparison_objs if is_in_objs(co, greatest_objs_ids)]
greatest_objs_weights = self._get_weights(greatest_objs)
# rearrange the weight to have the lowest equals to DEFAULT_COMPARISON_VALUE + 1
highest_weight = max(default_weight, len(in_between_weights))
for co_id in greatest_objs_weights:
greatest_objs_weights[co_id] = greatest_objs_weights[co_id] + highest_weight
for concept_id in greatest_objs_ids:
if concept_id not in greatest_objs_weights:
greatest_objs_weights[concept_id] = highest_weight + 1
return {**lesser_objs_weights, **in_between_weights, **greatest_objs_weights}
@staticmethod
def _get_partition(weighted_concepts):
res = {}
for k, v in weighted_concepts.items():
res.setdefault(v, []).append(k)
return res
def _add_comparison(self, context, comparison_obj):
key = self._compute_key(comparison_obj.property, comparison_obj.context)
previous = self.sheerka.om.get(self.COMPARISON_ENTRY, key)
new = previous.copy() if isinstance(previous, list) else []
for co in new:
if co.property == comparison_obj.property and \
co.a == comparison_obj.a and \
co.b == comparison_obj.b and \
co.op == comparison_obj.op and \
co.context == comparison_obj.context:
return self.sheerka.ret(self.NAME, False, self.sheerka.new(BuiltinConcepts.CONCEPT_ALREADY_DEFINED))
new.append(comparison_obj)
lesser_objs_ids = {co.a for co in new if co.op == "<<"}
greatest_objs_ids = {co.a for co in new if co.op == ">>"}
# check if it is a valid operation regarding other lesser and greatest concepts
if comparison_obj.op in ("<", ">"):
a_is_lesser = comparison_obj.a in lesser_objs_ids
b_is_lesser = comparison_obj.b in lesser_objs_ids
if a_is_lesser != b_is_lesser: # XOR operation
return self.sheerka.ret(self.NAME, False, self.sheerka.new(BuiltinConcepts.INVALID_LESSER_OPERATION))
a_is_greatest = comparison_obj.a in greatest_objs_ids
b_is_greatest = comparison_obj.b in greatest_objs_ids
if a_is_greatest != b_is_greatest: # XOR operation
return self.sheerka.ret(self.NAME, False, self.sheerka.new(BuiltinConcepts.INVALID_GREATEST_OPERATION))
if comparison_obj.op == "<<" and comparison_obj.a in greatest_objs_ids:
return self.sheerka.ret(self.NAME, False, self.sheerka.new(BuiltinConcepts.INVALID_GREATEST_OPERATION))
if comparison_obj.op == ">>" and comparison_obj.a in lesser_objs_ids:
return self.sheerka.ret(self.NAME, False, self.sheerka.new(BuiltinConcepts.INVALID_LESSER_OPERATION))
cycles = self.detect_cycles(new)
if cycles:
concepts_in_cycle = [self.sheerka.fast_resolve(c) for c in cycles]
chicken_an_egg = self.sheerka.new(BuiltinConcepts.CHICKEN_AND_EGG, body=concepts_in_cycle)
return self.sheerka.ret(self.NAME, False, chicken_an_egg)
self.sheerka.om.put(self.COMPARISON_ENTRY, key, comparison_obj)
self.sheerka.om.put(self.RESOLVED_COMPARISON_ENTRY, key, self._compute_weights(new,
lesser_objs_ids,
greatest_objs_ids))
if comparison_obj.property == BuiltinConcepts.PRECEDENCE:
if comparison_obj.context == CONCEPT_COMPARISON_CONTEXT:
self.sheerka.publish(context, EVENT_CONCEPT_PRECEDENCE_MODIFIED)
elif comparison_obj.context == RULE_COMPARISON_CONTEXT:
self.sheerka.publish(context, EVENT_RULE_PRECEDENCE_MODIFIED)
return self.sheerka.ret(self.NAME, True, self.sheerka.new(BuiltinConcepts.SUCCESS))
def initialize(self):
cache = ListCache().auto_configure(self.COMPARISON_ENTRY)
self.sheerka.om.register_cache(self.COMPARISON_ENTRY, cache, True, True)
@@ -202,7 +52,48 @@ class SheerkaComparisonManager(BaseService):
self.sheerka.bind_service_method(self.set_is_lesser, True)
self.sheerka.bind_service_method(self.set_is_greatest, True)
self.sheerka.bind_service_method(self.get_partition, False)
self.sheerka.bind_service_method(self.get_concepts_weights, False)
self.sheerka.bind_service_method(self.get_weights, False)
@staticmethod
def _compute_key(prop_name, comparison_context):
"""
Key to use to store the comparisons
:param prop_name:
:param comparison_context:
:return:
"""
prefix = SheerkaComparisonManager._get_real_prop_name(prop_name)
return f"{prefix}|{comparison_context}"
@staticmethod
def _get_real_prop_name(prop_name):
if isinstance(prop_name, Concept):
return prop_name.key if prop_name.get_metadata().is_builtin else prop_name.str_id
return str(prop_name)
def _add_comparison_new(self, context, comparison_obj):
key = self._compute_key(comparison_obj.property, comparison_obj.context)
previous_entries = self.sheerka.om.get(self.COMPARISON_ENTRY, key)
previous_entries = previous_entries.copy() if isinstance(previous_entries, list) else []
res = self.validate_new_entry(context, comparison_obj, previous_entries)
if not res.status:
return res
previous_entries.append(comparison_obj)
weights = self.compute_weights(previous_entries)
self.sheerka.om.put(self.COMPARISON_ENTRY, key, comparison_obj)
self.sheerka.om.put(self.RESOLVED_COMPARISON_ENTRY, key, weights)
if comparison_obj.property == BuiltinConcepts.PRECEDENCE:
if comparison_obj.context == CONCEPT_COMPARISON_CONTEXT:
self.sheerka.publish(context, EVENT_CONCEPT_PRECEDENCE_MODIFIED)
elif comparison_obj.context == RULE_COMPARISON_CONTEXT:
self.sheerka.publish(context, EVENT_RULE_PRECEDENCE_MODIFIED)
return self.sheerka.ret(self.NAME, True, self.sheerka.new(BuiltinConcepts.SUCCESS))
def set_is_greater_than(self, context, prop_name, item_a, item_b, comparison_context="#"):
"""
@@ -217,14 +108,15 @@ class SheerkaComparisonManager(BaseService):
context.log(f"Setting item {item_a} is greater than {item_b}", who=self.NAME)
ensure_concept_or_rule(item_a, item_b)
real_prop_name = self._get_real_prop_name(prop_name)
event_digest = context.event.get_digest()
comparison_obj = ComparisonObj(event_digest,
prop_name,
real_prop_name,
item_a.str_id,
item_b.str_id,
">",
comparison_context)
return self._add_comparison(context, comparison_obj)
return self._add_comparison_new(context, comparison_obj)
def set_is_less_than(self, context, prop_name, item_a, item_b, comparison_context="#"):
"""
@@ -239,14 +131,15 @@ class SheerkaComparisonManager(BaseService):
context.log(f"Setting item {item_a} is less than {item_b}", who=self.NAME)
ensure_concept_or_rule(item_a, item_b)
real_prop_name = self._get_real_prop_name(prop_name)
event_digest = context.event.get_digest()
comparison_obj = ComparisonObj(event_digest,
prop_name,
real_prop_name,
item_a.str_id,
item_b.str_id,
"<",
comparison_context)
return self._add_comparison(context, comparison_obj)
return self._add_comparison_new(context, comparison_obj)
def set_is_lesser(self, context, prop_name, item, comparison_context="#"):
"""
@@ -265,14 +158,15 @@ class SheerkaComparisonManager(BaseService):
context.log(f"Setting item {item} is lesser", who=self.NAME)
ensure_concept_or_rule(item)
real_prop_name = self._get_real_prop_name(prop_name)
event_digest = context.event.get_digest()
comparison_obj = ComparisonObj(event_digest,
prop_name,
real_prop_name,
item.str_id,
None,
"<<",
comparison_context)
return self._add_comparison(context, comparison_obj)
return self._add_comparison_new(context, comparison_obj)
def set_is_greatest(self, context, prop_name, item, comparison_context="#"):
"""
@@ -291,14 +185,15 @@ class SheerkaComparisonManager(BaseService):
context.log(f"Setting item {item} is greatest", who=self.NAME)
ensure_concept_or_rule(item)
real_prop_name = self._get_real_prop_name(prop_name)
event_digest = context.event.get_digest()
comparison_obj = ComparisonObj(event_digest,
prop_name,
real_prop_name,
item.str_id,
None,
">>",
comparison_context)
return self._add_comparison(context, comparison_obj)
return self._add_comparison_new(context, comparison_obj)
def set_are_equivalent(self, context, prop_name, concept_a, concept_b, comparison_context="#"):
"""
@@ -321,11 +216,14 @@ class SheerkaComparisonManager(BaseService):
:param comparison_context:
:return:
"""
weighted_concept = self.get_concepts_weights(prop_name, comparison_context)
weighted_concept = self.get_weights(prop_name, comparison_context)
return self._get_partition(weighted_concept)
res = {}
for k, v in weighted_concept.items():
res.setdefault(v, []).append(k)
return res
def get_concepts_weights(self, prop_name, comparison_context="#"):
def get_weights(self, prop_name, comparison_context="#"):
# KSI 2021-01-10 This implementation seems to be too complicated
# Chances are that there is a better way to implement this.
# Note that I don't want to use a DictionaryCache for the RESOLVED_COMPARISON_ENTRY
@@ -340,32 +238,245 @@ class SheerkaComparisonManager(BaseService):
else:
# otherwise, either it's not computed yet or it does not include the info of the current layer
# In both case, it is safer to recompute the weights
entries = self.sheerka.om.list_by_key(self.COMPARISON_ENTRY, key_to_use)
if entries is None:
entries = self.sheerka.om.get(self.COMPARISON_ENTRY, key_to_use)
if entries is NotFound:
weighted_concepts = {} # Why not put it in cache ???
else:
weighted_concepts = self._compute_weights(entries)
weighted_concepts = self.compute_weights(entries)
self.sheerka.om.put(self.RESOLVED_COMPARISON_ENTRY, key_to_use, weighted_concepts)
return weighted_concepts
@staticmethod
def detect_cycles(comparison_objs):
def toposort(comparison_objs):
"""
Topological sort or topological ordering of a directed graph is a linear ordering of its vertices
such that for every directed edge uv from vertex u to vertex v, u comes before v in the ordering
https://en.wikipedia.org/wiki/Topological_sorting
This implementation is taken from https://code.activestate.com/recipes/578272-topological-sort/
"""
if not comparison_objs:
return []
data = {}
for obj in comparison_objs:
data.setdefault(obj.a, set()).add(obj.b) if obj.op == ">" else data.setdefault(obj.b, set()).add(obj.a)
# Ignore self dependencies.
for k, v in data.items():
v.discard(k)
# Find all items that don't depend on anything.
extra_items_in_deps = reduce(set.union, data.values()) - set(data.keys())
# Add empty dependencies where needed
data.update({item: set() for item in extra_items_in_deps})
while True:
ordered = set(item for item, dep in data.items() if not dep)
if not ordered:
break
yield ordered
data = {item: (dep - ordered)
for item, dep in data.items()
if item not in ordered}
if len(data) != 0:
raise ChickenAndEggError(set(flatten(data.values())))
@staticmethod
def get_objs_groups(comparison_objs):
"""
Browse all ComparisonObj definitions group the obj in three categories
* those which are defined as greatest,
* those which are defined as lowest,
* those which are defined as equivalent as another one,
We also returns the list of all object
-> Which make it a tuple of four elements
"""
greatest = set()
lowest = set()
all_items = set()
for co in comparison_objs:
if co.op == ">>":
greatest.add(co.a)
elif co.op == "<<":
lowest.add(co.a)
if co.a is not None:
all_items.add(co.a)
if co.b is not None:
all_items.add(co.b)
return greatest, lowest, all_items
@staticmethod
def group_comparison_objs(comparison_objs, greatest_objs, lowest_objs):
"""
Groups the ComparisonObj in three categories
* those which express relation between greatest objects
* those which express relation between lowest objects
* those which express relation between objects which are neither greatest or lowest
To express a relation between greatest objects, both objects must be flagged as greatest
for example:
if the relation is 'a > b' with a is greatest, but not b,
this relation will NOT go in the greatest list
As a matter of fact, it does not fit in any list as it's an irrelevant information
Note that for efficiency, we consider that the settings are already valid
We cannot find something like
'a > b' with 'a' is lowest and 'b' is greatest
or 'a > b' 'b' lowest and 'a' is not
"""
greatest, lowest, equiv, other = [], [], [], [] # we keep the order
greatest_or_lowest_objs = set(greatest_objs)
greatest_or_lowest_objs.update(lowest_objs)
for co in comparison_objs:
if co.op in ("<", ">"):
if co.a in greatest_objs and co.b in greatest_objs:
greatest.append(co)
elif co.a in lowest_objs and co.b in lowest_objs:
lowest.append(co)
elif not (co.a in greatest_or_lowest_objs or co.b in greatest_or_lowest_objs):
other.append(co)
elif co.op == "=":
equiv.append(co)
return greatest, lowest, equiv, other
@staticmethod
def compute_weights(comparison_objs):
"""
Computes the weights of all items that appear in the comparison objs
"""
greatest_objs, lowest_objs, all_objs = SheerkaComparisonManager.get_objs_groups(comparison_objs)
# co stands for comparison_object
co_greatest, co_lowest, co_equiv, co_other = SheerkaComparisonManager.group_comparison_objs(comparison_objs,
greatest_objs,
lowest_objs)
weights = {obj: NotInit for obj in all_objs}
max_weight = SheerkaComparisonManager.DEFAULT_COMPARISON_VALUE
min_weight = SheerkaComparisonManager.DEFAULT_COMPARISON_VALUE
# manage lowest objects group
sorted_objs = list(SheerkaComparisonManager.toposort(co_lowest))
for group in reversed(sorted_objs):
group_weight = min_weight - 1
min_weight = group_weight
for item in group:
weights[item] = group_weight
# manage normal objects group
sorted_objs = SheerkaComparisonManager.toposort(co_other)
for i, group in enumerate(sorted_objs):
group_weight = i + SheerkaComparisonManager.DEFAULT_COMPARISON_VALUE
max_weight = group_weight
for item in group:
weights[item] = group_weight
# manage greatest objects group
sorted_objs = SheerkaComparisonManager.toposort(co_greatest)
for group in sorted_objs:
group_weight = max_weight + 1
max_weight = group_weight
for item in group:
weights[item] = group_weight
# manage greatest flagged objects that have not been initialized yet
for obj in greatest_objs:
if weights[obj] is NotInit:
weights[obj] = max_weight + 1
# manage lowest flagged objects that have not been initialized yet
for obj in lowest_objs:
if weights[obj] is NotInit:
weights[obj] = min_weight - 1
# manage equiv
for co in co_equiv:
if weights[co.a] is NotInit and weights[co.b] is not NotInit:
weights[co.a] = weights[co.b]
elif weights[co.b] is NotInit and weights[co.a] is not NotInit:
weights[co.b] = weights[co.a]
# set not initialized to default value
for k, v in weights.items():
if v is NotInit:
weights[k] = SheerkaComparisonManager.DEFAULT_COMPARISON_VALUE
return weights
@staticmethod
def validate_new_entry(context, new_entry, previous_entries):
sheerka = context.sheerka
name = SheerkaComparisonManager.NAME
# detect duplicates
for co in previous_entries:
if co.property == new_entry.property and \
co.a == new_entry.a and \
co.b == new_entry.b and \
co.op == new_entry.op and \
co.context == new_entry.context:
return sheerka.ret(name, False, sheerka.new(BuiltinConcepts.COMPARISON_ALREADY_DEFINED))
# detect cycle
cycles = SheerkaComparisonManager.detect_cycles(previous_entries, new_entry)
if cycles:
chicken_an_egg = context.sheerka.new(BuiltinConcepts.CHICKEN_AND_EGG, body=cycles)
return sheerka.ret(name, False, chicken_an_egg)
greatest_objs, lowest_objs, all_objs = SheerkaComparisonManager.get_objs_groups(previous_entries)
# detect IS_GREATEST_CONSTRAINT_ERROR and IS_LESSER_CONSTRAINT_ERROR
if new_entry.op == "<<" and new_entry.a in greatest_objs:
return sheerka.ret(name, False, sheerka.new(BuiltinConcepts.IS_GREATEST_CONSTRAINT_ERROR))
elif new_entry.op == ">>" and new_entry.a in lowest_objs:
return sheerka.ret(name, False, sheerka.new(BuiltinConcepts.IS_LESSER_CONSTRAINT_ERROR))
# implement the following matrix
# +-------+------+-----+-----+-----+
# | | a<< | b<< | a>> | b>> |
# | a < b | ok | nok | nok | ok |
# | a > b | nok | ok | ok | nok |
# |-------+------+-----+-----+-----+
# nok means that new rule cannot be accepted, unless there is a change on the other item
elif new_entry.op == "<":
if new_entry.a in greatest_objs and new_entry.b not in greatest_objs:
return sheerka.ret(name, False, sheerka.new(BuiltinConcepts.IS_GREATEST_CONSTRAINT_ERROR))
elif new_entry.b in lowest_objs and new_entry.a not in lowest_objs:
return sheerka.ret(name, False, sheerka.new(BuiltinConcepts.IS_LESSER_CONSTRAINT_ERROR))
elif new_entry.op == ">":
if new_entry.a in lowest_objs and new_entry.b not in lowest_objs:
return sheerka.ret(name, False, sheerka.new(BuiltinConcepts.IS_LESSER_CONSTRAINT_ERROR))
elif new_entry.b in greatest_objs and new_entry.a not in greatest_objs:
return sheerka.ret(name, False, sheerka.new(BuiltinConcepts.IS_GREATEST_CONSTRAINT_ERROR))
return sheerka.ret(name, True, sheerka.new(BuiltinConcepts.SUCCESS))
@staticmethod
def detect_cycles(previous_comparison_objs, new_comparison_obj: ComparisonObj):
"""
# Thanks to Divyanshu Mehta for contributing this code
# https://www.geeksforgeeks.org/detect-cycle-in-a-graph/?ref=lbp
:param comparison_objs:
:param previous_comparison_objs:
:param new_comparison_obj:
:return:
"""
latest = comparison_objs[-1]
if latest.op == "=":
if new_comparison_obj.op in ("<<", ">>", "="):
return None
def get_graph_and_vertices():
_graph = {}
_vertices = set()
for obj in comparison_objs:
if obj.op == "=":
for obj in previous_comparison_objs + [new_comparison_obj]:
if obj.op in ("<<", ">>", "="):
continue
_vertices.add(obj.a)
@@ -393,7 +504,7 @@ class SheerkaComparisonManager(BaseService):
elif rec_stack[neighbour]:
return True
# The node needs to be poped from
# The node needs to be popped from
# recursion stack before function ends
rec_stack[v] = False
return False
@@ -402,6 +513,7 @@ class SheerkaComparisonManager(BaseService):
visited = {k: False for k in vertices}
rec_stack = {k: False for k in vertices}
if is_cyclic(latest.a): # only need to check from the latest add, since the graph was not cyclic before
if is_cyclic(
new_comparison_obj.a): # only need to check from the latest add, since the graph was not cyclic before
return [k for k, v in rec_stack.items() if v]
return None
src/core/sheerka/services/SheerkaConceptManager.py
+228 -16
View File
@@ -1,14 +1,17 @@
from dataclasses import dataclass
from typing import Set
import core.utils
from cache.Cache import Cache
from cache.CacheManager import ConceptNotFound
from cache.DictionaryCache import DictionaryCache
from cache.ListIfNeededCache import ListIfNeededCache
from cache.SetCache import SetCache
from core.builtin_concepts import ErrorConcept
from core.builtin_concepts_ids import BuiltinConcepts, AllBuiltinConcepts, BuiltinUnique
from core.builtin_helpers import ensure_concept
from core.concept import Concept, DEFINITION_TYPE_DEF, DEFINITION_TYPE_BNF, freeze_concept_attrs, ConceptMetadata
from core.builtin_helpers import ensure_concept, ensure_bnf
from core.concept import Concept, DEFINITION_TYPE_DEF, DEFINITION_TYPE_BNF, freeze_concept_attrs, ConceptMetadata, \
VARIABLE_PREFIX
from core.error import ErrorObj
from core.global_symbols import EVENT_CONCEPT_CREATED, NotInit, NotFound
from core.sheerka.services.sheerka_service import BaseService
@@ -18,6 +21,17 @@ from sdp.sheerkaDataProvider import SheerkaDataProviderDuplicateKeyError
BASE_NODE_PARSER_CLASS = "parsers.BaseNodeParser.BaseNodeParser"
@dataclass
class ChickenAndEggError(Exception):
concepts: Set[str]
@dataclass
class NoFirstTokenError(ErrorObj):
concept: Concept
key: str
@dataclass
class NoModificationFound(ErrorObj):
"""
@@ -82,9 +96,11 @@ class SheerkaConceptManager(BaseService):
CONCEPTS_BY_HASH_ENTRY = "ConceptManager:Concepts_By_Hash" # store hash of concepts definitions (not values)
CONCEPTS_REFERENCES_ENTRY = "ConceptManager:Concepts_References" # tracks references between concepts
CONCEPTS_BY_FIRST_KEYWORD_ENTRY = "ConceptManager:Concepts_By_First_Keyword"
RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY = "ConceptManager:Resolved_Concepts_By_First_Keyword"
def __init__(self, sheerka):
super().__init__(sheerka)
self.bnp = core.utils.get_class(BASE_NODE_PARSER_CLASS) # BaseNodeParser
self.forbidden_meta = {"is_builtin", "key", "id", "props", "variables"}
self.allowed_meta = {attr for attr in vars(ConceptMetadata) if
not attr.startswith("_") and attr not in self.forbidden_meta}
@@ -101,6 +117,7 @@ class SheerkaConceptManager(BaseService):
self.sheerka.bind_service_method(self.get_by_hash, False, visible=False)
self.sheerka.bind_service_method(self.get_by_id, False, visible=False)
self.sheerka.bind_service_method(self.is_not_a_variable, False, visible=False)
self.sheerka.bind_service_method(self.get_concepts_by_first_token, False, visible=False)
register_concept_cache = self.sheerka.om.register_concept_cache
@@ -119,10 +136,22 @@ class SheerkaConceptManager(BaseService):
cache = SetCache().auto_configure(self.CONCEPTS_REFERENCES_ENTRY)
self.sheerka.om.register_cache(self.CONCEPTS_REFERENCES_ENTRY, cache)
cache = DictionaryCache().auto_configure(self.CONCEPTS_BY_FIRST_KEYWORD_ENTRY)
self.sheerka.om.register_cache(self.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, cache)
cache = DictionaryCache().auto_configure(self.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY)
self.sheerka.om.register_cache(self.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY, cache, persist=False)
def initialize_deferred(self, context, is_first_time):
if is_first_time:
self.sheerka.om.put(self.sheerka.OBJECTS_IDS_ENTRY, self.USER_CONCEPTS_IDS, 1000)
# initialize the dictionary of first tokens
self.sheerka.om.get(self.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, None) # to init the cache with the values from sdp
concepts_by_first_keyword = self.sheerka.om.current_cache_manager().copy(self.CONCEPTS_BY_FIRST_KEYWORD_ENTRY)
res = self.resolve_concepts_by_first_keyword(context, concepts_by_first_keyword)
self.sheerka.om.put(self.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY, False, res.body)
def initialize_builtin_concepts(self):
"""
Initializes the builtin concepts
@@ -181,18 +210,18 @@ class SheerkaConceptManager(BaseService):
# check if the bnf definition is correctly computed
try:
self.bnp.ensure_bnf(context, concept)
ensure_bnf(context, concept)
except Exception as ex:
return sheerka.ret(self.NAME, False, ex.args[0])
# compute new concepts_by_first_keyword
init_ret_value = self.bnp.compute_concepts_by_first_token(context, [concept], True)
init_ret_value = self.compute_concepts_by_first_token(context, [concept], True)
if not init_ret_value.status:
return sheerka.ret(self.NAME, False, ErrorConcept(init_ret_value.value))
concepts_by_first_keyword = init_ret_value.body
# computes resolved concepts_by_first_keyword
init_ret_value = self.bnp.resolve_concepts_by_first_keyword(context, concepts_by_first_keyword)
init_ret_value = self.resolve_concepts_by_first_keyword(context, concepts_by_first_keyword)
if not init_ret_value.status:
return sheerka.ret(self.NAME, False, ErrorConcept(init_ret_value.value))
resolved_concepts_by_first_keyword = init_ret_value.body
@@ -202,8 +231,8 @@ class SheerkaConceptManager(BaseService):
concept.freeze_definition_hash()
ontology.add_concept(concept)
ontology.put(sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, False, concepts_by_first_keyword)
ontology.put(sheerka.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY, False, resolved_concepts_by_first_keyword)
ontology.put(self.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, False, concepts_by_first_keyword)
ontology.put(self.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY, False, resolved_concepts_by_first_keyword)
if concept.get_metadata().definition_type == DEFINITION_TYPE_DEF and concept.get_metadata().definition != concept.name:
# allow search by definition when definition relevant
@@ -268,8 +297,8 @@ class SheerkaConceptManager(BaseService):
# To update concept by first keyword
# first remove the old references
keywords = self.bnp.get_first_tokens(sheerka, concept) # keyword of the old concept
concepts_by_first_keyword = cache_manager.copy(sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY)
keywords = self.get_first_tokens(sheerka, concept) # keyword of the old concept
concepts_by_first_keyword = cache_manager.copy(self.CONCEPTS_BY_FIRST_KEYWORD_ENTRY)
for keyword in keywords:
try:
concepts_by_first_keyword[keyword].remove(concept.id)
@@ -279,13 +308,13 @@ class SheerkaConceptManager(BaseService):
pass
# and then update
init_ret_value = self.bnp.compute_concepts_by_first_token(context, [new_concept], False, concepts_by_first_keyword)
init_ret_value = self.compute_concepts_by_first_token(context, [new_concept], False, concepts_by_first_keyword)
if not init_ret_value.status:
return sheerka.ret(self.NAME, False, ErrorConcept(init_ret_value.value))
concepts_by_first_keyword = init_ret_value.body
# computes resolved concepts_by_first_keyword
init_ret_value = self.bnp.resolve_concepts_by_first_keyword(context,
init_ret_value = self.resolve_concepts_by_first_keyword(context,
concepts_by_first_keyword,
{new_concept.id: new_concept})
if not init_ret_value.status:
@@ -302,9 +331,8 @@ class SheerkaConceptManager(BaseService):
cache_manager.put(self.CONCEPTS_REFERENCES_ENTRY, ref, new_concept.id)
cache_manager.update_concept(concept, new_concept)
cache_manager.put(sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, False, concepts_by_first_keyword)
cache_manager.put(sheerka.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY, False,
resolved_concepts_by_first_keyword)
cache_manager.put(self.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, False, concepts_by_first_keyword)
cache_manager.put(self.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY, False, resolved_concepts_by_first_keyword)
# everything seems to be fine. Update the list of attributes
# Caution. Must be done AFTER update_concept()
@@ -620,8 +648,192 @@ class SheerkaConceptManager(BaseService):
concept.get_metadata().key = None
if self._definition_has_changed(to_add) and concept.get_metadata().definition_type == DEFINITION_TYPE_BNF:
concept.set_bnf(None)
self.bnp.ensure_bnf(context, concept)
ensure_bnf(context, concept)
concept.init_key()
return
@staticmethod
def get_first_tokens(sheerka, concept):
"""
:param sheerka:
:param concept:
:return:
"""
if concept.get_bnf():
from parsers.BnfNodeParser import BnfNodeFirstTokenVisitor
bnf_visitor = BnfNodeFirstTokenVisitor(sheerka)
bnf_visitor.visit(concept.get_bnf())
return bnf_visitor.first_tokens
else:
keywords = concept.key.split()
for keyword in keywords:
if keyword.startswith(VARIABLE_PREFIX):
continue
return [keyword]
return None
@staticmethod
def compute_concepts_by_first_token(context, concepts, use_sheerka=False, previous_entries=None):
"""
Create the map describing the first token expected by a concept
eg the dictionary that goes into CONCEPTS_BY_FIRST_KEYWORD_ENTRY
:param context:
:param concepts: lists of concepts to parse
:param use_sheerka: if True, update concepts_by_first_keyword from sheerka
:param previous_entries:
:return:
"""
sheerka = context.sheerka
res = sheerka.om.copy(SheerkaConceptManager.CONCEPTS_BY_FIRST_KEYWORD_ENTRY) if use_sheerka \
else (previous_entries or {})
for concept in concepts:
keywords = SheerkaConceptManager.get_first_tokens(sheerka, concept)
if keywords is None:
# no first token found for a concept ?
return sheerka.ret(sheerka.name, False, NoFirstTokenError(concept, concept.key))
for keyword in keywords:
res.setdefault(keyword, []).append(concept.id)
# 'uniquify' the lists
for k, v in res.items():
res[k] = core.utils.make_unique(v)
return sheerka.ret("BaseNodeParser", True, res)
@staticmethod
def resolve_concepts_by_first_keyword(context, concepts_by_first_keyword, modified_concepts=None):
"""
From a dictionary of first tokens, create another dictionary where all references to other concepts
are resolved
fom example, from entries
{
'c:|1001:' : 'c:|1002:' # which means than the concept 1002 starts with the concept 1001
'foo': 'c:|1001:'
}
It will create
{
'foo': ['c:|1001:, 'c:|1002:'],
}
This dictionary is supposed to go into CONCEPTS_REFERENCES_ENTRY
"""
sheerka = context.sheerka
res = {}
def get_by_id(c_id):
if modified_concepts and c_id in modified_concepts:
return modified_concepts[c_id]
return sheerka.get_by_id(c_id)
def resolve_concepts(concept_str):
c_key, c_id = core.utils.unstr_concept(concept_str)
if c_id in already_seen:
return ChickenAndEggError(already_seen)
already_seen.add(c_id)
resolved = set()
to_resolve = set()
chicken_and_egg = set()
concept = get_by_id(c_id)
if sheerka.isaset(context, concept):
concepts = sheerka.get_set_elements(context, concept)
else:
concepts = [concept]
for concept in concepts:
ensure_bnf(context, concept) # need to make sure that it cannot fail
keywords = SheerkaConceptManager.get_first_tokens(sheerka, concept)
for keyword in keywords:
(to_resolve if keyword.startswith("c:|") else resolved).add(keyword)
for concept_to_resolve_str in to_resolve:
res = resolve_concepts(concept_to_resolve_str)
if isinstance(res, ChickenAndEggError):
chicken_and_egg |= res.concepts
else:
resolved |= res
to_resolve.clear()
if len(resolved) == 0 and len(chicken_and_egg) > 0:
raise ChickenAndEggError(chicken_and_egg)
else:
return resolved
for k, v in concepts_by_first_keyword.items():
if k.startswith("c:|"):
try:
already_seen = set()
resolved_keywords = resolve_concepts(k)
for resolved in resolved_keywords:
res.setdefault(resolved, []).extend(v)
except ChickenAndEggError as ex:
context.log(f"Chicken and egg detected for {k}, concepts={ex.concepts}")
concepts_in_recursion = ex.concepts
# make sure to have all the parents
for parent in v:
concepts_in_recursion.add(parent)
for concept_id in concepts_in_recursion:
# make sure we keep the longest chain
old = sheerka.chicken_and_eggs.get(concept_id)
if old is NotFound or len(old) < len(ex.concepts):
sheerka.chicken_and_eggs.put(concept_id, concepts_in_recursion)
else:
res.setdefault(k, []).extend(v)
# 'uniquify' the lists
for k, v in res.items():
res[k] = core.utils.make_unique(v)
return sheerka.ret("BaseNodeParser", True, res)
def get_concepts_by_first_token(self, token, to_keep, custom=None, to_map=None, strip_quotes=False, parser=None):
"""
Tries to find if there are concepts that match the value of the token
Caution: Returns the actual cache, not a copy
:param token:
:param to_keep: predicate to tell if the concept is eligible
:param custom: lambda name -> List[Concepts] that gives extra concepts, according to the name
:param to_map:
:param strip_quotes: Remove quotes from strings
:param parser: If needed, parser which requested the concepts
:return:
"""
if token.type == TokenKind.WHITESPACE:
return None
if token.type == TokenKind.STRING:
name = token.value[1:-1] if strip_quotes else token.value
else:
name = token.value
custom_concepts = custom(name) if custom else [] # to get extra concepts using an alternative method
result = []
concepts_ids = self.sheerka.om.get(self.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY, name)
if concepts_ids is NotFound:
return custom_concepts if custom else None
for concept_id in concepts_ids:
concept = self.sheerka.get_by_id(concept_id)
if not to_keep(concept):
continue
concept = to_map(concept, parser, self.sheerka) if to_map else concept
result.append(concept)
return core.utils.make_unique(result + custom_concepts,
lambda c: c.concept.id if hasattr(c, "concept") else c.id)
src/core/sheerka/services/SheerkaConceptsAlgebra.py
+1 -3
View File
@@ -119,9 +119,7 @@ class SheerkaConceptsAlgebra(BaseService):
if nb_props == 0:
return res
concepts_manager = self.sheerka.services[SheerkaConceptManager.NAME]
all_concepts = self.sheerka.om.list(concepts_manager.CONCEPTS_BY_ID_ENTRY)
all_concepts = self.sheerka.om.list(SheerkaConceptManager.CONCEPTS_BY_ID_ENTRY)
for c in all_concepts:
score = self._compute_score(c, concept, step_b=round(1 / nb_props, 2))
src/core/sheerka/services/SheerkaMemory.py
+1 -10
View File
@@ -139,14 +139,13 @@ class SheerkaMemory(BaseService):
self.add_to_memory(context, k, v)
self.registration.clear()
def memory(self, context, name=None):
def memory(self, context, name):
"""
Get the list of all memory_objects in memory
:param context:
:param name:
:return:
"""
if name:
name_to_use = name.name if isinstance(name, Concept) else name
self.unregister_object(context, name_to_use)
obj = self.get_from_memory(context, name_to_use)
@@ -158,14 +157,6 @@ class SheerkaMemory(BaseService):
return obj.obj
res = {}
for k, obj in self.sheerka.om.get_all(SheerkaMemory.OBJECTS_ENTRY).items():
if isinstance(obj, list):
obj = obj[-1]
res[k] = obj.obj
return res
def mem(self):
keys = sorted([k for k in self.sheerka.om.list(SheerkaMemory.OBJECTS_ENTRY)])
return {"keys": keys, "len": len(keys)}
src/core/sheerka/services/SheerkaResultManager.py
+32 -3
View File
@@ -10,10 +10,10 @@ from core.utils import as_bag
MAX_EXECUTION_HISTORY = 100
class SheerkaResultConcept(BaseService):
class SheerkaResultManager(BaseService):
NAME = "Result"
# SheerkaResultConcept seems to be a concept that must not support multiple ontology layers
# SheerkaResultManager seems to be a concept that must not support multiple ontology layers
# We must have always access to everything that was done, whatever the ontology
def __init__(self, sheerka, page_size=30):
@@ -23,7 +23,9 @@ class SheerkaResultConcept(BaseService):
self.last_execution = None
self.last_created_concept = None
self.last_created_concept_id = None
self.state_vars = ["last_created_concept_id"]
self.last_error_event_id = None
self.last_errors = None
self.state_vars = ["last_created_concept_id", "last_error_event_id"]
def initialize(self):
self.sheerka.bind_service_method(self.get_results_by_digest, True) # digest is recorded
@@ -33,6 +35,7 @@ class SheerkaResultConcept(BaseService):
self.sheerka.bind_service_method(self.get_execution_item, False)
self.sheerka.bind_service_method(self.get_last_return_value, False, as_name="last_ret")
self.sheerka.bind_service_method(self.get_last_created_concept, False, as_name="last_created_concept")
self.sheerka.bind_service_method(self.get_last_error, False, as_name="last_err")
def initialize_deferred(self, context, is_first_time):
self.restore_values(*self.state_vars)
@@ -44,6 +47,8 @@ class SheerkaResultConcept(BaseService):
self.last_execution = None
self.last_created_concept = None
self.last_created_concept_id = None
self.last_error_event_id = None
self.last_errors = None
def save_state(self, context):
self.store_values(context, *self.state_vars)
@@ -246,6 +251,12 @@ class SheerkaResultConcept(BaseService):
self.executions_contexts_cache.put(execution_context.event.get_digest(), execution_context)
self.last_execution = execution_context
in_error = [ret for ret in execution_context.values["return_values"] if not ret.status]
if in_error:
self.last_error_event_id = execution_context.event.get_digest()
self.last_errors = in_error[0] if len(in_error) == 1 else in_error
self.store_var(execution_context, "last_error_event_id")
def get_last_return_value(self, context):
"""
Return the last return value(s)
@@ -267,6 +278,24 @@ class SheerkaResultConcept(BaseService):
return self.sheerka.new(BuiltinConcepts.NOT_FOUND, body={"query": "last"})
def get_last_error(self):
if self.last_errors:
return self.last_errors
if self.last_error_event_id is None:
return self.sheerka.new(BuiltinConcepts.NOT_FOUND, body={"query": "get_last_error"})
# search in history
if self.last_error_event_id in self.executions_contexts_cache:
execution_result = self.executions_contexts_cache.get(self.last_error_event_id)
else:
execution_result = self.sheerka.om.current_sdp().load_result(self.last_error_event_id)
in_error = [ret for ret in execution_result.values["return_values"] if not ret.status]
self.last_errors = in_error[0] if len(in_error) == 1 else in_error
return self.last_errors
def new_concept_created(self, context, concept):
self.last_created_concept = concept
self.last_created_concept_id = concept.id
src/core/sheerka/services/SheerkaRuleManager.py
+47 -7
View File
@@ -522,6 +522,7 @@ class SheerkaRuleManager(BaseService):
self.sheerka.bind_service_method(self.get_rule_by_name, False)
self.sheerka.bind_service_method(self.dump_desc_rule, False, as_name="desc_rule")
self.sheerka.bind_service_method(self.get_format_rules, False, visible=False)
self.sheerka.bind_service_method(self.resolve_rule, False, visible=False)
cache = Cache().auto_configure(self.FORMAT_RULE_ENTRY)
self.sheerka.om.register_cache(self.FORMAT_RULE_ENTRY, cache, True, True)
@@ -566,7 +567,7 @@ class SheerkaRuleManager(BaseService):
:return:
"""
# get the priorities
rules_weights = self.sheerka.get_concepts_weights(BuiltinConcepts.PRECEDENCE, RULE_COMPARISON_CONTEXT)
rules_weights = self.sheerka.get_weights(BuiltinConcepts.PRECEDENCE, RULE_COMPARISON_CONTEXT)
# update the priorities
for rule in self.sheerka.om.list(self.FORMAT_RULE_ENTRY, cache_only=True):
@@ -712,6 +713,11 @@ class SheerkaRuleManager(BaseService):
Rule("print", "Display multiple success",
"isinstance(__ret_container, BuiltinConcepts.MULTIPLE_SUCCESS)",
"list(__ret_container.body)"),
# # index=[9] in code, id=10 in debug
# Rule("print", "Display simple result when only one success",
# "len(__rets)==1 and __rets[0].status == True",
# "{__rets[0].body}"),
]
for r in rules:
@@ -737,12 +743,7 @@ class SheerkaRuleManager(BaseService):
if rule_id is None:
return None
rule = self.sheerka.om.get(self.FORMAT_RULE_ENTRY, rule_id)
if rule is not NotFound:
return rule
rule = self.sheerka.om.get(self.EXEC_RULE_ENTRY, rule_id)
if rule is not NotFound:
if rule := self._inner_get_by_id(rule_id):
return rule
metadata = [("id", rule_id)]
@@ -815,3 +816,42 @@ class SheerkaRuleManager(BaseService):
else:
raise NotImplementedError(r)
return res
def resolve_rule(self, context, obj):
if obj is None:
return None
elif isinstance(obj, str):
# search by id first
if rule := self._inner_get_by_id(obj):
return rule
elif isinstance(obj, Token) and obj.type == TokenKind.RULE:
# search by id first
if rule := self._inner_get_by_id(obj.value[1]):
return rule
# try via indirection
if (rule_id := self.sheerka.get_from_short_term_memory(context, obj.value[1])) is not NotFound and \
(rule := self._inner_get_by_id(str(rule_id))) is not None:
return rule
elif isinstance(obj, Rule):
if obj.metadata.id_is_unresolved:
if (rule_id := self.sheerka.get_from_short_term_memory(context, obj.id)) is not NotFound and \
(rule := self._inner_get_by_id(str(rule_id))) is not None:
return rule
else:
return obj
return None
def _inner_get_by_id(self, rule_id):
rule = self.sheerka.om.get(self.FORMAT_RULE_ENTRY, rule_id)
if rule is not NotFound:
return rule
rule = self.sheerka.om.get(self.EXEC_RULE_ENTRY, rule_id)
if rule is not NotFound:
return rule
return None
src/core/sheerka/services/sheerka_service.py
+6
View File
@@ -40,3 +40,9 @@ class BaseService:
for prop_name in args:
if (value := self.sheerka.load_var(self.NAME, prop_name)) is not NotFound:
setattr(self, prop_name, value)
def store_var(self, context, var_name):
"""
Store/record the value of an attribute
"""
self.sheerka.record_var(context, self.NAME, var_name, getattr(self, var_name))
src/core/utils.py
+9 -1
View File
@@ -5,9 +5,10 @@ import os
import pkgutil
from copy import deepcopy
from pyparsing import *
from core.global_symbols import CustomType
from core.tokenizer import TokenKind, Tokenizer
from pyparsing import *
COLORS = {
"black",
@@ -600,6 +601,13 @@ def flatten_all_children(item, get_children):
return inner_get_all_children(item)
def flatten(list_of_lists):
"""
Flatten an list containing other lists
"""
return [item for sublist in list_of_lists for item in sublist]
def get_text_from_tokens(tokens, custom_switcher=None, tracker=None):
"""
Create the source code, from the list of token
src/evaluators/PythonEvaluator.py
+2 -3
View File
@@ -328,15 +328,14 @@ class PythonEvaluator(OneReturnValueEvaluator):
:return:
"""
if isinstance(name, Rule):
return name
return context.sheerka.resolve_rule(context, name)
if isinstance(name, Concept):
name = core.builtin_helpers.ensure_evaluated(context, name)
return name
if isinstance(name, Token) and name.type == TokenKind.RULE:
rule = context.sheerka.get_rule_by_id(name.value[1]) # TODO: need a resolve function for the rules
return rule if isinstance(rule, Rule) else None
return context.sheerka.resolve_rule(context, name)
if isinstance(name, tuple):
raise Exception()
src/evaluators/RuleEvaluator.py
+3 -2
View File
@@ -1,6 +1,6 @@
from core.builtin_concepts import BuiltinConcepts, ParserResultConcept
from core.global_symbols import NotFound
from core.rule import Rule, ACTION_TYPE_DEFERRED
from core.rule import Rule
from evaluators.BaseEvaluator import OneReturnValueEvaluator
@@ -34,7 +34,8 @@ class RuleEvaluator(OneReturnValueEvaluator):
success = True
for r in rules:
# Browse the rules to find possible deferred rules
if r.metadata.action_type == ACTION_TYPE_DEFERRED:
# I don't use resolve_rule because I need to have BuiltinConcepts.UNKNOWN_RULE when not found
if r.metadata.id_is_unresolved:
rule_id = sheerka.get_from_short_term_memory(context, r.id)
rule = sheerka.get_rule_by_id(str(rule_id if rule_id is not NotFound else r.id))
resolved.append(rule)
src/parsers/BaseNodeParser.py
+5 -257
View File
@@ -1,12 +1,9 @@
from collections import namedtuple
from dataclasses import dataclass
from enum import Enum
from typing import Set
import core.utils
from core.builtin_concepts import BuiltinConcepts
from core.concept import VARIABLE_PREFIX, Concept, DEFINITION_TYPE_BNF, ConceptParts
from core.global_symbols import NotFound
from core.concept import Concept, ConceptParts
from core.rule import Rule
from core.tokenizer import TokenKind, Token
from parsers.BaseParser import Node, BaseParser, ParsingError
@@ -14,17 +11,6 @@ from parsers.BaseParser import Node, BaseParser, ParsingError
DEBUG_COMPILED = True
@dataclass
class ChickenAndEggError(Exception):
concepts: Set[str]
@dataclass
class NoFirstTokenError(ParsingError):
concept: Concept
key: str
@dataclass()
class LexerNode(Node):
start: int # starting index in the tokens list
@@ -827,248 +813,10 @@ class BaseNodeParser(BaseParser):
:param concepts
:return:
"""
concepts_by_first_keyword = self.compute_concepts_by_first_token(context, concepts).body
resolved = self.resolve_concepts_by_first_keyword(context, concepts_by_first_keyword).body
from core.sheerka.services.SheerkaConceptManager import SheerkaConceptManager
concepts_by_first_keyword = SheerkaConceptManager.compute_concepts_by_first_token(context, concepts).body
resolved = SheerkaConceptManager.resolve_concepts_by_first_keyword(context, concepts_by_first_keyword).body
context.sheerka.om.put(context.sheerka.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY,
context.sheerka.om.put(SheerkaConceptManager.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY,
False,
resolved)
def get_concepts(self, token, to_keep, custom=None, to_map=None, strip_quotes=False):
"""
Tries to find if there are concepts that match the value of the token
Caution: Returns the actual cache, not a copy
:param token:
:param to_keep: predicate to tell if the concept is eligible
:param custom: lambda name -> List[Concepts] that gives extra concepts, according to the name
:param to_map:
:param strip_quotes: Remove quotes from strings
:return:
"""
if token.type == TokenKind.WHITESPACE:
return None
if token.type == TokenKind.STRING:
name = token.value[1:-1] if strip_quotes else token.value
else:
name = token.value
custom_concepts = custom(name) if custom else [] # to get extra concepts using an alternative method
result = []
concepts_ids = self.sheerka.om.get(self.sheerka.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY, name)
if concepts_ids is NotFound:
return custom_concepts if custom else None
for concept_id in concepts_ids:
concept = self.sheerka.get_by_id(concept_id)
if not to_keep(concept):
continue
concept = to_map(concept, self, self.sheerka) if to_map else concept
result.append(concept)
return core.utils.make_unique(result + custom_concepts,
lambda c: c.concept.id if hasattr(c, "concept") else c.id)
@staticmethod
def compute_concepts_by_first_token(context, concepts, use_sheerka=False, previous_entries=None):
"""
Create the map describing the first token expected by a concept
:param context:
:param concepts: lists of concepts to parse
:param use_sheerka: if True, update concepts_by_first_keyword from sheerka
:param previous_entries:
:return:
"""
sheerka = context.sheerka
res = sheerka.om.copy(sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY) if use_sheerka else (previous_entries or {})
for concept in concepts:
keywords = BaseNodeParser.get_first_tokens(sheerka, concept)
if keywords is None:
# no first token found for a concept ?
return sheerka.ret(sheerka.name, False, NoFirstTokenError(concept, concept.key))
for keyword in keywords:
res.setdefault(keyword, []).append(concept.id)
# 'uniquify' the lists
for k, v in res.items():
res[k] = core.utils.make_unique(v)
return sheerka.ret("BaseNodeParser", True, res)
@staticmethod
def resolve_concepts_by_first_keyword(context, concepts_by_first_keyword, modified_concepts=None):
sheerka = context.sheerka
res = {}
def get_by_id(c_id):
if modified_concepts and c_id in modified_concepts:
return modified_concepts[c_id]
return sheerka.get_by_id(c_id)
def resolve_concepts(concept_str):
c_key, c_id = core.utils.unstr_concept(concept_str)
if c_id in already_seen:
return ChickenAndEggError(already_seen)
already_seen.add(c_id)
resolved = set()
to_resolve = set()
chicken_and_egg = set()
concept = get_by_id(c_id)
if sheerka.isaset(context, concept):
concepts = sheerka.get_set_elements(context, concept)
else:
concepts = [concept]
for concept in concepts:
BaseNodeParser.ensure_bnf(context, concept) # need to make sure that it cannot fail
keywords = BaseNodeParser.get_first_tokens(sheerka, concept)
for keyword in keywords:
(to_resolve if keyword.startswith("c:|") else resolved).add(keyword)
for concept_to_resolve_str in to_resolve:
res = resolve_concepts(concept_to_resolve_str)
if isinstance(res, ChickenAndEggError):
chicken_and_egg |= res.concepts
else:
resolved |= res
to_resolve.clear()
if len(resolved) == 0 and len(chicken_and_egg) > 0:
raise ChickenAndEggError(chicken_and_egg)
else:
return resolved
for k, v in concepts_by_first_keyword.items():
if k.startswith("c:|"):
try:
already_seen = set()
resolved_keywords = resolve_concepts(k)
for resolved in resolved_keywords:
res.setdefault(resolved, []).extend(v)
except ChickenAndEggError as ex:
context.log(f"Chicken and egg detected for {k}, concepts={ex.concepts}")
concepts_in_recursion = ex.concepts
# make sure to have all the parents
for parent in v:
concepts_in_recursion.add(parent)
for concept_id in concepts_in_recursion:
# make sure we keep the longest chain
old = sheerka.chicken_and_eggs.get(concept_id)
if old is NotFound or len(old) < len(ex.concepts):
sheerka.chicken_and_eggs.put(concept_id, concepts_in_recursion)
else:
res.setdefault(k, []).extend(v)
# 'uniquify' the lists
for k, v in res.items():
res[k] = core.utils.make_unique(v)
return sheerka.ret("BaseNodeParser", True, res)
@staticmethod
def get_referenced_concepts(context, concept_id, already_seen):
"""
Gets all the tokens that may allow to recognize concept concept_id
Basically, it returns all the starting tokens for concept concept_id
CHICKEN_AND_EGG is returned when a circular references are found
:param context:
:param concept_id:
:param already_seen:
:return:
"""
if concept_id in already_seen:
return ChickenAndEggError(already_seen)
already_seen.add(concept_id)
resolved = set()
to_resolve = set()
chicken_and_egg = set()
sheerka = context.sheerka
concept = sheerka.get_by_id(concept_id)
if sheerka.isaset(context, concept):
concepts = sheerka.get_set_elements(context, concept)
else:
concepts = [concept]
for concept in concepts:
BaseNodeParser.ensure_bnf(context, concept) # need to make sure that it cannot fail
keywords = BaseNodeParser.get_first_tokens(sheerka, concept)
for keyword in keywords:
(to_resolve if keyword.startswith("c:|") else resolved).add(keyword)
for concept_to_resolve_str in to_resolve:
c_key, c_id = core.utils.unstr_concept(concept_to_resolve_str)
res = BaseNodeParser.get_referenced_concepts(context, c_id, already_seen)
if isinstance(res, ChickenAndEggError):
chicken_and_egg |= res.concepts
else:
resolved |= res
to_resolve.clear()
if len(resolved) == 0 and len(chicken_and_egg) > 0:
raise ChickenAndEggError(chicken_and_egg)
else:
return resolved
@staticmethod
def resolve_sya_associativity_and_precedence(context, sya):
pass
@staticmethod
def get_first_tokens(sheerka, concept):
"""
:param sheerka:
:param concept:
:return:
"""
if concept.get_bnf():
from parsers.BnfNodeParser import BnfNodeFirstTokenVisitor
bnf_visitor = BnfNodeFirstTokenVisitor(sheerka)
bnf_visitor.visit(concept.get_bnf())
return bnf_visitor.first_tokens
else:
keywords = concept.key.split()
for keyword in keywords:
if keyword.startswith(VARIABLE_PREFIX):
continue
return [keyword]
return None
@staticmethod
def ensure_bnf(context, concept, parser_name="BaseNodeParser"):
if concept.get_metadata().definition_type == DEFINITION_TYPE_BNF and not concept.get_bnf():
from parsers.BnfDefinitionParser import BnfDefinitionParser
regex_parser = BnfDefinitionParser()
desc = f"Resolving BNF '{concept.get_metadata().definition}'"
with context.push(BuiltinConcepts.INIT_BNF,
concept,
who=parser_name,
obj=concept,
desc=desc) as sub_context:
sub_context.add_inputs(parser_input=concept.get_metadata().definition)
bnf_parsing_ret_val = regex_parser.parse(sub_context, concept.get_metadata().definition)
sub_context.add_values(return_values=bnf_parsing_ret_val)
if not bnf_parsing_ret_val.status:
raise Exception(bnf_parsing_ret_val.value)
concept.set_bnf(bnf_parsing_ret_val.body.body)
if concept.id:
context.sheerka.get_by_id(concept.id).set_bnf(concept.get_bnf()) # update bnf in cache
src/parsers/BnfNodeParser.py
+2 -2
View File
@@ -1291,7 +1291,7 @@ class BnfNodeParser(BaseNodeParser):
debugger.debug_log(debug_prefix + ", all parsers are locked. Nothing to do.")
continue
concepts = self.get_concepts(token, self._is_eligible, strip_quotes=False)
concepts = context.sheerka.get_concepts_by_first_token(token, self._is_eligible, strip_quotes=False)
if not concepts:
if debugger.is_enabled():
@@ -1475,7 +1475,7 @@ class BnfNodeParser(BaseNodeParser):
desc = f"Resolve concept parsing expression for '{concept}'. {key_to_use=}"
with context.push(BuiltinConcepts.INIT_BNF, concept, who=self.name, obj=concept, desc=desc) as sub_context:
if not concept.get_bnf(): # 'if' is done outside to save a function call. Not sure it worth it.
BaseNodeParser.ensure_bnf(sub_context, concept, self.name)
core.builtin_helpers.ensure_bnf(sub_context, concept, self.name)
grammar[key_to_use] = UnderConstruction(concept.id)
src/parsers/RuleParser.py
+2 -2
View File
@@ -1,5 +1,5 @@
from core.builtin_concepts import BuiltinConcepts
from core.rule import Rule, ACTION_TYPE_DEFERRED
from core.rule import Rule
from core.sheerka.services.SheerkaExecute import ParserInput
from core.tokenizer import TokenKind
from parsers.BaseParser import BaseParser, ParsingError, UnexpectedTokenParsingError
@@ -70,7 +70,7 @@ class RuleParser(BaseParser):
rule = sheerka.get_rule_by_id(token.value[1])
else:
rule = Rule().set_id(token.value[1])
rule.metadata.action_type = ACTION_TYPE_DEFERRED
rule.metadata.id_is_unresolved = True
if sheerka.isinstance(rule, BuiltinConcepts.UNKNOWN_RULE):
return sheerka.ret(self.name,
src/parsers/SequenceNodeParser.py
+1 -1
View File
@@ -250,7 +250,7 @@ class SequenceNodeParser(BaseNodeParser):
return a if isinstance(a, list) else [a]
concepts_by_name = as_list(self.sheerka.resolve(token))
concepts_by_first_keyword = new_instances(super().get_concepts(token, self._is_eligible))
concepts_by_first_keyword = new_instances(self.sheerka.get_concepts_by_first_token(token, self._is_eligible))
if concepts_by_name is None:
return concepts_by_first_keyword
src/parsers/SyaNodeParser.py
+5 -2
View File
@@ -136,7 +136,7 @@ class SyaConceptDef:
# otherwise, use sheerka # KSI 20210109 otherwise or override ??
if sheerka:
concept_weight = parser.sheerka.get_concepts_weights(BuiltinConcepts.PRECEDENCE, CONCEPT_COMPARISON_CONTEXT)
concept_weight = parser.sheerka.get_weights(BuiltinConcepts.PRECEDENCE, CONCEPT_COMPARISON_CONTEXT)
if concept.str_id in concept_weight:
sya_concept_def.precedence = concept_weight[concept.str_id]
@@ -1210,7 +1210,10 @@ class SyaNodeParser(BaseNodeParser):
debugger.debug_log(debug_prefix + f", all parsers are locked")
continue
concepts_def = self.get_concepts(token, self._is_eligible, to_map=SyaConceptDef.get_sya_concept_def)
concepts_def = context.sheerka.get_concepts_by_first_token(token,
self._is_eligible,
to_map=SyaConceptDef.get_sya_concept_def,
parser=self)
if not concepts_def:
if debugger.is_enabled():
debugger.debug_log(debug_prefix + f", no concept found")
tests/cache/test_ListCache.py
Vendored
+8 -1
View File
@@ -6,7 +6,8 @@ from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
from tests.cache import FakeSdp
class TestListIfNeededCache(TestUsingMemoryBasedSheerka):
class TestListCache(TestUsingMemoryBasedSheerka):
def test_i_can_put_and_retrieve_value_from_list_cache(self):
cache = ListCache()
@@ -61,6 +62,12 @@ class TestListIfNeededCache(TestUsingMemoryBasedSheerka):
cache.put("key", "value2", alt_sdp=FakeSdp(get_alt_value=lambda cache_name, key: "xxx"))
assert cache.get("key") == ["value1", "value2"]
def test_i_can_get_when_alt_sdp(self):
cache = ListCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name")
cache.get("key", alt_sdp=FakeSdp(get_alt_value=lambda cache_name, key: ["value1"]))
assert cache.get("key") == ["value1"]
def test_i_can_update_from_list_cache(self):
cache = ListCache()
tests/core/test_SheerkaComparisonManager.py
+249 -31
View File
@@ -1,16 +1,35 @@
import pytest
from core.builtin_concepts import BuiltinConcepts
from core.concept import Concept
from core.global_symbols import EVENT_CONCEPT_PRECEDENCE_MODIFIED, CONCEPT_COMPARISON_CONTEXT, \
EVENT_RULE_PRECEDENCE_MODIFIED, \
RULE_COMPARISON_CONTEXT
EVENT_RULE_PRECEDENCE_MODIFIED, RULE_COMPARISON_CONTEXT
from core.sheerka.services.SheerkaComparisonManager import SheerkaComparisonManager, ComparisonObj
from core.sheerka.services.SheerkaConceptManager import ChickenAndEggError
from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
class TestSheerkaGreaterThanManager(TestUsingMemoryBasedSheerka):
@staticmethod
def get_comparison_objs(lst, prop_name="prop_name", context="#"):
for item in lst:
if ">>" in item:
a = item.split(">>")[0]
yield ComparisonObj("id", prop_name, a.strip(), None, ">>", context)
elif "<<" in item:
a = item.split("<<")[0]
yield ComparisonObj("id", prop_name, a.strip(), None, "<<", context)
elif ">" in item:
a, b = item.split(">")
yield ComparisonObj("id", prop_name, a.strip(), b.strip(), ">", context)
elif "<" in item:
a, b = item.split("<")
yield ComparisonObj("id", prop_name, a.strip(), b.strip(), "<", context)
else:
a, b = item.split("=")
yield ComparisonObj("id", prop_name, a.strip(), b.strip(), "=", context)
@staticmethod
def execution_definition(context, service, concepts_map, definition):
if ">>" in definition:
@@ -214,13 +233,13 @@ class TestSheerkaGreaterThanManager(TestUsingMemoryBasedSheerka):
for entry in entries:
self.execution_definition(context, service, concepts_map, entry)
assert service.get_concepts_weights("prop_name") == expected
assert service.get_weights("prop_name") == expected
def test_i_can_get_concept_weight_when_no_comparison_is_defined(self):
sheerka, context = self.init_concepts()
service = sheerka.services[SheerkaComparisonManager.NAME]
assert service.get_concepts_weights("prop_name") == {}
assert service.get_weights("prop_name") == {}
def test_i_can_recover_from_deleted_weight(self):
sheerka, context, one = self.init_concepts("one")
@@ -229,7 +248,7 @@ class TestSheerkaGreaterThanManager(TestUsingMemoryBasedSheerka):
service.set_is_lesser(context, "prop_name", one)
sheerka.om.clear(service.RESOLVED_COMPARISON_ENTRY)
assert service.get_concepts_weights("prop_name") == {"c:one|1001:": 0}
assert service.get_weights("prop_name") == {"c:one|1001:": 0}
def test_i_can_get_partition(self):
sheerka, context, one, two, three = self.init_concepts("one", "two", "three")
@@ -246,6 +265,38 @@ class TestSheerkaGreaterThanManager(TestUsingMemoryBasedSheerka):
3: ["c:three|1003:"],
}
def test_i_can_toposort(self):
# using sample test from https://code.activestate.com/recipes/578272-topological-sort/
comparison_objs = self.get_comparison_objs([
"2 < 11",
"11 > 9",
"9 < 8",
"10 < 11",
"3 > 10",
"11 < 7",
"11 < 5",
"7 > 8",
"3 > 8"])
assert list(SheerkaComparisonManager.toposort(comparison_objs)) == [{'2', '9', '10'},
{'8', '11'},
{'3', '5', '7'}]
def test_i_can_toposort_when_no_data(self):
assert list(SheerkaComparisonManager.toposort([])) == []
def test_i_cannot_toposort_when_cycle(self):
comparison_objs = self.get_comparison_objs([
"1 < 2",
"2 < 3",
"3 < 1",
"1 < 4", # no issue with this
"2 < 5", # no issue with this
])
with pytest.raises(ChickenAndEggError) as ex:
list(SheerkaComparisonManager.toposort(comparison_objs))
assert ex.value.concepts == {"1", "2", "3"}
def test_i_can_detect_chicken_and_egg(self):
sheerka, context, one, two = self.init_concepts("one", "two")
service = sheerka.services[SheerkaComparisonManager.NAME]
@@ -255,7 +306,7 @@ class TestSheerkaGreaterThanManager(TestUsingMemoryBasedSheerka):
assert not res.status
assert sheerka.isinstance(res.body, BuiltinConcepts.CHICKEN_AND_EGG)
assert set(res.body.body) == {one, two}
assert set(res.body.body) == {one.str_id, two.str_id}
def test_i_can_detect_more_complex_chicken_and_egg(self):
sheerka, context, one, two, three, four, five = self.init_concepts("one", "two", "three", "four", "five")
@@ -270,7 +321,7 @@ class TestSheerkaGreaterThanManager(TestUsingMemoryBasedSheerka):
assert not res.status
assert sheerka.isinstance(res.body, BuiltinConcepts.CHICKEN_AND_EGG)
assert set(res.body.body) == {one, two, five}
assert set(res.body.body) == {one.str_id, two.str_id, five.str_id}
def test_i_can_give_the_same_information_in_many_ways(self):
sheerka, context, one, two = self.init_concepts("one", "two")
@@ -292,10 +343,10 @@ class TestSheerkaGreaterThanManager(TestUsingMemoryBasedSheerka):
service = sheerka.services[SheerkaComparisonManager.NAME]
service.set_is_lesser(context, "prop_name", one)
assert service.get_concepts_weights("prop_name") == {"c:one|1001:": 0} # DEFAULT_COMPARISON_VALUE - 1
assert service.get_weights("prop_name") == {"c:one|1001:": 0} # DEFAULT_COMPARISON_VALUE - 1
sheerka.set_is_greater_than(context, "prop_name", three, two)
assert service.get_concepts_weights("prop_name") == {"c:one|1001:": 0, "c:two|1002:": 1, "c:three|1003:": 2}
assert service.get_weights("prop_name") == {"c:one|1001:": 0, "c:two|1002:": 1, "c:three|1003:": 2}
# I can commit
sheerka.om.commit(context)
@@ -315,7 +366,7 @@ class TestSheerkaGreaterThanManager(TestUsingMemoryBasedSheerka):
sheerka.set_is_less_than(context, "prop_name", less_l, lesser)
sheerka.set_is_greater_than(context, "prop_name", less_l, even_more_l)
assert service.get_concepts_weights("prop_name") == {"c:lesser|1001:": 0,
assert service.get_weights("prop_name") == {"c:lesser|1001:": 0,
"c:less_l|1002:": -1,
"c:even_less_l|1003:": -2}
@@ -325,46 +376,46 @@ class TestSheerkaGreaterThanManager(TestUsingMemoryBasedSheerka):
unless minus_one is defined as lesser itself
:return:
"""
sheerka, context, lesser, foo = self.init_concepts("lesser", "foo")
sheerka, context, lesser, foo, bar = self.init_concepts("lesser", "foo", "bar")
service = sheerka.services[SheerkaComparisonManager.NAME]
service.set_is_lesser(context, "prop_name", lesser)
res = sheerka.set_is_less_than(context, "prop_name", foo, lesser)
assert not res.status
assert sheerka.isinstance(res.body, BuiltinConcepts.INVALID_LESSER_OPERATION)
# sanity check
res = sheerka.set_is_greater_than(context, "prop_name", foo, lesser)
assert res.status
res = sheerka.set_is_less_than(context, "prop_name", bar, lesser)
assert not res.status
assert sheerka.isinstance(res.body, BuiltinConcepts.INVALID_LESSER_OPERATION)
assert sheerka.isinstance(res.body, BuiltinConcepts.IS_LESSER_CONSTRAINT_ERROR)
def test_a_greatest_concept_has_the_highest_weight(self):
sheerka, context, one, two, three = self.init_concepts("one", "two", "three")
service = sheerka.services[SheerkaComparisonManager.NAME]
service.set_is_greatest(context, "prop_name", three)
assert service.get_concepts_weights("prop_name") == {"c:three|1003:": 2} # DEFAULT_COMPARISON_VALUE + 1
assert service.get_weights("prop_name") == {"c:three|1003:": 2} # DEFAULT_COMPARISON_VALUE + 1
sheerka.set_is_greater_than(context, "prop_name", two, one)
assert service.get_concepts_weights("prop_name") == {"c:one|1001:": 1, "c:two|1002:": 2, "c:three|1003:": 3}
assert service.get_weights("prop_name") == {"c:one|1001:": 1, "c:two|1002:": 2, "c:three|1003:": 3}
def test_i_cannot_define_greater_than_a_greatest_if_not_a_greater_itself(self):
sheerka, context, greatest, foo = self.init_concepts("greatest", "foo")
sheerka, context, greatest, foo, bar = self.init_concepts("greatest", "foo", "bar")
service = sheerka.services[SheerkaComparisonManager.NAME]
service.set_is_greatest(context, "prop_name", greatest)
# sanity check
res = sheerka.set_is_less_than(context, "prop_name", foo, greatest)
assert not res.status
assert sheerka.isinstance(res.body, BuiltinConcepts.INVALID_GREATEST_OPERATION)
assert res.status
res = sheerka.set_is_greater_than(context, "prop_name", foo, greatest)
res = sheerka.set_is_greater_than(context, "prop_name", bar, greatest)
assert not res.status
assert sheerka.isinstance(res.body, BuiltinConcepts.INVALID_GREATEST_OPERATION)
assert sheerka.isinstance(res.body, BuiltinConcepts.IS_GREATEST_CONSTRAINT_ERROR)
@pytest.mark.parametrize("definitions, expected", [
(["foo >>", "foo <<"], BuiltinConcepts.INVALID_GREATEST_OPERATION),
(["foo <<", "foo >>"], BuiltinConcepts.INVALID_LESSER_OPERATION),
(["foo >>", "foo <<"], BuiltinConcepts.IS_GREATEST_CONSTRAINT_ERROR),
(["foo <<", "foo >>"], BuiltinConcepts.IS_LESSER_CONSTRAINT_ERROR),
])
def test_i_cannot_define_a_concept_as_lesser_and_greatest_at_the_same_time(self, definitions, expected):
sheerka, context, foo = self.init_concepts("foo")
@@ -386,7 +437,7 @@ class TestSheerkaGreaterThanManager(TestUsingMemoryBasedSheerka):
sheerka.set_is_less_than(context, "prop_name", greatest, more_g)
sheerka.set_is_greater_than(context, "prop_name", even_more_g, more_g)
assert service.get_concepts_weights("prop_name") == {"c:greatest|1001:": 2,
assert service.get_weights("prop_name") == {"c:greatest|1001:": 2,
"c:more_g|1002:": 3,
"c:even_more_g|1003:": 4}
@@ -405,7 +456,7 @@ class TestSheerkaGreaterThanManager(TestUsingMemoryBasedSheerka):
sheerka.set_is_less_than(context, "prop_name", three, four)
assert service.get_concepts_weights("prop_name") == {
assert service.get_weights("prop_name") == {
"c:one|1001:": -1,
"c:two|1002:": 0,
"c:three|1003:": 1,
@@ -417,7 +468,7 @@ class TestSheerkaGreaterThanManager(TestUsingMemoryBasedSheerka):
sheerka.set_is_less_than(context, "prop_name", three_and_half, four)
sheerka.set_is_greater_than(context, "prop_name", three_and_half, three)
assert service.get_concepts_weights("prop_name") == {
assert service.get_weights("prop_name") == {
"c:one|1001:": -1,
"c:two|1002:": 0,
"c:three|1003:": 1,
@@ -446,7 +497,7 @@ class TestSheerkaGreaterThanManager(TestUsingMemoryBasedSheerka):
service.set_is_greater_than(context, "prop_name", five, one)
service.set_is_greater_than(context, "prop_name", five, three)
assert service.get_concepts_weights("prop_name") == {
assert service.get_weights("prop_name") == {
"c:zero|1001:": 0,
"c:one|1002:": 1,
"c:two|1003:": 2,
@@ -471,7 +522,7 @@ class TestSheerkaGreaterThanManager(TestUsingMemoryBasedSheerka):
res = self.execution_definition(context, service, concepts_map, definition)
assert not res.status
assert sheerka.isinstance(res.body, BuiltinConcepts.CONCEPT_ALREADY_DEFINED)
assert sheerka.isinstance(res.body, BuiltinConcepts.COMPARISON_ALREADY_DEFINED)
def test_an_event_is_fired_when_modifying_concepts_precedence(self):
sheerka, context, one, two, foo = self.init_concepts("one", "two", "foo")
@@ -514,3 +565,170 @@ class TestSheerkaGreaterThanManager(TestUsingMemoryBasedSheerka):
sheerka.set_is_greater_than(context, BuiltinConcepts.PRECEDENCE, r1, r2, RULE_COMPARISON_CONTEXT)
assert event_received
@pytest.mark.parametrize("entries, expected", [
([], (set(), set(), set())),
(["a<<", "a<<", "b<<", "a<c"], (set(), {"a", "b"}, {"a", "b", "c"})),
(["a>>", "a>>", "b>>", "a<c"], ({"a", "b"}, set(), {"a", "b", "c"})),
(["a<<", "b>>", "c=d", "e<f", "g>h"], ({"b"}, {"a"}, {"a", "b", "c", "d", "e", "f", "g", "h"})),
])
def test_i_can_get_objs_groups(self, entries, expected):
comparison_objs = self.get_comparison_objs(entries)
assert SheerkaComparisonManager.get_objs_groups(comparison_objs) == expected
@pytest.mark.parametrize("entries, greatest, lowest, expected_tuple", [
# greatest
(["a < b"], {"a", "b"}, set(), (["a < b"], [], [], [])),
(["a > b"], {"a", "b"}, set(), (["a > b"], [], [], [])),
# lowest
(["a < b"], set(), {"a", "b"}, ([], ["a < b"], [], [])),
(["a > b"], set(), {"a", "b"}, ([], ["a > b"], [], [])),
# equiv
(["a = b"], set(), set(), ([], [], ["a = b"], [])),
# neither
(["a < b"], set(), set(), ([], [], [], ["a < b"])),
(["a > b"], set(), set(), ([], [], [], ["a > b"])),
# irrelevant information
(["a > b"], {"a"}, set(), ([], [], [], [])),
(["a > b"], set(), {"b"}, ([], [], [], [])),
(["a < b"], {"b"}, set(), ([], [], [], [])),
(["a < b"], set(), {"a"}, ([], [], [], [])),
# mixed
(["a<b", "c<d", "e<f", "z=b"], {"a", "b", "c", "d"}, set(), (["a<b", "c<d"], [], ["z=b"], ["e<f"])),
])
def test_i_can_group_comparison_objs(self, entries, greatest, lowest, expected_tuple):
comparison_objs = self.get_comparison_objs(entries)
g, l, e, o = SheerkaComparisonManager.group_comparison_objs(comparison_objs, greatest, lowest)
def compare_results(actual_result, expected_result):
assert len(actual_result) == len(expected_result)
for actual, expected in zip(actual_result, expected_result):
expected_comparison_obj = list(self.get_comparison_objs([expected]))[0]
assert actual.a == expected_comparison_obj.a
assert actual.b == expected_comparison_obj.b
assert actual.op == expected_comparison_obj.op
compare_results(g, expected_tuple[0])
compare_results(l, expected_tuple[1])
compare_results(e, expected_tuple[2])
compare_results(o, expected_tuple[3])
@pytest.mark.parametrize("entries, expected", [
([], {}),
(["a < b"], {"a": 1, "b": 2}),
(["a > b"], {"a": 2, "b": 1}),
(["a < b", "b > a"], {"a": 1, "b": 2}),
(["a < b", "b < c"], {"a": 1, "b": 2, "c": 3}),
(["a < b", "a < c"], {"a": 1, "b": 2, "c": 2}),
# greatest between themselves
(["a < b", "a>>", "b>>"], {"a": 2, "b": 3}),
(["a > b", "a>>", "b>>"], {"a": 3, "b": 2}),
(["a < b", "b > a", "a>>", "b>>"], {"a": 2, "b": 3}),
(["a < b", "b < c", "a>>", "b>>", "c>>"], {"a": 2, "b": 3, "c": 4}),
(["a < b", "a < c", "a>>", "b>>", "c>>"], {"a": 2, "b": 3, "c": 3}),
# lowest between themselves
(["a < b", "a<<", "b<<"], {"a": -1, "b": 0}),
(["a > b", "a<<", "b<<"], {"a": 0, "b": -1}),
(["a < b", "b > a", "a<<", "b<<"], {"a": -1, "b": 0}),
(["a < b", "b < c", "a<<", "b<<", "c<<"], {"a": -2, "b": -1, "c": 0}),
(["a < b", "a < c", "a<<", "b<<", "c<<"], {"a": -1, "b": 0, "c": 0}),
# greatest that does not appear in other relations
(["a >>", "b < c"], {"a": 3, "b": 1, "c": 2}),
(["a >>", "a > b"], {"a": 2, "b": 1}),
(["a >>", "a > b", "c > d", "d > e"], {"a": 4, "b": 1, "c": 3, "d": 2, "e": 1}),
# lowest that does not appear in other relations
(["a <<", "b < c"], {"a": 0, "b": 1, "c": 2}),
(["a <<", "a < b"], {"a": 0, "b": 1}),
(["a <<", "a < b", "c < d", "d < e"], {"a": 0, "b": 1, "c": 1, "d": 2, "e": 3}),
(["z <<", "a<<", "b<<", "c<<", "a < b", "b < c"], {"a": -2, "b": -1, "c": 0, "z": -3}),
# eq
(["a = b"], {"a": 1, "b": 1}),
(["a = b", "b > c"], {"a": 2, "b": 2, "c": 1}),
(["a = z", "z>>", "b<c", "c<d"], {"a": 4, "b": 1, "c": 2, "d": 3, "z": 4}),
(["b = a", "c = b", "a > d"], {"a": 2, "b": 2, "c": 2, "d": 1}),
(["a = b", "b = c", "a > d"], {"a": 2, "b": 2, "c": 2, "d": 1}),
#(["a = b", "b = c", "c > d"], {"a": 2, "b": 2, "c": 2, "d": 1}), # not working
# mix greatest and lesser
(["a >>", "b<<", "a > b"], {"a": 2, "b": 0}),
])
def test_i_can_compute_weight_new(self, entries, expected):
comparison_objs = list(self.get_comparison_objs(entries))
assert SheerkaComparisonManager.compute_weights(comparison_objs) == expected
@pytest.mark.parametrize("previous_entries, new_entry, items_in_cycle", [
(["a > b"], "b > a", {"a", "b"}),
(["a > b", "c > d"], "b > a", {"a", "b"}),
(["a < b", "b < c"], "c < a", {"a", "b", "c"}),
])
def test_validate_new_entry_i_can_detect_cycle(self, previous_entries, new_entry, items_in_cycle):
sheerka, context = self.init_test().unpack()
new_co = list(self.get_comparison_objs([new_entry]))[0]
previous_comparison_objs = list(self.get_comparison_objs(previous_entries))
res = SheerkaComparisonManager.validate_new_entry(context, new_co, previous_comparison_objs)
assert not res.status
assert sheerka.isinstance(res.body, BuiltinConcepts.CHICKEN_AND_EGG)
assert set(res.body.body) == items_in_cycle
@pytest.mark.parametrize("previous_entries, new_entry", [
(["a > b"], "a > b"),
(["a < b"], "a < b"),
(["a = b"], "a = b"),
(["a <<"], "a <<"),
(["a >>"], "a >>"),
])
def test_validate_new_entry_i_can_detect_duplicate_entries(self, previous_entries, new_entry):
sheerka, context = self.init_test().unpack()
new_co = list(self.get_comparison_objs([new_entry]))[0]
previous_comparison_objs = list(self.get_comparison_objs(previous_entries))
res = SheerkaComparisonManager.validate_new_entry(context, new_co, previous_comparison_objs)
assert not res.status
assert sheerka.isinstance(res.body, BuiltinConcepts.COMPARISON_ALREADY_DEFINED)
@pytest.mark.parametrize("previous_entries, new_entry", [
(["a>>"], "a<<"),
(["a>>"], "a < b"),
(["a>>"], "b > a"),
])
def test_validate_new_entry_i_can_detect_is_greatest_constraint_error(self, previous_entries, new_entry):
sheerka, context = self.init_test().unpack()
new_co = list(self.get_comparison_objs([new_entry]))[0]
previous_comparison_objs = list(self.get_comparison_objs(previous_entries))
res = SheerkaComparisonManager.validate_new_entry(context, new_co, previous_comparison_objs)
assert not res.status
assert sheerka.isinstance(res.body, BuiltinConcepts.IS_GREATEST_CONSTRAINT_ERROR)
@pytest.mark.parametrize("previous_entries, new_entry", [
(["a<<"], "a>>"),
(["a<<"], "a > b"),
(["a<<"], "b < a"),
])
def test_validate_new_entry_i_can_detect_is_lesser_constraint_error(self, previous_entries, new_entry):
sheerka, context = self.init_test().unpack()
new_co = list(self.get_comparison_objs([new_entry]))[0]
previous_comparison_objs = list(self.get_comparison_objs(previous_entries))
res = SheerkaComparisonManager.validate_new_entry(context, new_co, previous_comparison_objs)
assert not res.status
assert sheerka.isinstance(res.body, BuiltinConcepts.IS_LESSER_CONSTRAINT_ERROR)
tests/core/test_SheerkaConceptManager.py
+274 -20
View File
@@ -1,18 +1,20 @@
import pytest
from cache.CacheManager import ConceptNotFound
from core.builtin_concepts import BuiltinConcepts
from core.builtin_helpers import ensure_bnf
from core.concept import PROPERTIES_TO_SERIALIZE, Concept, DEFINITION_TYPE_DEF, get_concept_attrs, \
DEFINITION_TYPE_BNF
from core.global_symbols import NotInit, NotFound
from core.sheerka.Sheerka import Sheerka
from core.sheerka.services.SheerkaConceptManager import SheerkaConceptManager, NoModificationFound, ForbiddenAttribute, \
UnknownAttribute, CannotRemoveMeta, ValueNotFound, ConceptIsReferenced
from parsers.BaseNodeParser import BaseNodeParser
from parsers.BnfNodeParser import Sequence, StrMatch, ConceptExpression
UnknownAttribute, CannotRemoveMeta, ValueNotFound, ConceptIsReferenced, NoFirstTokenError
from parsers.BnfNodeParser import Sequence, StrMatch, ConceptExpression, OrderedChoice, Optional, ZeroOrMore, OneOrMore
from tests.TestUsingFileBasedSheerka import TestUsingFileBasedSheerka
from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
compute_concepts_by_first_token = SheerkaConceptManager.compute_concepts_by_first_token
resolve_concepts_by_first_keyword = SheerkaConceptManager.resolve_concepts_by_first_keyword
class TestSheerkaConceptManager(TestUsingMemoryBasedSheerka):
def test_i_can_create_a_concept(self):
@@ -48,15 +50,15 @@ class TestSheerkaConceptManager(TestUsingMemoryBasedSheerka):
assert sheerka.get_by_hash(concept.get_definition_hash()) == concept
# I can get by the first entry
assert sheerka.om.get(sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, "+") == [concept.id]
assert sheerka.om.get(sheerka.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY, "+") == [concept.id]
assert sheerka.om.get(service.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, "+") == [concept.id]
assert sheerka.om.get(service.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY, "+") == [concept.id]
# saved in sdp
assert sheerka.om.current_sdp().exists(service.CONCEPTS_BY_ID_ENTRY, concept.id)
assert sheerka.om.current_sdp().exists(service.CONCEPTS_BY_KEY_ENTRY, concept.key)
assert sheerka.om.current_sdp().exists(service.CONCEPTS_BY_NAME_ENTRY, concept.name)
assert sheerka.om.current_sdp().exists(service.CONCEPTS_BY_HASH_ENTRY, concept.get_definition_hash())
assert sheerka.om.current_sdp().exists(sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, "+")
assert sheerka.om.current_sdp().exists(service.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, "+")
def test_i_cannot_create_a_bnf_concept_that_references_a_concept_that_cannot_be_resolved(self):
sheerka, context, one_1, one_1_0 = self.init_concepts(Concept("one", body="1"), Concept("one", body="1.0"))
@@ -104,7 +106,7 @@ class TestSheerkaConceptManager(TestUsingMemoryBasedSheerka):
assert sheerka.om.current_sdp().exists(service.CONCEPTS_BY_KEY_ENTRY, concept.key)
assert sheerka.om.current_sdp().exists(service.CONCEPTS_BY_NAME_ENTRY, concept.name)
assert sheerka.om.current_sdp().exists(service.CONCEPTS_BY_HASH_ENTRY, concept.get_definition_hash())
assert sheerka.om.current_sdp().exists(sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, "hello")
assert sheerka.om.current_sdp().exists(service.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, "hello")
def test_i_cannot_add_the_same_concept_twice(self):
"""
@@ -185,8 +187,8 @@ class TestSheerkaConceptManager(TestUsingMemoryBasedSheerka):
assert res.status
# I can get by the first entry
assert sheerka.om.get(sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, "-") == [concept.id]
assert sheerka.om.get(sheerka.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY, "-") == [concept.id]
assert sheerka.om.get(SheerkaConceptManager.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, "-") == [concept.id]
assert sheerka.om.get(SheerkaConceptManager.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY, "-") == [concept.id]
@pytest.mark.parametrize("expression", [
"--'filter' ('one' | 'two') ",
@@ -198,8 +200,8 @@ class TestSheerkaConceptManager(TestUsingMemoryBasedSheerka):
create_new=True).unpack()
# I can get by the first entry
assert sheerka.om.get(sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, "-") == [bnf_concept.id]
assert sheerka.om.get(sheerka.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY, "-") == [bnf_concept.id]
assert sheerka.om.get(SheerkaConceptManager.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, "-") == [bnf_concept.id]
assert sheerka.om.get(SheerkaConceptManager.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY, "-") == [bnf_concept.id]
def test_concept_references_are_updated_1(self):
sheerka, context, one, two, number, twenty, twenties = self.init_test().with_concepts(
@@ -494,11 +496,11 @@ class TestSheerkaConceptManager(TestUsingMemoryBasedSheerka):
Concept("baz", definition="foo"),
create_new=True).unpack()
assert sheerka.om.copy(sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {
assert sheerka.om.copy(SheerkaConceptManager.CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {
"foo": ["1001"],
"bar": ["1002"],
'c:|1001:': ['1003']}
assert sheerka.om.copy(sheerka.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {
assert sheerka.om.copy(SheerkaConceptManager.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {
'foo': ['1001', '1003'],
'bar': ['1002']}
@@ -506,10 +508,10 @@ class TestSheerkaConceptManager(TestUsingMemoryBasedSheerka):
res = sheerka.modify_concept(context, foo, to_add)
assert res.status
assert sheerka.om.copy(sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {
assert sheerka.om.copy(SheerkaConceptManager.CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {
"bar": ["1002", "1001"],
'c:|1001:': ['1003']}
assert sheerka.om.copy(sheerka.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {
assert sheerka.om.copy(SheerkaConceptManager.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {
'bar': ['1002', '1001', '1003']}
def test_references_are_updated_after_concept_modification(self):
@@ -531,7 +533,7 @@ class TestSheerkaConceptManager(TestUsingMemoryBasedSheerka):
assert twenty_one.get_metadata().definition == "'twenty' one"
assert twenty_one.get_bnf() is None
BaseNodeParser.ensure_bnf(context, twenty_one)
ensure_bnf(context, twenty_one)
assert twenty_one.get_bnf() == Sequence(StrMatch('twenty'), ConceptExpression(modified, rule_name='one'))
def test_i_can_modify_on_top_of_a_new_ontology_layer(self):
@@ -730,6 +732,258 @@ class TestSheerkaConceptManager(TestUsingMemoryBasedSheerka):
assert res.status
assert sheerka.isinstance(res.body, BuiltinConcepts.NEW_CONCEPT)
@pytest.mark.parametrize("concept, expected", [
(Concept("foo"), {"foo": ["1001"]}),
(Concept("foo a").def_var("a"), {"foo": ["1001"]}),
(Concept("a b foo").def_var("a").def_var("b"), {"foo": ["1001"]}),
])
def test_i_can_get_concepts_by_first_keyword(self, concept, expected):
"""
Given a concept, i can find the first know token
example:
Concept("a foo b").def_var("a").def_var("b")
'a' and 'b' are properties
the first 'real' token is foo
:return:
"""
sheerka, context, *updated = self.init_concepts(concept)
res = SheerkaConceptManager.compute_concepts_by_first_token(context, updated)
assert res.status
assert res.body == expected
@pytest.mark.parametrize("bnf, expected", [
(StrMatch("foo"), {"foo": ["1002"]}),
(StrMatch("bar"), {"bar": ["1002"]}),
(ConceptExpression("bar"), {"c:|1001:": ["1002"]}),
(Sequence(StrMatch("foo"), StrMatch("bar")), {"foo": ["1002"]}),
(Sequence(StrMatch("foo"), ConceptExpression("bar")), {"foo": ["1002"]}),
(Sequence(ConceptExpression("bar"), StrMatch("foo")), {"c:|1001:": ["1002"]}),
(OrderedChoice(StrMatch("foo"), StrMatch("bar")), {"foo": ["1002"], "bar": ["1002"]}),
(Optional(StrMatch("foo")), {"foo": ["1002"]}),
(ZeroOrMore(StrMatch("foo")), {"foo": ["1002"]}),
(OneOrMore(StrMatch("foo")), {"foo": ["1002"]}),
(StrMatch("--filter"), {"--filter": ["1002"]}), # add both entries
])
def test_i_can_get_concepts_by_first_keyword_with_bnf(self, bnf, expected):
sheerka, context = self.init_test().unpack()
bar = Concept("bar").init_key()
sheerka.set_id_if_needed(bar, False)
sheerka.test_only_add_in_cache(bar)
concept = Concept("foo").init_key()
concept.set_bnf(bnf)
sheerka.set_id_if_needed(concept, False)
res = compute_concepts_by_first_token(context, [concept])
assert res.status
assert res.body == expected
def test_i_can_get_concepts_by_first_keyword_when_multiple_concepts(self):
sheerka = self.get_sheerka()
context = self.get_context(sheerka)
bar = Concept("bar").init_key()
sheerka.set_id_if_needed(bar, False)
sheerka.test_only_add_in_cache(bar)
baz = Concept("baz").init_key()
sheerka.set_id_if_needed(baz, False)
sheerka.test_only_add_in_cache(baz)
foo = Concept("foo").init_key()
foo.set_bnf(OrderedChoice(ConceptExpression("bar"), ConceptExpression("baz"), StrMatch("qux")))
sheerka.set_id_if_needed(foo, False)
res = compute_concepts_by_first_token(context, [bar, baz, foo])
assert res.status
assert res.body == {
"bar": ["1001"],
"baz": ["1002"],
"c:|1001:": ["1003"],
"c:|1002:": ["1003"],
"qux": ["1003"],
}
def test_i_can_get_concepts_by_first_keyword_using_sheerka(self):
sheerka, context, *updated = self.init_test().with_concepts(
"one",
"two",
Concept("twenty", definition="'twenty' (one|two)"),
create_new=True
).unpack()
bar = Concept("bar").init_key()
sheerka.set_id_if_needed(bar, False)
sheerka.test_only_add_in_cache(bar)
foo = Concept("foo").init_key()
foo.set_bnf(OrderedChoice(ConceptExpression("one"), ConceptExpression("bar"), StrMatch("qux")))
sheerka.set_id_if_needed(foo, False)
res = compute_concepts_by_first_token(context, [bar, foo], use_sheerka=True)
assert res.status
assert res.body == {
"one": ["1001"],
"two": ["1002"],
"twenty": ["1003"],
"bar": ["1004"],
"c:|1001:": ["1005"],
"c:|1004:": ["1005"],
"qux": ["1005"],
}
def test_i_cannot_get_concept_by_first_keyword_when_no_first_keyword(self):
sheerka, context, foo = self.init_concepts(Concept("x y", body="x y").def_var("x").def_var("y"))
res = compute_concepts_by_first_token(context, [foo])
assert not res.status
assert res.body == NoFirstTokenError(foo, foo.key)
def test_i_can_resolve_concepts_by_first_keyword(self):
sheerka, context, *updated = self.init_concepts(
"one",
Concept("two", definition="one"),
Concept("three", definition="two"))
concepts_by_first_keywords = {
"one": ["1001"],
"c:|1001:": ["1002"],
"c:|1002:": ["1003"],
}
resolved_ret_val = resolve_concepts_by_first_keyword(context, concepts_by_first_keywords)
assert resolved_ret_val.status
assert resolved_ret_val.body == {
"one": ["1001", "1002", "1003"],
}
def test_i_can_resolve_when_concepts_are_sets(self):
sheerka, context, number, *concepts = self.init_concepts(
"number",
"one",
"two",
"twenty",
"hundred",
Concept("twenties", definition="twenty number"),
Concept("hundreds", definition="number hundred"),
)
sheerka.set_isa(context, sheerka.new("one"), number)
sheerka.set_isa(context, sheerka.new("two"), number)
sheerka.set_isa(context, sheerka.new("twenty"), number)
sheerka.set_isa(context, sheerka.new("thirty"), number)
sheerka.set_isa(context, sheerka.new("hundred"), number)
sheerka.set_isa(context, sheerka.new("twenties"), number)
sheerka.set_isa(context, sheerka.new("hundreds"), number)
sheerka.concepts_grammars.clear() # reset all the grammar to simulate Sheerka restart
# cbft : concept_by_first_token (I usually don't use abbreviation)
cbft = compute_concepts_by_first_token(context, [number] + concepts).body
resolved_ret_val = resolve_concepts_by_first_keyword(context, cbft)
assert resolved_ret_val.status
assert resolved_ret_val.body == {
'number': ['1001'],
'one': ['1002', '1007'],
'two': ['1003', '1007'],
'twenty': ['1004', '1006', '1007'],
'hundred': ['1005', '1007'],
}
def test_concepts_are_defined_once(self):
sheerka = self.get_sheerka()
context = self.get_context(sheerka)
good = self.create_and_add_in_cache_concept(sheerka, "good")
foo = self.create_and_add_in_cache_concept(sheerka, "foo", bnf=ConceptExpression("good"))
bar = self.create_and_add_in_cache_concept(sheerka, "bar", bnf=ConceptExpression("good"))
baz = self.create_and_add_in_cache_concept(sheerka, "baz", bnf=OrderedChoice(
ConceptExpression("foo"),
ConceptExpression("bar")))
concepts_by_first_keywords = compute_concepts_by_first_token(context, [good, foo, bar, baz]).body
resolved_ret_val = resolve_concepts_by_first_keyword(context, concepts_by_first_keywords)
assert resolved_ret_val.status
assert resolved_ret_val.body == {
"good": ["1001", "1002", "1003", "1004"],
}
def test_i_can_resolve_more_complex(self):
sheerka = self.get_sheerka()
context = self.get_context(sheerka)
a = self.create_and_add_in_cache_concept(sheerka, "a", bnf=Sequence("one", "two"))
b = self.create_and_add_in_cache_concept(sheerka, "b", bnf=Sequence(ConceptExpression("a"), "two"))
concepts_by_first_keywords = compute_concepts_by_first_token(context, [a, b]).body
resolved_ret_val = resolve_concepts_by_first_keyword(context, concepts_by_first_keywords)
assert resolved_ret_val.status
assert resolved_ret_val.body == {
"one": ["1001", "1002"],
}
def tests_i_can_detect_direct_recursion(self):
sheerka, context, good, foo, bar = self.init_concepts(
"good",
self.bnf_concept("foo", ConceptExpression("bar")),
self.bnf_concept("bar", ConceptExpression("foo")),
)
concepts_by_first_keywords = compute_concepts_by_first_token(context, [good, foo, bar]).body
resolved_ret_val = resolve_concepts_by_first_keyword(context, concepts_by_first_keywords)
assert resolved_ret_val.status
assert resolved_ret_val.body == {
"good": ["1001"],
}
assert sheerka.chicken_and_eggs.get(foo.id) == {foo.id, bar.id}
assert sheerka.chicken_and_eggs.get(bar.id) == {foo.id, bar.id}
def test_i_can_detect_indirect_infinite_recursion(self):
sheerka, context, good, one, two, three = self.init_concepts(
"good",
self.bnf_concept("one", ConceptExpression("two")),
self.bnf_concept("two", ConceptExpression("three")),
self.bnf_concept("three", ConceptExpression("two")),
)
concepts_by_first_keywords = compute_concepts_by_first_token(context, [good, one, two, three]).body
resolved_ret_val = resolve_concepts_by_first_keyword(context, concepts_by_first_keywords)
assert resolved_ret_val.status
assert resolved_ret_val.body == {
"good": ["1001"],
}
assert sheerka.chicken_and_eggs.get(one.id) == {one.id, two.id, three.id}
assert sheerka.chicken_and_eggs.get(two.id) == {one.id, two.id, three.id}
assert sheerka.chicken_and_eggs.get(three.id) == {one.id, two.id, three.id}
def test_i_can_detect_the_longest_infinite_recursion_chain(self):
sheerka, context, good, one, two, three = self.init_concepts(
"good",
self.bnf_concept("two", ConceptExpression("three")),
self.bnf_concept("three", ConceptExpression("two")),
self.bnf_concept("one", ConceptExpression("three")),
)
concepts_by_first_keywords = compute_concepts_by_first_token(context, [good, one, two, three]).body
resolved_ret_val = resolve_concepts_by_first_keyword(context, concepts_by_first_keywords)
assert resolved_ret_val.status
assert resolved_ret_val.body == {
"good": ["1001"],
}
assert sheerka.chicken_and_eggs.get(one.id) == {one.id, two.id, three.id}
assert sheerka.chicken_and_eggs.get(two.id) == {one.id, two.id, three.id}
assert sheerka.chicken_and_eggs.get(three.id) == {one.id, two.id, three.id}
class TestSheerkaConceptManagerUsingFileBasedSheerka(TestUsingFileBasedSheerka):
def test_i_can_add_several_concepts(self):
@@ -759,8 +1013,8 @@ class TestSheerkaConceptManagerUsingFileBasedSheerka(TestUsingFileBasedSheerka):
assert sheerka.om.current_sdp().exists(service.CONCEPTS_BY_HASH_ENTRY, hello.get_definition_hash())
assert sheerka.om.current_sdp().exists(service.CONCEPTS_BY_HASH_ENTRY, greeting.get_definition_hash())
assert sheerka.om.current_sdp().exists(Sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, "Hello")
assert sheerka.om.current_sdp().exists(Sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, "Greeting")
assert sheerka.om.current_sdp().exists(service.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, "Hello")
assert sheerka.om.current_sdp().exists(service.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, "Greeting")
def test_i_cannot_add_the_same_concept_twice_using_sdp(self):
"""
tests/core/test_SheerkaDebugManager.py
+1
View File
@@ -973,3 +973,4 @@ class TestSheerkaDebugManager(TestUsingMemoryBasedSheerka):
assert service.debug_vars_settings == [DebugItem("v_name", "v_service", "v_method", 1, True, 1, False, True)]
assert service.debug_rules_settings == [DebugItem("r_name", "r_service", "r_method", 2, True, 2, False, True)]
assert service.debug_concepts_settings == [DebugItem("c_name", "c_serv", "c_method", 3, True, 3, False, True)]
tests/core/test_SheerkaMemory.py
+1 -12
View File
@@ -88,17 +88,6 @@ class TestSheerkaMemory(TestUsingMemoryBasedSheerka):
assert sheerka.om.copy(SheerkaMemory.OBJECTS_ENTRY) == {"a": MemoryObject(context.event.get_digest(), foo)}
assert id(sheerka.get_from_memory(context, "a").obj) == id(foo)
def test_i_can_use_memory_to_get_the_list_of_all_objects(self):
sheerka, context = self.init_test(cache_only=False).unpack()
foo = Concept("foo")
bar = Concept("bar")
sheerka.add_to_memory(context, "foo", 'value that will not appear')
sheerka.add_to_memory(context, "foo", foo)
sheerka.add_to_memory(context, "bar", bar)
sheerka.om.commit(context)
assert sheerka.memory(context) == {"foo": foo, "bar": bar}
def test_i_can_use_memory_with_a_string(self):
sheerka, context = self.init_test().unpack()
@@ -138,7 +127,7 @@ class TestSheerkaMemory(TestUsingMemoryBasedSheerka):
def test_object_are_not_added_in_memory_during_the_initialisation(self):
sheerka, context = self.init_test().unpack()
assert len(sheerka.memory(context)) == 0
assert len(sheerka.om.get_all(SheerkaMemory.OBJECTS_ENTRY)) == 0
class TestSheerkaMemoryUsingFileBase(TestUsingFileBasedSheerka):
tests/core/test_SheerkaRuleManager.py
+37 -2
View File
@@ -1,9 +1,10 @@
import ast
import pytest
from core.builtin_concepts import BuiltinConcepts
from core.concept import Concept, CMV
from core.global_symbols import RULE_COMPARISON_CONTEXT, NotFound
from core.global_symbols import RULE_COMPARISON_CONTEXT
from core.rule import Rule, ACTION_TYPE_PRINT, ACTION_TYPE_EXEC
from core.sheerka.services.SheerkaRuleManager import SheerkaRuleManager, FormatRuleParser, \
FormatAstRawText, FormatAstVariable, FormatAstSequence, FormatAstFunction, \
@@ -11,7 +12,6 @@ from core.sheerka.services.SheerkaRuleManager import SheerkaRuleManager, FormatR
from core.tokenizer import Token, TokenKind
from parsers.BaseNodeParser import SourceCodeWithConceptNode, SourceCodeNode
from parsers.PythonParser import PythonNode
from tests.TestUsingFileBasedSheerka import TestUsingFileBasedSheerka
from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
@@ -335,6 +335,41 @@ class TestSheerkaRuleManager(TestUsingMemoryBasedSheerka):
assert sheerka.get_rule_by_id(rule_true.id) == rule_true
assert not sheerka.is_known(sheerka.get_rule_by_id(rule_false.id))
def test_i_can_resolve_rule(self):
sheerka, context, rule = self.init_test().with_rules(("my rule", "True", "True")).unpack()
context.add_to_short_term_memory("x", rule.id)
# direct access by id
assert sheerka.resolve_rule(context, rule.id) == rule
# direct access by token
assert sheerka.resolve_rule(context, Token(TokenKind.RULE, ("i_do_not_care", rule.id), -1, -1, -1)) == rule
# indirect access by token
assert sheerka.resolve_rule(context, Token(TokenKind.RULE, ("i_do_not_care", "x"), -1, -1, -1)) == rule
# Simple returns the rule if id is resolved
assert sheerka.resolve_rule(context, rule) == rule
# look for the correct rule when id is unresolved
unresolved = Rule(rule_id="x")
unresolved.metadata.id_is_unresolved = True
assert sheerka.resolve_rule(context, unresolved) == rule
# look for the correct rule when id is unresolved
unresolved = Rule(rule_id="y")
unresolved.metadata.id_is_unresolved = True
assert sheerka.resolve_rule(context, unresolved) is None # no error raised
# I still can get the value when the indirection has the wrong type
context.add_to_short_term_memory("z", int(rule.id))
assert sheerka.resolve_rule(context, Token(TokenKind.RULE, ("i_do_not_care", "z"), -1, -1, -1)) == rule
unresolved = Rule(rule_id="z")
unresolved.metadata.id_is_unresolved = True
assert sheerka.resolve_rule(context, unresolved) == rule
# @pytest.mark.skip
# @pytest.mark.parametrize("text, expected", [
# ("cat is an animal", set()),
tests/core/test_sheerka.py
+4 -4
View File
@@ -539,21 +539,21 @@ class TestSheerkaUsingFileBasedSheerka(TestUsingFileBasedSheerka):
create_new=True).unpack()
sheerka.om.commit(context)
assert sheerka.om.copy(sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {
assert sheerka.om.copy(SheerkaConceptManager.CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {
'bar': ['1002'],
'c:|1001:': ['1003'],
'foo': ['1001']}
assert sheerka.om.copy(sheerka.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {
assert sheerka.om.copy(SheerkaConceptManager.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {
'bar': ['1002'],
'foo': ['1001', '1003']
}
sheerka = self.get_sheerka() # another instance
assert sheerka.om.copy(sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {
assert sheerka.om.copy(SheerkaConceptManager.CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {
'bar': ['1002'],
'c:|1001:': ['1003'],
'foo': ['1001']}
assert sheerka.om.copy(sheerka.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {
assert sheerka.om.copy(SheerkaConceptManager.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {
'bar': ['1002'],
'foo': ['1001', '1003']
}
tests/core/test_sheerkaResultManager.py
+68 -8
View File
@@ -1,12 +1,13 @@
import types
import pytest
from core.builtin_concepts import BuiltinConcepts
from core.sheerka.ExecutionContext import ExecutionContext
from core.sheerka.SheerkaOntologyManager import SheerkaOntologyManager
from core.sheerka.services.SheerkaResultManager import SheerkaResultConcept
from core.sheerka.services.SheerkaResultManager import SheerkaResultManager
from sdp.sheerkaDataProvider import Event
from tests.TestUsingFileBasedSheerka import TestUsingFileBasedSheerka
from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
@@ -25,7 +26,7 @@ class TestSheerkaResultManager(TestUsingMemoryBasedSheerka):
def init_service(self):
sheerka, context = self.init_test().unpack()
service = sheerka.services[SheerkaResultConcept.NAME]
service = sheerka.services[SheerkaResultManager.NAME]
return sheerka, context, service
@@ -64,7 +65,7 @@ class TestSheerkaResultManager(TestUsingMemoryBasedSheerka):
assert isinstance(res.body, types.GeneratorType)
# the digest is correctly recorded
assert sheerka.load_var(SheerkaResultConcept.NAME, "digest") == digest
assert sheerka.load_var(SheerkaResultManager.NAME, "digest") == digest
previous_results = list(res.body)
@@ -93,7 +94,7 @@ class TestSheerkaResultManager(TestUsingMemoryBasedSheerka):
assert isinstance(res.body, types.GeneratorType)
# the digest is correctly recorded
assert sheerka.load_var(SheerkaResultConcept.NAME, "digest") == digest
assert sheerka.load_var(SheerkaResultManager.NAME, "digest") == digest
previous_results = list(res.body)
@@ -148,7 +149,7 @@ class TestSheerkaResultManager(TestUsingMemoryBasedSheerka):
sheerka.evaluate_user_input("one")
sheerka.evaluate_user_input("one")
service = SheerkaResultConcept(sheerka, 2)
service = SheerkaResultManager(sheerka, 2)
res = service.get_results_by_command(context, "def concept")
assert sheerka.isinstance(res, BuiltinConcepts.EXPLANATION)
assert res.command == "def concept one as 1"
@@ -171,7 +172,7 @@ class TestSheerkaResultManager(TestUsingMemoryBasedSheerka):
sheerka.evaluate_user_input("one")
sheerka.evaluate_user_input("one")
service = SheerkaResultConcept(sheerka, 2)
service = SheerkaResultManager(sheerka, 2)
res = service.get_results_by_command(context, "fake command")
assert sheerka.isinstance(res, BuiltinConcepts.NOT_FOUND)
assert res.body == {'command': 'fake command'}
@@ -336,7 +337,7 @@ class TestSheerkaResultManager(TestUsingMemoryBasedSheerka):
def test_i_can_manage_invalid_predicates(self):
predicate = {"filter": "a b c"}
with pytest.raises(SyntaxError):
SheerkaResultConcept.get_predicate(**predicate)
SheerkaResultManager.get_predicate(**predicate)
def test_i_can_get_last_return_value(self):
sheerka, context, service = self.init_service()
@@ -368,3 +369,62 @@ class TestSheerkaResultManager(TestUsingMemoryBasedSheerka):
assert service.last_created_concept_id == new_concept.id
assert sheerka.get_last_created_concept(context) == new_concept
def test_last_error_is_recorded(self):
sheerka, context, foo = self.init_test().with_concepts("foo", create_new=True).unpack()
service = sheerka.services[SheerkaResultManager.NAME]
res = sheerka.evaluate_user_input("bar") # does not exist, will cause an error
sheerka.evaluate_user_input("foo") # exists, to validate that the error is not reset
assert service.last_error_event_id is not None
assert len(res) == 1
assert sheerka.get_last_error() == res[0]
def test_multiple_errors_are_recorded(self):
sheerka, context, foo = self.init_test().with_concepts("foo", create_new=True).unpack()
service = sheerka.services[SheerkaResultManager.NAME]
service.test_only_reset()
res = sheerka.evaluate_user_input("x x") # cannot be parsed
sheerka.evaluate_user_input("foo") # exists, to validate that the error is not reset
assert service.last_error_event_id is not None
assert len(res) > 1
assert sheerka.get_last_error() == [r for r in res if not r.status]
def test_i_cannot_get_last_error_when_no_error(self):
sheerka, context = self.init_test().unpack()
service = sheerka.services[SheerkaResultManager.NAME]
service.test_only_reset()
assert service.last_error_event_id is None
assert sheerka.get_last_error() == self.sheerka.new(BuiltinConcepts.NOT_FOUND, body={"query": "get_last_error"})
class TestSheerkaResultManagerFileBased(TestUsingFileBasedSheerka):
@classmethod
def setup_class(cls):
sheerka = cls().get_sheerka(cache_only=False, ontology="#TestSheerkaResultManager#")
sheerka.save_execution_context = True
cls.root_ontology_name = "#TestSheerkaResultManager#"
@classmethod
def teardown_class(cls):
cls.sheerka.pop_ontology()
cls.root_ontology_name = SheerkaOntologyManager.ROOT_ONTOLOGY_NAME
def test_i_can_retrieve_the_last_error_after_startup(self):
sheerka, context, foo = self.init_test().with_concepts("foo", create_new=True).unpack()
service = sheerka.services[SheerkaResultManager.NAME]
res_in_error = sheerka.evaluate_user_input("bar") # does not exist, will cause an error
sheerka.evaluate_user_input("foo") # exists, to validate that the error is not reset
assert service.last_error_event_id is not None
assert service.get_last_error() == res_in_error[0]
# simulate restart
sheerka = self.new_sheerka_instance(False)
service = sheerka.services[SheerkaResultManager.NAME]
assert service.last_error_event_id is not None
assert service.get_last_error() == res_in_error[0]
tests/core/test_sheerka_ontology.py
+100 -99
View File
@@ -1167,106 +1167,107 @@ class TestSheerkaOntology(TestUsingMemoryBasedSheerka):
"key5": "value5"
}
def test_i_can_list_by_key_when_dictionaries(self):
sheerka = self.get_sheerka(cache_only=False)
context = self.get_context(sheerka)
# def test_i_can_list_by_key_when_dictionaries(self):
# sheerka = self.get_sheerka(cache_only=False)
# context = self.get_context(sheerka)
#
# manager = SheerkaOntologyManager(sheerka, sheerka.root_folder, sheerka.cache_only)
# manager.register_cache("cache_name", Cache().auto_configure("cache_name"))
# manager.freeze()
#
# manager.put("cache_name", "key1", {"a": "value1", "b": "value2", "c": "value3"})
# manager.commit(context)
#
# manager.push_ontology("new ontology")
# manager.put("cache_name", "key1", {"a": "new value1", "d": "value4"}) # only in cache
#
# manager.push_ontology("another ontology")
# with manager.current_sdp().get_transaction(context.event) as transaction:
# transaction.add("cache_name", "key1", {"b": "new value2", "e": "value5"})
#
# assert manager.list_by_key("cache_name", "key1") == {
# "a": "new value1",
# "b": "new value2",
# "c": "value3",
# "d": "value4",
# "e": "value5",
# }
#
# def test_i_can_list_by_key_when_lists(self):
# sheerka = self.get_sheerka(cache_only=False)
# context = self.get_context(sheerka)
#
# manager = SheerkaOntologyManager(sheerka, sheerka.root_folder, sheerka.cache_only)
# manager.register_cache("cache_name", Cache().auto_configure("cache_name"))
# manager.freeze()
#
# manager.put("cache_name", "key1", ["a", "b", "c"])
# manager.commit(context)
#
# manager.push_ontology("new ontology")
# manager.put("cache_name", "key1", ["a", "d"]) # only in cache
#
# manager.push_ontology("another ontology")
# with manager.current_sdp().get_transaction(context.event) as transaction:
# transaction.add("cache_name", "key1", ["b", "e"])
#
# assert manager.list_by_key("cache_name", "key1") == ["a", "b", "c", "a", "d", "b", "e"]
#
# def test_i_can_list_by_key_when_dictionaries_and_entries_are_removed(self):
# sheerka = self.get_sheerka(cache_only=False)
# context = self.get_context(sheerka)
#
# manager = SheerkaOntologyManager(sheerka, sheerka.root_folder, sheerka.cache_only)
# manager.register_cache("cache_name", Cache().auto_configure("cache_name"))
# manager.freeze()
#
# manager.put("cache_name", "key1", {"a": "value1", "b": "value2", "c": "value3"})
# manager.put("cache_name", "key2", {"a": "value1", "b": "value2", "c": "value3"})
# manager.put("cache_name", "key3", {"a": "value1", "b": "value2", "c": "value3"})
# manager.commit(context)
#
# manager.push_ontology("new ontology")
# manager.put("cache_name", "key1", Removed) # removed in cache
# with manager.current_sdp().get_transaction(context.event) as transaction:
# transaction.add("cache_name", "key2", Removed) # removed in sdp
#
# manager.push_ontology("another ontology")
# manager.put("cache_name", "key1", {"e": "value1", "f": "value2", "g": "value3"})
# manager.put("cache_name", "key2", {"e": "value1", "f": "value2", "g": "value3"})
# manager.put("cache_name", "key3", {"e": "value1", "f": "value2", "g": "value3"})
#
# assert manager.list_by_key("cache_name", "key1") == {"e": "value1", "f": "value2", "g": "value3"}
# assert manager.list_by_key("cache_name", "key2") == {"e": "value1", "f": "value2", "g": "value3"}
# assert manager.list_by_key("cache_name", "key3") == {"a": "value1", "b": "value2", "c": "value3",
# "e": "value1", "f": "value2", "g": "value3"}
#
# def test_i_can_list_by_key_when_lists_and_entries_are_removed(self):
# sheerka = self.get_sheerka(cache_only=False)
# context = self.get_context(sheerka)
#
# manager = SheerkaOntologyManager(sheerka, sheerka.root_folder, sheerka.cache_only)
# manager.register_cache("cache_name", Cache().auto_configure("cache_name"))
# manager.freeze()
#
# manager.put("cache_name", "key1", ["a", "b", "c"])
# manager.put("cache_name", "key2", ["a", "b", "c"])
# manager.put("cache_name", "key3", ["a", "b", "c"])
# manager.commit(context)
#
# manager.push_ontology("new ontology")
# manager.put("cache_name", "key1", Removed) # removed in cache
# with manager.current_sdp().get_transaction(context.event) as transaction:
# transaction.add("cache_name", "key2", Removed) # removed in sdp
#
# manager.push_ontology("another ontology")
# manager.put("cache_name", "key1", ["e", "f", "g"])
# manager.put("cache_name", "key2", ["e", "f", "g"])
# manager.put("cache_name", "key3", ["e", "f", "g"])
#
# assert manager.list_by_key("cache_name", "key1") == ["e", "f", "g"]
# assert manager.list_by_key("cache_name", "key2") == ["e", "f", "g"]
# assert manager.list_by_key("cache_name", "key3") == ["a", "b", "c", "e", "f", "g"]
manager = SheerkaOntologyManager(sheerka, sheerka.root_folder, sheerka.cache_only)
manager.register_cache("cache_name", Cache().auto_configure("cache_name"))
manager.freeze()
manager.put("cache_name", "key1", {"a": "value1", "b": "value2", "c": "value3"})
manager.commit(context)
manager.push_ontology("new ontology")
manager.put("cache_name", "key1", {"a": "new value1", "d": "value4"}) # only in cache
manager.push_ontology("another ontology")
with manager.current_sdp().get_transaction(context.event) as transaction:
transaction.add("cache_name", "key1", {"b": "new value2", "e": "value5"})
assert manager.list_by_key("cache_name", "key1") == {
"a": "new value1",
"b": "new value2",
"c": "value3",
"d": "value4",
"e": "value5",
}
def test_i_can_list_by_key_when_lists(self):
sheerka = self.get_sheerka(cache_only=False)
context = self.get_context(sheerka)
manager = SheerkaOntologyManager(sheerka, sheerka.root_folder, sheerka.cache_only)
manager.register_cache("cache_name", Cache().auto_configure("cache_name"))
manager.freeze()
manager.put("cache_name", "key1", ["a", "b", "c"])
manager.commit(context)
manager.push_ontology("new ontology")
manager.put("cache_name", "key1", ["a", "d"]) # only in cache
manager.push_ontology("another ontology")
with manager.current_sdp().get_transaction(context.event) as transaction:
transaction.add("cache_name", "key1", ["b", "e"])
assert manager.list_by_key("cache_name", "key1") == ["a", "b", "c", "a", "d", "b", "e"]
def test_i_can_list_by_key_when_dictionaries_and_entries_are_removed(self):
sheerka = self.get_sheerka(cache_only=False)
context = self.get_context(sheerka)
manager = SheerkaOntologyManager(sheerka, sheerka.root_folder, sheerka.cache_only)
manager.register_cache("cache_name", Cache().auto_configure("cache_name"))
manager.freeze()
manager.put("cache_name", "key1", {"a": "value1", "b": "value2", "c": "value3"})
manager.put("cache_name", "key2", {"a": "value1", "b": "value2", "c": "value3"})
manager.put("cache_name", "key3", {"a": "value1", "b": "value2", "c": "value3"})
manager.commit(context)
manager.push_ontology("new ontology")
manager.put("cache_name", "key1", Removed) # removed in cache
with manager.current_sdp().get_transaction(context.event) as transaction:
transaction.add("cache_name", "key2", Removed) # removed in sdp
manager.push_ontology("another ontology")
manager.put("cache_name", "key1", {"e": "value1", "f": "value2", "g": "value3"})
manager.put("cache_name", "key2", {"e": "value1", "f": "value2", "g": "value3"})
manager.put("cache_name", "key3", {"e": "value1", "f": "value2", "g": "value3"})
assert manager.list_by_key("cache_name", "key1") == {"e": "value1", "f": "value2", "g": "value3"}
assert manager.list_by_key("cache_name", "key2") == {"e": "value1", "f": "value2", "g": "value3"}
assert manager.list_by_key("cache_name", "key3") == {"a": "value1", "b": "value2", "c": "value3",
"e": "value1", "f": "value2", "g": "value3"}
def test_i_can_list_by_key_when_lists_and_entries_are_removed(self):
sheerka = self.get_sheerka(cache_only=False)
context = self.get_context(sheerka)
manager = SheerkaOntologyManager(sheerka, sheerka.root_folder, sheerka.cache_only)
manager.register_cache("cache_name", Cache().auto_configure("cache_name"))
manager.freeze()
manager.put("cache_name", "key1", ["a", "b", "c"])
manager.put("cache_name", "key2", ["a", "b", "c"])
manager.put("cache_name", "key3", ["a", "b", "c"])
manager.commit(context)
manager.push_ontology("new ontology")
manager.put("cache_name", "key1", Removed) # removed in cache
with manager.current_sdp().get_transaction(context.event) as transaction:
transaction.add("cache_name", "key2", Removed) # removed in sdp
manager.push_ontology("another ontology")
manager.put("cache_name", "key1", ["e", "f", "g"])
manager.put("cache_name", "key2", ["e", "f", "g"])
manager.put("cache_name", "key3", ["e", "f", "g"])
assert manager.list_by_key("cache_name", "key1") == ["e", "f", "g"]
assert manager.list_by_key("cache_name", "key2") == ["e", "f", "g"]
assert manager.list_by_key("cache_name", "key3") == ["a", "b", "c", "e", "f", "g"]
def test_i_can_get_call_when_a_cache_is_cleared(self):
sheerka = self.get_sheerka(cache_only=False)
tests/evaluators/test_PythonEvaluator.py
+20 -2
View File
@@ -1,6 +1,7 @@
import ast
import pytest
from core.builtin_concepts import ReturnValueConcept, ParserResultConcept, BuiltinConcepts
from core.concept import Concept, CB
from core.sheerka.services.SheerkaConceptManager import SheerkaConceptManager
@@ -11,7 +12,6 @@ from parsers.BaseNodeParser import SourceCodeNode, SourceCodeWithConceptNode
from parsers.FunctionParser import FunctionParser
from parsers.PythonParser import PythonNode, PythonParser
from parsers.PythonWithConceptsParser import PythonWithConceptsParser
from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
@@ -226,7 +226,7 @@ class TestPythonEvaluator(TestUsingMemoryBasedSheerka):
evaluated = python_evaluator.eval(context, parsed)
assert evaluated.status
assert sheerka.get_concepts_weights(BuiltinConcepts.PRECEDENCE) == {'c:__var__0 plus __var__1|1001:': 1,
assert sheerka.get_weights(BuiltinConcepts.PRECEDENCE) == {'c:__var__0 plus __var__1|1001:': 1,
'c:__var__0 mult __var__1|1002:': 2}
def test_i_can_define_variables(self):
@@ -370,3 +370,21 @@ class TestPythonEvaluator(TestUsingMemoryBasedSheerka):
sequence = visitor.get_sequences(ast_, "foo")
assert sequence == expected
@pytest.mark.parametrize("parser, value", [
(PythonParser(), "3"),
(FunctionParser(), "3"),
(PythonParser(), 3),
(FunctionParser(), 3),
])
def test_i_can_eval_unresolved_rules(self, parser, value):
context = self.get_context()
context.add_to_short_term_memory("get_obj_name", get_obj_name)
context.add_to_short_term_memory("x", value)
parsed = parser.parse(context, ParserInput("get_obj_name(r:|x:)"))
python_evaluator = PythonEvaluator()
evaluated = python_evaluator.eval(context, parsed)
assert evaluated.status
assert evaluated.value == context.sheerka.get_rule_by_id(str(value)).name
tests/evaluators/test_RuleEvaluator.py
+4 -4
View File
@@ -1,6 +1,6 @@
import pytest
from core.builtin_concepts import ReturnValueConcept, ParserResultConcept, BuiltinConcepts
from core.rule import Rule, ACTION_TYPE_DEFERRED
from core.rule import Rule
from evaluators.RuleEvaluator import RuleEvaluator
from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
@@ -37,11 +37,11 @@ class TestRuleEvaluator(TestUsingMemoryBasedSheerka):
assert result.value == expected
assert result.parents == [ret_val]
def test_i_can_evaluate_deferred_rules(self):
def test_i_can_evaluate_unresolved_rules(self):
context = self.get_context()
rule = Rule().set_id("i")
rule.metadata.action_type = ACTION_TYPE_DEFERRED
rule.metadata.id_is_unresolved = True
ret_val = ReturnValueConcept("some_name", True, ParserResultConcept(value=[rule]), True)
context.add_to_short_term_memory("i", 1)
evaluator = RuleEvaluator()
@@ -57,7 +57,7 @@ class TestRuleEvaluator(TestUsingMemoryBasedSheerka):
sheerka, context = self.init_concepts()
rule = Rule().set_id("unknown variable")
rule.metadata.action_type = ACTION_TYPE_DEFERRED
rule.metadata.id_is_unresolved = True
ret_val = ReturnValueConcept("some_name", True, ParserResultConcept(value=[rule]), True)
evaluator = RuleEvaluator()
tests/non_reg/test_sheerka_display.py
+17
View File
@@ -87,4 +87,21 @@ ReturnValue(who=evaluators.Concept, status=True, value=(1002)foo)
test 1
#unit_test#
__default__
"""
def test_i_can_display_objects_in_memory(self, capsys):
init = [
"def concept one as 1",
"def concept two as 2",
"one",
"two"
]
sheerka = self.init_scenario(init)
capsys.readouterr()
sheerka.enable_process_return_values = True
sheerka.evaluate_user_input("in_memory()")
captured = capsys.readouterr()
assert captured.out == """one: (1001)one
two: (1002)two
"""
tests/non_reg/test_sheerka_non_reg.py
+18 -5
View File
@@ -116,7 +116,7 @@ as:
assert service.has_id(concept_saved.id)
assert service.has_name(concept_saved.name)
assert service.has_hash(concept_saved.get_definition_hash())
assert sheerka.om.copy(sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {'+': ['1001']}
assert sheerka.om.copy(SheerkaConceptManager.CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {'+': ['1001']}
# sdp is up to date
assert sheerka.om.current_sdp().exists(SheerkaConceptManager.CONCEPTS_BY_KEY_ENTRY, expected.key)
@@ -865,7 +865,7 @@ as:
res = sheerka.evaluate_user_input("set_is_less_than('some_prop', two, three)")
assert res[0].status
res = sheerka.evaluate_user_input("get_concepts_weights('some_prop')")
res = sheerka.evaluate_user_input("get_weights('some_prop')")
assert res[0].status
assert res[0].body == {'c:one|1001:': 1, 'c:two|1002:': 2, 'c:three|1003:': 3}
@@ -874,7 +874,7 @@ as:
res = sheerka.evaluate_user_input("eval four > three")
assert res[0].status
assert sheerka.get_concepts_weights("some_prop") == {'c:one|1001:': 1,
assert sheerka.get_weights("some_prop") == {'c:one|1001:': 1,
'c:two|1002:': 2,
'c:three|1003:': 3,
'c:four|1004:': 4}
@@ -898,14 +898,14 @@ as:
res = sheerka.evaluate_user_input(expression)
assert res[0].status
assert sheerka.isinstance(res[0].body, BuiltinConcepts.SUCCESS)
assert sheerka.get_concepts_weights("some_prop") == {'c:one|1001:': 1, 'c:two|1002:': 2}
assert sheerka.get_weights("some_prop") == {'c:one|1001:': 1, 'c:two|1002:': 2}
# it now also works using the concepts names
expression = "eval two < three"
res = sheerka.evaluate_user_input(expression)
assert res[0].status
assert sheerka.isinstance(res[0].body, BuiltinConcepts.SUCCESS)
assert sheerka.get_concepts_weights("some_prop") == {'c:one|1001:': 1, 'c:two|1002:': 2, 'c:three|1003:': 3}
assert sheerka.get_weights("some_prop") == {'c:one|1001:': 1, 'c:two|1002:': 2, 'c:three|1003:': 3}
def test_i_can_detect_multiple_errors_when_evaluating_a_concept(self):
sheerka, context, foo, plus_one = self.init_concepts(
@@ -1229,6 +1229,19 @@ as:
assert res[0].status
assert res[0].body == 21
def test_i_can_define_rules_priorities(self):
sheerka, context, r1, r2 = self.init_test().with_rules(("True", "True"), ("False", "False")).unpack()
sheerka.evaluate_user_input("def concept rule x > rule y where isinstance(x, int) and isinstance(y, int) as set_is_greater_than(__PRECEDENCE, r:|x:, r:|y:, 'Rule')")
res = sheerka.evaluate_user_input(f"eval rule {r1.id} > rule {r2.id}")
assert len(res) == 1
assert res[0].status
assert sheerka.isinstance(res[0].body, BuiltinConcepts.SUCCESS)
weights = sheerka.get_weights(BuiltinConcepts.PRECEDENCE, 'Rule')
assert weights[r1.str_id] == weights[r2.str_id] + 1
class TestSheerkaNonRegFile(TestUsingFileBasedSheerka):
def test_i_can_def_several_concepts(self):
tests/parsers/test_BaseNodeParser.py
-294
View File
@@ -1,294 +0,0 @@
import pytest
from core.concept import Concept
from parsers.BaseNodeParser import BaseNodeParser, NoFirstTokenError
from parsers.BnfNodeParser import StrMatch, Sequence, OrderedChoice, Optional, ZeroOrMore, OneOrMore, ConceptExpression
from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
class TestBaseNodeParser(TestUsingMemoryBasedSheerka):
@pytest.mark.parametrize("concept, expected", [
(Concept("foo"), {"foo": ["1001"]}),
(Concept("foo a").def_var("a"), {"foo": ["1001"]}),
(Concept("a b foo").def_var("a").def_var("b"), {"foo": ["1001"]}),
])
def test_i_can_get_concepts_by_first_keyword(self, concept, expected):
"""
Given a concept, i can find the first know token
example:
Concept("a foo b").def_var("a").def_var("b")
'a' and 'b' are properties
the first 'real' token is foo
:return:
"""
sheerka, context, *updated = self.init_concepts(concept)
res = BaseNodeParser.compute_concepts_by_first_token(context, updated)
assert res.status
assert res.body == expected
@pytest.mark.parametrize("bnf, expected", [
(StrMatch("foo"), {"foo": ["1002"]}),
(StrMatch("bar"), {"bar": ["1002"]}),
(ConceptExpression("bar"), {"c:|1001:": ["1002"]}),
(Sequence(StrMatch("foo"), StrMatch("bar")), {"foo": ["1002"]}),
(Sequence(StrMatch("foo"), ConceptExpression("bar")), {"foo": ["1002"]}),
(Sequence(ConceptExpression("bar"), StrMatch("foo")), {"c:|1001:": ["1002"]}),
(OrderedChoice(StrMatch("foo"), StrMatch("bar")), {"foo": ["1002"], "bar": ["1002"]}),
(Optional(StrMatch("foo")), {"foo": ["1002"]}),
(ZeroOrMore(StrMatch("foo")), {"foo": ["1002"]}),
(OneOrMore(StrMatch("foo")), {"foo": ["1002"]}),
(StrMatch("--filter"), {"--filter": ["1002"]}), # add both entries
])
def test_i_can_get_concepts_by_first_keyword_with_bnf(self, bnf, expected):
sheerka = self.get_sheerka()
context = self.get_context(sheerka)
bar = Concept("bar").init_key()
sheerka.set_id_if_needed(bar, False)
sheerka.test_only_add_in_cache(bar)
concept = Concept("foo").init_key()
concept.set_bnf(bnf)
sheerka.set_id_if_needed(concept, False)
res = BaseNodeParser.compute_concepts_by_first_token(context, [concept])
assert res.status
assert res.body == expected
def test_i_can_get_concepts_by_first_keyword_when_multiple_concepts(self):
sheerka = self.get_sheerka()
context = self.get_context(sheerka)
bar = Concept("bar").init_key()
sheerka.set_id_if_needed(bar, False)
sheerka.test_only_add_in_cache(bar)
baz = Concept("baz").init_key()
sheerka.set_id_if_needed(baz, False)
sheerka.test_only_add_in_cache(baz)
foo = Concept("foo").init_key()
foo.set_bnf(OrderedChoice(ConceptExpression("bar"), ConceptExpression("baz"), StrMatch("qux")))
sheerka.set_id_if_needed(foo, False)
res = BaseNodeParser.compute_concepts_by_first_token(context, [bar, baz, foo])
assert res.status
assert res.body == {
"bar": ["1001"],
"baz": ["1002"],
"c:|1001:": ["1003"],
"c:|1002:": ["1003"],
"qux": ["1003"],
}
def test_i_can_get_concepts_by_first_keyword_using_sheerka(self):
sheerka, context, *updated = self.init_test().with_concepts(
"one",
"two",
Concept("twenty", definition="'twenty' (one|two)"),
create_new=True
).unpack()
bar = Concept("bar").init_key()
sheerka.set_id_if_needed(bar, False)
sheerka.test_only_add_in_cache(bar)
foo = Concept("foo").init_key()
foo.set_bnf(OrderedChoice(ConceptExpression("one"), ConceptExpression("bar"), StrMatch("qux")))
sheerka.set_id_if_needed(foo, False)
res = BaseNodeParser.compute_concepts_by_first_token(context, [bar, foo], use_sheerka=True)
assert res.status
assert res.body == {
"one": ["1001"],
"two": ["1002"],
"twenty": ["1003"],
"bar": ["1004"],
"c:|1001:": ["1005"],
"c:|1004:": ["1005"],
"qux": ["1005"],
}
def test_i_cannot_get_concept_by_first_keyword_when_no_first_keyword(self):
sheerka, context, foo = self.init_concepts(Concept("x y", body="x y").def_var("x").def_var("y"))
res = BaseNodeParser.compute_concepts_by_first_token(context, [foo])
assert not res.status
assert res.body == NoFirstTokenError(foo, foo.key)
def test_i_can_resolve_concepts_by_first_keyword(self):
sheerka, context, *updated = self.init_concepts(
"one",
Concept("two", definition="one"),
Concept("three", definition="two"))
concepts_by_first_keywords = {
"one": ["1001"],
"c:|1001:": ["1002"],
"c:|1002:": ["1003"],
}
resolved_ret_val = BaseNodeParser.resolve_concepts_by_first_keyword(context, concepts_by_first_keywords)
assert resolved_ret_val.status
assert resolved_ret_val.body == {
"one": ["1001", "1002", "1003"],
}
def test_i_can_resolve_when_concepts_are_sets(self):
sheerka, context, number, *concepts = self.init_concepts(
"number",
"one",
"two",
"twenty",
"hundred",
Concept("twenties", definition="twenty number"),
Concept("hundreds", definition="number hundred"),
)
sheerka.set_isa(context, sheerka.new("one"), number)
sheerka.set_isa(context, sheerka.new("two"), number)
sheerka.set_isa(context, sheerka.new("twenty"), number)
sheerka.set_isa(context, sheerka.new("thirty"), number)
sheerka.set_isa(context, sheerka.new("hundred"), number)
sheerka.set_isa(context, sheerka.new("twenties"), number)
sheerka.set_isa(context, sheerka.new("hundreds"), number)
sheerka.concepts_grammars.clear() # reset all the grammar to simulate Sheerka restart
# cbft : concept_by_first_token (I usually don't use abbreviation)
cbft = BaseNodeParser.compute_concepts_by_first_token(context, [number] + concepts).body
resolved_ret_val = BaseNodeParser.resolve_concepts_by_first_keyword(context, cbft)
assert resolved_ret_val.status
assert resolved_ret_val.body == {
'number': ['1001'],
'one': ['1002', '1007'],
'two': ['1003', '1007'],
'twenty': ['1004', '1006', '1007'],
'hundred': ['1005', '1007'],
}
def test_concepts_are_defined_once(self):
sheerka = self.get_sheerka()
context = self.get_context(sheerka)
good = self.create_and_add_in_cache_concept(sheerka, "good")
foo = self.create_and_add_in_cache_concept(sheerka, "foo", bnf=ConceptExpression("good"))
bar = self.create_and_add_in_cache_concept(sheerka, "bar", bnf=ConceptExpression("good"))
baz = self.create_and_add_in_cache_concept(sheerka, "baz", bnf=OrderedChoice(
ConceptExpression("foo"),
ConceptExpression("bar")))
concepts_by_first_keywords = BaseNodeParser.compute_concepts_by_first_token(
context, [good, foo, bar, baz]).body
resolved_ret_val = BaseNodeParser.resolve_concepts_by_first_keyword(context, concepts_by_first_keywords)
assert resolved_ret_val.status
assert resolved_ret_val.body == {
"good": ["1001", "1002", "1003", "1004"],
}
def test_i_can_resolve_more_complex(self):
sheerka = self.get_sheerka()
context = self.get_context(sheerka)
a = self.create_and_add_in_cache_concept(sheerka, "a", bnf=Sequence("one", "two"))
b = self.create_and_add_in_cache_concept(sheerka, "b", bnf=Sequence(ConceptExpression("a"), "two"))
concepts_by_first_keywords = BaseNodeParser.compute_concepts_by_first_token(
context, [a, b]).body
resolved_ret_val = BaseNodeParser.resolve_concepts_by_first_keyword(context, concepts_by_first_keywords)
assert resolved_ret_val.status
assert resolved_ret_val.body == {
"one": ["1001", "1002"],
}
def tests_i_can_detect_direct_recursion(self):
sheerka, context, good, foo, bar = self.init_concepts(
"good",
self.bnf_concept("foo", ConceptExpression("bar")),
self.bnf_concept("bar", ConceptExpression("foo")),
)
concepts_by_first_keywords = BaseNodeParser.compute_concepts_by_first_token(context, [good, foo, bar]).body
resolved_ret_val = BaseNodeParser.resolve_concepts_by_first_keyword(context, concepts_by_first_keywords)
assert resolved_ret_val.status
assert resolved_ret_val.body == {
"good": ["1001"],
}
assert sheerka.chicken_and_eggs.get(foo.id) == {foo.id, bar.id}
assert sheerka.chicken_and_eggs.get(bar.id) == {foo.id, bar.id}
def test_i_can_detect_indirect_infinite_recursion(self):
sheerka, context, good, one, two, three = self.init_concepts(
"good",
self.bnf_concept("one", ConceptExpression("two")),
self.bnf_concept("two", ConceptExpression("three")),
self.bnf_concept("three", ConceptExpression("two")),
)
concepts_by_first_keywords = BaseNodeParser.compute_concepts_by_first_token(context, [good, one, two, three]).body
resolved_ret_val = BaseNodeParser.resolve_concepts_by_first_keyword(context, concepts_by_first_keywords)
assert resolved_ret_val.status
assert resolved_ret_val.body == {
"good": ["1001"],
}
assert sheerka.chicken_and_eggs.get(one.id) == {one.id, two.id, three.id}
assert sheerka.chicken_and_eggs.get(two.id) == {one.id, two.id, three.id}
assert sheerka.chicken_and_eggs.get(three.id) == {one.id, two.id, three.id}
def test_i_can_detect_the_longest_infinite_recursion_chain(self):
sheerka, context, good, one, two, three = self.init_concepts(
"good",
self.bnf_concept("two", ConceptExpression("three")),
self.bnf_concept("three", ConceptExpression("two")),
self.bnf_concept("one", ConceptExpression("three")),
)
concepts_by_first_keywords = BaseNodeParser.compute_concepts_by_first_token(context, [good, one, two, three]).body
resolved_ret_val = BaseNodeParser.resolve_concepts_by_first_keyword(context, concepts_by_first_keywords)
assert resolved_ret_val.status
assert resolved_ret_val.body == {
"good": ["1001"],
}
assert sheerka.chicken_and_eggs.get(one.id) == {one.id, two.id, three.id}
assert sheerka.chicken_and_eggs.get(two.id) == {one.id, two.id, three.id}
assert sheerka.chicken_and_eggs.get(three.id) == {one.id, two.id, three.id}
#
# def test_i_can_detect_infinite_recursion_from_ordered_choice(self):
# sheerka = self.get_sheerka()
# good = self.get_concept(sheerka, "good")
# one = self.get_concept(sheerka, "one", ConceptExpression("two"))
# two = self.get_concept(sheerka, "two", OrderedChoice(ConceptExpression("one"), ConceptExpression("two")))
#
# concepts_by_first_keywords = BaseNodeParser.get_concepts_by_first_token(sheerka, [good, one, two]).body
#
# resolved_ret_val = BaseNodeParser.resolve_concepts_by_first_keyword(sheerka, concepts_by_first_keywords)
# assert resolved_ret_val.status
# assert resolved_ret_val.body == {
# "good": ["1001"],
# BuiltinConcepts.CHICKEN_AND_EGG: ["1002", "1003"]
# }
#
# def test_i_can_detect_infinite_recursion_with_sequence(self):
# sheerka = self.get_sheerka()
# good = self.get_concept(sheerka, "good")
# one = self.get_concept(sheerka, "one", ConceptExpression("two"))
# two = self.get_concept(sheerka, "two", Sequence(StrMatch("yes"), ConceptExpression("one")))
#
# concepts_by_first_keywords = BaseNodeParser.get_concepts_by_first_token(sheerka, [good, one, two]).body
#
# resolved_ret_val = BaseNodeParser.resolve_concepts_by_first_keyword(sheerka, concepts_by_first_keywords)
# assert resolved_ret_val.status
# assert resolved_ret_val.body == {
# "good": ["1001"],
# BuiltinConcepts.CHICKEN_AND_EGG: ["1002", "1003"]
# }
tests/parsers/test_BnfNodeParser.py
+4 -3
View File
@@ -4,6 +4,7 @@ import tests.parsers.parsers_utils
from core.builtin_concepts import BuiltinConcepts
from core.concept import Concept, ConceptParts, DoNotResolve, CC, DEFINITION_TYPE_BNF
from core.global_symbols import NotInit
from core.sheerka.services.SheerkaConceptManager import SheerkaConceptManager
from core.sheerka.services.SheerkaExecute import ParserInput
from parsers.BaseNodeParser import CNC, UTN, CN
from parsers.BnfDefinitionParser import BnfDefinitionParser
@@ -833,7 +834,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
# every obvious cyclic recursion are removed from concept_by_first_keyword dict
parser.init_from_concepts(context, my_map.values())
assert sheerka.om.copy(sheerka.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == expected
assert sheerka.om.copy(SheerkaConceptManager.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == expected
# get_parsing_expression() also returns CHICKEN_AND_EGG
parsing_expression = parser.get_parsing_expression(context, my_map["foo"])
@@ -858,7 +859,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
# every obvious cyclic recursion are removed from concept_by_first_keyword dict
parser.init_from_concepts(context, my_map.values())
assert sheerka.om.copy(sheerka.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {}
assert sheerka.om.copy(SheerkaConceptManager.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {}
parsing_expression = parser.get_parsing_expression(context, my_map["foo"])
assert sheerka.isinstance(parsing_expression, BuiltinConcepts.CHICKEN_AND_EGG)
@@ -884,7 +885,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
# every obvious cyclic recursion are removed from concept_by_first_keyword dict
parser.init_from_concepts(context, my_map.values())
assert sheerka.om.copy(sheerka.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {}
assert sheerka.om.copy(SheerkaConceptManager.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {}
parsing_expression = parser.get_parsing_expression(context, my_map["foo"])
assert sheerka.isinstance(parsing_expression, BuiltinConcepts.CHICKEN_AND_EGG)
tests/parsers/test_RuleParser.py
+1 -1
View File
@@ -87,7 +87,7 @@ class TestRuleParser(TestUsingMemoryBasedSheerka):
assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT)
assert len(rules) == 1
assert rules[0].id == "xxx"
assert rules[0].metadata.action_type == "deferred"
assert rules[0].metadata.id_is_unresolved
@pytest.mark.parametrize("text", [
"r:|1:xxx",
tests/parsers/test_SyaNodeParser.py
+36 -3
View File
@@ -1,6 +1,8 @@
import pytest
import tests.parsers.parsers_utils
from core.builtin_concepts import BuiltinConcepts
from core.concept import Concept, CIO, ALL_ATTRIBUTES, CMV
from core.concept import Concept, CIO, CMV
from core.global_symbols import CONCEPT_COMPARISON_CONTEXT
from core.sheerka.services.SheerkaExecute import ParserInput
from core.tokenizer import Tokenizer
@@ -10,8 +12,7 @@ from parsers.BaseNodeParser import utnode, cnode, short_cnode, UnrecognizedToken
from parsers.PythonParser import PythonNode
from parsers.SyaNodeParser import SyaNodeParser, SyaConceptParserHelper, SyaAssociativity, \
NoneAssociativeSequenceError, TooManyParametersFoundError, InFixToPostFix, ParenthesisMismatchError
import tests.parsers.parsers_utils
from tests.TestUsingFileBasedSheerka import TestUsingFileBasedSheerka
from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
@@ -1356,3 +1357,35 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
assert res.status
assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
assert lexer_nodes == [CN(cmap["suffixed"], 0, 6, source=text)]
class TestFileBaseSyaNodeParser(TestUsingFileBasedSheerka):
def test_i_can_parse_after_restart(self):
sheerka, context, one, two, plus = self.init_test().with_concepts("one",
"two",
Concept("a plus b").def_var("a").def_var("b"),
create_new=True).unpack()
sheerka.om.commit(context)
parser = SyaNodeParser()
# sanity check
# Remove this commented section to make sure that the nominal case still works
# res = parser.parse(context, ParserInput("one plus two"))
# assert res.status
# assert sheerka.isinstance(res.body.body[0].concept, plus.key)
sheerka = self.new_sheerka_instance(False)
context = self.get_context(sheerka)
res = parser.parse(context, ParserInput("one plus two"))
assert res.status
assert sheerka.isinstance(res.body.body[0].concept, plus.key)
# adds an ontology layer and make the test again
sheerka.push_ontology(context, "new ontology")
sheerka = self.new_sheerka_instance(False)
context = self.get_context(sheerka)
res = parser.parse(context, ParserInput("one plus two"))
assert res.status
assert sheerka.isinstance(res.body.body[0].concept, plus.key)