First implementation of Debugger for SyaNodeParser

_concepts_admin.txt

@@ -8,39 +8,20 @@ set_isa(c:explain last:, __AUTO_EVAL)
 def concept explain x as get_results(id=x, depth=3)
 set_isa(c:explain x:, __AUTO_EVAL)
 
-def concept precedence a > precedence b as set_is_greater_than(__PRECEDENCE, a, b)
+def concept precedence a > precedence b as set_is_greater_than(__PRECEDENCE, a, b, 'Sya')
 set_isa(c:precedence a > precedence b:, __AUTO_EVAL)
 
 def concept x is a command as set_auto_eval(x, __AUTO_EVAL)
 set_auto_eval(c:x is a command:)
 
-def concept q from q ? as question(q) pre is_question()
-set_is_lesser(__PRECEDENCE, q)
-set_auto_eval(c:q:)
-
-def concept "x is a concept" as isinstance(x, Concept) pre is_question()
-
-def concept x is a y as set_isa(x, y)
-set_auto_eval(c:x is a y:)
-def concept x is an y as set_isa(x, y)
-set_auto_eval(c:x is an y:)
-def concept x is a y as isa(x,y) pre is_question()
-# no need to auto eval as it's a question
-def concept x is an y as isa(x,y) pre is_question()
-# no need to auto eval as it's a question
-def concept x has a y as set_hasa(x, y)
-set_auto_eval(c:x has a y:)
-def concept x has an y as set_hasa(x, y)
-set_auto_eval(c:x has an y:)
-def concept x has a y as hasa(x,y) pre is_question()
-# no need to auto eval as it's a question
-def concept x has an y as hasa(x,y) pre is_question()
-# no need to auto eval as it's a question
-
 def concept activate debug as set_debug(True)
 set_auto_eval(c:activate debug:)
 def concept deactivate debug as set_debug(False)
 set_auto_eval(c:deactivate debug:)
+def concept debug on as set_debug(True)
+set_auto_eval(c:debug on:)
+def concept debug off as set_debug(False)
+set_auto_eval(c:debug off:)
 
 def concept activate debug on x as debug_var(x)
 set_auto_eval(c:activate debug on x:)

_concepts_default.txt

@@ -0,0 +1,27 @@
+# question
+def concept q from q ? as question(q) pre is_question()
+set_is_lesser(__PRECEDENCE, q, 'Sya')
+set_auto_eval(c:q:)
+
+def concept "x is a concept" as isinstance(x, Concept) pre is_question()
+
+# is a
+def concept x is a y as set_isa(x, y)
+set_auto_eval(c:x is a y:)
+def concept x is an y as set_isa(x, y)
+set_auto_eval(c:x is an y:)
+def concept x is a y as isa(x,y) pre is_question()
+# no need to auto eval as it's a question
+def concept x is an y as isa(x,y) pre is_question()
+# no need to auto eval as it's a question
+
+
+# has a
+def concept x has a y as set_hasa(x, y)
+set_auto_eval(c:x has a y:)
+def concept x has an y as set_hasa(x, y)
+set_auto_eval(c:x has an y:)
+def concept x has a y as hasa(x,y) pre is_question()
+# no need to auto eval as it's a question
+def concept x has an y as hasa(x,y) pre is_question()
+# no need to auto eval as it's a question

_concepts_full.txt

@@ -1,98 +1,7 @@
 #import admin
-
-# define numbers
-def concept one as 1
-def concept two as 2
-def concept three as 3
-def concept four as 4
-def concept five as 5
-def concept six as 6
-def concept seven as 7
-def concept eight as 8
-def concept nine as 9
-def concept ten as 10
-def concept eleven as 11
-def concept twelve as 12
-def concept thirteen as 13
-def concept fourteen as 14
-def concept fifteen as 15
-def concept sixteen as 16
-def concept seventeen as 17
-def concept eighteen as 18
-def concept nineteen as 19
-def concept twenty as 20
-def concept number
-set_isa(one, number)
-set_isa(two, number)
-set_isa(three, number)
-set_isa(four, number)
-set_isa(five, number)
-set_isa(six, number)
-set_isa(seven, number)
-set_isa(eight, number)
-set_isa(nine, number)
-set_isa(ten, number)
-set_isa(eleven, number)
-set_isa(twelve, number)
-set_isa(thirteen, number)
-set_isa(fourteen, number)
-set_isa(fifteen, number)
-set_isa(sixteen, number)
-set_isa(seventeen, number)
-set_isa(eighteen, number)
-set_isa(nineteen, number)
-set_isa(twenty, number)
-def concept twenties from bnf twenty number where number < 10 as twenty + number
-set_isa(twenties, number)
-def concept thirty as 30
-set_isa(thirty, number)
-def concept thirties from bnf thirty number where number < 10 as thirty + number
-set_isa(thirties, number)
-def concept forty as 40
-set_isa(forty, number)
-def concept forties from bnf forty number where number < 10 as forty + number
-set_isa(forties, number)
-def concept fifty as 50
-set_isa(fifty, number)
-def concept fifties from bnf fifty number where number < 10 as fifty + number
-set_isa(fifties, number)
-def concept sixty as 60
-set_isa(sixty, number)
-def concept sixties from bnf sixty number where number < 10 as sixty + number
-set_isa(sixties, number)
-def concept seventy as 70
-set_isa(seventy, number)
-def concept seventies from bnf seventy number where number < 10 as seventy + number
-set_isa(seventies, number)
-def concept eighty as 80
-set_isa(eighty, number)
-def concept eighties from bnf eighty number where number < 10 as eighty + number
-set_isa(eighties, number)
-def concept ninety as 90
-set_isa(ninety, number)
-def concept nineties from bnf ninety number where number < 10 as ninety + number
-set_isa(nineties, number)
-def concept one hundred as 100
-set_isa(one hundred, number)
-def concept hundreds from bnf number=n1 'hundred' 'and' number=n2 where n1 < 10 and n2 < 100 as n1 * 100 + n2
-set_isa(hundreds, number)
-def concept hundreds from bnf number 'hundred' where number < 10 as number * 100
-set_isa(last_created_concept(), number)
-def concept thousands from bnf number 'thousand' where number < 1000 as number * 1000
-set_isa(thousands, number)
-def concept thousands from bnf number=n1 'thousand' 'and' number=n2 as n1 * 1000 + n2 where n1 < 1000 and n2 < 1000
-set_isa(last_created_concept(), number)
-
-# define basic operations on numbers
-def concept plus from a plus b as a + b
-def concept minus from a minus b as a - b
-def concept multiplied from a multiplied by b as a * b
-def concept divided from a divided by b as a * b
-set_is_greater_than(__PRECEDENCE, multiplied, plus)
-set_is_greater_than(__PRECEDENCE, divided, plus)
-set_is_greater_than(__PRECEDENCE, multiplied, minus)
-set_is_greater_than(__PRECEDENCE, divided, minus)
-
-activate return values processing
+#import default
+#import numbers
+
+
 
 

_concepts_lite.txt

@@ -1,4 +1,6 @@
 #import admin
+#import default
+
 def concept one as 1
 def concept two as 2
 def concept number
@@ -6,5 +8,5 @@ def concept apple
 def concept table
 def concept location
 def concept x is on y as set_attr(x, location, y)
-activate return values processing
+
 

_concepts_numbers.txt

@@ -0,0 +1,92 @@
+# define numbers
+def concept one as 1
+def concept two as 2
+def concept three as 3
+def concept four as 4
+def concept five as 5
+def concept six as 6
+def concept seven as 7
+def concept eight as 8
+def concept nine as 9
+def concept ten as 10
+def concept eleven as 11
+def concept twelve as 12
+def concept thirteen as 13
+def concept fourteen as 14
+def concept fifteen as 15
+def concept sixteen as 16
+def concept seventeen as 17
+def concept eighteen as 18
+def concept nineteen as 19
+def concept twenty as 20
+def concept number
+set_isa(one, number)
+set_isa(two, number)
+set_isa(three, number)
+set_isa(four, number)
+set_isa(five, number)
+set_isa(six, number)
+set_isa(seven, number)
+set_isa(eight, number)
+set_isa(nine, number)
+set_isa(ten, number)
+set_isa(eleven, number)
+set_isa(twelve, number)
+set_isa(thirteen, number)
+set_isa(fourteen, number)
+set_isa(fifteen, number)
+set_isa(sixteen, number)
+set_isa(seventeen, number)
+set_isa(eighteen, number)
+set_isa(nineteen, number)
+set_isa(twenty, number)
+def concept twenties from bnf twenty number where number < 10 as twenty + number
+set_isa(twenties, number)
+def concept thirty as 30
+set_isa(thirty, number)
+def concept thirties from bnf thirty number where number < 10 as thirty + number
+set_isa(thirties, number)
+def concept forty as 40
+set_isa(forty, number)
+def concept forties from bnf forty number where number < 10 as forty + number
+set_isa(forties, number)
+def concept fifty as 50
+set_isa(fifty, number)
+def concept fifties from bnf fifty number where number < 10 as fifty + number
+set_isa(fifties, number)
+def concept sixty as 60
+set_isa(sixty, number)
+def concept sixties from bnf sixty number where number < 10 as sixty + number
+set_isa(sixties, number)
+def concept seventy as 70
+set_isa(seventy, number)
+def concept seventies from bnf seventy number where number < 10 as seventy + number
+set_isa(seventies, number)
+def concept eighty as 80
+set_isa(eighty, number)
+def concept eighties from bnf eighty number where number < 10 as eighty + number
+set_isa(eighties, number)
+def concept ninety as 90
+set_isa(ninety, number)
+def concept nineties from bnf ninety number where number < 10 as ninety + number
+set_isa(nineties, number)
+def concept one hundred as 100
+set_isa(one hundred, number)
+def concept hundreds from bnf number=n1 'hundred' 'and' number=n2 where n1 < 10 and n2 < 100 as n1 * 100 + n2
+set_isa(hundreds, number)
+def concept hundreds from bnf number 'hundred' where number < 10 as number * 100
+set_isa(last_created_concept(), number)
+def concept thousands from bnf number 'thousand' where number < 1000 as number * 1000
+set_isa(thousands, number)
+def concept thousands from bnf number=n1 'thousand' 'and' number=n2 as n1 * 1000 + n2 where n1 < 1000 and n2 < 1000
+set_isa(last_created_concept(), number)
+
+# define basic operations on numbers
+def concept plus from a plus b as a + b
+def concept minus from a minus b as a - b
+def concept multiplied from a multiplied by b as a * b
+def concept divided from a divided by b as a * b
+set_is_greater_than(__PRECEDENCE, multiplied, plus)
+set_is_greater_than(__PRECEDENCE, divided, plus)
+set_is_greater_than(__PRECEDENCE, multiplied, minus)
+set_is_greater_than(__PRECEDENCE, divided, minus)

_concepts_test-adjective.txt

@@ -0,0 +1,9 @@
+#import full
+def concept size
+def concept little
+def concept girl
+def concept little x as set_attr(x, size, little) ret x
+def concept the x ret memory(x)
+def concept how is x as get_attr(x, size)
+
+

src/core/builtin_concepts.py

@@ -565,3 +565,24 @@ class ToListConcept(Concept):
         self.set_value("recurse_on", recurse_on)  # which sub items should we display
         self.set_value("tab", tab)  # customise tab (content and length)
         self._metadata.is_evaluated = True
+
+
+class NewConceptConcept(Concept):
+    ALL_ATTRIBUTES = ["concept"]
+
+    def __init__(self, concept=None):
+        Concept.__init__(self,
+                         BuiltinConcepts.NEW_CONCEPT,
+                         True,
+                         False,
+                         BuiltinConcepts.NEW_CONCEPT,
+                         bound_body="concept")
+
+        self.set_value("concept", concept)
+        self._metadata.is_evaluated = True
+
+    def __repr__(self):
+        if self.concept:
+            return f"NewConcept(concept={self.concept}, key='{self.concept.key}')"
+        else:
+            return super().__repr__()

src/core/concept.py

@@ -393,7 +393,7 @@ class Concept:
                 for name, value in other.get_metadata().variables:
                     self.def_var(name, value)
             elif prop == "props":
-                self._metadata.props = deepcopy(other.get_metadata().props)
+                self._metadata.props = core.utils.sheerka_deepcopy(other.get_metadata().props)
             else:
                 setattr(self._metadata, prop, getattr(other.get_metadata(), prop))
 
@@ -534,6 +534,27 @@ class Concept:
             bag[prop] = getattr(self, prop)
         return bag
 
+    def as_debug_bag(self, new_obj, recurse):
+
+        bag = {"id": self.id, "name": self.name, "key": self.key}
+
+        # add variable metadata
+        for k, v in [(k, v) for k, v in self._metadata.variables if v]:
+            bag[f"meta.{k}"] = f"'{v}'"
+
+        # add compiled info
+        for k, v in [(k, v) for k, v in self._compiled.items() if isinstance(v, Concept)]:
+            bag[f"compiled.{k}"] = new_obj(v) if recurse else v
+
+        # add values
+        for prop in [p for p in self.__dict__ if not p.startswith("_") and not p.startswith("##")]:
+            v = self.get_value(prop)
+            if v == NotInit:
+                continue
+            bag[f"value.{prop}"] = new_obj(v) if (isinstance(v, Concept) and recurse) else core.utils.escape_str(v)
+
+        return bag
+
     def get_format_instructions(self):
         from core.builtin_concepts import BuiltinConcepts
         return self.get_prop(BuiltinConcepts.FORMAT_INSTRUCTIONS)

src/core/global_symbols.py

@@ -1,7 +1,9 @@
 # events
-CONCEPT_PRECEDENCE_MODIFIED = "cpm"
-RULE_PRECEDENCE_MODIFIED = "rpm"
-CONTEXT_DISPOSED = "cd"
+EVENT_CONCEPT_PRECEDENCE_MODIFIED = "evt_cpm"
+EVENT_RULE_PRECEDENCE_MODIFIED = "evt_rpm"
+EVENT_CONTEXT_DISPOSED = "evt_cd"
+EVENT_USER_INPUT_EVALUATED = "evt_uie"
+EVENT_CONCEPT_CREATED = "evt_cc"
 
 # comparison context
 RULE_COMPARISON_CONTEXT = "Rule"

src/core/sheerka/ExecutionContext.py

@@ -5,18 +5,13 @@ import time
 
 from core.builtin_concepts import BuiltinConcepts, ParserResultConcept
 from core.concept import Concept, get_concept_attrs
-from core.global_symbols import CONTEXT_DISPOSED
+from core.global_symbols import EVENT_CONTEXT_DISPOSED
 from core.sheerka.services.SheerkaExecute import NO_MATCH
 from core.sheerka.services.SheerkaMemory import SheerkaMemory
-from core.utils import CONSOLE_COLORS_MAP as CCM
+from core.utils import CONSOLE_COLORS_MAP as CCM, CONSOLE_COLUMNS
 from sdp.sheerkaDataProvider import Event
 
-try:
-    rows, columns = os.popen('stty size', 'r').read().split()
-except ValueError:
-    rows, columns = 50, 80
-
-pp = pprint.PrettyPrinter(indent=2, width=columns)
+pp = pprint.PrettyPrinter(indent=2, width=CONSOLE_COLUMNS)
 
 DEBUG_TAB_SIZE = 4
 
@@ -134,7 +129,7 @@ class ExecutionContext:
             return
 
         if self.stm:
-            self.sheerka.publish(self, CONTEXT_DISPOSED)
+            self.sheerka.publish(self, EVENT_CONTEXT_DISPOSED)
 
         self._stop = time.time_ns()
 
@@ -207,7 +202,7 @@ class ExecutionContext:
     def activate_push(self):
         if self._push:
             if self._push.stm:
-                self.sheerka.publish(self._push, CONTEXT_DISPOSED)
+                self.sheerka.publish(self._push, EVENT_CONTEXT_DISPOSED)
             self._push._stop = time.time_ns()
 
             self._push = None

src/core/sheerka/Sheerka.py

@@ -14,9 +14,11 @@ from core.builtin_concepts import BuiltinConcepts, ErrorConcept, ReturnValueConc
     UnknownConcept, AllBuiltinConcepts
 from core.concept import Concept, ConceptParts, NotInit, get_concept_attrs
 from core.error import ErrorObj
+from core.global_symbols import EVENT_USER_INPUT_EVALUATED
 from core.profiling import profile
 from core.sheerka.ExecutionContext import ExecutionContext
 from core.sheerka_logger import console_handler
+from core.simple_debug import my_debug
 from core.tokenizer import Token, TokenKind
 from printer.SheerkaPrinter import SheerkaPrinter
 from sdp.sheerkaDataProvider import SheerkaDataProvider, Event
@@ -81,15 +83,6 @@ class Sheerka(Concept):
         self.return_value_concept_id = None
         self.error_concept_id = None
 
-        # a concept can be instantiated
-        # ex: File is a concept, but File('foo.txt') is an instance
-        # TODO: manage contexts
-        self.instances = []
-
-        # List of the known rules by the system
-        # ex: hello => say('hello')
-        self.rules = []
-
         self.sdp: SheerkaDataProvider = None
         self.cache_manager = CacheManager(cache_only)
 
@@ -105,10 +98,11 @@ class Sheerka(Concept):
         self.printer_handler = SheerkaPrinter(self)
 
         self.during_restore = False
+        self.during_initialisation = False
         self._builtins_classes_cache = None
 
         self.save_execution_context = True
-        self.enable_process_return_values = False
+        self.enable_process_return_values = True
 
         self.methods_with_context = {"test_using_context"}  # only the names, the method is defined in sheerka_methods
         self.sheerka_methods = {
@@ -120,10 +114,6 @@ class Sheerka(Concept):
 
         self.locals = {}
 
-        self.last_executions = []
-        self.last_return_values = []
-        self.execution_count = 0
-
     @property
     def resolved_concepts_by_first_keyword(self):
         """
@@ -196,6 +186,7 @@ class Sheerka(Concept):
             self.enable_process_return_values = enable_process_return_values
 
         try:
+            self.during_initialisation = True
             from sheerkapickle.sheerka_handlers import initialize_pickle_handlers
             initialize_pickle_handlers()
 
@@ -235,6 +226,9 @@ class Sheerka(Concept):
         except IOError as e:
             res = ReturnValueConcept(self, False, self.get(BuiltinConcepts.ERROR), e)
 
+        finally:
+            self.during_initialisation = False
+
         return res
 
     def initialize_caching(self):
@@ -445,6 +439,7 @@ class Sheerka(Concept):
         :return:
         """
         # self.log.debug(f"Processing user input '{text}', {user_name=}.")
+        my_debug(f"****************** Processing user input '{text}', {user_name=}.***********************************")
         event = Event(text, user_name)
         self.sdp.save_event(event)
 
@@ -456,6 +451,7 @@ class Sheerka(Concept):
                               desc=f"Evaluating '{text}'") as execution_context:
 
             user_input = self.ret(self.name, True, self.new(BuiltinConcepts.USER_INPUT, body=text, user_name=user_name))
+            execution_context.add_inputs(user_input=user_input)
 
             # TODO. Must be a context hint, not a return value
             reduce_requested = self.ret(self.name, True, self.new(BuiltinConcepts.REDUCE_REQUESTED))
@@ -466,32 +462,12 @@ class Sheerka(Concept):
             if self.cache_manager.is_dirty:
                 self.cache_manager.commit(execution_context)
 
-            # exec_count = ExecutionContext.ids[execution_context.event.get_digest()]
-            # print("Execution Context Count:", exec_count)
-            if self.save_execution_context:
-                try:
-                    # if exec_count > 3400:
-                    #     print("Saving result. digest=", execution_context.event.get_digest())
-                    self.sdp.save_result(execution_context)
-                except Exception as ex:
-                    print(f"Failed to save execution context. Reason: {ex}")
-                    pass
-                    # self.log.error(f"Failed to save execution context. Reason: {ex}")
+            self.publish(execution_context, EVENT_USER_INPUT_EVALUATED)
 
             # Do not save execution contexts from process_return_values
             if self.enable_process_return_values:
                 self.process_return_values(execution_context, ret)
 
-        self.execution_count += 1
-        self._last_execution = execution_context
-        if len(self.last_executions) == self.MAX_EXECUTION_HISTORY:
-            del self.last_executions[0]
-        self.last_executions.append(execution_context)
-
-        if len(self.last_return_values) == self.MAX_RETURN_VALUES_HISTORY:
-            del self.last_return_values[0]
-        self.last_return_values.append(ret)
-
         return ret
 
     def print(self, result, instructions=None):
@@ -887,24 +863,6 @@ class Sheerka(Concept):
 
         return self.parsers_prefix + name
 
-    # def concepts(self):
-    #     """
-    #     List of all known concepts (look up in sdp)
-    #     :return:
-    #     """
-    #     res = []
-    #     lst = self.sdp.list(self.CONCEPTS_BY_ID_ENTRY)
-    #     for item in lst:
-    #         if isinstance(item, list):
-    #             res.extend(item)
-    #         else:
-    #             res.append(item)
-    #
-    #     return sorted(res, key=lambda i: int(i.id))
-
-    def get_last_execution(self):
-        return self._last_execution
-
     def test(self):
         return f"I have access to Sheerka !"
 

src/core/sheerka/services/SheerkaAdmin.py

@@ -24,9 +24,6 @@ class SheerkaAdmin(BaseService):
         self.sheerka.bind_service_method(self.restore, True)
         self.sheerka.bind_service_method(self.concepts, False)
         self.sheerka.bind_service_method(self.desc, False)
-        self.sheerka.bind_service_method(self.last_created_concept, False)
-        self.sheerka.bind_service_method(self.last_ret, False)
-        self.sheerka.bind_service_method(self.last_error_ret, False)
         self.sheerka.bind_service_method(self.extended_isinstance, False)
         self.sheerka.bind_service_method(self.is_container, False)
         self.sheerka.bind_service_method(self.format_rules, False)
@@ -149,44 +146,6 @@ class SheerkaAdmin(BaseService):
     def format_rules(self):
         return self.sheerka.new(BuiltinConcepts.TO_LIST, items=self.sheerka.get_format_rules())
 
-    def last_created_concept(self, use_history=False):
-        for exec_result in reversed(self.sheerka.last_executions):
-            return_values = exec_result.values["return_values"]
-            for ret in return_values:
-                if ret.status and self.sheerka.isinstance(ret.value, BuiltinConcepts.NEW_CONCEPT):
-                    return ret.value.body
-
-        if use_history:
-            return self.sheerka.new(BuiltinConcepts.ERROR, body="Not yet implement")
-
-        return self.sheerka.new(BuiltinConcepts.NOT_FOUND)
-
-    def last_ret(self, context, index=-1):
-        try:
-            last = self.sheerka.last_return_values[index]
-            return last[0] if isinstance(last, list) and len(last) == 1 else last
-        except IndexError:
-            return None
-
-    def last_error_ret(self, context, index=-1):
-        while index >= -len(self.sheerka.last_return_values):
-            last = self.sheerka.last_return_values[index]
-            last = [last] if not hasattr(last, "__iter__") else last
-            last = [ret_val for ret_val in last if not ret_val.status]
-            if len(last) == 0:
-                index -= 1
-                continue
-
-            if len(last) > 1:
-                return context.sheerka.ret(SheerkaAdmin.NAME,
-                                           False,
-                                           context.sheerka.new(BuiltinConcepts.TOO_MANY_ERRORS, body=last))
-
-            return last[0]
-
-        return context.sheerka.ret(SheerkaAdmin.NAME,
-                                   False,
-                                   context.sheerka.new(BuiltinConcepts.NOT_FOUND))
 
     def extended_isinstance(self, a, b):
         """

src/core/sheerka/services/SheerkaComparisonManager.py

@@ -3,7 +3,7 @@ from dataclasses import dataclass
 from cache.Cache import Cache
 from cache.ListCache import ListCache
 from core.builtin_concepts import BuiltinConcepts
-from core.global_symbols import CONCEPT_PRECEDENCE_MODIFIED, RULE_PRECEDENCE_MODIFIED, RULE_COMPARISON_CONTEXT, \
+from core.global_symbols import EVENT_CONCEPT_PRECEDENCE_MODIFIED, EVENT_RULE_PRECEDENCE_MODIFIED, RULE_COMPARISON_CONTEXT, \
     CONCEPT_COMPARISON_CONTEXT
 from core.builtin_helpers import ensure_concept_or_rule
 from core.concept import Concept
@@ -183,9 +183,9 @@ class SheerkaComparisonManager(BaseService):
 
         if comparison_obj.property == BuiltinConcepts.PRECEDENCE:
             if comparison_obj.context == CONCEPT_COMPARISON_CONTEXT:
-                self.sheerka.publish(context, CONCEPT_PRECEDENCE_MODIFIED)
+                self.sheerka.publish(context, EVENT_CONCEPT_PRECEDENCE_MODIFIED)
             elif comparison_obj.context == RULE_COMPARISON_CONTEXT:
-                self.sheerka.publish(context, RULE_PRECEDENCE_MODIFIED)
+                self.sheerka.publish(context, EVENT_RULE_PRECEDENCE_MODIFIED)
 
         return self.sheerka.ret(self.NAME, True, self.sheerka.new(BuiltinConcepts.SUCCESS))
 

src/core/sheerka/services/SheerkaCreateNewConcept.py

@@ -2,6 +2,7 @@ import core.utils
 from core.builtin_concepts import BuiltinConcepts, ErrorConcept
 from core.builtin_helpers import ensure_concept
 from core.concept import Concept, DEFINITION_TYPE_DEF, DEFINITION_TYPE_BNF, freeze_concept_attrs
+from core.global_symbols import EVENT_CONCEPT_CREATED
 from core.sheerka.services.sheerka_service import BaseService
 from sdp.sheerkaDataProvider import SheerkaDataProviderDuplicateKeyError
 
@@ -85,6 +86,9 @@ class SheerkaCreateNewConcept(BaseService):
         if concept.get_bnf() and init_bnf_ret_value is not None and init_bnf_ret_value.status:
             sheerka.cache_manager.clear(sheerka.CONCEPTS_GRAMMARS_ENTRY)
 
+        # publish the new concept
+        sheerka.publish(context, EVENT_CONCEPT_CREATED, concept)
+
         # process the return if needed
         ret = sheerka.ret(self.NAME, True, sheerka.new(BuiltinConcepts.NEW_CONCEPT, body=concept))
         return ret

src/core/sheerka/services/SheerkaDebugManager.py

@@ -1,20 +1,57 @@
-import os
 import pprint
 import re
 from dataclasses import dataclass
 
 from core.builtin_concepts import BuiltinConcepts
+from core.concept import Concept, NotInit
 from core.sheerka.ExecutionContext import ExecutionContext
 from core.sheerka.services.sheerka_service import BaseService
-from core.utils import CONSOLE_COLORS_MAP as CCM
+from core.utils import CONSOLE_COLORS_MAP as CCM, CONSOLE_COLUMNS, PRIMITIVES_TYPES
 from core.utils import evaluate_expression, as_bag
 
-try:
-    rows, columns = os.popen('stty size', 'r').read().split()
-except ValueError:
-    rows, columns = 50, 80
+pp = pprint.PrettyPrinter(indent=2, width=CONSOLE_COLUMNS)
 
-pp = pprint.PrettyPrinter(indent=2, width=columns)
+NotFound = "** Not Found **"
+
+
+class ConceptDebugObj:
+    def __init__(self, concept):
+        self.concept = concept
+        self.attrs = concept.as_debug_bag(lambda x: ConceptDebugObj(x), True)
+
+    def __repr__(self):
+        return f"({self.concept_repr()})"
+
+    def __eq__(self, other):
+        if not isinstance(other, ConceptDebugObj):
+            return False
+
+        return self.attrs == other.attrs
+
+    def __hash__(self):
+        return hash(self.concept)
+
+    def concept_repr(self):
+        res = f":{self.concept.name}|{self.concept.id}:"
+
+        # print metadata
+        first = True
+        for k, v in [(k, v) for k, v in self.attrs.items() if k not in ("id", "name", "key")]:
+            if not first:
+                res += ", "
+            res += f"{k}={v}"
+            first = False
+
+        return res
+
+
+class ConceptNodeDebugObj:
+    def __init__(self, concept_node):
+        self.concept_node = concept_node
+        self.concept_debug = ConceptDebugObj(concept_node.concept)
+
+    def __repr__(self):
+        return f"ConceptNode({self.concept_debug.concept_repr()})"
 
 
 class BaseDebugLogger:
@@ -49,6 +86,9 @@ class BaseDebugLogger:
     def is_enabled(self):
         pass
 
+    def get_enabled_vars(self):
+        pass
+
 
 class NullDebugLogger(BaseDebugLogger):
     def __init__(self):
@@ -57,6 +97,9 @@ class NullDebugLogger(BaseDebugLogger):
     def is_enabled(self):
         return False
 
+    def get_enabled_vars(self):
+        pass
+
 
 class ConsoleDebugLogger(BaseDebugLogger):
 
@@ -70,9 +113,29 @@ class ConsoleDebugLogger(BaseDebugLogger):
         self.is_highlighted = ""
 
     def is_enabled(self):
+        """
+        True if the debug is activated for the current service, method and context
+        :return:
+        """
         return True
 
+    def get_enabled_vars(self):
+        """
+        Returns the list of all enabled variables for this console debugger
+        :return:
+        """
+        return self.debug_manager.get_enabled_items("vars",
+                                                    self.context,
+                                                    self.service_name,
+                                                    self.method_name,
+                                                    self.debug_id)
+
     def debug_entering(self, **kwargs):
+        """
+        Log that we start debugging a method (for a specified service and context)
+        :param kwargs:
+        :return:
+        """
         super().debug_entering(**kwargs)
 
         str_text = f"{CCM['blue']}Entering {self.service_name}.{self.method_name} with {CCM['reset']}"
@@ -84,10 +147,24 @@ class ConsoleDebugLogger(BaseDebugLogger):
             self.debug_manager.debug(self.prefix() + str_vars)
 
     def debug_log(self, text, is_error=False):
+        """
+        Prints a debug information (not related to a specific variable, concept or rule)
+        :param text:
+        :param is_error:
+        :return:
+        """
         color = 'red' if is_error else 'blue'
         self.debug_manager.debug(self.prefix() + f"{CCM[color]}..{text}{CCM['reset']}")
 
     def debug_var(self, name, value, is_error=False, hint=None):
+        """
+        Prints the value of a variable
+        :param name:
+        :param value:
+        :param is_error:
+        :param hint:
+        :return:
+        """
         enabled = is_error or self.debug_manager.compute_debug_var(self.context,
                                                                    self.service_name,
                                                                    self.method_name,
@@ -103,6 +180,12 @@ class ConsoleDebugLogger(BaseDebugLogger):
         self.debug(str_text, str_vars)
 
     def debug_rule(self, rule, results):
+        """
+        Prints debug information related to a specific rule id
+        :param rule:
+        :param results:
+        :return:
+        """
         if not self.debug_manager.compute_debug_rule(self.context,
                                                      self.service_name,
                                                      self.method_name,
@@ -115,6 +198,13 @@ class ConsoleDebugLogger(BaseDebugLogger):
         self.debug(str_text, str_vars)
 
     def debug_concept(self, concept, text=None, **kwargs):
+        """
+        Prints debug information related to a specific concept
+        :param concept:
+        :param text:
+        :param kwargs:
+        :return:
+        """
         raw = kwargs.pop('raw', None)
         if not self.debug_manager.compute_debug_concept(self.context,
                                                         self.service_name,
@@ -283,27 +373,6 @@ class SheerkaDebugManager(BaseService):
 
         return debug_for_self, debug_for_children
 
-    def inspect(self, context, context_id, *props):
-        """
-        Print
-        :param context:
-        :param context_id:
-        :return:
-        """
-        to_inspect = self.sheerka.get_execution_item(context, context_id)
-        if not isinstance(to_inspect, ExecutionContext):
-            return to_inspect
-
-        if not props:
-            props = ["inputs", "values.return_values"]
-
-        bag = as_bag(to_inspect)
-        res = {}
-        for prop in props:
-            res[prop] = evaluate_expression(prop, bag)
-
-        return self.sheerka.new(BuiltinConcepts.TO_DICT, body=res)
-
     def debug(self, *args, **kwargs):
         print(*args, **kwargs)
 
@@ -428,6 +497,24 @@ class SheerkaDebugManager(BaseService):
 
         return res
 
+    def get_enabled_items(self, item_type, context, service_name, method_name, debug_id):
+        if not self.activated:
+            return False
+
+        selected = set()
+        for setting in getattr(self, self.container_name(item_type)):
+            if not setting.enabled or setting.item is None:
+                continue
+
+            if (setting.service_name is None or setting.service_name == service_name) and \
+                (setting.method_name is None or setting.method_name == method_name) and \
+                (setting.context_id is None or setting.context_id == context.id or (
+                    setting.context_children and context.has_parent(setting.context_id))) and \
+                (setting.debug_id is None or setting.debug_id == debug_id):
+                selected.add(setting.item)
+
+        return selected
+
     def reset_debug(self, context):
         for item_type in ["vars", "rules", "concepts"]:
             setting_name = self.container_name(item_type)
@@ -487,6 +574,123 @@ class SheerkaDebugManager(BaseService):
     def compute_debug_rule(self, context, service_name, method_name, item, debug_id):
         return self.compute_debug_item("rules", context, service_name, method_name, item, debug_id)
 
+    def inspect(self, context, *args, **kwargs):
+        """
+        Print
+        :param context:
+        :param args: 1st parameter is what to display, the other are the properties to display
+        :param kwargs: how to display the result
+        :return:
+        """
+
+        if len(args) == 0:
+            return
+
+        forced_prop, props = None, None
+        if isinstance(args[0], int):
+            to_inspect = self.sheerka.get_execution_item(context, int(args[0]))
+            if isinstance(to_inspect, ExecutionContext):
+                if len(args) == 1:
+                    props = ["inputs", "values.return_values"]
+                else:
+                    props = args[1:]
+            else:
+                props = None
+        else:
+            to_inspect = args[0]
+            if isinstance(to_inspect, Concept):
+                if len(args) > 1:
+                    forced_prop = args[1:]
+                    props = forced_prop
+            else:
+                props = args[1:]
+
+        bag = self.as_debug_bag(to_inspect, False, forced_prop)
+        props = props or list(bag.keys())
+
+        res = self.inspect_object(bag, props, False, **kwargs)
+
+        # Attributes that are not found are probably directly requested by the user
+        # Let's try for the full names of these attributes
+        not_found = {k: v for k, v in [(k, v) for k, v in res.items() if v == NotFound]}
+        if len(not_found) > 0:
+            to_add = {}
+            to_remove = []
+
+            for k, v in not_found.items():
+                alternate_props = ["meta." + k, "compiled." + k, "value." + k]
+                res2 = self.inspect_object(bag, alternate_props, True, **kwargs)
+                if len(res2) > 0:
+                    to_add.update(res2)
+                    to_remove.append(k)
+
+            res.update(to_add)
+            for k in to_remove:
+                del res[k]
+
+        return self.sheerka.new(BuiltinConcepts.TO_DICT, body=res)
+
+    def inspect_object(self, bag, props, discard_not_found, **kwargs):
+        values_required = kwargs.get("values", False)
+        as_bag_required = kwargs.get("as_bag", False)
+
+        res = {}
+        for prop in props:
+            if prop in res:
+                # discard duplicates
+                continue
+
+            try:
+                value = bag[prop] if prop.startswith("#") else evaluate_expression(prop, bag)
+            except NameError:
+                if discard_not_found:
+                    continue
+                else:
+                    value = NotFound
+
+            if callable(value):
+                # discard methods
+                continue
+
+            if values_required:
+                value = self.get_inner_values(value, **kwargs)
+            elif as_bag_required:
+                value = self.get_debug_repr(value, **kwargs)
+
+            res[prop] = value
+        return res
+
+    def get_inner_values(self, obj, **kwargs):
+        if obj is None:
+            return None
+
+        if isinstance(obj, list):
+            return [self.get_inner_values(item, **kwargs) for item in obj]
+
+        if hasattr(obj, "get_obj_value"):
+            return obj.get_obj_value()
+
+        if not isinstance(obj, Concept):
+            return self.get_debug_repr(obj, **kwargs)
+
+        if obj.body is NotInit:
+            return self.get_debug_repr(obj, **kwargs)
+
+        return self.get_inner_values(obj.body, **kwargs)
+
+    def get_debug_repr(self, obj, **kwargs):
+        if kwargs.get("as_bag", False):
+            forced_props = self.get_concept_forced_props(obj) if isinstance(obj, Concept) else None
+            return self.as_debug_bag(obj, True, forced_props)
+
+        from parsers.BaseNodeParser import ConceptNode
+        if isinstance(obj, Concept):
+            return ConceptDebugObj(obj)
+        elif isinstance(obj, ConceptNode):
+            return ConceptNodeDebugObj(obj)
+        else:
+            return obj
+
     @staticmethod
     def container_name(item_type):
         return f"debug_{item_type}_settings"
@@ -505,7 +709,7 @@ class SheerkaDebugManager(BaseService):
                 if len(parts) > 1:
                     method_name = None if parts[1] == "*" else parts[1]
                 if len(parts) > 2:
-                    item = parts[2]
+                    item = ".".join(parts[2:])
 
         if len(args) > 1:
             context_part = args[1]
@@ -535,139 +739,22 @@ class SheerkaDebugManager(BaseService):
 
         return item, service, method_name, context_id, context_children, debug_id, enabled
 
-    # def debug_rule(self, context, rule=None, context_id=None, debug_id=None, enabled=True):
-    #     """
-    #     Add a debug rule request
-    #     :param context:
-    #     :param rule:
-    #     :param context_id:
-    #     :param debug_id:
-    #     :param enabled:
-    #     :return:
-    #     """
-    #     rule = str(rule) if rule is not None else None
-    #     for setting in self.debug_rules_settings:
-    #         if setting.rule_id == rule and \
-    #             setting.context_id == context_id and \
-    #             setting.debug_id == debug_id:
-    #             setting.enabled = enabled
-    #             break
-    #     else:
-    #         self.debug_rules_settings.append(DebugRuleSetting(rule,
-    #                                                           context_id,
-    #                                                           debug_id,
-    #                                                           enabled))
-    #
-    #     self.sheerka.record_var(context, self.NAME, "debug_rules_settings", self.debug_rules_settings)
-    #     return self.sheerka.ret(SheerkaDebugManager.NAME, True, self.sheerka.new(BuiltinConcepts.SUCCESS))
-    #
-    # def debug_concept(self, context, concept=None, context_id=None, debug_id=None, enabled=True):
-    #     """
-    #     Add a debug rule request
-    #     :param context:
-    #     :param concept:
-    #     :param context_id:
-    #     :param debug_id:
-    #     :param enabled:
-    #     :return:
-    #     """
-    #     concept_id = concept.id if isinstance(concept, Concept) else str(concept) if concept is not None else None
-    #     for setting in self.debug_concepts_settings:
-    #         if setting.concept_id == concept_id and \
-    #             setting.context_id == context_id and \
-    #             setting.debug_id == debug_id:
-    #             setting.enabled = enabled
-    #             break
-    #     else:
-    #         self.debug_concepts_settings.append(DebugConceptSetting(concept_id,
-    #                                                                 context_id,
-    #                                                                 debug_id,
-    #                                                                 enabled))
-    #
-    #     self.sheerka.record_var(context, self.NAME, "debug_concepts_settings", self.debug_concepts_settings)
-    #     return self.sheerka.ret(SheerkaDebugManager.NAME, True, self.sheerka.new(BuiltinConcepts.SUCCESS))
-    #
+    @staticmethod
+    def get_concept_forced_props(concept):
+        return ["id", "name", "key"] + [p for p in concept.__dict__ if not p.startswith("_")]
 
-    #
+    @staticmethod
+    def as_debug_bag(obj, recurse=True, forced_props=None):
+        if type(obj) in PRIMITIVES_TYPES:
+            return obj
 
-    # def compute_debug_var(self, service_name, method_name, context_id, variable_name, debug_id):
-    #     if not self.activated:
-    #         return False
-    #
-    #     selected = []
-    #     for setting in self.debug_vars_settings:
-    #         if setting.variable_name is None and setting.debug_id is None:
-    #             continue
-    #
-    #         if (setting.service_name is None or setting.service_name == service_name) and \
-    #             (setting.method_name is None or setting.method_name == method_name) and \
-    #             (setting.context_id is None or setting.context_id == context_id) and \
-    #             (setting.variable_name is None or
-    #              setting.variable_name == "*" or
-    #              setting.variable_name == variable_name) and \
-    #             (setting.debug_id is None or setting.debug_id == debug_id):
-    #             selected.append(setting.enabled)
-    #
-    #     if len(selected) == 0:
-    #         return False
-    #
-    #     res = selected[0]
-    #     for enabled in selected[1:]:
-    #         if res == False or enabled == False:
-    #             return False
-    #
-    #         if isinstance(res, str):
-    #             continue
-    #
-    #         res = enabled
-    #
-    #     return res
-    #
-    # def compute_debug_rule(self, rule_id, context_id, debug_id):
-    #     if not self.activated:
-    #         return False
-    #
-    #     selected = []
-    #     for setting in self.debug_rules_settings:
-    #         if (setting.rule_id is None or setting.rule_id == rule_id) and \
-    #             (setting.context_id is None or setting.context_id == context_id) and \
-    #             (setting.debug_id is None or setting.debug_id == debug_id):
-    #             selected.append(setting.enabled)
-    #
-    #     if len(selected) == 0:
-    #         return False
-    #
-    #     res = selected[0]
-    #     for enabled in selected[1:]:
-    #         res &= enabled
-    #
-    #     return res
-    #
-    # def compute_debug_concept(self, concept_id, context_id, debug_id):
-    #     if not self.activated:
-    #         return False
-    #
-    #     selected = []
-    #     for setting in self.debug_concepts_settings:
-    #         if (setting.concept_id is None or setting.concept_id == concept_id) and \
-    #             (setting.context_id is None or setting.context_id == context_id) and \
-    #             (setting.debug_id is None or setting.debug_id == debug_id):
-    #             selected.append(setting.enabled)
-    #
-    #     if len(selected) == 0:
-    #         return False
-    #
-    #     res = selected[0]
-    #     for enabled in selected[1:]:
-    #         res &= enabled
-    #
-    #     return res
-    #
-    # def reset_debug_rules(self, context):
-    #     self.debug_rules_settings.clear()
-    #     self.sheerka.record_var(context, self.NAME, "debug_rules_settings", self.debug_rules_settings)
-    #     return self.sheerka.ret(SheerkaDebugManager.NAME, True, self.sheerka.new(BuiltinConcepts.SUCCESS))
-    #
-    # def get_debug_settings(self):
-    #     lst = self.debug_vars_settings + self.debug_concepts_settings + self.debug_rules_settings
-    #     return self.sheerka.new(BuiltinConcepts.TO_LIST, body=lst)
+        res = {'#type#': type(obj).__name__}
+        if isinstance(obj, Concept) and not obj.get_metadata().is_builtin:
+            res.update(obj.as_debug_bag(SheerkaDebugManager.as_debug_bag, recurse))
+
+        else:
+            forced_props_to_use = [p for p in forced_props if p != "#type#"] if forced_props else None
+            res.update(as_bag(obj, forced_props_to_use))
+            del res["self"]
+
+        return res

src/core/sheerka/services/SheerkaDump.py

@@ -1,16 +1,15 @@
-import os
 import pprint
 
 from core.builtin_concepts import BuiltinConcepts
 from core.concept import Concept
 from core.sheerka.ExecutionContext import ExecutionContext
 from core.sheerka.services.sheerka_service import BaseService
+from core.utils import CONSOLE_COLUMNS
 from sdp.sheerkaDataProvider import SheerkaDataProvider, Event
 
 
 def get_pp():
-    rows, columns = os.popen('stty size', 'r').read().split()
-    pp = pprint.PrettyPrinter(width=columns, compact=True)
+    pp = pprint.PrettyPrinter(width=CONSOLE_COLUMNS, compact=True)
     return pp
 
 

src/core/sheerka/services/SheerkaMemory.py

@@ -4,7 +4,7 @@ from cache.FastCache import FastCache
 from cache.ListIfNeededCache import ListIfNeededCache
 from core.builtin_concepts import BuiltinConcepts
 from core.concept import Concept
-from core.global_symbols import CONTEXT_DISPOSED
+from core.global_symbols import EVENT_CONTEXT_DISPOSED
 from core.sheerka.services.sheerka_service import BaseService, ServiceObj
 
 
@@ -23,7 +23,7 @@ class SheerkaMemory(BaseService):
     def __init__(self, sheerka):
         super().__init__(sheerka)
         self.short_term_objects = FastCache()
-        self.objects = ListIfNeededCache(default=lambda k: self.sheerka.sdp.get(self.OBJECTS_ENTRY, k))
+        self.memory_objects = ListIfNeededCache(default=lambda k: self.sheerka.sdp.get(self.OBJECTS_ENTRY, k))
         self.registration = {}
 
     def initialize(self):
@@ -37,14 +37,16 @@ class SheerkaMemory(BaseService):
         self.sheerka.bind_service_method(self.unregister_object, True, visible=False)
         self.sheerka.bind_service_method(self.add_registered_objects, True, visible=False)
         self.sheerka.bind_service_method(self.memory, False)
+        self.sheerka.bind_service_method(self.mem, False)
 
-        self.sheerka.cache_manager.register_cache(self.OBJECTS_ENTRY, self.objects, persist=True, use_ref=True)
+        self.sheerka.cache_manager.register_cache(self.OBJECTS_ENTRY, self.memory_objects, persist=True, use_ref=True)
 
     def reset(self):
         self.short_term_objects.clear()
+        self.memory_objects.clear()
 
     def initialize_deferred(self, context, is_first_time):
-        self.sheerka.subscribe(CONTEXT_DISPOSED, self.remove_context)
+        self.sheerka.subscribe(EVENT_CONTEXT_DISPOSED, self.remove_context)
 
     def get_from_short_term_memory(self, context, key):
         while True:
@@ -90,16 +92,16 @@ class SheerkaMemory(BaseService):
         :param concept:
         :return:
         """
-        self.objects.put(key, MemoryObject(context.event.get_digest(), concept))
+        self.memory_objects.put(key, MemoryObject(context.event.get_digest(), concept))
 
     def get_from_memory(self, context, key):
         """"
         """
-        return self.objects.get(key)
+        return self.memory_objects.get(key)
 
     def register_object(self, context, key, concept):
         """
-        Before adding objects to memory, they first need to be registered
+        Before adding memory_objects to memory, they first need to be registered
         More:
         We don't want to add all evaluated concept into memory
         (because some of them may be ref to concept already in memory)
@@ -126,7 +128,7 @@ class SheerkaMemory(BaseService):
 
     def add_registered_objects(self, context):
         """
-        Adds all registered objects
+        Adds all registered memory_objects
         :param context:
         :return:
         """
@@ -136,7 +138,7 @@ class SheerkaMemory(BaseService):
 
     def memory(self, context, name=None):
         """
-        Get the list of all objects in memory
+        Get the list of all memory_objects in memory
         :param context:
         :param name:
         :return:
@@ -154,10 +156,14 @@ class SheerkaMemory(BaseService):
             return obj.obj
 
         res = {}
-        for k in self.objects:
-            obj = self.objects.get(k)
+        for k in self.memory_objects:
+            obj = self.memory_objects.get(k)
             if isinstance(obj, list):
                 obj = obj[-1]
             res[k] = obj.obj
 
         return res
+
+    def mem(self):
+        keys = sorted([k for k in self.memory_objects])
+        return {"keys": keys, "len": len(keys)}

src/core/sheerka/services/SheerkaResultManager.py

@@ -1,9 +1,14 @@
 import ast
 
+from cache.Cache import Cache
 from core.builtin_concepts import BuiltinConcepts
+from core.global_symbols import EVENT_USER_INPUT_EVALUATED, EVENT_CONCEPT_CREATED
 from core.sheerka.services.sheerka_service import BaseService
+from core.utils import CONSOLE_COLORS_MAP as CCM
 from core.utils import as_bag
 
+MAX_EXECUTION_HISTORY = 100
+
 
 class SheerkaResultConcept(BaseService):
     NAME = "Result"
@@ -11,6 +16,10 @@ class SheerkaResultConcept(BaseService):
     def __init__(self, sheerka, page_size=30):
         super().__init__(sheerka)
         self.page_size = page_size
+        self.executions_contexts_cache = Cache(MAX_EXECUTION_HISTORY)
+        self.last_execution = None
+        self.last_created_concept = None
+        self.last_created_concept_id = None
 
     def initialize(self):
         self.sheerka.bind_service_method(self.get_results_by_digest, True)  # digest is recorded
@@ -18,6 +27,19 @@ class SheerkaResultConcept(BaseService):
         self.sheerka.bind_service_method(self.get_last_results, True)  # digest is recorded
         self.sheerka.bind_service_method(self.get_results, False)
         self.sheerka.bind_service_method(self.get_execution_item, False)
+        self.sheerka.bind_service_method(self.get_last_ret, False, as_name="last_ret")
+        self.sheerka.bind_service_method(self.get_last_created_concept, False, as_name="last_created_concept")
+
+    def initialize_deferred(self, context, is_first_time):
+        self.restore_values("last_created_concept_id")
+        self.sheerka.subscribe(EVENT_USER_INPUT_EVALUATED, self.user_input_evaluated)
+        self.sheerka.subscribe(EVENT_CONCEPT_CREATED, self.new_concept_created)
+
+    def reset(self):
+        self.executions_contexts_cache.clear()
+        self.last_execution = None
+        self.last_created_concept = None
+        self.last_created_concept_id = None
 
     @staticmethod
     def get_predicate(**kwargs):
@@ -38,12 +60,24 @@ class SheerkaResultConcept(BaseService):
         predicate = " and ".join(res)
         return compile(ast.parse(predicate, mode="eval"), "<SheerkaResultManager.get_predicate>", mode="eval")
 
+    @staticmethod
+    def as_list(execution_context, predicate):
+        def _yield_result(lst):
+            for e in lst:
+                if predicate is None or eval(predicate, as_bag(e)):
+                    yield e
+
+                if e._children:
+                    yield from _yield_result(e._children)
+
+        return _yield_result([execution_context])
+
     def get_results_by_digest(self, context, digest, filter=None, record_digest=True, **kwargs):
         """
         Gets the entire execution tree for the given event digest
-        :param filter:
         :param context:
         :param digest:
+        :param filter:
         :param record_digest:
         :return:
         """
@@ -54,8 +88,12 @@ class SheerkaResultConcept(BaseService):
             kwargs["filter"] = filter
 
         try:
-            result = self.sheerka.sdp.load_result(digest)
-            event = self.sheerka.sdp.load_event(digest)
+            if digest in self.executions_contexts_cache:
+                result = self.executions_contexts_cache.get(digest)
+                event = result.event
+            else:
+                result = self.sheerka.sdp.load_result(digest)
+                event = self.sheerka.sdp.load_event(digest)  # there is no real need for a cache of the events
 
             if record_digest:
                 context.log(f"Recording digest '{digest}'")
@@ -89,7 +127,18 @@ class SheerkaResultConcept(BaseService):
         if command is None:
             return None
 
-        start = 0
+        # first, search in cache
+        for event_id in self.executions_contexts_cache:
+            execution_context = self.executions_contexts_cache.get(event_id)
+            if execution_context.event.message.startswith(command):
+                return self.get_results_by_digest(context,
+                                                  execution_context.event.get_digest(),
+                                                  filter,
+                                                  record_digest,
+                                                  **kwargs)
+
+        # not found, search in db
+        start = len(self.executions_contexts_cache)
         consumed = 0
         while True:
             for event in self.sheerka.sdp.load_events(self.page_size, start):
@@ -103,6 +152,7 @@ class SheerkaResultConcept(BaseService):
             start += self.page_size
             consumed = 0
 
+        # not found, return error
         return self.sheerka.new(BuiltinConcepts.NOT_FOUND, body={"command": command})
 
     def get_last_results(self, context, filter=None, record_digest=True, **kwargs):
@@ -114,23 +164,16 @@ class SheerkaResultConcept(BaseService):
         :return:
         """
 
-        start = 0
-        page_size = 2
-        consumed = 0
-        while True:
-            for event in self.sheerka.sdp.load_events(page_size, start):
-                consumed += 1
-                if self.sheerka.sdp.has_result(event.get_digest()):
-                    return self.get_results_by_digest(context, event.get_digest(), filter, record_digest, **kwargs)
+        if self.last_execution:
+            return self.get_results_by_digest(context,
+                                              self.last_execution.event.get_digest(),
+                                              filter,
+                                              record_digest,
+                                              **kwargs)
 
-            if consumed < page_size:
-                break
-
-            if page_size < 100:
-                page_size *= 2
-
-            start += page_size
-            consumed = 0
+        event_id = self._get_last_execution_result_event_id_from_db()
+        if event_id is not None:
+            return self.get_results_by_digest(context, event_id, filter, record_digest, **kwargs)
 
         return self.sheerka.new(BuiltinConcepts.NOT_FOUND, body={"query": "last"})
 
@@ -150,12 +193,21 @@ class SheerkaResultConcept(BaseService):
         return self.get_results_by_digest(context, digest, filter, False, **kwargs)
 
     def get_execution_item(self, context, item_id):
+        """
+        Return the item_id'th element of the execution result under investigation
+        :param context:
+        :param item_id:
+        :return:
+        """
         digest = self.sheerka.load_var(self.NAME, "digest")
         if digest is None:
             return self.sheerka.new(BuiltinConcepts.NOT_FOUND, body="no digest")
 
         try:
-            result = self.sheerka.sdp.load_result(digest)
+            if digest in self.executions_contexts_cache:
+                result = self.executions_contexts_cache.get(digest)
+            else:
+                result = self.sheerka.sdp.load_result(digest)
             items = list(self.as_list(result, self.get_predicate(id=item_id)))
 
             if len(items) == 0:
@@ -167,14 +219,77 @@ class SheerkaResultConcept(BaseService):
             context.log_error(f"Digest {digest} is not found.", self.NAME, ex)
             return self.sheerka.new(BuiltinConcepts.NOT_FOUND, body={"digest": digest})
 
-    @staticmethod
-    def as_list(execution_context, predicate):
-        def _yield_result(lst):
-            for e in lst:
-                if predicate is None or eval(predicate, as_bag(e)):
-                    yield e
+    def user_input_evaluated(self, execution_context):
+        """
+        Callback that updates the cache of execution contexts
+        :param execution_context:
+        :return:
+        """
+        if self.sheerka.save_execution_context:
+            try:
+                self.sheerka.sdp.save_result(execution_context)
+            except Exception as ex:
+                print(f"{CCM['red']}Failed to save execution context. Reason: {ex}{CCM['reset']}")
+                pass
+                # self.log.error(f"Failed to save execution context. Reason: {ex}")
 
-                if e._children:
-                    yield from _yield_result(e._children)
+        self.executions_contexts_cache.put(execution_context.event.get_digest(), execution_context)
+        self.last_execution = execution_context
 
-        return _yield_result([execution_context])
+    def get_last_ret(self, context):
+        """
+        Return the last return value(s)
+        :return:
+        """
+        if self.last_execution:
+            return self.last_execution.values["return_values"]
+
+        event_id = self._get_last_execution_result_event_id_from_db()
+        if event_id is not None:
+            try:
+
+                execution_result = self.sheerka.sdp.load_result(event_id)
+                return execution_result.values["return_values"]
+
+            except FileNotFoundError as ex:
+                context.log_error(f"Digest {event_id} is not found.", self.NAME, ex)
+                return self.sheerka.new(BuiltinConcepts.NOT_FOUND, body={"digest": event_id})
+
+        return self.sheerka.new(BuiltinConcepts.NOT_FOUND, body={"query": "last"})
+
+    def new_concept_created(self, context, concept):
+        self.last_created_concept = concept
+        self.last_created_concept_id = concept.id
+
+        self.sheerka.record_var(context, self.NAME, "last_created_concept_id", concept.id)
+
+    def get_last_created_concept(self, context):
+        if self.last_created_concept:
+            return self.last_created_concept
+
+        if self.last_created_concept_id:
+            self.last_created_concept = self.sheerka.new((None, self.last_created_concept_id))
+            return self.last_created_concept
+
+        return self.sheerka.new(BuiltinConcepts.NOT_FOUND, body={"query": "last_created_concept"})
+
+    def _get_last_execution_result_event_id_from_db(self):
+        start = 0
+        page_size = 2
+        consumed = 0
+        while True:
+            for event in self.sheerka.sdp.load_events(page_size, start):
+                consumed += 1
+                if self.sheerka.sdp.has_result(event.get_digest()):
+                    return event.get_digest()
+
+            if consumed < page_size:
+                break
+
+            if page_size < 100:
+                page_size *= 2
+
+            start += page_size
+            consumed = 0
+
+        return None

src/core/sheerka/services/SheerkaRuleManager.py

@@ -7,7 +7,7 @@ from cache.Cache import Cache
 from core.builtin_concepts import BuiltinConcepts, ReturnValueConcept
 from core.builtin_helpers import parse_unrecognized, only_successful, ensure_rule
 from core.concept import Concept
-from core.global_symbols import RULE_PRECEDENCE_MODIFIED, RULE_COMPARISON_CONTEXT
+from core.global_symbols import EVENT_RULE_PRECEDENCE_MODIFIED, RULE_COMPARISON_CONTEXT
 from core.rule import Rule
 from core.sheerka.services.sheerka_service import BaseService
 from core.tokenizer import Keywords, TokenKind, Token, IterParser
@@ -112,13 +112,14 @@ class FormatAstList(FormatAstNode):
     prefix: str = None
     suffix: str = None
     show_index: bool = False
+    index: object = None
 
     items: object = None
 
     def clone(self, **kwargs):
         return super().clone(
             FormatAstList(self.variable),
-            ("items_prop", "recurse_on", "recursion_depth", "debug", "prefix", "suffix", "show_index", "items"),
+            ("items_prop", "recurse_on", "recursion_depth", "debug", "prefix", "suffix", "show_index", "index", "items"),
             **kwargs)
 
 
@@ -496,10 +497,12 @@ class SheerkaRuleManager(BaseService):
         if is_first_time:
             # add builtin rules if it's the first initialization of Sheerka
             self.init_builtin_rules(context)
-
-        # adds the other rules (when it's not the first time)
-        self.format_rule_cache.populate(lambda: self.sheerka.sdp.list(self.FORMAT_RULE_ENTRY), lambda rule: rule.id)
-        self.exec_rule_cache.populate(lambda: self.sheerka.sdp.list(self.EXEC_RULE_ENTRY), lambda rule: rule.id)
+        else:
+            # adds the other rules (when it's not the first time)
+            self.format_rule_cache.populate(lambda: self.sheerka.sdp.list(self.FORMAT_RULE_ENTRY), lambda rule: rule.id)
+            self.exec_rule_cache.populate(lambda: self.sheerka.sdp.list(self.EXEC_RULE_ENTRY), lambda rule: rule.id)
+            self.format_rule_cache.reset_events()
+            self.exec_rule_cache.reset_events()
 
         # compile all the rules
         for rule_id in self.format_rule_cache:
@@ -508,7 +511,7 @@ class SheerkaRuleManager(BaseService):
         # update rules priorities
         self.update_rules_priorities(context)
 
-        self.sheerka.subscribe(RULE_PRECEDENCE_MODIFIED, self.update_rules_priorities)
+        self.sheerka.subscribe(EVENT_RULE_PRECEDENCE_MODIFIED, self.update_rules_priorities)
 
     def update_rules_priorities(self, context):
         """

src/core/sheerka/services/SheerkaSetsManager.py

@@ -43,6 +43,7 @@ class SheerkaSetsManager(BaseService):
         context.log(f"Setting concept {concept} is a {concept_set}", who=self.NAME)
         core.builtin_helpers.ensure_concept(concept, concept_set)
 
+
         if BuiltinConcepts.ISA in concept.get_metadata().props and concept_set in concept.get_metadata().props[
             BuiltinConcepts.ISA]:
             return self.sheerka.ret(

src/core/simple_debug.py

@@ -0,0 +1,46 @@
+default_debug_name = "*default*"
+debug_activated = set()
+
+
+def my_debug(*args, check_started=None):
+    """
+    Write one line per arg in 'debug.txt'
+    :param args:
+    :param check_started:
+        True : first check if start_debug() was called
+        <name> : first check if start_debug(name) was called
+        list of <names> : first check if start_debug() is called for all names
+    :return:
+    """
+    if check_started and default_debug_name not in debug_activated:
+        return
+
+    if isinstance(check_started, str) and check_started not in debug_activated:
+        return
+
+    if isinstance(check_started, list):
+        for debug_name in check_started:
+            if debug_name not in debug_activated:
+                return
+
+    # with open("debug.txt", "a") as f:
+    #     for arg in args:
+    #         if isinstance(arg, list):
+    #             for item in arg:
+    #                 f.write(f"{item}\n")
+    #         else:
+    #             f.write(f"{arg}\n")
+
+
+def start_debug(debug_name=default_debug_name, msg=None):
+    debug_activated.add(debug_name)
+    if msg:
+        with open("debug.txt", "a") as f:
+            f.write(f"{msg}\n")
+
+
+def stop_debug(debug_name=default_debug_name, msg=None):
+    if msg:
+        with open("debug.txt", "a") as f:
+            f.write(f"{msg}\n")
+    debug_activated.remove(debug_name)

src/core/utils.py

@@ -1,16 +1,15 @@
 import ast
 import importlib
 import inspect
+import os
 import pkgutil
+from copy import deepcopy
 
 from cache.Cache import Cache
 from core.ast_helpers import ast_to_props
 from core.tokenizer import TokenKind, Tokenizer
 from pyparsing import *
 
-default_debug_name = "*default*"
-debug_activated = set()
-
 COLORS = {
     "black",
     "red",
@@ -43,55 +42,17 @@ integer = Word(nums)
 escapeSeq = Combine(ESC + '[' + Optional(delimitedList(integer, ';')) +
                     oneOf(list(alphas)))
 
+try:
+    CONSOLE_ROWS, CONSOLE_COLUMNS = os.popen('stty size', 'r').read().split()
+    CONSOLE_ROWS, CONSOLE_COLUMNS = int(CONSOLE_ROWS), int(CONSOLE_COLUMNS)
+except ValueError:
+    CONSOLE_ROWS, CONSOLE_COLUMNS = 50, 80
+
 
 def no_color_str(text):
     return Suppress(escapeSeq).transformString(str(text))
 
 
-def my_debug(*args, check_started=None):
-    """
-    Write one line per arg in 'debug.txt'
-    :param args:
-    :param check_started:
-        True : first check if start_debug() was called
-        <name> : first check if start_debug(name) was called
-        list of <names> : first check if start_debug() is called for all names
-    :return:
-    """
-    if check_started and default_debug_name not in debug_activated:
-        return
-
-    if isinstance(check_started, str) and check_started not in debug_activated:
-        return
-
-    if isinstance(check_started, list):
-        for debug_name in check_started:
-            if debug_name not in debug_activated:
-                return
-
-    with open("debug.txt", "a") as f:
-        for arg in args:
-            if isinstance(arg, list):
-                for item in arg:
-                    f.write(f"{item}\n")
-            else:
-                f.write(f"{arg}\n")
-
-
-def start_debug(debug_name=default_debug_name, msg=None):
-    debug_activated.add(debug_name)
-    if msg:
-        with open("debug.txt", "a") as f:
-            f.write(f"{msg}\n")
-
-
-def stop_debug(debug_name=default_debug_name, msg=None):
-    if msg:
-        with open("debug.txt", "a") as f:
-            f.write(f"{msg}\n")
-    debug_activated.remove(debug_name)
-
-
 def sysarg_to_string(argv):
     """
     Transform a list of strings into a single string
@@ -606,13 +567,17 @@ def tokens_index(tokens, sub_tokens, skip=0):
     raise ValueError(f"sub tokens '{sub_tokens}' not found")
 
 
-def as_bag(obj):
+def as_bag(obj, forced_properties=None):
     """
     Get the properties of an object (static and dynamic)
     :param obj:
+    :param forced_properties:
     :return:
     """
-    if hasattr(obj, "as_bag"):
+
+    if forced_properties:
+        bag = {p: getattr(obj, p) for p in forced_properties}
+    elif hasattr(obj, "as_bag"):
         bag = obj.as_bag()
     else:
         bag = {} if type(obj) in PRIMITIVES_TYPES else {prop: getattr(obj, prop)
@@ -726,3 +691,77 @@ def dump_ast(node):
     for to_remove in [", ctx=Load()", ", kind=None", ", type_ignores=[]"]:
         dump = dump.replace(to_remove, "")
     return dump
+
+
+def sheerka_deepcopy(obj):
+    """
+    Internal implementation of deepcopy that can handle Concept circular references
+    :param obj:
+    :return:
+    """
+    already_seen = {}
+
+    def copy_concept(c):
+        id_c = id(c)
+        if id_c in already_seen:
+            ref = already_seen[id_c]
+            if ref == '_##_REF_##_':
+                raise Exception("Circular Ref not managed yet!")
+            else:
+                return ref
+
+        already_seen[id_c] = '_##_REF_##_'
+
+        cls = type(c)
+        instance = cls()
+        # update the metadata
+        for prop_name, prop_value in vars(c.get_metadata()).items():
+            if prop_name != "props":
+                setattr(instance.get_metadata(), prop_name, prop_value)
+            else:
+                setattr(instance.get_metadata(), prop_name, sheerka_deepcopy(prop_value))
+
+        # update the values
+        for prop_name, prop_value in c.values().items():
+            setattr(instance, prop_name, prop_value)
+
+        already_seen[id_c] = instance
+        return instance
+
+    from core.concept import Concept
+    if isinstance(obj, dict):
+        res = {sheerka_deepcopy(k): sheerka_deepcopy(v) for k, v in obj.items()}
+        return res
+    elif isinstance(obj, list):
+        return [sheerka_deepcopy(item) for item in obj]
+    elif isinstance(obj, set):
+        return {sheerka_deepcopy(item) for item in obj}
+    elif isinstance(obj, tuple):
+        return tuple((sheerka_deepcopy(item) for item in obj))
+    elif isinstance(obj, Concept):
+        return copy_concept(obj)
+    else:
+        return deepcopy(obj)
+
+
+def escape_str(x):
+    """
+    Returns a string representation that look like what would produce a debugger
+    :param x:
+    :return:
+    """
+    if isinstance(x, str):
+        return f"'{x}'"
+    return x
+
+
+class NextIdManager:
+    """
+    solely return the next integer
+    """
+    def __init__(self):
+        self.id = -1
+
+    def get_next_id(self):
+        self.id += 1
+        return self.id

src/evaluators/AddToMemoryEvaluator.py

@@ -23,5 +23,11 @@ class AddToMemoryEvaluator(OneReturnValueEvaluator):
         return len(context.sheerka.services[SheerkaMemory.NAME].registration) > 0
 
     def eval(self, context, return_value):
+        if context.sheerka.during_initialisation:
+            from core.sheerka.services.SheerkaMemory import SheerkaMemory
+            service = context.sheerka.services[SheerkaMemory.NAME]
+            service.registration.clear()
+            return None
+
         context.sheerka.add_registered_objects(context)
         return None  # no need to have a second pass

src/evaluators/MutipleSameSuccessEvaluator.py

@@ -36,10 +36,10 @@ class MultipleSameSuccessEvaluator(AllReturnValuesEvaluator):
                 to_process = True
                 self.eaten.append(ret)
             elif ret.who.startswith(BaseEvaluator.PREFIX):
+                self.eaten.append(ret)
                 if ret.status:
                     nb_successful_evaluators += 1
                     self.success.append(ret)
-                    self.eaten.append(ret)
             elif ret.who.startswith(BaseParser.PREFIX):
                 self.eaten.append(ret)
                 if ret.status:
@@ -63,7 +63,7 @@ class MultipleSameSuccessEvaluator(AllReturnValuesEvaluator):
         # I gave a random order to the other
         #
         # I guess that we need a proper algorithm to elect which return value to use if they have the same result
-        # I guts feeling is that, it will depend on the intent of the user
+        # My guts feeling is that, it will depend on the intent of the user
         # So it depends on the context
 
         # try to return a concept if possible

src/out/AsStrVisitor.py

@@ -1,16 +1,17 @@
 import re
 
 from core.sheerka.services.SheerkaRuleManager import FormatAstNode
-from core.utils import CONSOLE_COLORS_MAP as CCM, no_color_str
+from core.utils import CONSOLE_COLORS_MAP as CCM, no_color_str, CONSOLE_COLUMNS
 from out.OutVisitor import OutVisitor
 
-get_start = re.compile(r"^([\(\{\[]\w*)")
+get_starting_brace_bracket_parent = re.compile(r"^([\(\{\[]\w*)")
 
 
 class AsStrVisitor(OutVisitor):
 
-    def __init__(self, expand=False):
+    def __init__(self, expand=None, width=None):
         self.expand = expand
+        self.width = width
 
     def visit_FormatAstRawText(self, format_ast):
         return str(format_ast.text)
@@ -58,6 +59,7 @@ class AsStrVisitor(OutVisitor):
     def visit_FormatAstDict(self, format_ast):
         first = True
         result = ""
+        expand = self.expand or False
 
         keys_values = []
         max_len = 0
@@ -71,23 +73,36 @@ class AsStrVisitor(OutVisitor):
         if format_ast.prefix:
             result += format_ast.prefix
 
-        sep = ",\n" if self.expand else ", " if format_ast.debug else "\n"
+        sep = ",\n" if expand else ", " if format_ast.debug else "\n"
 
         for i, (k, v) in enumerate(format_ast.items):
             start = "" if first else sep
 
-            key = f"{keys_values[i]:<{max_len}}" if (self.expand or not format_ast.debug) else keys_values[i]
+            key = f"{keys_values[i]:<{max_len}}" if (expand or not format_ast.debug) else keys_values[i]
             colon = ": "
             indent = len(no_color_str(key)) + len(colon)
+
             value = self.visit(v)
-            if self.expand:
-                if m := get_start.match(no_color_str(value)):
+            if self.debug_activated(v) and self.width and len(no_color_str(value)) >= self.width:
+                value = forced_expanded_visitor.visit(v)
+                expand = True
+
+            if expand:
+                if m := get_starting_brace_bracket_parent.match(no_color_str(value)):
                     indent += len(m.group(1))
                 value = value.replace("\n", "\n" + " " * indent)
             first = False
 
             result += start + key + colon + value
+            expand = self.expand or False # reset expand
 
         if format_ast.suffix:
             result += format_ast.suffix
         return result
+
+    @staticmethod
+    def debug_activated(node):
+        return isinstance(node, FormatAstNode) and hasattr(node, "debug") and node.debug
+
+
+forced_expanded_visitor = AsStrVisitor(expand=True, width=CONSOLE_COLUMNS)

src/out/ConsoleVisistor.py

@@ -1,3 +1,4 @@
+from core.utils import CONSOLE_COLUMNS
 from out.AsStrVisitor import AsStrVisitor
 
 
@@ -6,7 +7,7 @@ class ConsoleVisitor(AsStrVisitor):
     Prints to the console
     """
 
-    def __init__(self, expand_mode="auto"):
+    def __init__(self, expand_mode="auto", console_width=None):
         """
         expand_mode:
         auto: not the first dict, the sub ones depends of the width
@@ -18,6 +19,7 @@ class ConsoleVisitor(AsStrVisitor):
         super().__init__()
         self.out = print
         self.expand_mode = expand_mode
+        self.console_width = console_width or CONSOLE_COLUMNS
 
     def visit_FormatAstRawText(self, format_ast):
         self.out(super().visit_FormatAstRawText(format_ast))
@@ -32,11 +34,11 @@ class ConsoleVisitor(AsStrVisitor):
         self.out(super().visit_FormatAstColor(format_ast))
 
     def visit_FormatAstSequence(self, format_ast):
-        visitor = AsStrVisitor()
+        visitor = AsStrVisitor(width=self.console_width)
         self.out(visitor.visit_FormatAstSequence(format_ast))
 
     def visit_FormatAstList(self, format_ast):
-        visitor = AsStrVisitor()
+        visitor = AsStrVisitor(width=self.console_width)
         res = visitor.visit_FormatAstList(format_ast)
         self.out(res)
 
@@ -46,6 +48,6 @@ class ConsoleVisitor(AsStrVisitor):
         else:
             expand = False
 
-        visitor = AsStrVisitor(expand)
+        visitor = AsStrVisitor(expand, width=self.console_width)
         res = visitor.visit_FormatAstDict(format_ast)
         self.out(res)

src/out/DeveloperVisitor.py

@@ -100,10 +100,20 @@ class DeveloperVisitor:
             for i, item in enumerate(items):
                 bag["__item"] = item
                 sub_visitor = DeveloperVisitor(self.sheerka_out, self.debugger, set(), self.list_recursion_depth)
-                result.append(sub_visitor.visit(context, FormatAstVariable("__item",
-                                                                           debug=format_ast.debug,
-                                                                           value=item,
-                                                                           index=i), bag))
+                to_visit = FormatAstDict("__item", debug=True, prefix='{', suffix='}') if isinstance(item, dict) else \
+                    FormatAstList("__item", debug=True, index=i, prefix='[', suffix=']') if isinstance(item, list) else \
+                        FormatAstVariable("__item", value=item, index=i, debug=format_ast.debug)
+                result.append(sub_visitor.visit(context, to_visit, bag))
+                # if isinstance(item, list):
+                #     format_ast_item = sub_visitor.visit(context,
+                #                                         FormatAstList("__item", debug=format_ast.debug, index=i),
+                #                                         bag)
+                # else:
+                #     format_ast_item = sub_visitor.visit(context,
+                #                                         FormatAstVariable("__item", debug=format_ast.debug, value=item,
+                #                                                           index=i),
+                #                                         bag)
+                # result.append(format_ast_item)
 
                 # recursion management
                 recursion_depth, recurse_on = self.get_recurse_info(item, recursion_depth, recurse_on)

src/parsers/BaseNodeParser.py

@@ -873,7 +873,7 @@ class BaseNodeParser(BaseParser):
                 if not to_keep(concept):
                     continue
 
-                concept = to_map(self, concept) if to_map else concept
+                concept = to_map(concept, self, self.sheerka) if to_map else concept
                 result.append(concept)
 
             return core.utils.make_unique(result + custom_concepts,

src/parsers/BnfNodeParser.py

@@ -1296,12 +1296,14 @@ class BnfNodeParser(BaseNodeParser):
                 if not concepts:
                     if debugger.is_enabled():
                         debugger.debug_log(debug_prefix + ", no concept found.")
+
                     for concept_parser in not_locked:
                         concept_parser.eat_unrecognized(token)
                     continue
 
                 if debugger.is_enabled():
                     debugger.debug_log(debug_prefix + f", concept(s) found={concepts}")
+
                 if len(concepts) == 1:
                     for concept_parser in not_locked:
                         concept_parser.eat_concept(concepts[0], token)

src/parsers/PythonParser.py

@@ -53,9 +53,6 @@ class PythonNode(Node):
             self.compiled = compile(self.ast_, "<string>", "eval")
         return self.compiled
 
-    # def __repr__(self):
-    #     return "PythonNode(parser_input='" + self.parser_input + "', ast=" + self.get_dump(self.ast_) + ")"
-
     def __repr__(self):
         ast_type = "expr" if isinstance(self.ast_, ast.Expression) else "module"
         return "PythonNode(" + ast_type + "='" + self.source + "')"

src/parsers/SyaNodeParser.py

@@ -11,7 +11,7 @@ from core.global_symbols import CONCEPT_COMPARISON_CONTEXT
 from core.sheerka.services.SheerkaComparisonManager import SheerkaComparisonManager
 from core.sheerka.services.SheerkaExecute import ParserInput
 from core.tokenizer import Token, TokenKind, Tokenizer
-from core.utils import get_n_clones
+from core.utils import get_n_clones, get_text_from_tokens, NextIdManager
 from parsers.BaseNodeParser import UnrecognizedTokensNode, ConceptNode, SourceCodeNode, SyaAssociativity, \
     SourceCodeWithConceptNode, BaseNodeParser
 from parsers.BaseParser import ErrorNode
@@ -25,6 +25,7 @@ DEBUG_PUSH_UNREC = "PUSH_UNREC"
 DEBUG_POP = "POP"
 DEBUG_EAT = "EAT"
 DEBUG_RECOG = "RECOG"
+DEBUG_CAN_POP = "CAN_POP"
 
 
 @dataclass()
@@ -42,12 +43,13 @@ class DebugInfo:
     token: Token = None  # current token
     concept: Concept = None  # current concept if ay
     action: str = None  # action taken
+    level: str = None
 
     def __repr__(self):
         token_repr = self.token.repr_value if isinstance(self.token, Token) else self.token
         msg = f"{self.pos:3}:{token_repr}" if self.pos != -1 else "  _:"
         if self.concept:
-            msg += f"({self.concept})"
+            msg += f" {self.concept.short_repr()}"
         return msg + f" => {self.action}"
 
 
@@ -118,6 +120,36 @@ class SyaConceptDef:
     precedence: int = SheerkaComparisonManager.DEFAULT_COMPARISON_VALUE
     associativity: SyaAssociativity = SyaAssociativity.Right
 
+    @staticmethod
+    def get_sya_concept_def(concept, parser, sheerka):
+        sya_concept_def = SyaConceptDef(concept)
+
+        # first, try to look in the parser
+        # it is where to find the data during the unit tests
+        if parser and concept.id in parser.sya_definitions:
+            # Manage when precedence and associativity are given in the unit tests
+            sya_def = parser.sya_definitions.get(concept.id)
+            if sya_def[0] is not None:
+                sya_concept_def.precedence = sya_def[0]
+            if sya_def[1] is not None:
+                sya_concept_def.associativity = sya_def[1]
+
+        # otherwise, use sheerka
+        if sheerka:
+            concept_weight = parser.sheerka.get_concepts_weights(BuiltinConcepts.PRECEDENCE, CONCEPT_COMPARISON_CONTEXT)
+            if concept.str_id in concept_weight:
+                sya_concept_def.precedence = concept_weight[concept.str_id]
+
+        # in the case of Sheerka, the associativity is managed by the concept itself
+        # There is no conflict with the settings of the unit test, as I don't use the props in the unit tests
+        if associativity := concept.get_prop(BuiltinConcepts.ASSOCIATIVITY):
+            sya_concept_def.associativity = SyaAssociativity(associativity)
+
+        return sya_concept_def
+
+    def short_repr(self):
+        return f"({self.concept}, prio={self.precedence}, assoc={self.associativity})"
+
 
 @dataclass()
 class SyaConceptParserHelper:
@@ -248,9 +280,19 @@ class SyaConceptParserHelper:
 
 
 class InFixToPostFix:
-    def __init__(self, context, debug_enabled=False):
+    def __init__(self, context, next_id_manager, debugger=None):
         self.context = context
-        self.debug_enabled = debug_enabled
+
+        self.next_id_manager = next_id_manager
+        self.id = self.next_id_manager.get_next_id()
+
+        self.debugger = debugger
+        if debugger:
+            self.debug_enabled = debugger.is_enabled()
+            self.enabled_debug_levels = debugger.get_enabled_vars()
+        else:
+            self.debug_enabled = False
+            self.enabled_debug_levels = None
 
         self.is_locked = False  # when locked, cannot process input
 
@@ -284,9 +326,15 @@ class InFixToPostFix:
 
     def _add_error(self, error):
         if self.debug_enabled:
-            self.debug.append(DebugInfo(action=f"=> ERROR {error}"))
+            self._add_debug(DebugInfo(action=f"=> ERROR {error}"))
         self.errors.append(error)
 
+    def _add_debug(self, debug_info: DebugInfo):
+        if debug_info.level is None or (self.enabled_debug_levels and
+                                        (f"#{self.id}.{debug_info.level}" in self.enabled_debug_levels or
+                                        "*" in self.enabled_debug_levels)):
+            self.debug.append(debug_info)
+
     def _is_lpar(self, token):
         """
         True if the token is a left parenthesis '('
@@ -337,10 +385,10 @@ class InFixToPostFix:
             else:
                 item.error = f"token '{item.expected[0].strip_quote}' not found"
             if self.debug_enabled:
-                self.debug.append(DebugInfo(action=f"ERROR {item.error}"))
+                self._add_debug(DebugInfo(action=f"ERROR {item.error}"))
 
         if self.debug_enabled:
-            self.debug.append(DebugInfo(action=f"{DEBUG_POP} {item}"))
+            self._add_debug(DebugInfo(action=f"{DEBUG_POP} {item}"))
         if isinstance(item, SyaConceptParserHelper) and item.potential_pos != -1:
             self.out.insert(item.potential_pos, item)
         else:
@@ -402,6 +450,11 @@ class InFixToPostFix:
             self.debug.pop()
 
     def _debug_nodes(self, nodes_sequences):
+        """
+        Returns a debug representation of a sequence of LexerNodes
+        :param nodes_sequences:
+        :return:
+        """
         res = "["
         first = True
         for sequence in nodes_sequences:
@@ -520,7 +573,7 @@ class InFixToPostFix:
                 # There are more than one solution found
                 # In the case, we create a new InfixToPostfix for each new possibility
                 if self.debug_enabled:
-                    self.debug.append(DebugInfo(action=f"{DEBUG_RECOG} {self._debug_nodes(nodes_sequences)}"))
+                    self._add_debug(DebugInfo(action=f"{DEBUG_RECOG} {self._debug_nodes(nodes_sequences)}"))
                 if len(nodes_sequences) > 1:
                     for node_sequence in nodes_sequences[1:]:
                         clone = self.clone()
@@ -599,33 +652,52 @@ class InFixToPostFix:
         self.stack.pop()
         self._put_to_out(item)
 
-    def i_can_pop(self, concept_node):
+    def i_can_pop(self, sya_parser_helper):
         """
         Validate the Shunting Yard Algorithm conditions to pop out from the stack
         Note that it's a custom implementation as I need to manage UnrecognizedTokensNode
-        :param concept_node:
+        :param sya_parser_helper:
         :return:
         """
         if len(self.stack) == 0:
+            if self.debug_enabled:
+                self._add_debug(DebugInfo(action=f"No stack. {DEBUG_CAN_POP} false.", level="can_pop"))
             return False
 
         stack_head = self.stack[-1]
 
         if not isinstance(stack_head, SyaConceptParserHelper):  # mostly left parenthesis
+            if self.debug_enabled:
+                self._add_debug(DebugInfo(action=f"No concept. {DEBUG_CAN_POP} false.", level="can_pop"))
             return False
 
-        current = concept_node.concept
+        current = sya_parser_helper.concept
         stack = stack_head.concept
 
         if stack.associativity == SyaAssociativity.No and current.associativity == SyaAssociativity.No:
-            self._add_error(NoneAssociativeSequenceErrorNode(current.concept, stack_head.start, concept_node.start))
+            self._add_error(
+                NoneAssociativeSequenceErrorNode(current.concept, stack_head.start, sya_parser_helper.start))
 
         if current.associativity == SyaAssociativity.Left and current.precedence <= stack.precedence:
+            if self.debug_enabled:
+                current_debug = f"{current.concept.id}({current.precedence})"
+                stack_debug = f"{stack.concept.id}({stack.precedence})"
+                self._add_debug(
+                    DebugInfo(action=f"assoc=Left and {current_debug} <= {stack_debug}. {DEBUG_CAN_POP} True.",
+                              level="can_pop"))
             return True
 
         if current.associativity == SyaAssociativity.Right and current.precedence < stack.precedence:
+            if self.debug_enabled:
+                current_debug = f"{current.concept.id}({current.precedence})"
+                stack_debug = f"{stack.concept.id}({stack.precedence})"
+                self._add_debug(
+                    DebugInfo(action=f"assoc=Right and {current_debug} < {stack_debug}. {DEBUG_CAN_POP} True.",
+                              level="can_pop"))
             return True
 
+        if self.debug_enabled:
+            self._add_debug(DebugInfo(action=f"No rule. {DEBUG_CAN_POP} False.", level="can_pop"))
         return False
 
     def handle_expected_token(self, token, pos):
@@ -693,7 +765,7 @@ class InFixToPostFix:
 
                 current_concept.end = pos
                 if self.debug_enabled:
-                    self.debug.append(DebugInfo(pos, token, None, "??"))
+                    self._add_debug(DebugInfo(pos, token, None, "??"))
                 self.manage_unrecognized()
                 # manage that some clones may have been forked
                 for forked in self.forked:
@@ -755,7 +827,7 @@ class InFixToPostFix:
 
         if self.parsing_function:
             if self.debug_enabled:
-                self.debug.append(DebugInfo(pos, token, None, DEBUG_PUSH_UNREC))
+                self._add_debug(DebugInfo(pos, token, None, DEBUG_PUSH_UNREC))
 
             self.unrecognized_tokens.add_token(token, pos)
 
@@ -790,13 +862,13 @@ class InFixToPostFix:
             # if the token 'bar' is found, it has to be considered as part of the concept foo
             if self.debug_enabled:
                 self._remove_debug_info_if_needed()
-                self.debug.append(DebugInfo(pos, token, None, DEBUG_EAT))
+                self._add_debug(DebugInfo(pos, token, None, DEBUG_EAT))
             return True
 
         elif self._is_lpar(token):
 
             if self.debug_enabled:
-                self.debug.append(DebugInfo(pos, token, None, DEBUG_PUSH_UNREC))
+                self._add_debug(DebugInfo(pos, token, None, DEBUG_PUSH_UNREC))
 
             if self.unrecognized_tokens.is_empty() or self.unrecognized_tokens.is_whitespace():
 
@@ -865,7 +937,7 @@ class InFixToPostFix:
 
         elif self._is_rpar(token):
             if self.debug_enabled:
-                self.debug.append(DebugInfo(pos, token, None, DEBUG_EAT))
+                self._add_debug(DebugInfo(pos, token, None, DEBUG_EAT))
 
             # first, remove what was in the buffer
             self.manage_unrecognized()
@@ -933,7 +1005,7 @@ class InFixToPostFix:
 
         if first_pass:
             if self.debug_enabled:
-                self.debug.append(DebugInfo(pos, token, sya_concept_def, "??"))
+                self._add_debug(DebugInfo(pos, token, sya_concept_def, "??"))
 
             if self.unrecognized_tokens.last_token_type() == TokenKind.WHITESPACE:
                 parser_helper.remember_whitespace = self.unrecognized_tokens.tokens[-1]
@@ -970,7 +1042,7 @@ class InFixToPostFix:
             else:
                 if self.debug_enabled:
                     self._remove_debug_info_if_needed()
-                    self.debug.append(DebugInfo(pos, token, sya_concept_def, DEBUG_PUSH))
+                    self._add_debug(DebugInfo(pos, token, sya_concept_def, DEBUG_PUSH))
                 self.stack.append(parser_helper)
                 self.manage_parameters_when_new_concept(parser_helper)
 
@@ -985,7 +1057,7 @@ class InFixToPostFix:
             return
 
         if self.debug_enabled:
-            self.debug.append(DebugInfo(pos, token, None, DEBUG_PUSH_UNREC))
+            self._add_debug(DebugInfo(pos, token, None, DEBUG_PUSH_UNREC))
 
         self.unrecognized_tokens.add_token(token, pos)
 
@@ -1005,7 +1077,7 @@ class InFixToPostFix:
             return  # no need to pop the buffer, as no concept is found
 
         if self.debug_enabled:
-            self.debug.append(DebugInfo(pos, "<EOF>", None, "??"))
+            self._add_debug(DebugInfo(pos, "<EOF>", None, "??"))
 
         while len(self.stack) > 0:
             parser_helper = self.stack[-1]
@@ -1036,7 +1108,7 @@ class InFixToPostFix:
             forked.finalize(pos)
 
     def clone(self):
-        clone = InFixToPostFix(self.context, self.debug_enabled)
+        clone = InFixToPostFix(self.context, self.next_id_manager, self.debugger)
         clone.is_locked = self.is_locked
         clone.out = self.out[:]
         clone.stack = [i.clone() if hasattr(i, "clone") else i for i in self.stack]
@@ -1054,6 +1126,7 @@ class PostFixToItem:
     start: int
     end: int
     has_unrecognized: bool
+    source: str
 
 
 class SyaNodeParser(BaseNodeParser):
@@ -1069,14 +1142,6 @@ class SyaNodeParser(BaseNodeParser):
             self.concepts_by_first_keyword = {}
             self.sya_definitions = {}
 
-        # self.token = None
-        # self.pos = -1
-        # self.tokens = None
-        #
-        # self.context: ExecutionContext = None
-        # self.text = None
-        # self.sheerka = None
-
     def init_from_concepts(self, context, concepts, **kwargs):
         super().init_from_concepts(context, concepts)
 
@@ -1093,27 +1158,8 @@ class SyaNodeParser(BaseNodeParser):
         """
         # We only concepts that has parameter (refuse atoms)
         # Bnf definitions are not supposed to be managed by this parser either
-        return len(concept.get_metadata().variables) > 0 and concept.get_metadata().definition_type != DEFINITION_TYPE_BNF
-
-    def _get_sya_concept_def(self, parser, concept):
-        sya_concept_def = SyaConceptDef(concept)
-        if concept.id in parser.sya_definitions:
-            # Manage when precedence and associativity are given in the unit tests
-            sya_def = parser.sya_definitions.get(concept.id)
-            if sya_def[0] is not None:
-                sya_concept_def.precedence = sya_def[0]
-            if sya_def[1] is not None:
-                sya_concept_def.associativity = sya_def[1]
-
-        if parser.sheerka:
-            concept_weight = parser.sheerka.get_concepts_weights(BuiltinConcepts.PRECEDENCE, CONCEPT_COMPARISON_CONTEXT)
-            if concept.str_id in concept_weight:
-                sya_concept_def.precedence = concept_weight[concept.str_id]
-
-        if associativity := concept.get_prop(BuiltinConcepts.ASSOCIATIVITY):
-            sya_concept_def.associativity = SyaAssociativity(associativity)
-
-        return sya_concept_def
+        return len(
+            concept.get_metadata().variables) > 0 and concept.get_metadata().definition_type != DEFINITION_TYPE_BNF
 
     def infix_to_postfix(self, context, parser_input: ParserInput):
         """
@@ -1126,6 +1172,9 @@ class SyaNodeParser(BaseNodeParser):
         if not self.reset_parser(context, parser_input):
             return None
 
+        debugger = context.get_debugger(self.NAME, "parse")
+        debugger.debug_entering(source=self.parser_input.as_text())
+
         forked = []
 
         def _add_forked_to_res():
@@ -1138,16 +1187,21 @@ class SyaNodeParser(BaseNodeParser):
                 res.extend(forked)
                 forked.clear()
 
-        res = [InFixToPostFix(context, context.debug_enabled)]
+        res = [InFixToPostFix(context, NextIdManager(), debugger)]
         while self.parser_input.next_token(False):
             for infix_to_postfix in res:
                 infix_to_postfix.reset()
 
             token = self.parser_input.token
+            if debugger.is_enabled():
+                debug_prefix = f"pos={self.parser_input.pos}, {token=}, {len(res)} parser(s)"
 
             try:
                 if token.type in (TokenKind.LPAR, TokenKind.RPAR):
                     # little optim, no need to lock, unlock or get the concept when parenthesis
+                    if debugger.is_enabled():
+                        debugger.debug_log(debug_prefix + ", eat token.")
+
                     for infix_to_postfix in res:
                         infix_to_postfix.eat_token(token, self.parser_input.pos)
                     continue
@@ -1156,21 +1210,34 @@ class SyaNodeParser(BaseNodeParser):
                     if infix_to_postfix.eat_token(token, self.parser_input.pos):
                         infix_to_postfix.lock()
 
-                concepts = self.get_concepts(token, self._is_eligible, to_map=self._get_sya_concept_def)
-                if not concepts:
+                nb_locked = len([itp for itp in res if itp.is_locked])
+                if nb_locked == len(res):
+                    if debugger.is_enabled():
+                        debugger.debug_log(debug_prefix + f", all parsers are locked")
+                    continue
+
+                concepts_def = self.get_concepts(token, self._is_eligible, to_map=SyaConceptDef.get_sya_concept_def)
+                if not concepts_def:
+                    if debugger.is_enabled():
+                        debugger.debug_log(debug_prefix + f", no concept found")
+
                     for infix_to_postfix in res:
                         infix_to_postfix.eat_unrecognized(token, self.parser_input.pos)
                     continue
 
-                if len(concepts) == 1:
+                if debugger.is_enabled():
+                    found = [cd.short_repr() for cd in concepts_def]
+                    debugger.debug_log(debug_prefix + f", concept(s) found={found}")
+
+                if len(concepts_def) == 1:
                     for infix_to_postfix in res:
-                        infix_to_postfix.eat_concept(concepts[0], token, self.parser_input.pos)
+                        infix_to_postfix.eat_concept(concepts_def[0], token, self.parser_input.pos)
                     continue
 
                 # make the cartesian product
                 temp_res = []
                 for infix_to_postfix in res:
-                    for concept in concepts:
+                    for concept in concepts_def:
                         clone = infix_to_postfix.clone()
                         temp_res.append(clone)
                         clone.eat_concept(concept, token, self.parser_input.pos)
@@ -1185,13 +1252,13 @@ class SyaNodeParser(BaseNodeParser):
             infix_to_postfix.finalize(self.parser_input.pos)
         _add_forked_to_res()
 
-        if context.debug_enabled:
-            context.debug(self.name, "infix_to_postfix", None, f"Parsing {parser_input}")
-            context.debug(self.name, "infix_to_postfix", "nb_found", f"{len(res)} InfixToPostFix(s) found")
-            for i, r in enumerate(res):
-                context.debug(self.name, "infix_to_postfix", "infix_to_postfix", f"#{i}")
+        if debugger.is_enabled():
+            for r in res:
                 for line in r.debug:
-                    context.debug(self.name, "infix_to_postfix", "infix_to_postfix", line)
+                    if line.level:
+                        debugger.debug_var(f"#{r.id}.{line.level}", line)
+                    else:
+                        debugger.debug_var(f"#{r.id}", line)
 
         return res
 
@@ -1222,6 +1289,7 @@ class SyaNodeParser(BaseNodeParser):
         end = item.end
         has_unrecognized = False
         concept = sheerka.new_from_template(item.concept, item.concept.key)
+        concept_metadata = []
         for param_index in reversed(range(len(concept.get_metadata().variables))):
             inner_item = self.postfix_to_item(sheerka, postfixed)
             if inner_item.start < start:
@@ -1237,8 +1305,20 @@ class SyaNodeParser(BaseNodeParser):
                     inner_item
 
             concept.get_compiled()[param_name] = param_value
+            concept_metadata.append((param_name, inner_item.source))
 
-        return PostFixToItem(concept, start, end, has_unrecognized)
+        # update the metadata
+        concept_metadata.reverse()
+
+        # ---- Sanity check. To remove at some point
+        assert len(concept_metadata) == len(concept.get_metadata().variables)
+        for meta_orig, meta_new in zip(concept.get_metadata().variables, concept_metadata):
+            assert meta_orig[0] == meta_new[0]
+        # ---- Sanity check. To remove at some point
+        concept.get_metadata().variables = concept_metadata
+
+        source = get_text_from_tokens(self.parser_input.tokens[start:end + 1])
+        return PostFixToItem(concept, start, end, has_unrecognized, source)
 
     def parse(self, context, parser_input: ParserInput):
         """

src/sdp/sheerkaDataProvider.py

@@ -232,6 +232,7 @@ class SheerkaDataProvider:
         """
 
         with self.lock:
+            self.log.debug(f"getting {entry=}, {key=}, {default=}, {load_origin=}")
             if entry not in self.state.data:
                 return default
 
@@ -464,6 +465,8 @@ class SheerkaDataProvider:
 
     def load_ref_if_needed_ex(self, item, load_origin):
         """
+        New version of the function.
+        The old one must be replaced at some point
         Make sure we return the real object, even inside a collection
         :param item:
         :param load_origin:

src/sdp/sheerkaSerializer.py

@@ -109,6 +109,7 @@ class Serializer:
             raise TypeError(f"Don't know how serializer name={header[0]}, version={header[1]}")
 
         serializer = serializers[0]
+        self.log.debug(f"deserializing using '{serializer}'")
         return serializer.load(stream, context)
 
 
@@ -298,7 +299,7 @@ class MemoryObjectSerializer(SheerkaPickleSerializer):
     CLASS_NAME = "core.sheerka.services.SheerkaMemory.MemoryObject"
 
     def __init__(self):
-        super().__init__(lambda obj: get_full_qualified_name(obj) == self.CLASS_NAME, "R", 1)
+        super().__init__(lambda obj: get_full_qualified_name(obj) == self.CLASS_NAME, "M", 1)
 
 
 class RuleSerializer(SheerkaPickleSerializer):

src/sheerkapickle/SheerkaPickler.py

@@ -4,11 +4,15 @@ from logging import Logger
 import core.utils
 from core.concept import Concept, NotInitialized
 from core.sheerka.services.SheerkaExecute import ParserInput
+from core.simple_debug import my_debug
 from sheerkapickle import utils, tags, handlers
 
 
 def encode(sheerka, obj):
-    return json.dumps(SheerkaPickler(sheerka).flatten(obj))
+    pickler = SheerkaPickler(sheerka)
+    flatten = pickler.flatten(obj)
+    my_debug(f"{obj} ids={len(pickler.ids)}, objs={len(pickler.objs)}")
+    return json.dumps(flatten)
 
 
 class ToReduce:

tests/core/test_SheerkaAdmin.py

@@ -7,59 +7,59 @@ class TestSheerkaAdmin(TestUsingMemoryBasedSheerka):
     def test_i_can_get_last_ret(self):
         pass
 
-    def test_i_can_get_last_error_ret(self):
-        sheerka, context = self.init_concepts()
-
-        # nothing in history
-        last_error_ret = sheerka.last_error_ret(context)
-        assert sheerka.isinstance(last_error_ret, BuiltinConcepts.RETURN_VALUE)
-        assert not last_error_ret.status
-        assert sheerka.isinstance(last_error_ret.body, BuiltinConcepts.NOT_FOUND)
-
-        # only success in history
-        r_success = sheerka.ret("Test", True, "success")
-        sheerka.last_return_values.append([r_success])
-        last_error_ret = sheerka.last_error_ret(context)
-        assert sheerka.isinstance(last_error_ret, BuiltinConcepts.RETURN_VALUE)
-        assert not last_error_ret.status
-        assert sheerka.isinstance(last_error_ret.body, BuiltinConcepts.NOT_FOUND)
-
-        # at least on error
-        r_failure = sheerka.ret("Test", False, "failure")
-        sheerka.last_return_values.append([r_failure])
-        assert sheerka.last_error_ret(context) == r_failure
-
-        # add another success and make sure we get the same error
-        sheerka.last_return_values.append([r_success])
-        assert sheerka.last_error_ret(context) == r_failure
-
-        # and I only get the last failure
-        r_failure2 = sheerka.ret("Test", False, "another failure")
-        sheerka.last_return_values.append([r_failure2])
-        assert sheerka.last_error_ret(context) == r_failure2
-
-        # but I still can get the previous error
-        assert sheerka.last_error_ret(context, -2) == r_failure
-
-    def test_i_can_get_last_error_ret_when_only_one_ret_has_failed(self):
-        sheerka, context = self.init_concepts()
-
-        r_success = sheerka.ret("Test", True, "success")
-        r_failure = sheerka.ret("Test", False, "False1")
-
-        sheerka.last_return_values.append([r_success, r_failure])
-        assert sheerka.last_error_ret(context) == r_failure
-
-    def test_i_cannot_get_last_error_ret_when_too_many_errors(self):
-        sheerka, context = self.init_concepts()
-
-        r1 = sheerka.ret("Test", True, "success")
-        r2 = sheerka.ret("Test", False, "False1")
-        r3 = sheerka.ret("Test", False, "False2")
-
-        sheerka.last_return_values.append([r1, r2, r3])
-        last_error_ret = sheerka.last_error_ret(context)
-        assert sheerka.isinstance(last_error_ret, BuiltinConcepts.RETURN_VALUE)
-        assert not last_error_ret.status
-        assert sheerka.isinstance(last_error_ret.body, BuiltinConcepts.TOO_MANY_ERRORS)
-        assert last_error_ret.body.body == [r2, r3]
+    # def test_i_can_get_last_error_ret(self):
+    #     sheerka, context = self.init_concepts()
+    #
+    #     # nothing in history
+    #     last_error_ret = sheerka.last_error_ret(context)
+    #     assert sheerka.isinstance(last_error_ret, BuiltinConcepts.RETURN_VALUE)
+    #     assert not last_error_ret.status
+    #     assert sheerka.isinstance(last_error_ret.body, BuiltinConcepts.NOT_FOUND)
+    #
+    #     # only success in history
+    #     r_success = sheerka.ret("Test", True, "success")
+    #     sheerka.last_return_values.append([r_success])
+    #     last_error_ret = sheerka.last_error_ret(context)
+    #     assert sheerka.isinstance(last_error_ret, BuiltinConcepts.RETURN_VALUE)
+    #     assert not last_error_ret.status
+    #     assert sheerka.isinstance(last_error_ret.body, BuiltinConcepts.NOT_FOUND)
+    #
+    #     # at least on error
+    #     r_failure = sheerka.ret("Test", False, "failure")
+    #     sheerka.last_return_values.append([r_failure])
+    #     assert sheerka.last_error_ret(context) == r_failure
+    #
+    #     # add another success and make sure we get the same error
+    #     sheerka.last_return_values.append([r_success])
+    #     assert sheerka.last_error_ret(context) == r_failure
+    #
+    #     # and I only get the last failure
+    #     r_failure2 = sheerka.ret("Test", False, "another failure")
+    #     sheerka.last_return_values.append([r_failure2])
+    #     assert sheerka.last_error_ret(context) == r_failure2
+    #
+    #     # but I still can get the previous error
+    #     assert sheerka.last_error_ret(context, -2) == r_failure
+    #
+    # def test_i_can_get_last_error_ret_when_only_one_ret_has_failed(self):
+    #     sheerka, context = self.init_concepts()
+    #
+    #     r_success = sheerka.ret("Test", True, "success")
+    #     r_failure = sheerka.ret("Test", False, "False1")
+    #
+    #     sheerka.last_return_values.append([r_success, r_failure])
+    #     assert sheerka.last_error_ret(context) == r_failure
+    #
+    # def test_i_cannot_get_last_error_ret_when_too_many_errors(self):
+    #     sheerka, context = self.init_concepts()
+    #
+    #     r1 = sheerka.ret("Test", True, "success")
+    #     r2 = sheerka.ret("Test", False, "False1")
+    #     r3 = sheerka.ret("Test", False, "False2")
+    #
+    #     sheerka.last_return_values.append([r1, r2, r3])
+    #     last_error_ret = sheerka.last_error_ret(context)
+    #     assert sheerka.isinstance(last_error_ret, BuiltinConcepts.RETURN_VALUE)
+    #     assert not last_error_ret.status
+    #     assert sheerka.isinstance(last_error_ret.body, BuiltinConcepts.TOO_MANY_ERRORS)
+    #     assert last_error_ret.body.body == [r2, r3]

tests/core/test_SheerkaComparisonManager.py

@@ -1,7 +1,7 @@
 import pytest
 from core.builtin_concepts import BuiltinConcepts
 from core.concept import Concept
-from core.global_symbols import CONCEPT_PRECEDENCE_MODIFIED, CONCEPT_COMPARISON_CONTEXT, RULE_PRECEDENCE_MODIFIED, \
+from core.global_symbols import EVENT_CONCEPT_PRECEDENCE_MODIFIED, CONCEPT_COMPARISON_CONTEXT, EVENT_RULE_PRECEDENCE_MODIFIED, \
     RULE_COMPARISON_CONTEXT
 from core.sheerka.services.SheerkaComparisonManager import SheerkaComparisonManager, ComparisonObj
 
@@ -446,7 +446,7 @@ class TestSheerkaGreaterThanManager(TestUsingMemoryBasedSheerka):
             nonlocal event_received
             event_received = True
 
-        sheerka.subscribe(CONCEPT_PRECEDENCE_MODIFIED, receive_event)
+        sheerka.subscribe(EVENT_CONCEPT_PRECEDENCE_MODIFIED, receive_event)
 
         sheerka.set_is_greater_than(context, foo, one, two)
         assert not event_received
@@ -471,7 +471,7 @@ class TestSheerkaGreaterThanManager(TestUsingMemoryBasedSheerka):
             nonlocal event_received
             event_received = True
 
-        sheerka.subscribe(RULE_PRECEDENCE_MODIFIED, receive_event)
+        sheerka.subscribe(EVENT_RULE_PRECEDENCE_MODIFIED, receive_event)
 
         sheerka.set_is_greater_than(context, foo, r1, r2)
         assert not event_received

tests/core/test_SheerkaDebugManager.py

@@ -1,12 +1,23 @@
 import pytest
 from core.builtin_concepts import BuiltinConcepts
+from core.concept import Concept, NotInit
 from core.sheerka.ExecutionContext import ExecutionContext
-from core.sheerka.services.SheerkaDebugManager import SheerkaDebugManager, DebugItem
+from core.sheerka.services.SheerkaDebugManager import SheerkaDebugManager, DebugItem, ConceptDebugObj
+from parsers.PythonParser import PythonNode
 from sdp.sheerkaDataProvider import Event
 
 from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
 
 
+class DummyObj:
+    def __init__(self, a, b):
+        self.a = a
+        self.b = b
+
+    def __repr__(self):
+        return f"DummyObj(a={self.a}, b={self.b})"
+
+
 class TestSheerkaDebugManager(TestUsingMemoryBasedSheerka):
 
     def test_i_can_activate_debug(self):
@@ -478,12 +489,85 @@ class TestSheerkaDebugManager(TestUsingMemoryBasedSheerka):
         assert len(service.debug_rules_settings) == 0
         assert len(service.debug_concepts_settings) == 0
 
+    @pytest.mark.parametrize("settings, expected", [
+        ({"service": "my_service", "item": "var_name"}, {"var_name"}),
+        ({"method": "my_method", "item": "var_name"}, {"var_name"}),
+        ({"debug_id": 0, "item": "var_name"}, {"var_name"}),
+        ({"service": "my_service", "item": "*"}, {"*"}),
+        ({"method": "my_method", "item": "*"}, {"*"}),
+        ({"debug_id": 0, "item": "*"}, {"*"}),
+        ({"service": "my_service"}, set()),
+        ({"method": "my_method"}, set()),
+        ({"debug_id": 0}, set()),
+    ])
+    def test_i_can_get_enabled_items(self, settings, expected):
+        sheerka, context = self.init_concepts()
+        service = sheerka.services[SheerkaDebugManager.NAME]
+        service.set_debug(context, True)
+
+        service.add_or_update_debug_item(context, "vars", **settings)
+
+        assert service.get_enabled_items("vars", context, "my_service", "my_method", 0) == expected
+
+    def test_i_can_get_enabled_items_for_context(self):
+        sheerka, context = self.init_concepts()
+        another_context = context.push(BuiltinConcepts.TESTING, None)
+        service = sheerka.services[SheerkaDebugManager.NAME]
+        service.set_debug(context, True)
+
+        service.add_or_update_debug_item(context, "rules", context_id=context.id, item="item", enabled=True)
+
+        assert service.get_enabled_items("rules", context, "my_service", "my_method", 10) == {"item"}
+        assert service.get_enabled_items("rules", another_context, "my_service", "my_method", 10) == set()
+
+    def test_i_can_get_enabled_items_for_sub_context(self):
+        sheerka, context = self.init_concepts()
+        sub_context = context.push(BuiltinConcepts.TESTING, None)
+        another_context = self.get_context(sheerka)
+        service = sheerka.services[SheerkaDebugManager.NAME]
+        service.set_debug(context, True)
+
+        service.add_or_update_debug_item(context, "rules", context_id=context.id, item="item", context_children=True)
+
+        assert service.get_enabled_items("rules", context, "my_service", "my_method", 10) == {"item"}
+        assert service.get_enabled_items("rules", sub_context, "my_service", "my_method", 10) == {"item"}
+        assert service.get_enabled_items("rules", another_context, "my_service", "my_method", 10) == set()
+
+    def test_i_can_get_all_enabled_items(self):
+        sheerka, context = self.init_concepts()
+        service = sheerka.services[SheerkaDebugManager.NAME]
+        service.set_debug(context, True)
+
+        service.add_or_update_debug_item(context, "vars", service="s1", item="v11")
+        service.add_or_update_debug_item(context, "vars", service="s2", item="v21")
+        service.add_or_update_debug_item(context, "vars", method="m1", item="v12")
+        service.add_or_update_debug_item(context, "vars", method="m2", item="v22")
+        service.add_or_update_debug_item(context, "vars", debug_id=1, item="v13")
+        service.add_or_update_debug_item(context, "vars", debug_id=2, item="v23")
+        service.add_or_update_debug_item(context, "vars", service="s1", method="m1", item="v111")
+
+        assert service.get_enabled_items("vars", context, "s1", "m1", 1) == {"v11", "v12", "v13", "v111"}
+
+    def test_i_can_manage_duplicates_when_get_enabled_items(self):
+        sheerka, context = self.init_concepts()
+        service = sheerka.services[SheerkaDebugManager.NAME]
+        service.set_debug(context, True)
+
+        service.add_or_update_debug_item(context, "vars", service="s1", item="var_name")
+        service.add_or_update_debug_item(context, "vars", method="m1", item="*")
+        service.add_or_update_debug_item(context, "vars", debug_id=1, item="var_name")
+        service.add_or_update_debug_item(context, "vars", service="s1", method="m1", item="*")
+
+        assert service.get_enabled_items("vars", context, "s1", "m1", 1) == {"var_name", "*"}
+
     @pytest.mark.parametrize("args, kwargs, expected", [
         (["my_service.my_method.my_var"], {}, ("my_var", "my_service", "my_method", None, False, None, True)),
         (["*.*.my_var"], {}, ("my_var", None, None, None, False, None, True)),
         (["my_service"], {}, (None, "my_service", None, None, False, None, True)),
         (["my_service.my_method"], {}, (None, "my_service", "my_method", None, False, None, True)),
         (["*.*.*"], {}, ("*", None, None, None, False, None, True)),
+        (["*.*.*.*.*"], {}, ("*.*.*", None, None, None, False, None, True)),
+        (["s.m.var.in.multi.parts"], {}, ("var.in.multi.parts", "s", "m", None, False, None, True)),
         ([1], {}, ("1", None, None, None, False, None, True)),
         (["", 1], {}, (None, None, None, 1, False, None, True)),
         ([None, 1], {}, (None, None, None, 1, False, None, True)),
@@ -566,3 +650,265 @@ class TestSheerkaDebugManager(TestUsingMemoryBasedSheerka):
             DebugItem('1', None, None, None, False, None, False, True)]
         assert another_service.debug_concepts_settings == [
             DebugItem('1001', None, None, None, False, None, False, True)]
+
+    def test_i_can_inspect_concept_all_attributes(self):
+        sheerka, context, foo = self.init_concepts("foo")
+        foo.values()  # freeze known attributes
+        foo.set_value("new_var", "var_value")
+
+        res = sheerka.inspect(context, foo)
+        assert res.body == {"#type#": "Concept",
+                            "id": foo.id,
+                            "name": foo.name,
+                            "key": foo.key,
+                            "value.new_var": "'var_value'"}
+
+    def test_i_can_inspect_concept_specified_attributes(self):
+        sheerka, context, foo = self.init_concepts("foo")
+        foo.values()  # freeze known attributes
+        foo.set_value("new_var", "var_value")
+
+        res = sheerka.inspect(context, foo, "id", "value.new_var")
+        assert res.body == {"id": foo.id, "value.new_var": "'var_value'"}
+
+    def test_i_can_inspect_concept_specified_attributes_using_short_name(self):
+        sheerka, context, foo, bar = self.init_concepts(Concept("foo").def_var("var_name", "default_value"), "bar")
+        foo.set_value("var_name", "var_value")
+        foo.get_compiled()["var_name"] = bar
+
+        res = sheerka.inspect(context, foo, "id", "var_name")
+        assert res.body == {"id": foo.id,
+                            "meta.var_name": "'default_value'",
+                            "compiled.var_name": bar,
+                            "value.var_name": "'var_value'"}
+
+    def test_i_can_inspect_object_all_attributes(self):
+        sheerka, context = self.init_concepts()
+        python_node = PythonNode("one + 1").init_ast()
+
+        res = sheerka.inspect(context, python_node)
+        assert set(res.body.keys()) == {"#type#", 'ast_', 'ast_str', 'compiled', 'objects', 'source'}
+
+    def test_i_can_inspect_object_specified_attributes(self):
+        sheerka, context = self.init_concepts()
+        python_node = PythonNode("one + 1").init_ast()
+
+        res = sheerka.inspect(context, python_node, "#type#", "source", "ast_str")
+        assert res.body == {
+            "#type#": "PythonNode",
+            'ast_str': "Expression(body=BinOp(left=Name(id='one'), op=Add(), right=Constant(value=1)))",
+            'source': 'one + 1'}
+
+    def test_i_can_inspect_object_with_as_bag_all_attributes(self):
+        sheerka, context = self.init_concepts()
+
+        res = sheerka.inspect(context, context)
+        assert res.body == {'#type#': 'ExecutionContext',
+                            '_children': [],
+                            'action': '__TESTING',
+                            'concepts': None,
+                            'context': None,
+                            'desc': None,
+                            'digest': 'xxx',
+                            'elapsed': 0,
+                            'elapsed_str': '0.0 ms',
+                            'id': context.id,
+                            'inputs': {},
+                            'obj': None,
+                            'status': None,
+                            'values': {},
+                            'who': 'test'}
+
+    def test_i_can_inspect_object_with_as_bag_specified_attributes(self):
+        sheerka, context = self.init_concepts()
+
+        res = sheerka.inspect(context, context, "who", "_children")
+        assert res.body == {'_children': [],
+                            'who': 'test'}
+
+    def test_i_can_inspect_object_with_concept(self):
+        sheerka, context, foo = self.init_concepts("foo")
+        foo.values()  # freeze known attributes
+        sheerka.set_attr(foo, "new_var", "var_value")
+
+        dummy = DummyObj(foo, "value")
+        res = sheerka.inspect(context, dummy)
+        assert res.body == {'#type#': 'DummyObj', 'a': foo, 'b': 'value'}
+
+    def test_i_can_inspect_object_with_concept_when_as_bag(self):
+        sheerka, context, foo = self.init_concepts("foo")
+        foo.values()  # freeze known attributes
+        sheerka.set_attr(foo, "new_var", "var_value")
+
+        dummy = DummyObj(foo, "value")
+        res = sheerka.inspect(context, dummy, as_bag=True)
+        assert res.body == {'#type#': 'DummyObj',
+                            'a': {'#type#': 'Concept',
+                                  'id': '1001',
+                                  'key': 'foo',
+                                  'name': 'foo',
+                                  'value.new_var': "'var_value'"},
+                            'b': 'value'}
+
+    def test_i_can_inspect_object_with_concept_when_values(self):
+        sheerka, context, foo = self.init_concepts("foo")
+        foo.values()  # freeze known attributes
+        sheerka.set_attr(foo, "new_var", "var_value")
+
+        dummy = DummyObj(foo, "value")
+        res = sheerka.inspect(context, dummy, values=True)
+        assert res.body == {'#type#': 'DummyObj', 'a': ConceptDebugObj(foo), 'b': 'value'}
+
+    def test_i_can_inspect_concept_with_concept(self):
+        sheerka, context, foo, bar = self.init_concepts(Concept("foo").def_var("var_name"), "bar")
+        foo.set_value("var_name", bar)
+
+        res = sheerka.inspect(context, foo)
+        assert res.body == {'#type#': 'Concept',
+                            'id': '1001',
+                            'key': 'foo',
+                            'name': 'foo',
+                            'value.var_name': bar}
+
+    def test_i_can_inspect_concept_with_concept_when_as_bag(self):
+        sheerka, context, foo, bar = self.init_concepts(Concept("foo").def_var("var_name"), "bar")
+        foo.set_value("var_name", bar)
+
+        res = sheerka.inspect(context, foo, as_bag=True)
+        assert res.body == {'#type#': 'Concept',
+                            'id': '1001',
+                            'key': 'foo',
+                            'name': 'foo',
+                            'value.var_name': {'#type#': 'Concept',
+                                               'id': '1002',
+                                               'key': 'bar',
+                                               'name': 'bar'}}
+
+    def test_i_can_inspect_concept_with_concept_when_values(self):
+        sheerka, context, foo, bar = self.init_concepts(Concept("foo").def_var("var_name"), "bar")
+        foo.set_value("var_name", bar)
+
+        res = sheerka.inspect(context, foo, values=True)
+        assert res.body == {'#type#': 'Concept',
+                            'id': '1001',
+                            'key': 'foo',
+                            'name': 'foo',
+                            'value.var_name': ConceptDebugObj(bar)}
+
+    def test_i_can_inspect_execution_context_item(self):
+        sheerka, context = self.init_concepts()
+        ExecutionContext.ids.clear()
+
+        sheerka.evaluate_user_input("def concept one as 1")
+        results = list(sheerka.get_last_results(context).body)
+        user_input_ret_val = results[0].inputs["user_input"]
+        return_values = results[0].values["return_values"]
+
+        res = sheerka.inspect(context, 0)
+        assert res.body == {'inputs': {'user_input': user_input_ret_val},
+                            'values.return_values': return_values}
+
+    def test_i_can_inspect_execution_context_values(self):
+        sheerka, context = self.init_concepts()
+        ExecutionContext.ids.clear()
+
+        sheerka.evaluate_user_input("def concept one as 1")
+        results = list(sheerka.get_last_results(context).body)
+        user_input_ret_val = results[0].inputs["user_input"]
+        return_values = results[0].values["return_values"]
+
+        res = sheerka.inspect(context, 0, values=True)
+
+        assert res.body == {'inputs': {'user_input': user_input_ret_val},
+                            'values.return_values': [ConceptDebugObj(return_values[0].body.body)]}
+
+    def test_i_can_inspect_when_a_property_does_not_exist(self):
+        sheerka, context, foo = self.init_concepts("foo")
+        foo.values()  # freeze known attributes
+        sheerka.set_attr(foo, "new_var", "var_value")
+
+        dummy = DummyObj(foo, "value")
+        res = sheerka.inspect(context, dummy, "#type#", "fake", "a", "b")
+        assert res.body == {'#type#': 'DummyObj',
+                            'fake': "** Not Found **",
+                            'a': foo,
+                            'b': 'value'}
+
+    def test_i_can_inspect_when_properties_are_specified_several_times(self):
+        sheerka, context, foo = self.init_concepts("foo")
+        foo.values()  # freeze known attributes
+        sheerka.set_attr(foo, "new_var", "var_value")
+
+        dummy = DummyObj(foo, "value")
+        res = sheerka.inspect(context, dummy, "#type#", "a", "b", "a")
+        assert res.body == {'#type#': 'DummyObj',
+                            'a': foo,
+                            'b': 'value'}
+
+    def test_i_cannot_inspect_execution_context_item_if_no_last_item(self):
+        sheerka, context = self.init_concepts()
+
+        res = sheerka.inspect(context, 0)
+
+        assert res.body == {'#type#': 'NotFound',
+                            'id': '70',
+                            'key': '__NOT_FOUND',
+                            'name': '__NOT_FOUND',
+                            'body': 'no digest'}
+
+    def test_i_can_inspect_values(self):
+        sheerka, context, table, how, little = self.init_concepts(
+            "table",
+            Concept("how is x").def_var("x"),
+            Concept("little x").def_var("x"),
+            create_new=True
+        )
+
+        return_values = sheerka.evaluate_user_input("how is little table")
+
+        res = sheerka.inspect(context, return_values[0], values=True)
+        concept_debug_obj = ConceptDebugObj(return_values[0].body)
+        assert res.body == {
+            'body': concept_debug_obj,
+            '#type#': 'ReturnValueConcept',
+            'id': '43',
+            'key': '__RETURN_VALUE',
+            'message': None,
+            'name': '__RETURN_VALUE',
+            'parents': [concept_debug_obj],
+            'status': True,
+            'value': concept_debug_obj,
+            'who': 'evaluators.OneSuccess'}
+
+        # I also can print it using bag
+        res = sheerka.inspect(context, return_values[0], '#type#', "who", "status", "value", values=True, as_bag=True)
+        assert res.body == {'#type#': 'ReturnValueConcept',
+                            'who': 'evaluators.OneSuccess',
+                            'status': True,
+                            'value': {'#type#': 'Concept',
+                                      'compiled.x': {'#type#': 'Concept',
+                                                     'compiled.x': {'#type#': 'Concept',
+                                                                    'id': '1001',
+                                                                    'key': 'table',
+                                                                    'name': 'table'},
+                                                     'id': '1003',
+                                                     'key': 'little __var__0',
+                                                     'meta.x': "'table'",
+                                                     'name': 'little x'},
+                                      'id': '1002',
+                                      'key': 'how is __var__0',
+                                      'meta.x': "'little table'",
+                                      'name': 'how is x'}}
+
+    def test_i_can_display_meta_and_compile_attributes_using_concept_debug_obj(self):
+        foo = Concept("foo", id=1001, key="foo_key").def_var("x", "x_meta").def_var("y", "y_meta")
+        foo.get_compiled()["x"] = Concept("bar", id=1002).def_var("a", "a_meta").set_value("a", "a_value")
+        foo.set_value("x", "x_value")
+        foo.set_value("y", NotInit)
+
+        foo.values()  # freeze attributes
+
+        foo.set_value("z", "extra_value")
+
+        assert str(ConceptDebugObj(foo)) == \
+               "(:foo|1001:meta.x='x_meta', meta.y='y_meta', compiled.x=(:bar|1002:meta.a='a_meta', value.a='a_value'), value.x='x_value', value.z='extra_value')"

tests/core/test_SheerkaMemory.py

@@ -85,7 +85,7 @@ class TestSheerkaMemory(TestUsingMemoryBasedSheerka):
         foo = Concept("foo")
         sheerka.add_to_memory(context, "a", foo)
 
-        assert service.objects.copy() == {"a": MemoryObject(context.event.get_digest(), foo)}
+        assert service.memory_objects.copy() == {"a": MemoryObject(context.event.get_digest(), foo)}
         assert id(sheerka.get_from_memory(context, "a").obj) == id(foo)
 
     def test_i_can_use_memory_to_get_the_list_of_all_objects(self):
@@ -134,6 +134,11 @@ class TestSheerkaMemory(TestUsingMemoryBasedSheerka):
 
         assert sheerka.memory(context, "item") == bar
 
+    def test_object_are_not_added_in_memory_during_the_initialisation(self):
+        sheerka, context = self.init_concepts()
+
+        assert len(sheerka.memory(context)) == 0
+
 
 class TestSheerkaMemoryUsingFileBase(TestUsingFileBasedSheerka):
     def test_i_can_record_memory_objects(self):

tests/core/test_SheerkaRuleManager.py

@@ -2,7 +2,7 @@ import ast
 
 import pytest
 from core.builtin_concepts import BuiltinConcepts
-from core.concept import Concept
+from core.concept import Concept, CMV
 from core.global_symbols import RULE_COMPARISON_CONTEXT
 from core.rule import Rule
 from core.sheerka.services.SheerkaRuleManager import SheerkaRuleManager, FormatRuleParser, \
@@ -227,8 +227,8 @@ class TestSheerkaRuleManager(TestUsingMemoryBasedSheerka):
         assert res[0].concept == expected
 
     @pytest.mark.parametrize("text, expected_variables", [
-        ("a cat is an animal", ["cat", "animal"]),
-        ("a cat is an b", ["a", "animal"]),
+        ("a cat is an animal", ["a cat", "animal"]),
+        ("a cat is an b", ["a cat", "b"]),
     ])
     def test_i_can_compile_predicate_when_sya_node_parser(self, text, expected_variables):
         sheerka, context, *concepts = self.init_concepts(
@@ -238,7 +238,7 @@ class TestSheerkaRuleManager(TestUsingMemoryBasedSheerka):
             create_new=True
         )
         service = sheerka.services[SheerkaRuleManager.NAME]
-        expected = concepts[0]
+        expected = CMV(concepts[0], x=expected_variables[0], y=expected_variables[1])
 
         res = service.compile_when(context, "test", text)
 
@@ -281,14 +281,14 @@ class TestSheerkaRuleManager(TestUsingMemoryBasedSheerka):
         assert isinstance(res[0], RulePredicate)
         assert res[0].evaluator == CONCEPT_EVALUATOR_NAME
         assert sheerka.isinstance(res[0].predicate, BuiltinConcepts.RETURN_VALUE)
-        assert sheerka.objvalue(res[0].predicate)[0].concept == concepts[0]
-        assert res[0].concept == concepts[0]
+        assert sheerka.objvalue(res[0].predicate)[0].concept == CMV(concepts[0], x="a", y="b")
+        assert res[0].concept == CMV(concepts[0], x="a", y="b")
 
         assert isinstance(res[1], RulePredicate)
         assert res[1].evaluator == CONCEPT_EVALUATOR_NAME
         assert sheerka.isinstance(res[1].predicate, BuiltinConcepts.RETURN_VALUE)
-        assert sheerka.objvalue(res[1].predicate)[0].concept == concepts[1]
-        assert res[1].concept == concepts[1]
+        assert sheerka.objvalue(res[1].predicate)[0].concept == CMV(concepts[1], x="a", y="b")
+        assert res[1].concept == CMV(concepts[1], x="a", y="b")
 
     # @pytest.mark.skip
     # @pytest.mark.parametrize("text, expected", [

tests/core/test_sheerkaResultManager.py

@@ -14,10 +14,8 @@ class TestSheerkaResultManager(TestUsingMemoryBasedSheerka):
 
     @classmethod
     def setup_class(cls):
-        sheerka = cls().get_sheerka()
+        sheerka = cls().get_sheerka(cache_only=False)
         sheerka.save_execution_context = True
-        sheerka.evaluate_user_input("def concept one as 1")
-        cls.io_cache = sheerka.sdp.io.cache.copy()
 
     @classmethod
     def teardown_class(cls):
@@ -26,21 +24,73 @@ class TestSheerkaResultManager(TestUsingMemoryBasedSheerka):
 
     def init_test(self):
         sheerka, context = self.init_concepts()
-        sheerka.sdp.io.cache = self.io_cache.copy()
-        return sheerka, context
+        service = sheerka.services[SheerkaResultConcept.NAME]
 
-    def test_i_can_get_the_result_by_digest(self):
-        sheerka, context = self.init_test()
+        return sheerka, context, service
 
-        digest = sheerka.get_last_execution().event.get_digest()
+    def test_i_can_record_execution_contexts(self):
+        sheerka, context, service = self.init_test()
+
+        sheerka.evaluate_user_input("foo")
+
+        executions_contexts_in_cache = service.executions_contexts_cache.copy()
+        assert len(executions_contexts_in_cache) == 1
+        event_id = list(executions_contexts_in_cache.keys())[0]
+        execution_context = list(executions_contexts_in_cache.values())[0]
+
+        assert execution_context.desc == "Evaluating 'foo'"
+
+        executions_contexts_in_db = sheerka.sdp.load_result(event_id)
+        assert executions_contexts_in_db is not None
+        assert executions_contexts_in_db.desc == "Evaluating 'foo'"
+
+        assert service.last_execution is not None
+        assert service.last_execution.desc == "Evaluating 'foo'"
+
+    def test_i_can_get_the_result_by_digest_using_cache(self):
+        sheerka, context, service = self.init_test()
+        sheerka.evaluate_user_input("def concept one as 1")
+
+        digest = service.last_execution.event.get_digest()
 
         res = sheerka.get_results_by_digest(context, digest)
 
+        # we get the result
         assert sheerka.isinstance(res, BuiltinConcepts.EXPLANATION)
         assert res.command == "def concept one as 1"
         assert res.digest == digest
         assert isinstance(res.body, types.GeneratorType)
 
+        # the digest is correctly recorded
+        assert sheerka.load_var(SheerkaResultConcept.NAME, "digest") == digest
+
+        previous_results = list(res.body)
+
+        # Second test,
+        # I can get the result from the recorded digest
+        res = sheerka.get_results(context)
+        assert sheerka.isinstance(res, BuiltinConcepts.EXPLANATION)
+        assert res.command == "def concept one as 1"
+        assert res.digest == digest
+        assert isinstance(res.body, types.GeneratorType)
+
+        assert list(res.body) == previous_results
+
+    def test_i_can_get_result_by_digest_using_db(self):
+        sheerka, context, service = self.init_test()
+        sheerka.evaluate_user_input("def concept one as 1")
+        digest = service.last_execution.event.get_digest()
+        service.reset()
+
+        res = sheerka.get_results_by_digest(context, digest)
+
+        # we get the result
+        assert sheerka.isinstance(res, BuiltinConcepts.EXPLANATION)
+        assert res.command == "def concept one as 1"
+        assert res.digest == digest
+        assert isinstance(res.body, types.GeneratorType)
+
+        # the digest is correctly recorded
         assert sheerka.load_var(SheerkaResultConcept.NAME, "digest") == digest
 
         previous_results = list(res.body)
@@ -67,9 +117,10 @@ class TestSheerkaResultManager(TestUsingMemoryBasedSheerka):
         assert sheerka.get_results(context) is None
 
     def test_i_can_get_the_result_by_command_name(self):
-        sheerka, context = self.init_test()
+        sheerka, context, service = self.init_test()
+        sheerka.evaluate_user_input("def concept one as 1")
 
-        digest = sheerka.get_last_execution().event.get_digest()
+        digest = service.last_execution.event.get_digest()
         sheerka.evaluate_user_input("one")  # another command
 
         res = sheerka.get_results_by_command(context, "def concept")
@@ -78,8 +129,18 @@ class TestSheerkaResultManager(TestUsingMemoryBasedSheerka):
         assert res.digest == digest
         assert isinstance(res.body, types.GeneratorType)
 
+    def test_i_can_get_the_result_by_command_name_using_db(self):
+        sheerka, context, service = self.init_test()
+        sheerka.evaluate_user_input("def concept one as 1")
+        sheerka.evaluate_user_input("one")  # another command
+        service.reset()
+
+        res = sheerka.get_results_by_command(context, "def concept")
+        assert res.command == "def concept one as 1"
+
     def test_i_can_get_the_result_by_command_when_not_in_the_same_page_size(self):
-        sheerka, context = self.init_test()
+        sheerka, context, service = self.init_test()
+        sheerka.evaluate_user_input("def concept one as 1")
 
         sheerka.evaluate_user_input("one")
         sheerka.evaluate_user_input("one")
@@ -98,7 +159,8 @@ class TestSheerkaResultManager(TestUsingMemoryBasedSheerka):
         assert res.body == {'command': 'def concept'}
 
     def test_i_cannot_get_result_from_command_if_the_command_does_not_exists_multiple_pages(self):
-        sheerka, context = self.init_test()
+        sheerka, context, service = self.init_test()
+        sheerka.evaluate_user_input("def concept one as 1")
 
         sheerka.evaluate_user_input("one")
         sheerka.evaluate_user_input("one")
@@ -110,7 +172,8 @@ class TestSheerkaResultManager(TestUsingMemoryBasedSheerka):
         assert res.body == {'command': 'fake command'}
 
     def test_i_can_get_last_results(self):
-        sheerka, context = self.init_test()
+        sheerka, context, service = self.init_test()
+        sheerka.evaluate_user_input("def concept one as 1")
 
         sheerka.evaluate_user_input("one")
 
@@ -118,6 +181,16 @@ class TestSheerkaResultManager(TestUsingMemoryBasedSheerka):
         assert sheerka.isinstance(res, BuiltinConcepts.EXPLANATION)
         assert res.command == "one"
 
+    def test_i_can_get_last_results_using_db(self):
+        sheerka, context, service = self.init_test()
+        sheerka.evaluate_user_input("def concept one as 1")
+        sheerka.evaluate_user_input("one")
+        service.reset()
+
+        res = sheerka.get_last_results(context)
+        assert sheerka.isinstance(res, BuiltinConcepts.EXPLANATION)
+        assert res.command == "one"
+
     def test_i_can_get_last_results_when_event_with_no_result(self):
         sheerka, context = self.init_concepts()
 
@@ -154,7 +227,8 @@ class TestSheerkaResultManager(TestUsingMemoryBasedSheerka):
         {"desc": "Evaluating 'def concept one as 1'", "id": 0}
     ])
     def test_i_can_get_last_results_using_kwarg(self, kwargs):
-        sheerka, context = self.init_test()
+        sheerka, context, service = self.init_test()
+        sheerka.evaluate_user_input("def concept one as 1")
         ExecutionContext.ids.clear()
 
         res = sheerka.get_last_results(context, **kwargs)
@@ -170,7 +244,8 @@ class TestSheerkaResultManager(TestUsingMemoryBasedSheerka):
         ("'def concept one as 1' in desc and id == 0", {"desc": "Evaluating 'def concept one as 1'", "id": 0})
     ])
     def test_i_can_get_last_results_using_filter(self, predicate, expected):
-        sheerka, context = self.init_test()
+        sheerka, context, service = self.init_test()
+        sheerka.evaluate_user_input("def concept one as 1")
         ExecutionContext.ids.clear()
 
         res = sheerka.get_last_results(context, filter=predicate)
@@ -186,7 +261,8 @@ class TestSheerkaResultManager(TestUsingMemoryBasedSheerka):
         ("'def concept one as 1' in desc and id == 0", {"desc": "Evaluating 'def concept one as 1'", "id": 0})
     ])
     def test_i_can_get_last_results_using_the_first_argument_to_filter(self, predicate, expected):
-        sheerka, context = self.init_test()
+        sheerka, context, service = self.init_test()
+        sheerka.evaluate_user_input("def concept one as 1")
         ExecutionContext.ids.clear()
 
         res = sheerka.get_last_results(context, predicate)
@@ -202,7 +278,8 @@ class TestSheerkaResultManager(TestUsingMemoryBasedSheerka):
         {"desc": "Evaluating 'def concept one as 1'", "id": 0}
     ])
     def test_i_can_get_results_using_kwarg(self, kwargs):
-        sheerka, context = self.init_test()
+        sheerka, context, service = self.init_test()
+        sheerka.evaluate_user_input("def concept one as 1")
         ExecutionContext.ids.clear()
         sheerka.get_last_results(context)
 
@@ -219,7 +296,8 @@ class TestSheerkaResultManager(TestUsingMemoryBasedSheerka):
         ("'def concept one as 1' in desc and id == 0", {"desc": "Evaluating 'def concept one as 1'", "id": 0})
     ])
     def test_i_can_get_results_using_filter(self, predicate, expected):
-        sheerka, context = self.init_test()
+        sheerka, context, service = self.init_test()
+        sheerka.evaluate_user_input("def concept one as 1")
         ExecutionContext.ids.clear()
         sheerka.get_last_results(context)
 
@@ -236,7 +314,8 @@ class TestSheerkaResultManager(TestUsingMemoryBasedSheerka):
         ("'def concept one as 1' in desc and id == 0", {"desc": "Evaluating 'def concept one as 1'", "id": 0})
     ])
     def test_i_can_get_results_using_the_first_argument_to_filter(self, predicate, expected):
-        sheerka, context = self.init_test()
+        sheerka, context, service = self.init_test()
+        sheerka.evaluate_user_input("def concept one as 1")
         ExecutionContext.ids.clear()
         sheerka.get_last_results(context)
 
@@ -251,3 +330,35 @@ class TestSheerkaResultManager(TestUsingMemoryBasedSheerka):
         predicate = {"filter": "a b c"}
         with pytest.raises(SyntaxError):
             SheerkaResultConcept.get_predicate(**predicate)
+
+    def test_i_can_get_last_return_value(self):
+        sheerka, context, service = self.init_test()
+
+        sheerka.evaluate_user_input("def concept one as 1")
+        ret = sheerka.last_ret(context)
+        assert sheerka.isinstance(ret[0].body, BuiltinConcepts.NEW_CONCEPT)
+
+        sheerka.evaluate_user_input("eval one")
+        ret = sheerka.last_ret(context)
+        assert ret[0].body == 1
+
+    def test_i_can_track_new_concept(self):
+        sheerka, context, service = self.init_test()
+
+        res = sheerka.evaluate_user_input("def concept one as 1")
+        new_concept = res[0].body.body
+
+        assert sheerka.last_created_concept(context) == new_concept
+        assert service.last_created_concept_id == new_concept.id
+
+    def test_last_created_concept_is_recorded(self):
+        sheerka, context, service = self.init_test()
+        res = sheerka.evaluate_user_input("def concept one as 1")
+        new_concept = res[0].body.body
+        service.reset()
+
+        service.initialize_deferred(context, False)
+
+        assert service.last_created_concept_id == new_concept.id
+        assert sheerka.last_created_concept(context) == new_concept
+

tests/core/test_utils.py

@@ -2,6 +2,7 @@ from dataclasses import dataclass
 
 import core.utils
 import pytest
+from core.builtin_concepts import BuiltinConcepts
 from core.concept import Concept
 from core.tokenizer import Token, TokenKind, Tokenizer, Keywords
 
@@ -338,3 +339,81 @@ def test_i_can_get_text_from_tokens(text, expected_text):
 def test_i_can_get_text_from_tokens_with_custom_switcher(text, custom, expected_text):
     tokens = list(Tokenizer(text))
     assert core.utils.get_text_from_tokens(tokens, custom) == expected_text
+
+
+def test_i_can_deep_copy_a_concept():
+    def check_are_the_same(actual, expected):
+        assert id(actual) != id(expected)
+
+        for k, v in vars(expected.get_metadata()).items():
+            assert getattr(actual.get_metadata(), k) == v
+
+        # test the values
+        for k, v in expected.values().items():
+            assert getattr(actual, k) == v
+
+    concept1 = Concept(name="concept1_name",
+                       is_builtin=True,
+                       is_unique=True,
+                       key="concept1_key",
+                       body="concept1_body",
+                       where='concept1_where',
+                       pre="concept1_pre",
+                       post="concept1_post",
+                       ret="concept1_ret",
+                       definition="concept1_definition",
+                       definition_type="concept1_definition_type",
+                       desc="concept1_desc",
+                       id="concept1_ids",
+                       props="concept1_props",
+                       variables=[],
+                       bound_body=None)
+
+    concept2 = Concept(name="concept2_name",
+                       is_builtin=True,
+                       is_unique=True,
+                       key="concept2_key",
+                       body="concept2_body",
+                       where='concept2_where',
+                       pre="concept2_pre",
+                       post="concept2_post",
+                       ret="concept2_ret",
+                       definition="concept2_definition",
+                       definition_type="concept2_definition_type",
+                       desc="concept2_desc",
+                       id="concept2_ids",
+                       props={"prop_name": concept1},
+                       variables=[("var1", "default_value1"), ("var2", "default_value2")],
+                       bound_body="var1")
+
+    concept = Concept(name="my_name",
+                      is_builtin=True,
+                      is_unique=True,
+                      key="my_key",
+                      body="my_body",
+                      where='my_where',
+                      pre="my_pre",
+                      post="my_post",
+                      ret="my_ret",
+                      definition="my_definition",
+                      definition_type="my_definition_type",
+                      desc="my_desc",
+                      id="my_ids",
+                      props={
+                          BuiltinConcepts.ISA: {concept1, concept2},
+                          "prop2": ["value1, value2"],
+                          "prop3": {"a": 1, "b": 2},
+                          "prop4": "a simple value"},
+                      variables=[("var1", "default_value1"), ("var2", "default_value2")])
+    concept.set_value("var1", "string_value")
+    concept.set_value("var2", 10)
+    concept.set_value("var3", concept1)
+
+    copied = core.utils.sheerka_deepcopy(concept)
+
+    check_are_the_same(copied, concept)
+
+    copied_props = sorted(list(copied.get_prop(BuiltinConcepts.ISA)), key=lambda o: o.id)
+    concept_props = sorted(list(concept.get_prop(BuiltinConcepts.ISA)), key=lambda o: o.id)
+    for copied_prop, concept_prop in zip(copied_props, concept_props):
+        check_are_the_same(copied_prop, concept_prop)

tests/non_reg/test_sheerka_display.py

@@ -0,0 +1,16 @@
+from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
+
+
+class TestSheerkaNonRegDisplay(TestUsingMemoryBasedSheerka):
+
+    def test_i_can_display_results_when_return_values_processing_is_on(self, capsys):
+        init = [
+            "def concept one as 1",
+        ]
+
+        sheerka = self.init_scenario(init)
+        sheerka.enable_process_return_values = True
+        sheerka.evaluate_user_input("one")
+
+        captured = capsys.readouterr()
+        assert captured.out == "ReturnValue(who=evaluators.OneSuccess, status=True, value=(1001)one)\n"

tests/non_reg/test_sheerka_non_reg.py

@@ -1092,7 +1092,7 @@ as:
         ]
 
         sheerka = self.init_scenario(init)
-        res = sheerka.evaluate_user_input("desc(the a)")
+        res = sheerka.evaluate_user_input("desc(c:the a:)")
 
         assert len(res) == 1
         assert res[0].status

tests/out/test_AsStrVisitor.py

@@ -142,7 +142,7 @@ class TestAsStrVisitor(TestUsingMemoryBasedSheerka):
             (FormatAstVariable('__key', index=0, value="key1", debug=True),
              FormatAstVariable('__value', index="key1", value=1, debug=True)),
 
-            (FormatAstVariable('__key', index=0, value="key2", debug=True),
+            (FormatAstVariable('__key', index=1, value="key2", debug=True),
              FormatAstDict("__value", debug=True, prefix="{", suffix="}", items=[
                  (FormatAstVariable('__key', index=0, value="sub_key1", debug=True),
                   FormatAstVariable('__value', index="sub_key1", value=1, debug=True)),
@@ -155,8 +155,14 @@ class TestAsStrVisitor(TestUsingMemoryBasedSheerka):
                   ])),
              ])),
 
-            (FormatAstVariable('__key', index=1, value="long_key3", debug=True),
+            (FormatAstVariable('__key', index=2, value="long_key3", debug=True),
              FormatAstVariable('__value', index="key2", value="value2", debug=True)),
+
+            (FormatAstVariable('__key', index=3, value="key3", debug=True),
+             FormatAstList("__value", debug=True, prefix="[", suffix="]", items=[
+                 FormatAstVariable('__item', index=0, value="first element", debug=True),
+                 FormatAstVariable('__item', index=1, value="second element", debug=True),
+             ])),
         ])
 
         res = visitor.visit(bag)
@@ -165,4 +171,6 @@ class TestAsStrVisitor(TestUsingMemoryBasedSheerka):
 'key2'     : {'sub_key1'     : 1,
               'sub_long_key2': {'sub_sub_key1': 1,
                                 'sub_sub_key2': 'sub_sub_value'}},
-'long_key3': 'value2'}"""
+'long_key3': 'value2',
+'key3'     : ['first element',
+              'second element']}"""

tests/out/test_DeveloperVisitor.py

@@ -0,0 +1,40 @@
+from core.sheerka.services.SheerkaDebugManager import NullDebugLogger
+from core.sheerka.services.SheerkaOut import SheerkaOut
+from core.sheerka.services.SheerkaRuleManager import FormatAstList, FormatAstVariable
+from out.DeveloperVisitor import DeveloperVisitor
+
+from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
+
+
+class TestDeveloperVisitor(TestUsingMemoryBasedSheerka):
+    def test_i_can_develop_list(self):
+        sheerka, context = self.init_concepts()
+        service_out = sheerka.services[SheerkaOut.NAME]
+        dev_visitor = DeveloperVisitor(service_out, NullDebugLogger(), set(), 0)
+
+        bag = {"a": ["a", "b", "c"]}
+
+        res = dev_visitor.visit(context, FormatAstList("a"), bag)
+        assert res == FormatAstList(variable="a", items=[
+            FormatAstVariable(name="__item", index=0, value="a"),
+            FormatAstVariable(name="__item", index=1, value="b"),
+            FormatAstVariable(name="__item", index=2, value="c"),
+        ])
+
+    def test_i_can_develop_list_of_list(self):
+        sheerka, context = self.init_concepts()
+        service_out = sheerka.services[SheerkaOut.NAME]
+        dev_visitor = DeveloperVisitor(service_out, NullDebugLogger(), set(), 0)
+
+        bag = {"a": [["a1", "a2"], ["b1"]]}
+
+        res = dev_visitor.visit(context, FormatAstList("a"), bag)
+        assert res == FormatAstList(variable="a", items=[
+            FormatAstList(variable="__item", index=0, debug=True, prefix='[', suffix=']', items=[
+                FormatAstVariable(name="__item", index=0, debug=True, value="a1"),
+                FormatAstVariable(name="__item", index=1, debug=True, value="a2"),
+            ]),
+            FormatAstList(variable="__item", index=1, debug=True, prefix='[', suffix=']', items=[
+                FormatAstVariable(name="__item", index=0, debug=True, value="b1"),
+            ]),
+        ])

tests/out/test_SheerkaOut.py

@@ -449,3 +449,51 @@ key2: value2
         captured = capsys.readouterr()
         assert captured.out == """{'\x1b[32mkey1\x1b[0m': 'value1', 'key2': 1, 'key3': DummyObj(prop_1=3.15, prop_2='a string'), 'key4': ['alpha', 0]}
 """
+
+    def test_i_can_print_out_dict_sub_items(self, capsys):
+        sheerka, context, service, *rules = self.init_service_with_rules(
+            ("isinstance(__obj, dict)", "dict(__obj)"),
+            ("__key=='key1'", "green(__key)")
+        )
+        obj = {
+            "key1": "value1",
+            "key2": 1,
+            "key3": DummyObj(prop_1=3.15, prop_2='a string'),
+            "key4": {"a": 1, "b": "value"},
+            "key5": ["alpha", 0]
+        }
+        service.process_return_values(context, obj)
+        captured = capsys.readouterr()
+        assert captured.out == """\x1b[32mkey1\x1b[0m: value1
+key2: 1
+key3: DummyObj(prop_1=3.15, prop_2='a string')
+key4: {'a': 1, 'b': 'value'}
+key5: ['alpha', 0]
+"""
+
+    def test_i_can_print_out_dict_with_expanded_sub_items(self, capsys):
+        sheerka, context, service, *rules = self.init_service_with_rules(
+            ("isinstance(__obj, dict)", "dict(__obj)"),
+            ("__key=='key1'", "green(__key)")
+        )
+        obj = {
+            "key1": "value1",
+            "key2": 1,
+            "key3": DummyObj(prop_1=3.15, prop_2='a string'),
+            "key4": {"a": 1, "b": "value"},
+            "key5": ["alpha", 0]
+        }
+
+        old_value = service.out_visitors[0].console_width
+        service.out_visitors[0].console_width = 5
+        service.process_return_values(context, obj)
+        captured = capsys.readouterr()
+        assert captured.out == """\x1b[32mkey1\x1b[0m: value1
+key2: 1
+key3: DummyObj(prop_1=3.15, prop_2='a string')
+key4: {'a': 1,
+       'b': 'value'}
+key5: ['alpha',
+       0]
+"""
+        service.out_visitors[0].console_width = old_value

tests/parsers/test_SyaNodeParser.py

@@ -1,10 +1,11 @@
 import pytest
 from core.builtin_concepts import BuiltinConcepts
-from core.concept import Concept, CIO, ALL_ATTRIBUTES
+from core.concept import Concept, CIO, ALL_ATTRIBUTES, CMV
 from core.global_symbols import CONCEPT_COMPARISON_CONTEXT
 from core.sheerka.services.SheerkaExecute import ParserInput
 from core.tokenizer import Tokenizer
-from parsers.BaseNodeParser import utnode, ConceptNode, cnode, short_cnode, UnrecognizedTokensNode, \
+from core.utils import NextIdManager
+from parsers.BaseNodeParser import utnode, cnode, short_cnode, UnrecognizedTokensNode, \
     SCWC, CNC, UTN, SCN, CN
 from parsers.PythonParser import PythonNode
 from parsers.SyaNodeParser import SyaNodeParser, SyaConceptParserHelper, SyaAssociativity, \
@@ -916,21 +917,21 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
 
     @pytest.mark.parametrize("expression, expected_debugs", [
         ("one", [["  0:one => PUSH_UNREC"]]),
-        ("one plus two", [[
-            '  0:one => PUSH_UNREC',
-            '  1:<ws> => PUSH_UNREC',
-            '  2:plus(SyaConceptDef(concept=(1005)a plus b, precedence=1, associativity=right)) => ??',
-            "  _: => RECOG [[CN((1001)one)]]",
-            "  _: => POP ConceptNode(concept='(1001)one', source='one', start=0, end=0)",
-            '  2:plus(SyaConceptDef(concept=(1005)a plus b, precedence=1, associativity=right)) => PUSH',
-            '  3:<ws> => EAT',
-            '  4:two => PUSH_UNREC',
-            '  5:<EOF> => ??',
-            "  _: => RECOG [[CN((1002)two)]]",
-            "  _: => POP ConceptNode(concept='(1002)two', source='two', start=4, end=4)",
-            '  _: => POP SyaConceptParserHelper(concept=(1005)a plus b, start=2, error=None)']]),
+        ("one plus two", [['  0:one => PUSH_UNREC',
+                           '  1:<ws> => PUSH_UNREC',
+                           '  2:plus ((1005)a plus b, prio=1, assoc=SyaAssociativity.Right) => ??',
+                           '  _: => RECOG [[CN((1001)one)]]',
+                           "  _: => POP ConceptNode(concept='(1001)one', source='one', start=0, end=0)",
+                           '  2:plus ((1005)a plus b, prio=1, assoc=SyaAssociativity.Right) => PUSH',
+                           '  3:<ws> => EAT',
+                           '  4:two => PUSH_UNREC',
+                           '  5:<EOF> => ??',
+                           '  _: => RECOG [[CN((1002)two)]]',
+                           "  _: => POP ConceptNode(concept='(1002)two', source='two', start=4, end=4)",
+                           '  _: => POP SyaConceptParserHelper(concept=(1005)a plus b, start=2, '
+                           'error=None)']]),
         ("suffixed one", [[
-            '  0:suffixed(SyaConceptDef(concept=(1009)suffixed a, precedence=1, associativity=right)) => PUSH',
+            '  0:suffixed ((1009)suffixed a, prio=1, assoc=SyaAssociativity.Right) => PUSH',
             '  1:<ws> => EAT',
             '  2:one => PUSH_UNREC',
             '  3:<EOF> => ??',
@@ -941,10 +942,10 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
         ("one ? twenty one : three", [[
             '  0:one => PUSH_UNREC',
             '  1:<ws> => PUSH_UNREC',
-            '  2:?(SyaConceptDef(concept=(1011)a ? b : c, precedence=1, associativity=right)) => ??',
-            "  _: => RECOG [[CN((1001)one)]]",
+            '  2:? ((1011)a ? b : c, prio=1, assoc=SyaAssociativity.Right) => ??',
+            '  _: => RECOG [[CN((1001)one)]]',
             "  _: => POP ConceptNode(concept='(1001)one', source='one', start=0, end=0)",
-            '  2:?(SyaConceptDef(concept=(1011)a ? b : c, precedence=1, associativity=right)) => PUSH',
+            '  2:? ((1011)a ? b : c, prio=1, assoc=SyaAssociativity.Right) => PUSH',
             '  3:<ws> => EAT',
             '  4:twenty => PUSH_UNREC',
             '  5:<ws> => PUSH_UNREC',
@@ -955,14 +956,13 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
             "  _: => POP UnrecognizedTokensNode(source='twenty ', start=4, end=5)",
             "  _: => POP ConceptNode(concept='(1001)one', source='one', start=6, end=6)",
             "  _: => => ERROR Too many parameters found for '(1011)a ? b : c' before token 'Token(:)'",
-            '  8:: => EAT',
-        ], [
+            '  8:: => EAT'], [
             '  0:one => PUSH_UNREC',
             '  1:<ws> => PUSH_UNREC',
-            '  2:?(SyaConceptDef(concept=(1011)a ? b : c, precedence=1, associativity=right)) => ??',
+            '  2:? ((1011)a ? b : c, prio=1, assoc=SyaAssociativity.Right) => ??',
             '  _: => RECOG [[CN((1001)one)]]',
             "  _: => POP ConceptNode(concept='(1001)one', source='one', start=0, end=0)",
-            '  2:?(SyaConceptDef(concept=(1011)a ? b : c, precedence=1, associativity=right)) => PUSH',
+            '  2:? ((1011)a ? b : c, prio=1, assoc=SyaAssociativity.Right) => PUSH',
             '  3:<ws> => EAT',
             '  4:twenty => PUSH_UNREC',
             '  5:<ws> => PUSH_UNREC',
@@ -976,12 +976,12 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
             ' 11:<EOF> => ??',
             '  _: => RECOG [[CN((1003)three)]]',
             "  _: => POP ConceptNode(concept='(1003)three', source='three', start=10, end=10)",
-            '  _: => POP SyaConceptParserHelper(concept=(1011)a ? b : c, start=2, error=None)'
-        ]]),
+            '  _: => POP SyaConceptParserHelper(concept=(1011)a ? b : c, start=2, error=None)']]),
     ])
     def test_i_can_debug(self, expression, expected_debugs):
         sheerka, context, parser = self.init_parser()
-        context.debug_enabled = True
+        sheerka.set_debug(context, True)
+        sheerka.debug_var(context, "Sya")
         res = parser.infix_to_postfix(context, ParserInput(expression))
 
         assert len(res) == len(expected_debugs)
@@ -989,6 +989,19 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
             actual_debug = [str(di) for di in res_i.debug]
             assert actual_debug == expected_debug
 
+    @pytest.mark.parametrize("settings", [
+        "Sya.*.*",
+        "Sya.*.#0.can_pop"
+    ])
+    def test_i_can_debug_can_pop_using_star(self, settings):
+        sheerka, context, parser = self.init_parser()
+        sheerka.set_debug(context, True)
+        sheerka.debug_var(context, settings)
+        res = parser.infix_to_postfix(context, ParserInput("one plus two mult three"))
+
+        debug = [str(di) for di in res[0].debug]
+        assert debug[5] == '  _: => No stack. CAN_POP false.'
+
     def test_i_can_parse_when_concept_atom_only(self):
         sheerka, context, parser = self.init_parser()
 
@@ -999,15 +1012,20 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
 
         assert res.status
         assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
-        assert lexer_nodes == [ConceptNode(cmap["plus"], 0, 8, source=text)]
+        assert lexer_nodes == [CN(cmap["plus"], 0, 8, source=text)]
 
         # check the compiled
         expected_concept = lexer_nodes[0].concept
         assert expected_concept.get_compiled()["a"] == cmap["one"]
-        assert expected_concept.get_compiled()["b"] == cmap["mult"]
+        assert expected_concept.get_compiled()["b"] == CMV(cmap["mult"], a="two", b="three")
         assert expected_concept.get_compiled()["b"].get_compiled()["a"] == cmap["two"]
         assert expected_concept.get_compiled()["b"].get_compiled()["b"] == cmap["three"]
 
+        # check the metadata
+        expected_concept = lexer_nodes[0].concept
+        assert expected_concept.get_metadata().variables == [("a", "one"), ("b", "two mult three")]
+        assert expected_concept.get_compiled()["b"].get_metadata().variables == [("a", "two"), ("b", "three")]
+
     def test_i_can_parse_when_python_code(self):
         sheerka, context, parser = self.init_parser()
 
@@ -1018,7 +1036,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
 
         assert res.status
         assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
-        assert lexer_nodes == [ConceptNode(cmap["suffixed"], 0, 6, source=text)]
+        assert lexer_nodes == [CN(cmap["suffixed"], 0, 6, source=text)]
 
         # check the compiled
         expected_concept = lexer_nodes[0].concept
@@ -1031,6 +1049,9 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
         assert return_value_a.body.source == "1 + 1"
         assert isinstance(return_value_a.body.body, PythonNode)
 
+        # check metadata
+        assert expected_concept.get_metadata().variables == [("a", "1 + 1")]
+
     def test_i_can_parse_when_bnf_concept(self):
         sheerka, context, parser = self.init_parser()
 
@@ -1043,13 +1064,16 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
         lexer_nodes = res[1].body.body
 
         assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
-        assert lexer_nodes == [ConceptNode(cmap["suffixed"], 0, 4, source=text)]
+        assert lexer_nodes == [CN(cmap["suffixed"], 0, 4, source=text)]
 
         # check the compiled
         expected_concept = lexer_nodes[0].concept
         assert sheerka.isinstance(expected_concept.get_compiled()["a"], "twenties")
         assert expected_concept.get_compiled()["a"].get_compiled()["unit"] == cmap["one"]
 
+        # check metadata
+        assert expected_concept.get_metadata().variables == [("a", "twenty one")]
+
     def test_i_can_parse_sequences(self):
         sheerka, context, parser = self.init_parser()
 
@@ -1061,8 +1085,8 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
         assert res.status
         assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
         assert lexer_nodes == [
-            ConceptNode(cmap["plus"], 0, 9, source="one plus 1 + 1 "),
-            ConceptNode(cmap["suffixed"], 10, 12, source="suffixed two")]
+            CN(cmap["plus"], 0, 9, source="one plus 1 + 1 "),
+            CN(cmap["suffixed"], 10, 12, source="suffixed two")]
 
         # check the compiled
         concept_plus_a = lexer_nodes[0].concept.get_compiled()["a"]
@@ -1198,7 +1222,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
 
         assert not res.status
         assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
-        assert lexer_nodes == [ConceptNode(cmap[expected_concept], 0, expected_end, source=text)]
+        assert lexer_nodes == [CN(cmap[expected_concept], 0, expected_end, source=text)]
 
         concept_found = lexer_nodes[0].concept
         for unrecognized in expected_unrecognized:
@@ -1278,7 +1302,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
     ])
     def test_i_can_get_functions_names_from_unrecognized(self, expression, expected):
         sheerka, context, parser = self.init_parser()
-        infix_to_postfix = InFixToPostFix(context)
+        infix_to_postfix = InFixToPostFix(context, NextIdManager())
 
         tokens = list(Tokenizer(expression, yield_eof=False))
         for pos, token in enumerate(tokens[:-1]):
@@ -1302,7 +1326,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
     ])
     def test_i_can_get_functions_names_from_unrecognized_when_multiple_results(self, expression, expected_list):
         sheerka, context, parser = self.init_parser()
-        infix_to_postfix = InFixToPostFix(context)
+        infix_to_postfix = InFixToPostFix(context, NextIdManager())
 
         tokens = list(Tokenizer(expression, yield_eof=False))
         for pos, token in enumerate(tokens[:-1]):