First but not optimized version of AstFormatDict

src/core/builtin_concepts.py

@@ -105,6 +105,7 @@ class BuiltinConcepts:
 
     # formatting
     TO_LIST = "__TO_LIST"
+    TO_DICT = "__TO_DICT"
 
 
 AllBuiltinConcepts = [v for n, v in BuiltinConcepts.__dict__.items() if not n.startswith("__")]
@@ -176,6 +177,7 @@ BuiltinContainers = [
     BuiltinConcepts.FILTERED,
     BuiltinConcepts.EXPLANATION,
     BuiltinConcepts.TO_LIST,
+    BuiltinConcepts.TO_DICT,
 ]
 
 """

src/core/sheerka/Sheerka.py

@@ -113,7 +113,8 @@ class Sheerka(Concept):
         self.methods_with_context = {"test_using_context"}  # only the names, the method is defined in sheerka_methods
         self.sheerka_methods = {
             "test": SheerkaMethod(self.test, False),
-            "test_using_context": SheerkaMethod(self.test_using_context, False)
+            "test_using_context": SheerkaMethod(self.test_using_context, False),
+            "test_dict": SheerkaMethod(self.test_dict, False)
         }
         self.sheerka_pipeables = {}
 
@@ -1028,6 +1029,20 @@ class Sheerka(Concept):
         # uncomment the following line to enable colors
         # logging.StreamHandler.emit = add_coloring_to_emit_ansi(logging.StreamHandler.emit)
 
+    def test_dict(self):
+        bag2 = {
+            "alpha": "value4",
+            "beta": ["item1", "item2", "item3", ]
+        }
+        bag = {
+            "a": "value1",
+            "baba": "value2",
+            "c": "value1",
+            "de": ["item1", "item2", "item3", ],
+            "e": bag2
+        }
+        return self.new(BuiltinConcepts.TO_DICT, body=bag)
+
 
 def to_profile():
     sheerka = Sheerka()

src/core/sheerka/services/SheerkaAdmin.py

@@ -3,6 +3,7 @@ import time
 from os import path
 
 from core.builtin_concepts import BuiltinConcepts, BuiltinContainers
+from core.builtin_helpers import ensure_concept
 from core.concept import Concept
 from core.sheerka.services.sheerka_service import BaseService
 
@@ -22,6 +23,7 @@ class SheerkaAdmin(BaseService):
         self.sheerka.bind_service_method(self.cache, False)
         self.sheerka.bind_service_method(self.restore, True)
         self.sheerka.bind_service_method(self.concepts, False)
+        self.sheerka.bind_service_method(self.desc, False)
         self.sheerka.bind_service_method(self.last_created_concept, False)
         self.sheerka.bind_service_method(self.last_ret, False)
         self.sheerka.bind_service_method(self.last_error_ret, False)
@@ -122,6 +124,28 @@ class SheerkaAdmin(BaseService):
     def concepts(self):
         return self.sheerka.new(BuiltinConcepts.TO_LIST, body=self.sheerka.sdp.list(self.sheerka.CONCEPTS_BY_ID_ENTRY))
 
+    def desc(self, *concepts):
+        ensure_concept(*concepts)
+        res = []
+        for c in concepts:
+            bag = {
+                "id": c.id,
+                "name": c.name,
+                "key": c.key,
+                "definition": c.get_metadata().definition,
+                "type": c.get_metadata().definition_type,
+                "body": c.get_metadata().body,
+                "where": c.get_metadata().where,
+                "pre": c.get_metadata().pre,
+                "post": c.get_metadata().post,
+                "ret": c.get_metadata().ret,
+                "vars": c.get_metadata().variables,
+                "props": c.get_metadata().props,
+            }
+            res.append(self.sheerka.new(BuiltinConcepts.TO_DICT, body=bag))
+
+        return res[0] if len(res) == 1 else self.sheerka.new(BuiltinConcepts.TO_LIST, body=res)
+
     def format_rules(self):
         return self.sheerka.new(BuiltinConcepts.TO_LIST, items=self.sheerka.get_format_rules())
 

src/core/sheerka/services/SheerkaDebugManager.py

@@ -34,13 +34,16 @@ class BaseDebugLogger:
     def debug_entering(self, **kwargs):
         pass
 
-    def debug_var(self, name, value, is_error=False):
+    def debug_var(self, name, value, is_error=False, hint=None):
         pass
 
     def debug_rule(self, rule, results):
         pass
 
-    def debug_log(self, text):
+    def debug_log(self, text, is_error=False):
+        pass
+
+    def is_enabled(self):
         pass
 
 
@@ -48,6 +51,9 @@ class NullDebugLogger(BaseDebugLogger):
     def __init__(self):
         pass
 
+    def is_enabled(self):
+        return False
+
 
 class ConsoleDebugLogger(BaseDebugLogger):
 
@@ -60,6 +66,9 @@ class ConsoleDebugLogger(BaseDebugLogger):
         self.debug_id = debug_id
         self.is_highlighted = ""
 
+    def is_enabled(self):
+        return True
+
     def debug_entering(self, **kwargs):
         super().debug_entering(**kwargs)
 
@@ -71,17 +80,18 @@ class ConsoleDebugLogger(BaseDebugLogger):
             self.debug_manager.debug(self.prefix() + str_text)
             self.debug_manager.debug(self.prefix() + str_vars)
 
-    def debug_var(self, name, value, is_error=False):
-        enabled = self.debug_manager.compute_var_debug(self.service_name,
-                                                       self.method_name,
-                                                       self.context_id,
-                                                       name,
-                                                       self.context_id)
+    def debug_var(self, name, value, is_error=False, hint=None):
+        enabled = is_error or self.debug_manager.compute_var_debug(self.service_name,
+                                                                   self.method_name,
+                                                                   self.context_id,
+                                                                   name,
+                                                                   self.context_id)
         if enabled == False:
             return
 
         color = 'red' if is_error else 'green'
-        str_text = f"{CCM[color]}..{name}={CCM['reset']}"
+        hint_str = f"({hint})" if hint is not None else ""
+        str_text = f"{CCM[color]}..{name}{hint_str}={CCM['reset']}"
         str_vars = "" if isinstance(enabled, str) else pp.pformat(value)
         if "\n" not in str(str_vars):
             self.debug_manager.debug(self.prefix() + str_text + str_vars)
@@ -101,8 +111,9 @@ class ConsoleDebugLogger(BaseDebugLogger):
             self.debug_manager.debug(self.prefix() + str_text)
             self.debug_manager.debug(self.prefix() + str_vars)
 
-    def debug_log(self, text):
-        self.debug_manager.debug(self.prefix() + f"{CCM['blue']}..{text}{CCM['reset']}")
+    def debug_log(self, text, is_error=False):
+        color = 'red' if is_error else 'blue'
+        self.debug_manager.debug(self.prefix() + f"{CCM[color]}..{text}{CCM['reset']}")
 
     def prefix(self):
         return f"[{self.context_id:2}][{self.debug_id:2}] {self.is_highlighted}"
@@ -271,12 +282,8 @@ class SheerkaDebugManager(BaseService):
         res = {}
         for prop in props:
             res[prop] = evaluate_expression(prop, bag)
-        # res = {
-        #     "return_values": to_inspect.values.get("return_values", None)
-        # }
 
-        pp.pprint(res)
-        return None
+        return self.sheerka.new(BuiltinConcepts.TO_DICT, body=res)
 
     def debug(self, *args, **kwargs):
         print(*args, **kwargs)
@@ -437,4 +444,4 @@ class SheerkaDebugManager(BaseService):
         return self.sheerka.ret(SheerkaDebugManager.NAME, True, self.sheerka.new(BuiltinConcepts.SUCCESS))
 
     def get_debug_settings(self):
-        return self.debug_vars_settings
+        return self.sheerka.new(BuiltinConcepts.TO_LIST, body=self.debug_vars_settings)

src/core/sheerka/services/SheerkaDump.py

@@ -21,7 +21,7 @@ class SheerkaDump(BaseService):
         super().__init__(sheerka)
 
     def initialize(self):
-        self.sheerka.bind_service_method(self.dump_desc, True, "desc")  # has_side_effect 'cause concept is evaluated
+        self.sheerka.bind_service_method(self.dump_desc, True, "old_desc")  # has_side_effect 'cause concept is evaluated
         self.sheerka.bind_service_method(self.dump_sdp, False, "dump_sdp")
 
     def dump_desc(self, *concept_names, eval=False):

src/core/sheerka/services/SheerkaOut.py

@@ -16,7 +16,8 @@ class SheerkaOut(BaseService):
         self.sheerka.bind_service_method(self.process_return_values, False)
 
     def create_out_tree(self, context, obj):
-        return self.create_out_tree_recursive(context, {'__obj': obj}, DeveloperVisitor(self, set(), 0))
+        debugger = context.get_debugger("Out.visitor", "create_out_tree")
+        return self.create_out_tree_recursive(context, {'__obj': obj}, DeveloperVisitor(self, debugger, set(), 0))
 
     def create_out_tree_recursive(self, context, bag, visitor):
         debugger = context.get_debugger(SheerkaOut.NAME, "create_out_tree")
@@ -40,8 +41,7 @@ class SheerkaOut(BaseService):
                 visitor.already_seen.add(rule.id)
 
                 bag.update(as_bag(current_obj))  # update with the current obj attributes
-                visitor.visit(context, rule.compiled_action, bag)
-                res = visitor.get_result()
+                res = visitor.visit(context, rule.compiled_action, bag)
 
         if res is None:
             debugger.debug_log(f"No matching rule.")
@@ -67,9 +67,7 @@ class SheerkaOut(BaseService):
 
         if out_tree:
             for visitor in self.out_visitors:
-                visitor.visit(context, out_tree, None)
-                if hasattr(visitor, "finalize"):
-                    visitor.finalize()
+                visitor.visit(out_tree)
 
             return self.sheerka.ret(self.NAME, True, self.sheerka.new(BuiltinConcepts.SUCCESS))
         return self.sheerka.ret(self.NAME, False, self.sheerka.new(BuiltinConcepts.NO_RESULT))

src/core/sheerka/services/SheerkaRuleManager.py

@@ -63,6 +63,15 @@ class FormatAstNode:
 
         return ", ".join(repr(item) for item in items)
 
+    def clone(self, instance, props, **kwargs):
+        for prop_name in props:
+            setattr(instance, prop_name, getattr(self, prop_name))
+
+        for k, v in kwargs.items():
+            setattr(instance, k, v)
+
+        return instance
+
 
 @dataclass
 class FormatAstRawText(FormatAstNode):
@@ -73,9 +82,15 @@ class FormatAstRawText(FormatAstNode):
 class FormatAstVariable(FormatAstNode):
     name: str
     format: Union[str, None] = None
+    debug: bool = False
     value: object = None
     index: object = None
 
+    def clone(self, **kwargs):
+        return super().clone(FormatAstVariable(self.name),
+                             ("format", "debug", "value", "index"),
+                             **kwargs)
+
 
 @dataclass
 class FormatAstVariableNotFound(FormatAstNode):
@@ -93,29 +108,50 @@ class FormatAstList(FormatAstNode):
     items_prop: str = None  # where to search the list if variable does not resolve to an iterable
     recurse_on: str = None
     recursion_depth: int = 0
+    debug: bool = False
+    prefix: str = None
+    suffix: str = None
+    show_index: bool = False
+
     items: object = None
 
+    def clone(self, **kwargs):
+        return super().clone(
+            FormatAstList(self.variable),
+            ("items_prop", "recurse_on", "recursion_depth", "debug", "prefix", "suffix", "show_index", "items"),
+            **kwargs)
+
+
+@dataclass
+class FormatAstDict(FormatAstNode):
+    variable: str
+    items_prop: str = None  # where to search the dict if variable does not resolve to an iterable
+    debug: bool = False
+    prefix: str = None
+    suffix: str = None
+
+    items: object = None
+
+    def clone(self, **kwargs):
+        return super().clone(
+            FormatAstDict(self.variable),
+            ("items_prop", "debug", "prefix", "suffix", "items"),
+            **kwargs)
+
 
 @dataclass
 class FormatAstColor(FormatAstNode):
-    def __init__(self, color, format_ast):
-        self.color = color
-        self.format_ast = format_ast
+    color: str
+    format_ast: FormatAstNode
 
     def __repr__(self):
         return f"{self.color}({self.format_ast})"
 
-    def __eq__(self, other):
-        if id(self) == id(other):
-            return True
-
-        if not isinstance(other, FormatAstColor):
-            return False
-
-        return self.color == other.color and self.format_ast == other.format_ast
-
-    def __hash__(self):
-        return hash((self.color, self.format_ast))
+    def clone(self, **kwargs):
+        return super().clone(
+            FormatAstColor(self.color, self.format_ast),
+            (),
+            **kwargs)
 
 
 @dataclass
@@ -125,21 +161,19 @@ class FormatAstFunction(FormatAstNode):
     kwargs: dict = None
 
 
+@dataclass
 class FormatAstSequence(FormatAstNode):
-    def __init__(self, items):
-        self.items = items
+    items: list
+    debug: bool = False
 
     def __repr__(self):
         return "FormatAstSequence(" + self.repr_value(self.items) + ")"
 
-    def __eq__(self, other):
-        if id(self) == id(other):
-            return True
-
-        if not isinstance(other, FormatAstSequence):
-            return False
-
-        return self.items == other.items
+    def clone(self, **kwargs):
+        return super().clone(
+            FormatAstSequence(self.items),
+            ("debug",),
+            **kwargs)
 
 
 class FormatRuleParser(IterParser):
@@ -170,6 +204,9 @@ class FormatRuleParser(IterParser):
         if value[0] in ("'", '"'):
             return value[1:-1]
 
+        if value in ("True", "False"):
+            return bool(value)
+
         try:
             return int(value)
         except ValueError:
@@ -278,6 +315,8 @@ class FormatRuleParser(IterParser):
             return self.return_color(func_name.value, args, kwargs)
         elif func_name.value == "list":
             return self.return_list(args, kwargs)
+        elif func_name.value == "dict":
+            return self.return_dict(args, kwargs)
 
         return FormatAstFunction(func_name.value, self.to_text(args), self.to_text(kwargs))
 
@@ -347,13 +386,19 @@ class FormatRuleParser(IterParser):
             return FormatAstColor(color, FormatAstVariable(variable, vformat))
 
     def return_list(self, args, kwargs):
+        """
+        Looking for variable_name, [recurse_on], [recursion_depth], [items_prop]
+        :param args:
+        :param kwargs:
+        :return:
+        """
         len_args = len(args)
         if len_args < 1:
             self.error_sink = FormatRuleSyntaxError("variable name not found", None)
             return None
 
-        if len_args > 3:
-            self.error_sink = FormatRuleSyntaxError("too many positional arguments", args[3][0])
+        if len_args > 4:
+            self.error_sink = FormatRuleSyntaxError("too many positional arguments", args[4][0])
             return None
 
         variable_name = get_text_from_tokens(args[0])
@@ -364,6 +409,10 @@ class FormatRuleParser(IterParser):
         elif len_args == 3:
             recursion_depth = self.to_value(args[1])
             recurse_on = self.to_value(args[2])
+        elif len_args == 4:
+            recursion_depth = self.to_value(args[1])
+            recurse_on = self.to_value(args[2])
+            items_prop = self.to_value(args[3])
 
         if "recurse_on" in kwargs:
             recurse_on = self.to_value(kwargs["recurse_on"])
@@ -383,6 +432,34 @@ class FormatRuleParser(IterParser):
 
         return FormatAstList(variable_name, items_prop, recurse_on, recursion_depth)
 
+    def return_dict(self, args, kwargs):
+        len_args = len(args)
+        if len_args < 1:
+            self.error_sink = FormatRuleSyntaxError("variable name not found", None)
+            return None
+
+        if len_args > 1:
+            self.error_sink = FormatRuleSyntaxError("too many positional arguments", args[1][0])
+            return None
+
+        variable_name = get_text_from_tokens(args[0])
+
+        kwargs_parameters = {}
+        for prop in ("items_prop", "prefix", "suffix", "debug"):
+            if prop in kwargs:
+                kwargs_parameters[prop] = self.to_value(kwargs[prop])
+
+        if "debug" in kwargs_parameters:
+            if "prefix" not in kwargs_parameters:
+                kwargs_parameters["prefix"] = "{"
+            if "suffix" not in kwargs_parameters:
+                kwargs_parameters["suffix"] = "}"
+
+        if self.error_sink:
+            return None
+
+        return FormatAstDict(variable_name, **kwargs_parameters)
+
 
 @dataclass()
 class RulePredicate:
@@ -556,6 +633,9 @@ class SheerkaRuleManager(BaseService):
             Rule("print", "Display formatted list",
                  "isinstance(__ret_container, BuiltinConcepts.TO_LIST)",
                  "list(__ret_container)"),
+            Rule("print", "Display formatted dict",
+                 "isinstance(__ret_container, BuiltinConcepts.TO_DICT)",
+                 "dict(__ret_container)"),
         ]
 
         for r in rules:

src/core/utils.py

@@ -2,11 +2,11 @@ import ast
 import importlib
 import inspect
 import pkgutil
-import re
 
 from cache.Cache import Cache
 from core.ast_helpers import ast_to_props
 from core.tokenizer import TokenKind, Tokenizer
+from pyparsing import *
 
 default_debug_name = "*default*"
 debug_activated = set()
@@ -38,6 +38,15 @@ PRIMITIVES_TYPES = (str, bool, type(None), int, float, list, dict, set, bytes, t
 
 expressions_cache = Cache()
 
+ESC = Literal('\x1b')
+integer = Word(nums)
+escapeSeq = Combine(ESC + '[' + Optional(delimitedList(integer, ';')) +
+                    oneOf(list(alphas)))
+
+
+def no_color_str(text):
+    return Suppress(escapeSeq).transformString(str(text))
+
 
 def my_debug(*args, check_started=None):
     """

src/evaluators/DefConceptEvaluator.py

@@ -129,12 +129,14 @@ class DefConceptEvaluator(OneReturnValueEvaluator):
         This function can only be a draft, as there may be tons of different situations
         I guess that it can only be complete when will we have access to Sheerka memory
         """
+        debugger = context.get_debugger(DefConceptEvaluator.NAME, "get_variables")
         #
         # Case of NameNode
         #
         if isinstance(ret_value, NameNode):
             names = [str(t.value) for t in ret_value.tokens if t.type in (
                 TokenKind.IDENTIFIER, TokenKind.STRING, TokenKind.KEYWORD)]
+            debugger.debug_var("names", names, hint="from NameNode")
             return set(filter(lambda x: x in concept_name and context.sheerka.not_is_variable(x), names))
 
         #
@@ -143,6 +145,7 @@ class DefConceptEvaluator(OneReturnValueEvaluator):
         if isinstance(ret_value.value, ParserResultConcept) and isinstance(ret_value.value.value, ParsingExpression):
             visitor = ConceptOrRuleNameVisitor()
             visitor.visit(ret_value.value.value)
+            debugger.debug_var("names", visitor.names, hint="from BNF")
             return set(visitor.names)
 
         #
@@ -152,6 +155,7 @@ class DefConceptEvaluator(OneReturnValueEvaluator):
             if len(concept_name) > 1:
                 visitor = UnreferencedVariablesVisitor(context)
                 names = visitor.get_names(python_node.ast_)
+                debugger.debug_var("names", names, hint="from python node")
                 return set(filter(lambda x: x in concept_name and context.sheerka.not_is_variable(x), names))
             else:
                 return set()
@@ -168,6 +172,7 @@ class DefConceptEvaluator(OneReturnValueEvaluator):
             for identifier in [i for i in concept_name if str(i).isalnum()]:
                 if identifier in tokens:
                     variables.add(identifier)
+            debugger.debug_var("names", variables, hint="from concept")
             return variables
 
         return set()

src/evaluators/PythonEvaluator.py

@@ -92,6 +92,9 @@ class PythonEvaluator(OneReturnValueEvaluator):
 
         debugger = context.get_debugger(PythonEvaluator.NAME, "eval")
         debugger.debug_entering(node=node)
+        exception_debugger = context.get_debugger("Exceptions", PythonEvaluator.NAME + ".eval")
+        get_trace_back = context.debug_enabled or exception_debugger.is_enabled()
+
         context.log(f"Evaluating python node {node}.", self.name)
 
         # If we evaluate a Concept metadata which is NOT the body ex (pre, post, where...)
@@ -136,9 +139,12 @@ class PythonEvaluator(OneReturnValueEvaluator):
             except Exception as ex:
                 if concepts_entries is None:
                     concepts_entries = self.get_concepts_entries_from_globals(my_globals)
-                errors.append(PythonEvalError(ex,
-                                              traceback.format_exc() if context.debug_enabled else None,
-                                              self.get_concepts_values_from_globals(globals_, concepts_entries)))
+                eval_error = PythonEvalError(ex,
+                                              traceback.format_exc() if get_trace_back else None,
+                                              self.get_concepts_values_from_globals(globals_, concepts_entries))
+                errors.append(eval_error)
+                exception_debugger.debug_var("exception", eval_error.error, is_error=True)
+                exception_debugger.debug_var("trace", eval_error.traceback, is_error=True)
 
         if evaluated == NotInit:
             if len(errors) == 1:

src/jupyter/SheerkaKernel.py

@@ -1,7 +1,8 @@
+from IPython.core.display import JSON
 from core.sheerka.Sheerka import Sheerka
 from ipykernel.ipkernel import IPythonKernel
 from ipykernel.kernelapp import IPKernelApp
-from sheerkapickle import encode
+from sheerkapickle.SheerkaPickler import SheerkaPickler
 
 
 class SheerkaKernel(IPythonKernel):
@@ -25,12 +26,19 @@ class SheerkaKernel(IPythonKernel):
 
     def do_execute(self, code, silent, store_history=True, user_expressions=None, allow_stdin=False):
         result = self.sheerka.evaluate_user_input(code)
+        # md = Markdown(f'<span style="color: {hexcolor}">{input_data}</span>')
+        as_json = JSON(SheerkaPickler(self.sheerka).flatten(result))
+        a = {
+            "a": "value1",
+            "b": "value2",
+        }
 
         if not silent:
             display_data_content = {
                 'data': {
-                    'text/plain': str(result),
-                    'application/json': encode(self.sheerka, result)
+                    # 'text/plain': str(result),
+                    # "text/markdown": md._repr_markdown_(),
+                    'application/json': JSON(a)._repr_json_(),
                 },
                 'metadata': {
                     'application/json': {'expanded': True}
@@ -38,9 +46,8 @@ class SheerkaKernel(IPythonKernel):
                 'execution_count': self.sheerka.execution_count,
 
             }
-            stream_content = {'name': 'stdout', 'text': display_data_content}
-            # self.send_response(self.iopub_socket, "stream", stream_content)
-            self.send_response(self.iopub_socket, "execute_result", display_data_content)
+
+            self.send_response(self.iopub_socket, "display_data", display_data_content)
 
         return {'status': 'ok',
                 # The base class increments the execution count

src/out/AsStrVisitor.py

@@ -0,0 +1,93 @@
+import re
+
+from core.sheerka.services.SheerkaRuleManager import FormatAstNode
+from core.utils import CONSOLE_COLORS_MAP as CCM, no_color_str
+from out.OutVisitor import OutVisitor
+
+get_start = re.compile(r"^([\(\{\[]\w*)")
+
+
+class AsStrVisitor(OutVisitor):
+
+    def __init__(self, expand=False):
+        self.expand = expand
+
+    def visit_FormatAstRawText(self, format_ast):
+        return str(format_ast.text)
+
+    def visit_FormatAstVariable(self, format_ast):
+        if isinstance(format_ast.value, FormatAstNode):
+            value = self.visit(format_ast.value)
+        else:
+            value = format_ast.value
+
+        if format_ast.debug and isinstance(value, str):
+            value = "'" + str(value).translate(str.maketrans({"'": r"\'"})) + "'"
+
+        return str(value)
+
+    def visit_FormatAstVariableNotFound(self, format_ast):
+        return CCM["red"] + format_ast.name + CCM["reset"]
+
+    def visit_FormatAstColor(self, format_ast):
+        return CCM[format_ast.color] + self.visit(format_ast.format_ast) + CCM["reset"]
+
+    def visit_FormatAstSequence(self, format_ast):
+        return "".join([self.visit(item) for item in format_ast.items])
+
+    def visit_FormatAstList(self, format_ast):
+        first = True
+        result = ""
+        if format_ast.prefix:
+            result += format_ast.prefix
+
+        sep = ",\n" if self.expand else ", " if format_ast.debug else "\n"
+
+        for item in format_ast.items:
+            if not first:
+                result += sep
+            if format_ast.show_index:
+                result += f"[{item.index}] "
+            result += self.visit(item)
+            first = False
+        if format_ast.suffix:
+            result += format_ast.suffix
+
+        return result
+
+    def visit_FormatAstDict(self, format_ast):
+        first = True
+        result = ""
+
+        keys_values = []
+        max_len = 0
+        for k, v in format_ast.items:
+            value = self.visit(k)
+            len_value = len(no_color_str(value))
+            if len_value > max_len:
+                max_len = len_value
+            keys_values.append(value)
+
+        if format_ast.prefix:
+            result += format_ast.prefix
+
+        sep = ",\n" if self.expand else ", " if format_ast.debug else "\n"
+
+        for i, (k, v) in enumerate(format_ast.items):
+            start = "" if first else sep
+
+            key = f"{keys_values[i]:<{max_len}}" if (self.expand or not format_ast.debug) else keys_values[i]
+            colon = ": "
+            indent = len(no_color_str(key)) + len(colon)
+            value = self.visit(v)
+            if self.expand:
+                if m := get_start.match(no_color_str(value)):
+                    indent += len(m.group(1))
+                value = value.replace("\n", "\n" + " " * indent)
+            first = False
+
+            result += start + key + colon + value
+
+        if format_ast.suffix:
+            result += format_ast.suffix
+        return result

src/out/ConsoleVisistor.py

@@ -1,55 +1,43 @@
-from out.OutVisitor import OutVisitor
-from core.sheerka.services.SheerkaRuleManager import FormatAstNode
+from out.AsStrVisitor import AsStrVisitor
 
 
-class ConsoleVisitor(OutVisitor):
+class ConsoleVisitor(AsStrVisitor):
     """
     Prints to the console
     """
-    COLORS = {
-        "reset": "\u001b[0m",
-        "black": "\u001b[30m",
-        "red": "\u001b[31m",
-        "green": "\u001b[32m",
-        "yellow": "\u001b[33m",
-        "blue": "\u001b[34m",
-        "magenta": "\u001b[35m",
-        "cyan": "\u001b[36m",
-        "white": "\u001b[37m",
-    }
 
-    def __init__(self):
+    def __init__(self, expand_mode="auto"):
+        super().__init__()
         self.out = print
+        self.expand_mode = expand_mode
 
-    def finalize(self):
-        self.out("")
+    def visit_FormatAstRawText(self, format_ast):
+        self.out(super().visit_FormatAstRawText(format_ast))
 
-    def visit_FormatAstRawText(self, context, format_ast, bag):
-        self.out(format_ast.text, end="")
+    def visit_FormatAstVariable(self, format_ast):
+        self.out(super().visit_FormatAstVariable(format_ast))
 
-    def visit_FormatAstVariable(self, context, format_ast, bag):
-        if isinstance(format_ast.value, FormatAstNode):
-            self.visit(context, format_ast.value, bag)
-            return
-        self.out(format_ast.value, end="")
+    def visit_FormatAstVariableNotFound(self, format_ast):
+        self.out(super().visit_FormatAstVariableNotFound(format_ast))
 
-    def visit_FormatAstVariableNotFound(self, context, format_ast, bag):
-        self.out(self.COLORS["red"] + format_ast.name + self.COLORS["reset"], end="")
+    def visit_FormatAstColor(self, format_ast):
+        self.out(super().visit_FormatAstColor(format_ast))
 
-    def visit_FormatAstSequence(self, context, format_ast, bag):
-        for item in format_ast.items:
-            self.visit(context, item, bag)
+    def visit_FormatAstSequence(self, format_ast):
+        visitor = AsStrVisitor()
+        self.out(visitor.visit_FormatAstSequence(format_ast))
 
-    def visit_FormatAstList(self, context, format_ast, bag):
-        first = True
-        for item in format_ast.items:
-            if not first:
-                self.out("")  # print new line
-            self.visit(context, item, bag)
-            first = False
+    def visit_FormatAstList(self, format_ast):
+        visitor = AsStrVisitor()
+        res = visitor.visit_FormatAstList(format_ast)
+        self.out(res)
 
-    def visit_FormatAstColor(self, context, format_ast, bag):
-        self.out(self.COLORS[format_ast.color], end="")
-        self.visit(context, format_ast.format_ast, bag)
-        self.out(self.COLORS["reset"], end="")
+    def visit_FormatAstDict(self, format_ast):
+        if self.expand_mode == "always":
+            expand = True
+        else:
+            expand = False
 
+        visitor = AsStrVisitor(expand)
+        res = visitor.visit_FormatAstDict(format_ast)
+        self.out(res)

src/out/DeveloperVisitor.py

@@ -1,31 +1,43 @@
-from core.sheerka.services.SheerkaRuleManager import FormatAstVariable, FormatAstVariableNotFound, FormatAstSequence, \
-    FormatAstColor, FormatAstList, FormatAstRawText
+from core.sheerka.services.SheerkaRuleManager import FormatAstVariable, FormatAstVariableNotFound, FormatAstColor, \
+    FormatAstList, FormatAstRawText, FormatAstDict
 from core.utils import evaluate_expression, as_bag
-from out.OutVisitor import OutVisitor
 
 fstring = compile('f"{value:{format}}"', "DeveloperVisitor.fstring", mode="eval")
 
 
-class DeveloperVisitor(OutVisitor):
+class DeveloperVisitor:
     """
     This visitor is used to resolve all the variables as well as all the lists
     Once completed, it will be passed to the ConsoleVisitor for console print
     """
 
-    def __init__(self, sheerka_out, already_seen, list_recursion_depth):
+    def __init__(self, sheerka_out, debugger, already_seen, list_recursion_depth):
         self._result = None
         self.sheerka_out = sheerka_out
+        self.debugger = debugger
         self.already_seen = already_seen
         self.list_recursion_depth = list_recursion_depth
 
+    def visit(self, context, format_ast, bag):
+        name = format_ast.__class__.__name__
+
+        method = 'visit_' + name
+        visit_method = getattr(self, method, self.generic_visit)
+        return visit_method(context, format_ast, bag)
+
+    def generic_visit(self, context, format_ast, bag):
+        pass
+
     def visit_FormatAstRawText(self, context, format_ast, bag):
-        if context.debug_enabled:
-            context.debug_entering("DeveloperVisitor", "visit_FormatAstRawText", format_ast=format_ast, bag=bag)
-        return self.set_result(format_ast)
+        if self.debugger.is_enabled():
+            debug_bag = {"format_ast": format_ast, "bag": bag}
+            self.debugger.debug_log(f"Entering visit_FormatAstRawText with {debug_bag}")
+        return format_ast
 
     def visit_FormatAstVariable(self, context, format_ast, bag):
-        if context.debug_enabled:
-            context.debug_entering("DeveloperVisitor", "visit_FormatAstVariable", format_ast=format_ast, bag=bag)
+        if self.debugger.is_enabled():
+            debug_bag = {"format_ast": format_ast, "bag": bag}
+            self.debugger.debug_log(f"Entering visit_FormatAstVariable with {debug_bag}")
 
         try:
             value = evaluate_expression(format_ast.name, bag)
@@ -33,6 +45,8 @@ class DeveloperVisitor(OutVisitor):
                 "__obj": value,
                 format_ast.name: value
             }
+            # create a bag entry with the last part of the variable
+            # eg: for __ret.status, creates an entry status
             try:
                 index = format_ast.name.rindex(".")
                 sub_bag[format_ast.name[index + 1:]] = value
@@ -43,32 +57,31 @@ class DeveloperVisitor(OutVisitor):
             if format_ast.format:
                 res = eval(fstring, {"value": res, "format": format_ast.format})
 
-            return self.set_result(FormatAstVariable(format_ast.name,
-                                                     format_ast.format,
-                                                     res,
-                                                     format_ast.index))
-
+            return format_ast.clone(value=res)
         except NameError as error:
             context.debug("DeveloperVisitor", "visit_FormatAstList", "evaluate_expression", error, is_error=True)
-            return self.set_result(FormatAstVariableNotFound(format_ast.name))
+            return FormatAstVariableNotFound(format_ast.name)
 
     def visit_FormatAstSequence(self, context, format_ast, bag):
-        if context.debug_enabled:
-            context.debug_entering("DeveloperVisitor", "visit_FormatAstSequence", format_ast=format_ast, bag=bag)
-        return self.set_result(FormatAstSequence([self.visit(context, item, bag) for item in format_ast.items]))
+        if self.debugger.is_enabled():
+            debug_bag = {"format_ast": format_ast, "bag": bag}
+            self.debugger.debug_log(f"Entering visit_FormatAstSequence with {debug_bag}")
+        return format_ast.clone(items=[self.visit(context, item, bag) for item in format_ast.items])
 
     def visit_FormatAstColor(self, context, format_ast, bag):
-        if context.debug_enabled:
-            context.debug_entering("DeveloperVisitor", "visit_FormatAstColor", format_ast=format_ast, bag=bag)
-        return self.set_result(FormatAstColor(format_ast.color, self.visit(context, format_ast.format_ast, bag)))
+        if self.debugger.is_enabled():
+            debug_bag = {"format_ast": format_ast, "bag": bag}
+            self.debugger.debug_log(f"Entering visit_FormatAstColor with {debug_bag}")
+        return format_ast.clone(format_ast=self.visit(context, format_ast.format_ast, bag))
 
     def visit_FormatAstList(self, context, format_ast, bag):
-        if context.debug_enabled:
-            context.debug_entering("DeveloperVisitor", "visit_FormatAstList", format_ast=format_ast, bag=bag)
+        if self.debugger.is_enabled():
+            debug_bag = {"format_ast": format_ast, "bag": bag}
+            self.debugger.debug_log(f"Entering visit_FormatAstList with {debug_bag}")
         try:
             value = evaluate_expression(format_ast.variable, bag)
             if value is None:
-                return self.set_result(FormatAstVariable(format_ast.variable, format_ast.format, None))
+                return FormatAstVariable(format_ast.variable, format_ast.format, value=None)
 
             if hasattr(value, "__iter__"):
                 items = value
@@ -78,22 +91,29 @@ class DeveloperVisitor(OutVisitor):
                 items = evaluate_expression(f"self.{items_props}", {"self": value})
                 if not hasattr(items, "__iter__"):
                     # Definition error ? No list found
-                    return self.set_result(FormatAstVariable(format_ast.variable, None, value))
+                    return FormatAstVariable(format_ast.variable, value=value)
 
-            recursion_depth, recurse_on = self.get_recurse_info(value, format_ast.recursion_depth, format_ast.recurse_on)
+            recursion_depth, recurse_on = self.get_recurse_info(value, format_ast.recursion_depth,
+                                                                format_ast.recurse_on)
 
             result = []  # TODO change into generator
             for i, item in enumerate(items):
                 bag["__item"] = item
-                sub_visitor = DeveloperVisitor(self.sheerka_out, set(), self.list_recursion_depth)
-                result.append(sub_visitor.visit(context, FormatAstVariable("__item", None, item, i), bag))
+                sub_visitor = DeveloperVisitor(self.sheerka_out, self.debugger, set(), self.list_recursion_depth)
+                result.append(sub_visitor.visit(context, FormatAstVariable("__item",
+                                                                           debug=format_ast.debug,
+                                                                           value=item,
+                                                                           index=i), bag))
 
                 # recursion management
                 recursion_depth, recurse_on = self.get_recurse_info(item, recursion_depth, recurse_on)
                 if recursion_depth > 0:
                     sub_items = evaluate_expression(recurse_on, as_bag(item))
                     if sub_items and hasattr(sub_items, "__iter__"):
-                        sub_visitor = DeveloperVisitor(self.sheerka_out, set(), self.list_recursion_depth + 1)
+                        sub_visitor = DeveloperVisitor(self.sheerka_out,
+                                                       self.debugger,
+                                                       set(),
+                                                       self.list_recursion_depth + 1)
                         bag[f"__{recurse_on}"] = sub_items
 
                         sub_items = sub_visitor.visit(context, FormatAstList(f"__{recurse_on}",
@@ -102,29 +122,64 @@ class DeveloperVisitor(OutVisitor):
                                                                              recursion_depth - 1), bag)
                         result.append(sub_items)
 
-            return self.set_result(FormatAstList(variable=format_ast.variable,
-                                                 items_prop=format_ast.items_prop,
-                                                 recurse_on=recurse_on,
-                                                 recursion_depth=recursion_depth,
-                                                 items=result))
+            return format_ast.clone(recurse_on=recurse_on, recursion_depth=recursion_depth, items=result)
         except NameError as error:
-            context.debug("DeveloperVisitor", "visit_FormatAstList", "evaluate_expression", error, is_error=True)
+            self.debugger.debug_log(error, is_error=True)
             var_name = format_ast.variable if error.args[0] == format_ast.variable else \
                 format_ast.variable + "." + error.args[0]
-            return self.set_result(FormatAstVariableNotFound(var_name))
+            return FormatAstVariableNotFound(var_name)
 
     def visit_FormatAstFunction(self, context, format_ast, bag):
-        if context.debug_enabled:
-            context.debug_entering("DeveloperVisitor", "visit_FormatAstFunction", format_ast=format_ast, bag=bag)
-        unknown = FormatAstColor("red", FormatAstRawText(f"function '{format_ast.name}' is unknown"))
-        return self.set_result(unknown)
+        if self.debugger.is_enabled():
+            debug_bag = {"format_ast": format_ast, "bag": bag}
+            self.debugger.debug_log(f"Entering visit_FormatAstFunction with {debug_bag}")
+        return FormatAstColor("red", FormatAstRawText(f"function '{format_ast.name}' is unknown"))
 
-    def set_result(self, result):
-        self._result = result
-        return result
+    def visit_FormatAstDict(self, context, format_ast, bag):
+        if self.debugger.is_enabled():
+            debug_bag = {"format_ast": format_ast, "bag": bag}
+            self.debugger.debug_log(f"Entering visit_FormatAstDict with {debug_bag}")
 
-    def get_result(self):
-        return self._result
+        try:
+            result = []
+
+            value = evaluate_expression(format_ast.variable, bag)
+            if value is None:
+                return FormatAstVariable(format_ast.variable, format_ast.format, value=None)
+
+            if isinstance(value, dict):
+                items = value
+            else:
+                # the variable does not resolve to a dictionary, let's look at one of its attribute
+                items_props = format_ast.items_prop or "body"
+                items = evaluate_expression(f"self.{items_props}", {"self": value})
+                if not isinstance(items, dict):
+                    # Definition error ? No Dictionary found
+                    return FormatAstVariable(format_ast.variable, value=value)
+
+            for i, (k, v) in enumerate(items.items()):
+                bag["__key"] = k
+                key_visitor = DeveloperVisitor(self.sheerka_out, self.debugger, set(), self.list_recursion_depth)
+                key_res = key_visitor.visit(context,
+                                            FormatAstVariable("__key", value=k, index=i, debug=format_ast.debug),
+                                            bag)
+
+                bag["__value"] = v
+                value_visitor = DeveloperVisitor(self.sheerka_out, self.debugger, set(), self.list_recursion_depth)
+                to_visit = FormatAstDict("__value", debug=True, prefix='{', suffix='}') if isinstance(v, dict) else \
+                    FormatAstList("__value", debug=True, prefix='[', suffix=']') if isinstance(v, list) else \
+                        FormatAstVariable("__value", value=v, index=k, debug=format_ast.debug)
+
+                value_res = value_visitor.visit(context, to_visit, bag)
+
+                result.append((key_res, value_res))
+
+            return format_ast.clone(items=result)
+        except NameError as error:
+            self.debugger.debug_log(error, is_error=True)
+            var_name = format_ast.variable if error.args[0] == format_ast.variable else \
+                format_ast.variable + "." + error.args[0]
+            return FormatAstVariableNotFound(var_name)
 
     @staticmethod
     def get_recurse_info(obj, recursion_depth, recurse_on):

src/out/OutVisitor.py

@@ -1,10 +1,10 @@
 class OutVisitor:
-    def visit(self, context, format_ast, bag):
+    def visit(self, format_ast):
         name = format_ast.__class__.__name__
 
         method = 'visit_' + name
         visit_method = getattr(self, method, self.generic_visit)
-        return visit_method(context, format_ast, bag)
+        return visit_method(format_ast)
 
-    def generic_visit(self, context, format_ast, bag):
+    def generic_visit(self, format_ast):
         pass

src/parsers/PythonParser.py

@@ -37,10 +37,15 @@ class PythonNode(Node):
 
     def __init__(self, source, ast_=None, objects=None):
         self.source = source
-        self.ast_ = ast_ if ast_ else ast.parse(source, mode="eval") if source else None
+        self.ast_ = ast_  # if ast_ else ast.parse(source, mode="eval") if source else None
         self.objects = objects or {}  # when objects (mainly concepts or rules) are recognized in the expression
         self.compiled = None
 
+    def init_ast(self):
+        if self.ast_ is None and self.source:
+            self.ast_ = ast.parse(self.source, mode="eval")
+        return self
+
     def get_compiled(self):
         if self.compiled is None:
             self.compiled = compile(self.ast_, "<string>", "eval")
@@ -60,10 +65,12 @@ class PythonNode(Node):
         if self.source != other.source:
             return False
 
-        self_dump = self.get_dump(self.ast_)
-        other_dump = self.get_dump(other.ast_)
+        if self.ast_ and other.ast_:
+            self_dump = self.get_dump(self.ast_)
+            other_dump = self.get_dump(other.ast_)
+            return self_dump == other_dump
 
-        return self_dump == other_dump
+        return True
 
     def __hash__(self):
         return hash((self.source, self.ast_.hash))