I can add tables

Refactoring DbEngine

Fixing unit tests

Fixing unit tests

Fixing unit tests

Refactored DbManager for datagrid

Improving front end performance

I can add new table

Fixed sidebar closing when clicking on it

Fix drag event rebinding, improve listener options, and add debug

Prevent duplicate drag event bindings with a dataset flag and ensure consistent scrollbar functionality. Change wheel event listener to passive mode for better performance. Refactor function naming for consistency, and add debug logs for event handling.

Refactor Datagrid bindings and default state handling.

Updated Javascript to conditionally rebind Datagrid on specific events. Improved Python components by handling empty DataFrame cases and removing redundant code. Revised default state initialization in settings for better handling of mutable fields.

Added Rowindex visualisation support

Working on Debugger with own implementation of JsonViewer

Working on JsonViewer.py

Fixed unit tests

Adding unit tests

I can fold and unfold

fixed unit tests

Adding css for debugger

Added tooltip management

Adding debugger functionalities

Refactor serializers and improve error handling in DB engine

Fixed error where tables were overwritten

I can display footer menu

Working on footer. Refactoring how heights are managed

Refactored scrollbars management

Working on footer menu

I can display footer menu + fixed unit tests

Fixed unit tests

Updated click management

I can display aggregations in footers

Added docker management

Refactor input handling and improve config defaults

Fixed scrollbars colors

Refactored tooltip management

Improved tooltip management

Improving FilterAll

src/components/debugger/DebuggerApp.py

@@ -3,15 +3,30 @@ import logging
 from fasthtml.fastapp import fast_app
 
 from components.debugger.constants import Routes
-from core.instance_manager import InstanceManager
+from core.instance_manager import InstanceManager, debug_session
 
 debugger_app, rt = fast_app()
 
 logger = logging.getLogger("Debugger")
 
 
-@rt(Routes.DbEngine)
-def post(session, _id: str, digest: str = None):
-  logger.debug(f"Entering {Routes.DbEngine} with args {_id=}, {digest=}")
+@rt(Routes.DbEngineData)
+def post(session, _id: str, user_id: str, digest: str = None):
+  logger.debug(f"Entering {Routes.DbEngineData} with args {debug_session(session)}, {_id=}, {user_id=}, {digest=}")
   instance = InstanceManager.get(session, _id)
-  return instance.add_tab(digest)
+  return instance.add_tab(user_id, digest)
+
+
+@rt(Routes.JsonViewerFold)
+def post(session, _id: str, node_id: str, folding: str):
+  logger.debug(f"Entering {Routes.JsonViewerFold} with args {debug_session(session)}, {_id=}, {node_id=}, {folding=}")
+  instance = InstanceManager.get(session, _id)
+  instance.set_node_folding(node_id, folding)
+  return instance.render_node(node_id)
+
+@rt(Routes.JsonOpenDigest)
+def post(session, _id: str, user_id: str, digest: str):
+  logger.debug(f"Entering {Routes.JsonOpenDigest} with args {debug_session(session)}, {_id=}, {user_id=}, {digest=}")
+  instance = InstanceManager.get(session, _id)
+  return instance.open_digest(user_id, digest)
+  

src/components/debugger/assets/Debugger.css

@@ -0,0 +1,72 @@
+:root:has(input.theme-controller[value=light]:checked),
+[data-theme="light"] {
+    --json-bool: oklch(75% 0.183 55.934); /* tailwindcss orange-400 */
+    --json-string: oklch(79.2% 0.209 151.711); /* tailwindcss green-400 */
+    --json-number: oklch(70.7% 0.165 254.624); /* tailwindcss blue-400 */
+    --json-object: oklch(57.7% 0.245 27.325); /* tailwindcss red-600 */
+    --json-null: var(--color-base-content);
+    --json-digest: var(--color-base-content);
+}
+
+:root:has(input.theme-controller[value=dark]:checked),
+[data-theme="dark"] {
+    --json-bool: oklch(88.5% 0.062 18.334); /* tailwindcss orange-200 */
+    --json-string: oklch(92.5% 0.084 155.995); /* tailwindcss green-200 */
+    --json-number: oklch(88.2% 0.059 254.128); /* tailwindcss blue-200 */
+    --json-object: oklch(44.4% 0.177 26.899); /* tailwindcss red-800 */
+    --json-null: var(--color-base-content);
+    --json-digest: var(--color-base-content);
+}
+
+
+:root:has(input.theme-controller[value=cupcake]:checked),
+[data-theme="cupcake"] {
+    --json-bool: oklch(75% 0.183 55.934); /* tailwindcss orange-400 */
+    --json-string: oklch(79.2% 0.209 151.711); /* tailwindcss green-400 */
+    --json-number: oklch(70.7% 0.165 254.624); /* tailwindcss blue-400 */
+    --json-object: oklch(57.7% 0.245 27.325); /* tailwindcss red-600 */
+    --json-null: var(--color-base-content); /* tailwindcss violet-400 */
+    --json-digest: var(--color-base-content);
+}
+
+
+.mmt-jsonviewer {
+    font-family: 'Consolas', 'Monaco', 'Courier New', monospace;
+    font-size: 14px;
+}
+
+/* Use inherited CSS variables for your custom theme */
+.mmt-jsonviewer-bool {
+    color: var(--json-bool);
+}
+
+.mmt-jsonviewer-string {
+    color: var(--json-string);
+}
+
+.mmt-jsonviewer-number {
+    color: var(--json-number);
+}
+
+.mmt-jsonviewer-null {
+    color: var(--json-null);
+}
+
+.mmt-jsonviewer-digest {
+    color: var(--json-digest);
+    cursor: pointer;
+
+}
+
+.mmt-jsonviewer-object {
+    color: var(--json-object);
+}
+
+/*:root:has(input.theme-controller[value=dark]:checked),*/
+/*[data-theme="dark"] {*/
+/*    --json-bool: oklch(40.8% 0.123 38.172); !* tailwindcss orange-900 *!*/
+/*    --json-string: oklch(39.3% 0.095 152.535); !* tailwindcss green-900 *!*/
+/*    --json-number: oklch(37.9% 0.146 265.522); !* tailwindcss blue-900 *!*/
+/*    --json-null: var(--color-base-content);*/
+/*    --json-digest: var(--color-base-content);*/
+/*}*/

src/components/debugger/assets/Debugger.js

@@ -1,5 +1,5 @@
 // Import the svelte-jsoneditor module
-import {createJSONEditor} from 'https://cdn.jsdelivr.net/npm/vanilla-jsoneditor/standalone.js';
+// import {createJSONEditor} from 'https://cdn.jsdelivr.net/npm/vanilla-jsoneditor/standalone.js';
 
 /**
  * Initializes and displays a JSON editor using the Svelte JSON Editor.

src/components/debugger/assets/icons.py

@@ -7,3 +7,33 @@ icon_dbengine = NotStr("""<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="h
     </path>
   </g>
 </svg>""")
+
+# Fluent CaretRight20Filled
+icon_collapsed = NotStr("""<svg name="collapsed" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 0 20 20">
+  <g fill="none">
+    <path d="M7 14.204a1 1 0 0 0 1.628.778l4.723-3.815a1.5 1.5 0 0 0 0-2.334L8.628 5.02A1 1 0 0 0 7 5.797v8.407z" fill="currentColor">
+    </path>
+  </g>
+</svg>""")
+
+# Fluent CaretDown20Filled
+icon_expanded = NotStr("""<svg name="expanded" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 0 20 20">
+  <g fill="none">
+    <path d="M5.797 7a1 1 0 0 0-.778 1.628l3.814 4.723a1.5 1.5 0 0 0 2.334 0l3.815-4.723A1 1 0 0 0 14.204 7H5.797z" fill="currentColor">
+    </path>
+  </g>
+</svg>""")
+
+icon_class = NotStr("""
+<svg name="expanded" viewBox="0 0 20 20" xmlns="http://www.w3.org/2000/svg">
+ <g fill="none" stroke="currentColor" stroke-width="1.5" >
+  <polygon points="5,2 2,8 8,8" />
+  <rect x="12" y="2" width="6" height="6"/>
+  <circle cx="5" cy="15" r="3" />
+  <polygon points="11.5,15 15,11.5 18.5,15 15,18.5" />
+  </g>
+</svg>
+
+
+"""
+                       )

src/components/debugger/commands.py

@@ -6,11 +6,40 @@ class Commands:
     self._owner = owner
     self._id = owner.get_id()
   
-  
-  def show_dbengine(self):
+  def db_engine_data(self, user_id: str):
     return {
-        "hx-post": f"{ROUTE_ROOT}{Routes.DbEngine}",
+        "hx-post": f"{ROUTE_ROOT}{Routes.DbEngineData}",
         "hx-target": f"#{self._owner.tabs_manager.get_id()}",
         "hx-swap": "outerHTML",
-        "hx-vals": f'{{"_id": "{self._id}"}}',
-    }
+        "hx-vals": f'{{"_id": "{self._id}", "user_id": "{user_id}"}}',
+    }
+  
+  def db_engine_refs(self, ref_id: str | None):
+    return {
+        "hx-post": f"{ROUTE_ROOT}{Routes.DbEngineRefs}",
+        "hx-target": f"#{self._owner.tabs_manager.get_id()}",
+        "hx-swap": "outerHTML",
+        "hx-vals": f'{{"_id": "{self._id}", "ref_id": "{ref_id}"}}',
+    }
+
+
+class JsonViewerCommands:
+  def __init__(self, owner):
+    self._owner = owner
+    self._id = owner.get_id()
+  
+  def fold(self, node_id: str, folding: str):
+    return {
+        "hx-post": f"{ROUTE_ROOT}{Routes.JsonViewerFold}",
+        "hx-target": f"#{node_id}",
+        "hx-swap": "outerHTML",
+        "hx-vals": f'{{"_id": "{self._id}", "node_id": "{node_id}", "folding": "{folding}"}}',
+    }
+  
+  def open_digest(self, user_id, digest):
+    return {
+        "hx-post": f"{ROUTE_ROOT}{Routes.JsonOpenDigest}",
+        "hx-target": f"#{self._owner.get_owner().tabs_manager.get_id()}",
+        "hx-swap": "outerHTML",
+        "hx-vals": f'{{"_id": "{self._id}", "user_id": "{user_id}", "digest": "{digest}"}}',
+    }

src/components/debugger/components/Debugger.py

@@ -1,4 +1,3 @@
-import json
 import logging
 
 from fasthtml.components import *
@@ -6,13 +5,15 @@ from fasthtml.components import *
 from components.BaseComponent import BaseComponent
 from components.debugger.assets.icons import icon_dbengine
 from components.debugger.commands import Commands
-from components.debugger.components.DbEngineDebugger import DbEngineDebugger
+from components.debugger.components.JsonViewer import JsonViewer
 from components.debugger.constants import DBENGINE_DEBUGGER_INSTANCE_ID
 from components_helpers import mk_ellipsis, mk_icon
+from core.instance_manager import InstanceManager
 from core.utils import get_unique_id
 
 logger = logging.getLogger("Debugger")
 
+
 class Debugger(BaseComponent):
   def __init__(self, session, _id, settings_manager, tabs_manager):
     super().__init__(session, _id)
@@ -21,26 +22,51 @@ class Debugger(BaseComponent):
     self.tabs_manager = tabs_manager
     self.commands = Commands(self)
   
-  def add_tab(self, digest):
-    content = self.mk_db_engine(digest)
+  def add_tab(self, user_id, digest):
+    content = self.mk_db_engine_object(user_id, digest)
     tab_key = f"debugger-dbengine-{digest}"
     title = f"DBEngine-{digest if digest else 'head'}"
     self.tabs_manager.add_tab(title, content, key=tab_key)
     return self.tabs_manager.render()
   
-  def mk_db_engine(self, digest):
-    data = self.db_engine.debug_load(digest) if digest else self.db_engine.debug_head()
+  def mk_db_engine_object(self, user_id, digest):
+    data = self.db_engine.debug_load(user_id, digest) if digest else self.db_engine.debug_head(user_id)
+
     logger.debug(f"mk_db_engine: {data}")
-    return DbEngineDebugger(self._session, self._id, self, json.dumps(data))
+    return InstanceManager.get(self._session,
+                               JsonViewer.create_component_id(self._session, prefix=self._id),
+                               JsonViewer,
+                               owner=self,
+                               user_id=user_id,
+                               data=data)
+  
+  def mk_db_engine(self, selected):
+    return Div(
+      Input(type="radio",
+            name=f"dbengine-accordion-{self._id}",
+            checked="checked" if selected else None,
+            cls="p-0! min-h-0!",
+            ),
+      Div(
+        mk_icon(icon_dbengine, can_select=False), mk_ellipsis("DbEngine", cls="text-sm"),
+        cls="collapse-title p-0 min-h-0 flex truncate",
+      ),
+      Div(
+        *[Div(user_id, **self.commands.db_engine_data(user_id)) for user_id in self.db_engine.debug_users()],
+        Div("refs", **self.commands.db_engine_refs(None)),
+        cls="collapse-content pr-0! truncate",
+      ),
+      cls="collapse mb-2",
+      id=f"db_engine_{self._id}",
+    )
   
   def __ft__(self):
     return Div(
       Div(cls="divider"),
       mk_ellipsis("Debugger", cls="text-sm font-medium mb-1"),
       Div(
-        mk_icon(icon_dbengine, can_select=False), mk_ellipsis("DbEngine"),
+        self.mk_db_engine(True),
         cls="flex truncate",
-        **self.commands.show_dbengine(),
       ),
       
       id=self._id,

src/components/debugger/components/JsonViewer.py

@@ -0,0 +1,286 @@
+import dataclasses
+from typing import Any
+
+from fasthtml.components import *
+from pandas import DataFrame
+
+from components.BaseComponent import BaseComponent
+from components.datagrid_new.components.DataGrid import DataGrid
+from components.debugger.assets.icons import icon_expanded, icon_collapsed, icon_class
+from components.debugger.commands import JsonViewerCommands
+from components.debugger.constants import INDENT_SIZE, MAX_TEXT_LENGTH, NODE_OBJECT, NODES_KEYS_TO_NOT_EXPAND
+from core.serializer import TAG_OBJECT
+from core.utils import get_unique_id
+
+
+class FoldingMode:
+  COLLAPSE = "collapse"
+  EXPAND = "expand"
+
+
+@dataclasses.dataclass
+class Node:
+  value: Any
+
+
+@dataclasses.dataclass
+class ValueNode(Node):
+  hint: str = None
+
+
+@dataclasses.dataclass
+class ListNode(Node):
+  node_id: str
+  level: int
+  children: list[Node] = dataclasses.field(default_factory=list)
+
+
+@dataclasses.dataclass
+class DictNode(Node):
+  node_id: str
+  level: int
+  children: dict[str, Node] = dataclasses.field(default_factory=dict)
+
+
+class JsonViewer(BaseComponent):
+  def __init__(self, session, _id, owner, user_id, data):
+    super().__init__(session, _id)
+    self._owner = owner  # debugger component
+    self.user_id = user_id
+    self.data = data
+    self._node_id = -1
+    self._commands = JsonViewerCommands(self)
+    
+    # A little explanation on how the folding / unfolding work
+    # all the nodes are either fold or unfold... except if there are not
+    # self._folding_mode keeps the current value (it's FoldingMode.COLLAPSE or FoldingMode.EXPAND
+    # self._nodes_to_track keeps track of the exceptions
+    # The idea is to minimize the memory usage
+    self._folding_mode = FoldingMode.COLLAPSE
+    self._nodes_to_track = set()  # all nodes that are expanded when _fold_mode and vice versa
+    
+    self._nodes_by_id = {}
+    
+    self.node = self._create_node(None, data)
+  
+  def set_node_folding(self, node_id, folding):
+    if folding == self._folding_mode:
+      self._nodes_to_track.remove(node_id)
+    else:
+      self._nodes_to_track.add(node_id)
+  
+  def render_node(self, node_id):
+    key, node = self._nodes_by_id[node_id]
+    return self._render_node(key, node)
+  
+  def set_folding_mode(self, folding_mode):
+    self._folding_mode = folding_mode
+    self._nodes_to_track.clear()
+  
+  def get_folding_mode(self):
+    return self._folding_mode
+  
+  def get_owner(self):
+    return self._owner
+  
+  def open_digest(self, user_id: str, digest: str):
+    return self._owner.add_tab(user_id, digest)
+  
+  def _create_node(self, key, data, level=0):
+    if isinstance(data, list):
+      node_id = self._get_next_id()
+      if level <= 1 and key not in NODES_KEYS_TO_NOT_EXPAND:
+        self._nodes_to_track.add(node_id)
+      node = ListNode(data, node_id, level)
+      self._nodes_by_id[node_id] = (key, node)
+      for index, item in enumerate(data):
+        node.children.append(self._create_node(index, item, level + 1))
+    
+    elif isinstance(data, dict):
+      node_id = self._get_next_id()
+      if level <= 1 and key not in NODES_KEYS_TO_NOT_EXPAND:
+        self._nodes_to_track.add(node_id)
+      node = DictNode(data, node_id, level)
+      self._nodes_by_id[node_id] = (key, node)
+      for key, value in data.items():
+        node.children[key] = self._create_node(key, value, level + 1)
+    
+    else:
+      if key == TAG_OBJECT:
+        hint = NODE_OBJECT
+      else:
+        hint = None
+      node = ValueNode(data, hint)
+    
+    return node
+  
+  def _must_expand(self, node):
+    if self._folding_mode == FoldingMode.COLLAPSE:
+      return node.node_id in self._nodes_to_track
+    else:
+      return node.node_id not in self._nodes_to_track
+  
+  def _mk_folding(self, node: Node):
+    if not isinstance(node, (ListNode, DictNode)):
+      return None
+    
+    must_expand = self._must_expand(node)
+    
+    return Span(icon_expanded if must_expand else icon_collapsed,
+                cls="icon-16-inline mmt-jsonviewer-folding",
+                style=f"margin-left: -{INDENT_SIZE}px;",
+                **self._commands.fold(node.node_id, FoldingMode.COLLAPSE if must_expand else FoldingMode.EXPAND)
+                )
+  
+  def _get_next_id(self):
+    self._node_id += 1
+    return f"{self._id}-{self._node_id}"
+  
+  def _render_value(self, node):
+    def _is_sha256(_value):
+      return isinstance(_value, str) and len(_value) == 64 and all(
+        c in '0123456789abcdefABCDEF' for c in _value)
+    
+    if isinstance(node, DictNode):
+      return self._render_dict(node)
+    elif isinstance(node, ListNode):
+      return self._render_list(node)
+    else:
+      data_tooltip = None
+      htmx_params = {}
+      icon = None
+      
+      if isinstance(node.value, bool):  # order is important bool is an int in Python !
+        str_value = "true" if node.value else "false"
+        data_class = "bool"
+      elif isinstance(node.value, (int, float)):
+        str_value = str(node.value)
+        data_class = "number"
+      elif node.value is None:
+        str_value = "null"
+        data_class = "null"
+      elif _is_sha256(node.value):
+        str_value = str(node.value)
+        data_class = "digest"
+        htmx_params = self._commands.open_digest(self.user_id, node.value)
+      elif node.hint == NODE_OBJECT:
+        icon = icon_class
+        str_value = node.value.split(".")[-1]
+        data_class = "object"
+      elif isinstance(node.value, DataFrame):
+        dg = DataGrid(self._session)
+        dg.init_from_dataframe(node.value)
+        str_value = dg
+        data_class = "dataframe"
+      else:
+        as_str = str(node.value)
+        if len(as_str) > MAX_TEXT_LENGTH:
+          str_value = as_str[:MAX_TEXT_LENGTH] + "..."
+          data_tooltip = as_str
+        else:
+          str_value = as_str
+        str_value = self.add_quotes(str_value)
+        data_class = "string"
+      
+      if data_tooltip is not None:
+        cls = f"mmt-jsonviewer-{data_class} mmt-tooltip"
+      else:
+        cls = f"mmt-jsonviewer-{data_class}"
+      
+      if icon is not None:
+        return Span(Span(icon, cls="icon-16-inline mr-1"),
+                    Span(str_value, data_tooltip=data_tooltip, **htmx_params),
+                    cls=cls)
+      
+      return Span(str_value, cls=cls, data_tooltip=data_tooltip, **htmx_params)
+  
+  def _render_dict(self, node: DictNode):
+    if self._must_expand(node):
+      return Span("{",
+                  *[
+                      self._render_node(key, value)
+                      for key, value in node.children.items()
+                  ],
+                  Div("}"),
+                  id=node.node_id)
+    else:
+      return Span("{...}", id=node.node_id)
+  
+  def _render_list(self, node: ListNode):
+    def _all_the_same(_node):
+      if len(_node.children) == 0:
+        return False
+      
+      sample_value = _node.children[0].value
+      
+      if sample_value is None:
+        return False
+      
+      type_ = type(sample_value)
+      if type_ in (int, float, str, bool, list, dict, ValueNode):
+        return False
+      
+      return all(type(item.value) == type_ for item in _node.children)
+    
+    def _render_as_grid(_node):
+      type_ = type(_node.children[0].value)
+      icon = icon_class
+      str_value = type_.__name__.split(".")[-1]
+      
+      data = [child.value.__dict__ for child in _node.children]
+      df = DataFrame(data)
+      dg = DataGrid(self._session)
+      dg.init_from_dataframe(df)
+      return Span(Span(Span(icon, cls="icon-16-inline mr-1"), Span(str_value), cls="mmt-jsonviewer-object"),
+                  dg,
+                  id=_node.node_id)
+    
+    def _render_as_list(_node):
+      return Span("[",
+                  *[
+                      self._render_node(index, item)
+                      for index, item in enumerate(_node.children)
+                  ],
+                  Div("]"),
+                  )
+    
+    if self._must_expand(node):
+      if _all_the_same(node):
+        return _render_as_grid(node)
+      return _render_as_list(node)
+    else:
+      return Span("[...]", id=node.node_id)
+  
+  def _render_node(self, key, node):
+    return Div(
+      self._mk_folding(node),
+      Span(f'{key} : ') if key is not None else None,
+      self._render_value(node),
+      style=f"margin-left: {INDENT_SIZE}px;",
+      id=node.node_id if hasattr(node, "node_id") else None,
+    )
+  
+  def __ft__(self):
+    return Div(
+      Div(self._render_node(None, self.node), id=f"{self._id}-root"),
+      cls="mmt-jsonviewer",
+      id=f"{self._id}")
+  
+  @staticmethod
+  def add_quotes(value: str):
+    if '"' in value and "'" in value:
+      # Value contains both double and single quotes, escape double quotes
+      return f'"{value.replace("\"", "\\\"")}"'
+    elif '"' in value:
+      # Value contains double quotes, use single quotes
+      return f"'{value}'"
+    else:
+      # Default case, use double quotes
+      return f'"{value}"'
+  
+  @staticmethod
+  def create_component_id(session, prefix=None, suffix=None):
+    if suffix is None:
+      suffix = get_unique_id()
+    
+    return f"{prefix}{suffix}"

src/components/debugger/constants.py

@@ -1,6 +1,14 @@
 DBENGINE_DEBUGGER_INSTANCE_ID = "debugger"
 ROUTE_ROOT = "/debugger"
 
+INDENT_SIZE = 20
+MAX_TEXT_LENGTH = 50
+
+NODE_OBJECT = "Object"
+NODES_KEYS_TO_NOT_EXPAND = ["Dataframe", "__parent__"]
+
 class Routes:
-  DbEngine = "/dbengine"  # request the filtering in the grid
-  
+  DbEngineData = "/dbengine-data"
+  DbEngineRefs = "/dbengine-refs"
+  JsonViewerFold = "/jsonviewer-fold"
+  JsonOpenDigest = "/jsonviewer-open-digest"