Added traceability

.gitignore

@@ -13,6 +13,7 @@ tools.db
 .mytools_db
 .idea/MyManagingTools.iml
 .idea/misc.xml
+**/*.prof
 
 # Created by .ignore support plugin (hsz.mobi)
 ### Python template

Makefile

@@ -18,6 +18,8 @@ clean:
 	rm -rf Untitled*.ipynb
 	rm -rf .ipynb_checkpoints
 	rm -rf src/tools.db
+	rm -rf src/*.out
+	rm -rf src/*.prof
 	find . -name '.sesskey' -exec rm -rf {} +
 	find . -name '.pytest_cache' -exec rm -rf {} +
 	find . -name '__pycache__' -exec rm -rf {} +

README.md

@@ -35,3 +35,10 @@ docker-compose down
 ```shell
 docker-compose build
 ```
+
+# Profiling
+```shell
+cd src
+python -m cProfile -o profile.out main.py
+snakeviz profile.out # 'pip install snakeviz' if snakeviz is not installed
+```

src/assets/main.js

@@ -25,11 +25,19 @@ function bindTooltipsWithDelegation() {
 
     // Add a single mouseenter and mouseleave listener to the parent element
     element.addEventListener("mouseenter", (event) => {
+        //console.debug("Entering element", event.target)
+
         const cell = event.target.closest("[data-tooltip]");
-        if (!cell) return;
+        if (!cell) {
+          // console.debug("  No 'data-tooltip' attribute found. Stopping.");
+          return;
+        }
 
         const no_tooltip = element.hasAttribute("mmt-no-tooltip");
-        if (no_tooltip) return;
+        if (no_tooltip) {
+          // console.debug("  Attribute 'mmt-no-tooltip' found. Cancelling.");
+          return;
+        }
 
         const content = cell.querySelector(".truncate") || cell;
         const isOverflowing = content.scrollWidth > content.clientWidth;

src/components/BaseComponent.py

@@ -1,4 +1,4 @@
-from core.utils import get_user_id
+from core.utils import get_user_id, get_unique_id
 
 
 class BaseComponent:
@@ -51,3 +51,12 @@ class BaseComponentSingleton(BaseComponent):
   @classmethod
   def create_component_id(cls, session):
     return f"{cls.COMPONENT_INSTANCE_ID}{session['user_id']}"
+
+
+class BaseComponentMultipleInstance(BaseComponent):
+  COMPONENT_INSTANCE_ID = None
+  
+  @classmethod
+  def create_component_id(cls, session):
+    component_id = cls.COMPONENT_INSTANCE_ID or cls.__name__
+    return get_unique_id(f"{component_id}{session['user_id']}")

src/components/datagrid_new/DataGridApp.py

@@ -136,3 +136,10 @@ def post(session, _id: str, state: str, args: str = None):
   logger.debug(f"Entering on_state_changed with args {_id=}, {state=}, {args=}")
   instance = InstanceManager.get(session, _id)
   return instance.manage_state_changed(state, args)
+
+
+@rt(Routes.GetPage)
+def get(session, _id: str, page_index: int):
+  logger.debug(f"Entering {Routes.GetPage} with args {_id=}, {page_index=}")
+  instance = InstanceManager.get(session, _id)
+  return instance.mk_body_content_page(page_index)

src/components/datagrid_new/assets/Datagrid.js

@@ -1,6 +1,6 @@
 function bindDatagrid(datagridId, allowColumnsReordering) {
-    bindScrollbars(datagridId);
+    manageScrollbars(datagridId, true);
-    makeResizable(datagridId)
+    makeResizable(datagridId);
 }
 
 function bindScrollbars(datagridId) {
@@ -21,7 +21,7 @@ function bindScrollbars(datagridId) {
     const table = datagrid.querySelector(".dt2-table");
 
     if (!verticalScrollbar || !verticalWrapper || !horizontalScrollbar || !horizontalWrapper || !body || !table) {
-        console.error("Essential scrollbar or content elements are missing in the datagrid.");
+        console.error("Essential scrollbars or content elements are missing in the datagrid.");
         return;
     }
 
@@ -176,6 +176,224 @@ function bindScrollbars(datagridId) {
     });
 }
 
+function manageScrollbars(datagridId, binding) {
+    console.debug("manageScrollbars on element " + datagridId + " with binding=" + binding);
+
+    const datagrid = document.getElementById(datagridId);
+
+    if (!datagrid) {
+        console.error(`Datagrid with id "${datagridId}" not found.`);
+        return;
+    }
+
+    const verticalScrollbar = datagrid.querySelector(".dt2-scrollbars-vertical");
+    const verticalWrapper = datagrid.querySelector(".dt2-scrollbars-vertical-wrapper");
+    const horizontalScrollbar = datagrid.querySelector(".dt2-scrollbars-horizontal");
+    const horizontalWrapper = datagrid.querySelector(".dt2-scrollbars-horizontal-wrapper");
+    const body = datagrid.querySelector(".dt2-body");
+    const table = datagrid.querySelector(".dt2-table");
+
+    if (!verticalScrollbar || !verticalWrapper || !horizontalScrollbar || !horizontalWrapper || !body || !table) {
+        console.error("Essential scrollbars or content elements are missing in the datagrid.");
+        return;
+    }
+
+    const computeScrollbarVisibility = () => {
+        // Determine if the content is clipped
+        const isVerticalRequired = body.scrollHeight > body.clientHeight;
+        const isHorizontalRequired = table.scrollWidth > table.clientWidth;
+
+        // Show or hide the scrollbar wrappers
+        requestAnimationFrame(() => {
+            verticalWrapper.style.display = isVerticalRequired ? "block" : "none";
+            horizontalWrapper.style.display = isHorizontalRequired ? "block" : "none";
+        });
+    };
+
+    const computeScrollbarSize = () => {
+        // Vertical scrollbar height
+        const visibleHeight = body.clientHeight;
+        const totalHeight = body.scrollHeight;
+        const wrapperHeight = verticalWrapper.offsetHeight;
+
+        let scrollbarHeight = 0;
+        if (totalHeight > 0) {
+            scrollbarHeight = (visibleHeight / totalHeight) * wrapperHeight;
+        }
+
+        // Horizontal scrollbar width
+        const visibleWidth = table.clientWidth;
+        const totalWidth = table.scrollWidth;
+        const wrapperWidth = horizontalWrapper.offsetWidth;
+
+        let scrollbarWidth = 0;
+        if (totalWidth > 0) {
+            scrollbarWidth = (visibleWidth / totalWidth) * wrapperWidth;
+        }
+
+        requestAnimationFrame(() => {
+            verticalScrollbar.style.height = `${scrollbarHeight}px`;
+            horizontalScrollbar.style.width = `${scrollbarWidth}px`;
+        });
+    };
+
+    const updateVerticalScrollbarForMouseWheel = () => {
+        const maxScrollTop = body.scrollHeight - body.clientHeight;
+        const wrapperHeight = verticalWrapper.offsetHeight;
+
+        if (maxScrollTop > 0) {
+            const scrollRatio = wrapperHeight / body.scrollHeight;
+            verticalScrollbar.style.top = `${body.scrollTop * scrollRatio}px`;
+        }
+    };
+
+    if (binding) {
+        // Clean up existing managers if they exist
+        if (datagrid._managers) {
+            // Remove drag events
+            if (datagrid._managers.dragManager) {
+                verticalScrollbar.removeEventListener("mousedown", datagrid._managers.dragManager.verticalMouseDown);
+                horizontalScrollbar.removeEventListener("mousedown", datagrid._managers.dragManager.horizontalMouseDown);
+                document.removeEventListener("mousemove", datagrid._managers.dragManager.mouseMove);
+                document.removeEventListener("mouseup", datagrid._managers.dragManager.mouseUp);
+            }
+
+            // Remove wheel events
+            if (datagrid._managers.wheelManager) {
+                body.removeEventListener("wheel", datagrid._managers.wheelManager.handleWheelScrolling);
+            }
+
+            // Remove resize events
+            if (datagrid._managers.resizeManager) {
+                window.removeEventListener("resize", datagrid._managers.resizeManager.handleResize);
+            }
+        }
+
+        // Create managers
+        const dragManager = {
+            isDragging: false,
+            startY: 0,
+            startX: 0,
+
+            updateVerticalScrollbar: (deltaX, deltaY) => {
+                const wrapperHeight = verticalWrapper.offsetHeight;
+                const scrollbarHeight = verticalScrollbar.offsetHeight;
+                const maxScrollTop = body.scrollHeight - body.clientHeight;
+                const scrollRatio = maxScrollTop / (wrapperHeight - scrollbarHeight);
+
+                let newTop = parseFloat(verticalScrollbar.style.top || "0") + deltaY;
+                newTop = Math.max(0, Math.min(newTop, wrapperHeight - scrollbarHeight));
+
+                verticalScrollbar.style.top = `${newTop}px`;
+                body.scrollTop = newTop * scrollRatio;
+            },
+
+            updateHorizontalScrollbar: (deltaX, deltaY) => {
+                const wrapperWidth = horizontalWrapper.offsetWidth;
+                const scrollbarWidth = horizontalScrollbar.offsetWidth;
+                const maxScrollLeft = table.scrollWidth - table.clientWidth;
+                const scrollRatio = maxScrollLeft / (wrapperWidth - scrollbarWidth);
+
+                let newLeft = parseFloat(horizontalScrollbar.style.left || "0") + deltaX;
+                newLeft = Math.max(0, Math.min(newLeft, wrapperWidth - scrollbarWidth));
+
+                horizontalScrollbar.style.left = `${newLeft}px`;
+                table.scrollLeft = newLeft * scrollRatio;
+            },
+
+            verticalMouseDown: (e) => {
+                disableTooltip();
+                dragManager.isDragging = true;
+                dragManager.startY = e.clientY;
+                dragManager.startX = e.clientX;
+                document.body.style.userSelect = "none";
+                verticalScrollbar.classList.add("dt2-dragging");
+            },
+
+            horizontalMouseDown: (e) => {
+                disableTooltip();
+                dragManager.isDragging = true;
+                dragManager.startY = e.clientY;
+                dragManager.startX = e.clientX;
+                document.body.style.userSelect = "none";
+                horizontalScrollbar.classList.add("dt2-dragging");
+            },
+
+            mouseMove: (e) => {
+                if (dragManager.isDragging) {
+                    const deltaY = e.clientY - dragManager.startY;
+                    const deltaX = e.clientX - dragManager.startX;
+
+                    // Determine which scrollbar is being dragged
+                    if (verticalScrollbar.classList.contains("dt2-dragging")) {
+                        dragManager.updateVerticalScrollbar(deltaX, deltaY);
+                    } else if (horizontalScrollbar.classList.contains("dt2-dragging")) {
+                        dragManager.updateHorizontalScrollbar(deltaX, deltaY);
+                    }
+
+                    // Reset start points for next update
+                    dragManager.startY = e.clientY;
+                    dragManager.startX = e.clientX;
+                }
+            },
+
+            mouseUp: () => {
+                dragManager.isDragging = false;
+                document.body.style.userSelect = "";
+                verticalScrollbar.classList.remove("dt2-dragging");
+                horizontalScrollbar.classList.remove("dt2-dragging");
+                enableTooltip();
+            }
+        };
+
+        const wheelManager = {
+            handleWheelScrolling: (event) => {
+                const deltaX = event.deltaX;
+                const deltaY = event.deltaY;
+
+                // Scroll the body and table content
+                body.scrollTop += deltaY; // Vertical scrolling
+                table.scrollLeft += deltaX; // Horizontal scrolling
+
+                // Update the vertical scrollbar position
+                updateVerticalScrollbarForMouseWheel();
+
+                // Prevent default behavior to fully manage the scroll
+                event.preventDefault();
+            }
+        };
+
+        const resizeManager = {
+            handleResize: () => {
+                computeScrollbarVisibility();
+                computeScrollbarSize();
+                updateVerticalScrollbarForMouseWheel();
+            }
+        };
+
+        // Store managers on datagrid for cleanup
+        datagrid._managers = {
+            dragManager,
+            wheelManager,
+            resizeManager
+        };
+
+        // Bind events
+        verticalScrollbar.addEventListener("mousedown", dragManager.verticalMouseDown);
+        horizontalScrollbar.addEventListener("mousedown", dragManager.horizontalMouseDown);
+        document.addEventListener("mousemove", dragManager.mouseMove);
+        document.addEventListener("mouseup", dragManager.mouseUp);
+
+        body.addEventListener("wheel", wheelManager.handleWheelScrolling, {passive: false});
+
+        window.addEventListener("resize", resizeManager.handleResize);
+    }
+
+    // Always execute computations
+    computeScrollbarVisibility();
+    computeScrollbarSize();
+}
+
 function makeResizable(datagridId) {
     console.debug("makeResizable on element " + datagridId);
 
@@ -495,3 +713,4 @@ function onAfterSettle(datagridId, event) {
         bindDatagrid(datagridId)
     }
 }
+

src/components/datagrid_new/components/DataGrid.py

@@ -1,4 +1,5 @@
 import copy
+import html
 import logging
 from io import BytesIO
 from typing import Literal, Any
@@ -20,9 +21,10 @@ from components.datagrid_new.db_management import DataGridDbManager
 from components.datagrid_new.settings import DataGridRowState, DataGridColumnState, \
   DataGridFooterConf, DataGridState, DataGridSettings, DatagridView
 from components_helpers import mk_icon, mk_ellipsis
+from core.fasthtml_helper import MyDiv, mk_my_ellipsis, MySpan, mk_my_icon
 from core.instance_manager import InstanceManager
 from core.settings_management import SettingsManager
-from core.utils import get_unique_id, make_safe_id
+from core.utils import get_unique_id, make_safe_id, timed
 
 logger = logging.getLogger("DataGrid")
 
@@ -59,6 +61,8 @@ class DataGrid(BaseComponent):
     self._state: DataGridState = self._db.load_state()
     self._settings: DataGridSettings = grid_settings or self._db.load_settings()
     self._df: DataFrame | None = self._db.load_dataframe()
+    self._fast_access = self._init_fast_access(self._df)
+    self._total_rows = len(self._df) if self._df is not None else 0
     
     # update boundaries if possible
     self.set_boundaries(boundaries)
@@ -118,14 +122,23 @@ class DataGrid(BaseComponent):
       else:
         return ColumnType.Text  # Default to Text if no match
     
-    self._df = df.copy()
+    def _init_columns(_df):
-    self._df.columns = self._df.columns.map(make_safe_id)  # make sure column names are trimmed
+      columns = [DataGridColumnState(make_safe_id(col_id),
-    self._state.rows = [DataGridRowState(row_id) for row_id in self._df.index]
-    self._state.columns = [DataGridColumnState(make_safe_id(col_id),
                                      col_index,
                                      col_id,
                                      _get_column_type(self._df[make_safe_id(col_id)].dtype))
-                           for col_index, col_id in enumerate(df.columns)]
+                 for col_index, col_id in enumerate(_df.columns)]
+      if self._state.row_index:
+        columns.insert(0, DataGridColumnState(make_safe_id(ROW_INDEX_ID), -1, " ", ColumnType.RowIndex))
+      
+      return columns
+    
+    self._df = df.copy()
+    self._df.columns = self._df.columns.map(make_safe_id)  # make sure column names are trimmed
+    self._state.rows = [DataGridRowState(row_id) for row_id in self._df.index]
+    self._state.columns = _init_columns(df)  # use df not self._df to keep the original title
+    self._fast_access = self._init_fast_access(self._df)
+    self._total_rows = len(self._df) if self._df is not None else 0
     
     if save_state:
       self._db.save_all(None, self._state, self._df)
@@ -205,6 +218,7 @@ class DataGrid(BaseComponent):
       
       self._state.columns = new_columns_states
     
+    self._fast_access = self._init_fast_access(self._df)
     self._views.recompute_need_save()
     
     self._db.save_all(self._settings, self._state, self._df if new_column else None)
@@ -439,7 +453,7 @@ class DataGrid(BaseComponent):
       _mk_keyboard_management(),
       Div(
         self.mk_table_header(),
-        self.mk_table_body(),
+        self.mk_table_body_page(),
         self.mk_table_footer(),
         cls="dt2-inner-table"),
       cls="dt2-table",
@@ -479,20 +493,18 @@ class DataGrid(BaseComponent):
       id=f"th_{self._id}"
     )
   
-  def mk_table_body(self):
+  def mk_table_body_page(self):
-    df = self._get_filtered_df()
+    """
+    This function is used to update the table body when the vertical scrollbar reaches the end
+    A new page is added when requested
+    """
     max_height = self._compute_body_max_height()
     
     return Div(
-      *[Div(
+      *self.mk_body_content_page(0),
-        *[self.mk_body_cell(col_pos, row_index, col_def) for col_pos, col_def in enumerate(self._state.columns)],
-        cls="dt2-row",
-        data_row=f"{row_index}",
-        id=f"tr_{self._id}-{row_index}",
-      ) for row_index in df.index],
       cls="dt2-body",
       style=f"max-height:{max_height}px;",
-      id=f"tb_{self._id}"
+      id=f"tb_{self._id}",
     )
   
   def mk_table_footer(self):
@@ -507,34 +519,55 @@ class DataGrid(BaseComponent):
       id=f"tf_{self._id}"
     )
   
+  def mk_body_content_page(self, page_index: int):
+    df = self._get_filtered_df()
+    start = page_index * DATAGRID_PAGE_SIZE
+    end = start + DATAGRID_PAGE_SIZE
+    if self._total_rows > end:
+      last_row = df.index[end - 1]
+    else:
+      last_row = None
+    
+    rows = [Div(
+      *[self.mk_body_cell(col_pos, row_index, col_def) for col_pos, col_def in enumerate(self._state.columns)],
+      cls="dt2-row",
+      data_row=f"{row_index}",
+      id=f"tr_{self._id}-{row_index}",
+      **self.commands.get_page(page_index + 1) if row_index == last_row else {}
+    ) for row_index in df.index[start:end]]
+    
+    rows.append(Script(f"manageScrollbars('{self._id}', false);"), )
+    
+    return rows
+  
   def mk_body_cell(self, col_pos, row_index, col_def: DataGridColumnState):
     if not col_def.usable:
       return None
     
     if not col_def.visible:
-      return Div(cls="dt2-col-hidden")
+      return MyDiv(cls="dt2-col-hidden")
     
     content = self.mk_body_cell_content(col_pos, row_index, col_def)
     
-    return Div(content,
+    return MyDiv(content,
                  data_col=col_def.col_id,
                  style=f"width:{col_def.width}px;",
                  cls="dt2-cell")
   
   def mk_body_cell_content(self, col_pos, row_index, col_def: DataGridColumnState):
     
-    def mk_bool(value):
+    def mk_bool(_value):
-      return Div(mk_icon(icon_checked if value else icon_unchecked, can_select=False),
+      return MyDiv(mk_my_icon(icon_checked if _value else icon_unchecked, can_select=False),
                    cls="dt2-cell-content-checkbox")
     
-    def mk_text(value):
+    def mk_text(_value):
-      return mk_ellipsis(value, cls="dt2-cell-content-text")
+      return mk_my_ellipsis(_value, cls="dt2-cell-content-text")
     
-    def mk_number(value):
+    def mk_number(_value):
-      return mk_ellipsis(value, cls="dt2-cell-content-number")
+      return mk_my_ellipsis(_value, cls="dt2-cell-content-number")
     
-    def process_cell_content(value):
+    def process_cell_content(_value):
-      value_str = str(value)
+      value_str = html.escape(str(_value))
       
       if FILTER_INPUT_CID not in self._state.filtered or (
           keyword := self._state.filtered[FILTER_INPUT_CID]) is None:
@@ -545,21 +578,22 @@ class DataGrid(BaseComponent):
         return value_str
       
       len_keyword = len(keyword)
-      res = [Span(value_str[:index])] if index > 0 else []
+      res = [MySpan(value_str[:index])] if index > 0 else []
-      res += [Span(value_str[index:index + len_keyword], cls="dt2-highlight-1")]
+      res += [MySpan(value_str[index:index + len_keyword], cls="dt2-highlight-1")]
-      res += [Span(value_str[index + len_keyword:])] if len(value_str) > len_keyword else []
+      res += [MySpan(value_str[index + len_keyword:])] if len(value_str) > len_keyword else []
       return tuple(res)
     
     column_type = col_def.type
+    value = self._fast_access[col_def.col_id][row_index]
     
     if column_type == ColumnType.Bool:
-      content = mk_bool(self._df.iloc[row_index, col_def.col_index])
+      content = mk_bool(value)
     elif column_type == ColumnType.Number:
-      content = mk_number(process_cell_content(self._df.iloc[row_index, col_def.col_index]))
+      content = mk_number(process_cell_content(value))
     elif column_type == ColumnType.RowIndex:
       content = mk_number(row_index)
     else:
-      content = mk_text(process_cell_content(self._df.iloc[row_index, col_def.col_index]))
+      content = mk_text(process_cell_content(value))
     
     return content
   
@@ -822,6 +856,31 @@ class DataGrid(BaseComponent):
     
     return True
   
+  @staticmethod
+  def _init_fast_access(df):
+    """
+    Generates a fast-access dictionary for a DataFrame.
+
+    This method converts the columns of the provided DataFrame into NumPy arrays
+    and stores them as values in a dictionary, using the column names as keys.
+    This allows for efficient access to the data stored in the DataFrame.
+
+    Args:
+        df (DataFrame): The input pandas DataFrame whose columns are to be converted
+        into a dictionary of NumPy arrays.
+
+    Returns:
+        dict: A dictionary where the keys are the column names of the input DataFrame
+        and the values are the corresponding column values as NumPy arrays.
+    """
+    if df is None:
+      return {}
+    
+    res = {col: df[col].to_numpy() for col in df.columns}
+    res[ROW_INDEX_ID] = df.index.to_numpy()
+    return res
+  
+  @timed
   def __ft__(self):
     return Div(
       Div(
@@ -844,7 +903,7 @@ class DataGrid(BaseComponent):
   @staticmethod
   def new(session, data, index=None):
     datagrid = DataGrid(session, DataGrid.create_component_id(session))
-    #dataframe = DataFrame(data, index=index)
+    # dataframe = DataFrame(data, index=index)
     dataframe = DataFrame(data)
     datagrid.init_from_dataframe(dataframe)
     return datagrid

src/components/datagrid_new/components/commands.py

@@ -91,12 +91,21 @@ class DataGridCommandManager(BaseCommandManager):
     return {
         "hx-post": f"{ROUTE_ROOT}{Routes.OnClick}",
         "hx-target": f"#tsm_{self._id}",
-        "hx-trigger" : "click",
+        "hx-trigger": "click",
         "hx-swap": "outerHTML",
         "hx-vals": f'js:{{_id: "{self._id}", cell_id:getCellId(event), modifier:getClickModifier(event), boundaries: getCellBoundaries(event)}}',
         "hx-on::before-request": f'validateOnClickRequest("{self._id}", event)',
     }
   
+  def get_page(self, page_index=0):
+    return {
+        "hx-get": f"{ROUTE_ROOT}{Routes.GetPage}",
+        "hx-target": f"#tb_{self._id}",
+        "hx-swap": "beforeend",
+        "hx-vals": f'{{"_id": "{self._id}", "page_index": "{page_index}"}}',
+        "hx-trigger": f"intersect root:#tb_{self._id} once",
+    }
+  
   def _get_hide_show_columns_attrs(self, mode, col_defs: list, new_value, cls=""):
     str_col_names = ", ".join(f"'{col_def.title}'" for col_def in col_defs)
     tooltip_msg = f"{mode} column{'s' if len(col_defs) > 1 else ''} {str_col_names}"
@@ -109,38 +118,6 @@ class DataGridCommandManager(BaseCommandManager):
         "data_tooltip": tooltip_msg,
         "cls": self.merge_class(cls, "mmt-tooltip")
     }
-  #
-  # @staticmethod
-  # def merge(*items):
-  #   """
-  #   Merges multiple dictionaries into a single dictionary by combining their key-value pairs.
-  #   If a key exists in multiple dictionaries and its value is a string, the values are concatenated.
-  #   If the key's value is not a string, an error is raised.
-  #
-  #   :param items: dictionaries to be merged. If all items are None, None is returned.
-  #   :return: A single dictionary containing the merged key-value pairs from all input dictionaries.
-  #   :raises NotImplementedError: If a key's value is not a string and exists in multiple input dictionaries.
-  #   """
-  #   if all(item is None for item in items):
-  #     return None
-  #
-  #   res = {}
-  #   for item in [item for item in items if item is not None]:
-  #
-  #     for key, value in item.items():
-  #       if not key in res:
-  #         res[key] = value
-  #       else:
-  #         if isinstance(res[key], str):
-  #           res[key] += " " + value
-  #         else:
-  #           raise NotImplementedError("")
-  #
-  #   return res
-  #
-  # @staticmethod
-  # def merge_class(cls1, cls2):
-  #   return (cls1 + " " + cls2) if cls2 else cls1
 
 
 class FilterAllCommands(BaseCommandManager):

src/components/datagrid_new/constants.py

@@ -17,6 +17,9 @@ CONTAINER_HEIGHT = "container_height"
 
 DATAGRID_STATE_FOOTER = "footer"
 
+DATAGRID_PAGE_SIZE = 50
+
+ROW_INDEX_ID = "__row_index__"
 
 class Routes:
   Filter = "/filter"  # request the filtering in the grid
@@ -33,6 +36,7 @@ class Routes:
   UpdateView = "/update_view"
   ShowFooterMenu = "/show_footer_menu"
   UpdateState = "/update_state"
+  GetPage = "/page"
 
 
 class ColumnType(Enum):
@@ -44,11 +48,13 @@ class ColumnType(Enum):
   Choice = "Choice"
   List = "List"
 
+
 class ViewType(Enum):
   Table = "Table"
   Chart = "Chart"
   Form = "Form"
 
+
 class FooterAggregation(Enum):
   Sum = "Sum"
   Mean = "Mean"

src/components/datagrid_new/settings.py

@@ -69,6 +69,7 @@ class DataGridSettings:
 class DataGridState:
   sidebar_visible: bool = False
   selected_view: str = None
+  row_index: bool = False
   columns: list[DataGridColumnState] = dataclasses.field(default_factory=list)
   rows: list[DataGridRowState] = dataclasses.field(default_factory=list)  # only the rows that have a specific state
   footers: list[DataGridFooterConf] = dataclasses.field(default_factory=list)

src/components/entryselector/EntrySelectorApp.py

@@ -0,0 +1,17 @@
+import logging
+
+from fasthtml.fastapp import fast_app
+
+from components.entryselector.constants import Routes
+from core.instance_manager import debug_session, InstanceManager
+
+logger = logging.getLogger("EntrySelectorApp")
+
+repositories_app, rt = fast_app()
+
+
+@rt(Routes.Select)
+def get(session, _id: str, entry: str):
+  logger.debug(f"Entering {Routes.Select} with args {debug_session(session)}, {_id=}, {entry=}")
+  instance = InstanceManager.get(session, _id)
+  return instance.select_entry(entry)

src/components/entryselector/assets/EntrySelector.css

@@ -0,0 +1,16 @@
+.es-container {
+  overflow-x: auto;
+  white-space: nowrap;
+
+}
+
+.es-entry {
+  border: 2px solid var(--color-base-300);
+  padding: 2px;
+  cursor: pointer;
+  display: inline-block; /* Ensure entries align horizontally if needed */
+}
+
+.es-entry:hover {
+  background-color: var(--color-base-300);
+}

src/components/entryselector/commands.py

@@ -0,0 +1,15 @@
+from components.BaseCommandManager import BaseCommandManager
+from components.entryselector.constants import Routes, ROUTE_ROOT
+
+
+class EntrySelectorCommandManager(BaseCommandManager):
+  def __init__(self, owner):
+    super().__init__(owner)
+  
+  def select_entry(self, entry):
+    return {
+        "hx-get": f"{ROUTE_ROOT}{Routes.Select}",
+        "hx-target": f"#{self._owner.content_id}",
+        "hx-swap": "innerHTML",
+        "hx-vals": f'{{"_id": "{self._id}", "entry": "{entry}"}}',
+    }

src/components/entryselector/components/EntrySelector.py

@@ -0,0 +1,45 @@
+import logging
+
+from fasthtml.components import *
+
+from components.BaseComponent import BaseComponentMultipleInstance
+from components.entryselector.commands import EntrySelectorCommandManager
+
+logger = logging.getLogger("EntrySelector")
+
+
+class EntrySelector(BaseComponentMultipleInstance):
+  def __init__(self, session, _id, owner, content_id, data=None, hooks=None, key=None, boundaries=None):
+    super().__init__(session, _id)
+    self._key = key
+    self._owner = owner  # debugger component
+    self.data = data
+    self.content_id = content_id
+    self.hooks = hooks
+    self._boundaries = boundaries if boundaries else {"width": "300"}
+    self._commands = EntrySelectorCommandManager(self)
+  
+  def set_data(self, data):
+    self.data = data
+  
+  def set_boundaries(self, boundaries):
+    self._boundaries = boundaries
+  
+  def select_entry(self, entry):
+    logger.debug(f"Selecting entry {entry}")
+    # return self._owner.select_entry(entry)
+  
+  def _mk_content(self):
+    if self.data is None:
+      return [Div("no entry")]
+    
+    return [Div(index,
+                **self._commands.select_entry(index),
+                cls="es-entry") for index in range(self.data)]
+  
+  def __ft__(self):
+    return Div(
+      *self._mk_content(),
+      cls="flex es-container",
+      id=f"{self._id}",
+    )

src/components/entryselector/constants.py

@@ -0,0 +1,5 @@
+ROUTE_ROOT = "/es"  # for EntrySelector
+
+
+class Routes:
+  Select = "/select"

src/components/repositories/RepositoriesApp.py

@@ -20,7 +20,7 @@ def get(session):
 
 
 @rt(Routes.AddRepository)
-def post(session, _id: str, tab_id: str, form_id: str, repository: str, table: str, tab_boundaries:str):
+def post(session, _id: str, tab_id: str, form_id: str, repository: str, table: str, tab_boundaries: str):
   logger.debug(
     f"Entering {Routes.AddRepository} with args {debug_session(session)}, {_id=}, {tab_id=}, {form_id=}, {repository=}, {table=}, {tab_boundaries=}")
   instance = InstanceManager.get(session, _id)  # Repository
@@ -34,8 +34,9 @@ def get(session, _id: str, repository_name: str):
 
 
 @rt(Routes.AddTable)
-def post(session, _id: str, tab_id: str, form_id: str, repository_name: str, table_name: str, tab_boundaries:str):
+def post(session, _id: str, tab_id: str, form_id: str, repository_name: str, table_name: str, tab_boundaries: str):
-  logger.debug(f"Entering {Routes.AddTable} with args {debug_session(session)}, {_id=}, {tab_id=}, {form_id=}, {repository_name=}, {table_name=}, {tab_boundaries=}")
+  logger.debug(
+    f"Entering {Routes.AddTable} with args {debug_session(session)}, {_id=}, {tab_id=}, {form_id=}, {repository_name=}, {table_name=}, {tab_boundaries=}")
   instance = InstanceManager.get(session, _id)
   return instance.add_new_table(tab_id, form_id, repository_name, table_name, json.loads(tab_boundaries))
 
@@ -48,7 +49,8 @@ def put(session, _id: str, repository: str):
 
 
 @rt(Routes.ShowTable)
-def get(session, _id: str, repository: str, table: str, tab_boundaries:str):
+def get(session, _id: str, repository: str, table: str, tab_boundaries: str):
-  logger.debug(f"Entering {Routes.ShowTable} with args {debug_session(session)}, {_id=}, {repository=}, {table=}, {tab_boundaries=}")
+  logger.debug(
+    f"Entering {Routes.ShowTable} with args {debug_session(session)}, {_id=}, {repository=}, {table=}, {tab_boundaries=}")
   instance = InstanceManager.get(session, _id)
   return instance.show_table(repository, table, json.loads(tab_boundaries))

src/components/workflows/components/WorkflowDesignerProperties.py

@@ -1,9 +1,10 @@
 from fasthtml.common import *
-from dataclasses import dataclass
 
 from components.BaseComponent import BaseComponent
+from components.entryselector.components.EntrySelector import EntrySelector
 from components.workflows.constants import COMPONENT_TYPES, PROCESSOR_TYPES
 from components_helpers import mk_dialog_buttons
+from core.instance_manager import InstanceManager
 from core.jira import JiraRequestTypes, DEFAULT_SEARCH_FIELDS
 from utils.DbManagementHelper import DbManagementHelper
 
@@ -25,6 +26,15 @@ class WorkflowDesignerProperties(BaseComponent):
     self._component = None
     self.update_layout()
     self.update_component(self._owner.get_state().selected_component_id)
+    self._input_entry_selector = InstanceManager.new(self._session,
+                                                     EntrySelector,
+                                                     owner=self,
+                                                     content_id=f"pic_{self._id}",
+                                                     data=100)
+    self._output_entry_selector = InstanceManager.new(self._session,
+                                                      EntrySelector,
+                                                      owner=self,
+                                                      content_id=f"poc_{self._id}")
   
   def update_layout(self):
     if self._owner.get_state().properties_input_width is None:
@@ -66,7 +76,8 @@ class WorkflowDesignerProperties(BaseComponent):
   
   def _mk_input(self):
     return Div(
-      "Input",
+      self._input_entry_selector,
+      Div(id=f"pic_{self._id}"),
       id=f"pi_{self._id}",
       style=f"width: {self.layout.input_width}px;",
       cls="wkf-properties-input"
@@ -74,7 +85,8 @@ class WorkflowDesignerProperties(BaseComponent):
   
   def _mk_output(self):
     return Div(
-      "Output",
+      self._output_entry_selector,
+      "Output Content",
       id=f"po_{self._id}",
       style=f"width: {self.layout.output_width}px;",
       cls="wkf-properties-output"

src/components/workflows/components/WorkflowPlayer.py

@@ -98,6 +98,8 @@ class WorkflowPlayer(BaseComponent):
         
         if component.id not in engine.errors:
           runtime_state.state = ComponentState.SUCCESS
+          runtime_state.input = engine.debug[component.id]["input"]
+          runtime_state.output = engine.debug[component.id]["output"]
           continue
         
         # the component failed
@@ -177,7 +179,7 @@ class WorkflowPlayer(BaseComponent):
     # Return sorted components
     return [components_by_id[cid] for cid in sorted_order]
   
-  def _get_engine(self, sorted_components):
+  def _get_engine(self, sorted_components) -> WorkflowEngine:
     # first reorder the component, according to the connection definitions
     engine = WorkflowEngine()
     for component in sorted_components:

src/components/workflows/db_management.py

@@ -48,6 +48,8 @@ class WorkflowComponentRuntimeState:
   id: str
   state: ComponentState = ComponentState.SUCCESS
   error_message: str | None = None
+  input: list = None
+  output: list = None
 
 
 @dataclass
@@ -62,7 +64,7 @@ class WorkflowsDesignerState:
   component_counter: int = 0
   designer_height: int = 230
   properties_input_width: int = None
-  properties_properties_width : int = None
+  properties_properties_width: int = None
   properties_output_width: int = None
   selected_component_id: str | None = None
 

src/core/fasthtml_helper.py

@@ -0,0 +1,76 @@
+from fastcore.basics import NotStr
+
+from core.utils import merge_classes
+
+attr_map = {
+    "cls": "class",
+    "_id": "id",
+}
+
+
+def safe_attr(attr_name):
+  attr_name = attr_name.replace("hx_", "hx-")
+  attr_name = attr_name.replace("data_", "data-")
+  return attr_map.get(attr_name, attr_name)
+
+
+def to_html(item):
+  if item is None:
+    return ""
+  elif isinstance(item, str):
+    return item
+  elif isinstance(item, (int, float, bool)):
+    return str(item)
+  elif isinstance(item, MyFt):
+    return item.to_html()
+  elif isinstance(item, NotStr):
+    return str(item)
+  else:
+    raise Exception(f"Unsupported type: {type(item)}, {item=}")
+
+
+class MyFt:
+  def __init__(self, tag, *args, **kwargs):
+    self.tag = tag
+    self.children = args
+    self.attrs = {safe_attr(k): v for k, v in kwargs.items()}
+  
+  def to_html(self):
+    body_items = [to_html(item) for item in self.children]
+    return f"<{self.tag} {' '.join(f'{k}="{v}"' for k, v in self.attrs.items())}>{' '.join(body_items)}</div>"
+  
+  def __ft__(self):
+    return NotStr(self.to_html())
+
+
+class MyDiv(MyFt):
+  def __init__(self, *args, **kwargs):
+    super().__init__("div", *args, **kwargs)
+
+
+class MySpan(MyFt):
+  def __init__(self, *args, **kwargs):
+    super().__init__("span", *args, **kwargs)
+
+
+def mk_my_ellipsis(txt: str, cls='', **kwargs):
+  merged_cls = merge_classes("truncate",
+                             cls,
+                             kwargs)
+  return MyDiv(txt, cls=merged_cls, data_tooltip=txt, **kwargs)
+
+
+def mk_my_icon(icon, size=20, can_select=True, can_hover=False, cls='', tooltip=None, **kwargs):
+  merged_cls = merge_classes(f"icon-{size}",
+                             'icon-btn' if can_select else '',
+                             'mmt-btn' if can_hover else '',
+                             cls,
+                             kwargs)
+  return mk_my_tooltip(icon, tooltip, cls=merged_cls, **kwargs) if tooltip else MyDiv(icon, cls=merged_cls, **kwargs)
+
+
+def mk_my_tooltip(element, tooltip: str, cls='', **kwargs):
+  merged_cls = merge_classes("mmt-tooltip",
+                             cls,
+                             kwargs)
+  return MyDiv(element, cls=merged_cls, data_tooltip=tooltip, **kwargs)

src/core/instance_manager.py

@@ -47,6 +47,10 @@ class InstanceManager:
     
     return InstanceManager._instances[key]
   
+  @staticmethod
+  def new(session, instance_type, **kwargs):
+    return InstanceManager.get(session, instance_type.create_component_id(session), instance_type, **kwargs)
+  
   @staticmethod
   def register(session: dict | None, instance, instance_id: str = None):
     """

src/core/utils.py

@@ -1,12 +1,16 @@
 import ast
 import base64
+import cProfile
+import functools
 import hashlib
 import importlib
 import inspect
 import pkgutil
 import re
+import time
 import types
 import uuid
+from datetime import datetime
 from enum import Enum
 from io import BytesIO
 from urllib.parse import urlparse
@@ -420,6 +424,66 @@ def split_host_port(url):
   return host, port
 
 
+def timed(func):
+  @functools.wraps(func)
+  def wrapper(*args, **kwargs):
+    start = time.perf_counter()
+    result = func(*args, **kwargs)
+    end = time.perf_counter()
+    
+    # get class name
+    class_name = None
+    if args:
+      # check the first argument to see if it's a class'
+      if inspect.isclass(args[0]):
+        class_name = args[0].__name__  # class method
+      elif hasattr(args[0], "__class__"):
+        class_name = args[0].__class__.__name__  # instance method
+    
+    if class_name:
+      print(f"[PERF] {class_name}.{func.__name__} took {end - start:.4f} sec")
+    else:
+      print(f"[PERF] {func.__name__} took {end - start:.4f} sec")
+    
+    return result
+  
+  return wrapper
+
+
+def profile_function(func):
+  @functools.wraps(func)
+  def wrapper(*args, **kwargs):
+    profiler = cProfile.Profile()
+    try:
+      profiler.enable()
+      result = func(*args, **kwargs)
+    finally:
+      profiler.disable()
+    
+    # Determine class name if any
+    class_name = None
+    if args:
+      if inspect.isclass(args[0]):
+        class_name = args[0].__name__  # class method
+      elif hasattr(args[0], "__class__"):
+        class_name = args[0].__class__.__name__  # instance method
+    
+    # Compose filename with timestamp
+    timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+    if class_name:
+      filename = f"{class_name}_{func.__name__}_{timestamp}.prof"
+    else:
+      filename = f"{func.__name__}_{timestamp}.prof"
+    
+    # Dump stats to file
+    profiler.dump_stats(filename)
+    print(f"[PROFILE] Profiling data saved to {filename}")
+    
+    return result
+  
+  return wrapper
+
+
 class UnreferencedNamesVisitor(ast.NodeVisitor):
   """
   Try to find symbols that will be requested by the ast
@@ -464,4 +528,3 @@ class UnreferencedNamesVisitor(ast.NodeVisitor):
     """
     self.names.add(node.arg)
     self.visit_selected(node, ["value"])
-

src/main.py

@@ -1,5 +1,4 @@
 # global layout
-import asyncio
 import logging.config
 
 import yaml
@@ -147,6 +146,7 @@ register_component("theme_controller", "components.themecontroller", "ThemeContr
 register_component("main_layout", "components.drawerlayout", "DrawerLayoutApp")
 register_component("undo_redo", "components.undo_redo", "UndoRedoApp")
 register_component("tabs", "components.tabs", "TabsApp")  # before repositories
+register_component("entryselector", "components.entryselector", "EntrySelectorApp")
 register_component("applications", "components.applications", "ApplicationsApp")
 register_component("repositories", "components.repositories", "RepositoriesApp")
 register_component("workflows", "components.workflows", "WorkflowsApp")
@@ -211,6 +211,25 @@ app, rt = fast_app(
   pico=False,
 )
 
+
+# -------------------------
+# Profiling middleware
+# -------------------------
+# @app.middleware("http")
+async def timing_middleware(request, call_next):
+  import time
+  start_total = time.perf_counter()
+  
+  # Call the next middleware or route handler
+  response = await call_next(request)
+  
+  end_total = time.perf_counter()
+  elapsed = end_total - start_total
+  
+  print(f"[PERF] Total server time: {elapsed:.4f} sec - Path: {request.url.path}")
+  return response
+
+
 settings_manager = SettingsManager()
 
 import_settings = AdminImportSettings(settings_manager, None)
@@ -253,6 +272,17 @@ def get(session):
           DrawerLayoutOld(pages),)
 
 
+@rt('/toasting')
+def get(session):
+  # Normally one toast is enough, this allows us to see
+  # different toast types in action.
+  add_toast(session, f"Toast is being cooked", "info")
+  add_toast(session, f"Toast is ready", "success")
+  add_toast(session, f"Toast is getting a bit crispy", "warning")
+  add_toast(session, f"Toast is burning!", "error")
+  return Titled("I like toast")
+
+
 # Error Handling
 @app.get("/{path:path}")
 def not_found(path: str, session=None):
@@ -275,18 +305,7 @@ def not_found(path: str, session=None):
 setup_toasts(app)
 
 
-@rt('/toasting')
+def main():
-def get(session):
-  # Normally one toast is enough, this allows us to see
-  # different toast types in action.
-  add_toast(session, f"Toast is being cooked", "info")
-  add_toast(session, f"Toast is ready", "success")
-  add_toast(session, f"Toast is getting a bit crispy", "warning")
-  add_toast(session, f"Toast is burning!", "error")
-  return Titled("I like toast")
-
-
-async def main():
   logger.info(f"  Starting FastHTML server on http://localhost:{APP_PORT}")
   serve(port=APP_PORT)
 
@@ -294,9 +313,4 @@ async def main():
 if __name__ == "__main__":
   # Start your application
   logger.info("Application starting...")
-  try:
+  main()
-    asyncio.run(main())
-  except KeyboardInterrupt:
-    logger.info("\nStopping application...")
-  except Exception as e:
-    logger.error(f"Error: {e}")

src/workflow/engine.py

@@ -1,6 +1,7 @@
 import ast
 import logging
 from abc import ABC, abstractmethod
+from dataclasses import dataclass
 from typing import Any, Generator
 
 from components.admin.admin_db_manager import AdminDbManager
@@ -11,6 +12,14 @@ from core.utils import UnreferencedNamesVisitor
 from utils.Datahelper import DataHelper
 
 
+@dataclass
+class WorkflowPayload:
+  processor_name: str
+  component_id: str
+  item_linkage_id: int
+  item: Any
+
+
 class DataProcessorError(Exception):
   def __init__(self, component_id, error):
     self.component_id = component_id
@@ -146,35 +155,56 @@ class WorkflowEngine:
     self.has_error = False
     self.global_error = None
     self.errors = {}
+    self.debug = {}
+    self.item_count = -1
   
   def add_processor(self, processor: DataProcessor) -> 'WorkflowEngine':
     """Add a data processor to the pipeline."""
     self.processors.append(processor)
     return self
   
-  def _process_single_item(self, item: Any, processor_index: int = 0) -> Generator[Any, None, None]:
+  def _process_single_item(self, item_linkage_id, item: Any, processor_index: int = 0) -> Generator[Any, None, None]:
     """Process a single item through the remaining processors."""
     if processor_index >= len(self.processors):
       yield item
       return
     
     processor = self.processors[processor_index]
+    if not processor.component_id in self.debug:
+      self.debug[processor.component_id] = {"input": [], "output": []}
+    
+    self.debug[processor.component_id]["input"].append(WorkflowPayload(
+      processor_name=processor.__class__.__name__,
+      component_id=processor.component_id,
+      item_linkage_id=item_linkage_id,
+      item=item))
     
     # Process the item through the current processor
     for processed_item in processor.process(item):
+      self.debug[processor.component_id]["output"].append(WorkflowPayload(
+        processor_name=processor.__class__.__name__,
+        component_id=processor.component_id,
+        item_linkage_id=item_linkage_id,
+        item=processed_item))
+      
       # Recursively process through remaining processors
-      yield from self._process_single_item(processed_item, processor_index + 1)
+      yield from self._process_single_item(item_linkage_id, processed_item, processor_index + 1)
   
   def run(self) -> Generator[Any, None, None]:
     """
     Run the workflow pipeline and yield results one by one.
     The first processor must be a DataProducer.
     """
+    
+    self.debug.clear()
+    
     if not self.processors:
       self.has_error = False
       self.global_error = "No processors in the pipeline"
+      self.item_count = -1
       raise ValueError(self.global_error)
     
+    self.item_count = 0
     first_processor = self.processors[0]
     
     if not isinstance(first_processor, DataProducer):
@@ -182,8 +212,16 @@ class WorkflowEngine:
       self.global_error = "First processor must be a DataProducer"
       raise ValueError(self.global_error)
     
-    for item in first_processor.process(None):
+    self.debug[first_processor.component_id] = {"input": [], "output": []}
-      yield from self._process_single_item(item, 1)
+    
+    for item_linkage_id, item in enumerate(first_processor.process(None)):
+      self.item_count += 1
+      self.debug[first_processor.component_id]["output"].append(WorkflowPayload(
+        processor_name=first_processor.__class__.__name__,
+        component_id=first_processor.component_id,
+        item_linkage_id=item_linkage_id,
+        item=item))
+      yield from self._process_single_item(item_linkage_id, item, 1)
   
   def run_to_list(self) -> list[Any]:
     """

tests/helpers.py

@@ -642,10 +642,10 @@ def extract_table_values_new(ft, header=True):
     # first, get the header
   
   if header:
-    header = search_elements_by_name(ft, attrs={"class": "dt2-header"}, comparison_method='contains')[0]
+    header_element = search_elements_by_name(ft, attrs={"class": "dt2-header"}, comparison_method='contains')[0]
     header_map = {}
     res = OrderedDict()
-    for row in header.children:
+    for row in header_element.children:
       col_id = row.attrs["data-col"]
       title = row.attrs["data-tooltip"]
       header_map[col_id] = title
@@ -654,6 +654,7 @@ def extract_table_values_new(ft, header=True):
     body = search_elements_by_name(ft, attrs={"class": "dt2-body"}, comparison_method='contains')[0]
     for row in body.children:
       for col in row.children:
+        if hasattr(col, "attrs"):
           col_id = col.attrs["data-col"]
           cell_value = _get_cell_content_value(col)
           res[header_map[col_id]].append(cell_value)

tests/test_datagrid_new.py

@@ -509,3 +509,18 @@ def test_i_can_compute_footer_menu_position_when_not_enough_space(dg):
   )
   
   assert matches(menu, expected)
+
+
+def test_the_content_of_the_cell_is_escaped(empty_dg):
+  df = pd.DataFrame({
+      'value': ['<div> My Content </div>'],
+      'value2': ['{My Content}'],
+  })
+  my_dg = empty_dg.init_from_dataframe(df)
+  
+  actual = my_dg.__ft__()
+  table_content = extract_table_values_new(actual, header=True)
+  
+  assert table_content == OrderedDict({
+      'value': ['&lt;div&gt; My Content &lt;/div&gt;'],
+      'value2': ['{My Content}']})