Fixed unit tests

Fixed content escaping

src/assets/main.js

@@ -25,11 +25,19 @@ function bindTooltipsWithDelegation() {
 
     // Add a single mouseenter and mouseleave listener to the parent element
     element.addEventListener("mouseenter", (event) => {
+        //console.debug("Entering element", event.target)
+
         const cell = event.target.closest("[data-tooltip]");
-        if (!cell) return;
+        if (!cell) {
+          // console.debug("  No 'data-tooltip' attribute found. Stopping.");
+          return;
+        }
 
         const no_tooltip = element.hasAttribute("mmt-no-tooltip");
-        if (no_tooltip) return;
+        if (no_tooltip) {
+          // console.debug("  Attribute 'mmt-no-tooltip' found. Cancelling.");
+          return;
+        }
 
         const content = cell.querySelector(".truncate") || cell;
         const isOverflowing = content.scrollWidth > content.clientWidth;

src/components/datagrid_new/DataGridApp.py

@@ -145,16 +145,10 @@ def post(session, _id: str, state: str, args: str = None):
 async def get(session, _id: str):
   logger.debug(f"Entering {Routes.YieldRow} with args {_id=}")
   instance = InstanceManager.get(session, _id)
-  return EventStream(instance.mk_lazy_body_content())
+  return EventStream(instance.mk_body_content_sse())
 
-
+@rt(Routes.GetPage)
-async def number_generator2():
+def get(session, _id: str, page_index: int):
-  for i in range(20):
+  logger.debug(f"Entering {Routes.GetPage} with args {_id=}, {page_index=}")
-    yield sse_message(Div(i * 5 + 1))
+  instance = InstanceManager.get(session, _id)
-    yield sse_message(Div(i * 5 + 2))
+  return instance.mk_body_content_page(page_index)
-    yield sse_message(Div(i * 5 + 3))
-    yield sse_message(Div(i * 5 + 4))
-    yield sse_message(Div(i * 5 + 5))
-    await asyncio.sleep(0.1)
-  
-  yield f"event: close\ndata: \n\n"

src/components/datagrid_new/assets/Datagrid.js

@@ -1,10 +1,6 @@
 function bindDatagrid(datagridId, allowColumnsReordering) {
-    bindScrollbars(datagridId);
+    manageScrollbars(datagridId, true);
-    makeResizable(datagridId)
+    makeResizable(datagridId);
-
-    document.body.addEventListener('htmx:sseBeforeMessage', function (e) {
-        console.log("htmx:sseBeforeMessage", e)
-    })
 }
 
 function bindScrollbars(datagridId) {
@@ -25,7 +21,7 @@ function bindScrollbars(datagridId) {
     const table = datagrid.querySelector(".dt2-table");
 
     if (!verticalScrollbar || !verticalWrapper || !horizontalScrollbar || !horizontalWrapper || !body || !table) {
-        console.error("Essential scrollbar or content elements are missing in the datagrid.");
+        console.error("Essential scrollbars or content elements are missing in the datagrid.");
         return;
     }
 
@@ -180,6 +176,224 @@ function bindScrollbars(datagridId) {
     });
 }
 
+function manageScrollbars(datagridId, binding) {
+    console.debug("manageScrollbars on element " + datagridId + " with binding=" + binding);
+
+    const datagrid = document.getElementById(datagridId);
+
+    if (!datagrid) {
+        console.error(`Datagrid with id "${datagridId}" not found.`);
+        return;
+    }
+
+    const verticalScrollbar = datagrid.querySelector(".dt2-scrollbars-vertical");
+    const verticalWrapper = datagrid.querySelector(".dt2-scrollbars-vertical-wrapper");
+    const horizontalScrollbar = datagrid.querySelector(".dt2-scrollbars-horizontal");
+    const horizontalWrapper = datagrid.querySelector(".dt2-scrollbars-horizontal-wrapper");
+    const body = datagrid.querySelector(".dt2-body");
+    const table = datagrid.querySelector(".dt2-table");
+
+    if (!verticalScrollbar || !verticalWrapper || !horizontalScrollbar || !horizontalWrapper || !body || !table) {
+        console.error("Essential scrollbars or content elements are missing in the datagrid.");
+        return;
+    }
+
+    const computeScrollbarVisibility = () => {
+        // Determine if the content is clipped
+        const isVerticalRequired = body.scrollHeight > body.clientHeight;
+        const isHorizontalRequired = table.scrollWidth > table.clientWidth;
+
+        // Show or hide the scrollbar wrappers
+        requestAnimationFrame(() => {
+            verticalWrapper.style.display = isVerticalRequired ? "block" : "none";
+            horizontalWrapper.style.display = isHorizontalRequired ? "block" : "none";
+        });
+    };
+
+    const computeScrollbarSize = () => {
+        // Vertical scrollbar height
+        const visibleHeight = body.clientHeight;
+        const totalHeight = body.scrollHeight;
+        const wrapperHeight = verticalWrapper.offsetHeight;
+
+        let scrollbarHeight = 0;
+        if (totalHeight > 0) {
+            scrollbarHeight = (visibleHeight / totalHeight) * wrapperHeight;
+        }
+
+        // Horizontal scrollbar width
+        const visibleWidth = table.clientWidth;
+        const totalWidth = table.scrollWidth;
+        const wrapperWidth = horizontalWrapper.offsetWidth;
+
+        let scrollbarWidth = 0;
+        if (totalWidth > 0) {
+            scrollbarWidth = (visibleWidth / totalWidth) * wrapperWidth;
+        }
+
+        requestAnimationFrame(() => {
+            verticalScrollbar.style.height = `${scrollbarHeight}px`;
+            horizontalScrollbar.style.width = `${scrollbarWidth}px`;
+        });
+    };
+
+    const updateVerticalScrollbarForMouseWheel = () => {
+        const maxScrollTop = body.scrollHeight - body.clientHeight;
+        const wrapperHeight = verticalWrapper.offsetHeight;
+
+        if (maxScrollTop > 0) {
+            const scrollRatio = wrapperHeight / body.scrollHeight;
+            verticalScrollbar.style.top = `${body.scrollTop * scrollRatio}px`;
+        }
+    };
+
+    if (binding) {
+        // Clean up existing managers if they exist
+        if (datagrid._managers) {
+            // Remove drag events
+            if (datagrid._managers.dragManager) {
+                verticalScrollbar.removeEventListener("mousedown", datagrid._managers.dragManager.verticalMouseDown);
+                horizontalScrollbar.removeEventListener("mousedown", datagrid._managers.dragManager.horizontalMouseDown);
+                document.removeEventListener("mousemove", datagrid._managers.dragManager.mouseMove);
+                document.removeEventListener("mouseup", datagrid._managers.dragManager.mouseUp);
+            }
+
+            // Remove wheel events
+            if (datagrid._managers.wheelManager) {
+                body.removeEventListener("wheel", datagrid._managers.wheelManager.handleWheelScrolling);
+            }
+
+            // Remove resize events
+            if (datagrid._managers.resizeManager) {
+                window.removeEventListener("resize", datagrid._managers.resizeManager.handleResize);
+            }
+        }
+
+        // Create managers
+        const dragManager = {
+            isDragging: false,
+            startY: 0,
+            startX: 0,
+
+            updateVerticalScrollbar: (deltaX, deltaY) => {
+                const wrapperHeight = verticalWrapper.offsetHeight;
+                const scrollbarHeight = verticalScrollbar.offsetHeight;
+                const maxScrollTop = body.scrollHeight - body.clientHeight;
+                const scrollRatio = maxScrollTop / (wrapperHeight - scrollbarHeight);
+
+                let newTop = parseFloat(verticalScrollbar.style.top || "0") + deltaY;
+                newTop = Math.max(0, Math.min(newTop, wrapperHeight - scrollbarHeight));
+
+                verticalScrollbar.style.top = `${newTop}px`;
+                body.scrollTop = newTop * scrollRatio;
+            },
+
+            updateHorizontalScrollbar: (deltaX, deltaY) => {
+                const wrapperWidth = horizontalWrapper.offsetWidth;
+                const scrollbarWidth = horizontalScrollbar.offsetWidth;
+                const maxScrollLeft = table.scrollWidth - table.clientWidth;
+                const scrollRatio = maxScrollLeft / (wrapperWidth - scrollbarWidth);
+
+                let newLeft = parseFloat(horizontalScrollbar.style.left || "0") + deltaX;
+                newLeft = Math.max(0, Math.min(newLeft, wrapperWidth - scrollbarWidth));
+
+                horizontalScrollbar.style.left = `${newLeft}px`;
+                table.scrollLeft = newLeft * scrollRatio;
+            },
+
+            verticalMouseDown: (e) => {
+                disableTooltip();
+                dragManager.isDragging = true;
+                dragManager.startY = e.clientY;
+                dragManager.startX = e.clientX;
+                document.body.style.userSelect = "none";
+                verticalScrollbar.classList.add("dt2-dragging");
+            },
+
+            horizontalMouseDown: (e) => {
+                disableTooltip();
+                dragManager.isDragging = true;
+                dragManager.startY = e.clientY;
+                dragManager.startX = e.clientX;
+                document.body.style.userSelect = "none";
+                horizontalScrollbar.classList.add("dt2-dragging");
+            },
+
+            mouseMove: (e) => {
+                if (dragManager.isDragging) {
+                    const deltaY = e.clientY - dragManager.startY;
+                    const deltaX = e.clientX - dragManager.startX;
+
+                    // Determine which scrollbar is being dragged
+                    if (verticalScrollbar.classList.contains("dt2-dragging")) {
+                        dragManager.updateVerticalScrollbar(deltaX, deltaY);
+                    } else if (horizontalScrollbar.classList.contains("dt2-dragging")) {
+                        dragManager.updateHorizontalScrollbar(deltaX, deltaY);
+                    }
+
+                    // Reset start points for next update
+                    dragManager.startY = e.clientY;
+                    dragManager.startX = e.clientX;
+                }
+            },
+
+            mouseUp: () => {
+                dragManager.isDragging = false;
+                document.body.style.userSelect = "";
+                verticalScrollbar.classList.remove("dt2-dragging");
+                horizontalScrollbar.classList.remove("dt2-dragging");
+                enableTooltip();
+            }
+        };
+
+        const wheelManager = {
+            handleWheelScrolling: (event) => {
+                const deltaX = event.deltaX;
+                const deltaY = event.deltaY;
+
+                // Scroll the body and table content
+                body.scrollTop += deltaY; // Vertical scrolling
+                table.scrollLeft += deltaX; // Horizontal scrolling
+
+                // Update the vertical scrollbar position
+                updateVerticalScrollbarForMouseWheel();
+
+                // Prevent default behavior to fully manage the scroll
+                event.preventDefault();
+            }
+        };
+
+        const resizeManager = {
+            handleResize: () => {
+                computeScrollbarVisibility();
+                computeScrollbarSize();
+                updateVerticalScrollbarForMouseWheel();
+            }
+        };
+
+        // Store managers on datagrid for cleanup
+        datagrid._managers = {
+            dragManager,
+            wheelManager,
+            resizeManager
+        };
+
+        // Bind events
+        verticalScrollbar.addEventListener("mousedown", dragManager.verticalMouseDown);
+        horizontalScrollbar.addEventListener("mousedown", dragManager.horizontalMouseDown);
+        document.addEventListener("mousemove", dragManager.mouseMove);
+        document.addEventListener("mouseup", dragManager.mouseUp);
+
+        body.addEventListener("wheel", wheelManager.handleWheelScrolling, {passive: false});
+
+        window.addEventListener("resize", resizeManager.handleResize);
+    }
+
+    // Always execute computations
+    computeScrollbarVisibility();
+    computeScrollbarSize();
+}
+
 function makeResizable(datagridId) {
     console.debug("makeResizable on element " + datagridId);
 

src/components/datagrid_new/components/DataGrid.py

@@ -1,5 +1,5 @@
-import asyncio
 import copy
+import html
 import logging
 from io import BytesIO
 from typing import Literal, Any
@@ -62,6 +62,7 @@ class DataGrid(BaseComponent):
     self._settings: DataGridSettings = grid_settings or self._db.load_settings()
     self._df: DataFrame | None = self._db.load_dataframe()
     self._fast_access = self._init_fast_access(self._df)
+    self._total_rows = len(self._df) if self._df is not None else 0
     
     # update boundaries if possible
     self.set_boundaries(boundaries)
@@ -121,15 +122,23 @@ class DataGrid(BaseComponent):
       else:
         return ColumnType.Text  # Default to Text if no match
     
+    def _init_columns(_df):
+      columns = [DataGridColumnState(make_safe_id(col_id),
+                                     col_index,
+                                     col_id,
+                                     _get_column_type(self._df[make_safe_id(col_id)].dtype))
+                 for col_index, col_id in enumerate(_df.columns)]
+      if self._state.row_index:
+        columns.insert(0, DataGridColumnState(make_safe_id(ROW_INDEX_ID), -1, " ", ColumnType.RowIndex))
+      
+      return columns
+    
     self._df = df.copy()
     self._df.columns = self._df.columns.map(make_safe_id)  # make sure column names are trimmed
     self._state.rows = [DataGridRowState(row_id) for row_id in self._df.index]
-    self._state.columns = [DataGridColumnState(make_safe_id(col_id),
+    self._state.columns = _init_columns(df)  # use df not self._df to keep the original title
-                                               col_index,
-                                               col_id,
-                                               _get_column_type(self._df[make_safe_id(col_id)].dtype))
-                           for col_index, col_id in enumerate(df.columns)]
     self._fast_access = self._init_fast_access(self._df)
+    self._total_rows = len(self._df) if self._df is not None else 0
     
     if save_state:
       self._db.save_all(None, self._state, self._df)
@@ -209,6 +218,7 @@ class DataGrid(BaseComponent):
       
       self._state.columns = new_columns_states
     
+    self._fast_access = self._init_fast_access(self._df)
     self._views.recompute_need_save()
     
     self._db.save_all(self._settings, self._state, self._df if new_column else None)
@@ -444,8 +454,7 @@ class DataGrid(BaseComponent):
       _mk_keyboard_management(),
       Div(
         self.mk_table_header(),
-        #self.mk_table_body(),
+        self.mk_table_body_page(),
-        self.mk_table_body_lazy(),
         self.mk_table_footer(),
         cls="dt2-inner-table"),
       cls="dt2-table",
@@ -480,43 +489,25 @@ class DataGrid(BaseComponent):
     
     header_class = "dt2-row dt2-header" + "" if self._settings.header_visible else " hidden"
     return Div(
-      Div(sse_swap="message"),
       *[_mk_header(col_def) for col_def in self._state.columns],
       cls=header_class,
       id=f"th_{self._id}"
     )
   
-  def mk_table_body_lazy(self):
+  def mk_table_body_page(self):
-    
+    """
+    This function is used to update the table body when the vertical scrollbar reaches the end
+    A new page is added when requested
+    """
     max_height = self._compute_body_max_height()
     
     return Div(
-      hx_ext="sse",
+      *self.mk_body_content_page(0),
-      sse_connect=f"{ROUTE_ROOT}{Routes.YieldRow}?_id={self._id}",
-      sse_close='close',
-      sse_swap="message",
-      hx_swap="beforeend",
       cls="dt2-body",
       style=f"max-height:{max_height}px;",
       id=f"tb_{self._id}",
     )
   
-  def mk_table_body(self):
-    df = self._get_filtered_df()
-    max_height = self._compute_body_max_height()
-    
-    return Div(
-      *[Div(
-        *[self.mk_body_cell(col_pos, row_index, col_def) for col_pos, col_def in enumerate(self._state.columns)],
-        cls="dt2-row",
-        data_row=f"{row_index}",
-        id=f"tr_{self._id}-{row_index}",
-      ) for row_index in df.index],
-      cls="dt2-body",
-      style=f"max-height:{max_height}px;",
-      id=f"tb_{self._id}"
-    )
-  
   def mk_table_footer(self):
     return Div(
       *[Div(
@@ -529,21 +520,26 @@ class DataGrid(BaseComponent):
       id=f"tf_{self._id}"
     )
   
-  async def mk_lazy_body_content(self):
+  def mk_body_content_page(self, page_index: int):
     df = self._get_filtered_df()
-    for i, row_index in enumerate(df.index):
+    start = page_index * DATAGRID_PAGE_SIZE
-      yield sse_message(Div(
+    end = start + DATAGRID_PAGE_SIZE
-        *[self.mk_body_cell(col_pos, row_index, col_def) for col_pos, col_def in enumerate(self._state.columns)],
+    if self._total_rows > end:
-        cls="dt2-row",
+      last_row = df.index[end - 1]
-        data_row=f"{row_index}",
+    else:
-        id=f"tr_{self._id}-{row_index}",
+      last_row = None
-      ))
-      if i % 50 == 0:
-        await asyncio.sleep(0.01)
-        logger.debug(f"yielding row {i}")
     
-    logger.debug(f"yielding close event")
+    rows = [Div(
-    yield f"event: close\ndata: \n\n"
+      *[self.mk_body_cell(col_pos, row_index, col_def) for col_pos, col_def in enumerate(self._state.columns)],
+      cls="dt2-row",
+      data_row=f"{row_index}",
+      id=f"tr_{self._id}-{row_index}",
+      **self.commands.get_page(page_index + 1) if row_index == last_row else {}
+    ) for row_index in df.index[start:end]]
+    
+    rows.append(Script(f"manageScrollbars('{self._id}', false);"), )
+    
+    return rows
   
   def mk_body_cell(self, col_pos, row_index, col_def: DataGridColumnState):
     if not col_def.usable:
@@ -572,7 +568,7 @@ class DataGrid(BaseComponent):
       return mk_my_ellipsis(_value, cls="dt2-cell-content-number")
     
     def process_cell_content(_value):
-      value_str = str(_value)
+      value_str = html.escape(str(_value))
       
       if FILTER_INPUT_CID not in self._state.filtered or (
           keyword := self._state.filtered[FILTER_INPUT_CID]) is None:
@@ -589,7 +585,6 @@ class DataGrid(BaseComponent):
       return tuple(res)
     
     column_type = col_def.type
-    # value = self._df.iloc[row_index, col_def.col_index]
     value = self._fast_access[col_def.col_id][row_index]
     
     if column_type == ColumnType.Bool:
@@ -879,7 +874,12 @@ class DataGrid(BaseComponent):
         dict: A dictionary where the keys are the column names of the input DataFrame
         and the values are the corresponding column values as NumPy arrays.
     """
-    return {col: df[col].to_numpy() for col in df.columns}
+    if df is None:
+      return {}
+    
+    res = {col: df[col].to_numpy() for col in df.columns}
+    res[ROW_INDEX_ID] = df.index.to_numpy()
+    return res
   
   @timed
   def __ft__(self):

src/components/datagrid_new/components/commands.py

@@ -91,12 +91,21 @@ class DataGridCommandManager(BaseCommandManager):
     return {
         "hx-post": f"{ROUTE_ROOT}{Routes.OnClick}",
         "hx-target": f"#tsm_{self._id}",
-        "hx-trigger" : "click",
+        "hx-trigger": "click",
         "hx-swap": "outerHTML",
         "hx-vals": f'js:{{_id: "{self._id}", cell_id:getCellId(event), modifier:getClickModifier(event), boundaries: getCellBoundaries(event)}}',
         "hx-on::before-request": f'validateOnClickRequest("{self._id}", event)',
     }
   
+  def get_page(self, page_index=0):
+    return {
+        "hx-get": f"{ROUTE_ROOT}{Routes.GetPage}",
+        "hx-target": f"#tb_{self._id}",
+        "hx-swap": "beforeend",
+        "hx-vals": f'{{"_id": "{self._id}", "page_index": "{page_index}"}}',
+        "hx-trigger": f"intersect root:#tb_{self._id} once",
+    }
+  
   def _get_hide_show_columns_attrs(self, mode, col_defs: list, new_value, cls=""):
     str_col_names = ", ".join(f"'{col_def.title}'" for col_def in col_defs)
     tooltip_msg = f"{mode} column{'s' if len(col_defs) > 1 else ''} {str_col_names}"
@@ -165,4 +174,4 @@ class FilterAllCommands(BaseCommandManager):
         "hx_vals": f'{{"_id": "{self._id}", "col_id":"{FILTER_INPUT_CID}"}}',
         "data_tooltip": "Reset filter",
         "cls": self.merge_class(cls, "mmt-tooltip"),
-    }
+    }

src/components/datagrid_new/constants.py

@@ -17,6 +17,9 @@ CONTAINER_HEIGHT = "container_height"
 
 DATAGRID_STATE_FOOTER = "footer"
 
+DATAGRID_PAGE_SIZE = 50
+
+ROW_INDEX_ID = "__row_index__"
 
 class Routes:
   Filter = "/filter"  # request the filtering in the grid
@@ -34,6 +37,7 @@ class Routes:
   ShowFooterMenu = "/show_footer_menu"
   UpdateState = "/update_state"
   YieldRow = "/yield-row"
+  GetPage = "/page"
 
 
 class ColumnType(Enum):

src/components/datagrid_new/settings.py

@@ -69,6 +69,7 @@ class DataGridSettings:
 class DataGridState:
   sidebar_visible: bool = False
   selected_view: str = None
+  row_index: bool = False
   columns: list[DataGridColumnState] = dataclasses.field(default_factory=list)
   rows: list[DataGridRowState] = dataclasses.field(default_factory=list)  # only the rows that have a specific state
   footers: list[DataGridFooterConf] = dataclasses.field(default_factory=list)

src/core/fasthtml_helper.py

@@ -8,6 +8,12 @@ attr_map = {
 }
 
 
+def safe_attr(attr_name):
+  attr_name = attr_name.replace("hx_", "hx-")
+  attr_name = attr_name.replace("data_", "data-")
+  return attr_map.get(attr_name, attr_name)
+
+
 def to_html(item):
   if item is None:
     return ""
@@ -24,14 +30,14 @@ def to_html(item):
 
 
 class MyFt:
-  def __init__(self, name, *args, **kwargs):
+  def __init__(self, tag, *args, **kwargs):
-    self.name = name
+    self.tag = tag
     self.children = args
-    self.attrs = kwargs
+    self.attrs = {safe_attr(k): v for k, v in kwargs.items()}
   
   def to_html(self):
     body_items = [to_html(item) for item in self.children]
-    return f"<{self.name} {' '.join(f'{attr_map.get(k, k)}="{v}"' for k, v in self.attrs.items())}>{' '.join(body_items)}</div>"
+    return f"<{self.tag} {' '.join(f'{k}="{v}"' for k, v in self.attrs.items())}>{' '.join(body_items)}</div>"
   
   def __ft__(self):
     return NotStr(self.to_html())

tests/helpers.py

@@ -642,10 +642,10 @@ def extract_table_values_new(ft, header=True):
     # first, get the header
   
   if header:
-    header = search_elements_by_name(ft, attrs={"class": "dt2-header"}, comparison_method='contains')[0]
+    header_element = search_elements_by_name(ft, attrs={"class": "dt2-header"}, comparison_method='contains')[0]
     header_map = {}
     res = OrderedDict()
-    for row in header.children:
+    for row in header_element.children:
       col_id = row.attrs["data-col"]
       title = row.attrs["data-tooltip"]
       header_map[col_id] = title
@@ -654,9 +654,10 @@ def extract_table_values_new(ft, header=True):
     body = search_elements_by_name(ft, attrs={"class": "dt2-body"}, comparison_method='contains')[0]
     for row in body.children:
       for col in row.children:
-        col_id = col.attrs["data-col"]
+        if hasattr(col, "attrs"):
-        cell_value = _get_cell_content_value(col)
+          col_id = col.attrs["data-col"]
-        res[header_map[col_id]].append(cell_value)
+          cell_value = _get_cell_content_value(col)
+          res[header_map[col_id]].append(cell_value)
     
     return res
   

tests/test_datagrid_new.py

@@ -509,3 +509,18 @@ def test_i_can_compute_footer_menu_position_when_not_enough_space(dg):
   )
   
   assert matches(menu, expected)
+
+
+def test_the_content_of_the_cell_is_escaped(empty_dg):
+  df = pd.DataFrame({
+      'value': ['<div> My Content </div>'],
+      'value2': ['{My Content}'],
+  })
+  my_dg = empty_dg.init_from_dataframe(df)
+  
+  actual = my_dg.__ft__()
+  table_content = extract_table_values_new(actual, header=True)
+  
+  assert table_content == OrderedDict({
+      'value': ['&lt;div&gt; My Content &lt;/div&gt;'],
+      'value2': ['{My Content}']})