Fixed performance issues by creating a dedicated store for dataframe and optimizing

src/myfasthtml/controls/DataGrid.py

@@ -79,11 +79,6 @@ class DatagridState(DbObject):
       self.selection: DatagridSelectionState = DatagridSelectionState()
       self.cell_formats: dict = {}
       self.table_format: list = []
-      self.ne_df = None
-      
-      self.ns_fast_access = None
-      self.ns_row_data = None
-      self.ns_total_rows = None
 
 
 class DatagridSettings(DbObject):
@@ -104,6 +99,16 @@ class DatagridSettings(DbObject):
       self.enable_formatting: bool = True
 
 
+class DatagridStore(DbObject):
+  def __init__(self, owner, save_state):
+    with self.initializing():
+      super().__init__(owner, name=f"{owner.get_id()}#df", save_state=save_state)
+      self.ne_df = None
+      self.ns_fast_access = None
+      self.ns_row_data = None
+      self.ns_total_rows = None
+
+
 class Commands(BaseCommands):
   def get_page(self, page_index: int):
     return Command("GetPage",
@@ -187,9 +192,10 @@ class DataGrid(MultipleInstance):
     name, namespace = (conf.name, conf.namespace) if conf else ("No name", "__default__")
     self._settings = DatagridSettings(self, save_state=save_state, name=name, namespace=namespace)
     self._state = DatagridState(self, save_state=self._settings.save_state)
+    self._df_store = DatagridStore(self, save_state=self._settings.save_state)
     self._formatting_engine = FormattingEngine()
     self.commands = Commands(self)
-    self.init_from_dataframe(self._state.ne_df, init_state=False)  # state comes from DatagridState
+    self.init_from_dataframe(self._df_store.ne_df, init_state=False)  # data comes from DatagridStore
     
     # add Panel
     self._panel = Panel(self, conf=PanelConf(show_display_right=False), _id="-panel")
@@ -250,7 +256,7 @@ class DataGrid(MultipleInstance):
   
   @property
   def _df(self):
-    return self._state.ne_df
+    return self._df_store.ne_df
   
   def _apply_sort(self, df):
     if df is None:
@@ -293,7 +299,7 @@ class DataGrid(MultipleInstance):
     df = self._df.copy()
     df = self._apply_sort(df)  # need to keep the real type to sort
     df = self._apply_filter(df)
-    self._state.ns_total_rows = len(df)
+    self._df_store.ns_total_rows = len(df)
     
     return df
   
@@ -391,14 +397,15 @@ class DataGrid(MultipleInstance):
       return _df.to_dict(orient='records')
     
     if df is not None:
-      self._state.ne_df = df
+      self._df_store.ne_df = df
       if init_state:
         self._df.columns = self._df.columns.map(make_safe_id)  # make sure column names are trimmed
-        self._state.rows = [DataGridRowState(row_id) for row_id in self._df.index]
+        self._df_store.save()
+        self._state.rows = []  # sparse: only rows with non-default state are stored
         self._state.columns = _init_columns(df)  # use df not self._df to keep the original title
-      self._state.ns_fast_access = _init_fast_access(self._df)
-      self._state.ns_row_data = _init_row_data(self._df)
-      self._state.ns_total_rows = len(self._df) if self._df is not None else 0
+      self._df_store.ns_fast_access = _init_fast_access(self._df)
+      self._df_store.ns_row_data = _init_row_data(self._df)
+      self._df_store.ns_total_rows = len(self._df) if self._df is not None else 0
     
     return self
   
@@ -427,10 +434,9 @@ class DataGrid(MultipleInstance):
     if cell_id in self._state.cell_formats:
       return self._state.cell_formats[cell_id]
     
-    if row_index < len(self._state.rows):
-      row_state = self._state.rows[row_index]
-      if row_state.format:
-        return row_state.format
+    row_state = next((r for r in self._state.rows if r.row_id == row_index), None)
+    if row_state and row_state.format:
+      return row_state.format
     
     if col_def.format:
       return col_def.format
@@ -502,7 +508,6 @@ class DataGrid(MultipleInstance):
     if (is_inside and
         cell_id_mousedown and cell_id_mouseup and
         cell_id_mousedown.startswith("tcell_") and cell_id_mouseup.startswith("tcell_")):
-      
       self._update_current_position(None)
       
       pos_mouse_down = self._get_pos_from_element_id(cell_id_mousedown)
@@ -615,7 +620,7 @@ class DataGrid(MultipleInstance):
       return Span(*res, cls=f"{css_class} truncate", style=style) if len(res) > 1 else res[0]
     
     column_type = col_def.type
-    value = self._state.ns_fast_access[col_def.col_id][row_index]
+    value = self._df_store.ns_fast_access[col_def.col_id][row_index]
     
     # Boolean type - uses cached HTML (only 2 possible values)
     if column_type == ColumnType.Bool:
@@ -630,7 +635,7 @@ class DataGrid(MultipleInstance):
     formatted_value = None
     rules = self._get_format_rules(col_pos, row_index, col_def)
     if rules:
-      row_data = self._state.ns_row_data[row_index] if row_index < len(self._state.ns_row_data) else None
+      row_data = self._df_store.ns_row_data[row_index] if row_index < len(self._df_store.ns_row_data) else None
       style, formatted_value = self._formatting_engine.apply_format(rules, value, row_data)
     
     # Use formatted value or convert to string
@@ -661,7 +666,7 @@ class DataGrid(MultipleInstance):
     if not col_def.visible:
       return None
     
-    value = self._state.ns_fast_access[col_def.col_id][row_index]
+    value = self._df_store.ns_fast_access[col_def.col_id][row_index]
     content = self.mk_body_cell_content(col_pos, row_index, col_def, filter_keyword_lower)
     
     return OptimizedDiv(content,
@@ -682,7 +687,7 @@ class DataGrid(MultipleInstance):
     
     start = page_index * DATAGRID_PAGE_SIZE
     end = start + DATAGRID_PAGE_SIZE
-    if self._state.ns_total_rows > end:
+    if self._df_store.ns_total_rows > end:
       last_row = df.index[end - 1]
     else:
       last_row = None
@@ -853,7 +858,7 @@ class DataGrid(MultipleInstance):
                )
   
   def render(self):
-    if self._state.ne_df is None:
+    if self._df_store.ne_df is None:
       return Div("No data to display !")
     
     return Div(

src/myfasthtml/controls/DataGridFormattingEditor.py

@@ -2,6 +2,7 @@ import logging
 from collections import defaultdict
 
 from myfasthtml.controls.DslEditor import DslEditor
+from myfasthtml.controls.datagrid_objects import DataGridRowState
 from myfasthtml.core.formatting.dsl import parse_dsl, DSLSyntaxError, ColumnScope, RowScope, CellScope, TableScope, TablesScope
 from myfasthtml.core.instances import InstancesManager
 
@@ -108,10 +109,11 @@ class DataGridFormattingEditor(DslEditor):
         logger.warning(f"Column '{column_name}' not found, skipping rules")
     
     for row_index, rules in rows_rules.items():
-      if row_index < len(state.rows):
-        state.rows[row_index].format = rules
-      else:
-        logger.warning(f"Row {row_index} out of range, skipping rules")
+      row_state = next((r for r in state.rows if r.row_id == row_index), None)
+      if row_state is None:
+        row_state = DataGridRowState(row_id=row_index)
+        state.rows.append(row_state)
+      row_state.format = rules
     
     for cell_id, rules in cells_rules.items():
       state.cell_formats[cell_id] = rules