Working on improving the perf

.idea/.gitignore

@@ -0,0 +1,3 @@
+# Default ignored files
+/shelf/
+/workspace.xml

.idea/inspectionProfiles/Project_Default.xml

@@ -0,0 +1,6 @@
+<component name="InspectionProjectProfileManager">
+  <profile version="1.0">
+    <option name="myName" value="Project Default" />
+    <inspection_tool class="PyInitNewSignatureInspection" enabled="false" level="WARNING" enabled_by_default="false" />
+  </profile>
+</component>

.idea/inspectionProfiles/profiles_settings.xml

@@ -0,0 +1,6 @@
+<component name="InspectionProjectProfileManager">
+  <settings>
+    <option name="USE_PROJECT_PROFILE" value="false" />
+    <version value="1.0" />
+  </settings>
+</component>

.idea/modules.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectModuleManager">
+    <modules>
+      <module fileurl="file://$PROJECT_DIR$/.idea/MyManagingTools.iml" filepath="$PROJECT_DIR$/.idea/MyManagingTools.iml" />
+    </modules>
+  </component>
+</project>

.idea/vcs.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="VcsDirectoryMappings">
+    <mapping directory="" vcs="Git" />
+  </component>
+</project>

src/components/datagrid_new/DataGridApp.py

@@ -1,6 +1,8 @@
+import asyncio
 import json
 import logging
 
+from fasthtml.components import Div, sse_message
 from fasthtml.core import EventStream
 from fasthtml.fastapp import fast_app
 from starlette.datastructures import UploadFile
@@ -138,9 +140,21 @@ def post(session, _id: str, state: str, args: str = None):
   instance = InstanceManager.get(session, _id)
   return instance.manage_state_changed(state, args)
 
+
 @rt(Routes.YieldRow)
 async def get(session, _id: str):
   logger.debug(f"Entering {Routes.YieldRow} with args {_id=}")
   instance = InstanceManager.get(session, _id)
-  return EventStream(instance.mk_async_lasy_body_content())
+  return EventStream(instance.mk_lazy_body_content())
 
+
+async def number_generator2():
+  for i in range(20):
+    yield sse_message(Div(i * 5 + 1))
+    yield sse_message(Div(i * 5 + 2))
+    yield sse_message(Div(i * 5 + 3))
+    yield sse_message(Div(i * 5 + 4))
+    yield sse_message(Div(i * 5 + 5))
+    await asyncio.sleep(0.1)
+  
+  yield f"event: close\ndata: \n\n"

src/components/datagrid_new/assets/Datagrid.js

@@ -1,6 +1,10 @@
 function bindDatagrid(datagridId, allowColumnsReordering) {
     bindScrollbars(datagridId);
     makeResizable(datagridId)
+
+    document.body.addEventListener('htmx:sseBeforeMessage', function (e) {
+        console.log("htmx:sseBeforeMessage", e)
+    })
 }
 
 function bindScrollbars(datagridId) {
@@ -495,3 +499,4 @@ function onAfterSettle(datagridId, event) {
         bindDatagrid(datagridId)
     }
 }
+

src/components/datagrid_new/components/DataGrid.py

@@ -1,3 +1,4 @@
+import asyncio
 import copy
 import logging
 from io import BytesIO
@@ -23,7 +24,7 @@ from components_helpers import mk_icon, mk_ellipsis
 from core.fasthtml_helper import MyDiv, mk_my_ellipsis, MySpan, mk_my_icon
 from core.instance_manager import InstanceManager
 from core.settings_management import SettingsManager
-from core.utils import get_unique_id, make_safe_id, timed, profile_function
+from core.utils import get_unique_id, make_safe_id, timed
 
 logger = logging.getLogger("DataGrid")
 
@@ -60,6 +61,7 @@ class DataGrid(BaseComponent):
     self._state: DataGridState = self._db.load_state()
     self._settings: DataGridSettings = grid_settings or self._db.load_settings()
     self._df: DataFrame | None = self._db.load_dataframe()
+    self._fast_access = self._init_fast_access(self._df)
     
     # update boundaries if possible
     self.set_boundaries(boundaries)
@@ -127,6 +129,7 @@ class DataGrid(BaseComponent):
                                                col_id,
                                                _get_column_type(self._df[make_safe_id(col_id)].dtype))
                            for col_index, col_id in enumerate(df.columns)]
+    self._fast_access = self._init_fast_access(self._df)
     
     if save_state:
       self._db.save_all(None, self._state, self._df)
@@ -387,7 +390,7 @@ class DataGrid(BaseComponent):
       id=f"scb_{self._id}",
     )
   
-  @profile_function
+  @timed
   def mk_table(self, oob=False):
     htmx_extra_params = {
         "hx-on::before-settle": f"onAfterSettle('{self._id}', event);",
@@ -442,7 +445,7 @@ class DataGrid(BaseComponent):
       Div(
         self.mk_table_header(),
         #self.mk_table_body(),
-        self.mk_table_body_lasy(),
+        self.mk_table_body_lazy(),
         self.mk_table_footer(),
         cls="dt2-inner-table"),
       cls="dt2-table",
@@ -477,25 +480,25 @@ class DataGrid(BaseComponent):
     
     header_class = "dt2-row dt2-header" + "" if self._settings.header_visible else " hidden"
     return Div(
+      Div(sse_swap="message"),
       *[_mk_header(col_def) for col_def in self._state.columns],
       cls=header_class,
       id=f"th_{self._id}"
     )
   
-  def mk_table_body_lasy(self):
-    df = self._get_filtered_df()
+  def mk_table_body_lazy(self):
+    
     max_height = self._compute_body_max_height()
     
     return Div(
-      # *self.mk_lasy_body_content(df),
       hx_ext="sse",
-      sse_connect=f"{ROUTE_ROOT}{Routes.YieldRow}",
-      hx_swap="beforeend show:bottom",
+      sse_connect=f"{ROUTE_ROOT}{Routes.YieldRow}?_id={self._id}",
+      sse_close='close',
       sse_swap="message",
+      hx_swap="beforeend",
       cls="dt2-body",
       style=f"max-height:{max_height}px;",
       id=f"tb_{self._id}",
-      hx_vals=f'{{"_id": "{self._id}"}}',
     )
   
   def mk_table_body(self):
@@ -526,22 +529,21 @@ class DataGrid(BaseComponent):
       id=f"tf_{self._id}"
     )
   
-  async def mk_async_lasy_body_content(self, df):
-    for row_index in df.index:
+  async def mk_lazy_body_content(self):
+    df = self._get_filtered_df()
+    for i, row_index in enumerate(df.index):
       yield sse_message(Div(
         *[self.mk_body_cell(col_pos, row_index, col_def) for col_pos, col_def in enumerate(self._state.columns)],
         cls="dt2-row",
         data_row=f"{row_index}",
         id=f"tr_{self._id}-{row_index}",
       ))
+      if i % 50 == 0:
+        await asyncio.sleep(0.01)
+        logger.debug(f"yielding row {i}")
     
-  def mk_lasy_body_content(self, df):
-    return [Div(
-      *[self.mk_body_cell(col_pos, row_index, col_def) for col_pos, col_def in enumerate(self._state.columns)],
-      cls="dt2-row",
-      data_row=f"{row_index}",
-      id=f"tr_{self._id}-{row_index}",
-    ) for row_index in df.index]
+    logger.debug(f"yielding close event")
+    yield f"event: close\ndata: \n\n"
   
   def mk_body_cell(self, col_pos, row_index, col_def: DataGridColumnState):
     if not col_def.usable:
@@ -557,21 +559,20 @@ class DataGrid(BaseComponent):
                  style=f"width:{col_def.width}px;",
                  cls="dt2-cell")
   
-  @profile_function
   def mk_body_cell_content(self, col_pos, row_index, col_def: DataGridColumnState):
     
-    def mk_bool(value):
-      return MyDiv(mk_my_icon(icon_checked if value else icon_unchecked, can_select=False),
+    def mk_bool(_value):
+      return MyDiv(mk_my_icon(icon_checked if _value else icon_unchecked, can_select=False),
                    cls="dt2-cell-content-checkbox")
     
-    def mk_text(value):
-      return mk_my_ellipsis(value, cls="dt2-cell-content-text")
+    def mk_text(_value):
+      return mk_my_ellipsis(_value, cls="dt2-cell-content-text")
     
-    def mk_number(value):
-      return mk_my_ellipsis(value, cls="dt2-cell-content-number")
+    def mk_number(_value):
+      return mk_my_ellipsis(_value, cls="dt2-cell-content-number")
     
-    def process_cell_content(value):
-      value_str = str(value)
+    def process_cell_content(_value):
+      value_str = str(_value)
       
       if FILTER_INPUT_CID not in self._state.filtered or (
           keyword := self._state.filtered[FILTER_INPUT_CID]) is None:
@@ -588,15 +589,17 @@ class DataGrid(BaseComponent):
       return tuple(res)
     
     column_type = col_def.type
+    # value = self._df.iloc[row_index, col_def.col_index]
+    value = self._fast_access[col_def.col_id][row_index]
     
     if column_type == ColumnType.Bool:
-      content = mk_bool(self._df.iloc[row_index, col_def.col_index])
+      content = mk_bool(value)
     elif column_type == ColumnType.Number:
-      content = mk_number(process_cell_content(self._df.iloc[row_index, col_def.col_index]))
+      content = mk_number(process_cell_content(value))
     elif column_type == ColumnType.RowIndex:
       content = mk_number(row_index)
     else:
-      content = mk_text(process_cell_content(self._df.iloc[row_index, col_def.col_index]))
+      content = mk_text(process_cell_content(value))
     
     return content
   
@@ -859,6 +862,25 @@ class DataGrid(BaseComponent):
     
     return True
   
+  @staticmethod
+  def _init_fast_access(df):
+    """
+    Generates a fast-access dictionary for a DataFrame.
+
+    This method converts the columns of the provided DataFrame into NumPy arrays
+    and stores them as values in a dictionary, using the column names as keys.
+    This allows for efficient access to the data stored in the DataFrame.
+
+    Args:
+        df (DataFrame): The input pandas DataFrame whose columns are to be converted
+        into a dictionary of NumPy arrays.
+
+    Returns:
+        dict: A dictionary where the keys are the column names of the input DataFrame
+        and the values are the corresponding column values as NumPy arrays.
+    """
+    return {col: df[col].to_numpy() for col in df.columns}
+  
   @timed
   def __ft__(self):
     return Div(

src/components/repositories/RepositoriesApp.py

@@ -35,7 +35,8 @@ def get(session, _id: str, repository_name: str):
 
 @rt(Routes.AddTable)
 def post(session, _id: str, tab_id: str, form_id: str, repository_name: str, table_name: str, tab_boundaries: str):
-  logger.debug(f"Entering {Routes.AddTable} with args {debug_session(session)}, {_id=}, {tab_id=}, {form_id=}, {repository_name=}, {table_name=}, {tab_boundaries=}")
+  logger.debug(
+    f"Entering {Routes.AddTable} with args {debug_session(session)}, {_id=}, {tab_id=}, {form_id=}, {repository_name=}, {table_name=}, {tab_boundaries=}")
   instance = InstanceManager.get(session, _id)
   return instance.add_new_table(tab_id, form_id, repository_name, table_name, json.loads(tab_boundaries))
 
@@ -49,6 +50,7 @@ def put(session, _id: str, repository: str):
 
 @rt(Routes.ShowTable)
 def get(session, _id: str, repository: str, table: str, tab_boundaries: str):
-  logger.debug(f"Entering {Routes.ShowTable} with args {debug_session(session)}, {_id=}, {repository=}, {table=}, {tab_boundaries=}")
+  logger.debug(
+    f"Entering {Routes.ShowTable} with args {debug_session(session)}, {_id=}, {repository=}, {table=}, {tab_boundaries=}")
   instance = InstanceManager.get(session, _id)
   return instance.show_table(repository, table, json.loads(tab_boundaries))

src/core/fasthtml_helper.py

@@ -26,12 +26,12 @@ def to_html(item):
 class MyFt:
   def __init__(self, name, *args, **kwargs):
     self.name = name
-    self.args = args
-    self.kwargs = kwargs
+    self.children = args
+    self.attrs = kwargs
   
   def to_html(self):
-    body_items = [to_html(item) for item in self.args]
-    return f"<{self.name} {' '.join(f'{attr_map.get(k, k)}="{v}"' for k, v in self.kwargs.items())}>{' '.join(body_items)}</div>"
+    body_items = [to_html(item) for item in self.children]
+    return f"<{self.name} {' '.join(f'{attr_map.get(k, k)}="{v}"' for k, v in self.attrs.items())}>{' '.join(body_items)}</div>"
   
   def __ft__(self):
     return NotStr(self.to_html())

src/core/utils.py

@@ -1,6 +1,7 @@
 import ast
 import base64
 import cProfile
+import functools
 import hashlib
 import importlib
 import inspect
@@ -424,6 +425,7 @@ def split_host_port(url):
 
 
 def timed(func):
+  @functools.wraps(func)
   def wrapper(*args, **kwargs):
     start = time.perf_counter()
     result = func(*args, **kwargs)
@@ -449,10 +451,13 @@ def timed(func):
 
 
 def profile_function(func):
+  @functools.wraps(func)
   def wrapper(*args, **kwargs):
     profiler = cProfile.Profile()
+    try:
       profiler.enable()
       result = func(*args, **kwargs)
+    finally:
       profiler.disable()
     
     # Determine class name if any

src/main.py

@@ -1,5 +1,4 @@
 # global layout
-import asyncio
 import logging.config
 import random
 from asyncio import sleep
@@ -216,21 +215,23 @@ app, rt = fast_app(
   pico=False,
 )
 
+
 # -------------------------
 # Profiling middleware
 # -------------------------
-# @app.middleware("http")
-# async def timing_middleware(request, call_next):
-#   start_total = time.perf_counter()
-#
-#   # Call the next middleware or route handler
-#   response = await call_next(request)
-#
-#   end_total = time.perf_counter()
-#   elapsed = end_total - start_total
-#
-#   print(f"[PERF] Total server time: {elapsed:.4f} sec - Path: {request.url.path}")
-#   return response
+@app.middleware("http")
+async def timing_middleware(request, call_next):
+  import time
+  start_total = time.perf_counter()
+
+  # Call the next middleware or route handler
+  response = await call_next(request)
+
+  end_total = time.perf_counter()
+  elapsed = end_total - start_total
+
+  print(f"[PERF] Total server time: {elapsed:.4f} sec - Path: {request.url.path}")
+  return response
 
 
 settings_manager = SettingsManager()
@@ -333,7 +334,7 @@ def not_found(path: str, session=None):
 setup_toasts(app)
 
 
-async def main():
+def main():
   logger.info(f"  Starting FastHTML server on http://localhost:{APP_PORT}")
   serve(port=APP_PORT)
 
@@ -341,9 +342,4 @@ async def main():
 if __name__ == "__main__":
   # Start your application
   logger.info("Application starting...")
-  try:
-    asyncio.run(main())
-  except KeyboardInterrupt:
-    logger.info("\nStopping application...")
-  except Exception as e:
-    logger.error(f"Error: {e}")
+  main()