Working on improving the perf

.gitignore

@@ -13,6 +13,7 @@ tools.db
 .mytools_db
 .idea/MyManagingTools.iml
 .idea/misc.xml
+**/*.prof
 
 # Created by .ignore support plugin (hsz.mobi)
 ### Python template

Makefile

@@ -18,6 +18,8 @@ clean:
 	rm -rf Untitled*.ipynb
 	rm -rf .ipynb_checkpoints
 	rm -rf src/tools.db
+	rm -rf src/*.out
+	rm -rf src/*.prof
 	find . -name '.sesskey' -exec rm -rf {} +
 	find . -name '.pytest_cache' -exec rm -rf {} +
 	find . -name '__pycache__' -exec rm -rf {} +

README.md

@@ -35,3 +35,10 @@ docker-compose down
 ```shell
 docker-compose build
 ```
+
+# Profiling
+```shell
+cd src
+python -m cProfile -o profile.out main.py
+snakeviz profile.out # 'pip install snakeviz' if snakeviz is not installed
+```

src/components/datagrid_new/DataGridApp.py

@@ -1,6 +1,9 @@
+import asyncio
 import json
 import logging
 
+from fasthtml.components import Div, sse_message
+from fasthtml.core import EventStream
 from fasthtml.fastapp import fast_app
 from starlette.datastructures import UploadFile
 
@@ -136,3 +139,22 @@ def post(session, _id: str, state: str, args: str = None):
   logger.debug(f"Entering on_state_changed with args {_id=}, {state=}, {args=}")
   instance = InstanceManager.get(session, _id)
   return instance.manage_state_changed(state, args)
+
+
+@rt(Routes.YieldRow)
+async def get(session, _id: str):
+  logger.debug(f"Entering {Routes.YieldRow} with args {_id=}")
+  instance = InstanceManager.get(session, _id)
+  return EventStream(instance.mk_lazy_body_content())
+
+
+async def number_generator2():
+  for i in range(20):
+    yield sse_message(Div(i * 5 + 1))
+    yield sse_message(Div(i * 5 + 2))
+    yield sse_message(Div(i * 5 + 3))
+    yield sse_message(Div(i * 5 + 4))
+    yield sse_message(Div(i * 5 + 5))
+    await asyncio.sleep(0.1)
+  
+  yield f"event: close\ndata: \n\n"

src/components/datagrid_new/assets/Datagrid.js

@@ -1,6 +1,10 @@
 function bindDatagrid(datagridId, allowColumnsReordering) {
     bindScrollbars(datagridId);
     makeResizable(datagridId)
+
+    document.body.addEventListener('htmx:sseBeforeMessage', function (e) {
+        console.log("htmx:sseBeforeMessage", e)
+    })
 }
 
 function bindScrollbars(datagridId) {
@@ -495,3 +499,4 @@ function onAfterSettle(datagridId, event) {
         bindDatagrid(datagridId)
     }
 }
+

src/components/datagrid_new/components/DataGrid.py

@@ -1,3 +1,4 @@
+import asyncio
 import copy
 import logging
 from io import BytesIO
@@ -20,9 +21,10 @@ from components.datagrid_new.db_management import DataGridDbManager
 from components.datagrid_new.settings import DataGridRowState, DataGridColumnState, \
   DataGridFooterConf, DataGridState, DataGridSettings, DatagridView
 from components_helpers import mk_icon, mk_ellipsis
+from core.fasthtml_helper import MyDiv, mk_my_ellipsis, MySpan, mk_my_icon
 from core.instance_manager import InstanceManager
 from core.settings_management import SettingsManager
-from core.utils import get_unique_id, make_safe_id
+from core.utils import get_unique_id, make_safe_id, timed
 
 logger = logging.getLogger("DataGrid")
 
@@ -59,6 +61,7 @@ class DataGrid(BaseComponent):
     self._state: DataGridState = self._db.load_state()
     self._settings: DataGridSettings = grid_settings or self._db.load_settings()
     self._df: DataFrame | None = self._db.load_dataframe()
+    self._fast_access = self._init_fast_access(self._df)
     
     # update boundaries if possible
     self.set_boundaries(boundaries)
@@ -126,6 +129,7 @@ class DataGrid(BaseComponent):
                                                col_id,
                                                _get_column_type(self._df[make_safe_id(col_id)].dtype))
                            for col_index, col_id in enumerate(df.columns)]
+    self._fast_access = self._init_fast_access(self._df)
     
     if save_state:
       self._db.save_all(None, self._state, self._df)
@@ -386,6 +390,7 @@ class DataGrid(BaseComponent):
       id=f"scb_{self._id}",
     )
   
+  @timed
   def mk_table(self, oob=False):
     htmx_extra_params = {
         "hx-on::before-settle": f"onAfterSettle('{self._id}', event);",
@@ -439,7 +444,8 @@ class DataGrid(BaseComponent):
       _mk_keyboard_management(),
       Div(
         self.mk_table_header(),
-        self.mk_table_body(),
+        #self.mk_table_body(),
+        self.mk_table_body_lazy(),
         self.mk_table_footer(),
         cls="dt2-inner-table"),
       cls="dt2-table",
@@ -474,11 +480,27 @@ class DataGrid(BaseComponent):
     
     header_class = "dt2-row dt2-header" + "" if self._settings.header_visible else " hidden"
     return Div(
+      Div(sse_swap="message"),
       *[_mk_header(col_def) for col_def in self._state.columns],
       cls=header_class,
       id=f"th_{self._id}"
     )
   
+  def mk_table_body_lazy(self):
+    
+    max_height = self._compute_body_max_height()
+    
+    return Div(
+      hx_ext="sse",
+      sse_connect=f"{ROUTE_ROOT}{Routes.YieldRow}?_id={self._id}",
+      sse_close='close',
+      sse_swap="message",
+      hx_swap="beforeend",
+      cls="dt2-body",
+      style=f"max-height:{max_height}px;",
+      id=f"tb_{self._id}",
+    )
+  
   def mk_table_body(self):
     df = self._get_filtered_df()
     max_height = self._compute_body_max_height()
@@ -507,34 +529,50 @@ class DataGrid(BaseComponent):
       id=f"tf_{self._id}"
     )
   
+  async def mk_lazy_body_content(self):
+    df = self._get_filtered_df()
+    for i, row_index in enumerate(df.index):
+      yield sse_message(Div(
+        *[self.mk_body_cell(col_pos, row_index, col_def) for col_pos, col_def in enumerate(self._state.columns)],
+        cls="dt2-row",
+        data_row=f"{row_index}",
+        id=f"tr_{self._id}-{row_index}",
+      ))
+      if i % 50 == 0:
+        await asyncio.sleep(0.01)
+        logger.debug(f"yielding row {i}")
+    
+    logger.debug(f"yielding close event")
+    yield f"event: close\ndata: \n\n"
+  
   def mk_body_cell(self, col_pos, row_index, col_def: DataGridColumnState):
     if not col_def.usable:
       return None
     
     if not col_def.visible:
-      return Div(cls="dt2-col-hidden")
+      return MyDiv(cls="dt2-col-hidden")
     
     content = self.mk_body_cell_content(col_pos, row_index, col_def)
     
-    return Div(content,
+    return MyDiv(content,
                  data_col=col_def.col_id,
                  style=f"width:{col_def.width}px;",
                  cls="dt2-cell")
   
   def mk_body_cell_content(self, col_pos, row_index, col_def: DataGridColumnState):
     
-    def mk_bool(value):
-      return Div(mk_icon(icon_checked if value else icon_unchecked, can_select=False),
+    def mk_bool(_value):
+      return MyDiv(mk_my_icon(icon_checked if _value else icon_unchecked, can_select=False),
                    cls="dt2-cell-content-checkbox")
     
-    def mk_text(value):
-      return mk_ellipsis(value, cls="dt2-cell-content-text")
+    def mk_text(_value):
+      return mk_my_ellipsis(_value, cls="dt2-cell-content-text")
     
-    def mk_number(value):
-      return mk_ellipsis(value, cls="dt2-cell-content-number")
+    def mk_number(_value):
+      return mk_my_ellipsis(_value, cls="dt2-cell-content-number")
     
-    def process_cell_content(value):
-      value_str = str(value)
+    def process_cell_content(_value):
+      value_str = str(_value)
       
       if FILTER_INPUT_CID not in self._state.filtered or (
           keyword := self._state.filtered[FILTER_INPUT_CID]) is None:
@@ -545,21 +583,23 @@ class DataGrid(BaseComponent):
         return value_str
       
       len_keyword = len(keyword)
-      res = [Span(value_str[:index])] if index > 0 else []
-      res += [Span(value_str[index:index + len_keyword], cls="dt2-highlight-1")]
-      res += [Span(value_str[index + len_keyword:])] if len(value_str) > len_keyword else []
+      res = [MySpan(value_str[:index])] if index > 0 else []
+      res += [MySpan(value_str[index:index + len_keyword], cls="dt2-highlight-1")]
+      res += [MySpan(value_str[index + len_keyword:])] if len(value_str) > len_keyword else []
       return tuple(res)
     
     column_type = col_def.type
+    # value = self._df.iloc[row_index, col_def.col_index]
+    value = self._fast_access[col_def.col_id][row_index]
     
     if column_type == ColumnType.Bool:
-      content = mk_bool(self._df.iloc[row_index, col_def.col_index])
+      content = mk_bool(value)
     elif column_type == ColumnType.Number:
-      content = mk_number(process_cell_content(self._df.iloc[row_index, col_def.col_index]))
+      content = mk_number(process_cell_content(value))
     elif column_type == ColumnType.RowIndex:
       content = mk_number(row_index)
     else:
-      content = mk_text(process_cell_content(self._df.iloc[row_index, col_def.col_index]))
+      content = mk_text(process_cell_content(value))
     
     return content
   
@@ -822,6 +862,26 @@ class DataGrid(BaseComponent):
     
     return True
   
+  @staticmethod
+  def _init_fast_access(df):
+    """
+    Generates a fast-access dictionary for a DataFrame.
+
+    This method converts the columns of the provided DataFrame into NumPy arrays
+    and stores them as values in a dictionary, using the column names as keys.
+    This allows for efficient access to the data stored in the DataFrame.
+
+    Args:
+        df (DataFrame): The input pandas DataFrame whose columns are to be converted
+        into a dictionary of NumPy arrays.
+
+    Returns:
+        dict: A dictionary where the keys are the column names of the input DataFrame
+        and the values are the corresponding column values as NumPy arrays.
+    """
+    return {col: df[col].to_numpy() for col in df.columns}
+  
+  @timed
   def __ft__(self):
     return Div(
       Div(
@@ -844,7 +904,7 @@ class DataGrid(BaseComponent):
   @staticmethod
   def new(session, data, index=None):
     datagrid = DataGrid(session, DataGrid.create_component_id(session))
-    #dataframe = DataFrame(data, index=index)
+    # dataframe = DataFrame(data, index=index)
     dataframe = DataFrame(data)
     datagrid.init_from_dataframe(dataframe)
     return datagrid

src/components/datagrid_new/constants.py

@@ -33,6 +33,7 @@ class Routes:
   UpdateView = "/update_view"
   ShowFooterMenu = "/show_footer_menu"
   UpdateState = "/update_state"
+  YieldRow = "/yield-row"
 
 
 class ColumnType(Enum):
@@ -44,11 +45,13 @@ class ColumnType(Enum):
   Choice = "Choice"
   List = "List"
 
+
 class ViewType(Enum):
   Table = "Table"
   Chart = "Chart"
   Form = "Form"
 
+
 class FooterAggregation(Enum):
   Sum = "Sum"
   Mean = "Mean"

src/components/repositories/RepositoriesApp.py

@@ -20,7 +20,7 @@ def get(session):
 
 
 @rt(Routes.AddRepository)
-def post(session, _id: str, tab_id: str, form_id: str, repository: str, table: str, tab_boundaries:str):
+def post(session, _id: str, tab_id: str, form_id: str, repository: str, table: str, tab_boundaries: str):
   logger.debug(
     f"Entering {Routes.AddRepository} with args {debug_session(session)}, {_id=}, {tab_id=}, {form_id=}, {repository=}, {table=}, {tab_boundaries=}")
   instance = InstanceManager.get(session, _id)  # Repository
@@ -34,8 +34,9 @@ def get(session, _id: str, repository_name: str):
 
 
 @rt(Routes.AddTable)
-def post(session, _id: str, tab_id: str, form_id: str, repository_name: str, table_name: str, tab_boundaries:str):
-  logger.debug(f"Entering {Routes.AddTable} with args {debug_session(session)}, {_id=}, {tab_id=}, {form_id=}, {repository_name=}, {table_name=}, {tab_boundaries=}")
+def post(session, _id: str, tab_id: str, form_id: str, repository_name: str, table_name: str, tab_boundaries: str):
+  logger.debug(
+    f"Entering {Routes.AddTable} with args {debug_session(session)}, {_id=}, {tab_id=}, {form_id=}, {repository_name=}, {table_name=}, {tab_boundaries=}")
   instance = InstanceManager.get(session, _id)
   return instance.add_new_table(tab_id, form_id, repository_name, table_name, json.loads(tab_boundaries))
 
@@ -48,7 +49,8 @@ def put(session, _id: str, repository: str):
 
 
 @rt(Routes.ShowTable)
-def get(session, _id: str, repository: str, table: str, tab_boundaries:str):
-  logger.debug(f"Entering {Routes.ShowTable} with args {debug_session(session)}, {_id=}, {repository=}, {table=}, {tab_boundaries=}")
+def get(session, _id: str, repository: str, table: str, tab_boundaries: str):
+  logger.debug(
+    f"Entering {Routes.ShowTable} with args {debug_session(session)}, {_id=}, {repository=}, {table=}, {tab_boundaries=}")
   instance = InstanceManager.get(session, _id)
   return instance.show_table(repository, table, json.loads(tab_boundaries))

src/core/fasthtml_helper.py

@@ -0,0 +1,70 @@
+from fastcore.basics import NotStr
+
+from core.utils import merge_classes
+
+attr_map = {
+    "cls": "class",
+    "_id": "id",
+}
+
+
+def to_html(item):
+  if item is None:
+    return ""
+  elif isinstance(item, str):
+    return item
+  elif isinstance(item, (int, float, bool)):
+    return str(item)
+  elif isinstance(item, MyFt):
+    return item.to_html()
+  elif isinstance(item, NotStr):
+    return str(item)
+  else:
+    raise Exception(f"Unsupported type: {type(item)}, {item=}")
+
+
+class MyFt:
+  def __init__(self, name, *args, **kwargs):
+    self.name = name
+    self.children = args
+    self.attrs = kwargs
+  
+  def to_html(self):
+    body_items = [to_html(item) for item in self.children]
+    return f"<{self.name} {' '.join(f'{attr_map.get(k, k)}="{v}"' for k, v in self.attrs.items())}>{' '.join(body_items)}</div>"
+  
+  def __ft__(self):
+    return NotStr(self.to_html())
+
+
+class MyDiv(MyFt):
+  def __init__(self, *args, **kwargs):
+    super().__init__("div", *args, **kwargs)
+
+
+class MySpan(MyFt):
+  def __init__(self, *args, **kwargs):
+    super().__init__("span", *args, **kwargs)
+
+
+def mk_my_ellipsis(txt: str, cls='', **kwargs):
+  merged_cls = merge_classes("truncate",
+                             cls,
+                             kwargs)
+  return MyDiv(txt, cls=merged_cls, data_tooltip=txt, **kwargs)
+
+
+def mk_my_icon(icon, size=20, can_select=True, can_hover=False, cls='', tooltip=None, **kwargs):
+  merged_cls = merge_classes(f"icon-{size}",
+                             'icon-btn' if can_select else '',
+                             'mmt-btn' if can_hover else '',
+                             cls,
+                             kwargs)
+  return mk_my_tooltip(icon, tooltip, cls=merged_cls, **kwargs) if tooltip else MyDiv(icon, cls=merged_cls, **kwargs)
+
+
+def mk_my_tooltip(element, tooltip: str, cls='', **kwargs):
+  merged_cls = merge_classes("mmt-tooltip",
+                             cls,
+                             kwargs)
+  return MyDiv(element, cls=merged_cls, data_tooltip=tooltip, **kwargs)

src/core/utils.py

@@ -1,12 +1,16 @@
 import ast
 import base64
+import cProfile
+import functools
 import hashlib
 import importlib
 import inspect
 import pkgutil
 import re
+import time
 import types
 import uuid
+from datetime import datetime
 from enum import Enum
 from io import BytesIO
 from urllib.parse import urlparse
@@ -420,6 +424,66 @@ def split_host_port(url):
   return host, port
 
 
+def timed(func):
+  @functools.wraps(func)
+  def wrapper(*args, **kwargs):
+    start = time.perf_counter()
+    result = func(*args, **kwargs)
+    end = time.perf_counter()
+    
+    # get class name
+    class_name = None
+    if args:
+      # check the first argument to see if it's a class'
+      if inspect.isclass(args[0]):
+        class_name = args[0].__name__  # class method
+      elif hasattr(args[0], "__class__"):
+        class_name = args[0].__class__.__name__  # instance method
+    
+    if class_name:
+      print(f"[PERF] {class_name}.{func.__name__} took {end - start:.4f} sec")
+    else:
+      print(f"[PERF] {func.__name__} took {end - start:.4f} sec")
+    
+    return result
+  
+  return wrapper
+
+
+def profile_function(func):
+  @functools.wraps(func)
+  def wrapper(*args, **kwargs):
+    profiler = cProfile.Profile()
+    try:
+      profiler.enable()
+      result = func(*args, **kwargs)
+    finally:
+      profiler.disable()
+    
+    # Determine class name if any
+    class_name = None
+    if args:
+      if inspect.isclass(args[0]):
+        class_name = args[0].__name__  # class method
+      elif hasattr(args[0], "__class__"):
+        class_name = args[0].__class__.__name__  # instance method
+    
+    # Compose filename with timestamp
+    timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+    if class_name:
+      filename = f"{class_name}_{func.__name__}_{timestamp}.prof"
+    else:
+      filename = f"{func.__name__}_{timestamp}.prof"
+    
+    # Dump stats to file
+    profiler.dump_stats(filename)
+    print(f"[PROFILE] Profiling data saved to {filename}")
+    
+    return result
+  
+  return wrapper
+
+
 class UnreferencedNamesVisitor(ast.NodeVisitor):
   """
   Try to find symbols that will be requested by the ast
@@ -464,4 +528,3 @@ class UnreferencedNamesVisitor(ast.NodeVisitor):
     """
     self.names.add(node.arg)
     self.visit_selected(node, ["value"])
-

src/main.py

@@ -1,6 +1,7 @@
 # global layout
-import asyncio
 import logging.config
+import random
+from asyncio import sleep
 
 import yaml
 from fasthtml.common import *
@@ -54,6 +55,9 @@ links = [
     Link(href="./assets/daisyui-5-themes.css", rel="stylesheet", type="text/css"),
     Script(src="./assets/tailwindcss-browser@4.js"),
     
+    # SSE
+    Script(src="https://unpkg.com/htmx-ext-sse@2.2.1/sse.js"),
+    
     # Old drawer layout
     Script(src="./assets/DrawerLayout.js", defer=True),
     Link(rel="stylesheet", href="./assets/DrawerLayout.css"),
@@ -211,6 +215,25 @@ app, rt = fast_app(
   pico=False,
 )
 
+
+# -------------------------
+# Profiling middleware
+# -------------------------
+@app.middleware("http")
+async def timing_middleware(request, call_next):
+  import time
+  start_total = time.perf_counter()
+
+  # Call the next middleware or route handler
+  response = await call_next(request)
+
+  end_total = time.perf_counter()
+  elapsed = end_total - start_total
+
+  print(f"[PERF] Total server time: {elapsed:.4f} sec - Path: {request.url.path}")
+  return response
+
+
 settings_manager = SettingsManager()
 
 import_settings = AdminImportSettings(settings_manager, None)
@@ -253,6 +276,42 @@ def get(session):
           DrawerLayoutOld(pages),)
 
 
+shutdown_event = signal_shutdown()
+
+
+async def number_generator():
+  while True:  # not shutdown_event.is_set():
+    data = Article(random.randint(1, 100))
+    print(data)
+    yield sse_message(data)
+    await sleep(1)
+
+
+@rt("/sse")
+def get():
+  return Titled("SSE Random Number Generator",
+                P("Generate pairs of random numbers, as the list grows scroll downwards."),
+                Div(hx_ext="sse",
+                    sse_connect="/number-stream",
+                    hx_swap="beforeend show:bottom",
+                    sse_swap="message"))
+
+
+@rt("/number-stream")
+async def get(): return EventStream(number_generator())
+
+
+@rt('/toasting')
+def get(session):
+  # Normally one toast is enough, this allows us to see
+  # different toast types in action.
+  add_toast(session, f"Toast is being cooked", "info")
+  add_toast(session, f"Toast is ready", "success")
+  add_toast(session, f"Toast is getting a bit crispy", "warning")
+  add_toast(session, f"Toast is burning!", "error")
+  return Titled("I like toast")
+
+
 # Error Handling
 @app.get("/{path:path}")
 def not_found(path: str, session=None):
@@ -275,18 +334,7 @@ def not_found(path: str, session=None):
 setup_toasts(app)
 
 
-@rt('/toasting')
-def get(session):
-  # Normally one toast is enough, this allows us to see
-  # different toast types in action.
-  add_toast(session, f"Toast is being cooked", "info")
-  add_toast(session, f"Toast is ready", "success")
-  add_toast(session, f"Toast is getting a bit crispy", "warning")
-  add_toast(session, f"Toast is burning!", "error")
-  return Titled("I like toast")
-
-
-async def main():
+def main():
   logger.info(f"  Starting FastHTML server on http://localhost:{APP_PORT}")
   serve(port=APP_PORT)
 
@@ -294,9 +342,4 @@ async def main():
 if __name__ == "__main__":
   # Start your application
   logger.info("Application starting...")
-  try:
-    asyncio.run(main())
-  except KeyboardInterrupt:
-    logger.info("\nStopping application...")
-  except Exception as e:
-    logger.error(f"Error: {e}")
+  main()