Updating git

.gitignore

@@ -13,6 +13,7 @@ tools.db
 .mytools_db
 .idea/MyManagingTools.iml
 .idea/misc.xml
+**/*.prof
 
 # Created by .ignore support plugin (hsz.mobi)
 ### Python template

.idea/.gitignore

@@ -1,3 +0,0 @@
-# Default ignored files
-/shelf/
-/workspace.xml

.idea/inspectionProfiles/Project_Default.xml

@@ -1,6 +0,0 @@
-<component name="InspectionProjectProfileManager">
-  <profile version="1.0">
-    <option name="myName" value="Project Default" />
-    <inspection_tool class="PyInitNewSignatureInspection" enabled="false" level="WARNING" enabled_by_default="false" />
-  </profile>
-</component>

.idea/inspectionProfiles/profiles_settings.xml

@@ -1,6 +0,0 @@
-<component name="InspectionProjectProfileManager">
-  <settings>
-    <option name="USE_PROJECT_PROFILE" value="false" />
-    <version value="1.0" />
-  </settings>
-</component>

.idea/modules.xml

@@ -1,8 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project version="4">
-  <component name="ProjectModuleManager">
-    <modules>
-      <module fileurl="file://$PROJECT_DIR$/.idea/MyManagingTools.iml" filepath="$PROJECT_DIR$/.idea/MyManagingTools.iml" />
-    </modules>
-  </component>
-</project>

.idea/vcs.xml

@@ -1,6 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project version="4">
-  <component name="VcsDirectoryMappings">
-    <mapping directory="" vcs="Git" />
-  </component>
-</project>

Makefile

@@ -18,7 +18,9 @@ clean:
 	rm -rf Untitled*.ipynb
 	rm -rf .ipynb_checkpoints
 	rm -rf src/tools.db
+	rm -rf src/*.out
+	rm -rf src/*.prof
 	find . -name '.sesskey' -exec rm -rf {} +
 	find . -name '.pytest_cache' -exec rm -rf {} +
 	find . -name '__pycache__' -exec rm -rf {} +
-	find . -name 'debug.txt' -exec rm -rf {}
+	find . -name 'debug.txt' -exec rm -rf {}

README.md

@@ -34,4 +34,11 @@ docker-compose down
 1. **Rebuild**:
 ```shell
 docker-compose build
+```
+
+# Profiling
+```shell
+cd src
+python -m cProfile -o profile.out main.py
+snakeviz profile.out # 'pip install snakeviz' if snakeviz is not installed
 ```

src/components/datagrid_new/DataGridApp.py

@@ -1,6 +1,7 @@
 import json
 import logging
 
+from fasthtml.core import EventStream
 from fasthtml.fastapp import fast_app
 from starlette.datastructures import UploadFile
 
@@ -136,3 +137,10 @@ def post(session, _id: str, state: str, args: str = None):
   logger.debug(f"Entering on_state_changed with args {_id=}, {state=}, {args=}")
   instance = InstanceManager.get(session, _id)
   return instance.manage_state_changed(state, args)
+
+@rt(Routes.YieldRow)
+async def get(session, _id: str):
+  logger.debug(f"Entering {Routes.YieldRow} with args {_id=}")
+  instance = InstanceManager.get(session, _id)
+  return EventStream(instance.mk_async_lasy_body_content())
+  

src/components/datagrid_new/components/DataGrid.py

@@ -20,9 +20,10 @@ from components.datagrid_new.db_management import DataGridDbManager
 from components.datagrid_new.settings import DataGridRowState, DataGridColumnState, \
   DataGridFooterConf, DataGridState, DataGridSettings, DatagridView
 from components_helpers import mk_icon, mk_ellipsis
+from core.fasthtml_helper import MyDiv, mk_my_ellipsis, MySpan, mk_my_icon
 from core.instance_manager import InstanceManager
 from core.settings_management import SettingsManager
-from core.utils import get_unique_id, make_safe_id
+from core.utils import get_unique_id, make_safe_id, timed, profile_function
 
 logger = logging.getLogger("DataGrid")
 
@@ -386,6 +387,7 @@ class DataGrid(BaseComponent):
       id=f"scb_{self._id}",
     )
   
+  @profile_function
   def mk_table(self, oob=False):
     htmx_extra_params = {
         "hx-on::before-settle": f"onAfterSettle('{self._id}', event);",
@@ -439,7 +441,8 @@ class DataGrid(BaseComponent):
       _mk_keyboard_management(),
       Div(
         self.mk_table_header(),
-        self.mk_table_body(),
+        # self.mk_table_body(),
+        self.mk_table_body_lasy(),
         self.mk_table_footer(),
         cls="dt2-inner-table"),
       cls="dt2-table",
@@ -479,6 +482,22 @@ class DataGrid(BaseComponent):
       id=f"th_{self._id}"
     )
   
+  def mk_table_body_lasy(self):
+    df = self._get_filtered_df()
+    max_height = self._compute_body_max_height()
+    
+    return Div(
+      # *self.mk_lasy_body_content(df),
+      hx_ext="sse",
+      sse_connect=f"{ROUTE_ROOT}{Routes.YieldRow}",
+      hx_swap="beforeend show:bottom",
+      sse_swap="message",
+      cls="dt2-body",
+      style=f"max-height:{max_height}px;",
+      id=f"tb_{self._id}",
+      hx_vals=f'{{"_id": "{self._id}"}}',
+    )
+  
   def mk_table_body(self):
     df = self._get_filtered_df()
     max_height = self._compute_body_max_height()
@@ -507,31 +526,49 @@ class DataGrid(BaseComponent):
       id=f"tf_{self._id}"
     )
   
+  async def mk_async_lasy_body_content(self, df):
+    for row_index in df.index:
+      yield sse_message(Div(
+        *[self.mk_body_cell(col_pos, row_index, col_def) for col_pos, col_def in enumerate(self._state.columns)],
+        cls="dt2-row",
+        data_row=f"{row_index}",
+        id=f"tr_{self._id}-{row_index}",
+      ))
+  
+  def mk_lasy_body_content(self, df):
+    return [Div(
+      *[self.mk_body_cell(col_pos, row_index, col_def) for col_pos, col_def in enumerate(self._state.columns)],
+      cls="dt2-row",
+      data_row=f"{row_index}",
+      id=f"tr_{self._id}-{row_index}",
+    ) for row_index in df.index]
+  
   def mk_body_cell(self, col_pos, row_index, col_def: DataGridColumnState):
     if not col_def.usable:
       return None
     
     if not col_def.visible:
-      return Div(cls="dt2-col-hidden")
+      return MyDiv(cls="dt2-col-hidden")
     
     content = self.mk_body_cell_content(col_pos, row_index, col_def)
     
-    return Div(content,
+    return MyDiv(content,
-               data_col=col_def.col_id,
+                 data_col=col_def.col_id,
-               style=f"width:{col_def.width}px;",
+                 style=f"width:{col_def.width}px;",
-               cls="dt2-cell")
+                 cls="dt2-cell")
   
+  @profile_function
   def mk_body_cell_content(self, col_pos, row_index, col_def: DataGridColumnState):
     
     def mk_bool(value):
-      return Div(mk_icon(icon_checked if value else icon_unchecked, can_select=False),
+      return MyDiv(mk_my_icon(icon_checked if value else icon_unchecked, can_select=False),
-                 cls="dt2-cell-content-checkbox")
+                   cls="dt2-cell-content-checkbox")
     
     def mk_text(value):
-      return mk_ellipsis(value, cls="dt2-cell-content-text")
+      return mk_my_ellipsis(value, cls="dt2-cell-content-text")
     
     def mk_number(value):
-      return mk_ellipsis(value, cls="dt2-cell-content-number")
+      return mk_my_ellipsis(value, cls="dt2-cell-content-number")
     
     def process_cell_content(value):
       value_str = str(value)
@@ -545,9 +582,9 @@ class DataGrid(BaseComponent):
         return value_str
       
       len_keyword = len(keyword)
-      res = [Span(value_str[:index])] if index > 0 else []
+      res = [MySpan(value_str[:index])] if index > 0 else []
-      res += [Span(value_str[index:index + len_keyword], cls="dt2-highlight-1")]
+      res += [MySpan(value_str[index:index + len_keyword], cls="dt2-highlight-1")]
-      res += [Span(value_str[index + len_keyword:])] if len(value_str) > len_keyword else []
+      res += [MySpan(value_str[index + len_keyword:])] if len(value_str) > len_keyword else []
       return tuple(res)
     
     column_type = col_def.type
@@ -822,6 +859,7 @@ class DataGrid(BaseComponent):
     
     return True
   
+  @timed
   def __ft__(self):
     return Div(
       Div(
@@ -844,7 +882,7 @@ class DataGrid(BaseComponent):
   @staticmethod
   def new(session, data, index=None):
     datagrid = DataGrid(session, DataGrid.create_component_id(session))
-    #dataframe = DataFrame(data, index=index)
+    # dataframe = DataFrame(data, index=index)
     dataframe = DataFrame(data)
     datagrid.init_from_dataframe(dataframe)
     return datagrid

src/components/datagrid_new/constants.py

@@ -33,6 +33,7 @@ class Routes:
   UpdateView = "/update_view"
   ShowFooterMenu = "/show_footer_menu"
   UpdateState = "/update_state"
+  YieldRow = "/yield-row"
 
 
 class ColumnType(Enum):
@@ -44,11 +45,13 @@ class ColumnType(Enum):
   Choice = "Choice"
   List = "List"
 
+
 class ViewType(Enum):
   Table = "Table"
   Chart = "Chart"
   Form = "Form"
 
+
 class FooterAggregation(Enum):
   Sum = "Sum"
   Mean = "Mean"
@@ -59,4 +62,4 @@ class FooterAggregation(Enum):
   FilteredMean = "FilteredMean"
   FilteredMin = "FilteredMin"
   FilteredMax = "FilteredMax"
-  FilteredCount = "FilteredCount"
+  FilteredCount = "FilteredCount"

src/core/fasthtml_helper.py

@@ -0,0 +1,70 @@
+from fastcore.basics import NotStr
+
+from core.utils import merge_classes
+
+attr_map = {
+    "cls": "class",
+    "_id": "id",
+}
+
+
+def to_html(item):
+  if item is None:
+    return ""
+  elif isinstance(item, str):
+    return item
+  elif isinstance(item, (int, float, bool)):
+    return str(item)
+  elif isinstance(item, MyFt):
+    return item.to_html()
+  elif isinstance(item, NotStr):
+    return str(item)
+  else:
+    raise Exception(f"Unsupported type: {type(item)}, {item=}")
+
+
+class MyFt:
+  def __init__(self, name, *args, **kwargs):
+    self.name = name
+    self.args = args
+    self.kwargs = kwargs
+  
+  def to_html(self):
+    body_items = [to_html(item) for item in self.args]
+    return f"<{self.name} {' '.join(f'{attr_map.get(k, k)}="{v}"' for k, v in self.kwargs.items())}>{' '.join(body_items)}</div>"
+  
+  def __ft__(self):
+    return NotStr(self.to_html())
+
+
+class MyDiv(MyFt):
+  def __init__(self, *args, **kwargs):
+    super().__init__("div", *args, **kwargs)
+
+
+class MySpan(MyFt):
+  def __init__(self, *args, **kwargs):
+    super().__init__("span", *args, **kwargs)
+
+
+def mk_my_ellipsis(txt: str, cls='', **kwargs):
+  merged_cls = merge_classes("truncate",
+                             cls,
+                             kwargs)
+  return MyDiv(txt, cls=merged_cls, data_tooltip=txt, **kwargs)
+
+
+def mk_my_icon(icon, size=20, can_select=True, can_hover=False, cls='', tooltip=None, **kwargs):
+  merged_cls = merge_classes(f"icon-{size}",
+                             'icon-btn' if can_select else '',
+                             'mmt-btn' if can_hover else '',
+                             cls,
+                             kwargs)
+  return mk_my_tooltip(icon, tooltip, cls=merged_cls, **kwargs) if tooltip else MyDiv(icon, cls=merged_cls, **kwargs)
+
+
+def mk_my_tooltip(element, tooltip: str, cls='', **kwargs):
+  merged_cls = merge_classes("mmt-tooltip",
+                             cls,
+                             kwargs)
+  return MyDiv(element, cls=merged_cls, data_tooltip=tooltip, **kwargs)

src/core/utils.py

@@ -1,12 +1,15 @@
 import ast
 import base64
+import cProfile
 import hashlib
 import importlib
 import inspect
 import pkgutil
 import re
+import time
 import types
 import uuid
+from datetime import datetime
 from enum import Enum
 from io import BytesIO
 from urllib.parse import urlparse
@@ -420,6 +423,62 @@ def split_host_port(url):
   return host, port
 
 
+def timed(func):
+  def wrapper(*args, **kwargs):
+    start = time.perf_counter()
+    result = func(*args, **kwargs)
+    end = time.perf_counter()
+    
+    # get class name
+    class_name = None
+    if args:
+      # check the first argument to see if it's a class'
+      if inspect.isclass(args[0]):
+        class_name = args[0].__name__  # class method
+      elif hasattr(args[0], "__class__"):
+        class_name = args[0].__class__.__name__  # instance method
+    
+    if class_name:
+      print(f"[PERF] {class_name}.{func.__name__} took {end - start:.4f} sec")
+    else:
+      print(f"[PERF] {func.__name__} took {end - start:.4f} sec")
+    
+    return result
+  
+  return wrapper
+
+
+def profile_function(func):
+  def wrapper(*args, **kwargs):
+    profiler = cProfile.Profile()
+    profiler.enable()
+    result = func(*args, **kwargs)
+    profiler.disable()
+    
+    # Determine class name if any
+    class_name = None
+    if args:
+      if inspect.isclass(args[0]):
+        class_name = args[0].__name__  # class method
+      elif hasattr(args[0], "__class__"):
+        class_name = args[0].__class__.__name__  # instance method
+    
+    # Compose filename with timestamp
+    timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+    if class_name:
+      filename = f"{class_name}_{func.__name__}_{timestamp}.prof"
+    else:
+      filename = f"{func.__name__}_{timestamp}.prof"
+    
+    # Dump stats to file
+    profiler.dump_stats(filename)
+    print(f"[PROFILE] Profiling data saved to {filename}")
+    
+    return result
+  
+  return wrapper
+
+
 class UnreferencedNamesVisitor(ast.NodeVisitor):
   """
   Try to find symbols that will be requested by the ast
@@ -464,4 +523,3 @@ class UnreferencedNamesVisitor(ast.NodeVisitor):
     """
     self.names.add(node.arg)
     self.visit_selected(node, ["value"])
-

src/main.py

@@ -1,6 +1,8 @@
 # global layout
 import asyncio
 import logging.config
+import random
+from asyncio import sleep
 
 import yaml
 from fasthtml.common import *
@@ -54,6 +56,9 @@ links = [
     Link(href="./assets/daisyui-5-themes.css", rel="stylesheet", type="text/css"),
     Script(src="./assets/tailwindcss-browser@4.js"),
     
+    # SSE
+    Script(src="https://unpkg.com/htmx-ext-sse@2.2.1/sse.js"),
+    
     # Old drawer layout
     Script(src="./assets/DrawerLayout.js", defer=True),
     Link(rel="stylesheet", href="./assets/DrawerLayout.css"),
@@ -211,6 +216,23 @@ app, rt = fast_app(
   pico=False,
 )
 
+# -------------------------
+# Profiling middleware
+# -------------------------
+# @app.middleware("http")
+# async def timing_middleware(request, call_next):
+#   start_total = time.perf_counter()
+#
+#   # Call the next middleware or route handler
+#   response = await call_next(request)
+#
+#   end_total = time.perf_counter()
+#   elapsed = end_total - start_total
+#
+#   print(f"[PERF] Total server time: {elapsed:.4f} sec - Path: {request.url.path}")
+#   return response
+
+
 settings_manager = SettingsManager()
 
 import_settings = AdminImportSettings(settings_manager, None)
@@ -253,6 +275,42 @@ def get(session):
           DrawerLayoutOld(pages),)
 
 
+shutdown_event = signal_shutdown()
+
+
+async def number_generator():
+  while True: # not shutdown_event.is_set():
+    data = Article(random.randint(1, 100))
+    print(data)
+    yield sse_message(data)
+    await sleep(1)
+
+
+@rt("/sse")
+def get():
+  return Titled("SSE Random Number Generator",
+                P("Generate pairs of random numbers, as the list grows scroll downwards."),
+                Div(hx_ext="sse",
+                    sse_connect="/number-stream",
+                    hx_swap="beforeend show:bottom",
+                    sse_swap="message"))
+
+
+@rt("/number-stream")
+async def get(): return EventStream(number_generator())
+
+
+@rt('/toasting')
+def get(session):
+  # Normally one toast is enough, this allows us to see
+  # different toast types in action.
+  add_toast(session, f"Toast is being cooked", "info")
+  add_toast(session, f"Toast is ready", "success")
+  add_toast(session, f"Toast is getting a bit crispy", "warning")
+  add_toast(session, f"Toast is burning!", "error")
+  return Titled("I like toast")
+
+
 # Error Handling
 @app.get("/{path:path}")
 def not_found(path: str, session=None):
@@ -275,17 +333,6 @@ def not_found(path: str, session=None):
 setup_toasts(app)
 
 
-@rt('/toasting')
-def get(session):
-  # Normally one toast is enough, this allows us to see
-  # different toast types in action.
-  add_toast(session, f"Toast is being cooked", "info")
-  add_toast(session, f"Toast is ready", "success")
-  add_toast(session, f"Toast is getting a bit crispy", "warning")
-  add_toast(session, f"Toast is burning!", "error")
-  return Titled("I like toast")
-
-
 async def main():
   logger.info(f"  Starting FastHTML server on http://localhost:{APP_PORT}")
   serve(port=APP_PORT)