diff --git a/.gitignore b/.gitignore
index 142f5be..2eb4f17 100644
--- a/.gitignore
+++ b/.gitignore
@@ -13,6 +13,7 @@ tools.db
.mytools_db
.idea/MyManagingTools.iml
.idea/misc.xml
+**/*.prof
# Created by .ignore support plugin (hsz.mobi)
### Python template
diff --git a/.idea/.gitignore b/.idea/.gitignore
deleted file mode 100644
index 26d3352..0000000
--- a/.idea/.gitignore
+++ /dev/null
@@ -1,3 +0,0 @@
-# Default ignored files
-/shelf/
-/workspace.xml
diff --git a/.idea/inspectionProfiles/Project_Default.xml b/.idea/inspectionProfiles/Project_Default.xml
deleted file mode 100644
index 4069cea..0000000
--- a/.idea/inspectionProfiles/Project_Default.xml
+++ /dev/null
@@ -1,6 +0,0 @@
-
-
-
-
-
-
\ No newline at end of file
diff --git a/.idea/inspectionProfiles/profiles_settings.xml b/.idea/inspectionProfiles/profiles_settings.xml
deleted file mode 100644
index 105ce2d..0000000
--- a/.idea/inspectionProfiles/profiles_settings.xml
+++ /dev/null
@@ -1,6 +0,0 @@
-
-
-
-
-
-
\ No newline at end of file
diff --git a/.idea/modules.xml b/.idea/modules.xml
deleted file mode 100644
index cab4603..0000000
--- a/.idea/modules.xml
+++ /dev/null
@@ -1,8 +0,0 @@
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/.idea/vcs.xml b/.idea/vcs.xml
deleted file mode 100644
index 35eb1dd..0000000
--- a/.idea/vcs.xml
+++ /dev/null
@@ -1,6 +0,0 @@
-
-
-
-
-
-
\ No newline at end of file
diff --git a/Makefile b/Makefile
index 85ef873..e885a8a 100644
--- a/Makefile
+++ b/Makefile
@@ -18,7 +18,9 @@ clean:
rm -rf Untitled*.ipynb
rm -rf .ipynb_checkpoints
rm -rf src/tools.db
+ rm -rf src/*.out
+ rm -rf src/*.prof
find . -name '.sesskey' -exec rm -rf {} +
find . -name '.pytest_cache' -exec rm -rf {} +
find . -name '__pycache__' -exec rm -rf {} +
- find . -name 'debug.txt' -exec rm -rf {}
\ No newline at end of file
+ find . -name 'debug.txt' -exec rm -rf {}
diff --git a/README.md b/README.md
index 4664d32..0eb3c4f 100644
--- a/README.md
+++ b/README.md
@@ -34,4 +34,11 @@ docker-compose down
1. **Rebuild**:
```shell
docker-compose build
+```
+
+# Profiling
+```shell
+cd src
+python -m cProfile -o profile.out main.py
+snakeviz profile.out # 'pip install snakeviz' if snakeviz is not installed
```
\ No newline at end of file
diff --git a/src/components/datagrid_new/DataGridApp.py b/src/components/datagrid_new/DataGridApp.py
index fda1ed0..5493530 100644
--- a/src/components/datagrid_new/DataGridApp.py
+++ b/src/components/datagrid_new/DataGridApp.py
@@ -1,6 +1,7 @@
import json
import logging
+from fasthtml.core import EventStream
from fasthtml.fastapp import fast_app
from starlette.datastructures import UploadFile
@@ -136,3 +137,10 @@ def post(session, _id: str, state: str, args: str = None):
logger.debug(f"Entering on_state_changed with args {_id=}, {state=}, {args=}")
instance = InstanceManager.get(session, _id)
return instance.manage_state_changed(state, args)
+
+@rt(Routes.YieldRow)
+async def get(session, _id: str):
+ logger.debug(f"Entering {Routes.YieldRow} with args {_id=}")
+ instance = InstanceManager.get(session, _id)
+ return EventStream(instance.mk_async_lasy_body_content())
+
\ No newline at end of file
diff --git a/src/components/datagrid_new/components/DataGrid.py b/src/components/datagrid_new/components/DataGrid.py
index 4709ac6..a13af75 100644
--- a/src/components/datagrid_new/components/DataGrid.py
+++ b/src/components/datagrid_new/components/DataGrid.py
@@ -20,9 +20,10 @@ from components.datagrid_new.db_management import DataGridDbManager
from components.datagrid_new.settings import DataGridRowState, DataGridColumnState, \
DataGridFooterConf, DataGridState, DataGridSettings, DatagridView
from components_helpers import mk_icon, mk_ellipsis
+from core.fasthtml_helper import MyDiv, mk_my_ellipsis, MySpan, mk_my_icon
from core.instance_manager import InstanceManager
from core.settings_management import SettingsManager
-from core.utils import get_unique_id, make_safe_id
+from core.utils import get_unique_id, make_safe_id, timed, profile_function
logger = logging.getLogger("DataGrid")
@@ -386,6 +387,7 @@ class DataGrid(BaseComponent):
id=f"scb_{self._id}",
)
+ @profile_function
def mk_table(self, oob=False):
htmx_extra_params = {
"hx-on::before-settle": f"onAfterSettle('{self._id}', event);",
@@ -439,7 +441,8 @@ class DataGrid(BaseComponent):
_mk_keyboard_management(),
Div(
self.mk_table_header(),
- self.mk_table_body(),
+ # self.mk_table_body(),
+ self.mk_table_body_lasy(),
self.mk_table_footer(),
cls="dt2-inner-table"),
cls="dt2-table",
@@ -479,6 +482,22 @@ class DataGrid(BaseComponent):
id=f"th_{self._id}"
)
+ def mk_table_body_lasy(self):
+ df = self._get_filtered_df()
+ max_height = self._compute_body_max_height()
+
+ return Div(
+ # *self.mk_lasy_body_content(df),
+ hx_ext="sse",
+ sse_connect=f"{ROUTE_ROOT}{Routes.YieldRow}",
+ hx_swap="beforeend show:bottom",
+ sse_swap="message",
+ cls="dt2-body",
+ style=f"max-height:{max_height}px;",
+ id=f"tb_{self._id}",
+ hx_vals=f'{{"_id": "{self._id}"}}',
+ )
+
def mk_table_body(self):
df = self._get_filtered_df()
max_height = self._compute_body_max_height()
@@ -507,31 +526,49 @@ class DataGrid(BaseComponent):
id=f"tf_{self._id}"
)
+ async def mk_async_lasy_body_content(self, df):
+ for row_index in df.index:
+ yield sse_message(Div(
+ *[self.mk_body_cell(col_pos, row_index, col_def) for col_pos, col_def in enumerate(self._state.columns)],
+ cls="dt2-row",
+ data_row=f"{row_index}",
+ id=f"tr_{self._id}-{row_index}",
+ ))
+
+ def mk_lasy_body_content(self, df):
+ return [Div(
+ *[self.mk_body_cell(col_pos, row_index, col_def) for col_pos, col_def in enumerate(self._state.columns)],
+ cls="dt2-row",
+ data_row=f"{row_index}",
+ id=f"tr_{self._id}-{row_index}",
+ ) for row_index in df.index]
+
def mk_body_cell(self, col_pos, row_index, col_def: DataGridColumnState):
if not col_def.usable:
return None
if not col_def.visible:
- return Div(cls="dt2-col-hidden")
+ return MyDiv(cls="dt2-col-hidden")
content = self.mk_body_cell_content(col_pos, row_index, col_def)
- return Div(content,
- data_col=col_def.col_id,
- style=f"width:{col_def.width}px;",
- cls="dt2-cell")
+ return MyDiv(content,
+ data_col=col_def.col_id,
+ style=f"width:{col_def.width}px;",
+ cls="dt2-cell")
+ @profile_function
def mk_body_cell_content(self, col_pos, row_index, col_def: DataGridColumnState):
def mk_bool(value):
- return Div(mk_icon(icon_checked if value else icon_unchecked, can_select=False),
- cls="dt2-cell-content-checkbox")
+ return MyDiv(mk_my_icon(icon_checked if value else icon_unchecked, can_select=False),
+ cls="dt2-cell-content-checkbox")
def mk_text(value):
- return mk_ellipsis(value, cls="dt2-cell-content-text")
+ return mk_my_ellipsis(value, cls="dt2-cell-content-text")
def mk_number(value):
- return mk_ellipsis(value, cls="dt2-cell-content-number")
+ return mk_my_ellipsis(value, cls="dt2-cell-content-number")
def process_cell_content(value):
value_str = str(value)
@@ -545,9 +582,9 @@ class DataGrid(BaseComponent):
return value_str
len_keyword = len(keyword)
- res = [Span(value_str[:index])] if index > 0 else []
- res += [Span(value_str[index:index + len_keyword], cls="dt2-highlight-1")]
- res += [Span(value_str[index + len_keyword:])] if len(value_str) > len_keyword else []
+ res = [MySpan(value_str[:index])] if index > 0 else []
+ res += [MySpan(value_str[index:index + len_keyword], cls="dt2-highlight-1")]
+ res += [MySpan(value_str[index + len_keyword:])] if len(value_str) > len_keyword else []
return tuple(res)
column_type = col_def.type
@@ -822,6 +859,7 @@ class DataGrid(BaseComponent):
return True
+ @timed
def __ft__(self):
return Div(
Div(
@@ -844,7 +882,7 @@ class DataGrid(BaseComponent):
@staticmethod
def new(session, data, index=None):
datagrid = DataGrid(session, DataGrid.create_component_id(session))
- #dataframe = DataFrame(data, index=index)
+ # dataframe = DataFrame(data, index=index)
dataframe = DataFrame(data)
datagrid.init_from_dataframe(dataframe)
return datagrid
diff --git a/src/components/datagrid_new/constants.py b/src/components/datagrid_new/constants.py
index e6d8ad7..ab406ff 100644
--- a/src/components/datagrid_new/constants.py
+++ b/src/components/datagrid_new/constants.py
@@ -33,6 +33,7 @@ class Routes:
UpdateView = "/update_view"
ShowFooterMenu = "/show_footer_menu"
UpdateState = "/update_state"
+ YieldRow = "/yield-row"
class ColumnType(Enum):
@@ -44,11 +45,13 @@ class ColumnType(Enum):
Choice = "Choice"
List = "List"
+
class ViewType(Enum):
Table = "Table"
Chart = "Chart"
Form = "Form"
+
class FooterAggregation(Enum):
Sum = "Sum"
Mean = "Mean"
@@ -59,4 +62,4 @@ class FooterAggregation(Enum):
FilteredMean = "FilteredMean"
FilteredMin = "FilteredMin"
FilteredMax = "FilteredMax"
- FilteredCount = "FilteredCount"
\ No newline at end of file
+ FilteredCount = "FilteredCount"
diff --git a/src/core/fasthtml_helper.py b/src/core/fasthtml_helper.py
new file mode 100644
index 0000000..eb0c633
--- /dev/null
+++ b/src/core/fasthtml_helper.py
@@ -0,0 +1,70 @@
+from fastcore.basics import NotStr
+
+from core.utils import merge_classes
+
+attr_map = {
+ "cls": "class",
+ "_id": "id",
+}
+
+
+def to_html(item):
+ if item is None:
+ return ""
+ elif isinstance(item, str):
+ return item
+ elif isinstance(item, (int, float, bool)):
+ return str(item)
+ elif isinstance(item, MyFt):
+ return item.to_html()
+ elif isinstance(item, NotStr):
+ return str(item)
+ else:
+ raise Exception(f"Unsupported type: {type(item)}, {item=}")
+
+
+class MyFt:
+ def __init__(self, name, *args, **kwargs):
+ self.name = name
+ self.args = args
+ self.kwargs = kwargs
+
+ def to_html(self):
+ body_items = [to_html(item) for item in self.args]
+ return f"<{self.name} {' '.join(f'{attr_map.get(k, k)}="{v}"' for k, v in self.kwargs.items())}>{' '.join(body_items)}"
+
+ def __ft__(self):
+ return NotStr(self.to_html())
+
+
+class MyDiv(MyFt):
+ def __init__(self, *args, **kwargs):
+ super().__init__("div", *args, **kwargs)
+
+
+class MySpan(MyFt):
+ def __init__(self, *args, **kwargs):
+ super().__init__("span", *args, **kwargs)
+
+
+def mk_my_ellipsis(txt: str, cls='', **kwargs):
+ merged_cls = merge_classes("truncate",
+ cls,
+ kwargs)
+ return MyDiv(txt, cls=merged_cls, data_tooltip=txt, **kwargs)
+
+
+def mk_my_icon(icon, size=20, can_select=True, can_hover=False, cls='', tooltip=None, **kwargs):
+ merged_cls = merge_classes(f"icon-{size}",
+ 'icon-btn' if can_select else '',
+ 'mmt-btn' if can_hover else '',
+ cls,
+ kwargs)
+ return mk_my_tooltip(icon, tooltip, cls=merged_cls, **kwargs) if tooltip else MyDiv(icon, cls=merged_cls, **kwargs)
+
+
+def mk_my_tooltip(element, tooltip: str, cls='', **kwargs):
+ merged_cls = merge_classes("mmt-tooltip",
+ cls,
+ kwargs)
+ return MyDiv(element, cls=merged_cls, data_tooltip=tooltip, **kwargs)
diff --git a/src/core/utils.py b/src/core/utils.py
index f466a5b..72a56ac 100644
--- a/src/core/utils.py
+++ b/src/core/utils.py
@@ -1,12 +1,15 @@
import ast
import base64
+import cProfile
import hashlib
import importlib
import inspect
import pkgutil
import re
+import time
import types
import uuid
+from datetime import datetime
from enum import Enum
from io import BytesIO
from urllib.parse import urlparse
@@ -420,6 +423,62 @@ def split_host_port(url):
return host, port
+def timed(func):
+ def wrapper(*args, **kwargs):
+ start = time.perf_counter()
+ result = func(*args, **kwargs)
+ end = time.perf_counter()
+
+ # get class name
+ class_name = None
+ if args:
+ # check the first argument to see if it's a class'
+ if inspect.isclass(args[0]):
+ class_name = args[0].__name__ # class method
+ elif hasattr(args[0], "__class__"):
+ class_name = args[0].__class__.__name__ # instance method
+
+ if class_name:
+ print(f"[PERF] {class_name}.{func.__name__} took {end - start:.4f} sec")
+ else:
+ print(f"[PERF] {func.__name__} took {end - start:.4f} sec")
+
+ return result
+
+ return wrapper
+
+
+def profile_function(func):
+ def wrapper(*args, **kwargs):
+ profiler = cProfile.Profile()
+ profiler.enable()
+ result = func(*args, **kwargs)
+ profiler.disable()
+
+ # Determine class name if any
+ class_name = None
+ if args:
+ if inspect.isclass(args[0]):
+ class_name = args[0].__name__ # class method
+ elif hasattr(args[0], "__class__"):
+ class_name = args[0].__class__.__name__ # instance method
+
+ # Compose filename with timestamp
+ timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+ if class_name:
+ filename = f"{class_name}_{func.__name__}_{timestamp}.prof"
+ else:
+ filename = f"{func.__name__}_{timestamp}.prof"
+
+ # Dump stats to file
+ profiler.dump_stats(filename)
+ print(f"[PROFILE] Profiling data saved to {filename}")
+
+ return result
+
+ return wrapper
+
+
class UnreferencedNamesVisitor(ast.NodeVisitor):
"""
Try to find symbols that will be requested by the ast
@@ -464,4 +523,3 @@ class UnreferencedNamesVisitor(ast.NodeVisitor):
"""
self.names.add(node.arg)
self.visit_selected(node, ["value"])
-
diff --git a/src/main.py b/src/main.py
index f7df448..7141fcb 100644
--- a/src/main.py
+++ b/src/main.py
@@ -1,6 +1,8 @@
# global layout
import asyncio
import logging.config
+import random
+from asyncio import sleep
import yaml
from fasthtml.common import *
@@ -54,6 +56,9 @@ links = [
Link(href="./assets/daisyui-5-themes.css", rel="stylesheet", type="text/css"),
Script(src="./assets/tailwindcss-browser@4.js"),
+ # SSE
+ Script(src="https://unpkg.com/htmx-ext-sse@2.2.1/sse.js"),
+
# Old drawer layout
Script(src="./assets/DrawerLayout.js", defer=True),
Link(rel="stylesheet", href="./assets/DrawerLayout.css"),
@@ -211,6 +216,23 @@ app, rt = fast_app(
pico=False,
)
+# -------------------------
+# Profiling middleware
+# -------------------------
+# @app.middleware("http")
+# async def timing_middleware(request, call_next):
+# start_total = time.perf_counter()
+#
+# # Call the next middleware or route handler
+# response = await call_next(request)
+#
+# end_total = time.perf_counter()
+# elapsed = end_total - start_total
+#
+# print(f"[PERF] Total server time: {elapsed:.4f} sec - Path: {request.url.path}")
+# return response
+
+
settings_manager = SettingsManager()
import_settings = AdminImportSettings(settings_manager, None)
@@ -253,6 +275,42 @@ def get(session):
DrawerLayoutOld(pages),)
+shutdown_event = signal_shutdown()
+
+
+async def number_generator():
+ while True: # not shutdown_event.is_set():
+ data = Article(random.randint(1, 100))
+ print(data)
+ yield sse_message(data)
+ await sleep(1)
+
+
+@rt("/sse")
+def get():
+ return Titled("SSE Random Number Generator",
+ P("Generate pairs of random numbers, as the list grows scroll downwards."),
+ Div(hx_ext="sse",
+ sse_connect="/number-stream",
+ hx_swap="beforeend show:bottom",
+ sse_swap="message"))
+
+
+@rt("/number-stream")
+async def get(): return EventStream(number_generator())
+
+
+@rt('/toasting')
+def get(session):
+ # Normally one toast is enough, this allows us to see
+ # different toast types in action.
+ add_toast(session, f"Toast is being cooked", "info")
+ add_toast(session, f"Toast is ready", "success")
+ add_toast(session, f"Toast is getting a bit crispy", "warning")
+ add_toast(session, f"Toast is burning!", "error")
+ return Titled("I like toast")
+
+
# Error Handling
@app.get("/{path:path}")
def not_found(path: str, session=None):
@@ -275,17 +333,6 @@ def not_found(path: str, session=None):
setup_toasts(app)
-@rt('/toasting')
-def get(session):
- # Normally one toast is enough, this allows us to see
- # different toast types in action.
- add_toast(session, f"Toast is being cooked", "info")
- add_toast(session, f"Toast is ready", "success")
- add_toast(session, f"Toast is getting a bit crispy", "warning")
- add_toast(session, f"Toast is burning!", "error")
- return Titled("I like toast")
-
-
async def main():
logger.info(f" Starting FastHTML server on http://localhost:{APP_PORT}")
serve(port=APP_PORT)